From 7780670ae44c5359e6e404075d52e65600e960bd Mon Sep 17 00:00:00 2001 From: David Arnold Date: Tue, 25 Jul 2023 03:37:46 -0500 Subject: [PATCH] build(deps): update node redis client to v4 (#21797) * build(deps): update redis client to v4 in legacy mode * fix: node17+ - prefer ipv4 * chore: use redis client v4 api (async) and adapt error handling * fix: timeout by exiting if not in watch mode * fix: parse message before republishing --------- Co-authored-by: Ankush Menat --- esbuild/esbuild.js | 67 ++++++++++++++++++++++++---------------------- node_utils.js | 8 +++++- package.json | 2 +- realtime/index.js | 29 ++++++++++---------- yarn.lock | 61 +++++++++++++++++------------------------ 5 files changed, 82 insertions(+), 85 deletions(-) diff --git a/esbuild/esbuild.js b/esbuild/esbuild.js index 8937a03216..e4a226281b 100644 --- a/esbuild/esbuild.js +++ b/esbuild/esbuild.js @@ -89,12 +89,10 @@ const NODE_PATHS = [].concat( app_list.map((app) => path.resolve(get_app_path(app), "..")).filter(fs.existsSync) ); -execute() - .then(() => RUN_BUILD_COMMAND && run_build_command_for_apps(APPS)) - .catch((e) => { - console.error(e); - process.exit(1); - }); +execute().catch((e) => { + console.error(e); + process.exit(1); +}); if (WATCH_MODE) { // listen for open files in editor event @@ -127,6 +125,10 @@ async function execute() { for (const result of results) { await write_assets_json(result.metafile); } + RUN_BUILD_COMMAND && run_build_command_for_apps(APPS); + if (!WATCH_MODE) { + process.exit(0); + } } function build_assets_for_apps(apps, files) { @@ -418,18 +420,18 @@ async function write_assets_json(metafile) { }; } -function update_assets_json_in_cache() { +async function update_assets_json_in_cache() { // update assets_json cache in redis, so that it can be read directly by python - return new Promise((resolve) => { - let client = get_redis_subscriber("redis_cache"); - // handle error event to avoid printing stack traces - client.on("error", (_) => { - log_warn("Cannot connect to redis_cache to update assets_json"); - }); - client.del("assets_json", (err) => { - client.unref(); - resolve(); - }); + let client = get_redis_subscriber("redis_cache"); + // handle error event to avoid printing stack traces + try { + await client.connect(); + } catch (e) { + log_warn("Cannot connect to redis_cache to update assets_json"); + } + client.del("assets_json", (err) => { + client.unref(); + resolve(); }); } @@ -458,9 +460,11 @@ function run_build_command_for_apps(apps) { async function notify_redis({ error, success, changed_files }) { // notify redis which in turns tells socketio to publish this to browser let subscriber = get_redis_subscriber("redis_queue"); - subscriber.on("error", (_) => { + try { + await subscriber.connect(); + } catch (e) { log_warn("Cannot connect to redis_queue for browser events"); - }); + } let payload = null; if (error) { @@ -483,7 +487,7 @@ async function notify_redis({ error, success, changed_files }) { }; } - subscriber.publish( + await subscriber.publish( "events", JSON.stringify({ event: "build_event", @@ -492,21 +496,20 @@ async function notify_redis({ error, success, changed_files }) { ); } -function open_in_editor() { +async function open_in_editor() { let subscriber = get_redis_subscriber("redis_queue"); - subscriber.on("error", (_) => { + try { + await subscriber.connect(); + } catch (e) { log_warn("Cannot connect to redis_queue for open_in_editor events"); + } + subscriber.subscribe("open_in_editor", (file) => { + file = JSON.parse(file); + let file_path = path.resolve(file.file); + log("Opening file in editor:", file_path); + let launch = require("launch-editor"); + launch(`${file_path}:${file.line}:${file.column}`); }); - subscriber.on("message", (event, file) => { - if (event === "open_in_editor") { - file = JSON.parse(file); - let file_path = path.resolve(file.file); - log("Opening file in editor:", file_path); - let launch = require("launch-editor"); - launch(`${file_path}:${file.line}:${file.column}`); - } - }); - subscriber.subscribe("open_in_editor"); } function get_rebuilt_assets(prev_assets, new_assets) { diff --git a/node_utils.js b/node_utils.js index 14b6f3576b..38c9b1c1cc 100644 --- a/node_utils.js +++ b/node_utils.js @@ -1,8 +1,14 @@ const fs = require("fs"); const path = require("path"); -const redis = require("redis"); +const redis = require("@redis/client"); const bench_path = path.resolve(__dirname, "..", ".."); +const dns = require("dns"); + +// Since node17, node resolves to ipv6 unless system is configured otherwise. +// In Frappe context using ipv4 - 127.0.0.1 is fine. +dns.setDefaultResultOrder("ipv4first"); + function get_conf() { // defaults var conf = { diff --git a/package.json b/package.json index 2cd1e6894e..0794ae5d1d 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "dependencies": { "@editorjs/editorjs": "^2.26.3", "@frappe/esbuild-plugin-postcss2": "^0.1.3", + "@redis/client": "^1.5.8", "@vue-flow/background": "^1.1.0", "@vue-flow/core": "^1.16.2", "@vue/component-compiler": "^4.2.4", @@ -63,7 +64,6 @@ "quill-image-resize": "^3.0.9", "quill-magic-url": "^3.0.0", "qz-tray": "^2.0.8", - "redis": "^3.1.1", "rtlcss": "^4.0.0", "sass": "^1.63.0", "showdown": "^2.1.0", diff --git a/realtime/index.js b/realtime/index.js index ecc6063ca4..c60ec66330 100644 --- a/realtime/index.js +++ b/realtime/index.js @@ -27,7 +27,8 @@ function on_connection(socket) { frappe_handlers(realtime, socket); // ESBUild "open in editor" on error - socket.on("open_in_editor", (data) => { + socket.on("open_in_editor", async (data) => { + await subscriber.connect(); subscriber.publish("open_in_editor", JSON.stringify(data)); }); } @@ -38,19 +39,19 @@ realtime.on("connection", on_connection); // Consume events sent from python via redis pub-sub channel. const subscriber = get_redis_subscriber(); -subscriber.on("message", function (_channel, message) { - message = JSON.parse(message); - - let namespace = "/" + message.namespace; - if (message.room) { - io.of(namespace).to(message.room).emit(message.event, message.message); - } else { - // publish to ALL sites only used for things like build event. - realtime.emit(message.event, message.message); - } -}); - -subscriber.subscribe("events"); +(async () => { + await subscriber.connect(); + subscriber.subscribe("events", (message) => { + message = JSON.parse(message); + let namespace = "/" + message.namespace; + if (message.room) { + io.of(namespace).to(message.room).emit(message.event, message.message); + } else { + // publish to ALL sites only used for things like build event. + realtime.emit(message.event, message.message); + } + }); +})(); // ======================= let port = conf.socketio_port; diff --git a/yarn.lock b/yarn.lock index e816b29656..f632157bbe 100644 --- a/yarn.lock +++ b/yarn.lock @@ -81,6 +81,15 @@ "@nodelib/fs.scandir" "2.1.5" fastq "^1.6.0" +"@redis/client@^1.5.8": + version "1.5.8" + resolved "https://registry.yarnpkg.com/@redis/client/-/client-1.5.8.tgz#a375ba7861825bd0d2dc512282b8bff7b98dbcb1" + integrity sha512-xzElwHIO6rBAqzPeVnCzgvrnBEcFL1P0w8P65VNLRkdVW8rOE58f52hdj0BDgmsdOm4f1EoXPZtH4Fh7M/qUpw== + dependencies: + cluster-key-slot "1.1.2" + generic-pool "3.9.0" + yallist "4.0.0" + "@socket.io/component-emitter@~3.1.0": version "3.1.0" resolved "https://registry.yarnpkg.com/@socket.io/component-emitter/-/component-emitter-3.1.0.tgz#96116f2a912e0c02817345b3c10751069920d553" @@ -592,6 +601,11 @@ clone@^2.1.1, clone@^2.1.2: resolved "https://registry.yarnpkg.com/clone/-/clone-2.1.2.tgz#1b7f4b9f591f1e8f83670401600345a02887435f" integrity sha512-3Pe/CF1Nn94hyhIYpjtiLhdCoEoz0DqQ+988E9gmeEdQZlojxnOb74wctFyuwWQHzqyf9X7C7MG8juUpqBJT8w== +cluster-key-slot@1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz#88ddaa46906e303b5de30d3153b7d9fe0a0c19ac" + integrity sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA== + color-convert@^1.9.0: version "1.9.3" resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.3.tgz#bb71850690e1f136567de629d2d5471deda4c1e8" @@ -973,11 +987,6 @@ delayed-stream@~1.0.0: resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" integrity sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ== -denque@^1.5.0: - version "1.5.1" - resolved "https://registry.yarnpkg.com/denque/-/denque-1.5.1.tgz#07f670e29c9a78f8faecb2566a1e2c11929c5cbf" - integrity sha512-XwE+iZ4D6ZUB7mfYRMb5wByE8L74HCn30FBN7sWnXksWc1LO1bPDl67pBR9o/kC4z/xSNAwkMYcGgqDV3BE3Hw== - dezalgo@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/dezalgo/-/dezalgo-1.0.4.tgz#751235260469084c132157dfa857f386d4c33d81" @@ -1457,6 +1466,11 @@ generic-names@^4.0.0: dependencies: loader-utils "^3.2.0" +generic-pool@3.9.0: + version "3.9.0" + resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-3.9.0.tgz#36f4a678e963f4fdb8707eab050823abc4e8f5e4" + integrity sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g== + get-caller-file@^2.0.5: version "2.0.5" resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-2.0.5.tgz#4f94412a82db32f36e3b0b9741f8a97feb031f7e" @@ -2874,33 +2888,6 @@ readdirp@~3.6.0: dependencies: picomatch "^2.2.1" -redis-commands@^1.7.0: - version "1.7.0" - resolved "https://registry.yarnpkg.com/redis-commands/-/redis-commands-1.7.0.tgz#15a6fea2d58281e27b1cd1acfb4b293e278c3a89" - integrity sha512-nJWqw3bTFy21hX/CPKHth6sfhZbdiHP6bTawSgQBlKOVRG7EZkfHbbHwQJnrE4vsQf0CMNE+3gJ4Fmm16vdVlQ== - -redis-errors@^1.0.0, redis-errors@^1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/redis-errors/-/redis-errors-1.2.0.tgz#eb62d2adb15e4eaf4610c04afe1529384250abad" - integrity sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w== - -redis-parser@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/redis-parser/-/redis-parser-3.0.0.tgz#b66d828cdcafe6b4b8a428a7def4c6bcac31c8b4" - integrity sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A== - dependencies: - redis-errors "^1.0.0" - -redis@^3.1.1: - version "3.1.2" - resolved "https://registry.yarnpkg.com/redis/-/redis-3.1.2.tgz#766851117e80653d23e0ed536254677ab647638c" - integrity sha512-grn5KoZLr/qrRQVwoSkmzdbw6pwF+/rwODtrOr6vuBRiR/f3rjSTGupbF90Zpqm2oenix8Do6RV7pYEkGwlKkw== - dependencies: - denque "^1.5.0" - redis-commands "^1.7.0" - redis-errors "^1.2.0" - redis-parser "^3.0.0" - regexp.prototype.flags@^1.2.0, regexp.prototype.flags@^1.5.0: version "1.5.0" resolved "https://registry.yarnpkg.com/regexp.prototype.flags/-/regexp.prototype.flags-1.5.0.tgz#fe7ce25e7e4cca8db37b6634c8a2c7009199b9cb" @@ -3479,16 +3466,16 @@ y18n@^5.0.5: resolved "https://registry.yarnpkg.com/y18n/-/y18n-5.0.8.tgz#7f4934d0f7ca8c56f95314939ddcd2dd91ce1d55" integrity sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA== +yallist@4.0.0, yallist@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72" + integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== + yallist@^2.1.2: version "2.1.2" resolved "https://registry.yarnpkg.com/yallist/-/yallist-2.1.2.tgz#1c11f9218f076089a47dd512f93c6699a6a81d52" integrity sha512-ncTzHV7NvsQZkYe1DW7cbDLm0YpzHmZF5r/iyP3ZnQtMiJ+pjzisCiMNI+Sj+xQF5pXhSHxSB3uDbsBTzY/c2A== -yallist@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72" - integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== - yaml@^1.10.2: version "1.10.2" resolved "https://registry.yarnpkg.com/yaml/-/yaml-1.10.2.tgz#2301c5ffbf12b467de8da2333a459e29e7920e4b"