diff --git a/.github/helper/install.sh b/.github/helper/install.sh index 39880e35e7..5cdcbebe1a 100644 --- a/.github/helper/install.sh +++ b/.github/helper/install.sh @@ -54,7 +54,7 @@ fi echo "Starting Bench..." -bench start &> bench_start.log & +bench start &> ~/frappe-bench/bench_start.log & if [ "$TYPE" == "server" ] then diff --git a/.github/workflows/patch-mariadb-tests.yml b/.github/workflows/patch-mariadb-tests.yml index 4b487d2aea..c8fe91d287 100644 --- a/.github/workflows/patch-mariadb-tests.yml +++ b/.github/workflows/patch-mariadb-tests.yml @@ -126,8 +126,10 @@ jobs: git checkout -q -f $branch_name pip install -U frappe-bench + pgrep honcho | xargs kill rm -rf ~/frappe-bench/env bench -v setup env + bench start &> ~/frappe-bench/bench_start.log & bench --site test_site migrate } @@ -143,3 +145,7 @@ jobs: rm -rf ~/frappe-bench/env bench -v setup env bench --site test_site migrate + + - name: Show bench output + if: ${{ always() }} + run: cat ~/frappe-bench/bench_start.log || true diff --git a/.github/workflows/server-tests.yml b/.github/workflows/server-tests.yml index 3b76da1973..8ae0be0197 100644 --- a/.github/workflows/server-tests.yml +++ b/.github/workflows/server-tests.yml @@ -136,6 +136,10 @@ jobs: BUILD_NUMBER: ${{ matrix.container }} TOTAL_BUILDS: 2 + - name: Show bench output + if: ${{ always() }} + run: cat ~/frappe-bench/bench_start.log || true + - name: Upload coverage data uses: actions/upload-artifact@v3 with: diff --git a/cypress/integration/control_currency.js b/cypress/integration/control_currency.js new file mode 100644 index 0000000000..5e6db86036 --- /dev/null +++ b/cypress/integration/control_currency.js @@ -0,0 +1,74 @@ +context("Control Currency", () => { + const fieldname = "currency_field"; + + before(() => { + cy.login(); + cy.visit("/app/website"); + }); + + function get_dialog_with_currency(df_options = {}) { + return cy.dialog({ + title: "Currency Check", + fields: [ + { + fieldname: fieldname, + fieldtype: "Currency", + Label: "Currency", + ...df_options, + }, + ], + }); + } + + it("check value changes", () => { + const TEST_CASES = [ + { + input: "10.101", + df_options: { precision: 1 }, + blur_expected: "10.1", + }, + { + input: "10.101", + df_options: { precision: "3" }, + blur_expected: "10.101", + }, + { + input: "10.101", + df_options: { precision: "" }, // default assumed to be 2; + blur_expected: "10.10", + }, + { + input: "10.101", + df_options: { precision: "0" }, + blur_expected: "10", + }, + { + input: "10.101", + df_options: { precision: 0 }, + blur_expected: "10", + }, + { + input: "10.101", + df_options: { precision: "" }, + blur_expected: "10.1", + default_precision: 1, + }, + ]; + + TEST_CASES.forEach((test_case) => { + cy.window() + .its("frappe") + .then((frappe) => { + frappe.boot.sysdefaults.currency = test_case.currency; + frappe.boot.sysdefaults.currency_precision = test_case.default_precision ?? 2; + }); + + get_dialog_with_currency(test_case.df_options).as("dialog"); + cy.get_field(fieldname, "Currency").clear(); + cy.wait(300); + cy.fill_field(fieldname, test_case.input, "Currency").blur(); + cy.get_field(fieldname, "Currency").should("have.value", test_case.blur_expected); + cy.hide_dialog(); + }); + }); +}); diff --git a/esbuild/esbuild.js b/esbuild/esbuild.js index 4804f0e25f..8c386c86e4 100644 --- a/esbuild/esbuild.js +++ b/esbuild/esbuild.js @@ -446,9 +446,9 @@ function run_build_command_for_apps(apps) { async function notify_redis({ error, success, changed_files }) { // notify redis which in turns tells socketio to publish this to browser - let subscriber = get_redis_subscriber("redis_socketio"); + let subscriber = get_redis_subscriber("redis_queue"); subscriber.on("error", (_) => { - log_warn("Cannot connect to redis_socketio for browser events"); + log_warn("Cannot connect to redis_queue for browser events"); }); let payload = null; @@ -482,9 +482,9 @@ async function notify_redis({ error, success, changed_files }) { } function open_in_editor() { - let subscriber = get_redis_subscriber("redis_socketio"); + let subscriber = get_redis_subscriber("redis_queue"); subscriber.on("error", (_) => { - log_warn("Cannot connect to redis_socketio for open_in_editor events"); + log_warn("Cannot connect to redis_queue for open_in_editor events"); }); subscriber.on("message", (event, file) => { if (event === "open_in_editor") { diff --git a/frappe/core/doctype/doctype/boilerplate/__init__.py b/frappe/core/doctype/doctype/boilerplate/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/frappe/custom/doctype/customize_form/customize_form.js b/frappe/custom/doctype/customize_form/customize_form.js index 3937079365..1a7bb9070a 100644 --- a/frappe/custom/doctype/customize_form/customize_form.js +++ b/frappe/custom/doctype/customize_form/customize_form.js @@ -179,7 +179,7 @@ frappe.ui.form.on("Customize Form", { () => { return frm.call({ doc: frm.doc, - method: "reset_to_defaults", + method: "reset_layout", callback: function (r) { if (!r.exc) { frappe.show_alert({ diff --git a/frappe/custom/doctype/customize_form/customize_form.py b/frappe/custom/doctype/customize_form/customize_form.py index 4da303a016..8930833760 100644 --- a/frappe/custom/doctype/customize_form/customize_form.py +++ b/frappe/custom/doctype/customize_form/customize_form.py @@ -578,12 +578,13 @@ class CustomizeForm(Document): filters={ "doc_type": self.doc_type, "property": "field_order", - "is_system_generated": False, }, ) - if property_setter: - frappe.delete_doc("Property Setter", property_setter) + if not property_setter: + return + + frappe.delete_doc("Property Setter", property_setter) self.fetch_to_customize() @classmethod diff --git a/frappe/desk/query_report.py b/frappe/desk/query_report.py index 6f4bc716aa..5b7c450ae9 100644 --- a/frappe/desk/query_report.py +++ b/frappe/desk/query_report.py @@ -171,6 +171,7 @@ def get_script(report_name): "script": render_include(script), "html_format": html_format, "execution_time": frappe.cache.hget("report_execution_time", report_name) or 0, + "filters": report.filters, } diff --git a/frappe/model/workflow.py b/frappe/model/workflow.py index 1a52077331..0e345a6515 100644 --- a/frappe/model/workflow.py +++ b/frappe/model/workflow.py @@ -210,7 +210,7 @@ def validate_workflow(doc): def get_workflow(doctype) -> "Workflow": - return frappe.get_doc("Workflow", get_workflow_name(doctype)) + return frappe.get_cached_doc("Workflow", get_workflow_name(doctype)) def has_approval_access(user, doc, transition): diff --git a/frappe/permissions.py b/frappe/permissions.py index 67ed972c32..b3380dacc0 100644 --- a/frappe/permissions.py +++ b/frappe/permissions.py @@ -97,6 +97,7 @@ def has_permission( if not perm: push_perm_check_log( _("User {0} does not have access to this document").format(frappe.bold(user)) + + f": {_(doc.doctype)} - {doc.name}" ) else: if ptype == "submit" and not cint(meta.is_submittable): diff --git a/frappe/public/js/form_builder/FormBuilder.vue b/frappe/public/js/form_builder/FormBuilder.vue index 2a1441c51a..c1ab90c4a4 100644 --- a/frappe/public/js/form_builder/FormBuilder.vue +++ b/frappe/public/js/form_builder/FormBuilder.vue @@ -166,8 +166,7 @@ onMounted(() => { } } - :deep([data-has-std-field="false"]), - :deep([data-is-custom="1"]) { + :deep([data-is-user-generated="1"]) { background-color: var(--yellow-highlight-color); } } @@ -175,7 +174,7 @@ onMounted(() => { :deep(.preview) { --field-placeholder-color: var(--fg-bg-color); - .tab, .column, .field, [data-is-custom="1"] { + .tab, .column, .field { background-color: var(--fg-color); } diff --git a/frappe/public/js/form_builder/components/Column.vue b/frappe/public/js/form_builder/components/Column.vue index acb1ff735e..1563d1033e 100644 --- a/frappe/public/js/form_builder/components/Column.vue +++ b/frappe/public/js/form_builder/components/Column.vue @@ -148,8 +148,6 @@ function move_columns_to_section() { :style="{ backgroundColor: column.fields.length ? '' : 'var(--field-placeholder-color)' }" v-model="column.fields" group="fields" - filter="[data-is-custom='0']" - :prevent-on-filter="false" :animation="200" :easing="store.get_animation" item-key="id" @@ -159,7 +157,7 @@ function move_columns_to_section() { diff --git a/frappe/public/js/form_builder/components/Section.vue b/frappe/public/js/form_builder/components/Section.vue index c97fe1e4d8..5131ff25d3 100644 --- a/frappe/public/js/form_builder/components/Section.vue +++ b/frappe/public/js/form_builder/components/Section.vue @@ -160,8 +160,6 @@ function move_sections_to_tab() { backgroundColor: section.columns.length ? null : 'var(--field-placeholder-color)' }" v-model="section.columns" - filter="[data-has-std-field='true']" - :prevent-on-filter="false" group="columns" item-key="id" :animation="200" @@ -172,8 +170,7 @@ function move_sections_to_tab() { diff --git a/frappe/public/js/form_builder/components/Tabs.vue b/frappe/public/js/form_builder/components/Tabs.vue index b587d9d37e..5c233dbd1b 100644 --- a/frappe/public/js/form_builder/components/Tabs.vue +++ b/frappe/public/js/form_builder/components/Tabs.vue @@ -114,8 +114,6 @@ function delete_tab(with_children) { class="tabs" v-model="store.form.layout.tabs" group="tabs" - filter="[data-has-std-field='true']" - :prevent-on-filter="false" :animation="200" :easing="store.get_animation" item-key="id" @@ -125,8 +123,7 @@ function delete_tab(with_children) {
diff --git a/frappe/public/js/form_builder/store.js b/frappe/public/js/form_builder/store.js index b60bdc7919..384baf646d 100644 --- a/frappe/public/js/form_builder/store.js +++ b/frappe/public/js/form_builder/store.js @@ -64,6 +64,10 @@ export const useStore = defineStore("form-builder-store", () => { }); } + function is_user_generated_field(field) { + return cint(field.df.is_custom_field && !field.df.is_system_generated); + } + async function fetch() { await frappe.model.clear_doc("DocType", doctype.value); await frappe.model.with_doctype(doctype.value); @@ -320,6 +324,7 @@ export const useStore = defineStore("form-builder-store", () => { selected, get_df, has_standard_field, + is_user_generated_field, fetch, reset_changes, validate_fields, diff --git a/frappe/public/js/frappe/file_uploader/FileUploader.vue b/frappe/public/js/frappe/file_uploader/FileUploader.vue index b17b13eb83..eba6bf1b2a 100644 --- a/frappe/public/js/frappe/file_uploader/FileUploader.vue +++ b/frappe/public/js/frappe/file_uploader/FileUploader.vue @@ -473,7 +473,20 @@ function upload_file(file, i) { } else if (xhr.status === 403) { file.failed = true; let response = JSON.parse(xhr.responseText); - file.error_message = `Not permitted. ${response._error_message || ''}`; + file.error_message = `Not permitted. ${response._error_message || ''}.`; + + try { + // Append server messages which are useful hint for perm issues + let server_messages = JSON.parse(response._server_messages); + + server_messages.forEach((m) => { + m = JSON.parse(m); + file.error_message += `\n ${m.message} ` + }) + } catch (e) { + console.warning("Failed to parse server message", e) + } + } else if (xhr.status === 413) { file.failed = true; diff --git a/frappe/public/js/frappe/form/controls/currency.js b/frappe/public/js/frappe/form/controls/currency.js index a7d30c071b..e2f79ba446 100644 --- a/frappe/public/js/frappe/form/controls/currency.js +++ b/frappe/public/js/frappe/form/controls/currency.js @@ -7,7 +7,7 @@ frappe.ui.form.ControlCurrency = class ControlCurrency extends frappe.ui.form.Co get_precision() { // always round based on field precision or currency's precision // this method is also called in this.parse() - if (!this.df.precision) { + if (typeof this.df.precision != "number" && !this.df.precision) { if (frappe.boot.sysdefaults.currency_precision) { this.df.precision = frappe.boot.sysdefaults.currency_precision; } else { diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js index 4622c58155..3d0b1bbc6f 100644 --- a/frappe/public/js/frappe/form/form.js +++ b/frappe/public/js/frappe/form/form.js @@ -1414,8 +1414,13 @@ frappe.ui.form.Form = class FrappeForm { if (selector.length) { frappe.utils.scroll_to(selector); } - } else if (window.location.hash && $(window.location.hash).length) { - frappe.utils.scroll_to(window.location.hash, true, 200, null, null, true); + } else if (window.location.hash) { + if ($(window.location.hash).length) { + frappe.utils.scroll_to(window.location.hash, true, 200, null, null, true); + } else { + this.scroll_to_field(window.location.hash.replace("#", "")) && + history.replaceState(null, null, " "); + } } } @@ -1926,11 +1931,12 @@ frappe.ui.form.Form = class FrappeForm { } // highlight control inside field - let control_element = $el.find(".form-control"); + let control_element = $el.closest(".frappe-control"); control_element.addClass("highlight"); setTimeout(() => { control_element.removeClass("highlight"); }, 2000); + return true; } setup_docinfo_change_listener() { diff --git a/frappe/public/js/frappe/form/formatters.js b/frappe/public/js/frappe/form/formatters.js index 637fd7063d..beddbf512d 100644 --- a/frappe/public/js/frappe/form/formatters.js +++ b/frappe/public/js/frappe/form/formatters.js @@ -103,9 +103,15 @@ frappe.form.formatters = { }, Currency: function (value, docfield, options, doc) { var currency = frappe.meta.get_field_currency(docfield, doc); - var precision = cint( - docfield.precision ?? frappe.boot.sysdefaults.currency_precision ?? 2 - ); + + let precision; + if (typeof docfield.precision == "number") { + precision = docfield.precision; + } else { + precision = cint( + docfield.precision || frappe.boot.sysdefaults.currency_precision || 2 + ); + } // If you change anything below, it's going to hurt a company in UAE, a bit. if (precision > 2) { diff --git a/frappe/public/js/frappe/router.js b/frappe/public/js/frappe/router.js index def04cf37e..9ad917a57a 100644 --- a/frappe/public/js/frappe/router.js +++ b/frappe/public/js/frappe/router.js @@ -31,20 +31,21 @@ window.addEventListener("popstate", (e) => { return false; }); -// routing v2, capture all clicks so that the target is managed with push-state +// Capture all clicks so that the target is managed with push-state $("body").on("click", "a", function (e) { - let override = (route) => { + const target_element = e.currentTarget; + const href = target_element.getAttribute("href"); + const is_on_same_host = target_element.hostname === window.location.hostname; + + const override = (route) => { e.preventDefault(); frappe.set_route(route); return false; }; - const target_element = e.currentTarget; - const href = target_element.getAttribute("href"); - const is_on_same_host = target_element.hostname === window.location.hostname; - // click handled, but not by href if ( + !is_on_same_host || // external link target_element.getAttribute("onclick") || // has a handler e.ctrlKey || e.metaKey || // open in a new tab @@ -53,18 +54,13 @@ $("body").on("click", "a", function (e) { return; } - if (href === "") { - return override("/app"); - } - if (href && href.startsWith("#")) { // target startswith "#", this is a v1 style route, so remake it. return override(target_element.hash); } - if (is_on_same_host && frappe.router.is_app_route(target_element.pathname)) { + if (frappe.router.is_app_route(target_element.pathname)) { // target has "/app, this is a v2 style route. - if (target_element.search) { frappe.route_options = {}; let params = new URLSearchParams(target_element.search); @@ -72,7 +68,10 @@ $("body").on("click", "a", function (e) { frappe.route_options[key] = value; } } - return override(target_element.pathname + target_element.hash); + if (target_element.hash) { + frappe.route_hash = target_element.hash; + } + return override(target_element.pathname); } }); @@ -352,8 +351,8 @@ frappe.router = { route = this.get_route_from_arguments(route); route = this.convert_from_standard_route(route); let sub_path = this.make_url(route); - // replace each # occurrences in the URL with encoded character except for last - // sub_path = sub_path.replace(/[#](?=.*[#])/g, "%23"); + sub_path += frappe.route_hash || ""; + frappe.route_hash = null; if (frappe.open_in_new_tab) { localStorage["route_options"] = JSON.stringify(frappe.route_options); window.open(sub_path, "_blank"); diff --git a/frappe/public/js/frappe/utils/utils.js b/frappe/public/js/frappe/utils/utils.js index 095b04c931..f9b6114539 100644 --- a/frappe/public/js/frappe/utils/utils.js +++ b/frappe/public/js/frappe/utils/utils.js @@ -340,9 +340,21 @@ Object.assign(frappe.utils, { scroll_top = 0; } + const highlight = () => { + if (highlight_element) { + $(element).addClass("highlight"); + document.addEventListener( + "click", + function () { + $(element).removeClass("highlight"); + }, + { once: true } + ); + } + }; // already there if (scroll_top == element_to_be_scrolled.scrollTop()) { - return; + return highlight(); } if (animate) { @@ -352,16 +364,7 @@ Object.assign(frappe.utils, { }) .promise() .then(() => { - if (highlight_element) { - $(element).addClass("highlight"); - document.addEventListener( - "click", - function () { - $(element).removeClass("highlight"); - }, - { once: true } - ); - } + highlight(); callback && callback(); }); } else { diff --git a/frappe/public/js/frappe/views/reports/report_utils.js b/frappe/public/js/frappe/views/reports/report_utils.js index 9713f8bb99..d75716541b 100644 --- a/frappe/public/js/frappe/views/reports/report_utils.js +++ b/frappe/public/js/frappe/views/reports/report_utils.js @@ -126,6 +126,13 @@ frappe.report_utils = { .then((r) => { frappe.dom.eval(r.script || ""); return frappe.after_ajax(() => { + if ( + frappe.query_reports[report_name] && + !frappe.query_reports[report_name].filter && + r.filters + ) { + return (frappe.query_reports[report_name].filters = r.filters); + } return ( frappe.query_reports[report_name] && frappe.query_reports[report_name].filters diff --git a/frappe/public/scss/desk/global.scss b/frappe/public/scss/desk/global.scss index b355dbdec2..98aeacf55d 100644 --- a/frappe/public/scss/desk/global.scss +++ b/frappe/public/scss/desk/global.scss @@ -597,11 +597,25 @@ details > summary:focus { display: none; } -.highlight { +.highlight:not(.frappe-control) { transition: 0.5s ease background-color; box-shadow: var(--highlight-shadow) !important; } +.frappe-control.highlight { + --wrap-padding: calc(-1 * var(--padding-sm)); + &::after { + content: " "; + border-radius: 5px; + box-shadow: var(--highlight-shadow) !important; + top: var(--wrap-padding); + position: absolute; + bottom: var(--wrap-padding); + left: var(--wrap-padding); + right: var(--wrap-padding); + } +} + .dropdown-menu.small { font-size: var(--text-sm); min-width: 140px; diff --git a/frappe/realtime.py b/frappe/realtime.py index fdb86546f3..410112b164 100644 --- a/frappe/realtime.py +++ b/frappe/realtime.py @@ -1,7 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors # License: MIT. See LICENSE -import os from contextlib import suppress import redis @@ -9,8 +8,6 @@ import redis import frappe from frappe.utils.data import cstr -redis_server = None - def publish_progress(percent, title=None, doctype=None, docname=None, description=None): publish_realtime( @@ -89,11 +86,12 @@ def flush_realtime_log(): for args in frappe.local._realtime_log: frappe.realtime.emit_via_redis(*args) - frappe.local._realtime_log = [] + clear_realtime_log() def clear_realtime_log(): - frappe.local._realtime_log = [] + if hasattr(frappe.local, "_realtime_log"): + del frappe.local._realtime_log def emit_via_redis(event, message, room): @@ -102,22 +100,13 @@ def emit_via_redis(event, message, room): :param event: Event name, like `task_progress` etc. :param message: JSON message object. For async must contain `task_id` :param room: name of the room""" + from frappe.utils.background_jobs import get_redis_conn with suppress(redis.exceptions.ConnectionError): - r = get_redis_server() + r = get_redis_conn() r.publish("events", frappe.as_json({"event": event, "message": message, "room": room})) -def get_redis_server(): - """returns redis_socketio connection.""" - global redis_server - if not redis_server: - from redis import Redis - - redis_server = Redis.from_url(frappe.conf.redis_socketio or "redis://localhost:12311") - return redis_server - - @frappe.whitelist(allow_guest=True) def can_subscribe_doc(doctype: str, docname: str) -> bool: from frappe.exceptions import PermissionError diff --git a/frappe/recorder.py b/frappe/recorder.py index 8229b862af..1f31181d53 100644 --- a/frappe/recorder.py +++ b/frappe/recorder.py @@ -169,7 +169,7 @@ def status(*args, **kwargs): @do_not_record @administrator_only def start(*args, **kwargs): - frappe.cache.set_value(RECORDER_INTERCEPT_FLAG, 1) + frappe.cache.set_value(RECORDER_INTERCEPT_FLAG, 1, expires_in_sec=60 * 60) @frappe.whitelist() diff --git a/frappe/tests/test_oauth20.py b/frappe/tests/test_oauth20.py index 8de652b888..52e9f1b0c5 100644 --- a/frappe/tests/test_oauth20.py +++ b/frappe/tests/test_oauth20.py @@ -107,7 +107,7 @@ class TestOAuth20(FrappeRequestTestCase): update_client_for_auth_code_grant(self.client_id) # Go to Authorize url - self.TEST_CLIENT.set_cookie(self.site, key="sid", value=self.sid) + self.TEST_CLIENT.set_cookie(key="sid", value=self.sid) resp = self.get( "/api/method/frappe.integrations.oauth2.authorize", { @@ -154,7 +154,7 @@ class TestOAuth20(FrappeRequestTestCase): update_client_for_auth_code_grant(self.client_id) # Go to Authorize url - self.TEST_CLIENT.set_cookie(self.site, key="sid", value=self.sid) + self.TEST_CLIENT.set_cookie(key="sid", value=self.sid) resp = self.get( "/api/method/frappe.integrations.oauth2.authorize", { @@ -203,7 +203,7 @@ class TestOAuth20(FrappeRequestTestCase): frappe.db.commit() # Go to Authorize url - self.TEST_CLIENT.set_cookie(self.site, key="sid", value=self.sid) + self.TEST_CLIENT.set_cookie(key="sid", value=self.sid) resp = self.get( "/api/method/frappe.integrations.oauth2.authorize", { @@ -321,7 +321,7 @@ class TestOAuth20(FrappeRequestTestCase): nonce = frappe.generate_hash() # Go to Authorize url - self.TEST_CLIENT.set_cookie(self.site, key="sid", value=self.sid) + self.TEST_CLIENT.set_cookie(key="sid", value=self.sid) resp = self.get( "/api/method/frappe.integrations.oauth2.authorize", { diff --git a/frappe/tests/test_pdf.py b/frappe/tests/test_pdf.py index 4a8fef253c..84004dc1f1 100644 --- a/frappe/tests/test_pdf.py +++ b/frappe/tests/test_pdf.py @@ -2,7 +2,7 @@ # License: MIT. See LICENSE import io -from PyPDF2 import PdfReader +from pypdf import PdfReader import frappe import frappe.utils.pdf as pdfgen @@ -43,7 +43,7 @@ class TestPdf(FrappeTestCase): password = "qwe" pdf = pdfgen.get_pdf(self.html, options={"password": password}) reader = PdfReader(io.BytesIO(pdf)) - self.assertTrue(reader.isEncrypted) + self.assertTrue(reader.is_encrypted) self.assertTrue(reader.decrypt(password)) def test_pdf_generation_as_a_user(self): diff --git a/frappe/utils/background_jobs.py b/frappe/utils/background_jobs.py index 6a203f8dc7..ea3ea4c191 100755 --- a/frappe/utils/background_jobs.py +++ b/frappe/utils/background_jobs.py @@ -28,6 +28,9 @@ RQ_JOB_FAILURE_TTL = 7 * 24 * 60 * 60 # 7 days instead of 1 year (default) RQ_RESULTS_TTL = 10 * 60 +_redis_queue_conn = None + + @lru_cache def get_queues_timeout(): common_site_config = frappe.get_conf() @@ -47,9 +50,6 @@ def get_queues_timeout(): } -redis_connection = None - - def enqueue( method: str | Callable, queue: str = "default", @@ -360,7 +360,7 @@ def get_redis_conn(username=None, password=None): elif not frappe.local.conf.redis_queue: raise Exception("redis_queue missing in common_site_config.json") - global redis_connection + global _redis_queue_conn cred = frappe._dict() if frappe.conf.get("use_rq_auth"): @@ -374,8 +374,14 @@ def get_redis_conn(username=None, password=None): elif os.environ.get("RQ_ADMIN_PASWORD"): cred["username"] = "default" cred["password"] = os.environ.get("RQ_ADMIN_PASWORD") + try: - redis_connection = RedisQueue.get_connection(**cred) + if not cred: + if not _redis_queue_conn: + _redis_queue_conn = RedisQueue.get_connection() + return _redis_queue_conn + else: + return RedisQueue.get_connection(**cred) except (redis.exceptions.AuthenticationError, redis.exceptions.ResponseError): log( f'Wrong credentials used for {cred.username or "default user"}. ' @@ -387,8 +393,6 @@ def get_redis_conn(username=None, password=None): log(f"Please make sure that Redis Queue runs @ {frappe.get_conf().redis_queue}", colour="red") raise - return redis_connection - def get_queues() -> list[Queue]: """Get all the queues linked to the current bench.""" diff --git a/frappe/utils/boilerplate.py b/frappe/utils/boilerplate.py index 0d786972fb..2cb1a6ab80 100644 --- a/frappe/utils/boilerplate.py +++ b/frappe/utils/boilerplate.py @@ -558,10 +558,6 @@ jobs: image: redis:alpine ports: - 11000:6379 - redis-socketio: - image: redis:alpine - ports: - - 12000:6379 mariadb: image: mariadb:10.6 env: diff --git a/frappe/utils/connections.py b/frappe/utils/connections.py index 6660e6ce19..020bc8b97f 100644 --- a/frappe/utils/connections.py +++ b/frappe/utils/connections.py @@ -3,7 +3,7 @@ from urllib.parse import urlparse from frappe import get_conf -REDIS_KEYS = ("redis_cache", "redis_queue", "redis_socketio") +REDIS_KEYS = ("redis_cache", "redis_queue") def is_open(ip, port, timeout=10): diff --git a/frappe/utils/pdf.py b/frappe/utils/pdf.py index 9b7c9a6ce4..721b061257 100644 --- a/frappe/utils/pdf.py +++ b/frappe/utils/pdf.py @@ -8,7 +8,7 @@ from distutils.version import LooseVersion import pdfkit from bs4 import BeautifulSoup -from PyPDF2 import PdfReader, PdfWriter +from pypdf import PdfReader, PdfWriter import frappe from frappe import _ diff --git a/frappe/utils/print_format.py b/frappe/utils/print_format.py index b96809f2c2..4faaf97780 100644 --- a/frappe/utils/print_format.py +++ b/frappe/utils/print_format.py @@ -1,6 +1,6 @@ import os -from PyPDF2 import PdfWriter +from pypdf import PdfWriter import frappe from frappe import _ diff --git a/frappe/utils/response.py b/frappe/utils/response.py index 79a6b16d1a..b0f8fbc6e6 100644 --- a/frappe/utils/response.py +++ b/frappe/utils/response.py @@ -68,7 +68,6 @@ def build_response(response_type=None): def as_csv(): response = Response() response.mimetype = "text/csv" - response.charset = "utf-8" response.headers["Content-Disposition"] = ( 'attachment; filename="%s.csv"' % frappe.response["doctype"].replace(" ", "_") ).encode("utf-8") @@ -79,7 +78,6 @@ def as_csv(): def as_txt(): response = Response() response.mimetype = "text" - response.charset = "utf-8" response.headers["Content-Disposition"] = ( 'attachment; filename="%s.txt"' % frappe.response["doctype"].replace(" ", "_") ).encode("utf-8") @@ -109,7 +107,6 @@ def as_json(): del frappe.local.response["http_status_code"] response.mimetype = "application/json" - response.charset = "utf-8" response.data = json.dumps(frappe.local.response, default=json_handler, separators=(",", ":")) return response diff --git a/frappe/website/utils.py b/frappe/website/utils.py index 922abbb751..de639d1709 100644 --- a/frappe/website/utils.py +++ b/frappe/website/utils.py @@ -12,7 +12,7 @@ from werkzeug.wrappers import Response import frappe from frappe import _ from frappe.model.document import Document -from frappe.utils import cint, get_assets_json, get_system_timezone, md_to_html +from frappe.utils import cint, cstr, get_assets_json, get_system_timezone, md_to_html FRONTMATTER_PATTERN = re.compile(r"^\s*(?:---|\+\+\+)(.*?)(?:---|\+\+\+)\s*(.+)$", re.S | re.M) H1_TAG_PATTERN = re.compile("

([^<]*)") @@ -529,14 +529,14 @@ def build_response(path, data, http_status_code, headers: dict | None = None): response = Response() response.data = set_content_type(response, data, path) response.status_code = http_status_code - response.headers["X-Page-Name"] = path.encode("ascii", errors="xmlcharrefreplace") + response.headers["X-Page-Name"] = cstr(path.encode("ascii", errors="xmlcharrefreplace")) response.headers["X-From-Cache"] = frappe.local.response.from_cache or False add_preload_for_bundled_assets(response) if headers: for key, val in headers.items(): - response.headers[key] = val.encode("ascii", errors="xmlcharrefreplace") + response.headers[key] = cstr(val.encode("ascii", errors="xmlcharrefreplace")) return response @@ -544,12 +544,10 @@ def build_response(path, data, http_status_code, headers: dict | None = None): def set_content_type(response, data, path): if isinstance(data, dict): response.mimetype = "application/json" - response.charset = "utf-8" data = json.dumps(data) return data response.mimetype = "text/html" - response.charset = "utf-8" # ignore paths ending with .com to avoid unnecessary download # https://bugs.python.org/issue22347 diff --git a/node_utils.js b/node_utils.js index 10744387ca..0b8c455875 100644 --- a/node_utils.js +++ b/node_utils.js @@ -38,7 +38,7 @@ function get_conf() { return conf; } -function get_redis_subscriber(kind = "redis_socketio", options = {}) { +function get_redis_subscriber(kind = "redis_queue", options = {}) { const conf = get_conf(); const host = conf[kind] || conf.redis_async_broker_port; return redis.createClient({ url: host, ...options }); diff --git a/pyproject.toml b/pyproject.toml index aa89eed928..687a08ac0f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,65 +13,65 @@ dependencies = [ "Click~=8.1.3", "filelock~=3.8.0", "filetype~=1.2.0", - "GitPython~=3.1.30", + "GitPython~=3.1.31", "Jinja2~=3.1.2", - "Pillow~=9.3.0", - "PyJWT~=2.4.0", + "Pillow~=9.5.0", + "PyJWT~=2.7.0", "PyMySQL==1.0.3", - "PyPDF2~=2.1.0", + "pypdf~=3.9.1", "PyPika~=0.48.9", "PyQRCode~=1.2.1", "PyYAML~=6.0", "RestrictedPython~=6.0", - "WeasyPrint==52.5", - "Werkzeug~=2.2.2", + "WeasyPrint==59.0", + "Werkzeug~=2.3.4", "Whoosh~=2.7.4", - "beautifulsoup4~=4.9.3", + "beautifulsoup4~=4.12.2", "bleach-allowlist~=1.0.3", "bleach~=3.3.0", - "cairocffi==1.2.0", + "cairocffi==1.5.1", "chardet~=5.1.0", - "croniter~=1.3.5", - "cryptography~=39.0.1", + "croniter~=1.3.15", + "cryptography~=41.0.1", "email-reply-parser~=0.5.12", "git-url-parse~=1.2.2", "gunicorn~=20.1.0", "html5lib~=1.1", "ipython~=8.10.0", "ldap3~=2.9", - "markdown2~=2.4.0", + "markdown2~=2.4.8", "MarkupSafe>=2.1.0,<3", "maxminddb-geolite2==2018.703", - "num2words~=0.5.10", - "oauthlib~=3.2.1", - "openpyxl~=3.0.7", + "num2words~=0.5.12", + "oauthlib~=3.2.2", + "openpyxl~=3.1.2", "passlib~=1.7.4", "pdfkit~=1.0.0", - "phonenumbers==8.12.40", + "phonenumbers==8.13.13", "premailer~=3.8.0", - "psutil~=5.9.1", + "psutil~=5.9.5", "psycopg2-binary~=2.9.1", - "pyOpenSSL~=23.0.0", - "pycryptodome~=3.10.1", - "pydantic~=1.10.2", - "pyotp~=2.6.0", - "python-dateutil~=2.8.1", - "pytz==2022.1", + "pyOpenSSL~=23.2.0", + "pycryptodome~=3.18.0", + "pydantic~=1.10.8", + "pyotp~=2.8.0", + "python-dateutil~=2.8.2", + "pytz==2023.3", "rauth~=0.7.3", - "redis~=4.5.4", - "hiredis~=2.0.0", - "requests-oauthlib~=1.3.0", + "redis~=4.5.5", + "hiredis~=2.2.3", + "requests-oauthlib~=1.3.1", "requests~=2.31.0", - "rq~=1.11.1", + "rq~=1.15.0", "rsa>=4.1", "semantic-version~=2.10.0", - "sqlparse~=0.4.1", - "tenacity~=8.0.1", - "terminaltables~=3.1.0", + "sqlparse~=0.4.4", + "tenacity~=8.2.2", + "terminaltables~=3.1.10", "traceback-with-variables~=2.0.4", "xlrd~=2.0.1", "zxcvbn~=4.4.28", - "markdownify~=0.11.2", + "markdownify~=0.11.6", # integration dependencies "boto3~=1.18.49", @@ -100,8 +100,8 @@ indent = "\t" [tool.bench.dev-dependencies] coverage = "~=6.5.0" -Faker = "~=13.12.1" -pyngrok = "~=5.0.5" -unittest-xml-reporting = "~=3.0.4" -watchdog = "~=2.1.9" -hypothesis = "~=6.68.2" +Faker = "~=18.10.1" +pyngrok = "~=6.0.0" +unittest-xml-reporting = "~=3.2.0" +watchdog = "~=3.0.0" +hypothesis = "~=6.77.0" diff --git a/setup.py b/setup.py deleted file mode 100644 index 7a90eed81a..0000000000 --- a/setup.py +++ /dev/null @@ -1,6 +0,0 @@ -# TODO: Remove this file when bench >=v5.11.0 is adopted / v15.0.0 is released -from setuptools import setup - -name = "frappe" - -setup() diff --git a/socketio.js b/socketio.js index 8e87a0cce1..31be39c847 100644 --- a/socketio.js +++ b/socketio.js @@ -181,7 +181,7 @@ io.on("connection", function (socket) { }); socket.on("open_in_editor", (data) => { - let s = get_redis_subscriber("redis_socketio"); + let s = get_redis_subscriber("redis_queue"); s.publish("open_in_editor", JSON.stringify(data)); }); });