diff --git a/.github/helper/consumer_db/mariadb.json b/.github/helper/db/mariadb.json similarity index 92% rename from .github/helper/consumer_db/mariadb.json rename to .github/helper/db/mariadb.json index 2e32157e1a..8bb654da66 100644 --- a/.github/helper/consumer_db/mariadb.json +++ b/.github/helper/db/mariadb.json @@ -1,7 +1,7 @@ { "db_host": "127.0.0.1", "db_port": 3306, - "db_name": "test_frappe_consumer", + "db_name": "test_frappe", "db_password": "test_frappe", "allow_tests": true, "db_type": "mariadb", diff --git a/.github/helper/consumer_db/postgres.json b/.github/helper/db/postgres.json similarity index 92% rename from .github/helper/consumer_db/postgres.json rename to .github/helper/db/postgres.json index 9532670029..6ca83b9e96 100644 --- a/.github/helper/consumer_db/postgres.json +++ b/.github/helper/db/postgres.json @@ -1,7 +1,7 @@ { "db_host": "127.0.0.1", "db_port": 5432, - "db_name": "test_frappe_consumer", + "db_name": "test_frappe", "db_password": "test_frappe", "db_type": "postgres", "allow_tests": true, diff --git a/.github/helper/install.sh b/.github/helper/install.sh index 1514236ecb..39880e35e7 100644 --- a/.github/helper/install.sh +++ b/.github/helper/install.sh @@ -17,37 +17,23 @@ fi echo "Setting Up Sites & Database..." mkdir ~/frappe-bench/sites/test_site -cp "${GITHUB_WORKSPACE}/.github/helper/consumer_db/$DB.json" ~/frappe-bench/sites/test_site/site_config.json - -if [ "$TYPE" == "server" ] -then - mkdir ~/frappe-bench/sites/test_site_producer - cp "${GITHUB_WORKSPACE}/.github/helper/producer_db/$DB.json" ~/frappe-bench/sites/test_site_producer/site_config.json -fi +cp "${GITHUB_WORKSPACE}/.github/helper/db/$DB.json" ~/frappe-bench/sites/test_site/site_config.json if [ "$DB" == "mariadb" ] then - mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "SET GLOBAL character_set_server = 'utf8mb4'" - mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "SET GLOBAL collation_server = 'utf8mb4_unicode_ci'" + mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "SET GLOBAL character_set_server = 'utf8mb4'"; + mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "SET GLOBAL collation_server = 'utf8mb4_unicode_ci'"; - mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "CREATE DATABASE test_frappe_consumer" - mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "CREATE USER 'test_frappe_consumer'@'localhost' IDENTIFIED BY 'test_frappe_consumer'" - mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "GRANT ALL PRIVILEGES ON \`test_frappe_consumer\`.* TO 'test_frappe_consumer'@'localhost'" + mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "CREATE DATABASE test_frappe"; + mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "CREATE USER 'test_frappe'@'localhost' IDENTIFIED BY 'test_frappe'"; + mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "GRANT ALL PRIVILEGES ON \`test_frappe\`.* TO 'test_frappe'@'localhost'"; - mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "CREATE DATABASE test_frappe_producer" - mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "CREATE USER 'test_frappe_producer'@'localhost' IDENTIFIED BY 'test_frappe_producer'" - mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "GRANT ALL PRIVILEGES ON \`test_frappe_producer\`.* TO 'test_frappe_producer'@'localhost'" - - mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "FLUSH PRIVILEGES" + mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "FLUSH PRIVILEGES"; fi - if [ "$DB" == "postgres" ] then - echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE DATABASE test_frappe_consumer" -U postgres - echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE USER test_frappe_consumer WITH PASSWORD 'test_frappe'" -U postgres - - echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE DATABASE test_frappe_producer" -U postgres - echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE USER test_frappe_producer WITH PASSWORD 'test_frappe'" -U postgres + echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE DATABASE test_frappe" -U postgres; + echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE USER test_frappe WITH PASSWORD 'test_frappe'" -U postgres; fi echo "Setting Up Procfile..." @@ -78,11 +64,6 @@ fi bench --site test_site reinstall --yes -if [ "$TYPE" == "server" ] -then - bench --site test_site_producer reinstall --yes -fi - if [ "$TYPE" == "server" ] then # wait till assets are built succesfully diff --git a/.github/helper/producer_db/mariadb.json b/.github/helper/producer_db/mariadb.json deleted file mode 100644 index c1db0d765f..0000000000 --- a/.github/helper/producer_db/mariadb.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_host": "127.0.0.1", - "db_port": 3306, - "db_name": "test_frappe_producer", - "db_password": "test_frappe", - "allow_tests": true, - "db_type": "mariadb", - "auto_email_id": "test@example.com", - "mail_server": "smtp.example.com", - "mail_login": "test@example.com", - "mail_password": "test", - "admin_password": "admin", - "root_login": "root", - "root_password": "travis", - "host_name": "http://test_site_producer:8000" -} diff --git a/.github/helper/producer_db/postgres.json b/.github/helper/producer_db/postgres.json deleted file mode 100644 index 8b9d2a20fd..0000000000 --- a/.github/helper/producer_db/postgres.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_host": "127.0.0.1", - "db_port": 5432, - "db_name": "test_frappe_producer", - "db_password": "test_frappe", - "db_type": "postgres", - "allow_tests": true, - "auto_email_id": "test@example.com", - "mail_server": "smtp.example.com", - "mail_login": "test@example.com", - "mail_password": "test", - "admin_password": "admin", - "root_login": "postgres", - "root_password": "travis", - "host_name": "http://test_site_producer:8000" -} diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml new file mode 100644 index 0000000000..3c6f8b744c --- /dev/null +++ b/.github/workflows/backport.yml @@ -0,0 +1,26 @@ +name: Backport +on: + pull_request_target: + types: + - closed + - labeled + +jobs: + main: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout Actions + uses: actions/checkout@v2 + with: + repository: "frappe/backport" + path: ./actions + ref: develop + - name: Install Actions + run: npm install --production --prefix ./actions + - name: Run backport + uses: ./actions/backport + with: + token: ${{secrets.RELEASE_TOKEN}} + labelsToAdd: "backport" + title: "{{originalTitle}}" diff --git a/.github/workflows/server-mariadb-tests.yml b/.github/workflows/server-mariadb-tests.yml index f6b6575a1f..7a38648e63 100644 --- a/.github/workflows/server-mariadb-tests.yml +++ b/.github/workflows/server-mariadb-tests.yml @@ -74,7 +74,6 @@ jobs: fi - uses: actions/setup-node@v3 - if: ${{ steps.check-build.outputs.build == 'strawberry' }} with: node-version: 16 check-latest: true @@ -82,7 +81,6 @@ jobs: - name: Add to Hosts run: | echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts - echo "127.0.0.1 test_site_producer" | sudo tee -a /etc/hosts - name: Cache pip uses: actions/cache@v3 diff --git a/.github/workflows/server-postgres-tests.yml b/.github/workflows/server-postgres-tests.yml index e0bb471d39..300f888de6 100644 --- a/.github/workflows/server-postgres-tests.yml +++ b/.github/workflows/server-postgres-tests.yml @@ -84,7 +84,6 @@ jobs: - name: Add to Hosts run: | echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts - echo "127.0.0.1 test_site_producer" | sudo tee -a /etc/hosts - name: Cache pip uses: actions/cache@v3 diff --git a/.github/workflows/ui-tests.yml b/.github/workflows/ui-tests.yml index cacb5ce833..4aa6ed0393 100644 --- a/.github/workflows/ui-tests.yml +++ b/.github/workflows/ui-tests.yml @@ -82,7 +82,6 @@ jobs: - name: Add to Hosts run: | echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts - echo "127.0.0.1 test_site_producer" | sudo tee -a /etc/hosts - name: Cache pip uses: actions/cache@v3 @@ -128,7 +127,10 @@ jobs: run: cd ~/frappe-bench/ && bench build --apps frappe - name: Site Setup - run: cd ~/frappe-bench/ && bench --site test_site execute frappe.utils.install.complete_setup_wizard + run: | + cd ~/frappe-bench/ + bench --site test_site execute frappe.utils.install.complete_setup_wizard + bench --site test_site execute frappe.tests.ui_test_helpers.create_test_user - name: UI Tests run: cd ~/frappe-bench/ && bench --site test_site run-ui-tests frappe --with-coverage --headless --parallel --ci-build-id $GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT diff --git a/.mergify.yml b/.mergify.yml index 0ea0f2a801..85b590ba76 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -4,9 +4,9 @@ pull_request_rules: - and: - and: - author!=surajshetty3416 - - author!=gavindsouza - author!=deepeshgarg007 - author!=ankush + - author!=frappe-pr-bot - author!=mergify[bot] - or: - base=version-15 diff --git a/CODEOWNERS b/CODEOWNERS index aff03e2082..861016710a 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -7,7 +7,6 @@ templates/ @surajshetty3416 www/ @surajshetty3416 patches/ @surajshetty3416 -event_streaming/ @ruchamahabal data_import* @netchampfaris core/ @surajshetty3416 workspace @shariquerik diff --git a/README.md b/README.md index 01f4199fdd..ca46ee2c1e 100644 --- a/README.md +++ b/README.md @@ -75,3 +75,5 @@ Full-stack web application framework that uses Python and MariaDB on the server ## License This repository has been released under the [MIT License](LICENSE). + +By contributing to Frappe, you agree that your contributions will be licensed under its MIT License. diff --git a/cypress.config.js b/cypress.config.js index f86354a06d..bfd0bc0025 100644 --- a/cypress.config.js +++ b/cypress.config.js @@ -3,6 +3,7 @@ const { defineConfig } = require("cypress"); module.exports = defineConfig({ projectId: "92odwv", adminPassword: "admin", + testUser: "frappe@example.com", defaultCommandTimeout: 20000, pageLoadTimeout: 15000, video: true, diff --git a/cypress/integration/control_link.js b/cypress/integration/control_link.js index 240515be45..a5281d9b09 100644 --- a/cypress/integration/control_link.js +++ b/cypress/integration/control_link.js @@ -177,14 +177,14 @@ context("Control Link", () => { cy.intercept("POST", "/api/method/frappe.client.validate_link").as("validate_link"); cy.get(".frappe-control[data-fieldname=assigned_by] input").focus().as("input"); - cy.get("@input").type("Administrator", { delay: 100 }).blur(); + cy.get("@input").type(cy.config("testUser"), { delay: 100 }).blur(); cy.wait("@validate_link"); cy.get(".frappe-control[data-fieldname=assigned_by_full_name] .control-value").should( "contain", - "Administrator" + "Frappe" ); - cy.window().its("cur_frm.doc.assigned_by").should("eq", "Administrator"); + cy.window().its("cur_frm.doc.assigned_by").should("eq", cy.config("testUser")); // invalid input cy.get("@input").clear().type("invalid input", { delay: 100 }).blur(); @@ -198,10 +198,10 @@ context("Control Link", () => { // set valid value again cy.get("@input").clear().focus(); cy.wait("@search_link"); - cy.get("@input").type("Administrator", { delay: 100 }).blur(); + cy.get("@input").type(cy.config("testUser"), { delay: 100 }).blur(); cy.wait("@validate_link"); - cy.window().its("cur_frm.doc.assigned_by").should("eq", "Administrator"); + cy.window().its("cur_frm.doc.assigned_by").should("eq", cy.config("testUser")); // clear input cy.get("@input").clear().blur(); diff --git a/cypress/integration/control_markdown_editor.js b/cypress/integration/control_markdown_editor.js index 16c3dac51f..24ab32f48a 100644 --- a/cypress/integration/control_markdown_editor.js +++ b/cypress/integration/control_markdown_editor.js @@ -7,9 +7,12 @@ context("Control Markdown Editor", () => { it("should allow inserting images by drag and drop", () => { cy.visit("/app/web-page/new"); cy.fill_field("content_type", "Markdown", "Select"); - cy.get_field("main_section_md", "Markdown Editor").attachFile("sample_image.jpg", { - subjectType: "drag-n-drop", - }); + cy.get_field("main_section_md", "Markdown Editor").selectFile( + "cypress/fixtures/sample_image.jpg", + { + action: "drag-drop", + } + ); cy.click_modal_primary_button("Upload"); cy.get_field("main_section_md", "Markdown Editor").should( "contain", diff --git a/cypress/integration/dashboard_links.js b/cypress/integration/dashboard_links.js index 995f8d0d9f..061899ec95 100644 --- a/cypress/integration/dashboard_links.js +++ b/cypress/integration/dashboard_links.js @@ -8,7 +8,7 @@ const child_table_doctype_name = child_table_doctype.name; context("Dashboard links", () => { before(() => { cy.visit("/login"); - cy.login(); + cy.login("Administrator"); cy.insert_doc("DocType", child_table_doctype, true); cy.insert_doc("DocType", child_table_doctype_1, true); cy.insert_doc("DocType", doctype_with_child_table, true); @@ -27,8 +27,7 @@ context("Dashboard links", () => { cy.visit("/app/contact"); cy.clear_filters(); - cy.visit("/app/user"); - cy.get(".list-row-col > .level-item > .ellipsis").eq(0).click({ force: true }); + cy.visit(`/app/user/${cy.config("testUser")}`); //To check if initially the dashboard contains only the "Contact" link and there is no counter cy.select_form_tab("Connections"); @@ -41,12 +40,11 @@ context("Dashboard links", () => { cy.findByRole("button", { name: "Add Contact" }).click(); cy.get('[data-doctype="Contact"][data-fieldname="first_name"]').type("Admin"); cy.findByRole("button", { name: "Save" }).click(); - cy.visit("/app/user"); - cy.get(".list-row-col > .level-item > .ellipsis").eq(0).click({ force: true }); + cy.visit(`/app/user/${cy.config("testUser")}`); - //To check if the counter for contact doc is "1" after adding the contact + //To check if the counter for contact doc is "2" after adding additional contact cy.select_form_tab("Connections"); - cy.get('[data-doctype="Contact"] > .count').should("contain", "1"); + cy.get('[data-doctype="Contact"] > .count').should("contain", "2"); cy.get('[data-doctype="Contact"]').contains("Contact").click(); //Deleting the newly created contact @@ -64,8 +62,7 @@ context("Dashboard links", () => { }); it("Report link in dashboard", () => { - cy.visit("/app/user"); - cy.visit("/app/user/Administrator"); + cy.visit(`/app/user/${cy.config("testUser")}`); cy.select_form_tab("Connections"); cy.get('.document-link[data-doctype="Contact"]').contains("Contact"); cy.window() diff --git a/cypress/integration/file_uploader.js b/cypress/integration/file_uploader.js index 669f9ba385..e1cf91d043 100644 --- a/cypress/integration/file_uploader.js +++ b/cypress/integration/file_uploader.js @@ -21,9 +21,11 @@ context("FileUploader", () => { it("should accept dropped files", () => { open_upload_dialog(); - cy.get_open_dialog().find(".file-upload-area").attachFile("example.json", { - subjectType: "drag-n-drop", - }); + cy.get_open_dialog() + .find(".file-upload-area") + .selectFile("cypress/fixtures/example.json", { + action: "drag-drop", + }); cy.get_open_dialog().find(".file-name").should("contain", "example.json"); cy.intercept("POST", "/api/method/upload_file").as("upload_file"); @@ -64,9 +66,11 @@ context("FileUploader", () => { it("should allow cropping and optimization for valid images", () => { open_upload_dialog(); - cy.get_open_dialog().find(".file-upload-area").attachFile("sample_image.jpg", { - subjectType: "drag-n-drop", - }); + cy.get_open_dialog() + .find(".file-upload-area") + .selectFile("cypress/fixtures/sample_image.jpg", { + action: "drag-drop", + }); cy.get_open_dialog().findAllByText("sample_image.jpg").should("exist"); cy.get_open_dialog().find(".btn-crop").first().click(); diff --git a/cypress/integration/login.js b/cypress/integration/login.js index 2db4b1fdcd..912f34c508 100644 --- a/cypress/integration/login.js +++ b/cypress/integration/login.js @@ -32,7 +32,7 @@ context("Login", () => { it("logs in using correct credentials", () => { cy.get("#login_email").type("Administrator"); - cy.get("#login_password").type(Cypress.config("adminPassword")); + cy.get("#login_password").type(Cypress.env("adminPassword")); cy.findByRole("button", { name: "Login" }).click(); cy.location("pathname").should("eq", "/app"); @@ -56,7 +56,7 @@ context("Login", () => { ); cy.get("#login_email").type("Administrator"); - cy.get("#login_password").type(Cypress.config("adminPassword")); + cy.get("#login_password").type(Cypress.env("adminPassword")); cy.findByRole("button", { name: "Login" }).click(); diff --git a/cypress/integration/sidebar.js b/cypress/integration/sidebar.js index ee724ffda2..0b2a21aa4f 100644 --- a/cypress/integration/sidebar.js +++ b/cypress/integration/sidebar.js @@ -4,9 +4,11 @@ const verify_attachment_visibility = (document, is_private) => { const assertion = is_private ? "be.checked" : "not.be.checked"; cy.findByRole("button", { name: "Attach File" }).click(); - cy.get_open_dialog().find(".file-upload-area").attachFile("sample_image.jpg", { - subjectType: "drag-n-drop", - }); + cy.get_open_dialog() + .find(".file-upload-area") + .selectFile("cypress/fixtures/sample_image.jpg", { + action: "drag-drop", + }); cy.get_open_dialog().findByRole("checkbox", { name: "Private" }).should(assertion); }; @@ -36,11 +38,6 @@ context("Sidebar", () => { //To check if no filter is available in "Assigned To" dropdown cy.get(".empty-state").should("contain", "No filters found"); - cy.click_sidebar_button("Created By"); - - //To check if "Created By" dropdown contains filter - cy.get(".group-by-item > .dropdown-item").should("contain", "Me"); - //Assigning a doctype to a user cy.visit("/app/doctype/ToDo"); cy.get(".form-assignments > .flex > .text-muted").click(); @@ -70,7 +67,7 @@ context("Sidebar", () => { cy.get(".condition").should("have.value", "like"); cy.get(".filter-field > .form-group > .input-with-feedback").should( "have.value", - "%Administrator%" + `%${cy.config("testUser")}%` ); cy.click_filter_button(); diff --git a/cypress/integration/table_multiselect.js b/cypress/integration/table_multiselect.js index 133af44d51..8261b5b384 100644 --- a/cypress/integration/table_multiselect.js +++ b/cypress/integration/table_multiselect.js @@ -13,7 +13,9 @@ context("Table MultiSelect", () => { cy.fill_field("assign_condition", 'status=="Open"', "Code"); cy.get('input[data-fieldname="users"]').focus().as("input"); cy.get('input[data-fieldname="users"] + ul').should("be.visible"); - cy.get("@input").type("test{enter}", { delay: 100 }); + cy.get("@input").type("test@erpnext", { delay: 100 }); + cy.wait(500); + cy.get("@input").type("{enter}"); cy.get( '.frappe-control[data-fieldname="users"] .form-control .tb-selected-value .btn-link-to-form' ).as("selected-value"); @@ -52,6 +54,6 @@ context("Table MultiSelect", () => { "existing_value" ); cy.get("@existing_value").find(".btn-link-to-form").click(); - cy.location("pathname").should("contain", "/user/test@erpnext.com"); + cy.location("pathname").should("contain", "/user/test%40erpnext.com"); }); }); diff --git a/cypress/integration/timeline.js b/cypress/integration/timeline.js index 5841891af6..7835819334 100644 --- a/cypress/integration/timeline.js +++ b/cypress/integration/timeline.js @@ -72,14 +72,14 @@ context("Timeline", () => { cy.click_listview_row_item(0); //To check if the submission of the documemt is visible in the timeline content - cy.get(".timeline-content").should("contain", "Administrator submitted this document"); + cy.get(".timeline-content").should("contain", "Frappe submitted this document"); cy.get('[id="page-Custom Submittable DocType"] .page-actions') .findByRole("button", { name: "Cancel" }) .click(); cy.get_open_dialog().findByRole("button", { name: "Yes" }).click(); //To check if the cancellation of the documemt is visible in the timeline content - cy.get(".timeline-content").should("contain", "Administrator cancelled this document"); + cy.get(".timeline-content").should("contain", "Frappe cancelled this document"); //Deleting the document cy.visit("/app/custom-submittable-doctype"); diff --git a/cypress/integration/web_form.js b/cypress/integration/web_form.js index d7f210e7d2..53f72ab013 100644 --- a/cypress/integration/web_form.js +++ b/cypress/integration/web_form.js @@ -1,6 +1,13 @@ context("Web Form", () => { before(() => { - cy.login(); + cy.login("Administrator"); + cy.visit("/app/"); + return cy + .window() + .its("frappe") + .then((frappe) => { + return frappe.xcall("frappe.tests.ui_test_helpers.clear_notes"); + }); }); it("Create Web Form", () => { @@ -42,7 +49,7 @@ context("Web Form", () => { }); it("Login Required", () => { - cy.login(); + cy.login("Administrator"); cy.visit("/app/web-form/note"); cy.findByRole("tab", { name: "Settings" }).click(); @@ -51,7 +58,6 @@ context("Web Form", () => { cy.save(); cy.visit("/note"); - cy.url().should("include", "/note/Note%201"); cy.call("logout"); @@ -62,7 +68,7 @@ context("Web Form", () => { }); it("Show List", () => { - cy.login(); + cy.login("Administrator"); cy.visit("/app/web-form/note"); cy.findByRole("tab", { name: "Settings" }).click(); @@ -156,7 +162,7 @@ context("Web Form", () => { }); it("Read Only", () => { - cy.login(); + cy.login("Administrator"); cy.visit("/note"); cy.url().should("include", "/note/list"); diff --git a/cypress/support/commands.js b/cypress/support/commands.js index 6548a427aa..a51e1daf17 100644 --- a/cypress/support/commands.js +++ b/cypress/support/commands.js @@ -1,4 +1,3 @@ -import "cypress-file-upload"; import "@testing-library/cypress/add-commands"; import "@4tw/cypress-drag-drop"; import "cypress-real-events/support"; @@ -30,7 +29,7 @@ import "cypress-real-events/support"; Cypress.Commands.add("login", (email, password) => { if (!email) { - email = "Administrator"; + email = Cypress.config("testUser") || "Administrator"; } if (!password) { password = Cypress.env("adminPassword"); diff --git a/esbuild/esbuild.js b/esbuild/esbuild.js index 56910cbcac..69e479a6ff 100644 --- a/esbuild/esbuild.js +++ b/esbuild/esbuild.js @@ -103,7 +103,7 @@ async function execute() { log_error("There were some problems during build"); log(); log(chalk.dim(e.stack)); - if (process.env.CI) { + if (process.env.CI || PRODUCTION) { process.kill(process.pid); } return; diff --git a/frappe/__init__.py b/frappe/__init__.py index 27862a6a55..5ddf1c55a9 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -203,6 +203,7 @@ def init(site: str, sites_path: str = ".", new_site: bool = False) -> None: "mute_emails": False, "has_dataurl": False, "new_site": new_site, + "read_only": False, } ) local.rollback_observers = [] @@ -238,7 +239,6 @@ def init(site: str, sites_path: str = ".", new_site: bool = False) -> None: local.jloader = None local.cache = {} local.document_cache = {} - local.meta_cache = {} local.form_dict = _dict() local.preload_assets = {"style": [], "script": []} local.session = _dict() @@ -284,9 +284,7 @@ def connect_replica(): user = local.conf.replica_db_name password = local.conf.replica_db_password - local.replica_db = get_db( - host=local.conf.replica_host, user=user, password=password, port=port, read_only=True - ) + local.replica_db = get_db(host=local.conf.replica_host, user=user, password=password, port=port) # swap db connections local.primary_db = local.db @@ -1065,21 +1063,9 @@ def set_value(doctype, docname, fieldname, value=None): return frappe.client.set_value(doctype, docname, fieldname, value) -@overload -def get_cached_doc(doctype, docname, _allow_dict=True) -> dict: - ... - - -@overload def get_cached_doc(*args, **kwargs) -> "Document": - ... - - -def get_cached_doc(*args, **kwargs): - allow_dict = kwargs.pop("_allow_dict", False) - def _respond(doc, from_redis=False): - if not allow_dict and isinstance(doc, dict): + if isinstance(doc, dict): local.document_cache[key] = doc = get_doc(doc) elif from_redis: @@ -1103,6 +1089,12 @@ def get_cached_doc(*args, **kwargs): if not key: key = get_document_cache_key(doc.doctype, doc.name) + _set_document_in_cache(key, doc) + + return doc + + +def _set_document_in_cache(key: str, doc: "Document") -> None: local.document_cache[key] = doc # Avoid setting in local.cache since we're already using local.document_cache above @@ -1112,8 +1104,6 @@ def get_cached_doc(*args, **kwargs): except Exception: cache().hset("document_cache", key, doc.as_dict(), cache_locally=False) - return doc - def can_cache_doc(args) -> str | None: """ @@ -1152,7 +1142,7 @@ def get_cached_value( doctype: str, name: str, fieldname: str = "name", as_dict: bool = False ) -> Any: try: - doc = get_cached_doc(doctype, name, _allow_dict=True) + doc = get_cached_doc(doctype, name) except DoesNotExistError: clear_last_message() return @@ -1188,13 +1178,9 @@ def get_doc(*args, **kwargs) -> "Document": doc = frappe.model.document.get_doc(*args, **kwargs) - # Replace cache - if key := can_cache_doc(args): - if key in local.document_cache: - local.document_cache[key] = doc - - if cache().hexists("document_cache", key): - cache().hset("document_cache", key, doc.as_dict()) + # Replace cache if stale one exists + if (key := can_cache_doc(args)) and cache().hexists("document_cache", key): + _set_document_in_cache(key, doc) return doc @@ -2224,13 +2210,18 @@ def log_error(title=None, message=None, reference_doctype=None, reference_name=N title = title or "Error" traceback = as_unicode(traceback or get_traceback(with_context=True)) - return get_doc( + error_log = get_doc( doctype="Error Log", error=traceback, method=title, reference_doctype=reference_doctype, reference_name=reference_name, - ).insert(ignore_permissions=True) + ) + + if flags.read_only: + error_log.deferred_insert() + else: + return error_log.insert(ignore_permissions=True) def get_desk_link(doctype, name): diff --git a/frappe/app.py b/frappe/app.py index 11c6308014..136b16bff5 100644 --- a/frappe/app.py +++ b/frappe/app.py @@ -33,17 +33,6 @@ SAFE_HTTP_METHODS = ("GET", "HEAD", "OPTIONS") UNSAFE_HTTP_METHODS = ("POST", "PUT", "DELETE", "PATCH") -class RequestContext: - def __init__(self, environ): - self.request = Request(environ) - - def __enter__(self): - init_request(self.request) - - def __exit__(self, type, value, traceback): - frappe.destroy() - - @local_manager.middleware @Request.application def application(request: Request): @@ -83,9 +72,6 @@ def application(request: Request): except HTTPException as e: return e - except frappe.SessionStopped as e: - response = frappe.utils.response.handle_session_stopped() - except Exception as e: response = handle_exception(e) @@ -118,9 +104,12 @@ def init_request(request): # site does not exist raise NotFound - if frappe.local.conf.get("maintenance_mode"): + if frappe.local.conf.maintenance_mode: frappe.connect() - raise frappe.SessionStopped("Session Stopped") + if frappe.local.conf.allow_reads_during_maintenance: + setup_read_only_mode() + else: + raise frappe.SessionStopped("Session Stopped") else: frappe.connect(set_admin_as_user=False) @@ -132,6 +121,24 @@ def init_request(request): frappe.local.http_request = frappe.auth.HTTPRequest() +def setup_read_only_mode(): + """During maintenance_mode reads to DB can still be performed to reduce downtime. This + function sets up read only mode + + - Setting global flag so other pages, desk and database can know that we are in read only mode. + - Setup read only database access either by: + - Connecting to read replica if one exists + - Or setting up read only SQL transactions. + """ + frappe.flags.read_only = True + + # If replica is available then just connect replica, else setup read only transaction. + if frappe.conf.read_from_replica: + frappe.connect_replica() + else: + frappe.db.begin(read_only=True) + + def log_request(request, response): if hasattr(frappe.local, "conf") and frappe.local.conf.enable_frappe_logger: frappe.logger("frappe.web", allow_site=frappe.local.site).info( @@ -233,11 +240,20 @@ def handle_exception(e): or (frappe.local.request.path.startswith("/api/") and not accept_header.startswith("text")) ) + if not frappe.session.user: + # If session creation fails then user won't be unset. This causes a lot of code that + # assumes presence of this to fail. Session creation fails => guest or expired login + # usually. + frappe.session.user = "Guest" + if respond_as_json: # handle ajax responses first # if the request is ajax, send back the trace or error message response = frappe.utils.response.report_error(http_status_code) + elif isinstance(e, frappe.SessionStopped): + response = frappe.utils.response.handle_session_stopped() + elif ( http_status_code == 500 and (frappe.db and isinstance(e, frappe.db.InternalError)) diff --git a/frappe/auth.py b/frappe/auth.py index 7ce2b17680..f7ff6f0fe5 100644 --- a/frappe/auth.py +++ b/frappe/auth.py @@ -6,9 +6,8 @@ import frappe import frappe.database import frappe.utils import frappe.utils.user -from frappe import _, conf +from frappe import _ from frappe.core.doctype.activity_log.activity_log import add_authentication_log -from frappe.modules.patch_handler import check_session_stopped from frappe.sessions import Session, clear_sessions, delete_session from frappe.translate import get_language from frappe.twofactor import ( @@ -30,9 +29,6 @@ class HTTPRequest: # load cookies self.set_cookies() - # set frappe.local.db - self.connect() - # login and start/resume user session self.set_session() @@ -45,9 +41,6 @@ class HTTPRequest: # write out latest cookies frappe.local.cookie_manager.init_cookies() - # check session status - check_session_stopped() - @property def domain(self): if not getattr(self, "_domain", None): @@ -97,16 +90,6 @@ class HTTPRequest: def set_lang(self): frappe.local.lang = get_language() - def get_db_name(self): - """get database name from conf""" - return conf.db_name - - def connect(self): - """connect to db, from ac_name or db_name""" - frappe.local.db = frappe.database.get_db( - user=self.get_db_name(), password=getattr(conf, "db_password", "") - ) - class LoginManager: diff --git a/frappe/automation/workspace/tools/tools.json b/frappe/automation/workspace/tools/tools.json index 40b265b34f..9962e27f55 100644 --- a/frappe/automation/workspace/tools/tools.json +++ b/frappe/automation/workspace/tools/tools.json @@ -1,6 +1,6 @@ { "charts": [], - "content": "[{\"type\":\"header\",\"data\":{\"text\":\"Your Shortcuts\",\"col\":12}},{\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"ToDo\",\"col\":3}},{\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"Note\",\"col\":3}},{\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"File\",\"col\":3}},{\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"Assignment Rule\",\"col\":3}},{\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"Auto Repeat\",\"col\":3}},{\"type\":\"spacer\",\"data\":{\"col\":12}},{\"type\":\"header\",\"data\":{\"text\":\"Reports & Masters\",\"col\":12}},{\"type\":\"card\",\"data\":{\"card_name\":\"Tools\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Email\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Automation\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Event Streaming\",\"col\":4}}]", + "content": "[{\"type\":\"header\",\"data\":{\"text\":\"Your Shortcuts\",\"col\":12}},{\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"ToDo\",\"col\":3}},{\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"Note\",\"col\":3}},{\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"File\",\"col\":3}},{\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"Assignment Rule\",\"col\":3}},{\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"Auto Repeat\",\"col\":3}},{\"type\":\"spacer\",\"data\":{\"col\":12}},{\"type\":\"header\",\"data\":{\"text\":\"Reports & Masters\",\"col\":12}},{\"type\":\"card\",\"data\":{\"card_name\":\"Tools\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Email\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Automation\",\"col\":4}}]", "creation": "2020-03-02 14:53:24.980279", "docstatus": 0, "doctype": "Workspace", @@ -107,7 +107,7 @@ "hidden": 0, "is_query_report": 0, "label": "Automation", - "link_count": 0, + "link_count": 3, "onboard": 0, "type": "Card Break" }, @@ -143,78 +143,16 @@ "link_type": "DocType", "onboard": 0, "type": "Link" - }, - { - "hidden": 0, - "is_query_report": 0, - "label": "Event Streaming", - "link_count": 0, - "onboard": 0, - "type": "Card Break" - }, - { - "dependencies": "", - "hidden": 0, - "is_query_report": 0, - "label": "Event Producer", - "link_count": 0, - "link_to": "Event Producer", - "link_type": "DocType", - "onboard": 0, - "type": "Link" - }, - { - "dependencies": "", - "hidden": 0, - "is_query_report": 0, - "label": "Event Consumer", - "link_count": 0, - "link_to": "Event Consumer", - "link_type": "DocType", - "onboard": 0, - "type": "Link" - }, - { - "dependencies": "", - "hidden": 0, - "is_query_report": 0, - "label": "Event Update Log", - "link_count": 0, - "link_to": "Event Update Log", - "link_type": "DocType", - "onboard": 0, - "type": "Link" - }, - { - "dependencies": "", - "hidden": 0, - "is_query_report": 0, - "label": "Event Sync Log", - "link_count": 0, - "link_to": "Event Sync Log", - "link_type": "DocType", - "onboard": 0, - "type": "Link" - }, - { - "dependencies": "", - "hidden": 0, - "is_query_report": 0, - "label": "Document Type Mapping", - "link_count": 0, - "link_to": "Document Type Mapping", - "link_type": "DocType", - "onboard": 0, - "type": "Link" } ], - "modified": "2022-01-13 17:48:48.456763", + "modified": "2022-08-23 14:42:58.364898", "modified_by": "Administrator", "module": "Automation", "name": "Tools", "owner": "Administrator", "parent_page": "", "public": 1, + "quick_lists": [], "restrict_to_domain": "", "roles": [], "sequence_id": 26.0, diff --git a/frappe/cache_manager.py b/frappe/cache_manager.py index 868329ec1e..aae7b804d0 100644 --- a/frappe/cache_manager.py +++ b/frappe/cache_manager.py @@ -12,7 +12,6 @@ doctype_map_keys = ( "energy_point_rule_map", "assignment_rule_map", "milestone_tracker_map", - "event_consumer_document_type_map", ) bench_cache_keys = ("assets_json",) @@ -59,8 +58,8 @@ user_cache_keys = ( ) doctype_cache_keys = ( - "meta", - "form_meta", + "doctype_meta", + "doctype_form_meta", "table_columns", "last_modified", "linked_doctypes", @@ -117,9 +116,6 @@ def clear_doctype_cache(doctype=None): clear_controller_cache(doctype) cache = frappe.cache() - if getattr(frappe.local, "meta_cache") and (doctype in frappe.local.meta_cache): - del frappe.local.meta_cache[doctype] - for key in ("is_table", "doctype_modules", "document_cache"): cache.delete_value(key) diff --git a/frappe/commands/site.py b/frappe/commands/site.py index eb5a732f04..13f642ea76 100644 --- a/frappe/commands/site.py +++ b/frappe/commands/site.py @@ -420,6 +420,9 @@ def install_app(context, apps, force=False): print(f"An error occurred while installing {app}{err_msg}") exit_code = 1 + if not exit_code: + frappe.db.commit() + frappe.destroy() sys.exit(exit_code) diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py index e1776ac8be..07061444b0 100644 --- a/frappe/commands/utils.py +++ b/frappe/commands/utils.py @@ -874,7 +874,6 @@ def run_ui_tests( node_bin = subprocess.getoutput("npm bin") cypress_path = f"{node_bin}/cypress" - plugin_path = f"{node_bin}/../cypress-file-upload" drag_drop_plugin_path = f"{node_bin}/../@4tw/cypress-drag-drop" real_events_plugin_path = f"{node_bin}/../cypress-real-events" testing_library_path = f"{node_bin}/../@testing-library" @@ -883,7 +882,6 @@ def run_ui_tests( # check if cypress in path...if not, install it. if not ( os.path.exists(cypress_path) - and os.path.exists(plugin_path) and os.path.exists(drag_drop_plugin_path) and os.path.exists(real_events_plugin_path) and os.path.exists(testing_library_path) @@ -894,10 +892,10 @@ def run_ui_tests( packages = " ".join( [ "cypress@^10", - "cypress-file-upload@^5", "@4tw/cypress-drag-drop@^2", "cypress-real-events", "@testing-library/cypress@^8", + "@testing-library/dom@8.17.1", "@cypress/code-coverage@^3", ] ) diff --git a/frappe/core/doctype/activity_log/activity_log.json b/frappe/core/doctype/activity_log/activity_log.json index ad12246a95..910baceb5e 100644 --- a/frappe/core/doctype/activity_log/activity_log.json +++ b/frappe/core/doctype/activity_log/activity_log.json @@ -102,8 +102,7 @@ "fetch_from": "reference_name.owner", "fieldname": "reference_owner", "fieldtype": "Read Only", - "label": "Reference Owner", - "search_index": 1 + "label": "Reference Owner" }, { "fieldname": "column_break_14", @@ -154,7 +153,7 @@ "icon": "fa fa-comment", "index_web_pages_for_search": 1, "links": [], - "modified": "2021-10-25 11:43:57.504565", + "modified": "2022-09-13 15:19:42.474114", "modified_by": "Administrator", "module": "Core", "name": "Activity Log", @@ -181,6 +180,7 @@ "search_fields": "subject", "sort_field": "modified", "sort_order": "DESC", + "states": [], "title_field": "subject", "track_seen": 1 -} +} \ No newline at end of file diff --git a/frappe/core/doctype/activity_log/activity_log.py b/frappe/core/doctype/activity_log/activity_log.py index 468b7f4473..e819683d0a 100644 --- a/frappe/core/doctype/activity_log/activity_log.py +++ b/frappe/core/doctype/activity_log/activity_log.py @@ -37,7 +37,6 @@ def on_doctype_update(): """Add indexes in `tabActivity Log`""" frappe.db.add_index("Activity Log", ["reference_doctype", "reference_name"]) frappe.db.add_index("Activity Log", ["timeline_doctype", "timeline_name"]) - frappe.db.add_index("Activity Log", ["link_doctype", "link_name"]) def add_authentication_log(subject, user, operation="Login", status="Success"): diff --git a/frappe/core/doctype/comment/comment.py b/frappe/core/doctype/comment/comment.py index 985dde8ce2..3d886cf93b 100644 --- a/frappe/core/doctype/comment/comment.py +++ b/frappe/core/doctype/comment/comment.py @@ -60,7 +60,6 @@ class Comment(Document): def on_doctype_update(): frappe.db.add_index("Comment", ["reference_doctype", "reference_name"]) - frappe.db.add_index("Comment", ["link_doctype", "link_name"]) def update_comment_in_doc(doc): diff --git a/frappe/core/doctype/custom_docperm/custom_docperm.json b/frappe/core/doctype/custom_docperm/custom_docperm.json index 93f5431903..2c594f5624 100644 --- a/frappe/core/doctype/custom_docperm/custom_docperm.json +++ b/frappe/core/doctype/custom_docperm/custom_docperm.json @@ -212,7 +212,8 @@ "fieldname": "parent", "fieldtype": "Data", "label": "Reference Document Type", - "read_only": 1 + "read_only": 1, + "search_index": 1 }, { "default": "0", diff --git a/frappe/core/doctype/data_import/data_import.py b/frappe/core/doctype/data_import/data_import.py index 5ad603e416..c055524fd1 100644 --- a/frappe/core/doctype/data_import/data_import.py +++ b/frappe/core/doctype/data_import/data_import.py @@ -9,7 +9,7 @@ from frappe.core.doctype.data_import.exporter import Exporter from frappe.core.doctype.data_import.importer import Importer from frappe.model.document import Document from frappe.modules.import_file import import_file_by_path -from frappe.utils.background_jobs import enqueue +from frappe.utils.background_jobs import enqueue, is_job_queued from frappe.utils.csvutils import validate_google_sheets_url @@ -59,15 +59,12 @@ class DataImport(Document): return i.get_data_for_import_preview() def start_import(self): - from frappe.core.page.background_jobs.background_jobs import get_info from frappe.utils.scheduler import is_scheduler_inactive if is_scheduler_inactive() and not frappe.flags.in_test: frappe.throw(_("Scheduler is inactive. Cannot import data."), title=_("Scheduler Inactive")) - enqueued_jobs = [d.get("job_name") for d in get_info()] - - if self.name not in enqueued_jobs: + if not is_job_queued(self.name): enqueue( start_import, queue="default", diff --git a/frappe/core/doctype/doctype/doctype.py b/frappe/core/doctype/doctype/doctype.py index f5700bd0e6..acc5c4871d 100644 --- a/frappe/core/doctype/doctype/doctype.py +++ b/frappe/core/doctype/doctype/doctype.py @@ -414,10 +414,6 @@ class DocType(Document): if not frappe.flags.in_install and hasattr(self, "before_update"): self.sync_global_search() - # clear from local cache - if self.name in frappe.local.meta_cache: - del frappe.local.meta_cache[self.name] - clear_linked_doctype_cache() def setup_autoincrement_and_sequence(self): @@ -1198,6 +1194,9 @@ def validate_fields(meta): frappe.throw(_("Precision should be between 1 and 6")) def check_unique_and_text(docname, d): + if meta.is_virtual: + return + if meta.issingle: d.unique = 0 d.search_index = 0 diff --git a/frappe/core/doctype/document_naming_settings/document_naming_settings.py b/frappe/core/doctype/document_naming_settings/document_naming_settings.py index 6fc4d9b23e..0cdba32dae 100644 --- a/frappe/core/doctype/document_naming_settings/document_naming_settings.py +++ b/frappe/core/doctype/document_naming_settings/document_naming_settings.py @@ -113,6 +113,8 @@ class DocumentNamingSettings(Document): option_string = "\n".join(options) + # Erase default first, it might not be in new options. + self.update_naming_series_property_setter(doctype, "default", "") self.update_naming_series_property_setter(doctype, "options", option_string) self.update_naming_series_property_setter(doctype, "default", default) diff --git a/frappe/core/doctype/error_log/error_log.py b/frappe/core/doctype/error_log/error_log.py index c7ab98e034..871ec8ebdd 100644 --- a/frappe/core/doctype/error_log/error_log.py +++ b/frappe/core/doctype/error_log/error_log.py @@ -9,7 +9,7 @@ from frappe.query_builder.functions import Now class ErrorLog(Document): def onload(self): - if not self.seen: + if not self.seen and not frappe.flags.read_only: self.db_set("seen", 1, update_modified=0) frappe.db.commit() diff --git a/frappe/core/doctype/file/file.json b/frappe/core/doctype/file/file.json index 3008e27aa0..cb727e48f0 100644 --- a/frappe/core/doctype/file/file.json +++ b/frappe/core/doctype/file/file.json @@ -64,8 +64,7 @@ "fieldname": "is_home_folder", "fieldtype": "Check", "hidden": 1, - "label": "Is Home Folder", - "search_index": 1 + "label": "Is Home Folder" }, { "default": "0", @@ -125,8 +124,7 @@ "in_standard_filter": 1, "label": "Attached To DocType", "options": "DocType", - "read_only": 1, - "search_index": 1 + "read_only": 1 }, { "fieldname": "column_break_10", @@ -136,8 +134,7 @@ "fieldname": "attached_to_name", "fieldtype": "Data", "label": "Attached To Name", - "read_only": 1, - "search_index": 1 + "read_only": 1 }, { "fieldname": "attached_to_field", @@ -175,7 +172,7 @@ "icon": "fa fa-file", "idx": 1, "links": [], - "modified": "2020-06-28 12:21:30.772386", + "modified": "2022-09-13 15:50:15.508250", "modified_by": "Administrator", "module": "Core", "name": "File", @@ -210,6 +207,7 @@ ], "sort_field": "modified", "sort_order": "ASC", + "states": [], "title_field": "file_name", "track_changes": 1 } \ No newline at end of file diff --git a/frappe/core/doctype/report/report.json b/frappe/core/doctype/report/report.json index 5b3593e658..4b98f1a3eb 100644 --- a/frappe/core/doctype/report/report.json +++ b/frappe/core/doctype/report/report.json @@ -128,7 +128,6 @@ "fieldtype": "Section Break" }, { - "depends_on": "eval:doc.is_standard == 'Yes'", "fieldname": "roles", "fieldtype": "Table", "label": "Roles", @@ -192,10 +191,11 @@ "idx": 1, "index_web_pages_for_search": 1, "links": [], - "modified": "2020-08-17 16:49:28.474274", + "modified": "2022-09-15 13:37:24.531848", "modified_by": "Administrator", "module": "Core", "name": "Report", + "naming_rule": "By fieldname", "owner": "Administrator", "permissions": [ { @@ -242,5 +242,6 @@ "show_name_in_global_search": 1, "sort_field": "modified", "sort_order": "DESC", + "states": [], "track_changes": 1 } \ No newline at end of file diff --git a/frappe/core/page/background_jobs/__init__.py b/frappe/core/doctype/rq_job/__init__.py similarity index 100% rename from frappe/core/page/background_jobs/__init__.py rename to frappe/core/doctype/rq_job/__init__.py diff --git a/frappe/core/doctype/rq_job/rq_job.js b/frappe/core/doctype/rq_job/rq_job.js new file mode 100644 index 0000000000..3f7a1a15b7 --- /dev/null +++ b/frappe/core/doctype/rq_job/rq_job.js @@ -0,0 +1,30 @@ +// Copyright (c) 2022, Frappe Technologies and contributors +// For license information, please see license.txt + +frappe.ui.form.on("RQ Job", { + refresh: function (frm) { + // Nothing in this form is supposed to be editable. + frm.disable_form(); + frm.dashboard.set_headline_alert( + "This is a virtual doctype and data is cleared periodically." + ); + + if (["started", "queued"].includes(frm.doc.status)) { + frm.add_custom_button(__("Force Stop job"), () => { + frappe.confirm( + "This will terminate the job immediately and might be dangerous, are you sure? ", + () => { + frappe + .xcall("frappe.core.doctype.rq_job.rq_job.stop_job", { + job_id: frm.doc.name, + }) + .then((r) => { + frappe.show_alert("Job Stopped Succefully"); + frm.reload_doc(); + }); + } + ); + }); + } + }, +}); diff --git a/frappe/core/doctype/rq_job/rq_job.json b/frappe/core/doctype/rq_job/rq_job.json new file mode 100644 index 0000000000..7cae15cf59 --- /dev/null +++ b/frappe/core/doctype/rq_job/rq_job.json @@ -0,0 +1,162 @@ +{ + "actions": [], + "allow_copy": 1, + "autoname": "field:job_id", + "creation": "2022-09-10 16:19:37.934903", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "job_info_section", + "job_id", + "job_name", + "queue", + "timeout", + "column_break_5", + "arguments", + "job_status_section", + "status", + "time_taken", + "column_break_11", + "started_at", + "ended_at", + "exception_section", + "exc_info" + ], + "fields": [ + { + "fieldname": "queue", + "fieldtype": "Select", + "in_list_view": 1, + "in_standard_filter": 1, + "label": "Queue", + "options": "default\nshort\nlong" + }, + { + "fieldname": "status", + "fieldtype": "Select", + "in_list_view": 1, + "in_standard_filter": 1, + "label": "Status", + "options": "queued\nstarted\nfinished\nfailed\ndeferred\nscheduled\ncanceled" + }, + { + "fieldname": "job_id", + "fieldtype": "Data", + "label": "Job ID", + "unique": 1 + }, + { + "fieldname": "exc_info", + "fieldtype": "Code", + "label": "Exception" + }, + { + "fieldname": "job_name", + "fieldtype": "Data", + "label": "Job Name" + }, + { + "fieldname": "arguments", + "fieldtype": "Code", + "label": "Arguments" + }, + { + "fieldname": "timeout", + "fieldtype": "Duration", + "label": "Timeout" + }, + { + "fieldname": "time_taken", + "fieldtype": "Duration", + "label": "Time Taken" + }, + { + "fieldname": "started_at", + "fieldtype": "Datetime", + "label": "Started At" + }, + { + "fieldname": "ended_at", + "fieldtype": "Datetime", + "label": "Ended At" + }, + { + "fieldname": "job_info_section", + "fieldtype": "Section Break", + "label": "Job Info" + }, + { + "fieldname": "job_status_section", + "fieldtype": "Section Break", + "label": "Job Status" + }, + { + "fieldname": "column_break_5", + "fieldtype": "Column Break" + }, + { + "fieldname": "column_break_11", + "fieldtype": "Column Break" + }, + { + "fieldname": "exception_section", + "fieldtype": "Section Break" + } + ], + "in_create": 1, + "is_virtual": 1, + "links": [], + "modified": "2022-09-11 05:27:50.878534", + "modified_by": "Administrator", + "module": "Core", + "name": "RQ Job", + "naming_rule": "By fieldname", + "owner": "Administrator", + "permissions": [ + { + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1 + }, + { + "delete": 1, + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "Administrator", + "share": 1 + } + ], + "sort_field": "modified", + "sort_order": "DESC", + "states": [ + { + "color": "Yellow", + "title": "queued" + }, + { + "color": "Blue", + "title": "started" + }, + { + "color": "Red", + "title": "failed" + }, + { + "color": "Green", + "title": "finished" + }, + { + "color": "Orange", + "title": "cancelled" + } + ], + "title_field": "job_name" +} \ No newline at end of file diff --git a/frappe/core/doctype/rq_job/rq_job.py b/frappe/core/doctype/rq_job/rq_job.py new file mode 100644 index 0000000000..7e1c35a0e6 --- /dev/null +++ b/frappe/core/doctype/rq_job/rq_job.py @@ -0,0 +1,193 @@ +# Copyright (c) 2022, Frappe Technologies and contributors +# For license information, please see license.txt + +import functools +import re + +from rq.command import send_stop_job_command +from rq.job import Job +from rq.queue import Queue + +import frappe +from frappe.model.document import Document +from frappe.utils import ( + cint, + compare, + convert_utc_to_user_timezone, + create_batch, + make_filter_dict, +) +from frappe.utils.background_jobs import get_queues, get_redis_conn + +QUEUES = ["default", "long", "short"] +JOB_STATUSES = ["queued", "started", "failed", "finished", "deferred", "scheduled", "canceled"] + + +def check_permissions(method): + @functools.wraps(method) + def wrapper(*args, **kwargs): + frappe.only_for("System Manager") + job = args[0].job + if not for_current_site(job): + raise frappe.PermissionError + + return method(*args, **kwargs) + + return wrapper + + +class RQJob(Document): + def load_from_db(self): + job = Job.fetch(self.name, connection=get_redis_conn()) + if not for_current_site(job): + raise frappe.PermissionError + super(Document, self).__init__(serialize_job(job)) + self._job_obj = job + + @property + def job(self): + return self._job_obj + + @staticmethod + def get_list(args): + + start = cint(args.get("start")) or 0 + page_length = cint(args.get("page_length")) or 20 + + order_desc = "desc" in args.get("order_by", "") + + matched_job_ids = RQJob.get_matching_job_ids(args) + + jobs = [] + for job_ids in create_batch(matched_job_ids, 100): + jobs.extend( + serialize_job(job) + for job in Job.fetch_many(job_ids=job_ids, connection=get_redis_conn()) + if job and for_current_site(job) + ) + if len(jobs) > start + page_length: + # we have fetched enough. This is inefficient but because of site filtering TINA + break + + return sorted(jobs, key=lambda j: j.modified, reverse=order_desc)[start : start + page_length] + + @staticmethod + def get_matching_job_ids(args): + filters = make_filter_dict(args.get("filters")) + + queues = _eval_filters(filters.get("queue"), QUEUES) + statuses = _eval_filters(filters.get("status"), JOB_STATUSES) + + matched_job_ids = [] + for queue in get_queues(): + if not queue.name.endswith(tuple(queues)): + continue + for status in statuses: + matched_job_ids.extend(fetch_job_ids(queue, status)) + + return matched_job_ids + + @check_permissions + def delete(self): + self.job.delete() + + @check_permissions + def stop_job(self): + send_stop_job_command(connection=get_redis_conn(), job_id=self.job_id) + + @staticmethod + def get_count(args) -> int: + # Can not be implemented efficiently due to site filtering hence ignored. + return 0 + + # None of these methods apply to virtual job doctype, overriden for sanity. + @staticmethod + def get_stats(args): + return {} + + def db_insert(self, *args, **kwargs): + pass + + def db_update(self, *args, **kwargs): + pass + + +def serialize_job(job: Job) -> frappe._dict: + modified = job.last_heartbeat or job.ended_at or job.started_at or job.created_at + job_name = job.kwargs.get("kwargs", {}).get("job_type") or str(job.kwargs.get("job_name")) + + # function objects have this repr: '' + # This regex just removes unnecessary things around it. + if matches := re.match(r".*) at 0x.*>", job_name): + job_name = matches.group("func_name") + + return frappe._dict( + name=job.id, + job_id=job.id, + queue=job.origin.rsplit(":", 1)[1], + job_name=job_name, + status=job.get_status(), + started_at=convert_utc_to_user_timezone(job.started_at) if job.started_at else "", + ended_at=convert_utc_to_user_timezone(job.ended_at) if job.ended_at else "", + time_taken=(job.ended_at - job.started_at).total_seconds() if job.ended_at else "", + exc_info=job.exc_info, + arguments=frappe.as_json(job.kwargs), + timeout=job.timeout, + creation=convert_utc_to_user_timezone(job.created_at), + modified=convert_utc_to_user_timezone(modified), + _comment_count=0, + ) + + +def for_current_site(job: Job) -> bool: + return job.kwargs.get("site") == frappe.local.site + + +def _eval_filters(filter, values: list[str]) -> list[str]: + if filter: + operator, operand = filter + return [val for val in values if compare(val, operator, operand)] + return values + + +def fetch_job_ids(queue: Queue, status: str) -> list[str]: + registry_map = { + "queued": queue, # self + "started": queue.started_job_registry, + "finished": queue.finished_job_registry, + "failed": queue.failed_job_registry, + "deferred": queue.deferred_job_registry, + "scheduled": queue.scheduled_job_registry, + "canceled": queue.canceled_job_registry, + } + + registry = registry_map.get(status) + if registry is not None: + job_ids = registry.get_job_ids() + return [j for j in job_ids if j] + + return [] + + +@frappe.whitelist() +def remove_failed_jobs(): + frappe.only_for("System Manager") + for queue in get_queues(): + fail_registry = queue.failed_job_registry + for job_ids in create_batch(fail_registry.get_job_ids(), 100): + for job in Job.fetch_many(job_ids=job_ids, connection=get_redis_conn()): + if job and for_current_site(job): + fail_registry.remove(job, delete_job=True) + + +def get_all_queued_jobs(): + jobs = [] + for q in get_queues(): + jobs.extend(q.get_jobs()) + + return [job for job in jobs if for_current_site(job)] + + +@frappe.whitelist() +def stop_job(job_id): + frappe.get_doc("RQ Job", job_id).stop_job() diff --git a/frappe/core/doctype/rq_job/rq_job_list.js b/frappe/core/doctype/rq_job/rq_job_list.js new file mode 100644 index 0000000000..5f6646cd65 --- /dev/null +++ b/frappe/core/doctype/rq_job/rq_job_list.js @@ -0,0 +1,32 @@ +frappe.listview_settings["RQ Job"] = { + hide_name_column: true, + + onload(listview) { + if (!has_common(frappe.user_roles, ["Administrator", "System Manager"])) return; + + listview.page.add_inner_button(__("Remove Failed Jobs"), () => { + frappe.confirm(__("Are you sure you want to remove all failed jobs?"), () => { + frappe.xcall("frappe.core.doctype.rq_job.rq_job.remove_failed_jobs"); + }); + }); + + if (listview.list_view_settings) { + listview.list_view_settings.disable_count = 1; + listview.list_view_settings.disable_sidebar_stats = 1; + } + + frappe.xcall("frappe.utils.scheduler.get_scheduler_status").then(({ status }) => { + if (status === "active") { + listview.page.set_indicator(__("Scheduler: Active"), "green"); + } else { + listview.page.set_indicator(__("Scheduler: Inactive"), "red"); + } + }); + + setInterval(() => { + if (!listview.list_view_settings.disable_auto_refresh) { + listview.refresh(); + } + }, 5000); + }, +}; diff --git a/frappe/core/doctype/rq_job/test_rq_job.py b/frappe/core/doctype/rq_job/test_rq_job.py new file mode 100644 index 0000000000..ae0691fa61 --- /dev/null +++ b/frappe/core/doctype/rq_job/test_rq_job.py @@ -0,0 +1,103 @@ +# Copyright (c) 2022, Frappe Technologies and Contributors + +# See license.txt + +import time + +from rq import exceptions as rq_exc +from rq.job import Job + +import frappe +from frappe.core.doctype.rq_job.rq_job import RQJob, remove_failed_jobs, stop_job +from frappe.tests.utils import FrappeTestCase, timeout +from frappe.utils.background_jobs import is_job_queued + + +class TestRQJob(FrappeTestCase): + + BG_JOB = "frappe.core.doctype.rq_job.test_rq_job.test_func" + + @timeout(seconds=20) + def check_status(self, job: Job, status, wait=True): + if wait: + while True: + if job.is_queued or job.is_started: + time.sleep(0.2) + else: + break + self.assertEqual(frappe.get_doc("RQ Job", job.id).status, status) + + def test_serialization(self): + + job = frappe.enqueue(method=self.BG_JOB, queue="short") + rq_job = frappe.get_doc("RQ Job", job.id) + + self.assertEqual(job, rq_job.job) + self.assertDocumentEqual( + { + "name": job.id, + "queue": "short", + "job_name": self.BG_JOB, + "status": "queued", + "exc_info": None, + }, + rq_job, + ) + self.check_status(job, "finished") + + def test_func_obj_serialization(self): + job = frappe.enqueue(method=test_func, queue="short") + rq_job = frappe.get_doc("RQ Job", job.id) + self.assertEqual(rq_job.job_name, "test_func") + + def test_get_list_filtering(self): + + # Check failed job clearning and filtering + remove_failed_jobs() + jobs = RQJob.get_list({"filters": [["RQ Job", "status", "=", "failed"]]}) + self.assertEqual(jobs, []) + + # Fail a job + job = frappe.enqueue(method=self.BG_JOB, queue="short", fail=True) + self.check_status(job, "failed") + jobs = RQJob.get_list({"filters": [["RQ Job", "status", "=", "failed"]]}) + self.assertEqual(len(jobs), 1) + self.assertTrue(jobs[0].exc_info) + + # Assert that non-failed job still exists + non_failed_jobs = RQJob.get_list({"filters": [["RQ Job", "status", "!=", "failed"]]}) + self.assertGreaterEqual(len(non_failed_jobs), 1) + + # Create a slow job and check if it's stuck in "Started" + job = frappe.enqueue(method=self.BG_JOB, queue="short", sleep=1000) + time.sleep(3) + self.check_status(job, "started", wait=False) + stop_job(job_id=job.id) + self.check_status(job, "stopped") + + def test_delete_doc(self): + job = frappe.enqueue(method=self.BG_JOB, queue="short") + frappe.get_doc("RQ Job", job.id).delete() + + with self.assertRaises(rq_exc.NoSuchJobError): + job.refresh() + + def test_is_enqueued(self): + + job_name = "uniq_test_job" + dummy_job = frappe.enqueue(self.BG_JOB, sleep=100, queue="short") + actual_job = frappe.enqueue(self.BG_JOB, job_name=job_name, queue="short") + + self.assertTrue(is_job_queued(job_name)) + stop_job(dummy_job.id) + self.check_status(actual_job, "finished") + self.assertFalse(is_job_queued(job_name)) + + +def test_func(fail=False, sleep=0): + if fail: + 42 / 0 + if sleep: + time.sleep(sleep) + + return True diff --git a/frappe/event_streaming/__init__.py b/frappe/core/doctype/rq_worker/__init__.py similarity index 100% rename from frappe/event_streaming/__init__.py rename to frappe/core/doctype/rq_worker/__init__.py diff --git a/frappe/core/doctype/rq_worker/rq_worker.js b/frappe/core/doctype/rq_worker/rq_worker.js new file mode 100644 index 0000000000..622cb30cb9 --- /dev/null +++ b/frappe/core/doctype/rq_worker/rq_worker.js @@ -0,0 +1,9 @@ +// Copyright (c) 2022, Frappe Technologies and contributors +// For license information, please see license.txt + +frappe.ui.form.on("RQ Worker", { + refresh: function (frm) { + // Nothing in this form is supposed to be editable. + frm.disable_form(); + }, +}); diff --git a/frappe/core/doctype/rq_worker/rq_worker.json b/frappe/core/doctype/rq_worker/rq_worker.json new file mode 100644 index 0000000000..ea65abd482 --- /dev/null +++ b/frappe/core/doctype/rq_worker/rq_worker.json @@ -0,0 +1,138 @@ +{ + "actions": [], + "allow_copy": 1, + "creation": "2022-09-10 14:54:57.342170", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "worker_information_section", + "queue", + "queue_type", + "column_break_4", + "worker_name", + "statistics_section", + "status", + "pid", + "current_job_id", + "successful_job_count", + "failed_job_count", + "column_break_12", + "birth_date", + "last_heartbeat", + "total_working_time" + ], + "fields": [ + { + "fieldname": "worker_name", + "fieldtype": "Data", + "label": "Worker Name", + "unique": 1 + }, + { + "fieldname": "status", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Status" + }, + { + "fieldname": "current_job_id", + "fieldtype": "Link", + "label": "Current Job ID", + "options": "RQ Job" + }, + { + "fieldname": "pid", + "fieldtype": "Data", + "label": "PID" + }, + { + "fieldname": "last_heartbeat", + "fieldtype": "Datetime", + "label": "Last Heartbeat" + }, + { + "fieldname": "birth_date", + "fieldtype": "Datetime", + "label": "Start Time" + }, + { + "fieldname": "successful_job_count", + "fieldtype": "Int", + "in_list_view": 1, + "label": "Successful Job Count" + }, + { + "fieldname": "failed_job_count", + "fieldtype": "Int", + "in_list_view": 1, + "label": "Failed Job Count" + }, + { + "fieldname": "total_working_time", + "fieldtype": "Duration", + "label": "Total Working Time" + }, + { + "fieldname": "queue", + "fieldtype": "Data", + "label": "Queue" + }, + { + "fieldname": "queue_type", + "fieldtype": "Select", + "in_list_view": 1, + "label": "Queue Type", + "options": "default\nlong\nshort" + }, + { + "fieldname": "worker_information_section", + "fieldtype": "Section Break", + "label": "Worker Information" + }, + { + "fieldname": "statistics_section", + "fieldtype": "Section Break", + "label": "Statistics" + }, + { + "fieldname": "column_break_4", + "fieldtype": "Column Break" + }, + { + "fieldname": "column_break_12", + "fieldtype": "Column Break" + } + ], + "in_create": 1, + "is_virtual": 1, + "links": [], + "modified": "2022-09-11 05:02:53.981705", + "modified_by": "Administrator", + "module": "Core", + "name": "RQ Worker", + "owner": "Administrator", + "permissions": [ + { + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1 + } + ], + "sort_field": "modified", + "sort_order": "DESC", + "states": [ + { + "color": "Blue", + "title": "idle" + }, + { + "color": "Yellow", + "title": "busy" + } + ] +} \ No newline at end of file diff --git a/frappe/core/doctype/rq_worker/rq_worker.py b/frappe/core/doctype/rq_worker/rq_worker.py new file mode 100644 index 0000000000..bc1326f522 --- /dev/null +++ b/frappe/core/doctype/rq_worker/rq_worker.py @@ -0,0 +1,67 @@ +# Copyright (c) 2022, Frappe Technologies and contributors +# For license information, please see license.txt + +from rq import Worker + +import frappe +from frappe.model.document import Document +from frappe.utils import cint, convert_utc_to_user_timezone +from frappe.utils.background_jobs import get_workers + + +class RQWorker(Document): + def load_from_db(self): + + all_workers = get_workers() + worker = [w for w in all_workers if w.pid == cint(self.name)][0] + d = serialize_worker(worker) + + super(Document, self).__init__(d) + + @staticmethod + def get_list(args): + start = cint(args.get("start")) or 0 + page_length = cint(args.get("page_length")) or 20 + + workers = get_workers()[start : start + page_length] + return [serialize_worker(worker) for worker in workers] + + @staticmethod + def get_count(args) -> int: + return len(get_workers()) + + # None of these methods apply to virtual workers, overriden for sanity. + @staticmethod + def get_stats(args): + return {} + + def db_insert(self, *args, **kwargs): + pass + + def db_update(self, *args, **kwargs): + pass + + def delete(self): + pass + + +def serialize_worker(worker: Worker) -> frappe._dict: + queue = ", ".join(worker.queue_names()) + + return frappe._dict( + name=worker.pid, + queue=queue, + queue_type=queue.rsplit(":", 1)[1], + worker_name=worker.name, + status=worker.get_state(), + pid=worker.pid, + current_job_id=worker.get_current_job_id(), + last_heartbeat=convert_utc_to_user_timezone(worker.last_heartbeat), + birth_date=convert_utc_to_user_timezone(worker.birth_date), + successful_job_count=worker.successful_job_count, + failed_job_count=worker.failed_job_count, + total_working_time=worker.total_working_time, + _comment_count=0, + modified=convert_utc_to_user_timezone(worker.last_heartbeat), + creation=convert_utc_to_user_timezone(worker.birth_date), + ) diff --git a/frappe/core/doctype/rq_worker/test_rq_worker.py b/frappe/core/doctype/rq_worker/test_rq_worker.py new file mode 100644 index 0000000000..5a43270681 --- /dev/null +++ b/frappe/core/doctype/rq_worker/test_rq_worker.py @@ -0,0 +1,17 @@ +# Copyright (c) 2022, Frappe Technologies and Contributors +# See license.txt + +import frappe +from frappe.core.doctype.rq_worker.rq_worker import RQWorker +from frappe.tests.utils import FrappeTestCase + + +class TestRQWorker(FrappeTestCase): + def test_get_worker_list(self): + workers = RQWorker.get_list({}) + self.assertGreaterEqual(len(workers), 1) + self.assertTrue(any(w.queue_type == "short" for w in workers)) + + def test_worker_serialization(self): + workers = RQWorker.get_list({}) + frappe.get_doc("RQ Worker", workers[0].pid) diff --git a/frappe/core/doctype/user/user.json b/frappe/core/doctype/user/user.json index c0d306f70f..64982c5707 100644 --- a/frappe/core/doctype/user/user.json +++ b/frappe/core/doctype/user/user.json @@ -131,7 +131,7 @@ { "fieldname": "middle_name", "fieldtype": "Data", - "label": "Middle Name (Optional)", + "label": "Middle Name", "oldfieldname": "middle_name", "oldfieldtype": "Data" }, @@ -496,7 +496,7 @@ { "description": "Restrict user from this IP address only. Multiple IP addresses can be added by separating with commas. Also accepts partial IP addresses like (111.111.111)", "fieldname": "restrict_ip", - "fieldtype": "Data", + "fieldtype": "Small Text", "label": "Restrict IP", "permlevel": 1 }, @@ -753,7 +753,7 @@ "link_fieldname": "user" } ], - "modified": "2022-08-11 14:47:04.100892", + "modified": "2022-09-19 16:05:46.485242", "modified_by": "Administrator", "module": "Core", "name": "User", diff --git a/frappe/core/doctype/view_log/view_log.json b/frappe/core/doctype/view_log/view_log.json index 6b19cdd507..a350ae835c 100644 --- a/frappe/core/doctype/view_log/view_log.json +++ b/frappe/core/doctype/view_log/view_log.json @@ -14,7 +14,6 @@ "fieldname": "viewed_by", "fieldtype": "Data", "label": "Viewed By", - "search_index": 1, "set_only_once": 1 }, { @@ -22,7 +21,6 @@ "fieldtype": "Link", "label": "Reference Document Type", "options": "DocType", - "search_index": 1, "set_only_once": 1 }, { @@ -35,7 +33,7 @@ } ], "links": [], - "modified": "2022-08-03 12:20:52.857103", + "modified": "2022-09-07 05:16:14.587628", "modified_by": "Administrator", "module": "Core", "name": "View Log", diff --git a/frappe/core/page/background_jobs/background_jobs.css b/frappe/core/page/background_jobs/background_jobs.css deleted file mode 100644 index 7716519113..0000000000 --- a/frappe/core/page/background_jobs/background_jobs.css +++ /dev/null @@ -1,47 +0,0 @@ - -.table-background-jobs { - margin-bottom: 0px; - margin-top: 0px; - font-size: var(--text-md); - table-layout: fixed; -} - -.table-background-jobs th { - font-weight: normal; - color: var(--text-muted); -} - -.table-background-jobs td { - color: var(--text-light); -} - -.table-background-jobs th, .table-background-jobs td { - padding: var(--padding-sm) var(--padding-md); -} - -.table-background-jobs tbody tr:hover { - background-color: var(--highlight-color); -} - -.job-name { - font-size: var(--text-md); - font-family: var(--font-family-monospace); - word-break: break-word; -} - -.no-background-jobs { - min-height: 320px; - display: flex; - align-items: center; - justify-content: center; - flex-direction: column; -} - -.no-background-jobs > img { - margin-bottom: var(--margin-md); - max-height: 100px; -} - -.footer { - padding: var(--padding-md); -} diff --git a/frappe/core/page/background_jobs/background_jobs.html b/frappe/core/page/background_jobs/background_jobs.html deleted file mode 100644 index e0c1a8f633..0000000000 --- a/frappe/core/page/background_jobs/background_jobs.html +++ /dev/null @@ -1,58 +0,0 @@ -{% if jobs.length %} - - - - - - - - - - - {% for j in jobs %} - - - - - - - {% endfor %} - -
{{ __("Queue") }}{{ __("Job") }}{{ __("Status") }}{{ __("Created") }}
- {{ toTitle(j.queue.split(":").slice(-1)[0]) }} - -
- - {{ frappe.utils.encode_tags(j.job_name) }} - -
- {% if j.exc_info %} -
- {{ __("Exception") }} -
-
{{ frappe.utils.encode_tags(j.exc_info) }}
-
-
- {% endif %} -
- - {{ toTitle(j.status) }} - - - {{ frappe.datetime.prettyDate(j.creation) }} -
-{% else %} -
- Empty State -

{{ __("No jobs found on this site") }}

-
-{% endif %} - diff --git a/frappe/core/page/background_jobs/background_jobs.js b/frappe/core/page/background_jobs/background_jobs.js deleted file mode 100644 index 94c7bbf3bc..0000000000 --- a/frappe/core/page/background_jobs/background_jobs.js +++ /dev/null @@ -1,136 +0,0 @@ -frappe.pages["background_jobs"].on_page_load = (wrapper) => { - const background_job = new BackgroundJobs(wrapper); - - $(wrapper).bind("show", () => { - background_job.show(); - }); - - window.background_jobs = background_job; -}; - -class BackgroundJobs { - constructor(wrapper) { - this.page = frappe.ui.make_app_page({ - parent: wrapper, - title: __("Background Jobs"), - single_column: true, - }); - - this.page.add_inner_button(__("Remove Failed Jobs"), () => { - frappe.confirm(__("Are you sure you want to remove all failed jobs?"), () => { - frappe - .call("frappe.core.page.background_jobs.background_jobs.remove_failed_jobs") - .then(() => this.refresh_jobs()); - }); - }); - - this.page.main.addClass("frappe-card"); - this.page.body.append('
'); - this.$content = $(this.page.body).find(".table-area"); - - this.make_filters(); - this.refresh_jobs = frappe.utils.throttle(this.refresh_jobs.bind(this), 1000); - } - - make_filters() { - this.view = this.page.add_field({ - label: __("View"), - fieldname: "view", - fieldtype: "Select", - options: ["Jobs", "Workers"], - default: "Jobs", - change: () => { - this.queue_timeout.toggle(this.view.get_value() === "Jobs"); - this.job_status.toggle(this.view.get_value() === "Jobs"); - }, - }); - this.queue_timeout = this.page.add_field({ - label: __("Queue"), - fieldname: "queue_timeout", - fieldtype: "Select", - options: [ - { label: "All Queues", value: "all" }, - { label: "Default", value: "default" }, - { label: "Short", value: "short" }, - { label: "Long", value: "long" }, - ], - default: "all", - }); - this.job_status = this.page.add_field({ - label: __("Job Status"), - fieldname: "job_status", - fieldtype: "Select", - options: [ - { label: "All Jobs", value: "all" }, - { label: "Queued", value: "queued" }, - { label: "Deferred", value: "deferred" }, - { label: "Started", value: "started" }, - { label: "Finished", value: "finished" }, - { label: "Failed", value: "failed" }, - ], - default: "all", - }); - this.auto_refresh = this.page.add_field({ - label: __("Auto Refresh"), - fieldname: "auto_refresh", - fieldtype: "Check", - default: 1, - change: () => { - if (this.auto_refresh.get_value()) { - this.refresh_jobs(); - } - }, - }); - } - - show() { - this.refresh_jobs(); - this.update_scheduler_status(); - } - - update_scheduler_status() { - frappe.call({ - method: "frappe.core.page.background_jobs.background_jobs.get_scheduler_status", - callback: (r) => { - let { status } = r.message; - if (status === "active") { - this.page.set_indicator(__("Scheduler: Active"), "green"); - } else { - this.page.set_indicator(__("Scheduler: Inactive"), "red"); - } - }, - }); - } - - refresh_jobs() { - let view = this.view.get_value(); - let args; - let { queue_timeout, job_status } = this.page.get_form_values(); - if (view === "Jobs") { - args = { view, queue_timeout, job_status }; - } else { - args = { view }; - } - - this.page.add_inner_message(__("Refreshing...")); - frappe.call({ - method: "frappe.core.page.background_jobs.background_jobs.get_info", - args, - callback: (res) => { - this.page.add_inner_message(""); - - let template = view === "Jobs" ? "background_jobs" : "background_workers"; - this.$content.html( - frappe.render_template(template, { - jobs: res.message || [], - }) - ); - - let auto_refresh = this.auto_refresh.get_value(); - if (frappe.get_route()[0] === "background_jobs" && auto_refresh) { - setTimeout(() => this.refresh_jobs(), 2000); - } - }, - }); - } -} diff --git a/frappe/core/page/background_jobs/background_jobs.json b/frappe/core/page/background_jobs/background_jobs.json deleted file mode 100644 index 6701cc54bc..0000000000 --- a/frappe/core/page/background_jobs/background_jobs.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "content": null, - "creation": "2016-08-18 16:44:14.322642", - "docstatus": 0, - "doctype": "Page", - "idx": 0, - "modified": "2016-08-18 16:48:11.577611", - "modified_by": "Administrator", - "module": "Core", - "name": "background_jobs", - "owner": "Administrator", - "page_name": "background_jobs", - "roles": [ - { - "role": "System Manager" - } - ], - "script": null, - "standard": "Yes", - "style": null, - "title": "Background Jobs" -} \ No newline at end of file diff --git a/frappe/core/page/background_jobs/background_jobs.py b/frappe/core/page/background_jobs/background_jobs.py deleted file mode 100644 index 8ef15b65eb..0000000000 --- a/frappe/core/page/background_jobs/background_jobs.py +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors -# License: MIT. See LICENSE - -from typing import TYPE_CHECKING - -import frappe -from frappe.utils import convert_utc_to_user_timezone -from frappe.utils.background_jobs import get_queues, get_workers -from frappe.utils.scheduler import is_scheduler_inactive - -if TYPE_CHECKING: - from rq.job import Job - -JOB_COLORS = {"queued": "orange", "failed": "red", "started": "blue", "finished": "green"} - - -@frappe.whitelist() -def get_info(view=None, queue_timeout=None, job_status=None) -> list[dict]: - jobs = [] - - def add_job(job: "Job", queue: str) -> None: - - if job.kwargs.get("site") == frappe.local.site: - job_info = { - "job_name": job.kwargs.get("kwargs", {}).get("playbook_method") - or job.kwargs.get("kwargs", {}).get("job_type") - or str(job.kwargs.get("job_name")), - "status": job.get_status(), - "queue": queue, - "creation": convert_utc_to_user_timezone(job.created_at), - "color": JOB_COLORS[job.get_status()], - } - - if job.exc_info: - job_info["exc_info"] = job.exc_info - - jobs.append(job_info) - - if view == "Jobs": - queues = get_queues() - for queue in queues: - for job in queue.jobs: - if job_status != "all" and job.get_status() != job_status: - return - if queue_timeout != "all" and not queue.name.endswith(f":{queue_timeout}"): - return - add_job(job, queue.name) - - elif view == "Workers": - workers = get_workers() - for worker in workers: - current_job = worker.get_current_job() - if current_job: - if hasattr(current_job, "kwargs") and current_job.kwargs.get("site") == frappe.local.site: - add_job(current_job, current_job.origin) - else: - jobs.append({"queue": worker.name, "job_name": "busy", "status": "", "creation": ""}) - else: - jobs.append({"queue": worker.name, "job_name": "idle", "status": "", "creation": ""}) - - return jobs - - -@frappe.whitelist() -def remove_failed_jobs(): - queues = get_queues() - for queue in queues: - fail_registry = queue.failed_job_registry - for job_id in fail_registry.get_job_ids(): - job = queue.fetch_job(job_id) - fail_registry.remove(job, delete_job=True) - - -@frappe.whitelist() -def get_scheduler_status(): - if is_scheduler_inactive(): - return {"status": "inactive"} - return {"status": "active"} diff --git a/frappe/core/page/background_jobs/background_workers.html b/frappe/core/page/background_jobs/background_workers.html deleted file mode 100644 index 1647cea4b4..0000000000 --- a/frappe/core/page/background_jobs/background_workers.html +++ /dev/null @@ -1,51 +0,0 @@ -{% if jobs.length %} - - - - - - - - - - - {% for j in jobs %} - - - - - - - {% endfor %} - -
{{ __("Worker") }}{{ __("Current Job") }}{{ __("Status") }}{{ __("Created") }}
- {{ j.queue }} - -
- - {{ frappe.utils.encode_tags(j.job_name) }} - -
- {% if j.exc_info %} -
- {{ __("Exception") }} -
-
{{ frappe.utils.encode_tags(j.exc_info) }}
-
-
- {% endif %} -
- {{ toTitle(j.status) }} - {{ frappe.datetime.prettyDate(j.creation) }}
-{% else %} -
- Empty State -

{{ __("No workers online on this site") }}

-
-{% endif %} - \ No newline at end of file diff --git a/frappe/core/workspace/build/build.json b/frappe/core/workspace/build/build.json index 9282c50e67..67dfae650f 100644 --- a/frappe/core/workspace/build/build.json +++ b/frappe/core/workspace/build/build.json @@ -217,8 +217,8 @@ "is_query_report": 0, "label": "Background Jobs", "link_count": 0, - "link_to": "background_jobs", - "link_type": "Page", + "link_to": "RQ Job", + "link_type": "DocType", "onboard": 0, "type": "Link" }, @@ -273,7 +273,7 @@ "type": "Link" } ], - "modified": "2022-09-02 01:48:28.029135", + "modified": "2022-09-11 06:41:31.095300", "modified_by": "Administrator", "module": "Core", "name": "Build", diff --git a/frappe/database/__init__.py b/frappe/database/__init__.py index d1b7729fee..76ad24b6e6 100644 --- a/frappe/database/__init__.py +++ b/frappe/database/__init__.py @@ -39,18 +39,14 @@ def drop_user_and_database(db_name, root_login=None, root_password=None): ) -def get_db(host=None, user=None, password=None, port=None, read_only=False): +def get_db(host=None, user=None, password=None, port=None): import frappe if frappe.conf.db_type == "postgres": import frappe.database.postgres.database - return frappe.database.postgres.database.PostgresDatabase( - host, user, password, port=port, read_only=read_only - ) + return frappe.database.postgres.database.PostgresDatabase(host, user, password, port=port) else: import frappe.database.mariadb.database - return frappe.database.mariadb.database.MariaDBDatabase( - host, user, password, port=port, read_only=read_only - ) + return frappe.database.mariadb.database.MariaDBDatabase(host, user, password, port=port) diff --git a/frappe/database/database.py b/frappe/database/database.py index 48b756b60e..c29861ab46 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -83,14 +83,12 @@ class Database: ac_name=None, use_default=0, port=None, - read_only=False, ): self.setup_type_map() self.host = host or frappe.conf.db_host or "127.0.0.1" self.port = port or frappe.conf.db_port or "" self.user = user or frappe.conf.db_name self.db_name = frappe.conf.db_name - self.read_only = read_only # Uses READ ONLY connection if set self._conn = None if ac_name: @@ -221,6 +219,15 @@ class Database: elif self.is_timedout(e): raise frappe.QueryTimeoutError(e) from e + elif self.is_read_only_mode_error(e): + frappe.throw( + _( + "Site is running in read only mode, this action can not be performed right now. Please try again later." + ), + title=_("In Read Only Mode"), + exc=frappe.InReadOnlyMode, + ) + # TODO: added temporarily elif self.db_type == "postgres": traceback.print_stack() @@ -956,8 +963,10 @@ class Database: return defaults.get(frappe.scrub(key)) - def begin(self): - self.sql("START TRANSACTION") + def begin(self, *, read_only=False): + read_only = read_only or frappe.flags.read_only + mode = "READ ONLY" if read_only else "" + self.sql(f"START TRANSACTION {mode}") def commit(self): """Commit current transaction. Calls SQL `COMMIT`.""" @@ -965,9 +974,7 @@ class Database: frappe.call(method[0], *(method[1] or []), **(method[2] or {})) self.sql("commit") - if self.db_type == "postgres": - # Postgres requires explicitly starting new transaction - self.begin() + self.begin() # explicitly start a new transaction frappe.local.rollback_observers = [] self.flush_realtime_log() @@ -1298,12 +1305,23 @@ class Database: def enqueue_jobs_after_commit(): - from frappe.utils.background_jobs import execute_job, get_queue + from frappe.utils.background_jobs import ( + RQ_JOB_FAILURE_TTL, + RQ_RESULTS_TTL, + execute_job, + get_queue, + ) if frappe.flags.enqueue_after_commit and len(frappe.flags.enqueue_after_commit) > 0: for job in frappe.flags.enqueue_after_commit: q = get_queue(job.get("queue"), is_async=job.get("is_async")) - q.enqueue_call(execute_job, timeout=job.get("timeout"), kwargs=job.get("queue_args")) + q.enqueue_call( + execute_job, + timeout=job.get("timeout"), + kwargs=job.get("queue_args"), + failure_ttl=RQ_JOB_FAILURE_TTL, + result_ttl=RQ_RESULTS_TTL, + ) frappe.flags.enqueue_after_commit = [] diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index bad00d9723..3fc241454e 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -32,6 +32,10 @@ class MariaDBExceptionUtil: def is_timedout(e: pymysql.Error) -> bool: return e.args[0] == ER.LOCK_WAIT_TIMEOUT + @staticmethod + def is_read_only_mode_error(e: pymysql.Error) -> bool: + return e.args[0] == 1792 + @staticmethod def is_table_missing(e: pymysql.Error) -> bool: return e.args[0] == ER.NO_SUCH_TABLE diff --git a/frappe/database/mariadb/schema.py b/frappe/database/mariadb/schema.py index 99297fbab2..8b1235d82e 100644 --- a/frappe/database/mariadb/schema.py +++ b/frappe/database/mariadb/schema.py @@ -6,7 +6,7 @@ from frappe.model import log_types class MariaDBTable(DBTable): def create(self): - additional_definitions = "" + additional_definitions = [] engine = self.meta.get("engine") or "InnoDB" varchar_len = frappe.db.VARCHAR_LEN name_column = f"name varchar({varchar_len}) primary key" @@ -14,26 +14,24 @@ class MariaDBTable(DBTable): # columns column_defs = self.get_column_definitions() if column_defs: - additional_definitions += ",\n".join(column_defs) + ",\n" + additional_definitions += column_defs # index index_defs = self.get_index_definitions() if index_defs: - additional_definitions += ",\n".join(index_defs) + ",\n" + additional_definitions += index_defs # child table columns if self.meta.get("istable") or 0: - additional_definitions += ( - ",\n".join( - ( - f"parent varchar({varchar_len})", - f"parentfield varchar({varchar_len})", - f"parenttype varchar({varchar_len})", - "index parent(parent)", - ) - ) - + ",\n" - ) + additional_definitions += [ + f"parent varchar({varchar_len})", + f"parentfield varchar({varchar_len})", + f"parenttype varchar({varchar_len})", + "index parent(parent)", + ] + else: + # parent types + additional_definitions.append("index modified(modified)") # creating sequence(s) if ( @@ -47,6 +45,8 @@ class MariaDBTable(DBTable): # issue link: https://jira.mariadb.org/browse/MDEV-20070 name_column = "name bigint primary key" + additional_definitions = ",\n".join(additional_definitions) + # create table query = f"""create table `{self.table_name}` ( {name_column}, @@ -56,8 +56,7 @@ class MariaDBTable(DBTable): owner varchar({varchar_len}), docstatus int(1) not null default '0', idx int(8) not null default '0', - {additional_definitions} - index modified(modified)) + {additional_definitions}) ENGINE={engine} ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 diff --git a/frappe/database/postgres/database.py b/frappe/database/postgres/database.py index cb566736ad..3b3612c0e4 100644 --- a/frappe/database/postgres/database.py +++ b/frappe/database/postgres/database.py @@ -12,7 +12,7 @@ from psycopg2.errorcodes import ( UNDEFINED_TABLE, UNIQUE_VIOLATION, ) -from psycopg2.errors import SequenceGeneratorLimitExceeded, SyntaxError +from psycopg2.errors import ReadOnlySqlTransaction, SequenceGeneratorLimitExceeded, SyntaxError from psycopg2.extensions import ISOLATION_LEVEL_REPEATABLE_READ import frappe @@ -55,6 +55,10 @@ class PostgresExceptionUtil: # http://initd.org/psycopg/docs/extensions.html?highlight=datatype#psycopg2.extensions.QueryCanceledError return isinstance(e, psycopg2.extensions.QueryCanceledError) + @staticmethod + def is_read_only_mode_error(e) -> bool: + return isinstance(e, ReadOnlySqlTransaction) + @staticmethod def is_syntax_error(e): return isinstance(e, SyntaxError) diff --git a/frappe/desk/desktop.py b/frappe/desk/desktop.py index 70fd3235fe..dfe7850349 100644 --- a/frappe/desk/desktop.py +++ b/frappe/desk/desktop.py @@ -291,12 +291,13 @@ class Workspace: quick_lists = self.doc.quick_lists for item in quick_lists: - new_item = item.as_dict().copy() + if self.is_item_allowed(item.document_type, "doctype"): + new_item = item.as_dict().copy() - # Translate label - new_item["label"] = _(item.label) if item.label else _(item.document_type) + # Translate label + new_item["label"] = _(item.label) if item.label else _(item.document_type) - items.append(new_item) + items.append(new_item) return items diff --git a/frappe/desk/doctype/dashboard_chart/dashboard_chart.js b/frappe/desk/doctype/dashboard_chart/dashboard_chart.js index b1c23eba28..6d23be79d7 100644 --- a/frappe/desk/doctype/dashboard_chart/dashboard_chart.js +++ b/frappe/desk/doctype/dashboard_chart/dashboard_chart.js @@ -532,25 +532,21 @@ frappe.ui.form.on("Dashboard Chart", { frm.set_df_property("parent_document_type", "hidden", !doc_is_table); if (document_type && doc_is_table) { - let parent = await frappe.db.get_list("DocField", { - filters: { - fieldtype: "Table", - options: document_type, - }, - fields: ["parent"], + let parents = await frappe.xcall( + "frappe.desk.doctype.dashboard_chart.dashboard_chart.get_parent_doctypes", + { child_type: document_type } + ); + + frm.set_query("parent_document_type", function () { + return { + filters: { + name: ["in", parents], + }, + }; }); - parent && - frm.set_query("parent_document_type", function () { - return { - filters: { - name: ["in", parent.map(({ parent }) => parent)], - }, - }; - }); - - if (parent.length === 1) { - frm.set_value("parent_document_type", parent[0].parent); + if (parents.length === 1) { + frm.set_value("parent_document_type", parents[0]); } } }, diff --git a/frappe/desk/doctype/dashboard_chart/dashboard_chart.py b/frappe/desk/doctype/dashboard_chart/dashboard_chart.py index fbf542f855..a85f269c8f 100644 --- a/frappe/desk/doctype/dashboard_chart/dashboard_chart.py +++ b/frappe/desk/doctype/dashboard_chart/dashboard_chart.py @@ -392,3 +392,25 @@ class DashboardChart(Document): json.loads(self.custom_options) except ValueError as error: frappe.throw(_("Invalid json added in the custom options: {0}").format(error)) + + +@frappe.whitelist() +def get_parent_doctypes(child_type: str) -> list[str]: + """Get all parent doctypes that have the child doctype.""" + assert isinstance(child_type, str) + + standard = frappe.get_all( + "DocField", + fields="parent", + filters={"fieldtype": "Table", "options": child_type}, + pluck="parent", + ) + + custom = frappe.get_all( + "Custom Field", + fields="dt", + filters={"fieldtype": "Table", "options": child_type}, + pluck="dt", + ) + + return standard + custom diff --git a/frappe/desk/doctype/event/event.json b/frappe/desk/doctype/event/event.json index bce3b1e65a..cb2e42aab2 100644 --- a/frappe/desk/doctype/event/event.json +++ b/frappe/desk/doctype/event/event.json @@ -221,7 +221,7 @@ "in_list_view": 1, "in_standard_filter": 1, "label": "Status", - "options": "Open\nClosed" + "options": "Open\nCompleted\nClosed" }, { "collapsible": 1, @@ -318,4 +318,4 @@ "track_changes": 1, "track_seen": 1, "track_views": 1 -} \ No newline at end of file +} diff --git a/frappe/desk/doctype/notification_log/notification_log.json b/frappe/desk/doctype/notification_log/notification_log.json index e188708277..f24a6447b4 100644 --- a/frappe/desk/doctype/notification_log/notification_log.json +++ b/frappe/desk/doctype/notification_log/notification_log.json @@ -22,18 +22,14 @@ "fieldname": "subject", "fieldtype": "Text", "in_list_view": 1, - "label": "Subject", - "show_days": 1, - "show_seconds": 1 + "label": "Subject" }, { "fieldname": "for_user", "fieldtype": "Link", "hidden": 1, "label": "For User", - "options": "User", - "show_days": 1, - "show_seconds": 1 + "options": "User" }, { "fieldname": "type", @@ -42,36 +38,26 @@ "in_list_view": 1, "in_standard_filter": 1, "label": "Type", - "options": "Mention\nEnergy Point\nAssignment\nShare\nAlert", - "search_index": 1, - "show_days": 1, - "show_seconds": 1 + "options": "Mention\nEnergy Point\nAssignment\nShare\nAlert" }, { "fieldname": "email_content", "fieldtype": "Text Editor", - "label": "Message", - "show_days": 1, - "show_seconds": 1 + "label": "Message" }, { "fieldname": "document_type", "fieldtype": "Link", "hidden": 1, "label": "Document Type", - "options": "DocType", - "search_index": 1, - "show_days": 1, - "show_seconds": 1 + "options": "DocType" }, { "fieldname": "document_name", "fieldtype": "Data", "hidden": 1, "label": "Document Link", - "search_index": 1, - "show_days": 1, - "show_seconds": 1 + "search_index": 1 }, { "fieldname": "from_user", @@ -79,9 +65,7 @@ "hidden": 1, "label": "From User", "options": "User", - "search_index": 1, - "show_days": 1, - "show_seconds": 1 + "search_index": 1 }, { "default": "0", @@ -89,38 +73,30 @@ "fieldtype": "Check", "hidden": 1, "ignore_user_permissions": 1, - "label": "Read", - "show_days": 1, - "show_seconds": 1 + "label": "Read" }, { "fieldname": "open_reference_document", "fieldtype": "Button", - "label": "Open Reference Document", - "show_days": 1, - "show_seconds": 1 + "label": "Open Reference Document" }, { "fieldname": "attached_file", "fieldtype": "Code", "hidden": 1, "label": "Attached File", - "options": "JSON", - "show_days": 1, - "show_seconds": 1 + "options": "JSON" }, { "fieldname": "attachment_link", "fieldtype": "HTML", - "label": "Attachment Link", - "show_days": 1, - "show_seconds": 1 + "label": "Attachment Link" } ], "hide_toolbar": 1, "in_create": 1, "links": [], - "modified": "2021-10-25 17:26:09.703215", + "modified": "2022-09-13 16:08:48.153934", "modified_by": "Administrator", "module": "Desk", "name": "Notification Log", @@ -138,6 +114,7 @@ ], "sort_field": "modified", "sort_order": "DESC", + "states": [], "title_field": "subject", "track_seen": 1 -} +} \ No newline at end of file diff --git a/frappe/desk/doctype/notification_log/notification_log.py b/frappe/desk/doctype/notification_log/notification_log.py index e3112b08a6..b46795dd8a 100644 --- a/frappe/desk/doctype/notification_log/notification_log.py +++ b/frappe/desk/doctype/notification_log/notification_log.py @@ -136,7 +136,7 @@ def get_email_header(doc): @frappe.whitelist() def get_notification_logs(limit=20): notification_logs = frappe.db.get_list( - "Notification Log", fields=["*"], limit=limit, order_by="creation desc" + "Notification Log", fields=["*"], limit=limit, order_by="modified desc" ) users = [log.from_user for log in notification_logs] diff --git a/frappe/desk/doctype/number_card/number_card.js b/frappe/desk/doctype/number_card/number_card.js index 77ab2b4ef8..b0c5456268 100644 --- a/frappe/desk/doctype/number_card/number_card.js +++ b/frappe/desk/doctype/number_card/number_card.js @@ -472,25 +472,21 @@ frappe.ui.form.on("Number Card", { frm.set_df_property("parent_document_type", "hidden", !doc_is_table); if (document_type && doc_is_table) { - let parent = await frappe.db.get_list("DocField", { - filters: { - fieldtype: "Table", - options: document_type, - }, - fields: ["parent"], + let parents = await frappe.xcall( + "frappe.desk.doctype.dashboard_chart.dashboard_chart.get_parent_doctypes", + { child_type: document_type } + ); + + frm.set_query("parent_document_type", function () { + return { + filters: { + name: ["in", parents], + }, + }; }); - parent && - frm.set_query("parent_document_type", function () { - return { - filters: { - name: ["in", parent.map(({ parent }) => parent)], - }, - }; - }); - - if (parent.length === 1) { - frm.set_value("parent_document_type", parent[0].parent); + if (parents.length === 1) { + frm.set_value("parent_document_type", parents[0]); } } }, diff --git a/frappe/desk/doctype/workspace/workspace.py b/frappe/desk/doctype/workspace/workspace.py index 82077cc59c..1577cd53d7 100644 --- a/frappe/desk/doctype/workspace/workspace.py +++ b/frappe/desk/doctype/workspace/workspace.py @@ -266,6 +266,7 @@ def duplicate_page(page_name, new_page): doc.public = new_page.get("is_public") doc.for_user = "" doc.label = doc.title + doc.module = "" if not doc.public: doc.for_user = doc.for_user or frappe.session.user doc.label = f"{doc.title}-{doc.for_user}" diff --git a/frappe/desk/form/load.py b/frappe/desk/form/load.py index 0b1e79208d..d7dfbb90d7 100644 --- a/frappe/desk/form/load.py +++ b/frappe/desk/form/load.py @@ -64,11 +64,9 @@ def getdoctype(doctype, with_parent=False, cached_timestamp=None): parent_dt = None # with parent (called from report builder) - if with_parent: - parent_dt = frappe.model.meta.get_parent_dt(doctype) - if parent_dt: - docs = get_meta_bundle(parent_dt) - frappe.response["parent_dt"] = parent_dt + if with_parent and (parent_dt := frappe.model.meta.get_parent_dt(doctype)): + docs = get_meta_bundle(parent_dt) + frappe.response["parent_dt"] = parent_dt if not docs: docs = get_meta_bundle(doctype) @@ -110,6 +108,8 @@ def get_docinfo(doc=None, doctype=None, name=None): docinfo.update( { + "doctype": doc.doctype, + "name": doc.name, "attachments": get_attachments(doc.doctype, doc.name), "communications": communications_except_auto_messages, "automated_messages": automated_messages, @@ -373,7 +373,7 @@ def run_onload(doc): def get_view_logs(doctype, docname): """get and return the latest view logs if available""" logs = [] - if hasattr(frappe.get_meta(doctype), "track_views") and frappe.get_meta(doctype).track_views: + if getattr(frappe.get_meta(doctype), "track_views", None): view_logs = frappe.get_all( "View Log", filters={ diff --git a/frappe/desk/form/meta.py b/frappe/desk/form/meta.py index ae7262e1b5..ee975c8326 100644 --- a/frappe/desk/form/meta.py +++ b/frappe/desk/form/meta.py @@ -35,12 +35,10 @@ ASSET_KEYS = ( def get_meta(doctype, cached=True): # don't cache for developer mode as js files, templates may be edited if cached and not frappe.conf.developer_mode: - meta = frappe.cache().hget("form_meta", doctype) - if meta: - meta = FormMeta(meta) - else: + meta = frappe.cache().hget("doctype_form_meta", doctype) + if not meta: meta = FormMeta(doctype) - frappe.cache().hset("form_meta", doctype, meta.as_dict()) + frappe.cache().hset("doctype_form_meta", doctype, meta) else: meta = FormMeta(doctype) diff --git a/frappe/email/doctype/auto_email_report/auto_email_report.js b/frappe/email/doctype/auto_email_report/auto_email_report.js index 6930078512..4ce4a63b03 100644 --- a/frappe/email/doctype/auto_email_report/auto_email_report.js +++ b/frappe/email/doctype/auto_email_report/auto_email_report.js @@ -35,6 +35,15 @@ frappe.ui.form.on("Auto Email Report", { frm.set_value("email_to", frappe.session.user); } } + + frm.set_query("sender", function () { + return { + filters: { + enable_outgoing: 1, + awaiting_password: 0, + }, + }; + }); }, report: function (frm) { frm.set_value("filters", ""); diff --git a/frappe/email/doctype/auto_email_report/auto_email_report.json b/frappe/email/doctype/auto_email_report/auto_email_report.json index 211e2e9662..75a9e99c96 100644 --- a/frappe/email/doctype/auto_email_report/auto_email_report.json +++ b/frappe/email/doctype/auto_email_report/auto_email_report.json @@ -1,238 +1,248 @@ { - "allow_rename": 1, - "creation": "2016-09-01 01:34:34.985457", - "doctype": "DocType", - "editable_grid": 1, - "engine": "InnoDB", - "field_order": [ - "report", - "user", - "enabled", - "column_break_4", - "report_type", - "reference_report", - "filter_data", - "send_if_data", - "data_modified_till", - "no_of_rows", - "report_filters", - "filters_display", - "filters", - "filter_meta", - "dynamic_report_filters_section", - "from_date_field", - "to_date_field", - "column_break_17", - "dynamic_date_period", - "email_settings", - "email_to", - "day_of_week", - "column_break_13", - "frequency", - "format", - "section_break_15", - "description" - ], - "fields": [ - { - "fieldname": "report", - "fieldtype": "Link", - "label": "Report", - "options": "Report", - "reqd": 1 - }, - { - "default": "User", - "fieldname": "user", - "fieldtype": "Link", - "label": "Based on Permissions For User", - "options": "User", - "reqd": 1 - }, - { - "default": "1", - "fieldname": "enabled", - "fieldtype": "Check", - "label": "Enabled" - }, - { - "fieldname": "column_break_4", - "fieldtype": "Column Break" - }, - { - "fetch_from": "report.report_type", - "fieldname": "report_type", - "fieldtype": "Read Only", - "label": "Report Type" - }, - { - "fieldname": "filter_data", - "fieldtype": "Section Break", - "label": "Filter Data" - }, - { - "default": "1", - "fieldname": "send_if_data", - "fieldtype": "Check", - "label": "Send only if there is any data" - }, - { - "depends_on": "eval:doc.report_type=='Report Builder'", - "description": "Zero means send records updated at anytime", - "fieldname": "data_modified_till", - "fieldtype": "Int", - "label": "Only Send Records Updated in Last X Hours" - }, - { - "default": "100", - "fieldname": "no_of_rows", - "fieldtype": "Int", - "label": "No of Rows (Max 500)" - }, - { - "collapsible": 1, - "depends_on": "eval:doc.report_type !== 'Report Builder'", - "fieldname": "report_filters", - "fieldtype": "Section Break", - "label": "Report Filters" - }, - { - "fieldname": "filters_display", - "fieldtype": "HTML", - "label": "Filters Display" - }, - { - "fieldname": "filters", - "fieldtype": "Text", - "hidden": 1, - "label": "Filters" - }, - { - "fieldname": "filter_meta", - "fieldtype": "Text", - "hidden": 1, - "label": "Filter Meta", - "read_only": 1 - }, - { - "collapsible": 1, - "depends_on": "eval:doc.report_type !== 'Report Builder'", - "fieldname": "dynamic_report_filters_section", - "fieldtype": "Section Break", - "label": "Dynamic Report Filters" - }, - { - "fieldname": "from_date_field", - "fieldtype": "Select", - "label": "From Date Field" - }, - { - "fieldname": "to_date_field", - "fieldtype": "Select", - "label": "To Date Field" - }, - { - "fieldname": "column_break_17", - "fieldtype": "Column Break" - }, - { - "fieldname": "dynamic_date_period", - "fieldtype": "Select", - "label": "Period", - "options": "\nDaily\nWeekly\nMonthly\nQuarterly\nHalf Yearly\nYearly" - }, - { - "fieldname": "email_settings", - "fieldtype": "Section Break", - "label": "Email Settings" - }, - { - "description": "For multiple addresses, enter the address on different line. e.g. test@test.com \u23ce test1@test.com", - "fieldname": "email_to", - "fieldtype": "Small Text", - "label": "Email To", - "reqd": 1 - }, - { - "default": "Monday", - "depends_on": "eval:doc.frequency=='Weekly'", - "fieldname": "day_of_week", - "fieldtype": "Select", - "label": "Day of Week", - "options": "Monday\nTuesday\nWednesday\nThursday\nFriday\nSaturday\nSunday" - }, - { - "fieldname": "column_break_13", - "fieldtype": "Column Break" - }, - { - "fieldname": "frequency", - "fieldtype": "Select", - "in_list_view": 1, - "in_standard_filter": 1, - "label": "Frequency", - "options": "Daily\nWeekdays\nWeekly\nMonthly", - "reqd": 1 - }, - { - "fieldname": "format", - "fieldtype": "Select", - "label": "Format", - "options": "HTML\nXLSX\nCSV", - "reqd": 1 - }, - { - "collapsible": 1, - "fieldname": "section_break_15", - "fieldtype": "Section Break", - "label": "Message" - }, - { - "fieldname": "description", - "fieldtype": "Text Editor", - "label": "Message" - }, - { - "fetch_from": "report.reference_report", - "fieldname": "reference_report", - "fieldtype": "Data", - "hidden": 1, - "label": "Reference Report", - "read_only": 1 - } - ], - "modified": "2021-01-28 15:59:43.151995", - "modified_by": "Administrator", - "module": "Email", - "name": "Auto Email Report", - "owner": "Administrator", - "permissions": [ - { - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "share": 1, - "write": 1 - }, - { - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "print": 1, - "read": 1, - "report": 1, - "role": "Report Manager", - "share": 1, - "write": 1 - } - ], - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1 - } \ No newline at end of file + "actions": [], + "allow_rename": 1, + "creation": "2016-09-01 01:34:34.985457", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "report", + "user", + "enabled", + "column_break_4", + "report_type", + "reference_report", + "filter_data", + "send_if_data", + "data_modified_till", + "no_of_rows", + "report_filters", + "filters_display", + "filters", + "filter_meta", + "dynamic_report_filters_section", + "from_date_field", + "to_date_field", + "column_break_17", + "dynamic_date_period", + "email_settings", + "email_to", + "day_of_week", + "column_break_13", + "sender", + "frequency", + "format", + "section_break_15", + "description" + ], + "fields": [ + { + "fieldname": "report", + "fieldtype": "Link", + "label": "Report", + "options": "Report", + "reqd": 1 + }, + { + "default": "User", + "fieldname": "user", + "fieldtype": "Link", + "label": "Based on Permissions For User", + "options": "User", + "reqd": 1 + }, + { + "default": "1", + "fieldname": "enabled", + "fieldtype": "Check", + "label": "Enabled" + }, + { + "fieldname": "column_break_4", + "fieldtype": "Column Break" + }, + { + "fetch_from": "report.report_type", + "fieldname": "report_type", + "fieldtype": "Read Only", + "label": "Report Type" + }, + { + "fieldname": "filter_data", + "fieldtype": "Section Break", + "label": "Filter Data" + }, + { + "default": "1", + "fieldname": "send_if_data", + "fieldtype": "Check", + "label": "Send only if there is any data" + }, + { + "depends_on": "eval:doc.report_type=='Report Builder'", + "description": "Zero means send records updated at anytime", + "fieldname": "data_modified_till", + "fieldtype": "Int", + "label": "Only Send Records Updated in Last X Hours" + }, + { + "default": "100", + "fieldname": "no_of_rows", + "fieldtype": "Int", + "label": "No of Rows (Max 500)" + }, + { + "collapsible": 1, + "depends_on": "eval:doc.report_type !== 'Report Builder'", + "fieldname": "report_filters", + "fieldtype": "Section Break", + "label": "Report Filters" + }, + { + "fieldname": "filters_display", + "fieldtype": "HTML", + "label": "Filters Display" + }, + { + "fieldname": "filters", + "fieldtype": "Text", + "hidden": 1, + "label": "Filters" + }, + { + "fieldname": "filter_meta", + "fieldtype": "Text", + "hidden": 1, + "label": "Filter Meta", + "read_only": 1 + }, + { + "collapsible": 1, + "depends_on": "eval:doc.report_type !== 'Report Builder'", + "fieldname": "dynamic_report_filters_section", + "fieldtype": "Section Break", + "label": "Dynamic Report Filters" + }, + { + "fieldname": "from_date_field", + "fieldtype": "Select", + "label": "From Date Field" + }, + { + "fieldname": "to_date_field", + "fieldtype": "Select", + "label": "To Date Field" + }, + { + "fieldname": "column_break_17", + "fieldtype": "Column Break" + }, + { + "fieldname": "dynamic_date_period", + "fieldtype": "Select", + "label": "Period", + "options": "\nDaily\nWeekly\nMonthly\nQuarterly\nHalf Yearly\nYearly" + }, + { + "fieldname": "email_settings", + "fieldtype": "Section Break", + "label": "Email Settings" + }, + { + "description": "For multiple addresses, enter the address on different line. e.g. test@test.com \u23ce test1@test.com", + "fieldname": "email_to", + "fieldtype": "Small Text", + "label": "Email To", + "reqd": 1 + }, + { + "default": "Monday", + "depends_on": "eval:doc.frequency=='Weekly'", + "fieldname": "day_of_week", + "fieldtype": "Select", + "label": "Day of Week", + "options": "Monday\nTuesday\nWednesday\nThursday\nFriday\nSaturday\nSunday" + }, + { + "fieldname": "column_break_13", + "fieldtype": "Column Break" + }, + { + "fieldname": "frequency", + "fieldtype": "Select", + "in_list_view": 1, + "in_standard_filter": 1, + "label": "Frequency", + "options": "Daily\nWeekdays\nWeekly\nMonthly", + "reqd": 1 + }, + { + "fieldname": "format", + "fieldtype": "Select", + "label": "Format", + "options": "HTML\nXLSX\nCSV", + "reqd": 1 + }, + { + "collapsible": 1, + "fieldname": "section_break_15", + "fieldtype": "Section Break", + "label": "Message" + }, + { + "fieldname": "description", + "fieldtype": "Text Editor", + "label": "Message" + }, + { + "fetch_from": "report.reference_report", + "fieldname": "reference_report", + "fieldtype": "Data", + "hidden": 1, + "label": "Reference Report", + "read_only": 1 + }, + { + "fieldname": "sender", + "fieldtype": "Link", + "label": "Sender", + "options": "Email Account" + } + ], + "links": [], + "modified": "2022-09-08 15:31:55.031023", + "modified_by": "Administrator", + "module": "Email", + "name": "Auto Email Report", + "owner": "Administrator", + "permissions": [ + { + "create": 1, + "delete": 1, + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1, + "write": 1 + }, + { + "create": 1, + "delete": 1, + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "Report Manager", + "share": 1, + "write": 1 + } + ], + "sort_field": "modified", + "sort_order": "DESC", + "states": [], + "track_changes": 1 +} \ No newline at end of file diff --git a/frappe/email/doctype/auto_email_report/auto_email_report.py b/frappe/email/doctype/auto_email_report/auto_email_report.py index b9b5e4e8d7..7b18c8632b 100644 --- a/frappe/email/doctype/auto_email_report/auto_email_report.py +++ b/frappe/email/doctype/auto_email_report/auto_email_report.py @@ -3,6 +3,7 @@ import calendar from datetime import timedelta +from email.utils import formataddr import frappe from frappe import _ @@ -37,6 +38,11 @@ class AutoEmailReport(Document): self.validate_report_format() self.validate_mandatory_fields() + @property + def sender_email(self): + email_id, login_id = frappe.db.get_value("Email Account", self.sender, ["email_id", "login_id"]) + return login_id if login_id else email_id + def validate_emails(self): """Cleanup list of emails""" if "," in self.email_to: @@ -203,6 +209,7 @@ class AutoEmailReport(Document): frappe.sendmail( recipients=self.email_to.split(), + sender=formataddr((self.sender, self.sender_email)) if self.sender else "", subject=self.name, message=message, attachments=attachments, diff --git a/frappe/email/doctype/email_queue_recipient/email_queue_recipient.json b/frappe/email/doctype/email_queue_recipient/email_queue_recipient.json index c217886ce6..406449e26d 100644 --- a/frappe/email/doctype/email_queue_recipient/email_queue_recipient.json +++ b/frappe/email/doctype/email_queue_recipient/email_queue_recipient.json @@ -22,7 +22,7 @@ "fieldtype": "Select", "in_list_view": 1, "label": "Status", - "options": "\nNot Sent\nSending\nSent\nError\nExpired", + "options": "\nNot Sent\nSent", "search_index": 1 }, { @@ -33,7 +33,7 @@ ], "istable": 1, "links": [], - "modified": "2022-07-11 16:38:10.644417", + "modified": "2022-09-06 13:38:10.644417", "modified_by": "Administrator", "module": "Email", "name": "Email Queue Recipient", diff --git a/frappe/email/doctype/newsletter/newsletter.py b/frappe/email/doctype/newsletter/newsletter.py index a6c87cbcac..30d51c4c03 100644 --- a/frappe/email/doctype/newsletter/newsletter.py +++ b/frappe/email/doctype/newsletter/newsletter.py @@ -128,15 +128,11 @@ class Newsletter(WebsiteGenerator): pluck="name", ) - def get_success_recipients(self) -> list[str]: - """Recipients who have already received the newsletter. - - Couldn't think of a better name ;) - """ + def get_queued_recipients(self) -> list[str]: + """Recipients who have already been queued for receiving the newsletter.""" return frappe.get_all( "Email Queue Recipient", filters={ - "status": ("in", ["Not Sent", "Sending", "Sent"]), "parent": ("in", self.get_linked_email_queue()), }, pluck="recipient", @@ -146,8 +142,7 @@ class Newsletter(WebsiteGenerator): """Get list of pending recipients of the newsletter. These recipients may not have receive the newsletter in the previous iteration. """ - success_recipients = set(self.get_success_recipients()) - return [x for x in self.newsletter_recipients if x not in success_recipients] + return [x for x in self.newsletter_recipients if x not in self.get_queued_recipients()] def queue_all(self): """Queue Newsletter to all the recipients generated from the `Email Group` table""" diff --git a/frappe/email/doctype/newsletter/test_newsletter.py b/frappe/email/doctype/newsletter/test_newsletter.py index c020c26454..f80d218bca 100644 --- a/frappe/email/doctype/newsletter/test_newsletter.py +++ b/frappe/email/doctype/newsletter/test_newsletter.py @@ -236,13 +236,15 @@ class TestNewsletter(TestNewsletterMixin, FrappeTestCase): email_queue_list = [frappe.get_doc("Email Queue", e.name) for e in frappe.get_all("Email Queue")] self.assertEqual(len(email_queue_list), 4) - # emulate partial send - email_queue_list[0].status = "Error" - email_queue_list[0].recipients[0].status = "Error" - email_queue_list[0].save() + # delete a queue document to emulate partial send + queue_recipient_name = email_queue_list[0].recipients[0].recipient + email_queue_list[0].delete() newsletter.email_sent = False + # make sure the pending recipient is only the one which has been deleted + self.assertEqual(newsletter.get_pending_recipients(), [queue_recipient_name]) + # retry newsletter.send_emails() - email_queue_list = [frappe.get_doc("Email Queue", e.name) for e in frappe.get_all("Email Queue")] - self.assertEqual(len(email_queue_list), 5) + self.assertEqual(frappe.db.count("Email Queue"), 4) + self.assertTrue(newsletter.email_sent) diff --git a/frappe/email/doctype/notification/notification.js b/frappe/email/doctype/notification/notification.js index 4e3b1eae53..a07dca4870 100644 --- a/frappe/email/doctype/notification/notification.js +++ b/frappe/email/doctype/notification/notification.js @@ -175,7 +175,7 @@ frappe.ui.form.on("Notification", { notification: frm.doc.name, }, callback: function (r) { - if (r.message) { + if (r.message && r.message.length > 0) { frappe.msgprint(r.message); } else { frappe.msgprint(__("No alerts for today")); diff --git a/frappe/event_streaming/doctype/__init__.py b/frappe/event_streaming/doctype/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/event_streaming/doctype/document_type_field_mapping/__init__.py b/frappe/event_streaming/doctype/document_type_field_mapping/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/event_streaming/doctype/document_type_field_mapping/document_type_field_mapping.json b/frappe/event_streaming/doctype/document_type_field_mapping/document_type_field_mapping.json deleted file mode 100644 index bba0a98237..0000000000 --- a/frappe/event_streaming/doctype/document_type_field_mapping/document_type_field_mapping.json +++ /dev/null @@ -1,73 +0,0 @@ -{ - "actions": [], - "creation": "2019-09-27 12:46:50.165135", - "doctype": "DocType", - "editable_grid": 1, - "engine": "InnoDB", - "field_order": [ - "local_fieldname", - "mapping_type", - "mapping", - "remote_value_filters", - "column_break_5", - "remote_fieldname", - "default_value" - ], - "fields": [ - { - "fieldname": "remote_fieldname", - "fieldtype": "Data", - "in_list_view": 1, - "label": "Remote Fieldname" - }, - { - "fieldname": "local_fieldname", - "fieldtype": "Data", - "in_list_view": 1, - "label": "Local Fieldname", - "reqd": 1 - }, - { - "fieldname": "column_break_5", - "fieldtype": "Column Break" - }, - { - "fieldname": "default_value", - "fieldtype": "Data", - "label": "Default Value" - }, - { - "fieldname": "mapping_type", - "fieldtype": "Select", - "label": "Mapping Type", - "options": "\nChild Table\nDocument" - }, - { - "depends_on": "eval:doc.mapping_type;", - "fieldname": "mapping", - "fieldtype": "Link", - "label": "Mapping", - "options": "Document Type Mapping" - }, - { - "depends_on": "eval:doc.mapping_type==\"Document\";", - "fieldname": "remote_value_filters", - "fieldtype": "Code", - "label": "Remote Value Filters", - "mandatory_depends_on": "eval:doc.mapping_type===\"Document\";", - "options": "JSON" - } - ], - "istable": 1, - "links": [], - "modified": "2020-03-19 13:56:36.223799", - "modified_by": "Administrator", - "module": "Event Streaming", - "name": "Document Type Field Mapping", - "owner": "Administrator", - "permissions": [], - "quick_entry": 1, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1 -} \ No newline at end of file diff --git a/frappe/event_streaming/doctype/document_type_field_mapping/document_type_field_mapping.py b/frappe/event_streaming/doctype/document_type_field_mapping/document_type_field_mapping.py deleted file mode 100644 index 96d9e0fcb3..0000000000 --- a/frappe/event_streaming/doctype/document_type_field_mapping/document_type_field_mapping.py +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright (c) 2019, Frappe Technologies and contributors -# License: MIT. See LICENSE - -# import frappe -from frappe.model.document import Document - - -class DocumentTypeFieldMapping(Document): - pass diff --git a/frappe/event_streaming/doctype/document_type_mapping/__init__.py b/frappe/event_streaming/doctype/document_type_mapping/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.js b/frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.js deleted file mode 100644 index ad9ab0f51d..0000000000 --- a/frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.js +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright (c) 2019, Frappe Technologies and contributors -// For license information, please see license.txt - -frappe.ui.form.on("Document Type Mapping", { - local_doctype: function (frm) { - if (frm.doc.local_doctype) { - frappe.model.clear_table(frm.doc, "field_mapping"); - let fields = frm.events.get_fields(frm); - $.each(fields, function (i, data) { - let row = frappe.model.add_child( - frm.doc, - "Document Type Field Mapping", - "field_mapping" - ); - row.local_fieldname = data; - }); - refresh_field("field_mapping"); - } - }, - - get_fields: function (frm) { - let filtered_fields = []; - frappe.model.with_doctype(frm.doc.local_doctype, () => { - frappe.get_meta(frm.doc.local_doctype).fields.map((field) => { - if ( - field.fieldname !== "remote_docname" && - field.fieldname !== "remote_site_name" && - frappe.model.is_value_type(field) && - !field.hidden - ) { - filtered_fields.push(field.fieldname); - } - }); - }); - return filtered_fields; - }, -}); diff --git a/frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.json b/frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.json deleted file mode 100644 index 6a59cf3b70..0000000000 --- a/frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.json +++ /dev/null @@ -1,71 +0,0 @@ -{ - "autoname": "field:mapping_name", - "creation": "2019-09-27 12:45:56.529124", - "doctype": "DocType", - "editable_grid": 1, - "engine": "InnoDB", - "field_order": [ - "mapping_name", - "local_doctype", - "remote_doctype", - "section_break_3", - "field_mapping" - ], - "fields": [ - { - "fieldname": "local_doctype", - "fieldtype": "Link", - "in_list_view": 1, - "label": "Local Document Type", - "options": "DocType", - "reqd": 1 - }, - { - "fieldname": "remote_doctype", - "fieldtype": "Data", - "in_list_view": 1, - "label": "Remote Document Type", - "reqd": 1 - }, - { - "fieldname": "section_break_3", - "fieldtype": "Section Break" - }, - { - "fieldname": "field_mapping", - "fieldtype": "Table", - "label": "Field Mapping", - "options": "Document Type Field Mapping" - }, - { - "fieldname": "mapping_name", - "fieldtype": "Data", - "label": "Mapping Name", - "reqd": 1, - "unique": 1 - } - ], - "modified": "2019-10-09 08:36:04.621397", - "modified_by": "Administrator", - "module": "Event Streaming", - "name": "Document Type Mapping", - "owner": "Administrator", - "permissions": [ - { - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "share": 1, - "write": 1 - } - ], - "quick_entry": 1, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1 -} \ No newline at end of file diff --git a/frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.py b/frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.py deleted file mode 100644 index 04b5015296..0000000000 --- a/frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.py +++ /dev/null @@ -1,181 +0,0 @@ -# Copyright (c) 2019, Frappe Technologies and contributors -# License: MIT. See LICENSE -import json - -import frappe -from frappe import _ -from frappe.model import child_table_fields, default_fields -from frappe.model.document import Document - - -class DocumentTypeMapping(Document): - def validate(self): - self.validate_inner_mapping() - - def validate_inner_mapping(self): - meta = frappe.get_meta(self.local_doctype) - for field_map in self.field_mapping: - if field_map.local_fieldname not in (default_fields + child_table_fields): - field = meta.get_field(field_map.local_fieldname) - if not field: - frappe.throw(_("Row #{0}: Invalid Local Fieldname").format(field_map.idx)) - - fieldtype = field.get("fieldtype") - if fieldtype in ["Link", "Dynamic Link", "Table"]: - if not field_map.mapping and not field_map.default_value: - msg = _( - "Row #{0}: Please set Mapping or Default Value for the field {1} since its a dependency field" - ).format(field_map.idx, frappe.bold(field_map.local_fieldname)) - frappe.throw(msg, title="Inner Mapping Missing") - - if field_map.mapping_type == "Document" and not field_map.remote_value_filters: - msg = _( - "Row #{0}: Please set remote value filters for the field {1} to fetch the unique remote dependency document" - ).format(field_map.idx, frappe.bold(field_map.remote_fieldname)) - frappe.throw(msg, title="Remote Value Filters Missing") - - def get_mapping(self, doc, producer_site, update_type): - remote_fields = [] - # list of tuples (local_fieldname, dependent_doc) - dependencies = [] - - for mapping in self.field_mapping: - if doc.get(mapping.remote_fieldname): - if mapping.mapping_type == "Document": - if not mapping.default_value: - dependency = self.get_mapped_dependency(mapping, producer_site, doc) - if dependency: - dependencies.append((mapping.local_fieldname, dependency)) - else: - doc[mapping.local_fieldname] = mapping.default_value - - if mapping.mapping_type == "Child Table" and update_type != "Update": - doc[mapping.local_fieldname] = get_mapped_child_table_docs( - mapping.mapping, doc[mapping.remote_fieldname], producer_site - ) - else: - # copy value into local fieldname key and remove remote fieldname key - doc[mapping.local_fieldname] = doc[mapping.remote_fieldname] - - if mapping.local_fieldname != mapping.remote_fieldname: - remote_fields.append(mapping.remote_fieldname) - - if not doc.get(mapping.remote_fieldname) and mapping.default_value and update_type != "Update": - doc[mapping.local_fieldname] = mapping.default_value - - # remove the remote fieldnames - for field in remote_fields: - doc.pop(field, None) - - if update_type != "Update": - doc["doctype"] = self.local_doctype - - mapping = {"doc": frappe.as_json(doc)} - if len(dependencies): - mapping["dependencies"] = dependencies - return mapping - - def get_mapped_update(self, update, producer_site): - update_diff = frappe._dict(json.loads(update.data)) - mapping = update_diff - dependencies = [] - if update_diff.changed: - doc_map = self.get_mapping(update_diff.changed, producer_site, "Update") - mapped_doc = doc_map.get("doc") - mapping.changed = json.loads(mapped_doc) - if doc_map.get("dependencies"): - dependencies += doc_map.get("dependencies") - - if update_diff.removed: - mapping = self.map_rows_removed(update_diff, mapping) - if update_diff.added: - mapping = self.map_rows(update_diff, mapping, producer_site, operation="added") - if update_diff.row_changed: - mapping = self.map_rows(update_diff, mapping, producer_site, operation="row_changed") - - update = {"doc": frappe.as_json(mapping)} - if len(dependencies): - update["dependencies"] = dependencies - return update - - def get_mapped_dependency(self, mapping, producer_site, doc): - inner_mapping = frappe.get_doc("Document Type Mapping", mapping.mapping) - filters = json.loads(mapping.remote_value_filters) - for key, value in filters.items(): - if value.startswith("eval:"): - val = frappe.safe_eval(value[5:], None, dict(doc=doc)) - filters[key] = val - if doc.get(value): - filters[key] = doc.get(value) - matching_docs = producer_site.get_doc(inner_mapping.remote_doctype, filters=filters) - if len(matching_docs): - remote_docname = matching_docs[0].get("name") - remote_doc = producer_site.get_doc(inner_mapping.remote_doctype, remote_docname) - doc = inner_mapping.get_mapping(remote_doc, producer_site, "Insert").get("doc") - return doc - return - - def map_rows_removed(self, update_diff, mapping): - removed = [] - mapping["removed"] = update_diff.removed - for key, value in update_diff.removed.copy().items(): - local_table_name = frappe.db.get_value( - "Document Type Field Mapping", - {"remote_fieldname": key, "parent": self.name}, - "local_fieldname", - ) - mapping.removed[local_table_name] = value - if local_table_name != key: - removed.append(key) - - # remove the remote fieldnames - for field in removed: - mapping.removed.pop(field, None) - return mapping - - def map_rows(self, update_diff, mapping, producer_site, operation): - remote_fields = [] - for tablename, entries in update_diff.get(operation).copy().items(): - local_table_name = frappe.db.get_value( - "Document Type Field Mapping", {"remote_fieldname": tablename}, "local_fieldname" - ) - table_map = frappe.db.get_value( - "Document Type Field Mapping", - {"local_fieldname": local_table_name, "parent": self.name}, - "mapping", - ) - table_map = frappe.get_doc("Document Type Mapping", table_map) - docs = [] - for entry in entries: - mapped_doc = table_map.get_mapping(entry, producer_site, "Update").get("doc") - docs.append(json.loads(mapped_doc)) - mapping.get(operation)[local_table_name] = docs - if local_table_name != tablename: - remote_fields.append(tablename) - - # remove the remote fieldnames - for field in remote_fields: - mapping.get(operation).pop(field, None) - - return mapping - - -def get_mapped_child_table_docs(child_map, table_entries, producer_site): - """Get mapping for child doctypes""" - child_map = frappe.get_doc("Document Type Mapping", child_map) - mapped_entries = [] - remote_fields = [] - for child_doc in table_entries: - for mapping in child_map.field_mapping: - if child_doc.get(mapping.remote_fieldname): - child_doc[mapping.local_fieldname] = child_doc[mapping.remote_fieldname] - if mapping.local_fieldname != mapping.remote_fieldname: - child_doc.pop(mapping.remote_fieldname, None) - mapped_entries.append(child_doc) - - # remove the remote fieldnames - for field in remote_fields: - child_doc.pop(field, None) - - child_doc["doctype"] = child_map.local_doctype - return mapped_entries diff --git a/frappe/event_streaming/doctype/document_type_mapping/test_document_type_mapping.py b/frappe/event_streaming/doctype/document_type_mapping/test_document_type_mapping.py deleted file mode 100644 index defaa8b9c6..0000000000 --- a/frappe/event_streaming/doctype/document_type_mapping/test_document_type_mapping.py +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright (c) 2019, Frappe Technologies and Contributors -# License: MIT. See LICENSE -# import frappe -from frappe.tests.utils import FrappeTestCase - - -class TestDocumentTypeMapping(FrappeTestCase): - pass diff --git a/frappe/event_streaming/doctype/event_consumer/__init__.py b/frappe/event_streaming/doctype/event_consumer/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/event_streaming/doctype/event_consumer/event_consumer.js b/frappe/event_streaming/doctype/event_consumer/event_consumer.js deleted file mode 100644 index 2bcf96f9f3..0000000000 --- a/frappe/event_streaming/doctype/event_consumer/event_consumer.js +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright (c) 2019, Frappe Technologies and contributors -// For license information, please see license.txt - -frappe.ui.form.on("Event Consumer", { - refresh: function (frm) { - // formatter for subscribed doctype approval status - frm.set_indicator_formatter("status", function (doc) { - let indicator = "orange"; - if (doc.status == "Approved") { - indicator = "green"; - } else if (doc.status == "Rejected") { - indicator = "red"; - } - return indicator; - }); - }, -}); diff --git a/frappe/event_streaming/doctype/event_consumer/event_consumer.json b/frappe/event_streaming/doctype/event_consumer/event_consumer.json deleted file mode 100644 index 42b47ce949..0000000000 --- a/frappe/event_streaming/doctype/event_consumer/event_consumer.json +++ /dev/null @@ -1,97 +0,0 @@ -{ - "actions": [], - "autoname": "field:callback_url", - "creation": "2019-08-26 17:45:15.479530", - "doctype": "DocType", - "editable_grid": 1, - "engine": "InnoDB", - "field_order": [ - "consumer_doctypes", - "callback_url", - "section_break_3", - "api_key", - "api_secret", - "column_break_6", - "user", - "incoming_change" - ], - "fields": [ - { - "fieldname": "callback_url", - "fieldtype": "Data", - "in_list_view": 1, - "label": "Callback URL", - "read_only": 1, - "reqd": 1, - "unique": 1 - }, - { - "fieldname": "api_key", - "fieldtype": "Data", - "label": "API Key", - "reqd": 1 - }, - { - "fieldname": "api_secret", - "fieldtype": "Password", - "label": "API Secret", - "reqd": 1 - }, - { - "fieldname": "user", - "fieldtype": "Link", - "label": "Event Subscriber", - "options": "User", - "read_only": 1, - "reqd": 1 - }, - { - "fieldname": "section_break_3", - "fieldtype": "Section Break" - }, - { - "fieldname": "column_break_6", - "fieldtype": "Column Break" - }, - { - "default": "0", - "fieldname": "incoming_change", - "fieldtype": "Check", - "hidden": 1, - "label": "Incoming Change", - "read_only": 1 - }, - { - "fieldname": "consumer_doctypes", - "fieldtype": "Table", - "label": "Event Consumer Document Types", - "options": "Event Consumer Document Type", - "reqd": 1 - } - ], - "in_create": 1, - "links": [], - "modified": "2020-09-08 16:42:39.828085", - "modified_by": "Administrator", - "module": "Event Streaming", - "name": "Event Consumer", - "owner": "Administrator", - "permissions": [ - { - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "share": 1, - "write": 1 - } - ], - "quick_entry": 1, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1 -} \ No newline at end of file diff --git a/frappe/event_streaming/doctype/event_consumer/event_consumer.py b/frappe/event_streaming/doctype/event_consumer/event_consumer.py deleted file mode 100644 index a2ae6f6651..0000000000 --- a/frappe/event_streaming/doctype/event_consumer/event_consumer.py +++ /dev/null @@ -1,216 +0,0 @@ -# Copyright (c) 2019, Frappe Technologies and contributors -# License: MIT. See LICENSE - -import json -import os - -import requests - -import frappe -from frappe import _ -from frappe.frappeclient import FrappeClient -from frappe.model.document import Document -from frappe.utils.background_jobs import get_jobs -from frappe.utils.data import get_url - - -class EventConsumer(Document): - def validate(self): - # approve subscribed doctypes for tests - # frappe.flags.in_test won't work here as tests are running on the consumer site - if os.environ.get("CI"): - for entry in self.consumer_doctypes: - entry.status = "Approved" - - def on_update(self): - if not self.incoming_change: - doc_before_save = self.get_doc_before_save() - if doc_before_save.api_key != self.api_key or doc_before_save.api_secret != self.api_secret: - return - - self.update_consumer_status() - else: - frappe.db.set_value(self.doctype, self.name, "incoming_change", 0) - - frappe.cache().delete_value("event_consumer_document_type_map") - - def on_trash(self): - for i in frappe.get_all("Event Update Log Consumer", {"consumer": self.name}): - frappe.delete_doc("Event Update Log Consumer", i.name) - frappe.cache().delete_value("event_consumer_document_type_map") - - def update_consumer_status(self): - consumer_site = get_consumer_site(self.callback_url) - event_producer = consumer_site.get_doc("Event Producer", get_url()) - event_producer = frappe._dict(event_producer) - config = event_producer.producer_doctypes - event_producer.producer_doctypes = [] - for entry in config: - if entry.get("has_mapping"): - ref_doctype = consumer_site.get_value( - "Document Type Mapping", "remote_doctype", entry.get("mapping") - ).get("remote_doctype") - else: - ref_doctype = entry.get("ref_doctype") - - entry["status"] = frappe.db.get_value( - "Event Consumer Document Type", {"parent": self.name, "ref_doctype": ref_doctype}, "status" - ) - - event_producer.producer_doctypes = config - # when producer doc is updated it updates the consumer doc - # set flag to avoid deadlock - event_producer.incoming_change = True - consumer_site.update(event_producer) - - def get_consumer_status(self): - response = requests.get(self.callback_url) - if response.status_code != 200: - return "offline" - return "online" - - -@frappe.whitelist() -def register_consumer(data): - """create an event consumer document for registering a consumer""" - data = json.loads(data) - # to ensure that consumer is created only once - if frappe.db.exists("Event Consumer", data["event_consumer"]): - return None - - user = data["user"] - if not frappe.db.exists("User", user): - frappe.throw(_("User {0} not found on the producer site").format(user)) - - if "System Manager" not in frappe.get_roles(user): - frappe.throw(_("Event Subscriber has to be a System Manager.")) - - consumer = frappe.new_doc("Event Consumer") - consumer.callback_url = data["event_consumer"] - consumer.user = data["user"] - consumer.api_key = data["api_key"] - consumer.api_secret = data["api_secret"] - consumer.incoming_change = True - consumer_doctypes = json.loads(data["consumer_doctypes"]) - - for entry in consumer_doctypes: - consumer.append( - "consumer_doctypes", - {"ref_doctype": entry.get("doctype"), "status": "Pending", "condition": entry.get("condition")}, - ) - - consumer.insert() - - # consumer's 'last_update' field should point to the latest update - # in producer's update log when subscribing - # so that, updates after subscribing are consumed and not the old ones. - last_update = str(get_last_update()) - return json.dumps({"last_update": last_update}) - - -def get_consumer_site(consumer_url): - """create a FrappeClient object for event consumer site""" - consumer_doc = frappe.get_doc("Event Consumer", consumer_url) - consumer_site = FrappeClient( - url=consumer_url, - api_key=consumer_doc.api_key, - api_secret=consumer_doc.get_password("api_secret"), - ) - return consumer_site - - -def get_last_update(): - """get the creation timestamp of last update consumed""" - updates = frappe.get_list( - "Event Update Log", "creation", ignore_permissions=True, limit=1, order_by="creation desc" - ) - if updates: - return updates[0].creation - return frappe.utils.now_datetime() - - -@frappe.whitelist() -def notify_event_consumers(doctype): - """get all event consumers and set flag for notification status""" - event_consumers = frappe.get_all( - "Event Consumer Document Type", ["parent"], {"ref_doctype": doctype, "status": "Approved"} - ) - for entry in event_consumers: - consumer = frappe.get_doc("Event Consumer", entry.parent) - consumer.flags.notified = False - notify(consumer) - - -@frappe.whitelist() -def notify(consumer): - """notify individual event consumers about a new update""" - consumer_status = consumer.get_consumer_status() - if consumer_status == "online": - try: - client = get_consumer_site(consumer.callback_url) - client.post_request( - { - "cmd": "frappe.event_streaming.doctype.event_producer.event_producer.new_event_notification", - "producer_url": get_url(), - } - ) - consumer.flags.notified = True - except Exception: - consumer.flags.notified = False - else: - consumer.flags.notified = False - - # enqueue another job if the site was not notified - if not consumer.flags.notified: - enqueued_method = "frappe.event_streaming.doctype.event_consumer.event_consumer.notify" - jobs = get_jobs() - if not jobs or enqueued_method not in jobs[frappe.local.site] and not consumer.flags.notifed: - frappe.enqueue( - enqueued_method, queue="long", enqueue_after_commit=True, **{"consumer": consumer} - ) - - -def has_consumer_access(consumer, update_log): - """Checks if consumer has completely satisfied all the conditions on the doc""" - - if isinstance(consumer, str): - consumer = frappe.get_doc("Event Consumer", consumer) - - if not frappe.db.exists(update_log.ref_doctype, update_log.docname): - # Delete Log - # Check if the last Update Log of this document was read by this consumer - last_update_log = frappe.get_all( - "Event Update Log", - filters={ - "ref_doctype": update_log.ref_doctype, - "docname": update_log.docname, - "creation": ["<", update_log.creation], - }, - order_by="creation desc", - limit_page_length=1, - ) - if not len(last_update_log): - return False - - last_update_log = frappe.get_doc("Event Update Log", last_update_log[0].name) - return len([x for x in last_update_log.consumers if x.consumer == consumer.name]) - - doc = frappe.get_doc(update_log.ref_doctype, update_log.docname) - try: - for dt_entry in consumer.consumer_doctypes: - if dt_entry.ref_doctype != update_log.ref_doctype: - continue - - if not dt_entry.condition: - return True - - condition: str = dt_entry.condition - if condition.startswith("cmd:"): - cmd = condition.split("cmd:")[1].strip() - args = {"consumer": consumer, "doc": doc, "update_log": update_log} - return frappe.call(cmd, **args) - else: - return frappe.safe_eval(condition, frappe._dict(doc=doc)) - except Exception as e: - consumer.log_error("has_consumer_access error") - return False diff --git a/frappe/event_streaming/doctype/event_consumer/test_event_consumer.py b/frappe/event_streaming/doctype/event_consumer/test_event_consumer.py deleted file mode 100644 index 54bf718f17..0000000000 --- a/frappe/event_streaming/doctype/event_consumer/test_event_consumer.py +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright (c) 2019, Frappe Technologies and Contributors -# License: MIT. See LICENSE -# import frappe -from frappe.tests.utils import FrappeTestCase - - -class TestEventConsumer(FrappeTestCase): - pass diff --git a/frappe/event_streaming/doctype/event_consumer_document_type/__init__.py b/frappe/event_streaming/doctype/event_consumer_document_type/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/event_streaming/doctype/event_consumer_document_type/event_consumer_document_type.json b/frappe/event_streaming/doctype/event_consumer_document_type/event_consumer_document_type.json deleted file mode 100644 index c243334a09..0000000000 --- a/frappe/event_streaming/doctype/event_consumer_document_type/event_consumer_document_type.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "actions": [], - "creation": "2019-10-03 21:10:54.754651", - "doctype": "DocType", - "editable_grid": 1, - "engine": "InnoDB", - "field_order": [ - "ref_doctype", - "status", - "unsubscribed", - "condition" - ], - "fields": [ - { - "columns": 4, - "fieldname": "ref_doctype", - "fieldtype": "Link", - "in_list_view": 1, - "label": "Document Type", - "options": "DocType", - "read_only": 1, - "reqd": 1 - }, - { - "columns": 4, - "default": "Pending", - "fieldname": "status", - "fieldtype": "Select", - "in_list_view": 1, - "label": "Approval Status", - "options": "Pending\nApproved\nRejected" - }, - { - "columns": 2, - "default": "0", - "fieldname": "unsubscribed", - "fieldtype": "Check", - "in_list_view": 1, - "label": "Unsubscribed", - "read_only": 1 - }, - { - "fieldname": "condition", - "fieldtype": "Code", - "label": "Condition", - "read_only": 1 - } - ], - "istable": 1, - "links": [], - "modified": "2020-11-07 09:26:49.894294", - "modified_by": "Administrator", - "module": "Event Streaming", - "name": "Event Consumer Document Type", - "owner": "Administrator", - "permissions": [], - "quick_entry": 1, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1 -} \ No newline at end of file diff --git a/frappe/event_streaming/doctype/event_consumer_document_type/event_consumer_document_type.py b/frappe/event_streaming/doctype/event_consumer_document_type/event_consumer_document_type.py deleted file mode 100644 index 1ed15c5a75..0000000000 --- a/frappe/event_streaming/doctype/event_consumer_document_type/event_consumer_document_type.py +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright (c) 2019, Frappe Technologies and contributors -# License: MIT. See LICENSE - -# import frappe -from frappe.model.document import Document - - -class EventConsumerDocumentType(Document): - pass diff --git a/frappe/event_streaming/doctype/event_producer/__init__.py b/frappe/event_streaming/doctype/event_producer/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/event_streaming/doctype/event_producer/event_producer.js b/frappe/event_streaming/doctype/event_producer/event_producer.js deleted file mode 100644 index 23ca482433..0000000000 --- a/frappe/event_streaming/doctype/event_producer/event_producer.js +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright (c) 2019, Frappe Technologies and contributors -// For license information, please see license.txt - -frappe.ui.form.on("Event Producer", { - refresh: function (frm) { - frm.set_query("ref_doctype", "producer_doctypes", function () { - return { - filters: { - issingle: 0, - istable: 0, - }, - }; - }); - - frm.set_indicator_formatter("status", function (doc) { - let indicator = "orange"; - if (doc.status == "Approved") { - indicator = "green"; - } else if (doc.status == "Rejected") { - indicator = "red"; - } - return indicator; - }); - }, -}); diff --git a/frappe/event_streaming/doctype/event_producer/event_producer.json b/frappe/event_streaming/doctype/event_producer/event_producer.json deleted file mode 100644 index d868f6c123..0000000000 --- a/frappe/event_streaming/doctype/event_producer/event_producer.json +++ /dev/null @@ -1,96 +0,0 @@ -{ - "actions": [], - "autoname": "field:producer_url", - "creation": "2019-08-26 19:17:24.919196", - "doctype": "DocType", - "editable_grid": 1, - "engine": "InnoDB", - "field_order": [ - "producer_url", - "producer_doctypes", - "section_break_3", - "api_key", - "api_secret", - "column_break_6", - "user", - "incoming_change" - ], - "fields": [ - { - "fieldname": "producer_url", - "fieldtype": "Data", - "in_list_view": 1, - "label": "Producer URL", - "reqd": 1, - "unique": 1 - }, - { - "description": "API Key of the user(Event Subscriber) on the producer site", - "fieldname": "api_key", - "fieldtype": "Data", - "label": "API Key", - "reqd": 1 - }, - { - "description": "API Secret of the user(Event Subscriber) on the producer site", - "fieldname": "api_secret", - "fieldtype": "Password", - "label": "API Secret", - "reqd": 1 - }, - { - "fieldname": "user", - "fieldtype": "Link", - "label": "Event Subscriber", - "options": "User", - "reqd": 1, - "set_only_once": 1 - }, - { - "fieldname": "column_break_6", - "fieldtype": "Column Break" - }, - { - "fieldname": "section_break_3", - "fieldtype": "Section Break" - }, - { - "default": "0", - "fieldname": "incoming_change", - "fieldtype": "Check", - "hidden": 1, - "label": "Incoming Change" - }, - { - "fieldname": "producer_doctypes", - "fieldtype": "Table", - "label": "Event Producer Document Types", - "options": "Event Producer Document Type", - "reqd": 1 - } - ], - "links": [], - "modified": "2020-10-26 13:00:15.361316", - "modified_by": "Administrator", - "module": "Event Streaming", - "name": "Event Producer", - "owner": "Administrator", - "permissions": [ - { - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "share": 1, - "write": 1 - } - ], - "quick_entry": 1, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1 -} \ No newline at end of file diff --git a/frappe/event_streaming/doctype/event_producer/event_producer.py b/frappe/event_streaming/doctype/event_producer/event_producer.py deleted file mode 100644 index f91c8a4fd4..0000000000 --- a/frappe/event_streaming/doctype/event_producer/event_producer.py +++ /dev/null @@ -1,569 +0,0 @@ -# Copyright (c) 2019, Frappe Technologies and contributors -# License: MIT. See LICENSE - -import json -import time - -import requests - -import frappe -from frappe import _ -from frappe.custom.doctype.custom_field.custom_field import create_custom_field -from frappe.frappeclient import FrappeClient -from frappe.model.document import Document -from frappe.utils.background_jobs import get_jobs -from frappe.utils.data import get_link_to_form, get_url -from frappe.utils.password import get_decrypted_password - - -class EventProducer(Document): - def before_insert(self): - self.check_url() - self.validate_event_subscriber() - self.incoming_change = True - self.create_event_consumer() - self.create_custom_fields() - - def validate(self): - self.validate_event_subscriber() - if frappe.flags.in_test: - for entry in self.producer_doctypes: - entry.status = "Approved" - - def validate_event_subscriber(self): - if not frappe.db.get_value("User", self.user, "api_key"): - frappe.throw( - _("Please generate keys for the Event Subscriber User {0} first.").format( - frappe.bold(get_link_to_form("User", self.user)) - ) - ) - - def on_update(self): - if not self.incoming_change: - if frappe.db.exists("Event Producer", self.name): - if not self.api_key or not self.api_secret: - frappe.throw(_("Please set API Key and Secret on the producer and consumer sites first.")) - else: - doc_before_save = self.get_doc_before_save() - if doc_before_save.api_key != self.api_key or doc_before_save.api_secret != self.api_secret: - return - - self.update_event_consumer() - self.create_custom_fields() - else: - # when producer doc is updated it updates the consumer doc, set flag to avoid deadlock - self.db_set("incoming_change", 0) - self.reload() - - def on_trash(self): - last_update = frappe.db.get_value("Event Producer Last Update", dict(event_producer=self.name)) - if last_update: - frappe.delete_doc("Event Producer Last Update", last_update) - - def check_url(self): - valid_url_schemes = ("http", "https") - frappe.utils.validate_url(self.producer_url, throw=True, valid_schemes=valid_url_schemes) - - # remove '/' from the end of the url like http://test_site.com/ - # to prevent mismatch in get_url() results - if self.producer_url.endswith("/"): - self.producer_url = self.producer_url[:-1] - - def create_event_consumer(self): - """register event consumer on the producer site""" - if self.is_producer_online(): - producer_site = FrappeClient( - url=self.producer_url, api_key=self.api_key, api_secret=self.get_password("api_secret") - ) - - response = producer_site.post_api( - "frappe.event_streaming.doctype.event_consumer.event_consumer.register_consumer", - params={"data": json.dumps(self.get_request_data())}, - ) - if response: - response = json.loads(response) - self.set_last_update(response["last_update"]) - else: - frappe.throw( - _( - "Failed to create an Event Consumer or an Event Consumer for the current site is already registered." - ) - ) - - def set_last_update(self, last_update): - last_update_doc_name = frappe.db.get_value( - "Event Producer Last Update", dict(event_producer=self.name) - ) - if not last_update_doc_name: - frappe.get_doc( - dict( - doctype="Event Producer Last Update", - event_producer=self.producer_url, - last_update=last_update, - ) - ).insert(ignore_permissions=True) - else: - frappe.db.set_value( - "Event Producer Last Update", last_update_doc_name, "last_update", last_update - ) - - def get_last_update(self): - return frappe.db.get_value( - "Event Producer Last Update", dict(event_producer=self.name), "last_update" - ) - - def get_request_data(self): - consumer_doctypes = [] - for entry in self.producer_doctypes: - if entry.has_mapping: - # if mapping, subscribe to remote doctype on consumer's site - dt = frappe.db.get_value("Document Type Mapping", entry.mapping, "remote_doctype") - else: - dt = entry.ref_doctype - consumer_doctypes.append({"doctype": dt, "condition": entry.condition}) - - user_key = frappe.db.get_value("User", self.user, "api_key") - user_secret = get_decrypted_password("User", self.user, "api_secret") - return { - "event_consumer": get_url(), - "consumer_doctypes": json.dumps(consumer_doctypes), - "user": self.user, - "api_key": user_key, - "api_secret": user_secret, - } - - def create_custom_fields(self): - """create custom field to store remote docname and remote site url""" - for entry in self.producer_doctypes: - if not entry.use_same_name: - if not frappe.db.exists( - "Custom Field", {"fieldname": "remote_docname", "dt": entry.ref_doctype} - ): - df = dict( - fieldname="remote_docname", - label="Remote Document Name", - fieldtype="Data", - read_only=1, - print_hide=1, - ) - create_custom_field(entry.ref_doctype, df) - if not frappe.db.exists( - "Custom Field", {"fieldname": "remote_site_name", "dt": entry.ref_doctype} - ): - df = dict( - fieldname="remote_site_name", - label="Remote Site", - fieldtype="Data", - read_only=1, - print_hide=1, - ) - create_custom_field(entry.ref_doctype, df) - - def update_event_consumer(self): - if self.is_producer_online(): - producer_site = get_producer_site(self.producer_url) - event_consumer = producer_site.get_doc("Event Consumer", get_url()) - event_consumer = frappe._dict(event_consumer) - if event_consumer: - config = event_consumer.consumer_doctypes - event_consumer.consumer_doctypes = [] - for entry in self.producer_doctypes: - if entry.has_mapping: - # if mapping, subscribe to remote doctype on consumer's site - ref_doctype = frappe.db.get_value("Document Type Mapping", entry.mapping, "remote_doctype") - else: - ref_doctype = entry.ref_doctype - - event_consumer.consumer_doctypes.append( - { - "ref_doctype": ref_doctype, - "status": get_approval_status(config, ref_doctype), - "unsubscribed": entry.unsubscribe, - "condition": entry.condition, - } - ) - event_consumer.user = self.user - event_consumer.incoming_change = True - producer_site.update(event_consumer) - - def is_producer_online(self): - """check connection status for the Event Producer site""" - retry = 3 - while retry > 0: - res = requests.get(self.producer_url) - if res.status_code == 200: - return True - retry -= 1 - time.sleep(5) - frappe.throw(_("Failed to connect to the Event Producer site. Retry after some time.")) - - -def get_producer_site(producer_url): - """create a FrappeClient object for event producer site""" - producer_doc = frappe.get_doc("Event Producer", producer_url) - producer_site = FrappeClient( - url=producer_url, - api_key=producer_doc.api_key, - api_secret=producer_doc.get_password("api_secret"), - ) - return producer_site - - -def get_approval_status(config, ref_doctype): - """check the approval status for consumption""" - for entry in config: - if entry.get("ref_doctype") == ref_doctype: - return entry.get("status") - return "Pending" - - -@frappe.whitelist() -def pull_producer_data(): - """Fetch data from producer node.""" - response = requests.get(get_url()) - if response.status_code == 200: - for event_producer in frappe.get_all("Event Producer"): - pull_from_node(event_producer.name) - return "success" - return None - - -@frappe.whitelist() -def pull_from_node(event_producer): - """pull all updates after the last update timestamp from event producer site""" - event_producer = frappe.get_doc("Event Producer", event_producer) - producer_site = get_producer_site(event_producer.producer_url) - last_update = event_producer.get_last_update() - - (doctypes, mapping_config, naming_config) = get_config(event_producer.producer_doctypes) - - updates = get_updates(producer_site, last_update, doctypes) - - for update in updates: - update.use_same_name = naming_config.get(update.ref_doctype) - mapping = mapping_config.get(update.ref_doctype) - if mapping: - update.mapping = mapping - update = get_mapped_update(update, producer_site) - if not update.update_type == "Delete": - update.data = json.loads(update.data) - - sync(update, producer_site, event_producer) - - -def get_config(event_config): - """get the doctype mapping and naming configurations for consumption""" - doctypes, mapping_config, naming_config = [], {}, {} - - for entry in event_config: - if entry.status == "Approved": - if entry.has_mapping: - (mapped_doctype, mapping) = frappe.db.get_value( - "Document Type Mapping", entry.mapping, ["remote_doctype", "name"] - ) - mapping_config[mapped_doctype] = mapping - naming_config[mapped_doctype] = entry.use_same_name - doctypes.append(mapped_doctype) - else: - naming_config[entry.ref_doctype] = entry.use_same_name - doctypes.append(entry.ref_doctype) - return (doctypes, mapping_config, naming_config) - - -def sync(update, producer_site, event_producer, in_retry=False): - """Sync the individual update""" - try: - if update.update_type == "Create": - set_insert(update, producer_site, event_producer.name) - if update.update_type == "Update": - set_update(update, producer_site) - if update.update_type == "Delete": - set_delete(update) - if in_retry: - return "Synced" - log_event_sync(update, event_producer.name, "Synced") - - except Exception: - if in_retry: - if frappe.flags.in_test: - print(frappe.get_traceback()) - return "Failed" - log_event_sync(update, event_producer.name, "Failed", frappe.get_traceback()) - - event_producer.set_last_update(update.creation) - frappe.db.commit() - - -def set_insert(update, producer_site, event_producer): - """Sync insert type update""" - if frappe.db.get_value(update.ref_doctype, update.docname): - # doc already created - return - doc = frappe.get_doc(update.data) - - if update.mapping: - if update.get("dependencies"): - dependencies_created = sync_mapped_dependencies(update.dependencies, producer_site) - for fieldname, value in dependencies_created.items(): - doc.update({fieldname: value}) - else: - sync_dependencies(doc, producer_site) - - if update.use_same_name: - doc.insert(set_name=update.docname, set_child_names=False) - else: - # if event consumer is not saving documents with the same name as the producer - # store the remote docname in a custom field for future updates - doc.remote_docname = update.docname - doc.remote_site_name = event_producer - doc.insert(set_child_names=False) - - -def set_update(update, producer_site): - """Sync update type update""" - local_doc = get_local_doc(update) - if local_doc: - data = frappe._dict(update.data) - - if data.changed: - local_doc.update(data.changed) - if data.removed: - local_doc = update_row_removed(local_doc, data.removed) - if data.row_changed: - update_row_changed(local_doc, data.row_changed) - if data.added: - local_doc = update_row_added(local_doc, data.added) - - if update.mapping: - if update.get("dependencies"): - dependencies_created = sync_mapped_dependencies(update.dependencies, producer_site) - for fieldname, value in dependencies_created.items(): - local_doc.update({fieldname: value}) - else: - sync_dependencies(local_doc, producer_site) - - local_doc.save() - local_doc.db_update_all() - - -def update_row_removed(local_doc, removed): - """Sync child table row deletion type update""" - for tablename, rownames in removed.items(): - table = local_doc.get_table_field_doctype(tablename) - for row in rownames: - table_rows = local_doc.get(tablename) - child_table_row = get_child_table_row(table_rows, row) - table_rows.remove(child_table_row) - local_doc.set(tablename, table_rows) - return local_doc - - -def get_child_table_row(table_rows, row): - for entry in table_rows: - if entry.get("name") == row: - return entry - - -def update_row_changed(local_doc, changed): - """Sync child table row updation type update""" - for tablename, rows in changed.items(): - old = local_doc.get(tablename) - for doc in old: - for row in rows: - if row["name"] == doc.get("name"): - doc.update(row) - - -def update_row_added(local_doc, added): - """Sync child table row addition type update""" - for tablename, rows in added.items(): - local_doc.extend(tablename, rows) - for child in rows: - child_doc = frappe.get_doc(child) - child_doc.parent = local_doc.name - child_doc.parenttype = local_doc.doctype - child_doc.insert(set_name=child_doc.name) - return local_doc - - -def set_delete(update): - """Sync delete type update""" - local_doc = get_local_doc(update) - if local_doc: - local_doc.delete() - - -def get_updates(producer_site, last_update, doctypes): - """Get all updates generated after the last update timestamp""" - docs = producer_site.post_request( - { - "cmd": "frappe.event_streaming.doctype.event_update_log.event_update_log.get_update_logs_for_consumer", - "event_consumer": get_url(), - "doctypes": frappe.as_json(doctypes), - "last_update": last_update, - } - ) - return [frappe._dict(d) for d in (docs or [])] - - -def get_local_doc(update): - """Get the local document if created with a different name""" - try: - if not update.use_same_name: - return frappe.get_doc(update.ref_doctype, {"remote_docname": update.docname}) - return frappe.get_doc(update.ref_doctype, update.docname) - except frappe.DoesNotExistError: - return None - - -def sync_dependencies(document, producer_site): - """ - dependencies is a dictionary to store all the docs - having dependencies and their sync status, - which is shared among all nested functions. - """ - dependencies = {document: True} - - def check_doc_has_dependencies(doc, producer_site): - """Sync child table link fields first, - then sync link fields, - then dynamic links""" - meta = frappe.get_meta(doc.doctype) - table_fields = meta.get_table_fields() - link_fields = meta.get_link_fields() - dl_fields = meta.get_dynamic_link_fields() - if table_fields: - sync_child_table_dependencies(doc, table_fields, producer_site) - if link_fields: - sync_link_dependencies(doc, link_fields, producer_site) - if dl_fields: - sync_dynamic_link_dependencies(doc, dl_fields, producer_site) - - def sync_child_table_dependencies(doc, table_fields, producer_site): - for df in table_fields: - child_table = doc.get(df.fieldname) - for entry in child_table: - child_doc = producer_site.get_doc(entry.doctype, entry.name) - if child_doc: - child_doc = frappe._dict(child_doc) - set_dependencies(child_doc, frappe.get_meta(entry.doctype).get_link_fields(), producer_site) - - def sync_link_dependencies(doc, link_fields, producer_site): - set_dependencies(doc, link_fields, producer_site) - - def sync_dynamic_link_dependencies(doc, dl_fields, producer_site): - for df in dl_fields: - docname = doc.get(df.fieldname) - linked_doctype = doc.get(df.options) - if docname and not check_dependency_fulfilled(linked_doctype, docname): - master_doc = producer_site.get_doc(linked_doctype, docname) - frappe.get_doc(master_doc).insert(set_name=docname) - - def set_dependencies(doc, link_fields, producer_site): - for df in link_fields: - docname = doc.get(df.fieldname) - linked_doctype = df.get_link_doctype() - if docname and not check_dependency_fulfilled(linked_doctype, docname): - master_doc = producer_site.get_doc(linked_doctype, docname) - try: - master_doc = frappe.get_doc(master_doc) - master_doc.insert(set_name=docname) - frappe.db.commit() - - # for dependency inside a dependency - except Exception: - dependencies[master_doc] = True - - def check_dependency_fulfilled(linked_doctype, docname): - return frappe.db.exists(linked_doctype, docname) - - while dependencies[document]: - # find the first non synced dependency - for item in reversed(list(dependencies.keys())): - if dependencies[item]: - dependency = item - break - - check_doc_has_dependencies(dependency, producer_site) - - # mark synced for nested dependency - if dependency != document: - dependencies[dependency] = False - dependency.insert() - - # no more dependencies left to be synced, the main doc is ready to be synced - # end the dependency loop - if not any(list(dependencies.values())[1:]): - dependencies[document] = False - - -def sync_mapped_dependencies(dependencies, producer_site): - dependencies_created = {} - for entry in dependencies: - doc = frappe._dict(json.loads(entry[1])) - docname = frappe.db.exists(doc.doctype, doc.name) - if not docname: - doc = frappe.get_doc(doc).insert(set_child_names=False) - dependencies_created[entry[0]] = doc.name - else: - dependencies_created[entry[0]] = docname - - return dependencies_created - - -def log_event_sync(update, event_producer, sync_status, error=None): - """Log event update received with the sync_status as Synced or Failed""" - doc = frappe.new_doc("Event Sync Log") - doc.update_type = update.update_type - doc.ref_doctype = update.ref_doctype - doc.status = sync_status - doc.event_producer = event_producer - doc.producer_doc = update.docname - doc.data = frappe.as_json(update.data) - doc.use_same_name = update.use_same_name - doc.mapping = update.mapping if update.mapping else None - if update.use_same_name: - doc.docname = update.docname - else: - doc.docname = frappe.db.get_value(update.ref_doctype, {"remote_docname": update.docname}, "name") - if error: - doc.error = error - doc.insert() - - -def get_mapped_update(update, producer_site): - """get the new update document with mapped fields""" - mapping = frappe.get_doc("Document Type Mapping", update.mapping) - if update.update_type == "Create": - doc = frappe._dict(json.loads(update.data)) - mapped_update = mapping.get_mapping(doc, producer_site, update.update_type) - update.data = mapped_update.get("doc") - update.dependencies = mapped_update.get("dependencies", None) - elif update.update_type == "Update": - mapped_update = mapping.get_mapped_update(update, producer_site) - update.data = mapped_update.get("doc") - update.dependencies = mapped_update.get("dependencies", None) - - update["ref_doctype"] = mapping.local_doctype - return update - - -@frappe.whitelist() -def new_event_notification(producer_url): - """Pull data from producer when notified""" - enqueued_method = "frappe.event_streaming.doctype.event_producer.event_producer.pull_from_node" - jobs = get_jobs() - if not jobs or enqueued_method not in jobs[frappe.local.site]: - frappe.enqueue(enqueued_method, queue="default", **{"event_producer": producer_url}) - - -@frappe.whitelist() -def resync(update): - """Retry syncing update if failed""" - update = frappe._dict(json.loads(update)) - producer_site = get_producer_site(update.event_producer) - event_producer = frappe.get_doc("Event Producer", update.event_producer) - if update.mapping: - update = get_mapped_update(update, producer_site) - update.data = json.loads(update.data) - return sync(update, producer_site, event_producer, in_retry=True) diff --git a/frappe/event_streaming/doctype/event_producer/test_event_producer.py b/frappe/event_streaming/doctype/event_producer/test_event_producer.py deleted file mode 100644 index 70e7483f92..0000000000 --- a/frappe/event_streaming/doctype/event_producer/test_event_producer.py +++ /dev/null @@ -1,438 +0,0 @@ -# Copyright (c) 2019, Frappe Technologies and Contributors -# License: MIT. See LICENSE -import json - -import frappe -from frappe.core.doctype.user.user import generate_keys -from frappe.event_streaming.doctype.event_producer.event_producer import pull_from_node -from frappe.frappeclient import FrappeClient -from frappe.query_builder.utils import db_type_is -from frappe.tests.test_query_builder import run_only_if -from frappe.tests.utils import FrappeTestCase - -producer_url = "http://test_site_producer:8000" - - -class TestEventProducer(FrappeTestCase): - def setUp(self): - create_event_producer(producer_url) - - def tearDown(self): - unsubscribe_doctypes(producer_url) - - def test_insert(self): - producer = get_remote_site() - producer_doc = insert_into_producer(producer, "test creation 1 sync") - self.pull_producer_data() - self.assertTrue(frappe.db.exists("ToDo", producer_doc.name)) - - def test_update(self): - producer = get_remote_site() - producer_doc = insert_into_producer(producer, "test update 1") - producer_doc["description"] = "test update 2" - producer_doc = producer.update(producer_doc) - self.pull_producer_data() - local_doc = frappe.get_doc(producer_doc.doctype, producer_doc.name) - self.assertEqual(local_doc.description, producer_doc.description) - - def test_delete(self): - producer = get_remote_site() - producer_doc = insert_into_producer(producer, "test delete sync") - self.pull_producer_data() - self.assertTrue(frappe.db.exists("ToDo", producer_doc.name)) - producer.delete("ToDo", producer_doc.name) - self.pull_producer_data() - self.assertFalse(frappe.db.exists("ToDo", producer_doc.name)) - - @run_only_if(db_type_is.MARIADB) - def test_multiple_doctypes_sync(self): - # TODO: This test is extremely flaky with Postgres. Rewrite this! - producer = get_remote_site() - - # insert todo and note in producer - producer_todo = insert_into_producer(producer, "test multiple doc sync") - producer_note1 = frappe._dict(doctype="Note", title="test multiple doc sync 1") - delete_on_remote_if_exists(producer, "Note", {"title": producer_note1["title"]}) - frappe.db.delete("Note", {"title": producer_note1["title"]}) - producer_note1 = producer.insert(producer_note1) - producer_note2 = frappe._dict(doctype="Note", title="test multiple doc sync 2") - delete_on_remote_if_exists(producer, "Note", {"title": producer_note2["title"]}) - frappe.db.delete("Note", {"title": producer_note2["title"]}) - producer_note2 = producer.insert(producer_note2) - - # update in producer - producer_todo["description"] = "test multiple doc update sync" - producer_todo = producer.update(producer_todo) - producer_note1["content"] = "testing update sync" - producer_note1 = producer.update(producer_note1) - - producer.delete("Note", producer_note2.name) - - self.pull_producer_data() - - # check inserted - self.assertTrue(frappe.db.exists("ToDo", producer_todo.name)) - - # check update - local_todo = frappe.get_doc("ToDo", producer_todo.name) - self.assertEqual(local_todo.description, producer_todo.description) - local_note1 = frappe.get_doc("Note", producer_note1.name) - self.assertEqual(local_note1.content, producer_note1.content) - - # check delete - self.assertFalse(frappe.db.exists("Note", producer_note2.name)) - - def test_child_table_sync_with_dependencies(self): - producer = get_remote_site() - producer_user = frappe._dict( - doctype="User", - email="test_user@sync.com", - send_welcome_email=0, - first_name="Test Sync User", - enabled=1, - roles=[{"role": "System Manager"}], - ) - delete_on_remote_if_exists(producer, "User", {"email": producer_user.email}) - frappe.db.delete("User", {"email": producer_user.email}) - producer_user = producer.insert(producer_user) - - producer_note = frappe._dict( - doctype="Note", title="test child table dependency sync", seen_by=[{"user": producer_user.name}] - ) - delete_on_remote_if_exists(producer, "Note", {"title": producer_note.title}) - frappe.db.delete("Note", {"title": producer_note.title}) - producer_note = producer.insert(producer_note) - - self.pull_producer_data() - self.assertTrue(frappe.db.exists("User", producer_user.name)) - if self.assertTrue(frappe.db.exists("Note", producer_note.name)): - local_note = frappe.get_doc("Note", producer_note.name) - self.assertEqual(len(local_note.seen_by), 1) - - def test_dynamic_link_dependencies_synced(self): - producer = get_remote_site() - # unsubscribe for Note to check whether dependency is fulfilled - event_producer = frappe.get_doc("Event Producer", producer_url, for_update=True) - event_producer.producer_doctypes = [] - event_producer.append("producer_doctypes", {"ref_doctype": "ToDo", "use_same_name": 1}) - event_producer.save() - - producer_link_doc = frappe._dict(doctype="Note", title="Test Dynamic Link 1") - - delete_on_remote_if_exists(producer, "Note", {"title": producer_link_doc.title}) - frappe.db.delete("Note", {"title": producer_link_doc.title}) - producer_link_doc = producer.insert(producer_link_doc) - producer_doc = frappe._dict( - doctype="ToDo", - description="Test Dynamic Link 2", - assigned_by="Administrator", - reference_type="Note", - reference_name=producer_link_doc.name, - ) - producer_doc = producer.insert(producer_doc) - - self.pull_producer_data() - - # check dynamic link dependency created - self.assertTrue(frappe.db.exists("Note", producer_link_doc.name)) - self.assertEqual( - producer_link_doc.name, frappe.db.get_value("ToDo", producer_doc.name, "reference_name") - ) - - reset_configuration(producer_url) - - def test_naming_configuration(self): - # test with use_same_name = 0 - producer = get_remote_site() - event_producer = frappe.get_doc("Event Producer", producer_url, for_update=True) - event_producer.producer_doctypes = [] - event_producer.append("producer_doctypes", {"ref_doctype": "ToDo", "use_same_name": 0}) - event_producer.save() - - producer_doc = insert_into_producer(producer, "test different name sync") - self.pull_producer_data() - self.assertTrue( - frappe.db.exists( - "ToDo", {"remote_docname": producer_doc.name, "remote_site_name": producer_url} - ) - ) - - reset_configuration(producer_url) - - def test_conditional_events(self): - producer = get_remote_site() - - # Add Condition - event_producer = frappe.get_doc("Event Producer", producer_url) - note_producer_entry = [x for x in event_producer.producer_doctypes if x.ref_doctype == "Note"][0] - note_producer_entry.condition = "doc.public == 1" - event_producer.save() - - # Make test doc - producer_note1 = frappe._dict(doctype="Note", public=0, title="test conditional sync") - delete_on_remote_if_exists(producer, "Note", {"title": producer_note1["title"]}) - producer_note1 = producer.insert(producer_note1) - - # Make Update - producer_note1["content"] = "Test Conditional Sync Content" - producer_note1 = producer.update(producer_note1) - - self.pull_producer_data() - - # Check if synced here - self.assertFalse(frappe.db.exists("Note", producer_note1.name)) - - # Lets satisfy the condition - producer_note1["public"] = 1 - producer_note1 = producer.update(producer_note1) - - self.pull_producer_data() - - # it should sync now - self.assertTrue(frappe.db.exists("Note", producer_note1.name)) - local_note = frappe.get_doc("Note", producer_note1.name) - self.assertEqual(local_note.content, producer_note1.content) - - reset_configuration(producer_url) - - def test_conditional_events_with_cmd(self): - producer = get_remote_site() - - # Add Condition - event_producer = frappe.get_doc("Event Producer", producer_url) - note_producer_entry = [x for x in event_producer.producer_doctypes if x.ref_doctype == "Note"][0] - note_producer_entry.condition = ( - "cmd: frappe.event_streaming.doctype.event_producer.test_event_producer.can_sync_note" - ) - event_producer.save() - - # Make test doc - producer_note1 = frappe._dict(doctype="Note", public=0, title="test conditional sync cmd") - delete_on_remote_if_exists(producer, "Note", {"title": producer_note1["title"]}) - producer_note1 = producer.insert(producer_note1) - - # Make Update - producer_note1["content"] = "Test Conditional Sync Content" - producer_note1 = producer.update(producer_note1) - - self.pull_producer_data() - - # Check if synced here - self.assertFalse(frappe.db.exists("Note", producer_note1.name)) - - # Lets satisfy the condition - producer_note1["public"] = 1 - producer_note1 = producer.update(producer_note1) - - self.pull_producer_data() - - # it should sync now - self.assertTrue(frappe.db.exists("Note", producer_note1.name)) - local_note = frappe.get_doc("Note", producer_note1.name) - self.assertEqual(local_note.content, producer_note1.content) - - reset_configuration(producer_url) - - def test_update_log(self): - producer = get_remote_site() - producer_doc = insert_into_producer(producer, "test update log") - update_log_doc = producer.get_value( - "Event Update Log", "docname", {"docname": producer_doc.get("name")} - ) - self.assertEqual(update_log_doc.get("docname"), producer_doc.get("name")) - - def test_event_sync_log(self): - producer = get_remote_site() - producer_doc = insert_into_producer(producer, "test event sync log") - self.pull_producer_data() - self.assertTrue(frappe.db.exists("Event Sync Log", {"docname": producer_doc.name})) - - def pull_producer_data(self): - pull_from_node(producer_url) - - def test_mapping(self): - producer = get_remote_site() - event_producer = frappe.get_doc("Event Producer", producer_url, for_update=True) - event_producer.producer_doctypes = [] - mapping = [{"local_fieldname": "description", "remote_fieldname": "content"}] - event_producer.append( - "producer_doctypes", - { - "ref_doctype": "ToDo", - "use_same_name": 1, - "has_mapping": 1, - "mapping": get_mapping("ToDo to Note", "ToDo", "Note", mapping), - }, - ) - event_producer.save() - - producer_note = frappe._dict(doctype="Note", title="Test Mapping", content="Test Mapping") - delete_on_remote_if_exists(producer, "Note", {"title": producer_note.title}) - producer_note = producer.insert(producer_note) - self.pull_producer_data() - # check inserted - self.assertTrue(frappe.db.exists("ToDo", {"description": producer_note.content})) - - # update in producer - producer_note["content"] = "test mapped doc update sync" - producer_note = producer.update(producer_note) - self.pull_producer_data() - - # check updated - self.assertTrue(frappe.db.exists("ToDo", {"description": producer_note["content"]})) - - producer.delete("Note", producer_note.name) - self.pull_producer_data() - # check delete - self.assertFalse(frappe.db.exists("ToDo", {"description": producer_note.content})) - - reset_configuration(producer_url) - - def test_inner_mapping(self): - producer = get_remote_site() - - setup_event_producer_for_inner_mapping() - producer_note = frappe._dict( - doctype="Note", title="Inner Mapping Tester", content="Test Inner Mapping" - ) - delete_on_remote_if_exists(producer, "Note", {"title": producer_note.title}) - producer_note = producer.insert(producer_note) - self.pull_producer_data() - - # check dependency inserted - self.assertTrue(frappe.db.exists("Role", {"role_name": producer_note.title})) - # check doc inserted - self.assertTrue(frappe.db.exists("ToDo", {"description": producer_note.content})) - - reset_configuration(producer_url) - - -def can_sync_note(consumer, doc, update_log): - return doc.public == 1 - - -def setup_event_producer_for_inner_mapping(): - event_producer = frappe.get_doc("Event Producer", producer_url, for_update=True) - event_producer.producer_doctypes = [] - inner_mapping = [{"local_fieldname": "role_name", "remote_fieldname": "title"}] - inner_map = get_mapping("Role to Note Dependency Creation", "Role", "Note", inner_mapping) - mapping = [ - { - "local_fieldname": "description", - "remote_fieldname": "content", - }, - { - "local_fieldname": "role", - "remote_fieldname": "title", - "mapping_type": "Document", - "mapping": inner_map, - "remote_value_filters": json.dumps({"title": "title"}), - }, - ] - event_producer.append( - "producer_doctypes", - { - "ref_doctype": "ToDo", - "use_same_name": 1, - "has_mapping": 1, - "mapping": get_mapping("ToDo to Note Mapping", "ToDo", "Note", mapping), - }, - ) - event_producer.save() - return event_producer - - -def insert_into_producer(producer, description): - # create and insert todo on remote site - todo = dict(doctype="ToDo", description=description, assigned_by="Administrator") - return producer.insert(todo) - - -def delete_on_remote_if_exists(producer, doctype, filters): - remote_doc = producer.get_value(doctype, "name", filters) - if remote_doc: - producer.delete(doctype, remote_doc.get("name")) - - -def get_mapping(mapping_name, local, remote, field_map): - name = frappe.db.exists("Document Type Mapping", mapping_name) - if name: - doc = frappe.get_doc("Document Type Mapping", name) - else: - doc = frappe.new_doc("Document Type Mapping") - - doc.mapping_name = mapping_name - doc.local_doctype = local - doc.remote_doctype = remote - for entry in field_map: - doc.append("field_mapping", entry) - doc.save() - return doc.name - - -def create_event_producer(producer_url): - if frappe.db.exists("Event Producer", producer_url): - event_producer = frappe.get_doc("Event Producer", producer_url) - for entry in event_producer.producer_doctypes: - entry.unsubscribe = 0 - event_producer.save() - return - - generate_keys("Administrator") - - producer_site = connect() - - response = producer_site.post_api( - "frappe.core.doctype.user.user.generate_keys", params={"user": "Administrator"} - ) - - api_secret = response.get("api_secret") - - response = producer_site.get_value("User", "api_key", {"name": "Administrator"}) - api_key = response.get("api_key") - - event_producer = frappe.new_doc("Event Producer") - event_producer.producer_doctypes = [] - event_producer.producer_url = producer_url - event_producer.append("producer_doctypes", {"ref_doctype": "ToDo", "use_same_name": 1}) - event_producer.append("producer_doctypes", {"ref_doctype": "Note", "use_same_name": 1}) - event_producer.user = "Administrator" - event_producer.api_key = api_key - event_producer.api_secret = api_secret - event_producer.save() - - -def reset_configuration(producer_url): - event_producer = frappe.get_doc("Event Producer", producer_url, for_update=True) - event_producer.producer_doctypes = [] - event_producer.conditions = [] - event_producer.producer_url = producer_url - event_producer.append("producer_doctypes", {"ref_doctype": "ToDo", "use_same_name": 1}) - event_producer.append("producer_doctypes", {"ref_doctype": "Note", "use_same_name": 1}) - event_producer.user = "Administrator" - event_producer.save() - - -def get_remote_site(): - producer_doc = frappe.get_doc("Event Producer", producer_url) - producer_site = FrappeClient( - url=producer_doc.producer_url, username="Administrator", password="admin", verify=False - ) - return producer_site - - -def unsubscribe_doctypes(producer_url): - event_producer = frappe.get_doc("Event Producer", producer_url) - for entry in event_producer.producer_doctypes: - entry.unsubscribe = 1 - event_producer.save() - - -def connect(): - def _connect(): - return FrappeClient(url=producer_url, username="Administrator", password="admin", verify=False) - - try: - return _connect() - except Exception: - return _connect() diff --git a/frappe/event_streaming/doctype/event_producer_document_type/__init__.py b/frappe/event_streaming/doctype/event_producer_document_type/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/event_streaming/doctype/event_producer_document_type/event_producer_document_type.json b/frappe/event_streaming/doctype/event_producer_document_type/event_producer_document_type.json deleted file mode 100644 index 17fd51d12d..0000000000 --- a/frappe/event_streaming/doctype/event_producer_document_type/event_producer_document_type.json +++ /dev/null @@ -1,86 +0,0 @@ -{ - "actions": [], - "creation": "2019-10-03 21:08:25.890352", - "doctype": "DocType", - "editable_grid": 1, - "engine": "InnoDB", - "field_order": [ - "ref_doctype", - "status", - "use_same_name", - "unsubscribe", - "has_mapping", - "mapping", - "condition" - ], - "fields": [ - { - "columns": 3, - "fieldname": "ref_doctype", - "fieldtype": "Link", - "in_list_view": 1, - "label": "Document Type", - "options": "DocType", - "reqd": 1, - "set_only_once": 1 - }, - { - "default": "0", - "description": "If the document has different field names on the Producer and Consumer's end check this and set up the Mapping", - "fieldname": "has_mapping", - "fieldtype": "Check", - "label": "Has Mapping" - }, - { - "depends_on": "eval: doc.has_mapping", - "fieldname": "mapping", - "fieldtype": "Link", - "label": "Mapping", - "options": "Document Type Mapping" - }, - { - "columns": 2, - "default": "0", - "description": "If this is checked the documents will have the same name as they have on the Event Producer's site", - "fieldname": "use_same_name", - "fieldtype": "Check", - "in_list_view": 1, - "label": "Use Same Name" - }, - { - "columns": 3, - "default": "Pending", - "fieldname": "status", - "fieldtype": "Select", - "in_list_view": 1, - "label": "Approval Status", - "options": "Pending\nApproved\nRejected", - "read_only": 1 - }, - { - "columns": 2, - "default": "0", - "fieldname": "unsubscribe", - "fieldtype": "Check", - "in_list_view": 1, - "label": "Unsubscribe" - }, - { - "fieldname": "condition", - "fieldtype": "Code", - "label": "Condition" - } - ], - "istable": 1, - "links": [], - "modified": "2020-11-07 09:26:58.463868", - "modified_by": "Administrator", - "module": "Event Streaming", - "name": "Event Producer Document Type", - "owner": "Administrator", - "permissions": [], - "quick_entry": 1, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1 -} \ No newline at end of file diff --git a/frappe/event_streaming/doctype/event_producer_document_type/event_producer_document_type.py b/frappe/event_streaming/doctype/event_producer_document_type/event_producer_document_type.py deleted file mode 100644 index 8f4c936792..0000000000 --- a/frappe/event_streaming/doctype/event_producer_document_type/event_producer_document_type.py +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright (c) 2019, Frappe Technologies and contributors -# License: MIT. See LICENSE - -# import frappe -from frappe.model.document import Document - - -class EventProducerDocumentType(Document): - pass diff --git a/frappe/event_streaming/doctype/event_producer_last_update/__init__.py b/frappe/event_streaming/doctype/event_producer_last_update/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/event_streaming/doctype/event_producer_last_update/event_producer_last_update.js b/frappe/event_streaming/doctype/event_producer_last_update/event_producer_last_update.js deleted file mode 100644 index 6d18be43e3..0000000000 --- a/frappe/event_streaming/doctype/event_producer_last_update/event_producer_last_update.js +++ /dev/null @@ -1,7 +0,0 @@ -// Copyright (c) 2020, Frappe Technologies and contributors -// For license information, please see license.txt - -frappe.ui.form.on("Event Producer Last Update", { - // refresh: function(frm) { - // } -}); diff --git a/frappe/event_streaming/doctype/event_producer_last_update/event_producer_last_update.json b/frappe/event_streaming/doctype/event_producer_last_update/event_producer_last_update.json deleted file mode 100644 index 27f8ed2f81..0000000000 --- a/frappe/event_streaming/doctype/event_producer_last_update/event_producer_last_update.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "actions": [], - "autoname": "field:event_producer", - "creation": "2020-10-26 12:53:11.940177", - "doctype": "DocType", - "editable_grid": 1, - "engine": "InnoDB", - "field_order": [ - "event_producer", - "last_update" - ], - "fields": [ - { - "fieldname": "event_producer", - "fieldtype": "Data", - "in_list_view": 1, - "label": "Event Producer", - "reqd": 1, - "unique": 1 - }, - { - "fieldname": "last_update", - "fieldtype": "Data", - "label": "Last Update" - } - ], - "in_create": 1, - "index_web_pages_for_search": 1, - "links": [], - "modified": "2020-10-26 13:22:27.056599", - "modified_by": "Administrator", - "module": "Event Streaming", - "name": "Event Producer Last Update", - "owner": "Administrator", - "permissions": [ - { - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "share": 1, - "write": 1 - } - ], - "read_only": 1, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1 -} \ No newline at end of file diff --git a/frappe/event_streaming/doctype/event_producer_last_update/event_producer_last_update.py b/frappe/event_streaming/doctype/event_producer_last_update/event_producer_last_update.py deleted file mode 100644 index ec5cee7e78..0000000000 --- a/frappe/event_streaming/doctype/event_producer_last_update/event_producer_last_update.py +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright (c) 2020, Frappe Technologies and contributors -# License: MIT. See LICENSE - -# import frappe -from frappe.model.document import Document - - -class EventProducerLastUpdate(Document): - pass diff --git a/frappe/event_streaming/doctype/event_producer_last_update/test_event_producer_last_update.py b/frappe/event_streaming/doctype/event_producer_last_update/test_event_producer_last_update.py deleted file mode 100644 index 3e68159790..0000000000 --- a/frappe/event_streaming/doctype/event_producer_last_update/test_event_producer_last_update.py +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright (c) 2020, Frappe Technologies and Contributors -# License: MIT. See LICENSE -# import frappe -from frappe.tests.utils import FrappeTestCase - - -class TestEventProducerLastUpdate(FrappeTestCase): - pass diff --git a/frappe/event_streaming/doctype/event_sync_log/__init__.py b/frappe/event_streaming/doctype/event_sync_log/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/event_streaming/doctype/event_sync_log/event_sync_log.js b/frappe/event_streaming/doctype/event_sync_log/event_sync_log.js deleted file mode 100644 index 7cc3198bae..0000000000 --- a/frappe/event_streaming/doctype/event_sync_log/event_sync_log.js +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright (c) 2019, Frappe Technologies and contributors -// For license information, please see license.txt - -frappe.ui.form.on("Event Sync Log", { - refresh: function (frm) { - if (frm.doc.status == "Failed") { - frm.add_custom_button(__("Resync"), function () { - frappe.call({ - method: "frappe.event_streaming.doctype.event_producer.event_producer.resync", - args: { - update: frm.doc, - }, - callback: function (r) { - if (r.message) { - frappe.msgprint(r.message); - frm.set_value("status", r.message); - frm.save(); - } - }, - }); - }); - } - }, -}); diff --git a/frappe/event_streaming/doctype/event_sync_log/event_sync_log.json b/frappe/event_streaming/doctype/event_sync_log/event_sync_log.json deleted file mode 100644 index f82128bd7b..0000000000 --- a/frappe/event_streaming/doctype/event_sync_log/event_sync_log.json +++ /dev/null @@ -1,137 +0,0 @@ -{ - "creation": "2019-09-24 22:22:05.845089", - "doctype": "DocType", - "editable_grid": 1, - "engine": "InnoDB", - "field_order": [ - "update_type", - "ref_doctype", - "docname", - "column_break_4", - "status", - "event_producer", - "producer_doc", - "event_configurations_section", - "use_same_name", - "column_break_9", - "mapping", - "section_break_8", - "data", - "error" - ], - "fields": [ - { - "fieldname": "update_type", - "fieldtype": "Select", - "in_list_view": 1, - "label": "Update Type", - "options": "Create\nUpdate\nDelete", - "read_only": 1 - }, - { - "fieldname": "ref_doctype", - "fieldtype": "Link", - "label": "Doctype", - "options": "DocType", - "read_only": 1 - }, - { - "fieldname": "docname", - "fieldtype": "Data", - "in_list_view": 1, - "label": "Document Name", - "options": "ref_doctype", - "read_only": 1 - }, - { - "fieldname": "column_break_4", - "fieldtype": "Column Break" - }, - { - "fieldname": "status", - "fieldtype": "Select", - "in_list_view": 1, - "label": "Status", - "options": "\nSynced\nFailed", - "read_only": 1 - }, - { - "fieldname": "event_producer", - "fieldtype": "Data", - "in_list_view": 1, - "label": "Event Producer", - "options": "Event Producer", - "read_only": 1 - }, - { - "fieldname": "section_break_8", - "fieldtype": "Section Break", - "label": "Data" - }, - { - "fieldname": "data", - "fieldtype": "Code", - "label": "Data", - "read_only": 1 - }, - { - "fieldname": "producer_doc", - "fieldtype": "Data", - "label": "Producer Document Name", - "read_only": 1 - }, - { - "depends_on": "eval:doc.status=='Failed'", - "fieldname": "error", - "fieldtype": "Code", - "label": "Error", - "read_only": 1 - }, - { - "fieldname": "event_configurations_section", - "fieldtype": "Section Break", - "label": "Event Configurations" - }, - { - "default": "0", - "fieldname": "use_same_name", - "fieldtype": "Data", - "label": "Use Same Name", - "read_only": 1 - }, - { - "fieldname": "column_break_9", - "fieldtype": "Column Break" - }, - { - "fieldname": "mapping", - "fieldtype": "Data", - "label": "Mapping", - "read_only": 1 - } - ], - "in_create": 1, - "modified": "2019-10-07 13:22:10.401479", - "modified_by": "Administrator", - "module": "Event Streaming", - "name": "Event Sync Log", - "owner": "Administrator", - "permissions": [ - { - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "share": 1, - "write": 1 - } - ], - "quick_entry": 1, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1 -} \ No newline at end of file diff --git a/frappe/event_streaming/doctype/event_sync_log/event_sync_log.py b/frappe/event_streaming/doctype/event_sync_log/event_sync_log.py deleted file mode 100644 index a1d82ad08f..0000000000 --- a/frappe/event_streaming/doctype/event_sync_log/event_sync_log.py +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright (c) 2019, Frappe Technologies and contributors -# License: MIT. See LICENSE - -# import frappe -from frappe.model.document import Document - - -class EventSyncLog(Document): - pass diff --git a/frappe/event_streaming/doctype/event_sync_log/event_sync_log_list.js b/frappe/event_streaming/doctype/event_sync_log/event_sync_log_list.js deleted file mode 100644 index 97d2ee0a1d..0000000000 --- a/frappe/event_streaming/doctype/event_sync_log/event_sync_log_list.js +++ /dev/null @@ -1,9 +0,0 @@ -frappe.listview_settings["Event Sync Log"] = { - get_indicator: function (doc) { - var colors = { - Failed: "red", - Synced: "green", - }; - return [__(doc.status), colors[doc.status], "status,=," + doc.status]; - }, -}; diff --git a/frappe/event_streaming/doctype/event_sync_log/test_event_sync_log.py b/frappe/event_streaming/doctype/event_sync_log/test_event_sync_log.py deleted file mode 100644 index 5efc030026..0000000000 --- a/frappe/event_streaming/doctype/event_sync_log/test_event_sync_log.py +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright (c) 2019, Frappe Technologies and Contributors -# License: MIT. See LICENSE -# import frappe -from frappe.tests.utils import FrappeTestCase - - -class TestEventSyncLog(FrappeTestCase): - pass diff --git a/frappe/event_streaming/doctype/event_update_log/__init__.py b/frappe/event_streaming/doctype/event_update_log/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/event_streaming/doctype/event_update_log/event_update_log.js b/frappe/event_streaming/doctype/event_update_log/event_update_log.js deleted file mode 100644 index d901799780..0000000000 --- a/frappe/event_streaming/doctype/event_update_log/event_update_log.js +++ /dev/null @@ -1,7 +0,0 @@ -// Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and contributors -// For license information, please see license.txt - -frappe.ui.form.on("Event Update Log", { - // refresh: function(frm) { - // } -}); diff --git a/frappe/event_streaming/doctype/event_update_log/event_update_log.json b/frappe/event_streaming/doctype/event_update_log/event_update_log.json deleted file mode 100644 index a42bc7ec87..0000000000 --- a/frappe/event_streaming/doctype/event_update_log/event_update_log.json +++ /dev/null @@ -1,77 +0,0 @@ -{ - "actions": [], - "creation": "2019-07-30 15:31:26.352527", - "doctype": "DocType", - "editable_grid": 1, - "engine": "InnoDB", - "field_order": [ - "update_type", - "ref_doctype", - "docname", - "data", - "consumers" - ], - "fields": [ - { - "fieldname": "update_type", - "fieldtype": "Select", - "in_list_view": 1, - "label": "Update Type", - "options": "Create\nUpdate\nDelete", - "read_only": 1 - }, - { - "fieldname": "ref_doctype", - "fieldtype": "Link", - "in_list_view": 1, - "label": "DocType", - "options": "DocType", - "read_only": 1 - }, - { - "fieldname": "docname", - "fieldtype": "Data", - "in_list_view": 1, - "label": "Document Name", - "read_only": 1 - }, - { - "fieldname": "data", - "fieldtype": "Code", - "label": "Data", - "read_only": 1 - }, - { - "fieldname": "consumers", - "fieldtype": "Table MultiSelect", - "label": "Consumers", - "options": "Event Update Log Consumer", - "read_only": 1 - } - ], - "in_create": 1, - "links": [], - "modified": "2020-09-04 07:31:52.599804", - "modified_by": "Administrator", - "module": "Event Streaming", - "name": "Event Update Log", - "owner": "Administrator", - "permissions": [ - { - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "share": 1, - "write": 1 - } - ], - "quick_entry": 1, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1 -} \ No newline at end of file diff --git a/frappe/event_streaming/doctype/event_update_log/event_update_log.py b/frappe/event_streaming/doctype/event_update_log/event_update_log.py deleted file mode 100644 index e40f600484..0000000000 --- a/frappe/event_streaming/doctype/event_update_log/event_update_log.py +++ /dev/null @@ -1,296 +0,0 @@ -# Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and contributors -# License: MIT. See LICENSE - -import frappe -from frappe.model import no_value_fields, table_fields -from frappe.model.document import Document -from frappe.utils.background_jobs import get_jobs - - -class EventUpdateLog(Document): - def after_insert(self): - """Send update notification updates to event consumers - whenever update log is generated""" - enqueued_method = ( - "frappe.event_streaming.doctype.event_consumer.event_consumer.notify_event_consumers" - ) - jobs = get_jobs() - if not jobs or enqueued_method not in jobs[frappe.local.site]: - frappe.enqueue( - enqueued_method, doctype=self.ref_doctype, queue="long", enqueue_after_commit=True - ) - - -def notify_consumers(doc, event): - """called via hooks""" - # make event update log for doctypes having event consumers - if frappe.flags.in_install or frappe.flags.in_migrate: - return - - consumers = check_doctype_has_consumers(doc.doctype) - if consumers: - if event == "after_insert": - doc.flags.event_update_log = make_event_update_log(doc, update_type="Create") - elif event == "on_trash": - make_event_update_log(doc, update_type="Delete") - else: - # on_update - # called after saving - if not doc.flags.event_update_log: # if not already inserted - diff = get_update(doc.get_doc_before_save(), doc) - if diff: - doc.diff = diff - make_event_update_log(doc, update_type="Update") - - -def check_doctype_has_consumers(doctype): - """Check if doctype has event consumers for event streaming""" - return frappe.cache_manager.get_doctype_map( - "Event Consumer Document Type", - doctype, - dict(ref_doctype=doctype, status="Approved", unsubscribed=0), - ) - - -def get_update(old, new, for_child=False): - """ - Get document objects with updates only - If there is a change, then returns a dict like: - { - "changed" : {fieldname1: new_value1, fieldname2: new_value2, }, - "added" : {table_fieldname1: [{row_dict1}, {row_dict2}], }, - "removed" : {table_fieldname1: [row_name1, row_name2], }, - "row_changed" : {table_fieldname1: - { - child_fieldname1: new_val, - child_fieldname2: new_val - }, - }, - } - """ - if not new: - return None - - out = frappe._dict(changed={}, added={}, removed={}, row_changed={}) - for df in new.meta.fields: - if df.fieldtype in no_value_fields and df.fieldtype not in table_fields: - continue - - old_value, new_value = old.get(df.fieldname), new.get(df.fieldname) - - if df.fieldtype in table_fields: - old_row_by_name, new_row_by_name = make_maps(old_value, new_value) - out = check_for_additions(out, df, new_value, old_row_by_name) - out = check_for_deletions(out, df, old_value, new_row_by_name) - - elif old_value != new_value: - out.changed[df.fieldname] = new_value - - out = check_docstatus(out, old, new, for_child) - if any((out.changed, out.added, out.removed, out.row_changed)): - return out - return None - - -def make_event_update_log(doc, update_type): - """Save update info for doctypes that have event consumers""" - if update_type != "Delete": - # diff for update type, doc for create type - data = frappe.as_json(doc) if not doc.get("diff") else frappe.as_json(doc.diff) - else: - data = None - return frappe.get_doc( - { - "doctype": "Event Update Log", - "update_type": update_type, - "ref_doctype": doc.doctype, - "docname": doc.name, - "data": data, - } - ).insert(ignore_permissions=True) - - -def make_maps(old_value, new_value): - """make maps""" - old_row_by_name, new_row_by_name = {}, {} - for d in old_value: - old_row_by_name[d.name] = d - for d in new_value: - new_row_by_name[d.name] = d - return old_row_by_name, new_row_by_name - - -def check_for_additions(out, df, new_value, old_row_by_name): - """check rows for additions, changes""" - for _i, d in enumerate(new_value): - if d.name in old_row_by_name: - diff = get_update(old_row_by_name[d.name], d, for_child=True) - if diff and diff.changed: - if not out.row_changed.get(df.fieldname): - out.row_changed[df.fieldname] = [] - diff.changed["name"] = d.name - out.row_changed[df.fieldname].append(diff.changed) - else: - if not out.added.get(df.fieldname): - out.added[df.fieldname] = [] - out.added[df.fieldname].append(d.as_dict()) - return out - - -def check_for_deletions(out, df, old_value, new_row_by_name): - """check for deletions""" - for d in old_value: - if d.name not in new_row_by_name: - if not out.removed.get(df.fieldname): - out.removed[df.fieldname] = [] - out.removed[df.fieldname].append(d.name) - return out - - -def check_docstatus(out, old, new, for_child): - """docstatus changes""" - if not for_child and old.docstatus != new.docstatus: - out.changed["docstatus"] = new.docstatus - return out - - -def is_consumer_uptodate(update_log, consumer): - """ - Checks if Consumer has read all the UpdateLogs before the specified update_log - :param update_log: The UpdateLog Doc in context - :param consumer: The EventConsumer doc - """ - if update_log.update_type == "Create": - # consumer is obviously up to date - return True - - prev_logs = frappe.get_all( - "Event Update Log", - filters={ - "ref_doctype": update_log.ref_doctype, - "docname": update_log.docname, - "creation": ["<", update_log.creation], - }, - order_by="creation desc", - limit_page_length=1, - ) - - if not len(prev_logs): - return False - - prev_log_consumers = frappe.get_all( - "Event Update Log Consumer", - fields=["consumer"], - filters={ - "parent": prev_logs[0].name, - "parenttype": "Event Update Log", - "consumer": consumer.name, - }, - ) - - return len(prev_log_consumers) > 0 - - -def mark_consumer_read(update_log_name, consumer_name): - """ - This function appends the Consumer to the list of Consumers that has 'read' an Update Log - """ - update_log = frappe.get_doc("Event Update Log", update_log_name) - if len([x for x in update_log.consumers if x.consumer == consumer_name]): - return - - frappe.get_doc( - frappe._dict( - doctype="Event Update Log Consumer", - consumer=consumer_name, - parent=update_log_name, - parenttype="Event Update Log", - parentfield="consumers", - ) - ).insert(ignore_permissions=True) - - -def get_unread_update_logs(consumer_name, dt, dn): - """ - Get old logs unread by the consumer on a particular document - """ - already_consumed = [ - x[0] - for x in frappe.db.sql( - """ - SELECT - update_log.name - FROM `tabEvent Update Log` update_log - JOIN `tabEvent Update Log Consumer` consumer ON consumer.parent = %(log_name)s - WHERE - consumer.consumer = %(consumer)s - AND update_log.ref_doctype = %(dt)s - AND update_log.docname = %(dn)s - """, - { - "consumer": consumer_name, - "dt": dt, - "dn": dn, - "log_name": "update_log.name" - if frappe.conf.db_type == "mariadb" - else "CAST(update_log.name AS VARCHAR)", - }, - as_dict=0, - ) - ] - - logs = frappe.get_all( - "Event Update Log", - fields=["update_type", "ref_doctype", "docname", "data", "name", "creation"], - filters={"ref_doctype": dt, "docname": dn, "name": ["not in", already_consumed]}, - order_by="creation", - ) - - return logs - - -@frappe.whitelist() -def get_update_logs_for_consumer(event_consumer, doctypes, last_update): - """ - Fetches all the UpdateLogs for the consumer - It will inject old un-consumed Update Logs if a doc was just found to be accessible to the Consumer - """ - - if isinstance(doctypes, str): - doctypes = frappe.parse_json(doctypes) - - from frappe.event_streaming.doctype.event_consumer.event_consumer import has_consumer_access - - consumer = frappe.get_doc("Event Consumer", event_consumer) - docs = frappe.get_list( - doctype="Event Update Log", - filters={"ref_doctype": ("in", doctypes), "creation": (">", last_update)}, - fields=["update_type", "ref_doctype", "docname", "data", "name", "creation"], - order_by="creation desc", - ) - - result = [] - to_update_history = [] - for d in docs: - if (d.ref_doctype, d.docname) in to_update_history: - # will be notified by background jobs - continue - - if not has_consumer_access(consumer=consumer, update_log=d): - continue - - if not is_consumer_uptodate(d, consumer): - to_update_history.append((d.ref_doctype, d.docname)) - # get_unread_update_logs will have the current log - old_logs = get_unread_update_logs(consumer.name, d.ref_doctype, d.docname) - if old_logs: - old_logs.reverse() - result.extend(old_logs) - else: - result.append(d) - - for d in result: - mark_consumer_read(update_log_name=d.name, consumer_name=consumer.name) - - result.reverse() - return result diff --git a/frappe/event_streaming/doctype/event_update_log/test_event_update_log.py b/frappe/event_streaming/doctype/event_update_log/test_event_update_log.py deleted file mode 100644 index a9065ab4ed..0000000000 --- a/frappe/event_streaming/doctype/event_update_log/test_event_update_log.py +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and Contributors -# License: MIT. See LICENSE -# import frappe -from frappe.tests.utils import FrappeTestCase - - -class TestEventUpdateLog(FrappeTestCase): - pass diff --git a/frappe/event_streaming/doctype/event_update_log_consumer/__init__.py b/frappe/event_streaming/doctype/event_update_log_consumer/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/event_streaming/doctype/event_update_log_consumer/event_update_log_consumer.json b/frappe/event_streaming/doctype/event_update_log_consumer/event_update_log_consumer.json deleted file mode 100644 index b3484c6481..0000000000 --- a/frappe/event_streaming/doctype/event_update_log_consumer/event_update_log_consumer.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "actions": [], - "creation": "2020-06-30 10:54:53.301787", - "doctype": "DocType", - "editable_grid": 1, - "engine": "InnoDB", - "field_order": [ - "consumer" - ], - "fields": [ - { - "fieldname": "consumer", - "fieldtype": "Link", - "in_list_view": 1, - "label": "Consumer", - "options": "Event Consumer", - "reqd": 1 - } - ], - "istable": 1, - "links": [], - "modified": "2020-06-30 10:54:53.301787", - "modified_by": "Administrator", - "module": "Event Streaming", - "name": "Event Update Log Consumer", - "owner": "Administrator", - "permissions": [], - "quick_entry": 1, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1 -} \ No newline at end of file diff --git a/frappe/event_streaming/doctype/event_update_log_consumer/event_update_log_consumer.py b/frappe/event_streaming/doctype/event_update_log_consumer/event_update_log_consumer.py deleted file mode 100644 index 69da7db92e..0000000000 --- a/frappe/event_streaming/doctype/event_update_log_consumer/event_update_log_consumer.py +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright (c) 2020, Frappe Technologies and contributors -# License: MIT. See LICENSE - -# import frappe -from frappe.model.document import Document - - -class EventUpdateLogConsumer(Document): - pass diff --git a/frappe/exceptions.py b/frappe/exceptions.py index c3bb45caea..2fe9de6be9 100644 --- a/frappe/exceptions.py +++ b/frappe/exceptions.py @@ -236,6 +236,10 @@ class QueryDeadlockError(Exception): pass +class InReadOnlyMode(ValidationError): + http_status_code = 503 # temporarily not available + + class TooManyWritesError(Exception): pass diff --git a/frappe/hooks.py b/frappe/hooks.py index 85aeea7418..9715508c78 100644 --- a/frappe/hooks.py +++ b/frappe/hooks.py @@ -138,16 +138,12 @@ standard_queries = {"User": "frappe.core.doctype.user.user.user_query"} doc_events = { "*": { - "after_insert": [ - "frappe.event_streaming.doctype.event_update_log.event_update_log.notify_consumers" - ], "on_update": [ "frappe.desk.notifications.clear_doctype_notifications", "frappe.core.doctype.activity_log.feed.update_feed", "frappe.workflow.doctype.workflow_action.workflow_action.process_workflow_actions", "frappe.automation.doctype.assignment_rule.assignment_rule.apply", "frappe.core.doctype.file.utils.attach_files_to_document", - "frappe.event_streaming.doctype.event_update_log.event_update_log.notify_consumers", "frappe.automation.doctype.assignment_rule.assignment_rule.update_due_date", "frappe.core.doctype.user_type.user_type.apply_permissions_for_non_standard_user_type", ], @@ -155,12 +151,10 @@ doc_events = { "on_cancel": [ "frappe.desk.notifications.clear_doctype_notifications", "frappe.workflow.doctype.workflow_action.workflow_action.process_workflow_actions", - "frappe.event_streaming.doctype.event_update_log.event_update_log.notify_consumers", ], "on_trash": [ "frappe.desk.notifications.clear_doctype_notifications", "frappe.workflow.doctype.workflow_action.workflow_action.process_workflow_actions", - "frappe.event_streaming.doctype.event_update_log.event_update_log.notify_consumers", ], "on_update_after_submit": [ "frappe.workflow.doctype.workflow_action.workflow_action.process_workflow_actions" diff --git a/frappe/installer.py b/frappe/installer.py index d7394ab3f2..4f1755c2a0 100644 --- a/frappe/installer.py +++ b/frappe/installer.py @@ -11,6 +11,7 @@ import click import frappe from frappe.defaults import _clear_cache from frappe.utils import cint, is_git_url +from frappe.utils.dashboard import sync_dashboards def _is_scheduler_enabled() -> bool: @@ -301,6 +302,7 @@ def install_app(name, verbose=False, set_as_patched=True, force=False): sync_jobs() sync_fixtures(name) sync_customizations(name) + sync_dashboards(name) for after_sync in app_hooks.after_sync or []: frappe.get_attr(after_sync)() # diff --git a/frappe/integrations/oauth2_logins.py b/frappe/integrations/oauth2_logins.py index c8252b0f70..b668211c2b 100644 --- a/frappe/integrations/oauth2_logins.py +++ b/frappe/integrations/oauth2_logins.py @@ -48,7 +48,7 @@ def custom(code, state): """ Callback for processing code and state for user added providers - process social login from /api/method/frappe.integrations.custom/ + process social login from /api/method/frappe.integrations.oauth2_logins.custom/ """ path = frappe.request.path[1:].split("/") if len(path) == 4 and path[3]: diff --git a/frappe/migrate.py b/frappe/migrate.py index 683252c625..6fa1fb43e9 100644 --- a/frappe/migrate.py +++ b/frappe/migrate.py @@ -13,6 +13,7 @@ from frappe.cache_manager import clear_global_cache from frappe.core.doctype.language.language import sync_languages from frappe.core.doctype.scheduled_job_type.scheduled_job_type import sync_jobs from frappe.database.schema import add_column +from frappe.deferred_insert import save_to_db as flush_deferred_inserts from frappe.desk.notifications import clear_notifications from frappe.modules.patch_handler import PatchType from frappe.modules.utils import sync_customizations @@ -123,6 +124,7 @@ class SiteMigration: * Sync in-Desk Module Dashboards * Sync customizations: Custom Fields, Property Setters, Custom Permissions * Sync Frappe's internal language master + * Flush deferred inserts made during maintenance mode. * Sync Portal Menu Items * Sync Installed Applications Version History * Execute `after_migrate` hooks @@ -132,6 +134,7 @@ class SiteMigration: sync_dashboards() sync_customizations() sync_languages() + flush_deferred_inserts() frappe.get_single("Portal Settings").sync_menu() frappe.get_single("Installed Applications").update_versions() diff --git a/frappe/model/__init__.py b/frappe/model/__init__.py index 29991fa403..a7f966ebd6 100644 --- a/frappe/model/__init__.py +++ b/frappe/model/__init__.py @@ -114,6 +114,8 @@ core_doctypes_list = ( "Client Script", ) +# NOTE: this is being used for dynamic autoincrement in new sites, +# removing any of these will require patches. log_types = ( "Version", "Error Log", diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py index de2a73bfb6..1ea9aae473 100644 --- a/frappe/model/db_query.py +++ b/frappe/model/db_query.py @@ -606,7 +606,10 @@ class DatabaseQuery: elif f.operator.lower() in ("in", "not in"): # if values contain '' or falsy values then only coalesce column - can_be_null = not f.value or any(v is None or v == "" for v in f.value) + # for `in` query this is only required if values contain '' or values are empty. + # for `not in` queries we can't be sure as column values might contain null. + if f.operator.lower() == "in": + can_be_null = not f.value or any(v is None or v == "" for v in f.value) values = f.value or "" if isinstance(values, str): diff --git a/frappe/model/document.py b/frappe/model/document.py index 2a82b5af9a..aa55eac30a 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -275,9 +275,6 @@ class Document(BaseDocument): if self.get("amended_from"): self.copy_attachments_from_amended_from() - # flag to prevent creation of event update log for create and update both - # during document creation - self.flags.update_log_for_doc_creation = True self.run_post_save_methods() self.flags.in_insert = False @@ -1369,7 +1366,7 @@ class Document(BaseDocument): if not user: user = frappe.session.user - if self.meta.track_seen: + if self.meta.track_seen and not frappe.flags.read_only: _seen = self.get("_seen") or [] _seen = frappe.parse_json(_seen) @@ -1384,15 +1381,19 @@ class Document(BaseDocument): user = frappe.session.user if hasattr(self.meta, "track_views") and self.meta.track_views: - frappe.get_doc( + view_log = frappe.get_doc( { "doctype": "View Log", "viewed_by": frappe.session.user, "reference_doctype": self.doctype, "reference_name": self.name, } - ).insert(ignore_permissions=True) - frappe.local.flags.commit = True + ) + if frappe.flags.read_only: + view_log.deferred_insert() + else: + view_log.insert(ignore_permissions=True) + frappe.local.flags.commit = True def log_error(self, title=None, message=None): """Helper function to create an Error Log""" @@ -1539,6 +1540,20 @@ class Document(BaseDocument): return DocTags(self.doctype).get_tags(self.name).split(",")[1:] + def deferred_insert(self) -> None: + """Push the document to redis temporarily and insert later. + + WARN: This doesn't guarantee insertion as redis can be restarted + before data is flushed to database. + """ + + from frappe.deferred_insert import deferred_insert + + self.set_user_and_timestamp() + + doc = self.get_valid_dict(convert_dates_to_str=True, ignore_virtual=True) + deferred_insert(doctype=self.doctype, records=doc) + def __repr__(self): name = self.name or "unsaved" doctype = self.__class__.__name__ diff --git a/frappe/model/meta.py b/frappe/model/meta.py index 906b5cf04c..6aa8d1d80e 100644 --- a/frappe/model/meta.py +++ b/frappe/model/meta.py @@ -40,21 +40,31 @@ from frappe.model.workflow import get_workflow_name from frappe.modules import load_doctype_module from frappe.utils import cast, cint, cstr +DEFAULT_FIELD_LABELS = { + "name": lambda: _("ID"), + "creation": lambda: _("Created On"), + "docstatus": lambda: _("Document Status"), + "idx": lambda: _("Index"), + "modified": lambda: _("Last Updated On"), + "modified_by": lambda: _("Last Updated By"), + "owner": lambda: _("Created By"), + "_user_tags": lambda: _("Tags"), + "_liked_by": lambda: _("Liked By"), + "_comments": lambda: _("Comments"), + "_assign": lambda: _("Assigned To"), +} + def get_meta(doctype, cached=True) -> "Meta": - if cached: - if not frappe.local.meta_cache.get(doctype): - meta = frappe.cache().hget("meta", doctype) - if meta: - meta = Meta(meta) - else: - meta = Meta(doctype) - frappe.cache().hset("meta", doctype, meta.as_dict()) - frappe.local.meta_cache[doctype] = meta + if not cached: + return Meta(doctype) - return frappe.local.meta_cache[doctype] - else: - return load_meta(doctype) + if meta := frappe.cache().hget("doctype_meta", doctype): + return meta + + meta = Meta(doctype) + frappe.cache().hset("doctype_meta", doctype, meta) + return meta def load_meta(doctype): @@ -86,7 +96,7 @@ def load_doctype_from_file(doctype): class Meta(Document): _metaclass = True default_fields = list(default_fields)[1:] - special_doctypes = ( + special_doctypes = { "DocField", "DocPerm", "DocType", @@ -94,24 +104,25 @@ class Meta(Document): "DocType Action", "DocType Link", "DocType State", - ) + } standard_set_once_fields = [ frappe._dict(fieldname="creation", fieldtype="Datetime"), frappe._dict(fieldname="owner", fieldtype="Data"), ] def __init__(self, doctype): - self._fields = {} + # from cache if isinstance(doctype, dict): super().__init__(doctype) + self.init_field_map() + return - elif isinstance(doctype, Document): + if isinstance(doctype, Document): super().__init__(doctype.as_dict()) - self.process() - else: super().__init__("DocType", doctype) - self.process() + + self.process() def load_from_db(self): try: @@ -126,10 +137,12 @@ class Meta(Document): # don't process for special doctypes # prevent's circular dependency if self.name in self.special_doctypes: + self.init_field_map() return self.add_custom_fields() self.apply_property_setters() + self.init_field_map() self.sort_fields() self.get_valid_columns() self.set_custom_permissions() @@ -233,36 +246,24 @@ class Meta(Document): def get_field(self, fieldname): """Return docfield from meta""" - if not self._fields: - for f in self.get("fields"): - self._fields[f.fieldname] = f return self._fields.get(fieldname) def has_field(self, fieldname): """Returns True if fieldname exists""" - return True if self.get_field(fieldname) else False + + return fieldname in self._fields def get_label(self, fieldname): """Get label of the given fieldname""" - df = self.get_field(fieldname) - if df: - label = df.label - else: - label = { - "name": _("ID"), - "creation": _("Created On"), - "docstatus": _("Document Status"), - "idx": _("Index"), - "modified": _("Last Updated On"), - "modified_by": _("Last Updated By"), - "owner": _("Created By"), - "_user_tags": _("Tags"), - "_liked_by": _("Liked By"), - "_comments": _("Comments"), - "_assign": _("Assigned To"), - }.get(fieldname) or _("No Label") - return label + + if df := self.get_field(fieldname): + return df.label + + if fieldname in DEFAULT_FIELD_LABELS: + return DEFAULT_FIELD_LABELS[fieldname]() + + return _("No Label") def get_options(self, fieldname): return self.get_field(fieldname).options @@ -273,12 +274,9 @@ class Meta(Document): if df.fieldtype == "Link": return df.options - elif df.fieldtype == "Dynamic Link": + if df.fieldtype == "Dynamic Link": return self.get_options(df.options) - else: - return None - def get_search_fields(self): search_fields = self.search_fields or "name" search_fields = [d.strip() for d in search_fields.split(",")] @@ -340,8 +338,9 @@ class Meta(Document): def is_translatable(self, fieldname): """Return true of false given a field""" - field = self.get_field(fieldname) - return field and field.translatable + + if field := self.get_field(fieldname): + return field.translatable def get_workflow(self): return get_workflow_name(self.name) @@ -349,11 +348,10 @@ class Meta(Document): def get_naming_series_options(self) -> list[str]: """Get list naming series options.""" - field = self.get_field("naming_series") - if field: + if field := self.get_field("naming_series"): options = field.options or "" - return options.split("\n") + return [] def add_custom_fields(self): @@ -450,6 +448,9 @@ class Meta(Document): self.set(fieldname, new_list) + def init_field_map(self): + self._fields = {field.fieldname: field for field in self.fields} + def sort_fields(self): """sort on basis of insert_after""" custom_fields = sorted(self.get_custom_fields(), key=lambda df: df.idx) @@ -666,10 +667,17 @@ def is_single(doctype): def get_parent_dt(dt): - parent_dt = frappe.get_all( - "DocField", "parent", dict(fieldtype=["in", frappe.model.table_fields], options=dt), limit=1 + if not frappe.is_table(dt): + return "" + + return ( + frappe.db.get_value( + "DocField", + {"fieldtype": ("in", frappe.model.table_fields), "options": dt}, + "parent", + ) + or "" ) - return parent_dt and parent_dt[0].parent or "" def set_fieldname(field_id, fieldname): diff --git a/frappe/model/rename_doc.py b/frappe/model/rename_doc.py index 46a239d0aa..b759a1ca00 100644 --- a/frappe/model/rename_doc.py +++ b/frappe/model/rename_doc.py @@ -527,7 +527,12 @@ def get_select_fields(old: str, new: str) -> list[dict]: standard_fields = ( frappe.qb.from_(df) .select(df.parent, df.fieldname, st_issingle) - .where((df.parent != new) & (df.fieldtype == "Select") & (df.options.like(f"%{old}%"))) + .where( + (df.parent != new) + & (df.fieldname != "fieldtype") + & (df.fieldtype == "Select") + & (df.options.like(f"%{old}%")) + ) .run(as_dict=True) ) diff --git a/frappe/model/sync.py b/frappe/model/sync.py index df3999054a..1eab050663 100644 --- a/frappe/model/sync.py +++ b/frappe/model/sync.py @@ -8,17 +8,17 @@ import os import frappe from frappe.modules.import_file import import_file_by_path -from frappe.modules.patch_handler import block_user +from frappe.modules.patch_handler import _patch_mode from frappe.utils import update_progress_bar def sync_all(force=0, reset_permissions=False): - block_user(True) + _patch_mode(True) for app in frappe.get_installed_apps(): sync_for(app, force, reset_permissions=reset_permissions) - block_user(False) + _patch_mode(False) frappe.clear_cache() diff --git a/frappe/modules.txt b/frappe/modules.txt index fb7817f6ba..863c448594 100644 --- a/frappe/modules.txt +++ b/frappe/modules.txt @@ -9,5 +9,4 @@ Integrations Printing Contacts Social -Automation -Event Streaming \ No newline at end of file +Automation \ No newline at end of file diff --git a/frappe/modules/patch_handler.py b/frappe/modules/patch_handler.py index f389312a4f..d5a37f52a5 100644 --- a/frappe/modules/patch_handler.py +++ b/frappe/modules/patch_handler.py @@ -154,7 +154,7 @@ def run_single(patchmodule=None, method=None, methodargs=None, force=False): def execute_patch(patchmodule, method=None, methodargs=None): """execute the patch""" - block_user(True) + _patch_mode(True) if patchmodule.startswith("execute:"): has_patch_file = False @@ -197,7 +197,7 @@ def execute_patch(patchmodule, method=None, methodargs=None): else: frappe.db.commit() end_time = time.time() - block_user(False) + _patch_mode(False) print(f"Success: Done in {round(end_time - start_time, 3)}s") return True @@ -216,18 +216,7 @@ def executed(patchmodule): return frappe.db.get_value("Patch Log", {"patch": patchmodule}) -def block_user(block, msg=None): +def _patch_mode(enable): """stop/start execution till patch is run""" - frappe.local.flags.in_patch = block - frappe.db.begin() - if not msg: - msg = "Patches are being executed in the system. Please try again in a few moments." - frappe.db.set_global("__session_status", block and "stop" or None) - frappe.db.set_global("__session_status_message", block and msg or None) + frappe.local.flags.in_patch = enable frappe.db.commit() - - -def check_session_stopped(): - if frappe.db.get_global("__session_status") == "stop": - frappe.msgprint(frappe.db.get_global("__session_status_message")) - raise frappe.SessionStopped("Session Stopped") diff --git a/frappe/patches.txt b/frappe/patches.txt index ca8f678e3d..2564a565b1 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -156,7 +156,6 @@ frappe.patches.v13_0.set_route_for_blog_category frappe.patches.v13_0.enable_custom_script frappe.patches.v13_0.update_newsletter_content_type execute:frappe.db.set_value('Website Settings', 'Website Settings', {'navbar_template': 'Standard Navbar', 'footer_template': 'Standard Footer'}) -frappe.patches.v13_0.delete_event_producer_and_consumer_keys frappe.patches.v13_0.web_template_set_module #2020-10-05 frappe.patches.v13_0.remove_custom_link execute:frappe.delete_doc("DocType", "Footer Item") @@ -195,6 +194,7 @@ frappe.patches.v14_0.log_settings_migration frappe.patches.v14_0.setup_likes_from_feedback frappe.patches.v14_0.update_webforms frappe.patches.v14_0.delete_payment_gateways +frappe.patches.v15_0.remove_event_streaming [post_model_sync] frappe.patches.v14_0.drop_data_import_legacy @@ -210,3 +210,6 @@ frappe.patches.v14_0.delete_data_migration_tool frappe.patches.v14_0.set_suspend_email_queue_default frappe.patches.v14_0.different_encryption_key frappe.patches.v14_0.update_multistep_webforms +execute:frappe.delete_doc('Page', 'background_jobs', ignore_missing=True, force=True) +frappe.patches.v14_0.drop_unused_indexes +frappe.patches.v15_0.drop_modified_index diff --git a/frappe/patches/v13_0/delete_event_producer_and_consumer_keys.py b/frappe/patches/v13_0/delete_event_producer_and_consumer_keys.py deleted file mode 100644 index 9cb081e15a..0000000000 --- a/frappe/patches/v13_0/delete_event_producer_and_consumer_keys.py +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors -# License: MIT. See LICENSE - -import frappe - - -def execute(): - if frappe.db.exists("DocType", "Event Producer"): - frappe.db.sql("""UPDATE `tabEvent Producer` SET api_key='', api_secret=''""") - if frappe.db.exists("DocType", "Event Consumer"): - frappe.db.sql("""UPDATE `tabEvent Consumer` SET api_key='', api_secret=''""") diff --git a/frappe/patches/v14_0/drop_unused_indexes.py b/frappe/patches/v14_0/drop_unused_indexes.py new file mode 100644 index 0000000000..896ea78fed --- /dev/null +++ b/frappe/patches/v14_0/drop_unused_indexes.py @@ -0,0 +1,56 @@ +""" +This patch just drops some known indexes which aren't being used anymore or never were used. +""" + +import click + +import frappe + +UNUSED_INDEXES = [ + ("Comment", ["link_doctype", "link_name"]), + ("Activity Log", ["link_doctype", "link_name"]), +] + + +def execute(): + if frappe.db.db_type == "postgres": + return + + db_tables = frappe.db.get_tables(cached=False) + + # All parent indexes + parent_doctypes = frappe.get_all( + "DocType", + {"istable": 0, "is_virtual": 0, "issingle": 0}, + pluck="name", + ) + db_tables = frappe.db.get_tables(cached=False) + + for doctype in parent_doctypes: + table = f"tab{doctype}" + if table not in db_tables: + continue + drop_index_if_exists(table, "parent") + + # Unused composite indexes + for doctype, index_fields in UNUSED_INDEXES: + table = f"tab{doctype}" + index_name = frappe.db.get_index_name(index_fields) + if table not in db_tables: + continue + drop_index_if_exists(table, index_name) + + +def drop_index_if_exists(table: str, index: str): + if not frappe.db.has_index(table, index): + click.echo(f"- Skipped {index} index for {table} because it doesn't exist") + return + + try: + frappe.db.sql_ddl(f"ALTER TABLE `{table}` DROP INDEX `{index}`") + except Exception as e: + frappe.log_error("Failed to drop index") + click.secho(f"x Failed to drop index {index} from {table}\n {str(e)}", fg="red") + return + + click.echo(f"✓ dropped {index} index from {table}") diff --git a/frappe/patches/v15_0/drop_modified_index.py b/frappe/patches/v15_0/drop_modified_index.py new file mode 100644 index 0000000000..8cdcf12ece --- /dev/null +++ b/frappe/patches/v15_0/drop_modified_index.py @@ -0,0 +1,21 @@ +import frappe +from frappe.patches.v14_0.drop_unused_indexes import drop_index_if_exists + + +def execute(): + if frappe.db.db_type == "postgres": + return + + db_tables = frappe.db.get_tables(cached=False) + + child_tables = frappe.get_all( + "DocType", + {"istable": 1, "is_virtual": 0}, + pluck="name", + ) + + for doctype in child_tables: + table = f"tab{doctype}" + if table not in db_tables: + continue + drop_index_if_exists(table, "modified") diff --git a/frappe/patches/v15_0/remove_event_streaming.py b/frappe/patches/v15_0/remove_event_streaming.py new file mode 100644 index 0000000000..4c6a1ce079 --- /dev/null +++ b/frappe/patches/v15_0/remove_event_streaming.py @@ -0,0 +1,22 @@ +import frappe + + +def execute(): + if "event_streaming" in frappe.get_installed_apps(): + return + + frappe.delete_doc_if_exists("Module Def", "Event Streaming", force=True) + + for doc in [ + "Event Consumer Document Type", + "Document Type Mapping", + "Event Producer", + "Event Producer Last Update", + "Event Producer Document Type", + "Event Consumer", + "Document Type Field Mapping", + "Event Update Log", + "Event Update Log Consumer", + "Event Sync Log", + ]: + frappe.delete_doc_if_exists("DocType", doc, force=True) diff --git a/frappe/public/js/frappe/desk.js b/frappe/public/js/frappe/desk.js index d942c3b849..c8d3d6d0d6 100644 --- a/frappe/public/js/frappe/desk.js +++ b/frappe/public/js/frappe/desk.js @@ -147,17 +147,6 @@ frappe.Application = class Application { this.link_preview = new frappe.ui.LinkPreview(); if (!frappe.boot.developer_mode) { - setInterval(function () { - frappe.call({ - method: "frappe.core.page.background_jobs.background_jobs.get_scheduler_status", - callback: function (r) { - if (r.message[0] == __("Inactive")) { - frappe.call("frappe.utils.scheduler.activate_scheduler"); - } - }, - }); - }, 300000); // check every 5 minutes - if (frappe.user.has_role("System Manager")) { setInterval(function () { frappe.call({ diff --git a/frappe/public/js/frappe/form/controls/link.js b/frappe/public/js/frappe/form/controls/link.js index e34fd01008..64942d2ac7 100644 --- a/frappe/public/js/frappe/form/controls/link.js +++ b/frappe/public/js/frappe/form/controls/link.js @@ -89,23 +89,19 @@ frappe.ui.form.ControlLink = class ControlLink extends frappe.ui.form.ControlDat is_translatable() { return in_list(frappe.boot?.translated_doctypes || [], this.get_options()); } - set_link_title(value) { - let doctype = this.get_options(); + async set_link_title(value) { + const doctype = this.get_options(); - if (!doctype) return; - - if (in_list(frappe.boot.link_title_doctypes, doctype)) { - let link_title = frappe.utils.get_link_title(doctype, value); - if (!link_title) { - link_title = frappe.utils.fetch_link_title(doctype, value).then((link_title) => { - this.translate_and_set_input_value(link_title, value); - }); - } else { - this.translate_and_set_input_value(link_title, value); - } - } else { + if (!doctype || !in_list(frappe.boot.link_title_doctypes, doctype)) { this.translate_and_set_input_value(value, value); + return; } + + const link_title = + frappe.utils.get_link_title(doctype, value) || + (await frappe.utils.fetch_link_title(doctype, value)); + + this.translate_and_set_input_value(link_title, value); } translate_and_set_input_value(link_title, value) { let translated_link_text = this.get_translated(link_title); diff --git a/frappe/public/js/frappe/form/dashboard.js b/frappe/public/js/frappe/form/dashboard.js index 53cef9bd8c..2e3fd9ba99 100644 --- a/frappe/public/js/frappe/form/dashboard.js +++ b/frappe/public/js/frappe/form/dashboard.js @@ -568,7 +568,7 @@ frappe.ui.form.Dashboard = class FormDashboard { this.chart_area.body.empty(); $.extend(args, { type: "line", - colors: ["green"], + colors: args.colors || ["green"], truncateLegends: 1, axisOptions: { shortenYAxisNumbers: 1, diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js index 9ef4b32915..8c642a73f0 100644 --- a/frappe/public/js/frappe/form/form.js +++ b/frappe/public/js/frappe/form/form.js @@ -450,6 +450,10 @@ frappe.ui.form.Form = class FrappeForm { .toggleClass("cancelled-form", this.doc.docstatus === 2); this.show_conflict_message(); + + if (frappe.boot.read_only) { + this.disable_form(); + } } } diff --git a/frappe/public/js/frappe/form/toolbar.js b/frappe/public/js/frappe/form/toolbar.js index 2173d18822..0052ccf5c2 100644 --- a/frappe/public/js/frappe/form/toolbar.js +++ b/frappe/public/js/frappe/form/toolbar.js @@ -197,10 +197,16 @@ frappe.ui.form.Toolbar = class Toolbar { // check if docname is updatable if (me.can_rename()) { + let label = __("New Name"); + if (me.frm.meta.autoname && me.frm.meta.autoname.startsWith("field:")) { + let fieldname = me.frm.meta.autoname.split(":")[1]; + label = __("New {0}", [me.frm.get_docfield(fieldname).label]); + } + fields.push( ...[ { - label: __("New Name"), + label: label, fieldname: "name", fieldtype: "Data", reqd: 1, diff --git a/frappe/public/js/frappe/icon_picker/icon_picker.js b/frappe/public/js/frappe/icon_picker/icon_picker.js index 2b638b1da2..b289a31ed0 100644 --- a/frappe/public/js/frappe/icon_picker/icon_picker.js +++ b/frappe/public/js/frappe/icon_picker/icon_picker.js @@ -16,7 +16,7 @@ class Picker { this.icon_picker_wrapper = $(`
- + ${frappe.utils.icon("search", "sm")}
diff --git a/frappe/public/js/frappe/list/list_view.js b/frappe/public/js/frappe/list/list_view.js index c7a2a9b72c..c6ff13f336 100644 --- a/frappe/public/js/frappe/list/list_view.js +++ b/frappe/public/js/frappe/list/list_view.js @@ -215,7 +215,7 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList { } set_primary_action() { - if (this.can_create) { + if (this.can_create && !frappe.boot.read_only) { const doctype_name = __(frappe.router.doctype_layout) || __(this.doctype); // Better style would be __("Add {0}", [doctype_name], "Primary action in list view") @@ -894,11 +894,9 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList { return this.settings.get_form_link(doc); } - const docname = cstr(doc.name).match(/[%'"#\s]/) ? encodeURIComponent(doc.name) : doc.name; - return `/app/${frappe.router.slug( frappe.router.doctype_layout || this.doctype - )}/${docname}`; + )}/${encodeURIComponent(cstr(doc.name))}`; } get_seen_class(doc) { diff --git a/frappe/public/js/frappe/list/list_view_select.js b/frappe/public/js/frappe/list/list_view_select.js index 4af1268f68..0e7033cf9a 100644 --- a/frappe/public/js/frappe/list/list_view_select.js +++ b/frappe/public/js/frappe/list/list_view_select.js @@ -248,19 +248,34 @@ frappe.views.ListViewSelect = class ListViewSelect { } setup_kanban_boards() { + function fetch_kanban_board(doctype) { + frappe.db.get_value( + "Kanban Board", + { reference_doctype: doctype }, + "name", + (board) => { + if (!$.isEmptyObject(board)) { + frappe.set_route("list", doctype, "kanban", board.name); + } else { + frappe.views.KanbanView.show_kanban_dialog(doctype); + } + } + ); + } + const last_opened_kanban = frappe.model.user_settings[this.doctype]["Kanban"]?.last_kanban_board; - if (!last_opened_kanban) { - return frappe.views.KanbanView.show_kanban_dialog(this.doctype, true); + fetch_kanban_board(this.doctype); + } else { + frappe.db.exists("Kanban Board", last_opened_kanban).then((exists) => { + if (exists) { + frappe.set_route("list", this.doctype, "kanban", last_opened_kanban); + } else { + fetch_kanban_board(this.doctype); + } + }); } - frappe.db.exists("Kanban Board", last_opened_kanban).then((exists) => { - if (exists) { - frappe.set_route("list", this.doctype, "kanban", last_opened_kanban); - } else { - frappe.views.KanbanView.show_kanban_dialog(this.doctype, true); - } - }); } get_calendars() { diff --git a/frappe/public/js/frappe/model/sync.js b/frappe/public/js/frappe/model/sync.js index a8f8851ebe..b9481dca96 100644 --- a/frappe/public/js/frappe/model/sync.js +++ b/frappe/public/js/frappe/model/sync.js @@ -56,16 +56,11 @@ Object.assign(frappe.model, { sync_docinfo: (r) => { // set docinfo (comments, assign, attachments) if (r.docinfo) { - var doc; - if (r.docs) { - doc = r.docs[0]; - } else { - if (cur_frm) doc = cur_frm.doc; - } - if (doc) { - if (!frappe.model.docinfo[doc.doctype]) frappe.model.docinfo[doc.doctype] = {}; - frappe.model.docinfo[doc.doctype][doc.name] = r.docinfo; + const { doctype, name } = r.docinfo; + if (!frappe.model.docinfo[doctype]) { + frappe.model.docinfo[doctype] = {}; } + frappe.model.docinfo[doctype][name] = r.docinfo; // copy values to frappe.boot.user_info Object.assign(frappe.boot.user_info, r.docinfo.user_info); diff --git a/frappe/public/js/frappe/router.js b/frappe/public/js/frappe/router.js index 6a0c745f29..0d1b815bce 100644 --- a/frappe/public/js/frappe/router.js +++ b/frappe/public/js/frappe/router.js @@ -369,12 +369,7 @@ frappe.router = { frappe.route_options = a; return null; } else { - a = String(a); - if (a && a.match(/[%'"#\s\t]/)) { - // if special chars, then encode - a = encodeURIComponent(a); - } - return a; + return encodeURIComponent(String(a)); } }).join("/"); let private_home = frappe.workspaces[`home-${frappe.user.name.toLowerCase()}`]; diff --git a/frappe/public/js/frappe/ui/toolbar/navbar.html b/frappe/public/js/frappe/ui/toolbar/navbar.html index 3276ca302a..ee070d4378 100644 --- a/frappe/public/js/frappe/ui/toolbar/navbar.html +++ b/frappe/public/js/frappe/ui/toolbar/navbar.html @@ -6,6 +6,11 @@ - {%- if not disable_signup -%} + {%- if not disable_signup and not disable_user_pass_login -%} - {%- if not disable_signup -%} + {%- if not disable_signup and not disable_user_pass_login -%}