diff --git a/.github/helper/db/mariadb.json b/.github/helper/db/mariadb.json index e86e701dc3..0a6c9890c4 100644 --- a/.github/helper/db/mariadb.json +++ b/.github/helper/db/mariadb.json @@ -6,7 +6,8 @@ "allow_tests": true, "db_type": "mariadb", "auto_email_id": "test@example.com", - "mail_server": "smtp.example.com", + "mail_server": "localhost", + "mail_port": 2525, "mail_login": "test@example.com", "mail_password": "test", "admin_password": "admin", diff --git a/.github/helper/db/postgres.json b/.github/helper/db/postgres.json index 6ca83b9e96..f830e717ed 100644 --- a/.github/helper/db/postgres.json +++ b/.github/helper/db/postgres.json @@ -6,7 +6,8 @@ "db_type": "postgres", "allow_tests": true, "auto_email_id": "test@example.com", - "mail_server": "smtp.example.com", + "mail_server": "localhost", + "mail_port": 2525, "mail_login": "test@example.com", "mail_password": "test", "admin_password": "admin", diff --git a/.github/workflows/server-tests.yml b/.github/workflows/server-tests.yml index fe68e33f8b..97000bff15 100644 --- a/.github/workflows/server-tests.yml +++ b/.github/workflows/server-tests.yml @@ -72,6 +72,12 @@ jobs: ports: - 5432:5432 + smtp_server: + image: rnwood/smtp4dev + ports: + - 2525:25 + - 3000:80 + steps: - name: Clone uses: actions/checkout@v4 diff --git a/cypress/integration/form_builder.js b/cypress/integration/form_builder.js index 53c45cd379..2a5865b904 100644 --- a/cypress/integration/form_builder.js +++ b/cypress/integration/form_builder.js @@ -56,12 +56,17 @@ context("Form Builder", () => { cy.visit(`/app/doctype/${doctype_name}`); cy.findByRole("tab", { name: "Form" }).click(); - let first_field = - ".tab-content.active .section-columns-container:first .column:first .field:first"; + let first_column = ".tab-content.active .section-columns-container:first .column:first"; - cy.get(".fields-container .field[title='Table']").drag(first_field, { - target: { x: 100, y: 10 }, - }); + let last_field = first_column + " .field:last"; + + let add_new_field_btn = first_column + " .add-new-field-btn button"; + + // add new field + cy.get(add_new_field_btn).click(); + + // type table and press enter + cy.get(".combo-box-options:visible .search-box > input").type("table{enter}"); // save cy.click_doc_primary_button("Save"); @@ -70,7 +75,7 @@ context("Form Builder", () => { cy.get_open_dialog().find(".msgprint").should("contain", "Options is required"); cy.hide_dialog(); - cy.get(first_field).click({ force: true }); + cy.get(last_field).click({ force: true }); cy.get(".sidebar-container .frappe-control[data-fieldname='options'] input") .click() @@ -78,13 +83,10 @@ context("Form Builder", () => { cy.get("@input").clear({ force: true }).type("Web Form Field", { delay: 200 }); cy.wait("@search_link"); - cy.get(first_field).click({ force: true }); + cy.get(last_field).click({ force: true }); - cy.get(first_field) - .find(".table-controls .table-column") - .contains("Field") - .should("exist"); - cy.get(first_field) + cy.get(last_field).find(".table-controls .table-column").contains("Field").should("exist"); + cy.get(last_field) .find(".table-controls .table-column") .contains("Fieldtype") .should("exist"); @@ -98,7 +100,7 @@ context("Form Builder", () => { cy.get_open_dialog().find(".msgprint").should("contain", "In List View"); cy.hide_dialog(); - cy.get(first_field).click({ force: true }); + cy.get(last_field).click({ force: true }); cy.get(".sidebar-container .field label .label-area").contains("In List View").click(); // validate In Global Search @@ -188,7 +190,7 @@ context("Form Builder", () => { // add new column cy.get(first_section).find(".column:first").click(15, 10); cy.get(first_section).find(".column:first .column-actions button:first").click(); - cy.get(first_section).find(".column").should("have.length", 3); + cy.get(first_section).find(".column").should("have.length", 2); }); it("Remove Tab/Section/Column", () => { @@ -197,7 +199,7 @@ context("Form Builder", () => { // remove column cy.get(first_section).find(".column:first").click(15, 10); cy.get(first_section).find(".column:first .column-actions button:last").click(); - cy.get(first_section).find(".column").should("have.length", 2); + cy.get(first_section).find(".column").should("have.length", 1); // remove section cy.get(first_section).click(15, 10); @@ -205,7 +207,7 @@ context("Form Builder", () => { cy.get(".tab-content.active .form-section-container").should("have.length", 1); // remove tab - cy.get(".tab-header").realHover().find(".tab-actions .remove-tab-btn").click(); + cy.get(".tab-header .tab:last").realHover().find(".remove-tab-btn").click(); cy.get(".tab-header .tabs .tab").should("have.length", 2); }); @@ -231,14 +233,19 @@ context("Form Builder", () => { cy.visit(`/app/doctype/${doctype_name}`); cy.findByRole("tab", { name: "Form" }).click(); - let first_field = - ".tab-content.active .section-columns-container:first .column:first .field:first"; + let first_column = ".tab-content.active .section-columns-container:first .column:first"; - cy.get(".fields-container .field[title='Data']").drag(first_field, { - target: { x: 100, y: 10 }, - }); + let last_field = first_column + " .field:last"; - cy.get(first_field).click(); + let add_new_field_btn = first_column + " .add-new-field-btn button"; + + // add new field + cy.get(add_new_field_btn).click(); + + // type data and press enter + cy.get(".combo-box-options:visible .search-box > input").type("data{enter}"); + + cy.get(last_field).click(); // validate duplicate name cy.get(".sidebar-container .frappe-control[data-fieldname='fieldname'] input") @@ -251,7 +258,7 @@ context("Form Builder", () => { cy.click_doc_primary_button("Save"); cy.get_open_dialog().find(".msgprint").should("contain", "appears multiple times"); cy.hide_dialog(); - cy.get(first_field).click(); + cy.get(last_field).click(); cy.get(".sidebar-container .frappe-control[data-fieldname='fieldname'] input").clear({ force: true, }); diff --git a/esbuild/sass_options.js b/esbuild/sass_options.js index 92b691cb46..1510a856d6 100644 --- a/esbuild/sass_options.js +++ b/esbuild/sass_options.js @@ -1,11 +1,11 @@ let path = require("path"); let { get_app_path, app_list } = require("./utils"); -let node_modules_path = path.resolve(get_app_path("frappe"), "..", "node_modules"); let app_paths = app_list.map(get_app_path).map((app_path) => path.resolve(app_path, "..")); +let node_modules_path = app_paths.map((app_path) => path.resolve(app_path, "node_modules")); module.exports = { - includePaths: [node_modules_path, ...app_paths], + includePaths: [...node_modules_path, ...app_paths], quietDeps: true, importer: function (url) { if (url.startsWith("~")) { diff --git a/frappe/__init__.py b/frappe/__init__.py index 30aa60bf32..9b3f256dec 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -136,16 +136,6 @@ def as_unicode(text: str, encoding: str = "utf-8") -> str: return str(text) -def get_lang_dict(fortype: str, name: str | None = None) -> dict[str, str]: - """Returns the translated language dict for the given type and name. - - :param fortype: must be one of `doctype`, `page`, `report`, `include`, `jsfile`, `boot` - :param name: name of the document for which assets are to be returned.""" - from frappe.translate import get_dict - - return get_dict(fortype, name) - - def set_user_lang(user: str, user_language: str | None = None) -> None: """Guess and set user language for the session. `frappe.local.lang`""" from frappe.translate import get_user_lang @@ -160,6 +150,7 @@ qb = local("qb") conf = local("conf") form = form_dict = local("form_dict") request = local("request") +job = local("job") response = local("response") session = local("session") user = local("user") diff --git a/frappe/auth.py b/frappe/auth.py index 35e07236bb..4b53e76533 100644 --- a/frappe/auth.py +++ b/frappe/auth.py @@ -20,7 +20,7 @@ from frappe.twofactor import ( ) from frappe.utils import cint, date_diff, datetime, get_datetime, today from frappe.utils.deprecations import deprecation_warning -from frappe.utils.password import check_password +from frappe.utils.password import check_password, get_decrypted_password from frappe.website.utils import get_home_page SAFE_HTTP_METHODS = frozenset(("GET", "HEAD", "OPTIONS")) @@ -574,6 +574,11 @@ def validate_auth(): validate_oauth(authorization_header) validate_auth_via_api_keys(authorization_header) + # If login via bearer, basic or keypair didn't work then authentication failed and we + # should terminate here. + if frappe.session.user in ("", "Guest"): + raise frappe.AuthenticationError + validate_auth_via_hooks() @@ -588,6 +593,9 @@ def validate_oauth(authorization_header): from frappe.integrations.oauth2 import get_oauth_server from frappe.oauth import get_url_delimiter + if authorization_header[0].lower() != "bearer": + return + form_dict = frappe.local.form_dict token = authorization_header[1] req = frappe.request @@ -613,7 +621,7 @@ def validate_oauth(authorization_header): frappe.set_user(frappe.db.get_value("OAuth Bearer Token", token, "user")) frappe.local.form_dict = form_dict except AttributeError: - pass + raise frappe.AuthenticationError def validate_auth_via_api_keys(authorization_header): @@ -639,15 +647,17 @@ def validate_auth_via_api_keys(authorization_header): frappe.InvalidAuthorizationToken, ) except (AttributeError, TypeError, ValueError): - pass + raise frappe.AuthenticationError def validate_api_key_secret(api_key, api_secret, frappe_authorization_source=None): """frappe_authorization_source to provide api key and secret for a doctype apart from User""" doctype = frappe_authorization_source or "User" doc = frappe.db.get_value(doctype=doctype, filters={"api_key": api_key}, fieldname=["name"]) + if not doc: + raise frappe.AuthenticationError form_dict = frappe.local.form_dict - doc_secret = frappe.utils.password.get_decrypted_password(doctype, doc, fieldname="api_secret") + doc_secret = get_decrypted_password(doctype, doc, fieldname="api_secret") if api_secret == doc_secret: if doctype == "User": user = frappe.db.get_value(doctype="User", filters={"api_key": api_key}, fieldname=["name"]) @@ -656,6 +666,8 @@ def validate_api_key_secret(api_key, api_secret, frappe_authorization_source=Non if frappe.local.login_manager.user in ("", "Guest"): frappe.set_user(user) frappe.local.form_dict = form_dict + else: + raise frappe.AuthenticationError def validate_auth_via_hooks(): diff --git a/frappe/client.py b/frappe/client.py index 6439e9d71d..91f531fe1e 100644 --- a/frappe/client.py +++ b/frappe/client.py @@ -28,6 +28,7 @@ def get_list( doctype, fields=None, filters=None, + group_by=None, order_by=None, limit_start=None, limit_page_length=20, @@ -53,6 +54,7 @@ def get_list( fields=fields, filters=filters, or_filters=or_filters, + group_by=group_by, order_by=order_by, limit_start=limit_start, limit_page_length=limit_page_length, diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py index f9add0a7c5..715c079af3 100644 --- a/frappe/commands/utils.py +++ b/frappe/commands/utils.py @@ -504,7 +504,7 @@ def postgres(context, extra_args): def _mariadb(extra_args=None): - mariadb = which("mariadb") + mariadb = which("mariadb") or which("mysql") command = [ mariadb, "--port", diff --git a/frappe/contacts/doctype/address/address.json b/frappe/contacts/doctype/address/address.json index 4b104c7ab0..0ec67103a7 100644 --- a/frappe/contacts/doctype/address/address.json +++ b/frappe/contacts/doctype/address/address.json @@ -148,7 +148,7 @@ "icon": "fa fa-map-marker", "idx": 5, "links": [], - "modified": "2023-10-11 11:48:26.954934", + "modified": "2023-10-30 05:50:23.912366", "modified_by": "Administrator", "module": "Contacts", "name": "Address", @@ -217,7 +217,6 @@ "write": 1 } ], - "quick_entry": 1, "search_fields": "country, state", "sort_field": "modified", "sort_order": "DESC", diff --git a/frappe/core/doctype/audit_trail/audit_trail.js b/frappe/core/doctype/audit_trail/audit_trail.js index ffd289257e..6fe3e46af4 100644 --- a/frappe/core/doctype/audit_trail/audit_trail.js +++ b/frappe/core/doctype/audit_trail/audit_trail.js @@ -3,6 +3,20 @@ frappe.ui.form.on("Audit Trail", { refresh(frm) { + let prev_route = frappe.get_prev_route(); + if ( + prev_route.length > 2 && + prev_route[0] == "Form" && + !prev_route.includes("Audit Trail") + ) { + frm.set_value("doctype_name", prev_route[1]); + frm.set_value("document", prev_route[2]); + frm.set_value("start_date", ""); + frm.set_value("end_date", ""); + if (frm.doc.doctype_name && frm.doc.document) + frm.events.get_audit_trail_for_document(frm); + } + frm.page.clear_indicator(); frm.disable_save(); @@ -16,17 +30,61 @@ frappe.ui.form.on("Audit Trail", { }; }); + frm.set_query("document", () => { + let filters = { + amended_from: ["!=", ""], + }; + if (frm.doc.start_date && frm.doc.end_date) + filters["creation"] = ["between", [frm.doc.start_date, frm.doc.end_date]]; + else if (frm.doc.start_date) filters["creation"] = [">=", frm.doc.start_date]; + else if (frm.doc.end_date) filters["creation"] = ["<=", frm.doc.end_date]; + return { + filters: filters, + }; + }); + frm.page.set_primary_action("Compare", () => { - frm.call({ - doc: frm.doc, - method: "compare_document", - callback: function (r) { - let document_names = r.message[0]; - let changed_fields = r.message[1]; - frm.events.render_changed_fields(frm, document_names, changed_fields); - frm.events.render_rows_added_or_removed(frm, changed_fields); - }, + frm.events.get_audit_trail_for_document(frm); + }); + }, + + start_date(frm) { + if (frm.doc.start_date > frm.doc.end_date) { + frm.doc.end_date = ""; + frm.refresh_fields(); + } + + frappe.db + .get_value(frm.doc.doctype_name, frm.doc.document, "creation") + .then((creation) => { + if (frappe.datetime.obj_to_str(creation) < frm.doc.start_date) { + frm.doc.document = ""; + frm.refresh_fields(); + } }); + }, + + end_date(frm) { + frappe.db + .get_value(frm.doc.doctype_name, frm.doc.document, "creation") + .then((creation) => { + if (frappe.datetime.obj_to_str(creation) > frm.doc.end_date) { + frm.doc.document = ""; + frm.refresh_fields(); + } + }); + }, + + get_audit_trail_for_document(frm) { + frm.call({ + doc: frm.doc, + method: "compare_document", + callback: function (r) { + let document_names = r.message[0]; + let changed_fields = r.message[1]; + frm.events.render_changed_fields(frm, document_names, changed_fields); + frm.events.render_rows_added_or_removed(frm, changed_fields); + }, }); }, diff --git a/frappe/core/doctype/audit_trail/audit_trail.json b/frappe/core/doctype/audit_trail/audit_trail.json index 4e47ce2562..762861eedb 100644 --- a/frappe/core/doctype/audit_trail/audit_trail.json +++ b/frappe/core/doctype/audit_trail/audit_trail.json @@ -9,6 +9,10 @@ "doctype_name", "column_break_peck", "document", + "section_break_dfrx", + "start_date", + "column_break_ytzm", + "end_date", "section_break_gppi", "version_table", "rows_added_section", @@ -68,13 +72,34 @@ "fieldtype": "Section Break", "hidden": 1, "label": "Rows Removed" + }, + { + "fieldname": "start_date", + "fieldtype": "Date", + "label": "Start Date" + }, + { + "fieldname": "end_date", + "fieldtype": "Date", + "label": "End Date" + }, + { + "collapsible": 1, + "collapsible_depends_on": "eval: doc.start_date || doc.end_date", + "fieldname": "section_break_dfrx", + "fieldtype": "Section Break", + "label": "Date Range" + }, + { + "fieldname": "column_break_ytzm", + "fieldtype": "Column Break" } ], "hide_toolbar": 1, "index_web_pages_for_search": 1, "issingle": 1, "links": [], - "modified": "2023-10-04 12:45:49.099121", + "modified": "2023-10-31 13:12:41.749483", "modified_by": "Administrator", "module": "Core", "name": "Audit Trail", diff --git a/frappe/core/doctype/audit_trail/audit_trail.py b/frappe/core/doctype/audit_trail/audit_trail.py index b8e4e48b43..2b2e843f46 100644 --- a/frappe/core/doctype/audit_trail/audit_trail.py +++ b/frappe/core/doctype/audit_trail/audit_trail.py @@ -7,6 +7,7 @@ import frappe from frappe import _ from frappe.core.doctype.version.version import get_diff from frappe.model.document import Document +from frappe.utils import compare class AuditTrail(Document): @@ -20,20 +21,31 @@ class AuditTrail(Document): doctype_name: DF.Link document: DF.DynamicLink + end_date: DF.Date | None + start_date: DF.Date | None # end: auto-generated types pass def validate(self): - self.validate_doctype_name() + self.validate_fields() self.validate_document() - def validate_doctype_name(self): - if not self.doctype_name: - frappe.throw(_("{} field cannot be empty.").format(frappe.bold("Doctype"))) + def validate_fields(self): + fields_dict = { + "DocType": self.doctype_name, + "Document": self.document, + } + for field in fields_dict: + if not fields_dict[field]: + frappe.throw(_("{} field cannot be empty.").format(frappe.bold(field))) def validate_document(self): - if not self.document: - frappe.throw(_("{} field cannot be empty.").format(frappe.bold("Document"))) + if not frappe.db.exists(self.doctype_name, self.document): + frappe.throw( + _("The selected document {0} is not a {1}.").format( + frappe.bold(self.document), frappe.bold(self.doctype_name) + ) + ) @frappe.whitelist() def compare_document(self): @@ -58,11 +70,18 @@ class AuditTrail(Document): } def get_amended_documents(self): + start_date = self.get("start_date") amended_document_names = [] curr_doc = self.document - while curr_doc and len(amended_document_names) < 5: + creation = frappe.db.get_value(self.doctype_name, self.document, "creation") + while ( + curr_doc + and len(amended_document_names) < 5 + and (start_date is None or compare(creation, ">=", start_date, "Date")) + ): amended_document_names.append(curr_doc) curr_doc = frappe.db.get_value(self.doctype_name, curr_doc, "amended_from") + creation = frappe.db.get_value(self.doctype_name, curr_doc, "creation") amended_document_names = amended_document_names[::-1] return amended_document_names diff --git a/frappe/core/doctype/audit_trail/test_audit_trail.py b/frappe/core/doctype/audit_trail/test_audit_trail.py index 45093de033..c5f195a9f6 100644 --- a/frappe/core/doctype/audit_trail/test_audit_trail.py +++ b/frappe/core/doctype/audit_trail/test_audit_trail.py @@ -3,6 +3,7 @@ import frappe from frappe.tests.utils import FrappeTestCase +from frappe.utils import today class TestAuditTrail(FrappeTestCase): @@ -129,6 +130,11 @@ def amend_document(amend_from, changed_fields, rows_updated, submit=False): def create_comparator_doc(doctype_name, document): comparator = frappe.new_doc("Audit Trail") - comparator.doctype_name = doctype_name - comparator.document = document + args_dict = { + "doctype_name": doctype_name, + "document": document, + "start_date": today(), + "end_date": today(), + } + comparator.update(args_dict) return comparator diff --git a/frappe/core/doctype/data_import/patches/__init__.py b/frappe/core/doctype/data_import/patches/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/frappe/core/doctype/data_import/patches/remove_stale_docfields_from_legacy_version.py b/frappe/core/doctype/data_import/patches/remove_stale_docfields_from_legacy_version.py new file mode 100644 index 0000000000..91d9c8f641 --- /dev/null +++ b/frappe/core/doctype/data_import/patches/remove_stale_docfields_from_legacy_version.py @@ -0,0 +1,6 @@ +import frappe + + +def execute(): + """Remove stale docfields from legacy version""" + frappe.db.delete("DocField", {"options": "Data Import", "parent": "Data Import Legacy"}) diff --git a/frappe/core/doctype/doctype/doctype.js b/frappe/core/doctype/doctype/doctype.js index 929244c977..c21654a109 100644 --- a/frappe/core/doctype/doctype/doctype.js +++ b/frappe/core/doctype/doctype/doctype.js @@ -2,6 +2,12 @@ // MIT License. See license.txt frappe.ui.form.on("DocType", { + onload: function (frm) { + if (frm.is_new()) { + frappe.listview_settings["DocType"].new_doctype_dialog(); + } + }, + before_save: function (frm) { let form_builder = frappe.form_builder; if (form_builder?.store) { @@ -13,6 +19,7 @@ frappe.ui.form.on("DocType", { } } }, + after_save: function (frm) { if ( frappe.form_builder && @@ -22,6 +29,7 @@ frappe.ui.form.on("DocType", { frappe.form_builder.store.fetch(); } }, + refresh: function (frm) { frm.set_query("role", "permissions", function (doc) { if (doc.custom && frappe.session.user != "Administrator") { @@ -119,6 +127,20 @@ frappe.ui.form.on("DocType", { setup_default_views: (frm) => { frappe.model.set_default_views_for_doctype(frm.doc.name, frm); }, + + on_tab_change: (frm) => { + let current_tab = frm.get_active_tab().label; + + if (current_tab === "Form") { + frm.footer.wrapper.hide(); + frm.form_wrapper.find(".form-message").hide(); + frm.form_wrapper.addClass("mb-1"); + } else { + frm.footer.wrapper.show(); + frm.form_wrapper.find(".form-message").show(); + frm.form_wrapper.removeClass("mb-1"); + } + }, }); frappe.ui.form.on("DocField", { diff --git a/frappe/core/doctype/doctype/doctype.json b/frappe/core/doctype/doctype/doctype.json index 21d5fbfac8..082471da7d 100644 --- a/frappe/core/doctype/doctype/doctype.json +++ b/frappe/core/doctype/doctype/doctype.json @@ -8,6 +8,9 @@ "document_type": "Document", "engine": "InnoDB", "field_order": [ + "form_builder_tab", + "form_builder", + "settings_tab", "sb0", "module", "is_submittable", @@ -32,32 +35,6 @@ "column_break_15", "description", "documentation", - "sb2", - "permissions", - "restrict_to_domain", - "read_only", - "in_create", - "actions_section", - "actions", - "links_section", - "links", - "document_states_section", - "states", - "web_view", - "has_web_view", - "allow_guest_to_view", - "index_web_pages_for_search", - "route", - "is_published_field", - "website_search_field", - "advanced", - "engine", - "migration_hash", - "form_builder_tab", - "form_builder", - "fields_section", - "fields", - "settings_tab", "form_settings_section", "image_field", "timeline_field", @@ -92,6 +69,29 @@ "email_append_to", "sender_field", "subject_field", + "sb2", + "permissions", + "restrict_to_domain", + "read_only", + "in_create", + "actions_section", + "actions", + "links_section", + "links", + "document_states_section", + "states", + "web_view", + "has_web_view", + "allow_guest_to_view", + "index_web_pages_for_search", + "route", + "is_published_field", + "website_search_field", + "advanced", + "engine", + "migration_hash", + "fields_section", + "fields", "connections_tab" ], "fields": [ @@ -640,6 +640,7 @@ "label": "Settings" }, { + "depends_on": "eval:!doc.__islocal", "fieldname": "form_builder_tab", "fieldtype": "Tab Break", "label": "Form" @@ -742,7 +743,7 @@ "link_fieldname": "reference_doctype" } ], - "modified": "2023-08-29 12:27:06.587523", + "modified": "2023-11-01 16:45:14.960949", "modified_by": "Administrator", "module": "Core", "name": "DocType", diff --git a/frappe/core/doctype/doctype/doctype.py b/frappe/core/doctype/doctype/doctype.py index cf5f608b4b..7b5c58dedd 100644 --- a/frappe/core/doctype/doctype/doctype.py +++ b/frappe/core/doctype/doctype/doctype.py @@ -350,8 +350,10 @@ class DocType(Document): self.flags.update_fields_to_fetch_queries = [] - if set(old_fields_to_fetch) != {df.fieldname for df in new_meta.get_fields_to_fetch()}: - for df in new_meta.get_fields_to_fetch(): + new_fields_to_fetch = new_meta.get_fields_to_fetch() + + if set(old_fields_to_fetch) != {df.fieldname for df in new_fields_to_fetch}: + for df in new_fields_to_fetch: if df.fieldname not in old_fields_to_fetch: link_fieldname, source_fieldname = df.fetch_from.split(".", 1) link_df = new_meta.get_field(link_fieldname) @@ -868,6 +870,7 @@ class DocType(Document): "read_only": 1, "print_hide": 1, "no_copy": 1, + "search_index": 1, }, ) diff --git a/frappe/core/doctype/doctype/doctype_list.js b/frappe/core/doctype/doctype/doctype_list.js new file mode 100644 index 0000000000..963e863380 --- /dev/null +++ b/frappe/core/doctype/doctype/doctype_list.js @@ -0,0 +1,122 @@ +frappe.listview_settings["DocType"] = { + primary_action: function () { + this.new_doctype_dialog(); + }, + + new_doctype_dialog() { + let non_developer = frappe.session.user !== "Administrator" || !frappe.boot.developer_mode; + let fields = [ + { + label: __("DocType Name"), + fieldname: "name", + fieldtype: "Data", + reqd: 1, + }, + { fieldtype: "Column Break" }, + { + label: __("Module"), + fieldname: "module", + fieldtype: "Link", + options: "Module Def", + reqd: 1, + }, + { fieldtype: "Section Break" }, + { + label: __("Is Submittable"), + fieldname: "is_submittable", + fieldtype: "Check", + description: __( + "Once submitted, submittable documents cannot be changed. They can only be Cancelled and Amended." + ), + depends_on: "eval:!doc.istable && !doc.issingle", + }, + { + label: __("Is Child Table"), + fieldname: "istable", + fieldtype: "Check", + description: __("Child Tables are shown as a Grid in other DocTypes"), + depends_on: "eval:!doc.is_submittable && !doc.issingle", + }, + { + label: __("Editable Grid"), + fieldname: "editable_grid", + fieldtype: "Check", + depends_on: "istable", + default: 1, + }, + { + label: __("Is Single"), + fieldname: "issingle", + fieldtype: "Check", + description: __( + "Single Types have only one record no tables associated. Values are stored in tabSingles" + ), + depends_on: "eval:!doc.istable && !doc.is_submittable", + }, + { + label: "Is Tree", + fieldname: "is_tree", + fieldtype: "Check", + default: "0", + depends_on: "eval:!doc.istable", + description: "Tree structures are implemented using Nested Set", + }, + { + label: __("Custom?"), + fieldname: "custom", + fieldtype: "Check", + default: non_developer, + read_only: non_developer, + }, + ]; + + if (!non_developer) { + fields.push({ + label: "Is Virtual", + fieldname: "is_virtual", + fieldtype: "Check", + default: "0", + }); + } + + let new_d = new frappe.ui.Dialog({ + title: __("Create New DocType"), + fields: fields, + primary_action_label: __("Create & Continue"), + primary_action(values) { + if (!values.istable) values.editable_grid = 0; + frappe.db + .insert({ + doctype: "DocType", + ...values, + permissions: [ + { + create: 1, + delete: 1, + email: 1, + export: 1, + print: 1, + read: 1, + report: 1, + role: "System Manager", + share: 1, + write: 1, + }, + ], + fields: [{ fieldtype: "Section Break" }], + }) + .then((doc) => { + frappe.set_route("Form", "DocType", doc.name); + }); + }, + secondary_action_label: __("Cancel"), + secondary_action() { + new_d.hide(); + if (frappe.get_route()[0] === "Form") { + frappe.set_route("List", "DocType"); + } + }, + }); + new_d.show(); + }, +}; diff --git a/frappe/core/doctype/page/page.json b/frappe/core/doctype/page/page.json index e913f126af..b5e9941a6d 100644 --- a/frappe/core/doctype/page/page.json +++ b/frappe/core/doctype/page/page.json @@ -102,14 +102,16 @@ "icon": "fa fa-file", "idx": 1, "links": [], - "modified": "2022-08-03 12:20:54.219236", + "modified": "2023-10-22 22:41:25.568952", "modified_by": "Administrator", "module": "Core", "name": "Page", + "naming_rule": "By fieldname", "owner": "Administrator", "permissions": [ { "create": 1, + "delete": 1, "email": 1, "print": 1, "read": 1, diff --git a/frappe/core/doctype/page/page.py b/frappe/core/doctype/page/page.py index 77029b8c67..270ece6fa5 100644 --- a/frappe/core/doctype/page/page.py +++ b/frappe/core/doctype/page/page.py @@ -2,9 +2,10 @@ # License: MIT. See LICENSE import os +import shutil import frappe -from frappe import _, conf, safe_decode +from frappe import _, conf, get_module_path, safe_decode from frappe.build import html_to_js_template from frappe.core.doctype.custom_role.custom_role import get_custom_allowed_roles from frappe.desk.form.meta import get_code_files_via_hooks, get_js @@ -103,7 +104,18 @@ class Page(Document): return d def on_trash(self): + if not frappe.conf.developer_mode and not frappe.flags.in_migrate: + frappe.throw(_("Deletion of this document is only permitted in developer mode.")) + delete_custom_role("page", self.name) + frappe.db.after_commit(self.delete_folder_with_contents) + + def delete_folder_with_contents(self): + module_path = get_module_path(self.module) + dir_path = os.path.join(module_path, "page", frappe.scrub(self.name)) + + if os.path.exists(dir_path): + shutil.rmtree(dir_path, ignore_errors=True) def is_permitted(self): """Returns true if Has Role is not set or the user is allowed.""" @@ -173,11 +185,6 @@ class Page(Document): # flag for not caching this page self._dynamic_page = True - if frappe.lang != "en": - from frappe.translate import get_lang_js - - self.script += get_lang_js("page", self.name) - for path in get_code_files_via_hooks("page_js", self.name): js = get_js(path) if js: diff --git a/frappe/core/doctype/page/test_page.py b/frappe/core/doctype/page/test_page.py index edf7f7c9b8..61e7ed99a0 100644 --- a/frappe/core/doctype/page/test_page.py +++ b/frappe/core/doctype/page/test_page.py @@ -1,5 +1,9 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +import os +import unittest +from unittest.mock import patch + import frappe from frappe.tests.utils import FrappeTestCase @@ -16,3 +20,18 @@ class TestPage(FrappeTestCase): frappe.NameError, frappe.get_doc(dict(doctype="Page", page_name="Settings", module="Core")).insert, ) + + @unittest.skipUnless( + os.access(frappe.get_app_path("frappe"), os.W_OK), "Only run if frappe app paths is writable" + ) + @patch.dict(frappe.conf, {"developer_mode": 1}) + def test_trashing(self): + page = frappe.new_doc("Page", page_name=frappe.generate_hash(), module="Core").insert() + + page.delete() + frappe.db.commit() + + module_path = frappe.get_module_path(page.module) + dir_path = os.path.join(module_path, "page", frappe.scrub(page.name)) + + self.assertFalse(os.path.exists(dir_path)) diff --git a/frappe/core/doctype/recorder/recorder.js b/frappe/core/doctype/recorder/recorder.js index 185f655786..37d387b711 100644 --- a/frappe/core/doctype/recorder/recorder.js +++ b/frappe/core/doctype/recorder/recorder.js @@ -2,6 +2,9 @@ // For license information, please see license.txt frappe.ui.form.on("Recorder", { + onload: function (frm) { + frm.fields_dict.sql_queries.grid.only_sortable(); + }, refresh: function (frm) { frm.disable_save(); frm._sort_order = {}; diff --git a/frappe/core/doctype/sms_settings/sms_settings.py b/frappe/core/doctype/sms_settings/sms_settings.py index 0047de1daa..1a68368ba0 100644 --- a/frappe/core/doctype/sms_settings/sms_settings.py +++ b/frappe/core/doctype/sms_settings/sms_settings.py @@ -30,7 +30,7 @@ def validate_receiver_nos(receiver_list): validated_receiver_list = [] for d in receiver_list: if not d: - break + continue # remove invalid character for x in [" ", "-", "(", ")"]: diff --git a/frappe/core/doctype/system_settings/system_settings.json b/frappe/core/doctype/system_settings/system_settings.json index 7e8664595c..54b23094f8 100644 --- a/frappe/core/doctype/system_settings/system_settings.json +++ b/frappe/core/doctype/system_settings/system_settings.json @@ -217,7 +217,7 @@ "label": "Security" }, { - "default": "60:00", + "default": "170:00", "description": "Example: Setting this to 24:00 will log out a user if they are not active for 24:00 hours.", "fieldname": "session_expiry", "fieldtype": "Data", diff --git a/frappe/custom/doctype/customize_form/customize_form.json b/frappe/custom/doctype/customize_form/customize_form.json index 8d4d194338..32fd3302ec 100644 --- a/frappe/custom/doctype/customize_form/customize_form.json +++ b/frappe/custom/doctype/customize_form/customize_form.json @@ -180,6 +180,7 @@ "depends_on": "doc_type", "fieldname": "fields_section_break", "fieldtype": "Section Break", + "hidden": 1, "label": "Fields" }, { @@ -393,7 +394,7 @@ "index_web_pages_for_search": 1, "issingle": 1, "links": [], - "modified": "2023-08-29 12:31:55.808848", + "modified": "2023-10-31 02:04:25.955931", "modified_by": "Administrator", "module": "Custom", "name": "Customize Form", diff --git a/frappe/database/db_manager.py b/frappe/database/db_manager.py index 7b244080b0..6431217585 100644 --- a/frappe/database/db_manager.py +++ b/frappe/database/db_manager.py @@ -57,6 +57,7 @@ class DbManager: esc = make_esc("$ ") pv = which("pv") + mariadb_cli = which("mariadb") or which("mysql") if pv: pipe = f"{pv} {source} |" @@ -68,7 +69,7 @@ class DbManager: if pipe: print("Restoring Database file...") - command = "{pipe} mariadb -u {user} -p{password} -h{host} -P{port} {target} {source}" + command = "{pipe} {mariadb_cli} -u {user} -p{password} -h{host} -P{port} {target} {source}" command = command.format( pipe=pipe, user=esc(user), @@ -77,6 +78,7 @@ class DbManager: target=esc(target), source=source, port=frappe.conf.db_port, + mariadb_cli=mariadb_cli, ) os.system(command) diff --git a/frappe/desk/doctype/dashboard_chart_source/dashboard_chart_source.js b/frappe/desk/doctype/dashboard_chart_source/dashboard_chart_source.js index 6f1fa36ffd..1349adaf74 100644 --- a/frappe/desk/doctype/dashboard_chart_source/dashboard_chart_source.js +++ b/frappe/desk/doctype/dashboard_chart_source/dashboard_chart_source.js @@ -1,4 +1,19 @@ // Copyright (c) 2019, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on("Dashboard Chart Source", {}); +frappe.ui.form.on("Dashboard Chart Source", { + refresh: function (frm) { + if (!frm.is_new()) { + frm.add_custom_button( + __("Dashboard Chart"), + function () { + let dashboard_chart = frappe.model.get_new_doc("Dashboard Chart"); + dashboard_chart.chart_type = "Custom"; + dashboard_chart.source = frm.doc.name; + frappe.set_route("Form", "Dashboard Chart", dashboard_chart.name); + }, + __("Create") + ); + } + }, +}); diff --git a/frappe/desk/form/linked_with.py b/frappe/desk/form/linked_with.py index 501953c030..cbe6dd7acc 100644 --- a/frappe/desk/form/linked_with.py +++ b/frappe/desk/form/linked_with.py @@ -71,7 +71,7 @@ class SubmittableDocumentTree: def get_all_children(self): """Get all nodes of a tree except the root node (all the nested submitted - documents those are present in referencing tables (dependent tables). + documents those are present in referencing tables dependent tables). """ while self.to_be_visited_documents: next_level_children = defaultdict(list) @@ -101,6 +101,10 @@ class SubmittableDocumentTree: child_docs = defaultdict(list) for field in referencing_fields: + if field["fieldname"] == "amended_from": + # perf: amended_from links are always linked to cancelled documents. + continue + links = ( get_referencing_documents( parent_dt, diff --git a/frappe/desk/form/meta.py b/frappe/desk/form/meta.py index d0a6b2501c..40497030a9 100644 --- a/frappe/desk/form/meta.py +++ b/frappe/desk/form/meta.py @@ -1,6 +1,5 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import io import os import frappe @@ -45,9 +44,6 @@ def get_meta(doctype, cached=True) -> "FormMeta": else: meta = FormMeta(doctype) - if frappe.local.lang != "en": - meta.set_translations(frappe.local.lang) - return meta @@ -256,18 +252,6 @@ class FormMeta(Meta): self.set("__form_grid_templates", templates) - def set_translations(self, lang): - from frappe.translate import extract_messages_from_code, make_dict_from_messages - - self.set("__messages", frappe.get_lang_dict("doctype", self.name)) - - # set translations for grid templates - if self.get("__form_grid_templates"): - for content in self.get("__form_grid_templates").values(): - messages = extract_messages_from_code(content) - messages = make_dict_from_messages(messages) - self.get("__messages").update(messages) - def load_dashboard(self): self.set("__dashboard", self.get_dashboard_data()) diff --git a/frappe/desk/page/setup_wizard/setup_wizard.js b/frappe/desk/page/setup_wizard/setup_wizard.js index c237624fff..8d42b804cd 100644 --- a/frappe/desk/page/setup_wizard/setup_wizard.js +++ b/frappe/desk/page/setup_wizard/setup_wizard.js @@ -404,7 +404,7 @@ frappe.setup.slides_settings = [ fieldname: "enable_telemetry", label: __("Allow sending usage data for improving applications"), fieldtype: "Check", - default: 1, + default: cint(frappe.telemetry.can_enable()), depends_on: "eval:frappe.telemetry.can_enable()", }, { diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index 24d548cb37..9bcf328116 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -348,14 +348,17 @@ class EmailAccount(Document): return frappe.get_doc(cls.DOCTYPE, name) @classmethod - def find_one_by_filters(cls, **kwargs): + def find_one_by_filters(cls, **kwargs) -> "EmailAccount": name = frappe.db.get_value(cls.DOCTYPE, kwargs) return cls.find(name) if name else None @classmethod def find_from_config(cls): config = cls.get_account_details_from_site_config() - return cls.from_record(config) if config else None + if config: + account = cls.from_record(config) + account._from_site_config = True + return account @classmethod def create_dummy(cls): @@ -475,9 +478,22 @@ class EmailAccount(Document): } def get_smtp_server(self): + """Get SMTPServer (wrapper around actual smtplib object) for this account. + + Implementation Detail: Since SMTPServer is same for each email connection, the same *instance* + is returned every time this function is called from same EmailAccount object. + This enables reusabilty of connection for better performance.""" + return self._smtp_server_instance + + @functools.cached_property + def _smtp_server_instance(self): config = self.sendmail_config() return SMTPServer(**config) + def remove_unpicklable_values(self, state): + super().remove_unpicklable_values(state) + state.pop("_smtp_server_instance", None) + def handle_incoming_connect_error(self, description): if test_internet(): if self.get_failed_attempts_count() > 2: diff --git a/frappe/email/doctype/email_queue/email_queue.py b/frappe/email/doctype/email_queue/email_queue.py index 5a4ac1ad70..da10ae5d16 100644 --- a/frappe/email/doctype/email_queue/email_queue.py +++ b/frappe/email/doctype/email_queue/email_queue.py @@ -31,6 +31,7 @@ from frappe.utils import ( sbool, split_emails, ) +from frappe.utils.deprecations import deprecated from frappe.utils.verified_command import get_signed_params @@ -86,7 +87,7 @@ class EmailQueue(Document): return duplicate @classmethod - def new(cls, doc_data, ignore_permissions=False): + def new(cls, doc_data, ignore_permissions=False) -> "EmailQueue": data = doc_data.copy() if not data.get("recipients"): return @@ -99,7 +100,7 @@ class EmailQueue(Document): return doc @classmethod - def find(cls, name): + def find(cls, name) -> "EmailQueue": return frappe.get_doc(cls.DOCTYPE, name) @classmethod @@ -166,14 +167,14 @@ class EmailQueue(Document): if method := get_hook_method("override_email_send"): method(self, self.sender, recipient.recipient, message) else: - if not frappe.flags.in_test: + if not frappe.flags.in_test or frappe.flags.testing_email: ctx.smtp_server.session.sendmail( from_addr=self.sender, to_addrs=recipient.recipient, msg=message ) ctx.update_recipient_status_to_sent(recipient) - if frappe.flags.in_test: + if frappe.flags.in_test and not frappe.flags.testing_email: frappe.flags.sent_mail = message return @@ -213,6 +214,7 @@ class EmailQueue(Document): @task(queue="short") +@deprecated def send_mail(email_queue_name, smtp_server_instance: SMTPServer = None): """This is equivalent to EmailQueue.send. @@ -231,11 +233,7 @@ class SendMailContext: self.queue_doc: EmailQueue = queue_doc self.email_account_doc = queue_doc.get_email_account() - self.smtp_server = smtp_server_instance or self.email_account_doc.get_smtp_server() - - # if smtp_server_instance is passed, then retain smtp session - # Note: smtp session will have to be manually closed - self.retain_smtp_session = bool(smtp_server_instance) + self.smtp_server: SMTPServer = smtp_server_instance or self.email_account_doc.get_smtp_server() self.sent_to_atleast_one_recipient = any( rec.recipient for rec in self.queue_doc.recipients if rec.is_mail_sent() @@ -246,9 +244,6 @@ class SendMailContext: return self def __exit__(self, exc_type, exc_val, exc_tb): - if not self.retain_smtp_session: - self.smtp_server.quit() - if exc_type: update_fields = {"error": "".join(traceback.format_tb(exc_tb))} if self.queue_doc.retry < get_email_retry_limit(): diff --git a/frappe/email/doctype/email_queue/test_email_queue.py b/frappe/email/doctype/email_queue/test_email_queue.py index dbd01019b9..7d76039b47 100644 --- a/frappe/email/doctype/email_queue/test_email_queue.py +++ b/frappe/email/doctype/email_queue/test_email_queue.py @@ -78,3 +78,20 @@ class TestEmailQueue(FrappeTestCase): {"subject": f"Failed to send email with subject: {subject}"}, ) self.assertTrue(notification_log) + + def test_perf_reusing_smtp_server(self): + """Ensure that same smtpserver instance is being returned when retrieved multiple times.""" + + self.assertTrue(frappe.new_doc("Email Queue").get_email_account()._from_site_config) + + def get_server(q): + return q.get_email_account().get_smtp_server() + + self.assertIs( + get_server(frappe.new_doc("Email Queue")), get_server(frappe.new_doc("Email Queue")) + ) + + q1 = frappe.new_doc("Email Queue", email_account="_Test Email Account 1") + q2 = frappe.new_doc("Email Queue", email_account="_Test Email Account 1") + self.assertIsNot(get_server(frappe.new_doc("Email Queue")), get_server(q1)) + self.assertIs(get_server(q1), get_server(q2)) diff --git a/frappe/email/doctype/newsletter/test_newsletter.py b/frappe/email/doctype/newsletter/test_newsletter.py index edbc8ff425..9677b94de3 100644 --- a/frappe/email/doctype/newsletter/test_newsletter.py +++ b/frappe/email/doctype/newsletter/test_newsletter.py @@ -151,7 +151,7 @@ class TestNewsletter(TestNewsletterMixin, FrappeTestCase): "Newsletter Email Group", filters={"parent": name}, fields=["email_group"] ) - flush(from_test=True) + flush() confirmed_unsubscribe(to_unsubscribe, group[0].email_group) name = self.send_newsletter() diff --git a/frappe/email/doctype/notification/notification.py b/frappe/email/doctype/notification/notification.py index 9cdb4bdcb0..a8c797a008 100644 --- a/frappe/email/doctype/notification/notification.py +++ b/frappe/email/doctype/notification/notification.py @@ -305,7 +305,7 @@ def get_context(context): def send_sms(self, doc, context): send_sms( receiver_list=self.get_receiver_list(doc, context), - msg=frappe.render_template(self.message, context), + msg=frappe.utils.strip_html_tags(frappe.render_template(self.message, context)), ) def get_list_of_recipients(self, doc, context): diff --git a/frappe/email/queue.py b/frappe/email/queue.py index b481fd21cd..6c78383b0c 100755 --- a/frappe/email/queue.py +++ b/frappe/email/queue.py @@ -7,6 +7,11 @@ from frappe.utils import cint, cstr, get_url, now_datetime from frappe.utils.data import getdate from frappe.utils.verified_command import get_signed_params, verify_request +# After this percent of failures in every batch, entire batch is aborted. +# This usually indicates a systemic failure so we shouldn't keep trying to send emails. +EMAIL_QUEUE_BATCH_FAILURE_THRESHOLD_PERCENT = 0.33 +EMAIL_QUEUE_BATCH_FAILURE_THRESHOLD_COUNT = 10 + def get_emails_sent_this_month(email_account=None): """Get count of emails sent from a specific email account. @@ -124,35 +129,45 @@ def return_unsubscribed_page(email, doctype, name): ) -def flush(from_test=False): - """flush email queue, every time: called from scheduler""" - from frappe.email.doctype.email_queue.email_queue import send_mail +def flush(): + """flush email queue, every time: called from scheduler. + + This should not be called outside of background jobs. + """ + from frappe.email.doctype.email_queue.email_queue import EmailQueue # To avoid running jobs inside unit tests if frappe.are_emails_muted(): msgprint(_("Emails are muted")) - from_test = True if cint(frappe.db.get_default("suspend_email_queue")) == 1: return - for row in get_queue(): + email_queue_batch = get_queue() + if not email_queue_batch: + return + + failed_email_queues = [] + for row in email_queue_batch: try: - frappe.enqueue( - method=send_mail, - email_queue_name=row.name, - now=from_test, - job_id=f"email_queue_sendmail_{row.name}", - queue="short", - deduplicate=True, - ) + email_queue: EmailQueue = frappe.get_doc("Email Queue", row.name) + email_queue.send() except Exception: frappe.get_doc("Email Queue", row.name).log_error() + failed_email_queues.append(row.name) + + if ( + len(failed_email_queues) / len(email_queue_batch) > EMAIL_QUEUE_BATCH_FAILURE_THRESHOLD_PERCENT + and len(failed_email_queues) > EMAIL_QUEUE_BATCH_FAILURE_THRESHOLD_COUNT + ): + frappe.throw(_("Email Queue flushing aborted due to too many failures.")) def get_queue(): + batch_size = cint(frappe.conf.email_queue_batch_size) or 500 + return frappe.db.sql( - """select + f"""select name, sender from `tabEmail Queue` @@ -160,8 +175,8 @@ def get_queue(): (status='Not Sent' or status='Partially Sent') and (send_after is null or send_after < %(now)s) order - by priority desc, creation asc - limit 500""", + by priority desc, retry asc, creation asc + limit {batch_size}""", {"now": now_datetime()}, as_dict=True, ) diff --git a/frappe/email/smtp.py b/frappe/email/smtp.py index 7b15440ccf..8fb65b7cd6 100644 --- a/frappe/email/smtp.py +++ b/frappe/email/smtp.py @@ -61,6 +61,10 @@ class SMTPServer: @property def session(self): + """Get SMTP session. + + We make best effort to revive connection if it's disconnected by checking the connection + health before returning it to user.""" if self.is_session_active(): return self._session @@ -86,14 +90,29 @@ class SMTPServer: frappe.msgprint(res[1], raise_exception=frappe.OutgoingEmailError) self._session = _session + self._enqueue_connection_closure() return self._session except smtplib.SMTPAuthenticationError: self.throw_invalid_credentials_exception() - except OSError: + except OSError as e: # Invalid mail server -- due to refusing connection - frappe.throw(_("Invalid Outgoing Mail Server or Port"), title=_("Incorrect Configuration")) + frappe.throw( + _("Invalid Outgoing Mail Server or Port: {0}").format(str(e)), + title=_("Incorrect Configuration"), + ) + + def _enqueue_connection_closure(self): + if frappe.request and hasattr(frappe.request, "after_response"): + frappe.request.after_response.add(self.quit) + elif frappe.job: + frappe.job.after_job.add(self.quit) + else: + # Console? + import atexit + + atexit.register(self.quit) def is_session_active(self): if self._session: diff --git a/frappe/geo/country_info.py b/frappe/geo/country_info.py index 3267149d4c..e07581c056 100644 --- a/frappe/geo/country_info.py +++ b/frappe/geo/country_info.py @@ -8,6 +8,7 @@ import os from functools import lru_cache import frappe +from frappe.utils.deprecations import deprecated from frappe.utils.momentjs import get_all_timezones @@ -38,29 +39,23 @@ def _get_country_timezone_info(): return {"country_info": get_all(), "all_timezones": get_all_timezones()} +@deprecated def get_translated_dict(): - from babel.dates import Locale, get_timezone, get_timezone_name + return get_translated_countries() + + +def get_translated_countries(): + from babel.dates import Locale translated_dict = {} locale = Locale.parse(frappe.local.lang, sep="-") - # timezones - for tz in get_all_timezones(): - timezone_name = get_timezone_name(get_timezone(tz), locale=locale, width="short") - if timezone_name: - translated_dict[tz] = timezone_name + " - " + tz - # country names && currencies for country, info in get_all().items(): country_name = locale.territories.get((info.get("code") or "").upper()) if country_name: translated_dict[country] = country_name - currency = info.get("currency") - currency_name = locale.currencies.get(currency) - if currency_name: - translated_dict[currency] = currency_name - return translated_dict diff --git a/frappe/handler.py b/frappe/handler.py index d24d548425..6db6a7600f 100644 --- a/frappe/handler.py +++ b/frappe/handler.py @@ -15,7 +15,7 @@ from frappe.core.doctype.server_script.server_script_utils import get_server_scr from frappe.monitor import add_data_to_monitor from frappe.utils import cint from frappe.utils.csvutils import build_csv_response -from frappe.utils.deprecations import deprecation_warning +from frappe.utils.deprecations import deprecated, deprecation_warning from frappe.utils.image import optimize_image from frappe.utils.response import build_response @@ -347,5 +347,4 @@ def run_doc_method(method, docs=None, dt=None, dn=None, arg=None, args=None): add_data_to_monitor(methodname=method) -# for backwards compatibility -runserverobj = run_doc_method +runserverobj = deprecated(run_doc_method) diff --git a/frappe/hooks.py b/frappe/hooks.py index 7e2d2537e3..988d0e5ee8 100644 --- a/frappe/hooks.py +++ b/frappe/hooks.py @@ -269,11 +269,6 @@ scheduler_events = { ], } -get_translated_dict = { - ("doctype", "System Settings"): "frappe.geo.country_info.get_translated_dict", - ("page", "setup-wizard"): "frappe.geo.country_info.get_translated_dict", -} - sounds = [ {"name": "email", "src": "/assets/frappe/sounds/email.mp3", "volume": 0.1}, {"name": "submit", "src": "/assets/frappe/sounds/submit.mp3", "volume": 0.1}, diff --git a/frappe/installer.py b/frappe/installer.py index 39e008dff2..93facf2b0e 100644 --- a/frappe/installer.py +++ b/frappe/installer.py @@ -726,7 +726,7 @@ def _guess_mariadb_version() -> tuple[int] | None: # in non-interactive mode. # Use db.sql("select version()") instead if connection is available. with suppress(Exception): - mariadb = which("mariadb") + mariadb = which("mariadb") or which("mysql") version_output = subprocess.getoutput(f"{mariadb} --version") version_regex = r"(?P\d+\.\d+\.\d+)-MariaDB" diff --git a/frappe/integrations/utils.py b/frappe/integrations/utils.py index 0f94d2ee29..9e80a9aa34 100644 --- a/frappe/integrations/utils.py +++ b/frappe/integrations/utils.py @@ -10,7 +10,7 @@ from frappe import _ from frappe.utils import get_request_session -def make_request(method, url, auth=None, headers=None, data=None, json=None): +def make_request(method, url, auth=None, headers=None, data=None, json=None, params=None): auth = auth or "" data = data or {} headers = headers or {} @@ -18,7 +18,7 @@ def make_request(method, url, auth=None, headers=None, data=None, json=None): try: s = get_request_session() frappe.flags.integration_request = s.request( - method, url, data=data, auth=auth, headers=headers, json=json + method, url, data=data, auth=auth, headers=headers, json=json, params=params ) frappe.flags.integration_request.raise_for_status() diff --git a/frappe/migrate.py b/frappe/migrate.py index 6b83521a7f..33c930e9da 100644 --- a/frappe/migrate.py +++ b/frappe/migrate.py @@ -1,6 +1,8 @@ # Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +import contextlib +import functools import json import os from textwrap import dedent @@ -36,14 +38,18 @@ BENCH_START_MESSAGE = dedent( def atomic(method): + @functools.wraps(method) def wrapper(*args, **kwargs): try: ret = method(*args, **kwargs) frappe.db.commit() return ret - except Exception: - frappe.db.rollback() - raise + except Exception as e: + # database itself can be gone while attempting rollback. + # We should preserve original exception in this case. + with contextlib.suppress(Exception): + frappe.db.rollback() + raise e return wrapper diff --git a/frappe/model/base_document.py b/frappe/model/base_document.py index a31114e112..78c20a2eae 100644 --- a/frappe/model/base_document.py +++ b/frappe/model/base_document.py @@ -782,8 +782,11 @@ class BaseDocument: else: values_to_fetch = ["name"] + [_df.fetch_from.split(".")[-1] for _df in fields_to_fetch] + # fallback to dict with field_to_fetch=None if link field value is not found + # (for compatibility, `values` must have same data type) + empty_values = _dict({value: None for value in values_to_fetch}) # don't cache if fetching other values too - values = frappe.db.get_value(doctype, docname, values_to_fetch, as_dict=True) + values = frappe.db.get_value(doctype, docname, values_to_fetch, as_dict=True) or empty_values if getattr(frappe.get_meta(doctype), "issingle", 0): values.name = doctype diff --git a/frappe/model/delete_doc.py b/frappe/model/delete_doc.py index f67585540a..e2202882b1 100644 --- a/frappe/model/delete_doc.py +++ b/frappe/model/delete_doc.py @@ -245,20 +245,28 @@ def check_if_doc_is_linked(doc, method="Delete"): from frappe.model.rename_doc import get_link_fields link_fields = get_link_fields(doc.doctype) - ignore_linked_doctypes = doc.get("ignore_linked_doctypes") or [] + ignored_doctypes = set() + + if method == "Cancel" and (doc_ignore_flags := doc.get("ignore_linked_doctypes")): + ignored_doctypes.update(doc_ignore_flags) + if method == "Delete": + ignored_doctypes.update(frappe.get_hooks("ignore_links_on_delete")) for lf in link_fields: link_dt, link_field, issingle = lf["parent"], lf["fieldname"], lf["issingle"] + if link_dt in ignored_doctypes or link_field == "amended_from": + continue try: meta = frappe.get_meta(link_dt) except frappe.DoesNotExistError: + frappe.clear_last_message() # This mostly happens when app do not remove their customizations, we shouldn't # prevent link checks from failing in those cases continue if issingle: - if frappe.db.get_value(link_dt, None, link_field) == doc.name: + if frappe.db.get_single_value(link_dt, link_field) == doc.name: raise_link_exists_exception(doc, link_dt, link_dt) continue @@ -270,12 +278,9 @@ def check_if_doc_is_linked(doc, method="Delete"): for item in frappe.db.get_values(link_dt, {link_field: doc.name}, fields, as_dict=True): # available only in child table cases item_parent = getattr(item, "parent", None) - linked_doctype = item.parenttype if item_parent else link_dt + linked_parent_doctype = item.parenttype if item_parent else link_dt - if linked_doctype in frappe.get_hooks("ignore_links_on_delete") or ( - linked_doctype in ignore_linked_doctypes and method == "Cancel" - ): - # don't check for communication and todo! + if linked_parent_doctype in ignored_doctypes: continue if method != "Delete" and (method != "Cancel" or not DocStatus(item.docstatus).is_submitted()): @@ -288,7 +293,7 @@ def check_if_doc_is_linked(doc, method="Delete"): continue else: reference_docname = item_parent or item.name - raise_link_exists_exception(doc, linked_doctype, reference_docname) + raise_link_exists_exception(doc, linked_parent_doctype, reference_docname) def check_if_doc_is_dynamically_linked(doc, method="Delete"): diff --git a/frappe/model/document.py b/frappe/model/document.py index 1b7a97cc97..4d5eec64fc 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -1584,6 +1584,12 @@ class Document(BaseDocument): DocTags(self.doctype).add(self.name, tag) + def remove_tag(self, tag): + """Remove a Tag to this document""" + from frappe.desk.doctype.tag.tag import DocTags + + DocTags(self.doctype).remove(self.name, tag) + def get_tags(self): """Return a list of Tags attached to this document""" from frappe.desk.doctype.tag.tag import DocTags diff --git a/frappe/parallel_test_runner.py b/frappe/parallel_test_runner.py index 250e8bec76..ebfa5d8c32 100644 --- a/frappe/parallel_test_runner.py +++ b/frappe/parallel_test_runner.py @@ -10,7 +10,7 @@ import requests import frappe -from .test_runner import SLOW_TEST_THRESHOLD, make_test_records, set_test_email_config +from .test_runner import SLOW_TEST_THRESHOLD, make_test_records click_ctx = click.get_current_context(True) if click_ctx: @@ -38,7 +38,6 @@ class ParallelTestRunner: frappe.flags.in_test = True frappe.clear_cache() frappe.utils.scheduler.disable_scheduler() - set_test_email_config() self.before_test_setup() def before_test_setup(self): diff --git a/frappe/patches.txt b/frappe/patches.txt index 83db10d905..d3f6e30aee 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -230,3 +230,4 @@ execute:frappe.delete_doc_if_exists("Workspace", "Customization") execute:frappe.db.set_single_value("Document Naming Settings", "default_amend_naming", "Amend Counter") frappe.patches.v15_0.move_event_cancelled_to_status frappe.patches.v15_0.set_file_type +frappe.core.doctype.data_import.patches.remove_stale_docfields_from_legacy_version diff --git a/frappe/patches/v14_0/drop_data_import_legacy.py b/frappe/patches/v14_0/drop_data_import_legacy.py index 16d366a845..452d45182c 100644 --- a/frappe/patches/v14_0/drop_data_import_legacy.py +++ b/frappe/patches/v14_0/drop_data_import_legacy.py @@ -8,7 +8,7 @@ def execute(): table = frappe.utils.get_table_name(doctype) # delete the doctype record to avoid broken links - frappe.db.delete("DocType", {"name": doctype}) + frappe.delete_doc("DocType", doctype, force=True) # leaving table in database for manual cleanup click.secho( diff --git a/frappe/printing/doctype/print_format/print_format.py b/frappe/printing/doctype/print_format/print_format.py index e1ad28ef7b..eda084968a 100644 --- a/frappe/printing/doctype/print_format/print_format.py +++ b/frappe/printing/doctype/print_format/print_format.py @@ -66,7 +66,7 @@ class PrintFormat(Document): if ( self.standard == "Yes" and not frappe.local.conf.get("developer_mode") - and not (frappe.flags.in_import or frappe.flags.in_test) + and not (frappe.flags.in_migrate or frappe.flags.in_test) ): frappe.throw(frappe._("Standard Print Format cannot be updated")) diff --git a/frappe/public/css/fonts/inter/inter.css b/frappe/public/css/fonts/inter/inter.css index 88057a55b6..3be525aba0 100644 --- a/frappe/public/css/fonts/inter/inter.css +++ b/frappe/public/css/fonts/inter/inter.css @@ -1,3 +1,5 @@ +/* This file is depricated use Inter.scss instead. */ +/* Backward compatibility */ @font-face { font-family: 'Inter V'; font-weight: 100 900; diff --git a/frappe/public/css/fonts/inter/inter.scss b/frappe/public/css/fonts/inter/inter.scss new file mode 100644 index 0000000000..d3565415a3 --- /dev/null +++ b/frappe/public/css/fonts/inter/inter.scss @@ -0,0 +1,164 @@ +// TODO instead of making copy of inter.css find a way to import it. +// workaround for css import as it fails for custom website_theme_template +@font-face { + font-family: 'Inter V'; + font-weight: 100 900; + font-display: swap; + font-style: normal; + src: url('/assets/frappe/css/fonts/inter/Inter.var.woff2?v=3.19') format('woff2-variations'), + url('/assets/frappe/css/fonts/inter/Inter.var.woff2?v=3.19') format('woff2'); + src: url('/assets/frappe/css/fonts/inter/Inter.var.woff2?v=3.19') format('woff2') tech('variations'); + } + @font-face { + font-family: 'Inter V'; + font-weight: 100 900; + font-display: swap; + font-style: italic; + src: url('/assets/frappe/css/fonts/inter/Inter-Italic.var.woff2?v=3.19') format('woff2-variations'), + url('/assets/frappe/css/fonts/inter/Inter-Italic.var.woff2?v=3.19') format('woff2'); + src: url('/assets/frappe/css/fonts/inter/Inter-Italic.var.woff2?v=3.19') format('woff2') tech('variations'); + } +@font-face { + font-family: 'Inter'; + font-display: swap; + font-style: italic; + font-weight: 100; + src: url("/assets/frappe/css/fonts/inter/inter_thinitalic.woff2") format("woff2"), + url("/assets/frappe/css/fonts/inter/inter_thinitalic.woff") format("woff"); +} + +@font-face { + font-family: 'Inter'; + font-display: swap; + font-style: normal; + font-weight: 200; + src: url("/assets/frappe/css/fonts/inter/inter_extralight.woff2") format("woff2"), + url("/assets/frappe/css/fonts/inter/inter_extralight.woff") format("woff"); +} +@font-face { + font-family: 'Inter'; + font-display: swap; + font-style: italic; + font-weight: 200; + src: url("/assets/frappe/css/fonts/inter/inter_extralightitalic.woff2") format("woff2"), + url("/assets/frappe/css/fonts/inter/inter_extralightitalic.woff") format("woff"); +} + +@font-face { + font-family: 'Inter'; + font-display: swap; + font-style: normal; + font-weight: 300; + src: url("/assets/frappe/css/fonts/inter/inter_light.woff2") format("woff2"), + url("/assets/frappe/css/fonts/inter/inter_light.woff") format("woff"); +} +@font-face { + font-family: 'Inter'; + font-display: swap; + font-style: italic; + font-weight: 300; + src: url("/assets/frappe/css/fonts/inter/inter_lightitalic.woff2") format("woff2"), + url("/assets/frappe/css/fonts/inter/inter_lightitalic.woff") format("woff"); +} + +@font-face { + font-family: 'Inter'; + font-display: swap; + font-style: normal; + font-weight: 400; + src: url("/assets/frappe/css/fonts/inter/inter_regular.woff2") format("woff2"), + url("/assets/frappe/css/fonts/inter/inter_regular.woff") format("woff"); +} +@font-face { + font-family: 'Inter'; + font-display: swap; + font-style: italic; + font-weight: 400; + src: url("/assets/frappe/css/fonts/inter/inter_italic.woff2") format("woff2"), + url("/assets/frappe/css/fonts/inter/inter_italic.woff") format("woff"); +} + +@font-face { + font-family: 'Inter'; + font-display: swap; + font-style: normal; + font-weight: 500; + src: url("/assets/frappe/css/fonts/inter/inter_medium.woff2") format("woff2"), + url("/assets/frappe/css/fonts/inter/inter_medium.woff") format("woff"); +} +@font-face { + font-family: 'Inter'; + font-display: swap; + font-style: italic; + font-weight: 500; + src: url("/assets/frappe/css/fonts/inter/inter_mediumitalic.woff2") format("woff2"), + url("/assets/frappe/css/fonts/inter/inter_mediumitalic.woff") format("woff"); +} + +@font-face { + font-family: 'Inter'; + font-display: swap; + font-style: normal; + font-weight: 600; + src: url("/assets/frappe/css/fonts/inter/inter_semibold.woff2") format("woff2"), + url("/assets/frappe/css/fonts/inter/inter_semibold.woff") format("woff"); +} +@font-face { + font-family: 'Inter'; + font-display: swap; + font-style: italic; + font-weight: 600; + src: url("/assets/frappe/css/fonts/inter/inter_semibolditalic.woff2") format("woff2"), + url("/assets/frappe/css/fonts/inter/inter_semibolditalic.woff") format("woff"); +} + +@font-face { + font-family: 'Inter'; + font-display: swap; + font-style: normal; + font-weight: 700; + src: url("/assets/frappe/css/fonts/inter/inter_bold.woff2") format("woff2"), + url("/assets/frappe/css/fonts/inter/inter_bold.woff") format("woff"); +} +@font-face { + font-family: 'Inter'; + font-display: swap; + font-style: italic; + font-weight: 700; + src: url("/assets/frappe/css/fonts/inter/inter_bolditalic.woff2") format("woff2"), + url("/assets/frappe/css/fonts/inter/inter_bolditalic.woff") format("woff"); +} + +@font-face { + font-family: 'Inter'; + font-display: swap; + font-style: normal; + font-weight: 800; + src: url("/assets/frappe/css/fonts/inter/inter_extrabold.woff2") format("woff2"), + url("/assets/frappe/css/fonts/inter/inter_extrabold.woff") format("woff"); +} +@font-face { + font-family: 'Inter'; + font-display: swap; + font-style: italic; + font-weight: 800; + src: url("/assets/frappe/css/fonts/inter/inter_extrabolditalic.woff2") format("woff2"), + url("/assets/frappe/css/fonts/inter/inter_extrabolditalic.woff") format("woff"); +} + +@font-face { + font-family: 'Inter'; + font-display: swap; + font-style: normal; + font-weight: 900; + src: url("/assets/frappe/css/fonts/inter/inter_black.woff2") format("woff2"), + url("/assets/frappe/css/fonts/inter/inter_black.woff") format("woff"); +} +@font-face { + font-family: 'Inter'; + font-display: swap; + font-style: italic; + font-weight: 900; + src: url("/assets/frappe/css/fonts/inter/inter_blackitalic.woff2") format("woff2"), + url("/assets/frappe/css/fonts/inter/inter_blackitalic.woff") format("woff"); +} diff --git a/frappe/public/js/form_builder/FormBuilder.vue b/frappe/public/js/form_builder/FormBuilder.vue index 427b93584b..63ed4ba25d 100644 --- a/frappe/public/js/form_builder/FormBuilder.vue +++ b/frappe/public/js/form_builder/FormBuilder.vue @@ -12,7 +12,9 @@ let should_render = computed(() => { }); let container = ref(null); -onClickOutside(container, () => (store.form.selected_field = null)); +onClickOutside(container, () => (store.form.selected_field = null), { + ignore: [".combo-box-options"], +}); watch( () => store.form.layout, @@ -30,22 +32,23 @@ onMounted(() => store.fetch()); class="form-builder-container" @click="store.form.selected_field = null" > -
-
- -
-
+
+
+ +
+
+
diff --git a/frappe/public/js/form_builder/components/AddFieldButton.vue b/frappe/public/js/form_builder/components/AddFieldButton.vue new file mode 100644 index 0000000000..92a141e4e6 --- /dev/null +++ b/frappe/public/js/form_builder/components/AddFieldButton.vue @@ -0,0 +1,139 @@ + + + + + diff --git a/frappe/public/js/form_builder/components/Autocomplete.vue b/frappe/public/js/form_builder/components/Autocomplete.vue new file mode 100644 index 0000000000..2ded948e27 --- /dev/null +++ b/frappe/public/js/form_builder/components/Autocomplete.vue @@ -0,0 +1,159 @@ + + + + + diff --git a/frappe/public/js/form_builder/components/Column.vue b/frappe/public/js/form_builder/components/Column.vue index 54dd49d5bf..f6a3cfbe51 100644 --- a/frappe/public/js/form_builder/components/Column.vue +++ b/frappe/public/js/form_builder/components/Column.vue @@ -1,15 +1,26 @@