Merge remote-tracking branch 'upstream/develop' into fix/non-html-notifications-rendered-from-files

.coveragerc

@@ -0,0 +1,23 @@
+[run]
+omit =
+    tests/*
+    .github/*
+    commands/*
+    **/test_*.py
+
+[report]
+exclude_lines =
+    pragma: no cover
+    if TYPE_CHECKING:
+
+exclude_also =
+    def __repr__
+    if self.debug:
+    if settings.DEBUG
+    raise AssertionError
+    raise NotImplementedError
+    if 0:
+    if __name__ == .__main__.:
+    if TYPE_CHECKING:
+    class .*\bProtocol\):
+    @(abc\.)?abstractmethod

.github/helper/db/mariadb.json

@@ -6,7 +6,8 @@
     "allow_tests": true,
     "db_type": "mariadb",
     "auto_email_id": "test@example.com",
-    "mail_server": "smtp.example.com",
+    "mail_server": "localhost",
+    "mail_port": 2525,
     "mail_login": "test@example.com",
     "mail_password": "test",
     "admin_password": "admin",

.github/helper/db/postgres.json

@@ -6,7 +6,8 @@
     "db_type": "postgres",
     "allow_tests": true,
     "auto_email_id": "test@example.com",
-    "mail_server": "smtp.example.com",
+    "mail_server": "localhost",
+    "mail_port": 2525,
     "mail_login": "test@example.com",
     "mail_password": "test",
     "admin_password": "admin",

.github/workflows/create-release.yml

@@ -17,7 +17,7 @@ jobs:
           fetch-depth: 0
           persist-credentials: false
       - name: Setup Node.js
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: 18
       - name: Setup dependencies

.github/workflows/initiate_release.yml

@@ -15,7 +15,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        version: ["13", "14"]
+        version: ["13", "14", "15"]
 
     steps:
       - uses: octokit/request-action@v2.x
@@ -30,23 +30,3 @@ jobs:
           head: version-${{ matrix.version }}-hotfix
         env:
           GITHUB_TOKEN: ${{ secrets.RELEASE_TOKEN }}
-
-  beta-release:
-    name: Release
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-
-    steps:
-      - uses: octokit/request-action@v2.x
-        with:
-          route: POST /repos/{owner}/{repo}/pulls
-          owner: frappe
-          repo: frappe
-          title: |-
-            "chore: release v15 beta"
-          body: "Automated beta release."
-          base: version-15-beta
-          head: develop
-        env:
-          GITHUB_TOKEN: ${{ secrets.RELEASE_TOKEN }}

.github/workflows/linters.yml

@@ -21,7 +21,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 200
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
         with:
           node-version: 18
           check-latest: true

.github/workflows/lock.yml

@@ -18,4 +18,4 @@ jobs:
         with:
           github-token: ${{ github.token }}
           issue-inactive-days: 14
-          pr-inactive-days: 14
+          pr-inactive-days: 14

.github/workflows/on_release.yml

@@ -20,7 +20,7 @@ jobs:
         with:
           path: 'frappe'
 
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
         with:
           node-version: 18
 

.github/workflows/patch-mariadb-tests.yml

@@ -67,7 +67,7 @@ jobs:
           python-version: "3.10"
 
       - name: Setup Node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: 18
           check-latest: true
@@ -134,6 +134,7 @@ jobs:
           }
 
           update_to_version 14
+          update_to_version 15
 
           echo "Updating to last commit"
           pgrep honcho | xargs kill

.github/workflows/publish-assets-develop.yml

@@ -14,7 +14,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           path: 'frappe'
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
         with:
           node-version: 18
       - uses: actions/setup-python@v4

.github/workflows/server-tests.yml

@@ -72,6 +72,12 @@ jobs:
         ports:
           - 5432:5432
 
+      smtp_server:
+        image: rnwood/smtp4dev
+        ports:
+          - 2525:25
+          - 3000:80
+
     steps:
       - name: Clone
         uses: actions/checkout@v4
@@ -89,7 +95,7 @@ jobs:
               exit 1
           fi
 
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
         with:
           node-version: 18
           check-latest: true
@@ -130,12 +136,14 @@ jobs:
           DB: ${{ matrix.db }}
 
       - name: Run Tests
-        run: cd ~/frappe-bench/sites && ../env/bin/python3 ../apps/frappe/.github/helper/ci.py
+        run: ../env/bin/python3 ../apps/frappe/.github/helper/ci.py
+        working-directory: /home/runner/frappe-bench/sites
         env:
           SITE: test_site
           CI_BUILD_ID: ${{ github.run_id }}
           BUILD_NUMBER: ${{ matrix.container }}
           TOTAL_BUILDS: 2
+          COVERAGE_RCFILE: /home/runner/frappe-bench/apps/frappe/.coveragerc
 
       - name: Show bench output
         if: ${{ always() }}

.github/workflows/ui-tests.yml

@@ -77,7 +77,7 @@ jobs:
               exit 1
           fi
 
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
         with:
           node-version: 18
           check-latest: true

.mergify.yml

@@ -9,6 +9,7 @@ pull_request_rules:
           - author!=frappe-pr-bot
           - author!=mergify[bot]
         - or:
+          - base=version-16
           - base=version-15
           - base=version-14
           - base=version-13
@@ -71,3 +72,13 @@ pull_request_rules:
         assignees:
           - "{{ author }}"
 
+  - name: backport to version-15-hotfix
+    conditions:
+      - label="backport version-15-hotfix"
+    actions:
+      backport:
+        branches:
+          - version-15-hotfix
+        assignees:
+          - "{{ author }}"
+

.pre-commit-config.yaml

@@ -35,7 +35,7 @@ repos:
     rev: v2.7.1
     hooks:
       - id: prettier
-        types_or: [javascript]
+        types_or: [javascript, vue, scss]
         # Ignore any files that might contain jinja / bundles
         exclude: |
             (?x)^(
@@ -44,7 +44,8 @@ repos:
                 .*boilerplate.*|
                 frappe/www/website_script.js|
                 frappe/templates/includes/.*|
-                frappe/public/js/lib/.*
+                frappe/public/js/lib/.*|
+                frappe/website/doctype/website_theme/website_theme_template.scss
             )$
 
 

cypress/integration/control_attach.js

@@ -65,6 +65,25 @@ context("Attach Control", () => {
 		//Clicking on the attach button which is displayed as part of creating a doctype with "Attach" fieldtype
 		cy.findByRole("button", { name: "Attach" }).click();
 
+		//Clicking on "Link" button to attach a file using the "Link" button
+		cy.findByRole("button", { name: "Link" }).click();
+		cy.findByPlaceholderText("Attach a web link").type(
+			"https://wallpaperplay.com/walls/full/8/2/b/72402.jpg",
+			{ force: true }
+		);
+
+		//Clicking on the Upload button to upload the file
+		cy.intercept("POST", "/api/method/upload_file").as("upload_image");
+		cy.get(".modal-footer").findByRole("button", { name: "Upload" }).click({ delay: 500 });
+		cy.wait("@upload_image");
+		cy.findByRole("button", { name: "Save" }).click();
+
+		//Navigating to the new form for the newly created doctype to check Library button
+		cy.new_form("Test Attach Control");
+
+		//Clicking on the attach button which is displayed as part of creating a doctype with "Attach" fieldtype
+		cy.findByRole("button", { name: "Attach" }).click();
+
 		//Clicking on "Library" button to attach a file using the "Library" button
 		cy.findByRole("button", { name: "Library" }).click();
 		cy.contains("72402.jpg").click();
@@ -86,9 +105,10 @@ context("Attach Control", () => {
 		//Checking if clicking on the clear button clears the field of the doctype form and again displays the attach button
 		cy.get(".control-input > .btn-sm").should("contain", "Attach");
 
-		//Deleting the doc
+		//Deleting both docs
 		cy.go_to_list("Test Attach Control");
 		cy.get(".list-row-checkbox").eq(0).click();
+		cy.get(".list-row-checkbox").eq(1).click();
 		cy.get(".actions-btn-group > .btn").contains("Actions").click();
 		cy.get('.actions-btn-group > .dropdown-menu [data-label="Delete"]').click();
 		cy.click_modal_primary_button("Yes");
@@ -107,7 +127,10 @@ context("Attach Control", () => {
 				};
 			},
 		});
-		cy.get("body").should("have.attr", "data-route", `Form/${doctype}/new-${dt_in_route}-1`);
+		cy.get("body").should(($body) => {
+			const dataRoute = $body.attr("data-route");
+			expect(dataRoute).to.match(new RegExp(`^Form/${doctype}/new-${dt_in_route}-`));
+		});
 		cy.get("body").should("have.attr", "data-ajax-state", "complete");
 
 		//Clicking on the attach button which is displayed as part of creating a doctype with "Attach" fieldtype
@@ -127,7 +150,10 @@ context("Attach Control", () => {
 				delete win.navigator.mediaDevices;
 			},
 		});
-		cy.get("body").should("have.attr", "data-route", `Form/${doctype}/new-${dt_in_route}-1`);
+		cy.get("body").should(($body) => {
+			const dataRoute = $body.attr("data-route");
+			expect(dataRoute).to.match(new RegExp(`^Form/${doctype}/new-${dt_in_route}-`));
+		});
 		cy.get("body").should("have.attr", "data-ajax-state", "complete");
 
 		//Clicking on the attach button which is displayed as part of creating a doctype with "Attach" fieldtype
@@ -137,3 +163,89 @@ context("Attach Control", () => {
 		cy.findByRole("button", { name: "Camera" }).should("not.exist");
 	});
 });
+context("Attach Control with Failed Document Save", () => {
+	before(() => {
+		cy.login();
+		cy.visit("/app/doctype");
+		return cy
+			.window()
+			.its("frappe")
+			.then((frappe) => {
+				return frappe.xcall("frappe.tests.ui_test_helpers.create_doctype", {
+					name: "Test Mandatory Attach Control",
+					fields: [
+						{
+							label: "Attach File or Image",
+							fieldname: "attach",
+							fieldtype: "Attach",
+							in_list_view: 1,
+						},
+						{
+							label: "Mandatory Text Field",
+							fieldname: "text_field",
+							fieldtype: "Text Editor",
+							in_list_view: 1,
+							reqd: 1,
+						},
+					],
+				});
+			});
+	});
+	let temp_name = "";
+	let docname = "";
+	it("Attaching a file on an unsaved document", () => {
+		//Navigating to the new form for the newly created doctype
+		cy.new_form("Test Mandatory Attach Control");
+		cy.get("body").should(($body) => {
+			temp_name = $body.attr("data-route").split("/")[2];
+		});
+
+		//Clicking on the attach button which is displayed as part of creating a doctype with "Attach" fieldtype
+		cy.findByRole("button", { name: "Attach" }).click();
+
+		//Clicking on "Link" button to attach a file using the "Link" button
+		cy.findByRole("button", { name: "Link" }).click();
+		cy.findByPlaceholderText("Attach a web link").type(
+			"https://wallpaperplay.com/walls/full/8/2/b/72402.jpg",
+			{ force: true }
+		);
+
+		//Clicking on the Upload button to upload the file
+		cy.intercept("POST", "/api/method/upload_file").as("upload_image");
+		cy.get(".modal-footer").findByRole("button", { name: "Upload" }).click({ delay: 500 });
+		cy.wait("@upload_image");
+		cy.get(".msgprint-dialog .modal-title").contains("Missing Fields").should("be.visible");
+		cy.hide_dialog();
+		cy.fill_field("text_field", "Random value", "Text Editor").wait(500);
+		cy.findByRole("button", { name: "Save" }).click().wait(500);
+
+		//Checking if the URL of the attached image is getting displayed in the field of the newly created doctype
+		cy.get(".attached-file > .ellipsis > .attached-file-link")
+			.should("have.attr", "href")
+			.and("equal", "https://wallpaperplay.com/walls/full/8/2/b/72402.jpg");
+
+		cy.get(".title-text").then(($value) => {
+			docname = $value.text();
+		});
+	});
+
+	it("Check if file was uploaded correctly", () => {
+		cy.go_to_list("File");
+		cy.open_list_filter();
+		cy.get(".fieldname-select-area .form-control")
+			.click()
+			.type("Attached To Name{enter}")
+			.blur()
+			.wait(500);
+		cy.get('input[data-fieldname="attached_to_name"]').click().type(docname).blur();
+		cy.get(".filter-popover .apply-filters").click({ force: true });
+		cy.get("header .level-right .list-count").should("contain.text", "1 of 1");
+	});
+
+	it("Check if file exists with temporary name", () => {
+		cy.open_list_filter();
+		cy.get('input[data-fieldname="attached_to_name"]').click().clear().type(temp_name).blur();
+		cy.get(".filter-popover .apply-filters").click({ force: true });
+		cy.get(".frappe-list > .no-result").should("be.visible");
+	});
+});

cypress/integration/control_data.js

@@ -49,7 +49,7 @@ context("Data Control", () => {
 		cy.new_form("Test Data Control");
 
 		//Checking the URL for the new form of the doctype
-		cy.location("pathname").should("eq", "/app/test-data-control/new-test-data-control-1");
+		cy.location("pathname").should("contains", "/app/test-data-control/new-test-data-control");
 		cy.get(".title-text").should("have.text", "New Test Data Control");
 		cy.get('.frappe-control[data-fieldname="name1"]')
 			.find("label")
@@ -128,7 +128,10 @@ context("Data Control", () => {
 		cy.fill_field("phone", "9432380001", "Data");
 		cy.findByRole("button", { name: "Save" }).click({ force: true });
 		//Checking if the fields contains the data which has been filled in
-		cy.location("pathname").should("not.be", "/app/test-data-control/new-test-data-control-1");
+		cy.location("pathname").should(
+			"not.contains",
+			"/app/test-data-control/new-test-data-control"
+		);
 		cy.get_field("name1").should("have.value", "Komal");
 		cy.get_field("email").should("have.value", "komal@test.com");
 		cy.get_field("phone").should("have.value", "9432380001");

cypress/integration/dashboard_chart.js

@@ -5,7 +5,7 @@ context("Dashboard Chart", () => {
 	});
 
 	it("Check filter populate for child table doctype", () => {
-		cy.visit("/app/dashboard-chart/new-dashboard-chart-1");
+		cy.new_form("Dashboard Chart");
 		cy.get('[data-fieldname="parent_document_type"]').should("have.css", "display", "none");
 
 		cy.get_field("document_type", "Link");

cypress/integration/form.js

@@ -35,7 +35,7 @@ context("Form", () => {
 		cy.visit("/app/todo/new");
 		cy.get_field("description", "Text Editor")
 			.type("this is a test todo", { force: true })
-			.wait(200);
+			.wait(1000);
 		cy.get(".page-title").should("contain", "Not Saved");
 		cy.intercept({
 			method: "POST",

cypress/integration/form_builder.js

@@ -43,7 +43,8 @@ context("Form Builder", () => {
 
 		// add new section
 		cy.get(first_section).click(15, 10);
-		cy.get(first_section).find(".section-actions button:first").click();
+		cy.get(first_section).find(".dropdown-btn:first").click();
+		cy.get(".dropdown-options:visible .dropdown-item:first").click();
 
 		// save
 		cy.click_doc_primary_button("Save");
@@ -56,12 +57,17 @@ context("Form Builder", () => {
 		cy.visit(`/app/doctype/${doctype_name}`);
 		cy.findByRole("tab", { name: "Form" }).click();
 
-		let first_field =
-			".tab-content.active .section-columns-container:first .column:first .field:first";
+		let first_column = ".tab-content.active .section-columns-container:first .column:first";
 
-		cy.get(".fields-container .field[title='Table']").drag(first_field, {
-			target: { x: 100, y: 10 },
-		});
+		let last_field = first_column + " .field:last";
+
+		let add_new_field_btn = first_column + " .add-new-field-btn button";
+
+		// add new field
+		cy.get(add_new_field_btn).click();
+
+		// type table and press enter
+		cy.get(".combo-box-options:visible .search-box > input").type("table{enter}");
 
 		// save
 		cy.click_doc_primary_button("Save");
@@ -70,20 +76,18 @@ context("Form Builder", () => {
 		cy.get_open_dialog().find(".msgprint").should("contain", "Options is required");
 		cy.hide_dialog();
 
-		cy.get(first_field).click({ force: true });
+		cy.get(last_field).click({ force: true });
 
 		cy.get(".sidebar-container .frappe-control[data-fieldname='options'] input")
 			.click()
 			.as("input");
 		cy.get("@input").clear({ force: true }).type("Web Form Field", { delay: 200 });
 		cy.wait("@search_link");
-		cy.get("@input").type("{enter}").blur();
 
-		cy.get(first_field)
-			.find(".table-controls .table-column")
-			.contains("Field")
-			.should("exist");
-		cy.get(first_field)
+		cy.get(last_field).click({ force: true });
+
+		cy.get(last_field).find(".table-controls .table-column").contains("Field").should("exist");
+		cy.get(last_field)
 			.find(".table-controls .table-column")
 			.contains("Fieldtype")
 			.should("exist");
@@ -97,7 +101,7 @@ context("Form Builder", () => {
 		cy.get_open_dialog().find(".msgprint").should("contain", "In List View");
 		cy.hide_dialog();
 
-		cy.get(first_field).click({ force: true });
+		cy.get(last_field).click({ force: true });
 		cy.get(".sidebar-container .field label .label-area").contains("In List View").click();
 
 		// validate In Global Search
@@ -181,30 +185,34 @@ context("Form Builder", () => {
 
 		// add new section
 		cy.get(first_section).click(15, 10);
-		cy.get(first_section).find(".section-actions button:first").click();
+		cy.get(first_section).find(".dropdown-btn:first").click();
+		cy.get(".dropdown-options:visible .dropdown-item:first").click();
 		cy.get(".tab-content.active .form-section-container").should("have.length", 2);
 
 		// add new column
-		cy.get(first_section).find(".column:first").click(15, 10);
-		cy.get(first_section).find(".column:first .column-actions button:first").click();
-		cy.get(first_section).find(".column").should("have.length", 3);
+		cy.get(first_section).click(15, 10);
+		cy.get(first_section).find(".dropdown-btn:first").click();
+		cy.get(".dropdown-options:visible .dropdown-item:last").click();
+		cy.get(first_section).find(".column").should("have.length", 2);
 	});
 
 	it("Remove Tab/Section/Column", () => {
 		let first_section = ".tab-content.active .form-section-container:first";
 
 		// remove column
-		cy.get(first_section).find(".column:first").click(15, 10);
-		cy.get(first_section).find(".column:first .column-actions button:last").click();
-		cy.get(first_section).find(".column").should("have.length", 2);
+		cy.get(first_section).click(15, 10);
+		cy.get(first_section).find(".dropdown-btn:first").click();
+		cy.get(".dropdown-options:visible .dropdown-item:last").click();
+		cy.get(first_section).find(".column").should("have.length", 1);
 
 		// remove section
 		cy.get(first_section).click(15, 10);
-		cy.get(first_section).find(".section-actions button:last").click();
+		cy.get(first_section).find(".dropdown-btn:first").click();
+		cy.get(".dropdown-options:visible .dropdown-item").eq(1).click();
 		cy.get(".tab-content.active .form-section-container").should("have.length", 1);
 
 		// remove tab
-		cy.get(".tab-header").realHover().find(".tab-actions .remove-tab-btn").click();
+		cy.get(".tab-header .tab:last").realHover().find(".remove-tab-btn").click();
 		cy.get(".tab-header .tabs .tab").should("have.length", 2);
 	});
 
@@ -230,14 +238,19 @@ context("Form Builder", () => {
 		cy.visit(`/app/doctype/${doctype_name}`);
 		cy.findByRole("tab", { name: "Form" }).click();
 
-		let first_field =
-			".tab-content.active .section-columns-container:first .column:first .field:first";
+		let first_column = ".tab-content.active .section-columns-container:first .column:first";
 
-		cy.get(".fields-container .field[title='Data']").drag(first_field, {
-			target: { x: 100, y: 10 },
-		});
+		let last_field = first_column + " .field:last";
 
-		cy.get(first_field).click();
+		let add_new_field_btn = first_column + " .add-new-field-btn button";
+
+		// add new field
+		cy.get(add_new_field_btn).click();
+
+		// type data and press enter
+		cy.get(".combo-box-options:visible .search-box > input").type("data{enter}");
+
+		cy.get(last_field).click();
 
 		// validate duplicate name
 		cy.get(".sidebar-container .frappe-control[data-fieldname='fieldname'] input")
@@ -250,7 +263,7 @@ context("Form Builder", () => {
 		cy.click_doc_primary_button("Save");
 		cy.get_open_dialog().find(".msgprint").should("contain", "appears multiple times");
 		cy.hide_dialog();
-		cy.get(first_field).click();
+		cy.get(last_field).click();
 		cy.get(".sidebar-container .frappe-control[data-fieldname='fieldname'] input").clear({
 			force: true,
 		});

cypress/integration/grid_keyboard_shortcut.js

@@ -1,18 +1,25 @@
 context("Grid Keyboard Shortcut", () => {
 	let total_count = 0;
+	let contact_email_name = null;
 	before(() => {
 		cy.login();
 	});
 	beforeEach(() => {
 		cy.reload();
-		cy.visit("/app/contact/new-contact-1");
+		cy.new_form("Contact");
 		cy.get('.frappe-control[data-fieldname="email_ids"]').find(".grid-add-row").click();
+		// as new names uses hash instead of numbers get row's data-name dynamically.
+		cy.get('.frappe-control[data-fieldname="email_ids"]')
+			.find(".grid-body .grid-row")
+			.should(($row) => {
+				contact_email_name = $row.attr("data-name");
+			});
 	});
 	it("Insert new row at the end", () => {
 		cy.add_new_row_in_grid(
 			"{ctrl}{shift}{downarrow}",
 			(cy, total_count) => {
-				cy.get('[data-name="new-contact-email-1"]').should(
+				cy.get(`[data-name="${contact_email_name}"]`).should(
 					"have.attr",
 					"data-idx",
 					`${total_count + 1}`
@@ -23,17 +30,17 @@ context("Grid Keyboard Shortcut", () => {
 	});
 	it("Insert new row at the top", () => {
 		cy.add_new_row_in_grid("{ctrl}{shift}{uparrow}", (cy) => {
-			cy.get('[data-name="new-contact-email-1"]').should("have.attr", "data-idx", "2");
+			cy.get(`[data-name="${contact_email_name}"]`).should("have.attr", "data-idx", "2");
 		});
 	});
 	it("Insert new row below", () => {
 		cy.add_new_row_in_grid("{ctrl}{downarrow}", (cy) => {
-			cy.get('[data-name="new-contact-email-1"]').should("have.attr", "data-idx", "1");
+			cy.get(`[data-name^="${contact_email_name}"]`).should("have.attr", "data-idx", "1");
 		});
 	});
 	it("Insert new row above", () => {
 		cy.add_new_row_in_grid("{ctrl}{uparrow}", (cy) => {
-			cy.get('[data-name="new-contact-email-1"]').should("have.attr", "data-idx", "2");
+			cy.get(`[data-name^="${contact_email_name}"]`).should("have.attr", "data-idx", "2");
 		});
 	});
 });

cypress/integration/number_card.js

@@ -5,7 +5,7 @@ context("Number Card", () => {
 	});
 
 	it("Check filter populate for child table doctype", () => {
-		cy.visit("/app/number-card/new-number-card-1");
+		cy.new_form("Number Card");
 		cy.get('[data-fieldname="parent_document_type"]').should("have.css", "display", "none");
 
 		cy.get_field("document_type", "Link");

cypress/integration/sidebar.js

@@ -2,7 +2,7 @@ const verify_attachment_visibility = (document, is_private) => {
 	cy.visit(`/app/${document}`);
 
 	const assertion = is_private ? "be.checked" : "not.be.checked";
-	cy.findByRole("button", { name: "Add File" }).click();
+	cy.get(".add-attachment-btn").click();
 
 	cy.get_open_dialog()
 		.find(".file-upload-area")
@@ -27,7 +27,7 @@ const attach_file = (file, no_of_files = 1) => {
 		);
 	}
 
-	cy.findByRole("button", { name: "Add File" }).click();
+	cy.get(".add-attachment-btn").click();
 	cy.get_open_dialog().find(".file-upload-area").selectFile(files, {
 		action: "drag-drop",
 	});
@@ -62,11 +62,8 @@ context("Sidebar", () => {
 		}).then((todo) => {
 			cy.visit(`/app/todo/${todo.message.name}`);
 
-			// explore icon btn should be hidden as there are no attachments
-			cy.get(".explore-btn").should("be.hidden");
-
 			attach_file("cypress/fixtures/sample_image.jpg");
-			cy.get(".explore-btn").should("be.visible");
+			cy.get(".explore-link").should("be.visible");
 			cy.get(".show-all-btn").should("be.hidden");
 
 			// attach 10 images
@@ -75,9 +72,8 @@ context("Sidebar", () => {
 
 			// attach 1 more image to reach attachment limit
 			attach_file("cypress/fixtures/sample_attachments/attachment-11.txt");
-			cy.get(".explore-full-btn").should("be.visible");
-			cy.get(".attachments-actions").should("be.hidden");
-			cy.get(".explore-btn").should("be.hidden");
+			cy.get(".add-attachment-btn").should("be.hidden");
+			cy.get(".explore-link").should("be.visible");
 
 			// test "Show All" button
 			cy.get(".attachment-row").should("have.length", 10);

cypress/integration/timeline.js

@@ -8,7 +8,7 @@ context("Timeline", () => {
 
 	it("Adding new ToDo, adding new comment, verifying comment addition & deletion and deleting ToDo", () => {
 		//Adding new ToDo
-		cy.visit("/app/todo/new-todo-1");
+		cy.new_form("ToDo");
 		cy.get('[data-fieldname="description"] .ql-editor.ql-blank')
 			.type("Test ToDo", { force: true })
 			.wait(200);

cypress/support/commands.js

@@ -254,7 +254,10 @@ Cypress.Commands.add("awesomebar", (text) => {
 Cypress.Commands.add("new_form", (doctype) => {
 	let dt_in_route = doctype.toLowerCase().replace(/ /g, "-");
 	cy.visit(`/app/${dt_in_route}/new`);
-	cy.get("body").should("have.attr", "data-route", `Form/${doctype}/new-${dt_in_route}-1`);
+	cy.get("body").should(($body) => {
+		const dataRoute = $body.attr("data-route");
+		expect(dataRoute).to.match(new RegExp(`^Form/${doctype}/new-${dt_in_route}-`));
+	});
 	cy.get("body").should("have.attr", "data-ajax-state", "complete");
 });
 

esbuild/sass_options.js

@@ -1,11 +1,11 @@
 let path = require("path");
 let { get_app_path, app_list } = require("./utils");
 
-let node_modules_path = path.resolve(get_app_path("frappe"), "..", "node_modules");
 let app_paths = app_list.map(get_app_path).map((app_path) => path.resolve(app_path, ".."));
+let node_modules_path = app_paths.map((app_path) => path.resolve(app_path, "node_modules"));
 
 module.exports = {
-	includePaths: [node_modules_path, ...app_paths],
+	includePaths: [...node_modules_path, ...app_paths],
 	quietDeps: true,
 	importer: function (url) {
 		if (url.startsWith("~")) {

frappe/__init__.py

@@ -132,16 +132,6 @@ def as_unicode(text: str, encoding: str = "utf-8") -> str:
 		return str(text)
 
 
-def get_lang_dict(fortype: str, name: str | None = None) -> dict[str, str]:
-	"""Returns the translated language dict for the given type and name.
-
-	:param fortype: must be one of `doctype`, `page`, `report`, `include`, `jsfile`, `boot`
-	:param name: name of the document for which assets are to be returned."""
-	from frappe.translate import get_dict
-
-	return get_dict(fortype, name)
-
-
 def set_user_lang(user: str, user_language: str | None = None) -> None:
 	"""Guess and set user language for the session. `frappe.local.lang`"""
 	from frappe.translate import get_user_lang
@@ -156,6 +146,7 @@ qb = local("qb")
 conf = local("conf")
 form = form_dict = local("form_dict")
 request = local("request")
+job = local("job")
 response = local("response")
 session = local("session")
 user = local("user")
@@ -169,7 +160,7 @@ lang = local("lang")
 
 # This if block is never executed when running the code. It is only used for
 # telling static code analyzer where to find dynamically defined attributes.
-if TYPE_CHECKING:
+if TYPE_CHECKING:  # pragma: no cover
 	from werkzeug.wrappers import Request
 
 	from frappe.database.mariadb.database import MariaDBDatabase
@@ -488,9 +479,12 @@ def msgprint(
 	def _raise_exception():
 		if raise_exception:
 			if inspect.isclass(raise_exception) and issubclass(raise_exception, Exception):
-				raise raise_exception(msg)
+				exc = raise_exception(msg)
 			else:
-				raise ValidationError(msg)
+				exc = ValidationError(msg)
+			if out.__frappe_exc_id:
+				exc.__frappe_exc_id = out.__frappe_exc_id
+			raise exc
 
 	if flags.mute_messages:
 		_raise_exception()
@@ -527,6 +521,7 @@ def msgprint(
 
 	if raise_exception:
 		out.raise_exception = 1
+		out.__frappe_exc_id = generate_hash()
 
 	if primary_action:
 		out.primary_action = primary_action
@@ -534,11 +529,7 @@ def msgprint(
 	if wide:
 		out.wide = wide
 
-	message_log.append(json.dumps(out))
-
-	if raise_exception and hasattr(raise_exception, "__name__"):
-		local.response["exc_type"] = raise_exception.__name__
-
+	message_log.append(out)
 	_raise_exception()
 
 
@@ -546,8 +537,8 @@ def clear_messages():
 	local.message_log = []
 
 
-def get_message_log():
-	return [json.loads(msg_out) for msg_out in local.message_log]
+def get_message_log() -> list[dict]:
+	return [msg_out for msg_out in local.message_log]
 
 
 def clear_last_message():
@@ -1225,7 +1216,7 @@ def get_doc(doctype: str, /) -> _SingleDocument:
 
 
 @overload
-def get_doc(doctype: str, name: str, /, for_update: bool | None = None) -> "Document":
+def get_doc(doctype: str, name: str, /, *, for_update: bool | None = None) -> "Document":
 	"""Retrieve DocType from DB, doctype and name must be positional argument."""
 	pass
 
@@ -1449,9 +1440,9 @@ def get_site_path(*joins):
 	"""Return path of current site.
 
 	:param *joins: Join additional path elements using `os.path.join`."""
-	from os.path import join, normpath
+	from os.path import join
 
-	return normpath(join(local.site_path, *joins))
+	return join(local.site_path, *joins)
 
 
 def get_pymodule_path(modulename, *joins):
@@ -1713,6 +1704,7 @@ def get_newargs(fn: Callable, kwargs: dict[str, Any]) -> dict[str, Any]:
 		if (a in fnargs) or varkw_exist:
 			newargs[a] = kwargs.get(a)
 
+	# WARNING: This behaviour is now  part of business logic in places, never remove.
 	newargs.pop("ignore_permissions", None)
 	newargs.pop("flags", None)
 

frappe/api.py

@@ -1,306 +0,0 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# License: MIT. See LICENSE
-import base64
-import binascii
-import json
-from typing import Literal
-from urllib.parse import urlencode, urlparse
-
-import frappe
-import frappe.client
-import frappe.handler
-from frappe import _
-from frappe.utils.data import sbool
-from frappe.utils.response import build_response
-
-
-def handle():
-	"""
-	Handler for `/api` methods
-
-	### Examples:
-
-	`/api/method/{methodname}` will call a whitelisted method
-
-	`/api/resource/{doctype}` will query a table
-	        examples:
-	        - `?fields=["name", "owner"]`
-	        - `?filters=[["Task", "name", "like", "%005"]]`
-	        - `?limit_start=0`
-	        - `?limit_page_length=20`
-
-	`/api/resource/{doctype}/{name}` will point to a resource
-	        `GET` will return doclist
-	        `POST` will insert
-	        `PUT` will update
-	        `DELETE` will delete
-
-	`/api/resource/{doctype}/{name}?run_method={method}` will run a whitelisted controller method
-	"""
-
-	parts = frappe.request.path[1:].split("/", 3)
-	call = doctype = name = None
-
-	if len(parts) > 1:
-		call = parts[1]
-
-	if len(parts) > 2:
-		doctype = parts[2]
-
-	if len(parts) > 3:
-		name = parts[3]
-
-	return _RESTAPIHandler(call, doctype, name).get_response()
-
-
-class _RESTAPIHandler:
-	def __init__(self, call: Literal["method", "resource"], doctype: str | None, name: str | None):
-		self.call = call
-		self.doctype = doctype
-		self.name = name
-
-	def get_response(self):
-		"""Prepare and get response based on URL and form body.
-
-		Note: most methods of this class directly operate on the response local.
-		"""
-		match self.call:
-			case "method":
-				return self.handle_method()
-			case "resource":
-				self.handle_resource()
-			case _:
-				raise frappe.DoesNotExistError
-
-		return build_response("json")
-
-	def handle_method(self):
-		frappe.local.form_dict.cmd = self.doctype
-		return frappe.handler.handle()
-
-	def handle_resource(self):
-		if self.doctype and self.name:
-			self.handle_document_resource()
-		elif self.doctype:
-			self.handle_doctype_resource()
-		else:
-			raise frappe.DoesNotExistError
-
-	def handle_document_resource(self):
-		if "run_method" in frappe.local.form_dict:
-			self.execute_doc_method()
-			return
-
-		match frappe.local.request.method:
-			case "GET":
-				self.get_doc()
-			case "PUT":
-				self.update_doc()
-			case "DELETE":
-				self.delete_doc()
-			case _:
-				raise frappe.DoesNotExistError
-
-	def handle_doctype_resource(self):
-		match frappe.local.request.method:
-			case "GET":
-				self.get_doc_list()
-			case "POST":
-				self.create_doc()
-			case _:
-				raise frappe.DoesNotExistError
-
-	def execute_doc_method(self):
-		method = frappe.local.form_dict.pop("run_method")
-		doc = frappe.get_doc(self.doctype, self.name)
-		doc.is_whitelisted(method)
-
-		if frappe.local.request.method == "GET":
-			if not doc.has_permission("read"):
-				frappe.throw(_("Not permitted"), frappe.PermissionError)
-			frappe.local.response.update({"data": doc.run_method(method, **frappe.local.form_dict)})
-
-		elif frappe.local.request.method == "POST":
-			if not doc.has_permission("write"):
-				frappe.throw(_("Not permitted"), frappe.PermissionError)
-
-			frappe.local.response.update({"data": doc.run_method(method, **frappe.local.form_dict)})
-			frappe.db.commit()
-
-	def get_doc(self):
-		doc = frappe.get_doc(self.doctype, self.name)
-		if not doc.has_permission("read"):
-			raise frappe.PermissionError
-		doc.apply_fieldlevel_read_permissions()
-		frappe.local.response.update({"data": doc})
-
-	def update_doc(self):
-		data = get_request_form_data()
-
-		doc = frappe.get_doc(self.doctype, self.name, for_update=True)
-
-		if "flags" in data:
-			del data["flags"]
-
-		# Not checking permissions here because it's checked in doc.save
-		doc.update(data)
-
-		frappe.local.response.update({"data": doc.save().as_dict()})
-
-		# check for child table doctype
-		if doc.get("parenttype"):
-			frappe.get_doc(doc.parenttype, doc.parent).save()
-		frappe.db.commit()
-
-	def delete_doc(self):
-		# Not checking permissions here because it's checked in delete_doc
-		frappe.delete_doc(self.doctype, self.name, ignore_missing=False)
-		frappe.local.response.http_status_code = 202
-		frappe.local.response.message = "ok"
-		frappe.db.commit()
-
-	def get_doc_list(self):
-		if frappe.local.form_dict.get("fields"):
-			frappe.local.form_dict["fields"] = json.loads(frappe.local.form_dict["fields"])
-
-		# set limit of records for frappe.get_list
-		frappe.local.form_dict.setdefault(
-			"limit_page_length",
-			frappe.local.form_dict.limit or frappe.local.form_dict.limit_page_length or 20,
-		)
-
-		# convert strings to native types - only as_dict and debug accept bool
-		for param in ["as_dict", "debug"]:
-			param_val = frappe.local.form_dict.get(param)
-			if param_val is not None:
-				frappe.local.form_dict[param] = sbool(param_val)
-
-		# evaluate frappe.get_list
-		data = frappe.call(frappe.client.get_list, self.doctype, **frappe.local.form_dict)
-
-		# set frappe.get_list result to response
-		frappe.local.response.update({"data": data})
-
-	def create_doc(self):
-		data = get_request_form_data()
-		data.update({"doctype": self.doctype})
-
-		# insert document from request data
-		doc = frappe.get_doc(data).insert()
-
-		# set response data
-		frappe.local.response.update({"data": doc.as_dict()})
-
-		# commit for POST requests
-		frappe.db.commit()
-
-
-def get_request_form_data():
-	if frappe.local.form_dict.data is None:
-		data = frappe.safe_decode(frappe.local.request.get_data())
-	else:
-		data = frappe.local.form_dict.data
-
-	try:
-		return frappe.parse_json(data)
-	except ValueError:
-		return frappe.local.form_dict
-
-
-def validate_auth():
-	"""
-	Authenticate and sets user for the request.
-	"""
-	authorization_header = frappe.get_request_header("Authorization", "").split(" ")
-
-	if len(authorization_header) == 2:
-		validate_oauth(authorization_header)
-		validate_auth_via_api_keys(authorization_header)
-
-	validate_auth_via_hooks()
-
-
-def validate_oauth(authorization_header):
-	"""
-	Authenticate request using OAuth and set session user
-
-	Args:
-	        authorization_header (list of str): The 'Authorization' header containing the prefix and token
-	"""
-
-	from frappe.integrations.oauth2 import get_oauth_server
-	from frappe.oauth import get_url_delimiter
-
-	form_dict = frappe.local.form_dict
-	token = authorization_header[1]
-	req = frappe.request
-	parsed_url = urlparse(req.url)
-	access_token = {"access_token": token}
-	uri = (
-		parsed_url.scheme + "://" + parsed_url.netloc + parsed_url.path + "?" + urlencode(access_token)
-	)
-	http_method = req.method
-	headers = req.headers
-	body = req.get_data()
-	if req.content_type and "multipart/form-data" in req.content_type:
-		body = None
-
-	try:
-		required_scopes = frappe.db.get_value("OAuth Bearer Token", token, "scopes").split(
-			get_url_delimiter()
-		)
-		valid, oauthlib_request = get_oauth_server().verify_request(
-			uri, http_method, body, headers, required_scopes
-		)
-		if valid:
-			frappe.set_user(frappe.db.get_value("OAuth Bearer Token", token, "user"))
-			frappe.local.form_dict = form_dict
-	except AttributeError:
-		pass
-
-
-def validate_auth_via_api_keys(authorization_header):
-	"""
-	Authenticate request using API keys and set session user
-
-	Args:
-	        authorization_header (list of str): The 'Authorization' header containing the prefix and token
-	"""
-
-	try:
-		auth_type, auth_token = authorization_header
-		authorization_source = frappe.get_request_header("Frappe-Authorization-Source")
-		if auth_type.lower() == "basic":
-			api_key, api_secret = frappe.safe_decode(base64.b64decode(auth_token)).split(":")
-			validate_api_key_secret(api_key, api_secret, authorization_source)
-		elif auth_type.lower() == "token":
-			api_key, api_secret = auth_token.split(":")
-			validate_api_key_secret(api_key, api_secret, authorization_source)
-	except binascii.Error:
-		frappe.throw(
-			_("Failed to decode token, please provide a valid base64-encoded token."),
-			frappe.InvalidAuthorizationToken,
-		)
-	except (AttributeError, TypeError, ValueError):
-		pass
-
-
-def validate_api_key_secret(api_key, api_secret, frappe_authorization_source=None):
-	"""frappe_authorization_source to provide api key and secret for a doctype apart from User"""
-	doctype = frappe_authorization_source or "User"
-	doc = frappe.db.get_value(doctype=doctype, filters={"api_key": api_key}, fieldname=["name"])
-	form_dict = frappe.local.form_dict
-	doc_secret = frappe.utils.password.get_decrypted_password(doctype, doc, fieldname="api_secret")
-	if api_secret == doc_secret:
-		if doctype == "User":
-			user = frappe.db.get_value(doctype="User", filters={"api_key": api_key}, fieldname=["name"])
-		else:
-			user = frappe.db.get_value(doctype, doc, "user")
-		if frappe.local.login_manager.user in ("", "Guest"):
-			frappe.set_user(user)
-		frappe.local.form_dict = form_dict
-
-
-def validate_auth_via_hooks():
-	for auth_hook in frappe.get_hooks("auth_hooks", []):
-		frappe.get_attr(auth_hook)()

frappe/api/__init__.py

@@ -0,0 +1,80 @@
+# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
+# License: MIT. See LICENSE
+from enum import Enum
+
+from werkzeug.exceptions import NotFound
+from werkzeug.routing import Map, Submount
+from werkzeug.wrappers import Request, Response
+
+import frappe
+import frappe.client
+from frappe import _
+from frappe.utils.response import build_response
+
+
+class ApiVersion(str, Enum):
+	V1 = "v1"
+	V2 = "v2"
+
+
+def handle(request: Request):
+	"""
+	Entry point for `/api` methods.
+
+	APIs are versioned using second part of path.
+	v1 -> `/api/v1/*`
+	v2 -> `/api/v2/*`
+
+	Different versions have different specification but broadly following things are supported:
+
+	- `/api/method/{methodname}` will call a whitelisted method
+	- `/api/resource/{doctype}` will query a table
+	        examples:
+	        - `?fields=["name", "owner"]`
+	        - `?filters=[["Task", "name", "like", "%005"]]`
+	        - `?limit_start=0`
+	        - `?limit_page_length=20`
+	- `/api/resource/{doctype}/{name}` will point to a resource
+	        `GET` will return document
+	        `POST` will insert
+	        `PUT` will update
+	        `DELETE` will delete
+	"""
+
+	try:
+		endpoint, arguments = API_URL_MAP.bind_to_environ(request.environ).match()
+	except NotFound:  # Wrap 404 - backward compatiblity
+		raise frappe.DoesNotExistError
+
+	data = endpoint(**arguments)
+	if isinstance(data, Response):
+		return data
+
+	if data is not None:
+		frappe.response["data"] = data
+	return build_response("json")
+
+
+# Merge all API version routing rules
+from frappe.api.v1 import url_rules as v1_rules
+from frappe.api.v2 import url_rules as v2_rules
+
+API_URL_MAP = Map(
+	[
+		# V1 routes
+		Submount("/api", v1_rules),
+		Submount(f"/api/{ApiVersion.V1.value}", v1_rules),
+		Submount(f"/api/{ApiVersion.V2.value}", v2_rules),
+	],
+	strict_slashes=False,  # Allows skipping trailing slashes
+	merge_slashes=False,
+)
+
+
+def get_api_version() -> ApiVersion | None:
+	if not frappe.request:
+		return
+
+	if frappe.request.path.startswith(f"/api/{ApiVersion.V2.value}"):
+		return ApiVersion.V2
+	return ApiVersion.V1

frappe/api/v1.py

@@ -0,0 +1,118 @@
+import json
+
+from werkzeug.routing import Rule
+
+import frappe
+from frappe import _
+from frappe.utils.data import sbool
+
+
+def document_list(doctype: str):
+	if frappe.form_dict.get("fields"):
+		frappe.form_dict["fields"] = json.loads(frappe.form_dict["fields"])
+
+	# set limit of records for frappe.get_list
+	frappe.form_dict.setdefault(
+		"limit_page_length",
+		frappe.form_dict.limit or frappe.form_dict.limit_page_length or 20,
+	)
+
+	# convert strings to native types - only as_dict and debug accept bool
+	for param in ["as_dict", "debug"]:
+		param_val = frappe.form_dict.get(param)
+		if param_val is not None:
+			frappe.form_dict[param] = sbool(param_val)
+
+	# evaluate frappe.get_list
+	return frappe.call(frappe.client.get_list, doctype, **frappe.form_dict)
+
+
+def handle_rpc_call(method: str):
+	import frappe.handler
+
+	method = method.split("/")[0]  # for backward compatiblity
+
+	frappe.form_dict.cmd = method
+	return frappe.handler.handle()
+
+
+def create_doc(doctype: str):
+	data = get_request_form_data()
+	data.pop("doctype", None)
+	return frappe.new_doc(doctype, **data).insert()
+
+
+def update_doc(doctype: str, name: str):
+	data = get_request_form_data()
+
+	doc = frappe.get_doc(doctype, name, for_update=True)
+	if "flags" in data:
+		del data["flags"]
+
+	doc.update(data)
+	doc.save()
+
+	# check for child table doctype
+	if doc.get("parenttype"):
+		frappe.get_doc(doc.parenttype, doc.parent).save()
+
+	return doc
+
+
+def delete_doc(doctype: str, name: str):
+	# TODO: child doc handling
+	frappe.delete_doc(doctype, name, ignore_missing=False)
+	frappe.response.http_status_code = 202
+	return "ok"
+
+
+def read_doc(doctype: str, name: str):
+	# Backward compatiblity
+	if "run_method" in frappe.form_dict:
+		return execute_doc_method(doctype, name)
+
+	doc = frappe.get_doc(doctype, name)
+	if not doc.has_permission("read"):
+		raise frappe.PermissionError
+	doc.apply_fieldlevel_read_permissions()
+	return doc
+
+
+def execute_doc_method(doctype: str, name: str, method: str | None = None):
+	method = method or frappe.form_dict.pop("run_method")
+	doc = frappe.get_doc(doctype, name)
+	doc.is_whitelisted(method)
+
+	if frappe.request.method == "GET":
+		if not doc.has_permission("read"):
+			frappe.throw(_("Not permitted"), frappe.PermissionError)
+		return doc.run_method(method, **frappe.form_dict)
+
+	elif frappe.request.method == "POST":
+		if not doc.has_permission("write"):
+			frappe.throw(_("Not permitted"), frappe.PermissionError)
+
+		return doc.run_method(method, **frappe.form_dict)
+
+
+def get_request_form_data():
+	if frappe.form_dict.data is None:
+		data = frappe.safe_decode(frappe.request.get_data())
+	else:
+		data = frappe.form_dict.data
+
+	try:
+		return frappe.parse_json(data)
+	except ValueError:
+		return frappe.form_dict
+
+
+url_rules = [
+	Rule("/method/<path:method>", endpoint=handle_rpc_call),
+	Rule("/resource/<doctype>", methods=["GET"], endpoint=document_list),
+	Rule("/resource/<doctype>", methods=["POST"], endpoint=create_doc),
+	Rule("/resource/<doctype>/<path:name>/", methods=["GET"], endpoint=read_doc),
+	Rule("/resource/<doctype>/<path:name>/", methods=["PUT"], endpoint=update_doc),
+	Rule("/resource/<doctype>/<path:name>/", methods=["DELETE"], endpoint=delete_doc),
+	Rule("/resource/<doctype>/<path:name>/", methods=["POST"], endpoint=execute_doc_method),
+]

frappe/api/v2.py

@@ -0,0 +1,193 @@
+"""REST API v2
+
+This file defines routes and implementation for REST API.
+
+Note:
+	- All functions in this file should be treated as "whitelisted" as they are exposed via routes
+	- None of the functions present here should be called from python code, their location and
+	  internal implementation can change without treating it as "breaking change".
+"""
+import json
+from typing import Any
+
+from werkzeug.routing import Rule
+
+import frappe
+import frappe.client
+from frappe import _, get_newargs, is_whitelisted
+from frappe.core.doctype.server_script.server_script_utils import get_server_script_map
+from frappe.handler import is_valid_http_method, run_server_script, upload_file
+
+PERMISSION_MAP = {
+	"GET": "read",
+	"POST": "write",
+}
+
+
+def handle_rpc_call(method: str, doctype: str | None = None):
+	from frappe.modules.utils import load_doctype_module
+
+	if doctype:
+		# Expand to run actual method from doctype controller
+		module = load_doctype_module(doctype)
+		method = module.__name__ + "." + method
+
+	for hook in reversed(frappe.get_hooks("override_whitelisted_methods", {}).get(method, [])):
+		# override using the last hook
+		method = hook
+		break
+
+	# via server script
+	server_script = get_server_script_map().get("_api", {}).get(method)
+	if server_script:
+		return run_server_script(server_script)
+
+	try:
+		method = frappe.get_attr(method)
+	except Exception as e:
+		frappe.throw(_("Failed to get method {0} with {1}").format(method, e))
+
+	is_whitelisted(method)
+	is_valid_http_method(method)
+
+	return frappe.call(method, **frappe.form_dict)
+
+
+def login():
+	"""Login happens implicitly, this function doesn't do anything."""
+	pass
+
+
+def logout():
+	frappe.local.login_manager.logout()
+	frappe.db.commit()
+
+
+def read_doc(doctype: str, name: str):
+	doc = frappe.get_doc(doctype, name)
+	doc.check_permission("read")
+	doc.apply_fieldlevel_read_permissions()
+	return doc
+
+
+def document_list(doctype: str):
+	if frappe.form_dict.get("fields"):
+		frappe.form_dict["fields"] = json.loads(frappe.form_dict["fields"])
+
+	# set limit of records for frappe.get_list
+	frappe.form_dict.limit_page_length = frappe.form_dict.limit or 20
+	# evaluate frappe.get_list
+	return frappe.call(frappe.client.get_list, doctype, **frappe.form_dict)
+
+
+def count(doctype: str) -> int:
+	from frappe.desk.reportview import get_count
+
+	frappe.form_dict.doctype = doctype
+
+	return get_count()
+
+
+def create_doc(doctype: str):
+	data = frappe.form_dict
+	data.pop("doctype", None)
+	return frappe.new_doc(doctype, **data).insert()
+
+
+def update_doc(doctype: str, name: str):
+	data = frappe.form_dict
+
+	doc = frappe.get_doc(doctype, name, for_update=True)
+	data.pop("flags", None)
+	doc.update(data)
+	doc.save()
+
+	# check for child table doctype
+	if doc.get("parenttype"):
+		frappe.get_doc(doc.parenttype, doc.parent).save()
+
+	return doc
+
+
+def delete_doc(doctype: str, name: str):
+	frappe.client.delete_doc(doctype, name)
+	frappe.response.http_status_code = 202
+	return "ok"
+
+
+def execute_doc_method(doctype: str, name: str, method: str | None = None):
+	"""Get a document from DB and execute method on it.
+
+	Use cases:
+	- Submitting/cancelling document
+	- Triggering some kind of update on a document
+	"""
+	method = method or frappe.form_dict.pop("run_method")
+	doc = frappe.get_doc(doctype, name)
+	doc.is_whitelisted(method)
+
+	doc.check_permission(PERMISSION_MAP[frappe.request.method])
+	return doc.run_method(method, **frappe.form_dict)
+
+
+def run_doc_method(method: str, document: dict[str, Any] | str, kwargs=None):
+	"""run a whitelisted controller method on in-memory document.
+
+
+	This is useful for building clients that don't necessarily encode all the business logic but
+	call server side function on object to validate and modify the doc.
+
+	The doc CAN exists in DB too and can write to DB as well if method is POST.
+	"""
+
+	if isinstance(document, str):
+		document = frappe.parse_json(document)
+
+	if kwargs is None:
+		kwargs = {}
+
+	doc = frappe.get_doc(document)
+	doc._original_modified = doc.modified
+	doc.check_if_latest()
+
+	doc.check_permission(PERMISSION_MAP[frappe.request.method])
+
+	method_obj = getattr(doc, method)
+	fn = getattr(method_obj, "__func__", method_obj)
+	is_whitelisted(fn)
+	is_valid_http_method(fn)
+
+	new_kwargs = get_newargs(fn, kwargs)
+	response = doc.run_method(method, **new_kwargs)
+	frappe.response.docs.append(doc)  # send modified document and result both.
+	return response
+
+
+url_rules = [
+	# RPC calls
+	Rule("/method/login", endpoint=login),
+	Rule("/method/logout", endpoint=logout),
+	Rule("/method/ping", endpoint=frappe.ping),
+	Rule("/method/upload_file", endpoint=upload_file),
+	Rule("/method/<method>", endpoint=handle_rpc_call),
+	Rule(
+		"/method/run_doc_method",
+		methods=["GET", "POST"],
+		endpoint=lambda: frappe.call(run_doc_method, **frappe.form_dict),
+	),
+	Rule("/method/<doctype>/<method>", endpoint=handle_rpc_call),
+	# Document level APIs
+	Rule("/document/<doctype>", methods=["GET"], endpoint=document_list),
+	Rule("/document/<doctype>", methods=["POST"], endpoint=create_doc),
+	Rule("/document/<doctype>/<path:name>/", methods=["GET"], endpoint=read_doc),
+	Rule("/document/<doctype>/<path:name>/", methods=["PATCH", "PUT"], endpoint=update_doc),
+	Rule("/document/<doctype>/<path:name>/", methods=["DELETE"], endpoint=delete_doc),
+	Rule(
+		"/document/<doctype>/<path:name>/method/<method>/",
+		methods=["GET", "POST"],
+		endpoint=execute_doc_method,
+	),
+	# Collection level APIs
+	Rule("/doctype/<doctype>/meta", methods=["GET"], endpoint=frappe.get_meta),
+	Rule("/doctype/<doctype>/count", methods=["GET"], endpoint=count),
+]

frappe/app.py

@@ -9,6 +9,7 @@ import re
 
 from werkzeug.exceptions import HTTPException, NotFound
 from werkzeug.middleware.profiler import ProfilerMiddleware
+from werkzeug.middleware.proxy_fix import ProxyFix
 from werkzeug.middleware.shared_data import SharedDataMiddleware
 from werkzeug.wrappers import Request, Response
 from werkzeug.wsgi import ClosingIterator
@@ -21,10 +22,11 @@ import frappe.rate_limiter
 import frappe.recorder
 import frappe.utils.response
 from frappe import _
-from frappe.auth import SAFE_HTTP_METHODS, UNSAFE_HTTP_METHODS, HTTPRequest
+from frappe.auth import SAFE_HTTP_METHODS, UNSAFE_HTTP_METHODS, HTTPRequest, validate_auth
 from frappe.middlewares import StaticDataMiddleware
-from frappe.utils import CallbackManager, cint, get_site_name, sanitize_html
+from frappe.utils import CallbackManager, cint, get_site_name
 from frappe.utils.data import escape_html
+from frappe.utils.deprecations import deprecation_warning
 from frappe.utils.error import log_error_snapshot
 from frappe.website.serve import get_response
 
@@ -92,16 +94,20 @@ def application(request: Request):
 
 		init_request(request)
 
-		frappe.api.validate_auth()
+		validate_auth()
 
 		if request.method == "OPTIONS":
 			response = Response()
 
 		elif frappe.form_dict.cmd:
-			response = frappe.handler.handle()
+			deprecation_warning(
+				f"{frappe.form_dict.cmd}: Sending `cmd` for RPC calls is deprecated, call REST API instead `/api/method/cmd`"
+			)
+			frappe.handler.handle()
+			response = frappe.utils.response.build_response("json")
 
 		elif request.path.startswith("/api/"):
-			response = frappe.api.handle()
+			response = frappe.api.handle(request)
 
 		elif request.path.startswith("/backups"):
 			response = frappe.utils.response.download_backup(request.path)
@@ -277,11 +283,11 @@ def set_cors_headers(response):
 	response.headers.extend(cors_headers)
 
 
-def make_form_dict(request):
+def make_form_dict(request: Request):
 	import json
 
 	request_data = request.get_data(as_text=True)
-	if "application/json" in (request.content_type or "") and request_data:
+	if request_data and request.is_json:
 		args = json.loads(request_data)
 	else:
 		args = {}
@@ -293,9 +299,8 @@ def make_form_dict(request):
 
 	frappe.local.form_dict = frappe._dict(args)
 
-	if "_" in frappe.local.form_dict:
-		# _ is passed by $.ajax so that the request is not cached by the browser. So, remove _ from form_dict
-		frappe.local.form_dict.pop("_")
+	# _ is passed by $.ajax so that the request is not cached by the browser. So, remove _ from form_dict
+	frappe.local.form_dict.pop("_", None)
 
 
 def handle_exception(e):
@@ -413,7 +418,13 @@ def sync_database(rollback: bool) -> bool:
 
 
 def serve(
-	port=8000, profile=False, no_reload=False, no_threading=False, site=None, sites_path="."
+	port=8000,
+	profile=False,
+	no_reload=False,
+	no_threading=False,
+	site=None,
+	sites_path=".",
+	proxy=False,
 ):
 	global application, _site, _sites_path
 	_site = site
@@ -427,6 +438,9 @@ def serve(
 	if not os.environ.get("NO_STATICS"):
 		application = application_with_statics()
 
+	if proxy or os.environ.get("USE_PROXY"):
+		application = ProxyFix(application, x_for=1, x_proto=1, x_host=1, x_port=1, x_prefix=1)
+
 	application.debug = True
 	application.config = {"SERVER_NAME": "127.0.0.1:8000"}
 

frappe/auth.py

@@ -1,6 +1,8 @@
 # Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors
 # MIT License. See LICENSE
-from urllib.parse import quote
+import base64
+import binascii
+from urllib.parse import quote, urlencode, urlparse
 
 import frappe
 import frappe.database
@@ -17,7 +19,8 @@ from frappe.twofactor import (
 	should_run_2fa,
 )
 from frappe.utils import cint, date_diff, datetime, get_datetime, today
-from frappe.utils.password import check_password
+from frappe.utils.deprecations import deprecation_warning
+from frappe.utils.password import check_password, get_decrypted_password
 from frappe.website.utils import get_home_page
 
 SAFE_HTTP_METHODS = frozenset(("GET", "HEAD", "OPTIONS"))
@@ -235,23 +238,28 @@ class LoginManager:
 		_raw_user_name = user
 		user = User.find_by_credentials(user, pwd)
 
+		ip_tracker = get_login_attempt_tracker(frappe.local.request_ip)
 		if not user:
+			ip_tracker and ip_tracker.add_failure_attempt()
 			self.fail("Invalid login credentials", user=_raw_user_name)
 
 		# Current login flow uses cached credentials for authentication while checking OTP.
 		# Incase of OTP check, tracker for auth needs to be disabled(If not, it can remove tracker history as it is going to succeed anyway)
 		# Tracker is activated for 2FA incase of OTP.
 		ignore_tracker = should_run_2fa(user.name) and ("otp" in frappe.form_dict)
-		tracker = None if ignore_tracker else get_login_attempt_tracker(user.name)
+		user_tracker = None if ignore_tracker else get_login_attempt_tracker(user.name)
 
 		if not user.is_authenticated:
-			tracker and tracker.add_failure_attempt()
+			user_tracker and user_tracker.add_failure_attempt()
+			ip_tracker and ip_tracker.add_failure_attempt()
 			self.fail("Invalid login credentials", user=user.name)
 		elif not (user.name == "Administrator" or user.enabled):
-			tracker and tracker.add_failure_attempt()
+			user_tracker and user_tracker.add_failure_attempt()
+			ip_tracker and ip_tracker.add_failure_attempt()
 			self.fail("User disabled or missing", user=user.name)
 		else:
-			tracker and tracker.add_success_attempt()
+			user_tracker and user_tracker.add_success_attempt()
+			ip_tracker and ip_tracker.add_success_attempt()
 		self.user = user.name
 
 	def force_user_to_reset_password(self):
@@ -433,7 +441,7 @@ def validate_ip_address(user):
 	frappe.throw(_("Access not allowed from this IP Address"), frappe.AuthenticationError)
 
 
-def get_login_attempt_tracker(user_name: str, raise_locked_exception: bool = True):
+def get_login_attempt_tracker(key: str, raise_locked_exception: bool = True):
 	"""Get login attempt tracker instance.
 
 	:param user_name: Name of the loggedin user
@@ -447,7 +455,7 @@ def get_login_attempt_tracker(user_name: str, raise_locked_exception: bool = Tru
 		tracker_kwargs["lock_interval"] = sys_settings.allow_login_after_fail
 		tracker_kwargs["max_consecutive_login_attempts"] = sys_settings.allow_consecutive_login_attempts
 
-	tracker = LoginAttemptTracker(user_name, **tracker_kwargs)
+	tracker = LoginAttemptTracker(key, **tracker_kwargs)
 
 	if raise_locked_exception and track_login_attempts and not tracker.is_user_allowed():
 		frappe.throw(
@@ -466,7 +474,12 @@ class LoginAttemptTracker:
 	"""
 
 	def __init__(
-		self, user_name: str, max_consecutive_login_attempts: int = 3, lock_interval: int = 5 * 60
+		self,
+		key: str,
+		max_consecutive_login_attempts: int = 3,
+		lock_interval: int = 5 * 60,
+		*,
+		user_name: str = None,
 	):
 		"""Initialize the tracker.
 
@@ -474,21 +487,23 @@ class LoginAttemptTracker:
 		:param max_consecutive_login_attempts: Maximum allowed consecutive failed login attempts
 		:param lock_interval: Locking interval incase of maximum failed attempts
 		"""
-		self.user_name = user_name
+		if user_name:
+			deprecation_warning("`username` parameter is deprecated, use `key` instead.")
+		self.key = key or user_name
 		self.lock_interval = datetime.timedelta(seconds=lock_interval)
 		self.max_failed_logins = max_consecutive_login_attempts
 
 	@property
 	def login_failed_count(self):
-		return frappe.cache.hget("login_failed_count", self.user_name)
+		return frappe.cache.hget("login_failed_count", self.key)
 
 	@login_failed_count.setter
 	def login_failed_count(self, count):
-		frappe.cache.hset("login_failed_count", self.user_name, count)
+		frappe.cache.hset("login_failed_count", self.key, count)
 
 	@login_failed_count.deleter
 	def login_failed_count(self):
-		frappe.cache.hdel("login_failed_count", self.user_name)
+		frappe.cache.hdel("login_failed_count", self.key)
 
 	@property
 	def login_failed_time(self):
@@ -496,15 +511,15 @@ class LoginAttemptTracker:
 
 		For every user we track only First failed login attempt time within lock interval of time.
 		"""
-		return frappe.cache.hget("login_failed_time", self.user_name)
+		return frappe.cache.hget("login_failed_time", self.key)
 
 	@login_failed_time.setter
 	def login_failed_time(self, timestamp):
-		frappe.cache.hset("login_failed_time", self.user_name, timestamp)
+		frappe.cache.hset("login_failed_time", self.key, timestamp)
 
 	@login_failed_time.deleter
 	def login_failed_time(self):
-		frappe.cache.hdel("login_failed_time", self.user_name)
+		frappe.cache.hdel("login_failed_time", self.key)
 
 	def add_failure_attempt(self):
 		"""Log user failure attempts into the system.
@@ -547,3 +562,114 @@ class LoginAttemptTracker:
 		):
 			return False
 		return True
+
+
+def validate_auth():
+	"""
+	Authenticate and sets user for the request.
+	"""
+	authorization_header = frappe.get_request_header("Authorization", "").split(" ")
+
+	if len(authorization_header) == 2:
+		validate_oauth(authorization_header)
+		validate_auth_via_api_keys(authorization_header)
+
+		# If login via bearer, basic or keypair didn't work then authentication failed and we
+		# should terminate here.
+		if frappe.session.user in ("", "Guest"):
+			raise frappe.AuthenticationError
+
+	validate_auth_via_hooks()
+
+
+def validate_oauth(authorization_header):
+	"""
+	Authenticate request using OAuth and set session user
+
+	Args:
+	        authorization_header (list of str): The 'Authorization' header containing the prefix and token
+	"""
+
+	from frappe.integrations.oauth2 import get_oauth_server
+	from frappe.oauth import get_url_delimiter
+
+	if authorization_header[0].lower() != "bearer":
+		return
+
+	form_dict = frappe.local.form_dict
+	token = authorization_header[1]
+	req = frappe.request
+	parsed_url = urlparse(req.url)
+	access_token = {"access_token": token}
+	uri = (
+		parsed_url.scheme + "://" + parsed_url.netloc + parsed_url.path + "?" + urlencode(access_token)
+	)
+	http_method = req.method
+	headers = req.headers
+	body = req.get_data()
+	if req.content_type and "multipart/form-data" in req.content_type:
+		body = None
+
+	try:
+		required_scopes = frappe.db.get_value("OAuth Bearer Token", token, "scopes").split(
+			get_url_delimiter()
+		)
+		valid, oauthlib_request = get_oauth_server().verify_request(
+			uri, http_method, body, headers, required_scopes
+		)
+		if valid:
+			frappe.set_user(frappe.db.get_value("OAuth Bearer Token", token, "user"))
+			frappe.local.form_dict = form_dict
+	except AttributeError:
+		raise frappe.AuthenticationError
+
+
+def validate_auth_via_api_keys(authorization_header):
+	"""
+	Authenticate request using API keys and set session user
+
+	Args:
+	        authorization_header (list of str): The 'Authorization' header containing the prefix and token
+	"""
+
+	try:
+		auth_type, auth_token = authorization_header
+		authorization_source = frappe.get_request_header("Frappe-Authorization-Source")
+		if auth_type.lower() == "basic":
+			api_key, api_secret = frappe.safe_decode(base64.b64decode(auth_token)).split(":")
+			validate_api_key_secret(api_key, api_secret, authorization_source)
+		elif auth_type.lower() == "token":
+			api_key, api_secret = auth_token.split(":")
+			validate_api_key_secret(api_key, api_secret, authorization_source)
+	except binascii.Error:
+		frappe.throw(
+			_("Failed to decode token, please provide a valid base64-encoded token."),
+			frappe.InvalidAuthorizationToken,
+		)
+	except (AttributeError, TypeError, ValueError):
+		raise frappe.AuthenticationError
+
+
+def validate_api_key_secret(api_key, api_secret, frappe_authorization_source=None):
+	"""frappe_authorization_source to provide api key and secret for a doctype apart from User"""
+	doctype = frappe_authorization_source or "User"
+	doc = frappe.db.get_value(doctype=doctype, filters={"api_key": api_key}, fieldname=["name"])
+	if not doc:
+		raise frappe.AuthenticationError
+	form_dict = frappe.local.form_dict
+	doc_secret = get_decrypted_password(doctype, doc, fieldname="api_secret")
+	if api_secret == doc_secret:
+		if doctype == "User":
+			user = frappe.db.get_value(doctype="User", filters={"api_key": api_key}, fieldname=["name"])
+		else:
+			user = frappe.db.get_value(doctype, doc, "user")
+		if frappe.local.login_manager.user in ("", "Guest"):
+			frappe.set_user(user)
+		frappe.local.form_dict = form_dict
+	else:
+		raise frappe.AuthenticationError
+
+
+def validate_auth_via_hooks():
+	for auth_hook in frappe.get_hooks("auth_hooks", []):
+		frappe.get_attr(auth_hook)()

frappe/automation/doctype/assignment_rule/assignment_rule.py

@@ -78,7 +78,7 @@ class AssignmentRule(Document):
 
 	def do_assignment(self, doc):
 		# clear existing assignment, to reassign
-		assign_to.clear(doc.get("doctype"), doc.get("name"))
+		assign_to.clear(doc.get("doctype"), doc.get("name"), ignore_permissions=True)
 
 		user = self.get_user(doc)
 
@@ -92,7 +92,8 @@ class AssignmentRule(Document):
 					assignment_rule=self.name,
 					notify=True,
 					date=doc.get(self.due_date_based_on) if self.due_date_based_on else None,
-				)
+				),
+				ignore_permissions=True,
 			)
 
 			# set for reference in round robin
@@ -104,12 +105,14 @@ class AssignmentRule(Document):
 	def clear_assignment(self, doc):
 		"""Clear assignments"""
 		if self.safe_eval("unassign_condition", doc):
-			return assign_to.clear(doc.get("doctype"), doc.get("name"))
+			return assign_to.clear(doc.get("doctype"), doc.get("name"), ignore_permissions=True)
 
 	def close_assignments(self, doc):
 		"""Close assignments"""
 		if self.safe_eval("close_condition", doc):
-			return assign_to.close_all_assignments(doc.get("doctype"), doc.get("name"))
+			return assign_to.close_all_assignments(
+				doc.get("doctype"), doc.get("name"), ignore_permissions=True
+			)
 
 	def get_user(self, doc):
 		"""

frappe/automation/doctype/assignment_rule/test_assignment_rule.py

@@ -113,6 +113,20 @@ class TestAutoAssign(FrappeTestCase):
 				len(frappe.get_all("ToDo", dict(allocated_to=user, reference_type=TEST_DOCTYPE))), 10
 			)
 
+	def test_assingment_on_guest_submissions(self):
+		"""Sometimes documents are inserted as guest, check if assignment rules run on them. Use case: Web Forms"""
+		with self.set_user("Guest"):
+			doc = _make_test_record(ignore_permissions=True, public=1)
+
+		# check assignment to *anyone*
+		self.assertTrue(
+			frappe.db.get_value(
+				"ToDo",
+				{"reference_type": TEST_DOCTYPE, "reference_name": doc.name, "status": "Open"},
+				"allocated_to",
+			),
+		)
+
 	def test_based_on_field(self):
 		self.assignment_rule.rule = "Based on Field"
 		self.assignment_rule.field = "owner"
@@ -373,13 +387,17 @@ def get_assignment_rule(days, assign=None):
 	return assignment_rule
 
 
-def _make_test_record(**kwargs):
+def _make_test_record(
+	*,
+	ignore_permissions=False,
+	**kwargs,
+):
 	doc = frappe.new_doc(TEST_DOCTYPE)
 
 	if kwargs:
 		doc.update(kwargs)
 
-	return doc.insert()
+	return doc.insert(ignore_permissions=ignore_permissions)
 
 
 def create_test_doctype(doctype: str):

frappe/boot.py

@@ -295,8 +295,7 @@ def add_home_page(bootinfo, docs):
 		docs.append(page)
 		bootinfo["home_page"] = page.name
 	except (frappe.DoesNotExistError, frappe.PermissionError):
-		if frappe.message_log:
-			frappe.message_log.pop()
+		frappe.clear_last_message()
 		bootinfo["home_page"] = "Workspaces"
 
 

frappe/client.py

@@ -28,12 +28,13 @@ def get_list(
 	doctype,
 	fields=None,
 	filters=None,
+	group_by=None,
 	order_by=None,
 	limit_start=None,
 	limit_page_length=20,
 	parent=None,
-	debug=False,
-	as_dict=True,
+	debug: bool = False,
+	as_dict: bool = True,
 	or_filters=None,
 ):
 	"""Returns a list of records by filters, fields, ordering and limit
@@ -53,6 +54,7 @@ def get_list(
 		fields=fields,
 		filters=filters,
 		or_filters=or_filters,
+		group_by=group_by,
 		order_by=order_by,
 		limit_start=limit_start,
 		limit_page_length=limit_page_length,

frappe/commands/utils.py

@@ -504,9 +504,9 @@ def postgres(context, extra_args):
 
 
 def _mariadb(extra_args=None):
-	mysql = which("mysql")
+	mariadb = which("mariadb") or which("mysql")
 	command = [
-		mysql,
+		mariadb,
 		"--port",
 		str(frappe.conf.db_port),
 		"-u",
@@ -521,7 +521,7 @@ def _mariadb(extra_args=None):
 	]
 	if extra_args:
 		command += list(extra_args)
-	os.execv(mysql, command)
+	os.execv(mariadb, command)
 
 
 def _psql(extra_args=None):
@@ -929,6 +929,12 @@ def run_ui_tests(
 @click.command("serve")
 @click.option("--port", default=8000)
 @click.option("--profile", is_flag=True, default=False)
+@click.option(
+	"--proxy",
+	is_flag=True,
+	default=False,
+	help="The development server may be run behind a proxy, e.g. ngrok / localtunnel",
+)
 @click.option("--noreload", "no_reload", is_flag=True, default=False)
 @click.option("--nothreading", "no_threading", is_flag=True, default=False)
 @click.option("--with-coverage", is_flag=True, default=False)
@@ -937,6 +943,7 @@ def serve(
 	context,
 	port=None,
 	profile=False,
+	proxy=False,
 	no_reload=False,
 	no_threading=False,
 	sites_path=".",
@@ -958,6 +965,7 @@ def serve(
 		frappe.app.serve(
 			port=port,
 			profile=profile,
+			proxy=proxy,
 			no_reload=no_reload,
 			no_threading=no_threading,
 			site=site,

frappe/contacts/doctype/address/address.json

@@ -148,7 +148,7 @@
  "icon": "fa fa-map-marker",
  "idx": 5,
  "links": [],
- "modified": "2020-10-21 16:14:37.284830",
+ "modified": "2023-10-30 05:50:23.912366",
  "modified_by": "Administrator",
  "module": "Contacts",
  "name": "Address",
@@ -206,9 +206,19 @@
    "role": "System Manager",
    "share": 1,
    "write": 1
+  },
+  {
+   "create": 1,
+   "export": 1,
+   "if_owner": 1,
+   "print": 1,
+   "read": 1,
+   "role": "All",
+   "write": 1
   }
  ],
  "search_fields": "country, state",
  "sort_field": "modified",
- "sort_order": "DESC"
-}
+ "sort_order": "DESC",
+ "states": []
+}

frappe/contacts/doctype/address/address.py

@@ -1,8 +1,6 @@
 # Copyright (c) 2015, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
-from typing import Optional
-
 from jinja2 import TemplateSyntaxError
 
 import frappe
@@ -166,18 +164,23 @@ def get_default_address(
 
 @frappe.whitelist()
 def get_address_display(address_dict: dict | str | None) -> str | None:
-	if not address_dict:
+	return render_address(address_dict)
+
+
+def render_address(address: dict | str | None, check_permissions=True) -> str | None:
+	if not address:
 		return
 
-	if not isinstance(address_dict, dict):
-		address = frappe.get_cached_doc("Address", address_dict)
-		address.check_permission()
-		address_dict = address.as_dict()
+	if not isinstance(address, dict):
+		address = frappe.get_cached_doc("Address", address)
+		if check_permissions:
+			address.check_permission()
+		address = address.as_dict()
 
-	name, template = get_address_templates(address_dict)
+	name, template = get_address_templates(address)
 
 	try:
-		return frappe.render_template(template, address_dict)
+		return frappe.render_template(template, address)
 	except TemplateSyntaxError:
 		frappe.throw(_("There is an error in your Address Template {0}").format(name))
 
@@ -258,7 +261,7 @@ def get_company_address(company):
 
 	if company:
 		ret.company_address = get_default_address("Company", company)
-		ret.company_address_display = get_address_display(ret.company_address)
+		ret.company_address_display = render_address(ret.company_address, check_permissions=False)
 
 	return ret
 

frappe/contacts/doctype/contact/contact.json

@@ -257,7 +257,7 @@
  "image_field": "image",
  "index_web_pages_for_search": 1,
  "links": [],
- "modified": "2022-10-27 10:40:50.097481",
+ "modified": "2023-10-02 12:00:27.299156",
  "modified_by": "Administrator",
  "module": "Contacts",
  "name": "Contact",
@@ -381,10 +381,13 @@
    "write": 1
   },
   {
-   "permlevel": 1,
+   "create": 1,
+   "delete": 1,
+   "if_owner": 1,
    "read": 1,
    "report": 1,
-   "role": "All"
+   "role": "All",
+   "write": 1
   }
  ],
  "show_title_field_in_link": 1,
@@ -392,4 +395,4 @@
  "sort_order": "ASC",
  "states": [],
  "title_field": "full_name"
-}
+}

frappe/core/doctype/activity_log/test_activity_log.py

@@ -20,6 +20,7 @@ class TestActivityLog(FrappeTestCase):
 			}
 		)
 
+		frappe.local.request_ip = "127.0.0.1"
 		frappe.local.cookie_manager = CookieManager()
 		frappe.local.login_manager = LoginManager()
 
@@ -60,6 +61,7 @@ class TestActivityLog(FrappeTestCase):
 			{"cmd": "login", "sid": "Guest", "pwd": "admin", "usr": "Administrator"}
 		)
 
+		frappe.local.request_ip = "127.0.0.1"
 		frappe.local.cookie_manager = CookieManager()
 		frappe.local.login_manager = LoginManager()
 

frappe/core/doctype/audit_trail/audit_trail.js

@@ -3,6 +3,20 @@
 
 frappe.ui.form.on("Audit Trail", {
 	refresh(frm) {
+		let prev_route = frappe.get_prev_route();
+		if (
+			prev_route.length > 2 &&
+			prev_route[0] == "Form" &&
+			!prev_route.includes("Audit Trail")
+		) {
+			frm.set_value("doctype_name", prev_route[1]);
+			frm.set_value("document", prev_route[2]);
+			frm.set_value("start_date", "");
+			frm.set_value("end_date", "");
+			if (frm.doc.doctype_name && frm.doc.document)
+				frm.events.get_audit_trail_for_document(frm);
+		}
+
 		frm.page.clear_indicator();
 
 		frm.disable_save();
@@ -16,17 +30,61 @@ frappe.ui.form.on("Audit Trail", {
 			};
 		});
 
+		frm.set_query("document", () => {
+			let filters = {
+				amended_from: ["!=", ""],
+			};
+			if (frm.doc.start_date && frm.doc.end_date)
+				filters["creation"] = ["between", [frm.doc.start_date, frm.doc.end_date]];
+			else if (frm.doc.start_date) filters["creation"] = [">=", frm.doc.start_date];
+			else if (frm.doc.end_date) filters["creation"] = ["<=", frm.doc.end_date];
+			return {
+				filters: filters,
+			};
+		});
+
 		frm.page.set_primary_action("Compare", () => {
-			frm.call({
-				doc: frm.doc,
-				method: "compare_document",
-				callback: function (r) {
-					let document_names = r.message[0];
-					let changed_fields = r.message[1];
-					frm.events.render_changed_fields(frm, document_names, changed_fields);
-					frm.events.render_rows_added_or_removed(frm, changed_fields);
-				},
+			frm.events.get_audit_trail_for_document(frm);
+		});
+	},
+
+	start_date(frm) {
+		if (frm.doc.start_date > frm.doc.end_date) {
+			frm.doc.end_date = "";
+			frm.refresh_fields();
+		}
+
+		frappe.db
+			.get_value(frm.doc.doctype_name, frm.doc.document, "creation")
+			.then((creation) => {
+				if (frappe.datetime.obj_to_str(creation) < frm.doc.start_date) {
+					frm.doc.document = "";
+					frm.refresh_fields();
+				}
 			});
+	},
+
+	end_date(frm) {
+		frappe.db
+			.get_value(frm.doc.doctype_name, frm.doc.document, "creation")
+			.then((creation) => {
+				if (frappe.datetime.obj_to_str(creation) > frm.doc.end_date) {
+					frm.doc.document = "";
+					frm.refresh_fields();
+				}
+			});
+	},
+
+	get_audit_trail_for_document(frm) {
+		frm.call({
+			doc: frm.doc,
+			method: "compare_document",
+			callback: function (r) {
+				let document_names = r.message[0];
+				let changed_fields = r.message[1];
+				frm.events.render_changed_fields(frm, document_names, changed_fields);
+				frm.events.render_rows_added_or_removed(frm, changed_fields);
+			},
 		});
 	},
 

frappe/core/doctype/audit_trail/audit_trail.json

@@ -9,6 +9,10 @@
   "doctype_name",
   "column_break_peck",
   "document",
+  "section_break_dfrx",
+  "start_date",
+  "column_break_ytzm",
+  "end_date",
   "section_break_gppi",
   "version_table",
   "rows_added_section",
@@ -21,7 +25,7 @@
    "fieldname": "doctype_name",
    "fieldtype": "Link",
    "in_list_view": 1,
-   "label": "Doctype",
+   "label": "DocType",
    "options": "DocType",
    "reqd": 1
   },
@@ -68,13 +72,34 @@
    "fieldtype": "Section Break",
    "hidden": 1,
    "label": "Rows Removed"
+  },
+  {
+   "fieldname": "start_date",
+   "fieldtype": "Date",
+   "label": "Start Date"
+  },
+  {
+   "fieldname": "end_date",
+   "fieldtype": "Date",
+   "label": "End Date"
+  },
+  {
+   "collapsible": 1,
+   "collapsible_depends_on": "eval: doc.start_date || doc.end_date",
+   "fieldname": "section_break_dfrx",
+   "fieldtype": "Section Break",
+   "label": "Date Range"
+  },
+  {
+   "fieldname": "column_break_ytzm",
+   "fieldtype": "Column Break"
   }
  ],
  "hide_toolbar": 1,
  "index_web_pages_for_search": 1,
  "issingle": 1,
  "links": [],
- "modified": "2023-08-22 12:12:59.780845",
+ "modified": "2023-10-31 13:12:41.749483",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "Audit Trail",
@@ -85,7 +110,8 @@
    "print": 1,
    "read": 1,
    "role": "System Manager",
-   "share": 1
+   "share": 1,
+   "write": 1
   }
  ],
  "sort_field": "modified",

frappe/core/doctype/audit_trail/audit_trail.py

@@ -7,6 +7,7 @@ import frappe
 from frappe import _
 from frappe.core.doctype.version.version import get_diff
 from frappe.model.document import Document
+from frappe.utils import compare
 
 
 class AuditTrail(Document):
@@ -20,20 +21,31 @@ class AuditTrail(Document):
 
 		doctype_name: DF.Link
 		document: DF.DynamicLink
+		end_date: DF.Date | None
+		start_date: DF.Date | None
 	# end: auto-generated types
 	pass
 
 	def validate(self):
-		self.validate_doctype_name()
+		self.validate_fields()
 		self.validate_document()
 
-	def validate_doctype_name(self):
-		if not self.doctype_name:
-			frappe.throw(_("{} field cannot be empty.").format(frappe.bold("Doctype")))
+	def validate_fields(self):
+		fields_dict = {
+			"DocType": self.doctype_name,
+			"Document": self.document,
+		}
+		for field in fields_dict:
+			if not fields_dict[field]:
+				frappe.throw(_("{} field cannot be empty.").format(frappe.bold(field)))
 
 	def validate_document(self):
-		if not self.document:
-			frappe.throw(_("{} field cannot be empty.").format(frappe.bold("Document")))
+		if not frappe.db.exists(self.doctype_name, self.document):
+			frappe.throw(
+				_("The selected document {0} is not a {1}.").format(
+					frappe.bold(self.document), frappe.bold(self.doctype_name)
+				)
+			)
 
 	@frappe.whitelist()
 	def compare_document(self):
@@ -58,11 +70,18 @@ class AuditTrail(Document):
 		}
 
 	def get_amended_documents(self):
+		start_date = self.get("start_date")
 		amended_document_names = []
 		curr_doc = self.document
-		while curr_doc and len(amended_document_names) < 5:
+		creation = frappe.db.get_value(self.doctype_name, self.document, "creation")
+		while (
+			curr_doc
+			and len(amended_document_names) < 5
+			and (start_date is None or compare(creation, ">=", start_date, "Date"))
+		):
 			amended_document_names.append(curr_doc)
 			curr_doc = frappe.db.get_value(self.doctype_name, curr_doc, "amended_from")
+			creation = frappe.db.get_value(self.doctype_name, curr_doc, "creation")
 		amended_document_names = amended_document_names[::-1]
 
 		return amended_document_names

frappe/core/doctype/audit_trail/test_audit_trail.py

@@ -3,6 +3,7 @@
 
 import frappe
 from frappe.tests.utils import FrappeTestCase
+from frappe.utils import today
 
 
 class TestAuditTrail(FrappeTestCase):
@@ -129,6 +130,11 @@ def amend_document(amend_from, changed_fields, rows_updated, submit=False):
 
 def create_comparator_doc(doctype_name, document):
 	comparator = frappe.new_doc("Audit Trail")
-	comparator.doctype_name = doctype_name
-	comparator.document = document
+	args_dict = {
+		"doctype_name": doctype_name,
+		"document": document,
+		"start_date": today(),
+		"end_date": today(),
+	}
+	comparator.update(args_dict)
 	return comparator

frappe/core/doctype/communication/communication.py

@@ -463,7 +463,7 @@ class Communication(Document, CommunicationEmailMixin):
 					duplicate = True
 
 			if duplicate:
-				del self.timeline_links[:]  # make it python 2 compatible as list.clear() is python 3 only
+				self.timeline_links.clear()
 				for l in links:
 					self.add_link(link_doctype=l[0], link_name=l[1])
 

frappe/core/doctype/data_import/patches/remove_stale_docfields_from_legacy_version.py

@@ -0,0 +1,6 @@
+import frappe
+
+
+def execute():
+	"""Remove stale docfields from legacy version"""
+	frappe.db.delete("DocField", {"options": "Data Import", "parent": "Data Import Legacy"})

frappe/core/doctype/data_import/test_importer.py

@@ -63,7 +63,7 @@ class TestImporter(FrappeTestCase):
 	def test_data_import_without_mandatory_values(self):
 		import_file = get_import_file("sample_import_file_without_mandatory")
 		data_import = self.get_importer(doctype_name, import_file)
-		frappe.local.message_log = []
+		frappe.clear_messages()
 		data_import.start_import()
 		data_import.reload()
 

frappe/core/doctype/deleted_document/deleted_document.py

@@ -73,11 +73,11 @@ def bulk_restore(docnames):
 			restored.append(d)
 
 		except frappe.DocumentAlreadyRestored:
-			frappe.message_log.pop()
+			frappe.clear_last_message()
 			invalid.append(d)
 
 		except Exception:
-			frappe.message_log.pop()
+			frappe.clear_last_message()
 			failed.append(d)
 			frappe.db.rollback()
 

frappe/core/doctype/docfield/docfield.json

@@ -244,9 +244,10 @@
   },
   {
    "default": "0",
+   "description": "If unchecked, the value will always be re-fetched on save.",
    "fieldname": "fetch_if_empty",
    "fieldtype": "Check",
-   "label": "Fetch only if value is not set"
+   "label": "Fetch on Save if Empty"
   },
   {
    "fieldname": "permissions",
@@ -415,7 +416,7 @@
    "width": "50px"
   },
   {
-   "description": "Number of columns for a field in a List View or a Grid (Total Columns should be less than 11)",
+   "description": "Number of columns for a field in a grid (total columns should be less than 11)",
    "fieldname": "columns",
    "fieldtype": "Int",
    "label": "Columns"
@@ -565,7 +566,7 @@
  "index_web_pages_for_search": 1,
  "istable": 1,
  "links": [],
- "modified": "2023-06-08 19:05:10.778371",
+ "modified": "2023-10-25 06:53:45.194081",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "DocField",

frappe/core/doctype/doctype/doctype.js

@@ -2,6 +2,12 @@
 // MIT License. See license.txt
 
 frappe.ui.form.on("DocType", {
+	onload: function (frm) {
+		if (frm.is_new()) {
+			frappe.listview_settings["DocType"].new_doctype_dialog();
+		}
+	},
+
 	before_save: function (frm) {
 		let form_builder = frappe.form_builder;
 		if (form_builder?.store) {
@@ -13,6 +19,7 @@ frappe.ui.form.on("DocType", {
 			}
 		}
 	},
+
 	after_save: function (frm) {
 		if (
 			frappe.form_builder &&
@@ -22,6 +29,7 @@ frappe.ui.form.on("DocType", {
 			frappe.form_builder.store.fetch();
 		}
 	},
+
 	refresh: function (frm) {
 		frm.set_query("role", "permissions", function (doc) {
 			if (doc.custom && frappe.session.user != "Administrator") {
@@ -119,6 +127,20 @@ frappe.ui.form.on("DocType", {
 	setup_default_views: (frm) => {
 		frappe.model.set_default_views_for_doctype(frm.doc.name, frm);
 	},
+
+	on_tab_change: (frm) => {
+		let current_tab = frm.get_active_tab().label;
+
+		if (current_tab === "Form") {
+			frm.footer.wrapper.hide();
+			frm.form_wrapper.find(".form-message").hide();
+			frm.form_wrapper.addClass("mb-1");
+		} else {
+			frm.footer.wrapper.show();
+			frm.form_wrapper.find(".form-message").show();
+			frm.form_wrapper.removeClass("mb-1");
+		}
+	},
 });
 
 frappe.ui.form.on("DocField", {
@@ -135,33 +157,6 @@ frappe.ui.form.on("DocField", {
 	},
 });
 
-function render_form_builder_message(frm) {
-	$(frm.fields_dict["try_form_builder_html"].wrapper).empty();
-	if (!frm.is_new() && frm.fields_dict["try_form_builder_html"]) {
-		let title = __("Use Form Builder to visually edit your form layout");
-		let msg = __(
-			"You can drag and drop fields to create your form layout, add tabs, sections and columns to organize your form and update field properties all from one screen."
-		);
-
-		let message = `
-		<div class="flex form-message blue p-3">
-			<div class="mr-3"><img style="border-radius: var(--border-radius-md)" width="360" src="/assets/frappe/images/form-builder.gif"></div>
-			<div>
-				<p style="font-size: var(--text-lg)">${title}</p>
-				<p>${msg}</p>
-				<div>
-					<a class="btn btn-primary btn-sm" href="/app/form-builder/${frm.doc.name}">
-						${__("Form Builder")} ${frappe.utils.icon("right", "xs")}
-					</a>
-				</div>
-			</div>
-		</div>
-		`;
-
-		$(frm.fields_dict["try_form_builder_html"].wrapper).html(message);
-	}
-}
-
 function render_form_builder(frm) {
 	if (frappe.form_builder && frappe.form_builder.doctype === frm.doc.name) {
 		frappe.form_builder.setup_page_actions();

frappe/core/doctype/doctype/doctype.json

@@ -8,6 +8,9 @@
  "document_type": "Document",
  "engine": "InnoDB",
  "field_order": [
+  "form_builder_tab",
+  "form_builder",
+  "settings_tab",
   "sb0",
   "module",
   "is_submittable",
@@ -32,32 +35,6 @@
   "column_break_15",
   "description",
   "documentation",
-  "sb2",
-  "permissions",
-  "restrict_to_domain",
-  "read_only",
-  "in_create",
-  "actions_section",
-  "actions",
-  "links_section",
-  "links",
-  "document_states_section",
-  "states",
-  "web_view",
-  "has_web_view",
-  "allow_guest_to_view",
-  "index_web_pages_for_search",
-  "route",
-  "is_published_field",
-  "website_search_field",
-  "advanced",
-  "engine",
-  "migration_hash",
-  "form_builder_tab",
-  "form_builder",
-  "fields_section",
-  "fields",
-  "settings_tab",
   "form_settings_section",
   "image_field",
   "timeline_field",
@@ -92,6 +69,29 @@
   "email_append_to",
   "sender_field",
   "subject_field",
+  "sb2",
+  "permissions",
+  "restrict_to_domain",
+  "read_only",
+  "in_create",
+  "actions_section",
+  "actions",
+  "links_section",
+  "links",
+  "document_states_section",
+  "states",
+  "web_view",
+  "has_web_view",
+  "allow_guest_to_view",
+  "index_web_pages_for_search",
+  "route",
+  "is_published_field",
+  "website_search_field",
+  "advanced",
+  "engine",
+  "migration_hash",
+  "fields_section",
+  "fields",
   "connections_tab"
  ],
  "fields": [
@@ -640,6 +640,7 @@
    "label": "Settings"
   },
   {
+   "depends_on": "eval:!doc.__islocal",
    "fieldname": "form_builder_tab",
    "fieldtype": "Tab Break",
    "label": "Form"
@@ -742,7 +743,7 @@
    "link_fieldname": "reference_doctype"
   }
  ],
- "modified": "2023-08-29 12:27:06.587523",
+ "modified": "2023-11-01 16:45:14.960949",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "DocType",

frappe/core/doctype/doctype/doctype.py

@@ -16,6 +16,7 @@ from frappe import _
 from frappe.cache_manager import clear_controller_cache, clear_user_cache
 from frappe.custom.doctype.custom_field.custom_field import create_custom_field
 from frappe.custom.doctype.property_setter.property_setter import make_property_setter
+from frappe.database import savepoint
 from frappe.database.schema import validate_column_length, validate_column_name
 from frappe.desk.notifications import delete_notification_count_for, get_filters_for
 from frappe.desk.utils import validate_route_conflict
@@ -349,8 +350,10 @@ class DocType(Document):
 
 		self.flags.update_fields_to_fetch_queries = []
 
-		if set(old_fields_to_fetch) != {df.fieldname for df in new_meta.get_fields_to_fetch()}:
-			for df in new_meta.get_fields_to_fetch():
+		new_fields_to_fetch = new_meta.get_fields_to_fetch()
+
+		if set(old_fields_to_fetch) != {df.fieldname for df in new_fields_to_fetch}:
+			for df in new_fields_to_fetch:
 				if df.fieldname not in old_fields_to_fetch:
 					link_fieldname, source_fieldname = df.fetch_from.split(".", 1)
 					link_df = new_meta.get_field(link_fieldname)
@@ -522,7 +525,9 @@ class DocType(Document):
 			if self.flags.in_insert:
 				self.run_module_method("after_doctype_insert")
 
+		self.sync_doctype_layouts()
 		delete_notification_count_for(doctype=self.name)
+
 		frappe.clear_cache(doctype=self.name)
 
 		# clear user cache so that on the next reload this doctype is included in boot
@@ -533,6 +538,17 @@ class DocType(Document):
 
 		clear_linked_doctype_cache()
 
+	@savepoint(catch=Exception)
+	def sync_doctype_layouts(self):
+		"""Sync Doctype Layout"""
+		doctype_layouts = frappe.get_all(
+			"DocType Layout", filters={"document_type": self.name}, pluck="name", ignore_ddl=True
+		)
+		for layout in doctype_layouts:
+			layout_doc = frappe.get_doc("DocType Layout", layout)
+			layout_doc.sync_fields()
+			layout_doc.save()
+
 	def setup_autoincrement_and_sequence(self):
 		"""Changes name type and makes sequence on change (if required)"""
 
@@ -854,6 +870,7 @@ class DocType(Document):
 						"read_only": 1,
 						"print_hide": 1,
 						"no_copy": 1,
+						"search_index": 1,
 					},
 				)
 

frappe/core/doctype/doctype/doctype_list.js

@@ -0,0 +1,122 @@
+frappe.listview_settings["DocType"] = {
+	primary_action: function () {
+		this.new_doctype_dialog();
+	},
+
+	new_doctype_dialog() {
+		let non_developer = frappe.session.user !== "Administrator" || !frappe.boot.developer_mode;
+		let fields = [
+			{
+				label: __("DocType Name"),
+				fieldname: "name",
+				fieldtype: "Data",
+				reqd: 1,
+			},
+			{ fieldtype: "Column Break" },
+			{
+				label: __("Module"),
+				fieldname: "module",
+				fieldtype: "Link",
+				options: "Module Def",
+				reqd: 1,
+			},
+			{ fieldtype: "Section Break" },
+			{
+				label: __("Is Submittable"),
+				fieldname: "is_submittable",
+				fieldtype: "Check",
+				description: __(
+					"Once submitted, submittable documents cannot be changed. They can only be Cancelled and Amended."
+				),
+				depends_on: "eval:!doc.istable && !doc.issingle",
+			},
+			{
+				label: __("Is Child Table"),
+				fieldname: "istable",
+				fieldtype: "Check",
+				description: __("Child Tables are shown as a Grid in other DocTypes"),
+				depends_on: "eval:!doc.is_submittable && !doc.issingle",
+			},
+			{
+				label: __("Editable Grid"),
+				fieldname: "editable_grid",
+				fieldtype: "Check",
+				depends_on: "istable",
+				default: 1,
+			},
+			{
+				label: __("Is Single"),
+				fieldname: "issingle",
+				fieldtype: "Check",
+				description: __(
+					"Single Types have only one record no tables associated. Values are stored in tabSingles"
+				),
+				depends_on: "eval:!doc.istable && !doc.is_submittable",
+			},
+			{
+				label: "Is Tree",
+				fieldname: "is_tree",
+				fieldtype: "Check",
+				default: "0",
+				depends_on: "eval:!doc.istable",
+				description: "Tree structures are implemented using Nested Set",
+			},
+			{
+				label: __("Custom?"),
+				fieldname: "custom",
+				fieldtype: "Check",
+				default: non_developer,
+				read_only: non_developer,
+			},
+		];
+
+		if (!non_developer) {
+			fields.push({
+				label: "Is Virtual",
+				fieldname: "is_virtual",
+				fieldtype: "Check",
+				default: "0",
+			});
+		}
+
+		let new_d = new frappe.ui.Dialog({
+			title: __("Create New DocType"),
+			fields: fields,
+			primary_action_label: __("Create & Continue"),
+			primary_action(values) {
+				if (!values.istable) values.editable_grid = 0;
+				frappe.db
+					.insert({
+						doctype: "DocType",
+						...values,
+						permissions: [
+							{
+								create: 1,
+								delete: 1,
+								email: 1,
+								export: 1,
+								print: 1,
+								read: 1,
+								report: 1,
+								role: "System Manager",
+								share: 1,
+								write: 1,
+							},
+						],
+						fields: [{ fieldtype: "Section Break" }],
+					})
+					.then((doc) => {
+						frappe.set_route("Form", "DocType", doc.name);
+					});
+			},
+			secondary_action_label: __("Cancel"),
+			secondary_action() {
+				new_d.hide();
+				if (frappe.get_route()[0] === "Form") {
+					frappe.set_route("List", "DocType");
+				}
+			},
+		});
+		new_d.show();
+	},
+};

frappe/core/doctype/document_naming_settings/document_naming_settings.py

@@ -248,8 +248,7 @@ class DocumentNamingSettings(Document):
 			doc = self._fetch_last_doc_if_available()
 			return "\n".join(NamingSeries(series).get_preview(doc=doc))
 		except Exception as e:
-			if frappe.message_log:
-				frappe.message_log.pop()
+			frappe.clear_last_message()
 			return _("Failed to generate names from the series") + f"\n{str(e)}"
 
 	def _fetch_last_doc_if_available(self):

frappe/core/doctype/file/file.js

@@ -40,14 +40,14 @@ frappe.ui.form.on("File", {
 			$preview = $(`<div class="img_preview">
 				<img
 					class="img-responsive"
-					src="${frm.doc.file_url}"
+					src="${frappe.utils.escape_html(frm.doc.file_url)}"
 					onerror="${frm.toggle_display("preview", false)}"
 				/>
 			</div>`);
 		} else if (frappe.utils.is_video_file(frm.doc.file_url)) {
 			$preview = $(`<div class="img_preview">
 				<video width="480" height="320" controls>
-					<source src="${frm.doc.file_url}">
+					<source src="${frappe.utils.escape_html(frm.doc.file_url)}">
 					${__("Your browser does not support the video element.")}
 				</video>
 			</div>`);
@@ -58,14 +58,14 @@ frappe.ui.form.on("File", {
 						style="background:#323639;"
 						width="100%"
 						height="1190"
-						src="${frm.doc.file_url}" type="application/pdf"
+						src="${frappe.utils.escape_html(frm.doc.file_url)}" type="application/pdf"
 					>
 				</object>
 			</div>`);
 		} else if (file_extension === "mp3") {
 			$preview = $(`<div class="img_preview">
 				<audio width="480" height="60" controls>
-					<source src="${frm.doc.file_url}" type="audio/mpeg">
+					<source src="${frappe.utils.escape_html(frm.doc.file_url)}" type="audio/mpeg">
 					${__("Your browser does not support the audio element.")}
 				</audio >
 			</div>`);

frappe/core/doctype/file/test_file.py

@@ -501,7 +501,7 @@ class TestFile(FrappeTestCase):
 		test_file.file_url = frappe.utils.get_url("unknown.jpg")
 		test_file.make_thumbnail(suffix="xs")
 		self.assertEqual(
-			json.loads(frappe.message_log[0]).get("message"),
+			frappe.message_log[0].get("message"),
 			f"File '{frappe.utils.get_url('unknown.jpg')}' not found",
 		)
 		self.assertEqual(test_file.thumbnail_url, None)

frappe/core/doctype/file/utils.py

@@ -13,7 +13,7 @@ from PIL import Image
 
 import frappe
 from frappe import _, safe_decode
-from frappe.utils import cstr, encode, get_files_path, random_string, strip
+from frappe.utils import cint, cstr, encode, get_files_path, random_string, strip
 from frappe.utils.file_manager import safe_b64decode
 from frappe.utils.image import optimize_image
 
@@ -339,6 +339,7 @@ def attach_files_to_document(doc: "Document", event) -> None:
 					"attached_to_name": doc.name,
 					"attached_to_doctype": doc.doctype,
 					"attached_to_field": df.fieldname,
+					"is_private": cint(value.startswith("/private")),
 				},
 			)
 			continue
@@ -357,6 +358,51 @@ def attach_files_to_document(doc: "Document", event) -> None:
 			doc.log_error("Error Attaching File")
 
 
+def relink_files(doc, fieldname, temp_doc_name):
+	if not temp_doc_name:
+		return
+	from frappe.utils.data import add_to_date, now_datetime
+
+	"""
+	Relink files attached to incorrect document name to the new document name
+	by check if file with temp name exists that was created in last 60 minutes
+	"""
+	mislinked_file = frappe.db.exists(
+		"File",
+		{
+			"file_url": doc.get(fieldname),
+			"attached_to_name": temp_doc_name,
+			"attached_to_doctype": doc.doctype,
+			"attached_to_field": fieldname,
+			"creation": (
+				"between",
+				[add_to_date(date=now_datetime(), minutes=-60), now_datetime()],
+			),
+		},
+	)
+	"""If file exists, attach it to the new docname"""
+	if mislinked_file:
+		frappe.db.set_value(
+			"File",
+			mislinked_file,
+			field={
+				"attached_to_name": doc.name,
+			},
+		)
+		return
+
+
+def relink_mismatched_files(doc: "Document") -> None:
+	if not doc.get("__temporary_name", None):
+		return
+	attach_fields = doc.meta.get("fields", {"fieldtype": ["in", ["Attach", "Attach Image"]]})
+	for df in attach_fields:
+		if doc.get(df.fieldname):
+			relink_files(doc, df.fieldname, doc.__temporary_name)
+	# delete temporary name after relinking is done
+	doc.delete_key("__temporary_name")
+
+
 def decode_file_content(content: bytes) -> bytes:
 	if isinstance(content, str):
 		content = content.encode("utf-8")

frappe/core/doctype/page/page.json

@@ -102,14 +102,16 @@
  "icon": "fa fa-file",
  "idx": 1,
  "links": [],
- "modified": "2022-08-03 12:20:54.219236",
+ "modified": "2023-10-22 22:41:25.568952",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "Page",
+ "naming_rule": "By fieldname",
  "owner": "Administrator",
  "permissions": [
   {
    "create": 1,
+   "delete": 1,
    "email": 1,
    "print": 1,
    "read": 1,

frappe/core/doctype/page/page.py

@@ -2,9 +2,10 @@
 # License: MIT. See LICENSE
 
 import os
+import shutil
 
 import frappe
-from frappe import _, conf, safe_decode
+from frappe import _, conf, get_module_path, safe_decode
 from frappe.build import html_to_js_template
 from frappe.core.doctype.custom_role.custom_role import get_custom_allowed_roles
 from frappe.desk.form.meta import get_code_files_via_hooks, get_js
@@ -103,7 +104,18 @@ class Page(Document):
 		return d
 
 	def on_trash(self):
+		if not frappe.conf.developer_mode and not frappe.flags.in_migrate:
+			frappe.throw(_("Deletion of this document is only permitted in developer mode."))
+
 		delete_custom_role("page", self.name)
+		frappe.db.after_commit(self.delete_folder_with_contents)
+
+	def delete_folder_with_contents(self):
+		module_path = get_module_path(self.module)
+		dir_path = os.path.join(module_path, "page", frappe.scrub(self.name))
+
+		if os.path.exists(dir_path):
+			shutil.rmtree(dir_path, ignore_errors=True)
 
 	def is_permitted(self):
 		"""Returns true if Has Role is not set or the user is allowed."""
@@ -173,11 +185,6 @@ class Page(Document):
 					# flag for not caching this page
 					self._dynamic_page = True
 
-		if frappe.lang != "en":
-			from frappe.translate import get_lang_js
-
-			self.script += get_lang_js("page", self.name)
-
 		for path in get_code_files_via_hooks("page_js", self.name):
 			js = get_js(path)
 			if js:

frappe/core/doctype/page/test_page.py

@@ -1,5 +1,9 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
+import os
+import unittest
+from unittest.mock import patch
+
 import frappe
 from frappe.tests.utils import FrappeTestCase
 
@@ -16,3 +20,18 @@ class TestPage(FrappeTestCase):
 			frappe.NameError,
 			frappe.get_doc(dict(doctype="Page", page_name="Settings", module="Core")).insert,
 		)
+
+	@unittest.skipUnless(
+		os.access(frappe.get_app_path("frappe"), os.W_OK), "Only run if frappe app paths is writable"
+	)
+	@patch.dict(frappe.conf, {"developer_mode": 1})
+	def test_trashing(self):
+		page = frappe.new_doc("Page", page_name=frappe.generate_hash(), module="Core").insert()
+
+		page.delete()
+		frappe.db.commit()
+
+		module_path = frappe.get_module_path(page.module)
+		dir_path = os.path.join(module_path, "page", frappe.scrub(page.name))
+
+		self.assertFalse(os.path.exists(dir_path))

frappe/core/doctype/prepared_report/prepared_report.py

@@ -1,7 +1,6 @@
 # Copyright (c) 2018, Frappe Technologies and contributors
 # License: MIT. See LICENSE
-
-
+import gzip
 import json
 from contextlib import suppress
 from typing import Any
@@ -13,7 +12,7 @@ from frappe.desk.form.load import get_attachments
 from frappe.desk.query_report import generate_report_result
 from frappe.model.document import Document
 from frappe.monitor import add_data_to_monitor
-from frappe.utils import add_to_date, gzip_compress, gzip_decompress, now
+from frappe.utils import add_to_date, now
 from frappe.utils.background_jobs import enqueue
 
 # If prepared report runs for longer than this time it's automatically considered as failed
@@ -84,8 +83,8 @@ class PreparedReport(Document):
 			attached_file = frappe.get_doc("File", attachment.name)
 
 			if with_file_name:
-				return (gzip_decompress(attached_file.get_content()), attachment.file_name)
-			return gzip_decompress(attached_file.get_content())
+				return (gzip.decompress(attached_file.get_content()), attachment.file_name)
+			return gzip.decompress(attached_file.get_content())
 
 
 def generate_report(prepared_report):
@@ -222,7 +221,7 @@ def create_json_gz_file(data, dt, dn):
 		frappe.utils.data.format_datetime(frappe.utils.now(), "Y-m-d-H:M")
 	)
 	encoded_content = frappe.safe_encode(frappe.as_json(data))
-	compressed_content = gzip_compress(encoded_content)
+	compressed_content = gzip.compress(encoded_content)
 
 	# Call save() file function to upload and attach the file
 	_file = frappe.get_doc(

frappe/core/doctype/recorder/recorder.js

@@ -2,6 +2,9 @@
 // For license information, please see license.txt
 
 frappe.ui.form.on("Recorder", {
+	onload: function (frm) {
+		frm.fields_dict.sql_queries.grid.only_sortable();
+	},
 	refresh: function (frm) {
 		frm.disable_save();
 		frm._sort_order = {};

frappe/core/doctype/rq_job/rq_job.py

@@ -87,7 +87,9 @@ class RQJob(Document):
 		matched_job_ids = RQJob.get_matching_job_ids(args)[start : start + page_length]
 
 		conn = get_redis_conn()
-		jobs = [serialize_job(job) for job in Job.fetch_many(job_ids=matched_job_ids, connection=conn)]
+		jobs = [
+			serialize_job(job) for job in Job.fetch_many(job_ids=matched_job_ids, connection=conn) if job
+		]
 
 		return sorted(jobs, key=lambda j: j.modified, reverse=order_desc)
 

frappe/core/doctype/scheduled_job_type/scheduled_job_type.json

@@ -53,6 +53,7 @@
   {
    "allow_in_quick_entry": 1,
    "depends_on": "eval:doc.frequency==='Cron'",
+   "description": "<pre>*  *  *  *  *\n\u252c  \u252c  \u252c  \u252c  \u252c\n\u2502  \u2502  \u2502  \u2502  \u2502\n\u2502  \u2502  \u2502  \u2502  \u2514 day of week (0 - 6) (0 is Sunday)\n\u2502  \u2502  \u2502  \u2514\u2500\u2500\u2500\u2500\u2500 month (1 - 12)\n\u2502  \u2502  \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500 day of month (1 - 31)\n\u2502  \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500 hour (0 - 23)\n\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500 minute (0 - 59)\n\n---\n\n* - Any value\n/ - Step values\n</pre>\n",
    "fieldname": "cron_format",
    "fieldtype": "Data",
    "label": "Cron Format",
@@ -100,7 +101,7 @@
    "link_fieldname": "scheduled_job_type"
   }
  ],
- "modified": "2022-06-28 02:55:12.470915",
+ "modified": "2023-10-14 11:26:05.005930",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "Scheduled Job Type",

frappe/core/doctype/scheduled_job_type/scheduled_job_type.py

@@ -105,9 +105,12 @@ class ScheduledJobType(Document):
 		if not self.cron_format:
 			self.cron_format = CRON_MAP[self.frequency]
 
-		return croniter(
-			self.cron_format, get_datetime(self.last_execution or datetime(2000, 1, 1))
-		).get_next(datetime)
+		# If this is a cold start then last_execution will not be set.
+		# Creation is set as fallback because if very old fallback is set job might trigger
+		# immediately, even when it's meant to be daily.
+		# A dynamic fallback like current time might miss the scheduler interval and job will never start.
+		last_execution = get_datetime(self.last_execution or self.creation)
+		return croniter(self.cron_format, last_execution).get_next(datetime)
 
 	def execute(self):
 		self.scheduler_log = None

frappe/core/doctype/scheduled_job_type/test_scheduled_job_type.py

@@ -1,9 +1,12 @@
 # Copyright (c) 2019, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
+from datetime import timedelta
+
 import frappe
 from frappe.core.doctype.scheduled_job_type.scheduled_job_type import sync_jobs
 from frappe.tests.utils import FrappeTestCase
 from frappe.utils import get_datetime
+from frappe.utils.data import add_to_date, now_datetime
 
 
 class TestScheduledJobType(FrappeTestCase):
@@ -68,6 +71,22 @@ class TestScheduledJobType(FrappeTestCase):
 		# runs every 15 mins
 		job = frappe.get_doc("Scheduled Job Type", dict(method="frappe.oauth.delete_oauth2_data"))
 		job.db_set("last_execution", "2019-01-01 00:00:00")
+		self.assertEqual(job.next_execution, get_datetime("2019-01-01 00:15:00"))
 		self.assertTrue(job.is_event_due(get_datetime("2019-01-01 00:15:01")))
 		self.assertFalse(job.is_event_due(get_datetime("2019-01-01 00:05:06")))
 		self.assertFalse(job.is_event_due(get_datetime("2019-01-01 00:14:59")))
+
+	def test_cold_start(self):
+		now = now_datetime()
+		just_before_12_am = now.replace(hour=11, minute=59, second=30)
+		just_after_12_am = now.replace(hour=0, minute=0, second=30) + timedelta(days=1)
+
+		job = frappe.new_doc("Scheduled Job Type")
+		job.frequency = "Daily"
+		job.set_user_and_timestamp()
+
+		with self.freeze_time(just_before_12_am):
+			self.assertFalse(job.is_event_due())
+
+		with self.freeze_time(just_after_12_am):
+			self.assertTrue(job.is_event_due())

frappe/core/doctype/server_script/server_script.json

@@ -136,6 +136,7 @@
   },
   {
    "depends_on": "eval:doc.event_frequency==='Cron'",
+   "description": "<pre>*  *  *  *  *\n\u252c  \u252c  \u252c  \u252c  \u252c\n\u2502  \u2502  \u2502  \u2502  \u2502\n\u2502  \u2502  \u2502  \u2502  \u2514 day of week (0 - 6) (0 is Sunday)\n\u2502  \u2502  \u2502  \u2514\u2500\u2500\u2500\u2500\u2500 month (1 - 12)\n\u2502  \u2502  \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500 day of month (1 - 31)\n\u2502  \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500 hour (0 - 23)\n\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500 minute (0 - 59)\n\n---\n\n* - Any value\n/ - Step values\n</pre>\n",
    "fieldname": "cron_format",
    "fieldtype": "Data",
    "label": "Cron Format"
@@ -148,7 +149,7 @@
    "link_fieldname": "server_script"
   }
  ],
- "modified": "2023-05-27 16:33:16.595424",
+ "modified": "2023-10-14 11:24:46.478533",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "Server Script",

frappe/core/doctype/server_script/server_script.py

@@ -75,6 +75,7 @@ class ServerScript(Document):
 		return super().clear_cache()
 
 	def on_trash(self):
+		frappe.cache.delete_value("server_script_map")
 		if self.script_type == "Scheduler Event":
 			for job in self.scheduled_jobs:
 				frappe.delete_doc("Scheduled Job Type", job.name)

frappe/core/doctype/sms_settings/sms_settings.py

@@ -30,7 +30,7 @@ def validate_receiver_nos(receiver_list):
 	validated_receiver_list = []
 	for d in receiver_list:
 		if not d:
-			break
+			continue
 
 		# remove invalid character
 		for x in [" ", "-", "(", ")"]:

frappe/core/doctype/system_settings/system_settings.json

@@ -217,7 +217,7 @@
    "label": "Security"
   },
   {
-   "default": "60:00",
+   "default": "170:00",
    "description": "Example: Setting this to 24:00 will log out a user if they are not active for 24:00 hours.",
    "fieldname": "session_expiry",
    "fieldtype": "Data",
@@ -292,6 +292,7 @@
    "label": "Brute Force Security"
   },
   {
+   "default": "10",
    "fieldname": "allow_consecutive_login_attempts",
    "fieldtype": "Int",
    "label": "Allow Consecutive Login Attempts "
@@ -602,7 +603,7 @@
  "icon": "fa fa-cog",
  "issingle": 1,
  "links": [],
- "modified": "2023-09-25 16:49:16.652874",
+ "modified": "2023-10-17 16:12:28.145496",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "System Settings",

frappe/core/doctype/user/test_user.py

@@ -420,7 +420,7 @@ class TestUser(FrappeTestCase):
 			self.assertEqual(update_password(new_password, key=test_user.reset_password_key), "/")
 			update_password(old_password, old_password=new_password)
 			self.assertEqual(
-				json.loads(frappe.message_log[0]).get("message"),
+				frappe.message_log[0].get("message"),
 				"Password reset instructions have been sent to your email",
 			)
 

frappe/core/doctype/user/user.js

@@ -123,7 +123,8 @@ frappe.ui.form.on("User", {
 			!doc.__unsaved &&
 			frappe.all_timezones &&
 			(hasChanged(doc.language, frappe.boot.user.language) ||
-				hasChanged(doc.time_zone, frappe.boot.time_zone.user))
+				hasChanged(doc.time_zone, frappe.boot.time_zone.user) ||
+				hasChanged(doc.desk_theme, frappe.boot.user.desk_theme))
 		) {
 			frappe.msgprint(__("Refreshing..."));
 			window.location.reload();

frappe/core/doctype/user/user.py

@@ -248,7 +248,7 @@ class User(Document):
 			frappe.local.login_manager.logout(user=self.name)
 
 		# toggle notifications based on the user's status
-		toggle_notifications(self.name, enable=cint(self.enabled))
+		toggle_notifications(self.name, enable=cint(self.enabled), ignore_permissions=True)
 
 	def add_system_manager_role(self):
 		if self.is_system_manager_disabled():
@@ -549,6 +549,10 @@ class User(Document):
 		# delete user permissions
 		frappe.db.delete("User Permission", {"user": self.name})
 
+		# Delete OAuth data
+		frappe.db.delete("OAuth Authorization Code", {"user": self.name})
+		frappe.db.delete("Token Cache", {"user": self.name})
+
 	def before_rename(self, old_name, new_name, merge=False):
 		frappe.clear_cache(user=old_name)
 		self.validate_rename(old_name, new_name)
@@ -775,7 +779,7 @@ def get_timezones():
 
 
 @frappe.whitelist()
-def get_all_roles(arg=None):
+def get_all_roles():
 	"""return all roles"""
 	active_domains = frappe.get_active_domains()
 
@@ -789,7 +793,7 @@ def get_all_roles(arg=None):
 		order_by="name",
 	)
 
-	return [role.get("name") for role in roles]
+	return sorted([role.get("name") for role in roles])
 
 
 @frappe.whitelist()

frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.py

@@ -59,6 +59,6 @@ def query_doctypes(doctype, txt, searchfield, start, page_len, filters):
 	return [
 		[dt]
 		for dt in can_read
-		if txt.lower().replace("%", "") in dt.lower()
+		if txt.lower().replace("%", "") in frappe._(dt).lower()
 		and (include_single_doctypes or dt not in single_doctypes)
 	]

frappe/custom/doctype/custom_field/custom_field.json

@@ -153,10 +153,10 @@
   },
   {
    "default": "0",
-   "description": "If checked, this field will be not overwritten based on Fetch From if a value already exists.",
+   "description": "If unchecked, the value will always be re-fetched on save.",
    "fieldname": "fetch_if_empty",
    "fieldtype": "Check",
-   "label": "Fetch If Empty"
+   "label": "Fetch on Save if Empty"
   },
   {
    "fieldname": "options_help",
@@ -450,7 +450,7 @@
  "idx": 1,
  "index_web_pages_for_search": 1,
  "links": [],
- "modified": "2023-06-08 19:05:51.737234",
+ "modified": "2023-10-25 06:55:10.713382",
  "modified_by": "Administrator",
  "module": "Custom",
  "name": "Custom Field",

frappe/custom/doctype/customize_form/customize_form.json

@@ -30,10 +30,10 @@
   "links",
   "document_states_section",
   "states",
-  "form_tab",
-  "form_builder",
   "fields_section_break",
   "fields",
+  "form_tab",
+  "form_builder",
   "settings_tab",
   "form_settings_section",
   "image_field",
@@ -393,7 +393,7 @@
  "index_web_pages_for_search": 1,
  "issingle": 1,
  "links": [],
- "modified": "2023-08-29 12:31:55.808848",
+ "modified": "2023-11-07 19:25:32.656641",
  "modified_by": "Administrator",
  "module": "Custom",
  "name": "Customize Form",

frappe/custom/doctype/customize_form_field/customize_form_field.json

@@ -193,10 +193,10 @@
   },
   {
    "default": "0",
-   "description": "If checked, this field will be not overwritten based on Fetch From if a value already exists.",
+   "description": "If unchecked, the value will always be re-fetched on save.",
    "fieldname": "fetch_if_empty",
    "fieldtype": "Check",
-   "label": "Fetch If Empty"
+   "label": "Fetch on Save if Empty"
   },
   {
    "fieldname": "permissions",
@@ -477,7 +477,7 @@
  "index_web_pages_for_search": 1,
  "istable": 1,
  "links": [],
- "modified": "2023-06-08 19:05:37.767838",
+ "modified": "2023-10-25 06:55:50.718441",
  "modified_by": "Administrator",
  "module": "Custom",
  "name": "Customize Form Field",

frappe/custom/doctype/doctype_layout/doctype_layout.py

@@ -32,7 +32,7 @@ class DocTypeLayout(Document):
 
 	@frappe.whitelist()
 	def sync_fields(self):
-		doctype_fields = frappe.get_meta(self.document_type).fields
+		doctype_fields = frappe.get_meta(self.document_type, cached=False).fields
 
 		if self.is_new():
 			added_fields = [field.fieldname for field in doctype_fields]

frappe/database/db_manager.py

@@ -57,6 +57,7 @@ class DbManager:
 
 		esc = make_esc("$ ")
 		pv = which("pv")
+		mariadb_cli = which("mariadb") or which("mysql")
 
 		if pv:
 			pipe = f"{pv} {source} |"
@@ -68,7 +69,7 @@ class DbManager:
 		if pipe:
 			print("Restoring Database file...")
 
-		command = "{pipe} mysql -u {user} -p{password} -h{host} -P{port} {target} {source}"
+		command = "{pipe} {mariadb_cli} -u {user} -p{password} -h{host} -P{port} {target} {source}"
 		command = command.format(
 			pipe=pipe,
 			user=esc(user),
@@ -77,6 +78,7 @@ class DbManager:
 			target=esc(target),
 			source=source,
 			port=frappe.conf.db_port,
+			mariadb_cli=mariadb_cli,
 		)
 
 		os.system(command)

frappe/database/postgres/database.py

@@ -160,11 +160,16 @@ class PostgresDatabase(PostgresExceptionUtil, Database):
 		return LazyDecode(self._cursor.query)
 
 	def get_connection(self):
-		conn = psycopg2.connect(
-			"host='{}' dbname='{}' user='{}' password='{}' port={}".format(
-				self.host, self.user, self.user, self.password, self.port
-			)
-		)
+		conn_settings = {
+			"user": self.user,
+			"dbname": self.user,
+			"host": self.host,
+			"password": self.password,
+		}
+		if self.port:
+			conn_settings["port"] = self.port
+
+		conn = psycopg2.connect(**conn_settings)
 		conn.set_isolation_level(ISOLATION_LEVEL_REPEATABLE_READ)
 
 		return conn

frappe/desk/desktop.py

@@ -22,8 +22,7 @@ def handle_not_exist(fn):
 		try:
 			return fn(*args, **kwargs)
 		except DoesNotExistError:
-			if frappe.message_log:
-				frappe.message_log.pop()
+			frappe.clear_last_message()
 			return []
 
 	return wrapper

frappe/desk/doctype/dashboard_chart_source/dashboard_chart_source.js

@@ -1,4 +1,19 @@
 // Copyright (c) 2019, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on("Dashboard Chart Source", {});
+frappe.ui.form.on("Dashboard Chart Source", {
+	refresh: function (frm) {
+		if (!frm.is_new()) {
+			frm.add_custom_button(
+				__("Dashboard Chart"),
+				function () {
+					let dashboard_chart = frappe.model.get_new_doc("Dashboard Chart");
+					dashboard_chart.chart_type = "Custom";
+					dashboard_chart.source = frm.doc.name;
+					frappe.set_route("Form", "Dashboard Chart", dashboard_chart.name);
+				},
+				__("Create")
+			);
+		}
+	},
+});

frappe/desk/doctype/desktop_icon/desktop_icon.py

@@ -283,8 +283,7 @@ def set_desktop_icons(visible_list, ignore_duplicate=True):
 						raise e
 					else:
 						visible_list.remove(module_name)
-						if frappe.message_log:
-							frappe.message_log.pop()
+						frappe.clear_last_message()
 
 	# set the order
 	set_order(visible_list)

frappe/desk/doctype/note/note.py

@@ -54,3 +54,7 @@ def get_permission_query_conditions(user):
 		user = frappe.session.user
 
 	return f"(`tabNote`.owner = {frappe.db.escape(user)} or `tabNote`.public = 1)"
+
+
+def has_permission(doc, user):
+	return doc.public or doc.owner == user

frappe/desk/doctype/notification_settings/notification_settings.py

@@ -70,7 +70,7 @@ def create_notification_settings(user):
 		_doc.insert(ignore_permissions=True)
 
 
-def toggle_notifications(user: str, enable: bool = False):
+def toggle_notifications(user: str, enable: bool = False, ignore_permissions=False):
 	try:
 		settings = frappe.get_doc("Notification Settings", user)
 	except frappe.DoesNotExistError:
@@ -79,7 +79,7 @@ def toggle_notifications(user: str, enable: bool = False):
 
 	if settings.enabled != enable:
 		settings.enabled = enable
-		settings.save()
+		settings.save(ignore_permissions=ignore_permissions)
 
 
 @frappe.whitelist()

frappe/desk/doctype/tag/tag.py

@@ -48,8 +48,6 @@ def add_tags(tags, dt, docs, color=None):
 		for tag in tags:
 			DocTags(dt).add(doc, tag)
 
-	# return tag
-
 
 @frappe.whitelist()
 def remove_tag(tag, dt, dn):
@@ -153,6 +151,7 @@ def update_tags(doc, tags):
 
 	:param doc: Document to be added to global tags
 	"""
+	doc.check_permission("write")
 	new_tags = {tag.strip() for tag in tags.split(",") if tag}
 	existing_tags = [
 		tag.tag

frappe/desk/doctype/todo/todo.json

@@ -62,8 +62,11 @@
    "label": "Color"
   },
   {
+   "allow_in_quick_entry": 1,
+   "default": "Today",
    "fieldname": "date",
    "fieldtype": "Date",
+   "in_list_view": 1,
    "in_standard_filter": 1,
    "label": "Due Date",
    "oldfieldname": "date",
@@ -158,7 +161,7 @@
  "icon": "fa fa-check",
  "idx": 2,
  "links": [],
- "modified": "2021-09-16 11:36:34.586898",
+ "modified": "2023-10-05 07:44:38.476400",
  "modified_by": "Administrator",
  "module": "Desk",
  "name": "ToDo",
@@ -196,4 +199,4 @@
  "title_field": "description",
  "track_changes": 1,
  "track_seen": 1
-}
+}

frappe/desk/doctype/workspace/workspace.py

@@ -242,6 +242,12 @@ def new_page(new_page):
 
 	if page.get("public") and not is_workspace_manager():
 		return
+	elif (
+		not page.get("public")
+		and page.get("for_user") != frappe.session.user
+		and not is_workspace_manager()
+	):
+		frappe.throw(_("Cannot create private workspace of other users"), frappe.PermissionError)
 
 	doc = frappe.new_doc("Workspace")
 	doc.title = page.get("title")
@@ -283,6 +289,16 @@ def update_page(name, title, icon, indicator_color, parent, public):
 	public = frappe.parse_json(public)
 	doc = frappe.get_doc("Workspace", name)
 
+	if (
+		not doc.get("public")
+		and doc.get("for_user") != frappe.session.user
+		and not is_workspace_manager()
+	):
+		frappe.throw(
+			_("Need Workspace Manager role to edit private workspace of other users"),
+			frappe.PermissionError,
+		)
+
 	if doc:
 		doc.title = title
 		doc.icon = icon
@@ -328,7 +344,11 @@ def hide_unhide_page(page_name: str, is_hidden: bool):
 			_("Need Workspace Manager role to hide/unhide public workspaces"), frappe.PermissionError
 		)
 
-	if not page.get("public") and page.get("for_user") != frappe.session.user:
+	if (
+		not page.get("public")
+		and page.get("for_user") != frappe.session.user
+		and not is_workspace_manager()
+	):
 		frappe.throw(_("Cannot update private workspace of other users"), frappe.PermissionError)
 
 	page.is_hidden = int(is_hidden)
@@ -387,7 +407,17 @@ def delete_page(page):
 	page = loads(page)
 
 	if page.get("public") and not is_workspace_manager():
-		return
+		frappe.throw(
+			_("Cannot delete public workspace without Workspace Manager role"),
+			frappe.PermissionError,
+		)
+	elif not page.get("public") and not is_workspace_manager():
+		workspace_owner = frappe.get_value("Workspace", page.get("name"), "for_user")
+		if workspace_owner != frappe.session.user:
+			frappe.throw(
+				_("Cannot delete private workspace of other users"),
+				frappe.PermissionError,
+			)
 
 	if frappe.db.exists("Workspace", page.get("name")):
 		frappe.get_doc("Workspace", page.get("name")).delete(ignore_permissions=True)

frappe/desk/form/assign_to.py

@@ -39,7 +39,7 @@ def get(args=None):
 
 
 @frappe.whitelist()
-def add(args=None):
+def add(args=None, *, ignore_permissions=False):
 	"""add in someone's to do list
 	args = {
 	        "assign_to": [],
@@ -63,6 +63,8 @@ def add(args=None):
 			"status": "Open",
 			"allocated_to": assign_to,
 		}
+		if not ignore_permissions:
+			frappe.get_doc(args["doctype"], args["name"]).check_permission()
 
 		if frappe.get_all("ToDo", filters=filters):
 			users_with_duplicate_todo.append(assign_to)
@@ -144,7 +146,7 @@ def add_multiple(args=None):
 		add(args)
 
 
-def close_all_assignments(doctype, name):
+def close_all_assignments(doctype, name, ignore_permissions=False):
 	assignments = frappe.get_all(
 		"ToDo",
 		fields=["allocated_to", "name"],
@@ -154,26 +156,42 @@ def close_all_assignments(doctype, name):
 		return False
 
 	for assign_to in assignments:
-		set_status(doctype, name, todo=assign_to.name, assign_to=assign_to.allocated_to, status="Closed")
+		set_status(
+			doctype,
+			name,
+			todo=assign_to.name,
+			assign_to=assign_to.allocated_to,
+			status="Closed",
+			ignore_permissions=ignore_permissions,
+		)
 
 	return True
 
 
 @frappe.whitelist()
-def remove(doctype, name, assign_to):
-	return set_status(doctype, name, "", assign_to, status="Cancelled")
+def remove(doctype, name, assign_to, ignore_permissions=False):
+	return set_status(
+		doctype, name, "", assign_to, status="Cancelled", ignore_permissions=ignore_permissions
+	)
 
 
 @frappe.whitelist()
-def close(doctype: str, name: str, assign_to: str):
+def close(doctype: str, name: str, assign_to: str, ignore_permissions=False):
 	if assign_to != frappe.session.user:
 		frappe.throw(_("Only the assignee can complete this to-do."))
 
-	return set_status(doctype, name, "", assign_to, status="Closed")
+	return set_status(
+		doctype, name, "", assign_to, status="Closed", ignore_permissions=ignore_permissions
+	)
 
 
-def set_status(doctype, name, todo=None, assign_to=None, status="Cancelled"):
+def set_status(
+	doctype, name, todo=None, assign_to=None, status="Cancelled", ignore_permissions=False
+):
 	"""remove from todo"""
+
+	if not ignore_permissions:
+		frappe.get_doc(doctype, name).check_permission()
 	try:
 		if not todo:
 			todo = frappe.db.get_value(
@@ -201,7 +219,7 @@ def set_status(doctype, name, todo=None, assign_to=None, status="Cancelled"):
 	return get({"doctype": doctype, "name": name})
 
 
-def clear(doctype, name):
+def clear(doctype, name, ignore_permissions=False):
 	"""
 	Clears assignments, return False if not assigned.
 	"""
@@ -215,7 +233,12 @@ def clear(doctype, name):
 
 	for assign_to in assignments:
 		set_status(
-			doctype, name, todo=assign_to.name, assign_to=assign_to.allocated_to, status="Cancelled"
+			doctype,
+			name,
+			todo=assign_to.name,
+			assign_to=assign_to.allocated_to,
+			status="Cancelled",
+			ignore_permissions=ignore_permissions,
 		)
 
 	return True

frappe/desk/form/linked_with.py

@@ -38,6 +38,7 @@ def get_submitted_linked_docs(doctype: str, name: str) -> list[tuple]:
 	3. Searching for links is going to be a tree like structure where at every level,
 	        you will be finding documents using parent document and parent document links.
 	"""
+	frappe.has_permission(doctype, doc=name)
 	tree = SubmittableDocumentTree(doctype, name)
 	visited_documents = tree.get_all_children()
 	docs = []
@@ -70,7 +71,7 @@ class SubmittableDocumentTree:
 
 	def get_all_children(self):
 		"""Get all nodes of a tree except the root node (all the nested submitted
-		documents those are present in referencing tables (dependent tables).
+		documents those are present in referencing tables dependent tables).
 		"""
 		while self.to_be_visited_documents:
 			next_level_children = defaultdict(list)
@@ -100,6 +101,10 @@ class SubmittableDocumentTree:
 
 		child_docs = defaultdict(list)
 		for field in referencing_fields:
+			if field["fieldname"] == "amended_from":
+				# perf: amended_from links are always linked to cancelled documents.
+				continue
+
 			links = (
 				get_referencing_documents(
 					parent_dt,
@@ -427,8 +432,7 @@ def get_linked_docs(doctype: str, name: str, linkinfo: dict | None = None) -> di
 			link_meta_bundle = frappe.desk.form.load.get_meta_bundle(dt)
 		except Exception as e:
 			if isinstance(e, frappe.DoesNotExistError):
-				if frappe.local.message_log:
-					frappe.local.message_log.pop()
+				frappe.clear_last_message()
 			continue
 		linkmeta = link_meta_bundle[0]
 
@@ -502,8 +506,7 @@ def get_linked_docs(doctype: str, name: str, linkinfo: dict | None = None) -> di
 						ret = None
 
 			except frappe.PermissionError:
-				if frappe.local.message_log:
-					frappe.local.message_log.pop()
+				frappe.clear_last_message()
 
 				continue
 
@@ -515,6 +518,7 @@ def get_linked_docs(doctype: str, name: str, linkinfo: dict | None = None) -> di
 
 @frappe.whitelist()
 def get(doctype, docname):
+	frappe.has_permission(doctype, doc=docname)
 	linked_doctypes = get_linked_doctypes(doctype=doctype)
 	return get_linked_docs(doctype=doctype, name=docname, linkinfo=linked_doctypes)
 

frappe/desk/form/meta.py

@@ -1,6 +1,5 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
-import io
 import os
 
 import frappe
@@ -45,9 +44,6 @@ def get_meta(doctype, cached=True) -> "FormMeta":
 	else:
 		meta = FormMeta(doctype)
 
-	if frappe.local.lang != "en":
-		meta.set_translations(frappe.local.lang)
-
 	return meta
 
 
@@ -256,18 +252,6 @@ class FormMeta(Meta):
 
 				self.set("__form_grid_templates", templates)
 
-	def set_translations(self, lang):
-		from frappe.translate import extract_messages_from_code, make_dict_from_messages
-
-		self.set("__messages", frappe.get_lang_dict("doctype", self.name))
-
-		# set translations for grid templates
-		if self.get("__form_grid_templates"):
-			for content in self.get("__form_grid_templates").values():
-				messages = extract_messages_from_code(content)
-				messages = make_dict_from_messages(messages)
-				self.get("__messages").update(messages)
-
 	def load_dashboard(self):
 		self.set("__dashboard", self.get_dashboard_data())
 

frappe/desk/form/save.py

@@ -17,6 +17,9 @@ def savedocs(doc, action):
 	"""save / submit / update doclist"""
 	doc = frappe.get_doc(json.loads(doc))
 	capture_doc(doc, action)
+	if doc.get("__islocal") and doc.name.startswith("new-" + doc.doctype.lower().replace(" ", "-")):
+		# required to relink missing attachments if they exist.
+		doc.__temporary_name = doc.name
 	set_local_name(doc)
 
 	# action

frappe/desk/page/setup_wizard/setup_wizard.js

@@ -404,7 +404,7 @@ frappe.setup.slides_settings = [
 				fieldname: "enable_telemetry",
 				label: __("Allow sending usage data for improving applications"),
 				fieldtype: "Check",
-				default: 1,
+				default: cint(frappe.telemetry.can_enable()),
 				depends_on: "eval:frappe.telemetry.can_enable()",
 			},
 			{

frappe/desk/query_report.py

@@ -573,7 +573,9 @@ def get_filtered_data(ref_doctype, columns, data, user):
 	if match_filters_per_doctype:
 		for row in data:
 			# Why linked_doctypes.get(ref_doctype)? because if column is empty, linked_doctypes[ref_doctype] is removed
-			if linked_doctypes.get(ref_doctype) and shared and row[linked_doctypes[ref_doctype]] in shared:
+			if (
+				linked_doctypes.get(ref_doctype) and shared and row.get(linked_doctypes[ref_doctype]) in shared
+			):
 				result.append(row)
 
 			elif has_match(

frappe/desk/reportview.py

@@ -46,7 +46,7 @@ def get_list():
 
 @frappe.whitelist()
 @frappe.read_only()
-def get_count():
+def get_count() -> int:
 	args = get_form_params()
 
 	if is_virtual_doctype(args.doctype):
@@ -65,7 +65,7 @@ def execute(doctype, *args, **kwargs):
 
 
 def get_form_params():
-	"""Stringify GET request parameters."""
+	"""parse GET request parameters."""
 	data = frappe._dict(frappe.local.form_dict)
 	clean_params(data)
 	validate_args(data)
@@ -215,12 +215,12 @@ def clean_params(data):
 
 
 def parse_json(data):
-	if isinstance(data.get("filters"), str):
-		data["filters"] = json.loads(data["filters"])
-	if isinstance(data.get("or_filters"), str):
-		data["or_filters"] = json.loads(data["or_filters"])
-	if isinstance(data.get("fields"), str):
-		data["fields"] = ["*"] if data["fields"] == "*" else json.loads(data["fields"])
+	if (filters := data.get("filters")) and isinstance(filters, str):
+		data["filters"] = json.loads(filters)
+	if (or_filters := data.get("or_filters")) and isinstance(or_filters, str):
+		data["or_filters"] = json.loads(or_filters)
+	if (fields := data.get("fields")) and isinstance(fields, str):
+		data["fields"] = ["*"] if fields == "*" else json.loads(fields)
 	if isinstance(data.get("docstatus"), str):
 		data["docstatus"] = json.loads(data["docstatus"])
 	if isinstance(data.get("save_user_settings"), str):

frappe/desk/search.py

@@ -1,20 +1,24 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
 
-import functools
 import json
 import re
+from typing import TypedDict
+
+from typing_extensions import NotRequired  # not required in 3.11+
 
 import frappe
 
 # Backward compatbility
 from frappe import _, is_whitelisted, validate_and_sanitize_search_inputs
 from frappe.database.schema import SPECIAL_CHAR_PATTERN
+from frappe.model.db_query import get_order_by
 from frappe.permissions import has_permission
 from frappe.utils import cint, cstr, unique
+from frappe.utils.data import make_filter_tuple
 
 
-def sanitize_searchfield(searchfield):
+def sanitize_searchfield(searchfield: str):
 	if not searchfield:
 		return
 
@@ -22,19 +26,25 @@ def sanitize_searchfield(searchfield):
 		frappe.throw(_("Invalid Search Field {0}").format(searchfield), frappe.DataError)
 
 
+class LinkSearchResults(TypedDict):
+	value: str
+	description: str
+	label: NotRequired[str]
+
+
 # this is called by the Link Field
 @frappe.whitelist()
 def search_link(
-	doctype,
-	txt,
-	query=None,
-	filters=None,
-	page_length=10,
-	searchfield=None,
-	reference_doctype=None,
-	ignore_user_permissions=False,
-):
-	search_widget(
+	doctype: str,
+	txt: str,
+	query: str | None = None,
+	filters: str | dict | list | None = None,
+	page_length: int = 10,
+	searchfield: str | None = None,
+	reference_doctype: str | None = None,
+	ignore_user_permissions: bool = False,
+) -> list[LinkSearchResults]:
+	results = search_widget(
 		doctype,
 		txt.strip(),
 		query,
@@ -44,25 +54,23 @@ def search_link(
 		reference_doctype=reference_doctype,
 		ignore_user_permissions=ignore_user_permissions,
 	)
-
-	frappe.response["results"] = build_for_autosuggest(frappe.response["values"], doctype=doctype)
-	del frappe.response["values"]
+	return build_for_autosuggest(results, doctype=doctype)
 
 
 # this is called by the search box
 @frappe.whitelist()
 def search_widget(
-	doctype,
-	txt,
-	query=None,
-	searchfield=None,
-	start=0,
-	page_length=10,
-	filters=None,
+	doctype: str,
+	txt: str,
+	query: str | None = None,
+	searchfield: str = None,
+	start: int = 0,
+	page_length: int = 10,
+	filters: str | None | dict | list = None,
 	filter_fields=None,
-	as_dict=False,
-	reference_doctype=None,
-	ignore_user_permissions=False,
+	as_dict: bool = False,
+	reference_doctype: str | None = None,
+	ignore_user_permissions: bool = False,
 ):
 
 	start = cint(start)
@@ -78,11 +86,13 @@ def search_widget(
 
 	standard_queries = frappe.get_hooks().standard_queries or {}
 
-	if query and query.split(maxsplit=1)[0].lower() != "select":
-		# by method
+	if not query and doctype in standard_queries:
+		query = standard_queries[doctype][-1]
+
+	if query:  # Query = custom search query i.e. python function
 		try:
 			is_whitelisted(frappe.get_attr(query))
-			frappe.response["values"] = frappe.call(
+			return frappe.call(
 				query,
 				doctype,
 				txt,
@@ -93,9 +103,9 @@ def search_widget(
 				as_dict=as_dict,
 				reference_doctype=reference_doctype,
 			)
-		except frappe.exceptions.PermissionError as e:
+		except (frappe.PermissionError, frappe.AppNotInstalledError, ImportError):
 			if frappe.local.conf.developer_mode:
-				raise e
+				raise
 			else:
 				frappe.respond_as_web_page(
 					title="Invalid Method",
@@ -103,153 +113,123 @@ def search_widget(
 					indicator_color="red",
 					http_status_code=404,
 				)
-			return
-		except Exception as e:
-			raise e
-	elif not query and doctype in standard_queries:
-		# from standard queries
-		search_widget(
-			doctype=doctype,
-			txt=txt,
-			query=standard_queries[doctype][0],
-			searchfield=searchfield,
-			start=start,
-			page_length=page_length,
-			filters=filters,
-			filter_fields=filter_fields,
-			as_dict=as_dict,
-			reference_doctype=reference_doctype,
-			ignore_user_permissions=ignore_user_permissions,
+				return []
+
+	meta = frappe.get_meta(doctype)
+
+	if isinstance(filters, dict):
+		filters_items = filters.items()
+		filters = []
+		for key, value in filters_items:
+			filters.append(make_filter_tuple(doctype, key, value))
+
+	if filters is None:
+		filters = []
+	or_filters = []
+
+	# build from doctype
+	if txt:
+		field_types = {
+			"Data",
+			"Text",
+			"Small Text",
+			"Long Text",
+			"Link",
+			"Select",
+			"Read Only",
+			"Text Editor",
+		}
+		search_fields = ["name"]
+		if meta.title_field:
+			search_fields.append(meta.title_field)
+
+		if meta.search_fields:
+			search_fields.extend(meta.get_search_fields())
+
+		for f in search_fields:
+			fmeta = meta.get_field(f.strip())
+			if not meta.translated_doctype and (f == "name" or (fmeta and fmeta.fieldtype in field_types)):
+				or_filters.append([doctype, f.strip(), "like", f"%{txt}%"])
+
+	if meta.get("fields", {"fieldname": "enabled", "fieldtype": "Check"}):
+		filters.append([doctype, "enabled", "=", 1])
+	if meta.get("fields", {"fieldname": "disabled", "fieldtype": "Check"}):
+		filters.append([doctype, "disabled", "!=", 1])
+
+	# format a list of fields combining search fields and filter fields
+	fields = get_std_fields_list(meta, searchfield or "name")
+	if filter_fields:
+		fields = list(set(fields + json.loads(filter_fields)))
+	formatted_fields = [f"`tab{meta.name}`.`{f.strip()}`" for f in fields]
+
+	# Insert title field query after name
+	if meta.show_title_field_in_link:
+		formatted_fields.insert(1, f"`tab{meta.name}`.{meta.title_field} as `label`")
+
+	order_by_based_on_meta = get_order_by(doctype, meta)
+	# `idx` is number of times a document is referred, check link_count.py
+	order_by = f"`tab{doctype}`.idx desc, {order_by_based_on_meta}"
+
+	if not meta.translated_doctype:
+		_txt = frappe.db.escape((txt or "").replace("%", "").replace("@", ""))
+		# locate returns 0 if string is not found, convert 0 to null and then sort null to end in order by
+		_relevance = f"(1 / nullif(locate({_txt}, `tab{doctype}`.`name`), 0))"
+		formatted_fields.append(f"""{_relevance} as `_relevance`""")
+		# Since we are sorting by alias postgres needs to know number of column we are sorting
+		if frappe.db.db_type == "mariadb":
+			order_by = f"ifnull(_relevance, -9999) desc, {order_by}"
+		elif frappe.db.db_type == "postgres":
+			# Since we are sorting by alias postgres needs to know number of column we are sorting
+			order_by = f"{len(formatted_fields)} desc nulls last, {order_by}"
+
+	ignore_permissions = doctype == "DocType" or (
+		cint(ignore_user_permissions)
+		and has_permission(
+			doctype,
+			ptype="select" if frappe.only_has_select_perm(doctype) else "read",
+			parent_doctype=reference_doctype,
 		)
-	else:
-		meta = frappe.get_meta(doctype)
+	)
 
-		if query:
-			frappe.throw(_("This query style is discontinued"))
-			# custom query
-			# frappe.response["values"] = frappe.db.sql(scrub_custom_query(query, searchfield, txt))
+	values = frappe.get_list(
+		doctype,
+		filters=filters,
+		fields=formatted_fields,
+		or_filters=or_filters,
+		limit_start=start,
+		limit_page_length=None if meta.translated_doctype else page_length,
+		order_by=order_by,
+		ignore_permissions=ignore_permissions,
+		reference_doctype=reference_doctype,
+		as_list=not as_dict,
+		strict=False,
+	)
+
+	if meta.translated_doctype:
+		# Filtering the values array so that query is included in very element
+		values = (
+			result
+			for result in values
+			if any(
+				re.search(f"{re.escape(txt)}.*", _(cstr(value)) or "", re.IGNORECASE)
+				for value in (result.values() if as_dict else result)
+			)
+		)
+
+	# Sorting the values array so that relevant results always come first
+	# This will first bring elements on top in which query is a prefix of element
+	# Then it will bring the rest of the elements and sort them in lexicographical order
+	values = sorted(values, key=lambda x: relevance_sorter(x, txt, as_dict))
+
+	# remove _relevance from results
+	if not meta.translated_doctype:
+		if as_dict:
+			for r in values:
+				r.pop("_relevance", None)
 		else:
-			if isinstance(filters, dict):
-				filters_items = filters.items()
-				filters = []
-				for f in filters_items:
-					if isinstance(f[1], (list, tuple)):
-						filters.append([doctype, f[0], f[1][0], f[1][1]])
-					else:
-						filters.append([doctype, f[0], "=", f[1]])
+			values = [r[:-1] for r in values]
 
-			if filters is None:
-				filters = []
-			or_filters = []
-
-			# build from doctype
-			if txt:
-				field_types = [
-					"Data",
-					"Text",
-					"Small Text",
-					"Long Text",
-					"Link",
-					"Select",
-					"Read Only",
-					"Text Editor",
-				]
-				search_fields = ["name"]
-				if meta.title_field:
-					search_fields.append(meta.title_field)
-
-				if meta.search_fields:
-					search_fields.extend(meta.get_search_fields())
-
-				for f in search_fields:
-					fmeta = meta.get_field(f.strip())
-					if not meta.translated_doctype and (
-						f == "name" or (fmeta and fmeta.fieldtype in field_types)
-					):
-						or_filters.append([doctype, f.strip(), "like", f"%{txt}%"])
-
-			if meta.get("fields", {"fieldname": "enabled", "fieldtype": "Check"}):
-				filters.append([doctype, "enabled", "=", 1])
-			if meta.get("fields", {"fieldname": "disabled", "fieldtype": "Check"}):
-				filters.append([doctype, "disabled", "!=", 1])
-
-			# format a list of fields combining search fields and filter fields
-			fields = get_std_fields_list(meta, searchfield or "name")
-			if filter_fields:
-				fields = list(set(fields + json.loads(filter_fields)))
-			formatted_fields = [f"`tab{meta.name}`.`{f.strip()}`" for f in fields]
-
-			# Insert title field query after name
-			if meta.show_title_field_in_link:
-				formatted_fields.insert(1, f"`tab{meta.name}`.{meta.title_field} as `label`")
-
-			# In order_by, `idx` gets second priority, because it stores link count
-			from frappe.model.db_query import get_order_by
-
-			order_by_based_on_meta = get_order_by(doctype, meta)
-			# 2 is the index of _relevance column
-			order_by = f"{order_by_based_on_meta}, `tab{doctype}`.idx desc"
-
-			if not meta.translated_doctype:
-				formatted_fields.append(
-					"""locate({_txt}, `tab{doctype}`.`name`) as `_relevance`""".format(
-						_txt=frappe.db.escape((txt or "").replace("%", "").replace("@", "")),
-						doctype=doctype,
-					)
-				)
-				order_by = f"_relevance, {order_by}"
-
-			ignore_permissions = (
-				True
-				if doctype == "DocType"
-				else (
-					cint(ignore_user_permissions)
-					and has_permission(
-						doctype,
-						ptype="select" if frappe.only_has_select_perm(doctype) else "read",
-					)
-				)
-			)
-
-			values = frappe.get_list(
-				doctype,
-				filters=filters,
-				fields=formatted_fields,
-				or_filters=or_filters,
-				limit_start=start,
-				limit_page_length=None if meta.translated_doctype else page_length,
-				order_by=order_by,
-				ignore_permissions=ignore_permissions,
-				reference_doctype=reference_doctype,
-				as_list=not as_dict,
-				strict=False,
-			)
-
-			if meta.translated_doctype:
-				# Filtering the values array so that query is included in very element
-				values = (
-					result
-					for result in values
-					if any(
-						re.search(f"{re.escape(txt)}.*", _(cstr(value)) or "", re.IGNORECASE)
-						for value in (result.values() if as_dict else result)
-					)
-				)
-
-			# Sorting the values array so that relevant results always come first
-			# This will first bring elements on top in which query is a prefix of element
-			# Then it will bring the rest of the elements and sort them in lexicographical order
-			values = sorted(values, key=lambda x: relevance_sorter(x, txt, as_dict))
-
-			# remove _relevance from results
-			if not meta.translated_doctype:
-				if as_dict:
-					for r in values:
-						r.pop("_relevance")
-				else:
-					values = [r[:-1] for r in values]
-
-			frappe.response["values"] = values
+	return values
 
 
 def get_std_fields_list(meta, key):
@@ -270,7 +250,7 @@ def get_std_fields_list(meta, key):
 	return sflist
 
 
-def build_for_autosuggest(res: list[tuple], doctype: str) -> list[dict]:
+def build_for_autosuggest(res: list[tuple], doctype: str) -> list[LinkSearchResults]:
 	def to_string(parts):
 		return ", ".join(
 			unique(_(cstr(part)) if meta.translated_doctype else cstr(part) for part in parts if part)

frappe/email/doctype/email_account/email_account.py

@@ -309,7 +309,7 @@ class EmailAccount(Document):
 		except OSError:
 			if in_receive:
 				# timeout while connecting, see receive.py connect method
-				description = frappe.message_log.pop() if frappe.message_log else "Socket Error"
+				description = frappe.clear_last_message() if frappe.message_log else "Socket Error"
 				if test_internet():
 					self.db_set("no_failed", self.no_failed + 1)
 					if self.no_failed > 2:
@@ -348,14 +348,17 @@ class EmailAccount(Document):
 		return frappe.get_doc(cls.DOCTYPE, name)
 
 	@classmethod
-	def find_one_by_filters(cls, **kwargs):
+	def find_one_by_filters(cls, **kwargs) -> "EmailAccount":
 		name = frappe.db.get_value(cls.DOCTYPE, kwargs)
 		return cls.find(name) if name else None
 
 	@classmethod
 	def find_from_config(cls):
 		config = cls.get_account_details_from_site_config()
-		return cls.from_record(config) if config else None
+		if config:
+			account = cls.from_record(config)
+			account._from_site_config = True
+			return account
 
 	@classmethod
 	def create_dummy(cls):
@@ -475,9 +478,22 @@ class EmailAccount(Document):
 		}
 
 	def get_smtp_server(self):
+		"""Get SMTPServer (wrapper around actual smtplib object) for this account.
+
+		Implementation Detail: Since SMTPServer is same for each email connection, the same *instance*
+		is returned every time this function is called from same EmailAccount object.
+		This enables reusabilty of connection for better performance."""
+		return self._smtp_server_instance
+
+	@functools.cached_property
+	def _smtp_server_instance(self):
 		config = self.sendmail_config()
 		return SMTPServer(**config)
 
+	def remove_unpicklable_values(self, state):
+		super().remove_unpicklable_values(state)
+		state.pop("_smtp_server_instance", None)
+
 	def handle_incoming_connect_error(self, description):
 		if test_internet():
 			if self.get_failed_attempts_count() > 2:
@@ -496,7 +512,7 @@ class EmailAccount(Document):
 							}
 						)
 					except assign_to.DuplicateToDoError:
-						frappe.message_log.pop()
+						frappe.clear_last_message()
 						pass
 			else:
 				self.set_failed_attempts_count(self.get_failed_attempts_count() + 1)