Merge branch 'develop' into po-translation

2023-12-02 19:43:32 +05:30 · 2023-12-02 19:43:32 +05:30 · 91cebdace8
commit 91cebdace8
parent 586fee979b 3198b4f490
455 changed files with 5951 additions and 4208 deletions
--- a/.github/frappe-framework-logo-dark.svg
+++ b/.github/frappe-framework-logo-dark.svg
--- a/.github/workflows/lock.yml
+++ b/.github/workflows/lock.yml
@ -14,7 +14,7 @@ jobs:
  lock:
    runs-on: ubuntu-latest
    steps:
-      - uses: dessant/lock-threads@v4
+      - uses: dessant/lock-threads@v5
        with:
          github-token: ${{ github.token }}
          issue-inactive-days: 14
--- a/.github/workflows/server-tests.yml
+++ b/.github/workflows/server-tests.yml
@ -85,7 +85,7 @@ jobs:
      - name: Setup Python
        uses: actions/setup-python@v4
        with:
-          python-version: '3.11'
+          python-version: '3.12'

      - name: Check for valid Python & Merge Conflicts
        run: |
@ -132,6 +132,7 @@ jobs:
        env:
          BEFORE: ${{ env.GITHUB_EVENT_PATH.before }}
          AFTER: ${{ env.GITHUB_EVENT_PATH.after }}
+          FRAPPE_SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
          TYPE: server
          DB: ${{ matrix.db }}

@ -142,6 +143,7 @@ jobs:
          SITE: test_site
          CI_BUILD_ID: ${{ github.run_id }}
          BUILD_NUMBER: ${{ matrix.container }}
+          FRAPPE_SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
          TOTAL_BUILDS: 2
          COVERAGE_RCFILE: /home/runner/frappe-bench/apps/frappe/.coveragerc

--- a/.github/workflows/ui-tests.yml
+++ b/.github/workflows/ui-tests.yml
@ -67,7 +67,7 @@ jobs:
      - name: Setup Python
        uses: actions/setup-python@v4
        with:
-          python-version: '3.11'
+          python-version: '3.12'

      - name: Check for valid Python & Merge Conflicts
        run: |
@ -120,6 +120,7 @@ jobs:
        env:
          BEFORE: ${{ env.GITHUB_EVENT_PATH.before }}
          AFTER: ${{ env.GITHUB_EVENT_PATH.after }}
+          FRAPPE_SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
          TYPE: ui
          DB: mariadb

--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@ -35,7 +35,7 @@ repos:
    rev: v2.7.1
    hooks:
      - id: prettier
-        types_or: [javascript]
+        types_or: [javascript, vue, scss]
        # Ignore any files that might contain jinja / bundles
        exclude: |
            (?x)^(
@ -44,7 +44,8 @@ repos:
                .*boilerplate.*|
                frappe/www/website_script.js|
                frappe/templates/includes/.*|
-                frappe/public/js/lib/.*
+                frappe/public/js/lib/.*|
+                frappe/website/doctype/website_theme/website_theme_template.scss
            )$


--- a/README.md
+++ b/README.md
@ -1,10 +1,8 @@
 <div align="center">
-	<h1>
-		<br>
-		<a href="https://frappeframework.com">
-			<img src=".github/frappe-framework-logo.svg" height="50">
-		</a>
-	</h1>
+	<picture>
+		<source media="(prefers-color-scheme: dark)" srcset=".github/frappe-framework-logo-dark.svg">
+		<img src=".github/frappe-framework-logo.svg" height="50">
+	</picture>
 	<h3>
 		a web framework with <a href="https://www.youtube.com/watch?v=LOjk3m0wTwg">"batteries included"</a>
 	</h3>
@ -71,12 +69,12 @@ Full-stack web application framework that uses Python and MariaDB on the server
 1. [Code of Conduct](CODE_OF_CONDUCT.md)
 1. [Contribution Guidelines](https://github.com/frappe/erpnext/wiki/Contribution-Guidelines)
 1. [Security Policy](SECURITY.md)
-1. [Translations](https://translate.erpnext.com)

 ## Resources

 1. [frappeframework.com](https://frappeframework.com) - Official documentation of the Frappe Framework.
 1. [frappe.school](https://frappe.school) - Pick from the various courses by the maintainers or from the community.
+1. [buildwithhussain.dev](https://buildwithhussain.dev) - Watch Frappe Framework being used in the wild to build world-class web apps.

 ## License
 This repository has been released under the [MIT License](LICENSE).
--- a/cypress/fixtures/form_builder_doctype.js
+++ b/cypress/fixtures/form_builder_doctype.js
@ -10,6 +10,12 @@ export default {
 			fieldtype: "Data",
 			label: "Data 3",
 		},
+		{
+			fieldname: "gender",
+			fieldtype: "Link",
+			label: "Gender",
+			options: "Gender",
+		},
 		{
 			fieldname: "tab",
 			fieldtype: "Tab Break",
--- a/cypress/integration/awesome_bar.js
+++ b/cypress/integration/awesome_bar.js
@ -2,7 +2,11 @@ context("Awesome Bar", () => {
 	before(() => {
 		cy.visit("/login");
 		cy.login();
-		cy.visit("/app/website");
+		cy.visit("/app/todo"); // Make sure ToDo filters are cleared.
+		cy.clear_filters();
+		cy.visit("/app/blog-post"); // Make sure Blog Post filters are cleared.
+		cy.clear_filters();
+		cy.visit("/app/website"); // Go to some other page.
 	});

 	beforeEach(() => {
@ -11,36 +15,61 @@ context("Awesome Bar", () => {
 		cy.get("@awesome_bar").type("{selectall}");
 	});

+	after(() => {
+		cy.visit("/app/todo"); // Make sure we're not bleeding any filters to the next spec.
+		cy.clear_filters();
+	});
+
 	it("navigates to doctype list", () => {
 		cy.get("@awesome_bar").type("todo");
-		cy.wait(100);
+		cy.wait(100); // Wait a bit before hitting enter.
 		cy.get(".awesomplete").findByRole("listbox").should("be.visible");
 		cy.get("@awesome_bar").type("{enter}");
 		cy.get(".title-text").should("contain", "To Do");
 		cy.location("pathname").should("eq", "/app/todo");
 	});

-	it("find text in doctype list", () => {
+	it("finds text in doctype list", () => {
 		cy.get("@awesome_bar").type("test in todo");
-		cy.wait(100);
+		cy.wait(150); // Wait a bit before hitting enter.
 		cy.get("@awesome_bar").type("{enter}");
 		cy.get(".title-text").should("contain", "To Do");
-		cy.wait(200);
-		const name_filter = cy.get('[data-original-title="ID"] > input');
-		name_filter.should("have.value", "%test%");
-		cy.clear_filters();
+		cy.wait(200); // Wait a bit longer before checking the filter.
+		cy.get('[data-original-title="ID"] > input').should("have.value", "%test%");
+	});
+
+	it("filter preserved, now finds something else", () => {
+		cy.visit("/app/todo");
+		cy.get(".title-text").should("contain", "To Do");
+		cy.wait(200); // Wait a bit longer before checking the filter.
+		cy.get('[data-original-title="ID"] > input').as("filter");
+		cy.get("@filter").should("have.value", "%test%");
+		cy.get("@awesome_bar").type("anothertest in todo");
+		cy.wait(200); // Wait a bit longer before hitting enter.
+		cy.get("@awesome_bar").type("{enter}");
+		cy.wait(200); // Wait a bit longer before checking the filter.
+		cy.get("@filter").should("have.value", "%anothertest%");
+	});
+
+	it("navigates to another doctype, filter not bleeding", () => {
+		cy.get("@awesome_bar").type("blog post");
+		cy.wait(150); // Wait a bit before hitting enter.
+		cy.get("@awesome_bar").type("{enter}");
+		cy.get(".title-text").should("contain", "Blog Post");
+		cy.wait(200); // Wait a bit longer before checking the filter.
+		cy.location("search").should("be.empty");
 	});

 	it("navigates to new form", () => {
 		cy.get("@awesome_bar").type("new blog post");
-		cy.wait(100);
+		cy.wait(150); // Wait a bit before hitting enter
 		cy.get("@awesome_bar").type("{enter}");
 		cy.get(".title-text:visible").should("have.text", "New Blog Post");
 	});

 	it("calculates math expressions", () => {
 		cy.get("@awesome_bar").type("55 + 32");
-		cy.wait(100);
+		cy.wait(150); // Wait a bit before hitting enter
 		cy.get("@awesome_bar").type("{downarrow}{enter}");
 		cy.get(".modal-title").should("contain", "Result");
 		cy.get(".msgprint").should("contain", "55 + 32 = 87");
--- a/cypress/integration/control_float.js
+++ b/cypress/integration/control_float.js
@ -32,10 +32,13 @@ context("Control Float", () => {
 				cy.wait(200);
 				cy.fill_field("float_number", d.input, "Float").blur();
 				cy.get_field("float_number", "Float").should("have.value", d.blur_expected);
-
+				cy.wait(100);
 				cy.get_field("float_number", "Float").focus();
+				cy.wait(100);
 				cy.get_field("float_number", "Float").blur();
+				cy.wait(100);
 				cy.get_field("float_number", "Float").focus();
+				cy.wait(100);
 				cy.get_field("float_number", "Float").should("have.value", d.focus_expected);
 			});
 		});
@ -49,17 +52,17 @@ context("Control Float", () => {
 					{
 						input: "364.87,334",
 						blur_expected: "36.487,334",
-						focus_expected: "36487.334",
+						focus_expected: "36.487,334",
 					},
 					{
-						input: "36487,334",
-						blur_expected: "36.487,334",
-						focus_expected: "36487.334",
+						input: "36487,335",
+						blur_expected: "36.487,335",
+						focus_expected: "36.487,335",
 					},
 					{
-						input: "100",
-						blur_expected: "100,000",
-						focus_expected: "100",
+						input: "2*(2+47)+1,5+1",
+						blur_expected: "100,500",
+						focus_expected: "100,500",
 					},
 				],
 			},
@ -67,19 +70,19 @@ context("Control Float", () => {
 				number_format: "#,###.##",
 				values: [
 					{
-						input: "364,87.334",
-						blur_expected: "36,487.334",
-						focus_expected: "36487.334",
+						input: "464,87.334",
+						blur_expected: "46,487.334",
+						focus_expected: "46,487.334",
 					},
 					{
-						input: "36487.334",
-						blur_expected: "36,487.334",
-						focus_expected: "36487.334",
+						input: "46487.335",
+						blur_expected: "46,487.335",
+						focus_expected: "46,487.335",
 					},
 					{
-						input: "100",
-						blur_expected: "100.000",
-						focus_expected: "100",
+						input: "3*(2+47)+1.5+1",
+						blur_expected: "149.500",
+						focus_expected: "149.500",
 					},
 				],
 			},
@ -90,13 +93,13 @@ context("Control Float", () => {
 					{
 						input: "12.345",
 						blur_expected: "12.345,000",
-						focus_expected: "12345",
+						focus_expected: "12.345,000",
 					},
 					{
 						// parseFloat would reduce 12,340 to 12,34 if this string was ever to be parsed
 						input: "12.340",
 						blur_expected: "12.340,000",
-						focus_expected: "12340",
+						focus_expected: "12.340,000",
 					},
 				],
 			},
--- a/cypress/integration/control_link.js
+++ b/cypress/integration/control_link.js
@ -152,7 +152,7 @@ context("Control Link", () => {

 		cy.get(".frappe-control[data-fieldname=link] input").focus().as("input");
 		cy.wait("@search_link");
-		cy.get("@input").type("todo for link");
+		cy.get("@input").type("todo for link", { delay: 200 });
 		cy.wait("@search_link");
 		cy.get(".frappe-control[data-fieldname=link] ul").should("be.visible");
 		cy.get(".frappe-control[data-fieldname=link] input").type("{enter}", { delay: 100 });
@ -260,7 +260,7 @@ context("Control Link", () => {

 			cy.get(".frappe-control[data-fieldname=link] input").focus().as("input");
 			cy.wait("@search_link");
-			cy.get("@input").type("Sonstiges", { delay: 100 });
+			cy.get("@input").type("Sonstiges", { delay: 200 });
 			cy.wait("@search_link");
 			cy.get(".frappe-control[data-fieldname=link] ul").should("be.visible");
 			cy.get(".frappe-control[data-fieldname=link] input").type("{enter}", { delay: 100 });
@ -291,7 +291,7 @@ context("Control Link", () => {

 		cy.get(".frappe-control[data-fieldname=link] input").focus().as("input");
 		cy.wait("@search_link");
-		cy.get("@input").type("Non-Conforming", { delay: 100 });
+		cy.get("@input").type("Non-Conforming", { delay: 200 });
 		cy.wait("@search_link");
 		cy.get(".frappe-control[data-fieldname=link] ul").should("be.visible");
 		cy.get(".frappe-control[data-fieldname=link] input").type("{enter}", { delay: 100 });
--- a/cypress/integration/customize_form.js
+++ b/cypress/integration/customize_form.js
@ -5,6 +5,8 @@ context("Customize Form", () => {
 	});
 	it("Changing to naming rule should update autoname", () => {
 		cy.fill_field("doc_type", "ToDo", "Link").blur();
+		cy.wait(2000);
+		cy.findByRole("tab", { name: "Details" }).click();
 		cy.click_form_section("Naming");
 		const naming_rule_default_autoname_map = {
 			"Set by user": "prompt",
--- a/cypress/integration/form.js
+++ b/cypress/integration/form.js
@ -116,50 +116,6 @@ context("Form", () => {
 		cy.get_field("location").should("have.value", "Bermuda");
 	});

-	it("let user undo/redo field value changes", { scrollBehavior: false }, () => {
-		const undo = () => cy.get("body").type("{esc}").type("{ctrl+z}").wait(500);
-		const redo = () => cy.get("body").type("{esc}").type("{ctrl+y}").wait(500);
-
-		cy.new_form("User");
-
-		jump_to_field("Email");
-		type_value("admin@example.com");
-
-		jump_to_field("Username");
-		type_value("admin42");
-
-		jump_to_field("Send Welcome Email");
-		cy.focused().uncheck();
-
-		// make a mistake
-		jump_to_field("Username");
-		type_value("admin24");
-
-		// undo behaviour
-		undo();
-		cy.get_field("username").should("have.value", "admin42");
-
-		// redo behaviour
-		redo();
-		cy.get_field("username").should("have.value", "admin24");
-
-		// undo everything & redo everything, ensure same values at the end
-		undo();
-		undo();
-		undo();
-		undo();
-		redo();
-		redo();
-		redo();
-		redo();
-
-		cy.compare_document({
-			username: "admin24",
-			email: "admin@example.com",
-			send_welcome_email: 0,
-		});
-	});
-
 	it("update docfield property using set_df_property in child table", () => {
 		cy.visit("/app/contact/Test Form Contact 1");
 		cy.window()
--- a/cypress/integration/form_builder.js
+++ b/cypress/integration/form_builder.js
@ -35,6 +35,40 @@ context("Form Builder", () => {
 		cy.get(".title-area .indicator-pill.orange").should("have.text", "Not Saved");
 	});

+	it("Check if Filters are applied to the link field", () => {
+		// Visit the Form Builder
+		cy.visit(`/app/doctype/${doctype_name}`);
+		cy.findByRole("tab", { name: "Form" }).click();
+
+		cy.get("[data-fieldname='gender']").click();
+
+		// click on filter action button
+		cy.get('[data-fieldname="gender"] .field-actions button:first').click();
+
+		// add filter
+		cy.get(".modal-body .clear-filters").click();
+		cy.get(".modal-body .filter-action-buttons .add-filter").click();
+		cy.wait(100);
+		cy.get(".modal-body .filter-box .list_filter .filter-field .link-field input").type(
+			"Male"
+		);
+		cy.get(".btn-modal-primary").click();
+
+		// Save the document
+		cy.click_doc_primary_button("Save");
+
+		// Open a new Form
+		cy.new_form(doctype_name);
+		// Click on the "salutation" field
+		cy.get_field("gender").clear().click();
+
+		cy.intercept("POST", "/api/method/frappe.desk.search.search_link").as("search_link");
+		cy.wait("@search_link").then((data) => {
+			expect(data.response.body.message.length).to.eq(1);
+			expect(data.response.body.message[0].value).to.eq("Male");
+		});
+	});
+
 	it("Add empty section and save", () => {
 		cy.visit(`/app/doctype/${doctype_name}`);
 		cy.findByRole("tab", { name: "Form" }).click();
@ -43,7 +77,8 @@ context("Form Builder", () => {

 		// add new section
 		cy.get(first_section).click(15, 10);
-		cy.get(first_section).find(".section-actions button:first").click();
+		cy.get(first_section).find(".dropdown-btn:first").click();
+		cy.get(".dropdown-options:visible .dropdown-item:first").click();

 		// save
 		cy.click_doc_primary_button("Save");
@ -184,12 +219,14 @@ context("Form Builder", () => {

 		// add new section
 		cy.get(first_section).click(15, 10);
-		cy.get(first_section).find(".section-actions button:first").click();
+		cy.get(first_section).find(".dropdown-btn:first").click();
+		cy.get(".dropdown-options:visible .dropdown-item:first").click();
 		cy.get(".tab-content.active .form-section-container").should("have.length", 2);

 		// add new column
-		cy.get(first_section).find(".column:first").click(15, 10);
-		cy.get(first_section).find(".column:first .column-actions button:first").click();
+		cy.get(first_section).click(15, 10);
+		cy.get(first_section).find(".dropdown-btn:first").click();
+		cy.get(".dropdown-options:visible .dropdown-item:last").click();
 		cy.get(first_section).find(".column").should("have.length", 2);
 	});

@ -197,13 +234,15 @@ context("Form Builder", () => {
 		let first_section = ".tab-content.active .form-section-container:first";

 		// remove column
-		cy.get(first_section).find(".column:first").click(15, 10);
-		cy.get(first_section).find(".column:first .column-actions button:last").click();
+		cy.get(first_section).click(15, 10);
+		cy.get(first_section).find(".dropdown-btn:first").click();
+		cy.get(".dropdown-options:visible .dropdown-item:last").click();
 		cy.get(first_section).find(".column").should("have.length", 1);

 		// remove section
 		cy.get(first_section).click(15, 10);
-		cy.get(first_section).find(".section-actions button:last").click();
+		cy.get(first_section).find(".dropdown-btn:first").click();
+		cy.get(".dropdown-options:visible .dropdown-item").eq(1).click();
 		cy.get(".tab-content.active .form-section-container").should("have.length", 1);

 		// remove tab
--- a/cypress/integration/grid_configuration.js
+++ b/cypress/integration/grid_configuration.js
@ -4,7 +4,7 @@ context("Grid Configuration", () => {
 		cy.visit("/app/doctype/User");
 	});
 	it("Set user wise grid settings", () => {
-		cy.findByRole("tab", { name: "Form" }).click();
+		cy.findByRole("tab", { name: "Settings" }).click();
 		cy.get('.form-section[data-fieldname="fields_section"]').click();
 		cy.wait(100);
 		cy.get('.frappe-control[data-fieldname="fields"]').as("table");
--- a/cypress/integration/multi_select_dialog.js
+++ b/cypress/integration/multi_select_dialog.js
@ -76,6 +76,11 @@ context("MultiSelectDialog", () => {
 	});

 	it("tests more button", () => {
+		cy.get_open_dialog()
+			.get(`.frappe-control[data-fieldname="search_term"]`)
+			.find('input[data-fieldname="search_term"]')
+			.should("exist")
+			.type("Test", { delay: 200 });
 		cy.get_open_dialog()
 			.get(`.frappe-control[data-fieldname="more_child_btn"]`)
 			.should("exist")
--- a/cypress/integration/permissions.js
+++ b/cypress/integration/permissions.js
@ -1,4 +1,4 @@
-context("Permissions API", () => {
+context.skip("Permissions API", () => {
 	before(() => {
 		cy.visit("/login");
 		cy.remove_role("frappe@example.com", "System Manager");
--- a/cypress/integration/theme_switcher_dialog.js
+++ b/cypress/integration/theme_switcher_dialog.js
@ -1,37 +0,0 @@
-context("Theme Switcher Shortcut", () => {
-	before(() => {
-		cy.login();
-		cy.visit("/app");
-	});
-	beforeEach(() => {
-		cy.reload();
-	});
-	it("Check Toggle", () => {
-		cy.open_theme_dialog();
-		cy.get(".modal-backdrop").should("exist");
-		cy.intercept("POST", "/api/method/frappe.core.doctype.user.user.switch_theme").as(
-			"set_theme"
-		);
-		cy.findByText("Timeless Night").click();
-		cy.wait("@set_theme");
-		cy.close_theme("{ctrl+shift+g}");
-		cy.get(".modal-backdrop").should("not.exist");
-	});
-	it("Check Enter", () => {
-		cy.open_theme_dialog();
-		cy.intercept("POST", "/api/method/frappe.core.doctype.user.user.switch_theme").as(
-			"set_theme"
-		);
-		cy.findByText("Frappe Light").click();
-		cy.wait("@set_theme");
-		cy.close_theme("{enter}");
-		cy.get(".modal-backdrop").should("not.exist");
-	});
-});
-
-Cypress.Commands.add("open_theme_dialog", () => {
-	cy.get("body").type("{ctrl+shift+g}");
-});
-Cypress.Commands.add("close_theme", (shortcut_keys) => {
-	cy.get(".modal-header").type(shortcut_keys);
-});
--- a/cypress/integration/timeline.js
+++ b/cypress/integration/timeline.js
@ -1,91 +0,0 @@
-import custom_submittable_doctype from "../fixtures/custom_submittable_doctype";
-
-context("Timeline", () => {
-	before(() => {
-		cy.visit("/login");
-		cy.login();
-	});
-
-	it("Adding new ToDo, adding new comment, verifying comment addition & deletion and deleting ToDo", () => {
-		//Adding new ToDo
-		cy.new_form("ToDo");
-		cy.get('[data-fieldname="description"] .ql-editor.ql-blank')
-			.type("Test ToDo", { force: true })
-			.wait(200);
-		cy.get(".page-head .page-actions").findByRole("button", { name: "Save" }).click();
-
-		cy.go_to_list("ToDo");
-		cy.clear_filters();
-		cy.click_listview_row_item(0);
-
-		//To check if the comment box is initially empty and tying some text into it
-		cy.get('[data-fieldname="comment"] .ql-editor')
-			.should("contain", "")
-			.type("Testing Timeline");
-
-		//Adding new comment
-		cy.get(".comment-box").findByRole("button", { name: "Comment" }).click();
-
-		//To check if the commented text is visible in the timeline content
-		cy.get(".timeline-content").should("contain", "Testing Timeline");
-
-		//Editing comment
-		cy.click_timeline_action_btn("Edit");
-		cy.get('.timeline-content [data-fieldname="comment"] .ql-editor').first().type(" 123");
-		cy.click_timeline_action_btn("Save");
-
-		//To check if the edited comment text is visible in timeline content
-		cy.get(".timeline-content").should("contain", "Testing Timeline 123");
-
-		//Discarding comment
-		cy.click_timeline_action_btn("Edit");
-		cy.click_timeline_action_btn("Dismiss");
-
-		//To check if after discarding the timeline content is same as previous
-		cy.get(".timeline-content").should("contain", "Testing Timeline 123");
-
-		//Deleting the added comment
-		cy.get(".timeline-message-box .more-actions > .action-btn").click(); //Menu button in timeline item
-		cy.get(".timeline-message-box .more-actions .dropdown-item")
-			.contains("Delete")
-			.click({ force: true });
-		cy.get_open_dialog().findByRole("button", { name: "Yes" }).click({ force: true });
-
-		cy.get(".timeline-content").should("not.contain", "Testing Timeline 123");
-	});
-
-	it("Timeline should have submit and cancel activity information", () => {
-		cy.visit("/app/doctype");
-
-		//Creating custom doctype
-		cy.insert_doc("DocType", custom_submittable_doctype, true);
-
-		cy.visit("/app/custom-submittable-doctype");
-		cy.click_listview_primary_button("Add Custom Submittable DocType");
-
-		//Adding a new entry for the created custom doctype
-		cy.fill_field("title", "Test");
-		cy.click_modal_primary_button("Save");
-		cy.click_modal_primary_button("Submit");
-
-		cy.visit("/app/custom-submittable-doctype");
-		cy.click_listview_row_item(0);
-
-		//To check if the submission of the documemt is visible in the timeline content
-		cy.get(".timeline-content").should("contain", "You submitted this document");
-		cy.get('[id="page-Custom Submittable DocType"] .page-actions')
-			.findByRole("button", { name: "Cancel" })
-			.click();
-		cy.get_open_dialog().findByRole("button", { name: "Yes" }).click();
-
-		//To check if the cancellation of the documemt is visible in the timeline content
-		cy.get(".timeline-content").should("contain", "You cancelled this document");
-
-		//Deleting the document
-		cy.visit("/app/custom-submittable-doctype");
-		cy.select_listview_row_checkbox(0);
-		cy.get(".page-actions").findByRole("button", { name: "Actions" }).click();
-		cy.get('.page-actions .actions-btn-group [data-label="Delete"]').click();
-		cy.click_modal_primary_button("Yes");
-	});
-});
--- a/cypress/integration/web_form.js
+++ b/cypress/integration/web_form.js
@ -156,6 +156,7 @@ context("Web Form", () => {

 		cy.findByRole("tab", { name: "Customization" }).click();
 		cy.fill_field("breadcrumbs", '[{"label": _("Notes"), "route":"note"}]', "Code");
+		cy.wait(2000);
 		cy.get(".form-tabs .nav-item .nav-link").contains("Customization").click();
 		cy.save();

--- a/cypress/integration/workspace.js
+++ b/cypress/integration/workspace.js
@ -20,7 +20,6 @@ context("Workspace 2.0", () => {
 		cy.get(".codex-editor__redactor .ce-block");
 		cy.get('.custom-actions button[data-label="Create%20Workspace"]').click();
 		cy.fill_field("title", "Test Private Page", "Data");
-		cy.fill_field("icon", "edit", "Icon");
 		cy.get_open_dialog().find(".modal-header").click();
 		cy.get_open_dialog().find(".btn-primary").click();

@ -52,7 +51,6 @@ context("Workspace 2.0", () => {
 		cy.get('.custom-actions button[data-label="Create%20Workspace"]').click();
 		cy.fill_field("title", "Test Child Page", "Data");
 		cy.fill_field("parent", "Test Private Page", "Select");
-		cy.fill_field("icon", "edit", "Icon");
 		cy.get_open_dialog().find(".modal-header").click();
 		cy.get_open_dialog().find(".btn-primary").click();

--- a/cypress/integration/workspace_blocks.js
+++ b/cypress/integration/workspace_blocks.js
@ -20,7 +20,6 @@ context("Workspace Blocks", () => {
 		cy.get(".codex-editor__redactor .ce-block");
 		cy.get('.custom-actions button[data-label="Create%20Workspace"]').click();
 		cy.fill_field("title", "Test Block Page", "Data");
-		cy.fill_field("icon", "edit", "Icon");
 		cy.get_open_dialog().find(".modal-header").click();
 		cy.get_open_dialog().find(".btn-primary").click();

--- a/cypress/support/commands.js
+++ b/cypress/support/commands.js
@ -37,22 +37,16 @@ Cypress.Commands.add("login", (email, password) => {
 	// cy.session clears all localStorage on new login, so we need to retain the last route
 	const session_last_route = window.localStorage.getItem("session_last_route");
 	return cy
-		.session(
-			[email, password] || "",
-			() => {
-				return cy.request({
-					url: "/api/method/login",
-					method: "POST",
-					body: {
-						usr: email,
-						pwd: password,
-					},
-				});
-			},
-			{
-				cacheAcrossSpecs: true,
-			}
-		)
+		.session([email, password] || "", () => {
+			return cy.request({
+				url: "/api/method/login",
+				method: "POST",
+				body: {
+					usr: email,
+					pwd: password,
+				},
+			});
+		})
 		.then(() => {
 			if (session_last_route) {
 				window.localStorage.setItem("session_last_route", session_last_route);
--- a/frappe/init.py
+++ b/frappe/init.py
@ -17,7 +17,6 @@ import inspect
 import json
 import os
 import re
-import unicodedata
 import warnings
 from collections.abc import Callable
 from typing import TYPE_CHECKING, Any, Literal, Optional, TypeAlias, overload
@ -43,9 +42,8 @@ from .utils.jinja import (
 	get_template,
 	render_template,
 )
-from .utils.lazy_loader import lazy_import

-__version__ = "15.0.0-dev"
+__version__ = "16.0.0-dev"
 __title__ = "Frappe Framework"

 controllers = {}
@ -61,6 +59,32 @@ if _dev_server:
 	warnings.simplefilter("always", DeprecationWarning)
 	warnings.simplefilter("always", PendingDeprecationWarning)

+# Always initialize sentry SDK if the DSN is sent
+if sentry_dsn := os.getenv("FRAPPE_SENTRY_DSN"):
+	import sentry_sdk
+	from sentry_sdk.integrations.argv import ArgvIntegration
+	from sentry_sdk.integrations.atexit import AtexitIntegration
+	from sentry_sdk.integrations.dedupe import DedupeIntegration
+	from sentry_sdk.integrations.excepthook import ExcepthookIntegration
+	from sentry_sdk.integrations.modules import ModulesIntegration
+
+	from frappe.utils.sentry import before_send
+
+	sentry_sdk.init(
+		dsn=sentry_dsn,
+		before_send=before_send,
+		release=__version__,
+		auto_enabling_integrations=False,
+		default_integrations=False,
+		integrations=[
+			AtexitIntegration(),
+			ExcepthookIntegration(),
+			DedupeIntegration(),
+			ModulesIntegration(),
+			ArgvIntegration(),
+		],
+	)
+

 class _dict(dict):
 	"""dict like object that exposes keys as attributes"""
@ -169,6 +193,7 @@ if TYPE_CHECKING:  # pragma: no cover

 	from frappe.database.mariadb.database import MariaDBDatabase
 	from frappe.database.postgres.database import PostgresDatabase
+	from frappe.email.doctype.email_queue.email_queue import EmailQueue
 	from frappe.model.document import Document
 	from frappe.query_builder.builder import MariaDB, Postgres
 	from frappe.utils.redis_wrapper import RedisWrapper
@ -243,7 +268,7 @@ def init(site: str, sites_path: str = ".", new_site: bool = False, force=False)
 	local.jloader = None
 	local.cache = {}
 	local.form_dict = _dict()
-	local.preload_assets = {"style": [], "script": []}
+	local.preload_assets = {"style": [], "script": [], "icons": []}
 	local.session = _dict()
 	local.dev_server = _dev_server
 	local.qb = get_query_builder(local.conf.db_type)
@ -422,7 +447,7 @@ def errprint(msg: str) -> None:

 	:param msg: Message."""
 	msg = as_unicode(msg)
-	if not request or (not "cmd" in local.form_dict) or conf.developer_mode:
+	if not request or ("cmd" not in local.form_dict) or conf.developer_mode:
 		print(msg)

 	error_log.append({"exc": msg})
@ -433,11 +458,11 @@ def print_sql(enable: bool = True) -> None:


 def log(msg: str) -> None:
-	"""Add to `debug_log`.
+	"""Add to `debug_log`

 	:param msg: Message."""
 	if not request:
-		if conf.get("logging") or False:
+		if conf.get("logging"):
 			print(repr(msg))

 	debug_log.append(as_unicode(msg))
@ -454,6 +479,8 @@ def msgprint(
 	primary_action: str = None,
 	is_minimizable: bool = False,
 	wide: bool = False,
+	*,
+	realtime=False,
 ) -> None:
 	"""Print a message to the user (via HTTP response).
 	Messages are sent in the `__server_messages` property in the
@ -467,6 +494,7 @@ def msgprint(
 	:param primary_action: [optional] Bind a primary server/client side action.
 	:param is_minimizable: [optional] Allow users to minimize the modal
 	:param wide: [optional] Show wide modal
+	:param realtime: Publish message immediately using websocket.
 	"""
 	import inspect
 	import sys
@ -533,7 +561,10 @@ def msgprint(
 	if wide:
 		out.wide = wide

-	message_log.append(out)
+	if realtime:
+		publish_realtime(event="msgprint", message=out)
+	else:
+		message_log.append(out)
 	_raise_exception()


@ -665,7 +696,7 @@ def sendmail(
 	print_letterhead=False,
 	with_container=False,
 	email_read_tracker_url=None,
-):
+) -> Optional["EmailQueue"]:
 	"""Send email using user's default **Email Account** or global default **Email Account**.


@ -750,7 +781,7 @@ def sendmail(
 	)

 	# build email queue and send the email if send_now is True.
-	builder.process(send_now=now)
+	return builder.process(send_now=now)


 whitelisted = []
@ -1963,7 +1994,7 @@ def get_all(doctype, *args, **kwargs):
 	        frappe.get_all("ToDo", fields=["*"], filters = [["modified", ">", "2014-01-01"]])
 	"""
 	kwargs["ignore_permissions"] = True
-	if not "limit_page_length" in kwargs:
+	if "limit_page_length" not in kwargs:
 		kwargs["limit_page_length"] = 0
 	return get_list(doctype, *args, **kwargs)

@ -2012,7 +2043,7 @@ def as_json(obj: dict | list, indent=1, separators=None, ensure_ascii=True) -> s


 def are_emails_muted():
-	return flags.mute_emails or cint(conf.get("mute_emails") or 0) or False
+	return flags.mute_emails or cint(conf.get("mute_emails"))


 def get_test_records(doctype):
--- a/frappe/app.py
+++ b/frappe/app.py
@ -22,7 +22,7 @@ import frappe.rate_limiter
 import frappe.recorder
 import frappe.utils.response
 from frappe import _
-from frappe.auth import SAFE_HTTP_METHODS, UNSAFE_HTTP_METHODS, HTTPRequest, validate_auth
+from frappe.auth import SAFE_HTTP_METHODS, UNSAFE_HTTP_METHODS, HTTPRequest, validate_auth  # noqa
 from frappe.middlewares import StaticDataMiddleware
 from frappe.utils import CallbackManager, cint, get_site_name
 from frappe.utils.data import escape_html
@ -179,9 +179,12 @@ def init_request(request):
 			raise frappe.SessionStopped("Session Stopped")
 	else:
 		frappe.connect(set_admin_as_user=False)
+	if request.path.startswith("/api/method/upload_file"):
+		from frappe.core.api.file import get_max_file_size

-	request.max_content_length = cint(frappe.local.conf.get("max_file_size")) or 10 * 1024 * 1024
-
+		request.max_content_length = get_max_file_size()
+	else:
+		request.max_content_length = cint(frappe.local.conf.get("max_file_size")) or 25 * 1024 * 1024
 	make_form_dict(request)

 	if request.method != "OPTIONS":
@ -283,11 +286,11 @@ def set_cors_headers(response):
 	response.headers.extend(cors_headers)


-def make_form_dict(request):
+def make_form_dict(request: Request):
 	import json

 	request_data = request.get_data(as_text=True)
-	if "application/json" in (request.content_type or "") and request_data:
+	if request_data and request.is_json:
 		args = json.loads(request_data)
 	else:
 		args = {}
@ -299,9 +302,8 @@ def make_form_dict(request):

 	frappe.local.form_dict = frappe._dict(args)

-	if "_" in frappe.local.form_dict:
-		# _ is passed by $.ajax so that the request is not cached by the browser. So, remove _ from form_dict
-		frappe.local.form_dict.pop("_")
+	# _ is passed by $.ajax so that the request is not cached by the browser. So, remove _ from form_dict
+	frappe.local.form_dict.pop("_", None)


 def handle_exception(e):
--- a/frappe/auth.py
+++ b/frappe/auth.py
@ -25,6 +25,7 @@ from frappe.website.utils import get_home_page

 SAFE_HTTP_METHODS = frozenset(("GET", "HEAD", "OPTIONS"))
 UNSAFE_HTTP_METHODS = frozenset(("POST", "PUT", "DELETE", "PATCH"))
+MAX_PASSWORD_SIZE = 512


 class HTTPRequest:
@ -96,7 +97,6 @@ class HTTPRequest:


 class LoginManager:
-
 	__slots__ = ("user", "info", "full_name", "user_type", "resume")

 	def __init__(self):
@ -235,6 +235,9 @@ class LoginManager:
 		if not (user and pwd):
 			self.fail(_("Incomplete login details"), user=user)

+		if len(pwd) > MAX_PASSWORD_SIZE:
+			self.fail(_("Password size exceeded the maximum allowed size"), user=user)
+
 		_raw_user_name = user
 		user = User.find_by_credentials(user, pwd)

@ -305,8 +308,8 @@ class LoginManager:

 	def validate_hour(self):
 		"""check if user is logging in during restricted hours"""
-		login_before = int(frappe.db.get_value("User", self.user, "login_before", ignore=True) or 0)
-		login_after = int(frappe.db.get_value("User", self.user, "login_after", ignore=True) or 0)
+		login_before = cint(frappe.db.get_value("User", self.user, "login_before", ignore=True))
+		login_after = cint(frappe.db.get_value("User", self.user, "login_after", ignore=True))

 		if not (login_before or login_after):
 			return
@ -574,13 +577,13 @@ def validate_auth():
 		validate_oauth(authorization_header)
 		validate_auth_via_api_keys(authorization_header)

-		# If login via bearer, basic or keypair didn't work then authentication failed and we
-		# should terminate here.
-		if frappe.session.user in ("", "Guest"):
-			raise frappe.AuthenticationError
-
 	validate_auth_via_hooks()

+	# If login via bearer, basic or keypair didn't work then authentication failed and we
+	# should terminate here.
+	if len(authorization_header) == 2 and frappe.session.user in ("", "Guest"):
+		raise frappe.AuthenticationError
+

 def validate_oauth(authorization_header):
 	"""
@ -621,7 +624,7 @@ def validate_oauth(authorization_header):
 			frappe.set_user(frappe.db.get_value("OAuth Bearer Token", token, "user"))
 			frappe.local.form_dict = form_dict
 	except AttributeError:
-		raise frappe.AuthenticationError
+		pass


 def validate_auth_via_api_keys(authorization_header):
@ -647,7 +650,7 @@ def validate_auth_via_api_keys(authorization_header):
 			frappe.InvalidAuthorizationToken,
 		)
 	except (AttributeError, TypeError, ValueError):
-		raise frappe.AuthenticationError
+		pass


 def validate_api_key_secret(api_key, api_secret, frappe_authorization_source=None):
--- a/frappe/boot.py
+++ b/frappe/boot.py
@ -122,12 +122,12 @@ def get_letter_heads():


 def load_conf_settings(bootinfo):
-	from frappe import conf
+	from frappe.core.api.file import get_max_file_size

-	bootinfo.max_file_size = conf.get("max_file_size") or 10485760
+	bootinfo.max_file_size = get_max_file_size()
 	for key in ("developer_mode", "socketio_port", "file_watcher_port"):
-		if key in conf:
-			bootinfo[key] = conf.get(key)
+		if key in frappe.conf:
+			bootinfo[key] = frappe.conf.get(key)


 def load_desktop_data(bootinfo):
--- a/frappe/commands/scheduler.py
+++ b/frappe/commands/scheduler.py
@ -202,7 +202,7 @@ def start_scheduler():
 def start_worker(
 	queue, quiet=False, rq_username=None, rq_password=None, burst=False, strategy=None
 ):
-	"""Start a backgrond worker"""
+	"""Start a background worker"""
 	from frappe.utils.background_jobs import start_worker

 	start_worker(
@ -225,7 +225,7 @@ def start_worker(
@click.option("--quiet", is_flag=True, default=False, help="Hide Log Outputs")
@click.option("--burst", is_flag=True, default=False, help="Run Worker in Burst mode.")
 def start_worker_pool(queue, quiet=False, num_workers=2, burst=False):
-	"""Start a backgrond worker"""
+	"""Start a pool of background workers"""
 	from frappe.utils.background_jobs import start_worker_pool

 	start_worker_pool(
--- a/frappe/commands/site.py
+++ b/frappe/commands/site.py
@ -48,6 +48,11 @@ from frappe.exceptions import SiteNotSpecifiedError
@click.option(
 	"--set-default", is_flag=True, default=False, help="Set the new site as default site"
 )
+@click.option(
+	"--setup-db/--no-setup-db",
+	default=True,
+	help="Create user and database in mariadb/postgres; only bootstrap if false",
+)
 def new_site(
 	site,
 	db_root_username=None,
@ -64,6 +69,7 @@ def new_site(
 	db_host=None,
 	db_port=None,
 	set_default=False,
+	setup_db=True,
 ):
 	"Create a new site"
 	from frappe.installer import _new_site, extract_sql_from_archive
@ -88,6 +94,7 @@ def new_site(
 		db_type=db_type,
 		db_host=db_host,
 		db_port=db_port,
+		setup_db=setup_db,
 	)

 	if set_default:
@ -406,7 +413,6 @@ def _reinstall(
 	verbose=False,
 ):
 	from frappe.installer import _new_site
-	from frappe.utils.synchronization import filelock

 	if not yes:
 		click.confirm("This will wipe your database. Are you sure you want to reinstall?", abort=True)
@ -944,9 +950,9 @@ def move(dest_dir, site):
 	site_dump_exists = True
 	count = 0
 	while site_dump_exists:
-		final_new_path = new_path + (count and str(count) or "")
+		final_new_path = new_path + str(count or "")
 		site_dump_exists = os.path.exists(final_new_path)
-		count = int(count or 0) + 1
+		count += 1

 	shutil.move(old_path, final_new_path)
 	frappe.destroy()
--- a/frappe/commands/utils.py
+++ b/frappe/commands/utils.py
@ -2,17 +2,16 @@ import json
 import os
 import subprocess
 import sys
-from shutil import which

 import click

 import frappe
+from frappe import _
 from frappe.commands import get_site, pass_context
 from frappe.coverage import CodeCoverage
 from frappe.exceptions import SiteNotSpecifiedError
 from frappe.utils import cint, update_progress_bar

-find_executable = which  # backwards compatibility
 EXTRA_ARGS_CTX = {"ignore_unknown_options": True, "allow_extra_args": True}


@ -465,19 +464,11 @@ def database(context, extra_args):
 	Enter into the Database console for given site.
 	"""
 	site = get_site(context)
-	if not site:
-		raise SiteNotSpecifiedError
 	frappe.init(site=site)
-	if frappe.conf.db_type == "mariadb":
-		_mariadb(extra_args=extra_args)
-	elif frappe.conf.db_type == "postgres":
-		_psql(extra_args=extra_args)
+	_enter_console(extra_args=extra_args)


-@click.command(
-	"mariadb",
-	context_settings=EXTRA_ARGS_CTX,
-)
+@click.command("mariadb", context_settings=EXTRA_ARGS_CTX)
@click.argument("extra_args", nargs=-1)
@pass_context
 def mariadb(context, extra_args):
@ -485,10 +476,9 @@ def mariadb(context, extra_args):
 	Enter into mariadb console for a given site.
 	"""
 	site = get_site(context)
-	if not site:
-		raise SiteNotSpecifiedError
 	frappe.init(site=site)
-	_mariadb(extra_args=extra_args)
+	frappe.conf.db_type = "mariadb"
+	_enter_console(extra_args=extra_args)


@click.command("postgres", context_settings=EXTRA_ARGS_CTX)
@ -500,42 +490,27 @@ def postgres(context, extra_args):
 	"""
 	site = get_site(context)
 	frappe.init(site=site)
-	_psql(extra_args=extra_args)
+	frappe.conf.db_type = "postgres"
+	_enter_console(extra_args=extra_args)


-def _mariadb(extra_args=None):
-	mariadb = which("mariadb") or which("mysql")
-	command = [
-		mariadb,
-		"--port",
-		str(frappe.conf.db_port),
-		"-u",
-		frappe.conf.db_name,
-		f"-p{frappe.conf.db_password}",
-		frappe.conf.db_name,
-		"-h",
-		frappe.conf.db_host,
-		"--pager=less -SFX",
-		"--safe-updates",
-		"-A",
-	]
-	if extra_args:
-		command += list(extra_args)
-	os.execv(mariadb, command)
+def _enter_console(extra_args=None):
+	from frappe.database import get_command

-
-def _psql(extra_args=None):
-	psql = which("psql")
-
-	host = frappe.conf.db_host
-	port = frappe.conf.db_port
-	env = os.environ.copy()
-	env["PGPASSWORD"] = frappe.conf.db_password
-	conn_string = f"postgresql://{frappe.conf.db_name}@{host}:{port}/{frappe.conf.db_name}"
-	psql_cmd = [psql, conn_string]
-	if extra_args:
-		psql_cmd = psql_cmd + list(extra_args)
-	subprocess.run(psql_cmd, check=True, env=env)
+	bin, args, bin_name = get_command(
+		host=frappe.conf.db_host,
+		port=frappe.conf.db_port,
+		user=frappe.conf.db_name,
+		password=frappe.conf.db_password,
+		db_name=frappe.conf.db_name,
+		extra=list(extra_args) if extra_args else [],
+	)
+	if not bin:
+		frappe.throw(
+			_("{} not found in PATH! This is required to access the console.").format(bin_name),
+			exc=frappe.ExecutableNotFound,
+		)
+	os.execv(bin, [bin] + args)


@click.command("jupyter")
--- a/frappe/contacts/doctype/address/address.json
+++ b/frappe/contacts/doctype/address/address.json
@ -51,12 +51,14 @@
   "fieldname": "address_line1",
   "fieldtype": "Data",
   "label": "Address Line 1",
+   "length": 240,
   "reqd": 1
  },
  {
   "fieldname": "address_line2",
   "fieldtype": "Data",
-   "label": "Address Line 2"
+   "label": "Address Line 2",
+   "length": 240
  },
  {
   "fieldname": "city",
@ -148,7 +150,7 @@
 "icon": "fa fa-map-marker",
 "idx": 5,
 "links": [],
- "modified": "2023-10-30 05:50:23.912366",
+ "modified": "2023-11-20 17:28:41.698356",
 "modified_by": "Administrator",
 "module": "Contacts",
 "name": "Address",
--- a/frappe/core/doctype/communication/communication.json
+++ b/frappe/core/doctype/communication/communication.json
@ -31,6 +31,7 @@
  "additional_info",
  "communication_date",
  "read_receipt",
+  "send_after",
  "column_break_14",
  "sender_full_name",
  "read_by_recipient",
@ -125,7 +126,7 @@
   "fieldtype": "Select",
   "hidden": 1,
   "label": "Delivery Status",
-   "options": "\nSent\nBounced\nOpened\nMarked As Spam\nRejected\nDelayed\nSoft-Bounced\nClicked\nRecipient Unsubscribed\nError\nExpired\nSending\nRead"
+   "options": "\nSent\nBounced\nOpened\nMarked As Spam\nRejected\nDelayed\nSoft-Bounced\nClicked\nRecipient Unsubscribed\nError\nExpired\nSending\nRead\nScheduled"
  },
  {
   "fieldname": "section_break_8",
@ -390,12 +391,17 @@
   "hidden": 1,
   "label": "IMAP Folder",
   "read_only": 1
+  },
+  {
+   "fieldname": "send_after",
+   "fieldtype": "Datetime",
+   "label": "Send After"
  }
 ],
 "icon": "fa fa-comment",
 "idx": 1,
 "links": [],
- "modified": "2023-08-29 17:20:52.541483",
+ "modified": "2023-11-27 20:38:27.467076",
 "modified_by": "Administrator",
 "module": "Core",
 "name": "Communication",
--- a/frappe/core/doctype/communication/communication.py
+++ b/frappe/core/doctype/communication/communication.py
@ -87,6 +87,7 @@ class Communication(Document, CommunicationEmailMixin):
 			"Expired",
 			"Sending",
 			"Read",
+			"Scheduled",
 		]
 		email_account: DF.Link | None
 		email_status: DF.Literal["Open", "Spam", "Trash"]
@ -106,6 +107,7 @@ class Communication(Document, CommunicationEmailMixin):
 		reference_name: DF.DynamicLink | None
 		reference_owner: DF.ReadOnly | None
 		seen: DF.Check
+		send_after: DF.Datetime | None
 		sender: DF.Data | None
 		sender_full_name: DF.Data | None
 		sent_or_received: DF.Literal["Sent", "Received"]
@ -162,7 +164,8 @@ class Communication(Document, CommunicationEmailMixin):
 			self.seen = 1
 			self.sent_or_received = "Sent"

-		self.set_status()
+		if not self.send_after:  # Handle empty string, always set NULL
+			self.send_after = None

 		validate_email(self)

@ -173,6 +176,10 @@ class Communication(Document, CommunicationEmailMixin):

 		self.set_sender_full_name()

+		if self.is_new():
+			self.set_status()
+			self.mark_email_as_spam()
+
 	def validate_reference(self):
 		if self.reference_doctype and self.reference_name:
 			if not self.reference_owner:
@ -333,9 +340,6 @@ class Communication(Document, CommunicationEmailMixin):
 		)

 	def set_status(self):
-		if not self.is_new():
-			return
-
 		if self.reference_doctype and self.reference_name:
 			self.status = "Linked"
 		elif self.communication_type == "Communication":
@ -343,15 +347,16 @@ class Communication(Document, CommunicationEmailMixin):
 		else:
 			self.status = "Closed"

-		# set email status to spam
-		email_rule = frappe.db.get_value("Email Rule", {"email_id": self.sender, "is_spam": 1})
+		if self.send_after and self.is_new():
+			self.delivery_status = "Scheduled"
+
+	def mark_email_as_spam(self):
 		if (
 			self.communication_type == "Communication"
 			and self.communication_medium == "Email"
-			and self.sent_or_received == "Sent"
-			and email_rule
+			and self.sent_or_received == "Received"
+			and frappe.db.exists("Email Rule", {"email_id": self.sender, "is_spam": 1})
 		):
-
 			self.email_status = "Spam"

 	@classmethod
@ -433,7 +438,18 @@ class Communication(Document, CommunicationEmailMixin):
 				frappe.db.commit()

 	def parse_email_for_timeline_links(self):
-		parse_email(self, [self.recipients, self.cc, self.bcc])
+		if not frappe.db.get_value("Email Account", self.email_account, "enable_automatic_linking"):
+			return
+
+		for doctype, docname in parse_email([self.recipients, self.cc, self.bcc]):
+			if not frappe.db.get_value(doctype, docname, ignore=True):
+				continue
+
+			self.add_link(doctype, docname)
+
+			if not self.reference_doctype:
+				self.reference_doctype = doctype
+				self.reference_name = docname

 	# Timeline Links
 	def set_timeline_links(self):
@ -452,20 +468,13 @@ class Communication(Document, CommunicationEmailMixin):
 			add_contact_links_to_communication(self, contact_name)

 	def deduplicate_timeline_links(self):
-		if self.timeline_links:
-			links, duplicate = [], False
+		if not self.timeline_links:
+			return

-			for l in self.timeline_links:
-				t = (l.link_doctype, l.link_name)
-				if not t in links:
-					links.append(t)
-				else:
-					duplicate = True
-
-			if duplicate:
-				self.timeline_links.clear()
-				for l in links:
-					self.add_link(link_doctype=l[0], link_name=l[1])
+		unique_links = {(link.link_doctype, link.link_name) for link in self.timeline_links}
+		self.timeline_links = []
+		for doctype, name in unique_links:
+			self.add_link(doctype, name)

 	def add_link(self, link_doctype, link_name, autosave=False):
 		self.append("timeline_links", {"link_doctype": link_doctype, "link_name": link_name})
@ -477,7 +486,7 @@ class Communication(Document, CommunicationEmailMixin):
 		return self.timeline_links

 	def remove_link(self, link_doctype, link_name, autosave=False, ignore_permissions=True):
-		for l in self.timeline_links:
+		for l in list(self.timeline_links):
 			if l.link_doctype == link_doctype and l.link_name == link_name:
 				self.timeline_links.remove(l)

@ -574,36 +583,35 @@ def add_contact_links_to_communication(communication, contact_name):
 			communication.add_link(contact_link.link_doctype, contact_link.link_name)


-def parse_email(communication, email_strings):
+def parse_email(email_strings):
 	"""
 	Parse email to add timeline links.
 	When automatic email linking is enabled, an email from email_strings can contain
 	a doctype and docname ie in the format `admin+doctype+docname@example.com` or `admin+doctype=docname@example.com`,
-	the email is parsed and doctype and docname is extracted and timeline link is added.
+	the email is parsed and doctype and docname is extracted.
 	"""
-	if not frappe.db.get_value("Email Account", filters={"enable_automatic_linking": 1}):
-		return
-
 	for email_string in email_strings:
-		if email_string:
-			for email in email_string.split(","):
-				email_username = email.split("@", 1)[0]
-				email_local_parts = email_username.split("+")
-				docname = doctype = None
-				if len(email_local_parts) == 3:
-					doctype = unquote(email_local_parts[1])
-					docname = unquote(email_local_parts[2])
+		if not email_string:
+			continue

-				elif len(email_local_parts) == 2:
-					document_parts = email_local_parts[1].split("=", 1)
-					if len(document_parts) != 2:
-						continue
+		for email in email_string.split(","):
+			email_username = email.split("@", 1)[0]
+			email_local_parts = email_username.split("+")
+			docname = doctype = None
+			if len(email_local_parts) == 3:
+				doctype = unquote(email_local_parts[1])
+				docname = unquote(email_local_parts[2])

-					doctype = unquote(document_parts[0])
-					docname = unquote(document_parts[1])
+			elif len(email_local_parts) == 2:
+				document_parts = email_local_parts[1].split("=", 1)
+				if len(document_parts) != 2:
+					continue

-				if doctype and docname and frappe.db.get_value(doctype, docname, ignore=True):
-					communication.add_link(doctype, docname)
+				doctype = unquote(document_parts[0])
+				docname = unquote(document_parts[1])
+
+			if doctype and docname:
+				yield doctype, docname


 def get_email_without_link(email):
--- a/frappe/core/doctype/communication/email.py
+++ b/frappe/core/doctype/communication/email.py
@ -46,6 +46,7 @@ def make(
 	print_letterhead=True,
 	email_template=None,
 	communication_type=None,
+	send_after=None,
 	**kwargs,
 ) -> dict[str, str]:
 	"""Make a new communication. Checks for email permissions for specified Document.
@ -64,6 +65,7 @@ def make(
 	:param attachments: List of File names or dicts with keys "fname" and "fcontent"
 	:param send_me_a_copy: Send a copy to the sender (default **False**).
 	:param email_template: Template which is used to compose mail .
+	:param send_after: Send after the given datetime.
 	"""
 	if kwargs:
 		from frappe.utils.commands import warn
@ -99,6 +101,7 @@ def make(
 		email_template=email_template,
 		communication_type=communication_type,
 		add_signature=False,
+		send_after=send_after,
 	)


@ -124,6 +127,7 @@ def _make(
 	email_template=None,
 	communication_type=None,
 	add_signature=True,
+	send_after=None,
 ) -> dict[str, str]:
 	"""Internal method to make a new communication that ignores Permission checks."""

@ -151,6 +155,7 @@ def _make(
 			"read_receipt": read_receipt,
 			"has_attachment": 1 if attachments else 0,
 			"communication_type": communication_type,
+			"send_after": send_after,
 		}
 	)
 	comm.flags.skip_add_signature = not add_signature
--- a/frappe/core/doctype/communication/mixins.py
+++ b/frappe/core/doctype/communication/mixins.py
@ -288,8 +288,9 @@ class CommunicationEmailMixin:
 			"delayed": True,
 			"communication": self.name,
 			"read_receipt": self.read_receipt,
-			"is_notification": (self.sent_or_received == "Received" and True) or False,
+			"is_notification": (self.sent_or_received == "Received"),
 			"print_letterhead": print_letterhead,
+			"send_after": self.send_after,
 		}

 	def send_email(
--- a/frappe/core/doctype/communication/test_communication.py
+++ b/frappe/core/doctype/communication/test_communication.py
@ -1,10 +1,9 @@
 # Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
 from typing import TYPE_CHECKING
-from urllib.parse import quote

 import frappe
-from frappe.core.doctype.communication.communication import Communication, get_emails
+from frappe.core.doctype.communication.communication import Communication, get_emails, parse_email
 from frappe.core.doctype.communication.email import add_attachments
 from frappe.email.doctype.email_queue.email_queue import EmailQueue
 from frappe.tests.utils import FrappeTestCase
@ -219,36 +218,25 @@ class TestCommunication(FrappeTestCase):
 		self.assertIn(comm_note_1.name, data)
 		self.assertIn(comm_note_2.name, data)

-	def test_link_in_email(self):
-		create_email_account()
+	def test_parse_email(self):
+		to = "Jon Doe <jon.doe@example.org>"
+		cc = """=?UTF-8?Q?Max_Mu=C3=9F?= <max.muss@examle.org>,
+	erp+Customer+that%20company@example.org"""
+		bcc = ""

-		notes = {}
-		for i in range(2):
-			frappe.delete_doc_if_exists("Note", f"test document link in email {i}")
-			notes[i] = frappe.get_doc(
-				{
-					"doctype": "Note",
-					"title": f"test document link in email {i}",
-				}
-			).insert(ignore_permissions=True)
+		results = list(parse_email([to, cc, bcc]))
+		self.assertEqual([("Customer", "that company")], results)

-		comm = frappe.get_doc(
-			{
-				"doctype": "Communication",
-				"communication_medium": "Email",
-				"subject": "Document Link in Email",
-				"sender": "comm_sender@example.com",
-				"recipients": f'comm_recipient+{quote("Note")}+{quote(notes[0].name)}@example.com,comm_recipient+{quote("Note")}={quote(notes[1].name)}@example.com',
-			}
-		).insert(ignore_permissions=True)
+		results = list(parse_email([to, bcc]))
+		self.assertEqual(results, [])

-		doc_links = [
-			(timeline_link.link_doctype, timeline_link.link_name) for timeline_link in comm.timeline_links
-		]
-		self.assertIn(("Note", notes[0].name), doc_links)
-		self.assertIn(("Note", notes[1].name), doc_links)
+		to = "jane.doe+A+Test@example.org"
+		cc = ""
+		bcc = "=?UTF-8?Q?Max_Mu=C3=9F?= <max.muss+Note=Very%20important@examle.org>"
+		results = list(parse_email([to, cc, bcc]))
+		self.assertEqual([("A", "Test"), ("Note", "Very important")], results)

-	def test_parse_emails(self):
+	def test_get_emails(self):
 		emails = get_emails(
 			[
 				"comm_recipient+DocType+DocName@example.com",
@ -293,6 +281,40 @@ class TestCommunication(FrappeTestCase):
 		self.assertEqual(comm_with_signature.content.count(signature), 1)
 		self.assertEqual(comm_without_signature.content.count(signature), 1)

+	def test_mark_as_spam(self):
+		frappe.get_doc(
+			{
+				"doctype": "Email Rule",
+				"email_id": "spammer@example.com",
+				"is_spam": 1,
+			}
+		).insert(ignore_permissions=True)
+
+		spam_comm: Communication = frappe.get_doc(
+			{
+				"doctype": "Communication",
+				"communication_medium": "Email",
+				"subject": "This is spam",
+				"sender": "spammer@example.com",
+				"recipients": "comm_recipient@example.com",
+				"sent_or_received": "Received",
+			}
+		).insert(ignore_permissions=True)
+
+		self.assertEqual(spam_comm.email_status, "Spam")
+
+		normal_comm: Communication = frappe.get_doc(
+			{
+				"doctype": "Communication",
+				"communication_medium": "Email",
+				"subject": "This is spam",
+				"sender": "friendlyhuman@example.com",
+				"recipients": "comm_recipient@example.com",
+				"sent_or_received": "Received",
+			}
+		).insert(ignore_permissions=True)
+		self.assertNotEqual(normal_comm.email_status, "Spam")
+

 class TestCommunicationEmailMixin(FrappeTestCase):
 	def new_communication(self, recipients=None, cc=None, bcc=None) -> Communication:
--- a/frappe/core/doctype/data_import/exporter.py
+++ b/frappe/core/doctype/data_import/exporter.py
@ -144,8 +144,7 @@ class Exporter:
 				value = doc.get(df.fieldname, None)

 				if df.fieldtype == "Duration":
-					value = flt(value or 0)
-					value = format_duration(value, df.hide_days)
+					value = format_duration(flt(value), df.hide_days)

 				row[i] = value
 		return rows
--- a/frappe/core/doctype/docfield/docfield.json
+++ b/frappe/core/doctype/docfield/docfield.json
@ -19,10 +19,12 @@
  "reqd",
  "is_virtual",
  "search_index",
+  "not_nullable",
  "column_break_18",
  "options",
  "sort_options",
  "show_dashboard",
+  "link_filters",
  "defaults_section",
  "default",
  "column_break_6",
@ -560,13 +562,25 @@
   "fieldname": "sort_options",
   "fieldtype": "Check",
   "label": "Sort Options"
+  },
+  {
+   "fieldname": "link_filters",
+   "fieldtype": "JSON",
+   "label": "Link Filters"
+  },
+  {
+   "default": "0",
+   "depends_on": "eval:!in_list([\"Check\", \"Currency\", \"Float\", \"Int\", \"Percent\", \"Rating\", \"Select\", \"Table\", \"Table MultiSelect\"], doc.fieldtype)",
+   "fieldname": "not_nullable",
+   "fieldtype": "Check",
+   "label": "Not Nullable"
  }
 ],
 "idx": 1,
 "index_web_pages_for_search": 1,
 "istable": 1,
 "links": [],
- "modified": "2023-10-25 06:53:45.194081",
+ "modified": "2023-11-16 11:26:56.364594",
 "modified_by": "Administrator",
 "module": "Core",
 "name": "DocField",
--- a/frappe/core/doctype/docfield/docfield.py
+++ b/frappe/core/doctype/docfield/docfield.py
@ -87,10 +87,12 @@ class DocField(Document):
 		is_virtual: DF.Check
 		label: DF.Data | None
 		length: DF.Int
+		link_filters: DF.JSON | None
 		mandatory_depends_on: DF.Code | None
 		max_height: DF.Data | None
 		no_copy: DF.Check
 		non_negative: DF.Check
+		not_nullable: DF.Check
 		oldfieldname: DF.Data | None
 		oldfieldtype: DF.Data | None
 		options: DF.SmallText | None
--- a/frappe/core/doctype/doctype/doctype.py
+++ b/frappe/core/doctype/doctype/doctype.py
@ -136,6 +136,7 @@ class DocType(Document):
 		is_virtual: DF.Check
 		issingle: DF.Check
 		istable: DF.Check
+		link_filters: DF.JSON
 		links: DF.Table[DocTypeLink]
 		make_attachments_public: DF.Check
 		max_attachments: DF.Int
@ -364,16 +365,23 @@ class DocType(Document):
 							SET `{fieldname}` = source.`{source_fieldname}`
 							FROM `tab{link_doctype}` as source
 							WHERE `{link_fieldname}` = source.name
-							AND ifnull(`{fieldname}`, '')=''
 						"""
+						if df.not_nullable:
+							update_query += "AND `{fieldname}`=''"
+						else:
+							update_query += "AND ifnull(`{fieldname}`, '')=''"
+
 					else:
 						update_query = """
 							UPDATE `tab{doctype}` as target
 							INNER JOIN `tab{link_doctype}` as source
 							ON `target`.`{link_fieldname}` = `source`.`name`
 							SET `target`.`{fieldname}` = `source`.`{source_fieldname}`
-							WHERE ifnull(`target`.`{fieldname}`, '')=""
 						"""
+						if df.not_nullable:
+							update_query += "WHERE `target`.`{fieldname}`=''"
+						else:
+							update_query += "WHERE ifnull(`target`.`{fieldname}`, '')=''"

 					self.flags.update_fields_to_fetch_queries.append(
 						update_query.format(
--- a/frappe/core/doctype/doctype/doctype_list.js
+++ b/frappe/core/doctype/doctype/doctype_list.js
@ -101,6 +101,7 @@ frappe.listview_settings["DocType"] = {
 								role: "System Manager",
 								share: 1,
 								write: 1,
+								submit: values.is_submittable ? 1 : 0,
 							},
 						],
 						fields: [{ fieldtype: "Section Break" }],
--- a/frappe/core/doctype/document_naming_rule/document_naming_rule.json
+++ b/frappe/core/doctype/document_naming_rule/document_naming_rule.json
@ -35,6 +35,7 @@
  {
   "fieldname": "prefix",
   "fieldtype": "Data",
+   "in_list_view": 1,
   "label": "Prefix",
   "mandatory_depends_on": "eval:doc.naming_by===\"Numbered\"",
   "reqd": 1
@ -44,6 +45,7 @@
   "description": "Warning: Updating counter may lead to document name conflicts if not done properly",
   "fieldname": "counter",
   "fieldtype": "Int",
+   "in_list_view": 1,
   "label": "Counter",
   "no_copy": 1
  },
@ -78,6 +80,7 @@
   "description": "Rules with higher priority number will be applied first.",
   "fieldname": "priority",
   "fieldtype": "Int",
+   "in_standard_filter": 1,
   "label": "Priority"
  },
  {
@ -87,7 +90,7 @@
 ],
 "index_web_pages_for_search": 1,
 "links": [],
- "modified": "2023-04-24 15:14:32.054272",
+ "modified": "2023-11-21 11:58:25.712375",
 "modified_by": "Administrator",
 "module": "Core",
 "name": "Document Naming Rule",
@ -107,7 +110,7 @@
  }
 ],
 "quick_entry": 1,
- "sort_field": "modified",
+ "sort_field": "priority",
 "sort_order": "DESC",
 "states": [],
 "title_field": "document_type",
--- a/frappe/core/doctype/document_naming_rule/document_naming_rule_list.js
+++ b/frappe/core/doctype/document_naming_rule/document_naming_rule_list.js
@ -0,0 +1,3 @@
+frappe.listview_settings["Document Naming Rule"] = {
+	hide_name_column: true,
+};
--- a/frappe/core/doctype/domain/domain.py
+++ b/frappe/core/doctype/domain/domain.py
@ -4,6 +4,7 @@
 import frappe
 from frappe.custom.doctype.custom_field.custom_field import create_custom_fields
 from frappe.model.document import Document
+from frappe.utils import cint


 class Domain(Document):
@ -28,7 +29,7 @@ class Domain(Document):
 		self.setup_properties()
 		self.set_values()

-		if not int(frappe.defaults.get_defaults().setup_complete or 0):
+		if not cint(frappe.defaults.get_defaults().setup_complete):
 			# if setup not complete, setup desktop etc.
 			self.setup_sidebar_items()
 			self.set_default_portal_role()
--- a/frappe/core/doctype/file/utils.py
+++ b/frappe/core/doctype/file/utils.py
@ -184,7 +184,7 @@ def remove_file_by_url(file_url: str, doctype: str = None, name: str = None) ->
 def get_content_hash(content: bytes | str) -> str:
 	if isinstance(content, str):
 		content = content.encode()
-	return hashlib.md5(content).hexdigest()  # nosec
+	return hashlib.md5(content, usedforsecurity=False).hexdigest()  # nosec


 def generate_file_name(name: str, suffix: str | None = None, is_private: bool = False) -> str:
@ -376,7 +376,7 @@ def relink_files(doc, fieldname, temp_doc_name):
 			"attached_to_field": fieldname,
 			"creation": (
 				"between",
-				[now_datetime() - add_to_date(date=now_datetime(), minutes=-60), now_datetime()],
+				[add_to_date(date=now_datetime(), minutes=-60), now_datetime()],
 			),
 		},
 	)
--- a/frappe/core/doctype/log_settings/log_settings.py
+++ b/frappe/core/doctype/log_settings/log_settings.py
@ -10,20 +10,6 @@ from frappe.model.document import Document
 from frappe.utils import cint
 from frappe.utils.caching import site_cache

-DEFAULT_LOGTYPES_RETENTION = {
-	"Error Log": 30,
-	"Activity Log": 90,
-	"Email Queue": 30,
-	"Scheduled Job Log": 90,
-	"Route History": 90,
-	"Submission Queue": 30,
-	"Prepared Report": 30,
-	"Webhook Request Log": 30,
-	"Integration Request": 90,
-	"Unhandled Email": 30,
-	"Reminder": 30,
-}
-

@runtime_checkable
 class LogType(Protocol):
@ -81,12 +67,14 @@ class LogSettings(Document):
 	def add_default_logtypes(self):
 		existing_logtypes = {d.ref_doctype for d in self.logs_to_clear}
 		added_logtypes = set()
-		for logtype, retention in DEFAULT_LOGTYPES_RETENTION.items():
+		default_logtypes_retention = frappe.get_hooks("default_log_clearing_doctypes", {})
+
+		for logtype, retentions in default_logtypes_retention.items():
 			if logtype not in existing_logtypes and _supports_log_clearing(logtype):
 				if not frappe.db.exists("DocType", logtype):
 					continue

-				self.append("logs_to_clear", {"ref_doctype": logtype, "days": cint(retention)})
+				self.append("logs_to_clear", {"ref_doctype": logtype, "days": cint(retentions[-1])})
 				added_logtypes.add(logtype)

 		if added_logtypes:
--- a/frappe/core/doctype/recorder/recorder.py
+++ b/frappe/core/doctype/recorder/recorder.py
@ -4,7 +4,7 @@
 import frappe
 from frappe.model.document import Document
 from frappe.recorder import get as get_recorder_data
-from frappe.utils import cint, evaluate_filters, make_filter_dict
+from frappe.utils import cint, evaluate_filters


 class Recorder(Document):
@ -27,6 +27,7 @@ class Recorder(Document):
 		sql_queries: DF.Table[RecorderQuery]
 		time: DF.Datetime | None
 		time_in_queries: DF.Float
+
 	# end: auto-generated types

 	def load_from_db(self):
@ -38,7 +39,7 @@ class Recorder(Document):

 	@staticmethod
 	def get_list(args):
-		start = cint(args.get("start")) or 0
+		start = cint(args.get("start"))
 		page_length = cint(args.get("page_length")) or 20
 		requests = Recorder.get_filtered_requests(args)[start : start + page_length]

--- a/frappe/core/doctype/report/report.py
+++ b/frappe/core/doctype/report/report.py
@ -45,6 +45,7 @@ class Report(Document):
 		report_script: DF.Code | None
 		report_type: DF.Literal["Report Builder", "Query Report", "Script Report", "Custom Report"]
 		roles: DF.Table[HasRole]
+
 	# end: auto-generated types
 	def validate(self):
 		"""only administrator can save standard report"""
@ -129,7 +130,7 @@ class Report(Document):
 		if frappe.flags.in_import:
 			return

-		if self.is_standard == "Yes" and (frappe.local.conf.get("developer_mode") or 0) == 1:
+		if self.is_standard == "Yes" and frappe.conf.developer_mode:
 			export_to_files(
 				record_list=[["Report", self.name]], record_module=self.module, create_init=True
 			)
@ -155,7 +156,6 @@ class Report(Document):
 	def execute_script_report(self, filters):
 		# save the timestamp to automatically set to prepared
 		threshold = 15
-		res = []

 		start_time = datetime.datetime.now()

@ -382,7 +382,7 @@ class Report(Document):


 def is_prepared_report_enabled(report):
-	return cint(frappe.db.get_value("Report", report, "prepared_report")) or 0
+	return cint(frappe.db.get_value("Report", report, "prepared_report"))


 def get_report_module_dotted_path(module, report_name):
--- a/frappe/core/doctype/rq_job/rq_job.py
+++ b/frappe/core/doctype/rq_job/rq_job.py
@ -59,6 +59,7 @@ class RQJob(Document):
 		]
 		time_taken: DF.Duration | None
 		timeout: DF.Duration | None
+
 	# end: auto-generated types
 	def load_from_db(self):
 		try:
@ -79,7 +80,7 @@ class RQJob(Document):
 	@staticmethod
 	def get_list(args):

-		start = cint(args.get("start")) or 0
+		start = cint(args.get("start"))
 		page_length = cint(args.get("page_length")) or 20

 		order_desc = "desc" in args.get("order_by", "")
@ -87,7 +88,9 @@ class RQJob(Document):
 		matched_job_ids = RQJob.get_matching_job_ids(args)[start : start + page_length]

 		conn = get_redis_conn()
-		jobs = [serialize_job(job) for job in Job.fetch_many(job_ids=matched_job_ids, connection=conn)]
+		jobs = [
+			serialize_job(job) for job in Job.fetch_many(job_ids=matched_job_ids, connection=conn) if job
+		]

 		return sorted(jobs, key=lambda j: j.modified, reverse=order_desc)

--- a/frappe/core/doctype/rq_worker/rq_worker.py
+++ b/frappe/core/doctype/rq_worker/rq_worker.py
@ -4,6 +4,7 @@
 import datetime
 from contextlib import suppress

+import pytz
 from rq import Worker

 import frappe
@ -33,6 +34,7 @@ class RQWorker(Document):
 		total_working_time: DF.Duration | None
 		utilization_percent: DF.Percent
 		worker_name: DF.Data | None
+
 	# end: auto-generated types
 	def load_from_db(self):

@ -46,7 +48,7 @@ class RQWorker(Document):

 	@staticmethod
 	def get_list(args):
-		start = cint(args.get("start")) or 0
+		start = cint(args.get("start"))
 		page_length = cint(args.get("page_length")) or 20

 		workers = get_workers()
@ -105,5 +107,7 @@ def serialize_worker(worker: Worker) -> frappe._dict:

 def compute_utilization(worker: Worker) -> float:
 	with suppress(Exception):
-		total_time = (datetime.datetime.utcnow() - worker.birth_date).total_seconds()
+		total_time = (
+			datetime.datetime.now(pytz.UTC) - worker.birth_date.replace(tzinfo=pytz.UTC)
+		).total_seconds()
 		return worker.total_working_time / total_time * 100
--- a/frappe/core/doctype/scheduled_job_log/scheduled_job_log.json
+++ b/frappe/core/doctype/scheduled_job_log/scheduled_job_log.json
@ -7,7 +7,8 @@
 "field_order": [
  "status",
  "scheduled_job_type",
-  "details"
+  "details",
+  "debug_log"
 ],
 "fields": [
  {
@ -35,10 +36,16 @@
   "options": "Scheduled Job Type",
   "read_only": 1,
   "reqd": 1
+  },
+  {
+   "fieldname": "debug_log",
+   "fieldtype": "Code",
+   "label": "Debug Log",
+   "read_only": 1
  }
 ],
 "links": [],
- "modified": "2022-06-13 05:41:21.090972",
+ "modified": "2023-11-09 12:06:41.781270",
 "modified_by": "Administrator",
 "module": "Core",
 "name": "Scheduled Job Log",
--- a/frappe/core/doctype/scheduled_job_log/scheduled_job_log.py
+++ b/frappe/core/doctype/scheduled_job_log/scheduled_job_log.py
@ -16,6 +16,7 @@ class ScheduledJobLog(Document):
 	if TYPE_CHECKING:
 		from frappe.types import DF

+		debug_log: DF.Code | None
 		details: DF.Code | None
 		scheduled_job_type: DF.Link
 		status: DF.Literal["Scheduled", "Complete", "Failed"]
--- a/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py
+++ b/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py
@ -2,7 +2,8 @@
 # License: MIT. See LICENSE

 import json
-from datetime import datetime
+from datetime import datetime, timedelta
+from random import randint

 import click
 from croniter import croniter
@ -110,7 +111,12 @@ class ScheduledJobType(Document):
 		# immediately, even when it's meant to be daily.
 		# A dynamic fallback like current time might miss the scheduler interval and job will never start.
 		last_execution = get_datetime(self.last_execution or self.creation)
-		return croniter(self.cron_format, last_execution).get_next(datetime)
+		next_execution = croniter(self.cron_format, last_execution).get_next(datetime)
+
+		jitter = 0
+		if self.frequency in ("Hourly Long", "Daily Long"):
+			jitter = randint(1, 600)
+		return next_execution + timedelta(seconds=jitter)

 	def execute(self):
 		self.scheduler_log = None
@ -145,6 +151,8 @@ class ScheduledJobType(Document):
 				dict(doctype="Scheduled Job Log", scheduled_job_type=self.name)
 			).insert(ignore_permissions=True)
 		self.scheduler_log.db_set("status", status)
+		if frappe.debug_log:
+			self.scheduler_log.db_set("debug_log", "\n".join(frappe.debug_log))
 		if status == "Failed":
 			self.scheduler_log.db_set("details", frappe.get_traceback())
 		if status == "Start":
--- a/frappe/core/doctype/system_settings/system_settings.json
+++ b/frappe/core/doctype/system_settings/system_settings.json
@ -23,38 +23,23 @@
  "float_precision",
  "currency_precision",
  "rounding_method",
-  "sec_backup_limit",
-  "backup_limit",
-  "encrypt_backup",
-  "background_workers",
-  "enable_scheduler",
-  "dormant_days",
  "permissions",
  "apply_strict_user_permissions",
  "column_break_21",
-  "allow_guests_to_upload_files",
-  "force_web_capture_mode_for_uploads",
+  "allow_older_web_view_links",
+  "security_tab",
  "security",
  "session_expiry",
  "document_share_key_expiry",
-  "column_break_13",
+  "column_break_txqh",
  "deny_multiple_sessions",
+  "disable_user_pass_login",
+  "login_methods_section",
  "allow_login_using_mobile_number",
  "allow_login_using_user_name",
-  "disable_user_pass_login",
+  "column_break_uhqk",
  "login_with_email_link",
  "login_with_email_link_expiry",
-  "allow_error_traceback",
-  "strip_exif_metadata_from_uploaded_images",
-  "allow_older_web_view_links",
-  "password_settings",
-  "logout_on_password_reset",
-  "force_user_to_reset_password",
-  "reset_password_link_expiry_duration",
-  "password_reset_limit",
-  "column_break_31",
-  "enable_password_policy",
-  "minimum_password_score",
  "brute_force_security",
  "allow_consecutive_login_attempts",
  "column_break_34",
@ -66,6 +51,16 @@
  "two_factor_method",
  "lifespan_qrcode_image",
  "otp_issuer_name",
+  "password_tab",
+  "password_settings",
+  "logout_on_password_reset",
+  "force_user_to_reset_password",
+  "reset_password_link_expiry_duration",
+  "password_reset_limit",
+  "column_break_31",
+  "enable_password_policy",
+  "minimum_password_score",
+  "email_tab",
  "email",
  "email_footer_address",
  "email_retry_limit",
@ -75,17 +70,31 @@
  "attach_view_link",
  "welcome_email_template",
  "reset_password_template",
-  "prepared_report_section",
-  "max_auto_email_report_per_user",
+  "files_tab",
+  "files_section",
+  "max_file_size",
+  "allow_guests_to_upload_files",
+  "force_web_capture_mode_for_uploads",
+  "strip_exif_metadata_from_uploaded_images",
+  "column_break_uqma",
+  "allowed_file_extensions",
+  "updates_tab",
  "system_updates_section",
  "disable_system_update_notification",
  "disable_change_log_notification",
+  "backups_tab",
+  "sec_backup_limit",
+  "backup_limit",
+  "encrypt_backup",
+  "advanced_tab",
+  "prepared_report_section",
+  "max_auto_email_report_per_user",
+  "background_workers",
+  "enable_scheduler",
+  "dormant_days",
  "telemetry_section",
-  "enable_telemetry",
-  "files_section",
-  "max_file_size",
-  "column_break_uqma",
-  "allowed_file_extensions"
+  "allow_error_traceback",
+  "enable_telemetry"
 ],
 "fields": [
  {
@ -126,7 +135,6 @@
   "read_only": 1
  },
  {
-   "collapsible": 1,
   "fieldname": "date_and_number_format",
   "fieldtype": "Section Break",
   "label": "Date and Number Format"
@ -171,10 +179,8 @@
   "options": "\n0\n1\n2\n3\n4\n5\n6\n7\n8\n9"
  },
  {
-   "collapsible": 1,
   "fieldname": "sec_backup_limit",
-   "fieldtype": "Section Break",
-   "label": "Backups"
+   "fieldtype": "Section Break"
  },
  {
   "default": "3",
@ -184,7 +190,6 @@
   "label": "Number of Backups"
  },
  {
-   "collapsible": 1,
   "fieldname": "background_workers",
   "fieldtype": "Section Break",
   "label": "Background Workers"
@ -198,7 +203,6 @@
   "label": "Enable Scheduled Jobs"
  },
  {
-   "collapsible": 1,
   "fieldname": "permissions",
   "fieldtype": "Section Break",
   "label": "Permissions"
@ -211,10 +215,8 @@
   "label": "Apply Strict User Permissions"
  },
  {
-   "collapsible": 1,
   "fieldname": "security",
-   "fieldtype": "Section Break",
-   "label": "Security"
+   "fieldtype": "Section Break"
  },
  {
   "default": "170:00",
@ -223,10 +225,6 @@
   "fieldtype": "Data",
   "label": "Session Expiry (idle timeout)"
  },
-  {
-   "fieldname": "column_break_13",
-   "fieldtype": "Column Break"
-  },
  {
   "default": "0",
   "description": "Note: Multiple sessions will be allowed in case of mobile device",
@ -255,7 +253,6 @@
   "label": "Show Full Error and Allow Reporting of Issues to the Developer"
  },
  {
-   "collapsible": 1,
   "fieldname": "password_settings",
   "fieldtype": "Section Break",
   "label": "Password"
@ -286,7 +283,6 @@
   "options": "2\n3\n4"
  },
  {
-   "collapsible": 1,
   "fieldname": "brute_force_security",
   "fieldtype": "Section Break",
   "label": "Brute Force Security"
@ -309,7 +305,6 @@
   "label": "Allow Login After Fail"
  },
  {
-   "collapsible": 1,
   "fieldname": "two_factor_authentication",
   "fieldtype": "Section Break",
   "label": "Two Factor Authentication"
@ -338,6 +333,7 @@
  },
  {
   "default": "OTP App",
+   "depends_on": "enable_two_factor_auth",
   "description": "Choose authentication method to be used by all users",
   "fieldname": "two_factor_method",
   "fieldtype": "Select",
@ -345,7 +341,7 @@
   "options": "OTP App\nSMS\nEmail"
  },
  {
-   "depends_on": "eval:doc.two_factor_method == \"OTP App\"",
+   "depends_on": "eval:doc.enable_two_factor_auth && doc.two_factor_method == \"OTP App\"",
   "description": "Time in seconds to retain QR code image on server. Min:<strong>240</strong>",
   "fieldname": "lifespan_qrcode_image",
   "fieldtype": "Int",
@ -359,10 +355,8 @@
   "label": "OTP Issuer Name"
  },
  {
-   "collapsible": 1,
   "fieldname": "email",
-   "fieldtype": "Section Break",
-   "label": "Email"
+   "fieldtype": "Section Break"
  },
  {
   "description": "Your organization name and address for the email footer.",
@ -430,7 +424,6 @@
   "label": "Include Web View Link in Email"
  },
  {
-   "collapsible": 1,
   "fieldname": "prepared_report_section",
   "fieldtype": "Section Break",
   "label": "Reports"
@ -456,10 +449,8 @@
   "label": "Encrypt Backups"
  },
  {
-   "collapsible": 1,
   "fieldname": "system_updates_section",
-   "fieldtype": "Section Break",
-   "label": "System Updates"
+   "fieldtype": "Section Break"
  },
  {
   "default": "0",
@ -547,7 +538,6 @@
   "label": "Disable Document Sharing"
  },
  {
-   "collapsible": 1,
   "fieldname": "telemetry_section",
   "fieldtype": "Section Break",
   "label": "Telemetry"
@ -578,10 +568,8 @@
   "label": "Force Web Capture Mode for Uploads"
  },
  {
-   "collapsible": 1,
   "fieldname": "files_section",
-   "fieldtype": "Section Break",
-   "label": "Files"
+   "fieldtype": "Section Break"
  },
  {
   "fieldname": "max_file_size",
@ -598,12 +586,60 @@
   "fieldname": "allowed_file_extensions",
   "fieldtype": "Small Text",
   "label": "Allowed File Extensions"
+  },
+  {
+   "fieldname": "security_tab",
+   "fieldtype": "Tab Break",
+   "label": "Login"
+  },
+  {
+   "fieldname": "email_tab",
+   "fieldtype": "Tab Break",
+   "label": "Email"
+  },
+  {
+   "fieldname": "files_tab",
+   "fieldtype": "Tab Break",
+   "label": "Files"
+  },
+  {
+   "fieldname": "updates_tab",
+   "fieldtype": "Tab Break",
+   "label": "Updates"
+  },
+  {
+   "fieldname": "backups_tab",
+   "fieldtype": "Tab Break",
+   "label": "Backups"
+  },
+  {
+   "fieldname": "advanced_tab",
+   "fieldtype": "Tab Break",
+   "label": "Advanced"
+  },
+  {
+   "fieldname": "password_tab",
+   "fieldtype": "Tab Break",
+   "label": "Password"
+  },
+  {
+   "fieldname": "column_break_txqh",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "login_methods_section",
+   "fieldtype": "Section Break",
+   "label": "Login Methods"
+  },
+  {
+   "fieldname": "column_break_uhqk",
+   "fieldtype": "Column Break"
  }
 ],
 "icon": "fa fa-cog",
 "issingle": 1,
 "links": [],
- "modified": "2023-10-17 16:12:28.145496",
+ "modified": "2023-11-27 14:08:01.927794",
 "modified_by": "Administrator",
 "module": "Core",
 "name": "System Settings",
--- a/frappe/core/doctype/system_settings/system_settings.py
+++ b/frappe/core/doctype/system_settings/system_settings.py
@ -97,8 +97,8 @@ class SystemSettings(Document):
 	def validate(self):
 		from frappe.twofactor import toggle_two_factor_auth

-		enable_password_policy = cint(self.enable_password_policy) and True or False
-		minimum_password_score = cint(getattr(self, "minimum_password_score", 0)) or 0
+		enable_password_policy = cint(self.enable_password_policy)
+		minimum_password_score = cint(getattr(self, "minimum_password_score", 0))
 		if enable_password_policy and minimum_password_score <= 0:
 			frappe.throw(_("Please select Minimum Password Score"))
 		elif not enable_password_policy:
@ -195,7 +195,7 @@ def update_last_reset_password_date():
 def load():
 	from frappe.utils.momentjs import get_all_timezones

-	if not "System Manager" in frappe.get_roles():
+	if "System Manager" not in frappe.get_roles():
 		frappe.throw(_("Not permitted"), frappe.PermissionError)

 	all_defaults = frappe.db.get_defaults()
--- a/frappe/core/doctype/user/user.py
+++ b/frappe/core/doctype/user/user.py
@ -9,6 +9,7 @@ import frappe.defaults
 import frappe.permissions
 import frappe.share
 from frappe import STANDARD_USERS, _, msgprint, throw
+from frappe.auth import MAX_PASSWORD_SIZE
 from frappe.core.doctype.user_type.user_type import user_linked_with_permission_on_doctype
 from frappe.desk.doctype.notification_settings.notification_settings import (
 	create_notification_settings,
@ -823,6 +824,9 @@ def update_password(
 	        old_password (str, optional): Old password. Defaults to None.
 	"""

+	if len(new_password) > MAX_PASSWORD_SIZE:
+		frappe.throw(_("Password size exceeded the maximum allowed size."))
+
 	result = test_password_strength(new_password)
 	feedback = result.get("feedback", None)

@ -872,7 +876,7 @@ def test_password_strength(
 			"Arguments `key` and `old_password` are deprecated in function `test_password_strength`."
 		)

-	enable_password_policy = frappe.get_system_settings("enable_password_policy") or 0
+	enable_password_policy = frappe.get_system_settings("enable_password_policy")

 	if not enable_password_policy:
 		return {}
@ -885,7 +889,7 @@ def test_password_strength(
 	if new_password:
 		result = _test_password_strength(new_password, user_inputs=user_data)
 		password_policy_validation_passed = False
-		minimum_password_score = cint(frappe.get_system_settings("minimum_password_score")) or 0
+		minimum_password_score = cint(frappe.get_system_settings("minimum_password_score"))

 		# score should be greater than 0 and minimum_password_score
 		if result.get("score") and result.get("score") >= minimum_password_score:
@ -1223,27 +1227,31 @@ def create_contact(user, ignore_links=False, ignore_mandatory=False):

 	contact_name = get_contact_name(user.email)
 	if not contact_name:
-		contact = frappe.get_doc(
-			{
-				"doctype": "Contact",
-				"first_name": user.first_name,
-				"last_name": user.last_name,
-				"user": user.name,
-				"gender": user.gender,
-			}
-		)
+		try:
+			contact = frappe.get_doc(
+				{
+					"doctype": "Contact",
+					"first_name": user.first_name,
+					"last_name": user.last_name,
+					"user": user.name,
+					"gender": user.gender,
+				}
+			)

-		if user.email:
-			contact.add_email(user.email, is_primary=True)
+			if user.email:
+				contact.add_email(user.email, is_primary=True)

-		if user.phone:
-			contact.add_phone(user.phone, is_primary_phone=True)
+			if user.phone:
+				contact.add_phone(user.phone, is_primary_phone=True)

-		if user.mobile_no:
-			contact.add_phone(user.mobile_no, is_primary_mobile_no=True)
-		contact.insert(
-			ignore_permissions=True, ignore_links=ignore_links, ignore_mandatory=ignore_mandatory
-		)
+			if user.mobile_no:
+				contact.add_phone(user.mobile_no, is_primary_mobile_no=True)
+
+			contact.insert(
+				ignore_permissions=True, ignore_links=ignore_links, ignore_mandatory=ignore_mandatory
+			)
+		except frappe.DuplicateEntryError:
+			pass
 	else:
 		contact = frappe.get_doc("Contact", contact_name)
 		contact.first_name = user.first_name
--- a/frappe/core/doctype/user_type/user_type.py
+++ b/frappe/core/doctype/user_type/user_type.py
@ -31,6 +31,7 @@ class UserType(Document):
 		user_doctypes: DF.Table[UserDocumentType]
 		user_id_field: DF.Literal
 		user_type_modules: DF.Table[UserTypeModule]
+
 	# end: auto-generated types
 	def validate(self):
 		self.set_modules()
@ -140,7 +141,7 @@ class UserType(Document):
 		for row in self.user_doctypes:
 			docperm = add_role_permissions(row.document_type, self.role)

-			values = {perm: row.get(perm) or 0 for perm in perms}
+			values = {perm: row.get(perm, default=0) for perm in perms}
 			for perm in ["print", "email", "share"]:
 				values[perm] = 1

--- a/frappe/custom/doctype/custom_field/custom_field.json
+++ b/frappe/custom/doctype/custom_field/custom_field.json
@ -15,6 +15,7 @@
  "fieldname",
  "insert_after",
  "length",
+  "link_filters",
  "column_break_6",
  "fieldtype",
  "precision",
@ -444,6 +445,12 @@
   "fieldname": "sort_options",
   "fieldtype": "Check",
   "label": "Sort Options"
+  },
+  {
+   "fieldname": "link_filters",
+   "fieldtype": "JSON",
+   "hidden": 1,
+   "label": "Link Filters"
  }
 ],
 "icon": "fa fa-glass",
--- a/frappe/custom/doctype/custom_field/custom_field.py
+++ b/frappe/custom/doctype/custom_field/custom_field.py
@ -94,6 +94,7 @@ class CustomField(Document):
 		is_virtual: DF.Check
 		label: DF.Data | None
 		length: DF.Int
+		link_filters: DF.JSON | None
 		mandatory_depends_on: DF.Code | None
 		module: DF.Link | None
 		no_copy: DF.Check
--- a/frappe/custom/doctype/customize_form/customize_form.js
+++ b/frappe/custom/doctype/customize_form/customize_form.js
@ -84,7 +84,7 @@ frappe.ui.form.on("Customize Form", {
 			if (!in_list(["Table", "Table MultiSelect"], f.fieldtype)) return;

 			frm.add_custom_button(
-				f.options,
+				__(f.options),
 				() => frm.set_value("doc_type", f.options),
 				__("Customize Child Table")
 			);
@ -97,7 +97,7 @@ frappe.ui.form.on("Customize Form", {

 		if (frm.doc.doc_type) {
 			frappe.model.with_doctype(frm.doc.doc_type).then(() => {
-				frm.page.set_title(__("Customize Form - {0}", [frm.doc.doc_type]));
+				frm.page.set_title(__("Customize Form - {0}", [__(frm.doc.doc_type)]));
 				frappe.customize_form.set_primary_action(frm);

 				frm.add_custom_button(
@ -149,6 +149,7 @@ frappe.ui.form.on("Customize Form", {
 				);

 				render_form_builder(frm);
+				frm.get_field("form_builder").tab.set_active();
 			});
 		}

--- a/frappe/custom/doctype/customize_form/customize_form.json
+++ b/frappe/custom/doctype/customize_form/customize_form.json
@ -7,10 +7,12 @@
 "editable_grid": 1,
 "engine": "InnoDB",
 "field_order": [
+  "details_tab",
  "doc_type",
  "properties",
  "label",
  "search_fields",
+  "link_filters",
  "column_break_5",
  "istable",
  "is_calendar_and_gantt",
@ -24,17 +26,6 @@
  "naming_section",
  "naming_rule",
  "autoname",
-  "document_actions_section",
-  "actions",
-  "document_links_section",
-  "links",
-  "document_states_section",
-  "states",
-  "form_tab",
-  "form_builder",
-  "fields_section_break",
-  "fields",
-  "settings_tab",
  "form_settings_section",
  "image_field",
  "max_attachments",
@ -59,7 +50,17 @@
  "section_break_8",
  "sort_field",
  "column_break_10",
-  "sort_order"
+  "sort_order",
+  "document_actions_section",
+  "actions",
+  "document_links_section",
+  "links",
+  "document_states_section",
+  "states",
+  "fields_section_break",
+  "fields",
+  "form_tab",
+  "form_builder"
 ],
 "fields": [
  {
@ -180,7 +181,6 @@
   "depends_on": "doc_type",
   "fieldname": "fields_section_break",
   "fieldtype": "Section Break",
-   "hidden": 1,
   "label": "Fields"
  },
  {
@ -372,11 +372,6 @@
   "fieldtype": "Check",
   "label": "Is Calendar and Gantt"
  },
-  {
-   "fieldname": "settings_tab",
-   "fieldtype": "Tab Break",
-   "label": "Settings"
-  },
  {
   "fieldname": "form_builder",
   "fieldtype": "HTML",
@ -386,6 +381,17 @@
   "fieldname": "form_tab",
   "fieldtype": "Tab Break",
   "label": "Form"
+  },
+  {
+   "fieldname": "link_filters",
+   "fieldtype": "JSON",
+   "hidden": 1,
+   "label": "Link Filters"
+  },
+  {
+   "fieldname": "details_tab",
+   "fieldtype": "Tab Break",
+   "label": "Details"
  }
 ],
 "hide_toolbar": 1,
@ -394,7 +400,7 @@
 "index_web_pages_for_search": 1,
 "issingle": 1,
 "links": [],
- "modified": "2023-10-31 02:04:25.955931",
+ "modified": "2023-11-16 11:23:06.427432",
 "modified_by": "Administrator",
 "module": "Custom",
 "name": "Customize Form",
--- a/frappe/custom/doctype/customize_form/customize_form.py
+++ b/frappe/custom/doctype/customize_form/customize_form.py
@ -54,6 +54,7 @@ class CustomizeForm(Document):
 		is_calendar_and_gantt: DF.Check
 		istable: DF.Check
 		label: DF.Data | None
+		link_filters: DF.JSON | None
 		links: DF.Table[DocTypeLink]
 		make_attachments_public: DF.Check
 		max_attachments: DF.Int
@ -681,6 +682,17 @@ def is_standard_or_system_generated_field(df):
 	return not df.get("is_custom_field") or df.get("is_system_generated")


+@frappe.whitelist()
+def get_link_filters_from_doc_without_customisations(doctype, fieldname):
+	"""Get the filters of a link field from a doc without customisations
+	In backend the customisations are not applied.
+	Customisations are applied in the client side.
+	"""
+	doc = frappe.get_doc("DocType", doctype)
+	field = list(filter(lambda x: x.fieldname == fieldname, doc.fields))
+	return field[0].link_filters
+
+
 doctype_properties = {
 	"search_fields": "Data",
 	"title_field": "Data",
@ -761,6 +773,7 @@ docfield_properties = {
 	"hide_days": "Check",
 	"hide_seconds": "Check",
 	"is_virtual": "Check",
+	"link_filters": "JSON",
 }

 doctype_link_properties = {
--- a/frappe/custom/doctype/customize_form_field/customize_form_field.json
+++ b/frappe/custom/doctype/customize_form_field/customize_form_field.json
@ -24,6 +24,7 @@
  "no_copy",
  "allow_in_quick_entry",
  "translatable",
+  "link_filters",
  "column_break_7",
  "default",
  "precision",
@ -204,7 +205,7 @@
   "label": "Permissions"
  },
  {
-   "description": "This field will appear only if the fieldname defined here has value OR the rules are true (examples): \nmyfield\neval:doc.myfield=='My Value'\neval:doc.age&gt;18",
+   "description": "This field will appear only if the fieldname defined here has value OR the rules are true (examples):\nmyfield\neval:doc.myfield=='My Value'\neval:doc.age&gt;18",
   "fieldname": "depends_on",
   "fieldtype": "Code",
   "label": "Depends On",
@ -471,13 +472,18 @@
   "fieldname": "sort_options",
   "fieldtype": "Check",
   "label": "Sort Options"
+  },
+  {
+   "fieldname": "link_filters",
+   "fieldtype": "JSON",
+   "label": "Link Filters"
  }
 ],
 "idx": 1,
 "index_web_pages_for_search": 1,
 "istable": 1,
 "links": [],
- "modified": "2023-10-25 06:55:50.718441",
+ "modified": "2023-11-07 13:17:21.373626",
 "modified_by": "Administrator",
 "module": "Custom",
 "name": "Customize Form Field",
--- a/frappe/custom/doctype/customize_form_field/customize_form_field.py
+++ b/frappe/custom/doctype/customize_form_field/customize_form_field.py
@ -86,6 +86,7 @@ class CustomizeFormField(Document):
 		is_virtual: DF.Check
 		label: DF.Data | None
 		length: DF.Int
+		link_filters: DF.JSON | None
 		mandatory_depends_on: DF.Code | None
 		no_copy: DF.Check
 		non_negative: DF.Check
--- a/frappe/database/init.py
+++ b/frappe/database/init.py
@ -3,25 +3,39 @@

 # Database Module
 # --------------------
+from shutil import which

 from frappe.database.database import savepoint


-def setup_database(force, source_sql=None, verbose=None, no_mariadb_socket=False):
+def setup_database(force, verbose=None, no_mariadb_socket=False):
 	import frappe

 	if frappe.conf.db_type == "postgres":
 		import frappe.database.postgres.setup_db

-		return frappe.database.postgres.setup_db.setup_database(force, source_sql, verbose)
+		return frappe.database.postgres.setup_db.setup_database()
 	else:
 		import frappe.database.mariadb.setup_db

 		return frappe.database.mariadb.setup_db.setup_database(
-			force, source_sql, verbose, no_mariadb_socket=no_mariadb_socket
+			force, verbose, no_mariadb_socket=no_mariadb_socket
 		)


+def bootstrap_database(db_name, verbose=None, source_sql=None):
+	import frappe
+
+	if frappe.conf.db_type == "postgres":
+		import frappe.database.postgres.setup_db
+
+		return frappe.database.postgres.setup_db.bootstrap_database(db_name, verbose, source_sql)
+	else:
+		import frappe.database.mariadb.setup_db
+
+		return frappe.database.mariadb.setup_db.bootstrap_database(db_name, verbose, source_sql)
+
+
 def drop_user_and_database(db_name, root_login=None, root_password=None):
 	import frappe

@ -50,3 +64,74 @@ def get_db(host=None, user=None, password=None, port=None):
 		import frappe.database.mariadb.database

 		return frappe.database.mariadb.database.MariaDBDatabase(host, user, password, port=port)
+
+
+def get_command(
+	host=None, port=None, user=None, password=None, db_name=None, extra=None, dump=False
+):
+	import frappe
+
+	if frappe.conf.db_type == "postgres":
+		if dump:
+			bin, bin_name = which("pg_dump"), "pg_dump"
+		else:
+			bin, bin_name = which("psql"), "psql"
+
+		host = frappe.utils.esc(host, "$ ")
+		user = frappe.utils.esc(user, "$ ")
+		db_name = frappe.utils.esc(db_name, "$ ")
+
+		if password:
+			password = frappe.utils.esc(password, "$ ")
+			conn_string = f"postgresql://{user}:{password}@{host}:{port}/{db_name}"
+		else:
+			conn_string = f"postgresql://{user}@{host}:{port}/{db_name}"
+
+		command = [conn_string]
+
+		if extra:
+			command.extend(extra)
+
+	else:
+		if dump:
+			bin, bin_name = which("mariadb-dump") or which("mysqldump"), "mariadb-dump"
+		else:
+			bin, bin_name = which("mariadb") or which("mysql"), "mariadb"
+
+		host = frappe.utils.esc(host, "$ ")
+		user = frappe.utils.esc(user, "$ ")
+		db_name = frappe.utils.esc(db_name, "$ ")
+
+		command = [
+			f"--user={user}",
+			f"--host={host}",
+			f"--port={port}",
+		]
+
+		if password:
+			password = frappe.utils.esc(password, "$ ")
+			command.append(f"--password={password}")
+
+		if dump:
+			command.extend(
+				[
+					"--single-transaction",
+					"--quick",
+					"--lock-tables=false",
+				]
+			)
+		else:
+			command.extend(
+				[
+					"--pager=less -SFX",
+					"--safe-updates",
+					"--no-auto-rehash",
+				]
+			)
+
+		command.append(db_name)
+
+		if extra:
+			command.extend(extra)
+
+	return bin, command, bin_name
--- a/frappe/database/database.py
+++ b/frappe/database/database.py
@ -1,7 +1,6 @@
 # Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE

-import datetime
 import itertools
 import json
 import random
@ -14,7 +13,6 @@ from time import time
 from typing import Any

 from pypika.dialects import MySQLQueryBuilder, PostgreSQLQueryBuilder
-from pypika.terms import Criterion, NullValue

 import frappe
 import frappe.defaults
@ -163,6 +161,8 @@ class Database:
 		:param auto_commit: Commit after executing the query.
 		:param update: Update this dict to all rows (if returned `as_dict`).
 		:param run: Returns query without executing it if False.
+		:param pluck: Get the plucked field only.
+		:param explain: Print `EXPLAIN` in error log.
 		Examples:

 		        # return customer names as dicts
@ -369,7 +369,7 @@ class Database:
 		self.commit()
 		self.sql(query, debug=debug)

-	def check_transaction_status(self, query):
+	def check_transaction_status(self, query: str):
 		"""Raises exception if more than 200,000 `INSERT`, `UPDATE` queries are
 		executed in one transaction. This is to ensure that writes are always flushed otherwise this
 		could cause the system to hang."""
@ -388,13 +388,13 @@ class Database:
 					msg += _("The changes have been reverted.") + "<br>"
 					raise frappe.TooManyWritesError(msg)

-	def check_implicit_commit(self, query):
+	def check_implicit_commit(self, query: str):
 		if (
 			self.transaction_writes
 			and query
 			and is_query_type(query, ("start", "alter", "drop", "create", "begin", "truncate"))
 		):
-			raise ImplicitCommitError("This statement can cause implicit commit")
+			raise ImplicitCommitError("This statement can cause implicit commit", query)

 	def fetch_as_dict(self) -> list[frappe._dict]:
 		"""Internal. Converts results to dict."""
--- a/frappe/database/db_manager.py
+++ b/frappe/database/db_manager.py
@ -1,4 +1,5 @@
 import frappe
+from frappe import _


 class DbManager:
@ -49,37 +50,38 @@ class DbManager:
 		return self.db.sql("SHOW DATABASES", pluck=True)

 	@staticmethod
-	def restore_database(target, source, user, password):
-		import os
+	def restore_database(verbose, target, source, user, password):
+		import shlex
 		from shutil import which

-		from frappe.utils import make_esc
+		from frappe.database import get_command
+		from frappe.utils import execute_in_shell

-		esc = make_esc("$ ")
 		pv = which("pv")
-		mariadb_cli = which("mariadb") or which("mysql")
+
+		command = []

 		if pv:
-			pipe = f"{pv} {source} |"
-			source = ""
-		else:
-			pipe = ""
-			source = f"< {source}"
-
-		if pipe:
+			command.extend([pv, source, "|"])
+			source = []
 			print("Restoring Database file...")
+		else:
+			source = ["<", source]

-		command = "{pipe} {mariadb_cli} -u {user} -p{password} -h{host} -P{port} {target} {source}"
-		command = command.format(
-			pipe=pipe,
-			user=esc(user),
-			password=esc(password),
-			host=esc(frappe.conf.db_host),
-			target=esc(target),
-			source=source,
+		bin, args, bin_name = get_command(
+			host=frappe.conf.db_host,
 			port=frappe.conf.db_port,
-			mariadb_cli=mariadb_cli,
+			user=user,
+			password=password,
+			db_name=target,
 		)
-
-		os.system(command)
+		if not bin:
+			frappe.throw(
+				_("{} not found in PATH! This is required to restore the database.").format(bin_name),
+				exc=frappe.ExecutableNotFound,
+			)
+		command.append(bin)
+		command.append(shlex.join(args))
+		command.extend(source)
+		execute_in_shell(" ".join(command), check_exit_code=True, verbose=verbose)
 		frappe.cache.delete_keys("")  # Delete all keys associated with this site.
--- a/frappe/database/mariadb/database.py
+++ b/frappe/database/mariadb/database.py
@ -310,7 +310,7 @@ class MariaDBDatabase(MariaDBConnectionUtil, MariaDBExceptionUtil, Database):
 		)

 	@staticmethod
-	def get_on_duplicate_update(key=None):
+	def get_on_duplicate_update():
 		return "ON DUPLICATE key UPDATE "

 	def get_table_columns_description(self, table_name):
@ -329,7 +329,8 @@ class MariaDBDatabase(MariaDBConnectionUtil, MariaDBExceptionUtil, Database):
 					and Seq_in_index = 1
 					limit 1
 			), 0) as 'index',
-			column_key = 'UNI' as 'unique'
+			column_key = 'UNI' as 'unique',
+			(is_nullable = 'NO') AS 'not_nullable'
 			from information_schema.columns as columns
 			where table_name = '{table_name}' """.format(
 				table_name=table_name
--- a/frappe/database/mariadb/schema.py
+++ b/frappe/database/mariadb/schema.py
@ -3,6 +3,7 @@ from pymysql.constants.ER import DUP_ENTRY
 import frappe
 from frappe import _
 from frappe.database.schema import DBTable
+from frappe.utils.defaults import get_not_null_defaults


 class MariaDBTable(DBTable):
@ -23,7 +24,7 @@ class MariaDBTable(DBTable):
 			additional_definitions += index_defs

 		# child table columns
-		if self.meta.get("istable") or 0:
+		if self.meta.get("istable", default=0):
 			additional_definitions += [
 				f"parent varchar({varchar_len})",
 				f"parentfield varchar({varchar_len})",
@ -69,7 +70,7 @@ class MariaDBTable(DBTable):
 		add_column_query = [
 			f"ADD COLUMN `{col.fieldname}` {col.get_definition()}" for col in self.add_column
 		]
-		columns_to_modify = set(self.change_type + self.set_default)
+		columns_to_modify = set(self.change_type + self.set_default + self.change_nullability)
 		modify_column_query = [
 			f"MODIFY `{col.fieldname}` {col.get_definition(for_modification=True)}"
 			for col in columns_to_modify
@ -102,12 +103,23 @@ class MariaDBTable(DBTable):
 				if index_record := frappe.db.get_column_index(self.table_name, col.fieldname, unique=False):
 					drop_index_query.append(f"DROP INDEX `{index_record.Key_name}`")

+		for col in self.change_nullability:
+			if col.not_nullable:
+				try:
+					table = frappe.qb.DocType(self.doctype)
+					frappe.qb.update(table).set(
+						col.fieldname, col.default or get_not_null_defaults(col.fieldtype)
+					).where(table[col.fieldname].isnull()).run()
+				except Exception:
+					print(f"Failed to update data in {self.table_name} for {col.fieldname}")
+					raise
 		try:
 			for query_parts in [add_column_query, modify_column_query, add_index_query, drop_index_query]:
 				if query_parts:
 					query_body = ", ".join(query_parts)
 					query = f"ALTER TABLE `{self.table_name}` {query_body}"
-					frappe.db.sql(query)
+					# nosemgrep
+					frappe.db.sql_ddl(query)

 		except Exception as e:
 			if query := locals().get("query"):  # this weirdness is to avoid potentially unbounded vars
--- a/frappe/database/mariadb/setup_db.py
+++ b/frappe/database/mariadb/setup_db.py
@ -23,7 +23,7 @@ def get_mariadb_version(version_string: str = ""):
 	return version.rsplit(".", 1)


-def setup_database(force, source_sql, verbose, no_mariadb_socket=False):
+def setup_database(force, verbose, no_mariadb_socket=False):
 	frappe.local.session = frappe._dict({"user": "Administrator"})

 	db_name = frappe.local.conf.db_name
@ -55,8 +55,6 @@ def setup_database(force, source_sql, verbose, no_mariadb_socket=False):
 	# close root connection
 	root_conn.close()

-	bootstrap_database(db_name, verbose, source_sql)
-

 def drop_user_and_database(db_name, root_login, root_password):
 	frappe.local.db = get_root_connection(root_login, root_password)
@ -75,8 +73,8 @@ def bootstrap_database(db_name, verbose, source_sql=None):
 		sys.exit(1)

 	import_db_from_sql(source_sql, verbose)
-
 	frappe.connect(db_name=db_name)
+
 	if "tabDefaultValue" not in frappe.db.get_tables(cached=False):
 		from click import secho

@ -97,7 +95,9 @@ def import_db_from_sql(source_sql=None, verbose=False):
 	db_name = frappe.conf.db_name
 	if not source_sql:
 		source_sql = os.path.join(os.path.dirname(__file__), "framework_mariadb.sql")
-	DbManager(frappe.local.db).restore_database(db_name, source_sql, db_name, frappe.conf.db_password)
+	DbManager(frappe.local.db).restore_database(
+		verbose, db_name, source_sql, db_name, frappe.conf.db_password
+	)
 	if verbose:
 		print("Imported from database %s" % source_sql)

--- a/frappe/database/postgres/database.py
+++ b/frappe/database/postgres/database.py
@ -160,11 +160,16 @@ class PostgresDatabase(PostgresExceptionUtil, Database):
 		return LazyDecode(self._cursor.query)

 	def get_connection(self):
-		conn = psycopg2.connect(
-			"host='{}' dbname='{}' user='{}' password='{}' port={}".format(
-				self.host, self.user, self.user, self.password, self.port
-			)
-		)
+		conn_settings = {
+			"user": self.user,
+			"dbname": self.user,
+			"host": self.host,
+			"password": self.password,
+		}
+		if self.port:
+			conn_settings["port"] = self.port
+
+		conn = psycopg2.connect(**conn_settings)
 		conn.set_isolation_level(ISOLATION_LEVEL_REPEATABLE_READ)

 		return conn
@ -387,7 +392,8 @@ class PostgresDatabase(PostgresExceptionUtil, Database):
 			END AS type,
 			BOOL_OR(b.index) AS index,
 			SPLIT_PART(COALESCE(a.column_default, NULL), '::', 1) AS default,
-			BOOL_OR(b.unique) AS unique
+			BOOL_OR(b.unique) AS unique,
+			COALESCE(a.is_nullable = 'NO', false) AS not_nullable
 			FROM information_schema.columns a
 			LEFT JOIN
 				(SELECT indexdef, tablename,
@ -397,7 +403,7 @@ class PostgresDatabase(PostgresExceptionUtil, Database):
 					WHERE tablename='{table_name}') b
 				ON SUBSTRING(b.indexdef, '(.*)') LIKE CONCAT('%', a.column_name, '%')
 			WHERE a.table_name = '{table_name}'
-			GROUP BY a.column_name, a.data_type, a.column_default, a.character_maximum_length;
+			GROUP BY a.column_name, a.data_type, a.column_default, a.character_maximum_length, a.is_nullable;
 		""".format(
 				table_name=table_name
 			),
--- a/frappe/database/postgres/schema.py
+++ b/frappe/database/postgres/schema.py
@ -2,6 +2,7 @@ import frappe
 from frappe import _
 from frappe.database.schema import DBTable, get_definition
 from frappe.utils import cint, flt
+from frappe.utils.defaults import get_not_null_defaults


 class PostgresTable(DBTable):
@ -16,7 +17,7 @@ class PostgresTable(DBTable):
 			additional_definitions += ",\n".join(column_defs)

 		# child table columns
-		if self.meta.get("istable") or 0:
+		if self.meta.get("istable", default=0):
 			if column_defs:
 				additional_definitions += ",\n"

@ -45,7 +46,7 @@ class PostgresTable(DBTable):
 			docstatus smallint not null default '0',
 			idx bigint not null default '0',
 			{additional_definitions}
-			)"""
+			)""",
 		)

 		self.create_indexes()
@ -139,11 +140,34 @@ class PostgresTable(DBTable):
 			if col.fieldname != "name":
 				# if index key exists
 				drop_contraint_query += f'DROP INDEX IF EXISTS "unique_{col.fieldname}" ;'
+
+		change_nullability = []
+		for col in self.change_nullability:
+			default = col.default or get_not_null_defaults(col.fieldtype)
+			if isinstance(default, str):
+				default = frappe.db.escape(default)
+			change_nullability.append(
+				f"ALTER COLUMN \"{col.fieldname}\" {'SET' if col.not_nullable else 'DROP'} NOT NULL"
+			)
+			change_nullability.append(f'ALTER COLUMN "{col.fieldname}" SET DEFAULT {default}')
+
+			if col.not_nullable:
+				try:
+					table = frappe.qb.DocType(self.doctype)
+					frappe.qb.update(table).set(
+						col.fieldname, col.default or get_not_null_defaults(col.fieldtype)
+					).where(table[col.fieldname].isnull()).run()
+				except Exception:
+					print(f"Failed to update data in {self.table_name} for {col.fieldname}")
+					raise
 		try:
 			if query:
 				final_alter_query = "ALTER TABLE `{}` {}".format(self.table_name, ", ".join(query))
 				# nosemgrep
 				frappe.db.sql(final_alter_query)
+			if change_nullability:
+				# nosemgrep
+				frappe.db.sql(f"ALTER TABLE `{self.table_name}` {','.join(change_nullability)}")
 			if create_contraint_query:
 				# nosemgrep
 				frappe.db.sql(create_contraint_query)
--- a/frappe/database/postgres/setup_db.py
+++ b/frappe/database/postgres/setup_db.py
@ -1,9 +1,10 @@
 import os

 import frappe
+from frappe import _


-def setup_database(force, source_sql=None, verbose=False):
+def setup_database():
 	root_conn = get_root_connection(frappe.flags.root_login, frappe.flags.root_password)
 	root_conn.commit()
 	root_conn.sql("end")
@ -14,9 +15,6 @@ def setup_database(force, source_sql=None, verbose=False):
 	root_conn.sql("GRANT ALL PRIVILEGES ON DATABASE `{0}` TO {0}".format(frappe.conf.db_name))
 	root_conn.close()

-	bootstrap_database(frappe.conf.db_name, verbose, source_sql=source_sql)
-	frappe.connect()
-

 def bootstrap_database(db_name, verbose, source_sql=None):
 	frappe.connect(db_name=db_name)
@ -38,13 +36,11 @@ def bootstrap_database(db_name, verbose, source_sql=None):


 def import_db_from_sql(source_sql=None, verbose=False):
+	import shlex
 	from shutil import which
-	from subprocess import PIPE, run

-	# we can't pass psql password in arguments in postgresql as mysql. So
-	# set password connection parameter in environment variable
-	subprocess_env = os.environ.copy()
-	subprocess_env["PGPASSWORD"] = str(frappe.conf.db_password)
+	from frappe.database import get_command
+	from frappe.utils import execute_in_shell

 	# bootstrap db
 	if not source_sql:
@ -52,27 +48,33 @@ def import_db_from_sql(source_sql=None, verbose=False):

 	pv = which("pv")

-	_command = (
-		f"psql {frappe.conf.db_name} "
-		f"-h {frappe.conf.db_host} -p {str(frappe.conf.db_port)} "
-		f"-U {frappe.conf.db_name}"
-	)
+	command = []

 	if pv:
-		command = f"{pv} {source_sql} | " + _command
+		command.extend([pv, source_sql, "|"])
+		source = []
+		print("Restoring Database file...")
 	else:
-		command = _command + f" -f {source_sql}"
+		source = ["-f", source_sql]

-	print("Restoring Database file...")
-	if verbose:
-		print(command)
+	bin, args, bin_name = get_command(
+		host=frappe.conf.db_host,
+		port=frappe.conf.db_port,
+		user=frappe.conf.db_name,
+		password=frappe.conf.db_password,
+		db_name=frappe.conf.db_name,
+	)

-	restore_proc = run(command, env=subprocess_env, shell=True, stdout=PIPE)
-
-	if verbose:
-		print(
-			f"\nSTDOUT by psql:\n{restore_proc.stdout.decode()}\nImported from Database File: {source_sql}"
+	if not bin:
+		frappe.throw(
+			_("{} not found in PATH! This is required to restore the database.").format(bin_name),
+			exc=frappe.ExecutableNotFound,
 		)
+	command.append(bin)
+	command.append(shlex.join(args))
+	command.extend(source)
+	execute_in_shell(" ".join(command), check_exit_code=True, verbose=verbose)
+	frappe.cache.delete_keys("")  # Delete all keys associated with this site.


 def get_root_connection(root_login=None, root_password=None):
--- a/frappe/database/schema.py
+++ b/frappe/database/schema.py
@ -3,6 +3,7 @@ import re
 import frappe
 from frappe import _
 from frappe.utils import cint, cstr, flt
+from frappe.utils.defaults import get_not_null_defaults

 SPECIAL_CHAR_PATTERN = re.compile(r"[\W]", flags=re.UNICODE)
 VARCHAR_CAST_PATTERN = re.compile(r"varchar\(([\d]+)\)")
@ -24,6 +25,7 @@ class DBTable:
 		self.add_column: list[DbColumn] = []
 		self.change_type: list[DbColumn] = []
 		self.change_name: list[DbColumn] = []
+		self.change_nullability: list[DbColumn] = []
 		self.add_unique: list[DbColumn] = []
 		self.add_index: list[DbColumn] = []
 		self.drop_unique: list[DbColumn] = []
@ -89,15 +91,16 @@ class DBTable:
 				continue

 			self.columns[field.get("fieldname")] = DbColumn(
-				self,
-				field.get("fieldname"),
-				field.get("fieldtype"),
-				field.get("length"),
-				field.get("default"),
-				field.get("search_index"),
-				field.get("options"),
-				field.get("unique"),
-				field.get("precision"),
+				table=self,
+				fieldname=field.get("fieldname"),
+				fieldtype=field.get("fieldtype"),
+				length=field.get("length"),
+				default=field.get("default"),
+				set_index=field.get("search_index"),
+				options=field.get("options"),
+				unique=field.get("unique"),
+				precision=field.get("precision"),
+				not_nullable=field.get("not_nullable"),
 			)

 	def validate(self):
@ -175,7 +178,18 @@ class DBTable:

 class DbColumn:
 	def __init__(
-		self, table, fieldname, fieldtype, length, default, set_index, options, unique, precision
+		self,
+		*,
+		table,
+		fieldname,
+		fieldtype,
+		length,
+		default,
+		set_index,
+		options,
+		unique,
+		precision,
+		not_nullable,
 	):
 		self.table = table
 		self.fieldname = fieldname
@ -186,6 +200,7 @@ class DbColumn:
 		self.options = options
 		self.unique = unique
 		self.precision = precision
+		self.not_nullable = not_nullable

 	def get_definition(self, for_modification=False):
 		column_def = get_definition(self.fieldtype, precision=self.precision, length=self.length)
@ -193,24 +208,43 @@ class DbColumn:
 		if not column_def:
 			return column_def

+		null = True
+		default = None
+		unique = False
+
 		if self.fieldtype in ("Check", "Int"):
-			default_value = cint(self.default) or 0
-			column_def += f" not null default {default_value}"
+			default = cint(self.default)
+			null = False

 		elif self.fieldtype in ("Currency", "Float", "Percent"):
-			default_value = flt(self.default) or 0
-			column_def += f" not null default {default_value}"
+			default = flt(self.default)
+			null = False

 		elif (
 			self.default
 			and (self.default not in frappe.db.DEFAULT_SHORTCUTS)
 			and not cstr(self.default).startswith(":")
 		):
-			column_def += f" default {frappe.db.escape(self.default)}"
+			default = frappe.db.escape(self.default)
+
+		if self.not_nullable and null:
+			if default is None:
+				default = get_not_null_defaults(self.fieldtype)
+				if isinstance(default, str):
+					default = frappe.db.escape(default)
+			null = False

 		if self.unique and not for_modification and (column_def not in ("text", "longtext")):
-			column_def += " unique"
+			unique = True

+		if not null:
+			column_def += " NOT NULL"
+
+		if default is not None:
+			column_def += f" DEFAULT {default}"
+
+		if unique:
+			column_def += " UNIQUE"
 		return column_def

 	def build_for_alter_table(self, current_def):
@ -250,11 +284,15 @@ class DbColumn:
 		):
 			self.table.set_default.append(self)

+		# nullability
+		if self.not_nullable is not None and (self.not_nullable != current_def["not_nullable"]):
+			self.table.change_nullability.append(self)
+
 		# index should be applied or dropped irrespective of type change
 		if (current_def["index"] and not self.set_index) and column_type not in ("text", "longtext"):
 			self.table.drop_index.append(self)

-		elif (not current_def["index"] and self.set_index) and not (column_type in ("text", "longtext")):
+		elif (not current_def["index"] and self.set_index) and column_type not in ("text", "longtext"):
 			self.table.add_index.append(self)

 	def default_changed(self, current_def):
--- a/frappe/database/utils.py
+++ b/frappe/database/utils.py
@ -1,7 +1,6 @@
 # Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE

-import typing
 from functools import cached_property
 from types import NoneType

@ -9,9 +8,6 @@ import frappe
 from frappe.query_builder.builder import MariaDB, Postgres
 from frappe.query_builder.functions import Function

-if typing.TYPE_CHECKING:
-	from frappe.query_builder import DocType
-
 Query = str | MariaDB | Postgres
 QueryValues = tuple | list | dict | NoneType

@ -27,7 +23,7 @@ NestedSetHierarchy = (
 )


-def is_query_type(query: str, query_type: str | tuple[str]) -> bool:
+def is_query_type(query: str, query_type: str | tuple[str, ...]) -> bool:
 	return query.lstrip().split(maxsplit=1)[0].lower().startswith(query_type)


--- a/frappe/desk/doctype/bulk_update/bulk_update.py
+++ b/frappe/desk/doctype/bulk_update/bulk_update.py
@ -46,8 +46,29 @@ class BulkUpdate(Document):

@frappe.whitelist()
 def submit_cancel_or_update_docs(doctype, docnames, action="submit", data=None):
-	docnames = frappe.parse_json(docnames)
+	if isinstance(docnames, str):
+		docnames = frappe.parse_json(docnames)

+	if len(docnames) < 20:
+		return _bulk_action(doctype, docnames, action, data)
+	elif len(docnames) <= 500:
+		frappe.msgprint(_("Bulk operation is enqueued in background."), alert=True)
+		frappe.enqueue(
+			_bulk_action,
+			doctype=doctype,
+			docnames=docnames,
+			action=action,
+			data=data,
+			queue="short",
+			timeout=1000,
+		)
+	else:
+		frappe.throw(
+			_("Bulk operations only support up to 500 documents."), title=_("Too Many Documents")
+		)
+
+
+def _bulk_action(doctype, docnames, action, data):
 	if data:
 		data = frappe.parse_json(data)

@ -85,5 +106,4 @@ def submit_cancel_or_update_docs(doctype, docnames, action="submit", data=None):

 def show_progress(docnames, message, i, description):
 	n = len(docnames)
-	if n >= 10:
-		frappe.publish_progress(float(i) * 100 / n, title=message, description=description)
+	frappe.publish_progress(float(i) * 100 / n, title=message, description=description)
--- a/frappe/desk/doctype/bulk_update/test_bulk_update.py
+++ b/frappe/desk/doctype/bulk_update/test_bulk_update.py
@ -0,0 +1,48 @@
+# Copyright (c) 2023, Frappe Technologies and Contributors
+# See LICENSE
+
+import time
+
+import frappe
+from frappe.core.doctype.doctype.test_doctype import new_doctype
+from frappe.desk.doctype.bulk_update.bulk_update import submit_cancel_or_update_docs
+from frappe.tests.utils import FrappeTestCase, timeout
+
+
+class TestBulkUpdate(FrappeTestCase):
+	@classmethod
+	def setUpClass(cls) -> None:
+		super().setUpClass()
+		cls.doctype = new_doctype(is_submittable=1, custom=1).insert().name
+		frappe.db.commit()
+		for _ in range(50):
+			frappe.new_doc(cls.doctype, some_fieldname=frappe.mock("name")).insert()
+
+	@timeout()
+	def wait_for_assertion(self, assertion):
+		"""Wait till an assertion becomes True"""
+		while True:
+			if assertion():
+				break
+			time.sleep(0.2)
+
+	def test_bulk_submit_in_background(self):
+		unsubmitted = frappe.get_all(self.doctype, {"docstatus": 0}, limit=5, pluck="name")
+		failed = submit_cancel_or_update_docs(self.doctype, unsubmitted, action="submit")
+		self.assertEqual(failed, [])
+
+		def check_docstatus(docs, status):
+			frappe.db.rollback()
+			matching_docs = frappe.get_all(
+				self.doctype, {"docstatus": status, "name": ("in", docs)}, pluck="name"
+			)
+			return set(matching_docs) == set(docs)
+
+		unsubmitted = frappe.get_all(self.doctype, {"docstatus": 0}, limit=20, pluck="name")
+		submit_cancel_or_update_docs(self.doctype, unsubmitted, action="submit")
+
+		self.wait_for_assertion(lambda: check_docstatus(unsubmitted, 1))
+
+		submitted = frappe.get_all(self.doctype, {"docstatus": 1}, limit=20, pluck="name")
+		submit_cancel_or_update_docs(self.doctype, submitted, action="cancel")
+		self.wait_for_assertion(lambda: check_docstatus(submitted, 2))
--- a/frappe/desk/doctype/dashboard_chart/dashboard_chart.js
+++ b/frappe/desk/doctype/dashboard_chart/dashboard_chart.js
@ -30,19 +30,21 @@ frappe.ui.form.on("Dashboard Chart", {
 			frm.disable_form();
 		}

-		frm.add_custom_button("Add Chart to Dashboard", () => {
-			const dialog = frappe.dashboard_utils.get_add_to_dashboard_dialog(
-				frm.doc.name,
-				"Dashboard Chart",
-				"frappe.desk.doctype.dashboard_chart.dashboard_chart.add_chart_to_dashboard"
-			);
+		if (!frm.is_new()) {
+			frm.add_custom_button("Add Chart to Dashboard", () => {
+				const dialog = frappe.dashboard_utils.get_add_to_dashboard_dialog(
+					frm.doc.name,
+					"Dashboard Chart",
+					"frappe.desk.doctype.dashboard_chart.dashboard_chart.add_chart_to_dashboard"
+				);

-			if (!frm.doc.chart_name) {
-				frappe.msgprint(__("Please create chart first"));
-			} else {
-				dialog.show();
-			}
-		});
+				if (!frm.doc.chart_name) {
+					frappe.msgprint(__("Please create chart first"));
+				} else {
+					dialog.show();
+				}
+			});
+		}

 		frm.set_df_property("filters_section", "hidden", 1);
 		frm.set_df_property("dynamic_filters_section", "hidden", 1);
--- a/frappe/desk/doctype/dashboard_chart/dashboard_chart.py
+++ b/frappe/desk/doctype/dashboard_chart/dashboard_chart.py
@ -209,10 +209,10 @@ def get_chart_config(chart, filters, timespan, timegrain, from_date, to_date):

 	data = frappe.db.get_list(
 		doctype,
-		fields=[f"{datefield} as _unit", f"SUM({value_field})", "COUNT(*)"],
+		fields=[datefield, f"SUM({value_field})", "COUNT(*)"],
 		filters=filters,
-		group_by="_unit",
-		order_by="_unit asc",
+		group_by=datefield,
+		order_by=datefield,
 		as_list=True,
 	)

--- a/frappe/desk/doctype/desktop_icon/desktop_icon.py
+++ b/frappe/desk/doctype/desktop_icon/desktop_icon.py
@ -37,6 +37,7 @@ class DesktopIcon(Document):
 		reverse: DF.Check
 		standard: DF.Check
 		type: DF.Literal["module", "list", "link", "page", "query-report"]
+
 	# end: auto-generated types
 	def validate(self):
 		if not self.label:
@ -225,7 +226,7 @@ def add_user_icon(_doctype, _report=None, label=None, link=None, type="link", st

 			icon_name = new_icon.name

-		except frappe.UniqueValidationError as e:
+		except frappe.UniqueValidationError:
 			frappe.throw(_("Desktop Icon already exists"))
 		except Exception as e:
 			raise e
@ -262,7 +263,7 @@ def set_desktop_icons(visible_list, ignore_duplicate=True):
 	an icon for the doctype"""

 	# clear all custom only if setup is not complete
-	if not int(frappe.defaults.get_defaults().setup_complete or 0):
+	if not frappe.defaults.get_defaults().get("setup_complete", 0):
 		frappe.db.delete("Desktop Icon", {"standard": 0})

 	# set standard as blocked and hidden if setting first active domain
--- a/frappe/desk/doctype/event/event.py
+++ b/frappe/desk/doctype/event/event.py
@ -121,9 +121,7 @@ class Event(Document):
 					["Communication Link", "link_doctype", "=", participant.reference_doctype],
 					["Communication Link", "link_name", "=", participant.reference_docname],
 				]
-				comms = frappe.get_all("Communication", filters=filters, fields=["name"])
-
-				if comms:
+				if comms := frappe.get_all("Communication", filters=filters, fields=["name"], distinct=True):
 					for comm in comms:
 						communication = frappe.get_doc("Communication", comm.name)
 						self.update_communication(participant, communication)
--- a/frappe/desk/doctype/notification_log/notification_log.js
+++ b/frappe/desk/doctype/notification_log/notification_log.js
@ -11,6 +11,10 @@ frappe.ui.form.on("Notification Log", {
 	},

 	open_reference_document: function (frm) {
+		if (frm.doc?.link) {
+			frappe.set_route(frm.doc.link);
+			return;
+		}
 		const dt = frm.doc.document_type;
 		const dn = frm.doc.document_name;
 		frappe.set_route("Form", dt, dn);
--- a/frappe/desk/doctype/notification_log/notification_log.json
+++ b/frappe/desk/doctype/notification_log/notification_log.json
@ -15,7 +15,8 @@
  "attached_file",
  "attachment_link",
  "open_reference_document",
-  "from_user"
+  "from_user",
+  "link"
 ],
 "fields": [
  {
@ -91,12 +92,18 @@
   "fieldname": "attachment_link",
   "fieldtype": "HTML",
   "label": "Attachment Link"
+  },
+  {
+   "fieldname": "link",
+   "fieldtype": "Data",
+   "hidden": 1,
+   "label": "Link"
  }
 ],
 "hide_toolbar": 1,
 "in_create": 1,
 "links": [],
- "modified": "2023-06-14 21:20:51.197943",
+ "modified": "2023-11-18 22:40:12.145940",
 "modified_by": "Administrator",
 "module": "Desk",
 "name": "Notification Log",
--- a/frappe/desk/doctype/notification_log/notification_log.py
+++ b/frappe/desk/doctype/notification_log/notification_log.py
@ -25,6 +25,7 @@ class NotificationLog(Document):
 		email_content: DF.TextEditor | None
 		for_user: DF.Link | None
 		from_user: DF.Link | None
+		link: DF.Data | None
 		read: DF.Check
 		subject: DF.Text | None
 		type: DF.Literal["Mention", "Energy Point", "Assignment", "Share", "Alert"]
@ -125,21 +126,24 @@ def send_notification_email(doc):
 	if not email:
 		return

-	doc_link = get_url_to_form(doc.document_type, doc.document_name)
 	header = get_email_header(doc)
 	email_subject = strip_html(doc.subject)
+	args = {
+		"body_content": doc.subject,
+		"description": doc.email_content,
+	}
+	if doc.link:
+		args["doc_link"] = doc.link
+	else:
+		args["document_type"] = doc.document_type
+		args["document_name"] = doc.document_name
+		args["doc_link"] = get_url_to_form(doc.document_type, doc.document_name)

 	frappe.sendmail(
 		recipients=email,
 		subject=email_subject,
 		template="new_notification",
-		args={
-			"body_content": doc.subject,
-			"description": doc.email_content,
-			"document_type": doc.document_type,
-			"document_name": doc.document_name,
-			"doc_link": doc_link,
-		},
+		args=args,
 		header=[header, "orange"],
 		now=frappe.flags.in_test,
 	)
--- a/frappe/desk/doctype/number_card/number_card.js
+++ b/frappe/desk/doctype/number_card/number_card.js
@ -4,8 +4,11 @@
 frappe.ui.form.on("Number Card", {
 	refresh: function (frm) {
 		if (!frappe.boot.developer_mode && frm.doc.is_standard) {
-			frm.disable_form();
+			frm.disable_save();
+		} else {
+			frm.enable_save();
 		}
+
 		frm.set_df_property("filters_section", "hidden", 1);
 		frm.set_df_property("dynamic_filters_section", "hidden", 1);
 		frm.trigger("set_options");
@ -19,12 +22,7 @@ frappe.ui.form.on("Number Card", {
 		}

 		if (frm.doc.type == "Custom") {
-			if (!frappe.boot.developer_mode) {
-				frm.disable_form();
-			}
 			frm.filters = eval(frm.doc.filters_config);
-			frm.trigger("set_filters_description");
-			frm.trigger("set_method_description");
 			frm.trigger("render_filters_table");
 		}
 		frm.trigger("set_parent_document_type");
@ -68,49 +66,7 @@ frappe.ui.form.on("Number Card", {
 		frm.set_df_property("dynamic_filters_section", "hidden", 1);
 	},

-	set_filters_description: function (frm) {
-		if (frm.doc.type == "Custom") {
-			frm.fields_dict.filters_config.set_description(`
-		Set the filters here. For example:
-<pre class="small text-muted">
-<code>
-[{
-	fieldname: "company",
-	label: __("Company"),
-	fieldtype: "Link",
-	options: "Company",
-	default: frappe.defaults.get_user_default("Company"),
-	reqd: 1
-},
-{
-	fieldname: "account",
-	label: __("Account"),
-	fieldtype: "Link",
-	options: "Account",
-	reqd: 1
-}]
-</code></pre>`);
-		}
-	},
-
-	set_method_description: function (frm) {
-		if (frm.doc.type == "Custom") {
-			frm.fields_dict.method.set_description(`
-		Set the path to a whitelisted function that will return the data for the number card in the format:
-<pre class="small text-muted">
-<code>
-{
-	"value": value,
-	"fieldtype": "Currency",
-	"route_options": {"from_date": "2023-05-23"},
-	"route": ["query-report", "Permitted Documents For User"]
-}
-</code></pre>`);
-		}
-	},
-
 	type: function (frm) {
-		frm.trigger("set_filters_description");
 		if (frm.doc.type == "Report") {
 			frm.set_query("report_name", () => {
 				return {
--- a/frappe/desk/doctype/number_card/number_card.json
+++ b/frappe/desk/doctype/number_card/number_card.json
@ -117,6 +117,7 @@
   "fieldname": "is_standard",
   "fieldtype": "Check",
   "label": "Is Standard",
+   "no_copy": 1,
   "read_only_depends_on": "eval: !frappe.boot.developer_mode"
  },
  {
@ -165,6 +166,7 @@
  },
  {
   "depends_on": "eval: doc.type == 'Custom'",
+   "description": "Set the path to a whitelisted function that will return the data for the number card in the format:\n\n<pre class=\"small text-muted\"><code>\n{\n\t\"value\": value,\n\t\"fieldtype\": \"Currency\",\n\t\"route_options\": {\"from_date\": \"2023-05-23\"},\n\t\"route\": [\"query-report\", \"Permitted Documents For User\"]\n}</code></pre>",
   "fieldname": "method",
   "fieldtype": "Data",
   "label": "Method",
@ -177,6 +179,7 @@
   "label": "Custom Configuration"
  },
  {
+   "description": "Set the filters here. For example:\n<pre class=\"small text-muted\"><code>\n[{\n\tfieldname: \"company\",\n\tlabel: __(\"Company\"),\n\tfieldtype: \"Link\",\n\toptions: \"Company\",\n\tdefault: frappe.defaults.get_user_default(\"Company\"),\n\treqd: 1\n},\n{\n\tfieldname: \"account\",\n\tlabel: __(\"Account\"),\n\tfieldtype: \"Link\",\n\toptions: \"Account\",\n\treqd: 1\n}]\n</code></pre>",
   "fieldname": "filters_config",
   "fieldtype": "Code",
   "label": "Filters Configuration",
@ -200,7 +203,7 @@
  }
 ],
 "links": [],
- "modified": "2023-08-28 22:23:56.286804",
+ "modified": "2023-11-09 13:44:00.280846",
 "modified_by": "Administrator",
 "module": "Desk",
 "name": "Number Card",
--- a/frappe/desk/doctype/system_console/system_console.json
+++ b/frappe/desk/doctype/system_console/system_console.json
@ -29,7 +29,7 @@
 ],
 "fields": [
  {
-   "description": "To print output use <code>log(text)</code>",
+   "description": "To print output use <code>print(text)</code>",
   "fieldname": "console",
   "fieldtype": "Code",
   "label": "Console",
@ -86,7 +86,7 @@
 "index_web_pages_for_search": 1,
 "issingle": 1,
 "links": [],
- "modified": "2022-04-15 14:15:58.398590",
+ "modified": "2023-11-03 13:02:00.706806",
 "modified_by": "Administrator",
 "module": "Desk",
 "name": "System Console",
--- a/frappe/desk/doctype/workspace_shortcut/workspace_shortcut.json
+++ b/frappe/desk/doctype/workspace_shortcut/workspace_shortcut.json
@ -44,7 +44,7 @@
   "fieldtype": "Select",
   "in_list_view": 1,
   "label": "DocType View",
-   "options": "\nList\nReport Builder\nDashboard\nTree\nNew\nCalendar\nKanban"
+   "options": "\nList\nReport Builder\nDashboard\nTree\nNew\nCalendar\nKanban\nImage"
  },
  {
   "fieldname": "column_break_4",
@ -102,8 +102,7 @@
   "fieldname": "url",
   "fieldtype": "Data",
   "in_list_view": 1,
-   "label": "URL",
-   "options": "URL"
+   "label": "URL"
  },
  {
   "depends_on": "eval:doc.doc_view == \"Kanban\"",
@ -116,7 +115,7 @@
 "index_web_pages_for_search": 1,
 "istable": 1,
 "links": [],
- "modified": "2023-07-18 16:12:53.546430",
+ "modified": "2023-11-27 14:13:38.489737",
 "modified_by": "Administrator",
 "module": "Desk",
 "name": "Workspace Shortcut",
--- a/frappe/desk/page/setup_wizard/setup_wizard.js
+++ b/frappe/desk/page/setup_wizard/setup_wizard.js
@ -196,7 +196,7 @@ frappe.setup.SetupWizard = class SetupWizard extends frappe.ui.Slides {
 					this.abort_setup(r.message.fail);
 				}
 			},
-			error: () => this.abort_setup("Error in setup"),
+			error: () => this.abort_setup(),
 		});
 	}

@ -213,7 +213,11 @@ frappe.setup.SetupWizard = class SetupWizard extends frappe.ui.Slides {

 	abort_setup(fail_msg) {
 		this.$working_state.find(".state-icon-container").html("");
-		fail_msg = fail_msg ? fail_msg : __("Failed to complete setup");
+		fail_msg = fail_msg
+			? fail_msg
+			: frappe.last_response.setup_wizard_failure_message
+			? frappe.last_response.setup_wizard_failure_message
+			: __("Failed to complete setup");

 		this.update_setup_message("Could not start up: " + fail_msg);

@ -463,7 +467,7 @@ frappe.setup.slides_settings = [
 				fieldtype: "Data",
 				options: "Email",
 			},
-			{ fieldname: "password", label: __("Password"), fieldtype: "Password" },
+			{ fieldname: "password", label: __("Password"), fieldtype: "Password", length: 512 },
 		],

 		onload: function (slide) {
--- a/frappe/desk/page/setup_wizard/setup_wizard.py
+++ b/frappe/desk/page/setup_wizard/setup_wizard.py
@ -83,11 +83,14 @@ def process_setup_stages(stages, user_input, is_background_task=False):
 				task.get("fn")(task.get("args"))
 	except Exception:
 		handle_setup_exception(user_input)
+		message = current_task.get("fail_msg") if current_task else "Failed to complete setup"
+		frappe.log_error(title=f"Setup failed: {message}")
 		if not is_background_task:
-			return {"status": "fail", "fail": current_task.get("fail_msg")}
+			frappe.response["setup_wizard_failure_message"] = message
+			raise
 		frappe.publish_realtime(
 			"setup_task",
-			{"status": "fail", "fail_msg": current_task.get("fail_msg")},
+			{"status": "fail", "fail_msg": message},
 			user=frappe.session.user,
 		)
 	else:
--- a/frappe/desk/reportview.py
+++ b/frappe/desk/reportview.py
@ -215,12 +215,12 @@ def clean_params(data):


 def parse_json(data):
-	if isinstance(data.get("filters"), str):
-		data["filters"] = json.loads(data["filters"])
-	if isinstance(data.get("or_filters"), str):
-		data["or_filters"] = json.loads(data["or_filters"])
-	if isinstance(data.get("fields"), str):
-		data["fields"] = ["*"] if data["fields"] == "*" else json.loads(data["fields"])
+	if (filters := data.get("filters")) and isinstance(filters, str):
+		data["filters"] = json.loads(filters)
+	if (or_filters := data.get("or_filters")) and isinstance(or_filters, str):
+		data["or_filters"] = json.loads(or_filters)
+	if (fields := data.get("fields")) and isinstance(fields, str):
+		data["fields"] = ["*"] if fields == "*" else json.loads(fields)
 	if isinstance(data.get("docstatus"), str):
 		data["docstatus"] = json.loads(data["docstatus"])
 	if isinstance(data.get("save_user_settings"), str):
--- a/frappe/email/doctype/email_account/email_account.js
+++ b/frappe/email/doctype/email_account/email_account.js
@ -159,6 +159,15 @@ frappe.ui.form.on("Email Account", {
 			delete frappe.route_flags.delete_user_from_locals;
 			delete locals["User"][frappe.route_flags.linked_user];
 		}
+
+		if (frappe.boot.developer_mode && !frm.is_dirty() && frm.doc.enable_incoming) {
+			frm.add_custom_button(__("Pull Emails"), () => {
+				frm.call({
+					method: "pull_emails",
+					args: { email_account: frm.doc.name },
+				});
+			});
+		}
 	},

 	authorize_api_access: function (frm) {
--- a/frappe/email/doctype/email_account/email_account.py
+++ b/frappe/email/doctype/email_account/email_account.py
@ -831,6 +831,14 @@ def pull(now=False):
 				)


+@frappe.whitelist()
+def pull_emails(email_account: str) -> None:
+	"""Pull emails from given email account."""
+	frappe.has_permission("Email Account", "read", throw=True)
+
+	pull_from_email_account(email_account)
+
+
 def pull_from_email_account(email_account):
 	"""Runs within a worker process"""
 	email_account = frappe.get_doc("Email Account", email_account)
--- a/frappe/email/doctype/email_domain/email_domain.py
+++ b/frappe/email/doctype/email_domain/email_domain.py
@ -73,6 +73,7 @@ class EmailDomain(Document):
 		use_ssl_for_outgoing: DF.Check
 		use_starttls: DF.Check
 		use_tls: DF.Check
+
 	# end: auto-generated types
 	def validate(self):
 		"""Validate POP3/IMAP and SMTP connections."""
@ -120,4 +121,4 @@ class EmailDomain(Document):
 		elif self.use_tls:
 			self.smtp_port = self.smtp_port or 587

-		conn_method((self.smtp_server or ""), cint(self.smtp_port) or 0).quit()
+		conn_method((self.smtp_server or ""), cint(self.smtp_port)).quit()
--- a/frappe/email/doctype/email_group/email_group.py
+++ b/frappe/email/doctype/email_group/email_group.py
@ -1,6 +1,8 @@
 # Copyright (c) 2015, Frappe Technologies and contributors
 # License: MIT. See LICENSE

+import contextlib
+
 import frappe
 from frappe import _
 from frappe.model.document import Document
@ -41,7 +43,7 @@ class EmailGroup(Document):
 		added = 0

 		for user in frappe.get_all(doctype, [email_field, unsubscribed_field or "name"]):
-			try:
+			with contextlib.suppress(frappe.UniqueValidationError, frappe.InvalidEmailAddressError):
 				email = parse_addr(user.get(email_field))[1] if user.get(email_field) else None
 				if email:
 					frappe.get_doc(
@ -52,10 +54,7 @@ class EmailGroup(Document):
 							"unsubscribed": user.get(unsubscribed_field) if unsubscribed_field else 0,
 						}
 					).insert(ignore_permissions=True)
-
 					added += 1
-			except frappe.UniqueValidationError:
-				pass

 		frappe.msgprint(_("{0} subscribers added").format(added))

@ -123,6 +122,5 @@ def send_welcome_email(welcome_email, email, email_group):
 		return

 	args = dict(email=email, email_group=email_group)
-	email_message = welcome_email.response or welcome_email.response_html
-	message = frappe.render_template(email_message, args)
+	message = frappe.render_template(welcome_email.response_, args)
 	frappe.sendmail(email, subject=welcome_email.subject, message=message)
--- a/frappe/email/doctype/email_group_member/email_group_member.json
+++ b/frappe/email/doctype/email_group_member/email_group_member.json
@ -28,6 +28,7 @@
   "in_global_search": 1,
   "in_list_view": 1,
   "label": "Email",
+   "options": "Email",
   "reqd": 1
  },
  {
@ -40,7 +41,7 @@
  }
 ],
 "links": [],
- "modified": "2022-07-11 16:38:34.165271",
+ "modified": "2023-11-25 16:54:59.828669",
 "modified_by": "Administrator",
 "module": "Email",
 "name": "Email Group Member",
--- a/frappe/email/doctype/email_queue/email_queue.py
+++ b/frappe/email/doctype/email_queue/email_queue.py
@ -687,13 +687,13 @@ class QueueBuilder:
 			mail.set_in_reply_to(self.in_reply_to)
 		return mail

-	def process(self, send_now=False):
+	def process(self, send_now=False) -> EmailQueue | None:
 		"""Build and return the email queues those are created.

 		Sends email incase if it is requested to send now.
 		"""
 		final_recipients = self.final_recipients()
-		queue_separately = (final_recipients and self.queue_separately) or len(final_recipients) > 20
+		queue_separately = (final_recipients and self.queue_separately) or len(final_recipients) > 100
 		if not (final_recipients + self.final_cc()):
 			return []

@ -705,6 +705,7 @@ class QueueBuilder:
 			recipients = list(set(final_recipients + self.final_cc() + self.bcc))
 			q = EmailQueue.new({**queue_data, **{"recipients": recipients}}, ignore_permissions=True)
 			send_now and q.send()
+			return q
 		else:
 			if send_now and len(final_recipients) >= 1000:
 				# force queueing if there are too many recipients to avoid timeouts
--- a/frappe/email/doctype/newsletter/newsletter.py
+++ b/frappe/email/doctype/newsletter/newsletter.py
@ -50,10 +50,10 @@ class Newsletter(WebsiteGenerator):
 		total_recipients: DF.Int
 		total_views: DF.Int
 	# end: auto-generated types
+
 	def validate(self):
 		self.route = f"newsletters/{self.name}"
 		self.validate_sender_address()
-		self.validate_recipient_address()
 		self.validate_publishing()
 		self.validate_scheduling_date()

@ -135,7 +135,6 @@ class Newsletter(WebsiteGenerator):
 	def validate_newsletter_recipients(self):
 		if not self.newsletter_recipients:
 			frappe.throw(_("Newsletter should have atleast one recipient"), exc=NoRecipientFoundError)
-		self.validate_recipient_address()

 	def validate_sender_address(self):
 		"""Validate self.send_from is a valid email address or not."""
@ -145,11 +144,6 @@ class Newsletter(WebsiteGenerator):
 				f"{self.sender_name} <{self.sender_email}>" if self.sender_name else self.sender_email
 			)

-	def validate_recipient_address(self):
-		"""Validate if self.newsletter_recipients are all valid email addresses or not."""
-		for recipient in self.newsletter_recipients:
-			frappe.utils.validate_email_address(recipient, throw=True)
-
 	def validate_publishing(self):
 		if self.send_webview_link and not self.published:
 			frappe.throw(_("Newsletter must be published to send webview link in email"))
@ -308,11 +302,11 @@ def confirmed_unsubscribe(email, group):

@frappe.whitelist(allow_guest=True)
@rate_limit(limit=10, seconds=60 * 60)
-def subscribe(email, email_group=None):  # noqa
+def subscribe(email, email_group=None):
 	"""API endpoint to subscribe an email to a particular email group. Triggers a confirmation email."""

 	if email_group is None:
-		email_group = _("Website")
+		email_group = get_default_email_group()

 	# build subscription confirmation URL
 	api_endpoint = frappe.utils.get_url(
@ -355,13 +349,16 @@ def subscribe(email, email_group=None):  # noqa


@frappe.whitelist(allow_guest=True)
-def confirm_subscription(email, email_group=_("Website")):  # noqa
+def confirm_subscription(email, email_group=None):
 	"""API endpoint to confirm email subscription.
 	This endpoint is called when user clicks on the link sent to their mail.
 	"""
 	if not verify_request():
 		return

+	if email_group is None:
+		email_group = get_default_email_group()
+
 	if not frappe.db.exists("Email Group", email_group):
 		frappe.get_doc({"doctype": "Email Group", "title": email_group}).insert(ignore_permissions=True)

@ -438,3 +435,7 @@ def newsletter_email_read(recipient_email=None, reference_doctype=None, referenc

 	finally:
 		frappe.response.update(frappe.utils.get_imaginary_pixel_response())
+
+
+def get_default_email_group():
+	return _("Website", lang=frappe.db.get_default("language"))
--- a/frappe/email/doctype/newsletter/newsletter_list.js
+++ b/frappe/email/doctype/newsletter/newsletter_list.js
@ -2,11 +2,11 @@ frappe.listview_settings["Newsletter"] = {
 	add_fields: ["subject", "email_sent", "schedule_sending"],
 	get_indicator: function (doc) {
 		if (doc.email_sent) {
-			return [__("Sent"), "green", "email_sent,=,Yes"];
+			return [__("Sent"), "green", "email_sent,=,1"];
 		} else if (doc.schedule_sending) {
-			return [__("Scheduled"), "purple", "email_sent,=,No|schedule_sending,=,Yes"];
+			return [__("Scheduled"), "purple", "email_sent,=,0|schedule_sending,=,1"];
 		} else {
-			return [__("Not Sent"), "gray", "email_sent,=,No"];
+			return [__("Not Sent"), "gray", "email_sent,=,0"];
 		}
 	},
 };
--- a/frappe/email/doctype/notification/notification.json
+++ b/frappe/email/doctype/notification/notification.json
@ -36,6 +36,7 @@
  "send_to_all_assignees",
  "recipients",
  "message_sb",
+  "message_type",
  "message",
  "message_examples",
  "view_properties",
@ -277,15 +278,24 @@
   "fieldname": "send_to_all_assignees",
   "fieldtype": "Check",
   "label": "Send To All Assignees"
+  },
+  {
+   "default": "Markdown",
+   "depends_on": "is_standard",
+   "fieldname": "message_type",
+   "fieldtype": "Select",
+   "label": "Message Type",
+   "options": "Markdown\nHTML\nPlain Text"
  }
 ],
 "icon": "fa fa-envelope",
 "index_web_pages_for_search": 1,
 "links": [],
- "modified": "2021-05-04 11:17:11.882314",
+ "modified": "2023-11-17 08:48:25.616203",
 "modified_by": "Administrator",
 "module": "Email",
 "name": "Notification",
+ "naming_rule": "Set by user",
 "owner": "Administrator",
 "permissions": [
  {
@ -301,6 +311,7 @@
 ],
 "sort_field": "modified",
 "sort_order": "DESC",
+ "states": [],
 "title_field": "subject",
 "track_changes": 1
 }
--- a/frappe/email/doctype/notification/notification.py
+++ b/frappe/email/doctype/notification/notification.py
@ -12,10 +12,12 @@ from frappe.desk.doctype.notification_log.notification_log import enqueue_create
 from frappe.integrations.doctype.slack_webhook_url.slack_webhook_url import send_slack_message
 from frappe.model.document import Document
 from frappe.modules.utils import export_module_json, get_doc_module
-from frappe.utils import add_to_date, cast, is_html, nowdate, validate_email_address
+from frappe.utils import add_to_date, cast, nowdate, validate_email_address
 from frappe.utils.jinja import validate_template
 from frappe.utils.safe_exec import get_safe_globals

+FORMATS = {"HTML": ".html", "Markdown": ".md", "Plain Text": ".txt"}
+

 class Notification(Document):
 	# begin: auto-generated types
@ -50,6 +52,7 @@ class Notification(Document):
 		]
 		is_standard: DF.Check
 		message: DF.Code | None
+		message_type: DF.Literal["Markdown", "HTML", "Plain Text"]
 		method: DF.Data | None
 		module: DF.Link | None
 		print_format: DF.Link | None
@ -93,11 +96,11 @@ class Notification(Document):
 	def on_update(self):
 		frappe.cache.hdel("notifications", self.document_type)
 		path = export_module_json(self, self.is_standard, self.module)
-		if path:
-			# js
-			if not os.path.exists(path + ".md") and not os.path.exists(path + ".html"):
-				with open(path + ".md", "w") as f:
-					f.write(self.message)
+		if path and self.message:
+			extension = FORMATS.get(self.message_type, ".md")
+			file_path = path + extension
+			with open(file_path, "w") as f:
+				f.write(self.message)

 			# py
 			if not os.path.exists(path + ".py"):
@ -399,18 +402,26 @@ def get_context(context):
 				}
 			]

-	def get_template(self):
+	def get_template(self, md_as_html=False):
 		module = get_doc_module(self.module, self.doctype, self.name)

-		def load_template(extn):
-			template = ""
-			template_path = os.path.join(os.path.dirname(module.__file__), frappe.scrub(self.name) + extn)
-			if os.path.exists(template_path):
-				with open(template_path) as f:
-					template = f.read()
-			return template
+		path = os.path.join(os.path.dirname(module.__file__), frappe.scrub(self.name))
+		extension = FORMATS.get(self.message_type, ".md")
+		file_path = path + extension

-		return load_template(".html") or load_template(".md")
+		template = ""
+
+		if os.path.exists(file_path):
+			with open(file_path) as f:
+				template = f.read()
+
+		if not template:
+			return
+
+		if extension == ".md":
+			return frappe.utils.md_to_html(template)
+
+		return template

 	def load_standard_properties(self, context):
 		"""load templates and run get_context"""
@ -421,10 +432,7 @@ def get_context(context):
 				if out:
 					context.update(out)

-		self.message = self.get_template()
-
-		if not is_html(self.message):
-			self.message = frappe.utils.md_to_html(self.message)
+		self.message = self.get_template(md_as_html=True)

 	def on_trash(self):
 		frappe.cache.hdel("notifications", self.document_type)
--- a/Show more
+++ b/Show more