diff --git a/.github/workflows/patch-mariadb-tests.yml b/.github/workflows/patch-mariadb-tests.yml index fe02351a5a..4875e6f5df 100644 --- a/.github/workflows/patch-mariadb-tests.yml +++ b/.github/workflows/patch-mariadb-tests.yml @@ -64,9 +64,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: | - 3.7 - 3.10 + python-version: "3.10" - name: Setup Node uses: actions/setup-node@v3 @@ -112,8 +110,8 @@ jobs: - name: Run Patch Tests run: | cd ~/frappe-bench/ - wget https://frappeframework.com/files/v10-frappe.sql.gz - bench --site test_site --force restore ~/frappe-bench/v10-frappe.sql.gz + wget https://frappeframework.com/files/v13-frappe.sql.gz + bench --site test_site --force restore ~/frappe-bench/v13-frappe.sql.gz source env/bin/activate cd apps/frappe/ @@ -121,7 +119,6 @@ jobs: function update_to_version() { version=$1 - py=$2 branch_name="version-$version-hotfix" echo "Updating to v$version" @@ -130,22 +127,22 @@ jobs: pgrep honcho | xargs kill rm -rf ~/frappe-bench/env - bench -v setup env --python $py - bench start &> ~/frappe-bench/bench_start.log & + bench -v setup env + bench start &>> ~/frappe-bench/bench_start.log & bench --site test_site migrate } - update_to_version 12 python3.7 - update_to_version 13 python3.7 - - update_to_version 14 python3.10 + update_to_version 14 echo "Updating to last commit" + pgrep honcho | xargs kill rm -rf ~/frappe-bench/env git checkout -q -f "$GITHUB_SHA" bench -v setup env + bench start &>> ~/frappe-bench/bench_start.log & bench --site test_site migrate + bench --site test_site execute frappe.tests.utils.check_orpahned_doctypes - name: Show bench output if: ${{ always() }} diff --git a/frappe/app.py b/frappe/app.py index 6f3846732b..168e454439 100644 --- a/frappe/app.py +++ b/frappe/app.py @@ -1,17 +1,18 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +import functools import gc import logging import os import re from werkzeug.exceptions import HTTPException, NotFound -from werkzeug.local import LocalManager from werkzeug.middleware.profiler import ProfilerMiddleware from werkzeug.middleware.proxy_fix import ProxyFix from werkzeug.middleware.shared_data import SharedDataMiddleware from werkzeug.wrappers import Request, Response +from werkzeug.wsgi import ClosingIterator import frappe import frappe.api @@ -23,12 +24,11 @@ import frappe.utils.response from frappe import _ from frappe.auth import SAFE_HTTP_METHODS, UNSAFE_HTTP_METHODS, HTTPRequest from frappe.middlewares import StaticDataMiddleware -from frappe.utils import cint, get_site_name, sanitize_html +from frappe.utils import CallbackManager, cint, get_site_name, sanitize_html +from frappe.utils.data import escape_html from frappe.utils.error import log_error_snapshot from frappe.website.serve import get_response -local_manager = LocalManager(frappe.local) - _site = None _sites_path = os.environ.get("SITES_PATH", ".") @@ -62,7 +62,28 @@ if frappe._tune_gc: # end: module pre-loading -@local_manager.middleware +def after_response_wrapper(app): + """Wrap a WSGI application to call after_response hooks after we have responded. + + This is done to reduce response time by deferring expensive tasks.""" + + @functools.wraps(app) + def application(environ, start_response): + return ClosingIterator( + app(environ, start_response), + ( + frappe.rate_limiter.update, + frappe.monitor.stop, + frappe.recorder.dump, + frappe.request.after_response.run, + frappe.destroy, + ), + ) + + return application + + +@after_response_wrapper @Request.application def application(request: Request): response = None @@ -109,7 +130,7 @@ def application(request: Request): # this function *must* always return a response, hence any exception thrown outside of # try..catch block like this finally block needs to be handled appropriately. - if request.method in UNSAFE_HTTP_METHODS and frappe.db and rollback: + if rollback and request.method in UNSAFE_HTTP_METHODS and frappe.db: frappe.db.rollback() try: @@ -120,8 +141,6 @@ def application(request: Request): log_request(request, response) process_response(response) - if frappe.db: - frappe.db.close() return response @@ -136,6 +155,8 @@ def run_after_request_hooks(request, response): def init_request(request): frappe.local.request = request + frappe.local.request.after_response = CallbackManager() + frappe.local.is_ajax = frappe.get_request_header("X-Requested-With") == "XMLHttpRequest" site = _site or request.headers.get("X-Frappe-Site-Name") or get_site_name(request.host) @@ -343,7 +364,7 @@ def handle_exception(e): response = frappe.rate_limiter.respond() else: - traceback = "
" + sanitize_html(frappe.get_traceback()) + "
" + traceback = "
" + escape_html(frappe.get_traceback()) + "
" # disable traceback in production if flag is set if frappe.local.flags.disable_traceback or not allow_traceback and not frappe.local.dev_server: traceback = "" @@ -393,6 +414,7 @@ def sync_database(rollback: bool) -> bool: def serve( + host=None, port=8000, profile=False, no_reload=False, @@ -417,7 +439,7 @@ def serve( application = ProxyFix(application, x_for=1, x_proto=1, x_host=1, x_port=1, x_prefix=1) application.debug = True - application.config = {"SERVER_NAME": "localhost:8000"} + application.config = {"SERVER_NAME": "127.0.0.1:8000"} log = logging.getLogger("werkzeug") log.propagate = False @@ -427,7 +449,7 @@ def serve( log.setLevel(logging.ERROR) run_simple( - "0.0.0.0", + host, int(port), application, exclude_patterns=["test_*"], diff --git a/frappe/commands/site.py b/frappe/commands/site.py index defc735ddd..263c5438bf 100644 --- a/frappe/commands/site.py +++ b/frappe/commands/site.py @@ -1169,7 +1169,7 @@ def start_ngrok(context, bind_tls, use_default_authtoken): port = frappe.conf.http_port or frappe.conf.webserver_port tunnel = ngrok.connect(addr=str(port), host_header=site, bind_tls=bind_tls) print(f"Public URL: {tunnel.public_url}") - print("Inspect logs at http://localhost:4040") + print("Inspect logs at http://127.0.0.1:4040") ngrok_process = ngrok.get_ngrok_process() try: diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py index 9c433f01a4..daa9859d4a 100644 --- a/frappe/commands/utils.py +++ b/frappe/commands/utils.py @@ -863,6 +863,7 @@ def run_ui_tests( ): "Run UI tests" site = get_site(context) + frappe.init(site) app_base_path = frappe.get_app_source_path(app) site_url = frappe.utils.get_site_url(site) admin_password = frappe.get_conf(site).admin_password @@ -921,6 +922,7 @@ def run_ui_tests( @click.command("serve") +@click.option("--host", default="127.0.0.1") @click.option("--port", default=8000) @click.option("--profile", is_flag=True, default=False) @click.option( @@ -935,6 +937,7 @@ def run_ui_tests( @pass_context def serve( context, + host="127.0.0.1", port=None, profile=False, proxy=False, @@ -957,6 +960,7 @@ def serve( no_threading = True no_reload = True frappe.app.serve( + host=host, port=port, profile=profile, proxy=proxy, diff --git a/frappe/core/api/file.py b/frappe/core/api/file.py index 1a616c3134..aa8be30707 100644 --- a/frappe/core/api/file.py +++ b/frappe/core/api/file.py @@ -84,7 +84,11 @@ def get_files_by_search_text(text: str) -> list[dict]: @frappe.whitelist(allow_guest=True) def get_max_file_size() -> int: - return cint(frappe.conf.get("max_file_size")) or 10485760 + return ( + cint(frappe.get_system_settings("max_file_size")) * 1024 * 1024 + or cint(frappe.conf.get("max_file_size")) + or 25 * 1024 * 1024 + ) @frappe.whitelist() diff --git a/frappe/translations/bo.csv b/frappe/core/doctype/audit_trail/__init__.py similarity index 100% rename from frappe/translations/bo.csv rename to frappe/core/doctype/audit_trail/__init__.py diff --git a/frappe/core/doctype/audit_trail/audit_trail.html b/frappe/core/doctype/audit_trail/audit_trail.html new file mode 100644 index 0000000000..74ed663e65 --- /dev/null +++ b/frappe/core/doctype/audit_trail/audit_trail.html @@ -0,0 +1,77 @@ +
+ +
+ {% if documents.length > 1 %} +

Documents to Compare : {{ documents.length }}

+ {% else %} +

Documents to Compare : {{ documents.length - 1 }}

+ {% endif %} +
+ + {% var field_keys = Object.keys(changed).sort(); %} + {% if field_keys.length > 0 %} +
+
Fields Changed
+ + + + {% for doc in documents %} + + {% endfor %} + + + {% for fieldname in field_keys %} + + + {% var values = changed[fieldname] %} + + {% for value in values %} + + {% endfor %} + + {% endfor %} + +
Fields {{ doc }}
{{ fieldname }} {{ value }}
+
+ {% endif %} + + {% var tables = Object.keys(row_changed).sort(); %} + {% if tables.length > 0 %} +
+
Rows Updated
+ + + + + + {% for table in tables %} + + + + {% for doc in documents %} + + {% endfor %} + + + {% var rows = Object.keys(row_changed[table]).sort(); %} + {% for idx in rows %} + + {% var fields = Object.keys(row_changed[table][idx]).sort(); %} + {% for field in fields %} + + + {% var values = row_changed[table][idx][field] %} + {% for value in values %} + + {% endfor %} + + {% endfor %} + {% endfor %} + + {% endfor %} + +
Fields
{{ table }}{{ doc }}
idx : {{ idx }}
{{ field }} {{ value }}
+
+ {% endif %} + +
\ No newline at end of file diff --git a/frappe/core/doctype/audit_trail/audit_trail.js b/frappe/core/doctype/audit_trail/audit_trail.js new file mode 100644 index 0000000000..ffd289257e --- /dev/null +++ b/frappe/core/doctype/audit_trail/audit_trail.js @@ -0,0 +1,67 @@ +// Copyright (c) 2023, Frappe Technologies and contributors +// For license information, please see license.txt + +frappe.ui.form.on("Audit Trail", { + refresh(frm) { + frm.page.clear_indicator(); + + frm.disable_save(); + + frm.set_query("doctype_name", () => { + return { + filters: { + track_changes: 1, + is_submittable: 1, + }, + }; + }); + + frm.page.set_primary_action("Compare", () => { + frm.call({ + doc: frm.doc, + method: "compare_document", + callback: function (r) { + let document_names = r.message[0]; + let changed_fields = r.message[1]; + frm.events.render_changed_fields(frm, document_names, changed_fields); + frm.events.render_rows_added_or_removed(frm, changed_fields); + }, + }); + }); + }, + + render_changed_fields(frm, document_names, changed_fields) { + let render_dict = { + documents: document_names, + changed: changed_fields.changed, + row_changed: changed_fields.row_changed, + }; + $(frappe.render_template("audit_trail", render_dict)).appendTo( + frm.fields_dict.version_table.$wrapper.empty() + ); + frm.set_df_property("version_table", "hidden", 0); + }, + + render_rows_added_or_removed(frm, changed_fields) { + let added_or_removed = { + rows_added: changed_fields.added, + rows_removed: changed_fields.removed, + }; + + let hide_section = 0; + let section_dict = {}; + + for (let key in added_or_removed) { + hide_section = 0; + section_dict = { + added_or_removed: added_or_removed[key], + }; + $(frappe.render_template("audit_trail_rows_added_removed", section_dict)).appendTo( + frm.fields_dict[key].$wrapper.empty() + ); + + if (!frm.fields_dict[key].disp_area.innerHTML.includes(" + {% var docs = Object.keys(added_or_removed) %} + {% for doc in docs %} +
+ {% if Object.keys(added_or_removed[doc]).length > 0 %} +
{{ doc }}
+
+ {% var tables = Object.keys(added_or_removed[doc]) %} + {% for table in tables %} +
{{ table }}
+ + + {% var fieldnames = Object.keys(added_or_removed[doc][table][0]) %} + {% for fieldname in fieldnames %} + + {% endfor %} + + + {% var rows = Object.keys(added_or_removed[doc][table]) %} + {% for row in rows %} + + {% var field_keys = Object.keys(added_or_removed[doc][table][row]) %} + {% for key in field_keys %} + + {% endfor %} + + {% endfor %} + +
{{ fieldname }}
{{ added_or_removed[doc][table][row][key] }}
+ {% endfor %} + {% endif %} +
+ {% endfor %} + \ No newline at end of file diff --git a/frappe/core/doctype/audit_trail/test_audit_trail.py b/frappe/core/doctype/audit_trail/test_audit_trail.py new file mode 100644 index 0000000000..45093de033 --- /dev/null +++ b/frappe/core/doctype/audit_trail/test_audit_trail.py @@ -0,0 +1,134 @@ +# Copyright (c) 2023, Frappe Technologies and Contributors +# See license.txt + +import frappe +from frappe.tests.utils import FrappeTestCase + + +class TestAuditTrail(FrappeTestCase): + def setUp(self): + self.child_doctype = create_custom_child_doctype() + self.custom_doctype = create_custom_doctype() + + def test_compare_changed_fields(self): + doc = frappe.new_doc("Test Custom Doctype for Doc Comparator") + doc.test_field = "first value" + doc.submit() + doc.cancel() + + changed_fields = frappe._dict(test_field="second value") + amended_doc = amend_document(doc, changed_fields, {}, 1) + amended_doc.cancel() + + changed_fields = frappe._dict(test_field="third value") + re_amended_doc = amend_document(amended_doc, changed_fields, {}, 1) + + comparator = create_comparator_doc("Test Custom Doctype for Doc Comparator", re_amended_doc.name) + documents, results = comparator.compare_document() + + test_field_values = results["changed"]["Field"] + self.check_expected_values(test_field_values, ["first value", "second value", "third value"]) + + def test_compare_rows(self): + doc = frappe.new_doc("Test Custom Doctype for Doc Comparator") + doc.append("child_table_field", {"test_table_field": "old row 1 value"}) + doc.submit() + doc.cancel() + + child_table_new = [{"test_table_field": "new row 1 value"}, {"test_table_field": "row 2 value"}] + rows_updated = frappe._dict(child_table_field=child_table_new) + amended_doc = amend_document(doc, {}, rows_updated, 1) + + comparator = create_comparator_doc("Test Custom Doctype for Doc Comparator", amended_doc.name) + documents, results = comparator.compare_document() + + results = frappe._dict(results) + self.check_rows_updated(results.row_changed) + self.check_rows_added(results.added[amended_doc.name]) + + def check_rows_updated(self, row_changed): + self.assertIn("Child Table Field", row_changed) + self.assertIn(0, row_changed["Child Table Field"]) + self.assertIn("Table Field", row_changed["Child Table Field"][0]) + table_field_values = row_changed["Child Table Field"][0]["Table Field"] + self.check_expected_values(table_field_values, ["old row 1 value", "new row 1 value"]) + + def check_rows_added(self, rows_added): + self.assertIn("Child Table Field", rows_added) + child_table = rows_added["Child Table Field"] + self.assertIn("Table Field", child_table[0]) + self.check_expected_values(child_table[0]["Table Field"], "row 2 value") + + def check_expected_values(self, values_to_check, expected_values): + for i in range(len(values_to_check)): + self.assertEqual(values_to_check[i], expected_values[i]) + + def tearDown(self): + self.child_doctype.delete() + self.custom_doctype.delete() + + +def create_custom_child_doctype(): + child_doctype = frappe.get_doc( + { + "doctype": "DocType", + "module": "Core", + "name": "Test Custom Child for Doc Comparator", + "custom": 1, + "istable": 1, + "fields": [ + { + "label": "Table Field", + "fieldname": "test_table_field", + "fieldtype": "Data", + "in_list_view": 1, + }, + ], + } + ).insert(ignore_if_duplicate=True) + return child_doctype + + +def create_custom_doctype(): + custom_doctype = frappe.get_doc( + { + "doctype": "DocType", + "module": "Core", + "name": "Test Custom Doctype for Doc Comparator", + "custom": 1, + "is_submittable": 1, + "fields": [ + { + "label": "Field", + "fieldname": "test_field", + "fieldtype": "Data", + }, + { + "label": "Child Table Field", + "fieldname": "child_table_field", + "fieldtype": "Table", + "options": "Test Custom Child for Doc Comparator", + }, + ], + "permissions": [{"role": "System Manager", "read": 1}], + } + ).insert(ignore_if_duplicate=True) + return custom_doctype + + +def amend_document(amend_from, changed_fields, rows_updated, submit=False): + amended_doc = frappe.copy_doc(amend_from) + amended_doc.amended_from = amend_from.name + amended_doc.update(changed_fields) + for child_table in rows_updated: + amended_doc.set(child_table, rows_updated[child_table]) + if submit: + amended_doc.submit() + return amended_doc + + +def create_comparator_doc(doctype_name, document): + comparator = frappe.new_doc("Audit Trail") + comparator.doctype_name = doctype_name + comparator.document = document + return comparator diff --git a/frappe/core/doctype/communication/email.py b/frappe/core/doctype/communication/email.py index 219568f7a6..5cb7bc668e 100755 --- a/frappe/core/doctype/communication/email.py +++ b/frappe/core/doctype/communication/email.py @@ -258,15 +258,17 @@ def add_attachments(name: str, attachments: Iterable[str | dict]) -> None: @frappe.whitelist(allow_guest=True, methods=("GET",)) def mark_email_as_seen(name: str = None): + frappe.request.after_response.add(lambda: _mark_email_as_seen(name)) + frappe.response.update(frappe.utils.get_imaginary_pixel_response()) + + +def _mark_email_as_seen(name): try: update_communication_as_read(name) - frappe.db.commit() # nosemgrep: this will be called in a GET request - except Exception: frappe.log_error("Unable to mark as seen", None, "Communication", name) - finally: - frappe.response.update(frappe.utils.get_imaginary_pixel_response()) + frappe.db.commit() # nosemgrep: after_response requires explicit commit def update_communication_as_read(name): diff --git a/frappe/core/doctype/data_import/importer.py b/frappe/core/doctype/data_import/importer.py index 60e7ee7f9f..fbbf12a978 100644 --- a/frappe/core/doctype/data_import/importer.py +++ b/frappe/core/doctype/data_import/importer.py @@ -45,6 +45,7 @@ class Importer: file_path or data_import.google_sheets_url or data_import.import_file, self.template_options, self.import_type, + console=self.console, ) def get_data_for_import_preview(self): @@ -393,12 +394,13 @@ class Importer: class ImportFile: - def __init__(self, doctype, file, template_options=None, import_type=None): + def __init__(self, doctype, file, template_options=None, import_type=None, *, console=False): self.doctype = doctype self.template_options = template_options or frappe._dict(column_to_field_map=frappe._dict()) self.column_to_field_map = self.template_options.column_to_field_map self.import_type = import_type self.warnings = [] + self.console = console self.file_doc = self.file_path = self.google_sheets_url = None if isinstance(file, str): diff --git a/frappe/core/doctype/doctype/test_doctype.py b/frappe/core/doctype/doctype/test_doctype.py index 54d0e5fb7d..c1c7589564 100644 --- a/frappe/core/doctype/doctype/test_doctype.py +++ b/frappe/core/doctype/doctype/test_doctype.py @@ -18,8 +18,11 @@ from frappe.core.doctype.doctype.doctype import ( WrongOptionsDoctypeLinkError, validate_links_table_fieldnames, ) +from frappe.core.doctype.rq_job.test_rq_job import wait_for_completion from frappe.custom.doctype.custom_field.custom_field import create_custom_fields from frappe.desk.form.load import getdoc +from frappe.model.delete_doc import delete_controllers +from frappe.model.sync import remove_orphan_doctypes from frappe.tests.utils import FrappeTestCase @@ -739,6 +742,21 @@ class TestDocType(FrappeTestCase): self.assertEqual(frappe.get_meta(doctype).get_field(field).default, "DELETETHIS") frappe.delete_doc("DocType", doctype) + @unittest.skipUnless( + os.access(frappe.get_app_path("frappe"), os.W_OK), "Only run if frappe app paths is writable" + ) + @patch.dict(frappe.conf, {"developer_mode": 1}) + def test_delete_orphaned_doctypes(self): + doctype = new_doctype(custom=0).insert() + frappe.db.commit() + + delete_controllers(doctype.name, doctype.module) + job = frappe.enqueue(remove_orphan_doctypes) + wait_for_completion(job) + + frappe.db.rollback() + self.assertFalse(frappe.db.exists("DocType", doctype.name)) + def test_not_in_list_view_for_not_allowed_mandatory_field(self): doctype = new_doctype( fields=[ diff --git a/frappe/core/doctype/file/file.py b/frappe/core/doctype/file/file.py index 985e5f50ba..0794852aed 100755 --- a/frappe/core/doctype/file/file.py +++ b/frappe/core/doctype/file/file.py @@ -659,10 +659,10 @@ class File(Document): file_size = len(self._content or b"") if file_size > max_file_size: - frappe.throw( - _("File size exceeded the maximum allowed size of {0} MB").format(max_file_size / 1048576), - exc=MaxFileSizeReachedError, - ) + msg = _("File size exceeded the maximum allowed size of {0} MB").format(max_file_size / 1048576) + if frappe.has_permission("System Settings", "write"): + msg += ".
" + _("You can increase the limit from System Settings.") + frappe.throw(msg, exc=MaxFileSizeReachedError) return file_size diff --git a/frappe/core/doctype/module_def/module_def.py b/frappe/core/doctype/module_def/module_def.py index 6b4b08c600..28f7b4f11a 100644 --- a/frappe/core/doctype/module_def/module_def.py +++ b/frappe/core/doctype/module_def/module_def.py @@ -3,6 +3,7 @@ import json import os +from pathlib import Path import frappe from frappe.model.document import Document @@ -63,21 +64,23 @@ class ModuleDef(Document): if not frappe.conf.get("developer_mode") or frappe.flags.in_uninstall or self.custom: return - modules = None if frappe.local.module_app.get(frappe.scrub(self.name)): - delete_folder(self.module_name, "Module Def", self.name) - with open(frappe.get_app_path(self.app_name, "modules.txt")) as f: - content = f.read() - if self.name in content.splitlines(): - modules = list(filter(None, content.splitlines())) - modules.remove(self.name) + frappe.db.after_commit.add(self.delete_module_from_file) - if modules: - with open(frappe.get_app_path(self.app_name, "modules.txt"), "w") as f: - f.write("\n".join(modules)) + def delete_module_from_file(self): + delete_folder(self.module_name, "Module Def", self.name) + modules = [] - frappe.clear_cache() - frappe.setup_module_map() + modules_txt = Path(frappe.get_app_path(self.app_name, "modules.txt")) + modules = [m for m in modules_txt.read_text().splitlines() if m] + + if self.name in modules: + modules.remove(self.name) + + if modules: + modules_txt.write_text("\n".join(modules)) + frappe.clear_cache() + frappe.setup_module_map() @frappe.whitelist() diff --git a/frappe/core/doctype/report/report.py b/frappe/core/doctype/report/report.py index 697aa9a300..0443766de1 100644 --- a/frappe/core/doctype/report/report.py +++ b/frappe/core/doctype/report/report.py @@ -86,7 +86,7 @@ class Report(Document): if ( self.is_standard == "Yes" and not cint(getattr(frappe.local.conf, "developer_mode", 0)) - and not frappe.flags.in_patch + and not (frappe.flags.in_migrate or frappe.flags.in_patch) ): frappe.throw(_("You are not allowed to delete Standard Report")) delete_custom_role("report", self.name) diff --git a/frappe/core/doctype/rq_job/test_rq_job.py b/frappe/core/doctype/rq_job/test_rq_job.py index a8e11269ee..fc191da233 100644 --- a/frappe/core/doctype/rq_job/test_rq_job.py +++ b/frappe/core/doctype/rq_job/test_rq_job.py @@ -15,16 +15,20 @@ from frappe.utils import cstr, execute_in_shell from frappe.utils.background_jobs import get_job_status, is_job_enqueued +@timeout(seconds=20) +def wait_for_completion(job: Job): + while True: + if not (job.is_queued or job.is_started): + break + time.sleep(0.2) + + class TestRQJob(FrappeTestCase): BG_JOB = "frappe.core.doctype.rq_job.test_rq_job.test_func" - @timeout(seconds=20) def check_status(self, job: Job, status, wait=True): - while wait: - if not (job.is_queued or job.is_started): - break - time.sleep(0.2) - + if wait: + wait_for_completion(job) self.assertEqual(frappe.get_doc("RQ Job", job.id).status, status) def test_serialization(self): diff --git a/frappe/core/doctype/system_settings/system_settings.json b/frappe/core/doctype/system_settings/system_settings.json index 1eea7dcecb..8970c518d1 100644 --- a/frappe/core/doctype/system_settings/system_settings.json +++ b/frappe/core/doctype/system_settings/system_settings.json @@ -81,7 +81,10 @@ "disable_system_update_notification", "disable_change_log_notification", "telemetry_section", - "enable_telemetry" + "enable_telemetry", + "files_section", + "max_file_size", + "column_break_uqma" ], "fields": [ { @@ -571,12 +574,28 @@ "fieldname": "force_web_capture_mode_for_uploads", "fieldtype": "Check", "label": "Force Web Capture Mode for Uploads" + }, + { + "collapsible": 1, + "fieldname": "files_section", + "fieldtype": "Section Break", + "label": "Files" + }, + { + "fieldname": "max_file_size", + "fieldtype": "Int", + "label": "Max File Size (MB)", + "non_negative": 1 + }, + { + "fieldname": "column_break_uqma", + "fieldtype": "Column Break" } ], "icon": "fa fa-cog", "issingle": 1, "links": [], - "modified": "2023-08-31 20:19:07.181041", + "modified": "2023-09-13 12:49:32.309521", "modified_by": "Administrator", "module": "Core", "name": "System Settings", diff --git a/frappe/core/doctype/system_settings/system_settings.py b/frappe/core/doctype/system_settings/system_settings.py index e0c0c58fe7..4fc99d29a2 100644 --- a/frappe/core/doctype/system_settings/system_settings.py +++ b/frappe/core/doctype/system_settings/system_settings.py @@ -64,6 +64,7 @@ class SystemSettings(Document): login_with_email_link_expiry: DF.Int logout_on_password_reset: DF.Check max_auto_email_report_per_user: DF.Int + max_file_size: DF.Int minimum_password_score: DF.Literal["2", "3", "4"] number_format: DF.Literal[ "#,###.##", diff --git a/frappe/core/doctype/user/user.js b/frappe/core/doctype/user/user.js index 7a72918f35..cd89a57dfe 100644 --- a/frappe/core/doctype/user/user.js +++ b/frappe/core/doctype/user/user.js @@ -301,7 +301,7 @@ frappe.ui.form.on("User", { email: frm.doc.email, }, callback: function (r) { - if (!Array.isArray(r.message)) { + if (!Array.isArray(r.message) || !r.message.length) { frappe.route_options = { email_id: frm.doc.email, awaiting_password: 1, diff --git a/frappe/core/doctype/version/version.py b/frappe/core/doctype/version/version.py index d68cda25b5..89b4a995ea 100644 --- a/frappe/core/doctype/version/version.py +++ b/frappe/core/doctype/version/version.py @@ -59,7 +59,7 @@ class Version(Document): return json.loads(self.data) -def get_diff(old, new, for_child=False): +def get_diff(old, new, for_child=False, compare_cancelled=False): """Get diff between 2 document objects If there is a change, then returns a dict like: @@ -112,6 +112,11 @@ def get_diff(old, new, for_child=False): # check rows for additions, changes for i, d in enumerate(new_value): old_row_name = getattr(d, old_row_name_field, None) + if compare_cancelled: + if amended_from: + if len(old_value) > i: + old_row_name = old_value[i].name + if old_row_name and old_row_name in old_rows_by_name: found_rows.add(old_row_name) diff --git a/frappe/core/page/permission_manager/permission_manager.js b/frappe/core/page/permission_manager/permission_manager.js index bc27106068..bbd2dda7b3 100644 --- a/frappe/core/page/permission_manager/permission_manager.js +++ b/frappe/core/page/permission_manager/permission_manager.js @@ -44,22 +44,23 @@ frappe.PermissionEngine = class PermissionEngine { } setup_page() { - this.doctype_select = this.wrapper.page - .add_select( - __("Document Type"), - [{ value: "", label: __("Select Document Type") + "..." }].concat( - this.options.doctypes - ) - ) - .change(function () { - frappe.set_route("permission-manager", $(this).val()); - }); + this.doctype_select = this.wrapper.page.add_field({ + fieldname: "doctype_select", + label: __("Document Type"), + fieldtype: "Link", + options: "DocType", + change: function () { + frappe.set_route("permission-manager", this.get_value()); + }, + }); - this.role_select = this.wrapper.page - .add_select(__("Roles"), [__("Select Role") + "..."].concat(this.options.roles)) - .change(() => { - this.refresh(); - }); + this.role_select = this.wrapper.page.add_field({ + fieldname: "role_select", + label: __("Roles"), + fieldtype: "Link", + options: "Role", + change: () => this.refresh(), + }); this.page.add_inner_button(__("Set User Permissions"), () => { return frappe.set_route("List", "User Permission"); @@ -76,13 +77,13 @@ frappe.PermissionEngine = class PermissionEngine { return; } if (frappe.get_route()[1]) { - this.doctype_select.val(frappe.get_route()[1]); + this.doctype_select.set_value(frappe.get_route()[1]); } else if (frappe.route_options) { if (frappe.route_options.doctype) { - this.doctype_select.val(frappe.route_options.doctype); + this.doctype_select.set_value(frappe.route_options.doctype); } if (frappe.route_options.role) { - this.role_select.val(frappe.route_options.role); + this.role_select.set_value(frappe.route_options.role); } frappe.route_options = null; } @@ -140,13 +141,11 @@ frappe.PermissionEngine = class PermissionEngine { } get_doctype() { - let doctype = this.doctype_select.val(); - return this.doctype_select.get(0).selectedIndex == 0 ? null : doctype; + return this.doctype_select.get_value(); } get_role() { - let role = this.role_select.val(); - return this.role_select.get(0).selectedIndex == 0 ? null : role; + return this.role_select.get_value(); } set_empty_message(message) { diff --git a/frappe/desk/desktop.py b/frappe/desk/desktop.py index eac90cee94..8964b8783c 100644 --- a/frappe/desk/desktop.py +++ b/frappe/desk/desktop.py @@ -72,9 +72,7 @@ class Workspace: """Returns true if Has Role is not set or the user is allowed.""" from frappe.utils import has_common - allowed = [ - d.role for d in frappe.get_all("Has Role", fields=["role"], filters={"parent": self.doc.name}) - ] + allowed = [d.role for d in self.doc.roles] custom_roles = get_custom_allowed_roles("page", self.doc.name) allowed.extend(custom_roles) diff --git a/frappe/desk/form_tour/main_workspace_tour/main_workspace_tour.json b/frappe/desk/form_tour/main_workspace_tour/main_workspace_tour.json index 0f936abae0..afd0583cfb 100644 --- a/frappe/desk/form_tour/main_workspace_tour/main_workspace_tour.json +++ b/frappe/desk/form_tour/main_workspace_tour/main_workspace_tour.json @@ -8,7 +8,7 @@ "include_name_field": 0, "is_standard": 1, "list_name": "", - "modified": "2023-08-24 11:01:18.688875", + "modified": "2023-05-24 12:43:43.741781", "modified_by": "Administrator", "module": "Desk", "name": "Main Workspace Tour", @@ -22,7 +22,7 @@ "steps": [ { "description": "This is Awesomebar, it helps you to navigate anywhere in the system, find documents, reports, settings, create new records and many more things.", - "element_selector": "#navbar-search[aria-expanded=\"true\"]", + "element_selector": "#navbar-search", "fieldtype": "0", "has_next_condition": 0, "hide_buttons": 0, diff --git a/frappe/desk/notifications.py b/frappe/desk/notifications.py index 3ae0619aab..f0470566ac 100644 --- a/frappe/desk/notifications.py +++ b/frappe/desk/notifications.py @@ -270,7 +270,9 @@ def get_open_count(doctype, name, items=None): } for d in items: - internal_link_for_doctype = links.get("internal_links", {}).get(d) + internal_link_for_doctype = links.get("internal_links", {}).get(d) or links.get( + "internal_and_external_links", {} + ).get(d) if internal_link_for_doctype: internal_links_data_for_d = get_internal_links(doc, internal_link_for_doctype, d) if internal_links_data_for_d["count"]: diff --git a/frappe/desk/query_report.py b/frappe/desk/query_report.py index dd07dced40..ce3d4ee3e7 100644 --- a/frappe/desk/query_report.py +++ b/frappe/desk/query_report.py @@ -25,8 +25,7 @@ def get_report_doc(report_name): if doc.report_type == "Custom Report": custom_report_doc = doc - reference_report = custom_report_doc.reference_report - doc = frappe.get_doc("Report", reference_report) + doc = get_reference_report(doc) doc.custom_report = report_name if custom_report_doc.json: data = json.loads(custom_report_doc.json) @@ -172,9 +171,17 @@ def get_script(report_name): "html_format": html_format, "execution_time": frappe.cache.hget("report_execution_time", report_name) or 0, "filters": report.filters, + "custom_report_name": report.name if report.get("is_custom_report") else None, } +def get_reference_report(report): + if report.report_type != "Custom Report": + return report + reference_report = frappe.get_doc("Report", report.reference_report) + return get_reference_report(reference_report) + + @frappe.whitelist() @frappe.read_only() def run( diff --git a/frappe/email/doctype/email_domain/email_domain.py b/frappe/email/doctype/email_domain/email_domain.py index 69a8701dbe..5b9f38615a 100644 --- a/frappe/email/doctype/email_domain/email_domain.py +++ b/frappe/email/doctype/email_domain/email_domain.py @@ -1,12 +1,13 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors # License: MIT. See LICENSE +import imaplib +import poplib import smtplib from functools import wraps import frappe from frappe import _ -from frappe.email.receive import Timed_IMAP4, Timed_IMAP4_SSL, Timed_POP3, Timed_POP3_SSL from frappe.email.utils import get_port from frappe.model.document import Document from frappe.utils import cint @@ -101,9 +102,9 @@ class EmailDomain(Document): self.incoming_port = get_port(self) if self.use_imap: - conn_method = Timed_IMAP4_SSL if self.use_ssl else Timed_IMAP4 + conn_method = imaplib.IMAP4_SSL if self.use_ssl else imaplib.IMAP4 else: - conn_method = Timed_POP3_SSL if self.use_ssl else Timed_POP3 + conn_method = poplib.POP3_SSL if self.use_ssl else poplib.POP3 self.use_starttls = cint(self.use_imap and self.use_starttls and not self.use_ssl) incoming_conn = conn_method(self.email_server, port=self.incoming_port) diff --git a/frappe/email/receive.py b/frappe/email/receive.py index 6af6c3cebe..39a5537ebf 100644 --- a/frappe/email/receive.py +++ b/frappe/email/receive.py @@ -49,10 +49,6 @@ class EmailSizeExceededError(frappe.ValidationError): pass -class EmailTimeoutError(frappe.ValidationError): - pass - - class LoginLimitExceeded(frappe.ValidationError): pass @@ -75,11 +71,11 @@ class EmailServer: """Connect to IMAP""" try: if cint(self.settings.use_ssl): - self.imap = Timed_IMAP4_SSL( + self.imap = imaplib.IMAP4_SSL( self.settings.host, self.settings.incoming_port, timeout=frappe.conf.get("pop_timeout") ) else: - self.imap = Timed_IMAP4( + self.imap = imaplib.IMAP4( self.settings.host, self.settings.incoming_port, timeout=frappe.conf.get("pop_timeout") ) @@ -109,11 +105,11 @@ class EmailServer: # this method return pop connection try: if cint(self.settings.use_ssl): - self.pop = Timed_POP3_SSL( + self.pop = poplib.POP3_SSL( self.settings.host, self.settings.incoming_port, timeout=frappe.conf.get("pop_timeout") ) else: - self.pop = Timed_POP3( + self.pop = poplib.POP3( self.settings.host, self.settings.incoming_port, timeout=frappe.conf.get("pop_timeout") ) @@ -170,7 +166,7 @@ class EmailServer: for i, uid in enumerate(email_list[:100]): try: self.retrieve_message(uid, i + 1) - except (EmailTimeoutError, LoginLimitExceeded): + except (_socket.timeout, LoginLimitExceeded): # get whatever messages were retrieved break @@ -263,7 +259,7 @@ class EmailServer: else: msg = self.pop.retr(msg_num) self.latest_messages.append(b"\n".join(msg[1])) - except EmailTimeoutError: + except _socket.timeout: # propagate this error to break the loop raise @@ -900,43 +896,3 @@ class InboundMail(Email): "has_attachment": 1 if self.attachments else 0, "seen": self.seen_status or 0, } - - -class TimerMixin: - def __init__(self, *args, **kwargs): - self.timeout = kwargs.pop("timeout", 0.0) - self.elapsed_time = 0.0 - self._super.__init__(self, *args, **kwargs) - if self.timeout: - # set per operation timeout to one-fifth of total pop timeout - self.sock.settimeout(self.timeout / 5.0) - - def _getline(self, *args, **kwargs): - start_time = time.monotonic() - ret = self._super._getline(self, *args, **kwargs) - - self.elapsed_time += time.monotonic() - start_time - if self.timeout and self.elapsed_time > self.timeout: - raise EmailTimeoutError - - return ret - - def quit(self, *args, **kwargs): - self.elapsed_time = 0.0 - return self._super.quit(self, *args, **kwargs) - - -class Timed_POP3(TimerMixin, poplib.POP3): - _super = poplib.POP3 - - -class Timed_POP3_SSL(TimerMixin, poplib.POP3_SSL): - _super = poplib.POP3_SSL - - -class Timed_IMAP4(TimerMixin, imaplib.IMAP4): - _super = imaplib.IMAP4 - - -class Timed_IMAP4_SSL(TimerMixin, imaplib.IMAP4_SSL): - _super = imaplib.IMAP4_SSL diff --git a/frappe/email/test_smtp.py b/frappe/email/test_smtp.py index b101f610a1..0495fd8596 100644 --- a/frappe/email/test_smtp.py +++ b/frappe/email/test_smtp.py @@ -73,7 +73,7 @@ def create_email_account(email_id, password, enable_outgoing, default_outgoing=0 "enable_incoming": 1, "append_to": append_to, "is_dummy_password": 1, - "smtp_server": "localhost", + "smtp_server": "127.0.0.1", "use_imap": 0, } diff --git a/frappe/hooks.py b/frappe/hooks.py index fd32ef9cf9..a417d19b79 100644 --- a/frappe/hooks.py +++ b/frappe/hooks.py @@ -422,7 +422,6 @@ before_request = [ "frappe.monitor.start", "frappe.rate_limiter.apply", ] -after_request = ["frappe.rate_limiter.update", "frappe.monitor.stop", "frappe.recorder.dump"] # Background Job Hooks before_job = [ diff --git a/frappe/integrations/doctype/google_calendar/google_calendar.py b/frappe/integrations/doctype/google_calendar/google_calendar.py index 695ae7db15..7125e243a9 100644 --- a/frappe/integrations/doctype/google_calendar/google_calendar.py +++ b/frappe/integrations/doctype/google_calendar/google_calendar.py @@ -214,7 +214,7 @@ def get_google_calendar_object(g_calendar): "token_uri": GoogleOAuth.OAUTH_URL, "client_id": google_settings.client_id, "client_secret": google_settings.get_password(fieldname="client_secret", raise_exception=False), - "scopes": "https://www.googleapis.com/auth/calendar/v3", + "scopes": ["https://www.googleapis.com/auth/calendar/v3"], } credentials = google.oauth2.credentials.Credentials(**credentials_dict) diff --git a/frappe/migrate.py b/frappe/migrate.py index 3241b14152..6b83521a7f 100644 --- a/frappe/migrate.py +++ b/frappe/migrate.py @@ -135,6 +135,7 @@ class SiteMigration: sync_customizations() sync_languages() flush_deferred_inserts() + frappe.model.sync.remove_orphan_doctypes() frappe.get_single("Portal Settings").sync_menu() frappe.get_single("Installed Applications").update_versions() diff --git a/frappe/model/meta.py b/frappe/model/meta.py index b26abef775..df04dc1eda 100644 --- a/frappe/model/meta.py +++ b/frappe/model/meta.py @@ -245,7 +245,7 @@ class Meta(Document): def get_label(self, fieldname): """Get label of the given fieldname""" if df := self.get_field(fieldname): - return df.label + return df.get("label") if fieldname in DEFAULT_FIELD_LABELS: return DEFAULT_FIELD_LABELS[fieldname]() diff --git a/frappe/model/sync.py b/frappe/model/sync.py index 0b344b892a..267a1667b5 100644 --- a/frappe/model/sync.py +++ b/frappe/model/sync.py @@ -7,6 +7,8 @@ import os import frappe +from frappe.cache_manager import clear_controller_cache +from frappe.model.base_document import get_controller from frappe.modules.import_file import import_file_by_path from frappe.modules.patch_handler import _patch_mode from frappe.utils import update_progress_bar @@ -135,3 +137,37 @@ def get_doc_files(files, start_path): files.append(doc_path) return files + + +def remove_orphan_doctypes(): + """Find and remove any orphaned doctypes. + + These are doctypes for which code and schema file is + deleted but entry is present in DocType table. + + Note: Deleting the entry doesn't delete any data. + So this is supposed to be non-destrictive operation. + """ + + doctype_names = frappe.get_all("DocType", {"custom": 0}, pluck="name") + orphan_doctypes = [] + + clear_controller_cache() + + for doctype in doctype_names: + try: + get_controller(doctype=doctype) + except ImportError: + orphan_doctypes.append(doctype) + except Exception: + continue + + if not orphan_doctypes: + return + + print(f"Orphaned DocType(s) found: {', '.join(orphan_doctypes)}") + for i, name in enumerate(orphan_doctypes): + frappe.delete_doc("DocType", name, force=True, ignore_missing=True) + update_progress_bar("Deleting orphaned DocTypes", i, len(orphan_doctypes)) + frappe.db.commit() + print() diff --git a/frappe/patches.txt b/frappe/patches.txt index 0a96030dcb..83db10d905 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -228,6 +228,5 @@ frappe.patches.v15_0.remove_background_jobs_from_dropdown frappe.desk.doctype.form_tour.patches.introduce_ui_tours execute:frappe.delete_doc_if_exists("Workspace", "Customization") execute:frappe.db.set_single_value("Document Naming Settings", "default_amend_naming", "Amend Counter") -execute:frappe.delete_doc_if_exists("DocType", "Error Snapshot") frappe.patches.v15_0.move_event_cancelled_to_status frappe.patches.v15_0.set_file_type diff --git a/frappe/printing/doctype/network_printer_settings/network_printer_settings.py b/frappe/printing/doctype/network_printer_settings/network_printer_settings.py index 7bae0996eb..ea359820d2 100644 --- a/frappe/printing/doctype/network_printer_settings/network_printer_settings.py +++ b/frappe/printing/doctype/network_printer_settings/network_printer_settings.py @@ -20,7 +20,7 @@ class NetworkPrinterSettings(Document): server_ip: DF.Data # end: auto-generated types @frappe.whitelist() - def get_printers_list(self, ip="localhost", port=631): + def get_printers_list(self, ip="127.0.0.1", port=631): printer_list = [] try: import cups diff --git a/frappe/public/icons/timeless/icons.svg b/frappe/public/icons/timeless/icons.svg index 29aa428d99..a67fadfd36 100644 --- a/frappe/public/icons/timeless/icons.svg +++ b/frappe/public/icons/timeless/icons.svg @@ -120,6 +120,12 @@ + + + + + + @@ -302,7 +308,7 @@ - + diff --git a/frappe/public/js/frappe/file_uploader/TreeNode.vue b/frappe/public/js/frappe/file_uploader/TreeNode.vue index 308bb2b825..9b9f673ec9 100644 --- a/frappe/public/js/frappe/file_uploader/TreeNode.vue +++ b/frappe/public/js/frappe/file_uploader/TreeNode.vue @@ -1,22 +1,40 @@