diff --git a/.github/helper/install.sh b/.github/helper/install.sh index 5cdcbebe1a..5a4d341a9b 100644 --- a/.github/helper/install.sh +++ b/.github/helper/install.sh @@ -54,6 +54,8 @@ fi echo "Starting Bench..." +export FRAPPE_TUNE_GC=True + bench start &> ~/frappe-bench/bench_start.log & if [ "$TYPE" == "server" ] diff --git a/.github/workflows/linters.yml b/.github/workflows/linters.yml index c563f9e43f..481041ed68 100644 --- a/.github/workflows/linters.yml +++ b/.github/workflows/linters.yml @@ -25,7 +25,7 @@ jobs: fetch-depth: 200 - uses: actions/setup-node@v3 with: - node-version: 16 + node-version: 18 check-latest: true - name: Check commit titles diff --git a/.github/workflows/on_release.yml b/.github/workflows/on_release.yml index 851b5b1d6a..c17a7c6639 100644 --- a/.github/workflows/on_release.yml +++ b/.github/workflows/on_release.yml @@ -22,7 +22,7 @@ jobs: - uses: actions/setup-node@v3 with: - node-version: 16 + node-version: 18 - uses: actions/setup-python@v4 with: diff --git a/.github/workflows/patch-mariadb-tests.yml b/.github/workflows/patch-mariadb-tests.yml index 864661ef54..87cd530538 100644 --- a/.github/workflows/patch-mariadb-tests.yml +++ b/.github/workflows/patch-mariadb-tests.yml @@ -62,14 +62,16 @@ jobs: fi - name: Setup Python - uses: "gabrielfalcao/pyenv-action@v10" + uses: actions/setup-python@v4 with: - versions: 3.10:latest, 3.7:latest + python-version: | + 3.7 + 3.10 - name: Setup Node uses: actions/setup-node@v3 with: - node-version: 16 + node-version: 18 check-latest: true - name: Add to Hosts @@ -100,7 +102,6 @@ jobs: run: | bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh pip install frappe-bench - pyenv global $(pyenv versions | grep '3.10') bash ${GITHUB_WORKSPACE}/.github/helper/install.sh env: BEFORE: ${{ env.GITHUB_EVENT_PATH.before }} @@ -120,25 +121,25 @@ jobs: function update_to_version() { version=$1 + py=$2 + branch_name="version-$version-hotfix" echo "Updating to v$version" git fetch --depth 1 upstream $branch_name:$branch_name git checkout -q -f $branch_name - pip install -U frappe-bench pgrep honcho | xargs kill rm -rf ~/frappe-bench/env - bench -v setup env + bench -v setup env --python $py bench start &> ~/frappe-bench/bench_start.log & + bench --site test_site migrate } - pyenv global $(pyenv versions | grep '3.7') - update_to_version 12 - update_to_version 13 + update_to_version 12 python3.7 + update_to_version 13 python3.7 - pyenv global $(pyenv versions | grep '3.10') - update_to_version 14 + update_to_version 14 python3.10 echo "Updating to last commit" rm -rf ~/frappe-bench/env diff --git a/.github/workflows/publish-assets-develop.yml b/.github/workflows/publish-assets-develop.yml index 4feaebe15d..f42c3bc55c 100644 --- a/.github/workflows/publish-assets-develop.yml +++ b/.github/workflows/publish-assets-develop.yml @@ -16,7 +16,7 @@ jobs: path: 'frappe' - uses: actions/setup-node@v3 with: - node-version: 16 + node-version: 18 - uses: actions/setup-python@v4 with: python-version: '3.11' diff --git a/.github/workflows/server-tests.yml b/.github/workflows/server-tests.yml index 8ae0be0197..f5eac8e380 100644 --- a/.github/workflows/server-tests.yml +++ b/.github/workflows/server-tests.yml @@ -90,7 +90,7 @@ jobs: - uses: actions/setup-node@v3 with: - node-version: 16 + node-version: 18 check-latest: true - name: Add to Hosts diff --git a/.github/workflows/ui-tests.yml b/.github/workflows/ui-tests.yml index 1b88bc73ce..bea00748e9 100644 --- a/.github/workflows/ui-tests.yml +++ b/.github/workflows/ui-tests.yml @@ -78,7 +78,7 @@ jobs: - uses: actions/setup-node@v3 with: - node-version: 16 + node-version: 18 check-latest: true - name: Add to Hosts diff --git a/.semgrepignore b/.semgrepignore new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cypress.config.js b/cypress.config.js index bfd0bc0025..2fdf10ca14 100644 --- a/cypress.config.js +++ b/cypress.config.js @@ -8,6 +8,8 @@ module.exports = defineConfig({ pageLoadTimeout: 15000, video: true, videoUploadOnPasses: false, + viewportHeight: 960, + viewportWidth: 1400, retries: { runMode: 2, openMode: 2, diff --git a/cypress/integration/awesome_bar.js b/cypress/integration/awesome_bar.js index 71e5e498cf..ecf8dcc718 100644 --- a/cypress/integration/awesome_bar.js +++ b/cypress/integration/awesome_bar.js @@ -7,50 +7,41 @@ context("Awesome Bar", () => { beforeEach(() => { cy.get(".navbar .navbar-home").click(); - cy.findByPlaceholderText("Search or type a command (Ctrl + G)").clear(); + cy.findByPlaceholderText("Search or type a command (Ctrl + G)").as("awesome_bar"); + cy.get("@awesome_bar").type("{selectall}"); }); it("navigates to doctype list", () => { - cy.findByPlaceholderText("Search or type a command (Ctrl + G)").type("todo", { - delay: 700, - }); + cy.get("@awesome_bar").type("todo"); + cy.wait(100); cy.get(".awesomplete").findByRole("listbox").should("be.visible"); - cy.findByPlaceholderText("Search or type a command (Ctrl + G)").type("{enter}", { - delay: 700, - }); - + cy.get("@awesome_bar").type("{enter}"); cy.get(".title-text").should("contain", "To Do"); - cy.location("pathname").should("eq", "/app/todo"); }); it("find text in doctype list", () => { - cy.findByPlaceholderText("Search or type a command (Ctrl + G)").type( - "test in todo{enter}", - { delay: 700 } - ); - + cy.get("@awesome_bar").type("test in todo"); + cy.wait(100); + cy.get("@awesome_bar").type("{enter}"); cy.get(".title-text").should("contain", "To Do"); - - cy.findByPlaceholderText("ID").should("have.value", "%test%"); + cy.wait(200); + const name_filter = cy.findByPlaceholderText("ID"); + name_filter.should("have.value", "%test%"); cy.clear_filters(); }); it("navigates to new form", () => { - cy.findByPlaceholderText("Search or type a command (Ctrl + G)").type( - "new blog post{enter}", - { delay: 700 } - ); - + cy.get("@awesome_bar").type("new blog post"); + cy.wait(100); + cy.get("@awesome_bar").type("{enter}"); cy.get(".title-text:visible").should("have.text", "New Blog Post"); }); it("calculates math expressions", () => { - cy.findByPlaceholderText("Search or type a command (Ctrl + G)").type( - "55 + 32{downarrow}{enter}", - { delay: 700 } - ); - + cy.get("@awesome_bar").type("55 + 32"); + cy.wait(100); + cy.get("@awesome_bar").type("{downarrow}{enter}"); cy.get(".modal-title").should("contain", "Result"); cy.get(".msgprint").should("contain", "55 + 32 = 87"); }); diff --git a/cypress/integration/form.js b/cypress/integration/form.js index 8186647a14..cdd6d7e9bd 100644 --- a/cypress/integration/form.js +++ b/cypress/integration/form.js @@ -59,11 +59,13 @@ context("Form", () => { .blur(); cy.click_listview_row_item_with_text("Test Form Contact 3"); + cy.scrollTo(0); cy.get("#page-Contact .page-head").findByTitle("Test Form Contact 3").should("exist"); cy.get(".prev-doc").should("be.visible").click(); cy.get(".msgprint-dialog .modal-body").contains("No further records").should("be.visible"); cy.hide_dialog(); + cy.scrollTo(0); cy.get("#page-Contact .page-head").findByTitle("Test Form Contact 3").should("exist"); cy.get(".next-doc").should("be.visible").click(); cy.get(".msgprint-dialog .modal-body").contains("No further records").should("be.visible"); diff --git a/cypress/integration/list_view.js b/cypress/integration/list_view.js index 3fa0758f0c..b07f18edc2 100644 --- a/cypress/integration/list_view.js +++ b/cypress/integration/list_view.js @@ -13,15 +13,8 @@ context("List View", () => { it("Keep checkbox checked after Refresh", { scrollBehavior: false }, () => { cy.go_to_list("ToDo"); cy.clear_filters(); - cy.get(".list-row-container .list-row-checkbox").click({ - multiple: true, - force: true, - }); - cy.get(".actions-btn-group button").contains("Actions").should("be.visible"); - cy.intercept("/api/method/frappe.desk.reportview.get").as("list-refresh"); - cy.wait(3000); // wait before you hit another refresh - cy.get('button[data-original-title="Refresh"]').click(); - cy.wait("@list-refresh"); + cy.get(".list-header-subject > .list-subject > .list-check-all").click(); + cy.get("button[data-original-title='Refresh']").click(); cy.get(".list-row-container .list-row-checkbox:checked").should("be.visible"); }); @@ -39,11 +32,8 @@ context("List View", () => { ]; cy.go_to_list("ToDo"); cy.clear_filters(); - cy.get('.list-row-container:contains("Pending") .list-row-checkbox').click({ - multiple: true, - force: true, - }); - cy.get(".actions-btn-group button").contains("Actions").should("be.visible").click(); + cy.get(".list-header-subject > .list-subject > .list-check-all").click(); + cy.findByRole("button", { name: "Actions" }).click(); cy.get(".dropdown-menu li:visible .dropdown-item") .should("have.length", 9) .each((el, index) => { @@ -56,8 +46,7 @@ context("List View", () => { }).as("bulk-approval"); cy.wrap(elements).contains("Approve").click(); cy.wait("@bulk-approval"); - cy.wait(300); - cy.get_open_dialog().find(".btn-modal-close").click(); + cy.hide_dialog(); cy.reload(); cy.clear_filters(); cy.get(".list-row-container:visible").should("contain", "Approved"); diff --git a/cypress/integration/navigation.js b/cypress/integration/navigation.js index cf1b5dc89d..5961702ba5 100644 --- a/cypress/integration/navigation.js +++ b/cypress/integration/navigation.js @@ -1,21 +1,26 @@ context("Navigation", () => { before(() => { + cy.visit("/login"); cy.login(); + cy.visit("/app/website"); }); it("Navigate to route with hash in document name", () => { - cy.insert_doc("ToDo", { - __newname: "ABC#123", - description: "Test this", - ignore_duplicate: true, - }); - cy.visit("/app/todo/ABC#123"); + cy.insert_doc( + "ToDo", + { + __newname: "ABC#123", + description: "Test this", + }, + true + ); + cy.visit(`/app/todo/${encodeURIComponent("ABC#123")}`); cy.title().should("eq", "Test this - ABC#123"); cy.get_field("description", "Text Editor").contains("Test this"); cy.go("back"); cy.title().should("eq", "Website"); }); - it.only("Navigate to previous page after login", () => { + it("Navigate to previous page after login", () => { cy.visit("/app/todo"); cy.get(".page-head").findByTitle("To Do").should("be.visible"); cy.clear_filters(); diff --git a/esbuild/esbuild.js b/esbuild/esbuild.js index e1594aa651..1476db3c20 100644 --- a/esbuild/esbuild.js +++ b/esbuild/esbuild.js @@ -60,6 +60,11 @@ const argv = yargs type: "boolean", description: "Run build command for apps", }) + .option("save-metafiles", { + type: "boolean", + description: + "Saves esbuild metafiles for built assets. Useful for analyzing bundle size. More info: https://esbuild.github.io/api/#metafile", + }) .example("node esbuild --apps frappe,erpnext", "Run build only for frappe and erpnext") .example( "node esbuild --files frappe/website.bundle.js,frappe/desk.bundle.js", @@ -401,6 +406,13 @@ async function write_assets_json(metafile) { await fs.promises.writeFile(assets_json_path, JSON.stringify(new_assets_json, null, 4)); await update_assets_json_in_cache(); + if (argv["save-metafiles"]) { + // use current timestamp in readable formate as a suffix for filename + let current_timestamp = new Date().getTime(); + const metafile_name = `meta-${current_timestamp}.json`; + await fs.promises.writeFile(`${metafile_name}`, JSON.stringify(metafile)); + log(`Saved metafile as ${metafile_name}`); + } return { new_assets_json, prev_assets_json, diff --git a/frappe/__init__.py b/frappe/__init__.py index 0f85c8a11b..998d881a13 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -11,6 +11,7 @@ be used to build database driven apps. Read the documentation: https://frappeframework.com/docs """ import functools +import gc import importlib import inspect import json @@ -57,6 +58,7 @@ re._MAXCACHE = ( 50 # reduced from default 512 given we are already maintaining this on parent worker ) +_tune_gc = bool(os.environ.get("FRAPPE_TUNE_GC", False)) if _dev_server: warnings.simplefilter("always", DeprecationWarning) @@ -2418,4 +2420,30 @@ def mock(type, size=1, locale="en"): return squashify(results) -from frappe.desk.search import validate_and_sanitize_search_inputs # noqa +def validate_and_sanitize_search_inputs(fn): + @functools.wraps(fn) + def wrapper(*args, **kwargs): + from frappe.desk.search import sanitize_searchfield + from frappe.utils import cint + + kwargs.update(dict(zip(fn.__code__.co_varnames, args))) + sanitize_searchfield(kwargs["searchfield"]) + kwargs["start"] = cint(kwargs["start"]) + kwargs["page_len"] = cint(kwargs["page_len"]) + + if kwargs["doctype"] and not db.exists("DocType", kwargs["doctype"]): + return [] + + return fn(**kwargs) + + return wrapper + + +if _tune_gc: + # generational GC gets triggered after certain allocs (g0) which is 700 by default. + # This number is quite small for frappe where a single query can potentially create 700+ + # objects easily. + # Bump this number higher, this will make GC less aggressive but that improves performance of + # everything else. + g0, g1, g2 = gc.get_threshold() # defaults are 700, 10, 10. + gc.set_threshold(g0 * 10, g1 * 2, g2 * 2) diff --git a/frappe/app.py b/frappe/app.py index ddde313ace..5113c858a5 100644 --- a/frappe/app.py +++ b/frappe/app.py @@ -1,6 +1,7 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +import gc import logging import os @@ -30,6 +31,30 @@ _site = None _sites_path = os.environ.get("SITES_PATH", ".") +# If gc.freeze is done then importing modules before forking allows us to share the memory +if frappe._tune_gc: + import frappe.boot + import frappe.client + import frappe.core.doctype.user.user + import frappe.database.mariadb.database # Load database related utils + import frappe.database.query + import frappe.desk.desktop # workspace + import frappe.model.db_query + import frappe.query_builder + import frappe.utils.background_jobs # Enqueue is very common + import frappe.utils.data # common utils + import frappe.utils.jinja # web page rendering + import frappe.utils.jinja_globals + import frappe.utils.redis_wrapper # Exact redis_wrapper + import frappe.utils.safe_exec + import frappe.utils.typing_validations # any whitelisted method uses this + import frappe.website.path_resolver # all the page types and resolver + import frappe.website.router # Website router + import frappe.website.website_generator # web page doctypes + +# end: module pre-loading + + @local_manager.middleware @Request.application def application(request: Request): @@ -394,3 +419,17 @@ def serve( use_evalex=not in_test_env, threaded=not no_threading, ) + + +# Both Gunicorn and RQ use forking to spawn workers. In an ideal world, the fork should be sharing +# most of the memory if there are no writes made to data because of Copy on Write, however, +# python's GC is not CoW friendly and writes to data even if user-code doesn't. Specifically, the +# generational GC which stores and mutates every python object: `PyGC_Head` +# +# Calling gc.freeze() moves all the objects imported so far into permanant generation and hence +# doesn't mutate `PyGC_Head` +# +# Refer to issue for more info: https://github.com/frappe/frappe/issues/18927 +if frappe._tune_gc: + gc.collect() # clean up any garbage created so far before freeze + gc.freeze() diff --git a/frappe/boot.py b/frappe/boot.py index 8881d25bd6..fb1bd3d7a2 100644 --- a/frappe/boot.py +++ b/frappe/boot.py @@ -21,7 +21,6 @@ from frappe.social.doctype.energy_point_log.energy_point_log import get_energy_p from frappe.social.doctype.energy_point_settings.energy_point_settings import ( is_energy_point_enabled, ) -from frappe.translate import get_lang_dict, get_messages_for_boot, get_translated_doctypes from frappe.utils import add_user_info, cstr, get_system_timezone from frappe.utils.change_log import get_versions from frappe.website.doctype.web_page_view.web_page_view import is_tracking_enabled @@ -29,6 +28,8 @@ from frappe.website.doctype.web_page_view.web_page_view import is_tracking_enabl def get_bootinfo(): """build and return boot info""" + from frappe.translate import get_lang_dict, get_translated_doctypes + frappe.set_user_lang(frappe.session.user) bootinfo = frappe._dict() hooks = frappe.get_hooks() @@ -257,6 +258,8 @@ def get_user_pages_or_reports(parent, cache=False): def load_translations(bootinfo): + from frappe.translate import get_messages_for_boot + bootinfo["lang"] = frappe.lang bootinfo["__messages"] = get_messages_for_boot() diff --git a/frappe/build.py b/frappe/build.py index 46301dadaf..5a9855ef16 100644 --- a/frappe/build.py +++ b/frappe/build.py @@ -9,9 +9,6 @@ from tempfile import mkdtemp, mktemp from urllib.parse import urlparse import click -import psutil -from requests import head -from requests.exceptions import HTTPError from semantic_version import Version import frappe @@ -27,7 +24,7 @@ class AssetsNotDownloadedError(Exception): pass -class AssetsDontExistError(HTTPError): +class AssetsDontExistError(Exception): pass @@ -78,6 +75,8 @@ def build_missing_files(): def get_assets_link(frappe_head) -> str: + import requests + tag = getoutput( r"cd ../apps/frappe && git show-ref --tags -d | grep %s | sed -e 's,.*" r" refs/tags/,,' -e 's/\^{}//'" % frappe_head @@ -89,7 +88,7 @@ def get_assets_link(frappe_head) -> str: else: url = f"http://assets.frappeframework.com/{frappe_head}.tar.gz" - if not head(url): + if not requests.head(url): reference = f"Release {tag}" if tag else f"Commit {frappe_head}" raise AssetsDontExistError(f"Assets for {reference} don't exist") @@ -230,6 +229,7 @@ def bundle( verbose=False, skip_frappe=False, files=None, + save_metafiles=False, ): """concat / minify js files""" setup() @@ -249,6 +249,9 @@ def bundle( command += " --run-build-command" + if save_metafiles: + command += " --save-metafiles" + check_node_executable() frappe_app_path = frappe.get_app_path("frappe", "..") frappe.commands.popen(command, cwd=frappe_app_path, env=get_node_env(), raise_err=True) @@ -275,8 +278,8 @@ def watch(apps=None): def check_node_executable(): node_version = Version(subprocess.getoutput("node -v")[1:]) warn = "⚠️ " - if node_version.major < 14: - click.echo(f"{warn} Please update your node version to 14") + if node_version.major < 18: + click.echo(f"{warn} Please update your node version to 18") if not shutil.which("yarn"): click.echo(f"{warn} Please install yarn using below command and try again.\nnpm install -g yarn") click.echo() @@ -288,6 +291,8 @@ def get_node_env(): def get_safe_max_old_space_size(): + import psutil + safe_max_old_space_size = 0 try: total_memory = psutil.virtual_memory().total / (1024 * 1024) diff --git a/frappe/cache_manager.py b/frappe/cache_manager.py index 6ee88d9d37..f47478d871 100644 --- a/frappe/cache_manager.py +++ b/frappe/cache_manager.py @@ -1,10 +1,7 @@ # Copyright (c) 2018, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import json - import frappe -from frappe.desk.notifications import clear_notifications, delete_notification_count_for common_default_keys = ["__default", "__global"] @@ -79,6 +76,8 @@ doctype_cache_keys = ( def clear_user_cache(user=None): + from frappe.desk.notifications import clear_notifications + # this will automatically reload the global cache # so it is important to clear this first clear_notifications(user) @@ -128,6 +127,8 @@ def clear_doctype_cache(doctype=None): def _clear_doctype_cache_form_redis(doctype: str | None = None): + from frappe.desk.notifications import delete_notification_count_for + for key in ("is_table", "doctype_modules"): frappe.cache.delete_value(key) diff --git a/frappe/commands/redis_utils.py b/frappe/commands/redis_utils.py index 1c3292b7fa..07ad8715c4 100644 --- a/frappe/commands/redis_utils.py +++ b/frappe/commands/redis_utils.py @@ -3,7 +3,6 @@ import os import click import frappe -from frappe.installer import update_site_config from frappe.utils.redis_queue import RedisQueue @@ -23,6 +22,8 @@ def create_rq_users(set_admin_password=False, use_rq_auth=False): acl config file will be used by redis server while starting the server and app config is used by app while connecting to redis server. """ + from frappe.installer import update_site_config + acl_file_path = os.path.abspath("../config/redis_queue.acl") with frappe.init_site(): diff --git a/frappe/commands/site.py b/frappe/commands/site.py index 25c8c3159d..d606bb78cf 100644 --- a/frappe/commands/site.py +++ b/frappe/commands/site.py @@ -9,7 +9,6 @@ import click # imports - module imports import frappe from frappe.commands import get_site, pass_context -from frappe.core.doctype.log_settings.log_settings import LOG_DOCTYPES from frappe.exceptions import SiteNotSpecifiedError @@ -1199,11 +1198,12 @@ def build_search_index(context): @click.command("clear-log-table") -@click.option("--doctype", required=True, type=click.Choice(LOG_DOCTYPES), help="Log DocType") +@click.option("--doctype", required=True, type=str, help="Log DocType") @click.option("--days", type=int, help="Keep records for days") @click.option("--no-backup", is_flag=True, default=False, help="Do not backup the table") @pass_context def clear_log_table(context, doctype, days, no_backup): + """If any logtype table grows too large then clearing it with DELETE query is not feasible in reasonable time. This command copies recent data to new table and replaces current table with new smaller table. @@ -1211,6 +1211,7 @@ def clear_log_table(context, doctype, days, no_backup): ref: https://mariadb.com/kb/en/big-deletes/#deleting-more-than-half-a-table """ + from frappe.core.doctype.log_settings.log_settings import LOG_DOCTYPES from frappe.core.doctype.log_settings.log_settings import clear_log_table as clear_logs from frappe.utils.backups import scheduled_backup diff --git a/frappe/commands/translate.py b/frappe/commands/translate.py index 69970d8d97..5042843405 100644 --- a/frappe/commands/translate.py +++ b/frappe/commands/translate.py @@ -102,10 +102,28 @@ def import_translations(context, lang, path): frappe.destroy() +@click.command("migrate-translations") +@click.argument("source-app") +@click.argument("target-app") +@pass_context +def migrate_translations(context, source_app, target_app): + "Migrate target-app-specific translations from source-app to target-app" + import frappe.translate + + site = get_site(context) + try: + frappe.init(site=site) + frappe.connect() + frappe.translate.migrate_translations(source_app, target_app) + finally: + frappe.destroy() + + commands = [ build_message_files, get_untranslated, import_translations, new_language, update_translations, + migrate_translations, ] diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py index 3f507514d4..e77376b693 100644 --- a/frappe/commands/utils.py +++ b/frappe/commands/utils.py @@ -31,6 +31,12 @@ EXTRA_ARGS_CTX = {"ignore_unknown_options": True, "allow_extra_args": True} @click.option( "--force", is_flag=True, default=False, help="Force build assets instead of downloading available" ) +@click.option( + "--save-metafiles", + is_flag=True, + default=False, + help="Saves esbuild metafiles for built assets. Useful for analyzing bundle size. More info: https://esbuild.github.io/api/#metafile", +) def build( app=None, apps=None, @@ -38,6 +44,7 @@ def build( production=False, verbose=False, force=False, + save_metafiles=False, ): "Compile JS and CSS source files" from frappe.build import bundle, download_frappe_assets @@ -62,7 +69,14 @@ def build( if production: mode = "production" - bundle(mode, apps=apps, hard_link=hard_link, verbose=verbose, skip_frappe=skip_frappe) + bundle( + mode, + apps=apps, + hard_link=hard_link, + verbose=verbose, + skip_frappe=skip_frappe, + save_metafiles=save_metafiles, + ) @click.command("watch") @@ -386,7 +400,14 @@ def import_doc(context, path, force=False): @click.command("data-import") @click.option( - "--file", "file_path", type=click.Path(), required=True, help="Path to import file (.csv, .xlsx)" + "--file", + "file_path", + type=click.Path(exists=True, dir_okay=False, resolve_path=True), + required=True, + help=( + "Path to import file (.csv, .xlsx)." + "Consider that relative paths will resolve from 'sites' directory" + ), ) @click.option("--doctype", type=str, required=True) @click.option( diff --git a/frappe/core/doctype/amended_document_naming_settings/__init__.py b/frappe/core/doctype/amended_document_naming_settings/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/frappe/core/doctype/amended_document_naming_settings/amended_document_naming_settings.json b/frappe/core/doctype/amended_document_naming_settings/amended_document_naming_settings.json new file mode 100644 index 0000000000..2892cc6091 --- /dev/null +++ b/frappe/core/doctype/amended_document_naming_settings/amended_document_naming_settings.json @@ -0,0 +1,44 @@ +{ + "actions": [], + "allow_rename": 1, + "creation": "2023-06-16 17:57:36.604672", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "document_type", + "action" + ], + "fields": [ + { + "default": "Amend Counter", + "fieldname": "action", + "fieldtype": "Select", + "in_list_view": 1, + "label": "Action", + "options": "Amend Counter\nDefault Naming", + "reqd": 1 + }, + { + "fieldname": "document_type", + "fieldtype": "Link", + "in_list_view": 1, + "label": "DocType", + "options": "DocType", + "reqd": 1, + "unique": 1 + } + ], + "index_web_pages_for_search": 1, + "istable": 1, + "links": [], + "modified": "2023-06-16 18:26:16.247475", + "modified_by": "Administrator", + "module": "Core", + "name": "Amended Document Naming Settings", + "owner": "Administrator", + "permissions": [], + "sort_field": "modified", + "sort_order": "DESC", + "states": [] +} \ No newline at end of file diff --git a/frappe/core/doctype/amended_document_naming_settings/amended_document_naming_settings.py b/frappe/core/doctype/amended_document_naming_settings/amended_document_naming_settings.py new file mode 100644 index 0000000000..91b31350b0 --- /dev/null +++ b/frappe/core/doctype/amended_document_naming_settings/amended_document_naming_settings.py @@ -0,0 +1,9 @@ +# Copyright (c) 2023, Frappe Technologies and contributors +# For license information, please see license.txt + +# import frappe +from frappe.model.document import Document + + +class AmendedDocumentNamingSettings(Document): + pass diff --git a/frappe/core/doctype/data_import/importer.py b/frappe/core/doctype/data_import/importer.py index 0b983d0be9..6d6e34d97d 100644 --- a/frappe/core/doctype/data_import/importer.py +++ b/frappe/core/doctype/data_import/importer.py @@ -579,6 +579,10 @@ class ImportFile: file_content = None + if self.console: + file_content = frappe.read_file(file_path, True) + return file_content, extn + file_name = frappe.db.get_value("File", {"file_url": file_path}) if file_name: file = frappe.get_doc("File", file_name) @@ -690,7 +694,7 @@ class Row: df = col.df if df.fieldtype == "Select": select_options = get_select_options(df) - if select_options and value not in select_options: + if select_options and cstr(value) not in select_options: options_string = ", ".join(frappe.bold(d) for d in select_options) msg = _("Value must be one of {0}").format(options_string) self.warnings.append( diff --git a/frappe/core/doctype/document_naming_settings/document_naming_settings.js b/frappe/core/doctype/document_naming_settings/document_naming_settings.js index 2a9ec4aae5..f19e197249 100644 --- a/frappe/core/doctype/document_naming_settings/document_naming_settings.js +++ b/frappe/core/doctype/document_naming_settings/document_naming_settings.js @@ -2,6 +2,16 @@ // For license information, please see license.txt frappe.ui.form.on("Document Naming Settings", { + setup: function (frm) { + frm.set_query("document_type", "amend_naming_override", () => { + return { + filters: { + is_submittable: 1, + }, + }; + }); + }, + refresh: function (frm) { frm.trigger("setup_transaction_autocomplete"); frm.disable_save(); diff --git a/frappe/core/doctype/document_naming_settings/document_naming_settings.json b/frappe/core/doctype/document_naming_settings/document_naming_settings.json index 9a12f3f77e..5a1991c14b 100644 --- a/frappe/core/doctype/document_naming_settings/document_naming_settings.json +++ b/frappe/core/doctype/document_naming_settings/document_naming_settings.json @@ -18,7 +18,11 @@ "update_series", "prefix", "current_value", - "update_series_start" + "update_series_start", + "amended_documents_section", + "default_amend_naming", + "amend_naming_override", + "update_amendment_naming" ], "fields": [ { @@ -105,13 +109,41 @@ "fieldtype": "Text", "label": "Preview of generated names", "read_only": 1 + }, + { + "collapsible": 1, + "description": "Configure how amended documents will be named.
\n\nDefault behaviour is to follow an amend counter which adds a number to the end of the original name indicating the amended version.
\n\nDefault Naming will make the amended document to behave same as new documents.", + "fieldname": "amended_documents_section", + "fieldtype": "Section Break", + "label": "Amended Documents" + }, + { + "default": "Amend Counter", + "fieldname": "default_amend_naming", + "fieldtype": "Select", + "in_list_view": 1, + "label": "Default Amendment Naming", + "options": "Amend Counter\nDefault Naming", + "reqd": 1 + }, + { + "fieldname": "amend_naming_override", + "fieldtype": "Table", + "label": "Amendment Naming Override", + "options": "Amended Document Naming Settings" + }, + { + "fieldname": "update_amendment_naming", + "fieldtype": "Button", + "label": "Update Amendment Naming", + "options": "update_amendment_rule" } ], "hide_toolbar": 1, "icon": "fa fa-sort-by-order", "issingle": 1, "links": [], - "modified": "2023-02-20 13:11:56.662100", + "modified": "2023-06-20 17:47:52.204139", "modified_by": "Administrator", "module": "Core", "name": "Document Naming Settings", diff --git a/frappe/core/doctype/document_naming_settings/document_naming_settings.py b/frappe/core/doctype/document_naming_settings/document_naming_settings.py index f8647bd74a..625b7cdd50 100644 --- a/frappe/core/doctype/document_naming_settings/document_naming_settings.py +++ b/frappe/core/doctype/document_naming_settings/document_naming_settings.py @@ -169,6 +169,23 @@ class DocumentNamingSettings(Document): self.current_value = NamingSeries(self.prefix).get_current_value() return self.current_value + @frappe.whitelist() + def update_amendment_rule(self): + self.db_set("default_amend_naming", self.default_amend_naming) + + existing_overrides = frappe.db.get_all( + "Amended Document Naming Settings", + filters={"name": ["not in", [d.name for d in self.amend_naming_override]]}, + pluck="name", + ) + for override in existing_overrides: + frappe.delete_doc("Amended Document Naming Settings", override) + + for row in self.amend_naming_override: + row.save() + + frappe.msgprint(_("Amendment naming rules updated."), indicator="green", alert=True) + @frappe.whitelist() def update_series_start(self): frappe.only_for("System Manager") diff --git a/frappe/core/doctype/document_naming_settings/test_document_naming_settings.py b/frappe/core/doctype/document_naming_settings/test_document_naming_settings.py index bcd3197112..d1a6fbe90d 100644 --- a/frappe/core/doctype/document_naming_settings/test_document_naming_settings.py +++ b/frappe/core/doctype/document_naming_settings/test_document_naming_settings.py @@ -26,6 +26,7 @@ class TestNamingSeries(FrappeTestCase): } ], autoname="naming_series:", + is_submittable=1, ) .insert() .name @@ -82,3 +83,36 @@ class TestNamingSeries(FrappeTestCase): self.dns.update_series_start() self.assertEqual(self.dns.get_current(), new_count, f"Incorrect update for {series}") + + def test_amended_naming(self): + self.dns.amend_naming_override = [] + self.dns.default_amend_naming = "Amend Counter" + self.dns.update_amendment_rule() + + submittable_doc = frappe.get_doc( + dict(doctype=self.ns_doctype, some_fieldname="test doc with submit") + ).submit() + submittable_doc.cancel() + + amended_doc = frappe.get_doc( + dict( + doctype=self.ns_doctype, + some_fieldname="test doc with submit", + amended_from=submittable_doc.name, + ) + ).insert() + + self.assertIn(submittable_doc.name, amended_doc.name) + amended_doc.delete() + + self.dns.default_amend_naming = "Default Naming" + self.dns.update_amendment_rule() + + new_amended_doc = frappe.get_doc( + dict( + doctype=self.ns_doctype, + some_fieldname="test doc with submit", + amended_from=submittable_doc.name, + ) + ).insert() + self.assertNotIn(submittable_doc.name, new_amended_doc.name) diff --git a/frappe/core/doctype/error_log/test_error_log.py b/frappe/core/doctype/error_log/test_error_log.py index 59670de8d2..3f19a6dd0c 100644 --- a/frappe/core/doctype/error_log/test_error_log.py +++ b/frappe/core/doctype/error_log/test_error_log.py @@ -1,7 +1,10 @@ # Copyright (c) 2015, Frappe Technologies and Contributors # License: MIT. See LICENSE +from ldap3.core.exceptions import LDAPException, LDAPInappropriateAuthenticationResult + import frappe from frappe.tests.utils import FrappeTestCase +from frappe.utils.error import _is_ldap_exception # test_records = frappe.get_test_records('Error Log') @@ -12,3 +15,9 @@ class TestErrorLog(FrappeTestCase): doc = frappe.new_doc("Error Log") error = doc.log_error("This is an error") self.assertEqual(error.doctype, "Error Log") + + def test_ldap_exceptions(self): + exc = [LDAPException, LDAPInappropriateAuthenticationResult] + + for e in exc: + self.assertTrue(_is_ldap_exception(e())) diff --git a/frappe/core/doctype/rq_job/test_rq_job.py b/frappe/core/doctype/rq_job/test_rq_job.py index 09a90f7445..c39717cfd8 100644 --- a/frappe/core/doctype/rq_job/test_rq_job.py +++ b/frappe/core/doctype/rq_job/test_rq_job.py @@ -124,6 +124,20 @@ class TestRQJob(FrappeTestCase): frappe.db.commit() self.assertIsNone(get_job_status(job_id)) + @timeout(20) + def test_memory_usage(self): + job = frappe.enqueue("frappe.utils.data._get_rss_memory_usage") + self.check_status(job, "finished") + + rss = job.latest_result().return_value + msg = """Memory usage of simple background job increased. Potential root cause can be a newly added python module import. Check and move them to approriate file/function to avoid loading the module by default.""" + + # If this starts failing analyze memory usage using memray or some equivalent tool to find + # offending imports/function calls. + # Refer this PR: https://github.com/frappe/frappe/pull/21467 + LAST_MEASURED_USAGE = 40 + self.assertLessEqual(rss, LAST_MEASURED_USAGE * 1.05, msg) + def test_func(fail=False, sleep=0): if fail: diff --git a/frappe/core/doctype/server_script/server_script_list.js b/frappe/core/doctype/server_script/server_script_list.js new file mode 100644 index 0000000000..0df447a0eb --- /dev/null +++ b/frappe/core/doctype/server_script/server_script_list.js @@ -0,0 +1,38 @@ +frappe.listview_settings["Server Script"] = { + onload: function (listview) { + add_github_star_cta(listview); + }, +}; + +function add_github_star_cta(listview) { + try { + const key = "show_github_star_banner"; + if (localStorage.getItem(key) == "false") { + return; + } + + if (listview.github_star_banner) { + listview.github_star_banner.remove(); + } + + const message = "Loving Frappe Framework?"; + const link = "https://github.com/frappe/frappe"; + const cta = "Star us on GitHub"; + + listview.github_star_banner = $(` +
+
+ ${message}
${cta} → +
+
+ + + +
+
+ `).appendTo(listview.page.sidebar); + } catch (error) { + console.error(error); + } +} diff --git a/frappe/core/doctype/system_settings/system_settings.py b/frappe/core/doctype/system_settings/system_settings.py index 2fec4e87af..0c842f9c7d 100644 --- a/frappe/core/doctype/system_settings/system_settings.py +++ b/frappe/core/doctype/system_settings/system_settings.py @@ -5,14 +5,13 @@ import frappe from frappe import _ from frappe.model import no_value_fields from frappe.model.document import Document -from frappe.translate import set_default_language -from frappe.twofactor import toggle_two_factor_auth from frappe.utils import cint, today -from frappe.utils.momentjs import get_all_timezones class SystemSettings(Document): def validate(self): + from frappe.twofactor import toggle_two_factor_auth + enable_password_policy = cint(self.enable_password_policy) and True or False minimum_password_score = cint(getattr(self, "minimum_password_score", 0)) or 0 if enable_password_policy and minimum_password_score <= 0: @@ -71,6 +70,8 @@ class SystemSettings(Document): update_last_reset_password_date() def set_defaults(self): + from frappe.translate import set_default_language + for df in self.meta.get("fields"): if df.fieldtype not in no_value_fields and self.has_value_changed(df.fieldname): frappe.db.set_default(df.fieldname, self.get(df.fieldname)) @@ -92,6 +93,8 @@ def update_last_reset_password_date(): @frappe.whitelist() def load(): + from frappe.utils.momentjs import get_all_timezones + if not "System Manager" in frappe.get_roles(): frappe.throw(_("Not permitted"), frappe.PermissionError) diff --git a/frappe/core/page/permission_manager/permission_manager.py b/frappe/core/page/permission_manager/permission_manager.py index 5ed3014778..fd879095c0 100644 --- a/frappe/core/page/permission_manager/permission_manager.py +++ b/frappe/core/page/permission_manager/permission_manager.py @@ -123,8 +123,15 @@ def update(doctype, role, permlevel, ptype, value=None): Returns: str: Refresh flag is permission is updated successfully """ + + def clear_cache(): + frappe.clear_cache(doctype=doctype) + frappe.only_for("System Manager") out = update_permission_property(doctype, role, permlevel, ptype, value) + + frappe.db.after_commit.add(clear_cache) + return "refresh" if out else None diff --git a/frappe/custom/doctype/custom_field/custom_field.py b/frappe/custom/doctype/custom_field/custom_field.py index 8953153be6..ed6296b6f2 100644 --- a/frappe/custom/doctype/custom_field/custom_field.py +++ b/frappe/custom/doctype/custom_field/custom_field.py @@ -40,8 +40,9 @@ class CustomField(Document): # remove special characters from fieldname self.fieldname = "".join( - filter(lambda x: x.isdigit() or x.isalpha() or "_", cstr(label).replace(" ", "_")) + [c for c in cstr(label).replace(" ", "_") if c.isdigit() or c.isalpha() or c == "_"] ) + self.fieldname = f"custom_{self.fieldname}" # fieldnames should be lowercase self.fieldname = self.fieldname.lower() diff --git a/frappe/custom/doctype/customize_form/test_customize_form.py b/frappe/custom/doctype/customize_form/test_customize_form.py index 149ef85e28..8a62d331be 100644 --- a/frappe/custom/doctype/customize_form/test_customize_form.py +++ b/frappe/custom/doctype/customize_form/test_customize_form.py @@ -14,10 +14,11 @@ test_dependencies = ["Custom Field", "Property Setter"] class TestCustomizeForm(FrappeTestCase): def insert_custom_field(self): - frappe.delete_doc_if_exists("Custom Field", "Event-test_custom_field") - frappe.get_doc( + frappe.delete_doc_if_exists("Custom Field", "Event-custom_test_field") + self.field = frappe.get_doc( { "doctype": "Custom Field", + "fieldname": "custom_test_field", "dt": "Event", "label": "Test Custom Field", "description": "A Custom Field for Testing", @@ -36,7 +37,7 @@ class TestCustomizeForm(FrappeTestCase): frappe.clear_cache(doctype="Event") def tearDown(self): - frappe.delete_doc("Custom Field", "Event-test_custom_field") + frappe.delete_doc("Custom Field", self.field.name) frappe.db.commit() frappe.clear_cache(doctype="Event") @@ -60,7 +61,7 @@ class TestCustomizeForm(FrappeTestCase): self.assertEqual(d.doc_type, "Event") self.assertEqual(len(d.get("fields")), len(frappe.get_doc("DocType", d.doc_type).fields) + 1) - self.assertEqual(d.get("fields")[-1].fieldname, "test_custom_field") + self.assertEqual(d.get("fields")[-1].fieldname, self.field.fieldname) self.assertEqual(d.get("fields", {"fieldname": "event_type"})[0].in_list_view, 1) return d @@ -129,21 +130,21 @@ class TestCustomizeForm(FrappeTestCase): def test_save_customization_custom_field_property(self): d = self.get_customize_form("Event") - self.assertEqual(frappe.db.get_value("Custom Field", "Event-test_custom_field", "reqd"), 0) + self.assertEqual(frappe.db.get_value("Custom Field", self.field.name, "reqd"), 0) - custom_field = d.get("fields", {"fieldname": "test_custom_field"})[0] + custom_field = d.get("fields", {"fieldname": self.field.fieldname})[0] custom_field.reqd = 1 custom_field.no_copy = 1 d.run_method("save_customization") - self.assertEqual(frappe.db.get_value("Custom Field", "Event-test_custom_field", "reqd"), 1) - self.assertEqual(frappe.db.get_value("Custom Field", "Event-test_custom_field", "no_copy"), 1) + self.assertEqual(frappe.db.get_value("Custom Field", self.field.name, "reqd"), 1) + self.assertEqual(frappe.db.get_value("Custom Field", self.field.name, "no_copy"), 1) custom_field = d.get("fields", {"is_custom_field": True})[0] custom_field.reqd = 0 custom_field.no_copy = 0 d.run_method("save_customization") - self.assertEqual(frappe.db.get_value("Custom Field", "Event-test_custom_field", "reqd"), 0) - self.assertEqual(frappe.db.get_value("Custom Field", "Event-test_custom_field", "no_copy"), 0) + self.assertEqual(frappe.db.get_value("Custom Field", self.field.name, "reqd"), 0) + self.assertEqual(frappe.db.get_value("Custom Field", self.field.name, "no_copy"), 0) def test_save_customization_new_field(self): d = self.get_customize_form("Event") @@ -157,28 +158,24 @@ class TestCustomizeForm(FrappeTestCase): }, ) d.run_method("save_customization") + + custom_field_name = "Event-custom_test_add_custom_field_via_customize_form" self.assertEqual( - frappe.db.get_value( - "Custom Field", "Event-test_add_custom_field_via_customize_form", "fieldtype" - ), + frappe.db.get_value("Custom Field", custom_field_name, "fieldtype"), "Data", ) self.assertEqual( - frappe.db.get_value( - "Custom Field", "Event-test_add_custom_field_via_customize_form", "insert_after" - ), + frappe.db.get_value("Custom Field", custom_field_name, "insert_after"), last_fieldname, ) - frappe.delete_doc("Custom Field", "Event-test_add_custom_field_via_customize_form") - self.assertEqual( - frappe.db.get_value("Custom Field", "Event-test_add_custom_field_via_customize_form"), None - ) + frappe.delete_doc("Custom Field", custom_field_name) + self.assertEqual(frappe.db.get_value("Custom Field", custom_field_name), None) def test_save_customization_remove_field(self): d = self.get_customize_form("Event") - custom_field = d.get("fields", {"fieldname": "test_custom_field"})[0] + custom_field = d.get("fields", {"fieldname": self.field.fieldname})[0] d.get("fields").remove(custom_field) d.run_method("save_customization") @@ -200,7 +197,7 @@ class TestCustomizeForm(FrappeTestCase): def test_set_allow_on_submit(self): d = self.get_customize_form("Event") d.get("fields", {"fieldname": "subject"})[0].allow_on_submit = 1 - d.get("fields", {"fieldname": "test_custom_field"})[0].allow_on_submit = 1 + d.get("fields", {"fieldname": "custom_test_field"})[0].allow_on_submit = 1 d.run_method("save_customization") d = self.get_customize_form("Event") @@ -209,7 +206,7 @@ class TestCustomizeForm(FrappeTestCase): self.assertEqual(d.get("fields", {"fieldname": "subject"})[0].allow_on_submit or 0, 0) # allow for custom field - self.assertEqual(d.get("fields", {"fieldname": "test_custom_field"})[0].allow_on_submit, 1) + self.assertEqual(d.get("fields", {"fieldname": "custom_test_field"})[0].allow_on_submit, 1) def test_title_field_pattern(self): d = self.get_customize_form("Web Form") @@ -406,7 +403,7 @@ class TestCustomizeForm(FrappeTestCase): def test_system_generated_fields(self): doctype = "Event" - custom_field_name = "test_custom_field" + custom_field_name = "custom_test_field" custom_field = frappe.get_doc("Custom Field", {"dt": doctype, "fieldname": custom_field_name}) custom_field.is_system_generated = 1 diff --git a/frappe/database/database.py b/frappe/database/database.py index 1f2a1eeba9..a264f39d47 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -17,7 +17,6 @@ from pypika.terms import Criterion, NullValue import frappe import frappe.defaults -import frappe.model.meta from frappe import _ from frappe.database.utils import ( DefaultOrderBy, diff --git a/frappe/defaults.py b/frappe/defaults.py index 0b86e99efa..3bcfbec1ce 100644 --- a/frappe/defaults.py +++ b/frappe/defaults.py @@ -3,7 +3,6 @@ import frappe from frappe.cache_manager import clear_defaults_cache, common_default_keys -from frappe.desk.notifications import clear_notifications from frappe.query_builder import DocType # Note: DefaultValue records are identified by parent (e.g. __default, __global) diff --git a/frappe/desk/doctype/module_onboarding/module_onboarding.js b/frappe/desk/doctype/module_onboarding/module_onboarding.js index 0e312025bf..831b29a660 100644 --- a/frappe/desk/doctype/module_onboarding/module_onboarding.js +++ b/frappe/desk/doctype/module_onboarding/module_onboarding.js @@ -13,6 +13,10 @@ frappe.ui.form.on("Module Onboarding", { if (!frappe.boot.developer_mode) { frm.trigger("disable_form"); } + + frm.add_custom_button(__("Reset"), () => { + frm.call("reset_progress"); + }); }, disable_form: function (frm) { diff --git a/frappe/desk/doctype/module_onboarding/module_onboarding.py b/frappe/desk/doctype/module_onboarding/module_onboarding.py index ea02f5911d..94805d05b6 100644 --- a/frappe/desk/doctype/module_onboarding/module_onboarding.py +++ b/frappe/desk/doctype/module_onboarding/module_onboarding.py @@ -2,6 +2,7 @@ # License: MIT. See LICENSE import frappe +from frappe import _ from frappe.model.document import Document from frappe.modules.export_file import export_to_files @@ -37,6 +38,16 @@ class ModuleOnboarding(Document): return False + @frappe.whitelist() + def reset_progress(self): + self.db_set("is_complete", 0) + + for step in self.get_steps(): + step.db_set("is_complete", 0) + step.db_set("is_skipped", 0) + + frappe.msgprint(_("Module onboarding progress reset"), alert=True) + def before_export(self, doc): doc.is_complete = 0 diff --git a/frappe/desk/form/meta.py b/frappe/desk/form/meta.py index 3ad65f7b13..6c338dbbbc 100644 --- a/frappe/desk/form/meta.py +++ b/frappe/desk/form/meta.py @@ -9,7 +9,6 @@ from frappe.build import scrub_html_template from frappe.model.meta import Meta from frappe.model.utils import render_include from frappe.modules import get_module_path, load_doctype_module, scrub -from frappe.translate import extract_messages_from_code, make_dict_from_messages from frappe.utils import get_html_format from frappe.utils.data import get_link_to_form @@ -260,6 +259,8 @@ class FormMeta(Meta): self.set("__form_grid_templates", templates) def set_translations(self, lang): + from frappe.translate import extract_messages_from_code, make_dict_from_messages + self.set("__messages", frappe.get_lang_dict("doctype", self.name)) # set translations for grid templates diff --git a/frappe/desk/form/save.py b/frappe/desk/form/save.py index 75335cb1ce..180717da40 100644 --- a/frappe/desk/form/save.py +++ b/frappe/desk/form/save.py @@ -16,7 +16,7 @@ from frappe.utils.telemetry import capture_doc def savedocs(doc, action): """save / submit / update doclist""" doc = frappe.get_doc(json.loads(doc)) - capture_doc(doc) + capture_doc(doc, action) set_local_name(doc) # action @@ -47,6 +47,8 @@ def savedocs(doc, action): def cancel(doctype=None, name=None, workflow_state_fieldname=None, workflow_state=None): """cancel a doclist""" doc = frappe.get_doc(doctype, name) + capture_doc(doc, "Cancel") + if workflow_state_fieldname and workflow_state: doc.set(workflow_state_fieldname, workflow_state) doc.cancel() diff --git a/frappe/desk/reportview.py b/frappe/desk/reportview.py index 326e9bb864..071b6e7e61 100644 --- a/frappe/desk/reportview.py +++ b/frappe/desk/reportview.py @@ -479,6 +479,7 @@ def delete_items(): def delete_bulk(doctype, items): + undeleted_items = [] for i, d in enumerate(items): try: frappe.delete_doc(doctype, d) @@ -493,7 +494,11 @@ def delete_bulk(doctype, items): except Exception: # rollback if any record failed to delete # if not rollbacked, queries get committed on after_request method in app.py + undeleted_items.append(d) frappe.db.rollback() + if undeleted_items and len(items) != len(undeleted_items): + frappe.clear_messages() + delete_bulk(doctype, undeleted_items) @frappe.whitelist() diff --git a/frappe/desk/search.py b/frappe/desk/search.py index d347cc188c..c4c11558dd 100644 --- a/frappe/desk/search.py +++ b/frappe/desk/search.py @@ -6,7 +6,9 @@ import json import re import frappe -from frappe import _, is_whitelisted + +# Backward compatbility +from frappe import _, is_whitelisted, validate_and_sanitize_search_inputs from frappe.database.schema import SPECIAL_CHAR_PATTERN from frappe.permissions import has_permission from frappe.utils import cint, cstr, unique @@ -293,22 +295,6 @@ def relevance_sorter(key, query, as_dict): return (cstr(value).casefold().startswith(query.casefold()) is not True, value) -def validate_and_sanitize_search_inputs(fn): - @functools.wraps(fn) - def wrapper(*args, **kwargs): - kwargs.update(dict(zip(fn.__code__.co_varnames, args))) - sanitize_searchfield(kwargs["searchfield"]) - kwargs["start"] = cint(kwargs["start"]) - kwargs["page_len"] = cint(kwargs["page_len"]) - - if kwargs["doctype"] and not frappe.db.exists("DocType", kwargs["doctype"]): - return [] - - return fn(**kwargs) - - return wrapper - - @frappe.whitelist() def get_names_for_mentions(search_term): users_for_mentions = frappe.cache.get_value("users_for_mentions", get_users_for_mentions) diff --git a/frappe/frappeclient.py b/frappe/frappeclient.py index 3f7577fac6..7ad016828a 100644 --- a/frappe/frappeclient.py +++ b/frappe/frappeclient.py @@ -4,8 +4,6 @@ FrappeClient is a library that helps you connect with other frappe systems import base64 import json -import requests - import frappe from frappe.utils.data import cstr @@ -37,6 +35,8 @@ class FrappeClient: api_secret=None, frappe_authorization_source=None, ): + import requests + self.headers = { "Accept": "application/json", "content-type": "application/x-www-form-urlencoded", @@ -390,42 +390,13 @@ class FrappeClient: class FrappeOAuth2Client(FrappeClient): def __init__(self, url, access_token, verify=True): + import requests + self.access_token = access_token self.headers = { "Authorization": "Bearer " + access_token, "content-type": "application/x-www-form-urlencoded", } self.verify = verify - self.session = OAuth2Session(self.headers) + self.session = requests.session() self.url = url - - def get_request(self, params): - res = requests.get( - self.url, params=self.preprocess(params), headers=self.headers, verify=self.verify - ) - res = self.post_process(res) - return res - - def post_request(self, data): - res = requests.post( - self.url, data=self.preprocess(data), headers=self.headers, verify=self.verify - ) - res = self.post_process(res) - return res - - -class OAuth2Session: - def __init__(self, headers): - self.headers = headers - - def get(self, url, params, verify): - res = requests.get(url, params=params, headers=self.headers, verify=verify) - return res - - def post(self, url, data, verify): - res = requests.post(url, data=data, headers=self.headers, verify=verify) - return res - - def put(self, url, data, verify): - res = requests.put(url, data=data, headers=self.headers, verify=verify) - return res diff --git a/frappe/installer.py b/frappe/installer.py index 9c2807d7cd..4f02e207bd 100644 --- a/frappe/installer.py +++ b/frappe/installer.py @@ -287,6 +287,9 @@ def install_app(name, verbose=False, set_as_patched=True, force=False): if out is False: return + for fn in frappe.get_hooks("before_app_install"): + frappe.get_attr(fn)(name) + if name != "frappe": add_module_defs(name, ignore_if_duplicate=force) @@ -302,6 +305,9 @@ def install_app(name, verbose=False, set_as_patched=True, force=False): for after_install in app_hooks.after_install or []: frappe.get_attr(after_install)() + for fn in frappe.get_hooks("after_app_install"): + frappe.get_attr(fn)(name) + sync_jobs() sync_fixtures(name) sync_customizations(name) @@ -369,6 +375,9 @@ def remove_app(app_name, dry_run=False, yes=False, no_backup=False, force=False) for before_uninstall in app_hooks.before_uninstall or []: frappe.get_attr(before_uninstall)() + for fn in frappe.get_hooks("before_app_uninstall"): + frappe.get_attr(fn)(app_name) + modules = frappe.get_all("Module Def", filters={"app_name": app_name}, pluck="name") drop_doctypes = _delete_modules(modules, dry_run=dry_run) @@ -382,6 +391,9 @@ def remove_app(app_name, dry_run=False, yes=False, no_backup=False, force=False) for after_uninstall in app_hooks.after_uninstall or []: frappe.get_attr(after_uninstall)() + for fn in frappe.get_hooks("after_app_uninstall"): + frappe.get_attr(fn)(app_name) + click.secho(f"Uninstalled App {app_name} from Site {site}", fg="green") frappe.flags.in_uninstall = False diff --git a/frappe/integrations/doctype/webhook/test_webhook.py b/frappe/integrations/doctype/webhook/test_webhook.py index 1701c418f7..d308ec95ab 100644 --- a/frappe/integrations/doctype/webhook/test_webhook.py +++ b/frappe/integrations/doctype/webhook/test_webhook.py @@ -3,6 +3,9 @@ import json from contextlib import contextmanager +import responses +from responses.matchers import json_params_matcher + import frappe from frappe.integrations.doctype.webhook.webhook import ( enqueue_webhook, @@ -94,9 +97,15 @@ class TestWebhook(FrappeTestCase): self.test_user.email = "user1@integration.webhooks.test.com" self.test_user.first_name = "user1" + self.responses = responses.RequestsMock() + self.responses.start() + def tearDown(self) -> None: self.user.delete() self.test_user.delete() + + self.responses.stop() + self.responses.reset() super().tearDown() def test_webhook_trigger_with_enabled_webhooks(self): @@ -172,6 +181,13 @@ class TestWebhook(FrappeTestCase): self.assertEqual(data, {"name": self.user.name}) def test_webhook_req_log_creation(self): + self.responses.add( + responses.POST, + "https://httpbin.org/post", + status=200, + json={}, + ) + if not frappe.db.get_value("User", "user2@integration.webhooks.test.com"): user = frappe.get_doc( {"doctype": "User", "email": "user2@integration.webhooks.test.com", "first_name": "user2"} @@ -185,6 +201,7 @@ class TestWebhook(FrappeTestCase): self.assertTrue(frappe.get_all("Webhook Request Log", pluck="name")) def test_webhook_with_array_body(self): + """Check if array request body are supported.""" wh_config = { "doctype": "Webhook", @@ -194,7 +211,7 @@ class TestWebhook(FrappeTestCase): "request_url": "https://httpbin.org/post", "request_method": "POST", "request_structure": "JSON", - "webhook_json": '[\r\n{% for n in range(3) %}\r\n {\r\n "title": "{{ doc.title }}",\r\n "n": {{ n }}\r\n }\r\n {%- if not loop.last -%}\r\n , \r\n {%endif%}\r\n{%endfor%}\r\n]', + "webhook_json": '[\r\n{% for n in range(3) %}\r\n {\r\n "title": "{{ doc.title }}" }\r\n {%- if not loop.last -%}\r\n , \r\n {%endif%}\r\n{%endfor%}\r\n]', "meets_condition": "Yes", "webhook_headers": [ { @@ -204,13 +221,22 @@ class TestWebhook(FrappeTestCase): ], } - with get_test_webhook(wh_config) as wh: - doc = frappe.new_doc("Note") - doc.title = "Test Webhook Note" + doc = frappe.new_doc("Note") + doc.title = "Test Webhook Note" + expected_req = [{"title": doc.title} for _ in range(3)] + self.responses.add( + responses.POST, + "https://httpbin.org/post", + status=200, + json=expected_req, + match=[json_params_matcher(expected_req)], + ) + + with get_test_webhook(wh_config) as wh: enqueue_webhook(doc, wh) log = frappe.get_last_doc("Webhook Request Log") - self.assertEqual(len(json.loads(log.response)["json"]), 3) + self.assertEqual(len(json.loads(log.response)), 3) def test_webhook_with_dynamic_url_enabled(self): wh_config = { @@ -232,12 +258,16 @@ class TestWebhook(FrappeTestCase): ], } + self.responses.add( + responses.POST, + "https://httpbin.org/anything/Note", + status=200, + ) + with get_test_webhook(wh_config) as wh: doc = frappe.new_doc("Note") doc.title = "Test Webhook Note" enqueue_webhook(doc, wh) - log = frappe.get_last_doc("Webhook Request Log") - self.assertEqual(json.loads(log.response)["url"], "https://httpbin.org/anything/Note") def test_webhook_with_dynamic_url_disabled(self): wh_config = { @@ -259,11 +289,13 @@ class TestWebhook(FrappeTestCase): ], } + self.responses.add( + responses.POST, + "https://httpbin.org/anything/{{doc.doctype}}", + status=200, + ) + with get_test_webhook(wh_config) as wh: doc = frappe.new_doc("Note") doc.title = "Test Webhook Note" enqueue_webhook(doc, wh) - log = frappe.get_last_doc("Webhook Request Log") - self.assertEqual( - json.loads(log.response)["url"], "https://httpbin.org/anything/{{doc.doctype}}" - ) diff --git a/frappe/model/naming.py b/frappe/model/naming.py index 73b5930563..0b76d18cff 100644 --- a/frappe/model/naming.py +++ b/frappe/model/naming.py @@ -151,7 +151,8 @@ def set_new_name(doc): if getattr(doc, "amended_from", None): _set_amended_name(doc) - return + if doc.name: + return elif getattr(doc.meta, "issingle", False): doc.name = doc.doctype @@ -506,6 +507,17 @@ def append_number_if_name_exists(doctype, value, fieldname="name", separator="-" def _set_amended_name(doc): + amend_naming_rule = frappe.db.get_value( + "Amended Document Naming Settings", {"document_type": doc.doctype}, "action", cache=True + ) + if not amend_naming_rule: + amend_naming_rule = frappe.db.get_single_value( + "Document Naming Settings", "default_amend_naming", cache=True + ) + + if amend_naming_rule == "Default Naming": + return + am_id = 1 am_prefix = doc.amended_from if frappe.db.get_value(doc.doctype, doc.amended_from, "amended_from"): diff --git a/frappe/model/rename_doc.py b/frappe/model/rename_doc.py index e3ca6de739..e8f5626af4 100644 --- a/frappe/model/rename_doc.py +++ b/frappe/model/rename_doc.py @@ -53,6 +53,7 @@ def update_document_title( # handle bad API usages merge = sbool(merge) enqueue = sbool(enqueue) + action_enqueued = enqueue and not is_scheduler_inactive() doc = frappe.get_doc(doctype, docname) doc.check_permission(permtype="write") @@ -65,7 +66,7 @@ def update_document_title( name_updated = updated_name and (updated_name != doc.name) if name_updated: - if enqueue and not is_scheduler_inactive(): + if action_enqueued: current_name = doc.name # before_name hook may have DocType specific validations or transformations @@ -90,18 +91,27 @@ def update_document_title( doc.rename(updated_name, merge=merge) if title_updated: - try: - setattr(doc, title_field, updated_title) - doc.save() - frappe.msgprint(_("Saved"), alert=True, indicator="green") - except Exception as e: - if frappe.db.is_duplicate_entry(e): - frappe.throw( - _("{0} {1} already exists").format(doctype, frappe.bold(docname)), - title=_("Duplicate Name"), - exc=frappe.DuplicateEntryError, - ) - raise + if action_enqueued and name_updated: + frappe.enqueue( + "frappe.client.set_value", + doctype=doc.doctype, + name=updated_name, + fieldname=title_field, + value=updated_title, + ) + else: + try: + setattr(doc, title_field, updated_title) + doc.save() + frappe.msgprint(_("Saved"), alert=True, indicator="green") + except Exception as e: + if frappe.db.is_duplicate_entry(e): + frappe.throw( + _("{0} {1} already exists").format(doctype, frappe.bold(docname)), + title=_("Duplicate Name"), + exc=frappe.DuplicateEntryError, + ) + raise return doc.name diff --git a/frappe/modules/import_file.py b/frappe/modules/import_file.py index 36e329409a..8c9a209501 100644 --- a/frappe/modules/import_file.py +++ b/frappe/modules/import_file.py @@ -34,6 +34,7 @@ ignore_values = { "Print Style": ["disabled"], "Module Onboarding": ["is_complete"], "Onboarding Step": ["is_complete", "is_skipped"], + "Workspace": ["is_hidden"], } ignore_doctypes = [""] diff --git a/frappe/patches.txt b/frappe/patches.txt index 436701e7bf..c26b1a74d7 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -226,3 +226,4 @@ frappe.patches.v14_0.remove_manage_subscriptions_from_navbar frappe.patches.v15_0.remove_background_jobs_from_dropdown frappe.desk.doctype.form_tour.patches.introduce_ui_tours execute:frappe.delete_doc_if_exists("Workspace", "Customization") +execute:frappe.db.set_single_value("Document Naming Settings", "default_amend_naming", "Amend Counter") diff --git a/frappe/permissions.py b/frappe/permissions.py index 4cd0d369e2..633d0e278d 100644 --- a/frappe/permissions.py +++ b/frappe/permissions.py @@ -532,7 +532,7 @@ def update_permission_property(doctype, role, permlevel, ptype, value=None, vali out = setup_custom_perms(doctype) - name = frappe.get_value("Custom DocPerm", dict(parent=doctype, role=role, permlevel=permlevel)) + name = frappe.db.get_value("Custom DocPerm", dict(parent=doctype, role=role, permlevel=permlevel)) table = DocType("Custom DocPerm") frappe.qb.update(table).set(ptype, value).where(table.name == name).run() diff --git a/frappe/public/js/form_builder/components/Column.vue b/frappe/public/js/form_builder/components/Column.vue index 1563d1033e..54dd49d5bf 100644 --- a/frappe/public/js/form_builder/components/Column.vue +++ b/frappe/public/js/form_builder/components/Column.vue @@ -6,7 +6,7 @@ import { ref } from "vue"; import { useStore } from "../store"; import { move_children_to_parent, confirm_dialog } from "../utils"; -let props = defineProps(["section", "column"]); +const props = defineProps(["section", "column"]); let store = useStore(); let hovered = ref(false); diff --git a/frappe/public/js/form_builder/components/EditableInput.vue b/frappe/public/js/form_builder/components/EditableInput.vue index 8964838f4a..21b517af3b 100644 --- a/frappe/public/js/form_builder/components/EditableInput.vue +++ b/frappe/public/js/form_builder/components/EditableInput.vue @@ -3,7 +3,7 @@ import { ref, nextTick, computed } from "vue"; import { useStore } from "../store"; let store = useStore(); -let props = defineProps({ +const props = defineProps({ text: { type: String }, @@ -46,7 +46,7 @@ function focus_on_label() { :disabled="store.read_only" type="text" :placeholder="placeholder" - v-model="text" + :value="text" :style="{ width: hidden_span_width }" @input="event => $emit('update:modelValue', event.target.value)" @keydown.enter="editing = false" diff --git a/frappe/public/js/form_builder/components/Field.vue b/frappe/public/js/form_builder/components/Field.vue index e0230765b5..b67d6db0c9 100644 --- a/frappe/public/js/form_builder/components/Field.vue +++ b/frappe/public/js/form_builder/components/Field.vue @@ -4,7 +4,7 @@ import { ref, computed } from "vue"; import { useStore } from "../store"; import { move_children_to_parent, clone_field } from "../utils"; -let props = defineProps(["column", "field"]); +const props = defineProps(["column", "field"]); let store = useStore(); let hovered = ref(false); diff --git a/frappe/public/js/form_builder/components/Section.vue b/frappe/public/js/form_builder/components/Section.vue index 5131ff25d3..cd675e5958 100644 --- a/frappe/public/js/form_builder/components/Section.vue +++ b/frappe/public/js/form_builder/components/Section.vue @@ -6,7 +6,7 @@ import { ref } from "vue"; import { useStore } from "../store"; import { section_boilerplate, move_children_to_parent, confirm_dialog } from "../utils"; -let props = defineProps(["tab", "section"]); +const props = defineProps(["tab", "section"]); let store = useStore(); let hovered = ref(false); diff --git a/frappe/public/js/form_builder/components/controls/AttachControl.vue b/frappe/public/js/form_builder/components/controls/AttachControl.vue index 86cdf7c5ac..6d8718d5dc 100644 --- a/frappe/public/js/form_builder/components/controls/AttachControl.vue +++ b/frappe/public/js/form_builder/components/controls/AttachControl.vue @@ -1,6 +1,6 @@