diff --git a/.github/helper/install.sh b/.github/helper/install.sh index 246bdbe096..3ef7db34f6 100644 --- a/.github/helper/install.sh +++ b/.github/helper/install.sh @@ -17,7 +17,7 @@ if [ "$TYPE" == "server" ]; then fi if [ "$DB" == "mariadb" ];then - sudo apt update && sudo apt install mariadb-client-10.3 + sudo apt install mariadb-client-10.3 mysql --host 127.0.0.1 --port 3306 -u root -e "SET GLOBAL character_set_server = 'utf8mb4'"; mysql --host 127.0.0.1 --port 3306 -u root -e "SET GLOBAL collation_server = 'utf8mb4_unicode_ci'"; diff --git a/.github/helper/install_dependencies.sh b/.github/helper/install_dependencies.sh index 666af13882..6a837f268a 100644 --- a/.github/helper/install_dependencies.sh +++ b/.github/helper/install_dependencies.sh @@ -16,10 +16,4 @@ sudo mv /tmp/wkhtmltox/bin/wkhtmltopdf /usr/local/bin/wkhtmltopdf sudo chmod o+x /usr/local/bin/wkhtmltopdf # install cups -sudo apt-get install libcups2-dev - -# install redis -sudo apt-get install redis-server - -# install redis -sudo apt-get install libmariadb-dev +sudo apt update && sudo apt install libcups2-dev libmariadb-dev redis-server diff --git a/.github/helper/roulette.py b/.github/helper/roulette.py index f68ef5046f..9165198012 100644 --- a/.github/helper/roulette.py +++ b/.github/helper/roulette.py @@ -5,8 +5,10 @@ import shlex import subprocess import sys import urllib.request +from functools import cache +@cache def fetch_pr_data(pr_number, repo, endpoint): api_url = f"https://api.github.com/repos/{repo}/pulls/{pr_number}" @@ -26,7 +28,16 @@ def get_output(command, shell=True): return subprocess.check_output(command, shell=shell, encoding="utf8").strip() def has_skip_ci_label(pr_number, repo="frappe/frappe"): - return any([label["name"] for label in fetch_pr_data(pr_number, repo, "")["labels"] if label["name"] == "Skip CI"]) + return has_label(pr_number, "Skip CI", repo) + +def has_run_server_tests_label(pr_number, repo="frappe/frappe"): + return has_label(pr_number, "Run Server Tests", repo) + +def has_run_ui_tests_label(pr_number, repo="frappe/frappe"): + return has_label(pr_number, "Run UI Tests", repo) + +def has_label(pr_number, label, repo="frappe/frappe"): + return any([label["name"] for label in fetch_pr_data(pr_number, repo, "")["labels"] if label["name"] == label]) def is_py(file): return file.endswith("py") @@ -77,11 +88,11 @@ if __name__ == "__main__": print("Only docs were updated, stopping build process.") sys.exit(0) - elif only_frontend_code_changed and build_type == "server": + elif only_frontend_code_changed and build_type == "server" and not has_run_server_tests_label(pr_number, repo): print("Only Frontend code was updated; Stopping Python build process.") sys.exit(0) - elif build_type == "ui" and only_py_changed: + elif build_type == "ui" and only_py_changed and not has_run_ui_tests_label(pr_number, repo): print("Only Python code was updated, stopping Cypress build process.") sys.exit(0) diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index dba13f9358..988c2dcc6c 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -2,8 +2,13 @@ name: 'Trigger Docker build on release' on: release: types: [released] +permissions: + contents: read + jobs: curl: + permissions: + contents: none name: 'Trigger Docker build on release' runs-on: ubuntu-latest container: diff --git a/.github/workflows/docs-checker.yml b/.github/workflows/docs-checker.yml index 5e91063698..a0f77b43fd 100644 --- a/.github/workflows/docs-checker.yml +++ b/.github/workflows/docs-checker.yml @@ -3,6 +3,9 @@ on: pull_request: types: [ opened, synchronize, reopened, edited ] +permissions: + contents: read + jobs: docs-required: name: 'Documentation Required' diff --git a/.github/workflows/patch-mariadb-tests.yml b/.github/workflows/patch-mariadb-tests.yml index c8294886a0..7dffc30dc0 100644 --- a/.github/workflows/patch-mariadb-tests.yml +++ b/.github/workflows/patch-mariadb-tests.yml @@ -7,6 +7,9 @@ concurrency: group: patch-mariadb-develop-${{ github.event.number }} cancel-in-progress: true +permissions: + contents: read + jobs: test: runs-on: ubuntu-latest @@ -56,7 +59,7 @@ jobs: uses: actions/cache@v2 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + key: ${{ runner.os }}-pip-${{ hashFiles('**/*requirements.txt', '**/pyproject.toml', '**/setup.py') }} restore-keys: | ${{ runner.os }}-pip- ${{ runner.os }}- @@ -121,7 +124,7 @@ jobs: git fetch --depth 1 upstream $branch_name:$branch_name git checkout -q -f $branch_name - pip install -q -r requirements.txt + bench setup requirements --python bench --site test_site migrate done diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9c7ecf989e..e9936482b0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,7 +2,10 @@ name: Generate Semantic Release on: push: branches: - - version-13 + - version-14-beta +permissions: + contents: read + jobs: release: name: Release diff --git a/.github/workflows/server-mariadb-tests.yml b/.github/workflows/server-mariadb-tests.yml index 4edf74ba71..33fc221e80 100644 --- a/.github/workflows/server-mariadb-tests.yml +++ b/.github/workflows/server-mariadb-tests.yml @@ -11,6 +11,9 @@ concurrency: cancel-in-progress: true +permissions: + contents: read + jobs: test: runs-on: ubuntu-latest @@ -67,7 +70,7 @@ jobs: uses: actions/cache@v2 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + key: ${{ runner.os }}-pip-${{ hashFiles('**/*requirements.txt', '**/pyproject.toml', '**/setup.py') }} restore-keys: | ${{ runner.os }}-pip- ${{ runner.os }}- diff --git a/.github/workflows/server-postgres-tests.yml b/.github/workflows/server-postgres-tests.yml index 895af5184e..2b4a2edae0 100644 --- a/.github/workflows/server-postgres-tests.yml +++ b/.github/workflows/server-postgres-tests.yml @@ -10,6 +10,9 @@ concurrency: group: server-postgres-develop-${{ github.event.number }} cancel-in-progress: true +permissions: + contents: read + jobs: test: runs-on: ubuntu-latest @@ -70,7 +73,7 @@ jobs: uses: actions/cache@v2 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + key: ${{ runner.os }}-pip-${{ hashFiles('**/*requirements.txt', '**/pyproject.toml', '**/setup.py') }} restore-keys: | ${{ runner.os }}-pip- ${{ runner.os }}- diff --git a/.github/workflows/ui-tests.yml b/.github/workflows/ui-tests.yml index fc8093444e..08bf3584f5 100644 --- a/.github/workflows/ui-tests.yml +++ b/.github/workflows/ui-tests.yml @@ -10,6 +10,9 @@ concurrency: group: ui-develop-${{ github.event.number }} cancel-in-progress: true +permissions: + contents: read + jobs: test: runs-on: ubuntu-latest @@ -66,7 +69,7 @@ jobs: uses: actions/cache@v2 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + key: ${{ runner.os }}-pip-${{ hashFiles('**/*requirements.txt', '**/pyproject.toml', '**/setup.py') }} restore-keys: | ${{ runner.os }}-pip- ${{ runner.os }}- diff --git a/.mergify.yml b/.mergify.yml index f1333362a8..97df91a927 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -7,6 +7,7 @@ pull_request_rules: - author!=gavindsouza - author!=deepeshgarg007 - author!=ankush + - author!=mergify[bot] - or: - base=version-13 - base=version-12 diff --git a/.releaserc b/.releaserc index 530a6c0767..c9ca71bbf5 100644 --- a/.releaserc +++ b/.releaserc @@ -1,11 +1,8 @@ { - "branches": ["version-13"], + "branches": ["develop", {"name": "version-14-beta", "channel": "beta", "prerelease": true}], "plugins": [ "@semantic-release/commit-analyzer", { - "preset": "angular", - "releaseRules": [ - {"breaking": true, "release": false} - ] + "preset": "angular" }, "@semantic-release/release-notes-generator", [ diff --git a/CODEOWNERS b/CODEOWNERS index 170334a4b4..59832e8636 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -12,7 +12,7 @@ data_import* @netchampfaris core/ @surajshetty3416 database @gavindsouza model @gavindsouza -requirements.txt @gavindsouza +pyproject.toml @gavindsouza query_builder/ @gavindsouza commands/ @gavindsouza workspace @shariquerik diff --git a/cypress/integration/awesome_bar.js b/cypress/integration/awesome_bar.js index 053d015366..e62ba6bec5 100644 --- a/cypress/integration/awesome_bar.js +++ b/cypress/integration/awesome_bar.js @@ -26,7 +26,7 @@ context('Awesome Bar', () => { cy.get('.title-text').should('contain', 'To Do'); - cy.findByPlaceholderText('Name') + cy.findByPlaceholderText('ID') .should('have.value', '%test%'); }); diff --git a/cypress/integration/control_data.js b/cypress/integration/control_data.js index 01f9168667..78cece627b 100644 --- a/cypress/integration/control_data.js +++ b/cypress/integration/control_data.js @@ -34,6 +34,12 @@ context('Data Control', () => { }); }); }); + + it('check custom formatters', () => { + cy.visit(`/app/doctype/User`); + cy.get('[data-fieldname="fields"] .grid-row[data-idx="2"] [data-fieldname="fieldtype"] .static-area').should('have.text', '🔵 Section Break'); + }); + it('Verifying data control by inputting different patterns for "Name" field', () => { cy.new_form('Test Data Control'); @@ -54,7 +60,7 @@ context('Data Control', () => { //Checking if the border color of the field changes to red cy.get('.frappe-control[data-fieldname="name1"]').should('have.class', 'has-error'); - cy.findByRole('button', {name: 'Save'}).click(); + cy.save(); //Checking for the error message cy.get('.modal-title').should('have.text', 'Message'); @@ -64,7 +70,7 @@ context('Data Control', () => { cy.get_field('name1', 'Data').clear({force: true}); cy.fill_field('name1', 'Komal{}/!', 'Data'); cy.get('.frappe-control[data-fieldname="name1"]').should('have.class', 'has-error'); - cy.findByRole('button', {name: 'Save'}).click(); + cy.save(); cy.get('.modal-title').should('have.text', 'Message'); cy.get('.msgprint').should('have.text', 'Komal{}/! is not a valid Name'); cy.hide_dialog(); @@ -76,14 +82,14 @@ context('Data Control', () => { cy.get_field('email', 'Data').clear({force: true}); cy.fill_field('email', 'komal', 'Data'); cy.get('.frappe-control[data-fieldname="email"]').should('have.class', 'has-error'); - cy.findByRole('button', {name: 'Save'}).click(); + cy.save(); cy.get('.modal-title').should('have.text', 'Message'); cy.get('.msgprint').should('have.text', 'komal is not a valid Email Address'); cy.hide_dialog(); cy.get_field('email', 'Data').clear({force: true}); cy.fill_field('email', 'komal@test', 'Data'); cy.get('.frappe-control[data-fieldname="email"]').should('have.class', 'has-error'); - cy.findByRole('button', {name: 'Save'}).click(); + cy.save(); cy.get('.modal-title').should('have.text', 'Message'); cy.get('.msgprint').should('have.text', 'komal@test is not a valid Email Address'); cy.hide_dialog(); @@ -125,4 +131,4 @@ context('Data Control', () => { cy.get('.actions-btn-group > .dropdown-menu [data-label="Delete"]').click(); cy.click_modal_primary_button('Yes'); }); -}); \ No newline at end of file +}); diff --git a/cypress/integration/custom_buttons.js b/cypress/integration/custom_buttons.js index e2f02668e9..6045d009c2 100644 --- a/cypress/integration/custom_buttons.js +++ b/cypress/integration/custom_buttons.js @@ -4,6 +4,7 @@ const test_button_names = [ "Porcupine Tree (the GOAT)", "AC / DC", `Electronic Dance "music"`, + "l'imperatrice", ]; const add_button = (label, group = "TestGroup") => { diff --git a/cypress/integration/customize_form.js b/cypress/integration/customize_form.js index 70615085c3..3857d7ccd8 100644 --- a/cypress/integration/customize_form.js +++ b/cypress/integration/customize_form.js @@ -1,5 +1,6 @@ context('Customize Form', () => { before(() => { + cy.login(); cy.visit('/app/customize-form'); }); it('Changing to naming rule should update autoname', () => { @@ -19,4 +20,4 @@ context('Customize Form', () => { cy.get_field("autoname", "Data").should("have.value", value); }); }); -}); \ No newline at end of file +}); diff --git a/cypress/integration/timeline_email.js b/cypress/integration/timeline_email.js index f2a239401d..993847bcb8 100644 --- a/cypress/integration/timeline_email.js +++ b/cypress/integration/timeline_email.js @@ -7,7 +7,7 @@ context('Timeline Email', () => { it('Adding new ToDo', () => { cy.click_listview_primary_button('Add ToDo'); - cy.get('.custom-actions:visible > .btn').contains("Edit in full page").click({delay: 500}); + cy.get('.custom-actions:visible > .btn').contains("Edit Full Form").click({delay: 500}); cy.fill_field("description", "Test ToDo", "Text Editor"); cy.wait(500); cy.get('.primary-action').contains('Save').click({force: true}); diff --git a/cypress/support/commands.js b/cypress/support/commands.js index 6398018e10..c168b0c201 100644 --- a/cypress/support/commands.js +++ b/cypress/support/commands.js @@ -27,6 +27,7 @@ import "cypress-real-events/support"; // // -- This is will overwrite an existing command -- // Cypress.Commands.overwrite("visit", (originalFn, url, options) => { ... }); + Cypress.Commands.add('login', (email, password) => { if (!email) { email = 'Administrator'; @@ -265,9 +266,14 @@ Cypress.Commands.add('get_open_dialog', () => { return cy.get('.modal:visible').last(); }); +Cypress.Commands.add('save', () => { + cy.intercept('/api').as('api'); + cy.get(`button[data-label="Save"]:visible`).click({scrollBehavior: false, force: true}); + cy.wait('@api'); +}); Cypress.Commands.add('hide_dialog', () => { cy.wait(300); - cy.get_open_dialog().find('.btn-modal-close').click(); + cy.get_open_dialog().focus().find('.btn-modal-close').click(); cy.get('.modal:visible').should('not.exist'); }); diff --git a/dev-requirements.txt b/dev-requirements.txt index f4045c6bed..b67e915a16 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,4 +1,4 @@ coverage==5.5 -Faker~=8.1.0 +Faker~=13.12.1 pyngrok~=5.0.5 unittest-xml-reporting~=3.0.4 diff --git a/frappe/__init__.py b/frappe/__init__.py index ec016c8b36..542c783319 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -10,18 +10,20 @@ be used to build database driven apps. Read the documentation: https://frappeframework.com/docs """ +import functools import importlib import inspect import json import os -import sys +import re import warnings -from typing import TYPE_CHECKING, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union import click from werkzeug.local import Local, release_local from frappe.query_builder import get_query_builder, patch_query_aggregation, patch_query_execute +from frappe.utils.caching import request_cache from frappe.utils.data import cstr, sbool # Local application imports @@ -44,6 +46,11 @@ STANDARD_USERS = ("Guest", "Administrator") DISABLE_DATABASE_POOLING = None _dev_server = int(sbool(os.environ.get("DEV_SERVER", False))) +_qb_patched = {} +re._MAXCACHE = ( + 50 # reduced from default 512 given we are already maintaining this on parent worker +) + if _dev_server: warnings.simplefilter("always", DeprecationWarning) @@ -236,8 +243,10 @@ def init(site, sites_path=None, new_site=False): local.qb = get_query_builder(local.conf.db_type or "mariadb") setup_module_map() - patch_query_execute() - patch_query_aggregation() + + if not _qb_patched.get(local.conf.db_type): + patch_query_execute() + patch_query_aggregation() local.initialised = True @@ -410,16 +419,22 @@ def msgprint( :param is_minimizable: [optional] Allow users to minimize the modal :param wide: [optional] Show wide modal """ + import inspect + import sys + from frappe.utils import strip_html_tags msg = safe_decode(msg) out = _dict(message=msg) + @functools.lru_cache(maxsize=1024) + def _strip_html_tags(message): + return strip_html_tags(message) + def _raise_exception(): if raise_exception: if flags.rollback_on_exception: db.rollback() - import inspect if inspect.isclass(raise_exception) and issubclass(raise_exception, Exception): raise raise_exception(msg) @@ -436,8 +451,11 @@ def msgprint( if as_list and type(msg) in (list, tuple): out.as_list = 1 + if sys.stdin.isatty(): + msg = _strip_html_tags(out.message) + if flags.print_messages and out.message: - print(f"Message: {strip_html_tags(out.message)}") + print(f"Message: {_strip_html_tags(out.message)}") out.title = title or _("Message", context="Default title of the message dialog") @@ -835,6 +853,7 @@ def clear_cache(user=None, doctype=None): :param user: If user is given, only user cache is cleared. :param doctype: If doctype is given, only DocType cache is cleared.""" import frappe.cache_manager + import frappe.utils.caching if doctype: frappe.cache_manager.clear_doctype_cache(doctype) @@ -854,7 +873,14 @@ def clear_cache(user=None, doctype=None): for fn in get_hooks("clear_cache"): get_attr(fn)() + frappe.utils.caching._SITE_CACHE.clear() local.role_permissions = {} + if hasattr(local, "request_cache"): + local.request_cache.clear() + if hasattr(local, "system_settings"): + del local.system_settings + if hasattr(local, "website_settings"): + del local.website_settings def only_has_select_perm(doctype, user=None, ignore_permissions=False): @@ -1024,7 +1050,7 @@ def get_cached_doc(*args, **kwargs): return doc if key := can_cache_doc(args): - # local cache + # local cache - has "ready" `Document` objects if doc := local.document_cache.get(key): return _respond(doc) @@ -1032,9 +1058,22 @@ def get_cached_doc(*args, **kwargs): if doc := cache().hget("document_cache", key): return _respond(doc, True) - # database + # Not found in local/redis, fetch from DB doc = get_doc(*args, **kwargs) + # Store in cache + if not key: + key = get_document_cache_key(doc.doctype, doc.name) + + local.document_cache[key] = doc + + # Avoid setting in local.cache since we're already using local.document_cache above + # Try pickling the doc object as-is first, else fallback to doc.as_dict() + try: + cache().hset("document_cache", key, doc, cache_locally=False) + except Exception: + cache().hset("document_cache", key, doc.as_dict(), cache_locally=False) + return doc @@ -1065,6 +1104,10 @@ def clear_document_cache(doctype, name): if key in local.document_cache: del local.document_cache[key] cache().hdel("document_cache", key) + if doctype == "System Settings" and hasattr(local, "system_settings"): + delattr(local, "system_settings") + if doctype == "Website Settings" and hasattr(local, "website_settings"): + delattr(local, "website_settings") def get_cached_value(doctype, name, fieldname="name", as_dict=False): @@ -1105,10 +1148,13 @@ def get_doc(*args, **kwargs): doc = frappe.model.document.get_doc(*args, **kwargs) - # set in cache + # Replace cache if key := can_cache_doc(args): - local.document_cache[key] = doc - cache().hset("document_cache", key, doc.as_dict()) + if key in local.document_cache: + local.document_cache[key] = doc + + if cache().hexists("document_cache", key): + cache().hset("document_cache", key, doc.as_dict()) return doc @@ -1163,7 +1209,7 @@ def delete_doc( :param delete_permanently: Do not create a Deleted Document for the document.""" import frappe.model.delete_doc - frappe.model.delete_doc.delete_doc( + return frappe.model.delete_doc.delete_doc( doctype, name, force, @@ -1259,8 +1305,10 @@ def get_module_path(module, *joins): :param module: Module name. :param *joins: Join additional path elements using `os.path.join`.""" - module = scrub(module) - return get_pymodule_path(local.module_app[module] + "." + module, *joins) + from frappe.modules.utils import get_module_app + + app = get_module_app(module) + return get_pymodule_path(app + "." + scrub(module), *joins) def get_app_path(app_name, *joins): @@ -1312,6 +1360,7 @@ def get_all_apps(with_internal_apps=True, sites_path=None): return apps +@request_cache def get_installed_apps(sort=False, frappe_last=False): """Get list of installed apps in current site.""" if getattr(flags, "in_install_db", True): @@ -1353,47 +1402,49 @@ def get_doc_hooks(): return local.doc_events_hooks -def get_hooks(hook=None, default=None, app_name=None): +@request_cache +def _load_app_hooks(app_name: Optional[str] = None): + hooks = {} + apps = [app_name] if app_name else get_installed_apps(sort=True) + + for app in apps: + try: + app_hooks = get_module(f"{app}.hooks") + except ImportError: + if local.flags.in_install_app: + # if app is not installed while restoring + # ignore it + pass + print(f'Could not find app "{app}"') + if not request: + raise SystemExit + raise + for key in dir(app_hooks): + if not key.startswith("_"): + append_hook(hooks, key, getattr(app_hooks, key)) + return hooks + + +def get_hooks( + hook: str = None, default: Optional[Any] = "_KEEP_DEFAULT_LIST", app_name: str = None +) -> _dict: """Get hooks via `app/hooks.py` :param hook: Name of the hook. Will gather all hooks for this name and return as a list. :param default: Default if no hook found. :param app_name: Filter by app.""" - def load_app_hooks(app_name=None): - hooks = {} - for app in [app_name] if app_name else get_installed_apps(sort=True): - app = "frappe" if app == "webnotes" else app - try: - app_hooks = get_module(app + ".hooks") - except ImportError: - if local.flags.in_install_app: - # if app is not installed while restoring - # ignore it - pass - print('Could not find app "{0}"'.format(app_name)) - if not request: - sys.exit(1) - raise - for key in dir(app_hooks): - if not key.startswith("_"): - append_hook(hooks, key, getattr(app_hooks, key)) - return hooks - - no_cache = conf.developer_mode or False - if app_name: - hooks = _dict(load_app_hooks(app_name)) + hooks = _dict(_load_app_hooks(app_name)) else: - if no_cache: - hooks = _dict(load_app_hooks()) + if conf.developer_mode: + hooks = _dict(_load_app_hooks()) else: - hooks = _dict(cache().get_value("app_hooks", load_app_hooks)) + hooks = _dict(cache().get_value("app_hooks", _load_app_hooks)) if hook: - return hooks.get(hook) or (default if default is not None else []) - else: - return hooks + return hooks.get(hook, ([] if default == "_KEEP_DEFAULT_LIST" else default)) + return hooks def append_hook(target, key, value): @@ -1501,19 +1552,35 @@ def call(fn, *args, **kwargs): return fn(*args, **newargs) -def get_newargs(fn, kwargs): +def get_newargs(fn: Callable, kwargs: Dict[str, Any]) -> Dict[str, Any]: + """Remove any kwargs that are not supported by the function. + + Example: + >>> def fn(a=1, b=2): pass + + >>> get_newargs(fn, {"a": 2, "c": 1}) + {"a": 2} + """ + + # if function has any **kwargs parameter that capture arbitrary keyword arguments + # Ref: https://docs.python.org/3/library/inspect.html#inspect.Parameter.kind + varkw_exist = False + if hasattr(fn, "fnargs"): fnargs = fn.fnargs else: signature = inspect.signature(fn) fnargs = list(signature.parameters) - varkw = "kwargs" in fnargs - if varkw: - fnargs.pop(-1) + + for param_name, parameter in signature.parameters.items(): + if parameter.kind == inspect.Parameter.VAR_KEYWORD: + varkw_exist = True + fnargs.remove(param_name) + break newargs = {} for a in kwargs: - if (a in fnargs) or varkw: + if (a in fnargs) or varkw_exist: newargs[a] = kwargs.get(a) newargs.pop("ignore_permissions", None) @@ -1809,18 +1876,21 @@ def get_value(*args, **kwargs): return db.get_value(*args, **kwargs) -def as_json(obj: Union[Dict, List], indent=1) -> str: +def as_json(obj: Union[Dict, List], indent=1, separators=None) -> str: from frappe.utils.response import json_handler + if separators is None: + separators = (",", ": ") + try: return json.dumps( - obj, indent=indent, sort_keys=True, default=json_handler, separators=(",", ": ") + obj, indent=indent, sort_keys=True, default=json_handler, separators=separators ) except TypeError: # this would break in case the keys are not all os "str" type - as defined in the JSON # adding this to ensure keys are sorted (expected behaviour) sorted_obj = dict(sorted(obj.items(), key=lambda kv: str(kv[0]))) - return json.dumps(sorted_obj, indent=indent, default=json_handler, separators=(",", ": ")) + return json.dumps(sorted_obj, indent=indent, default=json_handler, separators=separators) def are_emails_muted(): @@ -2158,8 +2228,18 @@ def safe_eval(code, eval_globals=None, eval_locals=None): return eval(code, eval_globals, eval_locals) +def get_website_settings(key): + if not hasattr(local, "website_settings"): + local.website_settings = db.get_singles_dict("Website Settings", cast=True) + + return local.website_settings[key] + + def get_system_settings(key): - return db.get_single_value("System Settings", key, cache=True) + if not hasattr(local, "system_settings"): + local.system_settings = db.get_singles_dict("System Settings", cast=True) + + return local.system_settings[key] def get_active_domains(): diff --git a/frappe/app.py b/frappe/app.py index e6df29fbd9..f8c81478c0 100644 --- a/frappe/app.py +++ b/frappe/app.py @@ -30,6 +30,8 @@ local_manager = LocalManager([frappe.local]) _site = None _sites_path = os.environ.get("SITES_PATH", ".") +SAFE_HTTP_METHODS = ("GET", "HEAD", "OPTIONS") +UNSAFE_HTTP_METHODS = ("POST", "PUT", "DELETE", "PATCH") class RequestContext(object): @@ -292,7 +294,10 @@ def handle_exception(e): def after_request(rollback): - if (frappe.local.request.method in ("POST", "PUT") or frappe.local.flags.commit) and frappe.db: + # if HTTP method would change server state, commit if necessary + if frappe.db and ( + frappe.local.flags.commit or frappe.local.request.method in UNSAFE_HTTP_METHODS + ): if frappe.db.transaction_writes: frappe.db.commit() rollback = False diff --git a/frappe/auth.py b/frappe/auth.py index dc53c20f28..80141d1d6c 100644 --- a/frappe/auth.py +++ b/frappe/auth.py @@ -165,7 +165,7 @@ class LoginManager: self.set_user_info() def get_user_info(self): - self.info = frappe.db.get_value( + self.info = frappe.get_cached_value( "User", self.user, ["user_type", "first_name", "last_name", "user_image"], as_dict=1 ) @@ -412,10 +412,16 @@ def clear_cookies(): def validate_ip_address(user): """check if IP Address is valid""" - user = ( - frappe.get_cached_doc("User", user) if not frappe.flags.in_test else frappe.get_doc("User", user) + from frappe.core.doctype.user.user import get_restricted_ip_list + + # Only fetch required fields - for perf + user_fields = ["restrict_ip", "bypass_restrict_ip_check_if_2fa_enabled"] + user_info = ( + frappe.get_cached_value("User", user, user_fields, as_dict=True) + if not frappe.flags.in_test + else frappe.db.get_value("User", user, user_fields, as_dict=True) ) - ip_list = user.get_restricted_ip_list() + ip_list = get_restricted_ip_list(user_info) if not ip_list: return @@ -430,7 +436,7 @@ def validate_ip_address(user): # check if two factor auth is enabled if system_settings.enable_two_factor_auth and not bypass_restrict_ip_check: # check if bypass restrict ip is enabled for login user - bypass_restrict_ip_check = user.bypass_restrict_ip_check_if_2fa_enabled + bypass_restrict_ip_check = user_info.bypass_restrict_ip_check_if_2fa_enabled for ip in ip_list: if frappe.local.request_ip.startswith(ip) or bypass_restrict_ip_check: diff --git a/frappe/automation/doctype/assignment_rule/assignment_rule.py b/frappe/automation/doctype/assignment_rule/assignment_rule.py index f3dfa4cf0a..0ca64e54c2 100644 --- a/frappe/automation/doctype/assignment_rule/assignment_rule.py +++ b/frappe/automation/doctype/assignment_rule/assignment_rule.py @@ -298,8 +298,6 @@ def apply(doc=None, method=None, doctype=None, name=None): if reopened: break - # print(f"Rule:{assignment_rule}\nDoc: {doc}\nReOpened: {reopened}") - assignment_rule.close_assignments(doc) diff --git a/frappe/boot.py b/frappe/boot.py index a23a7e6ac3..6cd86dc4fc 100644 --- a/frappe/boot.py +++ b/frappe/boot.py @@ -15,7 +15,7 @@ from frappe.geo.country_info import get_all from frappe.model.base_document import get_controller from frappe.query_builder import DocType from frappe.query_builder.functions import Count -from frappe.query_builder.terms import subqry +from frappe.query_builder.terms import SubQuery from frappe.social.doctype.energy_point_log.energy_point_log import get_energy_points from frappe.social.doctype.energy_point_settings.energy_point_settings import ( is_energy_point_enabled, @@ -211,7 +211,7 @@ def get_user_pages_or_reports(parent, cache=False): if parent == "Report": has_role[p.name].update({"ref_doctype": p.ref_doctype}) - no_of_roles = ( + no_of_roles = SubQuery( frappe.qb.from_(hasRole).select(Count("*")).where(hasRole.parent == parentTable.name) ) @@ -221,7 +221,7 @@ def get_user_pages_or_reports(parent, cache=False): pages_with_no_roles = ( frappe.qb.from_(parentTable) .select(parentTable.name, parentTable.modified, *columns) - .where(subqry(no_of_roles) == 0) + .where(no_of_roles == 0) ).run(as_dict=True) for p in pages_with_no_roles: @@ -327,7 +327,7 @@ def get_unseen_notes(): (note.notify_on_every_login == 1) & (note.expire_notification_on > frappe.utils.now()) & ( - subqry(frappe.qb.from_(nsb).select(nsb.user).where(nsb.parent == note.name)).notin( + SubQuery(frappe.qb.from_(nsb).select(nsb.user).where(nsb.parent == note.name)).notin( [frappe.session.user] ) ) diff --git a/frappe/build.py b/frappe/build.py index e20ee0d698..5923bd05ec 100644 --- a/frappe/build.py +++ b/frappe/build.py @@ -20,6 +20,8 @@ import frappe timestamps = {} app_paths = None sites_path = os.path.abspath(os.getcwd()) +WHITESPACE_PATTERN = re.compile(r"\s+") +HTML_COMMENT_PATTERN = re.compile(r"()") class AssetsNotDownloadedError(Exception): @@ -406,10 +408,10 @@ def link_assets_dir(source, target, hard_link=False): def scrub_html_template(content): """Returns HTML content with removed whitespace and comments""" # remove whitespace to a single space - content = re.sub(r"\s+", " ", content) + content = WHITESPACE_PATTERN.sub(" ", content) # strip comments - content = re.sub(r"()", "", content) + content = HTML_COMMENT_PATTERN.sub("", content) return content.replace("'", "'") diff --git a/frappe/client.py b/frappe/client.py index f753da6f57..4afe0898bc 100644 --- a/frappe/client.py +++ b/frappe/client.py @@ -100,7 +100,7 @@ def get_value(doctype, fieldname, filters=None, as_dict=True, debug=False, paren if frappe.is_table(doctype): check_parent_permission(parent, doctype) - if not frappe.has_permission(doctype): + if not frappe.has_permission(doctype, parent_doctype=parent): frappe.throw(_("No permission for {0}").format(doctype), frappe.PermissionError) filters = get_safe_filters(filters) @@ -385,7 +385,7 @@ def attach_file( is_private=None, docfield=None, ): - """Attach a file to Document (POST) + """Attach a file to Document :param filename: filename e.g. test-file.txt :param filedata: base64 encode filedata which must be urlencoded @@ -396,17 +396,10 @@ def attach_file( :param is_private: Attach file as private file (1 or 0) :param docfield: file to attach to (optional)""" - request_method = frappe.local.request.environ.get("REQUEST_METHOD") - - if request_method.upper() != "POST": - frappe.throw(_("Invalid Request")) - doc = frappe.get_doc(doctype, docname) + doc.check_permission() - if not doc.has_permission(): - frappe.throw(_("Not permitted"), frappe.PermissionError) - - _file = frappe.get_doc( + file = frappe.get_doc( { "doctype": "File", "file_name": filename, @@ -418,14 +411,13 @@ def attach_file( "content": filedata, "decode": decode_base64, } - ) - _file.save() + ).save() if docfield and doctype: - doc.set(docfield, _file.file_url) + doc.set(docfield, file.file_url) doc.save() - return _file.as_dict() + return file @frappe.whitelist() diff --git a/frappe/commands/site.py b/frappe/commands/site.py index 80acf647e0..13d702d6f3 100644 --- a/frappe/commands/site.py +++ b/frappe/commands/site.py @@ -9,6 +9,7 @@ import click # imports - module imports import frappe from frappe.commands import get_site, pass_context +from frappe.core.doctype.log_settings.log_settings import LOG_DOCTYPES from frappe.exceptions import SiteNotSpecifiedError @@ -143,10 +144,6 @@ def restore( ) from frappe.utils.backups import Backup - if not os.path.exists(sql_file_path): - print("Invalid path", sql_file_path) - sys.exit(1) - _backup = Backup(sql_file_path) site = get_site(context) @@ -1092,6 +1089,51 @@ def build_search_index(context): frappe.destroy() +@click.command("clear-log-table") +@click.option("--doctype", default="text", type=click.Choice(LOG_DOCTYPES), help="Log DocType") +@click.option("--days", type=int, help="Keep records for days") +@click.option("--no-backup", is_flag=True, default=False, help="Do not backup the table") +@pass_context +def clear_log_table(context, doctype, days, no_backup): + """If any logtype table grows too large then clearing it with DELETE query + is not feasible in reasonable time. This command copies recent data to new + table and replaces current table with new smaller table. + + + ref: https://mariadb.com/kb/en/big-deletes/#deleting-more-than-half-a-table + """ + from frappe.core.doctype.log_settings.log_settings import clear_log_table as clear_logs + from frappe.utils.backups import scheduled_backup + + if not context.sites: + raise SiteNotSpecifiedError + + if doctype not in LOG_DOCTYPES: + raise frappe.ValidationError(f"Unsupported logging DocType: {doctype}") + + for site in context.sites: + frappe.init(site=site) + frappe.connect() + + if not no_backup: + scheduled_backup( + ignore_conf=False, + include_doctypes=doctype, + ignore_files=True, + force=True, + ) + click.echo(f"Backed up {doctype}") + + try: + click.echo(f"Copying {doctype} records from last {days} days to temporary table.") + clear_logs(doctype, days=days) + except Exception as e: + click.echo(f"Log cleanup for {doctype} failed:\n{e}") + sys.exit(1) + else: + click.secho(f"Cleared {doctype} records older than {days} days", fg="green") + + @click.command("trim-database") @click.option("--dry-run", is_flag=True, default=False, help="Show what would be deleted") @click.option( @@ -1264,4 +1306,5 @@ commands = [ partial_restore, trim_tables, trim_database, + clear_log_table, ] diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py index 2d3916914d..41a4b27bcf 100644 --- a/frappe/commands/utils.py +++ b/frappe/commands/utils.py @@ -730,6 +730,7 @@ def transform_database(context, table, engine, row_format, failfast): @click.command("run-tests") @click.option("--app", help="For App") @click.option("--doctype", help="For DocType") +@click.option("--module-def", help="For all Doctypes in Module Def") @click.option("--case", help="Select particular TestCase") @click.option( "--doctype-list-path", @@ -754,6 +755,7 @@ def run_tests( app=None, module=None, doctype=None, + module_def=None, test=(), profile=False, coverage=False, @@ -790,6 +792,7 @@ def run_tests( app, module, doctype, + module_def, context.verbose, tests=tests, force=context.force, diff --git a/frappe/contacts/address_and_contact.py b/frappe/contacts/address_and_contact.py index 036594926e..a0f742c55a 100644 --- a/frappe/contacts/address_and_contact.py +++ b/frappe/contacts/address_and_contact.py @@ -3,6 +3,7 @@ import functools import re +from typing import Dict, List import frappe from frappe import _ @@ -169,29 +170,34 @@ def delete_contact_and_address(doctype, docname): @frappe.whitelist() @frappe.validate_and_sanitize_search_inputs -def filter_dynamic_link_doctypes(doctype, txt, searchfield, start, page_len, filters): - if not txt: - txt = "" +def filter_dynamic_link_doctypes(txt: str, filters: Dict) -> List[List[str]]: + from frappe.permissions import get_doctypes_with_read - doctypes = frappe.db.get_all( - "DocField", filters=filters, fields=["parent"], distinct=True, as_list=True + txt = txt or "" + filters = filters or {} + TXT_PATTERN = re.compile(f"{txt}.*") + + _doctypes_from_df = frappe.get_all( + "DocField", + filters=filters, + pluck="parent", + distinct=True, + order_by=None, ) + doctypes_from_df = {d for d in _doctypes_from_df if TXT_PATTERN.search(_(d), re.IGNORECASE)} - doctypes = tuple(d for d in doctypes if re.search(txt + ".*", _(d[0]), re.IGNORECASE)) + filters.update({"dt": ("not in", doctypes_from_df)}) + _doctypes_from_cdf = frappe.get_all( + "Custom Field", filters=filters, pluck="dt", distinct=True, order_by=None + ) + doctypes_from_cdf = {d for d in _doctypes_from_cdf if TXT_PATTERN.search(_(d), re.IGNORECASE)} - filters.update({"dt": ("not in", [d[0] for d in doctypes])}) + all_doctypes = doctypes_from_df.union(doctypes_from_cdf) + allowed_doctypes = set(get_doctypes_with_read()) - _doctypes = frappe.db.get_all("Custom Field", filters=filters, fields=["dt"], as_list=True) + valid_doctypes = sorted(all_doctypes.intersection(allowed_doctypes)) - _doctypes = tuple([d for d in _doctypes if re.search(txt + ".*", _(d[0]), re.IGNORECASE)]) - - all_doctypes = [d[0] for d in doctypes + _doctypes] - allowed_doctypes = frappe.permissions.get_doctypes_with_read() - - valid_doctypes = sorted(set(all_doctypes).intersection(set(allowed_doctypes))) - valid_doctypes = [[doctype] for doctype in valid_doctypes] - - return valid_doctypes + return [[doctype] for doctype in valid_doctypes] def set_link_title(doc): diff --git a/frappe/data_migration/__init__.py b/frappe/core/api/__init__.py similarity index 100% rename from frappe/data_migration/__init__.py rename to frappe/core/api/__init__.py diff --git a/frappe/core/api/file.py b/frappe/core/api/file.py new file mode 100644 index 0000000000..e558f2f7e3 --- /dev/null +++ b/frappe/core/api/file.py @@ -0,0 +1,122 @@ +import json +from typing import Dict, List + +import frappe +from frappe.core.doctype.file.file import File, setup_folder_path +from frappe.utils import cint, cstr + + +@frappe.whitelist() +def unzip_file(name: str): + """Unzip the given file and make file records for each of the extracted files""" + file: File = frappe.get_doc("File", name) + return file.unzip() + + +@frappe.whitelist() +def get_attached_images(doctype: str, names: List[str]) -> frappe._dict: + """get list of image urls attached in form + returns {name: ['image.jpg', 'image.png']}""" + + if isinstance(names, str): + names = json.loads(names) + + img_urls = frappe.db.get_list( + "File", + filters={ + "attached_to_doctype": doctype, + "attached_to_name": ("in", names), + "is_folder": 0, + }, + fields=["file_url", "attached_to_name as docname"], + ) + + out = frappe._dict() + for i in img_urls: + out[i.docname] = out.get(i.docname, []) + out[i.docname].append(i.file_url) + + return out + + +@frappe.whitelist() +def get_files_in_folder(folder: str, start: int = 0, page_length: int = 20) -> Dict: + start = cint(start) + page_length = cint(page_length) + + attachment_folder = frappe.db.get_value( + "File", + "Home/Attachments", + ["name", "file_name", "file_url", "is_folder", "modified"], + as_dict=1, + ) + + files = frappe.get_list( + "File", + {"folder": folder}, + ["name", "file_name", "file_url", "is_folder", "modified"], + start=start, + page_length=page_length + 1, + ) + + if folder == "Home" and attachment_folder not in files: + files.insert(0, attachment_folder) + + return {"files": files[:page_length], "has_more": len(files) > page_length} + + +@frappe.whitelist() +def get_files_by_search_text(text: str) -> List[Dict]: + if not text: + return [] + + text = "%" + cstr(text).lower() + "%" + return frappe.get_list( + "File", + fields=["name", "file_name", "file_url", "is_folder", "modified"], + filters={"is_folder": False}, + or_filters={ + "file_name": ("like", text), + "file_url": text, + "name": ("like", text), + }, + order_by="modified desc", + limit=20, + ) + + +@frappe.whitelist(allow_guest=True) +def get_max_file_size() -> int: + return cint(frappe.conf.get("max_file_size")) or 10485760 + + +@frappe.whitelist() +def create_new_folder(file_name: str, folder: str) -> File: + """create new folder under current parent folder""" + file = frappe.new_doc("File") + file.file_name = file_name + file.is_folder = 1 + file.folder = folder + file.insert(ignore_if_duplicate=True) + return file + + +@frappe.whitelist() +def move_file(file_list: List[File], new_parent: str, old_parent: str) -> None: + if isinstance(file_list, str): + file_list = json.loads(file_list) + + for file_obj in file_list: + setup_folder_path(file_obj.get("name"), new_parent) + + # recalculate sizes + frappe.get_doc("File", old_parent).save() + frappe.get_doc("File", new_parent).save() + + +@frappe.whitelist() +def zip_files(files: str): + files = frappe.parse_json(files) + frappe.response["filename"] = "files.zip" + frappe.response["filecontent"] = File.zip_files(files) + frappe.response["type"] = "download" diff --git a/frappe/core/doctype/access_log/access_log.json b/frappe/core/doctype/access_log/access_log.json index c5f1030266..69803ef05a 100644 --- a/frappe/core/doctype/access_log/access_log.json +++ b/frappe/core/doctype/access_log/access_log.json @@ -36,6 +36,7 @@ "fieldname": "user", "fieldtype": "Link", "in_list_view": 1, + "in_standard_filter": 1, "label": "User ", "options": "User", "read_only": 1 @@ -51,6 +52,7 @@ "fieldname": "reference_document", "fieldtype": "Data", "in_list_view": 1, + "in_standard_filter": 1, "label": "Reference Document", "read_only": 1 }, @@ -129,7 +131,7 @@ } ], "links": [], - "modified": "2022-05-03 09:34:19.337551", + "modified": "2022-06-13 05:59:26.866004", "modified_by": "Administrator", "module": "Core", "name": "Access Log", diff --git a/frappe/core/doctype/activity_log/activity_log.py b/frappe/core/doctype/activity_log/activity_log.py index 61dedd7bc0..468b7f4473 100644 --- a/frappe/core/doctype/activity_log/activity_log.py +++ b/frappe/core/doctype/activity_log/activity_log.py @@ -25,6 +25,13 @@ class ActivityLog(Document): if self.reference_doctype and self.reference_name: self.status = "Linked" + @staticmethod + def clear_old_logs(days=None): + if not days: + days = 90 + doctype = DocType("Activity Log") + frappe.db.delete(doctype, filters=(doctype.modified < (Now() - Interval(days=days)))) + def on_doctype_update(): """Add indexes in `tabActivity Log`""" @@ -43,12 +50,3 @@ def add_authentication_log(subject, user, operation="Login", status="Success"): "operation": operation, } ).insert(ignore_permissions=True, ignore_links=True) - - -def clear_activity_logs(days=None): - """clear 90 day old authentication logs or configured in log settings""" - - if not days: - days = 90 - doctype = DocType("Activity Log") - frappe.db.delete(doctype, filters=(doctype.creation < (Now() - Interval(days=days)))) diff --git a/frappe/core/doctype/activity_log/activity_log_list.js b/frappe/core/doctype/activity_log/activity_log_list.js index 111a230827..e3a75a1941 100644 --- a/frappe/core/doctype/activity_log/activity_log_list.js +++ b/frappe/core/doctype/activity_log/activity_log_list.js @@ -4,5 +4,10 @@ frappe.listview_settings['Activity Log'] = { return [__(doc.status), "green"]; else if(doc.operation == "Login" && doc.status == "Failed") return [__(doc.status), "red"]; - } -}; \ No newline at end of file + }, + onload: function(listview) { + frappe.require("logtypes.bundle.js", () => { + frappe.utils.logtypes.show_log_retention_message(cur_list.doctype); + }) + }, +}; diff --git a/frappe/core/doctype/communication/email.py b/frappe/core/doctype/communication/email.py index 464bc35a1c..2c8a65fafe 100755 --- a/frappe/core/doctype/communication/email.py +++ b/frappe/core/doctype/communication/email.py @@ -12,6 +12,7 @@ from frappe.utils import ( cint, get_datetime, get_formatted_email, + get_string_between, list_to_str, split_emails, validate_email_address, @@ -21,14 +22,6 @@ if TYPE_CHECKING: from frappe.core.doctype.communication.communication import Communication -OUTGOING_EMAIL_ACCOUNT_MISSING = _( - """ - Unable to send mail because of a missing email account. - Please setup default Email Account from Setup > Email > Email Account -""" -) - - @frappe.whitelist() def make( doctype=None, @@ -152,7 +145,7 @@ def _make( "reference_doctype": doctype, "reference_name": name, "email_template": email_template, - "message_id": get_message_id().strip(" <>"), + "message_id": get_string_between("<", get_message_id(), ">"), "read_receipt": read_receipt, "has_attachment": 1 if attachments else 0, "communication_type": communication_type, @@ -169,7 +162,12 @@ def _make( if cint(send_email): if not comm.get_outgoing_email_account(): - frappe.throw(msg=OUTGOING_EMAIL_ACCOUNT_MISSING, exc=frappe.OutgoingEmailError) + frappe.throw( + _( + "Unable to send mail because of a missing email account. Please setup default Email Account from Setup > Email > Email Account" + ), + exc=frappe.OutgoingEmailError, + ) comm.send_email( print_html=print_html, diff --git a/frappe/core/doctype/communication/mixins.py b/frappe/core/doctype/communication/mixins.py index 68abba3c13..0263cfeac5 100644 --- a/frappe/core/doctype/communication/mixins.py +++ b/frappe/core/doctype/communication/mixins.py @@ -152,7 +152,7 @@ class CommunicationEmailMixin: "doctype": self.reference_doctype, "name": self.reference_name, "print_format": print_format, - "key": get_parent_doc(self).get_signature(), + "key": get_parent_doc(self).get_document_share_key(), } ) diff --git a/frappe/core/doctype/data_import/importer.py b/frappe/core/doctype/data_import/importer.py index 01be69ea16..06d7588aef 100644 --- a/frappe/core/doctype/data_import/importer.py +++ b/frappe/core/doctype/data_import/importer.py @@ -4,6 +4,7 @@ import io import json import os +import re import timeit from datetime import date, datetime @@ -22,6 +23,7 @@ INVALID_VALUES = ("", None) MAX_ROWS_IN_PREVIEW = 10 INSERT = "Insert New Records" UPDATE = "Update Existing Records" +DURATION_PATTERN = re.compile(r"^(?:(\d+d)?((^|\s)\d+h)?((^|\s)\d+m)?((^|\s)\d+s)?)$") class Importer: @@ -725,10 +727,7 @@ class Row: ) return elif df.fieldtype == "Duration": - import re - - is_valid_duration = re.match(r"^(?:(\d+d)?((^|\s)\d+h)?((^|\s)\d+m)?((^|\s)\d+s)?)$", value) - if not is_valid_duration: + if not DURATION_PATTERN.match(value): self.warnings.append( { "row": self.row_number, diff --git a/frappe/core/doctype/deleted_document/deleted_document.json b/frappe/core/doctype/deleted_document/deleted_document.json index 1a612c7411..6b95a523c1 100644 --- a/frappe/core/doctype/deleted_document/deleted_document.json +++ b/frappe/core/doctype/deleted_document/deleted_document.json @@ -1,256 +1,81 @@ { - "allow_copy": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, - "creation": "2016-12-29 12:59:48.638970", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", + "actions": [], + "creation": "2016-12-29 12:59:48.638970", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "deleted_name", + "deleted_doctype", + "column_break_3", + "restored", + "new_name", + "section_break_6", + "data" + ], "fields": [ { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "deleted_name", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Deleted Name", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "deleted_name", + "fieldtype": "Data", + "in_list_view": 1, + "in_standard_filter": 1, + "label": "Deleted Name", + "read_only": 1 + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "deleted_doctype", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Deleted DocType", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "deleted_doctype", + "fieldtype": "Data", + "in_list_view": 1, + "in_standard_filter": 1, + "label": "Deleted DocType", + "read_only": 1 + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "column_break_3", - "fieldtype": "Column Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "column_break_3", + "fieldtype": "Column Break" + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "restored", - "fieldtype": "Check", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Restored", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "default": "0", + "fieldname": "restored", + "fieldtype": "Check", + "in_list_view": 1, + "label": "Restored", + "read_only": 1 + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "new_name", - "fieldtype": "Read Only", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "New Name", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "new_name", + "fieldtype": "Read Only", + "label": "New Name" + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "section_break_6", - "fieldtype": "Section Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "section_break_6", + "fieldtype": "Section Break" + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "data", - "fieldtype": "Code", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Data", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "fieldname": "data", + "fieldtype": "Code", + "label": "Data", + "read_only": 1 } - ], - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 1, - - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2016-12-29 14:39:45.724494", - "modified_by": "Administrator", - "module": "Core", - "name": "Deleted Document", - "name_case": "", - "owner": "Administrator", + ], + "in_create": 1, + "links": [], + "modified": "2022-06-13 05:50:58.314908", + "modified_by": "Administrator", + "module": "Core", + "name": "Deleted Document", + "owner": "Administrator", "permissions": [ { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 0, - "delete": 1, - "email": 0, - "export": 1, - "if_owner": 0, - "import": 0, - "is_custom": 0, - "permlevel": 0, - "print": 0, - "read": 1, - "report": 0, - "role": "System Manager", - "set_user_permissions": 0, - "share": 0, - "submit": 0, - "write": 0 + "delete": 1, + "export": 1, + "read": 1, + "role": "System Manager" } - ], - "quick_entry": 0, - "read_only": 0, - "read_only_onload": 0, - "sort_field": "modified", - "sort_order": "DESC", - "title_field": "deleted_name", - "track_changes": 1, - "track_seen": 0 + ], + "sort_field": "modified", + "sort_order": "DESC", + "states": [], + "title_field": "deleted_name", + "track_changes": 1 } \ No newline at end of file diff --git a/frappe/core/doctype/doctype/doctype.py b/frappe/core/doctype/doctype/doctype.py index 047c48e9d5..e834b698d5 100644 --- a/frappe/core/doctype/doctype/doctype.py +++ b/frappe/core/doctype/doctype/doctype.py @@ -8,6 +8,7 @@ import os # imports - standard imports import re import shutil +from typing import TYPE_CHECKING, Union # imports - module imports import frappe @@ -35,6 +36,15 @@ from frappe.query_builder.functions import Concat from frappe.utils import cint from frappe.website.utils import clear_cache +if TYPE_CHECKING: + from frappe.custom.doctype.customize_form.customize_form import CustomizeForm + +DEPENDS_ON_PATTERN = re.compile(r'[\w\.:_]+\s*={1}\s*[\w\.@\'"]+') +ILLEGAL_FIELDNAME_PATTERN = re.compile("""['",./%@()<>{}]""") +WHITESPACE_PADDING_PATTERN = re.compile(r"^[ \t\n\r]+|[ \t\n\r]+$", flags=re.ASCII) +START_WITH_LETTERS_PATTERN = re.compile(r"^(?![\W])[^\d_\s][\w -]+$", flags=re.ASCII) +FIELD_PATTERN = re.compile("{(.*?)}", flags=re.UNICODE) + class InvalidFieldNameError(frappe.ValidationError): pass @@ -357,8 +367,7 @@ class DocType(Document): else: if d.fieldname in restricted: frappe.throw(_("Fieldname {0} is restricted").format(d.fieldname), InvalidFieldNameError) - - d.fieldname = re.sub("""['",./%@()<>{}]""", "", d.fieldname) + d.fieldname = ILLEGAL_FIELDNAME_PATTERN.sub("", d.fieldname) # fieldnames should be lowercase d.fieldname = d.fieldname.lower() @@ -842,15 +851,13 @@ class DocType(Document): _("Doctype name is limited to {0} characters ({1})").format(max_length, name), frappe.NameError ) - flags = {"flags": re.ASCII} - # a DocType name should not start or end with an empty space - if re.search(r"^[ \t\n\r]+|[ \t\n\r]+$", name, **flags): + if WHITESPACE_PADDING_PATTERN.search(name): frappe.throw(_("DocType's name should not start or end with whitespace"), frappe.NameError) # a DocType's name should not start with a number or underscore # and should only contain letters, numbers, underscore, and hyphen - if not re.match(r"^(?![\W])[^\d_\s][\w -]+$", name, **flags): + if not START_WITH_LETTERS_PATTERN.match(name): frappe.throw( _( "A DocType's name should start with a letter and can only " @@ -913,11 +920,11 @@ def validate_series(dt, autoname=None, name=None): frappe.throw(_("Series {0} already used in {1}").format(prefix, used_in[0][0])) -def validate_autoincrement_autoname(dt: DocType) -> bool: +def validate_autoincrement_autoname(dt: Union[DocType, "CustomizeForm"]) -> bool: """Checks if can doctype can change to/from autoincrement autoname""" - def get_autoname_before_save(dt: DocType) -> str: - if dt.name == "Customize Form": + def get_autoname_before_save(dt: Union[DocType, "CustomizeForm"]) -> str: + if dt.doctype == "Customize Form": property_value = frappe.db.get_value( "Property Setter", {"doc_type": dt.doc_type, "property": "autoname"}, "value" ) @@ -940,10 +947,10 @@ def validate_autoincrement_autoname(dt: DocType) -> bool: or (not is_autoname_autoincrement and autoname_before_save == "autoincrement") ): - if frappe.get_meta(dt.name).issingle: - if dt.name == "Customize Form": - frappe.throw(_("Cannot change to/from autoincrement autoname in Customize Form")) + if dt.doctype == "Customize Form": + frappe.throw(_("Cannot change to/from autoincrement autoname in Customize Form")) + if frappe.get_meta(dt.name).issingle: return False if not frappe.get_all(dt.name, limit=1): @@ -1254,7 +1261,7 @@ def validate_fields(meta): if not pattern: return - for fieldname in re.findall("{(.*?)}", pattern, re.UNICODE): + for fieldname in FIELD_PATTERN.findall(pattern): if fieldname.startswith("{"): # edge case when double curlies are used for escape continue @@ -1336,9 +1343,7 @@ def validate_fields(meta): ] for field in depends_on_fields: depends_on = docfield.get(field, None) - if ( - depends_on and ("=" in depends_on) and re.match(r'[\w\.:_]+\s*={1}\s*[\w\.@\'"]+', depends_on) - ): + if depends_on and ("=" in depends_on) and DEPENDS_ON_PATTERN.match(depends_on): frappe.throw(_("Invalid {0} condition").format(frappe.unscrub(field)), frappe.ValidationError) def check_table_multiselect_option(docfield): diff --git a/frappe/core/doctype/doctype/test_doctype.py b/frappe/core/doctype/doctype/test_doctype.py index 11f5ef8a69..0bcd972c68 100644 --- a/frappe/core/doctype/doctype/test_doctype.py +++ b/frappe/core/doctype/doctype/test_doctype.py @@ -1,10 +1,14 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +import random +import string import unittest from typing import Dict, List, Optional +from unittest.mock import patch import frappe +from frappe.cache_manager import clear_doctype_cache from frappe.core.doctype.doctype.doctype import ( CannotIndexedError, DoctypeLinkError, @@ -15,8 +19,8 @@ from frappe.core.doctype.doctype.doctype import ( WrongOptionsDoctypeLinkError, validate_links_table_fieldnames, ) - -# test_records = frappe.get_test_records('DocType') +from frappe.custom.doctype.custom_field.custom_field import create_custom_fields +from frappe.desk.form.load import getdoc class TestDocType(unittest.TestCase): @@ -628,10 +632,55 @@ class TestDocType(unittest.TestCase): self.assertEqual(test_json.test_json_field["hello"], "world") + @patch.dict(frappe.conf, {"developer_mode": 1}) + def test_delete_doctype_with_customization(self): + from frappe.custom.doctype.property_setter.property_setter import make_property_setter + + custom_field = "customfield" + + doctype = new_doctype(custom=0).insert().name + + # Create property setter and custom field + field = "some_fieldname" + make_property_setter(doctype, field, "default", "DELETETHIS", "Data") + create_custom_fields({doctype: [{"fieldname": custom_field, "fieldtype": "Data"}]}) + + # Create 1 record + original_doc = frappe.get_doc(doctype=doctype, custom_field_name="wat").insert() + self.assertEqual(original_doc.some_fieldname, "DELETETHIS") + + # delete doctype + frappe.delete_doc("DocType", doctype) + clear_doctype_cache(doctype) + + # "restore" doctype by inserting doctype with same schema again + new_doctype(doctype, custom=0).insert() + + # Ensure basically same doctype getting "restored" + restored_doc = frappe.get_last_doc(doctype) + verify_fields = ["doctype", field, custom_field] + for f in verify_fields: + self.assertEqual(original_doc.get(f), restored_doc.get(f)) + + # Check form load of restored doctype + getdoc(doctype, restored_doc.name) + + # ensure meta - property setter + self.assertEqual(frappe.get_meta(doctype).get_field(field).default, "DELETETHIS") + frappe.delete_doc("DocType", doctype) + def new_doctype( - name, unique: bool = False, depends_on: str = "", fields: Optional[List[Dict]] = None, **kwargs + name: Optional[str] = None, + unique: bool = False, + depends_on: str = "", + fields: Optional[List[Dict]] = None, + **kwargs, ): + if not name: + # Test prefix is required to avoid coverage + name = "Test " + "".join(random.sample(string.ascii_lowercase, 10)) + doc = frappe.get_doc( { "doctype": "DocType", diff --git a/frappe/data_migration/doctype/__init__.py b/frappe/core/doctype/document_naming_settings/__init__.py similarity index 100% rename from frappe/data_migration/doctype/__init__.py rename to frappe/core/doctype/document_naming_settings/__init__.py diff --git a/frappe/core/doctype/document_naming_settings/document_naming_settings.js b/frappe/core/doctype/document_naming_settings/document_naming_settings.js new file mode 100644 index 0000000000..2dc5fc4d58 --- /dev/null +++ b/frappe/core/doctype/document_naming_settings/document_naming_settings.js @@ -0,0 +1,73 @@ +// Copyright (c) 2022, Frappe Technologies and contributors +// For license information, please see license.txt + +frappe.ui.form.on("Document Naming Settings", { + refresh: function(frm) { + frm.trigger("setup_transaction_autocomplete"); + frm.disable_save(); + }, + + setup_transaction_autocomplete: function(frm) { + frappe.call({ + method: "get_transactions_and_prefixes", + doc: frm.doc, + callback: function(r) { + frm.fields_dict.transaction_type.set_data(r.message.transactions); + frm.fields_dict.prefix.set_data(r.message.prefixes); + }, + }); + }, + + transaction_type: function(frm) { + frm.set_value("user_must_always_select", 0); + frappe.call({ + method: "get_options", + doc: frm.doc, + callback: function(r) { + frm.set_value("naming_series_options", r.message); + if (r.message && r.message.split("\n")[0] == "") + frm.set_value("user_must_always_select", 1); + }, + }); + }, + + prefix: function(frm) { + frappe.call({ + method: "get_current", + doc: frm.doc, + callback: function(r) { + frm.refresh_field("current_value"); + }, + }); + }, + + update: function(frm) { + frappe.call({ + method: "update_series", + doc: frm.doc, + freeze: true, + freeze_msg: __("Updating naming series options"), + callback: function(r) { + frm.trigger("setup_transaction_autocomplete"); + frm.trigger("transaction_type"); + }, + }); + }, + + try_naming_series(frm) { + frappe.call({ + method: "preview_series", + doc: frm.doc, + callback: function(r) { + if (!r.exc) { + frm.set_value("series_preview", r.message); + } else { + frm.set_value( + "series_preview", + __("Failed to generate preview of series") + ); + } + }, + }); + }, +}); diff --git a/frappe/core/doctype/document_naming_settings/document_naming_settings.json b/frappe/core/doctype/document_naming_settings/document_naming_settings.json new file mode 100644 index 0000000000..4c86b2ec1d --- /dev/null +++ b/frappe/core/doctype/document_naming_settings/document_naming_settings.json @@ -0,0 +1,133 @@ +{ + "actions": [], + "creation": "2022-05-30 07:24:07.736646", + "description": "Configure various aspects of how document naming works like naming series, current counter.", + "doctype": "DocType", + "engine": "InnoDB", + "field_order": [ + "naming_series_tab", + "setup_series", + "transaction_type", + "naming_series_options", + "user_must_always_select", + "update", + "column_break_9", + "try_naming_series", + "series_preview", + "help_html", + "update_series", + "prefix", + "current_value", + "update_series_start" + ], + "fields": [ + { + "collapsible": 1, + "description": "Set Naming Series options on your transactions.", + "fieldname": "setup_series", + "fieldtype": "Section Break", + "label": "Setup Series for transactions" + }, + { + "depends_on": "transaction_type", + "fieldname": "help_html", + "fieldtype": "HTML", + "label": "Help HTML", + "options": "
\n Edit list of Series in the box. Rules:\n \n Examples:\n \n
\n
\n" + }, + { + "default": "0", + "depends_on": "transaction_type", + "description": "Check this if you want to force the user to select a series before saving. There will be no default if you check this.", + "fieldname": "user_must_always_select", + "fieldtype": "Check", + "label": "User must always select" + }, + { + "depends_on": "transaction_type", + "fieldname": "update", + "fieldtype": "Button", + "label": "Update" + }, + { + "collapsible": 1, + "description": "Change the starting / current sequence number of an existing series.
\n\nWarning: Incorrectly updating counters can prevent documents from getting created. ", + "fieldname": "update_series", + "fieldtype": "Section Break", + "label": "Update Series Counter" + }, + { + "fieldname": "prefix", + "fieldtype": "Autocomplete", + "label": "Prefix" + }, + { + "description": "This is the number of the last created transaction with this prefix", + "fieldname": "current_value", + "fieldtype": "Int", + "label": "Current Value" + }, + { + "fieldname": "update_series_start", + "fieldtype": "Button", + "label": "Update Series Number", + "options": "update_series_start" + }, + { + "depends_on": "transaction_type", + "fieldname": "naming_series_options", + "fieldtype": "Text", + "label": "Series List for this Transaction" + }, + { + "depends_on": "transaction_type", + "description": "Generate 3 preview of names generate by any valid series.", + "fieldname": "try_naming_series", + "fieldtype": "Data", + "label": "Try a naming Series" + }, + { + "fieldname": "transaction_type", + "fieldtype": "Autocomplete", + "label": "Select Transaction" + }, + { + "fieldname": "column_break_9", + "fieldtype": "Column Break" + }, + { + "fieldname": "naming_series_tab", + "fieldtype": "Tab Break", + "label": "Naming Series" + }, + { + "fieldname": "series_preview", + "fieldtype": "Text", + "label": "Preview of generated names", + "read_only": 1 + } + ], + "hide_toolbar": 1, + "icon": "fa fa-sort-by-order", + "issingle": 1, + "links": [], + "modified": "2022-05-30 23:51:36.136535", + "modified_by": "Administrator", + "module": "Core", + "name": "Document Naming Settings", + "owner": "Administrator", + "permissions": [ + { + "create": 1, + "email": 1, + "print": 1, + "read": 1, + "role": "System Manager", + "share": 1, + "write": 1 + } + ], + "sort_field": "modified", + "sort_order": "DESC", + "states": [] +} diff --git a/frappe/core/doctype/document_naming_settings/document_naming_settings.py b/frappe/core/doctype/document_naming_settings/document_naming_settings.py new file mode 100644 index 0000000000..46def88b65 --- /dev/null +++ b/frappe/core/doctype/document_naming_settings/document_naming_settings.py @@ -0,0 +1,218 @@ +# Copyright (c) 2022, Frappe Technologies and contributors +# For license information, please see license.txt + +from typing import List, Set + +import frappe +from frappe import _ +from frappe.core.doctype.doctype.doctype import validate_series +from frappe.model.document import Document +from frappe.model.naming import NamingSeries +from frappe.permissions import get_doctypes_with_read +from frappe.utils import cint + + +class NamingSeriesNotSetError(frappe.ValidationError): + pass + + +class DocumentNamingSettings(Document): + @frappe.whitelist() + def get_transactions_and_prefixes(self): + + transactions = self._get_transactions() + prefixes = self._get_prefixes(transactions) + + return {"transactions": transactions, "prefixes": prefixes} + + def _get_transactions(self) -> List[str]: + + readable_doctypes = set(get_doctypes_with_read()) + + standard = frappe.get_all("DocField", {"fieldname": "naming_series"}, "parent", pluck="parent") + custom = frappe.get_all("Custom Field", {"fieldname": "naming_series"}, "dt", pluck="dt") + + return sorted(readable_doctypes.intersection(standard + custom)) + + def _get_prefixes(self, doctypes) -> List[str]: + """Get all prefixes for naming series. + + - For all templates prefix is evaluated considering today's date + - All existing prefix in DB are shared as is. + """ + series_templates = set() + for d in doctypes: + try: + options = frappe.get_meta(d).get_naming_series_options() + series_templates.update(options) + except frappe.DoesNotExistError: + frappe.msgprint(_("Unable to find DocType {0}").format(d)) + continue + + custom_templates = frappe.get_all( + "DocType", + fields=["autoname"], + filters={ + "name": ("not in", doctypes), + "autoname": ("like", "%.#%"), + "module": ("not in", ["Core"]), + }, + ) + if custom_templates: + series_templates.update([d.autoname.rsplit(".", 1)[0] for d in custom_templates]) + + return self._evaluate_and_clean_templates(series_templates) + + def _evaluate_and_clean_templates(self, series_templates: Set[str]) -> List[str]: + evalauted_prefix = set() + + series = frappe.qb.DocType("Series") + prefixes_from_db = frappe.qb.from_(series).select(series.name).run(pluck=True) + evalauted_prefix.update(prefixes_from_db) + + for series_template in series_templates: + prefix = NamingSeries(series_template).get_prefix() + if "{" in prefix: + # fieldnames can't be evalauted, rely on data in DB instead + continue + evalauted_prefix.add(prefix) + + return sorted(evalauted_prefix) + + def get_options_list(self, options: str) -> List[str]: + return [op.strip() for op in options.split("\n") if op.strip()] + + @frappe.whitelist() + def update_series(self): + """update series list""" + self.validate_set_series() + self.check_duplicate() + self.set_series_options_in_meta(self.transaction_type, self.naming_series_options) + + frappe.msgprint( + _("Series Updated for {}").format(self.transaction_type), alert=True, indicator="green" + ) + + def validate_set_series(self): + if self.transaction_type and not self.naming_series_options: + frappe.throw(_("Please set the series to be used.")) + + def set_series_options_in_meta(self, doctype: str, options: str) -> None: + options = self.get_options_list(options) + + # validate names + for series in options: + self.validate_series_name(series) + + if options and self.user_must_always_select: + options = [""] + options + + default = options[0] if options else "" + + option_string = "\n".join(options) + + self.update_naming_series_property_setter(doctype, "options", option_string) + self.update_naming_series_property_setter(doctype, "default", default) + + self.naming_series_options = option_string + + frappe.clear_cache(doctype=doctype) + + def update_naming_series_property_setter(self, doctype, property, value): + from frappe.custom.doctype.property_setter.property_setter import make_property_setter + + make_property_setter(doctype, "naming_series", property, value, "Text") + + def check_duplicate(self): + def stripped_series(s: str) -> str: + return s.strip().rstrip("#") + + standard = frappe.get_all("DocField", {"fieldname": "naming_series"}, "parent", pluck="parent") + custom = frappe.get_all("Custom Field", {"fieldname": "naming_series"}, "dt", pluck="dt") + + all_doctypes_with_naming_series = set(standard + custom) + all_doctypes_with_naming_series.remove(self.transaction_type) + + existing_series = {} + for doctype in all_doctypes_with_naming_series: + for series in frappe.get_meta(doctype).get_naming_series_options(): + existing_series[stripped_series(series)] = doctype + + dt = frappe.get_doc("DocType", self.transaction_type) + + options = self.get_options_list(self.naming_series_options) + for series in options: + if stripped_series(series) in existing_series: + frappe.throw(_("Series {0} already used in {1}").format(series, existing_series[series])) + validate_series(dt, series) + + def validate_series_name(self, series): + NamingSeries(series).validate() + + @frappe.whitelist() + def get_options(self, doctype=None): + doctype = doctype or self.transaction_type + if not doctype: + return + + if frappe.get_meta(doctype or self.transaction_type).get_field("naming_series"): + return frappe.get_meta(doctype or self.transaction_type).get_field("naming_series").options + + @frappe.whitelist() + def get_current(self): + """get series current""" + if self.prefix: + self.current_value = NamingSeries(self.prefix).get_current_value() + return self.current_value + + @frappe.whitelist() + def update_series_start(self): + frappe.only_for("System Manager") + + if not self.prefix: + frappe.throw(_("Please select prefix first")) + + naming_series = NamingSeries(self.prefix) + previous_value = naming_series.get_current_value() + naming_series.update_counter(self.current_value) + + self.create_version_log_for_change( + naming_series.get_prefix(), previous_value, self.current_value + ) + + frappe.msgprint( + _("Series counter for {} updated to {} successfully").format(self.prefix, self.current_value), + alert=True, + indicator="green", + ) + + def create_version_log_for_change(self, series, old, new): + version = frappe.new_doc("Version") + version.ref_doctype = "Series" + version.docname = series + version.data = frappe.as_json({"changed": [["current", old, new]]}) + version.flags.ignore_links = True # series is not a "real" doctype + version.flags.ignore_permissions = True + version.insert() + + @frappe.whitelist() + def preview_series(self) -> str: + """Preview what the naming series will generate.""" + + series = self.try_naming_series + if not series: + return "" + try: + doc = self._fetch_last_doc_if_available() + return "\n".join(NamingSeries(series).get_preview(doc=doc)) + except Exception as e: + if frappe.message_log: + frappe.message_log.pop() + return _("Failed to generate names from the series") + f"\n{str(e)}" + + def _fetch_last_doc_if_available(self): + """Fetch last doc for evaluating naming series with fields.""" + try: + return frappe.get_last_doc(self.transaction_type) + except Exception: + return None diff --git a/frappe/core/doctype/document_naming_settings/test_document_naming_settings.py b/frappe/core/doctype/document_naming_settings/test_document_naming_settings.py new file mode 100644 index 0000000000..98ce9e738b --- /dev/null +++ b/frappe/core/doctype/document_naming_settings/test_document_naming_settings.py @@ -0,0 +1,65 @@ +# Copyright (c) 2022, Frappe Technologies and Contributors +# See license.txt + +import frappe +from frappe.core.doctype.document_naming_settings.document_naming_settings import ( + DocumentNamingSettings, +) +from frappe.model.naming import NamingSeries, get_default_naming_series +from frappe.tests.utils import FrappeTestCase +from frappe.utils import cint + + +class TestNamingSeries(FrappeTestCase): + def setUp(self): + self.dns: DocumentNamingSettings = frappe.get_doc("Document Naming Settings") + + def tearDown(self): + frappe.db.rollback() + + def get_valid_serieses(self): + VALID_SERIES = ["SINV-", "SI-.{field}.", "SI-#.###", ""] + exisiting_series = self.dns.get_transactions_and_prefixes()["prefixes"] + return VALID_SERIES + exisiting_series + + def test_naming_preview(self): + self.dns.transaction_type = "Webhook" + + self.dns.try_naming_series = "AXBZ.####" + serieses = self.dns.preview_series().split("\n") + self.assertEqual(["AXBZ0001", "AXBZ0002", "AXBZ0003"], serieses) + + self.dns.try_naming_series = "AXBZ-.{currency}.-" + serieses = self.dns.preview_series().split("\n") + + def test_get_transactions(self): + + naming_info = self.dns.get_transactions_and_prefixes() + self.assertIn("Webhook", naming_info["transactions"]) + + existing_naming_series = frappe.get_meta("Webhook").get_field("naming_series").options + + for series in existing_naming_series.split("\n"): + self.assertIn(NamingSeries(series).get_prefix(), naming_info["prefixes"]) + + def test_default_naming_series(self): + self.assertIn("HOOK", get_default_naming_series("Webhook")) + self.assertIsNone(get_default_naming_series("DocType")) + + def test_updates_naming_options(self): + self.dns.transaction_type = "Webhook" + test_series = "KOOHBEW.###" + self.dns.naming_series_options = self.dns.get_options() + "\n" + test_series + self.dns.update_series() + self.assertIn(test_series, frappe.get_meta("Webhook").get_naming_series_options()) + + def test_update_series_counter(self): + for series in self.get_valid_serieses(): + if not series: + continue + self.dns.prefix = series + current_count = cint(self.dns.get_current()) + new_count = self.dns.current_value = current_count + 1 + self.dns.update_series_start() + + self.assertEqual(self.dns.get_current(), new_count, f"Incorrect update for {series}") diff --git a/frappe/data_migration/doctype/data_migration_connector/__init__.py b/frappe/core/doctype/document_share_key/__init__.py similarity index 100% rename from frappe/data_migration/doctype/data_migration_connector/__init__.py rename to frappe/core/doctype/document_share_key/__init__.py diff --git a/frappe/core/doctype/document_share_key/document_share_key.js b/frappe/core/doctype/document_share_key/document_share_key.js new file mode 100644 index 0000000000..c51233e10f --- /dev/null +++ b/frappe/core/doctype/document_share_key/document_share_key.js @@ -0,0 +1,8 @@ +// Copyright (c) 2021, Frappe Technologies and contributors +// For license information, please see license.txt + +frappe.ui.form.on('Document Share Key', { + // refresh: function(frm) { + + // } +}); diff --git a/frappe/core/doctype/document_share_key/document_share_key.json b/frappe/core/doctype/document_share_key/document_share_key.json new file mode 100644 index 0000000000..b96fe09f0b --- /dev/null +++ b/frappe/core/doctype/document_share_key/document_share_key.json @@ -0,0 +1,73 @@ +{ + "actions": [], + "allow_rename": 1, + "autoname": "hash", + "creation": "2022-01-14 13:40:49.487646", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "reference_doctype", + "reference_docname", + "key", + "expires_on" + ], + "fields": [ + { + "fieldname": "reference_doctype", + "fieldtype": "Link", + "label": "Reference Document Type", + "options": "DocType", + "read_only": 1, + "search_index": 1 + }, + { + "fieldname": "reference_docname", + "fieldtype": "Dynamic Link", + "label": "Reference Document Name", + "options": "reference_doctype", + "read_only": 1, + "search_index": 1 + }, + { + "fieldname": "key", + "fieldtype": "Data", + "label": "Key", + "read_only": 1 + }, + { + "fieldname": "expires_on", + "fieldtype": "Date", + "in_list_view": 1, + "label": "Expires On", + "read_only": 1 + } + ], + "in_create": 1, + "index_web_pages_for_search": 1, + "links": [], + "modified": "2022-01-14 13:57:28.050678", + "modified_by": "Administrator", + "module": "Core", + "name": "Document Share Key", + "naming_rule": "Expression", + "owner": "Administrator", + "permissions": [ + { + "create": 1, + "delete": 1, + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1, + "write": 1 + } + ], + "read_only": 1, + "sort_field": "modified", + "sort_order": "DESC", + "states": [] +} \ No newline at end of file diff --git a/frappe/core/doctype/document_share_key/document_share_key.py b/frappe/core/doctype/document_share_key/document_share_key.py new file mode 100644 index 0000000000..88608b992c --- /dev/null +++ b/frappe/core/doctype/document_share_key/document_share_key.py @@ -0,0 +1,20 @@ +# Copyright (c) 2021, Frappe Technologies and contributors +# For license information, please see license.txt + +from random import randrange + +import frappe +from frappe.model.document import Document + + +class DocumentShareKey(Document): + def before_insert(self): + self.key = frappe.generate_hash(length=randrange(25, 35)) + if not self.expires_on and not self.flags.no_expiry: + self.expires_on = frappe.utils.add_days( + None, days=frappe.get_system_settings("document_share_key_expiry") or 90 + ) + + +def is_expired(expires_on): + return expires_on and expires_on < frappe.utils.getdate() diff --git a/frappe/core/doctype/document_share_key/test_document_share_key.py b/frappe/core/doctype/document_share_key/test_document_share_key.py new file mode 100644 index 0000000000..10499fcc5d --- /dev/null +++ b/frappe/core/doctype/document_share_key/test_document_share_key.py @@ -0,0 +1,9 @@ +# Copyright (c) 2021, Frappe Technologies and Contributors +# See license.txt + +# import frappe +import unittest + + +class TestDocumentShareKey(unittest.TestCase): + pass diff --git a/frappe/core/doctype/error_log/error_log.js b/frappe/core/doctype/error_log/error_log.js index 4fe8fde5d6..1262002b04 100644 --- a/frappe/core/doctype/error_log/error_log.js +++ b/frappe/core/doctype/error_log/error_log.js @@ -1,8 +1,17 @@ -// Copyright (c) 2016, Frappe Technologies and contributors +// Copyright (c) 2022, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Error Log', { +frappe.ui.form.on("Error Log", { refresh: function(frm) { + frm.disable_save(); - } + if (frm.doc.reference_doctype && frm.doc.reference_name) { + frm.add_custom_button(__("Show Related Errors"), function() { + frappe.set_route("List", "Error Log", { + reference_doctype: frm.doc.reference_doctype, + reference_name: frm.doc.reference_name, + }); + }); + } + }, }); diff --git a/frappe/core/doctype/error_log/error_log.json b/frappe/core/doctype/error_log/error_log.json index e0ce109595..2ee86bd118 100644 --- a/frappe/core/doctype/error_log/error_log.json +++ b/frappe/core/doctype/error_log/error_log.json @@ -6,10 +6,12 @@ "engine": "MyISAM", "field_order": [ "seen", - "method", - "error", "reference_doctype", - "reference_name" + "column_break_3", + "reference_name", + "section_break_5", + "method", + "error" ], "fields": [ { @@ -47,12 +49,21 @@ "fieldtype": "Data", "label": "Reference Name", "read_only": 1 + }, + { + "fieldname": "column_break_3", + "fieldtype": "Column Break" + }, + { + "fieldname": "section_break_5", + "fieldtype": "Section Break" } ], "icon": "fa fa-warning-sign", "idx": 1, + "in_create": 1, "links": [], - "modified": "2022-05-19 05:32:16.026684", + "modified": "2022-06-13 06:34:05.158606", "modified_by": "Administrator", "module": "Core", "name": "Error Log", @@ -70,7 +81,6 @@ "write": 1 } ], - "quick_entry": 1, "sort_field": "modified", "sort_order": "ASC", "states": [], diff --git a/frappe/core/doctype/error_log/error_log.py b/frappe/core/doctype/error_log/error_log.py index d93029179c..224a5673a7 100644 --- a/frappe/core/doctype/error_log/error_log.py +++ b/frappe/core/doctype/error_log/error_log.py @@ -4,6 +4,8 @@ import frappe from frappe.model.document import Document +from frappe.query_builder import Interval +from frappe.query_builder.functions import Now class ErrorLog(Document): @@ -12,13 +14,10 @@ class ErrorLog(Document): self.db_set("seen", 1, update_modified=0) frappe.db.commit() - -def set_old_logs_as_seen(): - # set logs as seen - frappe.db.sql( - """UPDATE `tabError Log` SET `seen`=1 - WHERE `seen`=0 AND `creation` < (NOW() - INTERVAL '7' DAY)""" - ) + @staticmethod + def clear_old_logs(days=30): + table = frappe.qb.DocType("Error Log") + frappe.db.delete(table, filters=(table.modified < (Now() - Interval(days=days)))) @frappe.whitelist() diff --git a/frappe/core/doctype/error_log/error_log_list.js b/frappe/core/doctype/error_log/error_log_list.js index 91e69452ff..e92773a9de 100644 --- a/frappe/core/doctype/error_log/error_log_list.js +++ b/frappe/core/doctype/error_log/error_log_list.js @@ -1,7 +1,7 @@ -frappe.listview_settings['Error Log'] = { +frappe.listview_settings["Error Log"] = { add_fields: ["seen"], get_indicator: function(doc) { - if(cint(doc.seen)) { + if (cint(doc.seen)) { return [__("Seen"), "green", "seen,=,1"]; } else { return [__("Not Seen"), "red", "seen,=,0"]; @@ -11,11 +11,15 @@ frappe.listview_settings['Error Log'] = { onload: function(listview) { listview.page.add_menu_item(__("Clear Error Logs"), function() { frappe.call({ - method:'frappe.core.doctype.error_log.error_log.clear_error_logs', + method: "frappe.core.doctype.error_log.error_log.clear_error_logs", callback: function() { listview.refresh(); - } + }, }); }); - } + + frappe.require("logtypes.bundle.js", () => { + frappe.utils.logtypes.show_log_retention_message(cur_list.doctype); + }) + }, }; diff --git a/frappe/core/doctype/error_snapshot/error_snapshot.py b/frappe/core/doctype/error_snapshot/error_snapshot.py index 82f189217f..6e13b7a654 100644 --- a/frappe/core/doctype/error_snapshot/error_snapshot.py +++ b/frappe/core/doctype/error_snapshot/error_snapshot.py @@ -4,6 +4,8 @@ import frappe from frappe.model.document import Document +from frappe.query_builder import Interval +from frappe.query_builder.functions import Now class ErrorSnapshot(Document): @@ -32,3 +34,8 @@ class ErrorSnapshot(Document): frappe.db.set_value("Error Snapshot", parent["name"], "relapses", parent["relapses"] + 1) if parent["seen"]: frappe.db.set_value("Error Snapshot", parent["name"], "seen", False) + + @staticmethod + def clear_old_logs(days=30): + table = frappe.qb.DocType("Error Snapshot") + frappe.db.delete(table, filters=(table.modified < (Now() - Interval(days=days)))) diff --git a/frappe/core/doctype/error_snapshot/error_snapshot_list.js b/frappe/core/doctype/error_snapshot/error_snapshot_list.js index 1ba3e344ae..553495beb1 100644 --- a/frappe/core/doctype/error_snapshot/error_snapshot_list.js +++ b/frappe/core/doctype/error_snapshot/error_snapshot_list.js @@ -10,5 +10,10 @@ frappe.listview_settings["Error Snapshot"] = { } else { return [__("First Level"), !doc.seen ? "red" : "green", "parent_error_snapshot,=,"]; } - } + }, + onload: function(listview) { + frappe.require("logtypes.bundle.js", () => { + frappe.utils.logtypes.show_log_retention_message(cur_list.doctype); + }) + }, } diff --git a/frappe/core/doctype/file/__init__.py b/frappe/core/doctype/file/__init__.py index e69de29bb2..ad28c17e36 100644 --- a/frappe/core/doctype/file/__init__.py +++ b/frappe/core/doctype/file/__init__.py @@ -0,0 +1,2 @@ +from .exceptions import * +from .utils import * diff --git a/frappe/core/doctype/file/exceptions.py b/frappe/core/doctype/file/exceptions.py new file mode 100644 index 0000000000..d8939b69da --- /dev/null +++ b/frappe/core/doctype/file/exceptions.py @@ -0,0 +1,12 @@ +import frappe + + +class MaxFileSizeReachedError(frappe.ValidationError): + pass + + +class FolderNotEmpty(frappe.ValidationError): + pass + + +from frappe.exceptions import * diff --git a/frappe/core/doctype/file/file.js b/frappe/core/doctype/file/file.js index d40328d3cd..ecad8d884a 100644 --- a/frappe/core/doctype/file/file.js +++ b/frappe/core/doctype/file/file.js @@ -9,19 +9,9 @@ frappe.ui.form.on("File", "refresh", function(frm) { }, "fa fa-download"); } - var wrapper = frm.get_field("preview_html").$wrapper; - var is_viewable = frappe.utils.is_image_file(frm.doc.file_url); - - frm.toggle_display("preview", is_viewable); - frm.toggle_display("preview_html", is_viewable); - - if(is_viewable){ - wrapper.html('
\ - \ -
'); - } else { - wrapper.empty(); - } + frm.get_field("preview_html").$wrapper.html(`
+ +
`); var is_raster_image = (/\.(gif|jpg|jpeg|tiff|png)$/i).test(frm.doc.file_url); var is_optimizable = !frm.doc.is_folder && is_raster_image && frm.doc.file_size > 0; @@ -38,7 +28,7 @@ frappe.ui.form.on("File", "refresh", function(frm) { if(frm.doc.file_name && frm.doc.file_name.split('.').splice(-1)[0]==='zip') { frm.add_custom_button(__('Unzip'), function() { frappe.call({ - method: "frappe.core.doctype.file.file.unzip_file", + method: "frappe.core.api.file.unzip_file", args: { name: frm.doc.name, }, diff --git a/frappe/core/doctype/file/file.py b/frappe/core/doctype/file/file.py index 3352123146..e8b8da76ab 100755 --- a/frappe/core/doctype/file/file.py +++ b/frappe/core/doctype/file/file.py @@ -1,77 +1,49 @@ # Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -""" -record of files - -naming for same name files: file.gif, file-1.gif, file-2.gif etc -""" - -import hashlib -import imghdr import io -import json import mimetypes import os import re import shutil import zipfile -from io import BytesIO -from typing import TYPE_CHECKING, Tuple +from typing import List, Optional, Union from urllib.parse import quote, unquote -import requests from PIL import Image, ImageFile, ImageOps from requests.exceptions import HTTPError, SSLError import frappe -from frappe import _, conf, safe_decode +from frappe import _ from frappe.model.document import Document -from frappe.utils import ( - call_hook_method, - cint, - cstr, - encode, - get_files_path, - get_hook_method, - random_string, - strip, -) -from frappe.utils.file_manager import safe_b64decode +from frappe.utils import call_hook_method, cint, get_files_path, get_hook_method +from frappe.utils.file_manager import is_safe_path from frappe.utils.image import optimize_image, strip_exif_data -if TYPE_CHECKING: - from requests.models import Response - - -class MaxFileSizeReachedError(frappe.ValidationError): - pass - - -class FolderNotEmpty(frappe.ValidationError): - pass - +from .exceptions import AttachmentLimitReached, FolderNotEmpty, MaxFileSizeReachedError +from .utils import * exclude_from_linked_with = True ImageFile.LOAD_TRUNCATED_IMAGES = True +URL_PREFIXES = ("http://", "https://") class File(Document): no_feed_on_delete = True - def before_insert(self): - frappe.local.rollback_observers.append(self) - self.set_folder_name() - if self.file_name: - self.file_name = re.sub(r"/", "", self.file_name) - self.content = self.get("content", None) - self.decode = self.get("decode", False) - if self.content: - self.save_file(content=self.content, decode=self.decode) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # if content is set, file_url will be generated + # decode comes in the picture if content passed has to be decoded before writing to disk - def get_name_based_on_parent_folder(self): - if self.folder: - return "/".join([self.folder, self.file_name]) + self.content = self.get("content") or b"" + self.decode = self.get("decode", False) + + @property + def is_remote_file(self): + if self.file_url: + return self.file_url.startswith(URL_PREFIXES) + return not self.content def autoname(self): """Set name for folder""" @@ -82,85 +54,154 @@ class File(Document): # home self.name = self.file_name else: - self.name = frappe.generate_hash("", 10) + self.name = frappe.generate_hash(length=10) + + def before_insert(self): + self.set_folder_name() + self.set_file_name() + self.validate_attachment_limit() + + if not self.is_folder and not self.is_remote_file: + self.save_file(content=self.get_content()) + self.flags.new_file = True + frappe.local.rollback_observers.append(self) def after_insert(self): if not self.is_folder: - self.add_comment_in_reference_doc( - "Attachment", - _("Added {0}").format( - "{file_name}{icon}".format( - **{ - "icon": ' ' if self.is_private else "", - "file_url": quote(frappe.safe_encode(self.file_url)) if self.file_url else self.file_name, - "file_name": self.file_name or self.file_url, - } - ) - ), - ) - - def after_rename(self, olddn, newdn, merge=False): - for successor in self.get_successor(): - setup_folder_path(successor[0], self.name) - - def get_successor(self): - return frappe.db.get_values(doctype="File", filters={"folder": self.name}, fieldname="name") + self.create_attachment_record() + self.set_is_private() + self.set_file_name() + self.validate_duplicate_entry() def validate(self): - if self.is_new(): - self.set_is_private() - self.set_file_name() - self.validate_duplicate_entry() - self.validate_attachment_limit() + # Ensure correct formatting and type + self.file_url = unquote(self.file_url) if self.file_url else "" - self.validate_folder() + # when dict is passed to get_doc for creation of new_doc, is_new returns None + # this case is handled inside handle_is_private_changed + if not self.is_new() and self.has_value_changed("is_private"): + self.handle_is_private_changed() - if self.is_folder: - self.file_url = "" - else: - self.validate_url() + if not self.is_folder: + self.validate_file_path() + self.validate_file_url() + self.validate_file_on_disk() self.file_size = frappe.form_dict.file_size or self.file_size - def validate_url(self): - if not self.file_url or self.file_url.startswith(("http://", "https://")): - if not self.flags.ignore_file_validate: - self.validate_file() + def after_rename(self, *args, **kwargs): + for successor in self.get_successors(): + setup_folder_path(successor, self.name) + def on_trash(self): + if self.is_home_folder or self.is_attachments_folder: + frappe.throw(_("Cannot delete Home and Attachments folders")) + self.validate_empty_folder() + self._delete_file_on_disk() + if not self.is_folder: + self.add_comment_in_reference_doc("Attachment Removed", _("Removed {0}").format(self.file_name)) + + def on_rollback(self): + # following condition is only executed when an insert has been rolledback + if self.flags.new_file: + self._delete_file_on_disk() + self.flags.pop("new_file") return - # Probably an invalid web URL - if not self.file_url.startswith(("/files/", "/private/files/")): - frappe.throw(_("URL must start with http:// or https://"), title=_("Invalid URL")) + # if original_content flag is set, this rollback should revert the file to its original state + if self.flags.original_content: + file_path = self.get_full_path() - # Ensure correct formatting and type - self.file_url = unquote(self.file_url) - self.is_private = cint(self.is_private) + if isinstance(self.flags.original_content, bytes): + mode = "wb+" + elif isinstance(self.flags.original_content, str): + mode = "w+" - self.handle_is_private_changed() + with open(file_path, mode) as f: + f.write(self.flags.original_content) + os.fsync(f.fileno()) + self.flags.pop("original_content") + + # used in case file path (File.file_url) has been changed + if self.flags.original_path: + target = self.flags.original_path["old"] + source = self.flags.original_path["new"] + shutil.move(source, target) + self.flags.pop("original_path") + + def get_name_based_on_parent_folder(self) -> Union[str, None]: + if self.folder: + return os.path.join(self.folder, self.file_name) + + def get_successors(self): + return frappe.get_all("File", filters={"folder": self.name}, pluck="name") + + def validate_file_path(self): + if self.is_remote_file: + return base_path = os.path.realpath(get_files_path(is_private=self.is_private)) if not os.path.realpath(self.get_full_path()).startswith(base_path): - frappe.throw(_("The File URL you've entered is incorrect"), title=_("Invalid File URL")) + frappe.throw( + _("The File URL you've entered is incorrect"), + title=_("Invalid File URL"), + ) - def handle_is_private_changed(self): - if not frappe.db.exists("File", {"name": self.name, "is_private": cint(not self.is_private)}): + def validate_file_url(self): + if self.is_remote_file or not self.file_url: return + if not self.file_url.startswith(("/files/", "/private/files/")): + # Probably an invalid URL since it doesn't start with http either + frappe.throw( + _("URL must start with http:// or https://"), + title=_("Invalid URL"), + ) + + def handle_is_private_changed(self): + if self.is_remote_file: + return + + from pathlib import Path + old_file_url = self.file_url - file_name = self.file_url.split("/")[-1] - private_file_path = frappe.get_site_path("private", "files", file_name) - public_file_path = frappe.get_site_path("public", "files", file_name) + private_file_path = Path(frappe.get_site_path("private", "files", file_name)) + public_file_path = Path(frappe.get_site_path("public", "files", file_name)) - if self.is_private: - shutil.move(public_file_path, private_file_path) + if cint(self.is_private): + source = public_file_path + target = private_file_path url_starts_with = "/private/files/" else: - shutil.move(private_file_path, public_file_path) + source = private_file_path + target = public_file_path url_starts_with = "/files/" + updated_file_url = f"{url_starts_with}{file_name}" - self.file_url = "{0}{1}".format(url_starts_with, file_name) + # if a file document is created by passing dict throught get_doc and __local is not set, + # handle_is_private_changed would be executed; we're checking if updated_file_url is same + # as old_file_url to avoid a FileNotFoundError for this case. + if updated_file_url == old_file_url: + return + + if not source.exists(): + frappe.throw( + _("Cannot find file {} on disk").format(source), + exc=FileNotFoundError, + ) + if target.exists(): + frappe.throw( + _("A file with same name {} already exists").format(target), + exc=FileExistsError, + ) + + # Uses os.rename which is an atomic operation + shutil.move(source, target) + self.flags.original_path = {"old": source, "new": target} + frappe.local.rollback_observers.append(self) + + self.file_url = updated_file_url update_existing_file_docs(self) if ( @@ -171,7 +212,10 @@ class File(Document): return frappe.db.set_value( - self.attached_to_doctype, self.attached_to_name, self.attached_to_field, self.file_url + self.attached_to_doctype, + self.attached_to_name, + self.attached_to_field, + self.file_url, ) def fetch_attached_to_field(self, old_file_url): @@ -207,26 +251,26 @@ class File(Document): _("Maximum Attachment Limit of {0} has been reached for {1} {2}.").format( frappe.bold(attachment_limit), self.attached_to_doctype, self.attached_to_name ), - exc=frappe.exceptions.AttachmentLimitReached, + exc=AttachmentLimitReached, title=_("Attachment Limit Reached"), ) def set_folder_name(self): """Make parent folders if not exists based on reference doctype and name""" - if self.attached_to_doctype and not self.folder: + if self.folder: + return + + if self.attached_to_doctype: self.folder = frappe.db.get_value("File", {"is_attachments_folder": 1}) - def validate_folder(self): - if not self.is_home_folder and not self.folder and not self.flags.ignore_folder_validate: + elif not self.is_home_folder: self.folder = "Home" - def validate_file(self): - """Validates existence of public file - TODO: validate for private file - """ + def validate_file_on_disk(self): + """Validates existence file""" full_path = self.get_full_path() - if full_path.startswith("http"): + if full_path.startswith(URL_PREFIXES): return True if not os.path.exists(full_path): @@ -246,7 +290,10 @@ class File(Document): } if self.attached_to_doctype and self.attached_to_name: filters.update( - {"attached_to_doctype": self.attached_to_doctype, "attached_to_name": self.attached_to_name} + { + "attached_to_doctype": self.attached_to_doctype, + "attached_to_name": self.attached_to_name, + } ) duplicate_file = frappe.db.get_value("File", filters, ["name", "file_url"], as_dict=1) @@ -263,7 +310,7 @@ class File(Document): self.file_name = re.sub(r"/", "", self.file_name) def generate_content_hash(self): - if self.content_hash or not self.file_url or self.file_url.startswith("http"): + if self.content_hash or not self.file_url or self.is_remote_file: return file_name = self.file_url.split("/")[-1] try: @@ -273,79 +320,63 @@ class File(Document): except IOError: frappe.throw(_("File {0} does not exist").format(file_path)) - def on_trash(self): - if self.is_home_folder or self.is_attachments_folder: - frappe.throw(_("Cannot delete Home and Attachments folders")) - self.check_folder_is_empty() - self.call_delete_file() - if not self.is_folder: - self.add_comment_in_reference_doc("Attachment Removed", _("Removed {0}").format(self.file_name)) - def make_thumbnail( - self, set_as_thumbnail=True, width=300, height=300, suffix="small", crop=False - ): - if self.file_url: - try: - if self.file_url.startswith(("/files", "/private/files")): - image, filename, extn = get_local_image(self.file_url) - else: - image, filename, extn = get_web_image(self.file_url) - except (HTTPError, SSLError, IOError, TypeError): - return + self, + set_as_thumbnail: bool = True, + width: int = 300, + height: int = 300, + suffix: str = "small", + crop: bool = False, + ) -> str: + if not self.file_url: + return - size = width, height - if crop: - image = ImageOps.fit(image, size, Image.ANTIALIAS) + try: + if self.file_url.startswith(("/files", "/private/files")): + image, filename, extn = get_local_image(self.file_url) else: - image.thumbnail(size, Image.ANTIALIAS) + image, filename, extn = get_web_image(self.file_url) + except (HTTPError, SSLError, IOError, TypeError): + return - thumbnail_url = filename + "_" + suffix + "." + extn - path = os.path.abspath(frappe.get_site_path("public", thumbnail_url.lstrip("/"))) + size = width, height + if crop: + image = ImageOps.fit(image, size, Image.Resampling.LANCZOS) + else: + image.thumbnail(size, Image.Resampling.LANCZOS) - try: - image.save(path) - if set_as_thumbnail: - self.db_set("thumbnail_url", thumbnail_url) + thumbnail_url = f"{filename}_{suffix}.{extn}" + path = os.path.abspath(frappe.get_site_path("public", thumbnail_url.lstrip("/"))) - except IOError: - frappe.msgprint(_("Unable to write file format for {0}").format(path)) - return + try: + image.save(path) + if set_as_thumbnail: + self.db_set("thumbnail_url", thumbnail_url) - return thumbnail_url + except IOError: + frappe.msgprint(_("Unable to write file format for {0}").format(path)) + return - def check_folder_is_empty(self): + return thumbnail_url + + def validate_empty_folder(self): """Throw exception if folder is not empty""" - files = frappe.get_all("File", filters={"folder": self.name}, fields=("name", "file_name")) - - if self.is_folder and files: + if self.is_folder and frappe.get_all("File", filters={"folder": self.name}, limit=1): frappe.throw(_("Folder {0} is not empty").format(self.name), FolderNotEmpty) - def call_delete_file(self): + def _delete_file_on_disk(self): """If file not attached to any other record, delete it""" - if ( - self.file_name - and self.content_hash - and ( - not frappe.db.count("File", {"content_hash": self.content_hash, "name": ["!=", self.name]}) - ) - ): + on_disk_file_not_shared = self.content_hash and not frappe.get_all( + "File", + filters={"content_hash": self.content_hash, "name": ["!=", self.name]}, + limit=1, + ) + if on_disk_file_not_shared: self.delete_file_data_content() - elif self.file_url: + else: self.delete_file_data_content(only_thumbnail=True) - def on_rollback(self): - # if original_content flag is set, this rollback should revert the file to its original state - if self.flags.original_content: - file_path = self.get_full_path() - with open(file_path, "wb+") as f: - f.write(self.flags.original_content) - - # following condition is only executed when an insert has been rolledback - else: - self.flags.on_rollback = True - self.on_trash() - - def unzip(self): + def unzip(self) -> List["File"]: """Unzip current file and replace it by its children""" if not self.file_url.endswith(".zip"): frappe.throw(_("{0} is not a zip file").format(self.file_name)) @@ -378,31 +409,36 @@ class File(Document): return files def exists_on_disk(self): - exists = os.path.exists(self.get_full_path()) - return exists + return os.path.exists(self.get_full_path()) - def get_content(self): + def get_content(self) -> bytes: """Returns [`file_name`, `content`] for given file name `fname`""" if self.is_folder: frappe.throw(_("Cannot get file contents of a Folder")) if self.get("content"): - return self.content + self._content = self.content + if self.decode: + self._content = decode_file_content(self._content) + self.decode = False + # self.content = None # TODO: This needs to happen; make it happen somehow + return self._content - self.validate_url() + if self.file_url: + self.validate_file_url() file_path = self.get_full_path() # read the file - with io.open(encode(file_path), mode="rb") as f: - content = f.read() + with open(file_path, mode="rb") as f: + self._content = f.read() try: # for plain text files - content = content.decode() + self._content = self._content.decode() except UnicodeDecodeError: # for .png, .jpg, etc pass - return content + return self._content def get_full_path(self): """Returns file path from given file name""" @@ -410,7 +446,10 @@ class File(Document): file_path = self.file_url or self.file_name if "/" not in file_path: - file_path = "/files/" + file_path + if self.is_private: + file_path = f"/private/files/{file_path}" + else: + file_path = f"/files/{file_path}" if file_path.startswith("/private/files/"): file_path = get_files_path(*file_path.split("/private/files/", 1)[1].split("/"), is_private=1) @@ -418,61 +457,75 @@ class File(Document): elif file_path.startswith("/files/"): file_path = get_files_path(*file_path.split("/files/", 1)[1].split("/")) - elif file_path.startswith("http"): + elif file_path.startswith(URL_PREFIXES): pass elif not self.file_url: frappe.throw(_("There is some problem with the file url: {0}").format(file_path)) - return file_path - - def write_file(self): - """write file to disk with a random name (to compare)""" - file_path = get_files_path(is_private=self.is_private) + if not is_safe_path(file_path): + frappe.throw(_("Cannot access file path {0}").format(file_path)) if os.path.sep in self.file_name: frappe.throw(_("File name cannot have {0}").format(os.path.sep)) - # create directory (if not exists) - frappe.create_folder(file_path) - # write the file - self.content = self.get_content() - if isinstance(self.content, str): - self.content = self.content.encode() - with open(os.path.join(file_path.encode("utf-8"), self.file_name.encode("utf-8")), "wb+") as f: - f.write(self.content) + return file_path - return get_files_path(self.file_name, is_private=self.is_private) + def write_file(self): + """write file to disk with a random name (to compare)""" + if self.is_remote_file: + return + + file_path = self.get_full_path() + + if isinstance(self._content, str): + self._content = self._content.encode() + + with open(file_path, "wb+") as f: + f.write(self._content) + os.fsync(f.fileno()) + + frappe.local.rollback_observers.append(self) + + return file_path + + def save_file( + self, + content: Optional[Union[bytes, str]] = None, + decode=False, + ignore_existing_file_check=False, + overwrite=False, + ): + if self.is_remote_file: + return + + if not self.flags.new_file: + self.flags.original_content = self.get_content() + + if content: + self.content = content + self.decode = decode + self.get_content() + + if not self._content: + return - def save_file(self, content=None, decode=False, ignore_existing_file_check=False): file_exists = False - self.content = content - - if decode: - if isinstance(content, str): - self.content = content.encode("utf-8") - - if b"," in self.content: - self.content = self.content.split(b",")[1] - self.content = safe_b64decode(self.content) - - if not self.is_private: - self.is_private = 0 + duplicate_file = None + self.is_private = cint(self.is_private) self.content_type = mimetypes.guess_type(self.file_name)[0] - self.file_size = self.check_max_file_size() - + # transform file content based on site settings if ( self.content_type and self.content_type == "image/jpeg" and frappe.get_system_settings("strip_exif_metadata_from_uploaded_images") ): - self.content = strip_exif_data(self.content, self.content_type) + self._content = strip_exif_data(self._content, self.content_type) - self.content_hash = get_content_hash(self.content) - - duplicate_file = None + self.file_size = self.check_max_file_size() + self.content_hash = get_content_hash(self._content) # check if a file exists with the same content hash and is also in the same folder (public or private) if not ignore_existing_file_check: @@ -484,15 +537,18 @@ class File(Document): ) if duplicate_file: - file_doc = frappe.get_cached_doc("File", duplicate_file.name) + file_doc: "File" = frappe.get_cached_doc("File", duplicate_file.name) if file_doc.exists_on_disk(): self.file_url = duplicate_file.file_url file_exists = True - if os.path.exists(encode(get_files_path(self.file_name, is_private=self.is_private))): - self.file_name = get_file_name(self.file_name, self.content_hash[-6:]) - if not file_exists: + if not overwrite: + self.file_name = generate_file_name( + name=self.file_name, + suffix=self.content_hash[-6:], + is_private=self.is_private, + ) call_hook_method("before_write_file", file_size=self.file_size) write_file_method = get_hook_method("write_file") if write_file_method: @@ -500,23 +556,25 @@ class File(Document): return self.save_file_on_filesystem() def save_file_on_filesystem(self): - fpath = self.write_file() - if self.is_private: - self.file_url = "/private/files/{0}".format(self.file_name) + self.file_url = f"/private/files/{self.file_name}" else: - self.file_url = "/files/{0}".format(self.file_name) + self.file_url = f"/files/{self.file_name}" + + fpath = self.write_file() return {"file_name": os.path.basename(fpath), "file_url": self.file_url} def check_max_file_size(self): + from frappe.core.api.file import get_max_file_size + max_file_size = get_max_file_size() - file_size = len(self.content) + file_size = len(self._content or b"") if file_size > max_file_size: - frappe.msgprint( + frappe.throw( _("File size exceeded the maximum allowed size of {0} MB").format(max_file_size / 1048576), - raise_exception=MaxFileSizeReachedError, + exc=MaxFileSizeReachedError, ) return file_size @@ -543,6 +601,16 @@ class File(Document): """returns split filename and extension""" return os.path.splitext(self.file_name) + def create_attachment_record(self): + icon = ' ' if self.is_private else "" + file_url = quote(frappe.safe_encode(self.file_url)) if self.file_url else self.file_name + file_name = self.file_name or self.file_url + + self.add_comment_in_reference_doc( + "Attachment", + _("Added {0}").format(f"{file_name}{icon}"), + ) + def add_comment_in_reference_doc(self, comment_type, text): if self.attached_to_doctype and self.attached_to_name: try: @@ -570,18 +638,13 @@ class File(Document): if is_svg: raise TypeError("Optimization of SVG images is not supported") - content = self.get_content() - file_path = self.get_full_path() - optimized_content = optimize_image(content, content_type) + original_content = self.get_content() + optimized_content = optimize_image( + content=original_content, + content_type=content_type, + ) - with open(file_path, "wb+") as f: - f.write(optimized_content) - - self.file_size = len(optimized_content) - self.content_hash = get_content_hash(optimized_content) - # if rolledback, revert back to original - self.flags.original_content = content - frappe.local.rollback_observers.append(self) + self.save_file(content=optimized_content, overwrite=True) self.save() @staticmethod @@ -604,182 +667,6 @@ def on_doctype_update(): frappe.db.add_index("File", ["attached_to_doctype", "attached_to_name"]) -def make_home_folder(): - home = frappe.get_doc( - {"doctype": "File", "is_folder": 1, "is_home_folder": 1, "file_name": _("Home")} - ).insert(ignore_if_duplicate=True) - - frappe.get_doc( - { - "doctype": "File", - "folder": home.name, - "is_folder": 1, - "is_attachments_folder": 1, - "file_name": _("Attachments"), - } - ).insert(ignore_if_duplicate=True) - - -@frappe.whitelist() -def create_new_folder(file_name, folder): - """create new folder under current parent folder""" - file = frappe.new_doc("File") - file.file_name = file_name - file.is_folder = 1 - file.folder = folder - file.insert(ignore_if_duplicate=True) - return file - - -@frappe.whitelist() -def move_file(file_list, new_parent, old_parent): - - if isinstance(file_list, str): - file_list = json.loads(file_list) - - for file_obj in file_list: - setup_folder_path(file_obj.get("name"), new_parent) - - # recalculate sizes - frappe.get_doc("File", old_parent).save() - frappe.get_doc("File", new_parent).save() - - -@frappe.whitelist() -def zip_files(files): - files = frappe.parse_json(files) - zipped_files = File.zip_files(files) - frappe.response["filename"] = "files.zip" - frappe.response["filecontent"] = zipped_files - frappe.response["type"] = "download" - - -def setup_folder_path(filename, new_parent): - file = frappe.get_doc("File", filename) - file.folder = new_parent - file.save() - - if file.is_folder: - from frappe.model.rename_doc import rename_doc - - rename_doc("File", file.name, file.get_name_based_on_parent_folder(), ignore_permissions=True) - - -def get_extension(filename, extn, content: bytes = None, response: "Response" = None) -> str: - mimetype = None - - if response: - content_type = response.headers.get("Content-Type") - - if content_type: - _extn = mimetypes.guess_extension(content_type) - if _extn: - return _extn[1:] - - if extn: - # remove '?' char and parameters from extn if present - if "?" in extn: - extn = extn.split("?", 1)[0] - - mimetype = mimetypes.guess_type(filename + "." + extn)[0] - - if mimetype is None or not mimetype.startswith("image/") and content: - # detect file extension by reading image header properties - extn = imghdr.what(filename + "." + (extn or ""), h=content) - - return extn - - -def get_local_image(file_url): - if file_url.startswith("/private"): - file_url_path = (file_url.lstrip("/"),) - else: - file_url_path = ("public", file_url.lstrip("/")) - - file_path = frappe.get_site_path(*file_url_path) - - try: - image = Image.open(file_path) - except IOError: - frappe.throw(_("Unable to read file format for {0}").format(file_url)) - - content = None - - try: - filename, extn = file_url.rsplit(".", 1) - except ValueError: - # no extn - with open(file_path, "r") as f: - content = f.read() - - filename = file_url - extn = None - - extn = get_extension(filename, extn, content) - - return image, filename, extn - - -def get_web_image(file_url: str) -> Tuple["ImageFile", str, str]: - # download - file_url = frappe.utils.get_url(file_url) - r = requests.get(file_url, stream=True) - try: - r.raise_for_status() - except HTTPError: - if r.status_code == 404: - frappe.msgprint(_("File '{0}' not found").format(file_url)) - else: - frappe.msgprint(_("Unable to read file format for {0}").format(file_url)) - raise - - try: - image = Image.open(BytesIO(r.content)) - except Exception as e: - frappe.msgprint(_("Image link '{0}' is not valid").format(file_url), raise_exception=e) - - try: - filename, extn = file_url.rsplit("/", 1)[1].rsplit(".", 1) - except ValueError: - # the case when the file url doesn't have filename or extension - # but is fetched due to a query string. example: https://encrypted-tbn3.gstatic.com/images?q=something - filename = get_random_filename() - extn = None - - extn = get_extension(filename, extn, response=r) - if extn == "bin": - extn = get_extension(filename, extn, content=r.content) or "png" - - filename = "/files/" + strip(unquote(filename)) - - return image, filename, extn - - -def delete_file(path): - """Delete file from `public folder`""" - if path: - if ".." in path.split("/"): - frappe.throw( - _("It is risky to delete this file: {0}. Please contact your System Manager.").format(path) - ) - - parts = os.path.split(path.strip("/")) - if parts[0] == "files": - path = frappe.utils.get_site_path("public", "files", parts[-1]) - - else: - path = frappe.utils.get_site_path("private", "files", parts[-1]) - - path = encode(path) - if os.path.exists(path): - os.remove(path) - - -@frappe.whitelist() -def get_max_file_size(): - return cint(conf.get("max_file_size")) or 10485760 - - def has_permission(doc, ptype=None, user=None): has_access = False user = user or frappe.session.user @@ -817,238 +704,5 @@ def has_permission(doc, ptype=None, user=None): return has_access -def remove_file_by_url(file_url, doctype=None, name=None): - if doctype and name: - fid = frappe.db.get_value( - "File", {"file_url": file_url, "attached_to_doctype": doctype, "attached_to_name": name} - ) - else: - fid = frappe.db.get_value("File", {"file_url": file_url}) - - if fid: - from frappe.utils.file_manager import remove_file - - return remove_file(fid=fid) - - -def get_content_hash(content): - if isinstance(content, str): - content = content.encode() - return hashlib.md5(content).hexdigest() # nosec - - -def get_file_name(fname, optional_suffix): - # convert to unicode - fname = cstr(fname) - - f = fname.rsplit(".", 1) - if len(f) == 1: - partial, extn = f[0], "" - else: - partial, extn = f[0], "." + f[1] - return "{partial}{suffix}{extn}".format(partial=partial, extn=extn, suffix=optional_suffix) - - -@frappe.whitelist() -def download_file(file_url): - """ - Download file using token and REST API. Valid session or - token is required to download private files. - - Method : GET - Endpoint : frappe.core.doctype.file.file.download_file - URL Params : file_name = /path/to/file relative to site path - """ - file_doc = frappe.get_doc("File", {"file_url": file_url}) - file_doc.check_permission("read") - - frappe.local.response.filename = os.path.basename(file_url) - frappe.local.response.filecontent = file_doc.get_content() - frappe.local.response.type = "download" - - -def extract_images_from_doc(doc, fieldname): - content = doc.get(fieldname) - content = extract_images_from_html(doc, content) - if frappe.flags.has_dataurl: - doc.set(fieldname, content) - - -def extract_images_from_html(doc, content, is_private=False): - frappe.flags.has_dataurl = False - - def _save_file(match): - data = match.group(1) - data = data.split("data:")[1] - headers, content = data.split(",") - mtype = headers.split(";")[0] - - if isinstance(content, str): - content = content.encode("utf-8") - if b"," in content: - content = content.split(b",")[1] - content = safe_b64decode(content) - - content = optimize_image(content, mtype) - - if "filename=" in headers: - filename = headers.split("filename=")[-1] - filename = safe_decode(filename).split(";")[0] - - else: - filename = get_random_filename(content_type=mtype) - - # attaching a file to a child table doc, attaches it to the parent doc - doctype = doc.parenttype if doc.get("parent") else doc.doctype - name = doc.get("parent") or doc.name - - _file = frappe.get_doc( - { - "doctype": "File", - "file_name": filename, - "attached_to_doctype": doctype, - "attached_to_name": name, - "content": content, - "decode": False, - "is_private": is_private, - } - ) - _file.save(ignore_permissions=True) - file_url = _file.file_url - if not frappe.flags.has_dataurl: - frappe.flags.has_dataurl = True - - return ']*src\s*=\s*["\'](?=data:)(.*?)["\']', _save_file, content) - - return content - - -def get_random_filename(content_type=None): - extn = None - if content_type: - extn = mimetypes.guess_extension(content_type) - - return random_string(7) + (extn or "") - - -@frappe.whitelist() -def unzip_file(name): - """Unzip the given file and make file records for each of the extracted files""" - file_obj = frappe.get_doc("File", name) - files = file_obj.unzip() - return files - - -@frappe.whitelist() -def get_attached_images(doctype, names): - """get list of image urls attached in form - returns {name: ['image.jpg', 'image.png']}""" - - if isinstance(names, str): - names = json.loads(names) - - img_urls = frappe.db.get_list( - "File", - filters={"attached_to_doctype": doctype, "attached_to_name": ("in", names), "is_folder": 0}, - fields=["file_url", "attached_to_name as docname"], - ) - - out = frappe._dict() - for i in img_urls: - out[i.docname] = out.get(i.docname, []) - out[i.docname].append(i.file_url) - - return out - - -@frappe.whitelist() -def get_files_in_folder(folder, start=0, page_length=20): - start = cint(start) - page_length = cint(page_length) - - attachment_folder = frappe.db.get_value( - "File", "Home/Attachments", ["name", "file_name", "file_url", "is_folder", "modified"], as_dict=1 - ) - - files = frappe.db.get_list( - "File", - {"folder": folder}, - ["name", "file_name", "file_url", "is_folder", "modified"], - start=start, - page_length=page_length + 1, - ) - - if folder == "Home" and attachment_folder not in files: - files.insert(0, attachment_folder) - - return {"files": files[:page_length], "has_more": len(files) > page_length} - - -@frappe.whitelist() -def get_files_by_search_text(text): - if not text: - return [] - - text = "%" + cstr(text).lower() + "%" - return frappe.db.get_all( - "File", - fields=["name", "file_name", "file_url", "is_folder", "modified"], - filters={"is_folder": False}, - or_filters={"file_name": ("like", text), "file_url": text, "name": ("like", text)}, - order_by="modified desc", - limit=20, - ) - - -def update_existing_file_docs(doc): - # Update is private and file url of all file docs that point to the same file - file_doctype = frappe.qb.DocType("File") - ( - frappe.qb.update(file_doctype) - .set(file_doctype.file_url, doc.file_url) - .set(file_doctype.is_private, doc.is_private) - .where(file_doctype.content_hash == doc.content_hash) - .where(file_doctype.name != doc.name) - ).run() - - -def attach_files_to_document(doc, event): - """Runs on on_update hook of all documents. - Goes through every Attach and Attach Image field and attaches - the file url to the document if it is not already attached. - """ - - attach_fields = doc.meta.get("fields", {"fieldtype": ["in", ["Attach", "Attach Image"]]}) - - for df in attach_fields: - # this method runs in on_update hook of all documents - # we dont want the update to fail if file cannot be attached for some reason - try: - value = doc.get(df.fieldname) - if not (value or "").startswith(("/files", "/private/files")): - return - - if frappe.db.exists( - "File", - { - "file_url": value, - "attached_to_name": doc.name, - "attached_to_doctype": doc.doctype, - "attached_to_field": df.fieldname, - }, - ): - return - - file_doc = frappe.get_doc( - doctype="File", - file_url=value, - attached_to_name=doc.name, - attached_to_doctype=doc.doctype, - attached_to_field=df.fieldname, - folder="Home/Attachments", - ).insert() - except Exception: - file_doc.log_error("Error Attaching File") +# Note: kept at the end to not cause circular, partial imports & maintain backwards compatibility +from frappe.core.api.file import * diff --git a/frappe/core/doctype/file/test_file.py b/frappe/core/doctype/file/test_file.py index b02bb581ab..a9d40b35b1 100644 --- a/frappe/core/doctype/file/test_file.py +++ b/frappe/core/doctype/file/test_file.py @@ -4,18 +4,25 @@ import base64 import json import os import unittest +from contextlib import contextmanager +from typing import TYPE_CHECKING import frappe from frappe import _ -from frappe.core.doctype.file.file import ( - File, +from frappe.core.api.file import ( + create_new_folder, get_attached_images, get_files_in_folder, move_file, unzip_file, ) +from frappe.exceptions import ValidationError +from frappe.tests.utils import FrappeTestCase from frappe.utils import get_files_path +if TYPE_CHECKING: + from frappe.core.doctype.file.file import File + test_content1 = "Hello" test_content2 = "Hello World" @@ -28,7 +35,25 @@ def make_test_doc(): return d.doctype, d.name -class TestSimpleFile(unittest.TestCase): +@contextmanager +def make_test_image_file(): + file_path = frappe.get_app_path("frappe", "tests/data/sample_image_for_optimization.jpg") + with open(file_path, "rb") as f: + file_content = f.read() + + test_file = frappe.get_doc( + {"doctype": "File", "file_name": "sample_image_for_optimization.jpg", "content": file_content} + ).insert() + # remove those flags + _test_file: "File" = frappe.get_doc("File", test_file.name) + + try: + yield _test_file + finally: + _test_file.delete() + + +class TestSimpleFile(FrappeTestCase): def setUp(self): self.attached_to_doctype, self.attached_to_docname = make_test_doc() self.test_content = test_content1 @@ -50,11 +75,11 @@ class TestSimpleFile(unittest.TestCase): self.assertEqual(content, self.test_content) -class TestBase64File(unittest.TestCase): +class TestBase64File(FrappeTestCase): def setUp(self): self.attached_to_doctype, self.attached_to_docname = make_test_doc() self.test_content = base64.b64encode(test_content1.encode("utf-8")) - _file = frappe.get_doc( + _file: "File" = frappe.get_doc( { "doctype": "File", "file_name": "test_base64.txt", @@ -73,7 +98,7 @@ class TestBase64File(unittest.TestCase): self.assertEqual(content, test_content1) -class TestSameFileName(unittest.TestCase): +class TestSameFileName(FrappeTestCase): def test_saved_content(self): self.attached_to_doctype, self.attached_to_docname = make_test_doc() self.test_content1 = test_content1 @@ -133,7 +158,7 @@ class TestSameFileName(unittest.TestCase): self.assertEqual(_file.get_content(), test_content2) -class TestSameContent(unittest.TestCase): +class TestSameContent(FrappeTestCase): def setUp(self): self.attached_to_doctype1, self.attached_to_docname1 = make_test_doc() self.attached_to_doctype2, self.attached_to_docname2 = make_test_doc() @@ -201,7 +226,7 @@ class TestSameContent(unittest.TestCase): frappe.clear_cache(doctype="ToDo") -class TestFile(unittest.TestCase): +class TestFile(FrappeTestCase): def setUp(self): frappe.set_user("Administrator") self.delete_test_data() @@ -327,7 +352,7 @@ class TestFile(unittest.TestCase): _file.save() folder = frappe.get_doc("File", "Home/Test Folder 1/Test Folder 3") - self.assertRaises(frappe.ValidationError, folder.delete) + self.assertRaises(ValidationError, folder.delete) def test_same_file_url_update(self): attached_to_doctype1, attached_to_docname1 = make_test_doc() @@ -373,38 +398,35 @@ class TestFile(unittest.TestCase): { "doctype": "File", "file_name": "parent_dir.txt", - "attached_to_doctype": "", - "attached_to_name": "", "is_private": 1, "content": test_content1, } ).insert() file1.file_url = "/private/files/../test.txt" - self.assertRaises(frappe.exceptions.ValidationError, file1.save) + self.assertRaises(ValidationError, file1.save) # No validation to see if file exists file1.reload() file1.file_url = "/private/files/parent_dir2.txt" - file1.save() + self.assertRaises(OSError, file1.save) def test_file_url_validation(self): - test_file = frappe.get_doc( - {"doctype": "File", "file_name": "logo", "file_url": "https://frappe.io/files/frappe.png"} - ) + test_file: "File" = frappe.new_doc("File") + test_file.update({"file_name": "logo", "file_url": "https://frappe.io/files/frappe.png"}) self.assertIsNone(test_file.validate()) # bad path test_file.file_url = "/usr/bin/man" self.assertRaisesRegex( - frappe.exceptions.ValidationError, "URL must start with http:// or https://", test_file.validate + ValidationError, f"Cannot access file path {test_file.file_url}", test_file.validate ) test_file.file_url = None test_file.file_name = "/usr/bin/man" self.assertRaisesRegex( - frappe.exceptions.ValidationError, "There is some problem with the file url", test_file.validate + ValidationError, "There is some problem with the file url", test_file.validate ) test_file.file_url = None @@ -413,11 +435,11 @@ class TestFile(unittest.TestCase): test_file.file_url = None test_file.file_name = "/private/files/_file" - self.assertRaisesRegex(IOError, "does not exist", test_file.validate) + self.assertRaisesRegex(ValidationError, "File name cannot have", test_file.validate) def test_make_thumbnail(self): # test web image - test_file: File = frappe.get_doc( + test_file: "File" = frappe.get_doc( { "doctype": "File", "file_name": "logo", @@ -486,37 +508,36 @@ class TestFile(unittest.TestCase): "file_url": frappe.utils.get_url("/_test/assets/image.jpg"), } ).insert(ignore_permissions=True) - self.assertRaisesRegex(frappe.exceptions.ValidationError, "not a zip file", test_file.unzip) + self.assertRaisesRegex(ValidationError, "not a zip file", test_file.unzip) class TestAttachment(unittest.TestCase): test_doctype = "Test For Attachment" - def setUp(self): - if frappe.db.exists("DocType", self.test_doctype): - return - + @classmethod + def setUpClass(cls): frappe.get_doc( doctype="DocType", - name=self.test_doctype, + name=cls.test_doctype, module="Custom", custom=1, fields=[ {"label": "Title", "fieldname": "title", "fieldtype": "Data"}, {"label": "Attachment", "fieldname": "attachment", "fieldtype": "Attach"}, ], - ).insert() + ).insert(ignore_if_duplicate=True) - def tearDown(self): - frappe.delete_doc("DocType", self.test_doctype) + @classmethod + def tearDownClass(cls): + frappe.db.rollback() + frappe.delete_doc("DocType", cls.test_doctype) def test_file_attachment_on_update(self): doc = frappe.get_doc(doctype=self.test_doctype, title="test for attachment on update").insert() file = frappe.get_doc( {"doctype": "File", "file_name": "test_attach.txt", "content": "Test Content"} - ) - file.save() + ).save() doc.attachment = file.file_url doc.save() @@ -535,9 +556,11 @@ class TestAttachment(unittest.TestCase): self.assertTrue(exists) -class TestAttachmentsAccess(unittest.TestCase): - def test_attachments_access(self): +class TestAttachmentsAccess(FrappeTestCase): + def setUp(self) -> None: + frappe.db.delete("File", {"is_folder": 0}) + def test_attachments_access(self): frappe.set_user("test4@example.com") self.attached_to_doctype, self.attached_to_docname = make_test_doc() @@ -600,11 +623,12 @@ class TestAttachmentsAccess(unittest.TestCase): self.assertIn("test_user.txt", system_manager_attachments_files) self.assertIn("test_user.txt", user_attachments_files) + def tearDown(self) -> None: frappe.set_user("Administrator") frappe.db.rollback() -class TestFileUtils(unittest.TestCase): +class TestFileUtils(FrappeTestCase): def test_extract_images_from_doc(self): # with filename in data URI todo = frappe.get_doc( @@ -628,30 +652,22 @@ class TestFileUtils(unittest.TestCase): self.assertIn(f' None: + home = frappe.get_doc( + {"doctype": "File", "is_folder": 1, "is_home_folder": 1, "file_name": _("Home")} + ).insert(ignore_if_duplicate=True) + + frappe.get_doc( + { + "doctype": "File", + "folder": home.name, + "is_folder": 1, + "is_attachments_folder": 1, + "file_name": _("Attachments"), + } + ).insert(ignore_if_duplicate=True) + + +def setup_folder_path(filename: str, new_parent: str) -> None: + file: "File" = frappe.get_doc("File", filename) + file.folder = new_parent + file.save() + + if file.is_folder: + from frappe.model.rename_doc import rename_doc + + rename_doc("File", file.name, file.get_name_based_on_parent_folder(), ignore_permissions=True) + + +def get_extension( + filename, + extn: Optional[str] = None, + content: Optional[bytes] = None, + response: Optional["Response"] = None, +) -> str: + mimetype = None + + if response: + content_type = response.headers.get("Content-Type") + + if content_type: + _extn = mimetypes.guess_extension(content_type) + if _extn: + return _extn[1:] + + if extn: + # remove '?' char and parameters from extn if present + if "?" in extn: + extn = extn.split("?", 1)[0] + + mimetype = mimetypes.guess_type(filename + "." + extn)[0] + + if mimetype is None or not mimetype.startswith("image/") and content: + # detect file extension by reading image header properties + extn = imghdr.what(filename + "." + (extn or ""), h=content) + + return extn + + +def get_local_image(file_url: str) -> Tuple["ImageFile", str, str]: + if file_url.startswith("/private"): + file_url_path = (file_url.lstrip("/"),) + else: + file_url_path = ("public", file_url.lstrip("/")) + + file_path = frappe.get_site_path(*file_url_path) + + try: + image = Image.open(file_path) + except IOError: + frappe.throw(_("Unable to read file format for {0}").format(file_url)) + + content = None + + try: + filename, extn = file_url.rsplit(".", 1) + except ValueError: + # no extn + with open(file_path, "r") as f: + content = f.read() + + filename = file_url + extn = None + + extn = get_extension(filename, extn, content) + + return image, filename, extn + + +def get_web_image(file_url: str) -> Tuple["ImageFile", str, str]: + # download + file_url = frappe.utils.get_url(file_url) + r = requests.get(file_url, stream=True) + try: + r.raise_for_status() + except requests.exceptions.HTTPError as e: + if "404" in e.args[0]: + frappe.msgprint(_("File '{0}' not found").format(file_url)) + else: + frappe.msgprint(_("Unable to read file format for {0}").format(file_url)) + raise + + try: + image = Image.open(BytesIO(r.content)) + except Exception as e: + frappe.msgprint(_("Image link '{0}' is not valid").format(file_url), raise_exception=e) + + try: + filename, extn = file_url.rsplit("/", 1)[1].rsplit(".", 1) + except ValueError: + # the case when the file url doesn't have filename or extension + # but is fetched due to a query string. example: https://encrypted-tbn3.gstatic.com/images?q=something + filename = get_random_filename() + extn = None + + extn = get_extension(filename, extn, r.content) + filename = "/files/" + strip(unquote(filename)) + + return image, filename, extn + + +def delete_file(path: str) -> None: + """Delete file from `public folder`""" + if path: + if ".." in path.split("/"): + frappe.throw( + _("It is risky to delete this file: {0}. Please contact your System Manager.").format(path) + ) + + parts = os.path.split(path.strip("/")) + if parts[0] == "files": + path = frappe.utils.get_site_path("public", "files", parts[-1]) + + else: + path = frappe.utils.get_site_path("private", "files", parts[-1]) + + path = encode(path) + if os.path.exists(path): + os.remove(path) + + +def remove_file_by_url(file_url: str, doctype: str = None, name: str = None) -> "Document": + if doctype and name: + fid = frappe.db.get_value( + "File", {"file_url": file_url, "attached_to_doctype": doctype, "attached_to_name": name} + ) + else: + fid = frappe.db.get_value("File", {"file_url": file_url}) + + if fid: + from frappe.utils.file_manager import remove_file + + return remove_file(fid=fid) + + +def get_content_hash(content: Union[bytes, str]) -> str: + if isinstance(content, str): + content = content.encode() + return hashlib.md5(content).hexdigest() # nosec + + +def generate_file_name(name: str, suffix: Optional[str] = None, is_private: bool = False) -> str: + """Generate conflict-free file name. Suffix will be ignored if name available. If the + provided suffix doesn't result in an available path, a random suffix will be picked. + """ + + def path_exists(name, is_private): + return os.path.exists(encode(get_files_path(name, is_private=is_private))) + + if not path_exists(name, is_private): + return name + + candidate_path = get_file_name(name, suffix) + + if path_exists(candidate_path, is_private): + return generate_file_name(name, is_private=is_private) + return candidate_path + + +def get_file_name(fname: str, optional_suffix: Optional[str] = None) -> str: + # convert to unicode + fname = cstr(fname) + partial, extn = os.path.splitext(fname) + suffix = optional_suffix or frappe.generate_hash(length=6) + + return f"{partial}{suffix}{extn}" + + +def extract_images_from_doc(doc: "Document", fieldname: str): + content = doc.get(fieldname) + content = extract_images_from_html(doc, content) + if frappe.flags.has_dataurl: + doc.set(fieldname, content) + + +def extract_images_from_html(doc: "Document", content: str, is_private: bool = False): + frappe.flags.has_dataurl = False + + def _save_file(match): + data = match.group(1).split("data:")[1] + headers, content = data.split(",") + mtype = headers.split(";")[0] + + if isinstance(content, str): + content = content.encode("utf-8") + if b"," in content: + content = content.split(b",")[1] + content = safe_b64decode(content) + + content = optimize_image(content, mtype) + + if "filename=" in headers: + filename = headers.split("filename=")[-1] + filename = safe_decode(filename).split(";")[0] + + else: + filename = get_random_filename(content_type=mtype) + + if doc.meta.istable: + doctype = doc.parenttype + name = doc.parent + else: + doctype = doc.doctype + name = doc.name + + _file = frappe.get_doc( + { + "doctype": "File", + "file_name": filename, + "attached_to_doctype": doctype, + "attached_to_name": name, + "content": content, + "decode": False, + "is_private": is_private, + } + ) + _file.save(ignore_permissions=True) + file_url = _file.file_url + frappe.flags.has_dataurl = True + + return f']*src\s*=\s*["\'](?=data:)(.*?)["\']', _save_file, content) + + return content + + +def get_random_filename(content_type: str = None) -> str: + extn = None + if content_type: + extn = mimetypes.guess_extension(content_type) + + return random_string(7) + (extn or "") + + +def update_existing_file_docs(doc: "File") -> None: + # Update is private and file url of all file docs that point to the same file + file_doctype = frappe.qb.DocType("File") + ( + frappe.qb.update(file_doctype) + .set(file_doctype.file_url, doc.file_url) + .set(file_doctype.is_private, doc.is_private) + .where(file_doctype.content_hash == doc.content_hash) + .where(file_doctype.name != doc.name) + ).run() + + +def attach_files_to_document(doc: "File", event) -> None: + """Runs on on_update hook of all documents. + Goes through every Attach and Attach Image field and attaches + the file url to the document if it is not already attached. + """ + + attach_fields = doc.meta.get("fields", {"fieldtype": ["in", ["Attach", "Attach Image"]]}) + + for df in attach_fields: + # this method runs in on_update hook of all documents + # we dont want the update to fail if file cannot be attached for some reason + value = doc.get(df.fieldname) + if not (value or "").startswith(("/files", "/private/files")): + return + + if frappe.db.exists( + "File", + { + "file_url": value, + "attached_to_name": doc.name, + "attached_to_doctype": doc.doctype, + "attached_to_field": df.fieldname, + }, + ): + return + + file: "File" = frappe.get_doc( + doctype="File", + file_url=value, + attached_to_name=doc.name, + attached_to_doctype=doc.doctype, + attached_to_field=df.fieldname, + folder="Home/Attachments", + ) + try: + file.insert() + except Exception: + doc.log_error("Error Attaching File") + + +def decode_file_content(content: bytes) -> bytes: + if isinstance(content, str): + content = content.encode("utf-8") + if b"," in content: + content = content.split(b",")[1] + return safe_b64decode(content) diff --git a/frappe/core/doctype/log_settings/log_settings.js b/frappe/core/doctype/log_settings/log_settings.js index 09a2086a1d..dc7cc7eac2 100644 --- a/frappe/core/doctype/log_settings/log_settings.js +++ b/frappe/core/doctype/log_settings/log_settings.js @@ -1,8 +1,16 @@ // Copyright (c) 2020, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Log Settings', { - // refresh: function(frm) { - - // } +frappe.ui.form.on("Log Settings", { + refresh: (frm) => { + frm.set_query("ref_doctype", "logs_to_clear", () => { + const added_doctypes = frm.doc.logs_to_clear.map((r) => r.ref_doctype); + return { + query: "frappe.core.doctype.log_settings.log_settings.get_log_doctypes", + filters: [ + ["name", "not in", added_doctypes], + ], + }; + }); + }, }); diff --git a/frappe/core/doctype/log_settings/log_settings.json b/frappe/core/doctype/log_settings/log_settings.json index f06d14f16b..5a9dd159cc 100644 --- a/frappe/core/doctype/log_settings/log_settings.json +++ b/frappe/core/doctype/log_settings/log_settings.json @@ -5,61 +5,20 @@ "editable_grid": 1, "engine": "InnoDB", "field_order": [ - "error_log_notification_section", - "users_to_notify", - "log_cleanup_section", - "clear_error_log_after", - "clear_activity_log_after", - "column_break_4", - "clear_email_queue_after" + "logs_to_clear" ], "fields": [ { - "fieldname": "log_cleanup_section", - "fieldtype": "Section Break", - "label": "Log Cleanup" - }, - { - "fieldname": "column_break_4", - "fieldtype": "Column Break" - }, - { - "fieldname": "error_log_notification_section", - "fieldtype": "Section Break", - "label": "Error Log Notification" - }, - { - "fieldname": "users_to_notify", - "fieldtype": "Table MultiSelect", - "label": "Users To Notify", - "options": "Log Setting User" - }, - { - "default": "90", - "description": "In Days", - "fieldname": "clear_error_log_after", - "fieldtype": "Int", - "label": "Clear Error log After" - }, - { - "default": "90", - "description": "In Days", - "fieldname": "clear_activity_log_after", - "fieldtype": "Int", - "label": "Clear Activity Log After" - }, - { - "default": "30", - "description": "In Days", - "fieldname": "clear_email_queue_after", - "fieldtype": "Int", - "label": "Clear Email Queue After" + "fieldname": "logs_to_clear", + "fieldtype": "Table", + "label": "Logs to Clear", + "options": "Logs To Clear" } ], "index_web_pages_for_search": 1, "issingle": 1, "links": [], - "modified": "2020-10-13 12:18:48.649038", + "modified": "2022-06-11 02:17:30.803721", "modified_by": "Administrator", "module": "Core", "name": "Log Settings", @@ -79,5 +38,6 @@ "quick_entry": 1, "sort_field": "modified", "sort_order": "DESC", + "states": [], "track_changes": 1 -} +} \ No newline at end of file diff --git a/frappe/core/doctype/log_settings/log_settings.py b/frappe/core/doctype/log_settings/log_settings.py index 0fde168532..1a7ce532cd 100644 --- a/frappe/core/doctype/log_settings/log_settings.py +++ b/frappe/core/doctype/log_settings/log_settings.py @@ -2,44 +2,119 @@ # Copyright (c) 2020, Frappe Technologies and contributors # License: MIT. See LICENSE +from typing import Protocol, runtime_checkable + import frappe from frappe import _ +from frappe.model.base_document import get_controller from frappe.model.document import Document -from frappe.query_builder import DocType, Interval -from frappe.query_builder.functions import Now +from frappe.utils import cint +from frappe.utils.caching import site_cache + +DEFAULT_LOGTYPES_RETENTION = { + "Error Log": 30, + "Activity Log": 90, + "Email Queue": 30, + "Error Snapshot": 30, + "Scheduled Job Log": 90, +} + + +@runtime_checkable +class LogType(Protocol): + """Interface requirement for doctypes that can be cleared using log settings.""" + + @staticmethod + def clear_old_logs(days: int) -> None: + ... + + +@site_cache +def _supports_log_clearing(doctype: str) -> bool: + try: + controller = get_controller(doctype) + return issubclass(controller, LogType) + except Exception: + return False class LogSettings(Document): + def validate(self): + self.validate_supported_doctypes() + self.validate_duplicates() + self.add_default_logtypes() + + def validate_supported_doctypes(self): + for entry in self.logs_to_clear: + if _supports_log_clearing(entry.ref_doctype): + continue + + msg = _("{} does not support automated log clearing.").format(frappe.bold(entry.ref_doctype)) + if frappe.conf.developer_mode: + msg += "
" + _("Implement `clear_old_logs` method to enable auto error clearing.") + frappe.throw(msg, title=_("DocType not supported by Log Settings.")) + + def validate_duplicates(self): + seen = set() + for entry in self.logs_to_clear: + if entry.ref_doctype in seen: + frappe.throw( + _("{} appears more than once in configured log doctypes.").format(entry.ref_doctype) + ) + seen.add(entry.ref_doctype) + + def add_default_logtypes(self): + existing_logtypes = {d.ref_doctype for d in self.logs_to_clear} + added_logtypes = set() + for logtype, retention in DEFAULT_LOGTYPES_RETENTION.items(): + if logtype not in existing_logtypes and _supports_log_clearing(logtype): + self.append("logs_to_clear", {"ref_doctype": logtype, "days": cint(retention)}) + added_logtypes.add(logtype) + + if added_logtypes: + frappe.msgprint( + _("Added default log doctypes: {}").format(",".join(added_logtypes)), alert=True + ) + def clear_logs(self): - self.clear_email_queue() - self.clear_error_logs() - self.clear_activity_logs() + """ + Log settings can clear any log type that's registered to it and provides a method to delete old logs. - def clear_error_logs(self): - table = DocType("Error Log") - frappe.db.delete( - table, filters=(table.creation < (Now() - Interval(days=self.clear_error_log_after))) - ) + Check `LogDoctype` above for interface that doctypes need to implement. + """ - def clear_activity_logs(self): - from frappe.core.doctype.activity_log.activity_log import clear_activity_logs + for entry in self.logs_to_clear: + controller: LogType = get_controller(entry.ref_doctype) + func = controller.clear_old_logs - clear_activity_logs(days=self.clear_activity_log_after) + # Only pass what the method can handle, this is considering any + # future addition that might happen to the required interface. + kwargs = frappe.get_newargs(func, {"days": entry.days}) + func(**kwargs) + frappe.db.commit() - def clear_email_queue(self): - from frappe.email.queue import clear_outbox + def register_doctype(self, doctype: str, days=30): + existing_logtypes = {d.ref_doctype for d in self.logs_to_clear} - clear_outbox(days=self.clear_email_queue_after) + if doctype not in existing_logtypes and _supports_log_clearing(doctype): + self.append("logs_to_clear", {"ref_doctype": doctype, "days": cint(days)}) + else: + for entry in self.logs_to_clear: + if entry.ref_doctype == doctype: + entry.days = days + break def run_log_clean_up(): doc = frappe.get_doc("Log Settings") + doc.add_default_logtypes() + doc.save() doc.clear_logs() @frappe.whitelist() -def has_unseen_error_log(user): - def _get_response(show_alert=True): +def has_unseen_error_log(): + if frappe.get_all("Error Log", filters={"seen": 0}, limit=1): return { "show_alert": True, "message": _("You have unseen {0}").format( @@ -47,13 +122,67 @@ def has_unseen_error_log(user): ), } - if frappe.get_all("Error Log", filters={"seen": 0}, limit=1): - log_settings = frappe.get_cached_doc("Log Settings") - if log_settings.users_to_notify: - if user in [u.user for u in log_settings.users_to_notify]: - return _get_response() - else: - return _get_response(show_alert=False) - else: - return _get_response() +@frappe.whitelist() +@frappe.validate_and_sanitize_search_inputs +def get_log_doctypes(doctype, txt, searchfield, start, page_len, filters): + + filters = filters or {} + + filters.extend( + [ + ["istable", "=", 0], + ["issingle", "=", 0], + ["name", "like", f"%%{txt}%%"], + ] + ) + doctypes = frappe.get_list("DocType", filters=filters, pluck="name") + + supported_doctypes = [(d,) for d in doctypes if _supports_log_clearing(d)] + + return supported_doctypes[start:page_len] + + +LOG_DOCTYPES = [ + "Scheduled Job Log", + "Activity Log", + "Route History", + "Email Queue", + "Email Queue Recipient", + "Error Snapshot", + "Error Log", +] + + +def clear_log_table(doctype, days=90): + """If any logtype table grows too large then clearing it with DELETE query + is not feasible in reasonable time. This command copies recent data to new + table and replaces current table with new smaller table. + + ref: https://mariadb.com/kb/en/big-deletes/#deleting-more-than-half-a-table + """ + from frappe.utils import get_table_name + + if doctype not in LOG_DOCTYPES: + raise frappe.ValidationError(f"Unsupported logging DocType: {doctype}") + + original = get_table_name(doctype) + temporary = f"{original} temp_table" + backup = f"{original} backup_table" + + try: + frappe.db.sql_ddl(f"CREATE TABLE `{temporary}` LIKE `{original}`") + + # Copy all recent data to new table + frappe.db.sql( + f"""INSERT INTO `{temporary}` + SELECT * FROM `{original}` + WHERE `{original}`.`modified` > NOW() - INTERVAL '{days}' DAY""" + ) + frappe.db.sql_ddl(f"RENAME TABLE `{original}` TO `{backup}`, `{temporary}` TO `{original}`") + except Exception: + frappe.db.rollback() + frappe.db.sql_ddl(f"DROP TABLE IF EXISTS `{temporary}`") + raise + else: + frappe.db.sql_ddl(f"DROP TABLE `{backup}`") diff --git a/frappe/core/doctype/log_settings/test_log_settings.py b/frappe/core/doctype/log_settings/test_log_settings.py index 1b78745103..d7f43a181d 100644 --- a/frappe/core/doctype/log_settings/test_log_settings.py +++ b/frappe/core/doctype/log_settings/test_log_settings.py @@ -4,7 +4,7 @@ from datetime import datetime import frappe -from frappe.core.doctype.log_settings.log_settings import run_log_clean_up +from frappe.core.doctype.log_settings.log_settings import _supports_log_clearing, run_log_clean_up from frappe.tests.utils import FrappeTestCase from frappe.utils import add_to_date, now_datetime @@ -56,6 +56,23 @@ class TestLogSettings(FrappeTestCase): self.assertEqual(error_log_count, 0) self.assertEqual(email_queue_count, 0) + def test_logtype_identification(self): + supported_types = [ + "Error Log", + "Activity Log", + "Email Queue", + "Route History", + "Error Snapshot", + "Scheduled Job Log", + ] + + for lt in supported_types: + self.assertTrue(_supports_log_clearing(lt), f"{lt} should be recognized as log type") + + unsupported_types = ["DocType", "User", "Non Existing dt"] + for dt in unsupported_types: + self.assertFalse(_supports_log_clearing(dt), f"{dt} shouldn't be recognized as log type") + def setup_test_logs(past: datetime) -> None: activity_log = frappe.get_doc( diff --git a/frappe/data_migration/doctype/data_migration_connector/connectors/__init__.py b/frappe/core/doctype/logs_to_clear/__init__.py similarity index 100% rename from frappe/data_migration/doctype/data_migration_connector/connectors/__init__.py rename to frappe/core/doctype/logs_to_clear/__init__.py diff --git a/frappe/core/doctype/logs_to_clear/logs_to_clear.json b/frappe/core/doctype/logs_to_clear/logs_to_clear.json new file mode 100644 index 0000000000..df19ccd9e7 --- /dev/null +++ b/frappe/core/doctype/logs_to_clear/logs_to_clear.json @@ -0,0 +1,43 @@ +{ + "actions": [], + "autoname": "hash", + "creation": "2022-06-11 02:02:39.472511", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "ref_doctype", + "days" + ], + "fields": [ + { + "fieldname": "ref_doctype", + "fieldtype": "Link", + "in_list_view": 1, + "label": "Log DocType", + "options": "DocType", + "reqd": 1 + }, + { + "default": "30", + "fieldname": "days", + "fieldtype": "Int", + "in_list_view": 1, + "label": "Clear Logs After (days)", + "non_negative": 1, + "reqd": 1 + } + ], + "index_web_pages_for_search": 1, + "istable": 1, + "links": [], + "modified": "2022-06-13 02:51:36.857786", + "modified_by": "Administrator", + "module": "Core", + "name": "Logs To Clear", + "owner": "Administrator", + "permissions": [], + "sort_field": "modified", + "sort_order": "DESC", + "states": [] +} diff --git a/frappe/core/doctype/logs_to_clear/logs_to_clear.py b/frappe/core/doctype/logs_to_clear/logs_to_clear.py new file mode 100644 index 0000000000..3fb4f8e72a --- /dev/null +++ b/frappe/core/doctype/logs_to_clear/logs_to_clear.py @@ -0,0 +1,9 @@ +# Copyright (c) 2022, Frappe Technologies and contributors +# For license information, please see license.txt + +# import frappe +from frappe.model.document import Document + + +class LogsToClear(Document): + pass diff --git a/frappe/core/doctype/package_import/package_import.py b/frappe/core/doctype/package_import/package_import.py index 659017c498..43a985af9b 100644 --- a/frappe/core/doctype/package_import/package_import.py +++ b/frappe/core/doctype/package_import/package_import.py @@ -55,11 +55,11 @@ class PackageImport(Document): for module in os.listdir(package_path): module_path = os.path.join(package_path, module) if os.path.isdir(module_path): - get_doc_files(files, module_path) + files = get_doc_files(files, module_path) # import files for file in files: - import_file_by_path(file, force=self.force, ignore_version=True, for_sync=True) + import_file_by_path(file, force=self.force, ignore_version=True) log.append("Imported {}".format(file)) self.log = "\n".join(log) diff --git a/frappe/core/doctype/patch_log/patch_log.js b/frappe/core/doctype/patch_log/patch_log.js index 0080584a29..b52876ac97 100644 --- a/frappe/core/doctype/patch_log/patch_log.js +++ b/frappe/core/doctype/patch_log/patch_log.js @@ -3,6 +3,6 @@ frappe.ui.form.on('Patch Log', { refresh: function(frm) { - + frm.disable_save(); } }); diff --git a/frappe/core/doctype/patch_log/patch_log.json b/frappe/core/doctype/patch_log/patch_log.json index aa054f1360..9750c51279 100644 --- a/frappe/core/doctype/patch_log/patch_log.json +++ b/frappe/core/doctype/patch_log/patch_log.json @@ -1,87 +1,44 @@ { - "allow_copy": 0, - "allow_import": 0, - "allow_rename": 0, - "autoname": "PATCHLOG.#####", - "beta": 0, - "creation": "2013-01-17 11:36:45", - "custom": 0, - "description": "List of patches executed", - "docstatus": 0, - "doctype": "DocType", - "document_type": "System", - "editable_grid": 0, + "actions": [], + "autoname": "PATCHLOG.#####", + "creation": "2013-01-17 11:36:45", + "description": "List of patches executed", + "doctype": "DocType", + "document_type": "System", + "engine": "InnoDB", + "field_order": [ + "patch" + ], "fields": [ { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "patch", - "fieldtype": "Code", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Patch", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "fieldname": "patch", + "fieldtype": "Code", + "label": "Patch", + "read_only": 1 } - ], - "hide_heading": 0, - "hide_toolbar": 0, - "icon": "fa fa-cog", - "idx": 1, - "image_view": 0, - "in_create": 0, - - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2016-12-29 14:40:35.048570", - "modified_by": "Administrator", - "module": "Core", - "name": "Patch Log", - "owner": "Administrator", + ], + "icon": "fa fa-cog", + "idx": 1, + "links": [], + "modified": "2022-06-13 05:34:37.845368", + "modified_by": "Administrator", + "module": "Core", + "name": "Patch Log", + "naming_rule": "Expression (old style)", + "owner": "Administrator", "permissions": [ { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 0, - "delete": 0, - "email": 1, - "export": 0, - "if_owner": 0, - "import": 0, - "is_custom": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 1, - "role": "Administrator", - "set_user_permissions": 0, - "share": 0, - "submit": 0, - "write": 0 + "email": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "Administrator" } - ], - "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "track_changes": 1, - "track_seen": 0 + ], + "quick_entry": 1, + "sort_field": "modified", + "sort_order": "DESC", + "states": [], + "title_field": "patch", + "track_changes": 1 } \ No newline at end of file diff --git a/frappe/core/doctype/prepared_report/prepared_report.json b/frappe/core/doctype/prepared_report/prepared_report.json index 4663dcb463..cafe323519 100644 --- a/frappe/core/doctype/prepared_report/prepared_report.json +++ b/frappe/core/doctype/prepared_report/prepared_report.json @@ -23,15 +23,14 @@ { "fieldname": "report_name", "fieldtype": "Data", - "hidden": 1, "label": "Report Name", "read_only": 1 }, { "fieldname": "ref_report_doctype", "fieldtype": "Link", - "hidden": 1, - "label": "Ref Report DocType", + "in_standard_filter": 1, + "label": "Report Type", "options": "Report", "read_only": 1 }, @@ -41,6 +40,7 @@ "fieldtype": "Select", "hidden": 1, "in_list_view": 1, + "in_standard_filter": 1, "label": "Status", "options": "Error\nQueued\nCompleted", "read_only": 1 @@ -103,10 +103,11 @@ ], "in_create": 1, "links": [], - "modified": "2020-03-05 10:52:56.598365", + "modified": "2022-06-13 06:20:34.496412", "modified_by": "Administrator", "module": "Core", "name": "Prepared Report", + "naming_rule": "Expression (old style)", "owner": "Administrator", "permissions": [ { @@ -131,9 +132,9 @@ "share": 1 } ], - "quick_entry": 1, "sort_field": "modified", "sort_order": "DESC", - "title_field": "report_name", + "states": [], + "title_field": "ref_report_doctype", "track_changes": 1 } \ No newline at end of file diff --git a/frappe/core/doctype/report/report.py b/frappe/core/doctype/report/report.py index 9e6cc73f11..efb45f41c8 100644 --- a/frappe/core/doctype/report/report.py +++ b/frappe/core/doctype/report/report.py @@ -89,7 +89,9 @@ class Report(Document): ] custom_roles = get_custom_allowed_roles("report", self.name) - allowed.extend(custom_roles) + + if custom_roles: + allowed = custom_roles if not allowed: return True diff --git a/frappe/core/doctype/report/test_report.py b/frappe/core/doctype/report/test_report.py index 7b17a5a8d5..bbae616e93 100644 --- a/frappe/core/doctype/report/test_report.py +++ b/frappe/core/doctype/report/test_report.py @@ -186,6 +186,38 @@ class TestReport(FrappeTestCase): self.assertNotEqual(report.is_permitted(), True) frappe.set_user("Administrator") + def test_report_custom_permissions(self): + frappe.set_user("test@example.com") + frappe.db.delete("Custom Role", {"report": "Test Custom Role Report"}) + frappe.db.commit() # nosemgrep + if not frappe.db.exists("Report", "Test Custom Role Report"): + report = frappe.get_doc( + { + "doctype": "Report", + "ref_doctype": "User", + "report_name": "Test Custom Role Report", + "report_type": "Query Report", + "is_standard": "No", + "roles": [{"role": "_Test Role"}, {"role": "System Manager"}], + } + ).insert(ignore_permissions=True) + else: + report = frappe.get_doc("Report", "Test Custom Role Report") + + self.assertEqual(report.is_permitted(), True) + + frappe.get_doc( + { + "doctype": "Custom Role", + "report": "Test Custom Role Report", + "roles": [{"role": "_Test Role 2"}], + "ref_doctype": "User", + } + ).insert(ignore_permissions=True) + + self.assertNotEqual(report.is_permitted(), True) + frappe.set_user("Administrator") + # test for the `_format` method if report data doesn't have sort_by parameter def test_format_method(self): if frappe.db.exists("Report", "User Activity Report Without Sort"): diff --git a/frappe/core/doctype/scheduled_job_log/scheduled_job_log.json b/frappe/core/doctype/scheduled_job_log/scheduled_job_log.json index 396b32bdf9..451c4108a0 100644 --- a/frappe/core/doctype/scheduled_job_log/scheduled_job_log.json +++ b/frappe/core/doctype/scheduled_job_log/scheduled_job_log.json @@ -38,7 +38,7 @@ } ], "links": [], - "modified": "2021-10-25 00:00:00.000000", + "modified": "2022-06-13 05:41:21.090972", "modified_by": "Administrator", "module": "Core", "name": "Scheduled Job Log", @@ -59,5 +59,7 @@ ], "quick_entry": 1, "sort_field": "modified", - "sort_order": "DESC" -} + "sort_order": "DESC", + "states": [], + "title_field": "scheduled_job_type" +} \ No newline at end of file diff --git a/frappe/core/doctype/scheduled_job_log/scheduled_job_log.py b/frappe/core/doctype/scheduled_job_log/scheduled_job_log.py index bead463ba5..68541a36a0 100644 --- a/frappe/core/doctype/scheduled_job_log/scheduled_job_log.py +++ b/frappe/core/doctype/scheduled_job_log/scheduled_job_log.py @@ -2,9 +2,14 @@ # Copyright (c) 2019, Frappe Technologies and contributors # License: MIT. See LICENSE -# import frappe +import frappe from frappe.model.document import Document +from frappe.query_builder import Interval +from frappe.query_builder.functions import Now class ScheduledJobLog(Document): - pass + @staticmethod + def clear_old_logs(days=90): + table = frappe.qb.DocType("Scheduled Job Log") + frappe.db.delete(table, filters=(table.modified < (Now() - Interval(days=days)))) diff --git a/frappe/core/doctype/scheduled_job_log/scheduled_job_log_list.js b/frappe/core/doctype/scheduled_job_log/scheduled_job_log_list.js new file mode 100644 index 0000000000..5ddccb5d44 --- /dev/null +++ b/frappe/core/doctype/scheduled_job_log/scheduled_job_log_list.js @@ -0,0 +1,7 @@ +frappe.listview_settings["Scheduled Job Log"] = { + onload: function(listview) { + frappe.require("logtypes.bundle.js", () => { + frappe.utils.logtypes.show_log_retention_message(cur_list.doctype); + }) + }, +}; diff --git a/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py b/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py index 9665a20843..318b156dcd 100644 --- a/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py +++ b/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py @@ -185,9 +185,12 @@ def insert_single_event(frequency: str, event: str, cron_format: str = None): if not frappe.db.exists( "Scheduled Job Type", {"method": event, "frequency": frequency, **cron_expr} ): + savepoint = "scheduled_job_type_creation" try: + frappe.db.savepoint(savepoint) doc.insert() except frappe.DuplicateEntryError: + frappe.db.rollback(save_point=savepoint) doc.delete() doc.insert() diff --git a/frappe/core/doctype/server_script/server_script.json b/frappe/core/doctype/server_script/server_script.json index 9312ae178b..5446cc1a39 100644 --- a/frappe/core/doctype/server_script/server_script.json +++ b/frappe/core/doctype/server_script/server_script.json @@ -25,6 +25,7 @@ "fieldname": "script_type", "fieldtype": "Select", "in_list_view": 1, + "in_standard_filter": 1, "label": "Script Type", "options": "DocType Event\nScheduler Event\nPermission Query\nAPI", "reqd": 1 @@ -41,6 +42,7 @@ "fieldname": "reference_doctype", "fieldtype": "Link", "in_list_view": 1, + "in_standard_filter": 1, "label": "Reference Document Type", "options": "DocType" }, @@ -109,7 +111,7 @@ "link_fieldname": "server_script" } ], - "modified": "2022-04-27 11:42:52.032963", + "modified": "2022-06-13 06:04:20.937969", "modified_by": "Administrator", "module": "Core", "name": "Server Script", diff --git a/frappe/core/doctype/system_settings/system_settings.json b/frappe/core/doctype/system_settings/system_settings.json index 52ff5e38e4..c954e41202 100644 --- a/frappe/core/doctype/system_settings/system_settings.json +++ b/frappe/core/doctype/system_settings/system_settings.json @@ -1,6 +1,6 @@ { "actions": [], - "creation": "2014-04-17 16:53:52.640856", + "creation": "2022-01-06 03:18:16.326761", "doctype": "DocType", "document_type": "System", "engine": "InnoDB", @@ -34,12 +34,14 @@ "security", "session_expiry", "session_expiry_mobile", + "document_share_key_expiry", "column_break_13", "deny_multiple_sessions", "allow_login_using_mobile_number", "allow_login_using_user_name", "allow_error_traceback", "strip_exif_metadata_from_uploaded_images", + "allow_older_web_view_links", "password_settings", "logout_on_password_reset", "force_user_to_reset_password", @@ -482,6 +484,19 @@ "options": "Sunday\nMonday\nTuesday\nWednesday\nThursday\nFriday\nSaturday" }, { + "default": "30", + "description": "Number of days after which the document Web View link shared on email will be expired", + "fieldname": "document_share_key_expiry", + "fieldtype": "Int", + "label": "Document Share Key Expiry (in Days)" + }, + { + "default": "0", + "fieldname": "allow_older_web_view_links", + "fieldtype": "Check", + "label": "Allow Older Web View Links (Insecure)" + }, + { "fieldname": "column_break_64", "fieldtype": "Column Break" }, diff --git a/frappe/core/doctype/user/user.js b/frappe/core/doctype/user/user.js index 77c199cdd4..001aae4da0 100644 --- a/frappe/core/doctype/user/user.js +++ b/frappe/core/doctype/user/user.js @@ -59,6 +59,10 @@ frappe.ui.form.on('User', { onload: function(frm) { frm.can_edit_roles = has_access_to_edit_user(); + if (frm.is_new() && frm.roles_editor) { + frm.roles_editor.reset(); + } + if (frm.can_edit_roles && !frm.is_new() && in_list(['System User', 'Website User'], frm.doc.user_type)) { if (!frm.roles_editor) { const role_area = $('
') @@ -194,14 +198,14 @@ frappe.ui.form.on('User', { } } } - if (frm.doc.user_emails){ - var found =0; - for (var i = 0;i bool: + """Update changed info and return true if change contains useful data.""" + if not old: + # Check if doc has some information about creation source like data import + return self.for_insert(new) + else: + return self.set_diff(old, new) + + def set_diff(self, old: Document, new: Document) -> bool: """Set the data property with the diff of the docs if present""" diff = get_diff(old, new) if diff: self.ref_doctype = new.doctype self.docname = new.name - self.data = frappe.as_json(diff) + self.data = frappe.as_json(diff, indent=None, separators=(",", ":")) return True else: return False - def for_insert(self, doc): + def for_insert(self, doc: Document) -> bool: updater_reference = doc.flags.updater_reference + if not updater_reference: + return False + data = { "creation": doc.creation, "updater_reference": updater_reference, @@ -29,7 +41,8 @@ class Version(Document): } self.ref_doctype = doc.doctype self.docname = doc.name - self.data = frappe.as_json(data) + self.data = frappe.as_json(data, indent=None, separators=(",", ":")) + return True def get_data(self): return json.loads(self.data) diff --git a/frappe/custom/doctype/custom_field/custom_field.json b/frappe/custom/doctype/custom_field/custom_field.json index 045a0981f3..63be70c644 100644 --- a/frappe/custom/doctype/custom_field/custom_field.json +++ b/frappe/custom/doctype/custom_field/custom_field.json @@ -67,7 +67,8 @@ "fieldtype": "Link", "in_filter": 1, "in_list_view": 1, - "label": "Document", + "in_standard_filter": 1, + "label": "DocType", "oldfieldname": "dt", "oldfieldtype": "Link", "options": "DocType", @@ -94,6 +95,7 @@ "fieldname": "fieldname", "fieldtype": "Data", "in_list_view": 1, + "in_standard_filter": 1, "label": "Fieldname", "no_copy": 1, "oldfieldname": "fieldname", @@ -439,7 +441,7 @@ "idx": 1, "index_web_pages_for_search": 1, "links": [], - "modified": "2022-04-14 09:46:58.849765", + "modified": "2022-06-13 06:39:03.319667", "modified_by": "Administrator", "module": "Custom", "name": "Custom Field", diff --git a/frappe/custom/doctype/custom_field/custom_field.py b/frappe/custom/doctype/custom_field/custom_field.py index 10ee4a503f..8a2a2663de 100644 --- a/frappe/custom/doctype/custom_field/custom_field.py +++ b/frappe/custom/doctype/custom_field/custom_field.py @@ -161,6 +161,7 @@ def create_custom_field(doctype, df, ignore_validate=False, is_system_generated= custom_field.update(df) custom_field.flags.ignore_validate = ignore_validate custom_field.insert() + return custom_field def create_custom_fields(custom_fields, ignore_validate=False, update=True): diff --git a/frappe/data_migration/doctype/data_migration_connector/connectors/base.py b/frappe/data_migration/doctype/data_migration_connector/connectors/base.py deleted file mode 100644 index 7d2b320c59..0000000000 --- a/frappe/data_migration/doctype/data_migration_connector/connectors/base.py +++ /dev/null @@ -1,24 +0,0 @@ -from abc import ABCMeta, abstractmethod - -from frappe.utils.password import get_decrypted_password - - -class BaseConnection(metaclass=ABCMeta): - @abstractmethod - def get(self, remote_objectname, fields=None, filters=None, start=0, page_length=10): - pass - - @abstractmethod - def insert(self, doctype, doc): - pass - - @abstractmethod - def update(self, doctype, doc, migration_id): - pass - - @abstractmethod - def delete(self, doctype, migration_id): - pass - - def get_password(self): - return get_decrypted_password("Data Migration Connector", self.connector.name) diff --git a/frappe/data_migration/doctype/data_migration_connector/connectors/frappe_connection.py b/frappe/data_migration/doctype/data_migration_connector/connectors/frappe_connection.py deleted file mode 100644 index 8228529562..0000000000 --- a/frappe/data_migration/doctype/data_migration_connector/connectors/frappe_connection.py +++ /dev/null @@ -1,32 +0,0 @@ -import frappe -from frappe.frappeclient import FrappeClient - -from .base import BaseConnection - - -class FrappeConnection(BaseConnection): - def __init__(self, connector): - self.connector = connector - self.connection = FrappeClient( - self.connector.hostname, self.connector.username, self.get_password() - ) - self.name_field = "name" - - def insert(self, doctype, doc): - doc = frappe._dict(doc) - doc.doctype = doctype - return self.connection.insert(doc) - - def update(self, doctype, doc, migration_id): - doc = frappe._dict(doc) - doc.doctype = doctype - doc.name = migration_id - return self.connection.update(doc) - - def delete(self, doctype, migration_id): - return self.connection.delete(doctype, migration_id) - - def get(self, doctype, fields='"*"', filters=None, start=0, page_length=20): - return self.connection.get_list( - doctype, fields=fields, filters=filters, limit_start=start, limit_page_length=page_length - ) diff --git a/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.js b/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.js deleted file mode 100644 index 0898fcf4e7..0000000000 --- a/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.js +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright (c) 2017, Frappe Technologies and contributors -// For license information, please see license.txt - -frappe.ui.form.on('Data Migration Connector', { - onload(frm) { - if(frappe.boot.developer_mode) { - frm.add_custom_button(__('New Connection'), () => frm.events.new_connection(frm)); - } - }, - new_connection(frm) { - const d = new frappe.ui.Dialog({ - title: __('New Connection'), - fields: [ - { label: __('Module'), fieldtype: 'Link', options: 'Module Def', reqd: 1 }, - { label: __('Connection Name'), fieldtype: 'Data', description: 'For e.g: Shopify Connection', reqd: 1 }, - ], - primary_action_label: __('Create'), - primary_action: (values) => { - let { module, connection_name } = values; - - frm.events.create_new_connection(module, connection_name) - .then(r => { - if (r.message) { - const connector_name = connection_name - .replace('connection', 'Connector') - .replace('Connection', 'Connector') - .trim(); - - frm.set_value('connector_name', connector_name); - frm.set_value('connector_type', 'Custom'); - frm.set_value('python_module', r.message); - frm.save(); - frappe.show_alert(__("New module created {0}", [r.message])); - d.hide(); - } - }); - } - }); - - d.show(); - }, - create_new_connection(module, connection_name) { - return frappe.call('frappe.data_migration.doctype.data_migration_connector.data_migration_connector.create_new_connection', { - module, connection_name - }); - } -}); diff --git a/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.json b/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.json deleted file mode 100644 index 338d59aadd..0000000000 --- a/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.json +++ /dev/null @@ -1,307 +0,0 @@ -{ - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 1, - "autoname": "field:connector_name", - "beta": 1, - "creation": "2017-08-11 05:03:27.091416", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", - "fields": [ - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "connector_name", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Connector Name", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "depends_on": "eval:!doc.is_custom", - "fieldname": "connector_type", - "fieldtype": "Select", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Connector Type", - "length": 0, - "no_copy": 0, - "options": "\nFrappe\nCustom", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "depends_on": "eval:doc.connector_type == 'Custom'", - "fieldname": "python_module", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Python Module", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "authentication_credentials", - "fieldtype": "Section Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Authentication Credentials", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "default": "", - "fieldname": "hostname", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Hostname", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "database_name", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Database Name", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "username", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Username", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "password", - "fieldtype": "Password", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Password", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2017-12-01 13:38:55.992499", - "modified_by": "Administrator", - "module": "Data Migration", - "name": "Data Migration Connector", - "name_case": "", - "owner": "Administrator", - "permissions": [ - { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "set_user_permissions": 0, - "share": 1, - "submit": 0, - "write": 1 - } - ], - "quick_entry": 0, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1, - "track_seen": 0 -} \ No newline at end of file diff --git a/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.py b/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.py deleted file mode 100644 index 9db7fc2445..0000000000 --- a/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.py +++ /dev/null @@ -1,107 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Frappe Technologies and contributors -# License: MIT. See LICENSE - -import os - -import frappe -from frappe import _ -from frappe.model.document import Document -from frappe.modules.export_file import create_init_py - -from .connectors.base import BaseConnection -from .connectors.frappe_connection import FrappeConnection - - -class DataMigrationConnector(Document): - def validate(self): - if not (self.python_module or self.connector_type): - frappe.throw(_("Enter python module or select connector type")) - - if self.python_module: - try: - get_connection_class(self.python_module) - except: - frappe.throw(frappe._("Invalid module path")) - - def get_connection(self): - if self.python_module: - _class = get_connection_class(self.python_module) - return _class(self) - else: - self.connection = FrappeConnection(self) - - return self.connection - - -@frappe.whitelist() -def create_new_connection(module, connection_name): - if not frappe.conf.get("developer_mode"): - frappe.msgprint(_("Please enable developer mode to create new connection")) - return - # create folder - module_path = frappe.get_module_path(module) - connectors_folder = os.path.join(module_path, "connectors") - frappe.create_folder(connectors_folder) - - # create init py - create_init_py(module_path, "connectors", "") - - connection_class = connection_name.replace(" ", "") - file_name = frappe.scrub(connection_name) + ".py" - file_path = os.path.join(module_path, "connectors", file_name) - - # create boilerplate file - with open(file_path, "w") as f: - f.write(connection_boilerplate.format(connection_class=connection_class)) - - # get python module string from file_path - app_name = frappe.db.get_value("Module Def", module, "app_name") - python_module = os.path.relpath(file_path, "../apps/{0}".format(app_name)).replace( - os.path.sep, "." - )[:-3] - - return python_module - - -def get_connection_class(python_module): - filename = python_module.rsplit(".", 1)[-1] - classname = frappe.unscrub(filename).replace(" ", "") - module = frappe.get_module(python_module) - - raise_error = False - if hasattr(module, classname): - _class = getattr(module, classname) - if not issubclass(_class, BaseConnection): - raise_error = True - else: - raise_error = True - - if raise_error: - raise ImportError(filename) - - return _class - - -connection_boilerplate = """from frappe.data_migration.doctype.data_migration_connector.connectors.base import BaseConnection - -class {connection_class}(BaseConnection): - def __init__(self, connector): - # self.connector = connector - # self.connection = YourModule(self.connector.username, self.get_password()) - # self.name_field = 'id' - pass - - def get(self, remote_objectname, fields=None, filters=None, start=0, page_length=10): - pass - - def insert(self, doctype, doc): - pass - - def update(self, doctype, doc, migration_id): - pass - - def delete(self, doctype, migration_id): - pass - -""" diff --git a/frappe/data_migration/doctype/data_migration_connector/test_data_migration_connector.py b/frappe/data_migration/doctype/data_migration_connector/test_data_migration_connector.py deleted file mode 100644 index c4090796ab..0000000000 --- a/frappe/data_migration/doctype/data_migration_connector/test_data_migration_connector.py +++ /dev/null @@ -1,8 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Frappe Technologies and Contributors -# License: MIT. See LICENSE -import unittest - - -class TestDataMigrationConnector(unittest.TestCase): - pass diff --git a/frappe/data_migration/doctype/data_migration_mapping/__init__.py b/frappe/data_migration/doctype/data_migration_mapping/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.js b/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.js deleted file mode 100644 index 6c99b9a54d..0000000000 --- a/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.js +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright (c) 2017, Frappe Technologies and contributors -// For license information, please see license.txt - -frappe.ui.form.on('Data Migration Mapping', { - refresh: function() { - - } -}); diff --git a/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.json b/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.json deleted file mode 100644 index 998abdf6ca..0000000000 --- a/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.json +++ /dev/null @@ -1,456 +0,0 @@ -{ - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 1, - "autoname": "field:mapping_name", - "beta": 1, - "creation": "2017-08-11 05:11:49.975801", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", - "fields": [ - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "mapping_name", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Mapping Name", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "remote_objectname", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Remote Objectname", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "remote_primary_key", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Remote Primary Key", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "local_doctype", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Local DocType", - "length": 0, - "no_copy": 0, - "options": "DocType", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "local_primary_key", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Local Primary Key", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "column_break_5", - "fieldtype": "Column Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "mapping_type", - "fieldtype": "Select", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Mapping Type", - "length": 0, - "no_copy": 0, - "options": "Push\nPull\nSync", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "default": "10", - "fieldname": "page_length", - "fieldtype": "Int", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Page Length", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "migration_id_field", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Migration ID Field", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "mapping", - "fieldtype": "Section Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Mapping", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "fields", - "fieldtype": "Table", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Field Maps", - "length": 0, - "no_copy": 0, - "options": "Data Migration Mapping Detail", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 1, - "columns": 0, - "fieldname": "condition_detail", - "fieldtype": "Section Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Condition Detail", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "condition", - "fieldtype": "Code", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Condition", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2017-09-27 18:06:43.275207", - "modified_by": "Administrator", - "module": "Data Migration", - "name": "Data Migration Mapping", - "name_case": "", - "owner": "Administrator", - "permissions": [ - { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "set_user_permissions": 0, - "share": 1, - "submit": 0, - "write": 1 - } - ], - "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1, - "track_seen": 0 -} \ No newline at end of file diff --git a/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.py b/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.py deleted file mode 100644 index 49af65e99b..0000000000 --- a/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.py +++ /dev/null @@ -1,83 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Frappe Technologies and contributors -# License: MIT. See LICENSE - -import frappe -from frappe.model.document import Document -from frappe.utils.safe_exec import get_safe_globals - - -class DataMigrationMapping(Document): - def get_filters(self): - if self.condition: - return frappe.safe_eval(self.condition, get_safe_globals()) - - def get_fields(self): - fields = [] - for f in self.fields: - if not (f.local_fieldname[0] in ('"', "'") or f.local_fieldname.startswith("eval:")): - fields.append(f.local_fieldname) - - if frappe.db.has_column(self.local_doctype, self.migration_id_field): - fields.append(self.migration_id_field) - - if "name" not in fields: - fields.append("name") - - return fields - - def get_mapped_record(self, doc): - """Build a mapped record using information from the fields table""" - mapped = frappe._dict() - - key_fieldname = "remote_fieldname" - value_fieldname = "local_fieldname" - - if self.mapping_type == "Pull": - key_fieldname, value_fieldname = value_fieldname, key_fieldname - - for field_map in self.fields: - key = get_source_value(field_map, key_fieldname) - - if not field_map.is_child_table: - # field to field mapping - value = get_value_from_fieldname(field_map, value_fieldname, doc) - else: - # child table mapping - mapping_name = field_map.child_table_mapping - value = get_mapped_child_records( - mapping_name, doc.get(get_source_value(field_map, value_fieldname)) - ) - - mapped[key] = value - - return mapped - - -def get_mapped_child_records(mapping_name, child_docs): - mapped_child_docs = [] - mapping = frappe.get_doc("Data Migration Mapping", mapping_name) - for child_doc in child_docs: - mapped_child_docs.append(mapping.get_mapped_record(child_doc)) - - return mapped_child_docs - - -def get_value_from_fieldname(field_map, fieldname_field, doc): - field_name = get_source_value(field_map, fieldname_field) - - if field_name.startswith("eval:"): - value = frappe.safe_eval(field_name[5:], get_safe_globals()) - elif field_name[0] in ('"', "'"): - value = field_name[1:-1] - else: - value = get_source_value(doc, field_name) - return value - - -def get_source_value(source, key): - """Get value from source (object or dict) based on key""" - if isinstance(source, dict): - return source.get(key) - else: - return getattr(source, key) diff --git a/frappe/data_migration/doctype/data_migration_mapping/test_data_migration_mapping.py b/frappe/data_migration/doctype/data_migration_mapping/test_data_migration_mapping.py deleted file mode 100644 index 30d2a6bcfe..0000000000 --- a/frappe/data_migration/doctype/data_migration_mapping/test_data_migration_mapping.py +++ /dev/null @@ -1,8 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Frappe Technologies and Contributors -# License: MIT. See LICENSE -import unittest - - -class TestDataMigrationMapping(unittest.TestCase): - pass diff --git a/frappe/data_migration/doctype/data_migration_mapping_detail/__init__.py b/frappe/data_migration/doctype/data_migration_mapping_detail/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.json b/frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.json deleted file mode 100644 index ede9213f14..0000000000 --- a/frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.json +++ /dev/null @@ -1,163 +0,0 @@ -{ - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, - "creation": "2017-08-11 05:09:10.900237", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", - "fields": [ - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "remote_fieldname", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Remote Fieldname", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "local_fieldname", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Local Fieldname", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "is_child_table", - "fieldtype": "Check", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Is Child Table", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "depends_on": "is_child_table", - "fieldname": "child_table_mapping", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Child Table Mapping", - "length": 0, - "no_copy": 0, - "options": "Data Migration Mapping", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 1, - "max_attachments": 0, - "modified": "2017-09-28 17:13:31.337005", - "modified_by": "Administrator", - "module": "Data Migration", - "name": "Data Migration Mapping Detail", - "name_case": "", - "owner": "Administrator", - "permissions": [], - "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1, - "track_seen": 0 -} \ No newline at end of file diff --git a/frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.py b/frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.py deleted file mode 100644 index abd6348a26..0000000000 --- a/frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.py +++ /dev/null @@ -1,9 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Frappe Technologies and contributors -# License: MIT. See LICENSE - -from frappe.model.document import Document - - -class DataMigrationMappingDetail(Document): - pass diff --git a/frappe/data_migration/doctype/data_migration_plan/__init__.py b/frappe/data_migration/doctype/data_migration_plan/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.js b/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.js deleted file mode 100644 index 357ef2972f..0000000000 --- a/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.js +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright (c) 2017, Frappe Technologies and contributors -// For license information, please see license.txt - -frappe.ui.form.on('Data Migration Plan', { - onload(frm) { - frm.add_custom_button(__('Run'), () => frappe.new_doc('Data Migration Run', { - data_migration_plan: frm.doc.name - })); - } -}); diff --git a/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.json b/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.json deleted file mode 100644 index 2cfc2e3bd7..0000000000 --- a/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.json +++ /dev/null @@ -1,224 +0,0 @@ -{ - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "autoname": "field:plan_name", - "beta": 0, - "creation": "2017-08-11 05:15:51.482165", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", - "fields": [ - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "plan_name", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Plan Name", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 1 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "module", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Module", - "length": 0, - "no_copy": 0, - "options": "Module Def", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "mappings", - "fieldtype": "Table", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Mappings", - "length": 0, - "no_copy": 0, - "options": "Data Migration Plan Mapping", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "preprocess_method", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Preprocess Method", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "postprocess_method", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Postprocess Method", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2020-09-18 17:26:09.703215", - "modified_by": "Administrator", - "module": "Data Migration", - "name": "Data Migration Plan", - "name_case": "", - "owner": "Administrator", - "permissions": [ - { - "amend": 0, - "cancel": 0, - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "set_user_permissions": 0, - "share": 1, - "submit": 0, - "write": 1 - } - ], - "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1, - "track_seen": 0 -} \ No newline at end of file diff --git a/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.py b/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.py deleted file mode 100644 index 4118e8e7fe..0000000000 --- a/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.py +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright (c) 2021, Frappe Technologies and contributors -# License: MIT. See LICENSE - -import frappe -from frappe.custom.doctype.custom_field.custom_field import create_custom_field -from frappe.model.document import Document -from frappe.modules import get_module_path, scrub_dt_dn -from frappe.modules.export_file import create_init_py, export_to_files - - -def get_mapping_module(module, mapping_name): - app_name = frappe.db.get_value("Module Def", module, "app_name") - mapping_name = frappe.scrub(mapping_name) - module = frappe.scrub(module) - - try: - return frappe.get_module(f"{app_name}.{module}.data_migration_mapping.{mapping_name}") - except ImportError: - return None - - -class DataMigrationPlan(Document): - def on_update(self): - # update custom fields in mappings - self.make_custom_fields_for_mappings() - - if frappe.flags.in_import or frappe.flags.in_test: - return - - if frappe.local.conf.get("developer_mode"): - record_list = [["Data Migration Plan", self.name]] - - for m in self.mappings: - record_list.append(["Data Migration Mapping", m.mapping]) - - export_to_files(record_list=record_list, record_module=self.module) - - for m in self.mappings: - dt, dn = scrub_dt_dn("Data Migration Mapping", m.mapping) - create_init_py(get_module_path(self.module), dt, dn) - - def make_custom_fields_for_mappings(self): - frappe.flags.ignore_in_install = True - label = self.name + " ID" - fieldname = frappe.scrub(label) - - df = { - "label": label, - "fieldname": fieldname, - "fieldtype": "Data", - "hidden": 1, - "read_only": 1, - "unique": 1, - "no_copy": 1, - } - - for m in self.mappings: - mapping = frappe.get_doc("Data Migration Mapping", m.mapping) - create_custom_field(mapping.local_doctype, df) - mapping.migration_id_field = fieldname - mapping.save() - - # Create custom field in Deleted Document - create_custom_field("Deleted Document", df) - frappe.flags.ignore_in_install = False - - def pre_process_doc(self, mapping_name, doc): - module = get_mapping_module(self.module, mapping_name) - - if module and hasattr(module, "pre_process"): - return module.pre_process(doc) - return doc - - def post_process_doc(self, mapping_name, local_doc=None, remote_doc=None): - module = get_mapping_module(self.module, mapping_name) - - if module and hasattr(module, "post_process"): - return module.post_process(local_doc=local_doc, remote_doc=remote_doc) diff --git a/frappe/data_migration/doctype/data_migration_plan/test_data_migration_plan.py b/frappe/data_migration/doctype/data_migration_plan/test_data_migration_plan.py deleted file mode 100644 index ef3bfa3a70..0000000000 --- a/frappe/data_migration/doctype/data_migration_plan/test_data_migration_plan.py +++ /dev/null @@ -1,8 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Frappe Technologies and Contributors -# License: MIT. See LICENSE -import unittest - - -class TestDataMigrationPlan(unittest.TestCase): - pass diff --git a/frappe/data_migration/doctype/data_migration_plan_mapping/__init__.py b/frappe/data_migration/doctype/data_migration_plan_mapping/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.json b/frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.json deleted file mode 100644 index 5acf014715..0000000000 --- a/frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.json +++ /dev/null @@ -1,103 +0,0 @@ -{ - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 1, - "creation": "2017-08-11 05:15:38.390831", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", - "fields": [ - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "mapping", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Mapping", - "length": 0, - "no_copy": 0, - "options": "Data Migration Mapping", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "default": "1", - "fieldname": "enabled", - "fieldtype": "Check", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Enabled", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 1, - "max_attachments": 0, - "modified": "2017-09-20 21:43:04.908650", - "modified_by": "Administrator", - "module": "Data Migration", - "name": "Data Migration Plan Mapping", - "name_case": "", - "owner": "Administrator", - "permissions": [], - "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1, - "track_seen": 0 -} \ No newline at end of file diff --git a/frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.py b/frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.py deleted file mode 100644 index 0650f4b2c7..0000000000 --- a/frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.py +++ /dev/null @@ -1,9 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Frappe Technologies and contributors -# License: MIT. See LICENSE - -from frappe.model.document import Document - - -class DataMigrationPlanMapping(Document): - pass diff --git a/frappe/data_migration/doctype/data_migration_run/__init__.py b/frappe/data_migration/doctype/data_migration_run/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/data_migration/doctype/data_migration_run/data_migration_run.js b/frappe/data_migration/doctype/data_migration_run/data_migration_run.js deleted file mode 100644 index 82323c62f1..0000000000 --- a/frappe/data_migration/doctype/data_migration_run/data_migration_run.js +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright (c) 2017, Frappe Technologies and contributors -// For license information, please see license.txt - -frappe.ui.form.on('Data Migration Run', { - refresh: function(frm) { - if (frm.doc.status !== 'Success') { - frm.add_custom_button(__('Run'), () => frm.call('run')); - } - if (frm.doc.status === 'Started') { - frm.dashboard.add_progress(__('Percent Complete'), frm.doc.percent_complete, - __('Currently updating {0}', [frm.doc.current_mapping])); - } - } -}); diff --git a/frappe/data_migration/doctype/data_migration_run/data_migration_run.json b/frappe/data_migration/doctype/data_migration_run/data_migration_run.json deleted file mode 100644 index db77997928..0000000000 --- a/frappe/data_migration/doctype/data_migration_run/data_migration_run.json +++ /dev/null @@ -1,838 +0,0 @@ -{ - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, - "creation": "2017-09-11 12:55:27.597728", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", - "fields": [ - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "data_migration_plan", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Data Migration Plan", - "length": 0, - "no_copy": 0, - "options": "Data Migration Plan", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "data_migration_connector", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Data Migration Connector", - "length": 0, - "no_copy": 0, - "options": "Data Migration Connector", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "default": "Pending", - "fieldname": "status", - "fieldtype": "Select", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Status", - "length": 0, - "no_copy": 1, - "options": "Pending\nStarted\nPartial Success\nSuccess\nFail\nError", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "start_time", - "fieldtype": "Datetime", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Start Time", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "end_time", - "fieldtype": "Datetime", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "End Time", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "remote_id", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Remote ID", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "current_mapping", - "fieldtype": "Data", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Current Mapping", - "length": 0, - "no_copy": 1, - "options": "", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "current_mapping_start", - "fieldtype": "Int", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Current Mapping Start", - "length": 0, - "no_copy": 1, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "current_mapping_delete_start", - "fieldtype": "Int", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Current Mapping Delete Start", - "length": 0, - "no_copy": 1, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "current_mapping_type", - "fieldtype": "Select", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Current Mapping Type", - "length": 0, - "no_copy": 0, - "options": "Push\nPull", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "depends_on": "eval:(doc.status !== 'Pending')", - "fieldname": "current_mapping_action", - "fieldtype": "Select", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Current Mapping Action", - "length": 0, - "no_copy": 1, - "options": "Insert\nDelete", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "total_pages", - "fieldtype": "Int", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Total Pages", - "length": 0, - "no_copy": 1, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "percent_complete", - "fieldtype": "Percent", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Percent Complete", - "length": 0, - "no_copy": 1, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "trigger_name", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Trigger Name", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "depends_on": "eval:(doc.status !== 'Pending')", - "fieldname": "logs_sb", - "fieldtype": "Section Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Logs", - "length": 0, - "no_copy": 1, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "push_insert", - "fieldtype": "Int", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Push Insert", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "push_update", - "fieldtype": "Int", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Push Update", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "push_delete", - "fieldtype": "Int", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Push Delete", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "push_failed", - "fieldtype": "Code", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Push Failed", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "column_break_16", - "fieldtype": "Column Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "pull_insert", - "fieldtype": "Int", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Pull Insert", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "pull_update", - "fieldtype": "Int", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Pull Update", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "pull_failed", - "fieldtype": "Code", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Pull Failed", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "depends_on": "eval:doc.failed_log !== '[]'", - "fieldname": "log", - "fieldtype": "Code", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Log", - "length": 0, - "no_copy": 1, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2020-09-18 17:26:09.703215", - "modified_by": "Administrator", - "module": "Data Migration", - "name": "Data Migration Run", - "name_case": "", - "owner": "Administrator", - "permissions": [ - { - "amend": 0, - "cancel": 0, - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "set_user_permissions": 0, - "share": 1, - "submit": 0, - "write": 1 - } - ], - "quick_entry": 0, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1, - "track_seen": 0 -} \ No newline at end of file diff --git a/frappe/data_migration/doctype/data_migration_run/data_migration_run.py b/frappe/data_migration/doctype/data_migration_run/data_migration_run.py deleted file mode 100644 index c734cb105b..0000000000 --- a/frappe/data_migration/doctype/data_migration_run/data_migration_run.py +++ /dev/null @@ -1,514 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Frappe Technologies and contributors -# License: MIT. See LICENSE - -import json -import math - -import frappe -from frappe import _ -from frappe.data_migration.doctype.data_migration_mapping.data_migration_mapping import ( - get_source_value, -) -from frappe.model.document import Document -from frappe.utils import cstr - - -class DataMigrationRun(Document): - @frappe.whitelist() - def run(self): - self.begin() - if self.total_pages > 0: - self.enqueue_next_mapping() - else: - self.complete() - - def enqueue_next_mapping(self): - next_mapping_name = self.get_next_mapping_name() - if next_mapping_name: - next_mapping = self.get_mapping(next_mapping_name) - self.db_set( - dict( - current_mapping=next_mapping.name, - current_mapping_start=0, - current_mapping_delete_start=0, - current_mapping_action="Insert", - ), - notify=True, - commit=True, - ) - frappe.enqueue_doc(self.doctype, self.name, "run_current_mapping", now=frappe.flags.in_test) - else: - self.complete() - - def enqueue_next_page(self): - mapping = self.get_mapping(self.current_mapping) - percent_complete = self.percent_complete + (100.0 / self.total_pages) - fields = dict(percent_complete=percent_complete) - if self.current_mapping_action == "Insert": - start = self.current_mapping_start + mapping.page_length - fields["current_mapping_start"] = start - elif self.current_mapping_action == "Delete": - delete_start = self.current_mapping_delete_start + mapping.page_length - fields["current_mapping_delete_start"] = delete_start - - self.db_set(fields, notify=True, commit=True) - - if percent_complete < 100: - frappe.publish_realtime( - self.trigger_name, {"progress_percent": percent_complete}, user=frappe.session.user - ) - - frappe.enqueue_doc(self.doctype, self.name, "run_current_mapping", now=frappe.flags.in_test) - - def run_current_mapping(self): - try: - mapping = self.get_mapping(self.current_mapping) - - if mapping.mapping_type == "Push": - done = self.push() - elif mapping.mapping_type == "Pull": - done = self.pull() - - if done: - self.enqueue_next_mapping() - else: - self.enqueue_next_page() - - except Exception as e: - self.db_set("status", "Error", notify=True, commit=True) - print("Data Migration Run failed") - print(frappe.get_traceback()) - self.execute_postprocess("Error") - raise e - - def get_last_modified_condition(self): - last_run_timestamp = frappe.db.get_value( - "Data Migration Run", - dict( - data_migration_plan=self.data_migration_plan, - data_migration_connector=self.data_migration_connector, - name=("!=", self.name), - ), - "modified", - ) - if last_run_timestamp: - condition = dict(modified=(">", last_run_timestamp)) - else: - condition = {} - return condition - - def begin(self): - plan_active_mappings = [m for m in self.get_plan().mappings if m.enabled] - self.mappings = [ - frappe.get_doc("Data Migration Mapping", m.mapping) for m in plan_active_mappings - ] - - total_pages = 0 - for m in [mapping for mapping in self.mappings]: - if m.mapping_type == "Push": - count = float(self.get_count(m)) - page_count = math.ceil(count / m.page_length) - total_pages += page_count - if m.mapping_type == "Pull": - total_pages += 10 - - self.db_set( - dict( - status="Started", - current_mapping=None, - current_mapping_start=0, - current_mapping_delete_start=0, - percent_complete=0, - current_mapping_action="Insert", - total_pages=total_pages, - ), - notify=True, - commit=True, - ) - - def complete(self): - fields = dict() - - push_failed = self.get_log("push_failed", []) - pull_failed = self.get_log("pull_failed", []) - - status = "Partial Success" - - if not push_failed and not pull_failed: - status = "Success" - fields["percent_complete"] = 100 - - fields["status"] = status - - self.db_set(fields, notify=True, commit=True) - - self.execute_postprocess(status) - - frappe.publish_realtime(self.trigger_name, {"progress_percent": 100}, user=frappe.session.user) - - def execute_postprocess(self, status): - # Execute post process - postprocess_method_path = self.get_plan().postprocess_method - - if postprocess_method_path: - frappe.get_attr(postprocess_method_path)( - { - "status": status, - "stats": { - "push_insert": self.push_insert, - "push_update": self.push_update, - "push_delete": self.push_delete, - "pull_insert": self.pull_insert, - "pull_update": self.pull_update, - }, - } - ) - - def get_plan(self): - if not hasattr(self, "plan"): - self.plan = frappe.get_doc("Data Migration Plan", self.data_migration_plan) - return self.plan - - def get_mapping(self, mapping_name): - if hasattr(self, "mappings"): - for m in self.mappings: - if m.name == mapping_name: - return m - return frappe.get_doc("Data Migration Mapping", mapping_name) - - def get_next_mapping_name(self): - mappings = [m for m in self.get_plan().mappings if m.enabled] - if not self.current_mapping: - # first - return mappings[0].mapping - for i, d in enumerate(mappings): - if i == len(mappings) - 1: - # last - return None - if d.mapping == self.current_mapping: - return mappings[i + 1].mapping - - raise frappe.ValidationError("Mapping Broken") - - def get_data(self, filters): - mapping = self.get_mapping(self.current_mapping) - or_filters = self.get_or_filters(mapping) - start = self.current_mapping_start - - data = [] - doclist = frappe.get_all( - mapping.local_doctype, - filters=filters, - or_filters=or_filters, - start=start, - page_length=mapping.page_length, - ) - - for d in doclist: - doc = frappe.get_doc(mapping.local_doctype, d["name"]) - data.append(doc) - return data - - def get_new_local_data(self): - """Fetch newly inserted local data using `frappe.get_all`. Used during Push""" - mapping = self.get_mapping(self.current_mapping) - filters = mapping.get_filters() or {} - - # new docs dont have migration field set - filters.update({mapping.migration_id_field: ""}) - - return self.get_data(filters) - - def get_updated_local_data(self): - """Fetch local updated data using `frappe.get_all`. Used during Push""" - mapping = self.get_mapping(self.current_mapping) - filters = mapping.get_filters() or {} - - # existing docs must have migration field set - filters.update({mapping.migration_id_field: ("!=", "")}) - - return self.get_data(filters) - - def get_deleted_local_data(self): - """Fetch local deleted data using `frappe.get_all`. Used during Push""" - mapping = self.get_mapping(self.current_mapping) - filters = self.get_last_modified_condition() - filters.update({"deleted_doctype": mapping.local_doctype}) - - data = frappe.get_all("Deleted Document", fields=["name", "data"], filters=filters) - - _data = [] - for d in data: - doc = json.loads(d.data) - if doc.get(mapping.migration_id_field): - doc["_deleted_document_name"] = d["name"] - _data.append(doc) - - return _data - - def get_remote_data(self): - """Fetch data from remote using `connection.get`. Used during Pull""" - mapping = self.get_mapping(self.current_mapping) - start = self.current_mapping_start - filters = mapping.get_filters() or {} - connection = self.get_connection() - - return connection.get( - mapping.remote_objectname, - fields=["*"], - filters=filters, - start=start, - page_length=mapping.page_length, - ) - - def get_count(self, mapping): - filters = mapping.get_filters() or {} - or_filters = self.get_or_filters(mapping) - - to_insert = frappe.get_all( - mapping.local_doctype, ["count(name) as total"], filters=filters, or_filters=or_filters - )[0].total - - to_delete = frappe.get_all( - "Deleted Document", - ["count(name) as total"], - filters={"deleted_doctype": mapping.local_doctype}, - or_filters=or_filters, - )[0].total - - return to_insert + to_delete - - def get_or_filters(self, mapping): - or_filters = self.get_last_modified_condition() - - # docs whose migration_id_field is not set - # failed in the previous run, include those too - or_filters.update({mapping.migration_id_field: ("=", "")}) - - return or_filters - - def get_connection(self): - if not hasattr(self, "connection"): - self.connection = frappe.get_doc( - "Data Migration Connector", self.data_migration_connector - ).get_connection() - - return self.connection - - def push(self): - self.db_set("current_mapping_type", "Push") - done = True - - if self.current_mapping_action == "Insert": - done = self._push_insert() - - elif self.current_mapping_action == "Update": - done = self._push_update() - - elif self.current_mapping_action == "Delete": - done = self._push_delete() - - return done - - def _push_insert(self): - """Inserts new local docs on remote""" - mapping = self.get_mapping(self.current_mapping) - connection = self.get_connection() - data = self.get_new_local_data() - - for d in data: - # pre process before insert - doc = self.pre_process_doc(d) - doc = mapping.get_mapped_record(doc) - - try: - response_doc = connection.insert(mapping.remote_objectname, doc) - frappe.db.set_value( - mapping.local_doctype, - d.name, - mapping.migration_id_field, - response_doc[connection.name_field], - update_modified=False, - ) - frappe.db.commit() - self.update_log("push_insert", 1) - # post process after insert - self.post_process_doc(local_doc=d, remote_doc=response_doc) - except Exception as e: - self.update_log("push_failed", {d.name: cstr(e)}) - - # update page_start - self.db_set("current_mapping_start", self.current_mapping_start + mapping.page_length) - - if len(data) < mapping.page_length: - # done, no more new data to insert - self.db_set({"current_mapping_action": "Update", "current_mapping_start": 0}) - # not done with this mapping - return False - - def _push_update(self): - """Updates local modified docs on remote""" - mapping = self.get_mapping(self.current_mapping) - connection = self.get_connection() - data = self.get_updated_local_data() - - for d in data: - migration_id_value = d.get(mapping.migration_id_field) - # pre process before update - doc = self.pre_process_doc(d) - doc = mapping.get_mapped_record(doc) - try: - response_doc = connection.update(mapping.remote_objectname, doc, migration_id_value) - self.update_log("push_update", 1) - # post process after update - self.post_process_doc(local_doc=d, remote_doc=response_doc) - except Exception as e: - self.update_log("push_failed", {d.name: cstr(e)}) - - # update page_start - self.db_set("current_mapping_start", self.current_mapping_start + mapping.page_length) - - if len(data) < mapping.page_length: - # done, no more data to update - self.db_set({"current_mapping_action": "Delete", "current_mapping_start": 0}) - # not done with this mapping - return False - - def _push_delete(self): - """Deletes docs deleted from local on remote""" - mapping = self.get_mapping(self.current_mapping) - connection = self.get_connection() - data = self.get_deleted_local_data() - - for d in data: - # Deleted Document also has a custom field for migration_id - migration_id_value = d.get(mapping.migration_id_field) - # pre process before update - self.pre_process_doc(d) - try: - response_doc = connection.delete(mapping.remote_objectname, migration_id_value) - self.update_log("push_delete", 1) - # post process only when action is success - self.post_process_doc(local_doc=d, remote_doc=response_doc) - except Exception as e: - self.update_log("push_failed", {d.name: cstr(e)}) - - # update page_start - self.db_set("current_mapping_start", self.current_mapping_start + mapping.page_length) - - if len(data) < mapping.page_length: - # done, no more new data to delete - # done with this mapping - return True - - def pull(self): - self.db_set("current_mapping_type", "Pull") - - connection = self.get_connection() - mapping = self.get_mapping(self.current_mapping) - data = self.get_remote_data() - - for d in data: - migration_id_value = get_source_value(d, connection.name_field) - doc = self.pre_process_doc(d) - doc = mapping.get_mapped_record(doc) - - if migration_id_value: - try: - if not local_doc_exists(mapping, migration_id_value): - # insert new local doc - local_doc = insert_local_doc(mapping, doc) - - self.update_log("pull_insert", 1) - # set migration id - frappe.db.set_value( - mapping.local_doctype, - local_doc.name, - mapping.migration_id_field, - migration_id_value, - update_modified=False, - ) - frappe.db.commit() - else: - # update doc - local_doc = update_local_doc(mapping, doc, migration_id_value) - self.update_log("pull_update", 1) - # post process doc after success - self.post_process_doc(remote_doc=d, local_doc=local_doc) - except Exception as e: - # failed, append to log - self.update_log("pull_failed", {migration_id_value: cstr(e)}) - - if len(data) < mapping.page_length: - # last page, done with pull - return True - - def pre_process_doc(self, doc): - plan = self.get_plan() - doc = plan.pre_process_doc(self.current_mapping, doc) - return doc - - def post_process_doc(self, local_doc=None, remote_doc=None): - plan = self.get_plan() - doc = plan.post_process_doc(self.current_mapping, local_doc=local_doc, remote_doc=remote_doc) - return doc - - def set_log(self, key, value): - value = json.dumps(value) if "_failed" in key else value - self.db_set(key, value) - - def update_log(self, key, value=None): - """ - Helper for updating logs, - push_failed and pull_failed are stored as json, - other keys are stored as int - """ - if "_failed" in key: - # json - self.set_log(key, self.get_log(key, []) + [value]) - else: - # int - self.set_log(key, self.get_log(key, 0) + (value or 1)) - - def get_log(self, key, default=None): - value = self.db_get(key) - if "_failed" in key: - if not value: - value = json.dumps(default) - value = json.loads(value) - return value or default - - -def insert_local_doc(mapping, doc): - try: - # insert new doc - if not doc.doctype: - doc.doctype = mapping.local_doctype - doc = frappe.get_doc(doc).insert() - return doc - except Exception: - print("Data Migration Run failed: Error in Pull insert") - print(frappe.get_traceback()) - return None - - -def update_local_doc(mapping, remote_doc, migration_id_value): - try: - # migration id value is set in migration_id_field in mapping.local_doctype - docname = frappe.db.get_value( - mapping.local_doctype, filters={mapping.migration_id_field: migration_id_value} - ) - - doc = frappe.get_doc(mapping.local_doctype, docname) - doc.update(remote_doc) - doc.save() - return doc - except Exception: - print("Data Migration Run failed: Error in Pull update") - print(frappe.get_traceback()) - return None - - -def local_doc_exists(mapping, migration_id_value): - return frappe.db.exists(mapping.local_doctype, {mapping.migration_id_field: migration_id_value}) diff --git a/frappe/data_migration/doctype/data_migration_run/test_data_migration_run.py b/frappe/data_migration/doctype/data_migration_run/test_data_migration_run.py deleted file mode 100644 index 0357b1e0f5..0000000000 --- a/frappe/data_migration/doctype/data_migration_run/test_data_migration_run.py +++ /dev/null @@ -1,128 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Frappe Technologies and Contributors -# License: MIT. See LICENSE -import unittest - -import frappe - - -class TestDataMigrationRun(unittest.TestCase): - def test_run(self): - create_plan() - - description = "data migration todo" - new_todo = frappe.get_doc({"doctype": "ToDo", "description": description}).insert() - - event_subject = "data migration event" - frappe.get_doc( - dict( - doctype="Event", - subject=event_subject, - repeat_on="Monthly", - starts_on=frappe.utils.now_datetime(), - ) - ).insert() - - run = frappe.get_doc( - { - "doctype": "Data Migration Run", - "data_migration_plan": "ToDo Sync", - "data_migration_connector": "Local Connector", - } - ).insert() - - run.run() - self.assertEqual(run.db_get("status"), "Success") - - self.assertEqual(run.db_get("push_insert"), 1) - self.assertEqual(run.db_get("pull_insert"), 1) - - todo = frappe.get_doc("ToDo", new_todo.name) - self.assertTrue(todo.todo_sync_id) - - # Pushed Event - event = frappe.get_doc("Event", todo.todo_sync_id) - self.assertEqual(event.subject, description) - - # Pulled ToDo - created_todo = frappe.get_doc("ToDo", {"description": event_subject}) - self.assertEqual(created_todo.description, event_subject) - - todo_list = frappe.get_list( - "ToDo", filters={"description": "data migration todo"}, fields=["name"] - ) - todo_name = todo_list[0].name - - todo = frappe.get_doc("ToDo", todo_name) - todo.description = "data migration todo updated" - todo.save() - - run = frappe.get_doc( - { - "doctype": "Data Migration Run", - "data_migration_plan": "ToDo Sync", - "data_migration_connector": "Local Connector", - } - ).insert() - - run.run() - - # Update - self.assertEqual(run.db_get("status"), "Success") - self.assertEqual(run.db_get("pull_update"), 1) - - -def create_plan(): - frappe.get_doc( - { - "doctype": "Data Migration Mapping", - "mapping_name": "Todo to Event", - "remote_objectname": "Event", - "remote_primary_key": "name", - "mapping_type": "Push", - "local_doctype": "ToDo", - "fields": [ - {"remote_fieldname": "subject", "local_fieldname": "description"}, - { - "remote_fieldname": "starts_on", - "local_fieldname": "eval:frappe.utils.get_datetime_str(frappe.utils.get_datetime())", - }, - ], - "condition": '{"description": "data migration todo" }', - } - ).insert(ignore_if_duplicate=True) - - frappe.get_doc( - { - "doctype": "Data Migration Mapping", - "mapping_name": "Event to ToDo", - "remote_objectname": "Event", - "remote_primary_key": "name", - "local_doctype": "ToDo", - "local_primary_key": "name", - "mapping_type": "Pull", - "condition": '{"subject": "data migration event" }', - "fields": [{"remote_fieldname": "subject", "local_fieldname": "description"}], - } - ).insert(ignore_if_duplicate=True) - - frappe.get_doc( - { - "doctype": "Data Migration Plan", - "plan_name": "ToDo Sync", - "module": "Core", - "mappings": [{"mapping": "Todo to Event"}, {"mapping": "Event to ToDo"}], - } - ).insert(ignore_if_duplicate=True) - - frappe.get_doc( - { - "doctype": "Data Migration Connector", - "connector_name": "Local Connector", - "connector_type": "Frappe", - # connect to same host. - "hostname": frappe.conf.host_name or frappe.utils.get_site_url(frappe.local.site), - "username": "Administrator", - "password": frappe.conf.get("admin_password") or "admin", - } - ).insert(ignore_if_duplicate=True) diff --git a/frappe/database/database.py b/frappe/database/database.py index 2af890bdb1..cd53c3e1fd 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -19,12 +19,21 @@ import frappe import frappe.defaults import frappe.model.meta from frappe import _ +from frappe.exceptions import DoesNotExistError from frappe.model.utils.link_count import flush_local_link_count from frappe.query_builder.functions import Count from frappe.query_builder.utils import DocType -from frappe.utils import cast, get_datetime, get_table_name, getdate, now, sbool +from frappe.utils import cast as cast_fieldtype +from frappe.utils import get_datetime, get_table_name, getdate, now, sbool -from .query import Query +IFNULL_PATTERN = re.compile(r"ifnull\(", flags=re.IGNORECASE) +INDEX_PATTERN = re.compile(r"\s*\([^)]+\)\s*") +SINGLE_WORD_PATTERN = re.compile(r'([`"]?)(tab([A-Z]\w+))\1') +MULTI_WORD_PATTERN = re.compile(r'([`"])(tab([A-Z]\w+)( [A-Z]\w+)+)\1') + + +def is_query_type(query: str, query_type: Union[str, Tuple[str]]) -> bool: + return query.lstrip().split(maxsplit=1)[0].lower().startswith(query_type) class Database(object): @@ -66,7 +75,15 @@ class Database(object): self.password = password or frappe.conf.db_password self.value_cache = {} - self.query = Query() + + @property + def query(self): + if not hasattr(self, "_query"): + from .query import Query + + self._query = Query() + del Query + return self._query def setup_type_map(self): pass @@ -142,9 +159,8 @@ class Database(object): # remove whitespace / indentation from start and end of query query = query.strip() - if re.search(r"ifnull\(", query, flags=re.IGNORECASE): - # replaces ifnull in query with coalesce - query = re.sub(r"ifnull\(", "coalesce(", query, flags=re.IGNORECASE) + # replaces ifnull in query with coalesce + query = IFNULL_PATTERN.sub("coalesce(", query) if not self._conn: self.connect() @@ -192,6 +208,9 @@ class Database(object): elif frappe.conf.db_type == "postgres": # TODO: added temporarily + import traceback + + traceback.print_stack() frappe.errprint(f"Error in query:\n{e}") raise @@ -235,7 +254,7 @@ class Database(object): # debug if debug: - if explain and query.strip().lower().startswith("select"): + if explain and is_query_type(query, "select"): self.explain_query(query, values) mogrified_query = mogrified_query or self.mogrify(query, values) frappe.errprint(mogrified_query) @@ -278,7 +297,7 @@ class Database(object): """ return self.sql(query, values, **kwargs, debug=debug, pluck=True) - def sql_ddl(self, query, values=(), debug=False): + def sql_ddl(self, query, debug=False): """Commit and execute a query. DDL (Data Definition Language) queries that alter schema autocommit in MariaDB.""" self.commit() @@ -290,7 +309,7 @@ class Database(object): could cause the system to hang.""" self.check_implicit_commit(query) - if query and query.strip().lower() in ("commit", "rollback"): + if query and is_query_type(query, ("commit", "rollback")): self.transaction_writes = 0 if query[:6].lower() in ("update", "insert", "delete"): @@ -307,8 +326,7 @@ class Database(object): if ( self.transaction_writes and query - and query.strip().split()[0].lower() - in ["start", "alter", "drop", "create", "begin", "truncate"] + and is_query_type(query, ("start", "alter", "drop", "create", "begin", "truncate")) ): raise Exception("This statement can cause implicit commit") @@ -331,7 +349,7 @@ class Database(object): @staticmethod def clear_db_table_cache(query): - if query and query.strip().split()[0].lower() in {"drop", "create"}: + if query and is_query_type(query, ("drop", "create")): frappe.cache().delete_key("db_tables") @staticmethod @@ -600,24 +618,44 @@ class Database(object): else: return r and [[i[1] for i in r]] or [] - def get_singles_dict(self, doctype, debug=False, *, for_update=False): + def get_singles_dict(self, doctype, debug=False, *, for_update=False, cast=False): """Get Single DocType as dict. :param doctype: DocType of the single object whose value is requested + :param debug: Execute query in debug mode - print to STDOUT + :param for_update: Take `FOR UPDATE` lock on the records + :param cast: Cast values to Python data types based on field type Example: # Get coulmn and value of the single doctype Accounts Settings account_settings = frappe.db.get_singles_dict("Accounts Settings") """ - result = self.query.get_sql( + queried_result = self.query.get_sql( "Singles", filters={"doctype": doctype}, fields=["field", "value"], for_update=for_update, - ).run() + ).run(debug=debug) - return frappe._dict(result) + if not cast: + return frappe._dict(queried_result) + + try: + meta = frappe.get_meta(doctype) + except DoesNotExistError: + return frappe._dict(queried_result) + + return_value = frappe._dict() + + for fieldname, value in queried_result: + if df := meta.get_field(fieldname): + casted_value = cast_fieldtype(df.fieldtype, value) + else: + casted_value = value + return_value[fieldname] = casted_value + + return return_value @staticmethod def get_all(*args, **kwargs): @@ -680,7 +718,7 @@ class Database(object): _("Invalid field name: {0}").format(frappe.bold(fieldname)), self.InvalidColumnName ) - val = cast(df.fieldtype, val) + val = cast_fieldtype(df.fieldtype, val) self.value_cache[doctype][fieldname] = val @@ -907,6 +945,9 @@ class Database(object): frappe.call(method[0], *(method[1] or []), **(method[2] or {})) self.sql("commit") + if frappe.conf.db_type == "postgres": + # Postgres requires explicitly starting new transaction + self.begin() frappe.local.rollback_observers = [] self.flush_realtime_log() @@ -943,7 +984,7 @@ class Database(object): else: self.sql("rollback") self.begin() - for obj in frappe.local.rollback_observers: + for obj in dict.fromkeys(frappe.local.rollback_observers): if hasattr(obj, "on_rollback"): obj.on_rollback() frappe.local.rollback_observers = [] @@ -1102,8 +1143,7 @@ class Database(object): def get_index_name(fields): index_name = "_".join(fields) + "_index" # remove index length if present e.g. (10) from index name - index_name = re.sub(r"\s*\([^)]+\)\s*", r"", index_name) - return index_name + return INDEX_PATTERN.sub(r"", index_name) def get_system_setting(self, key): def _load_system_settings(): @@ -1153,12 +1193,11 @@ class Database(object): Doctype name can be passed directly, it will be pre-pended with `tab`. """ - values = () filters = filters or kwargs.get("conditions") query = self.query.build_conditions(table=doctype, filters=filters).delete() if "debug" not in kwargs: kwargs["debug"] = debug - return self.sql(query, values, **kwargs) + return query.run(**kwargs) def truncate(self, doctype: str): """Truncate a table in the database. This runs a DDL command `TRUNCATE TABLE`. @@ -1166,8 +1205,7 @@ class Database(object): Doctype name can be passed directly, it will be pre-pended with `tab`. """ - table = doctype if doctype.startswith("__") else f"tab{doctype}" - return self.sql_ddl(f"truncate `{table}`") + return self.sql_ddl(f"truncate `{get_table_name(doctype)}`") def clear_table(self, doctype): return self.truncate(doctype) @@ -1181,8 +1219,8 @@ class Database(object): def log_touched_tables(self, query, values=None): if values: - query = frappe.safe_decode(self.mogrify(query, values)) - if query.strip().lower().split()[0] in ("insert", "delete", "update", "alter", "drop", "rename"): + query = frappe.safe_decode(self._cursor.mogrify(query, values)) + if is_query_type(query, ("insert", "delete", "update", "alter", "drop", "rename")): # single_word_regex is designed to match following patterns # `tabXxx`, tabXxx and "tabXxx" @@ -1197,11 +1235,9 @@ class Database(object): # and are continued with multiple words that start with a captital letter # e.g. 'tabXxx' or 'tabXxx Xxx' or 'tabXxx Xxx Xxx' and so on - single_word_regex = r'([`"]?)(tab([A-Z]\w+))\1' - multi_word_regex = r'([`"])(tab([A-Z]\w+)( [A-Z]\w+)+)\1' tables = [] - for regex in (single_word_regex, multi_word_regex): - tables += [groups[1] for groups in re.findall(regex, query)] + for regex in (SINGLE_WORD_PATTERN, MULTI_WORD_PATTERN): + tables += [groups[1] for groups in regex.findall(query)] if frappe.flags.touched_tables is None: frappe.flags.touched_tables = set() diff --git a/frappe/database/postgres/database.py b/frappe/database/postgres/database.py index fc2a6b7941..0bcc2adcb2 100644 --- a/frappe/database/postgres/database.py +++ b/frappe/database/postgres/database.py @@ -30,6 +30,11 @@ DEC2FLOAT = psycopg2.extensions.new_type( psycopg2.extensions.register_type(DEC2FLOAT) +LOCATE_SUB_PATTERN = re.compile(r"locate\(([^,]+),([^)]+)(\)?)\)", flags=re.IGNORECASE) +LOCATE_QUERY_PATTERN = re.compile(r"locate\(", flags=re.IGNORECASE) +PG_TRANSFORM_PATTERN = re.compile(r"([=><]+)\s*([+-]?\d+)(\.0)?(?![a-zA-Z\.\d])") +FROM_TAB_PATTERN = re.compile(r"from tab([\w-]*)", flags=re.IGNORECASE) + class PostgresExceptionUtil: ProgrammingError = psycopg2.ProgrammingError @@ -383,12 +388,10 @@ class PostgresDatabase(PostgresExceptionUtil, Database): def modify_query(query): """ "Modifies query according to the requirements of postgres""" # replace ` with " for definitions - query = str(query) - query = query.replace("`", '"') + query = str(query).replace("`", '"') query = replace_locate_with_strpos(query) # select from requires "" - if re.search("from tab", query, flags=re.IGNORECASE): - query = re.sub(r"from tab([\w-]*)", r'from "tab\1"', query, flags=re.IGNORECASE) + query = FROM_TAB_PATTERN.sub(r'from "tab\1"', query) # only find int (with/without signs), ignore decimals (with/without signs), ignore hashes (which start with numbers), # drop .0 from decimals and add quotes around them @@ -397,8 +400,7 @@ def modify_query(query): # >>> re.sub(r"([=><]+)\s*([+-]?\d+)(\.0)?(?![a-zA-Z\.\d])", r"\1 '\2'", query) # "c='abcd' , a >= '45', b = '-45', c = '40', d= '4500', e=3500.53, f=40psdfsd, g= '9092094312', h=12.00023 - query = re.sub(r"([=><]+)\s*([+-]?\d+)(\.0)?(?![a-zA-Z\.\d])", r"\1 '\2'", query) - return query + return PG_TRANSFORM_PATTERN.sub(r"\1 '\2'", query) def modify_values(values): @@ -431,8 +433,6 @@ def modify_values(values): def replace_locate_with_strpos(query): # strpos is the locate equivalent in postgres - if re.search(r"locate\(", query, flags=re.IGNORECASE): - query = re.sub( - r"locate\(([^,]+),([^)]+)(\)?)\)", r"strpos(\2\3, \1)", query, flags=re.IGNORECASE - ) + if LOCATE_QUERY_PATTERN.search(query): + query = LOCATE_SUB_PATTERN.sub(r"strpos(\2\3, \1)", query) return query diff --git a/frappe/database/postgres/setup_db.py b/frappe/database/postgres/setup_db.py index 5584c098ce..9a7f2b43c4 100644 --- a/frappe/database/postgres/setup_db.py +++ b/frappe/database/postgres/setup_db.py @@ -6,6 +6,7 @@ import frappe def setup_database(force, source_sql=None, verbose=False): root_conn = get_root_connection(frappe.flags.root_login, frappe.flags.root_password) root_conn.commit() + root_conn.sql("end") root_conn.sql("DROP DATABASE IF EXISTS `{0}`".format(frappe.conf.db_name)) root_conn.sql("DROP USER IF EXISTS {0}".format(frappe.conf.db_name)) root_conn.sql("CREATE DATABASE `{0}`".format(frappe.conf.db_name)) diff --git a/frappe/database/query.py b/frappe/database/query.py index c2dd076f8f..bd5b05fb3c 100644 --- a/frappe/database/query.py +++ b/frappe/database/query.py @@ -8,6 +8,9 @@ from frappe import _ from frappe.model.db_query import get_timespan_date_range from frappe.query_builder import Criterion, Field, Order, Table +TAB_PATTERN = re.compile("^tab") +WORDS_PATTERN = re.compile(r"\w+") + def like(key: Field, value: str) -> frappe.qb: """Wrapper method for `LIKE` @@ -262,14 +265,7 @@ class Query: return conditions if isinstance(filters, list): for f in filters: - if not isinstance(f, (list, tuple)): - _operator = self.OPERATOR_MAP[filters[1].casefold()] - if not isinstance(filters[0], str): - conditions = make_function(filters[0], filters[2]) - break - conditions = conditions.where(_operator(Field(filters[0]), filters[2])) - break - else: + if isinstance(f, (list, tuple)): _operator = self.OPERATOR_MAP[f[-2].casefold()] if len(f) == 4: table_object = self.get_table(f[0]) @@ -277,6 +273,15 @@ class Query: else: _field = Field(f[0]) conditions = conditions.where(_operator(_field, f[-1])) + elif isinstance(f, dict): + conditions = self.dict_query(table, f, **kwargs) + else: + _operator = self.OPERATOR_MAP[filters[1].casefold()] + if not isinstance(filters[0], str): + conditions = make_function(filters[0], filters[2]) + break + conditions = conditions.where(_operator(Field(filters[0]), filters[2])) + break return self.add_conditions(conditions, **kwargs) @@ -383,7 +388,7 @@ class Permission: doctype = [doctype] for dt in doctype: - dt = re.sub("^tab", "", dt) + dt = TAB_PATTERN.sub("", dt) if not frappe.has_permission( dt, "select", @@ -399,4 +404,4 @@ class Permission: @staticmethod def get_tables_from_query(query: str): - return [table for table in re.findall(r"\w+", query) if table.startswith("tab")] + return [table for table in WORDS_PATTERN.findall(query) if table.startswith("tab")] diff --git a/frappe/database/schema.py b/frappe/database/schema.py index 19af447aae..9a8307ddae 100644 --- a/frappe/database/schema.py +++ b/frappe/database/schema.py @@ -4,6 +4,9 @@ import frappe from frappe import _ from frappe.utils import cint, cstr, flt +SPECIAL_CHAR_PATTERN = re.compile(r"[\W]", flags=re.UNICODE) +VARCHAR_CAST_PATTERN = re.compile(r"varchar\(([\d]+)\)") + class InvalidColumnName(frappe.ValidationError): pass @@ -130,7 +133,7 @@ class DBTable: if not current_col: continue current_type = self.current_columns[col.fieldname]["type"] - current_length = re.findall(r"varchar\(([\d]+)\)", current_type) + current_length = VARCHAR_CAST_PATTERN.findall(current_type) if not current_length: # case when the field is no longer a varchar continue @@ -304,8 +307,7 @@ class DbColumn: def validate_column_name(n): - special_characters = re.findall(r"[\W]", n, re.UNICODE) - if special_characters: + if special_characters := SPECIAL_CHAR_PATTERN.findall(n): special_characters = ", ".join('"{0}"'.format(c) for c in special_characters) frappe.throw( _("Fieldname {0} cannot have special characters like {1}").format( diff --git a/frappe/desk/doctype/dashboard_chart/dashboard_chart.py b/frappe/desk/doctype/dashboard_chart/dashboard_chart.py index 246c9ad4cd..ca29bad33b 100644 --- a/frappe/desk/doctype/dashboard_chart/dashboard_chart.py +++ b/frappe/desk/doctype/dashboard_chart/dashboard_chart.py @@ -14,6 +14,7 @@ from frappe.model.naming import append_number_if_name_exists from frappe.modules.export_file import export_to_files from frappe.utils import cint, get_datetime, getdate, now_datetime, nowdate from frappe.utils.dashboard import cache_source +from frappe.utils.data import format_date from frappe.utils.dateutils import ( get_dates_from_timegrain, get_from_date_from_timespan, @@ -221,13 +222,16 @@ def get_chart_config(chart, filters, timespan, timegrain, from_date, to_date): result = get_result(data, timegrain, from_date, to_date, chart.chart_type) - chart_config = { - "labels": [get_period(r[0], timegrain) for r in result], + return { + "labels": [ + format_date(get_period(r[0], timegrain)) + if timegrain in ("Daily", "Weekly") + else get_period(r[0], timegrain) + for r in result + ], "datasets": [{"name": chart.name, "values": [r[1] for r in result]}], } - return chart_config - def get_heatmap_chart_config(chart, filters, heatmap_year): aggregate_function = get_aggregate_function(chart.chart_type) diff --git a/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py b/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py index 94ea1af35c..ca84b2c301 100644 --- a/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py +++ b/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # License: MIT. See LICENSE -import unittest + from datetime import datetime from unittest.mock import patch @@ -9,11 +9,12 @@ from dateutil.relativedelta import relativedelta import frappe from frappe.desk.doctype.dashboard_chart.dashboard_chart import get +from frappe.tests.utils import FrappeTestCase from frappe.utils import formatdate, get_last_day, getdate from frappe.utils.dateutils import get_period, get_period_ending -class TestDashboardChart(unittest.TestCase): +class TestDashboardChart(FrappeTestCase): def test_period_ending(self): self.assertEqual(get_period_ending("2019-04-10", "Daily"), getdate("2019-04-10")) @@ -57,8 +58,6 @@ class TestDashboardChart(unittest.TestCase): self.assertEqual(result.get("labels")[idx], get_period(month)) cur_date += relativedelta(months=1) - frappe.db.rollback() - def test_empty_dashboard_chart(self): if frappe.db.exists("Dashboard Chart", "Test Empty Dashboard Chart"): frappe.delete_doc("Dashboard Chart", "Test Empty Dashboard Chart") @@ -89,8 +88,6 @@ class TestDashboardChart(unittest.TestCase): self.assertEqual(result.get("labels")[idx], get_period(month)) cur_date += relativedelta(months=1) - frappe.db.rollback() - def test_chart_wih_one_value(self): if frappe.db.exists("Dashboard Chart", "Test Empty Dashboard Chart 2"): frappe.delete_doc("Dashboard Chart", "Test Empty Dashboard Chart 2") @@ -127,8 +124,6 @@ class TestDashboardChart(unittest.TestCase): # only 1 data point with value self.assertEqual(result.get("datasets")[0].get("values")[2], 0) - frappe.db.rollback() - def test_group_by_chart_type(self): if frappe.db.exists("Dashboard Chart", "Test Group By Dashboard Chart"): frappe.delete_doc("Dashboard Chart", "Test Group By Dashboard Chart") @@ -151,8 +146,6 @@ class TestDashboardChart(unittest.TestCase): self.assertEqual(result.get("datasets")[0].get("values")[0], todo_status_count) - frappe.db.rollback() - def test_daily_dashboard_chart(self): insert_test_records() @@ -180,11 +173,10 @@ class TestDashboardChart(unittest.TestCase): self.assertEqual(result.get("datasets")[0].get("values"), [200.0, 400.0, 300.0, 0.0, 100.0, 0.0]) self.assertEqual( - result.get("labels"), ["06-01-19", "07-01-19", "08-01-19", "09-01-19", "10-01-19", "11-01-19"] + result.get("labels"), + ["06-01-2019", "07-01-2019", "08-01-2019", "09-01-2019", "10-01-2019", "11-01-2019"], ) - frappe.db.rollback() - def test_weekly_dashboard_chart(self): insert_test_records() @@ -212,9 +204,7 @@ class TestDashboardChart(unittest.TestCase): result = get(chart_name="Test Weekly Dashboard Chart", refresh=1) self.assertEqual(result.get("datasets")[0].get("values"), [50.0, 300.0, 800.0, 0.0]) - self.assertEqual(result.get("labels"), ["30-12-18", "06-01-19", "13-01-19", "20-01-19"]) - - frappe.db.rollback() + self.assertEqual(result.get("labels"), ["12-30-2018", "06-01-2019", "01-13-2019", "01-20-2019"]) def test_avg_dashboard_chart(self): insert_test_records() @@ -241,10 +231,39 @@ class TestDashboardChart(unittest.TestCase): with patch.object(frappe.utils.data, "get_first_day_of_the_week", return_value="Monday"): result = get(chart_name="Test Average Dashboard Chart", refresh=1) - self.assertEqual(result.get("labels"), ["30-12-18", "06-01-19", "13-01-19", "20-01-19"]) + self.assertEqual(result.get("labels"), ["12-30-2018", "06-01-2019", "01-13-2019", "01-20-2019"]) self.assertEqual(result.get("datasets")[0].get("values"), [50.0, 150.0, 266.6666666666667, 0.0]) - frappe.db.rollback() + def test_user_date_label_dashboard_chart(self): + frappe.delete_doc_if_exists("Dashboard Chart", "Test Dashboard Chart Date Label") + + frappe.get_doc( + dict( + doctype="Dashboard Chart", + chart_name="Test Dashboard Chart Date Label", + chart_type="Count", + document_type="DocType", + based_on="creation", + timespan="Select Date Range", + time_interval="Weekly", + from_date=datetime(2018, 12, 30), + to_date=datetime(2019, 1, 15), + filters_json="[]", + timeseries=1, + ) + ).insert() + + with patch.object(frappe.utils.data, "get_user_date_format", return_value="dd.mm.yyyy"): + result = get(chart_name="Test Dashboard Chart Date Label") + self.assertEqual( + sorted(result.get("labels")), sorted(["01.05.2019", "01.12.2019", "19.01.2019"]) + ) + + with patch.object(frappe.utils.data, "get_user_date_format", return_value="mm-dd-yyyy"): + result = get(chart_name="Test Dashboard Chart Date Label") + self.assertEqual( + sorted(result.get("labels")), sorted(["01-19-2019", "05-01-2019", "12-01-2019"]) + ) def insert_test_records(): diff --git a/frappe/desk/doctype/note/note.py b/frappe/desk/doctype/note/note.py index d67ecda594..a709b80f1d 100644 --- a/frappe/desk/doctype/note/note.py +++ b/frappe/desk/doctype/note/note.py @@ -1,16 +1,18 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +import re + import frappe from frappe.model.document import Document +NAME_PATTERN = re.compile("[%'\"#*?`]") + class Note(Document): def autoname(self): # replace forbidden characters - import re - - self.name = re.sub("[%'\"#*?`]", "", self.title.strip()) + self.name = NAME_PATTERN.sub("", self.title.strip()) def validate(self): if self.notify_on_login and not self.expire_notification_on: diff --git a/frappe/desk/doctype/notification_log/notification_log.py b/frappe/desk/doctype/notification_log/notification_log.py index c4a082ff11..def626513c 100644 --- a/frappe/desk/doctype/notification_log/notification_log.py +++ b/frappe/desk/doctype/notification_log/notification_log.py @@ -148,6 +148,6 @@ def trigger_indicator_hide(): def set_notifications_as_unseen(user): try: - frappe.db.set_value("Notification Settings", user, "seen", 0) + frappe.db.set_value("Notification Settings", user, "seen", 0, update_modified=False) except frappe.DoesNotExistError: return diff --git a/frappe/desk/doctype/notification_settings/notification_settings.py b/frappe/desk/doctype/notification_settings/notification_settings.py index ee425e154b..2bf7347a4f 100644 --- a/frappe/desk/doctype/notification_settings/notification_settings.py +++ b/frappe/desk/doctype/notification_settings/notification_settings.py @@ -48,9 +48,16 @@ def create_notification_settings(user): _doc.insert(ignore_permissions=True) -def toggle_notifications(user, enable=False): - if frappe.db.exists("Notification Settings", user): - frappe.db.set_value("Notification Settings", user, "enabled", enable) +def toggle_notifications(user: str, enable: bool = False): + try: + settings = frappe.get_doc("Notification Settings", user) + except frappe.DoesNotExistError: + frappe.clear_last_message() + return + + if settings.enabled != enable: + settings.enabled = enable + settings.save() @frappe.whitelist() diff --git a/frappe/desk/doctype/route_history/route_history.json b/frappe/desk/doctype/route_history/route_history.json index 09db2320ca..a5d73fc360 100644 --- a/frappe/desk/doctype/route_history/route_history.json +++ b/frappe/desk/doctype/route_history/route_history.json @@ -1,126 +1,52 @@ { - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, + "actions": [], "creation": "2018-10-05 11:26:04.601113", - "custom": 0, - "docstatus": 0, "doctype": "DocType", - "document_type": "", "editable_grid": 1, "engine": "InnoDB", + "field_order": [ + "route", + "user" + ], "fields": [ { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "route", "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, "in_list_view": 1, - "in_standard_filter": 0, - "label": "Route", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "label": "Route" }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "user", "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, "in_list_view": 1, - "in_standard_filter": 0, + "in_standard_filter": 1, "label": "User", - "length": 0, - "no_copy": 0, - "options": "User", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "options": "User" } ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2021-10-25 13:26:03.106050", + "links": [], + "modified": "2022-06-13 05:48:56.967244", "modified_by": "Administrator", "module": "Desk", "name": "Route History", - "name_case": "", "owner": "Administrator", "permissions": [ { - "amend": 0, - "cancel": 0, "create": 1, "delete": 1, "email": 1, "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, "print": 1, "read": 1, "report": 1, "role": "System Manager", - "set_user_permissions": 0, "share": 1, - "submit": 0, "write": 1 } ], "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, "sort_field": "modified", "sort_order": "DESC", - "track_seen": 0, - "track_views": 0 -} + "states": [], + "title_field": "route" +} \ No newline at end of file diff --git a/frappe/desk/doctype/route_history/route_history.py b/frappe/desk/doctype/route_history/route_history.py index e712a5bb11..c62311ae02 100644 --- a/frappe/desk/doctype/route_history/route_history.py +++ b/frappe/desk/doctype/route_history/route_history.py @@ -4,12 +4,15 @@ import frappe from frappe.deferred_insert import deferred_insert as _deferred_insert from frappe.model.document import Document -from frappe.query_builder import DocType -from frappe.query_builder.functions import Count +from frappe.query_builder import DocType, Interval +from frappe.query_builder.functions import Count, Now class RouteHistory(Document): - pass + @staticmethod + def clear_old_logs(days=30): + table = frappe.qb.DocType("Route History") + frappe.db.delete(table, filters=(table.modified < (Now() - Interval(days=days)))) def flush_old_route_records(): diff --git a/frappe/desk/doctype/route_history/route_history_list.js b/frappe/desk/doctype/route_history/route_history_list.js new file mode 100644 index 0000000000..84a441852c --- /dev/null +++ b/frappe/desk/doctype/route_history/route_history_list.js @@ -0,0 +1,7 @@ +frappe.listview_settings["Route History"] = { + onload: function(listview) { + frappe.require("logtypes.bundle.js", () => { + frappe.utils.logtypes.show_log_retention_message(cur_list.doctype); + }) + }, +}; diff --git a/frappe/desk/form/utils.py b/frappe/desk/form/utils.py index 2738d1f74a..573ec17979 100644 --- a/frappe/desk/form/utils.py +++ b/frappe/desk/form/utils.py @@ -2,42 +2,51 @@ # License: MIT. See LICENSE import json +from typing import TYPE_CHECKING import frappe import frappe.desk.form.load import frappe.desk.form.meta from frappe import _ -from frappe.core.doctype.file.file import extract_images_from_html +from frappe.core.doctype.file.utils import extract_images_from_html from frappe.desk.form.document_follow import follow_document +if TYPE_CHECKING: + from frappe.core.doctype.comment.comment import Comment -@frappe.whitelist() + +@frappe.whitelist(methods=["DELETE", "POST"]) def remove_attach(): """remove attachment""" fid = frappe.form_dict.get("fid") - file_name = frappe.form_dict.get("file_name") frappe.delete_doc("File", fid) -@frappe.whitelist() -def add_comment(reference_doctype, reference_name, content, comment_email, comment_by): - """allow any logged user to post a comment""" - doc = frappe.get_doc( - dict( - doctype="Comment", - reference_doctype=reference_doctype, - reference_name=reference_name, - comment_email=comment_email, - comment_type="Comment", - comment_by=comment_by, - ) - ) +@frappe.whitelist(methods=["POST", "PUT"]) +def add_comment( + reference_doctype: str, reference_name: str, content: str, comment_email: str, comment_by: str +) -> "Comment": + """Allow logged user with permission to read document to add a comment""" reference_doc = frappe.get_doc(reference_doctype, reference_name) - doc.content = extract_images_from_html(reference_doc, content, is_private=True) - doc.insert(ignore_permissions=True) + reference_doc.check_permission() + + comment = frappe.new_doc("Comment") + comment.update( + { + "comment_type": "Comment", + "reference_doctype": reference_doctype, + "reference_name": reference_name, + "comment_email": comment_email, + "comment_by": comment_by, + "content": extract_images_from_html(reference_doc, content, is_private=True), + } + ) + comment.insert(ignore_permissions=True) + if frappe.get_cached_value("User", frappe.session.user, "follow_commented_documents"): - follow_document(doc.reference_doctype, doc.reference_name, frappe.session.user) - return doc.as_dict() + follow_document(comment.reference_doctype, comment.reference_name, frappe.session.user) + + return comment @frappe.whitelist() diff --git a/frappe/desk/listview.py b/frappe/desk/listview.py index 88216d3998..5149f8bf86 100644 --- a/frappe/desk/listview.py +++ b/frappe/desk/listview.py @@ -1,6 +1,12 @@ -# Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and Contributors +# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +from typing import Dict, List + import frappe +from frappe.query_builder import Order +from frappe.query_builder.functions import Count +from frappe.query_builder.terms import SubQuery +from frappe.query_builder.utils import DocType @frappe.whitelist() @@ -24,37 +30,36 @@ def set_list_settings(doctype, values): @frappe.whitelist() -def get_group_by_count(doctype, current_filters, field): +def get_group_by_count(doctype: str, current_filters: str, field: str) -> List[Dict]: current_filters = frappe.parse_json(current_filters) - subquery_condition = "" - subquery = frappe.get_all(doctype, filters=current_filters, run=False) if field == "assigned_to": - subquery_condition = " and `tabToDo`.reference_name in ({subquery})".format(subquery=subquery) - return frappe.db.sql( - """select `tabToDo`.allocated_to as name, count(*) as count - from - `tabToDo`, `tabUser` - where - `tabToDo`.status!='Cancelled' and - `tabToDo`.allocated_to = `tabUser`.name and - `tabUser`.user_type = 'System User' - {subquery_condition} - group by - `tabToDo`.allocated_to - order by - count desc - limit 50""".format( - subquery_condition=subquery_condition - ), - as_dict=True, - ) - else: - return frappe.db.get_list( - doctype, - filters=current_filters, - group_by="`tab{0}`.{1}".format(doctype, field), - fields=["count(*) as count", "`{}` as name".format(field)], - order_by="count desc", - limit=50, + ToDo = DocType("ToDo") + User = DocType("User") + count = Count("*").as_("count") + filtered_records = frappe.db.query.build_conditions(doctype, current_filters).select("name") + + return ( + frappe.qb.from_(ToDo) + .from_(User) + .select(ToDo.allocated_to.as_("name"), count) + .where( + (ToDo.status != "Cancelled") + & (ToDo.allocated_to == User.name) + & (User.user_type == "System User") + & (ToDo.reference_name.isin(SubQuery(filtered_records))) + ) + .groupby(ToDo.allocated_to) + .orderby(count, order=Order.desc) + .limit(50) + .run(as_dict=True) ) + + return frappe.get_list( + doctype, + filters=current_filters, + group_by=f"`tab{doctype}`.{field}", + fields=["count(*) as count", f"`{field}` as name"], + order_by="count desc", + limit=50, + ) diff --git a/frappe/desk/page/setup_wizard/setup_wizard.py b/frappe/desk/page/setup_wizard/setup_wizard.py index 3f849bbcaa..8fc4b3f694 100755 --- a/frappe/desk/page/setup_wizard/setup_wizard.py +++ b/frappe/desk/page/setup_wizard/setup_wizard.py @@ -431,22 +431,13 @@ def make_records(records, debug=False): if doc.meta.get_field(parent_link_field) and not doc.get(parent_link_field): doc.flags.ignore_mandatory = True + savepoint = "setup_fixtures_creation" try: - doc.insert(ignore_permissions=True) - frappe.db.commit() - - except frappe.DuplicateEntryError as e: - # print("Failed to insert duplicate {0} {1}".format(doctype, doc.name)) - - # pass DuplicateEntryError and continue - if e.args and e.args[0] == doc.doctype and e.args[1] == doc.name: - # make sure DuplicateEntryError is for the exact same doc and not a related doc - frappe.clear_messages() - else: - raise - + frappe.db.savepoint(savepoint) + doc.insert(ignore_permissions=True, ignore_if_duplicate=True) except Exception as e: - frappe.db.rollback() + frappe.clear_last_message() + frappe.db.rollback(save_point=savepoint) exception = record.get("__exception") if exception: config = _dict(exception) @@ -461,3 +452,4 @@ def make_records(records, debug=False): def show_document_insert_error(): print("Document Insert Error") print(frappe.get_traceback()) + frappe.log_error("Exception during Setup") diff --git a/frappe/desk/reportview.py b/frappe/desk/reportview.py index b45f80f6ff..d6dce68399 100644 --- a/frappe/desk/reportview.py +++ b/frappe/desk/reportview.py @@ -14,6 +14,7 @@ from frappe.model import child_table_fields, default_fields, optional_fields from frappe.model.base_document import get_controller from frappe.model.db_query import DatabaseQuery from frappe.utils import add_user_info, cstr, format_duration +from frappe.utils.caching import site_cache @frappe.whitelist() @@ -171,7 +172,7 @@ def raise_invalid_field(fieldname): def is_standard(fieldname): if "." in fieldname: - parenttype, fieldname = get_parenttype_and_fieldname(fieldname, None) + fieldname = fieldname.split(".")[1].strip("`") return ( fieldname in default_fields or fieldname in optional_fields or fieldname in child_table_fields ) @@ -235,7 +236,16 @@ def parse_json(data): def get_parenttype_and_fieldname(field, data): if "." in field: - parenttype, fieldname = field.split(".")[0][4:-1], field.split(".")[1].strip("`") + parts = field.split(".") + parenttype = parts[0] + fieldname = parts[1] + if parenttype.startswith("`tab"): + # `tabChild DocType`.`fieldname` + parenttype = parenttype[4:-1] + fieldname = fieldname.strip("`") + else: + # tablefield.fieldname + parenttype = frappe.get_meta(data.doctype).get_field(parenttype).options else: parenttype = data.doctype fieldname = field.strip("`") @@ -724,5 +734,6 @@ def get_filters_cond( return cond +@site_cache(maxsize=128) def is_virtual_doctype(doctype): return frappe.db.get_value("DocType", doctype, "is_virtual") diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index 73ab13b851..a89b9a2747 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -23,10 +23,6 @@ from frappe.utils.error import raise_error_on_no_output from frappe.utils.jinja import render_template from frappe.utils.user import get_system_managers -OUTGOING_EMAIL_ACCOUNT_MISSING = _( - "Please setup default Email Account from Setup > Email > Email Account" -) - class SentEmailInInbox(Exception): pass @@ -319,7 +315,7 @@ class EmailAccount(Document): @classmethod @raise_error_on_no_output( keep_quiet=lambda: not cint(frappe.get_system_settings("setup_complete")), - error_message=OUTGOING_EMAIL_ACCOUNT_MISSING, + error_message=_("Please setup default Email Account from Setup > Email > Email Account"), error_type=frappe.OutgoingEmailError, ) # noqa @cache_email_account("outgoing_email_account") diff --git a/frappe/email/doctype/email_account/test_email_account.py b/frappe/email/doctype/email_account/test_email_account.py index a027a81bd7..537bf9eb7f 100644 --- a/frappe/email/doctype/email_account/test_email_account.py +++ b/frappe/email/doctype/email_account/test_email_account.py @@ -284,7 +284,7 @@ class TestEmailAccount(unittest.TestCase): messages = { # append_to = ToDo '"INBOX"': { - "latest_messages": [f.read().replace("{{ message_id }}", last_mail.message_id)], + "latest_messages": [f.read().replace("{{ message_id }}", "<" + last_mail.message_id + ">")], "seen_status": {2: "UNSEEN"}, "uid_list": [2], } diff --git a/frappe/email/doctype/email_queue/email_queue.py b/frappe/email/doctype/email_queue/email_queue.py index db2ca9e32b..c3002607b4 100644 --- a/frappe/email/doctype/email_queue/email_queue.py +++ b/frappe/email/doctype/email_queue/email_queue.py @@ -18,8 +18,17 @@ from frappe.email.doctype.email_account.email_account import EmailAccount from frappe.email.email_body import add_attachment, get_email, get_formatted_html from frappe.email.queue import get_unsubcribed_url, get_unsubscribe_message from frappe.model.document import Document -from frappe.query_builder.utils import DocType -from frappe.utils import add_days, cint, cstr, get_hook_method, nowdate, split_emails +from frappe.query_builder import DocType, Interval +from frappe.query_builder.functions import Now +from frappe.utils import ( + add_days, + cint, + cstr, + get_hook_method, + get_string_between, + nowdate, + split_emails, +) MAX_RETRY_COUNT = 3 @@ -136,6 +145,31 @@ class EmailQueue(Document): if ctx.email_account_doc.append_emails_to_sent_folder and ctx.sent_to: ctx.email_account_doc.append_email_to_sent_folder(message) + @staticmethod + def clear_old_logs(days=30): + """Remove low priority older than 31 days in Outbox or configured in Log Settings. + Note: Used separate query to avoid deadlock + """ + days = days or 31 + email_queue = frappe.qb.DocType("Email Queue") + email_recipient = frappe.qb.DocType("Email Queue Recipient") + + # Delete queue table + ( + frappe.qb.from_(email_queue) + .delete() + .where((email_queue.modified < (Now() - Interval(days=days)))) + ).run() + + # delete child tables, note that this has potential to leave some orphan + # child table behind if modified time was later than parent doc (rare). + # But it's safe since child table doesn't contain links. + ( + frappe.qb.from_(email_recipient) + .delete() + .where((email_recipient.modified < (Now() - Interval(days=days)))) + ).run() + @task(queue="short") def send_mail(email_queue_name, is_background_task=False): @@ -635,7 +669,7 @@ class QueueBuilder: d = { "priority": self.send_priority, "attachments": json.dumps(self.get_attachments()), - "message_id": mail.msg_root["Message-Id"].strip(" <>"), + "message_id": get_string_between("<", mail.msg_root["Message-Id"], ">"), "message": mail_to_string, "sender": self.sender, "reference_doctype": self.reference_doctype, diff --git a/frappe/email/doctype/email_queue/email_queue_list.js b/frappe/email/doctype/email_queue/email_queue_list.js index 0445a3ca19..edc6250714 100644 --- a/frappe/email/doctype/email_queue/email_queue_list.js +++ b/frappe/email/doctype/email_queue/email_queue_list.js @@ -19,5 +19,11 @@ frappe.listview_settings['Email Queue'] = { }) } } + }, + + onload: function(listview) { + frappe.require("logtypes.bundle.js", () => { + frappe.utils.logtypes.show_log_retention_message(cur_list.doctype); + }) } } diff --git a/frappe/email/doctype/email_queue/test_email_queue.py b/frappe/email/doctype/email_queue/test_email_queue.py index b3c5467085..435e4e691f 100644 --- a/frappe/email/doctype/email_queue/test_email_queue.py +++ b/frappe/email/doctype/email_queue/test_email_queue.py @@ -1,10 +1,42 @@ # -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and Contributors # License: MIT. See LICENSE -import unittest -# test_records = frappe.get_test_records('Email Queue') +import frappe +from frappe.tests.utils import FrappeTestCase -class TestEmailQueue(unittest.TestCase): - pass +class TestEmailQueue(FrappeTestCase): + def test_email_queue_deletion_based_on_modified_date(self): + from frappe.email.doctype.email_queue.email_queue import EmailQueue + + old_record = frappe.get_doc( + { + "doctype": "Email Queue", + "sender": "Test ", + "show_as_cc": "", + "message": "Test message", + "status": "Sent", + "priority": 1, + "recipients": [ + { + "recipient": "test_auth@test.com", + } + ], + } + ).insert() + + old_record.modified = "2010-01-01 00:00:01" + old_record.recipients[0].modified = old_record.modified + old_record.db_update_all() + + new_record = frappe.copy_doc(old_record) + new_record.insert() + + EmailQueue.clear_old_logs() + + self.assertFalse(frappe.db.exists("Email Queue", old_record.name)) + self.assertFalse(frappe.db.exists("Email Queue Recipient", {"parent": old_record.name})) + + self.assertTrue(frappe.db.exists("Email Queue", new_record.name)) + self.assertTrue(frappe.db.exists("Email Queue Recipient", {"parent": new_record.name})) diff --git a/frappe/email/doctype/newsletter/test_newsletter.py b/frappe/email/doctype/newsletter/test_newsletter.py index 9ec61194ef..550ee8164b 100644 --- a/frappe/email/doctype/newsletter/test_newsletter.py +++ b/frappe/email/doctype/newsletter/test_newsletter.py @@ -1,7 +1,6 @@ # Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See LICENSE -import unittest from random import choice from typing import Union from unittest.mock import MagicMock, PropertyMock, patch @@ -17,9 +16,9 @@ from frappe.email.doctype.newsletter.newsletter import ( send_scheduled_email, ) from frappe.email.queue import flush +from frappe.tests.utils import FrappeTestCase from frappe.utils import add_days, getdate -test_dependencies = ["Email Group"] emails = [ "test_subscriber1@example.com", "test_subscriber2@example.com", @@ -63,16 +62,23 @@ class TestNewsletterMixin: for email in emails: doctype = "Email Group Member" email_filters = {"email": email, "email_group": "_Test Email Group"} + + savepoint = "setup_email_group" + frappe.db.savepoint(savepoint) + try: frappe.get_doc( { "doctype": doctype, **email_filters, } - ).insert() + ).insert(ignore_if_duplicate=True) except Exception: + frappe.db.rollback(save_point=savepoint) frappe.db.update(doctype, email_filters, "unsubscribed", 0) + frappe.db.release_savepoint(savepoint) + def send_newsletter(self, published=0, schedule_send=None) -> Union[str, None]: frappe.db.delete("Email Queue") frappe.db.delete("Email Queue Recipient") @@ -127,7 +133,7 @@ class TestNewsletterMixin: return newsletter -class TestNewsletter(TestNewsletterMixin, unittest.TestCase): +class TestNewsletter(TestNewsletterMixin, FrappeTestCase): def test_send(self): self.send_newsletter() diff --git a/frappe/email/email_body.py b/frappe/email/email_body.py index 5e2f14d9bf..50c66e1ad2 100755 --- a/frappe/email/email_body.py +++ b/frappe/email/email_body.py @@ -24,6 +24,8 @@ from frappe.utils import ( ) from frappe.utils.pdf import get_pdf +EMBED_PATTERN = re.compile("""embed=["'](.*?)["']""") + def get_email( recipients, @@ -190,7 +192,7 @@ class EMail: def set_part_html(self, message, inline_images): from email.mime.text import MIMEText - has_inline_images = re.search("""embed=['"].*?['"]""", message) + has_inline_images = EMBED_PATTERN.search(message) if has_inline_images: # process inline images @@ -499,7 +501,7 @@ def replace_filename_with_cid(message): inline_images = [] while True: - matches = re.search("""embed=["'](.*?)["']""", message) + matches = EMBED_PATTERN.search(message) if not matches: break groups = matches.groups() @@ -510,7 +512,7 @@ def replace_filename_with_cid(message): filecontent = get_filecontent_from_path(img_path) if not filecontent: - message = re.sub("""embed=['"]{0}['"]""".format(img_path), "", message) + message = re.sub(f"""embed=['"]{img_path}['"]""", "", message) continue content_id = random_string(10) @@ -519,9 +521,7 @@ def replace_filename_with_cid(message): {"filename": filename, "filecontent": filecontent, "content_id": content_id} ) - message = re.sub( - """embed=['"]{0}['"]""".format(img_path), 'src="cid:{0}"'.format(content_id), message - ) + message = re.sub(f"""embed=['"]{img_path}['"]""", f'src="cid:{content_id}"', message) return (message, inline_images) diff --git a/frappe/email/queue.py b/frappe/email/queue.py index b92dea3e65..1519c26841 100755 --- a/frappe/email/queue.py +++ b/frappe/email/queue.py @@ -190,25 +190,6 @@ def get_queue(): ) -def clear_outbox(days: int = None) -> None: - """Remove low priority older than 31 days in Outbox or configured in Log Settings. - Note: Used separate query to avoid deadlock - """ - days = days or 31 - email_queue = DocType("Email Queue") - - email_queues = ( - frappe.qb.from_(email_queue) - .select(email_queue.name) - .where(email_queue.modified < (Now() - Interval(days=days))) - .run(pluck=True) - ) - - if email_queues: - frappe.db.delete("Email Queue", {"name": ("in", email_queues)}) - frappe.db.delete("Email Queue Recipient", {"parent": ("in", email_queues)}) - - def set_expiry_for_email_queue(): """Mark emails as expire that has not sent for 7 days. Called daily via scheduler. diff --git a/frappe/email/receive.py b/frappe/email/receive.py index 80c413faa1..12ab04eb4b 100644 --- a/frappe/email/receive.py +++ b/frappe/email/receive.py @@ -17,7 +17,7 @@ from email_reply_parser import EmailReplyParser import frappe from frappe import _, safe_decode, safe_encode -from frappe.core.doctype.file.file import MaxFileSizeReachedError, get_random_filename +from frappe.core.doctype.file import MaxFileSizeReachedError, get_random_filename from frappe.utils import ( add_days, cint, @@ -25,6 +25,7 @@ from frappe.utils import ( cstr, extract_email_id, get_datetime, + get_string_between, markdown, now, parse_addr, @@ -37,6 +38,9 @@ from frappe.utils.user import is_system_user # fix due to a python bug in poplib that limits it to 2048 poplib._MAXLINE = 20480 +THREAD_ID_PATTERN = re.compile(r"(?<=\[)[\w/-]+") +WORDS_PATTERN = re.compile(r"\w+") + class EmailSizeExceededError(frappe.ValidationError): pass @@ -272,7 +276,7 @@ class EmailServer: return def parse_imap_response(self, cmd, response): - pattern = r"(?<={cmd} )[0-9]*".format(cmd=cmd) + pattern = rf"(?<={cmd} )[0-9]*" match = re.search(pattern, response.decode("utf-8"), re.U | re.I) if match: @@ -331,8 +335,7 @@ class EmailServer: flags = [] for flag in imaplib.ParseFlags(flag_string) or []: - pattern = re.compile(r"\w+") - match = re.search(pattern, frappe.as_unicode(flag)) + match = WORDS_PATTERN.search(frappe.as_unicode(flag)) flags.append(match.group(0)) if "Seen" in flags: @@ -425,7 +428,9 @@ class Email: self.set_content_and_type() self.set_subject() self.set_from() - self.message_id = (self.mail.get("Message-ID") or "").strip(" <>") + + message_id = self.mail.get("Message-ID") or "" + self.message_id = get_string_between("<", message_id, ">") if self.mail["Date"]: try: @@ -441,7 +446,8 @@ class Email: @property def in_reply_to(self): - return (self.mail.get("In-Reply-To") or "").strip(" <>") + in_reply_to = self.mail.get("In-Reply-To") or "" + return get_string_between("<", in_reply_to, ">") def parse(self): """Walk and process multi-part email.""" @@ -618,7 +624,7 @@ class Email: def get_thread_id(self): """Extract thread ID from `[]`""" - l = re.findall(r"(?<=\[)[\w/-]+", self.subject) + l = THREAD_ID_PATTERN.findall(self.subject) return l and l[0] or None def is_reply(self): diff --git a/frappe/email/smtp.py b/frappe/email/smtp.py index d621fd2bba..1211419de1 100644 --- a/frappe/email/smtp.py +++ b/frappe/email/smtp.py @@ -1,24 +1,12 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors +# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE import smtplib -import _socket - import frappe from frappe import _ from frappe.utils import cint, cstr -CONNECTION_FAILED = _("Could not connect to outgoing email server") -AUTH_ERROR_TITLE = _("Invalid Credentials") -AUTH_ERROR = _("Incorrect email or password. Please check your login credentials.") -SOCKET_ERROR_TITLE = _("Incorrect Configuration") -SOCKET_ERROR = _("Invalid Outgoing Mail Server or Port") -SEND_MAIL_FAILED = _("Unable to send emails at this time") -EMAIL_ACCOUNT_MISSING = _( - "Email Account not setup. Please create a new Email Account from Setup > Email > Email Account" -) - class InvalidEmailCredentials(frappe.ValidationError): pass @@ -65,7 +53,12 @@ class SMTPServer: self._session = None if not self.server: - frappe.msgprint(EMAIL_ACCOUNT_MISSING, raise_exception=frappe.OutgoingEmailError) + frappe.msgprint( + _( + "Email Account not setup. Please create a new Email Account from Setup > Email > Email Account" + ), + raise_exception=frappe.OutgoingEmailError, + ) @property def port(self): @@ -93,7 +86,9 @@ class SMTPServer: try: _session = SMTP(self.server, self.port) if not _session: - frappe.msgprint(CONNECTION_FAILED, raise_exception=frappe.OutgoingEmailError) + frappe.msgprint( + _("Could not connect to outgoing email server"), raise_exception=frappe.OutgoingEmailError + ) self.secure_session(_session) if self.login and self.password: @@ -106,16 +101,12 @@ class SMTPServer: self._session = _session return self._session - except smtplib.SMTPAuthenticationError as e: + except smtplib.SMTPAuthenticationError: self.throw_invalid_credentials_exception() - except _socket.error as e: + except OSError: # Invalid mail server -- due to refusing connection - frappe.throw(SOCKET_ERROR, title=SOCKET_ERROR_TITLE) - - except smtplib.SMTPException: - frappe.msgprint(SEND_MAIL_FAILED) - raise + frappe.throw(_("Invalid Outgoing Mail Server or Port"), title=_("Incorrect Configuration")) def is_session_active(self): if self._session: @@ -130,4 +121,8 @@ class SMTPServer: @classmethod def throw_invalid_credentials_exception(cls): - frappe.throw(AUTH_ERROR, title=AUTH_ERROR_TITLE, exc=InvalidEmailCredentials) + frappe.throw( + _("Incorrect email or password. Please check your login credentials."), + title=_("Invalid Credentials"), + exc=InvalidEmailCredentials, + ) diff --git a/frappe/event_streaming/doctype/event_producer/test_event_producer.py b/frappe/event_streaming/doctype/event_producer/test_event_producer.py index 4464b0a434..eafa7a0b51 100644 --- a/frappe/event_streaming/doctype/event_producer/test_event_producer.py +++ b/frappe/event_streaming/doctype/event_producer/test_event_producer.py @@ -8,6 +8,8 @@ import frappe from frappe.core.doctype.user.user import generate_keys from frappe.event_streaming.doctype.event_producer.event_producer import pull_from_node from frappe.frappeclient import FrappeClient +from frappe.query_builder.utils import db_type_is +from frappe.tests.test_query_builder import run_only_if producer_url = "http://test_site_producer:8000" @@ -51,7 +53,9 @@ class TestEventProducer(unittest.TestCase): self.pull_producer_data() self.assertFalse(frappe.db.exists("ToDo", producer_doc.name)) + @run_only_if(db_type_is.MARIADB) def test_multiple_doctypes_sync(self): + # TODO: This test is extremely flaky with Postgres. Rewrite this! producer = get_remote_site() # insert todo and note in producer diff --git a/frappe/exceptions.py b/frappe/exceptions.py index a8569481d3..755c21c240 100644 --- a/frappe/exceptions.py +++ b/frappe/exceptions.py @@ -263,3 +263,15 @@ class ExecutableNotFound(FileNotFoundError): class InvalidRemoteException(Exception): pass + + +class LinkExpired(ValidationError): + http_status_code = 410 + title = "Link Expired" + message = "The link has expired" + + +class InvalidKeyError(ValidationError): + http_status_code = 401 + title = "Invalid Key" + message = "The document key is invalid" diff --git a/frappe/geo/country_info.json b/frappe/geo/country_info.json index 735dcddac3..94d1f3ed37 100644 --- a/frappe/geo/country_info.json +++ b/frappe/geo/country_info.json @@ -2412,7 +2412,7 @@ "Singapore": { "code": "sg", "currency": "SGD", - "currency_fraction": "Sen", + "currency_fraction": "Cent", "currency_fraction_units": 100, "currency_name": "Singapore Dollar", "currency_symbol": "$", diff --git a/frappe/handler.py b/frappe/handler.py index 44249323ef..c98f2c9b69 100644 --- a/frappe/handler.py +++ b/frappe/handler.py @@ -1,7 +1,9 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors +# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +import os from mimetypes import guess_type +from typing import TYPE_CHECKING from werkzeug.wrappers import Response @@ -15,6 +17,10 @@ from frappe.utils.csvutils import build_csv_response from frappe.utils.image import optimize_image from frappe.utils.response import build_response +if TYPE_CHECKING: + from frappe.core.doctype.file.file import File + from frappe.core.doctype.user.user import User + ALLOWED_MIMETYPES = ( "image/png", "image/jpeg", @@ -166,9 +172,9 @@ def upload_file(): if frappe.get_system_settings("allow_guests_to_upload_files"): ignore_permissions = True else: - return + raise frappe.PermissionError else: - user = frappe.get_doc("User", frappe.session.user) + user: "User" = frappe.get_doc("User", frappe.session.user) ignore_permissions = False files = frappe.request.files @@ -210,7 +216,7 @@ def upload_file(): is_whitelisted(method) return method() else: - ret = frappe.get_doc( + return frappe.get_doc( { "doctype": "File", "attached_to_doctype": doctype, @@ -222,9 +228,26 @@ def upload_file(): "is_private": cint(is_private), "content": content, } - ) - ret.save(ignore_permissions=ignore_permissions) - return ret + ).save(ignore_permissions=ignore_permissions) + + +@frappe.whitelist(allow_guest=True) +def download_file(file_url: str): + """ + Download file using token and REST API. Valid session or + token is required to download private files. + + Method : GET + Endpoints : download_file, frappe.core.doctype.file.file.download_file + URL Params : file_name = /path/to/file relative to site path + """ + file: "File" = frappe.get_doc("File", {"file_url": file_url}) + if not file.is_downloadable(): + raise frappe.PermissionError + + frappe.local.response.filename = os.path.basename(file_url) + frappe.local.response.filecontent = file.get_content() + frappe.local.response.type = "download" def get_attr(cmd): diff --git a/frappe/hooks.py b/frappe/hooks.py index ee8417a3ec..ae2abcec68 100644 --- a/frappe/hooks.py +++ b/frappe/hooks.py @@ -146,7 +146,7 @@ doc_events = { "frappe.core.doctype.activity_log.feed.update_feed", "frappe.workflow.doctype.workflow_action.workflow_action.process_workflow_actions", "frappe.automation.doctype.assignment_rule.assignment_rule.apply", - "frappe.core.doctype.file.file.attach_files_to_document", + "frappe.core.doctype.file.utils.attach_files_to_document", "frappe.event_streaming.doctype.event_update_log.event_update_log.notify_consumers", "frappe.automation.doctype.assignment_rule.assignment_rule.update_due_date", "frappe.core.doctype.user_type.user_type.apply_permissions_for_non_standard_user_type", @@ -219,7 +219,6 @@ scheduler_events = { "daily": [ "frappe.email.queue.set_expiry_for_email_queue", "frappe.desk.notifications.clear_notifications", - "frappe.core.doctype.error_log.error_log.set_old_logs_as_seen", "frappe.desk.doctype.event.event.send_event_digest", "frappe.sessions.clear_expired_sessions", "frappe.email.doctype.notification.notification.trigger_daily_alerts", @@ -368,4 +367,16 @@ global_search_doctypes = { ] } +override_whitelisted_methods = { + "frappe.core.doctype.file.file.download_file": "download_file", + "frappe.core.doctype.file.file.unzip_file": "frappe.core.api.file.unzip_file", + "frappe.core.doctype.file.file.get_attached_images": "frappe.core.api.file.get_attached_images", + "frappe.core.doctype.file.file.get_files_in_folder": "frappe.core.api.file.get_files_in_folder", + "frappe.core.doctype.file.file.get_files_by_search_text": "frappe.core.api.file.get_files_by_search_text", + "frappe.core.doctype.file.file.get_max_file_size": "frappe.core.api.file.get_max_file_size", + "frappe.core.doctype.file.file.create_new_folder": "frappe.core.api.file.create_new_folder", + "frappe.core.doctype.file.file.move_file": "frappe.core.api.file.move_file", + "frappe.core.doctype.file.file.zip_files": "frappe.core.api.file.zip_files", +} + translated_search_doctypes = ["DocType", "Role", "Country", "Gender", "Salutation"] diff --git a/frappe/installer.py b/frappe/installer.py index 42cc023c6a..ec7c8d4215 100644 --- a/frappe/installer.py +++ b/frappe/installer.py @@ -484,13 +484,20 @@ def set_all_patches_as_completed(app): def init_singles(): - singles = [single["name"] for single in frappe.get_all("DocType", filters={"issingle": True})] + singles = frappe.get_all("DocType", filters={"issingle": True}, pluck="name") for single in singles: - if not frappe.db.get_singles_dict(single): + if frappe.db.get_singles_dict(single): + continue + + try: doc = frappe.new_doc(single) doc.flags.ignore_mandatory = True doc.flags.ignore_validate = True doc.save() + except ImportError: + # The doctype exists, but controller is deleted, + # no need to attempt to init such single, ref: #16917 + continue def make_conf( diff --git a/frappe/integrations/doctype/google_drive/google_drive.py b/frappe/integrations/doctype/google_drive/google_drive.py index bbb1e8485e..347488ee44 100644 --- a/frappe/integrations/doctype/google_drive/google_drive.py +++ b/frappe/integrations/doctype/google_drive/google_drive.py @@ -259,13 +259,13 @@ def upload_system_backup_to_google_drive(): def daily_backup(): - drive_settings = frappe.db.get_singles_dict("Google Drive") + drive_settings = frappe.db.get_singles_dict("Google Drive", cast=True) if drive_settings.enable and drive_settings.frequency == "Daily": upload_system_backup_to_google_drive() def weekly_backup(): - drive_settings = frappe.db.get_singles_dict("Google Drive") + drive_settings = frappe.db.get_singles_dict("Google Drive", cast=True) if drive_settings.enable and drive_settings.frequency == "Weekly": upload_system_backup_to_google_drive() diff --git a/frappe/integrations/doctype/ldap_settings/ldap_settings.py b/frappe/integrations/doctype/ldap_settings/ldap_settings.py index a14124234f..96007ee918 100644 --- a/frappe/integrations/doctype/ldap_settings/ldap_settings.py +++ b/frappe/integrations/doctype/ldap_settings/ldap_settings.py @@ -120,7 +120,7 @@ class LDAPSettings(Document): def get_ldap_client_settings(): # return the settings to be used on the client side. result = {"enabled": False} - ldap = frappe.get_doc("LDAP Settings") + ldap = frappe.get_cached_doc("LDAP Settings") if ldap.enabled: result["enabled"] = True result["method"] = "frappe.integrations.doctype.ldap_settings.ldap_settings.login" diff --git a/frappe/model/base_document.py b/frappe/model/base_document.py index 22f8cc15df..d3cc662d52 100644 --- a/frappe/model/base_document.py +++ b/frappe/model/base_document.py @@ -1168,7 +1168,7 @@ class BaseDocument(object): return cast_fieldtype(df.fieldtype, value, show_warning=False) def _extract_images_from_text_editor(self): - from frappe.core.doctype.file.file import extract_images_from_doc + from frappe.core.doctype.file.utils import extract_images_from_doc if self.doctype != "DocType": for df in self.meta.get("fields", {"fieldtype": ("=", "Text Editor")}): diff --git a/frappe/model/create_new.py b/frappe/model/create_new.py index 8671008f82..51810c3e18 100644 --- a/frappe/model/create_new.py +++ b/frappe/model/create_new.py @@ -23,8 +23,6 @@ def get_new_doc(doctype, parent_doc=None, parentfield=None, as_dict=False): doc = copy.deepcopy(frappe.local.new_doc_templates[doctype]) - # doc = make_new_doc(doctype) - set_dynamic_default_values(doc, parent_doc, parentfield) if as_dict: diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py index 6a7f7fbd54..82913db98d 100644 --- a/frappe/model/db_query.py +++ b/frappe/model/db_query.py @@ -29,11 +29,34 @@ from frappe.utils import ( make_filter_tuple, ) +LOCATE_PATTERN = re.compile(r"locate\([^,]+,\s*[`\"]?name[`\"]?\s*\)", flags=re.IGNORECASE) +LOCATE_CAST_PATTERN = re.compile( + r"locate\(([^,]+),\s*([`\"]?name[`\"]?)\s*\)", flags=re.IGNORECASE +) +FUNC_IFNULL_PATTERN = re.compile( + r"(strpos|ifnull|coalesce)\(\s*[`\"]?name[`\"]?\s*,", flags=re.IGNORECASE +) +CAST_VARCHAR_PATTERN = re.compile( + r"([`\"]?tab[\w`\" -]+\.[`\"]?name[`\"]?)(?!\w)", flags=re.IGNORECASE +) +ORDER_BY_PATTERN = re.compile(r"\ order\ by\ |\ asc|\ ASC|\ desc|\ DESC", flags=re.IGNORECASE) +SUB_QUERY_PATTERN = re.compile("^.*[,();@].*") +IS_QUERY_PATTERN = re.compile(r"^(select|delete|update|drop|create)\s") +IS_QUERY_PREDICATE_PATTERN = re.compile( + r"\s*[0-9a-zA-z]*\s*( from | group by | order by | where | join )" +) +FIELD_QUOTE_PATTERN = re.compile(r"[0-9a-zA-Z]+\s*'") +FIELD_COMMA_PATTERN = re.compile(r"[0-9a-zA-Z]+\s*,") +STRICT_FIELD_PATTERN = re.compile(r".*/\*.*") +STRICT_UNION_PATTERN = re.compile(r".*\s(union).*\s") +ORDER_GROUP_PATTERN = re.compile(r".*[^a-z0-9-_ ,`'\"\.\(\)].*") + class DatabaseQuery(object): def __init__(self, doctype, user=None): self.doctype = doctype self.tables = [] + self.link_tables = [] self.conditions = [] self.or_conditions = [] self.fields = None @@ -216,6 +239,10 @@ class DatabaseQuery(object): parent_name = cast_name(f"{self.tables[0]}.name") args.tables += f" {self.join} {child} on ({child}.parent = {parent_name})" + # left join link tables + for link in self.link_tables: + args.tables += f" {self.join} `tab{link.doctype}` on (`tab{link.doctype}`.`name` = {self.tables[0]}.`{link.fieldname}`)" + if self.grouped_or_conditions: self.conditions.append(f"({' or '.join(self.grouped_or_conditions)})") @@ -261,7 +288,7 @@ class DatabaseQuery(object): return args def prepare_select_args(self, args): - order_field = re.sub(r"\ order\ by\ |\ asc|\ ASC|\ desc|\ DESC", "", args.order_by) + order_field = ORDER_BY_PATTERN.sub("", args.order_by) if order_field not in args.fields: extracted_column = order_column = order_field.replace("`", "") @@ -287,6 +314,23 @@ class DatabaseQuery(object): # remove empty strings / nulls in fields self.fields = [f for f in self.fields if f] + # convert child_table.fieldname to `tabChild DocType`.`fieldname` + for field in self.fields: + if "." in field and "tab" not in field: + original_field = field + alias = None + if " as " in field: + field, alias = field.split(" as ") + linked_fieldname, fieldname = field.split(".") + linked_field = frappe.get_meta(self.doctype).get_field(linked_fieldname) + linked_doctype = linked_field.options + if linked_field.fieldtype == "Link": + self.append_link_table(linked_doctype, linked_fieldname) + field = f"`tab{linked_doctype}`.`{fieldname}`" + if alias: + field = f"{field} as {alias}" + self.fields[self.fields.index(original_field)] = field + for filter_name in ["filters", "or_filters"]: filters = getattr(self, filter_name) if isinstance(filters, str): @@ -309,8 +353,6 @@ class DatabaseQuery(object): As field contains `,` and mysql function `version()`, with the help of regex the system will filter out this field. """ - - sub_query_regex = re.compile("^.*[,();@].*") blacklisted_keywords = ["select", "create", "insert", "delete", "drop", "update", "case", "show"] blacklisted_functions = [ "concat", @@ -334,19 +376,14 @@ class DatabaseQuery(object): frappe.throw(_("Use of sub-query or function is restricted"), frappe.DataError) def _is_query(field): - if re.compile(r"^(select|delete|update|drop|create)\s").match(field): + if IS_QUERY_PATTERN.match(field): _raise_exception() - elif re.compile(r"\s*[0-9a-zA-z]*\s*( from | group by | order by | where | join )").match( - field - ): + elif IS_QUERY_PREDICATE_PATTERN.match(field): _raise_exception() for field in self.fields: - if sub_query_regex.match(field): - if any(keyword in field.lower().split() for keyword in blacklisted_keywords): - _raise_exception() - + if SUB_QUERY_PATTERN.match(field): if any(f"({keyword}" in field.lower() for keyword in blacklisted_keywords): _raise_exception() @@ -357,19 +394,19 @@ class DatabaseQuery(object): # prevent access to global variables _raise_exception() - if re.compile(r"[0-9a-zA-Z]+\s*'").match(field): + if FIELD_QUOTE_PATTERN.match(field): _raise_exception() - if re.compile(r"[0-9a-zA-Z]+\s*,").match(field): + if FIELD_COMMA_PATTERN.match(field): _raise_exception() _is_query(field) if self.strict: - if re.compile(r".*/\*.*").match(field): + if STRICT_FIELD_PATTERN.match(field): frappe.throw(_("Illegal SQL Query")) - if re.compile(r".*\s(union).*\s").match(field.lower()): + if STRICT_UNION_PATTERN.match(field.lower()): frappe.throw(_("Illegal SQL Query")) def extract_tables(self): @@ -396,12 +433,27 @@ class DatabaseQuery(object): table_name = table_name[13:] if not table_name[0] == "`": table_name = f"`{table_name}`" - if table_name not in self.tables: + if table_name not in self.tables and table_name not in ( + d.table_name for d in self.link_tables + ): self.append_table(table_name) def append_table(self, table_name): self.tables.append(table_name) doctype = table_name[4:-1] + self.check_read_permission(doctype) + + def append_link_table(self, doctype, fieldname): + for d in self.link_tables: + if d.doctype == doctype and d.fieldname == fieldname: + return + + self.check_read_permission(doctype) + self.link_tables.append( + frappe._dict(doctype=doctype, fieldname=fieldname, table_name=f"`tab{doctype}`") + ) + + def check_read_permission(self, doctype): ptype = "select" if frappe.only_has_select_perm(doctype) else "read" if not self.flags.ignore_permissions and not frappe.has_permission( @@ -418,7 +470,7 @@ class DatabaseQuery(object): methods = ("count(", "avg(", "sum(", "extract(", "dayofyear(") return field.lower().startswith(methods) - if len(self.tables) > 1: + if len(self.tables) > 1 or len(self.link_tables) > 0: for idx, field in enumerate(self.fields): if "." not in field and not _in_standard_sql_methods(field): self.fields[idx] = f"{self.tables[0]}.{field}" @@ -861,7 +913,7 @@ class DatabaseQuery(object): if "select" in _lower and "from" in _lower: frappe.throw(_("Cannot use sub-query in order by")) - if re.compile(r".*[^a-z0-9-_ ,`'\"\.\(\)].*").match(_lower): + if ORDER_GROUP_PATTERN.match(_lower): frappe.throw(_("Illegal SQL Query")) for field in parameters.split(","): @@ -918,18 +970,16 @@ def cast_name(column: str) -> str: if frappe.db.db_type == "mariadb": return column - kwargs = {"string": column, "flags": re.IGNORECASE} + kwargs = {"string": column} if "cast(" not in column.lower() and "::" not in column: - if re.search(r"locate\([^,]+,\s*[`\"]?name[`\"]?\s*\)", **kwargs): - return re.sub( - r"locate\(([^,]+),\s*([`\"]?name[`\"]?)\s*\)", r"locate(\1, cast(\2 as varchar))", **kwargs - ) + if LOCATE_PATTERN.search(**kwargs): + return LOCATE_CAST_PATTERN.sub(r"locate(\1, cast(\2 as varchar))", **kwargs) - elif match := re.search(r"(strpos|ifnull|coalesce)\(\s*[`\"]?name[`\"]?\s*,", **kwargs): + elif match := FUNC_IFNULL_PATTERN.search(**kwargs): func = match.groups()[0] return re.sub(rf"{func}\(\s*([`\"]?name[`\"]?)\s*,", rf"{func}(cast(\1 as varchar),", **kwargs) - return re.sub(r"([`\"]?tab[\w`\" -]+\.[`\"]?name[`\"]?)(?!\w)", r"cast(\1 as varchar)", **kwargs) + return CAST_VARCHAR_PATTERN.sub(r"cast(\1 as varchar)", **kwargs) return column diff --git a/frappe/model/delete_doc.py b/frappe/model/delete_doc.py index 2eccc1e717..985cc53682 100644 --- a/frappe/model/delete_doc.py +++ b/frappe/model/delete_doc.py @@ -3,6 +3,7 @@ import os import shutil +from typing import List import frappe import frappe.defaults @@ -29,6 +30,7 @@ doctypes_to_skip = ( "Tag Link", "Notification Log", "Email Queue", + "Document Share Key", ) @@ -88,10 +90,6 @@ def delete_doc( update_flags(doc, flags, ignore_permissions) check_permission_and_not_submitted(doc) - frappe.db.delete("Custom Field", {"dt": name}) - frappe.db.delete("Client Script", {"dt": name}) - frappe.db.delete("Property Setter", {"doc_type": name}) - frappe.db.delete("Report", {"ref_doctype": name}) frappe.db.delete("Custom DocPerm", {"parent": name}) frappe.db.delete("__global_search", {"doctype": name}) @@ -192,39 +190,24 @@ def update_naming_series(doc): revert_series_if_last(doc.meta.autoname, doc.name, doc) -def delete_from_table(doctype, name, ignore_doctypes, doc): +def delete_from_table(doctype: str, name: str, ignore_doctypes: List[str], doc): if doctype != "DocType" and doctype == name: frappe.db.delete("Singles", {"doctype": name}) else: frappe.db.delete(doctype, {"name": name}) - # get child tables if doc: - tables = [d.options for d in doc.meta.get_table_fields()] - + child_doctypes = [d.options for d in doc.meta.get_table_fields()] else: + child_doctypes = frappe.get_all( + "DocField", + fields="options", + filters={"fieldtype": ["in", frappe.model.table_fields], "parent": doctype}, + pluck="options", + ) - def get_table_fields(field_doctype): - if field_doctype == "Custom Field": - return [] - - return [ - r[0] - for r in frappe.get_all( - field_doctype, - fields="options", - filters={"fieldtype": ["in", frappe.model.table_fields], "parent": doctype}, - as_list=1, - ) - ] - - tables = get_table_fields("DocField") - if not frappe.flags.in_install == "frappe": - tables += get_table_fields("Custom Field") - - # delete from child tables - for t in list(set(tables)): - if t not in ignore_doctypes: - frappe.db.delete(t, {"parenttype": doctype, "parent": name}) + child_doctypes_to_delete = set(child_doctypes) - set(ignore_doctypes) + for child_doctype in child_doctypes_to_delete: + frappe.db.delete(child_doctype, {"parenttype": doctype, "parent": name}) def update_flags(doc, flags=None, ignore_permissions=False): diff --git a/frappe/model/document.py b/frappe/model/document.py index f59fb81350..22514df75a 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -438,7 +438,7 @@ class Document(BaseDocument): def get_title(self): """Get the document title based on title_field or `title` or `name`""" - return self.get(self.meta.get_title_field()) + return self.get(self.meta.get_title_field()) or "" def set_title_field(self): """Set title field based on template""" @@ -1012,7 +1012,7 @@ class Document(BaseDocument): def delete(self, ignore_permissions=False): """Delete document.""" - frappe.delete_doc( + return frappe.delete_doc( self.doctype, self.name, ignore_permissions=ignore_permissions, flags=self.flags ) @@ -1198,11 +1198,10 @@ class Document(BaseDocument): return version = frappe.new_doc("Version") - if not self._doc_before_save: - version.for_insert(self) - version.insert(ignore_permissions=True) - elif version.set_diff(self._doc_before_save, self): + + if is_useful_diff := version.update_version_info(self._doc_before_save, self): version.insert(ignore_permissions=True) + if not frappe.flags.in_migrate: # follow since you made a change? if frappe.get_cached_value("User", frappe.session.user, "follow_created_documents"): @@ -1383,6 +1382,30 @@ class Document(BaseDocument): """Returns signature (hash) for private URL.""" return hashlib.sha224(get_datetime_str(self.creation).encode()).hexdigest() + def get_document_share_key(self, expires_on=None, no_expiry=False): + if no_expiry: + expires_on = None + + existing_key = frappe.db.exists( + "Document Share Key", + { + "reference_doctype": self.doctype, + "reference_docname": self.name, + "expires_on": expires_on, + }, + ) + if existing_key: + doc = frappe.get_doc("Document Share Key", existing_key) + else: + doc = frappe.new_doc("Document Share Key") + doc.reference_doctype = self.doctype + doc.reference_docname = self.name + doc.expires_on = expires_on + doc.flags.no_expiry = no_expiry + doc.insert(ignore_permissions=True) + + return doc.key + def get_liked_by(self): liked_by = getattr(self, "_liked_by", None) if liked_by: diff --git a/frappe/model/meta.py b/frappe/model/meta.py index 4f7dc01ea4..9f5c2e7611 100644 --- a/frappe/model/meta.py +++ b/frappe/model/meta.py @@ -17,6 +17,7 @@ Example: import json import os from datetime import datetime +from typing import List import click @@ -251,10 +252,15 @@ class Meta(Document): else: label = { "name": _("ID"), - "owner": _("Created By"), - "modified_by": _("Modified By"), "creation": _("Created On"), - "modified": _("Last Modified On"), + "docstatus": _("Document Status"), + "idx": _("Index"), + "modified": _("Last Updated On"), + "modified_by": _("Last Updated By"), + "owner": _("Created By"), + "_user_tags": _("Tags"), + "_liked_by": _("Liked By"), + "_comments": _("Comments"), "_assign": _("Assigned To"), }.get(fieldname) or _("No Label") return label @@ -341,6 +347,16 @@ class Meta(Document): def get_workflow(self): return get_workflow_name(self.name) + def get_naming_series_options(self) -> List[str]: + """Get list naming series options.""" + + field = self.get_field("naming_series") + if field: + options = field.options or "" + + return options.split("\n") + return [] + def add_custom_fields(self): if not frappe.db.table_exists("Custom Field"): return diff --git a/frappe/model/naming.py b/frappe/model/naming.py index bb93244a66..f6a3846699 100644 --- a/frappe/model/naming.py +++ b/frappe/model/naming.py @@ -2,7 +2,7 @@ # License: MIT. See LICENSE import re -from typing import TYPE_CHECKING, Optional, Union +from typing import TYPE_CHECKING, Callable, List, Optional, Union import frappe from frappe import _ @@ -11,6 +11,7 @@ from frappe.query_builder import DocType from frappe.utils import cint, cstr, now_datetime if TYPE_CHECKING: + from frappe.model.document import Document from frappe.model.meta import Meta @@ -18,6 +19,96 @@ if TYPE_CHECKING: # whether `log_types` have autoincremented naming set for the site or not. autoincremented_site_status_map = {} +NAMING_SERIES_PATTERN = re.compile(r"^[\w\- \/.#{}]+$", re.UNICODE) +BRACED_PARAMS_PATTERN = re.compile(r"(\{[\w | #]+\})") + + +class InvalidNamingSeriesError(frappe.ValidationError): + pass + + +class NamingSeries: + __slots__ = ("series",) + + def __init__(self, series: str): + self.series = series + + # Add default number part if missing + if "#" not in self.series: + self.series += ".#####" + + def validate(self): + if "." not in self.series: + frappe.throw( + _("Invalid naming series {}: dot (.) missing").format(frappe.bold(self.series)), + exc=InvalidNamingSeriesError, + ) + + if not NAMING_SERIES_PATTERN.match(self.series): + frappe.throw( + _( + 'Special Characters except "-", "#", ".", "/", "{" and "}" not allowed in naming series', + ), + exc=InvalidNamingSeriesError, + ) + + def generate_next_name(self, doc: "Document") -> str: + self.validate() + parts = self.series.split(".") + return parse_naming_series(parts, doc=doc) + + def get_prefix(self) -> str: + """Naming series stores prefix to maintain a counter in DB. This prefix can be used to update counter or validations. + + e.g. `SINV-.YY.-.####` has prefix of `SINV-22-` in database for year 2022. + """ + + prefix = None + + def fake_counter_backend(partial_series, digits): + nonlocal prefix + prefix = partial_series + return "#" * digits + + # This function evaluates all parts till we hit numerical parts and then + # sends prefix + digits to DB to find next number. + # Instead of reimplementing the whole parsing logic in multiple places we + # can just ask this function to give us the prefix. + parse_naming_series(self.series, number_generator=fake_counter_backend) + + if prefix is None: + frappe.throw(_("Invalid Naming Series")) + + return prefix + + def get_preview(self, doc=None) -> List[str]: + """Generate preview of naming series without using DB counters""" + generated_names = [] + for count in range(1, 4): + + def fake_counter(_prefix, digits): + return str(count).zfill(digits) + + generated_names.append(parse_naming_series(self.series, doc=doc, number_generator=fake_counter)) + return generated_names + + def update_counter(self, new_count: int) -> None: + """Warning: Incorrectly updating series can result in unusable transactions""" + Series = frappe.qb.DocType("Series") + prefix = self.get_prefix() + + # Initialize if not present in DB + if frappe.db.get_value("Series", prefix, "name", order_by="name") is None: + frappe.qb.into(Series).insert(prefix, 0).columns("name", "current").run() + + ( + frappe.qb.update(Series).set(Series.current, cint(new_count)).where(Series.name == prefix) + ).run() + + def get_current_value(self) -> int: + prefix = self.get_prefix() + return cint(frappe.db.get_value("Series", prefix, "current", order_by="name")) + def set_new_name(doc): """ @@ -175,24 +266,32 @@ def make_autoname(key="", doctype="", doc=""): if key == "hash": return frappe.generate_hash(doctype, 10) - if "#" not in key: - key = key + ".#####" - elif "." not in key: - error_message = _("Invalid naming series (. missing)") - if doctype: - error_message = _("Invalid naming series (. missing) for {0}").format(doctype) - - frappe.throw(error_message) - - parts = key.split(".") - n = parse_naming_series(parts, doctype, doc) - return n + series = NamingSeries(key) + return series.generate_next_name(doc) -def parse_naming_series(parts, doctype="", doc=""): - n = "" +def parse_naming_series( + parts: Union[List[str], str], + doctype=None, + doc: Optional["Document"] = None, + number_generator: Optional[Callable[[str, int], str]] = None, +) -> str: + + """Parse the naming series and get next name. + + args: + parts: naming series parts (split by `.`) + doc: document to use for series that have parts using fieldnames + number_generator: Use different counter backend other than `tabSeries`. Primarily used for testing. + """ + + name = "" if isinstance(parts, str): parts = parts.split(".") + + if not number_generator: + number_generator = getseries + series_set = False today = now_datetime() for e in parts: @@ -200,7 +299,7 @@ def parse_naming_series(parts, doctype="", doc=""): if e.startswith("#"): if not series_set: digits = len(e) - part = getseries(n, digits) + part = number_generator(name, digits) series_set = True elif e == "YY": part = today.strftime("%y") @@ -225,9 +324,9 @@ def parse_naming_series(parts, doctype="", doc=""): part = e if isinstance(part, str): - n += part + name += part - return n + return name def determine_consecutive_week_number(datetime): @@ -311,14 +410,15 @@ def revert_series_if_last(key, name, doc=None): frappe.db.sql("UPDATE `tabSeries` SET `current` = `current` - 1 WHERE `name`=%s", prefix) -def get_default_naming_series(doctype): +def get_default_naming_series(doctype: str) -> Optional[str]: """get default value for `naming_series` property""" - naming_series = frappe.get_meta(doctype).get_field("naming_series").options or "" - if naming_series: - naming_series = naming_series.split("\n") - return naming_series[0] or naming_series[1] - else: - return None + naming_series_options = frappe.get_meta(doctype).get_naming_series_options() + + # Return first truthy options + # Empty strings are used to avoid populating forms by default + for option in naming_series_options: + if option: + return option def validate_name(doctype: str, name: Union[int, str], case: Optional[str] = None): @@ -349,7 +449,7 @@ def validate_name(doctype: str, name: Union[int, str], case: Optional[str] = Non frappe.throw(_("Name of {0} cannot be {1}").format(doctype, name), frappe.NameError) special_characters = "<>" - if re.findall("[{0}]+".format(special_characters), name): + if re.findall(f"[{special_characters}]+", name): message = ", ".join("'{0}'".format(c) for c in special_characters) frappe.throw( _("Name cannot contain special characters like {0}").format(message), frappe.NameError @@ -436,6 +536,6 @@ def _format_autoname(autoname, doc): return parse_naming_series([trimmed_param], doc=doc) # Replace braced params with their parsed value - name = re.sub(r"(\{[\w | #]+\})", get_param_value_for_match, autoname_value) + name = BRACED_PARAMS_PATTERN.sub(get_param_value_for_match, autoname_value) return name diff --git a/frappe/model/sync.py b/frappe/model/sync.py index 4c535b2811..93b883cda6 100644 --- a/frappe/model/sync.py +++ b/frappe/model/sync.py @@ -53,22 +53,6 @@ def sync_for(app_name, force=0, reset_permissions=False): os.path.join(FRAPPE_PATH, "website", "doctype", website_module, f"{website_module}.json") ) - for data_migration_module in [ - "data_migration_mapping_detail", - "data_migration_mapping", - "data_migration_plan_mapping", - "data_migration_plan", - ]: - files.append( - os.path.join( - FRAPPE_PATH, - "data_migration", - "doctype", - data_migration_module, - f"{data_migration_module}.json", - ) - ) - for desk_module in [ "number_card", "dashboard_chart", @@ -124,8 +108,6 @@ def get_doc_files(files, start_path): "web_template", "notification", "print_style", - "data_migration_mapping", - "data_migration_plan", "workspace", "onboarding_step", "module_onboarding", diff --git a/frappe/model/utils/__init__.py b/frappe/model/utils/__init__.py index a0dd0d89e8..6385b61c38 100644 --- a/frappe/model/utils/__init__.py +++ b/frappe/model/utils/__init__.py @@ -21,10 +21,7 @@ STANDARD_FIELD_CONVERSION_MAP = { "_assign": "Text", "docstatus": "Int", } - -""" -Model utilities, unclassified functions -""" +INCLUDE_DIRECTIVE_PATTERN = re.compile(r"""{% include\s['"](.*)['"]\s%}""") def set_default(doc, key): @@ -67,7 +64,7 @@ def render_include(content): # try 5 levels of includes for i in range(5): if "{% include" in content: - paths = re.findall(r"""{% include\s['"](.*)['"]\s%}""", content) + paths = INCLUDE_DIRECTIVE_PATTERN.findall(content) if not paths: frappe.throw(_("Invalid include path"), InvalidIncludePath) @@ -78,7 +75,7 @@ def render_include(content): if path.endswith(".html"): include = html_to_js_template(path, include) - content = re.sub(r"""{{% include\s['"]{0}['"]\s%}}""".format(path), include, content) + content = re.sub(rf"""{{% include\s['"]{path}['"]\s%}}""", include, content) else: break diff --git a/frappe/modules.txt b/frappe/modules.txt index a707ca853e..fb7817f6ba 100644 --- a/frappe/modules.txt +++ b/frappe/modules.txt @@ -8,7 +8,6 @@ Desk Integrations Printing Contacts -Data Migration Social Automation Event Streaming \ No newline at end of file diff --git a/frappe/modules/utils.py b/frappe/modules/utils.py index 60eaefddc5..ad6a75e900 100644 --- a/frappe/modules/utils.py +++ b/frappe/modules/utils.py @@ -207,13 +207,18 @@ def export_doc(doctype, name, module=None): write_document_file(frappe.get_doc(doctype, name), module) -def get_doctype_module(doctype): +def get_doctype_module(doctype) -> str: """Returns **Module Def** name of given doctype.""" def make_modules_dict(): return dict(frappe.db.sql("select name, module from tabDocType")) - return frappe.cache().get_value("doctype_modules", make_modules_dict)[doctype] + doctype_module_map = frappe.cache().get_value("doctype_modules", make_modules_dict) + + if module_name := doctype_module_map.get(doctype): + return module_name + else: + frappe.throw(_("DocType {} not found").format(doctype), exc=frappe.DoesNotExistError) doctype_python_modules = {} @@ -234,9 +239,9 @@ def load_doctype_module(doctype, module=None, prefix="", suffix=""): if key not in doctype_python_modules: doctype_python_modules[key] = frappe.get_module(module_name) except ImportError as e: - raise ImportError( - "Module import failed for {0} ({1})".format(doctype, module_name + " Error: " + str(e)) - ) + msg = f"Module import failed for {doctype}, the DocType you're trying to open might be deleted." + msg += f"
Error: {e}" + raise ImportError(msg) from e return doctype_python_modules[key] @@ -251,12 +256,15 @@ def get_module_name(doctype, module, prefix="", suffix="", app=None): ) -def get_module_app(module): - return frappe.local.module_app[scrub(module)] +def get_module_app(module: str) -> str: + app = frappe.local.module_app.get(scrub(module)) + if app is None: + frappe.throw(_("Module {} not found").format(module), exc=frappe.DoesNotExistError) + return app -def get_app_publisher(module): - app = frappe.local.module_app[scrub(module)] +def get_app_publisher(module: str) -> str: + app = get_module_app(module) if not app: frappe.throw(_("App not found")) app_publisher = frappe.get_hooks(hook="app_publisher", app_name=app)[0] @@ -321,7 +329,7 @@ def make_boilerplate(template, doc, opts=None): base_class=base_class, doctype=doc.name, **opts, - custom_controller=custom_controller + custom_controller=custom_controller, ) ) ) diff --git a/frappe/patches.txt b/frappe/patches.txt index 6407453af7..d46d40655e 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -191,6 +191,8 @@ frappe.patches.v14_0.remove_post_and_post_comment frappe.patches.v14_0.reset_creation_datetime frappe.patches.v14_0.remove_is_first_startup frappe.patches.v14_0.reload_workspace_child_tables +frappe.patches.v14_0.clear_long_pending_stale_logs +frappe.patches.v14_0.log_settings_migration [post_model_sync] frappe.patches.v14_0.drop_data_import_legacy @@ -200,3 +202,5 @@ frappe.patches.v14_0.remove_db_aggregation frappe.patches.v14_0.update_color_names_in_kanban_board_column frappe.patches.v14_0.update_is_system_generated_flag frappe.patches.v14_0.update_auto_account_deletion_duration +frappe.patches.v14_0.set_document_expiry_default +frappe.patches.v14_0.delete_data_migration_tool diff --git a/frappe/patches/v14_0/clear_long_pending_stale_logs.py b/frappe/patches/v14_0/clear_long_pending_stale_logs.py new file mode 100644 index 0000000000..53127cb197 --- /dev/null +++ b/frappe/patches/v14_0/clear_long_pending_stale_logs.py @@ -0,0 +1,41 @@ +import frappe +from frappe.core.doctype.log_settings.log_settings import clear_log_table +from frappe.utils import add_to_date, today + + +def execute(): + """Due to large size of log tables on old sites some table cleanups never finished during daily log clean up. This patch discards such data by using "big delete" code. + + ref: https://github.com/frappe/frappe/issues/16971 + """ + + DOCTYPE_RETENTION_MAP = { + "Error Log": get_current_setting("clear_error_log_after") or 90, + "Activity Log": get_current_setting("clear_activity_log_after") or 90, + "Email Queue": get_current_setting("clear_email_queue_after") or 30, + # child table on email queue + "Email Queue Recipient": get_current_setting("clear_email_queue_after") or 30, + "Error Snapshot": get_current_setting("clear_error_log_after") or 90, + # newly added + "Scheduled Job Log": 90, + } + + for doctype, retention in DOCTYPE_RETENTION_MAP.items(): + if is_log_cleanup_stuck(doctype, retention): + print(f"Clearing old {doctype} records") + clear_log_table(doctype, retention) + + +def is_log_cleanup_stuck(doctype: str, retention: int) -> bool: + """Check if doctype has data significantly older than configured cleanup period""" + threshold = add_to_date(today(), days=retention * -2) + + return bool(frappe.db.exists(doctype, {"modified": ("<", threshold)})) + + +def get_current_setting(fieldname): + try: + return frappe.db.get_single_value("Log Settings", fieldname) + except Exception: + # Field might be gone if patch is reattempted + pass diff --git a/frappe/patches/v14_0/delete_data_migration_tool.py b/frappe/patches/v14_0/delete_data_migration_tool.py new file mode 100644 index 0000000000..d0416cb1e7 --- /dev/null +++ b/frappe/patches/v14_0/delete_data_migration_tool.py @@ -0,0 +1,12 @@ +# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors +# MIT License. See license.txt + +import frappe + + +def execute(): + doctypes = frappe.db.get_all("DocType", {"module": "Data Migration", "custom": 0}, pluck="name") + for doctype in doctypes: + frappe.delete_doc("DocType", doctype, ignore_missing=True) + + frappe.delete_doc("Module Def", "Data Migration", ignore_missing=True, force=True) diff --git a/frappe/patches/v14_0/log_settings_migration.py b/frappe/patches/v14_0/log_settings_migration.py new file mode 100644 index 0000000000..203405e69b --- /dev/null +++ b/frappe/patches/v14_0/log_settings_migration.py @@ -0,0 +1,29 @@ +import frappe + + +def execute(): + old_settings = { + "Error Log": get_current_setting("clear_error_log_after"), + "Activity Log": get_current_setting("clear_activity_log_after"), + "Email Queue": get_current_setting("clear_email_queue_after"), + } + + frappe.reload_doc("core", "doctype", "Logs To Clear") + frappe.reload_doc("core", "doctype", "Log Settings") + + log_settings = frappe.get_doc("Log Settings") + log_settings.add_default_logtypes() + + for doctype, retention in old_settings.items(): + if retention: + log_settings.register_doctype(doctype, retention) + + log_settings.save() + + +def get_current_setting(fieldname): + try: + return frappe.db.get_single_value("Log Settings", fieldname) + except Exception: + # Field might be gone if patch is reattempted + pass diff --git a/frappe/patches/v14_0/set_document_expiry_default.py b/frappe/patches/v14_0/set_document_expiry_default.py new file mode 100644 index 0000000000..59a9db6c4d --- /dev/null +++ b/frappe/patches/v14_0/set_document_expiry_default.py @@ -0,0 +1,9 @@ +import frappe + + +def execute(): + frappe.db.set_value( + "System Settings", + "System Settings", + {"document_share_key_expiry": 30, "allow_older_web_view_links": 1}, + ) diff --git a/frappe/patches/v14_0/update_github_endpoints.py b/frappe/patches/v14_0/update_github_endpoints.py index 8f9a06a043..5ea638f0a6 100644 --- a/frappe/patches/v14_0/update_github_endpoints.py +++ b/frappe/patches/v14_0/update_github_endpoints.py @@ -1,10 +1,9 @@ import frappe import json + def execute(): if frappe.db.exists("Social Login Key", "github"): - frappe.db.set_value("Social Login Key", "github", "auth_url_data", - json.dumps({ - "scope": "user:email" - }) + frappe.db.set_value( + "Social Login Key", "github", "auth_url_data", json.dumps({"scope": "user:email"}) ) diff --git a/frappe/printing/doctype/print_format/print_format.py b/frappe/printing/doctype/print_format/print_format.py index e54f46b487..2e12fdb8d8 100644 --- a/frappe/printing/doctype/print_format/print_format.py +++ b/frappe/printing/doctype/print_format/print_format.py @@ -56,7 +56,7 @@ class PrintFormat(Document): frappe.throw(_("{0} is required").format(frappe.bold(_("HTML"))), frappe.MandatoryError) def extract_images(self): - from frappe.core.doctype.file.file import extract_images_from_html + from frappe.core.doctype.file.utils import extract_images_from_html if self.print_format_builder_beta: return diff --git a/frappe/printing/page/print/print.js b/frappe/printing/page/print/print.js index 122aea9fa1..7db6930a60 100644 --- a/frappe/printing/page/print/print.js +++ b/frappe/printing/page/print/print.js @@ -8,7 +8,7 @@ frappe.pages['print'].on_page_load = function(wrapper) { $(wrapper).bind('show', () => { const route = frappe.get_route(); const doctype = route[1]; - const docname = route[2]; + const docname = route.slice(2).join("/"); if (!frappe.route_options || !frappe.route_options.frm) { frappe.model.with_doc(doctype, docname, () => { let frm = { doctype: doctype, docname: docname }; @@ -19,7 +19,8 @@ frappe.pages['print'].on_page_load = function(wrapper) { }); }); } else { - print_view.frm = frappe.route_options.frm; + print_view.frm = frappe.route_options.frm.doctype ? + frappe.route_options.frm : frappe.route_options.frm.frm; frappe.route_options.frm = null; print_view.show(print_view.frm); } @@ -85,6 +86,10 @@ frappe.ui.form.PrintView = class { () => this.refresh_print_format(), { icon: 'refresh' } ); + + this.page.add_action_icon("file", () => { + this.go_to_form_view(); + }, '', __("Form")); } setup_sidebar() { @@ -498,6 +503,13 @@ frappe.ui.form.PrintView = class { } } + go_to_form_view() { + frappe.route_options = { + frm: this, + }; + frappe.set_route('Form', this.frm.doctype, this.frm.docname); + } + show_footer() { // footer is hidden by default as reqd by pdf generation // simple hack to show it in print preview diff --git a/frappe/printing/page/print_format_builder/print_format_builder.js b/frappe/printing/page/print_format_builder/print_format_builder.js index 313e8da539..d12a7cc96c 100644 --- a/frappe/printing/page/print_format_builder/print_format_builder.js +++ b/frappe/printing/page/print_format_builder/print_format_builder.js @@ -441,7 +441,7 @@ frappe.PrintFormatBuilder = class PrintFormatBuilder { const field = $(e.currentTarget).parent(); // new dialog var d = new frappe.ui.Dialog({ - title: "Set Properties", + title: __("Set Properties"), fields: [ { label: __("Label"), @@ -452,7 +452,8 @@ frappe.PrintFormatBuilder = class PrintFormatBuilder { label: __("Align Value"), fieldname: "align", fieldtype: "Select", - options: [{'label': __('Left'), 'value': 'left'}, {'label': __('Right'), 'value': 'right'}] + options: [{'label': __('Left', null, 'alignment'), 'value': 'left'}, + {'label': __('Right', null, 'alignment'), 'value': 'right'}] }, { label: __("Remove Field"), diff --git a/frappe/public/icons/timeless/symbol-defs.svg b/frappe/public/icons/timeless/symbol-defs.svg index bf4e02a7af..fbd72d6fb5 100644 --- a/frappe/public/icons/timeless/symbol-defs.svg +++ b/frappe/public/icons/timeless/symbol-defs.svg @@ -2,11 +2,11 @@ - + - + @@ -14,15 +14,15 @@ - + - + - + @@ -52,22 +52,22 @@ - + - + - + - + @@ -77,7 +77,7 @@ - + @@ -95,19 +95,19 @@ - + - + - + @@ -303,7 +303,7 @@ - + @@ -947,5 +947,8 @@ + + + diff --git a/frappe/public/js/frappe/doctype/index.js b/frappe/public/js/frappe/doctype/index.js index 09f020f370..d8ffa1097a 100644 --- a/frappe/public/js/frappe/doctype/index.js +++ b/frappe/public/js/frappe/doctype/index.js @@ -5,6 +5,21 @@ frappe.provide("frappe.model"); apply to both DocType form and customize form. */ frappe.model.DocTypeController = class DocTypeController extends frappe.ui.form.Controller { + setup() { + // setup formatters for fieldtype + frappe.meta.docfield_map[this.frm.doctype==='DocType' ? 'DocField' : 'Customize Form Field'].fieldtype.formatter = (value) => { + const prefix = { + 'Tab Break': '🔴', + 'Section Break': '🔵', + 'Column Break': '🟡', + }; + if (prefix[value]) { + value = prefix[value] + ' ' + value; + } + return value; + }; + } + max_attachments() { if (!this.frm.doc.max_attachments) { return; diff --git a/frappe/public/js/frappe/file_uploader/FileBrowser.vue b/frappe/public/js/frappe/file_uploader/FileBrowser.vue index f3017261ee..910cfd3588 100644 --- a/frappe/public/js/frappe/file_uploader/FileBrowser.vue +++ b/frappe/public/js/frappe/file_uploader/FileBrowser.vue @@ -101,7 +101,7 @@ export default { }, get_files_in_folder(folder, start) { return frappe - .call("frappe.core.doctype.file.file.get_files_in_folder", { + .call("frappe.core.api.file.get_files_in_folder", { folder, start, page_length: this.page_length @@ -132,7 +132,7 @@ export default { if (this.search_text.length < 3) return; frappe .call( - "frappe.core.doctype.file.file.get_files_by_search_text", + "frappe.core.api.file.get_files_by_search_text", { text: this.search_text } diff --git a/frappe/public/js/frappe/file_uploader/FileUploader.vue b/frappe/public/js/frappe/file_uploader/FileUploader.vue index 6c816c1115..b940d8d773 100644 --- a/frappe/public/js/frappe/file_uploader/FileUploader.vue +++ b/frappe/public/js/frappe/file_uploader/FileUploader.vue @@ -222,11 +222,14 @@ export default { }); } if (this.restrictions.max_file_size == null) { - frappe.call('frappe.core.doctype.file.file.get_max_file_size') + frappe.call('frappe.core.api.file.get_max_file_size') .then(res => { this.restrictions.max_file_size = Number(res.message); }); } + if (this.restrictions.max_number_of_files == null && this.doctype) { + this.restrictions.max_number_of_files = frappe.get_meta(self.doctype).max_attachments; + } }, watch: { files(newvalue, oldvalue) { @@ -283,6 +286,21 @@ export default { return file; }); }, + show_max_files_number_warning(file) { + console.warn( + `File skipped because it exceeds the allowed specified limit of ${max_number_of_files} uploads`, + file, + ); + if (this.doctype) { + MSG = __('File "{0}" was skipped because only {1} uploads are allowed for DocType "{2}"', [file.name, max_number_of_files, this.doctype]) + } else { + MSG = __('File "{0}" was skipped because only {1} uploads are allowed', [file.name, max_number_of_files]) + } + frappe.show_alert({ + message: MSG, + indicator: "orange", + }); + }, add_files(file_array) { let files = Array.from(file_array) .filter(this.check_restrictions) @@ -303,8 +321,19 @@ export default { error_message: null, uploading: false, private: !is_image - } + }; }); + + // pop extra files as per FileUploader.restrictions.max_number_of_files + max_number_of_files = this.restrictions.max_number_of_files; + if (max_number_of_files && files.length > max_number_of_files) { + files.slice(max_number_of_files).forEach(file => { + this.show_max_files_number_warning(file, this.doctype); + }); + + files = files.slice(0, max_number_of_files); + } + this.files = this.files.concat(files); // if only one file is allowed and crop_image_aspect_ratio is set, open cropper immediately if (this.files.length === 1 && !this.allow_multiple && this.restrictions.crop_image_aspect_ratio != null) { @@ -316,13 +345,10 @@ export default { check_restrictions(file) { let { max_file_size, allowed_file_types = [] } = this.restrictions; - let mime_type = file.type; - let extension = '.' + file.name.split('.').pop(); - let is_correct_type = true; let valid_file_size = true; - if (allowed_file_types.length) { + if (allowed_file_types && allowed_file_types.length) { is_correct_type = allowed_file_types.some((type) => { // is this is a mime-type if (type.includes('/')) { diff --git a/frappe/public/js/frappe/form/controls/base_control.js b/frappe/public/js/frappe/form/controls/base_control.js index e22235f60f..f491fb1427 100644 --- a/frappe/public/js/frappe/form/controls/base_control.js +++ b/frappe/public/js/frappe/form/controls/base_control.js @@ -72,9 +72,12 @@ frappe.ui.form.Control = class BaseControl { status = "Read"; } + let value = this.value || this.get_model_value(); + value = this.get_parsed_value(value); + if ( status === "Read" && - is_null(this.value) && + is_null(value) && !in_list(["HTML", "Image", "Button"], this.df.fieldtype) ) status = "None"; @@ -93,9 +96,12 @@ frappe.ui.form.Control = class BaseControl { } } + let value = frappe.model.get_value(this.doctype, this.docname, this.df.fieldname); + value = this.get_parsed_value(value); + // hide if no value if (this.doctype && status==="Read" && !this.only_input - && is_null(frappe.model.get_value(this.doctype, this.docname, this.df.fieldname)) + && is_null(value) && !in_list(["HTML", "Image", "Button"], this.df.fieldtype)) { // eslint-disable-next-line @@ -159,14 +165,18 @@ frappe.ui.form.Control = class BaseControl { return this.doc[this.df.fieldname]; } } + get_parsed_value(value) { + if (this.parse) { + value = this.parse(value); + } + return value; + } set_value(value, force_set_value=false) { return this.validate_and_set_in_model(value, null, force_set_value); } parse_validate_and_set_in_model(value, e) { - if(this.parse) { - value = this.parse(value); - } + value = this.get_parsed_value(value); return this.validate_and_set_in_model(value, e); } validate_and_set_in_model(value, e, force_set_value=false) { diff --git a/frappe/public/js/frappe/form/controls/data.js b/frappe/public/js/frappe/form/controls/data.js index 95abba616a..f3576b0201 100644 --- a/frappe/public/js/frappe/form/controls/data.js +++ b/frappe/public/js/frappe/form/controls/data.js @@ -58,7 +58,7 @@ frappe.ui.form.ControlData = class ControlData extends frappe.ui.form.ControlInp this.has_input = true; this.bind_change_event(); this.setup_autoname_check(); - + this.setup_copy_button(); if (this.df.options == 'URL') { this.setup_url_field(); } @@ -112,6 +112,18 @@ frappe.ui.form.ControlData = class ControlData extends frappe.ui.form.ControlInp }); } + setup_copy_button() { + if (this.df.with_copy_button) { + this.$wrapper.find('.control-input').append( + `` + ).find(".action-btn").click(() => { + frappe.utils.copy_to_clipboard(this.value); + }); + } + } + setup_barcode_field() { this.$wrapper.find('.control-input').append( ` diff --git a/frappe/public/js/frappe/form/controls/date.js b/frappe/public/js/frappe/form/controls/date.js index bea7e77bd1..09517e742b 100644 --- a/frappe/public/js/frappe/form/controls/date.js +++ b/frappe/public/js/frappe/form/controls/date.js @@ -64,6 +64,8 @@ frappe.ui.form.ControlDate = class ControlDate extends frappe.ui.form.ControlDat dateFormat: date_format, startDate: this.get_start_date(), keyboardNav: false, + minDate: this.df.min_date, + maxDate: this.df.max_date, firstDay: frappe.datetime.get_first_day_of_the_week_index(), onSelect: () => { this.$input.trigger('change'); @@ -138,6 +140,9 @@ frappe.ui.form.ControlDate = class ControlDate extends frappe.ui.form.ControlDat } parse(value) { if (value) { + if (value == "Invalid date") { + return ""; + } return frappe.datetime.user_to_str(value, false, true); } } diff --git a/frappe/public/js/frappe/form/controls/date_range.js b/frappe/public/js/frappe/form/controls/date_range.js index 170404f575..02cac1cf28 100644 --- a/frappe/public/js/frappe/form/controls/date_range.js +++ b/frappe/public/js/frappe/form/controls/date_range.js @@ -41,9 +41,10 @@ frappe.ui.form.ControlDateRange = class ControlDateRange extends frappe.ui.form. this.set_mandatory && this.set_mandatory(value); } parse(value) { + if (!value || (value && !value.includes('to'))) return value; // replace the separator (which can be in user language) with comma const to = __('{0} to {1}').replace('{0}', '').replace('{1}', ''); - value = value.replace(to, ','); + value = value && value.replace(to, ','); if(value && value.includes(',')) { var vals = value.split(','); diff --git a/frappe/public/js/frappe/form/controls/datetime.js b/frappe/public/js/frappe/form/controls/datetime.js index 9b10465d7b..43873b3b1e 100644 --- a/frappe/public/js/frappe/form/controls/datetime.js +++ b/frappe/public/js/frappe/form/controls/datetime.js @@ -40,8 +40,11 @@ frappe.ui.form.ControlDatetime = class ControlDatetime extends frappe.ui.form.Co value = frappe.datetime.convert_to_system_tz(value, true); } - return value; + if (value == "Invalid date") { + value = ""; + } } + return value; } format_for_input(value) { if (!value) return ""; diff --git a/frappe/public/js/frappe/form/controls/duration.js b/frappe/public/js/frappe/form/controls/duration.js index 361d10982e..940ad9d58a 100644 --- a/frappe/public/js/frappe/form/controls/duration.js +++ b/frappe/public/js/frappe/form/controls/duration.js @@ -109,6 +109,10 @@ frappe.ui.form.ControlDuration = class ControlDuration extends frappe.ui.form.Co return cint(this.value); } + parse(value) { + return !value ? "" : value; + } + refresh_input() { super.refresh_input(); this.set_duration_options(); diff --git a/frappe/public/js/frappe/form/controls/html.js b/frappe/public/js/frappe/form/controls/html.js index b2f18d4ccc..4cc0e4ab50 100644 --- a/frappe/public/js/frappe/form/controls/html.js +++ b/frappe/public/js/frappe/form/controls/html.js @@ -28,5 +28,6 @@ frappe.ui.form.ControlHTML = class ControlHTML extends frappe.ui.form.Control { this.df.options = html; this.html(html); } + return Promise.resolve(); } }; diff --git a/frappe/public/js/frappe/form/controls/multiselect_pills.js b/frappe/public/js/frappe/form/controls/multiselect_pills.js index bf93ac0dd8..3f3e204039 100644 --- a/frappe/public/js/frappe/form/controls/multiselect_pills.js +++ b/frappe/public/js/frappe/form/controls/multiselect_pills.js @@ -38,6 +38,10 @@ frappe.ui.form.ControlMultiSelectPills = class ControlMultiSelectPills extends f } parse(value) { + if (typeof value == "object" || !this.rows) { + return value; + } + if (value) { this.rows.push(value); } diff --git a/frappe/public/js/frappe/form/controls/table_multiselect.js b/frappe/public/js/frappe/form/controls/table_multiselect.js index e106d8eed6..14bc0e297c 100644 --- a/frappe/public/js/frappe/form/controls/table_multiselect.js +++ b/frappe/public/js/frappe/form/controls/table_multiselect.js @@ -50,6 +50,10 @@ frappe.ui.form.ControlTableMultiSelect = class ControlTableMultiSelect extends f this.$input_area.find('.link-btn').remove(); } parse(value, label) { + if (typeof value == "object" || !this.rows) { + return value; + } + const link_field = this.get_link_field(); if (value) { diff --git a/frappe/public/js/frappe/form/controls/time.js b/frappe/public/js/frappe/form/controls/time.js index f7fcc4c618..bbdec69cf8 100644 --- a/frappe/public/js/frappe/form/controls/time.js +++ b/frappe/public/js/frappe/form/controls/time.js @@ -82,6 +82,9 @@ frappe.ui.form.ControlTime = class ControlTime extends frappe.ui.form.ControlDat } parse(value) { if (value) { + if (value == "Invalid date") { + value = ""; + } return frappe.datetime.user_to_str(value, true); } } diff --git a/frappe/public/js/frappe/form/dashboard.js b/frappe/public/js/frappe/form/dashboard.js index 0731bdf8fb..c057903a63 100644 --- a/frappe/public/js/frappe/form/dashboard.js +++ b/frappe/public/js/frappe/form/dashboard.js @@ -19,7 +19,7 @@ frappe.ui.form.Dashboard = class FormDashboard { }); this.heatmap_area = this.make_section({ - label: __("Overview"), + label: __("Activity"), css_class: 'form-heatmap', hidden: 1, collapsible: 1, diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js index 87d7d73814..eefc629b4d 100644 --- a/frappe/public/js/frappe/form/form.js +++ b/frappe/public/js/frappe/form/form.js @@ -39,6 +39,12 @@ frappe.ui.form.Form = class FrappeForm { this.parent = parent; this.doctype_layout = frappe.get_doc('DocType Layout', doctype_layout_name); this.setup_meta(doctype); + + this.beforeUnloadListener = (event) => { + event.preventDefault(); + // A String is returned for compatability with older Browsers. Return Value has to be truthy to trigger "Leave Site" Dialog + return event.returnValue = 'There are unsaved changes, are you sure you want to exit?'; + }; } setup_meta() { @@ -343,6 +349,8 @@ frappe.ui.form.Form = class FrappeForm { refresh(docname) { var switched = docname ? true : false; + removeEventListener("beforeunload", this.beforeUnloadListener, {capture: true}); + if(docname) { this.switch_doc(docname); } @@ -567,7 +575,7 @@ frappe.ui.form.Form = class FrappeForm { this.$wrapper.trigger('render_complete'); - this.cscript.is_onload && this.set_first_tab_as_active(); + this.layout.set_first_tab_as_active(switched || this.cscript.is_onload); if(!this.hidden) { this.layout.show_empty_form_message(); @@ -584,11 +592,6 @@ frappe.ui.form.Form = class FrappeForm { this.setup_image_autocompletions_in_markdown(); } - set_first_tab_as_active() { - this.layout.tabs[0] - && this.layout.tabs[0].set_active(); - } - focus_on_first_input() { let first = this.form_wrapper.find('.form-layout :input:visible:first'); if (!in_list(["Date", "Datetime"], first.attr("data-fieldtype"))) { @@ -913,7 +916,7 @@ frappe.ui.form.Form = class FrappeForm { } else { frappe.confirm(__("Permanently Cancel {0}?", [this.docname]), cancel_doc, me.handle_save_fail(btn, on_error)); } - }; + } savetrash() { this.validate_form_action("Delete"); @@ -1236,6 +1239,9 @@ frappe.ui.form.Form = class FrappeForm { dirty() { this.doc.__unsaved = 1; this.$wrapper.trigger('dirty'); + if (!frappe.boot.developer_mode) { + addEventListener("beforeunload", this.beforeUnloadListener, {capture: true}); + } } get_docinfo() { @@ -1482,15 +1488,21 @@ frappe.ui.form.Form = class FrappeForm { if(fieldobj) { if(!if_missing || !frappe.model.has_value(me.doctype, me.doc.name, f)) { if(frappe.model.table_fields.includes(fieldobj.df.fieldtype) && $.isArray(v)) { - + // set entire child table from specified array as value frappe.model.clear_table(me.doc, fieldobj.df.fieldname); - for (var i=0, j=v.length; i < j; i++) { - var d = v[i]; - var child = frappe.model.add_child(me.doc, fieldobj.df.options, - fieldobj.df.fieldname, i+1); - $.extend(child, d); - } + const standard_fields = [...frappe.model.std_fields_list, ...frappe.model.child_table_field_list]; + v.forEach((d, idx) => { + let child = frappe.model.add_child(me.doc, fieldobj.df.options, + fieldobj.df.fieldname, idx+1); + + // Don't set standard field, avoid mutating input too. + let doc_copy = {...d}; + standard_fields.forEach(field => { + delete doc_copy[field]; + }); + $.extend(child, doc_copy); + }); me.refresh_field(f); return Promise.resolve(); diff --git a/frappe/public/js/frappe/form/formatters.js b/frappe/public/js/frappe/form/formatters.js index 2b0f996661..3bf36c86af 100644 --- a/frappe/public/js/frappe/form/formatters.js +++ b/frappe/public/js/frappe/form/formatters.js @@ -15,17 +15,35 @@ frappe.form.formatters = { return "
" + value + "
"; } }, + _apply_custom_formatter: function(value, df) { + /* you can add a custom formatter in df.formatter + example: + frappe.meta.docfield_map[df.parent][df.fieldname].formatter = (value) => { + if (value==='Test') return '😜'; + } + */ + + if (df) { + const std_df = frappe.meta.docfield_map[df.parent] && frappe.meta.docfield_map[df.parent][df.fieldname]; + if (std_df && std_df.formatter && typeof std_df.formatter==='function') { + value = std_df.formatter(value); + } + } + return value; + }, Data: function(value, df) { if (df && df.options == "URL") { return `${value}`; } - return value==null ? "" : value; + value = value==null ? "" : value; + + return frappe.form.formatters._apply_custom_formatter(value, df); }, - Autocomplete: function(value) { - return __(frappe.form.formatters["Data"](value)); + Autocomplete: function(value, df) { + return __(frappe.form.formatters["Data"](value, df)); }, - Select: function(value) { - return __(frappe.form.formatters["Data"](value)); + Select: function(value, df) { + return __(frappe.form.formatters["Data"](value, df)); }, Float: function(value, docfield, options, doc) { // don't allow 0 precision for Floats, hence or'ing with null @@ -183,7 +201,7 @@ frappe.form.formatters = { return ""; } }, - Text: function(value) { + Text: function(value, df) { if(value) { var tags = [" { - if (!tab.wrapper.hasClass('hide') || !tab.parent.hasClass('hide')) { - tab.parent.removeClass('show hide'); - tab.wrapper.removeClass('show hide'); - if ( - tab.wrapper.find( - ".form-section:not(.hide-control, .empty-section), .form-dashboard-section:not(.hide-control, .empty-section)" - ).length - ) { - tab.toggle(true); - } else { - tab.toggle(false); - } - } - }); + for (let tab of this.tabs) { + tab.refresh(); + } const visible_tabs = this.tabs.filter(tab => !tab.hidden); if (visible_tabs && visible_tabs.length == 1) { @@ -349,6 +338,13 @@ frappe.ui.form.Layout = class Layout { } } + set_first_tab_as_active(switched) { + if (this.tabs.length && (switched || !this.frm.active_tab)) { + // set first tab as active when opening for first time, or new doc + this.tabs[0].set_active(); + } + } + refresh_fields(fields) { let fieldnames = fields.map((field) => { if (field.fieldname) return field.fieldname; diff --git a/frappe/public/js/frappe/form/quick_entry.js b/frappe/public/js/frappe/form/quick_entry.js index 86523d7088..f55f7139c9 100644 --- a/frappe/public/js/frappe/form/quick_entry.js +++ b/frappe/public/js/frappe/form/quick_entry.js @@ -267,7 +267,7 @@ frappe.ui.form.QuickEntryForm = class QuickEntryForm { render_edit_in_full_page_link() { var me = this; this.dialog.add_custom_action( - `${__("Edit in full page")}`, + `${__("Edit Full Form")}`, () => me.open_doc(true) ); } diff --git a/frappe/public/js/frappe/form/sidebar/attachments.js b/frappe/public/js/frappe/form/sidebar/attachments.js index 3663cfd9a5..effe3a575b 100644 --- a/frappe/public/js/frappe/form/sidebar/attachments.js +++ b/frappe/public/js/frappe/form/sidebar/attachments.js @@ -152,6 +152,7 @@ frappe.ui.form.Attachments = class Attachments { var me = this; return frappe.call({ method: 'frappe.desk.form.utils.remove_attach', + type: 'DELETE', args: { fid: fileid, dt: me.frm.doctype, @@ -182,6 +183,9 @@ frappe.ui.form.Attachments = class Attachments { folder: 'Home/Attachments', on_success: (file_doc) => { this.attachment_uploaded(file_doc); + }, + restrictions: { + max_number_of_files: this.frm.meta.max_attachments - this.frm.attachments.get_attachments().length, } }); } diff --git a/frappe/public/js/frappe/form/sidebar/user_image.js b/frappe/public/js/frappe/form/sidebar/user_image.js index f657cbca02..d08e17bced 100644 --- a/frappe/public/js/frappe/form/sidebar/user_image.js +++ b/frappe/public/js/frappe/form/sidebar/user_image.js @@ -65,15 +65,17 @@ frappe.ui.form.setup_user_image_event = function(frm) { }); } - frm.sidebar.image_wrapper.on('click', ':not(.sidebar-image-actions)', (e) => { - let $target = $(e.currentTarget); - if ($target.is('a.dropdown-toggle, .dropdown')) { - return; - } - let dropdown = frm.sidebar.image_wrapper.find('.sidebar-image-actions .dropdown'); - dropdown.toggleClass('open'); - e.stopPropagation(); - }); + if (frm.meta.image_field && !frm.fields_dict[frm.meta.image_field].df.read_only) { + frm.sidebar.image_wrapper.on('click', ':not(.sidebar-image-actions)', (e) => { + let $target = $(e.currentTarget); + if ($target.is('a.dropdown-toggle, .dropdown')) { + return; + } + let dropdown = frm.sidebar.image_wrapper.find('.sidebar-image-actions .dropdown'); + dropdown.toggleClass('open'); + e.stopPropagation(); + }); + } // bind click on image_wrapper frm.sidebar.image_wrapper.on('click', '.sidebar-image-change, .sidebar-image-remove', function(e) { diff --git a/frappe/public/js/frappe/form/tab.js b/frappe/public/js/frappe/form/tab.js index 0e740ce49c..69c573186b 100644 --- a/frappe/public/js/frappe/form/tab.js +++ b/frappe/public/js/frappe/form/tab.js @@ -3,7 +3,7 @@ export default class Tab { this.parent = parent; this.df = df || {}; this.frm = frm; - this.doctype = 'User'; + this.doctype = this.frm.doctype; this.label = this.df && this.df.label; this.tabs_list = tabs_list; this.tabs_content = tabs_content; @@ -36,10 +36,27 @@ export default class Tab { // hide if explicitly hidden let hide = this.df.hidden || this.df.hidden_due_to_dependency; + + // hide if dashboard and not saved + if (!hide && this.df.show_dashboard && this.frm.is_new() && !this.fields_list.length) { + hide = true; + } + + // hide if no read permission if (!hide && this.frm && !this.frm.get_perm(this.df.permlevel || 0, "read")) { hide = true; } + if (!hide && !this.df.show_dashboard) { + // show only if there is at least one visibe section or control + hide = true; + if (this.wrapper.find( + ".form-section:not(.hide-control, .empty-section), .form-dashboard-section:not(.hide-control, .empty-section)" + ).length) { + hide = false; + } + } + this.toggle(!hide); } @@ -61,7 +78,8 @@ export default class Tab { set_active() { this.parent.find('.nav-link').tab('show'); - this.wrapper.addClass('show'); + this.wrapper.addClass('active'); + this.frm.active_tab = this; } is_active() { diff --git a/frappe/public/js/frappe/form/toolbar.js b/frappe/public/js/frappe/form/toolbar.js index a19062d209..44820c2fa7 100644 --- a/frappe/public/js/frappe/form/toolbar.js +++ b/frappe/public/js/frappe/form/toolbar.js @@ -356,8 +356,9 @@ frappe.ui.form.Toolbar = class Toolbar { } make_customize_buttons() { - if (frappe.user_roles.includes("System Manager")) { - let is_doctype_form = this.frm.doctype === 'DocType'; + let is_doctype_form = this.frm.doctype === 'DocType'; + if (frappe.model.can_create("Custom Field") + && frappe.model.can_create("Property Setter")) { let doctype = is_doctype_form ? this.frm.docname : this.frm.doctype; let is_doctype_custom = is_doctype_form ? this.frm.doc.custom : false; @@ -372,7 +373,9 @@ frappe.ui.form.Toolbar = class Toolbar { } }, true); } + } + if (frappe.model.can_create("DocType")) { if (frappe.boot.developer_mode===1 && !is_doctype_form) { // edit doctype this.page.add_menu_item(__("Edit DocType"), () => { diff --git a/frappe/public/js/frappe/list/base_list.js b/frappe/public/js/frappe/list/base_list.js index f64ae39d1b..b91693257a 100644 --- a/frappe/public/js/frappe/list/base_list.js +++ b/frappe/public/js/frappe/list/base_list.js @@ -270,7 +270,6 @@ frappe.views.BaseList = class BaseList { doctype: this.doctype, stats: this.stats, parent: this.$page.find(".layout-side-section"), - // set_filter: this.set_filter.bind(this), page: this.page, list_view: this, }); @@ -428,7 +427,7 @@ frappe.views.BaseList = class BaseList { const filter = this.get_filters_for_args().filter(f => f[1] == fieldname)[0]; if (!filter) return; return { - 'like': filter[3].replace(/^%?|%$/g, ''), + 'like': filter[3]?.replace(/^%?|%$/g, ''), 'not set': null }[filter[2]] || filter[3]; } @@ -620,9 +619,7 @@ class FilterArea { filters = [filter]; } - filters = filters.filter((f) => { - return !this.exists(f); - }); + filters = filters.filter(f => !this.exists(f)); const { non_standard_filters, promise } = this.set_standard_filter( filters @@ -743,7 +740,7 @@ class FilterArea { let fields = [ { fieldtype: "Data", - label: "Name", + label: "ID", condition: "like", fieldname: "name", onchange: () => this.refresh_list_view(), diff --git a/frappe/public/js/frappe/list/bulk_operations.js b/frappe/public/js/frappe/list/bulk_operations.js index 94ec9d4e67..7c8c515643 100644 --- a/frappe/public/js/frappe/list/bulk_operations.js +++ b/frappe/public/js/frappe/list/bulk_operations.js @@ -208,7 +208,7 @@ export default class BulkOperations { const default_field = field_options.find(value => status_regex.test(value)); const dialog = new frappe.ui.Dialog({ - title: __('Edit'), + title: __('Bulk Edit'), fields: [ { 'fieldtype': 'Select', @@ -225,7 +225,9 @@ export default class BulkOperations { 'fieldtype': 'Data', 'label': __('Value'), 'fieldname': 'value', - 'reqd': 1 + onchange() { + show_help_text(); + } } ], primary_action: ({ value }) => { @@ -239,7 +241,7 @@ export default class BulkOperations { docnames: docnames, action: 'update', data: { - [fieldname]: value + [fieldname]: value || null } } }).then(r => { @@ -254,10 +256,11 @@ export default class BulkOperations { frappe.show_alert(__('Updated successfully')); }); }, - primary_action_label: __('Update') + primary_action_label: __('Update {0} records', [docnames.length]), }); if (default_field) set_value_field(dialog); // to set `Value` df based on default `Field` + show_help_text(); function set_value_field (dialogObj) { const new_df = Object.assign({}, @@ -275,9 +278,20 @@ export default class BulkOperations { new_df.default = options[0] || options[1]; } new_df.label = __('Value'); - new_df.reqd = 1; + new_df.onchange = show_help_text; + delete new_df.depends_on; dialogObj.replace_field('value', new_df); + show_help_text(); + } + + function show_help_text() { + let value = dialog.get_value('value'); + if (value == null || value === '') { + dialog.set_df_property('value', 'description', __('You have not entered a value. The field will be set to empty.')); + } else { + dialog.set_df_property('value', 'description', ''); + } } dialog.refresh(); diff --git a/frappe/public/js/frappe/list/list_settings.js b/frappe/public/js/frappe/list/list_settings.js index b7318ea780..6c48bd013a 100644 --- a/frappe/public/js/frappe/list/list_settings.js +++ b/frappe/public/js/frappe/list/list_settings.js @@ -310,7 +310,7 @@ export default class ListSettings { let me = this; me.subject_field = { - label: "Name", + label: "ID", fieldname: "name" }; diff --git a/frappe/public/js/frappe/list/list_sidebar.js b/frappe/public/js/frappe/list/list_sidebar.js index 28c6eaab0b..5feeb1928e 100644 --- a/frappe/public/js/frappe/list/list_sidebar.js +++ b/frappe/public/js/frappe/list/list_sidebar.js @@ -7,7 +7,6 @@ frappe.provide('frappe.views'); // stats = list of fields // doctype // parent -// set_filter = function called on click frappe.views.ListSidebar = class ListSidebar { constructor(opts) { diff --git a/frappe/public/js/frappe/list/list_view.js b/frappe/public/js/frappe/list/list_view.js index 4d172e9802..d57c57a0b6 100644 --- a/frappe/public/js/frappe/list/list_view.js +++ b/frappe/public/js/frappe/list/list_view.js @@ -331,7 +331,7 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList { this.columns.push({ type: "Subject", df: { - label: __("Name"), + label: __("ID"), fieldname: "name", }, }); @@ -398,7 +398,7 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList { this.columns.push({ type: "Field", df: { - label: __("Name"), + label: __("ID"), fieldname: "name", }, }); @@ -870,9 +870,7 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList {
`; } - const comment_count = ` + const comment_count = ` ${frappe.utils.icon('small-message')} ${doc._comment_count > 99 ? "99+" : doc._comment_count} `; @@ -1534,7 +1532,10 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList { }), standard: true, }); + } + if (frappe.model.can_create("Custom Field") + && frappe.model.can_create("Property Setter")) { items.push({ label: __("Customize", null, "Button in list view menu"), action: () => { diff --git a/frappe/public/js/frappe/logtypes.js b/frappe/public/js/frappe/logtypes.js new file mode 100644 index 0000000000..d6e386c4d2 --- /dev/null +++ b/frappe/public/js/frappe/logtypes.js @@ -0,0 +1,30 @@ +// Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors +// MIT License. See license.txt + +// Common utility functions for logging doctypes. + +frappe.provide("frappe.utils.logtypes"); + +frappe.utils.logtypes.show_log_retention_message = (doctype) => { + if (!frappe.model.can_write("Log Settings")) { + return; + } + + const add_sidebar_message = (message) => { + let sidebar_entry = $('').appendTo(cur_list.page.sidebar); + $(`
${message}
`).appendTo(sidebar_entry); + }; + + const log_settings_link = `${__('Log Settings')}`; + const cta = __("You can change the retention policy from {0}.", [log_settings_link,]); + let message = __("{0} records are not automatically deleted.", [__(doctype),]); + + frappe.db + .get_value("Logs To Clear", { ref_doctype: doctype }, "days", null, "Log Settings") + .then((r) => { + if (!r.exc && r.message && r.message.days) { + message = __("{0} records are retained for {1} days.", [__(doctype), r.message.days,]); + } + add_sidebar_message(`${message} ${cta}`); + }); +}; diff --git a/frappe/public/js/frappe/model/meta.js b/frappe/public/js/frappe/model/meta.js index 4a7295ed4e..b89374ab23 100644 --- a/frappe/public/js/frappe/model/meta.js +++ b/frappe/public/js/frappe/model/meta.js @@ -109,7 +109,7 @@ $.extend(frappe.meta, { var fields = $.map(frappe.meta.get_docfields(doctype, name), function(df) { return (df.fieldtype==="Link" && df.ignore_user_permissions!==1) ? df : null; }); - fields = fields.concat({label: "Name", fieldname: name, options: doctype}); + fields = fields.concat({label: "ID", fieldname: name, options: doctype}); return fields; }, @@ -177,12 +177,17 @@ $.extend(frappe.meta, { get_label: function(dt, fn, dn) { var standard = { - 'owner': __('Owner'), + 'name': __('ID'), 'creation': __('Created On'), - 'modified': __('Last Modified On'), - 'idx': __('Idx'), - 'name': __('Name'), - 'modified_by': __('Last Modified By') + 'docstatus': __('Document Status'), + 'idx': __('Index'), + 'modified': __('Last Updated On'), + 'modified_by': __('Last Updated By'), + 'owner': __('Created By'), + '_user_tags': __('Tags'), + '_liked_by': __('Liked By'), + '_comments': __('Comments'), + '_assign': __('Assigned To'), } if(standard[fn]) { return standard[fn]; diff --git a/frappe/public/js/frappe/roles_editor.js b/frappe/public/js/frappe/roles_editor.js index e23f808a82..05f58692f6 100644 --- a/frappe/public/js/frappe/roles_editor.js +++ b/frappe/public/js/frappe/roles_editor.js @@ -94,10 +94,14 @@ frappe.RoleEditor = class { .css("max-width", "80vw"); } show() { - let user_roles = this.frm.doc.roles.map(a => a.role); + this.reset(); + this.set_enable_disable(); + } + + reset() { + let user_roles = (this.frm.doc.roles || []).map(a => a.role); this.multicheck.selected_options = user_roles; this.multicheck.refresh_input(); - this.set_enable_disable(); } set_roles_in_table() { let roles = this.frm.doc.roles || []; diff --git a/frappe/public/js/frappe/ui/field_group.js b/frappe/public/js/frappe/ui/field_group.js index cba702407a..c226c4cbfb 100644 --- a/frappe/public/js/frappe/ui/field_group.js +++ b/frappe/public/js/frappe/ui/field_group.js @@ -132,7 +132,7 @@ frappe.ui.FieldGroup = class FieldGroup extends frappe.ui.form.Layout { var f = this.fields_dict[key]; if (f) { f.set_value(val).then(() => { - f.set_input(val); + f.set_input?.(val); this.refresh_dependency(); resolve(); }); diff --git a/frappe/public/js/frappe/ui/filters/filter.js b/frappe/public/js/frappe/ui/filters/filter.js index 4c21cacf90..c19d696008 100644 --- a/frappe/public/js/frappe/ui/filters/filter.js +++ b/frappe/public/js/frappe/ui/filters/filter.js @@ -249,14 +249,21 @@ frappe.ui.Filter = class { const filter_value = this.filter_list.get_filter_value(fieldname); args[field_name] = filter_value; } - frappe - .xcall(this.filters_config[condition].get_field, args) - .then(field => { - df.fieldtype = field.fieldtype; - df.options = field.options; - df.fieldname = fieldname; - this.make_field(df, cur.fieldtype); + let setup_field = (field) => { + df.fieldtype = field.fieldtype; + df.options = field.options; + df.fieldname = fieldname; + this.make_field(df, cur.fieldtype); + } + if (this.filters_config[condition].data) { + let field = this.filters_config[condition].data; + setup_field(field); + } else { + frappe.xcall(this.filters_config[condition].get_field, args).then(field => { + this.filters_config[condition].data = field; + setup_field(field); }); + } } else { this.make_field(df, cur.fieldtype); } @@ -436,15 +443,17 @@ frappe.ui.filter_utils = { val = val == 'Yes' ? 1 : 0; } - if (condition.indexOf('like', 'not like') !== -1) { + if (['like', 'not like'].includes(condition)) { // automatically append wildcards if (val && !(val.startsWith('%') || val.endsWith('%'))) { val = '%' + val + '%'; } - } else if (in_list(['in', 'not in'], condition)) { + } else if (['in', 'not in'].includes(condition)) { if (val) { val = val.split(',').map((v) => strip(v)); } + } else if (frappe.boot.additional_filters_config[condition]) { + val = field.value || val; } if (val === '%') { val = ''; diff --git a/frappe/public/js/frappe/ui/keyboard.js b/frappe/public/js/frappe/ui/keyboard.js index 85ce248175..f7ba2dd9ae 100644 --- a/frappe/public/js/frappe/ui/keyboard.js +++ b/frappe/public/js/frappe/ui/keyboard.js @@ -81,6 +81,7 @@ frappe.ui.keys.show_keyboard_shortcut_dialog = () => { } let html = shortcuts .filter(s => s.condition ? s.condition() : true) + .filter(s => !!s.description) .map(shortcut => { let shortcut_label = shortcut.shortcut .split('+') @@ -94,6 +95,8 @@ frappe.ui.keys.show_keyboard_shortcut_dialog = () => { ${shortcut.description || ''} `; }).join(''); + if (!html) return ''; + html = `
${heading}
${html} diff --git a/frappe/public/js/frappe/ui/page.js b/frappe/public/js/frappe/ui/page.js index 70e0862ba5..eded1aefc5 100644 --- a/frappe/public/js/frappe/ui/page.js +++ b/frappe/public/js/frappe/ui/page.js @@ -510,7 +510,7 @@ frappe.ui.Page = class Page { if (!label || !parent) return false; - const item_selector = `${selector}[data-label='${encodeURIComponent(label)}']`; + const item_selector = `${selector}[data-label="${encodeURIComponent(label)}"]`; const existing_items = $(parent).find(item_selector); return existing_items?.length > 0 && existing_items; diff --git a/frappe/public/js/frappe/ui/toolbar/about.js b/frappe/public/js/frappe/ui/toolbar/about.js index 47dc7ee851..464c7c4787 100644 --- a/frappe/public/js/frappe/ui/toolbar/about.js +++ b/frappe/public/js/frappe/ui/toolbar/about.js @@ -19,7 +19,7 @@ frappe.ui.misc.about = function() {

Installed Apps

\
Loading versions...
\
\ -

© Frappe Technologies Pvt. Ltd and contributors

\ +

© Frappe Technologies Pvt. Ltd. and contributors

\ ", frappe.app)); frappe.ui.misc.about_dialog = d; diff --git a/frappe/public/js/frappe/ui/toolbar/awesome_bar.js b/frappe/public/js/frappe/ui/toolbar/awesome_bar.js index 9ff8fe96f3..97cfb78d93 100644 --- a/frappe/public/js/frappe/ui/toolbar/awesome_bar.js +++ b/frappe/public/js/frappe/ui/toolbar/awesome_bar.js @@ -106,6 +106,7 @@ frappe.search.AwesomeBar = class AwesomeBar { frappe.set_route(item.route); } $input.val(""); + $input.trigger('blur'); }); $input.on("awesomplete-selectcomplete", function(e) { diff --git a/frappe/public/js/frappe/utils/datetime.js b/frappe/public/js/frappe/utils/datetime.js index 2bf64fd88e..12cefb4fd9 100644 --- a/frappe/public/js/frappe/utils/datetime.js +++ b/frappe/public/js/frappe/utils/datetime.js @@ -191,7 +191,7 @@ $.extend(frappe.datetime, { global_date_format: function(d) { var m = moment(d); if(m._f && m._f.indexOf("HH")!== -1) { - return m.format("Do MMMM YYYY, h:mma") + return m.format("Do MMMM YYYY, hh:mm A"); } else { return m.format('Do MMMM YYYY'); } diff --git a/frappe/public/js/frappe/utils/file_manager.js b/frappe/public/js/frappe/utils/file_manager.js index f57cb8200a..e366920f55 100644 --- a/frappe/public/js/frappe/utils/file_manager.js +++ b/frappe/public/js/frappe/utils/file_manager.js @@ -20,7 +20,7 @@ frappe.file_manager = function() { new_folder = new_folder_; frappe.call({ - method:"frappe.core.doctype.file.file.move_file", + method: "frappe.core.api.file.move_file", args: { file_list: files_to_move, new_parent: new_folder, diff --git a/frappe/public/js/frappe/views/breadcrumbs.js b/frappe/public/js/frappe/views/breadcrumbs.js index 4026f9b47b..30b7f544fc 100644 --- a/frappe/public/js/frappe/views/breadcrumbs.js +++ b/frappe/public/js/frappe/views/breadcrumbs.js @@ -150,7 +150,7 @@ frappe.breadcrumbs = { set_form_breadcrumb(breadcrumbs, view) { const doctype = breadcrumbs.doctype; - const docname = frappe.get_route()[2]; + const docname = frappe.get_route().slice(2).join("/"); let form_route = `/app/${frappe.router.slug(doctype)}/${docname}`; $(`
  • ${__(docname)}
    • `) .appendTo(this.$breadcrumbs); diff --git a/frappe/public/js/frappe/views/file/file_view.js b/frappe/public/js/frappe/views/file/file_view.js index de06e6013e..5bfa210ed9 100644 --- a/frappe/public/js/frappe/views/file/file_view.js +++ b/frappe/public/js/frappe/views/file/file_view.js @@ -109,7 +109,7 @@ frappe.views.FileView = class FileView extends frappe.views.ListView { }; frappe.call({ method: - "frappe.core.doctype.file.file.create_new_folder", + "frappe.core.api.file.create_new_folder", args: data }); }, @@ -130,7 +130,7 @@ frappe.views.FileView = class FileView extends frappe.views.ListView { frappe.show_alert(__("Unzipping files...")); frappe .call( - "frappe.core.doctype.file.file.unzip_file", + "frappe.core.api.file.unzip_file", { name: file.name } @@ -173,7 +173,7 @@ frappe.views.FileView = class FileView extends frappe.views.ListView { this.page.add_actions_menu_item(__('Export as zip'), () => { let docnames = this.get_checked_items(true); if (docnames.length) { - open_url_post('/api/method/frappe.core.doctype.file.file.zip_files', { + open_url_post('/api/method/frappe.core.api.file.zip_files', { files: JSON.stringify(docnames) }); } diff --git a/frappe/public/js/frappe/views/image/image_view.js b/frappe/public/js/frappe/views/image/image_view.js index c499f122a9..57647642a3 100644 --- a/frappe/public/js/frappe/views/image/image_view.js +++ b/frappe/public/js/frappe/views/image/image_view.js @@ -145,7 +145,7 @@ frappe.views.ImageView = class ImageView extends frappe.views.ListView { get_attached_images() { return frappe .call({ - method: "frappe.core.doctype.file.file.get_attached_images", + method: "frappe.core.api.file.get_attached_images", args: { doctype: this.doctype, names: this.items.map(i => i.name) diff --git a/frappe/public/js/frappe/views/reports/report_view.js b/frappe/public/js/frappe/views/reports/report_view.js index 4c545d1ce7..6880d472d3 100644 --- a/frappe/public/js/frappe/views/reports/report_view.js +++ b/frappe/public/js/frappe/views/reports/report_view.js @@ -624,9 +624,11 @@ frappe.views.ReportView = class ReportView extends frappe.views.ListView { title: __('Edit {0}', [col.docfield.label]), fields: [col.docfield], primary_action: () => { - this.datatable.cellmanager.submitEditing(); this.datatable.cellmanager.deactivateEditing(); d.hide(); + }, + on_hide: () => { + this.datatable.cellmanager.deactivateEditing(false); } }); d.show(); diff --git a/frappe/public/js/frappe/views/workspace/blocks/block.js b/frappe/public/js/frappe/views/workspace/blocks/block.js index 1df6b707fe..7b9d547b6b 100644 --- a/frappe/public/js/frappe/views/workspace/blocks/block.js +++ b/frappe/public/js/frappe/views/workspace/blocks/block.js @@ -214,10 +214,6 @@ export default class Block { $button.find('.dropdown-list').toggleClass('hidden'); }); - $(document).click(() => { - $button.find('.dropdown-list').addClass('hidden'); - }); - $widget_control.prepend($button); this.dropdown_list.forEach((item) => { diff --git a/frappe/public/js/frappe/views/workspace/workspace.js b/frappe/public/js/frappe/views/workspace/workspace.js index 31e4f27e1f..d95925eea6 100644 --- a/frappe/public/js/frappe/views/workspace/workspace.js +++ b/frappe/public/js/frappe/views/workspace/workspace.js @@ -37,6 +37,7 @@ frappe.views.Workspace = class Workspace { this.prepare_container(); this.setup_pages(); + this.register_awesomebar_shortcut(); } prepare_container() { @@ -692,11 +693,6 @@ frappe.views.Workspace = class Workspace { $button.filter('.dropdown-list').toggleClass('hidden'); }); - $(document).click(event => { - event.stopPropagation(); - $('.dropdown-list:not(.hidden)').addClass('hidden'); - }); - sidebar_control.append($button); this.dropdown_list.forEach((i) => { @@ -1228,4 +1224,18 @@ frappe.views.Workspace = class Workspace { $('.desk-sidebar').removeClass('hidden'); $('.list-sidebar').find('.workspace-sidebar-skeleton').remove(); } + + register_awesomebar_shortcut() { + 'abcdefghijklmnopqrstuvwxyz'.split('').forEach(letter => { + const default_shortcut = { + action: (e) => { + $("#navbar-search").focus(); + return false; // don't prevent default = type the letter in awesomebar + }, + page: this.page, + }; + frappe.ui.keys.add_shortcut({shortcut: letter, ...default_shortcut}); + frappe.ui.keys.add_shortcut({shortcut: `shift+${letter}`, ...default_shortcut}); + }); + } }; diff --git a/frappe/public/js/frappe/widgets/onboarding_widget.js b/frappe/public/js/frappe/widgets/onboarding_widget.js index 7d379d4531..e560551f79 100644 --- a/frappe/public/js/frappe/widgets/onboarding_widget.js +++ b/frappe/public/js/frappe/widgets/onboarding_widget.js @@ -99,12 +99,7 @@ export default class OnboardingWidget extends Widget { const toggle_content = () => { this.step_body.empty(); this.step_footer.empty(); - - this.step_body.html( - step.description ? - frappe.markdown(step.description) - : `

    ${step.title}

    ` - ); + set_description(); if (step.intro_video_url) { $(``) @@ -117,6 +112,21 @@ export default class OnboardingWidget extends Widget { } }; + const set_description = () => { + let content = step.description ? + frappe.markdown(step.description) : `

    ${step.title}

    `; + + if (step.action === 'Create Entry') { + // add a secondary action to view list + content += `

    + + ${ __('Show {0} List', [step.reference_document])} +

    `; + } + + this.step_body.html(content); + }; + const toggle_video = () => { this.step_body.empty(); this.step_footer.empty(); @@ -222,7 +232,7 @@ export default class OnboardingWidget extends Widget { const on_finish = () => { let msg_dialog = frappe.msgprint({ message: __("Let's take you back to onboarding"), - title: __("Great Job"), + title: __("Onboarding complete"), primary_action: { action: () => { frappe.set_route(current_route).then(() => { @@ -265,7 +275,7 @@ export default class OnboardingWidget extends Widget { if (success) { args.message = __("Let's take you back to onboarding"); - args.title = __("Looks Great"); + args.title = __("Action Complete"); args.primary_action = { action: () => { frappe.set_route(current_route).then(() => { @@ -278,7 +288,7 @@ export default class OnboardingWidget extends Widget { custom_onhide = () => args.primary_action.action(); } else { args.message = __("Looks like you didn't change the value"); - args.title = __("Oops"); + args.title = __("Try Again"); args.secondary_action = { action: () => frappe.set_route(current_route), label: __("Go Back"), @@ -314,7 +324,7 @@ export default class OnboardingWidget extends Widget { const on_finish = () => { frappe.msgprint({ message: __("Awesome, now try making an entry yourself"), - title: __("Great"), + title: __("Document Saved"), primary_action: { action: () => { frappe.set_route(current_route).then(() => { @@ -337,8 +347,8 @@ export default class OnboardingWidget extends Widget { let callback = () => { frappe.msgprint({ - message: __("You're doing great, let's take you back to the onboarding page."), - title: __("Good Work 🎉"), + message: __("Let's take you back to onboarding"), + title: __("Action Complete"), primary_action: { action: () => { frappe.set_route(current_route).then(() => { @@ -358,7 +368,7 @@ export default class OnboardingWidget extends Widget { frappe.route_hooks.after_save = () => { frappe.msgprint({ message: __("Submit this document to complete this step."), - title: __("Great") + title: __("Document Saved") }); }; frappe.route_hooks.after_submit = callback; @@ -377,7 +387,7 @@ export default class OnboardingWidget extends Widget { if (frappe.get_route_str() != current_route) { let success_dialog = frappe.msgprint({ message: __("Let's take you back to onboarding"), - title: __("Looks Great"), + title: __("Document Saved"), primary_action: { action: () => { success_dialog.hide(); @@ -397,7 +407,7 @@ export default class OnboardingWidget extends Widget { } else { frappe.msgprint({ message: __("Let us continue with the onboarding"), - title: __("Looks Great") + title: __("Document Saved") }); this.mark_complete(step); } diff --git a/frappe/public/js/logtypes.bundle.js b/frappe/public/js/logtypes.bundle.js new file mode 100644 index 0000000000..775ac730ad --- /dev/null +++ b/frappe/public/js/logtypes.bundle.js @@ -0,0 +1 @@ +import "./frappe/logtypes" diff --git a/frappe/public/js/print_format_builder/PrintFormatControls.vue b/frappe/public/js/print_format_builder/PrintFormatControls.vue index 2eefc22409..7a4e9c81e7 100644 --- a/frappe/public/js/print_format_builder/PrintFormatControls.vue +++ b/frappe/public/js/print_format_builder/PrintFormatControls.vue @@ -181,8 +181,8 @@ export default { return [ { label: __("Top"), fieldname: "margin_top" }, { label: __("Bottom"), fieldname: "margin_bottom" }, - { label: __("Left"), fieldname: "margin_left" }, - { label: __("Right"), fieldname: "margin_right" } + { label: __("Left", null, 'alignment'), fieldname: "margin_left" }, + { label: __("Right", null, 'alignment'), fieldname: "margin_right" } ]; }, fields() { diff --git a/frappe/public/scss/common/controls.scss b/frappe/public/scss/common/controls.scss index 1135fbb23d..9685c66ed9 100644 --- a/frappe/public/scss/common/controls.scss +++ b/frappe/public/scss/common/controls.scss @@ -127,6 +127,24 @@ select.form-control { margin-bottom: 0; } } + .action-btn { + position: absolute; + top: 4px; + right: 4px; + padding: 3px; + z-index: 3; + } + + button.action-btn { + padding: 3px 5px; + background-color: var(--fg-color); + } + + .link-btn { + @extend .action-btn; + background-color: none; + display: none; + } } .frappe-control:not([data-fieldtype='MultiSelectPills']):not([data-fieldtype='Table MultiSelect']) { @@ -289,15 +307,6 @@ textarea.form-control { position: relative; } -.link-btn { - position: absolute; - top: 4px; - right: 4px; - padding: 3px; - display: none; - z-index: 3; -} - .phone-btn { position: absolute; top: 4px; diff --git a/frappe/public/scss/common/css_variables.scss b/frappe/public/scss/common/css_variables.scss index a89ebb3b50..ab52c10e45 100644 --- a/frappe/public/scss/common/css_variables.scss +++ b/frappe/public/scss/common/css_variables.scss @@ -1,3 +1,5 @@ +$input-height: 28px !default; + :root, [data-theme="light"] { --brand-color: #0089FF; @@ -16,7 +18,7 @@ --blue-900: #1A4469; --blue-800: #154875; --blue-700: #1366AE; - --blue-600: #1579D0; + --blue-600: #1673C5; --blue-500: #2490EF; --blue-400: #50A6F2; --blue-300: #7CBCF5; @@ -143,7 +145,7 @@ --btn-shadow: var(--shadow-xs); // Type Colors - --text-muted: var(--gray-600); + --text-muted: var(--gray-700); --text-light: var(--gray-800); --text-color: var(--gray-900); --heading-color: var(--gray-900); @@ -177,19 +179,18 @@ --text-2xl: 20px; --text-3xl: 22px; - --text-on-blue: var(--blue-600); - --text-on-light-blue: var(--blue-500); - --text-on-dark-blue: var(--blue-700); - --text-on-blue: var(--blue-600); - --text-on-green: var(--dark-green-500); - --text-on-yellow: var(--yellow-500); - --text-on-orange: var(--orange-500); - --text-on-red: var(--red-500); + --text-on-blue: var(--blue-700); + --text-on-light-blue: var(--blue-600); + --text-on-dark-blue: var(--blue-800); + --text-on-green: var(--dark-green-700); + --text-on-yellow: var(--yellow-700); + --text-on-orange: var(--orange-600); + --text-on-red: var(--red-600); --text-on-gray: var(--gray-600); --text-on-light-gray: var(--gray-800); - --text-on-purple: var(--purple-500); - --text-on-pink: var(--pink-500); - --text-on-cyan: var(--cyan-600); + --text-on-purple: var(--purple-700); + --text-on-pink: var(--pink-600); + --text-on-cyan: var(--cyan-800); // alert colors --alert-text-danger: var(--red-600); @@ -249,7 +250,11 @@ --border-radius-full: 999px; --primary-color: #2490EF; - --btn-height: 28px; + --btn-height: 30px; + + // input + --input-height: #{$input-height}; + --input-disabled-bg: var(--gray-200); // Checkbox --checkbox-right-margin: var(--margin-xs); diff --git a/frappe/public/scss/common/form.scss b/frappe/public/scss/common/form.scss index fcea603994..a01ff54c28 100644 --- a/frappe/public/scss/common/form.scss +++ b/frappe/public/scss/common/form.scss @@ -13,6 +13,7 @@ font-weight: normal; font-size: var(--text-sm); } + min-height: var(--input-height); border-radius: $border-radius; font-weight: 400; padding: 6px 12px; diff --git a/frappe/public/scss/common/grid.scss b/frappe/public/scss/common/grid.scss index d1f89abbcd..07ab6d75a9 100644 --- a/frappe/public/scss/common/grid.scss +++ b/frappe/public/scss/common/grid.scss @@ -201,7 +201,7 @@ } .link-btn { - top: 8px; + top: 2px; } .form-control:focus { diff --git a/frappe/public/scss/desk/css_variables.scss b/frappe/public/scss/desk/css_variables.scss index a06ba3e9b0..aceaa3e1e6 100644 --- a/frappe/public/scss/desk/css_variables.scss +++ b/frappe/public/scss/desk/css_variables.scss @@ -1,7 +1,5 @@ @import '../common/css_variables.scss'; -$input-height: 28px !default; - :root, [data-theme="light"] { // breakpoints @@ -31,10 +29,6 @@ $input-height: 28px !default; --page-head-height: 75px; --page-bottom-margin: 60px; - // input - --input-height: #{$input-height}; - --input-disabled-bg: var(--gray-200); - // checkbox --checkbox-right-margin: var(--margin-xs); --checkbox-size: 14px; diff --git a/frappe/public/scss/desk/desktop.scss b/frappe/public/scss/desk/desktop.scss index 8be8abed35..c0fef60162 100644 --- a/frappe/public/scss/desk/desktop.scss +++ b/frappe/public/scss/desk/desktop.scss @@ -421,8 +421,8 @@ body { display: none; } - i { - color: var(--green-600); + .icon { + stroke: var(--white); } span { diff --git a/frappe/public/scss/desk/filters.scss b/frappe/public/scss/desk/filters.scss index 0552311ee2..ffaea7a9bd 100644 --- a/frappe/public/scss/desk/filters.scss +++ b/frappe/public/scss/desk/filters.scss @@ -1,6 +1,6 @@ .filter-icon.active { use { - stroke: var(--blue-500); + stroke: var(--text-on-blue); } } diff --git a/frappe/public/scss/desk/list.scss b/frappe/public/scss/desk/list.scss index c71dbdca89..31d1661abb 100644 --- a/frappe/public/scss/desk/list.scss +++ b/frappe/public/scss/desk/list.scss @@ -204,8 +204,8 @@ $level-margin-right: 8px; border: 1px solid var(--dark-border-color); &.btn-info { - background-color: var(--gray-400); - border-color: var(--gray-400); + background-color: var(--gray-600); + border-color: var(--gray-600); color: var(--white); font-weight: var(--text-bold); } @@ -401,7 +401,7 @@ input.list-check-all { } .filter-button.btn-primary-light { - color: var(--blue-500); + color: var(--text-on-blue); } .sort-selector { diff --git a/frappe/public/scss/desk/report.scss b/frappe/public/scss/desk/report.scss index f8666602ff..8ed0fb740c 100644 --- a/frappe/public/scss/desk/report.scss +++ b/frappe/public/scss/desk/report.scss @@ -104,12 +104,12 @@ } .group-by-button.btn-primary-light { - color: var(--blue-500); + color: var(--text-on-blue); } .group-by-icon.active { use { - stroke: var(--blue-500); + stroke: var(--text-on-blue); } } diff --git a/frappe/public/scss/desk/sidebar.scss b/frappe/public/scss/desk/sidebar.scss index e30e0c3b94..25dcceec5b 100644 --- a/frappe/public/scss/desk/sidebar.scss +++ b/frappe/public/scss/desk/sidebar.scss @@ -492,4 +492,4 @@ body[data-route^="Module"] .main-menu { .shared-user { margin-bottom: 10px; -} +} \ No newline at end of file diff --git a/frappe/public/scss/login.bundle.scss b/frappe/public/scss/login.bundle.scss index 8d0a32846f..488dd4106e 100644 --- a/frappe/public/scss/login.bundle.scss +++ b/frappe/public/scss/login.bundle.scss @@ -16,7 +16,7 @@ body { .for-forgot, .for-signup, .for-email-login { - padding: max(15vh, 70px) 0; + padding: max(10vh, 60px) 0; @include media-breakpoint-up(sm) { .page-card { @@ -177,6 +177,7 @@ body { } h4 { + margin-top: 1rem; font-size: var(--text-xl); color: var(--text-color); } diff --git a/frappe/public/scss/print.bundle.scss b/frappe/public/scss/print.bundle.scss index 61f56beaf8..3e8baddcb6 100644 --- a/frappe/public/scss/print.bundle.scss +++ b/frappe/public/scss/print.bundle.scss @@ -1,5 +1,8 @@ @import "frappe/public/css/bootstrap.css"; @import "./common/quill"; + +@import "./desk/variables"; +@import "~bootstrap/scss/utilities/spacing"; @import "./desk/css_variables"; @import "./element/checkbox"; @@ -12,4 +15,23 @@ svg[data-barcode-value] > g { fill: black !important; } + .print-hide { + display: none !important; + } } + +.action-banner { + display: flex; + justify-content: flex-end; + padding-right: 20px; + font-size: var(--text-md); +} + +.invalid-state { + display: grid; + place-content: center; + height: 100vh; + img { + margin: auto; + } +} \ No newline at end of file diff --git a/frappe/public/scss/website/index.scss b/frappe/public/scss/website/index.scss index f25e4d6cc6..109bc8cbb4 100644 --- a/frappe/public/scss/website/index.scss +++ b/frappe/public/scss/website/index.scss @@ -125,6 +125,10 @@ align-items: center; } +.page_content { + min-height: 50vh; +} + .breadcrumb-container { margin-top: 1rem; padding-top: 0.25rem; diff --git a/frappe/query_builder/terms.py b/frappe/query_builder/terms.py index 8d64d2ddcd..64a4707983 100644 --- a/frappe/query_builder/terms.py +++ b/frappe/query_builder/terms.py @@ -100,7 +100,7 @@ class ParameterizedFunction(Function): return function_sql -class subqry(Criterion): +class SubQuery(Criterion): def __init__( self, subq: QueryBuilder, @@ -112,3 +112,6 @@ class subqry(Criterion): def get_sql(self, **kwg: Any) -> str: kwg["subquery"] = True return self.subq.get_sql(**kwg) + + +subqry = SubQuery diff --git a/frappe/query_builder/utils.py b/frappe/query_builder/utils.py index 69aee9b350..10bab38a63 100644 --- a/frappe/query_builder/utils.py +++ b/frappe/query_builder/utils.py @@ -64,7 +64,6 @@ def patch_query_execute(): This excludes the use of `frappe.db.sql` method while executing the query object """ - from frappe.utils.safe_exec import check_safe_sql_query def execute_query(query, *args, **kwargs): query, params = prepare_query(query) @@ -73,6 +72,8 @@ def patch_query_execute(): def prepare_query(query): import inspect + from frappe.utils.safe_exec import check_safe_sql_query + param_collector = NamedParameterWrapper() query = query.get_sql(param_wrapper=param_collector) if frappe.flags.in_safe_exec and not check_safe_sql_query(query, throw=False): @@ -103,6 +104,7 @@ def patch_query_execute(): builder_class.run = execute_query builder_class.walk = prepare_query + frappe._qb_patched[frappe.conf.db_type] = True def patch_query_aggregation(): @@ -113,3 +115,4 @@ def patch_query_aggregation(): frappe.qb.min = _min frappe.qb.avg = _avg frappe.qb.sum = _sum + frappe._qb_patched[frappe.conf.db_type] = True diff --git a/frappe/recorder.py b/frappe/recorder.py index 95b78dd085..87e001fe31 100644 --- a/frappe/recorder.py +++ b/frappe/recorder.py @@ -16,6 +16,7 @@ from frappe import _ RECORDER_INTERCEPT_FLAG = "recorder-intercept" RECORDER_REQUEST_SPARSE_HASH = "recorder-requests-sparse" RECORDER_REQUEST_HASH = "recorder-requests" +TRACEBACK_PATH_PATTERN = re.compile(".*/apps/") def sql(*args, **kwargs): @@ -58,7 +59,7 @@ def get_current_stack_frames(): for frame, filename, lineno, function, context, index in list(reversed(frames))[:-2]: if "/apps/" in filename: yield { - "filename": re.sub(".*/apps/", "", filename), + "filename": TRACEBACK_PATH_PATTERN.sub("", filename), "lineno": lineno, "function": function, } diff --git a/frappe/sessions.py b/frappe/sessions.py index 6e0ce73732..67b58e1d89 100644 --- a/frappe/sessions.py +++ b/frappe/sessions.py @@ -187,7 +187,7 @@ def get(): bootinfo["translated_search_doctypes"] = frappe.get_hooks("translated_search_doctypes") bootinfo["disable_async"] = frappe.conf.disable_async - bootinfo["setup_complete"] = cint(frappe.db.get_single_value("System Settings", "setup_complete")) + bootinfo["setup_complete"] = cint(frappe.get_system_settings("setup_complete")) bootinfo["desk_theme"] = frappe.db.get_value("User", frappe.session.user, "desk_theme") or "Light" diff --git a/frappe/share.py b/frappe/share.py index 01d1412b8d..3edcb1be38 100644 --- a/frappe/share.py +++ b/frappe/share.py @@ -93,7 +93,7 @@ def set_permission(doctype, name, user, permission_to, value=1, everyone=0): if not (share.read or share.write or share.submit or share.share): share.delete() - share = {} + share = None return share diff --git a/frappe/templates/includes/comments/comments.py b/frappe/templates/includes/comments/comments.py index dd94cfa989..b98106c0dc 100644 --- a/frappe/templates/includes/comments/comments.py +++ b/frappe/templates/includes/comments/comments.py @@ -10,6 +10,11 @@ from frappe.utils.html_utils import clean_html from frappe.website.doctype.blog_settings.blog_settings import get_comment_limit from frappe.website.utils import clear_cache +URLS_COMMENT_PATTERN = re.compile( + r"http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+", re.IGNORECASE +) +EMAIL_PATTERN = re.compile(r"(^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$)", re.IGNORECASE) + @frappe.whitelist(allow_guest=True) @rate_limit(key="reference_name", limit=get_comment_limit, seconds=60 * 60) @@ -23,12 +28,7 @@ def add_comment(comment, comment_email, comment_by, reference_doctype, reference frappe.msgprint(_("The comment cannot be empty")) return False - url_regex = re.compile( - r"http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+", re.IGNORECASE - ) - email_regex = re.compile(r"(^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$)", re.IGNORECASE) - - if url_regex.search(comment) or email_regex.search(comment): + if URLS_COMMENT_PATTERN.search(comment) or EMAIL_PATTERN.search(comment): frappe.msgprint(_("Comments cannot have links or email addresses")) return False diff --git a/frappe/templates/includes/integrations/razorpay_checkout.js b/frappe/templates/includes/integrations/razorpay_checkout.js index 2986fcb0fc..3df6ed68ea 100644 --- a/frappe/templates/includes/integrations/razorpay_checkout.js +++ b/frappe/templates/includes/integrations/razorpay_checkout.js @@ -3,6 +3,7 @@ $(document).ready(function(){ var options = { "key": "{{ api_key }}", "amount": cint({{ amount }} * 100), // 2000 paise = INR 20 + "currency": "{{ currency }}", "name": "{{ title }}", "description": "{{ description }}", "subscription_id": "{{ subscription_id }}", diff --git a/frappe/templates/pages/integrations/razorpay_checkout.py b/frappe/templates/pages/integrations/razorpay_checkout.py index aed832119b..b4f9e74a03 100644 --- a/frappe/templates/pages/integrations/razorpay_checkout.py +++ b/frappe/templates/pages/integrations/razorpay_checkout.py @@ -17,6 +17,7 @@ expected_keys = ( "payer_name", "payer_email", "order_id", + "currency", ) diff --git a/frappe/templates/styles/card_style.css b/frappe/templates/styles/card_style.css index 9e38ad70bf..a9639d8133 100644 --- a/frappe/templates/styles/card_style.css +++ b/frappe/templates/styles/card_style.css @@ -2,33 +2,32 @@ background-color: var(--bg-color); } - - .page-card { - max-width: 360px; - padding: 15px; - margin: 70px auto; - border-radius: 4px; - background-color: var(--fg-color); - /* box-shadow: var(--shadow-base); */ + max-width: 360px; + padding: 15px; + margin: 70px auto; + border-radius: 4px; + background-color: var(--fg-color); + box-shadow: var(--shadow-base); } .for-reset-password { - margin: 80px 0; + margin: 80px 0; } .for-reset-password .page-card { - border: 0; - max-width: 450px; - margin: auto; - border-radius: 10px; + border: 0; + max-width: 450px; + margin: auto; + border-radius: var(--border-radius-md); + padding: 40px 60px; } -@media (min-width: 567px) { +@media (max-width: 425px) { .for-reset-password .page-card { - box-shadow: var(--shadow-base); - padding: 40px 60px; - + box-shadow: none; + background: none; + padding: 0px; } } diff --git a/frappe/test_runner.py b/frappe/test_runner.py index 96feac532f..82179d8fac 100644 --- a/frappe/test_runner.py +++ b/frappe/test_runner.py @@ -41,6 +41,7 @@ def main( app=None, module=None, doctype=None, + module_def=None, verbose=False, tests=(), force=False, @@ -97,6 +98,13 @@ def main( ret = run_tests_for_doctype( doctype, verbose, tests, force, profile, failfast=failfast, junit_xml_output=junit_xml_output ) + elif module_def: + doctypes = frappe.db.get_list( + "DocType", filters={"module": module_def, "istable": 0}, pluck="name" + ) + ret = run_tests_for_doctype( + doctypes, verbose, tests, force, profile, failfast=failfast, junit_xml_output=junit_xml_output + ) elif module: ret = run_tests_for_module( module, diff --git a/frappe/tests/__init__.py b/frappe/tests/__init__.py index 5a44cae5f1..eda83bd14b 100644 --- a/frappe/tests/__init__.py +++ b/frappe/tests/__init__.py @@ -1,11 +1,13 @@ import frappe -def update_system_settings(args): +def update_system_settings(args, commit=False): doc = frappe.get_doc("System Settings") doc.update(args) doc.flags.ignore_mandatory = 1 doc.save() + if commit: + frappe.db.commit() def get_system_setting(key): diff --git a/frappe/tests/test_caching.py b/frappe/tests/test_caching.py new file mode 100644 index 0000000000..6a1390f10c --- /dev/null +++ b/frappe/tests/test_caching.py @@ -0,0 +1,95 @@ +import time +import unittest +from typing import Dict, List, Tuple, Union +from unittest.mock import MagicMock + +import frappe +from frappe.tests.test_api import FrappeAPITestCase +from frappe.utils.caching import request_cache, site_cache + +CACHE_TTL = 4 +external_service = MagicMock(return_value=30) +register_with_external_service = MagicMock(return_value=True) + + +@request_cache +def request_specific_api(a: Union[List, Tuple, Dict, int], b: int) -> int: + # API that takes very long to return a result + todays_value = external_service() + if not isinstance(a, (int, float)): + a = 1 + return a**b * todays_value + + +@frappe.whitelist(allow_guest=True) +@site_cache +def ping() -> str: + register_with_external_service(frappe.local.site) + return frappe.local.site + + +@frappe.whitelist(allow_guest=True) +@site_cache(ttl=CACHE_TTL) +def ping_with_ttl() -> str: + register_with_external_service(frappe.local.site) + return frappe.local.site + + +class TestCachingUtils(unittest.TestCase): + def test_request_cache(self): + retval = [] + acceptable_args = [ + [1, 2, 3, 4], + range(10), + {"abc": "test-key"}, + frappe.get_last_doc("DocType"), + frappe._dict(), + ] + same_output_received = lambda: all([x for x in set(retval) if x == retval[0]]) + + # ensure that external service was called only once + # thereby return value of request_specific_api is cached + for _ in range(5): + retval.append(request_specific_api(120, 23)) + external_service.assert_called_once() + self.assertTrue(same_output_received()) + + # ensure that cache differentiates between int & float + # types. Giving different return values for both + retval.append(request_specific_api(120.0, 23)) + self.assertTrue(external_service.call_count, 2) + + # ensure that function is executed when call isn't + # already cached + retval.clear() + for _ in range(10): + request_specific_api(120, 13) + self.assertTrue(external_service.call_count, 3) + self.assertTrue(same_output_received()) + + # ensure key generation capacity for different types + retval.clear() + for arg in acceptable_args: + external_service.call_count = 0 + for _ in range(2): + request_specific_api(arg, 13) + self.assertTrue(external_service.call_count, 1) + self.assertTrue(same_output_received()) + + +class TestSiteCache(FrappeAPITestCase): + def test_site_cache(self): + module = __name__ + api_with_ttl = f"{module}.ping_with_ttl" + api_without_ttl = f"{module}.ping" + + start = time.monotonic() + for _ in range(5): + self.get(f"/api/method/{api_with_ttl}") + self.get(f"/api/method/{api_without_ttl}") + end = time.monotonic() + + self.assertEqual(register_with_external_service.call_count, 2) + time.sleep(CACHE_TTL - (end - start)) + self.get(f"/api/method/{api_with_ttl}") + self.assertEqual(register_with_external_service.call_count, 3) diff --git a/frappe/tests/test_commands.py b/frappe/tests/test_commands.py index be968511a8..aeb7f364bc 100644 --- a/frappe/tests/test_commands.py +++ b/frappe/tests/test_commands.py @@ -27,6 +27,8 @@ import frappe.commands.site import frappe.commands.utils import frappe.recorder from frappe.installer import add_to_installed_apps, remove_app +from frappe.query_builder.utils import db_type_is +from frappe.tests.test_query_builder import run_only_if from frappe.utils import add_to_date, get_bench_path, get_bench_relative_path, now from frappe.utils.backups import fetch_latest_backups @@ -518,6 +520,23 @@ class TestBackups(BaseTestCommands): self.assertIsNotNone(after_backup["public"]) self.assertIsNotNone(after_backup["private"]) + @run_only_if(db_type_is.MARIADB) + def test_clear_log_table(self): + d = frappe.get_doc(doctype="Error Log", title="Something").insert() + d.db_set("modified", "2010-01-01", update_modified=False) + frappe.db.commit() + + tables_before = frappe.db.get_tables(cached=False) + + self.execute("bench --site {site} clear-log-table --days=30 --doctype='Error Log'") + self.assertEqual(self.returncode, 0) + frappe.db.commit() + + self.assertFalse(frappe.db.exists("Error Log", d.name)) + tables_after = frappe.db.get_tables(cached=False) + + self.assertEqual(set(tables_before), set(tables_after)) + def test_backup_with_custom_path(self): """Backup to a custom path (--backup-path)""" backup_path = os.path.join(self.home, "backups") diff --git a/frappe/tests/test_db.py b/frappe/tests/test_db.py index 86e54cb866..73b5446404 100644 --- a/frappe/tests/test_db.py +++ b/frappe/tests/test_db.py @@ -182,10 +182,12 @@ class TestDB(unittest.TestCase): self.assertIn("tabToDo", frappe.flags.touched_tables) frappe.flags.touched_tables = set() - create_custom_field("ToDo", {"label": "ToDo Custom Field"}) - + cf = create_custom_field("ToDo", {"label": "ToDo Custom Field"}) self.assertIn("tabToDo", frappe.flags.touched_tables) self.assertIn("tabCustom Field", frappe.flags.touched_tables) + if cf: + cf.delete() + frappe.db.commit() frappe.flags.in_migrate = False frappe.flags.touched_tables.clear() @@ -867,3 +869,18 @@ class TestDDLCommandsPost(unittest.TestCase): self.assertIn( "is null", frappe.db.get_values(user, filters={user.name: ("is", "not set")}, run=False).lower() ) + + +@run_only_if(db_type_is.POSTGRES) +class TestTransactionManagement(unittest.TestCase): + def test_create_proper_transactions(self): + def _get_transaction_id(): + return frappe.db.sql("select txid_current()", pluck=True) + + self.assertEqual(_get_transaction_id(), _get_transaction_id()) + + frappe.db.rollback() + self.assertEqual(_get_transaction_id(), _get_transaction_id()) + + frappe.db.commit() + self.assertEqual(_get_transaction_id(), _get_transaction_id()) diff --git a/frappe/tests/test_db_query.py b/frappe/tests/test_db_query.py index dd67d68cd2..c1b2e05266 100644 --- a/frappe/tests/test_db_query.py +++ b/frappe/tests/test_db_query.py @@ -35,6 +35,35 @@ class TestReportview(unittest.TestCase): clear_custom_fields("DocType") + def test_child_table_field_syntax(self): + note = frappe.get_doc( + doctype="Note", + title=f"Test {frappe.utils.random_string(8)}", + content="test", + seen_by=[{"user": "Administrator"}], + ).insert() + result = frappe.db.get_all( + "Note", + filters={"name": note.name}, + fields=["name", "seen_by.user as seen_by"], + limit=1, + ) + self.assertEqual(result[0].seen_by, "Administrator") + note.delete() + + def test_link_field_syntax(self): + todo = frappe.get_doc( + doctype="ToDo", description="Test ToDo", allocated_to="Administrator" + ).insert() + result = frappe.db.get_all( + "ToDo", + filters={"name": todo.name}, + fields=["name", "allocated_to.email as allocated_user_email"], + limit=1, + ) + self.assertEqual(result[0].allocated_user_email, "admin@example.com") + todo.delete() + def test_build_match_conditions(self): clear_user_permissions_for_doctype("Blog Post", "test2@example.com") diff --git a/frappe/tests/test_document.py b/frappe/tests/test_document.py index 5bda6a1d9d..00bca40268 100644 --- a/frappe/tests/test_document.py +++ b/frappe/tests/test_document.py @@ -6,9 +6,13 @@ from datetime import timedelta from unittest.mock import patch import frappe +from frappe.app import make_form_dict from frappe.desk.doctype.note.note import Note from frappe.model.naming import make_autoname, parse_naming_series, revert_series_if_last -from frappe.utils import cint, now_datetime +from frappe.utils import cint, now_datetime, set_request +from frappe.website.serve import get_response + +from . import update_system_settings class CustomTestNote(Note): @@ -357,3 +361,49 @@ class TestDocument(unittest.TestCase): # setting None should init a table field to empty list doc.set("user_emails", None) self.assertEqual(doc.user_emails, []) + + +class TestDocumentWebView(unittest.TestCase): + def get(self, path, user="Guest"): + frappe.set_user(user) + set_request(method="GET", path=path) + make_form_dict(frappe.local.request) + response = get_response() + frappe.set_user("Administrator") + return response + + def test_web_view_link_authentication(self): + todo = frappe.get_doc({"doctype": "ToDo", "description": "Test"}).insert() + document_key = todo.get_document_share_key() + + # with old-style signature key + update_system_settings({"allow_older_web_view_links": True}, True) + old_document_key = todo.get_signature() + url = f"/ToDo/{todo.name}?key={old_document_key}" + self.assertEqual(self.get(url).status, "200 OK") + + update_system_settings({"allow_older_web_view_links": False}, True) + self.assertEqual(self.get(url).status, "401 UNAUTHORIZED") + + # with valid key + url = f"/ToDo/{todo.name}?key={document_key}" + self.assertEqual(self.get(url).status, "200 OK") + + # with invalid key + invalid_key_url = f"/ToDo/{todo.name}?key=INVALID_KEY" + self.assertEqual(self.get(invalid_key_url).status, "401 UNAUTHORIZED") + + # expire the key + document_key_doc = frappe.get_doc("Document Share Key", {"key": document_key}) + document_key_doc.expires_on = "2020-01-01" + document_key_doc.save(ignore_permissions=True) + + # with expired key + self.assertEqual(self.get(url).status, "410 GONE") + + # without key + url_without_key = f"/ToDo/{todo.name}" + self.assertEqual(self.get(url_without_key).status, "403 FORBIDDEN") + + # Logged-in user can access the page without key + self.assertEqual(self.get(url_without_key, "Administrator").status, "200 OK") diff --git a/frappe/tests/test_form_load.py b/frappe/tests/test_form_load.py index e92b8c3ff2..22db56eeef 100644 --- a/frappe/tests/test_form_load.py +++ b/frappe/tests/test_form_load.py @@ -181,7 +181,7 @@ class TestFormLoad(unittest.TestCase): self.assertEqual(len(docinfo.comments), 1) self.assertIn("test", docinfo.comments[0].content) - self.assertGreaterEqual(len(docinfo.versions), 2) + self.assertGreaterEqual(len(docinfo.versions), 1) self.assertEqual(set(docinfo.tags.split(",")), {"more_tag", "test_tag"}) diff --git a/frappe/tests/test_naming.py b/frappe/tests/test_naming.py index e217f24154..d966fd5ce8 100644 --- a/frappe/tests/test_naming.py +++ b/frappe/tests/test_naming.py @@ -1,20 +1,21 @@ # Copyright (c) 2018, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import unittest - import frappe from frappe.core.doctype.doctype.test_doctype import new_doctype from frappe.model.naming import ( + InvalidNamingSeriesError, + NamingSeries, append_number_if_name_exists, determine_consecutive_week_number, getseries, revert_series_if_last, ) +from frappe.tests.utils import FrappeTestCase from frappe.utils import now_datetime -class TestNaming(unittest.TestCase): +class TestNaming(FrappeTestCase): def setUp(self): frappe.db.delete("Note") @@ -52,16 +53,13 @@ class TestNaming(unittest.TestCase): self.assertEqual(country.name, country.country_name) def test_child_table_naming(self): - child_dt_with_naming = new_doctype( - "childtable_with_autonaming", istable=1, autoname="field:some_fieldname" - ).insert() + child_dt_with_naming = new_doctype(istable=1, autoname="field:some_fieldname").insert() dt_with_child_autoname = new_doctype( - "dt_with_childtable_naming", fields=[ { "label": "table with naming", "fieldname": "table_with_naming", - "options": "childtable_with_autonaming", + "options": child_dt_with_naming.name, "fieldtype": "Table", } ], @@ -69,7 +67,7 @@ class TestNaming(unittest.TestCase): name = frappe.generate_hash(length=10) - doc = frappe.new_doc("dt_with_childtable_naming") + doc = frappe.new_doc(dt_with_child_autoname.name) doc.append("table_with_naming", {"some_fieldname": name}) doc.save() self.assertEqual(doc.table_with_naming[0].name, name) @@ -89,31 +87,18 @@ class TestNaming(unittest.TestCase): """ Test if braced params are replaced in format autoname """ - doctype = "ToDo" - - todo_doctype = frappe.get_doc("DocType", doctype) - todo_doctype.autoname = "format:TODO-{MM}-{status}-{##}" - todo_doctype.save() + doctype = new_doctype(autoname="format:TODO-{MM}-{some_fieldname}-{##}").insert() description = "Format" - todo = frappe.new_doc(doctype) - todo.description = description - todo.insert() + doc = frappe.new_doc(doctype.name) + doc.some_fieldname = description + doc.insert() series = getseries("", 2) + series = int(series) - 1 - series = str(int(series) - 1) - - if len(series) < 2: - series = "0" + series - - self.assertEqual( - todo.name, - "TODO-{month}-{status}-{series}".format( - month=now_datetime().strftime("%m"), status=todo.status, series=series - ), - ) + self.assertEqual(doc.name, f"TODO-{now_datetime().strftime('%m')}-{description}-{series:02}") def test_format_autoname_for_consecutive_week_number(self): """ @@ -303,6 +288,46 @@ class TestNaming(unittest.TestCase): dt.delete(ignore_permissions=True) + def test_naming_series_prefix(self): + today = now_datetime() + year = today.strftime("%y") + month = today.strftime("%m") + + prefix_test_cases = { + "SINV-.YY.-.####": f"SINV-{year}-", + "SINV-.YY.-.MM.-.####": f"SINV-{year}-{month}-", + "SINV": "SINV", + "SINV-.": "SINV-", + } + + for series, prefix in prefix_test_cases.items(): + self.assertEqual(prefix, NamingSeries(series).get_prefix()) + + def test_naming_series_validation(self): + dns = frappe.get_doc("Document Naming Settings") + exisiting_series = dns.get_transactions_and_prefixes()["prefixes"] + valid = ["SINV-", "SI-.{field}.", "SI-#.###", ""] + exisiting_series + invalid = ["$INV-", r"WINDOWS\NAMING"] + + for series in valid: + if series.strip(): + try: + NamingSeries(series).validate() + except Exception as e: + self.fail(f"{series} should be valid\n{e}") + + for series in invalid: + self.assertRaises(InvalidNamingSeriesError, NamingSeries(series).validate) + + def test_naming_using_fields(self): + + webhook = frappe.new_doc("Webhook") + webhook.webhook_docevent = "on_update" + name = NamingSeries("KOOH-.{webhook_docevent}.").generate_next_name(webhook) + self.assertTrue( + name.startswith("KOOH-on_update"), f"incorrect name generated {name}, missing field value" + ) + def make_invalid_todo(): frappe.get_doc({"doctype": "ToDo", "description": "Test"}).insert(set_name="ToDo") diff --git a/frappe/tests/test_pdf.py b/frappe/tests/test_pdf.py index 497546ebd5..8f2a2c1cfa 100644 --- a/frappe/tests/test_pdf.py +++ b/frappe/tests/test_pdf.py @@ -3,7 +3,7 @@ import io import unittest -from PyPDF2 import PdfFileReader +from PyPDF2 import PdfReader import frappe import frappe.utils.pdf as pdfgen @@ -42,7 +42,7 @@ class TestPdf(unittest.TestCase): def test_pdf_encryption(self): password = "qwe" pdf = pdfgen.get_pdf(self.html, options={"password": password}) - reader = PdfFileReader(io.BytesIO(pdf)) + reader = PdfReader(io.BytesIO(pdf)) self.assertTrue(reader.isEncrypted) self.assertTrue(reader.decrypt(password)) diff --git a/frappe/tests/test_permissions.py b/frappe/tests/test_permissions.py index 4164b0be36..26d5c714ef 100644 --- a/frappe/tests/test_permissions.py +++ b/frappe/tests/test_permissions.py @@ -672,3 +672,31 @@ class TestPermissions(FrappeTestCase): doctype="Has Role", parent_doctype="Has Role", ) + + def test_select_user(self): + """If test3@example.com is restricted by a User Permission to see only + users linked to a certain doctype (in this case: Gender "Female"), he + should not be able to query other users (Gender "Male"). + """ + # ensure required genders exist + for gender in ("Male", "Female"): + if frappe.db.exists("Gender", gender): + continue + + frappe.get_doc({"doctype": "Gender", "gender": gender}).insert() + + # asssign gender to test users + frappe.db.set_value("User", "test1@example.com", "gender", "Male") + frappe.db.set_value("User", "test2@example.com", "gender", "Female") + frappe.db.set_value("User", "test3@example.com", "gender", "Female") + + # restrict test3@example.com to see only female users + add_user_permission("Gender", "Female", "test3@example.com") + + # become user test3@example.com and see what users he can query + frappe.set_user("test3@example.com") + users = frappe.get_list("User", pluck="name") + + self.assertNotIn("test1@example.com", users) + self.assertIn("test2@example.com", users) + self.assertIn("test3@example.com", users) diff --git a/frappe/tests/test_rename_doc.py b/frappe/tests/test_rename_doc.py index 8bf76b3e13..928953fe1c 100644 --- a/frappe/tests/test_rename_doc.py +++ b/frappe/tests/test_rename_doc.py @@ -100,8 +100,6 @@ class TestRenameDoc(unittest.TestCase): frappe.delete_doc("DocType", dt) frappe.db.sql_ddl(f"DROP TABLE IF EXISTS `tab{dt}`") - frappe.delete_doc_if_exists("Renamed Doc", "ToDo") - # reset original value of developer_mode conf frappe.conf.developer_mode = self._original_developer_flag diff --git a/frappe/tests/test_scheduler.py b/frappe/tests/test_scheduler.py index 5161e1e80f..c6b381e487 100644 --- a/frappe/tests/test_scheduler.py +++ b/frappe/tests/test_scheduler.py @@ -1,18 +1,16 @@ +import os import time from unittest import TestCase +from unittest.mock import patch import frappe -from frappe.core.doctype.scheduled_job_type.scheduled_job_type import sync_jobs +from frappe.core.doctype.scheduled_job_type.scheduled_job_type import ScheduledJobType, sync_jobs from frappe.utils import add_days, get_datetime from frappe.utils.background_jobs import enqueue from frappe.utils.doctor import purge_pending_jobs from frappe.utils.scheduler import enqueue_events, is_dormant, schedule_jobs_based_on_activity -def test_timeout(): - time.sleep(100) - - def test_timeout_10(): time.sleep(10) @@ -23,6 +21,11 @@ def test_method(): class TestScheduler(TestCase): def setUp(self): + frappe.db.rollback() + + if not os.environ.get("CI"): + return + purge_pending_jobs() if not frappe.get_all("Scheduled Job Type", limit=1): sync_jobs() @@ -44,15 +47,9 @@ class TestScheduler(TestCase): def test_queue_peeking(self): job = get_test_job() - self.assertTrue(job.enqueue()) - job.db_set("last_execution", "2010-01-01 00:00:00") - frappe.db.commit() - - time.sleep(0.5) - - # 1st job is in the queue (or running), don't enqueue it again - self.assertFalse(job.enqueue()) - frappe.db.delete("Scheduled Job Log", {"scheduled_job_type": job.name}) + with patch.object(job, "is_job_in_queue", return_value=True): + # 1st job is in the queue (or running), don't enqueue it again + self.assertFalse(job.enqueue()) def test_is_dormant(self): self.assertTrue(is_dormant(check_time=get_datetime("2100-01-01 00:00:00"))) @@ -88,22 +85,10 @@ class TestScheduler(TestCase): ) ) - frappe.db.rollback() - def test_job_timeout(self): - return - job = enqueue(test_timeout, timeout=10) - count = 5 - while count > 0: - count -= 1 - time.sleep(5) - if job.get_status() == "failed": - break - - self.assertTrue(job.is_failed) - - -def get_test_job(method="frappe.tests.test_scheduler.test_timeout_10", frequency="All"): +def get_test_job( + method="frappe.tests.test_scheduler.test_timeout_10", frequency="All" +) -> ScheduledJobType: if not frappe.db.exists("Scheduled Job Type", dict(method=method)): job = frappe.get_doc( dict( diff --git a/frappe/tests/test_utils.py b/frappe/tests/test_utils.py index 04f9d16fd1..2a8d27cd19 100644 --- a/frappe/tests/test_utils.py +++ b/frappe/tests/test_utils.py @@ -30,13 +30,17 @@ from frappe.utils import ( validate_url, ) from frappe.utils.data import ( + add_to_date, cast, + get_first_day_of_week, get_time, get_timedelta, + getdate, now_datetime, nowtime, validate_python_code, ) +from frappe.utils.dateutils import get_dates_from_timegrain from frappe.utils.diff import _get_value_from_version, get_version_diff, version_query from frappe.utils.image import optimize_image, strip_exif_data from frappe.utils.response import json_handler @@ -445,6 +449,31 @@ class TestDateUtils(unittest.TestCase): self.assertIsInstance(get_timedelta(str(timedelta_input)), timedelta) self.assertIsInstance(get_timedelta(str(time_input)), timedelta) + def test_date_from_timegrain(self): + start_date = getdate("2021-01-01") + + daily = get_dates_from_timegrain(start_date, add_to_date(start_date, days=6), "Daily") + self.assertEqual(len(daily), 7) + for idx, d in enumerate(daily): + self.assertEqual(d, add_to_date(start_date, days=idx)) + + start = get_first_day_of_week(start_date) + end = add_to_date(add_to_date(start, weeks=52), days=-1) + weekly = get_dates_from_timegrain(start, end, "Weekly") + self.assertEqual(len(weekly), 52) + for idx, d in enumerate(weekly, start=1): + self.assertEqual(d, add_to_date(start, days=7 * idx - 1)) + + quarterly = get_dates_from_timegrain(start_date, add_to_date(start_date, months=5), "Quarterly") + self.assertEqual(len(quarterly), 2) + for idx, d in enumerate(quarterly, start=1): + self.assertEqual(d, add_to_date(start_date, months=idx * 3, days=-1)) + + yearly = get_dates_from_timegrain(start_date, add_to_date(start_date, years=2), "Yearly") + self.assertEqual(len(yearly), 3) + for idx, d in enumerate(yearly, start=1): + self.assertEqual(d, add_to_date(start_date, years=idx, days=-1)) + class TestResponse(unittest.TestCase): def test_json_handler(self): @@ -615,3 +644,29 @@ class TestAppParser(unittest.TestCase): self.assertEqual("healthcare", parse_app_name("https://github.com/frappe/healthcare.git")) self.assertEqual("healthcare", parse_app_name("git@github.com:frappe/healthcare.git")) self.assertEqual("healthcare", parse_app_name("frappe/healthcare@develop")) + + +class TestIntrospectionMagic(unittest.TestCase): + """Test utils that inspect live objects""" + + def test_get_newargs(self): + # `kwargs` is just convention any **varname should work. + def f(a, b=2, **args): + pass + + safe_kwargs = {"company": "Wind Power", "b": 1} + self.assertEqual(frappe.get_newargs(f, safe_kwargs), safe_kwargs) + + unsafe_args = dict(safe_kwargs) + unsafe_args.update({"ignore_permissions": True, "flags": {"ignore_mandatory": True}}) + self.assertEqual(frappe.get_newargs(f, unsafe_args), safe_kwargs) + + def test_strip_off_kwargs_when_not_supported(self): + def f(a, b=2): + pass + + args = {"company": "Wind Power", "b": 1} + self.assertEqual(frappe.get_newargs(f, args), {"b": 1}) + + # No args + self.assertEqual(frappe.get_newargs(lambda: None, args), {}) diff --git a/frappe/tests/test_website.py b/frappe/tests/test_website.py index 37ac611b4e..9478c4cf5f 100644 --- a/frappe/tests/test_website.py +++ b/frappe/tests/test_website.py @@ -118,7 +118,7 @@ class TestWebsite(unittest.TestCase): def test_error_page(self): set_request(method="GET", path="/_test/problematic_page") response = get_response() - self.assertEqual(response.status_code, 500) + self.assertEqual(response.status_code, 417) def test_login(self): set_request(method="GET", path="/login") diff --git a/frappe/tests/utils.py b/frappe/tests/utils.py index 7d00a0c1f9..fc26694d46 100644 --- a/frappe/tests/utils.py +++ b/frappe/tests/utils.py @@ -82,6 +82,8 @@ def _restore_thread_locals(flags): frappe.local.realtime_log = [] frappe.local.conf = frappe._dict(frappe.get_site_config()) frappe.local.cache = {} + frappe.local.lang = "en" + frappe.local.lang_full_dict = None @contextmanager diff --git a/frappe/translate.py b/frappe/translate.py index 3123eade48..eb26124ba8 100644 --- a/frappe/translate.py +++ b/frappe/translate.py @@ -48,6 +48,8 @@ TRANSLATE_PATTERN = re.compile( # END: JS context search r"[\s\n]*\)" # Closing function call ignore leading whitespace/newlines ) +REPORT_TRANSLATE_PATTERN = re.compile('"([^:,^"]*):') +CSV_STRIP_WHITESPACE_PATTERN = re.compile(r"{\s?([0-9]+)\s?}") def get_language(lang_list: List = None) -> str: @@ -602,7 +604,7 @@ def get_messages_from_report(name): messages.extend( [ (None, message) - for message in re.findall('"([^:,^"]*):', report.query) + for message in REPORT_TRANSLATE_PATTERN.findall(report.query) if is_translatable(message) ] ) @@ -801,7 +803,7 @@ def write_csv_file(path, app_messages, lang_dict): t = lang_dict.get(message, "") # strip whitespaces - translated_string = re.sub(r"{\s?([0-9]+)\s?}", r"{\g<1>}", t) + translated_string = CSV_STRIP_WHITESPACE_PATTERN.sub(r"{\g<1>}", t) if translated_string: w.writerow([message, translated_string, context]) diff --git a/frappe/translations/fr.csv b/frappe/translations/fr.csv index 06a2473d3b..69bc47d4f6 100644 --- a/frappe/translations/fr.csv +++ b/frappe/translations/fr.csv @@ -419,6 +419,7 @@ Also adding the dependent currency field {0},Ajout également du champ de devise Always use Account's Email Address as Sender,Toujours utiliser l'adresse Email du compte comme Expéditeur, Always use Account's Name as Sender's Name,Toujours utiliser le nom du compte comme nom de l'expéditeur, Amend,Nouv. version +amend,Nouv. version Amending,Nouv. version en cours, Amount Based On Field,Montant Basé sur le Champ, Amount Field,Champ du Montant, @@ -799,9 +800,9 @@ DESC,DESC, Daily Event Digest is sent for Calendar Events where reminders are set.,Un Récapitulatif Quotidien est envoyé pour les Événements du Calendrier ayant des rappels., Danger,Danger, Dark Color,Couleur sombre, -Dashboard Chart,Tableau de bord, -Dashboard Chart Link,Lien de tableau de bord, -Dashboard Chart Source,Source du graphique du tableau de bord, +Dashboard Chart,Tableau de bord - indicateur, +Dashboard Chart Link,Lien de tableau de bord - indicateur, +Dashboard Chart Source,Source du graphique du tableau de bord - indicateur, Dashboard Name,Nom du tableau de bord, Dashboards,Tableaux de bord, Data,Données, @@ -2296,7 +2297,7 @@ Show Line Breaks after Sections,Afficher les Sauts de Ligne après Sections, Show Permissions,Afficher les Autorisations, Show Preview Popup,Afficher l'aperçu Popup, Show Relapses,Afficher les Rechutes, -Show Report,Rapport d'émission, +Show Report,Afficher le rapport, Show Section Headings,Voir la Section Titres, Show Sidebar,Afficher la Barre Latérale, Show Title,Afficher le Titre, @@ -4641,7 +4642,7 @@ Not permitted to view {0},Non autorisé à afficher {0}, Camera,Caméra, Invalid filter: {0},Filtre non valide: {0}, Let's Get Started,Commençons, -Reports & Masters,Rapports et masters, +Reports & Masters,Ecrans principaux et Rapports, New {0} {1} added to Dashboard {2},Nouveau {0} {1} ajouté au tableau de bord {2}, New {0} {1} created,Nouveau {0} {1} créé, New {0} Created,Nouveau {0} créé, @@ -4701,8 +4702,8 @@ Value cannot be negative for,La valeur ne peut pas être négative pour, Value cannot be negative for {0}: {1},La valeur ne peut pas être négative pour {0}: {1}, Negative Value,Valeur négative, Authentication failed while receiving emails from Email Account: {0}.,L'authentification a échoué lors de la réception des e-mails du compte de messagerie: {0}., -Message from server: {0},Message du serveur: {0}, -{0} edited this {1},{0} a édité {1}, +Message from server: {0},Message du serveur: {0} +{0} edited this {1},{0} a édité {1} {0} created this {1},{0} a créé {1} Report an Issue,Signaler une anomalie User Forum,Forum utilisateur @@ -4717,3 +4718,11 @@ Document has been cancelled,Document annulé Document is in draft state,Document au statut brouillon Copy to Clipboard,Copier vers le presse-papiers Don't have an account?,Vous n'avez pas de compte? +Left:alignment,Gauche +Right:alignment,Droite +Set Properties,Gérer les proriétés +Create Workspace,Créer un espace de travail +Always use this email address as sender address,Toujours utiliser cet email comme expediteur +Always use this name as sender name,Toujours utiliser ce nom comme expediteur +Login to {0},Se connecter à {0} +Add / Remove Fields,Ajouter / Supprimer des colonnes diff --git a/frappe/translations/ru.csv b/frappe/translations/ru.csv index 3fdeab5546..94a87bdcf8 100644 --- a/frappe/translations/ru.csv +++ b/frappe/translations/ru.csv @@ -840,7 +840,7 @@ Default Sending and Inbox,По умолчанию отправка и получ Default Sort Field,Поле сортировки по умолчанию, Default Sort Order,Порядок сортировки по умолчанию, Default Value,Значение по умолчанию, -"Default: ""Contact Us""","По умолчанию: ""Обратная связь""", +"Default: ""Contact Us""","По умолчанию: ""Contact Us""", DefaultValue,DefaultValue, Define workflows for forms.,Определите рабочие процессы для форм., Defines actions on states and the next step and allowed roles.,"Определяет действия на статусах, следующий шаг и роли, обладающие правами перевода статусов.", @@ -849,7 +849,7 @@ Delayed,Задерживается, Delete Data,Удалить данные, Delete comment?,Удалить комментарий?, Delete this record to allow sending to this email address,"Удалить эту запись, чтобы разрешить отправку на этот адрес электронной почты", -Delete {0} items permanently?,Удалить {0} продуктов навсегда?, +Delete {0} items permanently?,Удалить {0} объектов навсегда?, Deleted,Удаленный, Deleted DocType,Удаленный DocType, Deleted Document,Удаленный документ, @@ -914,7 +914,7 @@ Document can't saved.,Документ не может быть сохранен Document {0} has been set to state {1} by {2},Документ {0} установлен в состояние {1} на {2}, Documents,Документы, Documents assigned to you and by you.,"Документы, назначенные вам и вами.", -Domain Settings,Настройки домена, +Domain Settings,Настройка сфер деятельности, Domains HTML,Домены HTML, "Don't HTML Encode HTML tags like <script> or just characters like < or >, as they could be intentionally used in this field","Не HTML Кодировать HTML-теги, такие как <скрипт> или просто символы, такие как <или>, так как они могут быть преднамеренно использованы в этой области", Don't Override Status,Не переопределять статус, @@ -990,7 +990,7 @@ Enable Auto Reply,Включить автоматический ответ, Enable Automatic Backup,Включить автоматическое резервное копирование, Enable Chat,Включить чат, Enable Comments,Включить комментарии, -Enable Incoming,Включение входящей, +Enable Incoming,Включить входящие, Enable Outgoing,Включить исходящие, Enable Password Policy,Включить политику паролей, Enable Print Server,Включить сервер печати, @@ -1000,7 +1000,7 @@ Enable Scheduled Jobs,Включить запланированных задан Enable Social Login,Включить социальный вход, Enable Two Factor Auth,Включить двухфакторный аут, Enabled email inbox for user {0},Включен почтовый ящик для пользователя {0}, -"Encryption key is invalid, Please check site_config.json","Ключ шифрования недействителен, проверьте сайт_config.json", +"Encryption key is invalid, Please check site_config.json","Ключ шифрования недействителен, проверьте site_config.json", End Date Field,Поле конечной даты, End Date cannot be before Start Date!,Дата окончания не может быть до даты начала!, Endpoint URL,URL конечной точки, @@ -1029,8 +1029,8 @@ Error in Notification: {},Ошибка в уведомлении: {}, Error while connecting to email account {0},Ошибка при подключении к учетной записи электронной почты {0}, Error while evaluating Notification {0}. Please fix your template.,Ошибка при оценке уведомления {0}. Исправьте шаблон., Error: Document has been modified after you have opened it,"Ошибка: документ был изменен после того, как вы открыли его", -Error: Value missing for {0}: {1},Ошибка: значение отсутствует для {0}: {1}, -Errors in Background Events,Ошибки в фоновых событий, +Error: Value missing for {0}: {1},Ошибка: отсутствует значение для {0}: {1}, +Errors in Background Events,Ошибки в фоновых событиях, Event Category,Категория события, Event Participants,Участники мероприятия, Event Type,Тип события, @@ -1184,7 +1184,7 @@ Get Contacts,Получить контакты, Get Fields,Получить поля, Get your globally recognized avatar from Gravatar.com,Получить всемирно признанный аватара из Gravatar.com, GitHub,GitHub, -Give Review Points,Дайте очки обзора, +Give Review Points,Дайте баллы обзора, Global Unsubscribe,Глобальная отписка, Go to the document,Перейти к документу, Go to this URL after completing the form (only for Guest users),Перейдите по этому URL-адресу после заполнения формы (только для гостевых пользователей), @@ -1461,8 +1461,8 @@ Letter Head Name,Название заголовка письма, Letter Head in HTML,Заголовок письма в HTML, Level Name,Название уровня, Liked,Понравилось, -Liked By,В избранное К, -Liked by {0},В избранное {0}, +Liked By,Нравится, +Liked by {0},Нравится {0}, Likes,Понравившееся, Limit Number of DB Backups,Ограничение количества резервных копий БД, Line,Линия, @@ -1470,7 +1470,7 @@ Link DocType,Ссылка DocType, Link Expired,Срок действия ссылки, Link Name,Имя ссылки, Link Title,Название ссылки, -"Link that is the website home page. Standard Links (index, login, products, blog, about, contact)","Ссылка, которая является стартовой страницей сайта. Стандартные ссылки (индекс, логин, продукты, блог, о, контакт)", +"Link that is the website home page. Standard Links (index, login, products, blog, about, contact)","Ссылка, которая является стартовой страницей сайта. Стандартные ссылки (index, login, products, blog, about, contact)", Link to the page you want to open. Leave blank if you want to make it a group parent.,"Ссылка на страницу, которую вы хотите открыть. Оставьте пустым, если хотите сделать его родительским элементом группы.", Linked,Связанный, Linked With,Связанные с, @@ -2096,7 +2096,7 @@ Revert Of,Вернуть из, Reverted,Отменено, Review Level,Уровень обзора, Review Levels,Уровни обзора, -Review Points,Очки обзора, +Review Points,Баллы обзора, Reviews,Отзывы, Revoke,Аннулировать, Revoked,Аннулировано, @@ -2141,10 +2141,10 @@ SMS sent to following numbers: {0},SMS отправлено следующим SMTP Server,SMTP-сервер, SMTP Settings for outgoing emails,Настройки SMTP для исходящих писем, "SQL Conditions. Example: status=""Open""",SQL условия. Пример: статус = "Открыть", -SSL/TLS Mode,Режим SSL / TLS, +SSL/TLS Mode,Режим SSL/TLS, Salesforce,Salesforce, Same Field is entered more than once,Одно и то же поле вводится не один раз, -Save API Secret: ,Сохранить API-интерфейс:, +Save API Secret: ,Сохранить API секрет: , Save As,Сохранить как, Save Filter,Сохранить фильтр, Save Report,Сохранить отчет, @@ -2258,7 +2258,7 @@ Set Property After Alert,Задать свойство после оповеще Set Quantity,Установите Количество, Set Role For,Установить роль для, Set User Permissions,Задание разрешений пользователя, -Set Value,Задать значение, +Set Value,Установить значение, Set custom roles for page and report,Набор пользовательских ролей для страницы и отчета, "Set default format, page size, print style etc.","Установить форму, размер страницы, стиль печати и т.д., используюмых по умолчанию", Set non-standard precision for a Float or Currency field,Установите нестандартные точность для поплавка или валютной области, @@ -2337,7 +2337,7 @@ Slideshow like display for the website,"Слайд-шоу, как дисплей Small Text,Маленьикий текст, Smallest Currency Fraction Value,Минимальное дробное значение, Smallest circulating fraction unit (coin). For e.g. 1 cent for USD and it should be entered as 0.01,"Минимальная разменная денежная единица (монета). Например, для доллара — 1 цент, и его нужно ввести как 0,01", -Snapshot View,Снимок Посмотреть, +Snapshot View,Просмотр снимка, Social,Сообщество, Social Login Key,Ключ социального входа, Social Login Provider,Социальный провайдер, @@ -2418,7 +2418,7 @@ Suspend Sending,Приостановить Отправка, Switch To Desk,Переключение на рабочий стол, Symbol,Символ, Sync,Синхронизация, -Sync on Migrate,Синхронизация по Migrate, +Sync on Migrate,Синхронизировать при переносе, Syntax error in template,Синтаксическая ошибка в шаблоне, System,Система, System Page,Страница системы, @@ -2450,7 +2450,7 @@ Thank you for your interest in subscribing to our updates,Спасибо за в Thank you for your message,Спасибо за ваше сообщение, The CSV format is case sensitive,Формат CSV чувствителен к регистру, The Condition '{0}' is invalid,Условие '{0}' является недействительным, -The First User: You,Первый пользователя: Вы, +The First User: You,Первый пользователь: Вы, "The application has been updated to a new version, please refresh this page","Приложение был обновлен до новой версии, пожалуйста, обновите эту страницу", The attachments could not be correctly linked to the new document,Вложения не могут быть правильно связаны с новым документом, The document could not be correctly assigned,Документ не может быть правильно назначен, @@ -2653,7 +2653,7 @@ User Field,Поле пользователя, User ID of a Blogger,ID пользователя-блоггера, User Image,Изображение пользователя, User Name,Имя пользователя, -User Permission,Пользователь Введено, +User Permission,Разрешения пользователя, User Permissions,Разрешения пользователей, User Permissions are used to limit users to specific records.,Пользовательские разрешения используются для ограничения пользователей конкретными записями., User Permissions created sucessfully,Пользовательские разрешения созданы успешно, @@ -3068,8 +3068,8 @@ zoom-out,отдалить, {0} or {1},{0} или {1}, {0} record deleted,{0} запись удалена, {0} records deleted,{0} записей удалено, -{0} reverted your point on {1},{0} вернул вашу точку на {1}, -{0} reverted your points on {1},{0} вернул ваши очки на {1}, +{0} reverted your point on {1},{0} вернул ваш балл на {1}, +{0} reverted your points on {1},{0} вернул ваши баллы на {1}, {0} reverted {1},{0} вернул {1}, {0} room must have atmost one user.,{0} номер должен иметь самого одного пользователя., {0} rows for {1},{0} строк для {1}, @@ -3148,7 +3148,7 @@ Access not allowed from this IP Address,Доступ с этого IP-адрес Action Type,Тип действия, Activity Log by ,Активность Журнал по, Add Fields,Добавить поля, -Administration,Администрация, +Administration,Администрирование, After Cancel,После отмены, After Delete,После удаления, After Save,После сохранения, @@ -3157,7 +3157,7 @@ After Submit,После отправки, Aggregate Function Based On,"Агрегатная функция, основанная на", Aggregate Function field is required to create a dashboard chart,Поле Aggregate Function необходимо для создания диаграммы панели мониторинга., All Records,Все записи, -Allot Points To Assigned Users,Выделить очки назначенным пользователям, +Allot Points To Assigned Users,Выделить баллы назначенным пользователям, Allow Auto Repeat,Разрешить автоматическое повторение, Allow Google Calendar Access,Разрешить доступ к Календарю Google, Allow Google Contacts Access,Разрешить доступ к контактам Google, @@ -3380,7 +3380,7 @@ Invalid field name: {0},Неверное имя поля: {0}, Invalid file URL. Please contact System Administrator.,"Неверный URL файла. Пожалуйста, свяжитесь с системным администратором.", Invalid include path,Неверный путь включения, Invalid username or password,неправильное имя пользователя или пароль, -Is Primary,Первичный, +Is Primary,Основной, Is Primary Mobile,Основной мобильный, Is Primary Phone,Основной телефон, Is Tree,Дерево, @@ -3667,7 +3667,7 @@ via Data Import,через импорт данных, {0} are mandatory fields,{0} обязательные поля, {0} are required,{0} требуется, {0} assigned a new task {1} {2} to you,{0} назначил вам новое задание {1} {2}, -{0} gained {1} point for {2} {3},{0} набрал {1} очко за {2} {3}, +{0} gained {1} point for {2} {3},{0} получил {1} балл за {2} {3}, {0} gained {1} points for {2} {3},{0} набрал {1} баллов за {2} {3}, {0} has no versions tracked.,{0} не отслеживает версии., {0} is not a valid report format. Report format should one of the following {1},{0} не является допустимым форматом отчета. Формат отчета должен быть одним из следующих {1}, @@ -4045,7 +4045,7 @@ No Permitted Charts on this Dashboard,На этой панели инструм No Permitted Charts,Нет разрешенных графиков, Reset Chart,Сбросить график, via {0},через {0}, -{0} is not a valid Phone Number,{0} не является действительным номером телефона, +{0} is not a valid Phone Number,{0} недействительный номер телефона, Failed Transactions,Неудачные транзакции, Value for field {0} is too long in {1}. Length should be lesser than {2} characters,Значение поля {0} слишком длинное в {1}. Длина должна быть меньше {2} симв., Data Too Long,Данные слишком длинные, @@ -4121,8 +4121,8 @@ Using this console may allow attackers to impersonate you and steal your informa {0} w,{0} н, {0} M,{0} М, {0} y,{0} г, -yesterday,вчерашний день, -{0} years ago,{0} лет назад, +yesterday,вчера, +{0} years ago,{0} год назад, New Chart,Новый график, New Shortcut,Новый ярлык, Edit Chart,Изменить диаграмму, @@ -4700,3 +4700,11 @@ Value cannot be negative for {0}: {1},Значение не может быть Negative Value,Отрицательное значение, Authentication failed while receiving emails from Email Account: {0}.,Ошибка аутентификации при получении писем из учетной записи электронной почты: {0}., Message from server: {0},Сообщение с сервера: {0}, +Documentation,Документация, +User Forum,Форум пользователей, +Report an issue,Сообщить об ошибке, +My Profile,Мой профиль, +My Settings,Мои настройки, +Toggle Full Width,Переключить ширину, +Toggle Theme,Переключить тему, +Modules,Модули, diff --git a/frappe/twofactor.py b/frappe/twofactor.py index 5a2799bc54..9e9a2c5d76 100644 --- a/frappe/twofactor.py +++ b/frappe/twofactor.py @@ -277,7 +277,9 @@ def get_email_subject_for_qr_code(kwargs_dict): def get_email_body_for_qr_code(kwargs_dict): """Get QRCode email body.""" - body_template = "Please click on the following link and follow the instructions on the page.

    {{qrcode_link}}" + body_template = _( + "Please click on the following link and follow the instructions on the page. {0}" + ).format("

    {{qrcode_link}}") body = frappe.render_template(body_template, kwargs_dict) return body diff --git a/frappe/utils/__init__.py b/frappe/utils/__init__.py index 1af0ec6a39..6872ace7d8 100644 --- a/frappe/utils/__init__.py +++ b/frappe/utils/__init__.py @@ -27,6 +27,16 @@ import frappe from frappe.utils.data import * from frappe.utils.html_utils import sanitize_html +EMAIL_NAME_PATTERN = re.compile(r"[^A-Za-z0-9\u00C0-\u024F\/\_\' ]+") +EMAIL_STRING_PATTERN = re.compile(r"([a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)") +NON_MD_HTML_PATTERN = re.compile(r"|") +HTML_TAGS_PATTERN = re.compile(r"\<[^>]*\>") +INCLUDE_DIRECTIVE_PATTERN = re.compile("""({% include ['"]([^'"]*)['"] %})""") +PHONE_NUMBER_PATTERN = re.compile(r"([0-9\ \+\_\-\,\.\*\#\(\)]){1,20}$") +PERSON_NAME_PATTERN = re.compile(r"^[\w][\w\'\-]*( \w[\w\'\-]*)*$") +WHITESPACE_PATTERN = re.compile(r"[\t\n\r]") +MULTI_EMAIL_STRING_PATTERN = re.compile(r'[,\n](?=(?:[^"]|"[^"]*")*$)') + def get_fullname(user=None): """get the full name (first name + last name) of the user from User""" @@ -116,7 +126,7 @@ def validate_phone_number(phone_number, throw=False): return False phone_number = phone_number.strip() - match = re.match(r"([0-9\ \+\_\-\,\.\*\#\(\)]){1,20}$", phone_number) + match = PHONE_NUMBER_PATTERN.match(phone_number) if not match and throw: frappe.throw( @@ -135,7 +145,7 @@ def validate_name(name, throw=False): return False name = name.strip() - match = re.match(r"^[\w][\w\'\-]*( \w[\w\'\-]*)*$", name) + match = PERSON_NAME_PATTERN.match(name) if not match and throw: frappe.throw(frappe._("{0} is not a valid Name").format(name), frappe.InvalidNameError) @@ -201,8 +211,8 @@ def split_emails(txt): email_list = [] # emails can be separated by comma or newline - s = re.sub(r"[\t\n\r]", " ", cstr(txt)) - for email in re.split(r'[,\n](?=(?:[^"]|"[^"]*")*$)', s): + s = WHITESPACE_PATTERN.sub(" ", cstr(txt)) + for email in MULTI_EMAIL_STRING_PATTERN.split(s): email = strip(cstr(email)) if email: email_list.append(email) @@ -360,7 +370,7 @@ def remove_blanks(d): def strip_html_tags(text): """Remove html tags from text""" - return re.sub(r"\<[^>]*\>", "", text) + return HTML_TAGS_PATTERN.sub("", text) def get_file_timestamp(fn): @@ -584,7 +594,7 @@ def get_html_format(print_path): with open(print_path, "r") as f: html_format = f.read() - for include_directive, path in re.findall("""({% include ['"]([^'"]*)['"] %})""", html_format): + for include_directive, path in INCLUDE_DIRECTIVE_PATTERN.findall(html_format): for app_name in frappe.get_installed_apps(): include_path = frappe.get_app_path(app_name, *path.split(os.path.sep)) if os.path.exists(include_path): @@ -601,7 +611,7 @@ def is_markdown(text): elif "" in text: return False else: - return not re.search(r"|", text) + return not NON_MD_HTML_PATTERN.search(text) def get_sites(sites_path=None): @@ -670,8 +680,7 @@ def parse_addr(email_string): name = get_name_from_email_string(email_string, email, name) return (name, email) else: - email_regex = re.compile(r"([a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)") - email_list = re.findall(email_regex, email_string) + email_list = EMAIL_STRING_PATTERN.findall(email_string) if len(email_list) > 0 and check_format(email_list[0]): # take only first email address email = email_list[0] @@ -698,7 +707,7 @@ def check_format(email_id): def get_name_from_email_string(email_string, email_id, name): name = email_string.replace(email_id, "") - name = re.sub(r"[^A-Za-z0-9\u00C0-\u024F\/\_\' ]+", "", name).strip() + name = EMAIL_NAME_PATTERN.sub("", name).strip() if not name: name = email_id return name diff --git a/frappe/utils/background_jobs.py b/frappe/utils/background_jobs.py index ce8e44665a..f49c641673 100755 --- a/frappe/utils/background_jobs.py +++ b/frappe/utils/background_jobs.py @@ -3,7 +3,7 @@ import socket import time from collections import defaultdict from functools import lru_cache -from typing import List +from typing import TYPE_CHECKING, List from uuid import uuid4 import redis @@ -19,6 +19,9 @@ from frappe.utils import cstr, get_bench_id from frappe.utils.commands import log from frappe.utils.redis_queue import RedisQueue +if TYPE_CHECKING: + from rq.job import Job + @lru_cache() def get_queues_timeout(): @@ -52,7 +55,7 @@ def enqueue( *, at_front=False, **kwargs, -): +) -> "Job": """ Enqueue method to be executed using a background worker diff --git a/frappe/utils/boilerplate.py b/frappe/utils/boilerplate.py index 22ca64eb1a..edb742feb4 100644 --- a/frappe/utils/boilerplate.py +++ b/frappe/utils/boilerplate.py @@ -11,6 +11,8 @@ import git import frappe from frappe.utils import touch_file +APP_TITLE_PATTERN = re.compile(r"^(?![\W])[^\d_\s][\w -]+$", flags=re.UNICODE) + def make_boilerplate(dest, app_name, no_git=False): if not os.path.exists(dest): @@ -67,7 +69,7 @@ def _get_user_inputs(app_name): def is_valid_title(title) -> bool: - if not re.match(r"^(?![\W])[^\d_\s][\w -]+$", title, re.UNICODE): + if not APP_TITLE_PATTERN.match(title): print( "App Title should start with a letter and it can only consist of letters, numbers, spaces and underscores" ) @@ -488,7 +490,7 @@ jobs: uses: actions/cache@v2 with: path: ~/.cache/pip - key: ${{{{ runner.os }}}}-pip-${{{{ hashFiles('**/*requirements.txt') }}}} + key: ${{{{ runner.os }}}}-pip-${{{{ hashFiles('**/*requirements.txt', '**/pyproject.toml', '**/setup.py', '**/setup.cfg') }}}} restore-keys: | ${{{{ runner.os }}}}-pip- ${{{{ runner.os }}}}- diff --git a/frappe/utils/caching.py b/frappe/utils/caching.py new file mode 100644 index 0000000000..326dacfd5a --- /dev/null +++ b/frappe/utils/caching.py @@ -0,0 +1,130 @@ +# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors +# License: MIT. Check LICENSE + +import json +from collections import defaultdict +from datetime import datetime, timedelta +from functools import wraps +from typing import Callable, Dict, Optional, Tuple + +import frappe + +_SITE_CACHE = defaultdict(lambda: defaultdict(dict)) + + +def __generate_request_cache_key(args: Tuple, kwargs: Dict): + """Generate a key for the cache.""" + if not kwargs: + return hash(args) + return hash((args, frozenset(kwargs.items()))) + + +def request_cache(func: Callable) -> Callable: + """Decorator to cache function calls mid-request. Cache is stored in + frappe.local.request_cache. The cache only persists for the current request + and is cleared when the request is over. The function is called just once + per request with the same set of (kw)arguments. + + Usage: + from frappe.utils.caching import request_cache + + @request_cache + def calculate_pi(num_terms=0): + import math, time + print(f"{num_terms = }") + time.sleep(10) + return math.pi + + calculate_pi(10) # will calculate value + calculate_pi(10) # will return value from cache + """ + + @wraps(func) + def wrapper(*args, **kwargs): + if not getattr(frappe.local, "initialised", None): + return func(*args, **kwargs) + if not hasattr(frappe.local, "request_cache"): + frappe.local.request_cache = defaultdict(dict) + + try: + args_key = __generate_request_cache_key(args, kwargs) + except Exception: + return func(*args, **kwargs) + + try: + return frappe.local.request_cache[func][args_key] + except KeyError: + return_val = func(*args, **kwargs) + frappe.local.request_cache[func][args_key] = return_val + return return_val + + return wrapper + + +def site_cache(ttl: Optional[int] = None, maxsize: Optional[int] = None) -> Callable: + """Decorator to cache method calls across requests. The cache is stored in + frappe.utils.caching._SITE_CACHE. The cache persists on the parent process. + It offers a light-weight cache for the current process without the additional + overhead of serializing / deserializing Python objects. + + Note: This cache isn't shared among workers. If you need to share data across + workers, use redis (frappe.cache API) instead. + + Usage: + from frappe.utils.caching import site_cache + + @site_cache + def calculate_pi(): + import math, time + precision = get_precision("Math Constant", "Pi") # depends on site data + return round(math.pi, precision) + + calculate_pi(10) # will calculate value + calculate_pi(10) # will return value from cache + calculate_pi.clear_cache() # clear this function's cache for all sites + calculate_pi(10) # will calculate value + """ + + def time_cache_wrapper(func: Callable = None) -> Callable: + func_key = f"{func.__module__}.{func.__name__}" + + def clear_cache(): + """Clear cache for this function for all sites if not specified.""" + _SITE_CACHE[func_key].clear() + + func.clear_cache = clear_cache + + if ttl is not None and not callable(ttl): + func.ttl = ttl + func.expiration = datetime.utcnow() + timedelta(seconds=func.ttl) + + if maxsize is not None and not callable(maxsize): + func.maxsize = maxsize + + @wraps(func) + def site_cache_wrapper(*args, **kwargs): + if getattr(frappe.local, "initialised", None): + func_call_key = json.dumps((args, kwargs)) + + if hasattr(func, "ttl") and datetime.utcnow() >= func.expiration: + func.clear_cache() + func.expiration = datetime.utcnow() + timedelta(seconds=func.ttl) + + if hasattr(func, "maxsize") and len(_SITE_CACHE[func_key][frappe.local.site]) >= func.maxsize: + _SITE_CACHE[func_key][frappe.local.site].pop( + next(iter(_SITE_CACHE[func_key][frappe.local.site])), None + ) + + if func_call_key not in _SITE_CACHE[func_key][frappe.local.site]: + _SITE_CACHE[func_key][frappe.local.site][func_call_key] = func(*args, **kwargs) + + return _SITE_CACHE[func_key][frappe.local.site][func_call_key] + + return func(*args, **kwargs) + + return site_cache_wrapper + + if callable(ttl): + return time_cache_wrapper(ttl) + + return time_cache_wrapper diff --git a/frappe/utils/data.py b/frappe/utils/data.py index 6a9ffc81a6..49f9ead437 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -29,7 +29,22 @@ if typing.TYPE_CHECKING: DATE_FORMAT = "%Y-%m-%d" TIME_FORMAT = "%H:%M:%S.%f" -DATETIME_FORMAT = DATE_FORMAT + " " + TIME_FORMAT +DATETIME_FORMAT = f"{DATE_FORMAT} {TIME_FORMAT}" +TIMEDELTA_DAY_PATTERN = re.compile( + r"(?P[-\d]+) day[s]*, (?P\d+):(?P\d+):(?P\d[\.\d+]*)" +) +TIMEDELTA_BASE_PATTERN = re.compile(r"(?P\d+):(?P\d+):(?P\d[\.\d+]*)") +URLS_HTTP_TAG_PATTERN = re.compile( + r'(href|src){1}([\s]*=[\s]*[\'"]?)((?:http)[^\'">]+)([\'"]?)' +) # href='https://... +URLS_NOT_HTTP_TAG_PATTERN = re.compile( + r'(href|src){1}([\s]*=[\s]*[\'"]?)((?!http)[^\'" >]+)([\'"]?)' +) # href=/assets/... +URL_NOTATION_PATTERN = re.compile( + r'(:[\s]?url)(\([\'"]?)((?!http)[^\'" >]+)([\'"]?\))' +) # background-image: url('/assets/...') +DURATION_PATTERN = re.compile(r"^(?:(\d+d)?((^|\s)\d+h)?((^|\s)\d+m)?((^|\s)\d+s)?)$") +HTML_TAG_PATTERN = re.compile("<[^>]+>") class Weekday(Enum): @@ -692,10 +707,7 @@ def duration_to_seconds(duration): def validate_duration_format(duration): - import re - - is_valid_duration = re.match(r"^(?:(\d+d)?((^|\s)\d+h)?((^|\s)\d+m)?((^|\s)\d+s)?)$", duration) - if not is_valid_duration: + if not DURATION_PATTERN.match(duration): frappe.throw( frappe._("Value {0} must be in the valid duration format: d h m s").format( frappe.bold(duration) @@ -1297,7 +1309,7 @@ def in_words(integer: int, in_million=True) -> str: def is_html(text: str) -> bool: if not isinstance(text, str): return False - return re.search("<[^>]+>", text) + return HTML_TAG_PATTERN.search(text) def is_image(filepath: str) -> bool: @@ -1314,7 +1326,7 @@ def get_thumbnail_base64_for_image(src): from PIL import Image from frappe import cache, safe_decode - from frappe.core.doctype.file.file import get_local_image + from frappe.core.doctype.file.utils import get_local_image if not src: frappe.throw("Invalid source for image: {0}".format(src)) @@ -1337,7 +1349,7 @@ def get_thumbnail_base64_for_image(src): original_size = image.size size = 50, 50 - image.thumbnail(size, Image.ANTIALIAS) + image.thumbnail(size, Image.Resampling.LANCZOS) base64_string = image_to_base64(image, extn) return { @@ -1851,12 +1863,8 @@ def expand_relative_urls(html: str) -> str: return "".join(to_expand) - html = re.sub( - r'(href|src){1}([\s]*=[\s]*[\'"]?)((?!http)[^\'" >]+)([\'"]?)', _expand_relative_urls, html - ) - - # background-image: url('/assets/...') - html = re.sub(r'(:[\s]?url)(\([\'"]?)((?!http)[^\'" >]+)([\'"]?\))', _expand_relative_urls, html) + html = URLS_NOT_HTTP_TAG_PATTERN.sub(_expand_relative_urls, html) + html = URL_NOTATION_PATTERN.sub(_expand_relative_urls, html) return html @@ -1870,7 +1878,7 @@ def quote_urls(html: str) -> str: groups[2] = quoted(groups[2]) return "".join(groups) - return re.sub(r'(href|src){1}([\s]*=[\s]*[\'"]?)((?:http)[^\'">]+)([\'"]?)', _quote_url, html) + return URLS_HTTP_TAG_PATTERN.sub(_quote_url, html) def unique(seq: typing.Sequence["T"]) -> List["T"]: @@ -1887,6 +1895,15 @@ def strip(val: str, chars: Optional[str] = None) -> str: return (val or "").replace("\ufeff", "").replace("\u200b", "").strip(chars) +def get_string_between(start: str, string: str, end: str) -> str: + if not string: + return "" + + out = re.search(f"{start}(.*){end}", string) + + return out.group(1) if out else string + + def to_markdown(html: str) -> str: from html.parser import HTMLParser @@ -2088,10 +2105,8 @@ def format_timedelta(o: datetime.timedelta) -> str: def parse_timedelta(s: str) -> datetime.timedelta: # ref: https://stackoverflow.com/a/21074460/10309266 if "day" in s: - m = re.match( - r"(?P[-\d]+) day[s]*, (?P\d+):(?P\d+):(?P\d[\.\d+]*)", s - ) + m = TIMEDELTA_DAY_PATTERN.match(s) else: - m = re.match(r"(?P\d+):(?P\d+):(?P\d[\.\d+]*)", s) + m = TIMEDELTA_BASE_PATTERN.match(s) return datetime.timedelta(**{key: float(val) for key, val in m.groupdict().items()}) diff --git a/frappe/utils/dateutils.py b/frappe/utils/dateutils.py index b8f22e7ed7..d7412a444f 100644 --- a/frappe/utils/dateutils.py +++ b/frappe/utils/dateutils.py @@ -106,11 +106,10 @@ def get_dates_from_timegrain(from_date, to_date, timegrain="Daily"): months = 1 elif "Quarterly" == timegrain: months = 3 + elif "Yearly" == timegrain: + months = 1 - if "Weekly" == timegrain: - dates = [get_last_day_of_week(from_date)] - else: - dates = [get_period_ending(from_date, timegrain)] + dates = [get_period_ending(from_date, timegrain)] while getdate(dates[-1]) < getdate(to_date): if "Weekly" == timegrain: @@ -163,16 +162,12 @@ def get_period_beginning(date, timegrain, as_str=True): def get_period_ending(date, timegrain): - date = getdate(date) - if timegrain == "Daily": - return date - else: - return getdate( - { - "Daily": date, - "Weekly": get_last_day_of_week(date), - "Monthly": get_last_day(date), - "Quarterly": get_quarter_ending(date), - "Yearly": get_year_ending(date), - }[timegrain] - ) + return getdate( + { + "Daily": date, + "Weekly": get_last_day_of_week(date), + "Monthly": get_last_day(date), + "Quarterly": get_quarter_ending(date), + "Yearly": get_year_ending(date), + }[timegrain] + ) diff --git a/frappe/utils/file_manager.py b/frappe/utils/file_manager.py index ca5589c906..620962004a 100644 --- a/frappe/utils/file_manager.py +++ b/frappe/utils/file_manager.py @@ -449,3 +449,15 @@ def add_attachments(doctype, name, attachments): files.append(f) return files + + +def is_safe_path(path: str) -> bool: + if path.startswith(("http://", "https://")): + return True + + basedir = frappe.get_site_path() + # ref: https://docs.python.org/3/library/os.path.html#os.path.commonpath + matchpath = os.path.realpath(os.path.abspath(path)) + basedir = os.path.realpath(os.path.abspath(basedir)) + + return basedir == os.path.commonpath((basedir, matchpath)) diff --git a/frappe/utils/formatters.py b/frappe/utils/formatters.py index adf551580c..575a05a5c2 100644 --- a/frappe/utils/formatters.py +++ b/frappe/utils/formatters.py @@ -20,6 +20,8 @@ from frappe.utils import ( formatdate, ) +BLOCK_TAGS_PATTERN = re.compile(r"(") elif df.get("fieldtype") == "Markdown Editor": diff --git a/frappe/utils/global_search.py b/frappe/utils/global_search.py index 6e482baa78..af6d9a3c28 100644 --- a/frappe/utils/global_search.py +++ b/frappe/utils/global_search.py @@ -13,6 +13,8 @@ from frappe.utils import cint, strip_html_tags from frappe.utils.data import cstr from frappe.utils.html_utils import unescape_html +HTML_TAGS_PATTERN = re.compile(r"(?s)<[\s]*(script|style).*?") + def setup_global_search_table(): """ @@ -360,7 +362,7 @@ def get_formatted_value(value, field): if getattr(field, "fieldtype", None) in ["Text", "Text Editor"]: value = unescape_html(frappe.safe_decode(value)) - value = re.subn(r"(?s)<[\s]*(script|style).*?", "", str(value))[0] + value = HTML_TAGS_PATTERN.subn("", str(value))[0] value = " ".join(value.split()) return field.label + " : " + strip_html_tags(str(value)) diff --git a/frappe/utils/html_utils.py b/frappe/utils/html_utils.py index 8eac761220..b9d0e8dfe2 100644 --- a/frappe/utils/html_utils.py +++ b/frappe/utils/html_utils.py @@ -5,6 +5,16 @@ from bleach_allowlist import bleach_allowlist import frappe +EMOJI_PATTERN = re.compile( + "(\ud83d[\ude00-\ude4f])|" + "(\ud83c[\udf00-\uffff])|" + "(\ud83d[\u0000-\uddff])|" + "(\ud83d[\ude80-\udeff])|" + "(\ud83c[\udde0-\uddff])" + "+", + flags=re.UNICODE, +) + def clean_html(html): import bleach @@ -181,28 +191,17 @@ def is_json(text): def get_icon_html(icon, small=False): from frappe.utils import is_image - emoji_pattern = re.compile( - "(\ud83d[\ude00-\ude4f])|" - "(\ud83c[\udf00-\uffff])|" - "(\ud83d[\u0000-\uddff])|" - "(\ud83d[\ude80-\udeff])|" - "(\ud83c[\udde0-\uddff])" - "+", - flags=re.UNICODE, - ) - icon = icon or "" - if icon and emoji_pattern.match(icon): - return '' + icon + "" + + if icon and EMOJI_PATTERN.match(icon): + return f'{icon}' if is_image(icon): return ( - ''.format(icon=icon) - if small - else ''.format(icon=icon) + f'' if small else f'' ) else: - return "".format(icon=icon) + return f"" def unescape_html(value): diff --git a/frappe/utils/image.py b/frappe/utils/image.py index 0cbc02fb31..8823ea3dfe 100644 --- a/frappe/utils/image.py +++ b/frappe/utils/image.py @@ -7,8 +7,6 @@ from PIL import Image def resize_images(path, maxdim=700): - from PIL import Image - size = (maxdim, maxdim) for basepath, folders, files in os.walk(path): for fname in files: @@ -16,7 +14,7 @@ def resize_images(path, maxdim=700): if extn in ("jpg", "jpeg", "png", "gif"): im = Image.open(os.path.join(basepath, fname)) if im.size[0] > size[0] or im.size[1] > size[1]: - im.thumbnail(size, Image.ANTIALIAS) + im.thumbnail(size, Image.Resampling.LANCZOS) im.save(os.path.join(basepath, fname)) print("resized {0}".format(os.path.join(basepath, fname))) @@ -56,7 +54,7 @@ def optimize_image( image = Image.open(io.BytesIO(content)) image_format = content_type.split("/")[1] size = max_width, max_height - image.thumbnail(size, Image.LANCZOS) + image.thumbnail(size, Image.Resampling.LANCZOS) output = io.BytesIO() image.save( diff --git a/frappe/utils/install.py b/frappe/utils/install.py index 2ce067a018..fcf8f9d436 100644 --- a/frappe/utils/install.py +++ b/frappe/utils/install.py @@ -21,7 +21,7 @@ def after_install(): create_user_type() install_basic_docs() - from frappe.core.doctype.file.file import make_home_folder + from frappe.core.doctype.file.utils import make_home_folder make_home_folder() diff --git a/frappe/utils/logger.py b/frappe/utils/logger.py index bdae92da51..f2ce8a14c3 100755 --- a/frappe/utils/logger.py +++ b/frappe/utils/logger.py @@ -7,10 +7,7 @@ from logging.handlers import RotatingFileHandler import frappe from frappe.utils import get_sites -# imports - third party imports - - -default_log_level = logging.DEBUG +default_log_level = logging.WARNING if frappe._dev_server else logging.ERROR def get_logger( @@ -21,7 +18,7 @@ def get_logger( max_size=100_000, file_count=20, stream_only=False, -): +) -> "logging.Logger": """Application Logger for your given module Args: @@ -90,7 +87,7 @@ def get_logger( class SiteContextFilter(logging.Filter): """This is a filter which injects request information (if available) into the log.""" - def filter(self, record): + def filter(self, record) -> bool: if "Form Dict" not in str(record.msg): site = getattr(frappe.local, "site", None) form_dict = getattr(frappe.local, "form_dict", None) @@ -98,7 +95,7 @@ class SiteContextFilter(logging.Filter): return True -def set_log_level(level): +def set_log_level(level: int) -> None: """Use this method to set log level to something other than the default DEBUG""" frappe.log_level = getattr(logging, (level or "").upper(), None) or default_log_level frappe.loggers = {} diff --git a/frappe/utils/nestedset.py b/frappe/utils/nestedset.py index d3067973ef..681cd6439d 100644 --- a/frappe/utils/nestedset.py +++ b/frappe/utils/nestedset.py @@ -17,6 +17,7 @@ from frappe import _ from frappe.model.document import Document from frappe.query_builder import Order from frappe.query_builder.functions import Coalesce, Max +from frappe.query_builder.terms import SubQuery from frappe.query_builder.utils import DocType @@ -336,14 +337,15 @@ class NestedSet(Document): def get_root_of(doctype): """Get root element of a DocType with a tree structure""" from frappe.query_builder.functions import Count - from frappe.query_builder.terms import subqry Table = DocType(doctype) t1 = Table.as_("t1") t2 = Table.as_("t2") - subq = frappe.qb.from_(t2).select(Count("*")).where((t2.lft < t1.lft) & (t2.rgt > t1.rgt)) - result = frappe.qb.from_(t1).select(t1.name).where((subqry(subq) == 0) & (t1.rgt > t1.lft)).run() + node_query = SubQuery( + frappe.qb.from_(t2).select(Count("*")).where((t2.lft < t1.lft) & (t2.rgt > t1.rgt)) + ) + result = frappe.qb.from_(t1).select(t1.name).where((node_query == 0) & (t1.rgt > t1.lft)).run() return result[0][0] if result else None diff --git a/frappe/utils/password.py b/frappe/utils/password.py index f2c4b9685a..c539891ac7 100644 --- a/frappe/utils/password.py +++ b/frappe/utils/password.py @@ -213,21 +213,16 @@ def decrypt(txt, encryption_key=None): try: cipher_suite = Fernet(encode(encryption_key or get_encryption_key())) - plain_text = cstr(cipher_suite.decrypt(encode(txt))) - return plain_text + return cstr(cipher_suite.decrypt(encode(txt))) except InvalidToken: # encryption_key in site_config is changed and not valid - frappe.throw( - _("Encryption key is invalid") + "!" - if encryption_key - else _(", please check site_config.json.") - ) + frappe.throw(_("Encryption key is invalid! Please check site_config.json")) def get_encryption_key(): - from frappe.installer import update_site_config - if "encryption_key" not in frappe.local.conf: + from frappe.installer import update_site_config + encryption_key = Fernet.generate_key().decode() update_site_config("encryption_key", encryption_key) frappe.local.conf.encryption_key = encryption_key diff --git a/frappe/utils/password_strength.py b/frappe/utils/password_strength.py index 1f7a171ce9..59c784e5b4 100644 --- a/frappe/utils/password_strength.py +++ b/frappe/utils/password_strength.py @@ -177,9 +177,9 @@ def get_dictionary_match_feedback(match, is_sole_match): word = match.get("token") # Variations of the match like UPPERCASES - if re.match(scoring.START_UPPER, word): + if scoring.START_UPPER.match(word): suggestions.append(_("Capitalization doesn't help very much.")) - elif re.match(scoring.ALL_UPPER, word): + elif scoring.ALL_UPPER.match(word): suggestions.append(_("All-uppercase is almost as easy to guess as all-lowercase.")) # Match contains l33t speak substitutions diff --git a/frappe/utils/pdf.py b/frappe/utils/pdf.py index 952717434c..811a6511fd 100644 --- a/frappe/utils/pdf.py +++ b/frappe/utils/pdf.py @@ -5,10 +5,11 @@ import os import re import subprocess from distutils.version import LooseVersion +from typing import Optional import pdfkit from bs4 import BeautifulSoup -from PyPDF2 import PdfFileReader, PdfFileWriter +from PyPDF2 import PdfReader, PdfWriter import frappe from frappe import _ @@ -23,7 +24,7 @@ PDF_CONTENT_ERRORS = [ ] -def get_pdf(html, options=None, output=None): +def get_pdf(html, options=None, output: Optional[PdfWriter] = None): html = scrub_urls(html) html, options = prepare_options(html, options) @@ -35,11 +36,10 @@ def get_pdf(html, options=None, output=None): try: # Set filename property to false, so no file is actually created - filedata = pdfkit.from_string(html, False, options=options or {}) + filedata = pdfkit.from_string(html, options=options or {}, verbose=True) - # https://pythonhosted.org/PyPDF2/PdfFileReader.html - # create in-memory binary streams from filedata and create a PdfFileReader object - reader = PdfFileReader(io.BytesIO(filedata)) + # create in-memory binary streams from filedata and create a PdfReader object + reader = PdfReader(io.BytesIO(filedata)) except OSError as e: if any([error in str(e) for error in PDF_CONTENT_ERRORS]): if not filedata: @@ -47,8 +47,8 @@ def get_pdf(html, options=None, output=None): frappe.throw(_("PDF generation failed because of broken image links")) # allow pdfs with missing images if file got created - if output: # output is a PdfFileWriter object - output.appendPagesFromReader(reader) + if output: + output.append_pages_from_reader(reader) else: raise finally: @@ -58,11 +58,11 @@ def get_pdf(html, options=None, output=None): password = options["password"] if output: - output.appendPagesFromReader(reader) + output.append_pages_from_reader(reader) return output - writer = PdfFileWriter() - writer.appendPagesFromReader(reader) + writer = PdfWriter() + writer.append_pages_from_reader(reader) if "password" in options: writer.encrypt(password) diff --git a/frappe/utils/print_format.py b/frappe/utils/print_format.py index 13989490a5..a48d7ab84f 100644 --- a/frappe/utils/print_format.py +++ b/frappe/utils/print_format.py @@ -1,6 +1,6 @@ import os -from PyPDF2 import PdfFileWriter +from PyPDF2 import PdfWriter import frappe from frappe import _ @@ -12,6 +12,8 @@ no_cache = 1 base_template_path = "www/printview.html" standard_format = "templates/print_formats/standard.html" +from frappe.www.printview import validate_print_permission + @frappe.whitelist() def download_multi_pdf(doctype, name, format=None, no_letterhead=False, options=None): @@ -56,7 +58,7 @@ def download_multi_pdf(doctype, name, format=None, no_letterhead=False, options= import json - output = PdfFileWriter() + output = PdfWriter() if isinstance(options, str): options = json.loads(options) @@ -115,8 +117,11 @@ def read_multi_pdf(output): return filedata -@frappe.whitelist() +@frappe.whitelist(allow_guest=True) def download_pdf(doctype, name, format=None, doc=None, no_letterhead=0): + doc = doc or frappe.get_doc(doctype, name) + validate_print_permission(doc) + html = frappe.get_print(doctype, name, format, doc=doc, no_letterhead=no_letterhead) frappe.local.response.filename = "{name}.pdf".format( name=name.replace(" ", "-").replace("/", "-") @@ -147,7 +152,7 @@ def print_by_server( cups.setServer(print_settings.server_ip) cups.setPort(print_settings.port) conn = cups.Connection() - output = PdfFileWriter() + output = PdfWriter() output = frappe.get_print( doctype, name, print_format, doc=doc, no_letterhead=no_letterhead, as_pdf=True, output=output ) diff --git a/frappe/utils/redis_wrapper.py b/frappe/utils/redis_wrapper.py index 0101355174..06480f0b7b 100644 --- a/frappe/utils/redis_wrapper.py +++ b/frappe/utils/redis_wrapper.py @@ -30,7 +30,7 @@ class RedisWrapper(redis.Redis): return "{0}|{1}".format(frappe.conf.db_name, key).encode("utf-8") - def set_value(self, key, val, user=None, expires_in_sec=None, shared=False): + def set_value(self, key, val, user=None, expires_in_sec=None, shared=False, cache_locally=True): """Sets cache value. :param key: Cache key @@ -40,7 +40,7 @@ class RedisWrapper(redis.Redis): """ key = self.make_key(key, user, shared) - if not expires_in_sec: + if not expires_in_sec and cache_locally: frappe.local.cache[key] = val try: @@ -151,16 +151,15 @@ class RedisWrapper(redis.Redis): def ltrim(self, key, start, stop): return super(RedisWrapper, self).ltrim(self.make_key(key), start, stop) - def hset(self, name, key, value, shared=False): + def hset(self, name: str, key: str, value, shared: bool = False, cache_locally: bool = True): if key is None: return _name = self.make_key(name, shared=shared) # set in local - if _name not in frappe.local.cache: - frappe.local.cache[_name] = {} - frappe.local.cache[_name][key] = value + if cache_locally: + frappe.local.cache.setdefault(_name, {})[key] = value # set in redis try: @@ -168,6 +167,15 @@ class RedisWrapper(redis.Redis): except redis.exceptions.ConnectionError: pass + def hexists(self, name: str, key: str, shared: bool = False) -> bool: + if key is None: + return False + _name = self.make_key(name, shared=shared) + try: + return super(RedisWrapper, self).hexists(_name, key) + except redis.exceptions.ConnectionError: + return False + def hgetall(self, name): value = super(RedisWrapper, self).hgetall(self.make_key(name)) return {key: pickle.loads(value) for key, value in value.items()} diff --git a/frappe/utils/response.py b/frappe/utils/response.py index c537460713..80aeaa2ad0 100644 --- a/frappe/utils/response.py +++ b/frappe/utils/response.py @@ -6,6 +6,7 @@ import decimal import json import mimetypes import os +from typing import TYPE_CHECKING from urllib.parse import quote import werkzeug.utils @@ -22,6 +23,9 @@ from frappe import _ from frappe.core.doctype.access_log.access_log import make_access_log from frappe.utils import cint, format_timedelta +if TYPE_CHECKING: + from frappe.core.doctype.file.file import File + def report_error(status_code): """Build error. Show traceback in developer mode""" @@ -209,28 +213,27 @@ def download_backup(path): return send_private_file(path) -def download_private_file(path): +def download_private_file(path: str) -> Response: """Checks permissions and sends back private file""" - files = frappe.db.get_all("File", {"file_url": path}) can_access = False + files = frappe.get_all("File", filters={"file_url": path}, pluck="name") # this file might be attached to multiple documents # if the file is accessible from any one of those documents # then it should be downloadable - for f in files: - _file = frappe.get_doc("File", f) - can_access = _file.is_downloadable() - if can_access: - make_access_log(doctype="File", document=_file.name, file_type=os.path.splitext(path)[-1][1:]) + for fname in files: + file: "File" = frappe.get_doc("File", fname) + if can_access := file.is_downloadable(): break if not can_access: raise Forbidden(_("You don't have permission to access this file")) + make_access_log(doctype="File", document=file.name, file_type=os.path.splitext(path)[-1][1:]) return send_private_file(path.split("/private", 1)[1]) -def send_private_file(path): +def send_private_file(path: str) -> Response: path = os.path.join(frappe.local.conf.get("private_path", "private"), path.strip("/")) filename = os.path.basename(path) diff --git a/frappe/utils/xlsxutils.py b/frappe/utils/xlsxutils.py index ad02cd8327..1b898f69a2 100644 --- a/frappe/utils/xlsxutils.py +++ b/frappe/utils/xlsxutils.py @@ -40,7 +40,7 @@ def make_xlsx(data, sheet_name, wb=None, column_widths=None): if isinstance(item, str) and next(ILLEGAL_CHARACTERS_RE.finditer(value), None): # Remove illegal characters from the string - value = re.sub(ILLEGAL_CHARACTERS_RE, "", value) + value = ILLEGAL_CHARACTERS_RE.sub("", value) clean_row.append(value) diff --git a/frappe/website/doctype/personal_data_download_request/test_personal_data_download_request.py b/frappe/website/doctype/personal_data_download_request/test_personal_data_download_request.py index 4f115325df..6518cda5ed 100644 --- a/frappe/website/doctype/personal_data_download_request/test_personal_data_download_request.py +++ b/frappe/website/doctype/personal_data_download_request/test_personal_data_download_request.py @@ -2,17 +2,17 @@ # Copyright (c) 2019, Frappe Technologies and Contributors # License: MIT. See LICENSE import json -import unittest import frappe from frappe.contacts.doctype.contact.contact import get_contact_name from frappe.core.doctype.user.user import create_contact +from frappe.tests.utils import FrappeTestCase from frappe.website.doctype.personal_data_download_request.personal_data_download_request import ( get_user_data, ) -class TestRequestPersonalData(unittest.TestCase): +class TestRequestPersonalData(FrappeTestCase): def setUp(self): create_user_if_not_exists(email="test_privacy@example.com") @@ -48,7 +48,7 @@ class TestRequestPersonalData(unittest.TestCase): email_queue = frappe.get_all( "Email Queue", fields=["message"], order_by="creation DESC", limit=1 ) - self.assertTrue("Subject: Download Your Data" in email_queue[0].message) + self.assertIn(frappe._("Download Your Data"), email_queue[0].message) frappe.db.delete("Email Queue") diff --git a/frappe/website/doctype/web_form/templates/web_form.html b/frappe/website/doctype/web_form/templates/web_form.html index 7dd532421d..96072a19ea 100644 --- a/frappe/website/doctype/web_form/templates/web_form.html +++ b/frappe/website/doctype/web_form/templates/web_form.html @@ -22,7 +22,7 @@ data-web-form="{{ name }}" data-web-form-doctype="{{ doc_type }}" data-login-req {% if is_list %}
    -

    {{ _(title) }}

    +

    {{ _(title) }}

    diff --git a/frappe/website/doctype/web_form/web_form.py b/frappe/website/doctype/web_form/web_form.py index 5304fbdfab..e07aa19162 100644 --- a/frappe/website/doctype/web_form/web_form.py +++ b/frappe/website/doctype/web_form/web_form.py @@ -6,7 +6,8 @@ import os import frappe from frappe import _, scrub -from frappe.core.doctype.file.file import get_max_file_size, remove_file_by_url +from frappe.core.api.file import get_max_file_size +from frappe.core.doctype.file import remove_file_by_url from frappe.custom.doctype.customize_form.customize_form import docfield_properties from frappe.desk.form.meta import get_code_files_via_hooks from frappe.integrations.utils import get_payment_gateway_controller diff --git a/frappe/website/doctype/web_page/web_page.py b/frappe/website/doctype/web_page/web_page.py index f74af1d8c7..bd7bcb8de4 100644 --- a/frappe/website/doctype/web_page/web_page.py +++ b/frappe/website/doctype/web_page/web_page.py @@ -19,6 +19,8 @@ from frappe.website.utils import ( ) from frappe.website.website_generator import WebsiteGenerator +H_TAG_PATTERN = re.compile("") + class WebPage(WebsiteGenerator): def validate(self): @@ -114,7 +116,7 @@ class WebPage(WebsiteGenerator): context.header = context.title # add h1 tag to header - if context.get("header") and not re.findall("", context.header): + if context.get("header") and not H_TAG_PATTERN.findall(context.header): context.header = "

    " + context.header + "

    " # if title not set, set title from header diff --git a/frappe/website/doctype/website_settings/website_settings.json b/frappe/website/doctype/website_settings/website_settings.json index b628437315..8c46ef30bf 100644 --- a/frappe/website/doctype/website_settings/website_settings.json +++ b/frappe/website/doctype/website_settings/website_settings.json @@ -21,6 +21,7 @@ "website_theme_image_link", "brand", "banner_image", + "splash_image", "brand_html", "set_banner_from_image", "favicon", @@ -271,7 +272,7 @@ }, { "default": "0", - "description": "To use Google Indexing, enable Google Settings.", + "description": "To use Google Indexing, enable Google Settings.", "fieldname": "enable_google_indexing", "fieldtype": "Check", "label": "Enable Google Indexing" @@ -413,6 +414,11 @@ "fieldname": "footer_powered", "fieldtype": "Small Text", "label": "Footer \"Powered By\"" + }, + { + "fieldname": "splash_image", + "fieldtype": "Attach Image", + "label": "Splash Image" } ], "icon": "fa fa-cog", @@ -420,7 +426,7 @@ "index_web_pages_for_search": 1, "issingle": 1, "links": [], - "modified": "2022-03-09 01:47:31.094462", + "modified": "2022-05-27 12:33:29.019998", "modified_by": "Administrator", "module": "Website", "name": "Website Settings", diff --git a/frappe/website/doctype/website_settings/website_settings.py b/frappe/website/doctype/website_settings/website_settings.py index e8f15290c4..f249778c58 100644 --- a/frappe/website/doctype/website_settings/website_settings.py +++ b/frappe/website/doctype/website_settings/website_settings.py @@ -1,5 +1,6 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors +# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +from typing import Dict, List from urllib.parse import quote import frappe @@ -7,7 +8,6 @@ from frappe import _ from frappe.integrations.doctype.google_settings.google_settings import get_auth_url from frappe.model.document import Document from frappe.utils import encode, get_request_site_address -from frappe.website.doctype.website_theme.website_theme import add_website_theme INDEXING_SCOPES = "https://www.googleapis.com/auth/indexing" @@ -124,7 +124,9 @@ class WebsiteSettings(Document): def get_website_settings(context=None): hooks = frappe.get_hooks() - context = context or frappe._dict() + context = frappe._dict(context or {}) + settings: "WebsiteSettings" = frappe.get_single("Website Settings") + context = context.update( { "top_bar_items": get_items("top_bar_items"), @@ -136,7 +138,6 @@ def get_website_settings(context=None): } ) - settings = frappe.get_single("Website Settings") for k in [ "banner_html", "banner_image", @@ -161,11 +162,8 @@ def get_website_settings(context=None): "show_language_picker", "footer_powered", ]: - if hasattr(settings, k): - context[k] = settings.get(k) - - if settings.address: - context["footer_address"] = settings.address + if setting_value := settings.get(k): + context[k] = setting_value for k in [ "facebook_share", @@ -176,6 +174,9 @@ def get_website_settings(context=None): ]: context[k] = int(context.get(k) or 0) + if settings.address: + context["footer_address"] = settings.address + if frappe.request: context.url = quote(str(get_request_site_address(full_address=True)), safe="/:") @@ -185,7 +186,7 @@ def get_website_settings(context=None): context.web_include_css = hooks.web_include_css or [] - via_hooks = frappe.get_hooks("website_context") + via_hooks = hooks.website_context or [] for key in via_hooks: context[key] = via_hooks[key] if key not in ("top_bar_items", "footer_items", "post_login") and isinstance( @@ -193,7 +194,13 @@ def get_website_settings(context=None): ): context[key] = context[key][-1] - add_website_theme(context) + if context.disable_website_theme: + context.theme = frappe._dict() + + else: + from frappe.website.doctype.website_theme.website_theme import get_active_theme + + context.theme = get_active_theme() or frappe._dict() if not context.get("favicon"): context["favicon"] = "/assets/frappe/images/frappe-favicon.svg" @@ -203,30 +210,37 @@ def get_website_settings(context=None): context["hide_login"] = settings.hide_login + if settings.splash_image: + context["splash_image"] = settings.splash_image + return context -def get_items(parentfield): - all_top_items = frappe.db.sql( - """\ - select * from `tabTop Bar Item` - where parent='Website Settings' and parentfield= %s - order by idx asc""", - parentfield, - as_dict=1, +def get_items(parentfield: str) -> List[Dict]: + _items = frappe.get_all( + "Top Bar Item", + filters={"parent": "Website Settings", "parentfield": parentfield}, + order_by="idx asc", + fields="*", ) - - top_items = all_top_items[:] + top_items = _items.copy() # attach child items to top bar - for d in all_top_items: - if d["parent_label"]: - for t in top_items: - if t["label"] == d["parent_label"]: - if not "child_items" in t: - t["child_items"] = [] - t["child_items"].append(d) - break + for item in _items: + if not item["parent_label"]: + continue + + for top_bar_item in top_items: + if top_bar_item["label"] != item["parent_label"]: + continue + + if "child_items" not in top_bar_item: + top_bar_item["child_items"] = [] + + top_bar_item["child_items"].append(item) + + break + return top_items diff --git a/frappe/website/doctype/website_theme/website_theme.py b/frappe/website/doctype/website_theme/website_theme.py index c833430534..9cb7095f40 100644 --- a/frappe/website/doctype/website_theme/website_theme.py +++ b/frappe/website/doctype/website_theme/website_theme.py @@ -5,6 +5,7 @@ from os.path import abspath from os.path import exists as path_exists from os.path import join as join_path from os.path import splitext +from typing import Optional import frappe from frappe import _ @@ -131,17 +132,8 @@ class WebsiteTheme(Document): return out -def add_website_theme(context): - context.theme = frappe._dict() - - if not context.disable_website_theme: - website_theme = get_active_theme() - context.theme = website_theme or frappe._dict() - - -def get_active_theme(): - website_theme = frappe.db.get_single_value("Website Settings", "website_theme") - if website_theme: +def get_active_theme() -> Optional["WebsiteTheme"]: + if website_theme := frappe.db.get_single_value("Website Settings", "website_theme"): try: return frappe.get_doc("Website Theme", website_theme) except frappe.DoesNotExistError: diff --git a/frappe/website/page_renderers/error_page.py b/frappe/website/page_renderers/error_page.py index 613809bfdc..6a3925967c 100644 --- a/frappe/website/page_renderers/error_page.py +++ b/frappe/website/page_renderers/error_page.py @@ -5,7 +5,13 @@ class ErrorPage(TemplatePage): def __init__(self, path=None, http_status_code=None, exception=None): path = "error" super().__init__(path=path, http_status_code=http_status_code) - self.http_status_code = getattr(exception, "http_status_code", None) or http_status_code or 500 + self.exception = exception def can_render(self): return True + + def init_context(self): + super().init_context() + self.context.http_status_code = getattr(self.exception, "http_status_code", None) or 500 + self.context.error_title = getattr(self.exception, "title", None) + self.context.error_message = getattr(self.exception, "message", None) diff --git a/frappe/website/page_renderers/template_page.py b/frappe/website/page_renderers/template_page.py index 2ed8a62119..83f68d3716 100644 --- a/frappe/website/page_renderers/template_page.py +++ b/frappe/website/page_renderers/template_page.py @@ -212,19 +212,13 @@ class TemplatePage(BaseTemplatePage): def run_pymodule_method(self, method_name): if hasattr(self.pymodule, method_name): - try: - import inspect + import inspect - method = getattr(self.pymodule, method_name) - if inspect.getfullargspec(method).args: - return method(self.context) - else: - return method() - except (frappe.PermissionError, frappe.DoesNotExistError, frappe.Redirect): - raise - except Exception: - if not frappe.flags.in_migrate: - frappe.errprint(frappe.utils.get_traceback()) + method = getattr(self.pymodule, method_name) + if inspect.getfullargspec(method).args: + return method(self.context) + else: + return method() def render_template(self): if self.template_path.endswith("min.js"): diff --git a/frappe/website/router.py b/frappe/website/router.py index e9f0d0f09c..8c21501a4e 100644 --- a/frappe/website/router.py +++ b/frappe/website/router.py @@ -8,7 +8,7 @@ import re from werkzeug.routing import Map, NotFound, Rule import frappe -from frappe.website.utils import extract_title +from frappe.website.utils import extract_title, get_frontmatter def get_page_info_from_web_page_with_dynamic_routes(path): @@ -161,26 +161,6 @@ def get_page_info(path, app, start, basepath=None, app_path=None, fname=None): return page_info -def get_frontmatter(string): - """ - Reference: https://github.com/jonbeebe/frontmatter - """ - import yaml - - fmatter = "" - body = "" - result = re.compile(r"^\s*(?:---|\+\+\+)(.*?)(?:---|\+\+\+)\s*(.+)$", re.S | re.M).search(string) - - if result: - fmatter = result.group(1) - body = result.group(2) - - return { - "attributes": yaml.safe_load(fmatter), - "body": body, - } - - def setup_source(page_info): """Get the HTML source of the template""" jenv = frappe.get_jenv() diff --git a/frappe/website/utils.py b/frappe/website/utils.py index f673a20656..6e34c05d40 100644 --- a/frappe/website/utils.py +++ b/frappe/website/utils.py @@ -15,6 +15,13 @@ from frappe import _ from frappe.model.document import Document from frappe.utils import md_to_html +FRONTMATTER_PATTERN = re.compile(r"^\s*(?:---|\+\+\+)(.*?)(?:---|\+\+\+)\s*(.+)$", re.S | re.M) +H1_TAG_PATTERN = re.compile("

    ([^<]*)") +IMAGE_TAG_PATTERN = re.compile(r"""]*src\s?=\s?['"]([^'"]*)['"]""") +CLEANUP_PATTERN_1 = re.compile(r'[~!@#$%^&*+()<>,."\'\?]') +CLEANUP_PATTERN_2 = re.compile("[:/]") +CLEANUP_PATTERN_3 = re.compile(r"(-)\1+") + def delete_page_cache(path): cache = frappe.cache() @@ -29,7 +36,7 @@ def delete_page_cache(path): def find_first_image(html): - m = re.finditer(r"""]*src\s?=\s?['"]([^'"]*)['"]""", html) + m = IMAGE_TAG_PATTERN.finditer(html) try: return next(m).groups()[0] except StopIteration: @@ -156,17 +163,17 @@ def is_signup_disabled(): return frappe.db.get_single_value("Website Settings", "disable_signup", True) -def cleanup_page_name(title): +def cleanup_page_name(title: str) -> str: """make page name from title""" if not title: return "" name = title.lower() - name = re.sub(r'[~!@#$%^&*+()<>,."\'\?]', "", name) - name = re.sub("[:/]", "-", name) + name = CLEANUP_PATTERN_1.sub("", name) + name = CLEANUP_PATTERN_2.sub("-", name) name = "-".join(name.split()) # replace repeating hyphens - name = re.sub(r"(-)\1+", r"\1", name) + name = CLEANUP_PATTERN_3.sub(r"\1", name) return name[:140] @@ -287,8 +294,8 @@ def extract_title(source, path): if not title and "

    " in source: # extract title from h1 - match = re.findall("

    ([^<]*)", source) - title_content = match[0].strip()[:300] + match = H1_TAG_PATTERN.search(source).group() + title_content = match.strip()[:300] if "{{" not in title_content: title = title_content @@ -308,17 +315,14 @@ def extract_title(source, path): return title -def extract_comment_tag(source, tag): +def extract_comment_tag(source: str, tag: str): """Extract custom tags in comments from source. :param source: raw template source in HTML :param title: tag to search, example "title" """ - - if "".format(tag), source)[0].strip() - else: - return None + matched_pattern = re.search(f"", source) + return matched_pattern.groups()[0].strip() if matched_pattern else None def get_html_content_based_on_type(doc, fieldname, content_type): @@ -378,7 +382,8 @@ def get_frontmatter(string): "Reference: https://github.com/jonbeebe/frontmatter" frontmatter = "" body = "" - result = re.compile(r"^\s*(?:---|\+\+\+)(.*?)(?:---|\+\+\+)\s*(.+)$", re.S | re.M).search(string) + result = FRONTMATTER_PATTERN.search(string) + if result: frontmatter = result.group(1) body = result.group(2) diff --git a/frappe/website/web_template/hero/hero.html b/frappe/website/web_template/hero/hero.html index 4f2f8d89ed..454a3a8f98 100644 --- a/frappe/website/web_template/hero/hero.html +++ b/frappe/website/web_template/hero/hero.html @@ -1,21 +1,21 @@
    -

    {{ title }}

    +

    {{ _(title) }}

    {%- if subtitle -%}

    - {{ subtitle }} + {{ _(subtitle) }}

    {%- endif -%} {%- if primary_action or secondary_action -%}
    {%- if primary_action -%} - {{ primary_action_label }} + {{ _(primary_action_label) }} {%- endif -%} {%- if secondary_action -%} - {{ secondary_action_label }} + {{ _(secondary_action_label) }} {%- endif -%}
    diff --git a/frappe/website/web_template/hero_with_right_image/hero_with_right_image.html b/frappe/website/web_template/hero_with_right_image/hero_with_right_image.html index baee262451..69a9fff702 100644 --- a/frappe/website/web_template/hero_with_right_image/hero_with_right_image.html +++ b/frappe/website/web_template/hero_with_right_image/hero_with_right_image.html @@ -2,22 +2,22 @@

    - {{ title }} + {{ _(title) }}

    {%- if subtitle -%}

    - {{ subtitle }} + {{ _(subtitle) }}

    {%- endif -%} diff --git a/frappe/website/web_template/markdown/markdown.html b/frappe/website/web_template/markdown/markdown.html index 049e7da83f..b27114ff88 100644 --- a/frappe/website/web_template/markdown/markdown.html +++ b/frappe/website/web_template/markdown/markdown.html @@ -1,5 +1,5 @@
    - {{ frappe.utils.md_to_html(content) }} + {{ frappe.utils.md_to_html(_(content)) }}
    diff --git a/frappe/website/web_template/section_with_cards/section_with_cards.html b/frappe/website/web_template/section_with_cards/section_with_cards.html index d0eb164288..b5577795a8 100644 --- a/frappe/website/web_template/section_with_cards/section_with_cards.html +++ b/frappe/website/web_template/section_with_cards/section_with_cards.html @@ -18,7 +18,7 @@ }) }} {%- endif -%}
    -

    {{ title or '' }}

    +

    {{ title or '' }}

    {{ content or '' }}

    diff --git a/frappe/website/web_template/section_with_collapsible_content/section_with_collapsible_content.html b/frappe/website/web_template/section_with_collapsible_content/section_with_collapsible_content.html index 0f0353f115..adb58e4094 100644 --- a/frappe/website/web_template/section_with_collapsible_content/section_with_collapsible_content.html +++ b/frappe/website/web_template/section_with_collapsible_content/section_with_collapsible_content.html @@ -1,7 +1,7 @@
    -

    {{ title }}

    +

    {{ _(title) }}

    {%- if subtitle -%} -

    {{ subtitle }}

    +

    {{ _(subtitle) }}

    {%- endif -%}
    @@ -10,7 +10,7 @@ {%- set collapse_id = 'id-' + frappe.utils.generate_hash('Collapse', 12) -%}

    {{ cta_label }} + {{ _(cta_label) }}

    {%- if cta_description -%}
    - {{ cta_description }} + {{ _(cta_description) }}
    {%- endif -%}
    diff --git a/frappe/website/web_template/section_with_embed/section_with_embed.html b/frappe/website/web_template/section_with_embed/section_with_embed.html index e9205eb100..daf253afb2 100644 --- a/frappe/website/web_template/section_with_embed/section_with_embed.html +++ b/frappe/website/web_template/section_with_embed/section_with_embed.html @@ -1,10 +1,10 @@
    -

    {{ title }}

    +

    {{ _(title) }}

    {%- if subtitle -%} -

    {{ subtitle }}

    +

    {{ _(subtitle) }}

    {%- endif -%}
    - {{ frappe.utils.md_to_html(embed_html) }} + {{ frappe.utils.md_to_html(_(embed_html)) }}
    diff --git a/frappe/website/web_template/section_with_features/section_with_features.html b/frappe/website/web_template/section_with_features/section_with_features.html index b8cc3c2b90..3b21794165 100644 --- a/frappe/website/web_template/section_with_features/section_with_features.html +++ b/frappe/website/web_template/section_with_features/section_with_features.html @@ -1,9 +1,9 @@
    {%- if title -%} -

    {{ title }}

    +

    {{ _(title) }}

    {%- endif -%} {%- if subtitle -%} -

    {{ subtitle }}

    +

    {{ _(subtitle) }}

    {%- endif -%}
    {%- endif -%} {%- if feature.title -%} -

    {{ feature.title }}

    +

    {{ _(feature.title) }}

    {%- endif -%} {%- if feature.content -%} -

    {{ frappe.utils.md_to_html(feature.content) }}

    +

    {{ frappe.utils.md_to_html(_(feature.content)) }}

    {%- endif -%}
    diff --git a/frappe/website/web_template/section_with_image/section_with_image.html b/frappe/website/web_template/section_with_image/section_with_image.html index b485ad118c..61c4010808 100644 --- a/frappe/website/web_template/section_with_image/section_with_image.html +++ b/frappe/website/web_template/section_with_image/section_with_image.html @@ -1,7 +1,7 @@
    -

    {{ title }}

    +

    {{ _(title) }}

    {%- if subtitle -%} -

    {{ subtitle }}

    +

    {{ _(subtitle) }}

    {%- endif -%} {%- if image -%} diff --git a/frappe/website/web_template/section_with_videos/section_with_videos.html b/frappe/website/web_template/section_with_videos/section_with_videos.html index 30bea5dead..304f77993c 100644 --- a/frappe/website/web_template/section_with_videos/section_with_videos.html +++ b/frappe/website/web_template/section_with_videos/section_with_videos.html @@ -13,7 +13,7 @@
    {%- if video.title -%} -

    {{ video.title }}

    +

    {{ video.title }}

    {%- endif -%} {%- if video.content -%}

    {{ video.content }}

    diff --git a/frappe/website/web_template/split_section_with_image/split_section_with_image.html b/frappe/website/web_template/split_section_with_image/split_section_with_image.html index 5ebeef3912..ced68ccf0a 100644 --- a/frappe/website/web_template/split_section_with_image/split_section_with_image.html +++ b/frappe/website/web_template/split_section_with_image/split_section_with_image.html @@ -15,7 +15,7 @@
    {%- endif -%}
    -

    {{ title }}

    +

    {{ title }}

    {%- if content -%}

    {{ content }}

    {%- endif -%} diff --git a/frappe/www/about.py b/frappe/www/about.py index fd2331d9f6..a233bfd311 100644 --- a/frappe/www/about.py +++ b/frappe/www/about.py @@ -7,6 +7,6 @@ sitemap = 1 def get_context(context): - context.doc = frappe.get_doc("About Us Settings", "About Us Settings") + context.doc = frappe.get_cached_doc("About Us Settings") return context diff --git a/frappe/www/app.py b/frappe/www/app.py index ae0dad3326..9a8c80d6b1 100644 --- a/frappe/www/app.py +++ b/frappe/www/app.py @@ -10,6 +10,9 @@ import frappe.sessions from frappe import _ from frappe.utils.jinja_globals import is_rtl +SCRIPT_TAG_PATTERN = re.compile(r"\") +CLOSING_SCRIPT_TAG_PATTERN = re.compile(r"") + def get_context(context): if frappe.session.user == "Guest": @@ -29,15 +32,13 @@ def get_context(context): frappe.db.commit() - desk_theme = frappe.db.get_value("User", frappe.session.user, "desk_theme") - boot_json = frappe.as_json(boot) # remove script tags from boot - boot_json = re.sub(r"\", "", boot_json) + boot_json = SCRIPT_TAG_PATTERN.sub("", boot_json) # TODO: Find better fix - boot_json = re.sub(r"", "", boot_json) + boot_json = CLOSING_SCRIPT_TAG_PATTERN.sub("", boot_json) context.update( { @@ -49,7 +50,7 @@ def get_context(context): "lang": frappe.local.lang, "sounds": hooks["sounds"], "boot": boot if context.get("for_mobile") else boot_json, - "desk_theme": desk_theme or "Light", + "desk_theme": boot.get("desk_theme") or "Light", "csrf_token": csrf_token, "google_analytics_id": frappe.conf.get("google_analytics_id"), "google_analytics_anonymize_ip": frappe.conf.get("google_analytics_anonymize_ip"), diff --git a/frappe/www/error.html b/frappe/www/error.html index d63daec759..142897c35a 100644 --- a/frappe/www/error.html +++ b/frappe/www/error.html @@ -23,15 +23,15 @@
    - {{_("Uncaught Server Exception")}} + {{ error_title }}
    -

    {{_("There was an error building this page")}}

    +

    {{ error_message }}

    - {{ _("Error Code: {0}").format('500') }} + {{ _("Error Code: {0}").format(http_status_code) }}