diff --git a/.eslintrc b/.eslintrc index d123023a68..8a509f0df4 100644 --- a/.eslintrc +++ b/.eslintrc @@ -143,6 +143,7 @@ "Cypress": true, "cy": true, "it": true, + "describe": true, "expect": true, "context": true, "before": true, diff --git a/.github/helper/semgrep_rules/frappe_correctness.py b/.github/helper/semgrep_rules/frappe_correctness.py new file mode 100644 index 0000000000..37889fbbb1 --- /dev/null +++ b/.github/helper/semgrep_rules/frappe_correctness.py @@ -0,0 +1,28 @@ +import frappe +from frappe import _, flt + +from frappe.model.document import Document + + +def on_submit(self): + if self.value_of_goods == 0: + frappe.throw(_('Value of goods cannot be 0')) + # ruleid: frappe-modifying-after-submit + self.status = 'Submitted' + +def on_submit(self): # noqa + if flt(self.per_billed) < 100: + self.update_billing_status() + else: + # todook: frappe-modifying-after-submit + self.status = "Completed" + self.db_set("status", "Completed") + +class TestDoc(Document): + pass + + def validate(self): + #ruleid: frappe-modifying-child-tables-while-iterating + for item in self.child_table: + if item.value < 0: + self.remove(item) diff --git a/.github/helper/semgrep_rules/frappe_correctness.yml b/.github/helper/semgrep_rules/frappe_correctness.yml new file mode 100644 index 0000000000..faab3344a6 --- /dev/null +++ b/.github/helper/semgrep_rules/frappe_correctness.yml @@ -0,0 +1,135 @@ +# This file specifies rules for correctness according to how frappe doctype data model works. + +rules: +- id: frappe-modifying-but-not-comitting + patterns: + - pattern: | + def $METHOD(self, ...): + ... + self.$ATTR = ... + - pattern-not: | + def $METHOD(self, ...): + ... + self.$ATTR = ... + ... + self.db_set(..., self.$ATTR, ...) + - pattern-not: | + def $METHOD(self, ...): + ... + self.$ATTR = $SOME_VAR + ... + self.db_set(..., $SOME_VAR, ...) + - pattern-not: | + def $METHOD(self, ...): + ... + self.$ATTR = $SOME_VAR + ... + self.save() + - metavariable-regex: + metavariable: '$ATTR' + # this is negative look-ahead, add more attrs to ignore like (ignore|ignore_this_too|ignore_me) + regex: '^(?!ignore_linked_doctypes|status_updater)(.*)$' + - metavariable-regex: + metavariable: "$METHOD" + regex: "(on_submit|on_cancel)" + message: | + DocType modified in self.$METHOD. Please check if modification of self.$ATTR is commited to database. + languages: [python] + severity: ERROR + +- id: frappe-modifying-but-not-comitting-other-method + patterns: + - pattern: | + class $DOCTYPE(...): + def $METHOD(self, ...): + ... + self.$ANOTHER_METHOD() + ... + + def $ANOTHER_METHOD(self, ...): + ... + self.$ATTR = ... + - pattern-not: | + class $DOCTYPE(...): + def $METHOD(self, ...): + ... + self.$ANOTHER_METHOD() + ... + + def $ANOTHER_METHOD(self, ...): + ... + self.$ATTR = ... + ... + self.db_set(..., self.$ATTR, ...) + - pattern-not: | + class $DOCTYPE(...): + def $METHOD(self, ...): + ... + self.$ANOTHER_METHOD() + ... + + def $ANOTHER_METHOD(self, ...): + ... + self.$ATTR = $SOME_VAR + ... + self.db_set(..., $SOME_VAR, ...) + - pattern-not: | + class $DOCTYPE(...): + def $METHOD(self, ...): + ... + self.$ANOTHER_METHOD() + ... + self.save() + def $ANOTHER_METHOD(self, ...): + ... + self.$ATTR = ... + - metavariable-regex: + metavariable: "$METHOD" + regex: "(on_submit|on_cancel)" + message: | + self.$ANOTHER_METHOD is called from self.$METHOD, check if changes to self.$ATTR are commited to database. + languages: [python] + severity: ERROR + +- id: frappe-print-function-in-doctypes + pattern: print(...) + message: | + Did you mean to leave this print statement in? Consider using msgprint or logger instead of print statement. + languages: [python] + severity: WARNING + paths: + exclude: + - test_*.py + include: + - "*/**/doctype/*" + +- id: frappe-modifying-child-tables-while-iterating + pattern-either: + - pattern: | + for $ROW in self.$TABLE: + ... + self.remove(...) + - pattern: | + for $ROW in self.$TABLE: + ... + self.append(...) + message: | + Child table being modified while iterating on it. + languages: [python] + severity: ERROR + paths: + include: + - "*/**/doctype/*" + +- id: frappe-same-key-assigned-twice + pattern-either: + - pattern: | + {..., $X: $A, ..., $X: $B, ...} + - pattern: | + dict(..., ($X, $A), ..., ($X, $B), ...) + - pattern: | + _dict(..., ($X, $A), ..., ($X, $B), ...) + message: | + key `$X` is uselessly assigned twice. This could be a potential bug. + languages: [python] + severity: ERROR diff --git a/.github/helper/semgrep_rules/security.yml b/.github/helper/semgrep_rules/security.yml index 1937fc0e52..b2cc4b16fc 100644 --- a/.github/helper/semgrep_rules/security.yml +++ b/.github/helper/semgrep_rules/security.yml @@ -12,3 +12,18 @@ rules: exclude: - frappe/__init__.py - frappe/commands/utils.py + +- id: frappe-sqli-format-strings + patterns: + - pattern-inside: | + @frappe.whitelist() + def $FUNC(...): + ... + - pattern-either: + - pattern: frappe.db.sql("..." % ...) + - pattern: frappe.db.sql(f"...", ...) + - pattern: frappe.db.sql("...".format(...), ...) + message: | + Detected use of raw string formatting for SQL queries. This can lead to sql injection vulnerabilities. Refer security guidelines - https://github.com/frappe/erpnext/wiki/Code-Security-Guidelines + languages: [python] + severity: WARNING diff --git a/.github/helper/semgrep_rules/translate.yml b/.github/helper/semgrep_rules/translate.yml index 3737da5a7e..df55089b9f 100644 --- a/.github/helper/semgrep_rules/translate.yml +++ b/.github/helper/semgrep_rules/translate.yml @@ -44,7 +44,8 @@ rules: pattern-either: - pattern: _(...) + ... + _(...) - pattern: _("..." + "...") - - pattern-regex: '_\([^\)]*\\\s*' + - pattern-regex: '_\([^\)]*\\\s*' # lines broken by `\` + - pattern-regex: '_\(\s*\n' # line breaks allowed by python for using ( ) message: | Do not split strings inside translate function. Do not concatenate using translate functions. Please refer: https://frappeframework.com/docs/user/en/translations diff --git a/.github/helper/semgrep_rules/ux.py b/.github/helper/semgrep_rules/ux.py new file mode 100644 index 0000000000..4a74457435 --- /dev/null +++ b/.github/helper/semgrep_rules/ux.py @@ -0,0 +1,31 @@ +import frappe +from frappe import msgprint, throw, _ + + +# ruleid: frappe-missing-translate-function +throw("Error Occured") + +# ruleid: frappe-missing-translate-function +frappe.throw("Error Occured") + +# ruleid: frappe-missing-translate-function +frappe.msgprint("Useful message") + +# ruleid: frappe-missing-translate-function +msgprint("Useful message") + + +# ok: frappe-missing-translate-function +translatedmessage = _("Hello") + +# ok: frappe-missing-translate-function +throw(translatedmessage) + +# ok: frappe-missing-translate-function +msgprint(translatedmessage) + +# ok: frappe-missing-translate-function +msgprint(_("Helpful message")) + +# ok: frappe-missing-translate-function +frappe.throw(_("Error occured")) diff --git a/.github/helper/semgrep_rules/ux.yml b/.github/helper/semgrep_rules/ux.yml new file mode 100644 index 0000000000..ed06a6a80c --- /dev/null +++ b/.github/helper/semgrep_rules/ux.yml @@ -0,0 +1,15 @@ +rules: +- id: frappe-missing-translate-function + pattern-either: + - patterns: + - pattern: frappe.msgprint("...", ...) + - pattern-not: frappe.msgprint(_("..."), ...) + - pattern-not: frappe.msgprint(__("..."), ...) + - patterns: + - pattern: frappe.throw("...", ...) + - pattern-not: frappe.throw(_("..."), ...) + - pattern-not: frappe.throw(__("..."), ...) + message: | + All user facing text must be wrapped in translate function. Please refer to translation documentation. https://frappeframework.com/docs/user/en/guides/basics/translations + languages: [python, javascript, json] + severity: ERROR diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 363191fd05..d2a00ab05f 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -151,7 +151,8 @@ jobs: cd ${GITHUB_WORKSPACE} pip install coveralls==3.0.1 pip install coverage==5.5 - coveralls --service=github-actions + coveralls --service=github env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_TOKEN }} + COVERALLS_SERVICE_NAME: github diff --git a/.github/workflows/semgrep.yml b/.github/workflows/semgrep.yml index 1d5694f521..5092bf4705 100644 --- a/.github/workflows/semgrep.yml +++ b/.github/workflows/semgrep.yml @@ -14,9 +14,19 @@ jobs: uses: actions/setup-python@v2 with: python-version: 3.8 - - name: Run semgrep + + - name: Setup semgrep run: | python -m pip install -q semgrep git fetch origin $GITHUB_BASE_REF:$GITHUB_BASE_REF -q + + - name: Semgrep errors + run: | files=$(git diff --name-only --diff-filter=d $GITHUB_BASE_REF) - [[ -d .github/helper/semgrep_rules ]] && semgrep --config=.github/helper/semgrep_rules --quiet --error $files + [[ -d .github/helper/semgrep_rules ]] && semgrep --severity ERROR --config=.github/helper/semgrep_rules --quiet --error $files + semgrep --config="r/python.lang.correctness" --quiet --error $files + + - name: Semgrep warnings + run: | + files=$(git diff --name-only --diff-filter=d $GITHUB_BASE_REF) + [[ -d .github/helper/semgrep_rules ]] && semgrep --severity WARNING --severity INFO --config=.github/helper/semgrep_rules --quiet $files diff --git a/README.md b/README.md index b00d291b96..e00bea7857 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,8 @@ Full-stack web application framework that uses Python and MariaDB on the server ### Installation -[Install via Frappe Bench](https://github.com/frappe/bench) +* [Install via Docker](https://github.com/frappe/frappe_docker) +* [Install via Frappe Bench](https://github.com/frappe/bench) ## Contributing diff --git a/frappe/api.py b/frappe/api.py index 6a09b795b0..4a120f228a 100644 --- a/frappe/api.py +++ b/frappe/api.py @@ -1,12 +1,10 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import base64 import binascii import json - -from six.moves.urllib.parse import urlencode, urlparse +from urllib.parse import urlencode, urlparse import frappe import frappe.client @@ -14,6 +12,7 @@ import frappe.handler from frappe import _ from frappe.utils.response import build_response + def handle(): """ Handler for `/api` methods @@ -38,9 +37,6 @@ def handle(): `/api/resource/{doctype}/{name}?run_method={method}` will run a whitelisted controller method """ - - validate_auth() - parts = frappe.request.path[1:].split("/",3) call = doctype = name = None @@ -116,7 +112,7 @@ def handle(): frappe.local.form_dict['fields'] = json.loads(frappe.local.form_dict['fields']) frappe.local.form_dict.setdefault('limit_page_length', 20) frappe.local.response.update({ - "data": frappe.call( + "data": frappe.call( frappe.client.get_list, doctype, **frappe.local.form_dict @@ -140,6 +136,7 @@ def handle(): return build_response("json") + def get_request_form_data(): if frappe.local.form_dict.data is None: data = frappe.safe_decode(frappe.local.request.get_data()) @@ -148,25 +145,18 @@ def get_request_form_data(): return frappe.parse_json(data) + def validate_auth(): - if frappe.get_request_header("Authorization") is None: - return - - VALID_AUTH_PREFIX_TYPES = ['basic', 'bearer', 'token'] - VALID_AUTH_PREFIX_STRING = ", ".join(VALID_AUTH_PREFIX_TYPES).title() - + """ + Authenticate and sets user for the request. + """ authorization_header = frappe.get_request_header("Authorization", str()).split(" ") - authorization_type = authorization_header[0].lower() - if len(authorization_header) == 1: - frappe.throw(_('Invalid Authorization headers, add a token with a prefix from one of the following: {0}.').format(VALID_AUTH_PREFIX_STRING), frappe.InvalidAuthorizationHeader) - - if authorization_type == "bearer": + if len(authorization_header) == 2: validate_oauth(authorization_header) - elif authorization_type in VALID_AUTH_PREFIX_TYPES: validate_auth_via_api_keys(authorization_header) - else: - frappe.throw(_('Invalid Authorization Type {0}, must be one of {1}.').format(authorization_type, VALID_AUTH_PREFIX_STRING), frappe.InvalidAuthorizationPrefix) + + validate_auth_via_hooks() def validate_oauth(authorization_header): @@ -177,8 +167,8 @@ def validate_oauth(authorization_header): authorization_header (list of str): The 'Authorization' header containing the prefix and token """ - from frappe.oauth import get_url_delimiter from frappe.integrations.oauth2 import get_oauth_server + from frappe.oauth import get_url_delimiter form_dict = frappe.local.form_dict token = authorization_header[1] @@ -192,14 +182,13 @@ def validate_oauth(authorization_header): try: required_scopes = frappe.db.get_value("OAuth Bearer Token", token, "scopes").split(get_url_delimiter()) + valid, oauthlib_request = get_oauth_server().verify_request(uri, http_method, body, headers, required_scopes) + if valid: + frappe.set_user(frappe.db.get_value("OAuth Bearer Token", token, "user")) + frappe.local.form_dict = form_dict except AttributeError: - frappe.throw(_("Invalid Bearer token, please provide a valid access token with prefix 'Bearer'."), frappe.InvalidAuthorizationToken) + pass - valid, oauthlib_request = get_oauth_server().verify_request(uri, http_method, body, headers, required_scopes) - - if valid: - frappe.set_user(frappe.db.get_value("OAuth Bearer Token", token, "user")) - frappe.local.form_dict = form_dict def validate_auth_via_api_keys(authorization_header): @@ -222,8 +211,7 @@ def validate_auth_via_api_keys(authorization_header): except binascii.Error: frappe.throw(_("Failed to decode token, please provide a valid base64-encoded token."), frappe.InvalidAuthorizationToken) except (AttributeError, TypeError, ValueError): - frappe.throw(_("Invalid token, please provide a valid token with prefix 'Basic' or 'Token'."), frappe.InvalidAuthorizationToken) - + pass def validate_api_key_secret(api_key, api_secret, frappe_authorization_source=None): @@ -248,3 +236,8 @@ def validate_api_key_secret(api_key, api_secret, frappe_authorization_source=Non if frappe.local.login_manager.user in ('', 'Guest'): frappe.set_user(user) frappe.local.form_dict = form_dict + + +def validate_auth_via_hooks(): + for auth_hook in frappe.get_hooks('auth_hooks', []): + frappe.get_attr(auth_hook)() diff --git a/frappe/app.py b/frappe/app.py index 607479ad52..c9e993a853 100644 --- a/frappe/app.py +++ b/frappe/app.py @@ -56,6 +56,7 @@ def application(request): frappe.recorder.record() frappe.monitor.start() frappe.rate_limiter.apply() + frappe.api.validate_auth() if request.method == "OPTIONS": response = Response() diff --git a/frappe/commands/__init__.py b/frappe/commands/__init__.py index b9ae02e112..61ee62d352 100644 --- a/frappe/commands/__init__.py +++ b/frappe/commands/__init__.py @@ -62,11 +62,24 @@ def popen(command, *args, **kwargs): if env: env = dict(environ, **env) + def set_low_prio(): + import psutil + if psutil.LINUX: + psutil.Process().nice(19) + psutil.Process().ionice(psutil.IOPRIO_CLASS_IDLE) + elif psutil.WINDOWS: + psutil.Process().nice(psutil.IDLE_PRIORITY_CLASS) + psutil.Process().ionice(psutil.IOPRIO_VERYLOW) + else: + psutil.Process().nice(19) + # ionice not supported + proc = subprocess.Popen(command, stdout=None if output else subprocess.PIPE, stderr=None if output else subprocess.PIPE, shell=shell, cwd=cwd, + preexec_fn=set_low_prio, env=env ) diff --git a/frappe/commands/scheduler.py b/frappe/commands/scheduler.py index bd9c9d2cb0..e9638800cd 100755 --- a/frappe/commands/scheduler.py +++ b/frappe/commands/scheduler.py @@ -18,22 +18,33 @@ def _is_scheduler_enabled(): return enable_scheduler -@click.command('trigger-scheduler-event') -@click.argument('event') + +@click.command("trigger-scheduler-event", help="Trigger a scheduler event") +@click.argument("event") @pass_context def trigger_scheduler_event(context, event): - "Trigger a scheduler event" import frappe.utils.scheduler + + exit_code = 0 + for site in context.sites: try: frappe.init(site=site) frappe.connect() - frappe.utils.scheduler.trigger(site, event, now=True) + try: + frappe.get_doc("Scheduled Job Type", {"method": event}).execute() + except frappe.DoesNotExistError: + click.secho(f"Event {event} does not exist!", fg="red") + exit_code = 1 finally: frappe.destroy() + if not context.sites: raise SiteNotSpecifiedError + sys.exit(exit_code) + + @click.command('enable-scheduler') @pass_context def enable_scheduler(context): diff --git a/frappe/email/doctype/newsletter/newsletter.py b/frappe/email/doctype/newsletter/newsletter.py index c792347c09..6412338e96 100755 --- a/frappe/email/doctype/newsletter/newsletter.py +++ b/frappe/email/doctype/newsletter/newsletter.py @@ -24,6 +24,7 @@ class Newsletter(WebsiteGenerator): if self.send_from: validate_email_address(self.send_from, True) + @frappe.whitelist() def test_send(self, doctype="Lead"): self.recipients = frappe.utils.split_emails(self.test_email_id) self.queue_all(test_email=True) diff --git a/frappe/handler.py b/frappe/handler.py index 82c1ea65c6..a38feb90fa 100755 --- a/frappe/handler.py +++ b/frappe/handler.py @@ -9,7 +9,6 @@ import frappe import frappe.utils import frappe.sessions from frappe.utils import cint -from frappe.api import validate_auth from frappe import _, is_whitelisted from frappe.utils.response import build_response from frappe.utils.csvutils import build_csv_response @@ -24,7 +23,7 @@ ALLOWED_MIMETYPES = ('image/png', 'image/jpeg', 'application/pdf', 'application/ def handle(): """handle request""" - validate_auth() + cmd = frappe.local.form_dict.cmd data = None diff --git a/frappe/hooks.py b/frappe/hooks.py index 99339bad55..f587c59b4a 100644 --- a/frappe/hooks.py +++ b/frappe/hooks.py @@ -132,6 +132,16 @@ has_website_permission = { "Address": "frappe.contacts.doctype.address.address.has_website_permission" } +jinja = { + "methods": "frappe.utils.jinja_globals", + "filters": [ + "frappe.utils.data.global_date_format", + "frappe.utils.markdown", + "frappe.website.utils.get_shade", + "frappe.website.utils.abs_url", + ] +} + standard_queries = { "User": "frappe.core.doctype.user.user.user_query" } diff --git a/frappe/patches.txt b/frappe/patches.txt index 516ddb6094..60c3112f4a 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -335,3 +335,4 @@ frappe.patches.v13_0.rename_list_view_setting_to_list_view_settings frappe.patches.v13_0.remove_twilio_settings frappe.patches.v12_0.rename_uploaded_files_with_proper_name frappe.patches.v13_0.queryreport_columns +frappe.patches.v13_0.jinja_hook diff --git a/frappe/patches/v13_0/jinja_hook.py b/frappe/patches/v13_0/jinja_hook.py new file mode 100644 index 0000000000..84ed6e6cff --- /dev/null +++ b/frappe/patches/v13_0/jinja_hook.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors +# MIT License. See license.txt + +from __future__ import unicode_literals +import frappe +from click import secho + +def execute(): + if frappe.get_hooks('jenv'): + print() + secho('WARNING: The hook "jenv" is deprecated. Follow the migration guide to use the new "jinja" hook.', fg='yellow') + secho('https://github.com/frappe/frappe/wiki/Migrating-to-Version-13', fg='yellow') + print() diff --git a/frappe/public/js/frappe/form/controls/base_control.js b/frappe/public/js/frappe/form/controls/base_control.js index d98bc97915..d6c268a28a 100644 --- a/frappe/public/js/frappe/form/controls/base_control.js +++ b/frappe/public/js/frappe/form/controls/base_control.js @@ -159,9 +159,13 @@ frappe.ui.form.Control = class BaseControl { } validate_and_set_in_model(value, e) { var me = this; - if(this.inside_change_event) { + let force_value_set = (this.doc && this.doc.__run_link_triggers); + let is_value_same = (this.get_model_value() === value); + + if (this.inside_change_event || (!force_value_set && is_value_same)) { return Promise.resolve(); } + this.inside_change_event = true; var set = function(value) { me.inside_change_event = false; diff --git a/frappe/public/js/frappe/form/controls/table_multiselect.js b/frappe/public/js/frappe/form/controls/table_multiselect.js index 2dab759eae..6d0064e617 100644 --- a/frappe/public/js/frappe/form/controls/table_multiselect.js +++ b/frappe/public/js/frappe/form/controls/table_multiselect.js @@ -66,6 +66,10 @@ frappe.ui.form.ControlTableMultiSelect = class ControlTableMultiSelect extends f this._rows_list = this.rows.map(row => row[link_field.fieldname]); return this.rows; } + get_model_value() { + let value = this._super(); + return value ? value.filter(d => !d.__islocal) : value; + } validate(value) { const rows = (value || []).slice(); diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js index 33124dcf51..2aca3fa1b3 100644 --- a/frappe/public/js/frappe/form/form.js +++ b/frappe/public/js/frappe/form/form.js @@ -1203,8 +1203,7 @@ frappe.ui.form.Form = class FrappeForm { $.each(grid_field_label_map, function(fname, label) { fname = fname.split("-"); - var df = frappe.meta.get_docfield(fname[0], fname[1], me.doc.name); - if(df) df.label = label; + me.fields_dict[parentfield].grid.update_docfield_property(fname[1], 'label', label); }); } diff --git a/frappe/public/js/frappe/form/grid.js b/frappe/public/js/frappe/form/grid.js index 86feefed7a..4d381c9be7 100644 --- a/frappe/public/js/frappe/form/grid.js +++ b/frappe/public/js/frappe/form/grid.js @@ -387,6 +387,8 @@ export default class Grid { this.wrapper.find('.grid-footer').toggle(false); } + this.wrapper.find('.grid-add-row, .grid-add-multiple-rows').toggle(this.is_editable()); + } truncate_rows() { diff --git a/frappe/public/js/frappe/form/grid_row.js b/frappe/public/js/frappe/form/grid_row.js index 4afa251c27..9a689fabf4 100644 --- a/frappe/public/js/frappe/form/grid_row.js +++ b/frappe/public/js/frappe/form/grid_row.js @@ -557,13 +557,10 @@ export default class GridRow { this.row.toggle(false); // this.form_panel.toggle(true); - if (this.grid.cannot_add_rows || (this.grid.df && this.grid.df.cannot_add_rows)) { - this.wrapper.find('.grid-insert-row-below, .grid-insert-row, .grid-duplicate-row') - .addClass('hidden'); - } else { - this.wrapper.find('.grid-insert-row-below, .grid-insert-row, .grid-duplicate-row') - .removeClass('hidden'); - } + let cannot_add_rows = this.grid.cannot_add_rows || (this.grid.df && this.grid.df.cannot_add_rows); + this.wrapper + .find('.grid-insert-row-below, .grid-insert-row, .grid-duplicate-row, .grid-append-row') + .toggle(!cannot_add_rows); frappe.dom.freeze("", "dark"); if (cur_frm) cur_frm.cur_grid = this; diff --git a/frappe/public/js/frappe/form/grid_row_form.js b/frappe/public/js/frappe/form/grid_row_form.js index 68e4178ae7..f5a4af206f 100644 --- a/frappe/public/js/frappe/form/grid_row_form.js +++ b/frappe/public/js/frappe/form/grid_row_form.js @@ -119,7 +119,7 @@ export default class GridRowForm { }); } toggle_add_delete_button_display($parent) { - $parent.find(".row-actions") + $parent.find(".row-actions, .grid-append-row") .toggle(this.row.grid.is_editable()); } refresh_field(fieldname) { diff --git a/frappe/templates/base.html b/frappe/templates/base.html index 1ce333c986..e9b17eb221 100644 --- a/frappe/templates/base.html +++ b/frappe/templates/base.html @@ -60,7 +60,7 @@ window.is_chat_enabled = {{ chat_enable }}; - + {% include "public/icons/timeless/symbol-defs.svg" %} {%- block banner -%} {% include "templates/includes/banner_extension.html" ignore missing %} diff --git a/frappe/utils/__init__.py b/frappe/utils/__init__.py index efa69d4453..251a095343 100644 --- a/frappe/utils/__init__.py +++ b/frappe/utils/__init__.py @@ -307,14 +307,23 @@ def unesc(s, esc_chars): s = s.replace(esc_str, c) return s -def execute_in_shell(cmd, verbose=0): +def execute_in_shell(cmd, verbose=0, low_priority=False): # using Popen instead of os.system - as recommended by python docs import tempfile from subprocess import Popen with tempfile.TemporaryFile() as stdout: with tempfile.TemporaryFile() as stderr: - p = Popen(cmd, shell=True, stdout=stdout, stderr=stderr) + kwargs = { + "shell": True, + "stdout": stdout, + "stderr": stderr + } + + if low_priority: + kwargs["preexec_fn"] = lambda: os.nice(10) + + p = Popen(cmd, **kwargs) p.wait() stdout.seek(0) diff --git a/frappe/utils/backups.py b/frappe/utils/backups.py index 9a6747a0cf..b21efc5e89 100644 --- a/frappe/utils/backups.py +++ b/frappe/utils/backups.py @@ -315,8 +315,6 @@ class BackupGenerator: print(template.format(_type.title(), info["path"], info["size"])) def backup_files(self): - import subprocess - for folder in ("public", "private"): files_path = frappe.get_site_path(folder, "files") backup_path = ( @@ -327,12 +325,12 @@ class BackupGenerator: cmd_string = "tar cf - {1} | gzip > {0}" else: cmd_string = "tar -cf {0} {1}" - output = subprocess.check_output( - cmd_string.format(backup_path, files_path), shell=True - ) - if self.verbose and output: - print(output.decode("utf8")) + frappe.utils.execute_in_shell( + cmd_string.format(backup_path, files_path), + verbose=self.verbose, + low_priority=True + ) def copy_site_config(self): site_config_backup_path = self.backup_path_conf @@ -436,7 +434,7 @@ class BackupGenerator: if self.verbose: print(command + "\n") - err, out = frappe.utils.execute_in_shell(command) + frappe.utils.execute_in_shell(command, low_priority=True) def send_email(self): """ diff --git a/frappe/utils/boilerplate.py b/frappe/utils/boilerplate.py index e59f579f75..f54482e7f2 100755 --- a/frappe/utils/boilerplate.py +++ b/frappe/utils/boilerplate.py @@ -190,6 +190,15 @@ app_license = "{app_license}" # automatically create page for each record of this doctype # website_generators = ["Web Page"] +# Jinja +# ---------- + +# add methods and filters to jinja environment +# jinja = {{ +# "methods": "{app_name}.utils.jinja_methods", +# "filters": "{app_name}.utils.jinja_filters" +# }} + # Installation # ------------ @@ -303,6 +312,13 @@ user_data_fields = [ }} ] +# Authentication and authorization +# -------------------------------- + +# auth_hooks = [ +# "{app_name}.auth.validate" +# ] + """ desktop_template = """# -*- coding: utf-8 -*- diff --git a/frappe/utils/jinja.py b/frappe/utils/jinja.py index 5d1bf923cd..a77eca4977 100644 --- a/frappe/utils/jinja.py +++ b/frappe/utils/jinja.py @@ -18,16 +18,10 @@ def get_jenv(): set_filters(jenv) jenv.globals.update(get_safe_globals()) - jenv.globals.update(get_jenv_customization('methods')) - jenv.globals.update({ - 'resolve_class': resolve_class, - 'inspect': inspect, - 'web_blocks': web_blocks, - 'web_block': web_block, - 'script': script, - 'style': style, - 'assets_url': assets_url - }) + + methods, filters = get_jinja_hooks() + jenv.globals.update(methods or {}) + jenv.filters.update(filters or {}) frappe.local.jenv = jenv @@ -130,125 +124,49 @@ def get_jloader(): def set_filters(jenv): import frappe - from frappe.utils import global_date_format, cint, cstr, flt, markdown - from frappe.website.utils import get_shade, abs_url + from frappe.utils import cint, cstr, flt - jenv.filters["global_date_format"] = global_date_format - jenv.filters["markdown"] = markdown jenv.filters["json"] = frappe.as_json - jenv.filters["get_shade"] = get_shade jenv.filters["len"] = len jenv.filters["int"] = cint jenv.filters["str"] = cstr jenv.filters["flt"] = flt - jenv.filters["abs_url"] = abs_url if frappe.flags.in_setup_help: return - jenv.filters.update(get_jenv_customization('filters')) - - -def get_jenv_customization(customization_type): - '''Returns a dict with filter/method name as key and definition as value''' +def get_jinja_hooks(): + """Returns a tuple of (methods, filters) each containing a dict of method name and method definition pair.""" import frappe - out = {} if not getattr(frappe.local, "site", None): + return (None, None) + + from types import FunctionType, ModuleType + from inspect import getmembers, isfunction + + def get_obj_dict_from_paths(object_paths): + out = {} + for obj_path in object_paths: + try: + obj = frappe.get_module(obj_path) + except ModuleNotFoundError: + obj = frappe.get_attr(obj_path) + + if isinstance(obj, ModuleType): + functions = getmembers(obj, isfunction) + for function_name, function in functions: + out[function_name] = function + elif isinstance(obj, FunctionType): + function_name = obj.__name__ + out[function_name] = obj return out - values = frappe.get_hooks("jenv", {}).get(customization_type) - if not values: - return out + values = frappe.get_hooks("jinja") + methods, filters = values.get("methods", []), values.get("filters", []) - for value in values: - fn_name, fn_string = value.split(":") - out[fn_name] = frappe.get_attr(fn_string) + method_dict = get_obj_dict_from_paths(methods) + filter_dict = get_obj_dict_from_paths(filters) - return out - - -def resolve_class(classes): - import frappe - - if classes is None: - return '' - - if isinstance(classes, frappe.string_types): - return classes - - if isinstance(classes, (list, tuple)): - return ' '.join([resolve_class(c) for c in classes]).strip() - - if isinstance(classes, dict): - return ' '.join([classname for classname in classes if classes[classname]]).strip() - - return classes - - -def inspect(var, render=True): - context = { "var": var } - if render: - html = "
{{ var | pprint | e }}
" - else: - html = "" - return get_jenv().from_string(html).render(context) - - -def web_block(template, values=None, **kwargs): - options = {"template": template, "values": values} - options.update(kwargs) - return web_blocks([options]) - - -def web_blocks(blocks): - from frappe import throw, _dict - from frappe.website.doctype.web_page.web_page import get_web_blocks_html - - web_blocks = [] - for block in blocks: - if not block.get('template'): - throw('Web Template is not specified') - - doc = _dict({ - 'doctype': 'Web Page Block', - 'web_template': block['template'], - 'web_template_values': block.get('values', {}), - 'add_top_padding': 1, - 'add_bottom_padding': 1, - 'add_container': 1, - 'hide_block': 0, - 'css_class': '' - }) - doc.update(block) - web_blocks.append(doc) - - out = get_web_blocks_html(web_blocks) - - html = out.html - for script in out.scripts: - html += ''.format(script) - - return html - -def script(path): - path = assets_url(path) - if '/public/' in path: - path = path.replace('/public/', '/dist/') - return f'' - -def style(path): - path = assets_url(path) - if '/public/' in path: - path = path.replace('/public/', '/dist/') - if path.endswith(('.scss', '.sass', '.less', '.styl')): - path = path.rsplit('.', 1)[0] + '.css' - return f'' - -def assets_url(path): - if not path.startswith('/'): - path = '/' + path - if not path.startswith('/assets'): - path = '/assets' + path - return path + return method_dict, filter_dict diff --git a/frappe/utils/jinja_globals.py b/frappe/utils/jinja_globals.py new file mode 100644 index 0000000000..dff69bb1f8 --- /dev/null +++ b/frappe/utils/jinja_globals.py @@ -0,0 +1,91 @@ +# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors +# MIT License. See license.txt + +from __future__ import unicode_literals +from frappe.utils.jinja import get_jenv +import frappe + + +def resolve_class(classes): + if classes is None: + return "" + + if isinstance(classes, frappe.string_types): + return classes + + if isinstance(classes, (list, tuple)): + return " ".join([resolve_class(c) for c in classes]).strip() + + if isinstance(classes, dict): + return " ".join([classname for classname in classes if classes[classname]]).strip() + + return classes + + +def inspect(var, render=True): + context = {"var": var} + if render: + html = "
{{ var | pprint | e }}
" + else: + return "" + return get_jenv().from_string(html).render(context) + + +def web_block(template, values=None, **kwargs): + options = {"template": template, "values": values} + options.update(kwargs) + return web_blocks([options]) + + +def web_blocks(blocks): + from frappe import throw, _dict + from frappe.website.doctype.web_page.web_page import get_web_blocks_html + + web_blocks = [] + for block in blocks: + if not block.get("template"): + throw("Web Template is not specified") + + doc = _dict( + { + "doctype": "Web Page Block", + "web_template": block["template"], + "web_template_values": block.get("values", {}), + "add_top_padding": 1, + "add_bottom_padding": 1, + "add_container": 1, + "hide_block": 0, + "css_class": "", + } + ) + doc.update(block) + web_blocks.append(doc) + + out = get_web_blocks_html(web_blocks) + + html = out.html + for script in out.scripts: + html += "".format(script) + + return html + +def script(path): + path = assets_url(path) + if '/public/' in path: + path = path.replace('/public/', '/dist/') + return f'' + +def style(path): + path = assets_url(path) + if '/public/' in path: + path = path.replace('/public/', '/dist/') + if path.endswith(('.scss', '.sass', '.less', '.styl')): + path = path.rsplit('.', 1)[0] + '.css' + return f'' + +def assets_url(path): + if not path.startswith('/'): + path = '/' + path + if not path.startswith('/assets'): + path = '/assets' + path + return path diff --git a/frappe/website/doctype/web_page/web_page.py b/frappe/website/doctype/web_page/web_page.py index 86774c79c4..cce00564ff 100644 --- a/frappe/website/doctype/web_page/web_page.py +++ b/frappe/website/doctype/web_page/web_page.py @@ -242,11 +242,11 @@ def extract_script_and_style_tags(html): styles = [] for script in soup.find_all('script'): - scripts.append(script.text) + scripts.append(script.string) script.extract() for style in soup.find_all('style'): - styles.append(style.text) + styles.append(style.string) style.extract() return str(soup), scripts, styles