From 6df8479525c4e7af1cfd3b0428700c40cad0c9a9 Mon Sep 17 00:00:00 2001 From: Richard Case Date: Thu, 11 Mar 2021 01:18:16 +0000 Subject: [PATCH 01/27] fix: build priority on computers with low memory fixes:frappe/bench#1135 --- frappe/commands/__init__.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/frappe/commands/__init__.py b/frappe/commands/__init__.py index b9ae02e112..61ee62d352 100644 --- a/frappe/commands/__init__.py +++ b/frappe/commands/__init__.py @@ -62,11 +62,24 @@ def popen(command, *args, **kwargs): if env: env = dict(environ, **env) + def set_low_prio(): + import psutil + if psutil.LINUX: + psutil.Process().nice(19) + psutil.Process().ionice(psutil.IOPRIO_CLASS_IDLE) + elif psutil.WINDOWS: + psutil.Process().nice(psutil.IDLE_PRIORITY_CLASS) + psutil.Process().ionice(psutil.IOPRIO_VERYLOW) + else: + psutil.Process().nice(19) + # ionice not supported + proc = subprocess.Popen(command, stdout=None if output else subprocess.PIPE, stderr=None if output else subprocess.PIPE, shell=shell, cwd=cwd, + preexec_fn=set_low_prio, env=env ) From 1fd08d39606d71b9fc2a3be368e3558d65cf8b36 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Sat, 17 Apr 2021 07:31:24 +0530 Subject: [PATCH 02/27] refactor: Jinja hooks - Rename hook from "jenv" to "jinja" - You can now pass the path to the module and all of the methods in that module will be added as methods - You can also pass module path of a method BREAKING CHANGE: Previous use of "jenv" hook won't work anymore --- frappe/hooks.py | 4 ++ frappe/utils/jinja.py | 111 +++++++++------------------------- frappe/utils/jinja_globals.py | 71 ++++++++++++++++++++++ 3 files changed, 102 insertions(+), 84 deletions(-) create mode 100644 frappe/utils/jinja_globals.py diff --git a/frappe/hooks.py b/frappe/hooks.py index 74c538c5df..c47afadf58 100644 --- a/frappe/hooks.py +++ b/frappe/hooks.py @@ -130,6 +130,10 @@ has_website_permission = { "Address": "frappe.contacts.doctype.address.address.has_website_permission" } +jinja = { + "methods": "frappe.utils.jinja_globals" +} + standard_queries = { "User": "frappe.core.doctype.user.user.user_query" } diff --git a/frappe/utils/jinja.py b/frappe/utils/jinja.py index cd74b2a283..1b2ef9f47f 100644 --- a/frappe/utils/jinja.py +++ b/frappe/utils/jinja.py @@ -18,13 +18,10 @@ def get_jenv(): set_filters(jenv) jenv.globals.update(get_safe_globals()) - jenv.globals.update(get_jenv_customization('methods')) - jenv.globals.update({ - 'resolve_class': resolve_class, - 'inspect': inspect, - 'web_blocks': web_blocks, - 'web_block': web_block - }) + + methods, filters = get_jinja_hooks('methods') + jenv.globals.update(methods or {}) + jenv.filters.update(filters or {}) frappe.local.jenv = jenv @@ -143,88 +140,34 @@ def set_filters(jenv): if frappe.flags.in_setup_help: return - jenv.filters.update(get_jenv_customization('filters')) - - -def get_jenv_customization(customization_type): - '''Returns a dict with filter/method name as key and definition as value''' +def get_jinja_hooks(): + """Returns a tuple of (methods, filters) each containing a dict of method name and method definition pair.""" import frappe - out = {} if not getattr(frappe.local, "site", None): + return (None, None) + + from types import FunctionType, ModuleType + from inspect import getmembers, isfunction + + def get_obj_dict_from_paths(object_paths): + out = {} + for obj_path in object_paths: + obj = frappe.get_attr(obj_path) + if isinstance(obj, ModuleType): + functions = getmembers(obj, isfunction) + for function_name, function in functions: + out[function_name] = function + elif isinstance(obj, FunctionType): + function_name = obj.__name__ + out[function_name] = obj return out - values = frappe.get_hooks("jenv", {}).get(customization_type) - if not values: - return out + values = frappe.get_hooks("jinja") + methods, filters = values.get("methods", []), values.get("filters", []) - for value in values: - fn_name, fn_string = value.split(":") - out[fn_name] = frappe.get_attr(fn_string) + method_dict = get_obj_dict_from_paths(methods) + filter_dict = get_obj_dict_from_paths(filters) - return out - - -def resolve_class(classes): - import frappe - - if classes is None: - return '' - - if isinstance(classes, frappe.string_types): - return classes - - if isinstance(classes, (list, tuple)): - return ' '.join([resolve_class(c) for c in classes]).strip() - - if isinstance(classes, dict): - return ' '.join([classname for classname in classes if classes[classname]]).strip() - - return classes - - -def inspect(var, render=True): - context = { "var": var } - if render: - html = "
{{ var | pprint | e }}
" - else: - html = "" - return get_jenv().from_string(html).render(context) - - -def web_block(template, values=None, **kwargs): - options = {"template": template, "values": values} - options.update(kwargs) - return web_blocks([options]) - - -def web_blocks(blocks): - from frappe import throw, _dict - from frappe.website.doctype.web_page.web_page import get_web_blocks_html - - web_blocks = [] - for block in blocks: - if not block.get('template'): - throw('Web Template is not specified') - - doc = _dict({ - 'doctype': 'Web Page Block', - 'web_template': block['template'], - 'web_template_values': block.get('values', {}), - 'add_top_padding': 1, - 'add_bottom_padding': 1, - 'add_container': 1, - 'hide_block': 0, - 'css_class': '' - }) - doc.update(block) - web_blocks.append(doc) - - out = get_web_blocks_html(web_blocks) - - html = out.html - for script in out.scripts: - html += ''.format(script) - - return html + return method_dict, filter_dict diff --git a/frappe/utils/jinja_globals.py b/frappe/utils/jinja_globals.py new file mode 100644 index 0000000000..e63926a109 --- /dev/null +++ b/frappe/utils/jinja_globals.py @@ -0,0 +1,71 @@ +# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors +# MIT License. See license.txt + +from __future__ import unicode_literals +from frappe.utils.jinja import get_jenv +import frappe + + +def resolve_class(classes): + if classes is None: + return "" + + if isinstance(classes, frappe.string_types): + return classes + + if isinstance(classes, (list, tuple)): + return " ".join([resolve_class(c) for c in classes]).strip() + + if isinstance(classes, dict): + return " ".join([classname for classname in classes if classes[classname]]).strip() + + return classes + + +def inspect(var, render=True): + context = {"var": var} + if render: + html = "
{{ var | pprint | e }}
" + else: + return "" + return get_jenv().from_string(html).render(context) + + +def web_block(template, values=None, **kwargs): + options = {"template": template, "values": values} + options.update(kwargs) + return web_blocks([options]) + + +def web_blocks(blocks): + from frappe import throw, _dict + from frappe.website.doctype.web_page.web_page import get_web_blocks_html + + web_blocks = [] + for block in blocks: + if not block.get("template"): + throw("Web Template is not specified") + + doc = _dict( + { + "doctype": "Web Page Block", + "web_template": block["template"], + "web_template_values": block.get("values", {}), + "add_top_padding": 1, + "add_bottom_padding": 1, + "add_container": 1, + "hide_block": 0, + "css_class": "", + } + ) + doc.update(block) + web_blocks.append(doc) + + out = get_web_blocks_html(web_blocks) + + html = out.html + for script in out.scripts: + html += "".format(script) + + return html + From a62ef80cddedec855e592fbe77a4ab7cdede8bad Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Sat, 17 Apr 2021 07:41:04 +0530 Subject: [PATCH 03/27] fix: Add jinja hook boilerplate --- frappe/utils/boilerplate.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/frappe/utils/boilerplate.py b/frappe/utils/boilerplate.py index e59f579f75..80eda0af13 100755 --- a/frappe/utils/boilerplate.py +++ b/frappe/utils/boilerplate.py @@ -190,6 +190,15 @@ app_license = "{app_license}" # automatically create page for each record of this doctype # website_generators = ["Web Page"] +# Jinja +# ---------- + +# add methods and filters to jinja environment +# jinja = {{ +# "methods": "{app_name}.utils.jinja_methods", +# "filters": "{app_name}.utils.jinja_filters" +# }} + # Installation # ------------ From a78fed4ffccbe3faa628eb7d6a25affc9887023e Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Sat, 17 Apr 2021 07:52:04 +0530 Subject: [PATCH 04/27] fix: Move standard filters to jinja hooks --- frappe/hooks.py | 8 +++++++- frappe/utils/jinja.py | 7 +------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/frappe/hooks.py b/frappe/hooks.py index c47afadf58..1c78d47755 100644 --- a/frappe/hooks.py +++ b/frappe/hooks.py @@ -131,7 +131,13 @@ has_website_permission = { } jinja = { - "methods": "frappe.utils.jinja_globals" + "methods": "frappe.utils.jinja_globals", + "filters": [ + "frappe.utils.data.global_date_format", + "frappe.utils.markdown", + "frappe.website.utils.get_shade", + "frappe.website.utils.abs_url", + ] } standard_queries = { diff --git a/frappe/utils/jinja.py b/frappe/utils/jinja.py index 1b2ef9f47f..6d25f4a405 100644 --- a/frappe/utils/jinja.py +++ b/frappe/utils/jinja.py @@ -124,18 +124,13 @@ def get_jloader(): def set_filters(jenv): import frappe - from frappe.utils import global_date_format, cint, cstr, flt, markdown - from frappe.website.utils import get_shade, abs_url + from frappe.utils import cint, cstr, flt - jenv.filters["global_date_format"] = global_date_format - jenv.filters["markdown"] = markdown jenv.filters["json"] = frappe.as_json - jenv.filters["get_shade"] = get_shade jenv.filters["len"] = len jenv.filters["int"] = cint jenv.filters["str"] = cstr jenv.filters["flt"] = flt - jenv.filters["abs_url"] = abs_url if frappe.flags.in_setup_help: return From b32db6e329a006c1e0eedd93ceb53930db1753f5 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Sat, 17 Apr 2021 16:03:25 +0530 Subject: [PATCH 05/27] fix: method call --- frappe/utils/jinja.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/utils/jinja.py b/frappe/utils/jinja.py index 6d25f4a405..42ab267381 100644 --- a/frappe/utils/jinja.py +++ b/frappe/utils/jinja.py @@ -19,7 +19,7 @@ def get_jenv(): jenv.globals.update(get_safe_globals()) - methods, filters = get_jinja_hooks('methods') + methods, filters = get_jinja_hooks() jenv.globals.update(methods or {}) jenv.filters.update(filters or {}) From d309343c6fa43b8a2a28fa615a534647a6c3e069 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Sun, 18 Apr 2021 18:56:45 +0530 Subject: [PATCH 06/27] feat(hooks): auth hooks hook for request authentication --- frappe/api.py | 33 +++++++++++++++------------------ frappe/utils/boilerplate.py | 7 +++++++ 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/frappe/api.py b/frappe/api.py index 6a09b795b0..3a1be2593e 100644 --- a/frappe/api.py +++ b/frappe/api.py @@ -149,24 +149,17 @@ def get_request_form_data(): return frappe.parse_json(data) def validate_auth(): - if frappe.get_request_header("Authorization") is None: - return - VALID_AUTH_PREFIX_TYPES = ['basic', 'bearer', 'token'] VALID_AUTH_PREFIX_STRING = ", ".join(VALID_AUTH_PREFIX_TYPES).title() authorization_header = frappe.get_request_header("Authorization", str()).split(" ") authorization_type = authorization_header[0].lower() - if len(authorization_header) == 1: - frappe.throw(_('Invalid Authorization headers, add a token with a prefix from one of the following: {0}.').format(VALID_AUTH_PREFIX_STRING), frappe.InvalidAuthorizationHeader) - - if authorization_type == "bearer": + if len(authorization_header) == 2: validate_oauth(authorization_header) - elif authorization_type in VALID_AUTH_PREFIX_TYPES: validate_auth_via_api_keys(authorization_header) - else: - frappe.throw(_('Invalid Authorization Type {0}, must be one of {1}.').format(authorization_type, VALID_AUTH_PREFIX_STRING), frappe.InvalidAuthorizationPrefix) + + validate_auth_via_hooks() def validate_oauth(authorization_header): @@ -192,14 +185,13 @@ def validate_oauth(authorization_header): try: required_scopes = frappe.db.get_value("OAuth Bearer Token", token, "scopes").split(get_url_delimiter()) + valid, oauthlib_request = get_oauth_server().verify_request(uri, http_method, body, headers, required_scopes) + + if valid: + frappe.set_user(frappe.db.get_value("OAuth Bearer Token", token, "user")) + frappe.local.form_dict = form_dict except AttributeError: - frappe.throw(_("Invalid Bearer token, please provide a valid access token with prefix 'Bearer'."), frappe.InvalidAuthorizationToken) - - valid, oauthlib_request = get_oauth_server().verify_request(uri, http_method, body, headers, required_scopes) - - if valid: - frappe.set_user(frappe.db.get_value("OAuth Bearer Token", token, "user")) - frappe.local.form_dict = form_dict + pass def validate_auth_via_api_keys(authorization_header): @@ -222,7 +214,7 @@ def validate_auth_via_api_keys(authorization_header): except binascii.Error: frappe.throw(_("Failed to decode token, please provide a valid base64-encoded token."), frappe.InvalidAuthorizationToken) except (AttributeError, TypeError, ValueError): - frappe.throw(_("Invalid token, please provide a valid token with prefix 'Basic' or 'Token'."), frappe.InvalidAuthorizationToken) + pass @@ -248,3 +240,8 @@ def validate_api_key_secret(api_key, api_secret, frappe_authorization_source=Non if frappe.local.login_manager.user in ('', 'Guest'): frappe.set_user(user) frappe.local.form_dict = form_dict + + +def validate_auth_via_hooks(): + for auth_hook in frappe.get_hooks('auth_hooks', []): + frappe.get_attr(auth_hook)() diff --git a/frappe/utils/boilerplate.py b/frappe/utils/boilerplate.py index e59f579f75..ffcb64cff2 100755 --- a/frappe/utils/boilerplate.py +++ b/frappe/utils/boilerplate.py @@ -303,6 +303,13 @@ user_data_fields = [ }} ] +# Authentication and authorization +# -------------------------------- + +# auth_hooks = [ +# "{app_name}.auth.validate" +# ] + """ desktop_template = """# -*- coding: utf-8 -*- From 557235471ed3540f12d5b9ac883e682f9dbf3daa Mon Sep 17 00:00:00 2001 From: Rohan Bansal Date: Mon, 19 Apr 2021 13:39:39 +0530 Subject: [PATCH 07/27] fix: remove unused variables --- frappe/api.py | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/frappe/api.py b/frappe/api.py index 3a1be2593e..59f14b54c8 100644 --- a/frappe/api.py +++ b/frappe/api.py @@ -1,12 +1,10 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import base64 import binascii import json - -from six.moves.urllib.parse import urlencode, urlparse +from urllib.parse import urlencode, urlparse import frappe import frappe.client @@ -14,6 +12,7 @@ import frappe.handler from frappe import _ from frappe.utils.response import build_response + def handle(): """ Handler for `/api` methods @@ -38,7 +37,6 @@ def handle(): `/api/resource/{doctype}/{name}?run_method={method}` will run a whitelisted controller method """ - validate_auth() parts = frappe.request.path[1:].split("/",3) @@ -116,7 +114,7 @@ def handle(): frappe.local.form_dict['fields'] = json.loads(frappe.local.form_dict['fields']) frappe.local.form_dict.setdefault('limit_page_length', 20) frappe.local.response.update({ - "data": frappe.call( + "data": frappe.call( frappe.client.get_list, doctype, **frappe.local.form_dict @@ -140,6 +138,7 @@ def handle(): return build_response("json") + def get_request_form_data(): if frappe.local.form_dict.data is None: data = frappe.safe_decode(frappe.local.request.get_data()) @@ -148,12 +147,9 @@ def get_request_form_data(): return frappe.parse_json(data) -def validate_auth(): - VALID_AUTH_PREFIX_TYPES = ['basic', 'bearer', 'token'] - VALID_AUTH_PREFIX_STRING = ", ".join(VALID_AUTH_PREFIX_TYPES).title() +def validate_auth(): authorization_header = frappe.get_request_header("Authorization", str()).split(" ") - authorization_type = authorization_header[0].lower() if len(authorization_header) == 2: validate_oauth(authorization_header) @@ -170,8 +166,8 @@ def validate_oauth(authorization_header): authorization_header (list of str): The 'Authorization' header containing the prefix and token """ - from frappe.oauth import get_url_delimiter from frappe.integrations.oauth2 import get_oauth_server + from frappe.oauth import get_url_delimiter form_dict = frappe.local.form_dict token = authorization_header[1] @@ -185,14 +181,14 @@ def validate_oauth(authorization_header): try: required_scopes = frappe.db.get_value("OAuth Bearer Token", token, "scopes").split(get_url_delimiter()) - valid, oauthlib_request = get_oauth_server().verify_request(uri, http_method, body, headers, required_scopes) - - if valid: - frappe.set_user(frappe.db.get_value("OAuth Bearer Token", token, "user")) - frappe.local.form_dict = form_dict except AttributeError: pass + valid, oauthlib_request = get_oauth_server().verify_request(uri, http_method, body, headers, required_scopes) + if valid: + frappe.set_user(frappe.db.get_value("OAuth Bearer Token", token, "user")) + frappe.local.form_dict = form_dict + def validate_auth_via_api_keys(authorization_header): """ @@ -217,7 +213,6 @@ def validate_auth_via_api_keys(authorization_header): pass - def validate_api_key_secret(api_key, api_secret, frappe_authorization_source=None): """frappe_authorization_source to provide api key and secret for a doctype apart from User""" doctype = frappe_authorization_source or 'User' From 8c74df6cc3b5bf4b1664b6ba76f38d9fc8d6b19f Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Mon, 19 Apr 2021 15:18:15 +0530 Subject: [PATCH 08/27] fix: duplicate validate_auth calls --- frappe/api.py | 8 ++++---- frappe/handler.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/frappe/api.py b/frappe/api.py index 59f14b54c8..4117c49333 100644 --- a/frappe/api.py +++ b/frappe/api.py @@ -181,13 +181,13 @@ def validate_oauth(authorization_header): try: required_scopes = frappe.db.get_value("OAuth Bearer Token", token, "scopes").split(get_url_delimiter()) + valid, oauthlib_request = get_oauth_server().verify_request(uri, http_method, body, headers, required_scopes) + if valid: + frappe.set_user(frappe.db.get_value("OAuth Bearer Token", token, "user")) + frappe.local.form_dict = form_dict except AttributeError: pass - valid, oauthlib_request = get_oauth_server().verify_request(uri, http_method, body, headers, required_scopes) - if valid: - frappe.set_user(frappe.db.get_value("OAuth Bearer Token", token, "user")) - frappe.local.form_dict = form_dict def validate_auth_via_api_keys(authorization_header): diff --git a/frappe/handler.py b/frappe/handler.py index 82c1ea65c6..1897abe019 100755 --- a/frappe/handler.py +++ b/frappe/handler.py @@ -24,7 +24,7 @@ ALLOWED_MIMETYPES = ('image/png', 'image/jpeg', 'application/pdf', 'application/ def handle(): """handle request""" - validate_auth() + cmd = frappe.local.form_dict.cmd data = None From 1d0e72834d3f67ec84089190a32ddb28b53963e5 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Mon, 19 Apr 2021 15:54:16 +0530 Subject: [PATCH 09/27] fix: remove unused imports --- frappe/api.py | 5 +++-- frappe/app.py | 1 + frappe/handler.py | 1 - 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/frappe/api.py b/frappe/api.py index 4117c49333..4a120f228a 100644 --- a/frappe/api.py +++ b/frappe/api.py @@ -37,8 +37,6 @@ def handle(): `/api/resource/{doctype}/{name}?run_method={method}` will run a whitelisted controller method """ - validate_auth() - parts = frappe.request.path[1:].split("/",3) call = doctype = name = None @@ -149,6 +147,9 @@ def get_request_form_data(): def validate_auth(): + """ + Authenticate and sets user for the request. + """ authorization_header = frappe.get_request_header("Authorization", str()).split(" ") if len(authorization_header) == 2: diff --git a/frappe/app.py b/frappe/app.py index 607479ad52..c9e993a853 100644 --- a/frappe/app.py +++ b/frappe/app.py @@ -56,6 +56,7 @@ def application(request): frappe.recorder.record() frappe.monitor.start() frappe.rate_limiter.apply() + frappe.api.validate_auth() if request.method == "OPTIONS": response = Response() diff --git a/frappe/handler.py b/frappe/handler.py index 1897abe019..a38feb90fa 100755 --- a/frappe/handler.py +++ b/frappe/handler.py @@ -9,7 +9,6 @@ import frappe import frappe.utils import frappe.sessions from frappe.utils import cint -from frappe.api import validate_auth from frappe import _, is_whitelisted from frappe.utils.response import build_response from frappe.utils.csvutils import build_csv_response From 976ebd5b2c2d4188db1bfe38b29f9badd709eb5b Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Thu, 22 Apr 2021 06:03:35 +0530 Subject: [PATCH 10/27] fix: Check if path is a module first then function --- frappe/utils/jinja.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/frappe/utils/jinja.py b/frappe/utils/jinja.py index 42ab267381..a77eca4977 100644 --- a/frappe/utils/jinja.py +++ b/frappe/utils/jinja.py @@ -149,7 +149,11 @@ def get_jinja_hooks(): def get_obj_dict_from_paths(object_paths): out = {} for obj_path in object_paths: - obj = frappe.get_attr(obj_path) + try: + obj = frappe.get_module(obj_path) + except ModuleNotFoundError: + obj = frappe.get_attr(obj_path) + if isinstance(obj, ModuleType): functions = getmembers(obj, isfunction) for function_name, function in functions: From 1a30e11b5f54cb7f13309ae3851c3e7f3394d7ab Mon Sep 17 00:00:00 2001 From: Anand Chitipothu Date: Thu, 22 Apr 2021 09:01:59 +0530 Subject: [PATCH 11/27] fix: Invalid HTML generated by the base template (#12953) Closes #12952 --- frappe/templates/base.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/templates/base.html b/frappe/templates/base.html index 78aa573c99..d59c4b0f2b 100644 --- a/frappe/templates/base.html +++ b/frappe/templates/base.html @@ -60,7 +60,7 @@ window.is_chat_enabled = {{ chat_enable }}; - + {% include "public/icons/timeless/symbol-defs.svg" %} {%- block banner -%} {% include "templates/includes/banner_extension.html" ignore missing %} From edf92d4450b95fab8cce11e809570597c24ec167 Mon Sep 17 00:00:00 2001 From: gavin Date: Thu, 22 Apr 2021 12:41:31 +0530 Subject: [PATCH 12/27] fix(cli): Trigger Scheduler Event (#12955) * Triggers events via Scheduled Job Type's execute method * Exits with code 1 if no event with that name found or process termination * Added feedback if event not found --- frappe/commands/scheduler.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/frappe/commands/scheduler.py b/frappe/commands/scheduler.py index bd9c9d2cb0..e9638800cd 100755 --- a/frappe/commands/scheduler.py +++ b/frappe/commands/scheduler.py @@ -18,22 +18,33 @@ def _is_scheduler_enabled(): return enable_scheduler -@click.command('trigger-scheduler-event') -@click.argument('event') + +@click.command("trigger-scheduler-event", help="Trigger a scheduler event") +@click.argument("event") @pass_context def trigger_scheduler_event(context, event): - "Trigger a scheduler event" import frappe.utils.scheduler + + exit_code = 0 + for site in context.sites: try: frappe.init(site=site) frappe.connect() - frappe.utils.scheduler.trigger(site, event, now=True) + try: + frappe.get_doc("Scheduled Job Type", {"method": event}).execute() + except frappe.DoesNotExistError: + click.secho(f"Event {event} does not exist!", fg="red") + exit_code = 1 finally: frappe.destroy() + if not context.sites: raise SiteNotSpecifiedError + sys.exit(exit_code) + + @click.command('enable-scheduler') @pass_context def enable_scheduler(context): From 165607565ee29985490c6f3f059e0c6f6a05ad61 Mon Sep 17 00:00:00 2001 From: shariquerik Date: Thu, 22 Apr 2021 12:43:30 +0530 Subject: [PATCH 13/27] fix: Hide grid Add Row & Add Multiple buttons when document grid is not editable --- frappe/public/js/frappe/form/grid.js | 2 ++ frappe/public/js/frappe/form/grid_row.js | 4 ++-- frappe/public/js/frappe/form/grid_row_form.js | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/frappe/public/js/frappe/form/grid.js b/frappe/public/js/frappe/form/grid.js index 86feefed7a..30a3597ec7 100644 --- a/frappe/public/js/frappe/form/grid.js +++ b/frappe/public/js/frappe/form/grid.js @@ -387,6 +387,8 @@ export default class Grid { this.wrapper.find('.grid-footer').toggle(false); } + this.wrapper.find('.grid-add-row, .grid-add-multiple-rows').toggle(this.is_editable()) + } truncate_rows() { diff --git a/frappe/public/js/frappe/form/grid_row.js b/frappe/public/js/frappe/form/grid_row.js index 4afa251c27..08267112de 100644 --- a/frappe/public/js/frappe/form/grid_row.js +++ b/frappe/public/js/frappe/form/grid_row.js @@ -558,10 +558,10 @@ export default class GridRow { // this.form_panel.toggle(true); if (this.grid.cannot_add_rows || (this.grid.df && this.grid.df.cannot_add_rows)) { - this.wrapper.find('.grid-insert-row-below, .grid-insert-row, .grid-duplicate-row') + this.wrapper.find('.grid-insert-row-below, .grid-insert-row, .grid-duplicate-row, .grid-append-row') .addClass('hidden'); } else { - this.wrapper.find('.grid-insert-row-below, .grid-insert-row, .grid-duplicate-row') + this.wrapper.find('.grid-insert-row-below, .grid-insert-row, .grid-duplicate-row, .grid-append-row') .removeClass('hidden'); } diff --git a/frappe/public/js/frappe/form/grid_row_form.js b/frappe/public/js/frappe/form/grid_row_form.js index 68e4178ae7..f5a4af206f 100644 --- a/frappe/public/js/frappe/form/grid_row_form.js +++ b/frappe/public/js/frappe/form/grid_row_form.js @@ -119,7 +119,7 @@ export default class GridRowForm { }); } toggle_add_delete_button_display($parent) { - $parent.find(".row-actions") + $parent.find(".row-actions, .grid-append-row") .toggle(this.row.grid.is_editable()); } refresh_field(fieldname) { From 9d3be5160fc21eaefcb655aa5ca38bc51e18b424 Mon Sep 17 00:00:00 2001 From: Sagar Vora Date: Thu, 22 Apr 2021 12:24:12 +0530 Subject: [PATCH 14/27] perf: low priority for backup processes --- frappe/utils/__init__.py | 13 +++++++++++-- frappe/utils/backups.py | 14 ++++++-------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/frappe/utils/__init__.py b/frappe/utils/__init__.py index efa69d4453..251a095343 100644 --- a/frappe/utils/__init__.py +++ b/frappe/utils/__init__.py @@ -307,14 +307,23 @@ def unesc(s, esc_chars): s = s.replace(esc_str, c) return s -def execute_in_shell(cmd, verbose=0): +def execute_in_shell(cmd, verbose=0, low_priority=False): # using Popen instead of os.system - as recommended by python docs import tempfile from subprocess import Popen with tempfile.TemporaryFile() as stdout: with tempfile.TemporaryFile() as stderr: - p = Popen(cmd, shell=True, stdout=stdout, stderr=stderr) + kwargs = { + "shell": True, + "stdout": stdout, + "stderr": stderr + } + + if low_priority: + kwargs["preexec_fn"] = lambda: os.nice(10) + + p = Popen(cmd, **kwargs) p.wait() stdout.seek(0) diff --git a/frappe/utils/backups.py b/frappe/utils/backups.py index 9a6747a0cf..90a6b94ff0 100644 --- a/frappe/utils/backups.py +++ b/frappe/utils/backups.py @@ -315,8 +315,6 @@ class BackupGenerator: print(template.format(_type.title(), info["path"], info["size"])) def backup_files(self): - import subprocess - for folder in ("public", "private"): files_path = frappe.get_site_path(folder, "files") backup_path = ( @@ -327,12 +325,12 @@ class BackupGenerator: cmd_string = "tar cf - {1} | gzip > {0}" else: cmd_string = "tar -cf {0} {1}" - output = subprocess.check_output( - cmd_string.format(backup_path, files_path), shell=True - ) - if self.verbose and output: - print(output.decode("utf8")) + frappe.utils.execute_in_shell( + cmd_string.format(backup_path, files_path), + verbose=self.verbose, + low_priority=True + ) def copy_site_config(self): site_config_backup_path = self.backup_path_conf @@ -436,7 +434,7 @@ class BackupGenerator: if self.verbose: print(command + "\n") - err, out = frappe.utils.execute_in_shell(command) + err, out = frappe.utils.execute_in_shell(command, low_priority=True) def send_email(self): """ From ab3be339fdc6c6aef314f46e6bcb73e0f1cb8a6b Mon Sep 17 00:00:00 2001 From: Sagar Vora Date: Thu, 22 Apr 2021 12:56:21 +0530 Subject: [PATCH 15/27] fix: remove unsused variables --- frappe/utils/backups.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/utils/backups.py b/frappe/utils/backups.py index 90a6b94ff0..b21efc5e89 100644 --- a/frappe/utils/backups.py +++ b/frappe/utils/backups.py @@ -434,7 +434,7 @@ class BackupGenerator: if self.verbose: print(command + "\n") - err, out = frappe.utils.execute_in_shell(command, low_priority=True) + frappe.utils.execute_in_shell(command, low_priority=True) def send_email(self): """ From b4fa6d56df154f3b659a46243b2f73c733d85121 Mon Sep 17 00:00:00 2001 From: shariquerik Date: Thu, 22 Apr 2021 13:34:45 +0530 Subject: [PATCH 16/27] refactor: Using toggle instead of addClass-removeClass --- frappe/public/js/frappe/form/grid.js | 2 +- frappe/public/js/frappe/form/grid_row.js | 11 ++++------- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/frappe/public/js/frappe/form/grid.js b/frappe/public/js/frappe/form/grid.js index 30a3597ec7..4d381c9be7 100644 --- a/frappe/public/js/frappe/form/grid.js +++ b/frappe/public/js/frappe/form/grid.js @@ -387,7 +387,7 @@ export default class Grid { this.wrapper.find('.grid-footer').toggle(false); } - this.wrapper.find('.grid-add-row, .grid-add-multiple-rows').toggle(this.is_editable()) + this.wrapper.find('.grid-add-row, .grid-add-multiple-rows').toggle(this.is_editable()); } diff --git a/frappe/public/js/frappe/form/grid_row.js b/frappe/public/js/frappe/form/grid_row.js index 08267112de..9a689fabf4 100644 --- a/frappe/public/js/frappe/form/grid_row.js +++ b/frappe/public/js/frappe/form/grid_row.js @@ -557,13 +557,10 @@ export default class GridRow { this.row.toggle(false); // this.form_panel.toggle(true); - if (this.grid.cannot_add_rows || (this.grid.df && this.grid.df.cannot_add_rows)) { - this.wrapper.find('.grid-insert-row-below, .grid-insert-row, .grid-duplicate-row, .grid-append-row') - .addClass('hidden'); - } else { - this.wrapper.find('.grid-insert-row-below, .grid-insert-row, .grid-duplicate-row, .grid-append-row') - .removeClass('hidden'); - } + let cannot_add_rows = this.grid.cannot_add_rows || (this.grid.df && this.grid.df.cannot_add_rows); + this.wrapper + .find('.grid-insert-row-below, .grid-insert-row, .grid-duplicate-row, .grid-append-row') + .toggle(!cannot_add_rows); frappe.dom.freeze("", "dark"); if (cur_frm) cur_frm.cur_grid = this; From 162f191b7727bc2b8359aff2fcf5efe849d35e4a Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Thu, 22 Apr 2021 14:21:05 +0530 Subject: [PATCH 17/27] fix(control): Check if same value is set to avoid unnecessary change trigger --- .eslintrc | 1 + frappe/public/js/frappe/form/controls/base_control.js | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.eslintrc b/.eslintrc index d123023a68..8a509f0df4 100644 --- a/.eslintrc +++ b/.eslintrc @@ -143,6 +143,7 @@ "Cypress": true, "cy": true, "it": true, + "describe": true, "expect": true, "context": true, "before": true, diff --git a/frappe/public/js/frappe/form/controls/base_control.js b/frappe/public/js/frappe/form/controls/base_control.js index 9981398b84..b17ce973ec 100644 --- a/frappe/public/js/frappe/form/controls/base_control.js +++ b/frappe/public/js/frappe/form/controls/base_control.js @@ -159,9 +159,10 @@ frappe.ui.form.Control = Class.extend({ }, validate_and_set_in_model: function(value, e) { var me = this; - if(this.inside_change_event) { + if (this.inside_change_event || this.get_model_value() === value) { return Promise.resolve(); } + this.inside_change_event = true; var set = function(value) { me.inside_change_event = false; From d61f5afcdd1b71d9d7bc3c261369626e79fd550c Mon Sep 17 00:00:00 2001 From: Mohammad Hasnain Mohsin Rajan Date: Thu, 22 Apr 2021 15:53:52 +0530 Subject: [PATCH 18/27] ci: Set COVERALLS_SERVICE_NAME as github (#12961) Co-authored-by: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> --- .github/workflows/ci-tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 363191fd05..d2a00ab05f 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -151,7 +151,8 @@ jobs: cd ${GITHUB_WORKSPACE} pip install coveralls==3.0.1 pip install coverage==5.5 - coveralls --service=github-actions + coveralls --service=github env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_TOKEN }} + COVERALLS_SERVICE_NAME: github From 1ef4a58aa8b4720592124b991051363f1a98a2a2 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Thu, 22 Apr 2021 15:59:33 +0530 Subject: [PATCH 19/27] fix: Override get_model_value for table multiselect --- frappe/public/js/frappe/form/controls/table_multiselect.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frappe/public/js/frappe/form/controls/table_multiselect.js b/frappe/public/js/frappe/form/controls/table_multiselect.js index c306146f90..eb3f1bce6e 100644 --- a/frappe/public/js/frappe/form/controls/table_multiselect.js +++ b/frappe/public/js/frappe/form/controls/table_multiselect.js @@ -66,6 +66,10 @@ frappe.ui.form.ControlTableMultiSelect = frappe.ui.form.ControlLink.extend({ this._rows_list = this.rows.map(row => row[link_field.fieldname]); return this.rows; }, + get_model_value() { + let value = this._super(); + return value ? value.filter(d => !d.__islocal) : value; + }, validate(value) { const rows = (value || []).slice(); From aaa165c7840959e54d2905f9e050b5a765211017 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Thu, 22 Apr 2021 17:00:55 +0530 Subject: [PATCH 20/27] fix: Show upgrade message in patch - Will only be shown if the old style "jenv" hook is being used --- frappe/patches.txt | 1 + frappe/patches/v13_0/jinja_hook.py | 13 +++++++++++++ 2 files changed, 14 insertions(+) create mode 100644 frappe/patches/v13_0/jinja_hook.py diff --git a/frappe/patches.txt b/frappe/patches.txt index 516ddb6094..60c3112f4a 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -335,3 +335,4 @@ frappe.patches.v13_0.rename_list_view_setting_to_list_view_settings frappe.patches.v13_0.remove_twilio_settings frappe.patches.v12_0.rename_uploaded_files_with_proper_name frappe.patches.v13_0.queryreport_columns +frappe.patches.v13_0.jinja_hook diff --git a/frappe/patches/v13_0/jinja_hook.py b/frappe/patches/v13_0/jinja_hook.py new file mode 100644 index 0000000000..3e9efc5896 --- /dev/null +++ b/frappe/patches/v13_0/jinja_hook.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors +# MIT License. See license.txt + +from __future__ import unicode_literals +import frappe +from click import secho + +def execute(): + if frappe.get_hooks('jenv') or True: + print() + secho('WARNING: The hook "jenv" is deprecated. Follow the migration guide to use the new "jinja" hook.', fg='yellow') + secho('https://github.com/frappe/frappe/wiki/Migrating-to-Version-13', fg='yellow') + print() From 87f3038272a33954437d50759caa05ec3916df12 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Thu, 22 Apr 2021 17:31:34 +0530 Subject: [PATCH 21/27] fix: remove hardcoding --- frappe/patches/v13_0/jinja_hook.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/patches/v13_0/jinja_hook.py b/frappe/patches/v13_0/jinja_hook.py index 3e9efc5896..84ed6e6cff 100644 --- a/frappe/patches/v13_0/jinja_hook.py +++ b/frappe/patches/v13_0/jinja_hook.py @@ -6,7 +6,7 @@ import frappe from click import secho def execute(): - if frappe.get_hooks('jenv') or True: + if frappe.get_hooks('jenv'): print() secho('WARNING: The hook "jenv" is deprecated. Follow the migration guide to use the new "jinja" hook.', fg='yellow') secho('https://github.com/frappe/frappe/wiki/Migrating-to-Version-13', fg='yellow') From c3b087643b4842d5c74654875f2bc157540b1a8f Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Thu, 22 Apr 2021 23:31:51 +0530 Subject: [PATCH 22/27] fix: Use node.string to extract style and script - node.text stopped working in beautifulsoup 4.9.x --- frappe/website/doctype/web_page/web_page.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/website/doctype/web_page/web_page.py b/frappe/website/doctype/web_page/web_page.py index 86774c79c4..cce00564ff 100644 --- a/frappe/website/doctype/web_page/web_page.py +++ b/frappe/website/doctype/web_page/web_page.py @@ -242,11 +242,11 @@ def extract_script_and_style_tags(html): styles = [] for script in soup.find_all('script'): - scripts.append(script.text) + scripts.append(script.string) script.extract() for style in soup.find_all('style'): - styles.append(style.text) + styles.append(style.string) style.extract() return str(soup), scripts, styles From 5881c5dbecfa3a21f558c6d93985dd139c999b94 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Fri, 23 Apr 2021 09:04:34 +0530 Subject: [PATCH 23/27] ci(semgrep): add more rules, r/python.correctness (#12876) * ci(semgrep): add more rules, r/python.correctness - Added file for defining rules as per frappe data model: frappe_correctness.yml - Add rule for SQLi, with WARNING only for now - Add rule file for UX - WARNING | INFO do not fail the build now * ci(semgrep): on_cancel, on_submit correctness rule * ci(semgrep): split workflow in steps * ci(semgrep): catch line breaks in _() * chore: fix sider issue --- .../semgrep_rules/frappe_correctness.py | 28 ++++ .../semgrep_rules/frappe_correctness.yml | 135 ++++++++++++++++++ .github/helper/semgrep_rules/security.yml | 15 ++ .github/helper/semgrep_rules/translate.yml | 3 +- .github/helper/semgrep_rules/ux.py | 31 ++++ .github/helper/semgrep_rules/ux.yml | 15 ++ .github/workflows/semgrep.yml | 14 +- 7 files changed, 238 insertions(+), 3 deletions(-) create mode 100644 .github/helper/semgrep_rules/frappe_correctness.py create mode 100644 .github/helper/semgrep_rules/frappe_correctness.yml create mode 100644 .github/helper/semgrep_rules/ux.py create mode 100644 .github/helper/semgrep_rules/ux.yml diff --git a/.github/helper/semgrep_rules/frappe_correctness.py b/.github/helper/semgrep_rules/frappe_correctness.py new file mode 100644 index 0000000000..37889fbbb1 --- /dev/null +++ b/.github/helper/semgrep_rules/frappe_correctness.py @@ -0,0 +1,28 @@ +import frappe +from frappe import _, flt + +from frappe.model.document import Document + + +def on_submit(self): + if self.value_of_goods == 0: + frappe.throw(_('Value of goods cannot be 0')) + # ruleid: frappe-modifying-after-submit + self.status = 'Submitted' + +def on_submit(self): # noqa + if flt(self.per_billed) < 100: + self.update_billing_status() + else: + # todook: frappe-modifying-after-submit + self.status = "Completed" + self.db_set("status", "Completed") + +class TestDoc(Document): + pass + + def validate(self): + #ruleid: frappe-modifying-child-tables-while-iterating + for item in self.child_table: + if item.value < 0: + self.remove(item) diff --git a/.github/helper/semgrep_rules/frappe_correctness.yml b/.github/helper/semgrep_rules/frappe_correctness.yml new file mode 100644 index 0000000000..faab3344a6 --- /dev/null +++ b/.github/helper/semgrep_rules/frappe_correctness.yml @@ -0,0 +1,135 @@ +# This file specifies rules for correctness according to how frappe doctype data model works. + +rules: +- id: frappe-modifying-but-not-comitting + patterns: + - pattern: | + def $METHOD(self, ...): + ... + self.$ATTR = ... + - pattern-not: | + def $METHOD(self, ...): + ... + self.$ATTR = ... + ... + self.db_set(..., self.$ATTR, ...) + - pattern-not: | + def $METHOD(self, ...): + ... + self.$ATTR = $SOME_VAR + ... + self.db_set(..., $SOME_VAR, ...) + - pattern-not: | + def $METHOD(self, ...): + ... + self.$ATTR = $SOME_VAR + ... + self.save() + - metavariable-regex: + metavariable: '$ATTR' + # this is negative look-ahead, add more attrs to ignore like (ignore|ignore_this_too|ignore_me) + regex: '^(?!ignore_linked_doctypes|status_updater)(.*)$' + - metavariable-regex: + metavariable: "$METHOD" + regex: "(on_submit|on_cancel)" + message: | + DocType modified in self.$METHOD. Please check if modification of self.$ATTR is commited to database. + languages: [python] + severity: ERROR + +- id: frappe-modifying-but-not-comitting-other-method + patterns: + - pattern: | + class $DOCTYPE(...): + def $METHOD(self, ...): + ... + self.$ANOTHER_METHOD() + ... + + def $ANOTHER_METHOD(self, ...): + ... + self.$ATTR = ... + - pattern-not: | + class $DOCTYPE(...): + def $METHOD(self, ...): + ... + self.$ANOTHER_METHOD() + ... + + def $ANOTHER_METHOD(self, ...): + ... + self.$ATTR = ... + ... + self.db_set(..., self.$ATTR, ...) + - pattern-not: | + class $DOCTYPE(...): + def $METHOD(self, ...): + ... + self.$ANOTHER_METHOD() + ... + + def $ANOTHER_METHOD(self, ...): + ... + self.$ATTR = $SOME_VAR + ... + self.db_set(..., $SOME_VAR, ...) + - pattern-not: | + class $DOCTYPE(...): + def $METHOD(self, ...): + ... + self.$ANOTHER_METHOD() + ... + self.save() + def $ANOTHER_METHOD(self, ...): + ... + self.$ATTR = ... + - metavariable-regex: + metavariable: "$METHOD" + regex: "(on_submit|on_cancel)" + message: | + self.$ANOTHER_METHOD is called from self.$METHOD, check if changes to self.$ATTR are commited to database. + languages: [python] + severity: ERROR + +- id: frappe-print-function-in-doctypes + pattern: print(...) + message: | + Did you mean to leave this print statement in? Consider using msgprint or logger instead of print statement. + languages: [python] + severity: WARNING + paths: + exclude: + - test_*.py + include: + - "*/**/doctype/*" + +- id: frappe-modifying-child-tables-while-iterating + pattern-either: + - pattern: | + for $ROW in self.$TABLE: + ... + self.remove(...) + - pattern: | + for $ROW in self.$TABLE: + ... + self.append(...) + message: | + Child table being modified while iterating on it. + languages: [python] + severity: ERROR + paths: + include: + - "*/**/doctype/*" + +- id: frappe-same-key-assigned-twice + pattern-either: + - pattern: | + {..., $X: $A, ..., $X: $B, ...} + - pattern: | + dict(..., ($X, $A), ..., ($X, $B), ...) + - pattern: | + _dict(..., ($X, $A), ..., ($X, $B), ...) + message: | + key `$X` is uselessly assigned twice. This could be a potential bug. + languages: [python] + severity: ERROR diff --git a/.github/helper/semgrep_rules/security.yml b/.github/helper/semgrep_rules/security.yml index 1937fc0e52..b2cc4b16fc 100644 --- a/.github/helper/semgrep_rules/security.yml +++ b/.github/helper/semgrep_rules/security.yml @@ -12,3 +12,18 @@ rules: exclude: - frappe/__init__.py - frappe/commands/utils.py + +- id: frappe-sqli-format-strings + patterns: + - pattern-inside: | + @frappe.whitelist() + def $FUNC(...): + ... + - pattern-either: + - pattern: frappe.db.sql("..." % ...) + - pattern: frappe.db.sql(f"...", ...) + - pattern: frappe.db.sql("...".format(...), ...) + message: | + Detected use of raw string formatting for SQL queries. This can lead to sql injection vulnerabilities. Refer security guidelines - https://github.com/frappe/erpnext/wiki/Code-Security-Guidelines + languages: [python] + severity: WARNING diff --git a/.github/helper/semgrep_rules/translate.yml b/.github/helper/semgrep_rules/translate.yml index 3737da5a7e..df55089b9f 100644 --- a/.github/helper/semgrep_rules/translate.yml +++ b/.github/helper/semgrep_rules/translate.yml @@ -44,7 +44,8 @@ rules: pattern-either: - pattern: _(...) + ... + _(...) - pattern: _("..." + "...") - - pattern-regex: '_\([^\)]*\\\s*' + - pattern-regex: '_\([^\)]*\\\s*' # lines broken by `\` + - pattern-regex: '_\(\s*\n' # line breaks allowed by python for using ( ) message: | Do not split strings inside translate function. Do not concatenate using translate functions. Please refer: https://frappeframework.com/docs/user/en/translations diff --git a/.github/helper/semgrep_rules/ux.py b/.github/helper/semgrep_rules/ux.py new file mode 100644 index 0000000000..4a74457435 --- /dev/null +++ b/.github/helper/semgrep_rules/ux.py @@ -0,0 +1,31 @@ +import frappe +from frappe import msgprint, throw, _ + + +# ruleid: frappe-missing-translate-function +throw("Error Occured") + +# ruleid: frappe-missing-translate-function +frappe.throw("Error Occured") + +# ruleid: frappe-missing-translate-function +frappe.msgprint("Useful message") + +# ruleid: frappe-missing-translate-function +msgprint("Useful message") + + +# ok: frappe-missing-translate-function +translatedmessage = _("Hello") + +# ok: frappe-missing-translate-function +throw(translatedmessage) + +# ok: frappe-missing-translate-function +msgprint(translatedmessage) + +# ok: frappe-missing-translate-function +msgprint(_("Helpful message")) + +# ok: frappe-missing-translate-function +frappe.throw(_("Error occured")) diff --git a/.github/helper/semgrep_rules/ux.yml b/.github/helper/semgrep_rules/ux.yml new file mode 100644 index 0000000000..ed06a6a80c --- /dev/null +++ b/.github/helper/semgrep_rules/ux.yml @@ -0,0 +1,15 @@ +rules: +- id: frappe-missing-translate-function + pattern-either: + - patterns: + - pattern: frappe.msgprint("...", ...) + - pattern-not: frappe.msgprint(_("..."), ...) + - pattern-not: frappe.msgprint(__("..."), ...) + - patterns: + - pattern: frappe.throw("...", ...) + - pattern-not: frappe.throw(_("..."), ...) + - pattern-not: frappe.throw(__("..."), ...) + message: | + All user facing text must be wrapped in translate function. Please refer to translation documentation. https://frappeframework.com/docs/user/en/guides/basics/translations + languages: [python, javascript, json] + severity: ERROR diff --git a/.github/workflows/semgrep.yml b/.github/workflows/semgrep.yml index 1d5694f521..5092bf4705 100644 --- a/.github/workflows/semgrep.yml +++ b/.github/workflows/semgrep.yml @@ -14,9 +14,19 @@ jobs: uses: actions/setup-python@v2 with: python-version: 3.8 - - name: Run semgrep + + - name: Setup semgrep run: | python -m pip install -q semgrep git fetch origin $GITHUB_BASE_REF:$GITHUB_BASE_REF -q + + - name: Semgrep errors + run: | files=$(git diff --name-only --diff-filter=d $GITHUB_BASE_REF) - [[ -d .github/helper/semgrep_rules ]] && semgrep --config=.github/helper/semgrep_rules --quiet --error $files + [[ -d .github/helper/semgrep_rules ]] && semgrep --severity ERROR --config=.github/helper/semgrep_rules --quiet --error $files + semgrep --config="r/python.lang.correctness" --quiet --error $files + + - name: Semgrep warnings + run: | + files=$(git diff --name-only --diff-filter=d $GITHUB_BASE_REF) + [[ -d .github/helper/semgrep_rules ]] && semgrep --severity WARNING --severity INFO --config=.github/helper/semgrep_rules --quiet $files From f911b2d9ce82d1fc8ef41c3c157af662145b1a77 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Fri, 23 Apr 2021 14:40:47 +0530 Subject: [PATCH 24/27] docs: add docker repo link in README --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b00d291b96..e00bea7857 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,8 @@ Full-stack web application framework that uses Python and MariaDB on the server ### Installation -[Install via Frappe Bench](https://github.com/frappe/bench) +* [Install via Docker](https://github.com/frappe/frappe_docker) +* [Install via Frappe Bench](https://github.com/frappe/bench) ## Contributing From 669fead7991bb57a6c7996276850e62f4d56d7fe Mon Sep 17 00:00:00 2001 From: rohitwaghchaure Date: Fri, 23 Apr 2021 20:03:41 +0530 Subject: [PATCH 25/27] fix: Default values were not triggering change event (#12975) --- frappe/public/js/frappe/form/controls/base_control.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/form/controls/base_control.js b/frappe/public/js/frappe/form/controls/base_control.js index b17ce973ec..8c2c5c4338 100644 --- a/frappe/public/js/frappe/form/controls/base_control.js +++ b/frappe/public/js/frappe/form/controls/base_control.js @@ -159,7 +159,10 @@ frappe.ui.form.Control = Class.extend({ }, validate_and_set_in_model: function(value, e) { var me = this; - if (this.inside_change_event || this.get_model_value() === value) { + let force_value_set = (this.doc && this.doc.__run_link_triggers); + let is_value_same = (this.get_model_value() === value); + + if (this.inside_change_event || (!force_value_set && is_value_same)) { return Promise.resolve(); } From 2c7136761eefc976eb280b45d386ecd1d80f0dfc Mon Sep 17 00:00:00 2001 From: Saqib Date: Fri, 23 Apr 2021 20:35:55 +0530 Subject: [PATCH 26/27] fix: Currency labels in grids (#12974) --- frappe/public/js/frappe/form/form.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js index de9331a726..2b7562f836 100644 --- a/frappe/public/js/frappe/form/form.js +++ b/frappe/public/js/frappe/form/form.js @@ -1203,8 +1203,7 @@ frappe.ui.form.Form = class FrappeForm { $.each(grid_field_label_map, function(fname, label) { fname = fname.split("-"); - var df = frappe.meta.get_docfield(fname[0], fname[1], me.doc.name); - if(df) df.label = label; + me.fields_dict[parentfield].grid.update_docfield_property(fname[1], 'label', label); }); } From 796d241bb174f84e98873934668e3042f8139173 Mon Sep 17 00:00:00 2001 From: walstanb Date: Sat, 24 Apr 2021 13:48:40 +0530 Subject: [PATCH 27/27] chore: frappe.whitelist for doc methods --- frappe/email/doctype/newsletter/newsletter.py | 1 + 1 file changed, 1 insertion(+) diff --git a/frappe/email/doctype/newsletter/newsletter.py b/frappe/email/doctype/newsletter/newsletter.py index c792347c09..6412338e96 100755 --- a/frappe/email/doctype/newsletter/newsletter.py +++ b/frappe/email/doctype/newsletter/newsletter.py @@ -24,6 +24,7 @@ class Newsletter(WebsiteGenerator): if self.send_from: validate_email_address(self.send_from, True) + @frappe.whitelist() def test_send(self, doctype="Lead"): self.recipients = frappe.utils.split_emails(self.test_email_id) self.queue_all(test_email=True)