diff --git a/.eslintrc b/.eslintrc index d123023a68..8a509f0df4 100644 --- a/.eslintrc +++ b/.eslintrc @@ -143,6 +143,7 @@ "Cypress": true, "cy": true, "it": true, + "describe": true, "expect": true, "context": true, "before": true, diff --git a/.github/helper/semgrep_rules/frappe_correctness.py b/.github/helper/semgrep_rules/frappe_correctness.py new file mode 100644 index 0000000000..37889fbbb1 --- /dev/null +++ b/.github/helper/semgrep_rules/frappe_correctness.py @@ -0,0 +1,28 @@ +import frappe +from frappe import _, flt + +from frappe.model.document import Document + + +def on_submit(self): + if self.value_of_goods == 0: + frappe.throw(_('Value of goods cannot be 0')) + # ruleid: frappe-modifying-after-submit + self.status = 'Submitted' + +def on_submit(self): # noqa + if flt(self.per_billed) < 100: + self.update_billing_status() + else: + # todook: frappe-modifying-after-submit + self.status = "Completed" + self.db_set("status", "Completed") + +class TestDoc(Document): + pass + + def validate(self): + #ruleid: frappe-modifying-child-tables-while-iterating + for item in self.child_table: + if item.value < 0: + self.remove(item) diff --git a/.github/helper/semgrep_rules/frappe_correctness.yml b/.github/helper/semgrep_rules/frappe_correctness.yml new file mode 100644 index 0000000000..faab3344a6 --- /dev/null +++ b/.github/helper/semgrep_rules/frappe_correctness.yml @@ -0,0 +1,135 @@ +# This file specifies rules for correctness according to how frappe doctype data model works. + +rules: +- id: frappe-modifying-but-not-comitting + patterns: + - pattern: | + def $METHOD(self, ...): + ... + self.$ATTR = ... + - pattern-not: | + def $METHOD(self, ...): + ... + self.$ATTR = ... + ... + self.db_set(..., self.$ATTR, ...) + - pattern-not: | + def $METHOD(self, ...): + ... + self.$ATTR = $SOME_VAR + ... + self.db_set(..., $SOME_VAR, ...) + - pattern-not: | + def $METHOD(self, ...): + ... + self.$ATTR = $SOME_VAR + ... + self.save() + - metavariable-regex: + metavariable: '$ATTR' + # this is negative look-ahead, add more attrs to ignore like (ignore|ignore_this_too|ignore_me) + regex: '^(?!ignore_linked_doctypes|status_updater)(.*)$' + - metavariable-regex: + metavariable: "$METHOD" + regex: "(on_submit|on_cancel)" + message: | + DocType modified in self.$METHOD. Please check if modification of self.$ATTR is commited to database. + languages: [python] + severity: ERROR + +- id: frappe-modifying-but-not-comitting-other-method + patterns: + - pattern: | + class $DOCTYPE(...): + def $METHOD(self, ...): + ... + self.$ANOTHER_METHOD() + ... + + def $ANOTHER_METHOD(self, ...): + ... + self.$ATTR = ... + - pattern-not: | + class $DOCTYPE(...): + def $METHOD(self, ...): + ... + self.$ANOTHER_METHOD() + ... + + def $ANOTHER_METHOD(self, ...): + ... + self.$ATTR = ... + ... + self.db_set(..., self.$ATTR, ...) + - pattern-not: | + class $DOCTYPE(...): + def $METHOD(self, ...): + ... + self.$ANOTHER_METHOD() + ... + + def $ANOTHER_METHOD(self, ...): + ... + self.$ATTR = $SOME_VAR + ... + self.db_set(..., $SOME_VAR, ...) + - pattern-not: | + class $DOCTYPE(...): + def $METHOD(self, ...): + ... + self.$ANOTHER_METHOD() + ... + self.save() + def $ANOTHER_METHOD(self, ...): + ... + self.$ATTR = ... + - metavariable-regex: + metavariable: "$METHOD" + regex: "(on_submit|on_cancel)" + message: | + self.$ANOTHER_METHOD is called from self.$METHOD, check if changes to self.$ATTR are commited to database. + languages: [python] + severity: ERROR + +- id: frappe-print-function-in-doctypes + pattern: print(...) + message: | + Did you mean to leave this print statement in? Consider using msgprint or logger instead of print statement. + languages: [python] + severity: WARNING + paths: + exclude: + - test_*.py + include: + - "*/**/doctype/*" + +- id: frappe-modifying-child-tables-while-iterating + pattern-either: + - pattern: | + for $ROW in self.$TABLE: + ... + self.remove(...) + - pattern: | + for $ROW in self.$TABLE: + ... + self.append(...) + message: | + Child table being modified while iterating on it. + languages: [python] + severity: ERROR + paths: + include: + - "*/**/doctype/*" + +- id: frappe-same-key-assigned-twice + pattern-either: + - pattern: | + {..., $X: $A, ..., $X: $B, ...} + - pattern: | + dict(..., ($X, $A), ..., ($X, $B), ...) + - pattern: | + _dict(..., ($X, $A), ..., ($X, $B), ...) + message: | + key `$X` is uselessly assigned twice. This could be a potential bug. + languages: [python] + severity: ERROR diff --git a/.github/helper/semgrep_rules/security.yml b/.github/helper/semgrep_rules/security.yml index 1937fc0e52..b2cc4b16fc 100644 --- a/.github/helper/semgrep_rules/security.yml +++ b/.github/helper/semgrep_rules/security.yml @@ -12,3 +12,18 @@ rules: exclude: - frappe/__init__.py - frappe/commands/utils.py + +- id: frappe-sqli-format-strings + patterns: + - pattern-inside: | + @frappe.whitelist() + def $FUNC(...): + ... + - pattern-either: + - pattern: frappe.db.sql("..." % ...) + - pattern: frappe.db.sql(f"...", ...) + - pattern: frappe.db.sql("...".format(...), ...) + message: | + Detected use of raw string formatting for SQL queries. This can lead to sql injection vulnerabilities. Refer security guidelines - https://github.com/frappe/erpnext/wiki/Code-Security-Guidelines + languages: [python] + severity: WARNING diff --git a/.github/helper/semgrep_rules/translate.yml b/.github/helper/semgrep_rules/translate.yml index 3737da5a7e..df55089b9f 100644 --- a/.github/helper/semgrep_rules/translate.yml +++ b/.github/helper/semgrep_rules/translate.yml @@ -44,7 +44,8 @@ rules: pattern-either: - pattern: _(...) + ... + _(...) - pattern: _("..." + "...") - - pattern-regex: '_\([^\)]*\\\s*' + - pattern-regex: '_\([^\)]*\\\s*' # lines broken by `\` + - pattern-regex: '_\(\s*\n' # line breaks allowed by python for using ( ) message: | Do not split strings inside translate function. Do not concatenate using translate functions. Please refer: https://frappeframework.com/docs/user/en/translations diff --git a/.github/helper/semgrep_rules/ux.py b/.github/helper/semgrep_rules/ux.py new file mode 100644 index 0000000000..4a74457435 --- /dev/null +++ b/.github/helper/semgrep_rules/ux.py @@ -0,0 +1,31 @@ +import frappe +from frappe import msgprint, throw, _ + + +# ruleid: frappe-missing-translate-function +throw("Error Occured") + +# ruleid: frappe-missing-translate-function +frappe.throw("Error Occured") + +# ruleid: frappe-missing-translate-function +frappe.msgprint("Useful message") + +# ruleid: frappe-missing-translate-function +msgprint("Useful message") + + +# ok: frappe-missing-translate-function +translatedmessage = _("Hello") + +# ok: frappe-missing-translate-function +throw(translatedmessage) + +# ok: frappe-missing-translate-function +msgprint(translatedmessage) + +# ok: frappe-missing-translate-function +msgprint(_("Helpful message")) + +# ok: frappe-missing-translate-function +frappe.throw(_("Error occured")) diff --git a/.github/helper/semgrep_rules/ux.yml b/.github/helper/semgrep_rules/ux.yml new file mode 100644 index 0000000000..ed06a6a80c --- /dev/null +++ b/.github/helper/semgrep_rules/ux.yml @@ -0,0 +1,15 @@ +rules: +- id: frappe-missing-translate-function + pattern-either: + - patterns: + - pattern: frappe.msgprint("...", ...) + - pattern-not: frappe.msgprint(_("..."), ...) + - pattern-not: frappe.msgprint(__("..."), ...) + - patterns: + - pattern: frappe.throw("...", ...) + - pattern-not: frappe.throw(_("..."), ...) + - pattern-not: frappe.throw(__("..."), ...) + message: | + All user facing text must be wrapped in translate function. Please refer to translation documentation. https://frappeframework.com/docs/user/en/guides/basics/translations + languages: [python, javascript, json] + severity: ERROR diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index bfe2002f69..d2a00ab05f 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -149,9 +149,10 @@ jobs: run: | cp ~/frappe-bench/sites/.coverage ${GITHUB_WORKSPACE} cd ${GITHUB_WORKSPACE} - pip install coveralls==2.2.0 - pip install coverage==4.5.4 - coveralls + pip install coveralls==3.0.1 + pip install coverage==5.5 + coveralls --service=github env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_TOKEN }} + COVERALLS_SERVICE_NAME: github diff --git a/.github/workflows/semgrep.yml b/.github/workflows/semgrep.yml index 1d5694f521..5092bf4705 100644 --- a/.github/workflows/semgrep.yml +++ b/.github/workflows/semgrep.yml @@ -14,9 +14,19 @@ jobs: uses: actions/setup-python@v2 with: python-version: 3.8 - - name: Run semgrep + + - name: Setup semgrep run: | python -m pip install -q semgrep git fetch origin $GITHUB_BASE_REF:$GITHUB_BASE_REF -q + + - name: Semgrep errors + run: | files=$(git diff --name-only --diff-filter=d $GITHUB_BASE_REF) - [[ -d .github/helper/semgrep_rules ]] && semgrep --config=.github/helper/semgrep_rules --quiet --error $files + [[ -d .github/helper/semgrep_rules ]] && semgrep --severity ERROR --config=.github/helper/semgrep_rules --quiet --error $files + semgrep --config="r/python.lang.correctness" --quiet --error $files + + - name: Semgrep warnings + run: | + files=$(git diff --name-only --diff-filter=d $GITHUB_BASE_REF) + [[ -d .github/helper/semgrep_rules ]] && semgrep --severity WARNING --severity INFO --config=.github/helper/semgrep_rules --quiet $files diff --git a/README.md b/README.md index b00d291b96..e00bea7857 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,8 @@ Full-stack web application framework that uses Python and MariaDB on the server ### Installation -[Install via Frappe Bench](https://github.com/frappe/bench) +* [Install via Docker](https://github.com/frappe/frappe_docker) +* [Install via Frappe Bench](https://github.com/frappe/bench) ## Contributing diff --git a/cypress/integration/relative_time_filters.js b/cypress/integration/relative_time_filters.js index 80e6387d99..cbb0524c24 100644 --- a/cypress/integration/relative_time_filters.js +++ b/cypress/integration/relative_time_filters.js @@ -1,7 +1,4 @@ context('Relative Timeframe', () => { - beforeEach(() => { - cy.login(); - }); before(() => { cy.login(); cy.visit('/app/website'); diff --git a/cypress/integration/table_multiselect.js b/cypress/integration/table_multiselect.js index faa72d63a5..25cab78ba2 100644 --- a/cypress/integration/table_multiselect.js +++ b/cypress/integration/table_multiselect.js @@ -1,5 +1,5 @@ context('Table MultiSelect', () => { - beforeEach(() => { + before(() => { cy.login(); }); diff --git a/frappe/__init__.py b/frappe/__init__.py index cab9b0da76..1bcf47e986 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -34,7 +34,7 @@ if PY2: reload(sys) sys.setdefaultencoding("utf-8") -__version__ = '13.0.0-dev' +__version__ = '13.1.0' __title__ = "Frappe Framework" @@ -975,7 +975,7 @@ def get_pymodule_path(modulename, *joins): :param *joins: Join additional path elements using `os.path.join`.""" if not "public" in joins: joins = [scrub(part) for part in joins] - return os.path.join(os.path.dirname(get_module(scrub(modulename)).__file__), *joins) + return os.path.join(os.path.dirname(get_module(scrub(modulename)).__file__ or ''), *joins) def get_module_list(app_name): """Get list of modules for given all via `app/modules.txt`.""" diff --git a/frappe/api.py b/frappe/api.py index 6a09b795b0..9039ae0e5f 100644 --- a/frappe/api.py +++ b/frappe/api.py @@ -1,12 +1,10 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import base64 import binascii import json - -from six.moves.urllib.parse import urlencode, urlparse +from urllib.parse import urlencode, urlparse import frappe import frappe.client @@ -14,6 +12,7 @@ import frappe.handler from frappe import _ from frappe.utils.response import build_response + def handle(): """ Handler for `/api` methods @@ -38,9 +37,6 @@ def handle(): `/api/resource/{doctype}/{name}?run_method={method}` will run a whitelisted controller method """ - - validate_auth() - parts = frappe.request.path[1:].split("/",3) call = doctype = name = None @@ -116,7 +112,7 @@ def handle(): frappe.local.form_dict['fields'] = json.loads(frappe.local.form_dict['fields']) frappe.local.form_dict.setdefault('limit_page_length', 20) frappe.local.response.update({ - "data": frappe.call( + "data": frappe.call( frappe.client.get_list, doctype, **frappe.local.form_dict @@ -140,6 +136,7 @@ def handle(): return build_response("json") + def get_request_form_data(): if frappe.local.form_dict.data is None: data = frappe.safe_decode(frappe.local.request.get_data()) @@ -148,25 +145,18 @@ def get_request_form_data(): return frappe.parse_json(data) + def validate_auth(): - if frappe.get_request_header("Authorization") is None: - return - - VALID_AUTH_PREFIX_TYPES = ['basic', 'bearer', 'token'] - VALID_AUTH_PREFIX_STRING = ", ".join(VALID_AUTH_PREFIX_TYPES).title() - + """ + Authenticate and sets user for the request. + """ authorization_header = frappe.get_request_header("Authorization", str()).split(" ") - authorization_type = authorization_header[0].lower() - if len(authorization_header) == 1: - frappe.throw(_('Invalid Authorization headers, add a token with a prefix from one of the following: {0}.').format(VALID_AUTH_PREFIX_STRING), frappe.InvalidAuthorizationHeader) - - if authorization_type == "bearer": + if len(authorization_header) == 2: validate_oauth(authorization_header) - elif authorization_type in VALID_AUTH_PREFIX_TYPES: validate_auth_via_api_keys(authorization_header) - else: - frappe.throw(_('Invalid Authorization Type {0}, must be one of {1}.').format(authorization_type, VALID_AUTH_PREFIX_STRING), frappe.InvalidAuthorizationPrefix) + + validate_auth_via_hooks() def validate_oauth(authorization_header): @@ -177,8 +167,8 @@ def validate_oauth(authorization_header): authorization_header (list of str): The 'Authorization' header containing the prefix and token """ - from frappe.oauth import get_url_delimiter from frappe.integrations.oauth2 import get_oauth_server + from frappe.oauth import get_url_delimiter form_dict = frappe.local.form_dict token = authorization_header[1] @@ -187,19 +177,20 @@ def validate_oauth(authorization_header): access_token = {"access_token": token} uri = parsed_url.scheme + "://" + parsed_url.netloc + parsed_url.path + "?" + urlencode(access_token) http_method = req.method - body = req.get_data() headers = req.headers + body = req.get_data() + if req.content_type and "multipart/form-data" in req.content_type: + body = None try: required_scopes = frappe.db.get_value("OAuth Bearer Token", token, "scopes").split(get_url_delimiter()) + valid, oauthlib_request = get_oauth_server().verify_request(uri, http_method, body, headers, required_scopes) + if valid: + frappe.set_user(frappe.db.get_value("OAuth Bearer Token", token, "user")) + frappe.local.form_dict = form_dict except AttributeError: - frappe.throw(_("Invalid Bearer token, please provide a valid access token with prefix 'Bearer'."), frappe.InvalidAuthorizationToken) + pass - valid, oauthlib_request = get_oauth_server().verify_request(uri, http_method, body, headers, required_scopes) - - if valid: - frappe.set_user(frappe.db.get_value("OAuth Bearer Token", token, "user")) - frappe.local.form_dict = form_dict def validate_auth_via_api_keys(authorization_header): @@ -222,8 +213,7 @@ def validate_auth_via_api_keys(authorization_header): except binascii.Error: frappe.throw(_("Failed to decode token, please provide a valid base64-encoded token."), frappe.InvalidAuthorizationToken) except (AttributeError, TypeError, ValueError): - frappe.throw(_("Invalid token, please provide a valid token with prefix 'Basic' or 'Token'."), frappe.InvalidAuthorizationToken) - + pass def validate_api_key_secret(api_key, api_secret, frappe_authorization_source=None): @@ -248,3 +238,8 @@ def validate_api_key_secret(api_key, api_secret, frappe_authorization_source=Non if frappe.local.login_manager.user in ('', 'Guest'): frappe.set_user(user) frappe.local.form_dict = form_dict + + +def validate_auth_via_hooks(): + for auth_hook in frappe.get_hooks('auth_hooks', []): + frappe.get_attr(auth_hook)() diff --git a/frappe/app.py b/frappe/app.py index 3bbb68b45e..1695c1f735 100644 --- a/frappe/app.py +++ b/frappe/app.py @@ -55,6 +55,7 @@ def application(request): frappe.recorder.record() frappe.monitor.start() frappe.rate_limiter.apply() + frappe.api.validate_auth() if request.method == "OPTIONS": response = Response() diff --git a/frappe/auth.py b/frappe/auth.py index ca97bbc17d..73cb8e8c15 100644 --- a/frappe/auth.py +++ b/frappe/auth.py @@ -120,6 +120,7 @@ class LoginManager: self.make_session() self.set_user_info() + @frappe.whitelist() def login(self): # clear cache frappe.clear_cache(user = frappe.form_dict.get('usr')) diff --git a/frappe/cache_manager.py b/frappe/cache_manager.py index bad879d2fa..4e0fe0cf44 100644 --- a/frappe/cache_manager.py +++ b/frappe/cache_manager.py @@ -18,7 +18,7 @@ global_cache_keys = ("app_hooks", "installed_apps", 'all_apps', 'scheduler_events', 'time_zone', 'webhooks', 'active_domains', 'active_modules', 'assignment_rule', 'server_script_map', 'wkhtmltopdf_version', 'domain_restricted_doctypes', 'domain_restricted_pages', 'information_schema:counts', - 'sitemap_routes', 'db_tables') + doctype_map_keys + 'sitemap_routes', 'db_tables', 'server_script_autocompletion_items') + doctype_map_keys user_cache_keys = ("bootinfo", "user_recent", "roles", "user_doc", "lang", "defaults", "user_permissions", "home_page", "linked_with", diff --git a/frappe/change_log/v13/v13_1_0.md b/frappe/change_log/v13/v13_1_0.md new file mode 100644 index 0000000000..87c3bd0906 --- /dev/null +++ b/frappe/change_log/v13/v13_1_0.md @@ -0,0 +1,22 @@ +# Version 13.1.0 Release Notes + +### Features & Enhancements + +- Automated mail notifications will be shown in timeline ([#12693](https://github.com/frappe/frappe/pull/12693)) +- Introduced Client Script for List views ([#12590](https://github.com/frappe/frappe/pull/12590)) +- Introduced language switcher for guest users on website navbar ([#12813](https://github.com/frappe/frappe/pull/12813)) +- Option to give submit permission while sharing a document ([#12799](https://github.com/frappe/frappe/pull/12799)) +- Added option to set `autoname` in Customize Form ([#12413](https://github.com/frappe/frappe/pull/12413)) +- Virtual DocType ([#12121](https://github.com/frappe/frappe/pull/12121)) + +### Fixes + +- Workspace fixes ([#12650](https://github.com/frappe/frappe/pull/12650)) ([#12655](https://github.com/frappe/frappe/pull/12655)) ([#12869](https://github.com/frappe/frappe/pull/12869)) +- Fixed an issue where select options were not getting updated in Grid ([#12839](https://github.com/frappe/frappe/pull/12839)) +- Webform Fixes ([#12630](https://github.com/frappe/frappe/pull/12630)) ([#12756](https://github.com/frappe/frappe/pull/12756)) ([#12819](https://github.com/frappe/frappe/pull/12819)) +- Fixed timespan filter for next and last timespans ([#12509](https://github.com/frappe/frappe/pull/12509)) +- System Notification fixes ([#12719](https://github.com/frappe/frappe/pull/12719)) +- Design Fixes ([#12669](https://github.com/frappe/frappe/pull/12669)) ([#12591](https://github.com/frappe/frappe/pull/12591)) ([#12557](https://github.com/frappe/frappe/pull/12557)) ([#12751](https://github.com/frappe/frappe/pull/12751)) ([#12864](https://github.com/frappe/frappe/pull/12864)) +- Fixed Multi-column paste in grid ([#12861](https://github.com/frappe/frappe/pull/12861)) +- Fixed grid validation ([#12744](https://github.com/frappe/frappe/pull/12744)) +- Fixed currency value formatting in dashboard chart ([#12613](https://github.com/frappe/frappe/pull/12613)) diff --git a/frappe/commands/__init__.py b/frappe/commands/__init__.py index b9ae02e112..61ee62d352 100644 --- a/frappe/commands/__init__.py +++ b/frappe/commands/__init__.py @@ -62,11 +62,24 @@ def popen(command, *args, **kwargs): if env: env = dict(environ, **env) + def set_low_prio(): + import psutil + if psutil.LINUX: + psutil.Process().nice(19) + psutil.Process().ionice(psutil.IOPRIO_CLASS_IDLE) + elif psutil.WINDOWS: + psutil.Process().nice(psutil.IDLE_PRIORITY_CLASS) + psutil.Process().ionice(psutil.IOPRIO_VERYLOW) + else: + psutil.Process().nice(19) + # ionice not supported + proc = subprocess.Popen(command, stdout=None if output else subprocess.PIPE, stderr=None if output else subprocess.PIPE, shell=shell, cwd=cwd, + preexec_fn=set_low_prio, env=env ) diff --git a/frappe/commands/scheduler.py b/frappe/commands/scheduler.py index bd9c9d2cb0..e9638800cd 100755 --- a/frappe/commands/scheduler.py +++ b/frappe/commands/scheduler.py @@ -18,22 +18,33 @@ def _is_scheduler_enabled(): return enable_scheduler -@click.command('trigger-scheduler-event') -@click.argument('event') + +@click.command("trigger-scheduler-event", help="Trigger a scheduler event") +@click.argument("event") @pass_context def trigger_scheduler_event(context, event): - "Trigger a scheduler event" import frappe.utils.scheduler + + exit_code = 0 + for site in context.sites: try: frappe.init(site=site) frappe.connect() - frappe.utils.scheduler.trigger(site, event, now=True) + try: + frappe.get_doc("Scheduled Job Type", {"method": event}).execute() + except frappe.DoesNotExistError: + click.secho(f"Event {event} does not exist!", fg="red") + exit_code = 1 finally: frappe.destroy() + if not context.sites: raise SiteNotSpecifiedError + sys.exit(exit_code) + + @click.command('enable-scheduler') @pass_context def enable_scheduler(context): diff --git a/frappe/commands/site.py b/frappe/commands/site.py index 0fadf2a294..0102d3ac40 100755 --- a/frappe/commands/site.py +++ b/frappe/commands/site.py @@ -676,10 +676,8 @@ def start_ngrok(context): frappe.init(site=site) port = frappe.conf.http_port or frappe.conf.webserver_port - public_url = ngrok.connect(port=port, options={ - 'host_header': site - }) - print(f'Public URL: {public_url}') + tunnel = ngrok.connect(addr=str(port), host_header=site) + print(f'Public URL: {tunnel.public_url}') print('Inspect logs at http://localhost:4040') ngrok_process = ngrok.get_ngrok_process() diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py index 5ff66171fc..a203c8c6d9 100644 --- a/frappe/commands/utils.py +++ b/frappe/commands/utils.py @@ -11,7 +11,7 @@ import click import frappe from frappe.commands import get_site, pass_context from frappe.exceptions import SiteNotSpecifiedError -from frappe.utils import get_bench_path, update_progress_bar +from frappe.utils import get_bench_path, update_progress_bar, cint @click.command('build') @@ -567,11 +567,14 @@ def run_ui_tests(context, app, headless=False): node_bin = subprocess.getoutput("npm bin") cypress_path = "{0}/cypress".format(node_bin) - plugin_path = "{0}/cypress-file-upload".format(node_bin) + plugin_path = "{0}/../cypress-file-upload".format(node_bin) # check if cypress in path...if not, install it. - if not (os.path.exists(cypress_path) or os.path.exists(plugin_path)) \ - or not subprocess.getoutput("npm view cypress version").startswith("6."): + if not ( + os.path.exists(cypress_path) + and os.path.exists(plugin_path) + and cint(subprocess.getoutput("npm view cypress version")[:1]) >= 6 + ): # install cypress click.secho("Installing Cypress...", fg="yellow") frappe.commands.popen("yarn add cypress@^6 cypress-file-upload@^5 --no-lockfile") diff --git a/frappe/core/doctype/report/report.py b/frappe/core/doctype/report/report.py index af2c4e5dc2..8a0f9a99f5 100644 --- a/frappe/core/doctype/report/report.py +++ b/frappe/core/doctype/report/report.py @@ -325,9 +325,8 @@ def get_group_by_field(args, doctype): if args['aggregate_function'] == 'count': group_by_field = 'count(*) as _aggregate_column' else: - group_by_field = '{0}(`tab{1}`.{2}) as _aggregate_column'.format( + group_by_field = '{0}({1}) as _aggregate_column'.format( args.aggregate_function, - doctype, args.aggregate_on ) diff --git a/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py b/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py index 92493a593a..59089d12ad 100644 --- a/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py +++ b/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py @@ -2,14 +2,15 @@ # Copyright (c) 2019, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals +import json +from datetime import datetime from typing import Dict, List -import frappe, json -from frappe.model.document import Document -from frappe.utils import now_datetime, get_datetime -from datetime import datetime from croniter import croniter + +import frappe +from frappe.model.document import Document +from frappe.utils import get_datetime, now_datetime from frappe.utils.background_jobs import enqueue, get_jobs diff --git a/frappe/core/doctype/server_script/server_script.js b/frappe/core/doctype/server_script/server_script.js index 95a63780f8..dda39115bf 100644 --- a/frappe/core/doctype/server_script/server_script.js +++ b/frappe/core/doctype/server_script/server_script.js @@ -9,6 +9,12 @@ frappe.ui.form.on('Server Script', { if (frm.doc.script_type != 'Scheduler Event') { frm.dashboard.hide(); } + + frm.call('get_autocompletion_items') + .then(r => r.message) + .then(items => { + frm.set_df_property('script', 'autocompletions', items); + }); }, setup_help(frm) { diff --git a/frappe/core/doctype/server_script/server_script.py b/frappe/core/doctype/server_script/server_script.py index 8838d9e954..f80a067cf1 100644 --- a/frappe/core/doctype/server_script/server_script.py +++ b/frappe/core/doctype/server_script/server_script.py @@ -5,11 +5,12 @@ from __future__ import unicode_literals import ast +from types import FunctionType, MethodType, ModuleType from typing import Dict, List import frappe from frappe.model.document import Document -from frappe.utils.safe_exec import safe_exec +from frappe.utils.safe_exec import get_safe_globals, safe_exec, NamespaceDict from frappe import _ @@ -122,6 +123,51 @@ class ServerScript(Document): if locals["conditions"]: return locals["conditions"] + @frappe.whitelist() + def get_autocompletion_items(self): + """Generates a list of a autocompletion strings from the context dict + that is used while executing a Server Script. + + Returns: + list: Returns list of autocompletion items. + For e.g., ["frappe.utils.cint", "frappe.db.get_all", ...] + """ + def get_keys(obj): + out = [] + for key in obj: + if key.startswith('_'): + continue + value = obj[key] + if isinstance(value, (NamespaceDict, dict)) and value: + if key == 'form_dict': + out.append(['form_dict', 7]) + continue + for subkey, score in get_keys(value): + fullkey = f'{key}.{subkey}' + out.append([fullkey, score]) + else: + if isinstance(value, type) and issubclass(value, Exception): + score = 0 + elif isinstance(value, ModuleType): + score = 10 + elif isinstance(value, (FunctionType, MethodType)): + score = 9 + elif isinstance(value, type): + score = 8 + elif isinstance(value, dict): + score = 7 + else: + score = 6 + out.append([key, score]) + return out + + items = frappe.cache().get_value('server_script_autocompletion_items') + if not items: + items = get_keys(get_safe_globals()) + items = [{'value': d[0], 'score': d[1]} for d in items] + frappe.cache().set_value('server_script_autocompletion_items', items) + return items + @frappe.whitelist() def setup_scheduler_events(script_name, frequency): diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index f9997d1526..7d1d92408c 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -1,17 +1,13 @@ -from __future__ import unicode_literals - -import frappe import warnings import pymysql -from pymysql.times import TimeDelta -from pymysql.constants import ER, FIELD_TYPE -from pymysql.converters import conversions +from pymysql.constants import ER, FIELD_TYPE +from pymysql.converters import conversions, escape_string -from frappe.utils import get_datetime, cstr, UnicodeWithAttrs +import frappe from frappe.database.database import Database -from six import PY2, binary_type, text_type, string_types from frappe.database.mariadb.schema import MariaDBTable +from frappe.utils import UnicodeWithAttrs, cstr, get_datetime class MariaDBDatabase(Database): @@ -72,22 +68,20 @@ class MariaDBDatabase(Database): conversions.update({ FIELD_TYPE.NEWDECIMAL: float, FIELD_TYPE.DATETIME: get_datetime, - UnicodeWithAttrs: conversions[text_type] + UnicodeWithAttrs: conversions[str] }) - if PY2: - conversions.update({ - TimeDelta: conversions[binary_type] - }) - - if usessl: - conn = pymysql.connect(self.host, self.user or '', self.password or '', - port=self.port, charset='utf8mb4', use_unicode = True, ssl=ssl_params, - conv = conversions, local_infile = frappe.conf.local_infile) - else: - conn = pymysql.connect(self.host, self.user or '', self.password or '', - port=self.port, charset='utf8mb4', use_unicode = True, conv = conversions, - local_infile = frappe.conf.local_infile) + conn = pymysql.connect( + user=self.user or '', + password=self.password or '', + host=self.host, + port=self.port, + charset='utf8mb4', + use_unicode=True, + ssl=ssl_params if usessl else None, + conv=conversions, + local_infile=frappe.conf.local_infile + ) # MYSQL_OPTION_MULTI_STATEMENTS_OFF = 1 # # self._conn.set_server_option(MYSQL_OPTION_MULTI_STATEMENTS_OFF) @@ -111,7 +105,7 @@ class MariaDBDatabase(Database): def escape(s, percent=True): """Excape quotes and percent in given string.""" # pymysql expects unicode argument to escape_string with Python 3 - s = frappe.as_unicode(pymysql.escape_string(frappe.as_unicode(s)), "utf-8").replace("`", "\\`") + s = frappe.as_unicode(escape_string(frappe.as_unicode(s)), "utf-8").replace("`", "\\`") # NOTE separating % escape, because % escape should only be done when using LIKE operator # or when you use python format string to generate query that already has a %s @@ -260,7 +254,7 @@ class MariaDBDatabase(Database): ADD INDEX `%s`(%s)""" % (table_name, index_name, ", ".join(fields))) def add_unique(self, doctype, fields, constraint_name=None): - if isinstance(fields, string_types): + if isinstance(fields, str): fields = [fields] if not constraint_name: constraint_name = "unique_" + "_".join(fields) diff --git a/frappe/desk/reportview.py b/frappe/desk/reportview.py index 3d04c171a7..86f8ec0aa7 100644 --- a/frappe/desk/reportview.py +++ b/frappe/desk/reportview.py @@ -126,13 +126,14 @@ def setup_group_by(data): if data.group_by: if data.aggregate_function.lower() not in ('count', 'sum', 'avg'): frappe.throw(_('Invalid aggregate function')) - if '`' in data.aggregate_on: - raise_invalid_field(data.aggregate_on) - data.fields.append('{aggregate_function}(`tab{doctype}`.`{aggregate_on}`) AS _aggregate_column'.format(**data)) - if data.aggregate_on: - data.fields.append(data.aggregate_on) - data.pop('aggregate_on') + if frappe.db.has_column(data.aggregate_on_doctype, data.aggregate_on_field): + data.fields.append('{aggregate_function}(`tab{aggregate_on_doctype}`.`{aggregate_on_field}`) AS _aggregate_column'.format(**data)) + else: + raise_invalid_field(data.aggregate_on_field) + + data.pop('aggregate_on_doctype') + data.pop('aggregate_on_field') data.pop('aggregate_function') def raise_invalid_field(fieldname): diff --git a/frappe/desk/treeview.py b/frappe/desk/treeview.py index d479b71b52..6f0d7d3d5f 100644 --- a/frappe/desk/treeview.py +++ b/frappe/desk/treeview.py @@ -36,7 +36,7 @@ def get_all_nodes(doctype, label, parent, tree_method, **filters): return out @frappe.whitelist() -def get_children(doctype, parent=''): +def get_children(doctype, parent='', **filters): return _get_children(doctype, parent) def _get_children(doctype, parent='', ignore_permissions=False): diff --git a/frappe/email/doctype/newsletter/newsletter.py b/frappe/email/doctype/newsletter/newsletter.py index c792347c09..6412338e96 100755 --- a/frappe/email/doctype/newsletter/newsletter.py +++ b/frappe/email/doctype/newsletter/newsletter.py @@ -24,6 +24,7 @@ class Newsletter(WebsiteGenerator): if self.send_from: validate_email_address(self.send_from, True) + @frappe.whitelist() def test_send(self, doctype="Lead"): self.recipients = frappe.utils.split_emails(self.test_email_id) self.queue_all(test_email=True) diff --git a/frappe/email/receive.py b/frappe/email/receive.py index cf6c13ee76..949da4a343 100644 --- a/frappe/email/receive.py +++ b/frappe/email/receive.py @@ -1,18 +1,27 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals -import six -from six import iteritems, text_type -from six.moves import range -import time, _socket, poplib, imaplib, email, email.utils, datetime, chardet, re -from email_reply_parser import EmailReplyParser +import datetime +import email +import email.utils +import imaplib +import poplib +import re +import time from email.header import decode_header + +import _socket +import chardet +import six +from email_reply_parser import EmailReplyParser + import frappe from frappe import _, safe_decode, safe_encode -from frappe.utils import (extract_email_id, convert_utc_to_user_timezone, now, - cint, cstr, strip, markdown, parse_addr) -from frappe.core.doctype.file.file import get_random_filename, MaxFileSizeReachedError +from frappe.core.doctype.file.file import (MaxFileSizeReachedError, + get_random_filename) +from frappe.utils import (cint, convert_utc_to_user_timezone, cstr, + extract_email_id, markdown, now, parse_addr, strip) + class EmailSizeExceededError(frappe.ValidationError): pass class EmailTimeoutError(frappe.ValidationError): pass @@ -337,7 +346,7 @@ class EmailServer: return self.imap.select("Inbox") - for uid, operation in iteritems(uid_list): + for uid, operation in uid_list.items(): if not uid: continue op = "+FLAGS" if operation == "Read" else "-FLAGS" @@ -473,7 +482,7 @@ class Email: self.html_content += markdown(text_content) def get_charset(self, part): - """Detect chartset.""" + """Detect charset.""" charset = part.get_content_charset() if not charset: charset = chardet.detect(safe_encode(cstr(part)))['encoding'] @@ -484,7 +493,7 @@ class Email: charset = self.get_charset(part) try: - return text_type(part.get_payload(decode=True), str(charset), "ignore") + return str(part.get_payload(decode=True), str(charset), "ignore") except LookupError: return part.get_payload() diff --git a/frappe/handler.py b/frappe/handler.py index 82c1ea65c6..a38feb90fa 100755 --- a/frappe/handler.py +++ b/frappe/handler.py @@ -9,7 +9,6 @@ import frappe import frappe.utils import frappe.sessions from frappe.utils import cint -from frappe.api import validate_auth from frappe import _, is_whitelisted from frappe.utils.response import build_response from frappe.utils.csvutils import build_csv_response @@ -24,7 +23,7 @@ ALLOWED_MIMETYPES = ('image/png', 'image/jpeg', 'application/pdf', 'application/ def handle(): """handle request""" - validate_auth() + cmd = frappe.local.form_dict.cmd data = None diff --git a/frappe/hooks.py b/frappe/hooks.py index 74c538c5df..1c78d47755 100644 --- a/frappe/hooks.py +++ b/frappe/hooks.py @@ -130,6 +130,16 @@ has_website_permission = { "Address": "frappe.contacts.doctype.address.address.has_website_permission" } +jinja = { + "methods": "frappe.utils.jinja_globals", + "filters": [ + "frappe.utils.data.global_date_format", + "frappe.utils.markdown", + "frappe.website.utils.get_shade", + "frappe.website.utils.abs_url", + ] +} + standard_queries = { "User": "frappe.core.doctype.user.user.user_query" } diff --git a/frappe/integrations/doctype/dropbox_settings/dropbox_settings.py b/frappe/integrations/doctype/dropbox_settings/dropbox_settings.py index 09da1ecc42..53f0935c80 100644 --- a/frappe/integrations/doctype/dropbox_settings/dropbox_settings.py +++ b/frappe/integrations/doctype/dropbox_settings/dropbox_settings.py @@ -2,22 +2,23 @@ # Copyright (c) 2015, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals -import dropbox import json -import frappe import os -from frappe import _ -from frappe.model.document import Document -from frappe.integrations.offsite_backup_utils import get_latest_backup_file, send_email, validate_file_size, get_chunk_site -from frappe.integrations.utils import make_post_request -from frappe.utils import (cint, get_request_site_address, - get_files_path, get_backups_path, get_url, encode) -from frappe.utils.backups import new_backup -from frappe.utils.background_jobs import enqueue -from six.moves.urllib.parse import urlparse, parse_qs +from urllib.parse import parse_qs, urlparse + +import dropbox from rq.timeouts import JobTimeoutException -from six import text_type + +import frappe +from frappe import _ +from frappe.integrations.offsite_backup_utils import (get_chunk_site, + get_latest_backup_file, send_email, validate_file_size) +from frappe.integrations.utils import make_post_request +from frappe.model.document import Document +from frappe.utils import (cint, encode, get_backups_path, get_files_path, + get_request_site_address, get_url) +from frappe.utils.background_jobs import enqueue +from frappe.utils.backups import new_backup ignore_list = [".DS_Store"] @@ -91,7 +92,10 @@ def backup_to_dropbox(upload_db_backup=True): dropbox_settings['access_token'] = access_token['oauth2_token'] set_dropbox_access_token(access_token['oauth2_token']) - dropbox_client = dropbox.Dropbox(dropbox_settings['access_token'], timeout=None) + dropbox_client = dropbox.Dropbox( + oauth2_access_token=dropbox_settings['access_token'], + timeout=None + ) if upload_db_backup: if frappe.flags.create_new_backup: @@ -127,7 +131,7 @@ def upload_from_folder(path, is_private, dropbox_folder, dropbox_client, did_not else: response = frappe._dict({"entries": []}) - path = text_type(path) + path = str(path) for f in frappe.get_all("File", filters={"is_folder": 0, "is_private": is_private, "uploaded_to_dropbox": 0}, fields=['file_url', 'name', 'file_name']): @@ -286,11 +290,11 @@ def get_redirect_url(): def get_dropbox_authorize_url(): app_details = get_dropbox_settings(redirect_uri=True) dropbox_oauth_flow = dropbox.DropboxOAuth2Flow( - app_details["app_key"], - app_details["app_secret"], - app_details["redirect_uri"], - {}, - "dropbox-auth-csrf-token" + consumer_key=app_details["app_key"], + redirect_uri=app_details["redirect_uri"], + session={}, + csrf_token_session_key="dropbox-auth-csrf-token", + consumer_secret=app_details["app_secret"] ) auth_url = dropbox_oauth_flow.start() @@ -307,13 +311,13 @@ def dropbox_auth_finish(return_access_token=False): close = '

' + _('Please close this window') + '

' dropbox_oauth_flow = dropbox.DropboxOAuth2Flow( - app_details["app_key"], - app_details["app_secret"], - app_details["redirect_uri"], - { + consumer_key=app_details["app_key"], + redirect_uri=app_details["redirect_uri"], + session={ 'dropbox-auth-csrf-token': callback.state }, - "dropbox-auth-csrf-token" + csrf_token_session_key="dropbox-auth-csrf-token", + consumer_secret=app_details["app_secret"] ) if callback.state or callback.code: diff --git a/frappe/integrations/doctype/google_calendar/google_calendar.py b/frappe/integrations/doctype/google_calendar/google_calendar.py index fbedd75029..f93be35aa7 100644 --- a/frappe/integrations/doctype/google_calendar/google_calendar.py +++ b/frappe/integrations/doctype/google_calendar/google_calendar.py @@ -2,22 +2,23 @@ # Copyright (c) 2019, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals -import frappe -import requests -import googleapiclient.discovery -import google.oauth2.credentials -from frappe import _ -from frappe.model.document import Document -from frappe.utils import get_request_site_address -from googleapiclient.errors import HttpError -from frappe.utils.password import set_encrypted_password -from frappe.utils import add_days, get_datetime, get_weekdays, now_datetime, add_to_date, get_time_zone -from dateutil import parser from datetime import datetime, timedelta -from six.moves.urllib.parse import quote +from urllib.parse import quote + +import google.oauth2.credentials +import requests +from dateutil import parser +from googleapiclient.discovery import build +from googleapiclient.errors import HttpError + +import frappe +from frappe import _ from frappe.integrations.doctype.google_settings.google_settings import get_auth_url +from frappe.model.document import Document +from frappe.utils import (add_days, add_to_date, get_datetime, + get_request_site_address, get_time_zone, get_weekdays, now_datetime) +from frappe.utils.password import set_encrypted_password SCOPES = "https://www.googleapis.com/auth/calendar" @@ -171,7 +172,12 @@ def get_google_calendar_object(g_calendar): } credentials = google.oauth2.credentials.Credentials(**credentials_dict) - google_calendar = googleapiclient.discovery.build("calendar", "v3", credentials=credentials) + google_calendar = build( + serviceName="calendar", + version="v3", + credentials=credentials, + static_discovery=False + ) check_google_calendar(account, google_calendar) diff --git a/frappe/integrations/doctype/google_contacts/google_contacts.py b/frappe/integrations/doctype/google_contacts/google_contacts.py index 4c8c3b67f6..1705f98e91 100644 --- a/frappe/integrations/doctype/google_contacts/google_contacts.py +++ b/frappe/integrations/doctype/google_contacts/google_contacts.py @@ -2,17 +2,17 @@ # Copyright (c) 2019, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals -import frappe -import requests -import googleapiclient.discovery -import google.oauth2.credentials -from frappe.model.document import Document -from frappe import _ +import google.oauth2.credentials +import requests +from googleapiclient.discovery import build from googleapiclient.errors import HttpError -from frappe.utils import get_request_site_address + +import frappe +from frappe import _ from frappe.integrations.doctype.google_settings.google_settings import get_auth_url +from frappe.model.document import Document +from frappe.utils import get_request_site_address SCOPES = "https://www.googleapis.com/auth/contacts" @@ -118,7 +118,12 @@ def get_google_contacts_object(g_contact): } credentials = google.oauth2.credentials.Credentials(**credentials_dict) - google_contacts = googleapiclient.discovery.build("people", "v1", credentials=credentials) + google_contacts = build( + serviceName="people", + version="v1", + credentials=credentials, + static_discovery=False + ) return google_contacts, account diff --git a/frappe/integrations/doctype/google_drive/google_drive.py b/frappe/integrations/doctype/google_drive/google_drive.py index 859c769018..93b6fa3f8d 100644 --- a/frappe/integrations/doctype/google_drive/google_drive.py +++ b/frappe/integrations/doctype/google_drive/google_drive.py @@ -2,27 +2,29 @@ # Copyright (c) 2019, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals -import frappe -import requests -import googleapiclient.discovery -import google.oauth2.credentials import os +from urllib.parse import quote -from frappe import _ -from googleapiclient.errors import HttpError -from frappe.model.document import Document -from frappe.utils import get_request_site_address -from frappe.utils.background_jobs import enqueue -from six.moves.urllib.parse import quote +import google.oauth2.credentials +import requests from apiclient.http import MediaFileUpload -from frappe.utils import get_backups_path, get_bench_path -from frappe.utils.backups import new_backup +from googleapiclient.discovery import build +from googleapiclient.errors import HttpError + +import frappe +from frappe import _ from frappe.integrations.doctype.google_settings.google_settings import get_auth_url -from frappe.integrations.offsite_backup_utils import get_latest_backup_file, send_email, validate_file_size +from frappe.integrations.offsite_backup_utils import (get_latest_backup_file, + send_email, validate_file_size) +from frappe.model.document import Document +from frappe.utils import (get_backups_path, get_bench_path, + get_request_site_address) +from frappe.utils.background_jobs import enqueue +from frappe.utils.backups import new_backup SCOPES = "https://www.googleapis.com/auth/drive" + class GoogleDrive(Document): def validate(self): @@ -126,7 +128,12 @@ def get_google_drive_object(): } credentials = google.oauth2.credentials.Credentials(**credentials_dict) - google_drive = googleapiclient.discovery.build("drive", "v3", credentials=credentials) + google_drive = build( + serviceName="drive", + version="v3", + credentials=credentials, + static_discovery=False + ) return google_drive, account diff --git a/frappe/integrations/doctype/token_cache/test_token_cache.py b/frappe/integrations/doctype/token_cache/test_token_cache.py index 73c9f38fce..7aa069647d 100644 --- a/frappe/integrations/doctype/token_cache/test_token_cache.py +++ b/frappe/integrations/doctype/token_cache/test_token_cache.py @@ -13,7 +13,7 @@ class TestTokenCache(unittest.TestCase): def setUp(self): self.token_cache = frappe.get_last_doc('Token Cache') self.token_cache.update({'connected_app': frappe.get_last_doc('Connected App').name}) - self.token_cache.save() + self.token_cache.save(ignore_permissions=True) def test_get_auth_header(self): self.token_cache.get_auth_header() diff --git a/frappe/integrations/oauth2.py b/frappe/integrations/oauth2.py index c444964a16..3ebaaffcff 100644 --- a/frappe/integrations/oauth2.py +++ b/frappe/integrations/oauth2.py @@ -133,7 +133,7 @@ def get_token(*args, **kwargs): } id_token_encoded = jwt.encode(id_token, client_secret, algorithm='HS256', headers=id_token_header) - out.update({"id_token": str(id_token_encoded)}) + out.update({"id_token": frappe.safe_decode(id_token_encoded)}) frappe.local.response = out diff --git a/frappe/patches.txt b/frappe/patches.txt index 516ddb6094..60c3112f4a 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -335,3 +335,4 @@ frappe.patches.v13_0.rename_list_view_setting_to_list_view_settings frappe.patches.v13_0.remove_twilio_settings frappe.patches.v12_0.rename_uploaded_files_with_proper_name frappe.patches.v13_0.queryreport_columns +frappe.patches.v13_0.jinja_hook diff --git a/frappe/patches/v13_0/jinja_hook.py b/frappe/patches/v13_0/jinja_hook.py new file mode 100644 index 0000000000..84ed6e6cff --- /dev/null +++ b/frappe/patches/v13_0/jinja_hook.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors +# MIT License. See license.txt + +from __future__ import unicode_literals +import frappe +from click import secho + +def execute(): + if frappe.get_hooks('jenv'): + print() + secho('WARNING: The hook "jenv" is deprecated. Follow the migration guide to use the new "jinja" hook.', fg='yellow') + secho('https://github.com/frappe/frappe/wiki/Migrating-to-Version-13', fg='yellow') + print() diff --git a/frappe/public/js/frappe/form/controls/base_control.js b/frappe/public/js/frappe/form/controls/base_control.js index 9981398b84..8c2c5c4338 100644 --- a/frappe/public/js/frappe/form/controls/base_control.js +++ b/frappe/public/js/frappe/form/controls/base_control.js @@ -159,9 +159,13 @@ frappe.ui.form.Control = Class.extend({ }, validate_and_set_in_model: function(value, e) { var me = this; - if(this.inside_change_event) { + let force_value_set = (this.doc && this.doc.__run_link_triggers); + let is_value_same = (this.get_model_value() === value); + + if (this.inside_change_event || (!force_value_set && is_value_same)) { return Promise.resolve(); } + this.inside_change_event = true; var set = function(value) { me.inside_change_event = false; diff --git a/frappe/public/js/frappe/form/controls/code.js b/frappe/public/js/frappe/form/controls/code.js index eec450b390..9600763588 100644 --- a/frappe/public/js/frappe/form/controls/code.js +++ b/frappe/public/js/frappe/form/controls/code.js @@ -31,6 +31,57 @@ frappe.ui.form.ControlCode = frappe.ui.form.ControlText.extend({ const input_value = this.get_input_value(); this.parse_validate_and_set_in_model(input_value); }, 300)); + + // setup autocompletion when it is set the first time + Object.defineProperty(this.df, 'autocompletions', { + get() { + return this._autocompletions || []; + }, + set: (value) => { + this.setup_autocompletion(); + this.df._autocompletions = value; + } + }); + }, + + setup_autocompletion() { + if (this._autocompletion_setup) return; + + const ace = window.ace; + const get_autocompletions = () => this.df.autocompletions; + + ace.config.loadModule("ace/ext/language_tools", langTools => { + this.editor.setOptions({ + enableBasicAutocompletion: true, + enableLiveAutocompletion: true + }); + + langTools.addCompleter({ + getCompletions: function(editor, session, pos, prefix, callback) { + if (prefix.length === 0) { + callback(null, []); + return; + } + let autocompletions = get_autocompletions(); + if (autocompletions.length) { + callback( + null, + autocompletions.map(a => { + if (typeof a === 'string') { + a = { value: a }; + } + return { + name: 'frappe', + value: a.value, + score: a.score + }; + }) + ); + } + } + }); + }); + this._autocompletion_setup = true; }, refresh_height() { diff --git a/frappe/public/js/frappe/form/controls/link.js b/frappe/public/js/frappe/form/controls/link.js index 1a483c5968..c0ff128088 100644 --- a/frappe/public/js/frappe/form/controls/link.js +++ b/frappe/public/js/frappe/form/controls/link.js @@ -462,9 +462,10 @@ frappe.ui.form.ControlLink = frappe.ui.form.ControlData.extend({ if(this.frm && this.frm.fetch_dict[df.fieldname]) { fetch = this.frm.fetch_dict[df.fieldname].columns.join(', '); } - // if default and no fetch, no need to validate - if (!fetch && df.__default_value && df.__default_value===value) return value; + if (!fetch && df.__default_value && df.__default_value===value) { + resolve(value); + } this.fetch_and_validate_link(resolve, df, doctype, docname, value, fetch); }); diff --git a/frappe/public/js/frappe/form/controls/table_multiselect.js b/frappe/public/js/frappe/form/controls/table_multiselect.js index c306146f90..eb3f1bce6e 100644 --- a/frappe/public/js/frappe/form/controls/table_multiselect.js +++ b/frappe/public/js/frappe/form/controls/table_multiselect.js @@ -66,6 +66,10 @@ frappe.ui.form.ControlTableMultiSelect = frappe.ui.form.ControlLink.extend({ this._rows_list = this.rows.map(row => row[link_field.fieldname]); return this.rows; }, + get_model_value() { + let value = this._super(); + return value ? value.filter(d => !d.__islocal) : value; + }, validate(value) { const rows = (value || []).slice(); diff --git a/frappe/public/js/frappe/form/dashboard.js b/frappe/public/js/frappe/form/dashboard.js index 9b6d15c1fc..c1c95d94cf 100644 --- a/frappe/public/js/frappe/form/dashboard.js +++ b/frappe/public/js/frappe/form/dashboard.js @@ -290,7 +290,7 @@ frappe.ui.form.Dashboard = class FormDashboard { // bind links transactions_area_body.find(".badge-link").on('click', function() { - me.open_document_list($(this).parent()); + me.open_document_list($(this).closest('.document-link')); }); // bind reports diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js index ef728e730e..2b7562f836 100644 --- a/frappe/public/js/frappe/form/form.js +++ b/frappe/public/js/frappe/form/form.js @@ -360,6 +360,7 @@ frappe.ui.form.Form = class FrappeForm { grid_obj.grid.grid_pagination.go_to_page(1, true); }); frappe.ui.form.close_grid_form(); + this.viewers && this.viewers.parent.empty(); this.docname = docname; this.setup_docinfo_change_listener(); } @@ -1202,8 +1203,7 @@ frappe.ui.form.Form = class FrappeForm { $.each(grid_field_label_map, function(fname, label) { fname = fname.split("-"); - var df = frappe.meta.get_docfield(fname[0], fname[1], me.doc.name); - if(df) df.label = label; + me.fields_dict[parentfield].grid.update_docfield_property(fname[1], 'label', label); }); } diff --git a/frappe/public/js/frappe/form/grid.js b/frappe/public/js/frappe/form/grid.js index 86feefed7a..4d381c9be7 100644 --- a/frappe/public/js/frappe/form/grid.js +++ b/frappe/public/js/frappe/form/grid.js @@ -387,6 +387,8 @@ export default class Grid { this.wrapper.find('.grid-footer').toggle(false); } + this.wrapper.find('.grid-add-row, .grid-add-multiple-rows').toggle(this.is_editable()); + } truncate_rows() { diff --git a/frappe/public/js/frappe/form/grid_row.js b/frappe/public/js/frappe/form/grid_row.js index 5e3a2b8ccd..9a689fabf4 100644 --- a/frappe/public/js/frappe/form/grid_row.js +++ b/frappe/public/js/frappe/form/grid_row.js @@ -6,6 +6,7 @@ export default class GridRow { this.on_grid_fields = []; $.extend(this, opts); if (this.doc && this.parent_df.options) { + frappe.meta.make_docfield_copy_for(this.parent_df.options, this.doc.name, this.docfields); this.docfields = frappe.meta.get_docfields(this.parent_df.options, this.doc.name); } this.columns = {}; @@ -555,6 +556,12 @@ export default class GridRow { this.grid_form.render(); this.row.toggle(false); // this.form_panel.toggle(true); + + let cannot_add_rows = this.grid.cannot_add_rows || (this.grid.df && this.grid.df.cannot_add_rows); + this.wrapper + .find('.grid-insert-row-below, .grid-insert-row, .grid-duplicate-row, .grid-append-row') + .toggle(!cannot_add_rows); + frappe.dom.freeze("", "dark"); if (cur_frm) cur_frm.cur_grid = this; this.wrapper.addClass("grid-row-open"); diff --git a/frappe/public/js/frappe/form/grid_row_form.js b/frappe/public/js/frappe/form/grid_row_form.js index 68e4178ae7..f5a4af206f 100644 --- a/frappe/public/js/frappe/form/grid_row_form.js +++ b/frappe/public/js/frappe/form/grid_row_form.js @@ -119,7 +119,7 @@ export default class GridRowForm { }); } toggle_add_delete_button_display($parent) { - $parent.find(".row-actions") + $parent.find(".row-actions, .grid-append-row") .toggle(this.row.grid.is_editable()); } refresh_field(fieldname) { diff --git a/frappe/public/js/frappe/form/toolbar.js b/frappe/public/js/frappe/form/toolbar.js index 145b8d3eed..22787b70c1 100644 --- a/frappe/public/js/frappe/form/toolbar.js +++ b/frappe/public/js/frappe/form/toolbar.js @@ -211,7 +211,6 @@ frappe.ui.form.Toolbar = class Toolbar { make_viewers() { if (this.frm.viewers) { - this.frm.viewers.parent.empty(); return; } this.frm.viewers = new frappe.ui.form.FormViewers({ diff --git a/frappe/public/js/frappe/model/create_new.js b/frappe/public/js/frappe/model/create_new.js index dc6ee56fca..1b09a451eb 100644 --- a/frappe/public/js/frappe/model/create_new.js +++ b/frappe/public/js/frappe/model/create_new.js @@ -177,7 +177,9 @@ $.extend(frappe.model, { // Use User Permission value when only when it has a single value user_default = user_defaults[0]; } - } else if (!user_default) { + } + + if (!user_default) { user_default = frappe.defaults.get_user_default(df.fieldname); } else if ( !user_default && diff --git a/frappe/public/js/frappe/model/meta.js b/frappe/public/js/frappe/model/meta.js index c2fd6b1ae6..6ee9084adc 100644 --- a/frappe/public/js/frappe/model/meta.js +++ b/frappe/public/js/frappe/model/meta.js @@ -38,14 +38,14 @@ $.extend(frappe.meta, { frappe.meta.docfield_list[df.parent].push(df); }, - make_docfield_copy_for: function(doctype, docname) { + make_docfield_copy_for: function(doctype, docname, docfield_list=null) { var c = frappe.meta.docfield_copy; if(!c[doctype]) c[doctype] = {}; if(!c[doctype][docname]) c[doctype][docname] = {}; - var docfield_list = frappe.meta.docfield_list[doctype] || []; + docfield_list = docfield_list || frappe.meta.docfield_list[doctype] || []; for(var i=0, j=docfield_list.length; i { - this.dashboard_chart && this.dashboard_chart.draw(true); - }); - - $(document.body).on('toggleListSidebar', () => { - this.dashboard_chart && this.dashboard_chart.draw(true); - }); - - $(document.body).on('toggleFullWidth', () => { - this.dashboard_chart && this.dashboard_chart.draw(true); - }); - } } diff --git a/frappe/templates/base.html b/frappe/templates/base.html index 78aa573c99..d59c4b0f2b 100644 --- a/frappe/templates/base.html +++ b/frappe/templates/base.html @@ -60,7 +60,7 @@ window.is_chat_enabled = {{ chat_enable }}; - + {% include "public/icons/timeless/symbol-defs.svg" %} {%- block banner -%} {% include "templates/includes/banner_extension.html" ignore missing %} diff --git a/frappe/translate.py b/frappe/translate.py index 62ee733f5f..4baf4bdd89 100644 --- a/frappe/translate.py +++ b/frappe/translate.py @@ -115,7 +115,7 @@ def get_dict(fortype, name=None): messages.extend(get_server_messages(app)) messages = deduplicate_messages(messages) - messages += frappe.db.sql("""select "navbar", item_label from `tabNavbar Item` where item_label is not null""") + messages += frappe.db.sql("""select 'navbar', item_label from `tabNavbar Item` where item_label is not null""") messages = get_messages_from_include_files() messages += frappe.db.sql("select 'Print Format:', name from `tabPrint Format`") messages += frappe.db.sql("select 'DocType:', name from tabDocType") @@ -518,8 +518,13 @@ def get_messages_from_file(path): apps_path = get_bench_dir() if os.path.exists(path): with open(path, 'r') as sourcefile: + try: + file_contents = sourcefile.read() + except Exception: + print("Could not scan file for translation: {0}".format(path)) + return [] data = [(os.path.relpath(path, apps_path), message, context, line) \ - for line, message, context in extract_messages_from_code(sourcefile.read())] + for line, message, context in extract_messages_from_code(file_contents)] return data else: # print "Translate: {0} missing".format(os.path.abspath(path)) @@ -601,11 +606,23 @@ def write_csv_file(path, app_messages, lang_dict): from csv import writer with open(path, 'w', newline='') as msgfile: w = writer(msgfile, lineterminator='\n') - for p, m in app_messages: - t = lang_dict.get(m, '') + + for app_message in app_messages: + context = None + if len(app_message) == 2: + path, message = app_message + elif len(app_message) == 3: + path, message, lineno = app_message + elif len(app_message) == 4: + path, message, context, lineno = app_message + else: + continue + + t = lang_dict.get(message, '') # strip whitespaces - t = re.sub('{\s?([0-9]+)\s?}', "{\g<1>}", t) - w.writerow([p if p else '', m, t]) + translated_string = re.sub('{\s?([0-9]+)\s?}', "{\g<1>}", t) + if translated_string: + w.writerow([message, translated_string, context]) def get_untranslated(lang, untranslated_file, get_all=False): """Returns all untranslated strings for a language and writes in a file @@ -827,7 +844,7 @@ def get_all_languages(with_language_name=False): return frappe.db.sql_list('select name from tabLanguage') def get_all_language_with_name(): - return frappe.db.get_all('language', ['language_code', 'language_name']) + return frappe.db.get_all('Language', ['language_code', 'language_name']) if not frappe.db: frappe.connect() diff --git a/frappe/utils/__init__.py b/frappe/utils/__init__.py index efa69d4453..251a095343 100644 --- a/frappe/utils/__init__.py +++ b/frappe/utils/__init__.py @@ -307,14 +307,23 @@ def unesc(s, esc_chars): s = s.replace(esc_str, c) return s -def execute_in_shell(cmd, verbose=0): +def execute_in_shell(cmd, verbose=0, low_priority=False): # using Popen instead of os.system - as recommended by python docs import tempfile from subprocess import Popen with tempfile.TemporaryFile() as stdout: with tempfile.TemporaryFile() as stderr: - p = Popen(cmd, shell=True, stdout=stdout, stderr=stderr) + kwargs = { + "shell": True, + "stdout": stdout, + "stderr": stderr + } + + if low_priority: + kwargs["preexec_fn"] = lambda: os.nice(10) + + p = Popen(cmd, **kwargs) p.wait() stdout.seek(0) diff --git a/frappe/utils/backups.py b/frappe/utils/backups.py index 77c5761527..b21efc5e89 100644 --- a/frappe/utils/backups.py +++ b/frappe/utils/backups.py @@ -15,7 +15,7 @@ import click # imports - module imports import frappe from frappe import _, conf -from frappe.utils import get_file_size, get_url, now, now_datetime +from frappe.utils import get_file_size, get_url, now, now_datetime, cint # backup variable for backwards compatibility verbose = False @@ -315,8 +315,6 @@ class BackupGenerator: print(template.format(_type.title(), info["path"], info["size"])) def backup_files(self): - import subprocess - for folder in ("public", "private"): files_path = frappe.get_site_path(folder, "files") backup_path = ( @@ -327,12 +325,12 @@ class BackupGenerator: cmd_string = "tar cf - {1} | gzip > {0}" else: cmd_string = "tar -cf {0} {1}" - output = subprocess.check_output( - cmd_string.format(backup_path, files_path), shell=True - ) - if self.verbose and output: - print(output.decode("utf8")) + frappe.utils.execute_in_shell( + cmd_string.format(backup_path, files_path), + verbose=self.verbose, + low_priority=True + ) def copy_site_config(self): site_config_backup_path = self.backup_path_conf @@ -436,7 +434,7 @@ class BackupGenerator: if self.verbose: print(command + "\n") - err, out = frappe.utils.execute_in_shell(command) + frappe.utils.execute_in_shell(command, low_priority=True) def send_email(self): """ @@ -474,29 +472,6 @@ download only after 24 hours.""" % { return recipient_list -@frappe.whitelist() -def get_backup(): - """ - This function is executed when the user clicks on - Toos > Download Backup - """ - delete_temp_backups() - odb = BackupGenerator( - frappe.conf.db_name, - frappe.conf.db_name, - frappe.conf.db_password, - db_host=frappe.db.host, - db_type=frappe.conf.db_type, - db_port=frappe.conf.db_port, - ) - odb.get_backup() - recipient_list = odb.send_email() - frappe.msgprint( - _( - "Download link for your backup will be emailed on the following email address: {0}" - ).format(", ".join(recipient_list)) - ) - @frappe.whitelist() def fetch_latest_backups(partial=False): """Fetches paths of the latest backup taken in the last 30 days @@ -570,7 +545,7 @@ def new_backup( force=False, verbose=False, ): - delete_temp_backups(older_than=frappe.conf.keep_backups_for_hours or 24) + delete_temp_backups() odb = BackupGenerator( frappe.conf.db_name, frappe.conf.db_name, @@ -595,8 +570,9 @@ def new_backup( def delete_temp_backups(older_than=24): """ - Cleans up the backup_link_path directory by deleting files older than 24 hours + Cleans up the backup_link_path directory by deleting older files """ + older_than = cint(frappe.conf.keep_backups_for_hours) or older_than backup_path = get_backup_path() if os.path.exists(backup_path): file_list = os.listdir(get_backup_path()) diff --git a/frappe/utils/boilerplate.py b/frappe/utils/boilerplate.py index 0b4e5dec21..7c7df1cb8e 100755 --- a/frappe/utils/boilerplate.py +++ b/frappe/utils/boilerplate.py @@ -190,6 +190,15 @@ app_license = "{app_license}" # automatically create page for each record of this doctype # website_generators = ["Web Page"] +# Jinja +# ---------- + +# add methods and filters to jinja environment +# jinja = {{ +# "methods": "{app_name}.utils.jinja_methods", +# "filters": "{app_name}.utils.jinja_filters" +# }} + # Installation # ------------ @@ -303,6 +312,13 @@ user_data_fields = [ }} ] +# Authentication and authorization +# -------------------------------- + +# auth_hooks = [ +# "{app_name}.auth.validate" +# ] + """ desktop_template = """# -*- coding: utf-8 -*- diff --git a/frappe/utils/jinja.py b/frappe/utils/jinja.py index cd74b2a283..a77eca4977 100644 --- a/frappe/utils/jinja.py +++ b/frappe/utils/jinja.py @@ -18,13 +18,10 @@ def get_jenv(): set_filters(jenv) jenv.globals.update(get_safe_globals()) - jenv.globals.update(get_jenv_customization('methods')) - jenv.globals.update({ - 'resolve_class': resolve_class, - 'inspect': inspect, - 'web_blocks': web_blocks, - 'web_block': web_block - }) + + methods, filters = get_jinja_hooks() + jenv.globals.update(methods or {}) + jenv.filters.update(filters or {}) frappe.local.jenv = jenv @@ -127,104 +124,49 @@ def get_jloader(): def set_filters(jenv): import frappe - from frappe.utils import global_date_format, cint, cstr, flt, markdown - from frappe.website.utils import get_shade, abs_url + from frappe.utils import cint, cstr, flt - jenv.filters["global_date_format"] = global_date_format - jenv.filters["markdown"] = markdown jenv.filters["json"] = frappe.as_json - jenv.filters["get_shade"] = get_shade jenv.filters["len"] = len jenv.filters["int"] = cint jenv.filters["str"] = cstr jenv.filters["flt"] = flt - jenv.filters["abs_url"] = abs_url if frappe.flags.in_setup_help: return - jenv.filters.update(get_jenv_customization('filters')) - - -def get_jenv_customization(customization_type): - '''Returns a dict with filter/method name as key and definition as value''' +def get_jinja_hooks(): + """Returns a tuple of (methods, filters) each containing a dict of method name and method definition pair.""" import frappe - out = {} if not getattr(frappe.local, "site", None): + return (None, None) + + from types import FunctionType, ModuleType + from inspect import getmembers, isfunction + + def get_obj_dict_from_paths(object_paths): + out = {} + for obj_path in object_paths: + try: + obj = frappe.get_module(obj_path) + except ModuleNotFoundError: + obj = frappe.get_attr(obj_path) + + if isinstance(obj, ModuleType): + functions = getmembers(obj, isfunction) + for function_name, function in functions: + out[function_name] = function + elif isinstance(obj, FunctionType): + function_name = obj.__name__ + out[function_name] = obj return out - values = frappe.get_hooks("jenv", {}).get(customization_type) - if not values: - return out + values = frappe.get_hooks("jinja") + methods, filters = values.get("methods", []), values.get("filters", []) - for value in values: - fn_name, fn_string = value.split(":") - out[fn_name] = frappe.get_attr(fn_string) + method_dict = get_obj_dict_from_paths(methods) + filter_dict = get_obj_dict_from_paths(filters) - return out - - -def resolve_class(classes): - import frappe - - if classes is None: - return '' - - if isinstance(classes, frappe.string_types): - return classes - - if isinstance(classes, (list, tuple)): - return ' '.join([resolve_class(c) for c in classes]).strip() - - if isinstance(classes, dict): - return ' '.join([classname for classname in classes if classes[classname]]).strip() - - return classes - - -def inspect(var, render=True): - context = { "var": var } - if render: - html = "
{{ var | pprint | e }}
" - else: - html = "" - return get_jenv().from_string(html).render(context) - - -def web_block(template, values=None, **kwargs): - options = {"template": template, "values": values} - options.update(kwargs) - return web_blocks([options]) - - -def web_blocks(blocks): - from frappe import throw, _dict - from frappe.website.doctype.web_page.web_page import get_web_blocks_html - - web_blocks = [] - for block in blocks: - if not block.get('template'): - throw('Web Template is not specified') - - doc = _dict({ - 'doctype': 'Web Page Block', - 'web_template': block['template'], - 'web_template_values': block.get('values', {}), - 'add_top_padding': 1, - 'add_bottom_padding': 1, - 'add_container': 1, - 'hide_block': 0, - 'css_class': '' - }) - doc.update(block) - web_blocks.append(doc) - - out = get_web_blocks_html(web_blocks) - - html = out.html - for script in out.scripts: - html += ''.format(script) - - return html + return method_dict, filter_dict diff --git a/frappe/utils/jinja_globals.py b/frappe/utils/jinja_globals.py new file mode 100644 index 0000000000..e63926a109 --- /dev/null +++ b/frappe/utils/jinja_globals.py @@ -0,0 +1,71 @@ +# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors +# MIT License. See license.txt + +from __future__ import unicode_literals +from frappe.utils.jinja import get_jenv +import frappe + + +def resolve_class(classes): + if classes is None: + return "" + + if isinstance(classes, frappe.string_types): + return classes + + if isinstance(classes, (list, tuple)): + return " ".join([resolve_class(c) for c in classes]).strip() + + if isinstance(classes, dict): + return " ".join([classname for classname in classes if classes[classname]]).strip() + + return classes + + +def inspect(var, render=True): + context = {"var": var} + if render: + html = "
{{ var | pprint | e }}
" + else: + return "" + return get_jenv().from_string(html).render(context) + + +def web_block(template, values=None, **kwargs): + options = {"template": template, "values": values} + options.update(kwargs) + return web_blocks([options]) + + +def web_blocks(blocks): + from frappe import throw, _dict + from frappe.website.doctype.web_page.web_page import get_web_blocks_html + + web_blocks = [] + for block in blocks: + if not block.get("template"): + throw("Web Template is not specified") + + doc = _dict( + { + "doctype": "Web Page Block", + "web_template": block["template"], + "web_template_values": block.get("values", {}), + "add_top_padding": 1, + "add_bottom_padding": 1, + "add_container": 1, + "hide_block": 0, + "css_class": "", + } + ) + doc.update(block) + web_blocks.append(doc) + + out = get_web_blocks_html(web_blocks) + + html = out.html + for script in out.scripts: + html += "".format(script) + + return html + diff --git a/frappe/utils/oauth.py b/frappe/utils/oauth.py index 6596701ee3..6a92737a0d 100644 --- a/frappe/utils/oauth.py +++ b/frappe/utils/oauth.py @@ -64,8 +64,6 @@ def get_oauth2_authorize_url(provider, redirect_to): state = { "site": frappe.utils.get_url(), "token": frappe.generate_hash(), "redirect_to": redirect_to } - frappe.cache().set_value("{0}:{1}".format(provider, state["token"]), True, expires_in_sec=120) - # relative to absolute url data = { "redirect_uri": get_redirect_uri(provider), @@ -176,11 +174,6 @@ def login_oauth_user(data=None, provider=None, state=None, email_id=None, key=No frappe.respond_as_web_page(_("Invalid Request"), _("Token is missing"), http_status_code=417) return - token = frappe.cache().get_value("{0}:{1}".format(provider, state["token"]), expires=True) - if not token: - frappe.respond_as_web_page(_("Invalid Request"), _("Invalid Token"), http_status_code=417) - return - user = get_email(data) if not user: diff --git a/frappe/utils/xlsxutils.py b/frappe/utils/xlsxutils.py index 3c7b027470..356e2ddfdb 100644 --- a/frappe/utils/xlsxutils.py +++ b/frappe/utils/xlsxutils.py @@ -1,18 +1,19 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals - -import frappe +import re +from io import BytesIO import openpyxl import xlrd -import re -from openpyxl.styles import Font from openpyxl import load_workbook +from openpyxl.styles import Font from openpyxl.utils import get_column_letter -from six import BytesIO, string_types + +import frappe ILLEGAL_CHARACTERS_RE = re.compile(r'[\000-\010]|[\013-\014]|[\016-\037]') + + # return xlsx file object def make_xlsx(data, sheet_name, wb=None, column_widths=None): column_widths = column_widths or [] @@ -31,12 +32,12 @@ def make_xlsx(data, sheet_name, wb=None, column_widths=None): for row in data: clean_row = [] for item in row: - if isinstance(item, string_types) and (sheet_name not in ['Data Import Template', 'Data Export']): + if isinstance(item, str) and (sheet_name not in ['Data Import Template', 'Data Export']): value = handle_html(item) else: value = item - if isinstance(item, string_types) and next(ILLEGAL_CHARACTERS_RE.finditer(value), None): + if isinstance(item, str) and next(ILLEGAL_CHARACTERS_RE.finditer(value), None): # Remove illegal characters from the string value = re.sub(ILLEGAL_CHARACTERS_RE, '', value) @@ -80,12 +81,12 @@ def handle_html(data): return value + def read_xlsx_file_from_attached_file(file_url=None, fcontent=None, filepath=None): if file_url: _file = frappe.get_doc("File", {"file_url": file_url}) filename = _file.get_full_path() elif fcontent: - from io import BytesIO filename = BytesIO(fcontent) elif filepath: filename = filepath @@ -102,6 +103,7 @@ def read_xlsx_file_from_attached_file(file_url=None, fcontent=None, filepath=Non rows.append(tmp_list) return rows + def read_xls_file_from_attached_file(content): book = xlrd.open_workbook(file_contents=content) sheets = book.sheets() @@ -111,6 +113,7 @@ def read_xls_file_from_attached_file(content): rows.append(sheet.row_values(i)) return rows + def build_xlsx_response(data, filename): xlsx_file = make_xlsx(data, filename) # write out response as a xlsx type diff --git a/frappe/website/doctype/web_page/web_page.py b/frappe/website/doctype/web_page/web_page.py index 86774c79c4..cce00564ff 100644 --- a/frappe/website/doctype/web_page/web_page.py +++ b/frappe/website/doctype/web_page/web_page.py @@ -242,11 +242,11 @@ def extract_script_and_style_tags(html): styles = [] for script in soup.find_all('script'): - scripts.append(script.text) + scripts.append(script.string) script.extract() for style in soup.find_all('style'): - styles.append(style.text) + styles.append(style.string) style.extract() return str(soup), scripts, styles diff --git a/frappe/website/doctype/website_settings/google_indexing.py b/frappe/website/doctype/website_settings/google_indexing.py index 599de5a2b6..75095bd7df 100644 --- a/frappe/website/doctype/website_settings/google_indexing.py +++ b/frappe/website/doctype/website_settings/google_indexing.py @@ -2,17 +2,18 @@ # Copyright (c) 2020, Frappe Technologies and contributors # For license information, please see license.txt -from __future__ import unicode_literals -import frappe -import requests -import googleapiclient.discovery -import google.oauth2.credentials -from frappe import _ +from urllib.parse import quote + +import google.oauth2.credentials +import requests +from googleapiclient.discovery import build from googleapiclient.errors import HttpError -from frappe.utils import get_request_site_address -from six.moves.urllib.parse import quote + +import frappe +from frappe import _ from frappe.integrations.doctype.google_settings.google_settings import get_auth_url +from frappe.utils import get_request_site_address SCOPES = "https://www.googleapis.com/auth/indexing" @@ -82,7 +83,12 @@ def get_google_indexing_object(): } credentials = google.oauth2.credentials.Credentials(**credentials_dict) - google_indexing = googleapiclient.discovery.build("indexing", "v3", credentials=credentials) + google_indexing = build( + serviceName="indexing", + version="v3", + credentials=credentials, + static_discovery=False + ) return google_indexing diff --git a/frappe/www/login.py b/frappe/www/login.py index 76b232f8ee..1ce25a81d9 100644 --- a/frappe/www/login.py +++ b/frappe/www/login.py @@ -95,14 +95,6 @@ def login_via_frappe(code, state): def login_via_office365(code, state): login_via_oauth2_id_token("office_365", code, state, decoder=decoder_compat) -@frappe.whitelist(allow_guest=True) -def login_oauth_user(data=None, provider=None, state=None, email_id=None, key=None, generate_login_token=False): - if not ((data and provider and state) or (email_id and key)): - frappe.respond_as_web_page(_("Invalid Request"), _("Missing parameters for login"), http_status_code=417) - return - - _login_oauth_user(data, provider, state, email_id, key, generate_login_token) - @frappe.whitelist(allow_guest=True) def login_via_token(login_token): sid = frappe.cache().get_value("login_token:{0}".format(login_token), expires=True) diff --git a/package.json b/package.json index 3c8da66242..6e82890617 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "driver.js": "^0.9.8", "express": "^4.17.1", "fast-deep-equal": "^2.0.1", - "frappe-charts": "^2.0.0-rc11", + "frappe-charts": "^2.0.0-rc13", "frappe-datatable": "^1.15.3", "frappe-gantt": "^0.5.0", "fuse.js": "^3.4.6", diff --git a/requirements.txt b/requirements.txt index 0f88a48f73..91f235159f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,79 +1,79 @@ -Babel==2.6.0 -beautifulsoup4==4.8.2 -bleach-whitelist==0.0.10 -bleach==3.3.0 -boto3==1.10.18 -braintree==3.57.1 -chardet==3.0.4 -Click==7.0 -coverage==4.5.4 -croniter==0.3.31 -cryptography==3.3.2 -dropbox==9.1.0 -email-reply-parser==0.5.9 -Faker==2.0.4 +Babel~=2.9.0 +beautifulsoup4~=4.9.3 +bleach-whitelist~=0.0.11 +bleach~=3.3.0 +boto3~=1.17.53 +braintree~=4.8.0 +chardet~=4.0.0 +Click~=7.1.2 +coverage~=5.5 +croniter~=1.0.11 +cryptography~=3.4.7 +dropbox~=11.7.0 +email-reply-parser~=0.5.12 +Faker~=8.1.0 future==0.18.2 -gitdb2==2.0.6;python_version<'3.4' -GitPython==2.1.15 -git-url-parse==1.2.2 -google-api-python-client==1.9.3 -google-auth-httplib2==0.0.3 -google-auth-oauthlib==0.4.1 -google-auth==1.18.0 -googlemaps==3.1.1 -gunicorn==19.10.0 -html2text==2016.9.19 -html5lib==1.0.1 -ipython==7.14.0 -jedi==0.17.2 # not directly required. Pinned to fix upstream issue with ipython. -Jinja2==2.11.3 -ldap3==2.7 -markdown2==2.4.0 +git-url-parse~=1.2.2 +gitdb~=4.0.7 +GitPython~=3.1.14 +google-api-python-client~=2.2.0 +google-auth-httplib2~=0.1.0 +google-auth-oauthlib~=0.4.4 +google-auth~=1.29.0 +googlemaps~=4.4.5 +gunicorn~=20.1.0 +html2text==2020.1.16 +html5lib~=1.1 +ipython~=7.16.1 +jedi==0.17.2 # not directly required. Pinned to fix upstream IPython issue (https://github.com/ipython/ipython/issues/12740) +Jinja2~=2.11.3 +ldap3~=2.9 +markdown2~=2.4.0 maxminddb-geolite2==2018.703 -ndg-httpsclient==0.5.1 -num2words==0.5.10 -oauthlib==3.1.0 -openpyxl==2.6.4 -passlib==1.7.3 -pdfkit==0.6.1 -Pillow>=8.0.0 -premailer==3.6.1 -psutil==5.7.2 -psycopg2-binary==2.8.4 -pyasn1==0.4.8 -PyJWT==1.7.1 -PyMySQL==0.9.3 -pyngrok==4.1.6 -pyOpenSSL==19.1.0 -pyotp==2.3.0 -PyPDF2==1.26.0 -pypng==0.0.20 -PyQRCode==1.2.1 -python-dateutil==2.8.1 -pytz==2019.3 -PyYAML==5.4 -rauth==0.7.3 -redis==3.5.3 -requests-oauthlib==1.3.0 -requests==2.23.0 -RestrictedPython==5.0 -rq>=1.1.0 -schedule==0.6.0 -semantic-version==2.8.4 -simple-chalk==0.1.0 -six==1.14.0 -sqlparse==0.2.4 -stripe==2.40.0 -terminaltables==3.1.0 -unittest-xml-reporting==2.5.2 -urllib3==1.25.9 -watchdog==0.8.0 -Werkzeug==0.16.1 -Whoosh==2.7.4 -xlrd==1.2.0 -zxcvbn-python==4.4.24 -pycryptodome==3.9.8 -paytmchecksum==1.7.0 -wrapt==1.10.11 -razorpay==1.2.0 +ndg-httpsclient~=0.5.1 +num2words~=0.5.10 +oauthlib~=3.1.0 +openpyxl~=3.0.7 +passlib~=1.7.4 +paytmchecksum~=1.7.0 +pdfkit~=0.6.1 +Pillow~=8.2.0 +premailer~=3.8.0 +psutil~=5.8.0 +psycopg2-binary~=2.8.6 +pyasn1~=0.4.8 +pycryptodome~=3.10.1 +PyJWT~=1.7.1 +PyMySQL~=1.0.2 +pyngrok~=5.0.5 +pyOpenSSL~=20.0.1 +pyotp~=2.6.0 +PyPDF2~=1.26.0 +pypng~=0.0.20 +PyQRCode~=1.2.1 +python-dateutil~=2.8.1 +pytz==2021.1 +PyYAML~=5.4.1 +rauth~=0.7.3 +razorpay~=1.2.0 +redis~=3.5.3 +requests-oauthlib~=1.3.0 +requests~=2.25.1 +RestrictedPython~=5.1 +rq~=1.8.0 rsa>=4.1 # not directly required, pinned by Snyk to avoid a vulnerability +schedule~=1.1.0 +semantic-version~=2.8.5 +simple-chalk~=0.1.0 +six~=1.15.0 +sqlparse~=0.4.1 +stripe~=2.56.0 +terminaltables~=3.1.0 +unittest-xml-reporting~=3.0.4 +urllib3~=1.26.4 +watchdog~=2.0.2 +Werkzeug~=0.16.1 +Whoosh~=2.7.4 +wrapt~=1.12.1 +xlrd~=2.0.1 +zxcvbn-python~=4.4.24 diff --git a/yarn.lock b/yarn.lock index 4f6f62ac0a..8ac348011d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2699,10 +2699,10 @@ fragment-cache@^0.2.1: dependencies: map-cache "^0.2.2" -frappe-charts@^2.0.0-rc11: - version "2.0.0-rc11" - resolved "https://registry.yarnpkg.com/frappe-charts/-/frappe-charts-2.0.0-rc11.tgz#0724fa0d43593362c075c3805ebbbe1a608fcef7" - integrity sha512-DY3tThT1lNGcJlRMOtIhnILtSm5h1iKysWhZAyj7yrGiOnOWbZpYx/NZzXZYwtRrWwMlYiLX2ylV76qo31ONsg== +frappe-charts@^2.0.0-rc13: + version "2.0.0-rc13" + resolved "https://registry.yarnpkg.com/frappe-charts/-/frappe-charts-2.0.0-rc13.tgz#fdb251d7ae311c41e38f90a3ae108070ec6b9072" + integrity sha512-Bv7IfllIrjRbKWHn5b769dOSenqdBixAr6m5kurf8ZUOJSLOgK4HOXItJ7BA8n9PvviH9/k5DaloisjLM2Bm1w== frappe-datatable@^1.15.3: version "1.15.3"