diff --git a/.flake8 b/.flake8 index 399b176e1d..56c9b9a369 100644 --- a/.flake8 +++ b/.flake8 @@ -29,4 +29,5 @@ ignore = B950, W191, -max-line-length = 200 \ No newline at end of file +max-line-length = 200 +exclude=.github/helper/semgrep_rules diff --git a/.github/helper/semgrep_rules/frappe_correctness.py b/.github/helper/semgrep_rules/frappe_correctness.py index 37889fbbb1..745e6463b8 100644 --- a/.github/helper/semgrep_rules/frappe_correctness.py +++ b/.github/helper/semgrep_rules/frappe_correctness.py @@ -4,25 +4,61 @@ from frappe import _, flt from frappe.model.document import Document +# ruleid: frappe-modifying-but-not-comitting def on_submit(self): if self.value_of_goods == 0: frappe.throw(_('Value of goods cannot be 0')) - # ruleid: frappe-modifying-after-submit self.status = 'Submitted' -def on_submit(self): # noqa - if flt(self.per_billed) < 100: - self.update_billing_status() - else: - # todook: frappe-modifying-after-submit - self.status = "Completed" - self.db_set("status", "Completed") -class TestDoc(Document): - pass +# ok: frappe-modifying-but-not-comitting +def on_submit(self): + if self.value_of_goods == 0: + frappe.throw(_('Value of goods cannot be 0')) + self.status = 'Submitted' + self.db_set('status', 'Submitted') - def validate(self): - #ruleid: frappe-modifying-child-tables-while-iterating - for item in self.child_table: - if item.value < 0: - self.remove(item) +# ok: frappe-modifying-but-not-comitting +def on_submit(self): + if self.value_of_goods == 0: + frappe.throw(_('Value of goods cannot be 0')) + x = "y" + self.status = x + self.db_set('status', x) + + +# ok: frappe-modifying-but-not-comitting +def on_submit(self): + x = "y" + self.status = x + self.save() + +# ruleid: frappe-modifying-but-not-comitting-other-method +class DoctypeClass(Document): + def on_submit(self): + self.good_method() + self.tainted_method() + + def tainted_method(self): + self.status = "uptate" + + +# ok: frappe-modifying-but-not-comitting-other-method +class DoctypeClass(Document): + def on_submit(self): + self.good_method() + self.tainted_method() + + def tainted_method(self): + self.status = "update" + self.db_set("status", "update") + +# ok: frappe-modifying-but-not-comitting-other-method +class DoctypeClass(Document): + def on_submit(self): + self.good_method() + self.tainted_method() + self.save() + + def tainted_method(self): + self.status = "uptate" diff --git a/.github/helper/semgrep_rules/translate.js b/.github/helper/semgrep_rules/translate.js index 7b92fe2dff..9cdfb75d0b 100644 --- a/.github/helper/semgrep_rules/translate.js +++ b/.github/helper/semgrep_rules/translate.js @@ -35,3 +35,10 @@ __('You have' + 'subscribers in your mailing list.') // ruleid: frappe-translation-js-splitting __('You have {0} subscribers' + 'in your mailing list', [subscribers.length]) + +// ok: frappe-translation-js-splitting +__("Ctrl+Enter to add comment") + +// ruleid: frappe-translation-js-splitting +__('You have {0} subscribers \ + in your mailing list', [subscribers.length]) diff --git a/.github/helper/semgrep_rules/translate.py b/.github/helper/semgrep_rules/translate.py index bd6cd9126c..9de6aa94f0 100644 --- a/.github/helper/semgrep_rules/translate.py +++ b/.github/helper/semgrep_rules/translate.py @@ -51,3 +51,11 @@ _(f"what" + f"this is also not cool") _("") # ruleid: frappe-translation-empty-string _('') + + +class Test: + # ok: frappe-translation-python-splitting + def __init__( + args + ): + pass diff --git a/.github/helper/semgrep_rules/translate.yml b/.github/helper/semgrep_rules/translate.yml index 7754b52efc..5f03fb9fd0 100644 --- a/.github/helper/semgrep_rules/translate.yml +++ b/.github/helper/semgrep_rules/translate.yml @@ -44,8 +44,8 @@ rules: pattern-either: - pattern: _(...) + _(...) - pattern: _("..." + "...") - - pattern-regex: '_\([^\)]*\\\s*' # lines broken by `\` - - pattern-regex: '_\(\s*\n' # line breaks allowed by python for using ( ) + - pattern-regex: '[\s\.]_\([^\)]*\\\s*' # lines broken by `\` + - pattern-regex: '[\s\.]_\(\s*\n' # line breaks allowed by python for using ( ) message: | Do not split strings inside translate function. Do not concatenate using translate functions. Please refer: https://frappeframework.com/docs/user/en/translations @@ -54,8 +54,8 @@ rules: - id: frappe-translation-js-splitting pattern-either: - - pattern-regex: '__\([^\)]*[\+\\]\s*' - - pattern: __('...' + '...') + - pattern-regex: '__\([^\)]*[\\]\s+' + - pattern: __('...' + '...', ...) - pattern: __('...') + __('...') message: | Do not split strings inside translate function. Do not concatenate using translate functions. diff --git a/.github/workflows/semgrep.yml b/.github/workflows/semgrep.yml index 5092bf4705..389524e968 100644 --- a/.github/workflows/semgrep.yml +++ b/.github/workflows/semgrep.yml @@ -4,6 +4,8 @@ on: pull_request: branches: - develop + - version-13-hotfix + - version-13-pre-release jobs: semgrep: name: Frappe Linter diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/server-mariadb-tests.yml similarity index 52% rename from .github/workflows/ci-tests.yml rename to .github/workflows/server-mariadb-tests.yml index 7ac80c5708..1742e813c6 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/server-mariadb-tests.yml @@ -1,10 +1,8 @@ -name: CI +name: Server on: pull_request: - types: [opened, synchronize, reopened, labeled, unlabeled] workflow_dispatch: - push: jobs: test: @@ -13,23 +11,9 @@ jobs: strategy: fail-fast: false matrix: - include: - - DB: "mariadb" - TYPE: "server" - JOB_NAME: "Python MariaDB" - RUN_COMMAND: bench --site test_site run-tests --coverage + container: [1, 2] - - DB: "postgres" - TYPE: "server" - JOB_NAME: "Python PostgreSQL" - RUN_COMMAND: bench --site test_site run-tests --coverage - - - DB: "mariadb" - TYPE: "ui" - JOB_NAME: "UI MariaDB" - RUN_COMMAND: bench --site test_site run-ui-tests frappe --headless - - name: ${{ matrix.JOB_NAME }} + name: Python Unit Tests (MariaDB) services: mysql: @@ -40,18 +24,6 @@ jobs: - 3306:3306 options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=3 - postgres: - image: postgres:12.4 - env: - POSTGRES_PASSWORD: travis - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 5432:5432 - steps: - name: Clone uses: actions/checkout@v2 @@ -104,68 +76,54 @@ jobs: restore-keys: | ${{ runner.os }}-yarn- - - name: Cache cypress binary - if: matrix.TYPE == 'ui' - uses: actions/cache@v2 - with: - path: ~/.cache - key: ${{ runner.os }}-cypress- - restore-keys: | - ${{ runner.os }}-cypress- - ${{ runner.os }}- - - name: Install Dependencies run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh env: BEFORE: ${{ env.GITHUB_EVENT_PATH.before }} AFTER: ${{ env.GITHUB_EVENT_PATH.after }} - TYPE: ${{ matrix.TYPE }} + TYPE: server - name: Install run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh env: - DB: ${{ matrix.DB }} - TYPE: ${{ matrix.TYPE }} + DB: mariadb + TYPE: server - - name: Run Set-Up - if: matrix.TYPE == 'ui' - run: cd ~/frappe-bench/ && bench --site test_site execute frappe.utils.install.complete_setup_wizard - env: - DB: ${{ matrix.DB }} - TYPE: ${{ matrix.TYPE }} - - - name: Setup tmate session - if: contains(github.event.pull_request.labels.*.name, 'debug-gha') - uses: mxschmitt/action-tmate@v3 - name: Run Tests - run: cd ~/frappe-bench/ && ${{ matrix.RUN_COMMAND }} + run: cd ~/frappe-bench/ && bench --site test_site run-parallel-tests --use-orchestrator --with-coverage env: - DB: ${{ matrix.DB }} - TYPE: ${{ matrix.TYPE }} + CI_BUILD_ID: ${{ github.run_id }} + ORCHESTRATOR_URL: http://test-orchestrator.frappe.io - - name: Coverage - Pull Request - if: matrix.TYPE == 'server' && github.event_name == 'pull_request' + - name: Upload Coverage Data run: | cp ~/frappe-bench/sites/.coverage ${GITHUB_WORKSPACE} cd ${GITHUB_WORKSPACE} - pip install coveralls==2.2.0 - pip install coverage==4.5.4 - coveralls --service=github + pip3 install coverage==5.5 + pip3 install coveralls==3.0.1 + coveralls env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_TOKEN }} - COVERALLS_SERVICE_NAME: github + COVERALLS_FLAG_NAME: run-${{ matrix.container }} + COVERALLS_SERVICE_NAME: ${{ github.event_name == 'pull_request' && 'github' || 'github-actions' }} + COVERALLS_PARALLEL: true - - name: Coverage - Push - if: matrix.TYPE == 'server' && github.event_name == 'push' + coveralls: + name: Coverage Wrap Up + needs: test + container: python:3-slim + runs-on: ubuntu-18.04 + steps: + - name: Clone + uses: actions/checkout@v2 + + - name: Coveralls Finished run: | - cp ~/frappe-bench/sites/.coverage ${GITHUB_WORKSPACE} cd ${GITHUB_WORKSPACE} - pip install coveralls==2.2.0 - pip install coverage==4.5.4 - coveralls --service=github-actions + pip3 install coverage==5.5 + pip3 install coveralls==3.0.1 + coveralls --finish env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_TOKEN }} - COVERALLS_SERVICE_NAME: github-actions diff --git a/.github/workflows/server-postgres-tests.yml b/.github/workflows/server-postgres-tests.yml new file mode 100644 index 0000000000..4325eebaad --- /dev/null +++ b/.github/workflows/server-postgres-tests.yml @@ -0,0 +1,100 @@ +name: Server + +on: + pull_request: + workflow_dispatch: + +jobs: + test: + runs-on: ubuntu-18.04 + + strategy: + fail-fast: false + matrix: + container: [1, 2] + + name: Python Unit Tests (Postgres) + + services: + postgres: + image: postgres:12.4 + env: + POSTGRES_PASSWORD: travis + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + + steps: + - name: Clone + uses: actions/checkout@v2 + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.7 + + - uses: actions/setup-node@v2 + with: + node-version: '14' + check-latest: true + + - name: Add to Hosts + run: | + echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts + echo "127.0.0.1 test_site_producer" | sudo tee -a /etc/hosts + + - name: Cache pip + uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + ${{ runner.os }}- + + - name: Cache node modules + uses: actions/cache@v2 + env: + cache-name: cache-node-modules + with: + path: ~/.npm + key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-build-${{ env.cache-name }}- + ${{ runner.os }}-build- + ${{ runner.os }}- + + - name: Get yarn cache directory path + id: yarn-cache-dir-path + run: echo "::set-output name=dir::$(yarn cache dir)" + + - uses: actions/cache@v2 + id: yarn-cache + with: + path: ${{ steps.yarn-cache-dir-path.outputs.dir }} + key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} + restore-keys: | + ${{ runner.os }}-yarn- + + - name: Install Dependencies + run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh + env: + BEFORE: ${{ env.GITHUB_EVENT_PATH.before }} + AFTER: ${{ env.GITHUB_EVENT_PATH.after }} + TYPE: server + + - name: Install + run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh + env: + DB: postgres + TYPE: server + + - name: Run Tests + run: cd ~/frappe-bench/ && bench --site test_site run-parallel-tests --use-orchestrator + env: + CI_BUILD_ID: ${{ github.run_id }} + ORCHESTRATOR_URL: http://test-orchestrator.frappe.io diff --git a/.github/workflows/ui-tests.yml b/.github/workflows/ui-tests.yml new file mode 100644 index 0000000000..9eea128cd1 --- /dev/null +++ b/.github/workflows/ui-tests.yml @@ -0,0 +1,105 @@ +name: UI + +on: + pull_request: + workflow_dispatch: + +jobs: + test: + runs-on: ubuntu-18.04 + + strategy: + fail-fast: false + matrix: + containers: [1, 2] + + name: UI Tests (Cypress) + + services: + mysql: + image: mariadb:10.3 + env: + MYSQL_ALLOW_EMPTY_PASSWORD: YES + ports: + - 3306:3306 + options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=3 + + steps: + - name: Clone + uses: actions/checkout@v2 + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.7 + + - uses: actions/setup-node@v2 + with: + node-version: '12' + check-latest: true + + - name: Add to Hosts + run: | + echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts + echo "127.0.0.1 test_site_producer" | sudo tee -a /etc/hosts + + - name: Cache pip + uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + ${{ runner.os }}- + + - name: Cache node modules + uses: actions/cache@v2 + env: + cache-name: cache-node-modules + with: + path: ~/.npm + key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-build-${{ env.cache-name }}- + ${{ runner.os }}-build- + ${{ runner.os }}- + + - name: Get yarn cache directory path + id: yarn-cache-dir-path + run: echo "::set-output name=dir::$(yarn cache dir)" + + - uses: actions/cache@v2 + id: yarn-cache + with: + path: ${{ steps.yarn-cache-dir-path.outputs.dir }} + key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} + restore-keys: | + ${{ runner.os }}-yarn- + + - name: Cache cypress binary + uses: actions/cache@v2 + with: + path: ~/.cache + key: ${{ runner.os }}-cypress- + restore-keys: | + ${{ runner.os }}-cypress- + ${{ runner.os }}- + + - name: Install Dependencies + run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh + env: + BEFORE: ${{ env.GITHUB_EVENT_PATH.before }} + AFTER: ${{ env.GITHUB_EVENT_PATH.after }} + TYPE: ui + + - name: Install + run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh + env: + DB: mariadb + TYPE: ui + + - name: Site Setup + run: cd ~/frappe-bench/ && bench --site test_site execute frappe.utils.install.complete_setup_wizard + + - name: UI Tests + run: cd ~/frappe-bench/ && bench --site test_site run-ui-tests frappe --headless --parallel --ci-build-id $GITHUB_RUN_ID diff --git a/.mergify.yml b/.mergify.yml index 82f710a5a8..c759c1e3ec 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -3,9 +3,12 @@ pull_request_rules: conditions: - status-success=Sider - status-success=Semantic Pull Request - - status-success=Python MariaDB - - status-success=Python PostgreSQL - - status-success=UI MariaDB + - status-success=Python Unit Tests (MariaDB) (1) + - status-success=Python Unit Tests (MariaDB) (2) + - status-success=Python Unit Tests (Postgres) (1) + - status-success=Python Unit Tests (Postgres) (2) + - status-success=UI Tests (Cypress) (1) + - status-success=UI Tests (Cypress) (2) - status-success=security/snyk (frappe) - label!=dont-merge - label!=squash @@ -16,9 +19,12 @@ pull_request_rules: - name: Automatic squash on CI success and review conditions: - status-success=Sider - - status-success=Python MariaDB - - status-success=Python PostgreSQL - - status-success=UI MariaDB + - status-success=Python Unit Tests (MariaDB) (1) + - status-success=Python Unit Tests (MariaDB) (2) + - status-success=Python Unit Tests (Postgres) (1) + - status-success=Python Unit Tests (Postgres) (2) + - status-success=UI Tests (Cypress) (1) + - status-success=UI Tests (Cypress) (2) - status-success=security/snyk (frappe) - label!=dont-merge - label=squash diff --git a/frappe/app.py b/frappe/app.py index a72f343532..64befdf531 100644 --- a/frappe/app.py +++ b/frappe/app.py @@ -99,17 +99,7 @@ def application(request): frappe.monitor.stop(response) frappe.recorder.dump() - if hasattr(frappe.local, 'conf') and frappe.local.conf.enable_frappe_logger: - frappe.logger("frappe.web", allow_site=frappe.local.site).info({ - "site": get_site_name(request.host), - "remote_addr": getattr(request, "remote_addr", "NOTFOUND"), - "base_url": getattr(request, "base_url", "NOTFOUND"), - "full_path": getattr(request, "full_path", "NOTFOUND"), - "method": getattr(request, "method", "NOTFOUND"), - "scheme": getattr(request, "scheme", "NOTFOUND"), - "http_status_code": getattr(response, "status_code", "NOTFOUND") - }) - + log_request(request, response) process_response(response) frappe.destroy() @@ -137,6 +127,19 @@ def init_request(request): if request.method != "OPTIONS": frappe.local.http_request = frappe.auth.HTTPRequest() +def log_request(request, response): + if hasattr(frappe.local, 'conf') and frappe.local.conf.enable_frappe_logger: + frappe.logger("frappe.web", allow_site=frappe.local.site).info({ + "site": get_site_name(request.host), + "remote_addr": getattr(request, "remote_addr", "NOTFOUND"), + "base_url": getattr(request, "base_url", "NOTFOUND"), + "full_path": getattr(request, "full_path", "NOTFOUND"), + "method": getattr(request, "method", "NOTFOUND"), + "scheme": getattr(request, "scheme", "NOTFOUND"), + "http_status_code": getattr(response, "status_code", "NOTFOUND") + }) + + def process_response(response): if not response: return diff --git a/frappe/build.py b/frappe/build.py index 10c70de9e4..db9599fdb6 100644 --- a/frappe/build.py +++ b/frappe/build.py @@ -1,14 +1,11 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import print_function, unicode_literals - import os import re import json import shutil -import warnings -import tempfile +from tempfile import mkdtemp, mktemp from distutils.spawn import find_executable import frappe @@ -16,8 +13,8 @@ from frappe.utils.minify import JavascriptMinify import click import psutil -from six import iteritems, text_type -from six.moves.urllib.parse import urlparse +from urllib.parse import urlparse +from simple_chalk import green timestamps = {} @@ -76,8 +73,8 @@ def get_assets_link(frappe_head): from requests import head tag = getoutput( - "cd ../apps/frappe && git show-ref --tags -d | grep %s | sed -e 's,.*" - " refs/tags/,,' -e 's/\^{}//'" + r"cd ../apps/frappe && git show-ref --tags -d | grep %s | sed -e 's,.*" + r" refs/tags/,,' -e 's/\^{}//'" % frappe_head ) @@ -98,9 +95,7 @@ def download_frappe_assets(verbose=True): commit HEAD. Returns True if correctly setup else returns False. """ - from simple_chalk import green from subprocess import getoutput - from tempfile import mkdtemp assets_setup = False frappe_head = getoutput("cd ../apps/frappe && git rev-parse HEAD") @@ -167,7 +162,7 @@ def symlink(target, link_name, overwrite=False): # Create link to target with temporary filename while True: - temp_link_name = tempfile.mktemp(dir=link_dir) + temp_link_name = mktemp(dir=link_dir) # os.* functions mimic as closely as possible system functions # The POSIX symlink() returns EEXIST if link_name already exists @@ -194,7 +189,8 @@ def symlink(target, link_name, overwrite=False): def setup(): - global app_paths + global app_paths, assets_path + pymodules = [] for app in frappe.get_all_apps(True): try: @@ -202,12 +198,13 @@ def setup(): except ImportError: pass app_paths = [os.path.dirname(pymodule.__file__) for pymodule in pymodules] + assets_path = os.path.join(frappe.local.sites_path, "assets") -def bundle(mode, apps=None, make_copy=False, restore=False, verbose=False, skip_frappe=False, files=None): +def bundle(mode, apps=None, hard_link=False, make_copy=False, restore=False, verbose=False, skip_frappe=False, files=None): """concat / minify js files""" setup() - make_asset_dirs(make_copy=make_copy, restore=restore) + make_asset_dirs(hard_link=hard_link) mode = "production" if mode == "production" else "build" command = "yarn run {mode}".format(mode=mode) @@ -264,75 +261,101 @@ def get_safe_max_old_space_size(): return safe_max_old_space_size -def make_asset_dirs(make_copy=False, restore=False): - # don't even think of making assets_path absolute - rm -rf ahead. - assets_path = os.path.join(frappe.local.sites_path, "assets") +def generate_assets_map(): + symlinks = {} - for dir_path in [os.path.join(assets_path, "js"), os.path.join(assets_path, "css")]: - if not os.path.exists(dir_path): - os.makedirs(dir_path) + for app_name in frappe.get_all_apps(): + app_doc_path = None - for app_name in frappe.get_all_apps(True): pymodule = frappe.get_module(app_name) app_base_path = os.path.abspath(os.path.dirname(pymodule.__file__)) - - symlinks = [] app_public_path = os.path.join(app_base_path, "public") - # app/public > assets/app - symlinks.append([app_public_path, os.path.join(assets_path, app_name)]) - # app/node_modules > assets/app/node_modules - if os.path.exists(os.path.abspath(app_public_path)): - symlinks.append( - [ - os.path.join(app_base_path, "..", "node_modules"), - os.path.join(assets_path, app_name, "node_modules"), - ] - ) + app_node_modules_path = os.path.join(app_base_path, "..", "node_modules") + app_docs_path = os.path.join(app_base_path, "docs") + app_www_docs_path = os.path.join(app_base_path, "www", "docs") - app_doc_path = None - if os.path.isdir(os.path.join(app_base_path, "docs")): + app_assets = os.path.abspath(app_public_path) + app_node_modules = os.path.abspath(app_node_modules_path) + + # {app}/public > assets/{app} + if os.path.isdir(app_assets): + symlinks[app_assets] = os.path.join(assets_path, app_name) + + # {app}/node_modules > assets/{app}/node_modules + if os.path.isdir(app_node_modules): + symlinks[app_node_modules] = os.path.join(assets_path, app_name, "node_modules") + + # {app}/docs > assets/{app}_docs + if os.path.isdir(app_docs_path): app_doc_path = os.path.join(app_base_path, "docs") - - elif os.path.isdir(os.path.join(app_base_path, "www", "docs")): + elif os.path.isdir(app_www_docs_path): app_doc_path = os.path.join(app_base_path, "www", "docs") - if app_doc_path: - symlinks.append([app_doc_path, os.path.join(assets_path, app_name + "_docs")]) + app_docs = os.path.abspath(app_doc_path) + symlinks[app_docs] = os.path.join(assets_path, app_name + "_docs") - for source, target in symlinks: - source = os.path.abspath(source) - if os.path.exists(source): - if restore: - if os.path.exists(target): - if os.path.islink(target): - os.unlink(target) - else: - shutil.rmtree(target) - shutil.copytree(source, target) - elif make_copy: - if os.path.exists(target): - warnings.warn("Target {target} already exists.".format(target=target)) - else: - shutil.copytree(source, target) - else: - if os.path.exists(target): - if os.path.islink(target): - os.unlink(target) - else: - shutil.rmtree(target) - try: - symlink(source, target, overwrite=True) - except OSError: - print("Cannot link {} to {}".format(source, target)) - else: - warnings.warn('Source {source} does not exist.'.format(source = source)) - pass + return symlinks + + +def setup_assets_dirs(): + for dir_path in (os.path.join(assets_path, x) for x in ("js", "css")): + os.makedirs(dir_path, exist_ok=True) + + +def clear_broken_symlinks(): + for path in os.listdir(assets_path): + path = os.path.join(assets_path, path) + if os.path.islink(path) and not os.path.exists(path): + os.remove(path) + + + +def unstrip(message): + try: + max_str = os.get_terminal_size().columns + except Exception: + max_str = 80 + _len = len(message) + _rem = max_str - _len + return f"{message}{' ' * _rem}" + + +def make_asset_dirs(hard_link=False): + setup_assets_dirs() + clear_broken_symlinks() + symlinks = generate_assets_map() + + for source, target in symlinks.items(): + start_message = unstrip(f"{'Copying assets from' if hard_link else 'Linking'} {source} to {target}") + fail_message = unstrip(f"Cannot {'copy' if hard_link else 'link'} {source} to {target}") + + try: + print(start_message, end="\r") + link_assets_dir(source, target, hard_link=hard_link) + except Exception: + print(fail_message, end="\r") + + print(unstrip(f"{green('✔')} Application Assets Linked") + "\n") + + +def link_assets_dir(source, target, hard_link=False): + if not os.path.exists(source): + return + + if os.path.exists(target): + if os.path.islink(target): + os.unlink(target) + else: + shutil.rmtree(target) + + if hard_link: + shutil.copytree(source, target, dirs_exist_ok=True) + else: + symlink(source, target, overwrite=True) def build(no_compress=False, verbose=False): - assets_path = os.path.join(frappe.local.sites_path, "assets") - - for target, sources in iteritems(get_build_maps()): + for target, sources in get_build_maps().items(): pack(os.path.join(assets_path, target), sources, no_compress, verbose) @@ -346,7 +369,7 @@ def get_build_maps(): if os.path.exists(path): with open(path) as f: try: - for target, sources in iteritems(json.loads(f.read())): + for target, sources in (json.loads(f.read() or "{}")).items(): # update app path source_paths = [] for source in sources: @@ -379,7 +402,7 @@ def pack(target, sources, no_compress, verbose): timestamps[f] = os.path.getmtime(f) try: with open(f, "r") as sourcefile: - data = text_type(sourcefile.read(), "utf-8", errors="ignore") + data = str(sourcefile.read(), "utf-8", errors="ignore") extn = f.rsplit(".", 1)[1] @@ -394,7 +417,7 @@ def pack(target, sources, no_compress, verbose): jsm.minify(tmpin, tmpout) minified = tmpout.getvalue() if minified: - outtxt += text_type(minified or "", "utf-8").strip("\n") + ";" + outtxt += str(minified or "", "utf-8").strip("\n") + ";" if verbose: print("{0}: {1}k".format(f, int(len(minified) / 1024))) @@ -424,16 +447,16 @@ def html_to_js_template(path, content): def scrub_html_template(content): """Returns HTML content with removed whitespace and comments""" # remove whitespace to a single space - content = re.sub("\s+", " ", content) + content = re.sub(r"\s+", " ", content) # strip comments - content = re.sub("()", "", content) + content = re.sub(r"()", "", content) return content.replace("'", "\'") def files_dirty(): - for target, sources in iteritems(get_build_maps()): + for target, sources in get_build_maps().items(): for f in sources: if ":" in f: f, suffix = f.split(":") diff --git a/frappe/commands/__init__.py b/frappe/commands/__init__.py index 61ee62d352..e521acc9ad 100644 --- a/frappe/commands/__init__.py +++ b/frappe/commands/__init__.py @@ -28,6 +28,10 @@ def pass_context(f): except frappe.exceptions.SiteNotSpecifiedError as e: click.secho(str(e), fg='yellow') sys.exit(1) + except frappe.exceptions.IncorrectSitePath: + site = ctx.obj.get("sites", "")[0] + click.secho(f'Site {site} does not exist!', fg='yellow') + sys.exit(1) if profile: pr.disable() diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py index f82c94999b..2fff120852 100644 --- a/frappe/commands/utils.py +++ b/frappe/commands/utils.py @@ -15,17 +15,17 @@ from frappe.utils import get_bench_path, update_progress_bar, cint @click.command('build') -@click.option('--app', help='Build assets for specific app') +@click.option('--app', help='Build assets for app') @click.option('--apps', help='Build assets for specific apps') -@click.option('--make-copy', is_flag=True, default=False, help='Copy the files instead of symlinking') -@click.option('--restore', is_flag=True, default=False, help='Copy the files instead of symlinking with force') +@click.option('--hard-link', is_flag=True, default=False, help='Copy the files instead of symlinking') +@click.option('--make-copy', is_flag=True, default=False, help='[DEPRECATED] Copy the files instead of symlinking') +@click.option('--restore', is_flag=True, default=False, help='[DEPRECATED] Copy the files instead of symlinking with force') @click.option('--production', is_flag=True, default=False, help='Build assets in production mode') -@click.option('--force', is_flag=True, default=False, help='Force build assets instead of downloading available') @click.option('--verbose', is_flag=True, default=False, help='Verbose') -def build(app=None, apps=None, make_copy=False, restore=False, production=False, verbose=False, force=False): +@click.option('--force', is_flag=True, default=False, help='Force build assets instead of downloading available') +def build(app=None, apps=None, hard_link=False, make_copy=False, restore=False, production=False, verbose=False, force=False): "Compile JS and CSS source files" from frappe.build import bundle, download_frappe_assets - frappe.init('') if not apps and app: @@ -44,7 +44,15 @@ def build(app=None, apps=None, make_copy=False, restore=False, production=False, if production: mode = "production" - bundle(mode, apps=apps, make_copy=make_copy, restore=restore, verbose=verbose, skip_frappe=skip_frappe) + if make_copy or restore: + hard_link = make_copy or restore + click.secho( + "bench build: --make-copy and --restore options are deprecated in favour of --hard-link", + fg="yellow", + ) + + bundle(mode, apps=apps, hard_link=hard_link, verbose=verbose, skip_frappe=skip_frappe) + @click.command('watch') @@ -499,6 +507,8 @@ frappe.db.connect() @pass_context def console(context): "Start ipython console for a site" + import warnings + site = get_site(context) frappe.init(site=site) frappe.connect() @@ -519,6 +529,7 @@ def console(context): if failed_to_import: print("\nFailed to import:\n{}".format(", ".join(failed_to_import))) + warnings.simplefilter('ignore') IPython.embed(display_banner="", header="", colors="neutral") @@ -596,12 +607,29 @@ def run_tests(context, app=None, module=None, doctype=None, test=(), profile=Fal if os.environ.get('CI'): sys.exit(ret) +@click.command('run-parallel-tests') +@click.option('--app', help="For App", default='frappe') +@click.option('--build-number', help="Build number", default=1) +@click.option('--total-builds', help="Total number of builds", default=1) +@click.option('--with-coverage', is_flag=True, help="Build coverage file") +@click.option('--use-orchestrator', is_flag=True, help="Use orchestrator to run parallel tests") +@pass_context +def run_parallel_tests(context, app, build_number, total_builds, with_coverage=False, use_orchestrator=False): + site = get_site(context) + if use_orchestrator: + from frappe.parallel_test_runner import ParallelTestWithOrchestrator + ParallelTestWithOrchestrator(app, site=site, with_coverage=with_coverage) + else: + from frappe.parallel_test_runner import ParallelTestRunner + ParallelTestRunner(app, site=site, build_number=build_number, total_builds=total_builds, with_coverage=with_coverage) @click.command('run-ui-tests') @click.argument('app') @click.option('--headless', is_flag=True, help="Run UI Test in headless mode") +@click.option('--parallel', is_flag=True, help="Run UI Test in parallel mode") +@click.option('--ci-build-id') @pass_context -def run_ui_tests(context, app, headless=False): +def run_ui_tests(context, app, headless=False, parallel=True, ci_build_id=None): "Run UI tests" site = get_site(context) app_base_path = os.path.abspath(os.path.join(frappe.get_app_path(app), '..')) @@ -633,6 +661,12 @@ def run_ui_tests(context, app, headless=False): command = '{site_env} {password_env} {cypress} {run_or_open}' formatted_command = command.format(site_env=site_env, password_env=password_env, cypress=cypress_path, run_or_open=run_or_open) + if parallel: + formatted_command += ' --parallel' + + if ci_build_id: + formatted_command += ' --ci-build-id {}'.format(ci_build_id) + click.secho("Running Cypress...", fg="yellow") frappe.commands.popen(formatted_command, cwd=app_base_path, raise_err=True) @@ -808,5 +842,6 @@ commands = [ watch, bulk_rename, add_to_email_queue, - rebuild_global_search + rebuild_global_search, + run_parallel_tests ] diff --git a/frappe/contacts/doctype/contact/test_contact.py b/frappe/contacts/doctype/contact/test_contact.py index 4929873dc4..b131428696 100644 --- a/frappe/contacts/doctype/contact/test_contact.py +++ b/frappe/contacts/doctype/contact/test_contact.py @@ -5,7 +5,8 @@ from __future__ import unicode_literals import frappe import unittest -from frappe.exceptions import ValidationError + +test_dependencies = ['Contact', 'Salutation'] class TestContact(unittest.TestCase): @@ -52,4 +53,4 @@ def create_contact(name, salutation, emails=None, phones=None, save=True): if save: doc.insert() - return doc \ No newline at end of file + return doc diff --git a/frappe/core/doctype/activity_log/test_activity_log.py b/frappe/core/doctype/activity_log/test_activity_log.py index 05ece76c7f..f33c7a1c85 100644 --- a/frappe/core/doctype/activity_log/test_activity_log.py +++ b/frappe/core/doctype/activity_log/test_activity_log.py @@ -90,4 +90,5 @@ class TestActivityLog(unittest.TestCase): def update_system_settings(args): doc = frappe.get_doc('System Settings') doc.update(args) + doc.flags.ignore_mandatory = 1 doc.save() diff --git a/frappe/core/doctype/data_export/exporter.py b/frappe/core/doctype/data_export/exporter.py index bec8cde7ea..5d600cc0db 100644 --- a/frappe/core/doctype/data_export/exporter.py +++ b/frappe/core/doctype/data_export/exporter.py @@ -282,7 +282,7 @@ class DataExporter: try: sflags = self.docs_to_export.get("flags", "I,U").upper() flags = 0 - for a in re.split('\W+',sflags): + for a in re.split(r'\W+', sflags): flags = flags | reflags.get(a,0) c = re.compile(names, flags) diff --git a/frappe/core/doctype/data_import/importer.py b/frappe/core/doctype/data_import/importer.py index 720fe1dda7..d3f981add4 100644 --- a/frappe/core/doctype/data_import/importer.py +++ b/frappe/core/doctype/data_import/importer.py @@ -641,7 +641,7 @@ class Row: return elif df.fieldtype == "Duration": import re - is_valid_duration = re.match("^(?:(\d+d)?((^|\s)\d+h)?((^|\s)\d+m)?((^|\s)\d+s)?)$", value) + is_valid_duration = re.match(r"^(?:(\d+d)?((^|\s)\d+h)?((^|\s)\d+m)?((^|\s)\d+s)?)$", value) if not is_valid_duration: self.warnings.append( { @@ -929,10 +929,7 @@ class Column: self.warnings.append( { "col": self.column_number, - "message": _( - "Date format could not be determined from the values in" - " this column. Defaulting to yyyy-mm-dd." - ), + "message": _("Date format could not be determined from the values in this column. Defaulting to yyyy-mm-dd."), "type": "info", } ) diff --git a/frappe/core/doctype/docshare/test_docshare.py b/frappe/core/doctype/docshare/test_docshare.py index d4ef1f92f8..9c424eb4d7 100644 --- a/frappe/core/doctype/docshare/test_docshare.py +++ b/frappe/core/doctype/docshare/test_docshare.py @@ -7,6 +7,8 @@ import frappe.share import unittest from frappe.automation.doctype.auto_repeat.test_auto_repeat import create_submittable_doctype +test_dependencies = ['User'] + class TestDocShare(unittest.TestCase): def setUp(self): self.user = "test@example.com" @@ -112,4 +114,4 @@ class TestDocShare(unittest.TestCase): self.assertTrue(frappe.has_permission(doctype, "read", doc=submittable_doc.name, user=self.user)) self.assertTrue(frappe.has_permission(doctype, "write", doc=submittable_doc.name, user=self.user)) - frappe.share.remove(doctype, submittable_doc.name, self.user) \ No newline at end of file + frappe.share.remove(doctype, submittable_doc.name, self.user) diff --git a/frappe/core/doctype/doctype/doctype.py b/frappe/core/doctype/doctype/doctype.py index 6eef5a4023..84673f990a 100644 --- a/frappe/core/doctype/doctype/doctype.py +++ b/frappe/core/doctype/doctype/doctype.py @@ -671,12 +671,12 @@ class DocType(Document): flags = {"flags": re.ASCII} if six.PY3 else {} # a DocType name should not start or end with an empty space - if re.search("^[ \t\n\r]+|[ \t\n\r]+$", name, **flags): + if re.search(r"^[ \t\n\r]+|[ \t\n\r]+$", name, **flags): frappe.throw(_("DocType's name should not start or end with whitespace"), frappe.NameError) # a DocType's name should not start with a number or underscore # and should only contain letters, numbers and underscore - if not re.match("^(?![\W])[^\d_\s][\w ]+$", name, **flags): + if not re.match(r"^(?![\W])[^\d_\s][\w ]+$", name, **flags): frappe.throw(_("DocType's name should start with a letter and it can only consist of letters, numbers, spaces and underscores"), frappe.NameError) validate_route_conflict(self.doctype, self.name) @@ -964,7 +964,7 @@ def validate_fields(meta): for field in depends_on_fields: depends_on = docfield.get(field, None) if depends_on and ("=" in depends_on) and \ - re.match("""[\w\.:_]+\s*={1}\s*[\w\.@'"]+""", depends_on): + re.match(r'[\w\.:_]+\s*={1}\s*[\w\.@\'"]+', depends_on): frappe.throw(_("Invalid {0} condition").format(frappe.unscrub(field)), frappe.ValidationError) def check_table_multiselect_option(docfield): diff --git a/frappe/core/doctype/doctype/test_doctype.py b/frappe/core/doctype/doctype/test_doctype.py index bfa9d0ec8a..9c492d2c36 100644 --- a/frappe/core/doctype/doctype/test_doctype.py +++ b/frappe/core/doctype/doctype/test_doctype.py @@ -92,7 +92,7 @@ class TestDocType(unittest.TestCase): fields=["parent", "depends_on", "collapsible_depends_on", "mandatory_depends_on",\ "read_only_depends_on", "fieldname", "fieldtype"]) - pattern = """[\w\.:_]+\s*={1}\s*[\w\.@'"]+""" + pattern = r'[\w\.:_]+\s*={1}\s*[\w\.@\'"]+' for field in docfields: for depends_on in ["depends_on", "collapsible_depends_on", "mandatory_depends_on", "read_only_depends_on"]: condition = field.get(depends_on) @@ -517,4 +517,4 @@ def new_doctype(name, unique=0, depends_on='', fields=None): for f in fields: doc.append('fields', f) - return doc \ No newline at end of file + return doc diff --git a/frappe/core/doctype/file/file.py b/frappe/core/doctype/file/file.py index 017106e6f5..c4c37e6d13 100755 --- a/frappe/core/doctype/file/file.py +++ b/frappe/core/doctype/file/file.py @@ -498,7 +498,7 @@ class File(Document): self.file_size = self.check_max_file_size() if ( - self.content_type and "image" in self.content_type + self.content_type and self.content_type == "image/jpeg" and frappe.get_system_settings("strip_exif_metadata_from_uploaded_images") ): self.content = strip_exif_data(self.content, self.content_type) @@ -912,7 +912,7 @@ def extract_images_from_html(doc, content): return ']*src\s*=\s*["\'](?=data:)(.*?)["\']', _save_file, content) + content = re.sub(r']*src\s*=\s*["\'](?=data:)(.*?)["\']', _save_file, content) return content diff --git a/frappe/core/doctype/file/test_file.py b/frappe/core/doctype/file/test_file.py index 2f8f437fc9..2596fe94d0 100644 --- a/frappe/core/doctype/file/test_file.py +++ b/frappe/core/doctype/file/test_file.py @@ -193,6 +193,7 @@ class TestSameContent(unittest.TestCase): class TestFile(unittest.TestCase): def setUp(self): + frappe.set_user('Administrator') self.delete_test_data() self.upload_file() diff --git a/frappe/core/doctype/role_profile/test_role_profile.py b/frappe/core/doctype/role_profile/test_role_profile.py index 624b85c315..975453e8d1 100644 --- a/frappe/core/doctype/role_profile/test_role_profile.py +++ b/frappe/core/doctype/role_profile/test_role_profile.py @@ -5,6 +5,8 @@ from __future__ import unicode_literals import frappe import unittest +test_dependencies = ['Role'] + class TestRoleProfile(unittest.TestCase): def test_make_new_role_profile(self): new_role_profile = frappe.get_doc(dict(doctype='Role Profile', role_profile='Test 1')).insert() @@ -21,4 +23,4 @@ class TestRoleProfile(unittest.TestCase): # clear roles new_role_profile.roles = [] new_role_profile.save() - self.assertEqual(new_role_profile.roles, []) \ No newline at end of file + self.assertEqual(new_role_profile.roles, []) diff --git a/frappe/core/doctype/system_settings/system_settings.py b/frappe/core/doctype/system_settings/system_settings.py index d102526a9e..05aaca81de 100644 --- a/frappe/core/doctype/system_settings/system_settings.py +++ b/frappe/core/doctype/system_settings/system_settings.py @@ -42,7 +42,7 @@ class SystemSettings(Document): def on_update(self): for df in self.meta.get("fields"): - if df.fieldtype not in no_value_fields: + if df.fieldtype not in no_value_fields and self.has_value_changed(df.fieldname): frappe.db.set_default(df.fieldname, self.get(df.fieldname)) if self.language: diff --git a/frappe/desk/doctype/todo/test_todo.py b/frappe/desk/doctype/todo/test_todo.py index b767fd4aef..de5b6724a6 100644 --- a/frappe/desk/doctype/todo/test_todo.py +++ b/frappe/desk/doctype/todo/test_todo.py @@ -9,8 +9,7 @@ from frappe.model.db_query import DatabaseQuery from frappe.permissions import add_permission, reset_perms from frappe.core.doctype.doctype.doctype import clear_permissions_cache -# test_records = frappe.get_test_records('ToDo') -test_user_records = frappe.get_test_records('User') +test_dependencies = ['User'] class TestToDo(unittest.TestCase): def test_delete(self): @@ -77,7 +76,7 @@ class TestToDo(unittest.TestCase): frappe.set_user('test4@example.com') #owner and assigned_by is test4 todo3 = create_new_todo('Test3', 'test4@example.com') - + # user without any role to read or write todo document self.assertFalse(todo1.has_permission("read")) self.assertFalse(todo1.has_permission("write")) diff --git a/frappe/desk/doctype/workspace_link/workspace_link.json b/frappe/desk/doctype/workspace_link/workspace_link.json index 010fb3f316..53dadad83d 100644 --- a/frappe/desk/doctype/workspace_link/workspace_link.json +++ b/frappe/desk/doctype/workspace_link/workspace_link.json @@ -8,13 +8,13 @@ "type", "label", "icon", + "only_for", "hidden", "link_details_section", "link_type", "link_to", "column_break_7", "dependencies", - "only_for", "onboard", "is_query_report" ], @@ -84,7 +84,7 @@ { "fieldname": "only_for", "fieldtype": "Link", - "label": "Only for ", + "label": "Only for", "options": "Country" }, { @@ -104,7 +104,7 @@ "index_web_pages_for_search": 1, "istable": 1, "links": [], - "modified": "2021-01-12 13:13:12.379443", + "modified": "2021-05-13 13:10:18.128512", "modified_by": "Administrator", "module": "Desk", "name": "Workspace Link", diff --git a/frappe/email/doctype/notification/test_notification.py b/frappe/email/doctype/notification/test_notification.py index 87c4b2527a..31d5d9d1cc 100644 --- a/frappe/email/doctype/notification/test_notification.py +++ b/frappe/email/doctype/notification/test_notification.py @@ -7,9 +7,7 @@ import frappe, frappe.utils, frappe.utils.scheduler from frappe.desk.form import assign_to import unittest -test_records = frappe.get_test_records('Notification') - -test_dependencies = ["User"] +test_dependencies = ["User", "Notification"] class TestNotification(unittest.TestCase): def setUp(self): diff --git a/frappe/email/receive.py b/frappe/email/receive.py index 949da4a343..6d60007cdb 100644 --- a/frappe/email/receive.py +++ b/frappe/email/receive.py @@ -284,7 +284,7 @@ class EmailServer: flags = [] for flag in imaplib.ParseFlags(flag_string) or []: - pattern = re.compile("\w+") + pattern = re.compile(r"\w+") match = re.search(pattern, frappe.as_unicode(flag)) flags.append(match.group(0)) @@ -555,7 +555,7 @@ class Email: def get_thread_id(self): """Extract thread ID from `[]`""" - l = re.findall('(?<=\[)[\w/-]+', self.subject) + l = re.findall(r'(?<=\[)[\w/-]+', self.subject) return l and l[0] or None diff --git a/frappe/hooks.py b/frappe/hooks.py index 6fff9ac2a1..d0968ce051 100644 --- a/frappe/hooks.py +++ b/frappe/hooks.py @@ -228,7 +228,6 @@ scheduler_events = { "frappe.desk.doctype.event.event.send_event_digest", "frappe.sessions.clear_expired_sessions", "frappe.email.doctype.notification.notification.trigger_daily_alerts", - "frappe.realtime.remove_old_task_logs", "frappe.utils.scheduler.restrict_scheduler_events_if_dormant", "frappe.email.doctype.auto_email_report.auto_email_report.send_daily", "frappe.website.doctype.personal_data_deletion_request.personal_data_deletion_request.remove_unverified_record", diff --git a/frappe/installer.py b/frappe/installer.py index 0cd5b136ae..d7d885d60e 100755 --- a/frappe/installer.py +++ b/frappe/installer.py @@ -390,19 +390,16 @@ def get_conf_params(db_name=None, db_password=None): def make_site_dirs(): - site_public_path = os.path.join(frappe.local.site_path, 'public') - site_private_path = os.path.join(frappe.local.site_path, 'private') - for dir_path in ( - os.path.join(site_private_path, 'backups'), - os.path.join(site_public_path, 'files'), - os.path.join(site_private_path, 'files'), - os.path.join(frappe.local.site_path, 'logs'), - os.path.join(frappe.local.site_path, 'task-logs')): - if not os.path.exists(dir_path): - os.makedirs(dir_path) - locks_dir = frappe.get_site_path('locks') - if not os.path.exists(locks_dir): - os.makedirs(locks_dir) + for dir_path in [ + os.path.join("public", "files"), + os.path.join("private", "backups"), + os.path.join("private", "files"), + "error-snapshots", + "locks", + "logs", + ]: + path = frappe.get_site_path(dir_path) + os.makedirs(path, exist_ok=True) def add_module_defs(app): diff --git a/frappe/model/base_document.py b/frappe/model/base_document.py index 154a091b8a..54d77ba988 100644 --- a/frappe/model/base_document.py +++ b/frappe/model/base_document.py @@ -870,7 +870,7 @@ class BaseDocument(object): from frappe.model.meta import get_default_df df = get_default_df(fieldname) - if not currency: + if not currency and df: currency = self.get(df.get("options")) if not frappe.db.exists('Currency', currency, cache=True): currency = None diff --git a/frappe/model/naming.py b/frappe/model/naming.py index 359b8e2367..b8d6a6f8d7 100644 --- a/frappe/model/naming.py +++ b/frappe/model/naming.py @@ -283,7 +283,7 @@ def append_number_if_name_exists(doctype, value, fieldname="name", separator="-" filters.update({fieldname: value}) exists = frappe.db.exists(doctype, filters) - regex = "^{value}{separator}\d+$".format(value=re.escape(value), separator=separator) + regex = "^{value}{separator}\\d+$".format(value=re.escape(value), separator=separator) if exists: last = frappe.db.sql("""SELECT `{fieldname}` FROM `tab{doctype}` diff --git a/frappe/parallel_test_runner.py b/frappe/parallel_test_runner.py new file mode 100644 index 0000000000..1dbb24f191 --- /dev/null +++ b/frappe/parallel_test_runner.py @@ -0,0 +1,282 @@ +import json +import os +import re +import sys +import time +import unittest +import click +import frappe +import requests + +from .test_runner import (SLOW_TEST_THRESHOLD, make_test_records, set_test_email_config) + +click_ctx = click.get_current_context(True) +if click_ctx: + click_ctx.color = True + +class ParallelTestRunner(): + def __init__(self, app, site, build_number=1, total_builds=1, with_coverage=False): + self.app = app + self.site = site + self.with_coverage = with_coverage + self.build_number = frappe.utils.cint(build_number) or 1 + self.total_builds = frappe.utils.cint(total_builds) + self.setup_test_site() + self.run_tests() + + def setup_test_site(self): + frappe.init(site=self.site) + if not frappe.db: + frappe.connect() + + frappe.flags.in_test = True + frappe.clear_cache() + frappe.utils.scheduler.disable_scheduler() + set_test_email_config() + self.before_test_setup() + + def before_test_setup(self): + start_time = time.time() + for fn in frappe.get_hooks("before_tests", app_name=self.app): + frappe.get_attr(fn)() + + test_module = frappe.get_module(f'{self.app}.tests') + + if hasattr(test_module, "global_test_dependencies"): + for doctype in test_module.global_test_dependencies: + make_test_records(doctype) + + elapsed = time.time() - start_time + elapsed = click.style(f' ({elapsed:.03}s)', fg='red') + click.echo(f'Before Test {elapsed}') + + def run_tests(self): + self.test_result = ParallelTestResult(stream=sys.stderr, descriptions=True, verbosity=2) + + self.start_coverage() + + for test_file_info in self.get_test_file_list(): + self.run_tests_for_file(test_file_info) + + self.save_coverage() + self.print_result() + + def run_tests_for_file(self, file_info): + if not file_info: return + + frappe.set_user('Administrator') + path, filename = file_info + module = self.get_module(path, filename) + self.create_test_dependency_records(module, path, filename) + test_suite = unittest.TestSuite() + module_test_cases = unittest.TestLoader().loadTestsFromModule(module) + test_suite.addTest(module_test_cases) + test_suite(self.test_result) + + def create_test_dependency_records(self, module, path, filename): + if hasattr(module, "test_dependencies"): + for doctype in module.test_dependencies: + make_test_records(doctype) + + if os.path.basename(os.path.dirname(path)) == "doctype": + # test_data_migration_connector.py > data_migration_connector.json + test_record_filename = re.sub('^test_', '', filename).replace(".py", ".json") + test_record_file_path = os.path.join(path, test_record_filename) + if os.path.exists(test_record_file_path): + with open(test_record_file_path, 'r') as f: + doc = json.loads(f.read()) + doctype = doc["name"] + make_test_records(doctype) + + def get_module(self, path, filename): + app_path = frappe.get_pymodule_path(self.app) + relative_path = os.path.relpath(path, app_path) + if relative_path == '.': + module_name = self.app + else: + relative_path = relative_path.replace('/', '.') + module_name = os.path.splitext(filename)[0] + module_name = f'{self.app}.{relative_path}.{module_name}' + + return frappe.get_module(module_name) + + def print_result(self): + self.test_result.printErrors() + click.echo(self.test_result) + if self.test_result.failures or self.test_result.errors: + if os.environ.get('CI'): + sys.exit(1) + + def start_coverage(self): + if self.with_coverage: + from coverage import Coverage + from frappe.utils import get_bench_path + + # Generate coverage report only for app that is being tested + source_path = os.path.join(get_bench_path(), 'apps', self.app) + omit=['*.html', '*.js', '*.xml', '*.css', '*.less', '*.scss', + '*.vue', '*/doctype/*/*_dashboard.py', '*/patches/*'] + + if self.app == 'frappe': + omit.append('*/commands/*') + + self.coverage = Coverage(source=[source_path], omit=omit) + self.coverage.start() + + def save_coverage(self): + if not self.with_coverage: + return + self.coverage.stop() + self.coverage.save() + + def get_test_file_list(self): + test_list = get_all_tests(self.app) + split_size = frappe.utils.ceil(len(test_list) / self.total_builds) + # [1,2,3,4,5,6] to [[1,2], [3,4], [4,6]] if split_size is 2 + test_chunks = [test_list[x:x+split_size] for x in range(0, len(test_list), split_size)] + return test_chunks[self.build_number - 1] + + +class ParallelTestResult(unittest.TextTestResult): + def startTest(self, test): + self._started_at = time.time() + super(unittest.TextTestResult, self).startTest(test) + test_class = unittest.util.strclass(test.__class__) + if not hasattr(self, 'current_test_class') or self.current_test_class != test_class: + click.echo(f"\n{unittest.util.strclass(test.__class__)}") + self.current_test_class = test_class + + def getTestMethodName(self, test): + return test._testMethodName if hasattr(test, '_testMethodName') else str(test) + + def addSuccess(self, test): + super(unittest.TextTestResult, self).addSuccess(test) + elapsed = time.time() - self._started_at + threshold_passed = elapsed >= SLOW_TEST_THRESHOLD + elapsed = click.style(f' ({elapsed:.03}s)', fg='red') if threshold_passed else '' + click.echo(f" {click.style(' ✔ ', fg='green')} {self.getTestMethodName(test)}{elapsed}") + + def addError(self, test, err): + super(unittest.TextTestResult, self).addError(test, err) + click.echo(f" {click.style(' ✖ ', fg='red')} {self.getTestMethodName(test)}") + + def addFailure(self, test, err): + super(unittest.TextTestResult, self).addFailure(test, err) + click.echo(f" {click.style(' ✖ ', fg='red')} {self.getTestMethodName(test)}") + + def addSkip(self, test, reason): + super(unittest.TextTestResult, self).addSkip(test, reason) + click.echo(f" {click.style(' = ', fg='white')} {self.getTestMethodName(test)}") + + def addExpectedFailure(self, test, err): + super(unittest.TextTestResult, self).addExpectedFailure(test, err) + click.echo(f" {click.style(' ✖ ', fg='red')} {self.getTestMethodName(test)}") + + def addUnexpectedSuccess(self, test): + super(unittest.TextTestResult, self).addUnexpectedSuccess(test) + click.echo(f" {click.style(' ✔ ', fg='green')} {self.getTestMethodName(test)}") + + def printErrors(self): + click.echo('\n') + self.printErrorList(' ERROR ', self.errors, 'red') + self.printErrorList(' FAIL ', self.failures, 'red') + + def printErrorList(self, flavour, errors, color): + for test, err in errors: + click.echo(self.separator1) + click.echo(f"{click.style(flavour, bg=color)} {self.getDescription(test)}") + click.echo(self.separator2) + click.echo(err) + + def __str__(self): + return f"Tests: {self.testsRun}, Failing: {len(self.failures)}, Errors: {len(self.errors)}" + +def get_all_tests(app): + test_file_list = [] + for path, folders, files in os.walk(frappe.get_pymodule_path(app)): + for dontwalk in ('locals', '.git', 'public', '__pycache__'): + if dontwalk in folders: + folders.remove(dontwalk) + + # for predictability + folders.sort() + files.sort() + + if os.path.sep.join(["doctype", "doctype", "boilerplate"]) in path: + # in /doctype/doctype/boilerplate/ + continue + + for filename in files: + if filename.startswith("test_") and filename.endswith(".py") \ + and filename != 'test_runner.py': + test_file_list.append([path, filename]) + + return test_file_list + + +class ParallelTestWithOrchestrator(ParallelTestRunner): + ''' + This can be used to balance-out test time across multiple instances + This is dependent on external orchestrator which returns next test to run + + orchestrator endpoints + - register-instance (, , test_spec_list) + - get-next-test-spec (, ) + - test-completed (, ) + ''' + def __init__(self, app, site, with_coverage=False): + self.orchestrator_url = os.environ.get('ORCHESTRATOR_URL') + if not self.orchestrator_url: + click.echo('ORCHESTRATOR_URL environment variable not found!') + click.echo('Pass public URL after hosting https://github.com/frappe/test-orchestrator') + sys.exit(1) + + self.ci_build_id = os.environ.get('CI_BUILD_ID') + self.ci_instance_id = os.environ.get('CI_INSTANCE_ID') or frappe.generate_hash(length=10) + if not self.ci_build_id: + click.echo('CI_BUILD_ID environment variable not found!') + sys.exit(1) + + ParallelTestRunner.__init__(self, app, site, with_coverage=with_coverage) + + def run_tests(self): + self.test_status = 'ongoing' + self.register_instance() + super().run_tests() + + def get_test_file_list(self): + while self.test_status == 'ongoing': + yield self.get_next_test() + + def register_instance(self): + test_spec_list = get_all_tests(self.app) + response_data = self.call_orchestrator('register-instance', data={ + 'test_spec_list': test_spec_list + }) + self.is_master = response_data.get('is_master') + + def get_next_test(self): + response_data = self.call_orchestrator('get-next-test-spec') + self.test_status = response_data.get('status') + return response_data.get('next_test') + + def print_result(self): + self.call_orchestrator('test-completed') + return super().print_result() + + def call_orchestrator(self, endpoint, data={}): + # add repo token header + # build id in header + headers = { + 'CI-BUILD-ID': self.ci_build_id, + 'CI-INSTANCE-ID': self.ci_instance_id, + 'REPO-TOKEN': '2948288382838DE' + } + url = f'{self.orchestrator_url}/{endpoint}' + res = requests.get(url, json=data, headers=headers) + res.raise_for_status() + response_data = {} + if 'application/json' in res.headers.get('content-type'): + response_data = res.json() + + return response_data diff --git a/frappe/patches/v5_0/fix_text_editor_file_urls.py b/frappe/patches/v5_0/fix_text_editor_file_urls.py index d91aad0234..a6d7d2fb9a 100644 --- a/frappe/patches/v5_0/fix_text_editor_file_urls.py +++ b/frappe/patches/v5_0/fix_text_editor_file_urls.py @@ -33,8 +33,7 @@ def execute(): def scrub_relative_urls(html): """prepend a slash before a relative url""" try: - return re.sub("""src[\s]*=[\s]*['"]files/([^'"]*)['"]""", 'src="/files/\g<1>"', html) - # return re.sub("""(src|href)[^\w'"]*['"](?!http|ftp|mailto|/|#|%|{|cid:|\.com/www\.)([^'" >]+)['"]""", '\g<1>="/\g<2>"', html) + return re.sub(r'src[\s]*=[\s]*[\'"]files/([^\'"]*)[\'"]', r'src="/files/\g<1>"', html) except: print("Error", html) raise diff --git a/frappe/printing/doctype/print_format/test_print_format.py b/frappe/printing/doctype/print_format/test_print_format.py index 7e30bda23e..121916ae5f 100644 --- a/frappe/printing/doctype/print_format/test_print_format.py +++ b/frappe/printing/doctype/print_format/test_print_format.py @@ -12,13 +12,13 @@ class TestPrintFormat(unittest.TestCase): def test_print_user(self, style=None): print_html = frappe.get_print("User", "Administrator", style=style) self.assertTrue("" in print_html) - self.assertTrue(re.findall('
[\s]*administrator[\s]*
', print_html)) + self.assertTrue(re.findall(r'
[\s]*administrator[\s]*
', print_html)) return print_html def test_print_user_standard(self): print_html = self.test_print_user("Standard") - self.assertTrue(re.findall('\.print-format {[\s]*font-size: 9pt;', print_html)) - self.assertFalse(re.findall('th {[\s]*background-color: #eee;[\s]*}', print_html)) + self.assertTrue(re.findall(r'\.print-format {[\s]*font-size: 9pt;', print_html)) + self.assertFalse(re.findall(r'th {[\s]*background-color: #eee;[\s]*}', print_html)) self.assertFalse("font-family: serif;" in print_html) def test_print_user_modern(self): diff --git a/frappe/public/js/frappe/form/controls/date.js b/frappe/public/js/frappe/form/controls/date.js index 1f5f6f5f25..9ad81c7e46 100644 --- a/frappe/public/js/frappe/form/controls/date.js +++ b/frappe/public/js/frappe/form/controls/date.js @@ -13,9 +13,11 @@ frappe.ui.form.ControlDate = class ControlDate extends frappe.ui.form.ControlDat super.set_formatted_input(value); if (this.timepicker_only) return; if (!this.datepicker) return; - if(!value) { + if (!value) { this.datepicker.clear(); return; + } else if (value === "Today") { + value = this.get_now_date(); } let should_refresh = this.last_value && this.last_value !== value; diff --git a/frappe/public/js/frappe/form/grid.js b/frappe/public/js/frappe/form/grid.js index 4d381c9be7..dd7c339395 100644 --- a/frappe/public/js/frappe/form/grid.js +++ b/frappe/public/js/frappe/form/grid.js @@ -910,6 +910,10 @@ export default class Grid { update_docfield_property(fieldname, property, value) { // update the docfield of each row + if (!this.grid_rows) { + return; + } + for (let row of this.grid_rows) { let docfield = row.docfields.find(d => d.fieldname === fieldname); if (docfield) { diff --git a/frappe/public/js/frappe/form/layout.js b/frappe/public/js/frappe/form/layout.js index 9ef068d9c7..282655b589 100644 --- a/frappe/public/js/frappe/form/layout.js +++ b/frappe/public/js/frappe/form/layout.js @@ -510,7 +510,7 @@ frappe.ui.form.Layout = class Layout { form_obj = this; } if (form_obj) { - if (this.doc && this.doc.parent) { + if (this.doc && this.doc.parent && this.doc.parentfield) { form_obj.setting_dependency = true; form_obj.set_df_property(this.doc.parentfield, property, value, this.doc.parent, fieldname, this.doc.name); form_obj.setting_dependency = false; diff --git a/frappe/realtime.py b/frappe/realtime.py index f546703e58..6c812e8868 100644 --- a/frappe/realtime.py +++ b/frappe/realtime.py @@ -1,56 +1,23 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors # For license information, please see license.txt -from __future__ import unicode_literals - - import frappe import os -import time import redis -from io import FileIO -from frappe.utils import get_site_path -from frappe import conf -END_LINE = '' -TASK_LOG_MAX_AGE = 86400 # 1 day in seconds redis_server = None + @frappe.whitelist() def get_pending_tasks_for_doc(doctype, docname): return frappe.db.sql_list("select name from `tabAsync Task` where status in ('Queued', 'Running') and reference_doctype=%s and reference_name=%s", (doctype, docname)) -def set_task_status(task_id, status, response=None): - if not response: - response = {} - response.update({ - "status": status, - "task_id": task_id - }) - emit_via_redis("task_status_change", response, room="task:" + task_id) - - -def remove_old_task_logs(): - logs_path = get_site_path('task-logs') - - def full_path(_file): - return os.path.join(logs_path, _file) - - files_to_remove = [full_path(_file) for _file in os.listdir(logs_path)] - files_to_remove = [_file for _file in files_to_remove if is_file_old(_file) and os.path.isfile(_file)] - for _file in files_to_remove: - os.remove(_file) - - -def is_file_old(file_path): - return ((time.time() - os.stat(file_path).st_mtime) > TASK_LOG_MAX_AGE) - def publish_progress(percent, title=None, doctype=None, docname=None, description=None): publish_realtime('progress', {'percent': percent, 'title': title, 'description': description}, user=frappe.session.user, doctype=doctype, docname=docname) + def publish_realtime(event=None, message=None, room=None, user=None, doctype=None, docname=None, task_id=None, after_commit=False): @@ -103,6 +70,7 @@ def publish_realtime(event=None, message=None, room=None, else: emit_via_redis(event, message, room) + def emit_via_redis(event, message, room): """Publish real-time updates via redis @@ -117,57 +85,17 @@ def emit_via_redis(event, message, room): # print(frappe.get_traceback()) pass -def put_log(line_no, line, task_id=None): - r = get_redis_server() - if not task_id: - task_id = frappe.local.task_id - task_progress_room = get_task_progress_room(task_id) - task_log_key = "task_log:" + task_id - publish_realtime('task_progress', { - "message": { - "lines": {line_no: line} - }, - "task_id": task_id - }, room=task_progress_room) - r.hset(task_log_key, line_no, line) - r.expire(task_log_key, 3600) - def get_redis_server(): """returns redis_socketio connection.""" global redis_server if not redis_server: from redis import Redis - redis_server = Redis.from_url(conf.get("redis_socketio") + redis_server = Redis.from_url(frappe.conf.redis_socketio or "redis://localhost:12311") return redis_server -class FileAndRedisStream(FileIO): - def __init__(self, *args, **kwargs): - ret = super(FileAndRedisStream, self).__init__(*args, **kwargs) - self.count = 0 - return ret - - def write(self, data): - ret = super(FileAndRedisStream, self).write(data) - if frappe.local.task_id: - put_log(self.count, data, task_id=frappe.local.task_id) - self.count += 1 - return ret - - -def get_std_streams(task_id): - stdout = FileAndRedisStream(get_task_log_file_path(task_id, 'stdout'), 'w') - # stderr = FileAndRedisStream(get_task_log_file_path(task_id, 'stderr'), 'w') - return stdout, stdout - - -def get_task_log_file_path(task_id, stream_type): - logs_dir = frappe.utils.get_site_path('task-logs') - return os.path.join(logs_dir, task_id + '.' + stream_type) - - @frappe.whitelist(allow_guest=True) def can_subscribe_doc(doctype, docname): if os.environ.get('CI'): @@ -201,9 +129,7 @@ def get_site_room(): def get_task_progress_room(task_id): return "".join([frappe.local.site, ":task_progress:", task_id]) -# frappe.chat def get_chat_room(room): room = ''.join([frappe.local.site, ":room:", room]) return room -# end frappe.chat room \ No newline at end of file diff --git a/frappe/search/full_text_search.py b/frappe/search/full_text_search.py index ecb018dbb4..9dd181323e 100644 --- a/frappe/search/full_text_search.py +++ b/frappe/search/full_text_search.py @@ -1,8 +1,8 @@ # Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals import frappe +from frappe.utils import update_progress_bar from whoosh.index import create_in, open_dir, EmptyIndexError from whoosh.fields import TEXT, ID, Schema @@ -95,9 +95,10 @@ class FullTextSearch: ix = self.create_index() writer = ix.writer() - for document in self.documents: + for i, document in enumerate(self.documents): if document: writer.add_document(**document) + update_progress_bar("Building Index", i, len(self.documents)) writer.commit(optimize=True) diff --git a/frappe/search/website_search.py b/frappe/search/website_search.py index 87ef4b08ad..452ea2a427 100644 --- a/frappe/search/website_search.py +++ b/frappe/search/website_search.py @@ -1,15 +1,16 @@ # Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -from __future__ import unicode_literals -import frappe -from bs4 import BeautifulSoup -from whoosh.fields import TEXT, ID, Schema -from frappe.search.full_text_search import FullTextSearch -from frappe.website.render import render_page -from frappe.utils import set_request import os +from bs4 import BeautifulSoup +from whoosh.fields import ID, TEXT, Schema + +import frappe +from frappe.search.full_text_search import FullTextSearch +from frappe.utils import set_request, update_progress_bar +from frappe.website.render import render_page + INDEX_NAME = "web_routes" class WebsiteSearch(FullTextSearch): @@ -30,11 +31,21 @@ class WebsiteSearch(FullTextSearch): Returns: self (object): FullTextSearch Instance """ - routes = get_static_pages_from_all_apps() - routes += slugs_with_web_view() - documents = [self.get_document_to_index(route) for route in routes] - return documents + if getattr(self, "_items_to_index", False): + return self._items_to_index + + routes = get_static_pages_from_all_apps() + slugs_with_web_view() + + self._items_to_index = [] + + for i, route in enumerate(routes): + update_progress_bar("Retrieving Routes", i, len(routes)) + self._items_to_index += [self.get_document_to_index(route)] + + print() + + return self.get_items_to_index() def get_document_to_index(self, route): """Render a page and parse it using BeautifulSoup @@ -114,4 +125,4 @@ def remove_document_from_index(path): def build_index_for_all_routes(): ws = WebsiteSearch(INDEX_NAME) - return ws.build() \ No newline at end of file + return ws.build() diff --git a/frappe/test_runner.py b/frappe/test_runner.py index fd8e38e587..cd71dd46c5 100644 --- a/frappe/test_runner.py +++ b/frappe/test_runner.py @@ -9,7 +9,6 @@ import time import xmlrunner import importlib from frappe.modules import load_doctype_module, get_module_name -from frappe.utils import cstr import frappe.utils.scheduler import cProfile, pstats from six import StringIO @@ -308,6 +307,8 @@ def get_dependencies(doctype): if doctype_name in options_list: options_list.remove(doctype_name) + options_list.sort() + return options_list def make_test_records_for_doctype(doctype, verbose=0, force=False): diff --git a/frappe/tests/__init__.py b/frappe/tests/__init__.py index f4dc7f33e2..a310864d83 100644 --- a/frappe/tests/__init__.py +++ b/frappe/tests/__init__.py @@ -3,7 +3,10 @@ import frappe def update_system_settings(args): doc = frappe.get_doc('System Settings') doc.update(args) + doc.flags.ignore_mandatory = 1 doc.save() def get_system_setting(key): return frappe.db.get_single_value("System Settings", key) + +global_test_dependencies = ['User'] diff --git a/frappe/tests/test_auth.py b/frappe/tests/test_auth.py index 086602ea01..bbe9c36aea 100644 --- a/frappe/tests/test_auth.py +++ b/frappe/tests/test_auth.py @@ -110,7 +110,7 @@ class TestLoginAttemptTracker(unittest.TestCase): def test_account_unlock(self): """Make sure that locked account gets unlocked after lock_interval of time. """ - lock_interval = 10 # In sec + lock_interval = 2 # In sec tracker = LoginAttemptTracker(user_name='tester', max_consecutive_login_attempts=1, lock_interval=lock_interval) # Clear the cache by setting attempt as success tracker.add_success_attempt() diff --git a/frappe/tests/test_document.py b/frappe/tests/test_document.py index 2be92be1f5..1a5a8721fd 100644 --- a/frappe/tests/test_document.py +++ b/frappe/tests/test_document.py @@ -6,9 +6,8 @@ import os import unittest import frappe -from frappe.utils import cint, add_to_date, now +from frappe.utils import cint from frappe.model.naming import revert_series_if_last, make_autoname, parse_naming_series -from frappe.exceptions import DoesNotExistError class TestDocument(unittest.TestCase): @@ -87,13 +86,13 @@ class TestDocument(unittest.TestCase): d.insert() self.assertEqual(frappe.db.get_value("User", d.name), d.name) - def test_confict_validation(self): + def test_conflict_validation(self): d1 = self.test_insert() d2 = frappe.get_doc(d1.doctype, d1.name) d1.save() self.assertRaises(frappe.TimestampMismatchError, d2.save) - def test_confict_validation_single(self): + def test_conflict_validation_single(self): d1 = frappe.get_doc("Website Settings", "Website Settings") d1.home_page = "test-web-page-1" @@ -110,7 +109,7 @@ class TestDocument(unittest.TestCase): def test_permission_single(self): frappe.set_user("Guest") - d = frappe.get_doc("Website Settings", "Website Settigns") + d = frappe.get_doc("Website Settings", "Website Settings") self.assertRaises(frappe.PermissionError, d.save) frappe.set_user("Administrator") @@ -196,41 +195,6 @@ class TestDocument(unittest.TestCase): self.assertTrue(xss not in d.subject) self.assertTrue(escaped_xss in d.subject) - def test_link_count(self): - if os.environ.get('CI'): - # cannot run this test reliably in travis due to its handling - # of parallelism - return - - from frappe.model.utils.link_count import update_link_count - - update_link_count() - - doctype, name = 'User', 'test@example.com' - - d = self.test_insert() - d.append('event_participants', {"reference_doctype": doctype, "reference_docname": name}) - - d.save() - - link_count = frappe.cache().get_value('_link_count') or {} - old_count = link_count.get((doctype, name)) or 0 - - frappe.db.commit() - - link_count = frappe.cache().get_value('_link_count') or {} - new_count = link_count.get((doctype, name)) or 0 - - self.assertEqual(old_count + 1, new_count) - - before_update = frappe.db.get_value(doctype, name, 'idx') - - update_link_count() - - after_update = frappe.db.get_value(doctype, name, 'idx') - - self.assertEqual(before_update + new_count, after_update) - def test_naming_series(self): data = ["TEST-", "TEST/17-18/.test_data./.####", "TEST.YYYY.MM.####"] diff --git a/frappe/tests/test_email.py b/frappe/tests/test_email.py index ff7e6d534c..af90ee7a6b 100644 --- a/frappe/tests/test_email.py +++ b/frappe/tests/test_email.py @@ -6,11 +6,7 @@ from __future__ import unicode_literals import unittest, frappe, re, email from six import PY3 -from frappe.test_runner import make_test_records - -make_test_records("User") -make_test_records("Email Account") - +test_dependencies = ['Email Account'] class TestEmail(unittest.TestCase): def setUp(self): diff --git a/frappe/tests/test_fmt_datetime.py b/frappe/tests/test_fmt_datetime.py index 20f2af88ba..e19eb25fe6 100644 --- a/frappe/tests/test_fmt_datetime.py +++ b/frappe/tests/test_fmt_datetime.py @@ -45,6 +45,7 @@ class TestFmtDatetime(unittest.TestCase): frappe.db.set_default("time_format", self.pre_test_time_format) frappe.local.user_date_format = None frappe.local.user_time_format = None + frappe.db.rollback() # Test utility functions @@ -97,28 +98,12 @@ class TestFmtDatetime(unittest.TestCase): self.assertEqual(formatdate(test_date), valid_fmt) # Test time formatters - def test_format_time_forced(self): # Test with forced time formats self.assertEqual( format_time(test_time, 'ss:mm:HH'), test_date_obj.strftime('%S:%M:%H')) - @unittest.expectedFailure - def test_format_time_forced_broken_locale(self): - # Test with forced time formats - # Currently format_time defaults to HH:mm:ss if the locale is - # broken, so this is an expected failure. - lang = frappe.local.lang - try: - # Force fallback from Babel - frappe.local.lang = 'FAKE' - self.assertEqual( - format_time(test_time, 'ss:mm:HH'), - test_date_obj.strftime('%S:%M:%H')) - finally: - frappe.local.lang = lang - def test_format_time(self): # Test format_time with various default time formats set for fmt, valid_fmt in test_time_formats.items(): @@ -135,21 +120,6 @@ class TestFmtDatetime(unittest.TestCase): format_datetime(test_datetime, 'dd-yyyy-MM ss:mm:HH'), test_date_obj.strftime('%d-%Y-%m %S:%M:%H')) - @unittest.expectedFailure - def test_format_datetime_forced_broken_locale(self): - # Test with forced datetime formats - # Currently format_datetime defaults to yyyy-MM-dd HH:mm:ss - # if the locale is broken, so this is an expected failure. - lang = frappe.local.lang - # Force fallback from Babel - try: - frappe.local.lang = 'FAKE' - self.assertEqual( - format_datetime(test_datetime, 'dd-yyyy-MM ss:mm:HH'), - test_date_obj.strftime('%d-%Y-%m %S:%M:%H')) - finally: - frappe.local.lang = lang - def test_format_datetime(self): # Test formatdate with various default date formats set for date_fmt, valid_date in test_date_formats.items(): diff --git a/frappe/tests/test_seen.py b/frappe/tests/test_seen.py index 402bc5f4bf..8eea30d773 100644 --- a/frappe/tests/test_seen.py +++ b/frappe/tests/test_seen.py @@ -41,6 +41,7 @@ class TestSeen(unittest.TestCase): self.assertTrue('test1@example.com' in json.loads(ev._seen)) ev.save() + ev = frappe.get_doc('Event', ev.name) self.assertFalse('test@example.com' in json.loads(ev._seen)) self.assertTrue('test1@example.com' in json.loads(ev._seen)) diff --git a/frappe/tests/test_twofactor.py b/frappe/tests/test_twofactor.py index 7acb0a36e8..709b88b8f3 100644 --- a/frappe/tests/test_twofactor.py +++ b/frappe/tests/test_twofactor.py @@ -8,7 +8,7 @@ from frappe.utils import cint from frappe.utils import set_request from frappe.auth import validate_ip_address, get_login_attempt_tracker from frappe.twofactor import (should_run_2fa, authenticate_for_2factor, get_cached_user_pass, - two_factor_is_enabled_for_, confirm_otp_token, get_otpsecret_for_, get_verification_obj) + two_factor_is_enabled_for_, confirm_otp_token, get_otpsecret_for_, get_verification_obj, ExpiredLoginException) from . import update_system_settings, get_system_setting import time @@ -111,6 +111,7 @@ class TestTwoFactor(unittest.TestCase): def test_confirm_otp_token(self): '''Ensure otp is confirmed''' + frappe.flags.otp_expiry = 2 authenticate_for_2factor(self.user) tmp_id = frappe.local.response['tmp_id'] otp = 'wrongotp' @@ -118,10 +119,11 @@ class TestTwoFactor(unittest.TestCase): confirm_otp_token(self.login_manager,otp=otp,tmp_id=tmp_id) otp = get_otp(self.user) self.assertTrue(confirm_otp_token(self.login_manager,otp=otp,tmp_id=tmp_id)) + frappe.flags.otp_expiry = None if frappe.flags.tests_verbose: - print('Sleeping for 30secs to confirm token expires..') - time.sleep(30) - with self.assertRaises(frappe.AuthenticationError): + print('Sleeping for 2 secs to confirm token expires..') + time.sleep(2) + with self.assertRaises(ExpiredLoginException): confirm_otp_token(self.login_manager,otp=otp,tmp_id=tmp_id) def test_get_verification_obj(self): @@ -208,12 +210,14 @@ def enable_2fa(bypass_two_factor_auth=0, bypass_restrict_ip_check=0): system_settings.bypass_2fa_for_retricted_ip_users = cint(bypass_two_factor_auth) system_settings.bypass_restrict_ip_check_if_2fa_enabled = cint(bypass_restrict_ip_check) system_settings.two_factor_method = 'OTP App' + system_settings.flags.ignore_mandatory = True system_settings.save(ignore_permissions=True) frappe.db.commit() def disable_2fa(): system_settings = frappe.get_doc('System Settings') system_settings.enable_two_factor_auth = 0 + system_settings.flags.ignore_mandatory = True system_settings.save(ignore_permissions=True) frappe.db.commit() diff --git a/frappe/twofactor.py b/frappe/twofactor.py index 0a120d5287..4e098c3075 100644 --- a/frappe/twofactor.py +++ b/frappe/twofactor.py @@ -73,11 +73,11 @@ def cache_2fa_data(user, token, otp_secret, tmp_id): # set increased expiry time for SMS and Email if verification_method in ['SMS', 'Email']: - expiry_time = 300 + expiry_time = frappe.flags.token_expiry or 300 frappe.cache().set(tmp_id + '_token', token) frappe.cache().expire(tmp_id + '_token', expiry_time) else: - expiry_time = 180 + expiry_time = frappe.flags.otp_expiry or 180 for k, v in iteritems({'_usr': user, '_pwd': pwd, '_otp_secret': otp_secret}): frappe.cache().set("{0}{1}".format(tmp_id, k), v) frappe.cache().expire("{0}{1}".format(tmp_id, k), expiry_time) diff --git a/frappe/utils/__init__.py b/frappe/utils/__init__.py index 0d59aa2197..8dcc5f527a 100644 --- a/frappe/utils/__init__.py +++ b/frappe/utils/__init__.py @@ -161,7 +161,7 @@ def validate_url(txt, throw=False, valid_schemes=None): Parameters: throw (`bool`): throws a validationError if URL is not valid - valid_schemes (`str` or `list`): if provided checks the given URL's scheme against this + valid_schemes (`str` or `list`): if provided checks the given URL's scheme against this Returns: bool: if `txt` represents a valid URL @@ -225,14 +225,17 @@ def get_gravatar(email): return gravatar_url -def get_traceback(): +def get_traceback() -> str: """ Returns the traceback of the Exception """ exc_type, exc_value, exc_tb = sys.exc_info() + + if not any([exc_type, exc_value, exc_tb]): + return "" + trace_list = traceback.format_exception(exc_type, exc_value, exc_tb) - body = "".join(cstr(t) for t in trace_list) - return body + return "".join(cstr(t) for t in trace_list) def log(event, details): frappe.logger().info(details) @@ -425,7 +428,7 @@ def get_test_client(): return Client(application) def get_hook_method(hook_name, fallback=None): - method = (frappe.get_hooks().get(hook_name)) + method = frappe.get_hooks().get(hook_name) if method: method = frappe.get_attr(method[0]) return method @@ -439,6 +442,16 @@ def call_hook_method(hook, *args, **kwargs): return out +def is_cli() -> bool: + """Returns True if current instance is being run via a terminal + """ + invoked_from_terminal = False + try: + invoked_from_terminal = bool(os.get_terminal_size()) + except Exception: + invoked_from_terminal = sys.stdin.isatty() + return invoked_from_terminal + def update_progress_bar(txt, i, l): if os.environ.get("CI"): if i == 0: @@ -448,7 +461,7 @@ def update_progress_bar(txt, i, l): sys.stdout.flush() return - if not getattr(frappe.local, 'request', None): + if not getattr(frappe.local, 'request', None) or is_cli(): lt = len(txt) try: col = 40 if os.get_terminal_size().columns > 80 else 20 diff --git a/frappe/utils/boilerplate.py b/frappe/utils/boilerplate.py index 20e98ac67f..d950d9f082 100755 --- a/frappe/utils/boilerplate.py +++ b/frappe/utils/boilerplate.py @@ -40,7 +40,7 @@ def make_boilerplate(dest, app_name): if hook_key=="app_name" and hook_val.lower().replace(" ", "_") != hook_val: print("App Name must be all lowercase and without spaces") hook_val = "" - elif hook_key=="app_title" and not re.match("^(?![\W])[^\d_\s][\w -]+$", hook_val, re.UNICODE): + elif hook_key=="app_title" and not re.match(r"^(?![\W])[^\d_\s][\w -]+$", hook_val, re.UNICODE): print("App Title should start with a letter and it can only consist of letters, numbers, spaces and underscores") hook_val = "" diff --git a/frappe/utils/data.py b/frappe/utils/data.py index 1e09be7ad0..5063783733 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -1278,7 +1278,9 @@ def make_filter_dict(filters): def sanitize_column(column_name): from frappe import _ + import sqlparse regex = re.compile("^.*[,'();].*") + column_name = sqlparse.format(column_name, strip_comments=True, keyword_case="lower") blacklisted_keywords = ['select', 'create', 'insert', 'delete', 'drop', 'update', 'case', 'and', 'or'] def _raise_exception(): diff --git a/frappe/utils/file_manager.py b/frappe/utils/file_manager.py index e165a4e338..2177e67274 100644 --- a/frappe/utils/file_manager.py +++ b/frappe/utils/file_manager.py @@ -418,7 +418,7 @@ def extract_images_from_html(doc, content): return ']*src\s*=\s*["\'](?=data:)(.*?)["\']', _save_file, content) + content = re.sub(r']*src\s*=\s*["\'](?=data:)(.*?)["\']', _save_file, content) return content diff --git a/frappe/utils/formatters.py b/frappe/utils/formatters.py index 5d1e9bdb19..7913413878 100644 --- a/frappe/utils/formatters.py +++ b/frappe/utils/formatters.py @@ -78,7 +78,7 @@ def format_value(value, df=None, doc=None, currency=None, translated=False): return "{}%".format(flt(value, 2)) elif df.get("fieldtype") in ("Text", "Small Text"): - if not re.search("(\") elif df.get("fieldtype") == "Markdown Editor": diff --git a/frappe/utils/global_search.py b/frappe/utils/global_search.py index 1c067d0146..c20f3b29d4 100644 --- a/frappe/utils/global_search.py +++ b/frappe/utils/global_search.py @@ -348,7 +348,7 @@ def get_formatted_value(value, field): if getattr(field, 'fieldtype', None) in ["Text", "Text Editor"]: value = unescape_html(frappe.safe_decode(value)) - value = (re.subn(r'<[\s]*(script|style).*?(?s)', '', text_type(value))[0]) + value = (re.subn(r'(?s)<[\s]*(script|style).*?', '', text_type(value))[0]) value = ' '.join(value.split()) return field.label + " : " + strip_html_tags(text_type(value)) diff --git a/frappe/utils/redis_wrapper.py b/frappe/utils/redis_wrapper.py index fe75e67d8e..678a61ca6e 100644 --- a/frappe/utils/redis_wrapper.py +++ b/frappe/utils/redis_wrapper.py @@ -98,7 +98,7 @@ class RedisWrapper(redis.Redis): return self.keys(key) except redis.exceptions.ConnectionError: - regex = re.compile(cstr(key).replace("|", "\|").replace("*", "[\w]*")) + regex = re.compile(cstr(key).replace("|", r"\|").replace("*", r"[\w]*")) return [k for k in list(frappe.local.cache) if regex.match(cstr(k))] def delete_keys(self, key): diff --git a/frappe/website/context.py b/frappe/website/context.py index b39770dcd4..c898d39869 100644 --- a/frappe/website/context.py +++ b/frappe/website/context.py @@ -61,7 +61,7 @@ def update_controller_context(context, controller): except (frappe.PermissionError, frappe.PageDoesNotExistError, frappe.Redirect): raise except: - if not frappe.flags.in_migrate: + if not any([frappe.flags.in_migrate, frappe.flags.in_website_search_build]): frappe.errprint(frappe.utils.get_traceback()) if hasattr(module, "get_children"): diff --git a/frappe/website/doctype/blog_post/test_blog_post.py b/frappe/website/doctype/blog_post/test_blog_post.py index f0fc484a31..9ecac07ee5 100644 --- a/frappe/website/doctype/blog_post/test_blog_post.py +++ b/frappe/website/doctype/blog_post/test_blog_post.py @@ -13,6 +13,8 @@ from frappe.website.doctype.blog_post.blog_post import get_blog_list from frappe.website.website_generator import WebsiteGenerator from frappe.custom.doctype.customize_form.customize_form import reset_customization +test_dependencies = ['Blog Post'] + class TestBlogPost(unittest.TestCase): def setUp(self): reset_customization('Blog Post') diff --git a/frappe/website/doctype/web_form/test_web_form.py b/frappe/website/doctype/web_form/test_web_form.py index d16a613546..78f7fd6337 100644 --- a/frappe/website/doctype/web_form/test_web_form.py +++ b/frappe/website/doctype/web_form/test_web_form.py @@ -8,7 +8,7 @@ import unittest, json from frappe.website.render import build_page from frappe.website.doctype.web_form.web_form import accept -test_records = frappe.get_test_records('Web Form') +test_dependencies = ['Web Form'] class TestWebForm(unittest.TestCase): def setUp(self): diff --git a/frappe/website/doctype/web_page/test_web_page.py b/frappe/website/doctype/web_page/test_web_page.py index a481337978..7a2ddc6961 100644 --- a/frappe/website/doctype/web_page/test_web_page.py +++ b/frappe/website/doctype/web_page/test_web_page.py @@ -39,7 +39,7 @@ class TestWebPage(unittest.TestCase): published = 1, content_type = 'Rich Text', main_section = 'rich text', - main_section_md = '# h1\n\markdown content', + main_section_md = '# h1\nmarkdown content', main_section_html = '
html content
' )).insert() diff --git a/frappe/website/router.py b/frappe/website/router.py index 4acb0c6b9b..f3518e179e 100644 --- a/frappe/website/router.py +++ b/frappe/website/router.py @@ -158,7 +158,7 @@ def evaluate_dynamic_routes(rules, path): route_map = Map(rules) endpoint = None - if frappe.local.request: + if hasattr(frappe.local, 'request') and frappe.local.request.environ: urls = route_map.bind_to_environ(frappe.local.request.environ) try: endpoint, args = urls.match("/" + path) diff --git a/frappe/www/app.py b/frappe/www/app.py index 6088c413dc..5f19712cd3 100644 --- a/frappe/www/app.py +++ b/frappe/www/app.py @@ -34,10 +34,10 @@ def get_context(context): boot_json = frappe.as_json(boot) # remove script tags from boot - boot_json = re.sub("\", "", boot_json) + boot_json = re.sub(r"\", "", boot_json) # TODO: Find better fix - boot_json = re.sub("", "", boot_json) + boot_json = re.sub(r"", "", boot_json) context.update({ "no_cache": 1, diff --git a/frappe/www/printview.py b/frappe/www/printview.py index 04e846c41d..3ddf032e9e 100644 --- a/frappe/www/printview.py +++ b/frappe/www/printview.py @@ -409,7 +409,7 @@ def get_print_style(style=None, print_format=None, for_legacy=False): css = css + '\n' + frappe.db.get_value('Print Style', style, 'css') # move @import to top - for at_import in list(set(re.findall("(@import url\([^\)]+\)[;]?)", css))): + for at_import in list(set(re.findall(r"(@import url\([^\)]+\)[;]?)", css))): css = css.replace(at_import, "") # prepend css with at_import diff --git a/requirements.txt b/requirements.txt index 193c5a86b6..769d8c3e7b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,8 @@ boto3~=1.17.53 braintree~=4.8.0 chardet~=4.0.0 Click~=7.1.2 -coverage~=4.5.4 +colorama~=0.4.4 +coverage==5.5 croniter~=1.0.11 cryptography~=3.4.7 dropbox~=11.7.0