vassili/seitime-frappe
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Merge remote-tracking branch 'upstream/develop' into esbuild

Browse source
This commit is contained in:
Faris Ansari 2021-05-16 11:26:43 +05:30
commit f8ca990a83
66 changed files with 907 additions and 440 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
.flake8
View file

@ -29,4 +29,5 @@ ignore =
B950,
W191,
max-line-length = 200
max-line-length = 200
exclude=.github/helper/semgrep_rules

66
.github/helper/semgrep_rules/frappe_correctness.py vendored
View file

@ -4,25 +4,61 @@ from frappe import _, flt
from frappe.model.document import Document
# ruleid: frappe-modifying-but-not-comitting
def on_submit(self):
if self.value_of_goods == 0:
frappe.throw(_('Value of goods cannot be 0'))
# ruleid: frappe-modifying-after-submit
self.status = 'Submitted'
def on_submit(self): # noqa
if flt(self.per_billed) < 100:
self.update_billing_status()
else:
# todook: frappe-modifying-after-submit
self.status = "Completed"
self.db_set("status", "Completed")
class TestDoc(Document):
pass
# ok: frappe-modifying-but-not-comitting
def on_submit(self):
if self.value_of_goods == 0:
frappe.throw(_('Value of goods cannot be 0'))
self.status = 'Submitted'
self.db_set('status', 'Submitted')
def validate(self):
#ruleid: frappe-modifying-child-tables-while-iterating
for item in self.child_table:
if item.value < 0:
self.remove(item)
# ok: frappe-modifying-but-not-comitting
def on_submit(self):
if self.value_of_goods == 0:
frappe.throw(_('Value of goods cannot be 0'))
x = "y"
self.status = x
self.db_set('status', x)
# ok: frappe-modifying-but-not-comitting
def on_submit(self):
x = "y"
self.status = x
self.save()
# ruleid: frappe-modifying-but-not-comitting-other-method
class DoctypeClass(Document):
def on_submit(self):
self.good_method()
self.tainted_method()
def tainted_method(self):
self.status = "uptate"
# ok: frappe-modifying-but-not-comitting-other-method
class DoctypeClass(Document):
def on_submit(self):
self.good_method()
self.tainted_method()
def tainted_method(self):
self.status = "update"
self.db_set("status", "update")
# ok: frappe-modifying-but-not-comitting-other-method
class DoctypeClass(Document):
def on_submit(self):
self.good_method()
self.tainted_method()
self.save()
def tainted_method(self):
self.status = "uptate"

7
.github/helper/semgrep_rules/translate.js vendored
View file

@ -35,3 +35,10 @@ __('You have' + 'subscribers in your mailing list.')
// ruleid: frappe-translation-js-splitting
__('You have {0} subscribers' +
'in your mailing list', [subscribers.length])
// ok: frappe-translation-js-splitting
__("Ctrl+Enter to add comment")
// ruleid: frappe-translation-js-splitting
__('You have {0} subscribers \
in your mailing list', [subscribers.length])

8
.github/helper/semgrep_rules/translate.py vendored
View file

@ -51,3 +51,11 @@ _(f"what" + f"this is also not cool")
_("")
# ruleid: frappe-translation-empty-string
_('')
class Test:
# ok: frappe-translation-python-splitting
def __init__(
args
):
pass

8
.github/helper/semgrep_rules/translate.yml vendored
View file

@ -44,8 +44,8 @@ rules:
pattern-either:
- pattern: _(...) + _(...)
- pattern: _("..." + "...")
- pattern-regex: '_\([^\)]*\\\s*' # lines broken by `\`
- pattern-regex: '_\(\s*\n' # line breaks allowed by python for using ( )
- pattern-regex: '[\s\.]_\([^\)]*\\\s*' # lines broken by `\`
- pattern-regex: '[\s\.]_\(\s*\n' # line breaks allowed by python for using ( )
message: |
Do not split strings inside translate function. Do not concatenate using translate functions.
Please refer: https://frappeframework.com/docs/user/en/translations
@ -54,8 +54,8 @@ rules:
- id: frappe-translation-js-splitting
pattern-either:
- pattern-regex: '__\([^\)]*[\+\\]\s*'
- pattern: __('...' + '...')
- pattern-regex: '__\([^\)]*[\\]\s+'
- pattern: __('...' + '...', ...)
- pattern: __('...') + __('...')
message: |
Do not split strings inside translate function. Do not concatenate using translate functions.

2
.github/workflows/semgrep.yml vendored
View file

@ -4,6 +4,8 @@ on:
pull_request:
branches:
- develop
- version-13-hotfix
- version-13-pre-release
jobs:
semgrep:
name: Frappe Linter

100
.github/workflows/ci-tests.yml → .github/workflows/server-mariadb-tests.yml vendored
View file

@ -1,10 +1,8 @@
name: CI
name: Server
on:
pull_request:
types: [opened, synchronize, reopened, labeled, unlabeled]
workflow_dispatch:
push:
jobs:
test:
@ -13,23 +11,9 @@ jobs:
strategy:
fail-fast: false
matrix:
include:
- DB: "mariadb"
TYPE: "server"
JOB_NAME: "Python MariaDB"
RUN_COMMAND: bench --site test_site run-tests --coverage
container: [1, 2]
- DB: "postgres"
TYPE: "server"
JOB_NAME: "Python PostgreSQL"
RUN_COMMAND: bench --site test_site run-tests --coverage
- DB: "mariadb"
TYPE: "ui"
JOB_NAME: "UI MariaDB"
RUN_COMMAND: bench --site test_site run-ui-tests frappe --headless
name: ${{ matrix.JOB_NAME }}
name: Python Unit Tests (MariaDB)
services:
mysql:
@ -40,18 +24,6 @@ jobs:
- 3306:3306
options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=3
postgres:
image: postgres:12.4
env:
POSTGRES_PASSWORD: travis
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
ports:
- 5432:5432
steps:
- name: Clone
uses: actions/checkout@v2
@ -104,68 +76,54 @@ jobs:
restore-keys: |
${{ runner.os }}-yarn-
- name: Cache cypress binary
if: matrix.TYPE == 'ui'
uses: actions/cache@v2
with:
path: ~/.cache
key: ${{ runner.os }}-cypress-
restore-keys: |
${{ runner.os }}-cypress-
${{ runner.os }}-
- name: Install Dependencies
run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh
env:
BEFORE: ${{ env.GITHUB_EVENT_PATH.before }}
AFTER: ${{ env.GITHUB_EVENT_PATH.after }}
TYPE: ${{ matrix.TYPE }}
TYPE: server
- name: Install
run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh
env:
DB: ${{ matrix.DB }}
TYPE: ${{ matrix.TYPE }}
DB: mariadb
TYPE: server
- name: Run Set-Up
if: matrix.TYPE == 'ui'
run: cd ~/frappe-bench/ && bench --site test_site execute frappe.utils.install.complete_setup_wizard
env:
DB: ${{ matrix.DB }}
TYPE: ${{ matrix.TYPE }}
- name: Setup tmate session
if: contains(github.event.pull_request.labels.*.name, 'debug-gha')
uses: mxschmitt/action-tmate@v3
- name: Run Tests
run: cd ~/frappe-bench/ && ${{ matrix.RUN_COMMAND }}
run: cd ~/frappe-bench/ && bench --site test_site run-parallel-tests --use-orchestrator --with-coverage
env:
DB: ${{ matrix.DB }}
TYPE: ${{ matrix.TYPE }}
CI_BUILD_ID: ${{ github.run_id }}
ORCHESTRATOR_URL: http://test-orchestrator.frappe.io
- name: Coverage - Pull Request
if: matrix.TYPE == 'server' && github.event_name == 'pull_request'
- name: Upload Coverage Data
run: |
cp ~/frappe-bench/sites/.coverage ${GITHUB_WORKSPACE}
cd ${GITHUB_WORKSPACE}
pip install coveralls==2.2.0
pip install coverage==4.5.4
coveralls --service=github
pip3 install coverage==5.5
pip3 install coveralls==3.0.1
coveralls
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_TOKEN }}
COVERALLS_SERVICE_NAME: github
COVERALLS_FLAG_NAME: run-${{ matrix.container }}
COVERALLS_SERVICE_NAME: ${{ github.event_name == 'pull_request' && 'github' || 'github-actions' }}
COVERALLS_PARALLEL: true
- name: Coverage - Push
if: matrix.TYPE == 'server' && github.event_name == 'push'
coveralls:
name: Coverage Wrap Up
needs: test
container: python:3-slim
runs-on: ubuntu-18.04
steps:
- name: Clone
uses: actions/checkout@v2
- name: Coveralls Finished
run: |
cp ~/frappe-bench/sites/.coverage ${GITHUB_WORKSPACE}
cd ${GITHUB_WORKSPACE}
pip install coveralls==2.2.0
pip install coverage==4.5.4
coveralls --service=github-actions
pip3 install coverage==5.5
pip3 install coveralls==3.0.1
coveralls --finish
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_TOKEN }}
COVERALLS_SERVICE_NAME: github-actions

100
.github/workflows/server-postgres-tests.yml vendored Normal file
View file

@ -0,0 +1,100 @@
name: Server
on:
pull_request:
workflow_dispatch:
jobs:
test:
runs-on: ubuntu-18.04
strategy:
fail-fast: false
matrix:
container: [1, 2]
name: Python Unit Tests (Postgres)
services:
postgres:
image: postgres:12.4
env:
POSTGRES_PASSWORD: travis
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
ports:
- 5432:5432
steps:
- name: Clone
uses: actions/checkout@v2
- name: Setup Python
uses: actions/setup-python@v2
with:
python-version: 3.7
- uses: actions/setup-node@v2
with:
node-version: '14'
check-latest: true
- name: Add to Hosts
run: |
echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts
echo "127.0.0.1 test_site_producer" | sudo tee -a /etc/hosts
- name: Cache pip
uses: actions/cache@v2
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
restore-keys: |
${{ runner.os }}-pip-
${{ runner.os }}-
- name: Cache node modules
uses: actions/cache@v2
env:
cache-name: cache-node-modules
with:
path: ~/.npm
key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
restore-keys: |
${{ runner.os }}-build-${{ env.cache-name }}-
${{ runner.os }}-build-
${{ runner.os }}-
- name: Get yarn cache directory path
id: yarn-cache-dir-path
run: echo "::set-output name=dir::$(yarn cache dir)"
- uses: actions/cache@v2
id: yarn-cache
with:
path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn-
- name: Install Dependencies
run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh
env:
BEFORE: ${{ env.GITHUB_EVENT_PATH.before }}
AFTER: ${{ env.GITHUB_EVENT_PATH.after }}
TYPE: server
- name: Install
run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh
env:
DB: postgres
TYPE: server
- name: Run Tests
run: cd ~/frappe-bench/ && bench --site test_site run-parallel-tests --use-orchestrator
env:
CI_BUILD_ID: ${{ github.run_id }}
ORCHESTRATOR_URL: http://test-orchestrator.frappe.io

105
.github/workflows/ui-tests.yml vendored Normal file
View file

@ -0,0 +1,105 @@
name: UI
on:
pull_request:
workflow_dispatch:
jobs:
test:
runs-on: ubuntu-18.04
strategy:
fail-fast: false
matrix:
containers: [1, 2]
name: UI Tests (Cypress)
services:
mysql:
image: mariadb:10.3
env:
MYSQL_ALLOW_EMPTY_PASSWORD: YES
ports:
- 3306:3306
options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=3
steps:
- name: Clone
uses: actions/checkout@v2
- name: Setup Python
uses: actions/setup-python@v2
with:
python-version: 3.7
- uses: actions/setup-node@v2
with:
node-version: '12'
check-latest: true
- name: Add to Hosts
run: |
echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts
echo "127.0.0.1 test_site_producer" | sudo tee -a /etc/hosts
- name: Cache pip
uses: actions/cache@v2
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
restore-keys: |
${{ runner.os }}-pip-
${{ runner.os }}-
- name: Cache node modules
uses: actions/cache@v2
env:
cache-name: cache-node-modules
with:
path: ~/.npm
key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
restore-keys: |
${{ runner.os }}-build-${{ env.cache-name }}-
${{ runner.os }}-build-
${{ runner.os }}-
- name: Get yarn cache directory path
id: yarn-cache-dir-path
run: echo "::set-output name=dir::$(yarn cache dir)"
- uses: actions/cache@v2
id: yarn-cache
with:
path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn-
- name: Cache cypress binary
uses: actions/cache@v2
with:
path: ~/.cache
key: ${{ runner.os }}-cypress-
restore-keys: |
${{ runner.os }}-cypress-
${{ runner.os }}-
- name: Install Dependencies
run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh
env:
BEFORE: ${{ env.GITHUB_EVENT_PATH.before }}
AFTER: ${{ env.GITHUB_EVENT_PATH.after }}
TYPE: ui
- name: Install
run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh
env:
DB: mariadb
TYPE: ui
- name: Site Setup
run: cd ~/frappe-bench/ && bench --site test_site execute frappe.utils.install.complete_setup_wizard
- name: UI Tests
run: cd ~/frappe-bench/ && bench --site test_site run-ui-tests frappe --headless --parallel --ci-build-id $GITHUB_RUN_ID

18
.mergify.yml
View file

@ -3,9 +3,12 @@ pull_request_rules:
conditions:
- status-success=Sider
- status-success=Semantic Pull Request
- status-success=Python MariaDB
- status-success=Python PostgreSQL
- status-success=UI MariaDB
- status-success=Python Unit Tests (MariaDB) (1)
- status-success=Python Unit Tests (MariaDB) (2)
- status-success=Python Unit Tests (Postgres) (1)
- status-success=Python Unit Tests (Postgres) (2)
- status-success=UI Tests (Cypress) (1)
- status-success=UI Tests (Cypress) (2)
- status-success=security/snyk (frappe)
- label!=dont-merge
- label!=squash
@ -16,9 +19,12 @@ pull_request_rules:
- name: Automatic squash on CI success and review
conditions:
- status-success=Sider
- status-success=Python MariaDB
- status-success=Python PostgreSQL
- status-success=UI MariaDB
- status-success=Python Unit Tests (MariaDB) (1)
- status-success=Python Unit Tests (MariaDB) (2)
- status-success=Python Unit Tests (Postgres) (1)
- status-success=Python Unit Tests (Postgres) (2)
- status-success=UI Tests (Cypress) (1)
- status-success=UI Tests (Cypress) (2)
- status-success=security/snyk (frappe)
- label!=dont-merge
- label=squash

25
frappe/app.py
View file

@ -99,17 +99,7 @@ def application(request):
frappe.monitor.stop(response)
frappe.recorder.dump()
if hasattr(frappe.local, 'conf') and frappe.local.conf.enable_frappe_logger:
frappe.logger("frappe.web", allow_site=frappe.local.site).info({
"site": get_site_name(request.host),
"remote_addr": getattr(request, "remote_addr", "NOTFOUND"),
"base_url": getattr(request, "base_url", "NOTFOUND"),
"full_path": getattr(request, "full_path", "NOTFOUND"),
"method": getattr(request, "method", "NOTFOUND"),
"scheme": getattr(request, "scheme", "NOTFOUND"),
"http_status_code": getattr(response, "status_code", "NOTFOUND")
})
log_request(request, response)
process_response(response)
frappe.destroy()
@ -137,6 +127,19 @@ def init_request(request):
if request.method != "OPTIONS":
frappe.local.http_request = frappe.auth.HTTPRequest()
def log_request(request, response):
if hasattr(frappe.local, 'conf') and frappe.local.conf.enable_frappe_logger:
frappe.logger("frappe.web", allow_site=frappe.local.site).info({
"site": get_site_name(request.host),
"remote_addr": getattr(request, "remote_addr", "NOTFOUND"),
"base_url": getattr(request, "base_url", "NOTFOUND"),
"full_path": getattr(request, "full_path", "NOTFOUND"),
"method": getattr(request, "method", "NOTFOUND"),
"scheme": getattr(request, "scheme", "NOTFOUND"),
"http_status_code": getattr(response, "status_code", "NOTFOUND")
})
def process_response(response):
if not response:
return

175
frappe/build.py
View file

@ -1,14 +1,11 @@
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt
from __future__ import print_function, unicode_literals
import os
import re
import json
import shutil
import warnings
import tempfile
from tempfile import mkdtemp, mktemp
from distutils.spawn import find_executable
import frappe
@ -16,8 +13,8 @@ from frappe.utils.minify import JavascriptMinify
import click
import psutil
from six import iteritems, text_type
from six.moves.urllib.parse import urlparse
from urllib.parse import urlparse
from simple_chalk import green
timestamps = {}
@ -76,8 +73,8 @@ def get_assets_link(frappe_head):
from requests import head
tag = getoutput(
"cd ../apps/frappe && git show-ref --tags -d | grep %s | sed -e 's,.*"
" refs/tags/,,' -e 's/\^{}//'"
r"cd ../apps/frappe && git show-ref --tags -d | grep %s | sed -e 's,.*"
r" refs/tags/,,' -e 's/\^{}//'"
% frappe_head
)
@ -98,9 +95,7 @@ def download_frappe_assets(verbose=True):
commit HEAD.
Returns True if correctly setup else returns False.
"""
from simple_chalk import green
from subprocess import getoutput
from tempfile import mkdtemp
assets_setup = False
frappe_head = getoutput("cd ../apps/frappe && git rev-parse HEAD")
@ -167,7 +162,7 @@ def symlink(target, link_name, overwrite=False):
# Create link to target with temporary filename
while True:
temp_link_name = tempfile.mktemp(dir=link_dir)
temp_link_name = mktemp(dir=link_dir)
# os.* functions mimic as closely as possible system functions
# The POSIX symlink() returns EEXIST if link_name already exists
@ -194,7 +189,8 @@ def symlink(target, link_name, overwrite=False):
def setup():
global app_paths
global app_paths, assets_path
pymodules = []
for app in frappe.get_all_apps(True):
try:
@ -202,12 +198,13 @@ def setup():
except ImportError:
pass
app_paths = [os.path.dirname(pymodule.__file__) for pymodule in pymodules]
assets_path = os.path.join(frappe.local.sites_path, "assets")
def bundle(mode, apps=None, make_copy=False, restore=False, verbose=False, skip_frappe=False, files=None):
def bundle(mode, apps=None, hard_link=False, make_copy=False, restore=False, verbose=False, skip_frappe=False, files=None):
"""concat / minify js files"""
setup()
make_asset_dirs(make_copy=make_copy, restore=restore)
make_asset_dirs(hard_link=hard_link)
mode = "production" if mode == "production" else "build"
command = "yarn run {mode}".format(mode=mode)
@ -264,75 +261,101 @@ def get_safe_max_old_space_size():
return safe_max_old_space_size
def make_asset_dirs(make_copy=False, restore=False):
# don't even think of making assets_path absolute - rm -rf ahead.
assets_path = os.path.join(frappe.local.sites_path, "assets")
def generate_assets_map():
symlinks = {}
for dir_path in [os.path.join(assets_path, "js"), os.path.join(assets_path, "css")]:
if not os.path.exists(dir_path):
os.makedirs(dir_path)
for app_name in frappe.get_all_apps():
app_doc_path = None
for app_name in frappe.get_all_apps(True):
pymodule = frappe.get_module(app_name)
app_base_path = os.path.abspath(os.path.dirname(pymodule.__file__))
symlinks = []
app_public_path = os.path.join(app_base_path, "public")
# app/public > assets/app
symlinks.append([app_public_path, os.path.join(assets_path, app_name)])
# app/node_modules > assets/app/node_modules
if os.path.exists(os.path.abspath(app_public_path)):
symlinks.append(
[
os.path.join(app_base_path, "..", "node_modules"),
os.path.join(assets_path, app_name, "node_modules"),
]
)
app_node_modules_path = os.path.join(app_base_path, "..", "node_modules")
app_docs_path = os.path.join(app_base_path, "docs")
app_www_docs_path = os.path.join(app_base_path, "www", "docs")
app_doc_path = None
if os.path.isdir(os.path.join(app_base_path, "docs")):
app_assets = os.path.abspath(app_public_path)
app_node_modules = os.path.abspath(app_node_modules_path)
# {app}/public > assets/{app}
if os.path.isdir(app_assets):
symlinks[app_assets] = os.path.join(assets_path, app_name)
# {app}/node_modules > assets/{app}/node_modules
if os.path.isdir(app_node_modules):
symlinks[app_node_modules] = os.path.join(assets_path, app_name, "node_modules")
# {app}/docs > assets/{app}_docs
if os.path.isdir(app_docs_path):
app_doc_path = os.path.join(app_base_path, "docs")
elif os.path.isdir(os.path.join(app_base_path, "www", "docs")):
elif os.path.isdir(app_www_docs_path):
app_doc_path = os.path.join(app_base_path, "www", "docs")
if app_doc_path:
symlinks.append([app_doc_path, os.path.join(assets_path, app_name + "_docs")])
app_docs = os.path.abspath(app_doc_path)
symlinks[app_docs] = os.path.join(assets_path, app_name + "_docs")
for source, target in symlinks:
source = os.path.abspath(source)
if os.path.exists(source):
if restore:
if os.path.exists(target):
if os.path.islink(target):
os.unlink(target)
else:
shutil.rmtree(target)
shutil.copytree(source, target)
elif make_copy:
if os.path.exists(target):
warnings.warn("Target {target} already exists.".format(target=target))
else:
shutil.copytree(source, target)
else:
if os.path.exists(target):
if os.path.islink(target):
os.unlink(target)
else:
shutil.rmtree(target)
try:
symlink(source, target, overwrite=True)
except OSError:
print("Cannot link {} to {}".format(source, target))
else:
warnings.warn('Source {source} does not exist.'.format(source = source))
pass
return symlinks
def setup_assets_dirs():
for dir_path in (os.path.join(assets_path, x) for x in ("js", "css")):
os.makedirs(dir_path, exist_ok=True)
def clear_broken_symlinks():
for path in os.listdir(assets_path):
path = os.path.join(assets_path, path)
if os.path.islink(path) and not os.path.exists(path):
os.remove(path)
def unstrip(message):
try:
max_str = os.get_terminal_size().columns
except Exception:
max_str = 80
_len = len(message)
_rem = max_str - _len
return f"{message}{' ' * _rem}"
def make_asset_dirs(hard_link=False):
setup_assets_dirs()
clear_broken_symlinks()
symlinks = generate_assets_map()
for source, target in symlinks.items():
start_message = unstrip(f"{'Copying assets from' if hard_link else 'Linking'} {source} to {target}")
fail_message = unstrip(f"Cannot {'copy' if hard_link else 'link'} {source} to {target}")
try:
print(start_message, end="\r")
link_assets_dir(source, target, hard_link=hard_link)
except Exception:
print(fail_message, end="\r")
print(unstrip(f"{green('✔')} Application Assets Linked") + "\n")
def link_assets_dir(source, target, hard_link=False):
if not os.path.exists(source):
return
if os.path.exists(target):
if os.path.islink(target):
os.unlink(target)
else:
shutil.rmtree(target)
if hard_link:
shutil.copytree(source, target, dirs_exist_ok=True)
else:
symlink(source, target, overwrite=True)
def build(no_compress=False, verbose=False):
assets_path = os.path.join(frappe.local.sites_path, "assets")
for target, sources in iteritems(get_build_maps()):
for target, sources in get_build_maps().items():
pack(os.path.join(assets_path, target), sources, no_compress, verbose)
@ -346,7 +369,7 @@ def get_build_maps():
if os.path.exists(path):
with open(path) as f:
try:
for target, sources in iteritems(json.loads(f.read())):
for target, sources in (json.loads(f.read() or "{}")).items():
# update app path
source_paths = []
for source in sources:
@ -379,7 +402,7 @@ def pack(target, sources, no_compress, verbose):
timestamps[f] = os.path.getmtime(f)
try:
with open(f, "r") as sourcefile:
data = text_type(sourcefile.read(), "utf-8", errors="ignore")
data = str(sourcefile.read(), "utf-8", errors="ignore")
extn = f.rsplit(".", 1)[1]
@ -394,7 +417,7 @@ def pack(target, sources, no_compress, verbose):
jsm.minify(tmpin, tmpout)
minified = tmpout.getvalue()
if minified:
outtxt += text_type(minified or "", "utf-8").strip("\n") + ";"
outtxt += str(minified or "", "utf-8").strip("\n") + ";"
if verbose:
print("{0}: {1}k".format(f, int(len(minified) / 1024)))
@ -424,16 +447,16 @@ def html_to_js_template(path, content):
def scrub_html_template(content):
"""Returns HTML content with removed whitespace and comments"""
# remove whitespace to a single space
content = re.sub("\s+", " ", content)
content = re.sub(r"\s+", " ", content)
# strip comments
content = re.sub("(<!--.*?-->)", "", content)
content = re.sub(r"(<!--.*?-->)", "", content)
return content.replace("'", "\'")
def files_dirty():
for target, sources in iteritems(get_build_maps()):
for target, sources in get_build_maps().items():
for f in sources:
if ":" in f:
f, suffix = f.split(":")

4
frappe/commands/__init__.py
View file

@ -28,6 +28,10 @@ def pass_context(f):
except frappe.exceptions.SiteNotSpecifiedError as e:
click.secho(str(e), fg='yellow')
sys.exit(1)
except frappe.exceptions.IncorrectSitePath:
site = ctx.obj.get("sites", "")[0]
click.secho(f'Site {site} does not exist!', fg='yellow')
sys.exit(1)
if profile:
pr.disable()

53
frappe/commands/utils.py
View file

@ -15,17 +15,17 @@ from frappe.utils import get_bench_path, update_progress_bar, cint
@click.command('build')
@click.option('--app', help='Build assets for specific app')
@click.option('--app', help='Build assets for app')
@click.option('--apps', help='Build assets for specific apps')
@click.option('--make-copy', is_flag=True, default=False, help='Copy the files instead of symlinking')
@click.option('--restore', is_flag=True, default=False, help='Copy the files instead of symlinking with force')
@click.option('--hard-link', is_flag=True, default=False, help='Copy the files instead of symlinking')
@click.option('--make-copy', is_flag=True, default=False, help='[DEPRECATED] Copy the files instead of symlinking')
@click.option('--restore', is_flag=True, default=False, help='[DEPRECATED] Copy the files instead of symlinking with force')
@click.option('--production', is_flag=True, default=False, help='Build assets in production mode')
@click.option('--force', is_flag=True, default=False, help='Force build assets instead of downloading available')
@click.option('--verbose', is_flag=True, default=False, help='Verbose')
def build(app=None, apps=None, make_copy=False, restore=False, production=False, verbose=False, force=False):
@click.option('--force', is_flag=True, default=False, help='Force build assets instead of downloading available')
def build(app=None, apps=None, hard_link=False, make_copy=False, restore=False, production=False, verbose=False, force=False):
"Compile JS and CSS source files"
from frappe.build import bundle, download_frappe_assets
frappe.init('')
if not apps and app:
@ -44,7 +44,15 @@ def build(app=None, apps=None, make_copy=False, restore=False, production=False,
if production:
mode = "production"
bundle(mode, apps=apps, make_copy=make_copy, restore=restore, verbose=verbose, skip_frappe=skip_frappe)
if make_copy or restore:
hard_link = make_copy or restore
click.secho(
"bench build: --make-copy and --restore options are deprecated in favour of --hard-link",
fg="yellow",
)
bundle(mode, apps=apps, hard_link=hard_link, verbose=verbose, skip_frappe=skip_frappe)
@click.command('watch')
@ -499,6 +507,8 @@ frappe.db.connect()
@pass_context
def console(context):
"Start ipython console for a site"
import warnings
site = get_site(context)
frappe.init(site=site)
frappe.connect()
@ -519,6 +529,7 @@ def console(context):
if failed_to_import:
print("\nFailed to import:\n{}".format(", ".join(failed_to_import)))
warnings.simplefilter('ignore')
IPython.embed(display_banner="", header="", colors="neutral")
@ -596,12 +607,29 @@ def run_tests(context, app=None, module=None, doctype=None, test=(), profile=Fal
if os.environ.get('CI'):
sys.exit(ret)
@click.command('run-parallel-tests')
@click.option('--app', help="For App", default='frappe')
@click.option('--build-number', help="Build number", default=1)
@click.option('--total-builds', help="Total number of builds", default=1)
@click.option('--with-coverage', is_flag=True, help="Build coverage file")
@click.option('--use-orchestrator', is_flag=True, help="Use orchestrator to run parallel tests")
@pass_context
def run_parallel_tests(context, app, build_number, total_builds, with_coverage=False, use_orchestrator=False):
site = get_site(context)
if use_orchestrator:
from frappe.parallel_test_runner import ParallelTestWithOrchestrator
ParallelTestWithOrchestrator(app, site=site, with_coverage=with_coverage)
else:
from frappe.parallel_test_runner import ParallelTestRunner
ParallelTestRunner(app, site=site, build_number=build_number, total_builds=total_builds, with_coverage=with_coverage)
@click.command('run-ui-tests')
@click.argument('app')
@click.option('--headless', is_flag=True, help="Run UI Test in headless mode")
@click.option('--parallel', is_flag=True, help="Run UI Test in parallel mode")
@click.option('--ci-build-id')
@pass_context
def run_ui_tests(context, app, headless=False):
def run_ui_tests(context, app, headless=False, parallel=True, ci_build_id=None):
"Run UI tests"
site = get_site(context)
app_base_path = os.path.abspath(os.path.join(frappe.get_app_path(app), '..'))
@ -633,6 +661,12 @@ def run_ui_tests(context, app, headless=False):
command = '{site_env} {password_env} {cypress} {run_or_open}'
formatted_command = command.format(site_env=site_env, password_env=password_env, cypress=cypress_path, run_or_open=run_or_open)
if parallel:
formatted_command += ' --parallel'
if ci_build_id:
formatted_command += ' --ci-build-id {}'.format(ci_build_id)
click.secho("Running Cypress...", fg="yellow")
frappe.commands.popen(formatted_command, cwd=app_base_path, raise_err=True)
@ -808,5 +842,6 @@ commands = [
watch,
bulk_rename,
add_to_email_queue,
rebuild_global_search
rebuild_global_search,
run_parallel_tests
]

5
frappe/contacts/doctype/contact/test_contact.py
View file

@ -5,7 +5,8 @@ from __future__ import unicode_literals
import frappe
import unittest
from frappe.exceptions import ValidationError
test_dependencies = ['Contact', 'Salutation']
class TestContact(unittest.TestCase):
@ -52,4 +53,4 @@ def create_contact(name, salutation, emails=None, phones=None, save=True):
if save:
doc.insert()
return doc
return doc

1
frappe/core/doctype/activity_log/test_activity_log.py
View file

@ -90,4 +90,5 @@ class TestActivityLog(unittest.TestCase):
def update_system_settings(args):
doc = frappe.get_doc('System Settings')
doc.update(args)
doc.flags.ignore_mandatory = 1
doc.save()

2
frappe/core/doctype/data_export/exporter.py
View file

@ -282,7 +282,7 @@ class DataExporter:
try:
sflags = self.docs_to_export.get("flags", "I,U").upper()
flags = 0
for a in re.split('\W+',sflags):
for a in re.split(r'\W+', sflags):
flags = flags | reflags.get(a,0)
c = re.compile(names, flags)

7
frappe/core/doctype/data_import/importer.py
View file

@ -641,7 +641,7 @@ class Row:
return
elif df.fieldtype == "Duration":
import re
is_valid_duration = re.match("^(?:(\d+d)?((^|\s)\d+h)?((^|\s)\d+m)?((^|\s)\d+s)?)$", value)
is_valid_duration = re.match(r"^(?:(\d+d)?((^|\s)\d+h)?((^|\s)\d+m)?((^|\s)\d+s)?)$", value)
if not is_valid_duration:
self.warnings.append(
{
@ -929,10 +929,7 @@ class Column:
self.warnings.append(
{
"col": self.column_number,
"message": _(
"Date format could not be determined from the values in"
" this column. Defaulting to yyyy-mm-dd."
),
"message": _("Date format could not be determined from the values in this column. Defaulting to yyyy-mm-dd."),
"type": "info",
}
)

4
frappe/core/doctype/docshare/test_docshare.py
View file

@ -7,6 +7,8 @@ import frappe.share
import unittest
from frappe.automation.doctype.auto_repeat.test_auto_repeat import create_submittable_doctype
test_dependencies = ['User']
class TestDocShare(unittest.TestCase):
def setUp(self):
self.user = "test@example.com"
@ -112,4 +114,4 @@ class TestDocShare(unittest.TestCase):
self.assertTrue(frappe.has_permission(doctype, "read", doc=submittable_doc.name, user=self.user))
self.assertTrue(frappe.has_permission(doctype, "write", doc=submittable_doc.name, user=self.user))
frappe.share.remove(doctype, submittable_doc.name, self.user)
frappe.share.remove(doctype, submittable_doc.name, self.user)

6
frappe/core/doctype/doctype/doctype.py
View file

@ -671,12 +671,12 @@ class DocType(Document):
flags = {"flags": re.ASCII} if six.PY3 else {}
# a DocType name should not start or end with an empty space
if re.search("^[ \t\n\r]+|[ \t\n\r]+$", name, **flags):
if re.search(r"^[ \t\n\r]+|[ \t\n\r]+$", name, **flags):
frappe.throw(_("DocType's name should not start or end with whitespace"), frappe.NameError)
# a DocType's name should not start with a number or underscore
# and should only contain letters, numbers and underscore
if not re.match("^(?![\W])[^\d_\s][\w ]+$", name, **flags):
if not re.match(r"^(?![\W])[^\d_\s][\w ]+$", name, **flags):
frappe.throw(_("DocType's name should start with a letter and it can only consist of letters, numbers, spaces and underscores"), frappe.NameError)
validate_route_conflict(self.doctype, self.name)
@ -964,7 +964,7 @@ def validate_fields(meta):
for field in depends_on_fields:
depends_on = docfield.get(field, None)
if depends_on and ("=" in depends_on) and \
re.match("""[\w\.:_]+\s*={1}\s*[\w\.@'"]+""", depends_on):
re.match(r'[\w\.:_]+\s*={1}\s*[\w\.@\'"]+', depends_on):
frappe.throw(_("Invalid {0} condition").format(frappe.unscrub(field)), frappe.ValidationError)
def check_table_multiselect_option(docfield):

4
frappe/core/doctype/doctype/test_doctype.py
View file

@ -92,7 +92,7 @@ class TestDocType(unittest.TestCase):
fields=["parent", "depends_on", "collapsible_depends_on", "mandatory_depends_on",\
"read_only_depends_on", "fieldname", "fieldtype"])
pattern = """[\w\.:_]+\s*={1}\s*[\w\.@'"]+"""
pattern = r'[\w\.:_]+\s*={1}\s*[\w\.@\'"]+'
for field in docfields:
for depends_on in ["depends_on", "collapsible_depends_on", "mandatory_depends_on", "read_only_depends_on"]:
condition = field.get(depends_on)
@ -517,4 +517,4 @@ def new_doctype(name, unique=0, depends_on='', fields=None):
for f in fields:
doc.append('fields', f)
return doc
return doc

4
frappe/core/doctype/file/file.py
View file

@ -498,7 +498,7 @@ class File(Document):
self.file_size = self.check_max_file_size()
if (
self.content_type and "image" in self.content_type
self.content_type and self.content_type == "image/jpeg"
and frappe.get_system_settings("strip_exif_metadata_from_uploaded_images")
):
self.content = strip_exif_data(self.content, self.content_type)
@ -912,7 +912,7 @@ def extract_images_from_html(doc, content):
return '<img src="{file_url}"'.format(file_url=file_url)
if content and isinstance(content, string_types):
content = re.sub('<img[^>]*src\s*=\s*["\'](?=data:)(.*?)["\']', _save_file, content)
content = re.sub(r'<img[^>]*src\s*=\s*["\'](?=data:)(.*?)["\']', _save_file, content)
return content

1
frappe/core/doctype/file/test_file.py
View file

@ -193,6 +193,7 @@ class TestSameContent(unittest.TestCase):
class TestFile(unittest.TestCase):
def setUp(self):
frappe.set_user('Administrator')
self.delete_test_data()
self.upload_file()

4
frappe/core/doctype/role_profile/test_role_profile.py
View file

@ -5,6 +5,8 @@ from __future__ import unicode_literals
import frappe
import unittest
test_dependencies = ['Role']
class TestRoleProfile(unittest.TestCase):
def test_make_new_role_profile(self):
new_role_profile = frappe.get_doc(dict(doctype='Role Profile', role_profile='Test 1')).insert()
@ -21,4 +23,4 @@ class TestRoleProfile(unittest.TestCase):
# clear roles
new_role_profile.roles = []
new_role_profile.save()
self.assertEqual(new_role_profile.roles, [])
self.assertEqual(new_role_profile.roles, [])

2
frappe/core/doctype/system_settings/system_settings.py
View file

@ -42,7 +42,7 @@ class SystemSettings(Document):
def on_update(self):
for df in self.meta.get("fields"):
if df.fieldtype not in no_value_fields:
if df.fieldtype not in no_value_fields and self.has_value_changed(df.fieldname):
frappe.db.set_default(df.fieldname, self.get(df.fieldname))
if self.language:

5
frappe/desk/doctype/todo/test_todo.py
View file

@ -9,8 +9,7 @@ from frappe.model.db_query import DatabaseQuery
from frappe.permissions import add_permission, reset_perms
from frappe.core.doctype.doctype.doctype import clear_permissions_cache
# test_records = frappe.get_test_records('ToDo')
test_user_records = frappe.get_test_records('User')
test_dependencies = ['User']
class TestToDo(unittest.TestCase):
def test_delete(self):
@ -77,7 +76,7 @@ class TestToDo(unittest.TestCase):
frappe.set_user('test4@example.com')
#owner and assigned_by is test4
todo3 = create_new_todo('Test3', 'test4@example.com')
# user without any role to read or write todo document
self.assertFalse(todo1.has_permission("read"))
self.assertFalse(todo1.has_permission("write"))

6
frappe/desk/doctype/workspace_link/workspace_link.json
View file

@ -8,13 +8,13 @@
"type",
"label",
"icon",
"only_for",
"hidden",
"link_details_section",
"link_type",
"link_to",
"column_break_7",
"dependencies",
"only_for",
"onboard",
"is_query_report"
],
@ -84,7 +84,7 @@
{
"fieldname": "only_for",
"fieldtype": "Link",
"label": "Only for ",
"label": "Only for",
"options": "Country"
},
{
@ -104,7 +104,7 @@
"index_web_pages_for_search": 1,
"istable": 1,
"links": [],
"modified": "2021-01-12 13:13:12.379443",
"modified": "2021-05-13 13:10:18.128512",
"modified_by": "Administrator",
"module": "Desk",
"name": "Workspace Link",

4
frappe/email/doctype/notification/test_notification.py
View file

@ -7,9 +7,7 @@ import frappe, frappe.utils, frappe.utils.scheduler
from frappe.desk.form import assign_to
import unittest
test_records = frappe.get_test_records('Notification')
test_dependencies = ["User"]
test_dependencies = ["User", "Notification"]
class TestNotification(unittest.TestCase):
def setUp(self):

4
frappe/email/receive.py
View file

@ -284,7 +284,7 @@ class EmailServer:
flags = []
for flag in imaplib.ParseFlags(flag_string) or []:
pattern = re.compile("\w+")
pattern = re.compile(r"\w+")
match = re.search(pattern, frappe.as_unicode(flag))
flags.append(match.group(0))
@ -555,7 +555,7 @@ class Email:
def get_thread_id(self):
"""Extract thread ID from `[]`"""
l = re.findall('(?<=\[)[\w/-]+', self.subject)
l = re.findall(r'(?<=\[)[\w/-]+', self.subject)
return l and l[0] or None

1
frappe/hooks.py
View file

@ -228,7 +228,6 @@ scheduler_events = {
"frappe.desk.doctype.event.event.send_event_digest",
"frappe.sessions.clear_expired_sessions",
"frappe.email.doctype.notification.notification.trigger_daily_alerts",
"frappe.realtime.remove_old_task_logs",
"frappe.utils.scheduler.restrict_scheduler_events_if_dormant",
"frappe.email.doctype.auto_email_report.auto_email_report.send_daily",
"frappe.website.doctype.personal_data_deletion_request.personal_data_deletion_request.remove_unverified_record",

23
frappe/installer.py
View file

@ -390,19 +390,16 @@ def get_conf_params(db_name=None, db_password=None):
def make_site_dirs():
site_public_path = os.path.join(frappe.local.site_path, 'public')
site_private_path = os.path.join(frappe.local.site_path, 'private')
for dir_path in (
os.path.join(site_private_path, 'backups'),
os.path.join(site_public_path, 'files'),
os.path.join(site_private_path, 'files'),
os.path.join(frappe.local.site_path, 'logs'),
os.path.join(frappe.local.site_path, 'task-logs')):
if not os.path.exists(dir_path):
os.makedirs(dir_path)
locks_dir = frappe.get_site_path('locks')
if not os.path.exists(locks_dir):
os.makedirs(locks_dir)
for dir_path in [
os.path.join("public", "files"),
os.path.join("private", "backups"),
os.path.join("private", "files"),
"error-snapshots",
"locks",
"logs",
]:
path = frappe.get_site_path(dir_path)
os.makedirs(path, exist_ok=True)
def add_module_defs(app):

2
frappe/model/base_document.py
View file

@ -870,7 +870,7 @@ class BaseDocument(object):
from frappe.model.meta import get_default_df
df = get_default_df(fieldname)
if not currency:
if not currency and df:
currency = self.get(df.get("options"))
if not frappe.db.exists('Currency', currency, cache=True):
currency = None

2
frappe/model/naming.py
View file

@ -283,7 +283,7 @@ def append_number_if_name_exists(doctype, value, fieldname="name", separator="-"
filters.update({fieldname: value})
exists = frappe.db.exists(doctype, filters)
regex = "^{value}{separator}\d+$".format(value=re.escape(value), separator=separator)
regex = "^{value}{separator}\\d+$".format(value=re.escape(value), separator=separator)
if exists:
last = frappe.db.sql("""SELECT `{fieldname}` FROM `tab{doctype}`

282
frappe/parallel_test_runner.py Normal file
View file

@ -0,0 +1,282 @@
import json
import os
import re
import sys
import time
import unittest
import click
import frappe
import requests
from .test_runner import (SLOW_TEST_THRESHOLD, make_test_records, set_test_email_config)
click_ctx = click.get_current_context(True)
if click_ctx:
click_ctx.color = True
class ParallelTestRunner():
def __init__(self, app, site, build_number=1, total_builds=1, with_coverage=False):
self.app = app
self.site = site
self.with_coverage = with_coverage
self.build_number = frappe.utils.cint(build_number) or 1
self.total_builds = frappe.utils.cint(total_builds)
self.setup_test_site()
self.run_tests()
def setup_test_site(self):
frappe.init(site=self.site)
if not frappe.db:
frappe.connect()
frappe.flags.in_test = True
frappe.clear_cache()
frappe.utils.scheduler.disable_scheduler()
set_test_email_config()
self.before_test_setup()
def before_test_setup(self):
start_time = time.time()
for fn in frappe.get_hooks("before_tests", app_name=self.app):
frappe.get_attr(fn)()
test_module = frappe.get_module(f'{self.app}.tests')
if hasattr(test_module, "global_test_dependencies"):
for doctype in test_module.global_test_dependencies:
make_test_records(doctype)
elapsed = time.time() - start_time
elapsed = click.style(f' ({elapsed:.03}s)', fg='red')
click.echo(f'Before Test {elapsed}')
def run_tests(self):
self.test_result = ParallelTestResult(stream=sys.stderr, descriptions=True, verbosity=2)
self.start_coverage()
for test_file_info in self.get_test_file_list():
self.run_tests_for_file(test_file_info)
self.save_coverage()
self.print_result()
def run_tests_for_file(self, file_info):
if not file_info: return
frappe.set_user('Administrator')
path, filename = file_info
module = self.get_module(path, filename)
self.create_test_dependency_records(module, path, filename)
test_suite = unittest.TestSuite()
module_test_cases = unittest.TestLoader().loadTestsFromModule(module)
test_suite.addTest(module_test_cases)
test_suite(self.test_result)
def create_test_dependency_records(self, module, path, filename):
if hasattr(module, "test_dependencies"):
for doctype in module.test_dependencies:
make_test_records(doctype)
if os.path.basename(os.path.dirname(path)) == "doctype":
# test_data_migration_connector.py > data_migration_connector.json
test_record_filename = re.sub('^test_', '', filename).replace(".py", ".json")
test_record_file_path = os.path.join(path, test_record_filename)
if os.path.exists(test_record_file_path):
with open(test_record_file_path, 'r') as f:
doc = json.loads(f.read())
doctype = doc["name"]
make_test_records(doctype)
def get_module(self, path, filename):
app_path = frappe.get_pymodule_path(self.app)
relative_path = os.path.relpath(path, app_path)
if relative_path == '.':
module_name = self.app
else:
relative_path = relative_path.replace('/', '.')
module_name = os.path.splitext(filename)[0]
module_name = f'{self.app}.{relative_path}.{module_name}'
return frappe.get_module(module_name)
def print_result(self):
self.test_result.printErrors()
click.echo(self.test_result)
if self.test_result.failures or self.test_result.errors:
if os.environ.get('CI'):
sys.exit(1)
def start_coverage(self):
if self.with_coverage:
from coverage import Coverage
from frappe.utils import get_bench_path
# Generate coverage report only for app that is being tested
source_path = os.path.join(get_bench_path(), 'apps', self.app)
omit=['*.html', '*.js', '*.xml', '*.css', '*.less', '*.scss',
'*.vue', '*/doctype/*/*_dashboard.py', '*/patches/*']
if self.app == 'frappe':
omit.append('*/commands/*')
self.coverage = Coverage(source=[source_path], omit=omit)
self.coverage.start()
def save_coverage(self):
if not self.with_coverage:
return
self.coverage.stop()
self.coverage.save()
def get_test_file_list(self):
test_list = get_all_tests(self.app)
split_size = frappe.utils.ceil(len(test_list) / self.total_builds)
# [1,2,3,4,5,6] to [[1,2], [3,4], [4,6]] if split_size is 2
test_chunks = [test_list[x:x+split_size] for x in range(0, len(test_list), split_size)]
return test_chunks[self.build_number - 1]
class ParallelTestResult(unittest.TextTestResult):
def startTest(self, test):
self._started_at = time.time()
super(unittest.TextTestResult, self).startTest(test)
test_class = unittest.util.strclass(test.__class__)
if not hasattr(self, 'current_test_class') or self.current_test_class != test_class:
click.echo(f"\n{unittest.util.strclass(test.__class__)}")
self.current_test_class = test_class
def getTestMethodName(self, test):
return test._testMethodName if hasattr(test, '_testMethodName') else str(test)
def addSuccess(self, test):
super(unittest.TextTestResult, self).addSuccess(test)
elapsed = time.time() - self._started_at
threshold_passed = elapsed >= SLOW_TEST_THRESHOLD
elapsed = click.style(f' ({elapsed:.03}s)', fg='red') if threshold_passed else ''
click.echo(f" {click.style(' ✔ ', fg='green')} {self.getTestMethodName(test)}{elapsed}")
def addError(self, test, err):
super(unittest.TextTestResult, self).addError(test, err)
click.echo(f" {click.style(' ✖ ', fg='red')} {self.getTestMethodName(test)}")
def addFailure(self, test, err):
super(unittest.TextTestResult, self).addFailure(test, err)
click.echo(f" {click.style(' ✖ ', fg='red')} {self.getTestMethodName(test)}")
def addSkip(self, test, reason):
super(unittest.TextTestResult, self).addSkip(test, reason)
click.echo(f" {click.style(' = ', fg='white')} {self.getTestMethodName(test)}")
def addExpectedFailure(self, test, err):
super(unittest.TextTestResult, self).addExpectedFailure(test, err)
click.echo(f" {click.style(' ✖ ', fg='red')} {self.getTestMethodName(test)}")
def addUnexpectedSuccess(self, test):
super(unittest.TextTestResult, self).addUnexpectedSuccess(test)
click.echo(f" {click.style(' ✔ ', fg='green')} {self.getTestMethodName(test)}")
def printErrors(self):
click.echo('\n')
self.printErrorList(' ERROR ', self.errors, 'red')
self.printErrorList(' FAIL ', self.failures, 'red')
def printErrorList(self, flavour, errors, color):
for test, err in errors:
click.echo(self.separator1)
click.echo(f"{click.style(flavour, bg=color)} {self.getDescription(test)}")
click.echo(self.separator2)
click.echo(err)
def __str__(self):
return f"Tests: {self.testsRun}, Failing: {len(self.failures)}, Errors: {len(self.errors)}"
def get_all_tests(app):
test_file_list = []
for path, folders, files in os.walk(frappe.get_pymodule_path(app)):
for dontwalk in ('locals', '.git', 'public', '__pycache__'):
if dontwalk in folders:
folders.remove(dontwalk)
# for predictability
folders.sort()
files.sort()
if os.path.sep.join(["doctype", "doctype", "boilerplate"]) in path:
# in /doctype/doctype/boilerplate/
continue
for filename in files:
if filename.startswith("test_") and filename.endswith(".py") \
and filename != 'test_runner.py':
test_file_list.append([path, filename])
return test_file_list
class ParallelTestWithOrchestrator(ParallelTestRunner):
'''
This can be used to balance-out test time across multiple instances
This is dependent on external orchestrator which returns next test to run
orchestrator endpoints
- register-instance (<build_id>, <instance_id>, test_spec_list)
- get-next-test-spec (<build_id>, <instance_id>)
- test-completed (<build_id>, <instance_id>)
'''
def __init__(self, app, site, with_coverage=False):
self.orchestrator_url = os.environ.get('ORCHESTRATOR_URL')
if not self.orchestrator_url:
click.echo('ORCHESTRATOR_URL environment variable not found!')
click.echo('Pass public URL after hosting https://github.com/frappe/test-orchestrator')
sys.exit(1)
self.ci_build_id = os.environ.get('CI_BUILD_ID')
self.ci_instance_id = os.environ.get('CI_INSTANCE_ID') or frappe.generate_hash(length=10)
if not self.ci_build_id:
click.echo('CI_BUILD_ID environment variable not found!')
sys.exit(1)
ParallelTestRunner.__init__(self, app, site, with_coverage=with_coverage)
def run_tests(self):
self.test_status = 'ongoing'
self.register_instance()
super().run_tests()
def get_test_file_list(self):
while self.test_status == 'ongoing':
yield self.get_next_test()
def register_instance(self):
test_spec_list = get_all_tests(self.app)
response_data = self.call_orchestrator('register-instance', data={
'test_spec_list': test_spec_list
})
self.is_master = response_data.get('is_master')
def get_next_test(self):
response_data = self.call_orchestrator('get-next-test-spec')
self.test_status = response_data.get('status')
return response_data.get('next_test')
def print_result(self):
self.call_orchestrator('test-completed')
return super().print_result()
def call_orchestrator(self, endpoint, data={}):
# add repo token header
# build id in header
headers = {
'CI-BUILD-ID': self.ci_build_id,
'CI-INSTANCE-ID': self.ci_instance_id,
'REPO-TOKEN': '2948288382838DE'
}
url = f'{self.orchestrator_url}/{endpoint}'
res = requests.get(url, json=data, headers=headers)
res.raise_for_status()
response_data = {}
if 'application/json' in res.headers.get('content-type'):
response_data = res.json()
return response_data

3
frappe/patches/v5_0/fix_text_editor_file_urls.py
View file

@ -33,8 +33,7 @@ def execute():
def scrub_relative_urls(html):
"""prepend a slash before a relative url"""
try:
return re.sub("""src[\s]*=[\s]*['"]files/([^'"]*)['"]""", 'src="/files/\g<1>"', html)
# return re.sub("""(src|href)[^\w'"]*['"](?!http|ftp|mailto|/|#|%|{|cid:|\.com/www\.)([^'" >]+)['"]""", '\g<1>="/\g<2>"', html)
return re.sub(r'src[\s]*=[\s]*[\'"]files/([^\'"]*)[\'"]', r'src="/files/\g<1>"', html)
except:
print("Error", html)
raise

6
frappe/printing/doctype/print_format/test_print_format.py
View file

@ -12,13 +12,13 @@ class TestPrintFormat(unittest.TestCase):
def test_print_user(self, style=None):
print_html = frappe.get_print("User", "Administrator", style=style)
self.assertTrue("<label>First Name: </label>" in print_html)
self.assertTrue(re.findall('<div class="col-xs-[^"]*">[\s]*administrator[\s]*</div>', print_html))
self.assertTrue(re.findall(r'<div class="col-xs-[^"]*">[\s]*administrator[\s]*</div>', print_html))
return print_html
def test_print_user_standard(self):
print_html = self.test_print_user("Standard")
self.assertTrue(re.findall('\.print-format {[\s]*font-size: 9pt;', print_html))
self.assertFalse(re.findall('th {[\s]*background-color: #eee;[\s]*}', print_html))
self.assertTrue(re.findall(r'\.print-format {[\s]*font-size: 9pt;', print_html))
self.assertFalse(re.findall(r'th {[\s]*background-color: #eee;[\s]*}', print_html))
self.assertFalse("font-family: serif;" in print_html)
def test_print_user_modern(self):

4
frappe/public/js/frappe/form/controls/date.js
View file

@ -13,9 +13,11 @@ frappe.ui.form.ControlDate = class ControlDate extends frappe.ui.form.ControlDat
super.set_formatted_input(value);
if (this.timepicker_only) return;
if (!this.datepicker) return;
if(!value) {
if (!value) {
this.datepicker.clear();
return;
} else if (value === "Today") {
value = this.get_now_date();
}
let should_refresh = this.last_value && this.last_value !== value;

4
frappe/public/js/frappe/form/grid.js
View file

@ -910,6 +910,10 @@ export default class Grid {
update_docfield_property(fieldname, property, value) {
// update the docfield of each row
if (!this.grid_rows) {
return;
}
for (let row of this.grid_rows) {
let docfield = row.docfields.find(d => d.fieldname === fieldname);
if (docfield) {

2
frappe/public/js/frappe/form/layout.js
View file

@ -510,7 +510,7 @@ frappe.ui.form.Layout = class Layout {
form_obj = this;
}
if (form_obj) {
if (this.doc && this.doc.parent) {
if (this.doc && this.doc.parent && this.doc.parentfield) {
form_obj.setting_dependency = true;
form_obj.set_df_property(this.doc.parentfield, property, value, this.doc.parent, fieldname, this.doc.name);
form_obj.setting_dependency = false;

82
frappe/realtime.py
View file

@ -1,56 +1,23 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors
# For license information, please see license.txt
from __future__ import unicode_literals
import frappe
import os
import time
import redis
from io import FileIO
from frappe.utils import get_site_path
from frappe import conf
END_LINE = '<!-- frappe: end-file -->'
TASK_LOG_MAX_AGE = 86400 # 1 day in seconds
redis_server = None
@frappe.whitelist()
def get_pending_tasks_for_doc(doctype, docname):
return frappe.db.sql_list("select name from `tabAsync Task` where status in ('Queued', 'Running') and reference_doctype=%s and reference_name=%s", (doctype, docname))
def set_task_status(task_id, status, response=None):
if not response:
response = {}
response.update({
"status": status,
"task_id": task_id
})
emit_via_redis("task_status_change", response, room="task:" + task_id)
def remove_old_task_logs():
logs_path = get_site_path('task-logs')
def full_path(_file):
return os.path.join(logs_path, _file)
files_to_remove = [full_path(_file) for _file in os.listdir(logs_path)]
files_to_remove = [_file for _file in files_to_remove if is_file_old(_file) and os.path.isfile(_file)]
for _file in files_to_remove:
os.remove(_file)
def is_file_old(file_path):
return ((time.time() - os.stat(file_path).st_mtime) > TASK_LOG_MAX_AGE)
def publish_progress(percent, title=None, doctype=None, docname=None, description=None):
publish_realtime('progress', {'percent': percent, 'title': title, 'description': description},
user=frappe.session.user, doctype=doctype, docname=docname)
def publish_realtime(event=None, message=None, room=None,
user=None, doctype=None, docname=None, task_id=None,
after_commit=False):
@ -103,6 +70,7 @@ def publish_realtime(event=None, message=None, room=None,
else:
emit_via_redis(event, message, room)
def emit_via_redis(event, message, room):
"""Publish real-time updates via redis
@ -117,57 +85,17 @@ def emit_via_redis(event, message, room):
# print(frappe.get_traceback())
pass
def put_log(line_no, line, task_id=None):
r = get_redis_server()
if not task_id:
task_id = frappe.local.task_id
task_progress_room = get_task_progress_room(task_id)
task_log_key = "task_log:" + task_id
publish_realtime('task_progress', {
"message": {
"lines": {line_no: line}
},
"task_id": task_id
}, room=task_progress_room)
r.hset(task_log_key, line_no, line)
r.expire(task_log_key, 3600)
def get_redis_server():
"""returns redis_socketio connection."""
global redis_server
if not redis_server:
from redis import Redis
redis_server = Redis.from_url(conf.get("redis_socketio")
redis_server = Redis.from_url(frappe.conf.redis_socketio
or "redis://localhost:12311")
return redis_server
class FileAndRedisStream(FileIO):
def __init__(self, *args, **kwargs):
ret = super(FileAndRedisStream, self).__init__(*args, **kwargs)
self.count = 0
return ret
def write(self, data):
ret = super(FileAndRedisStream, self).write(data)
if frappe.local.task_id:
put_log(self.count, data, task_id=frappe.local.task_id)
self.count += 1
return ret
def get_std_streams(task_id):
stdout = FileAndRedisStream(get_task_log_file_path(task_id, 'stdout'), 'w')
# stderr = FileAndRedisStream(get_task_log_file_path(task_id, 'stderr'), 'w')
return stdout, stdout
def get_task_log_file_path(task_id, stream_type):
logs_dir = frappe.utils.get_site_path('task-logs')
return os.path.join(logs_dir, task_id + '.' + stream_type)
@frappe.whitelist(allow_guest=True)
def can_subscribe_doc(doctype, docname):
if os.environ.get('CI'):
@ -201,9 +129,7 @@ def get_site_room():
def get_task_progress_room(task_id):
return "".join([frappe.local.site, ":task_progress:", task_id])
# frappe.chat
def get_chat_room(room):
room = ''.join([frappe.local.site, ":room:", room])
return room
# end frappe.chat room

5
frappe/search/full_text_search.py
View file

@ -1,8 +1,8 @@
# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt
from __future__ import unicode_literals
import frappe
from frappe.utils import update_progress_bar
from whoosh.index import create_in, open_dir, EmptyIndexError
from whoosh.fields import TEXT, ID, Schema
@ -95,9 +95,10 @@ class FullTextSearch:
ix = self.create_index()
writer = ix.writer()
for document in self.documents:
for i, document in enumerate(self.documents):
if document:
writer.add_document(**document)
update_progress_bar("Building Index", i, len(self.documents))
writer.commit(optimize=True)

35
frappe/search/website_search.py
View file

@ -1,15 +1,16 @@
# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt
from __future__ import unicode_literals
import frappe
from bs4 import BeautifulSoup
from whoosh.fields import TEXT, ID, Schema
from frappe.search.full_text_search import FullTextSearch
from frappe.website.render import render_page
from frappe.utils import set_request
import os
from bs4 import BeautifulSoup
from whoosh.fields import ID, TEXT, Schema
import frappe
from frappe.search.full_text_search import FullTextSearch
from frappe.utils import set_request, update_progress_bar
from frappe.website.render import render_page
INDEX_NAME = "web_routes"
class WebsiteSearch(FullTextSearch):
@ -30,11 +31,21 @@ class WebsiteSearch(FullTextSearch):
Returns:
self (object): FullTextSearch Instance
"""
routes = get_static_pages_from_all_apps()
routes += slugs_with_web_view()
documents = [self.get_document_to_index(route) for route in routes]
return documents
if getattr(self, "_items_to_index", False):
return self._items_to_index
routes = get_static_pages_from_all_apps() + slugs_with_web_view()
self._items_to_index = []
for i, route in enumerate(routes):
update_progress_bar("Retrieving Routes", i, len(routes))
self._items_to_index += [self.get_document_to_index(route)]
print()
return self.get_items_to_index()
def get_document_to_index(self, route):
"""Render a page and parse it using BeautifulSoup
@ -114,4 +125,4 @@ def remove_document_from_index(path):
def build_index_for_all_routes():
ws = WebsiteSearch(INDEX_NAME)
return ws.build()
return ws.build()

3
frappe/test_runner.py
View file

@ -9,7 +9,6 @@ import time
import xmlrunner
import importlib
from frappe.modules import load_doctype_module, get_module_name
from frappe.utils import cstr
import frappe.utils.scheduler
import cProfile, pstats
from six import StringIO
@ -308,6 +307,8 @@ def get_dependencies(doctype):
if doctype_name in options_list:
options_list.remove(doctype_name)
options_list.sort()
return options_list
def make_test_records_for_doctype(doctype, verbose=0, force=False):

3
frappe/tests/__init__.py
View file

@ -3,7 +3,10 @@ import frappe
def update_system_settings(args):
doc = frappe.get_doc('System Settings')
doc.update(args)
doc.flags.ignore_mandatory = 1
doc.save()
def get_system_setting(key):
return frappe.db.get_single_value("System Settings", key)
global_test_dependencies = ['User']

2
frappe/tests/test_auth.py
View file

@ -110,7 +110,7 @@ class TestLoginAttemptTracker(unittest.TestCase):
def test_account_unlock(self):
"""Make sure that locked account gets unlocked after lock_interval of time.
"""
lock_interval = 10 # In sec
lock_interval = 2 # In sec
tracker = LoginAttemptTracker(user_name='tester', max_consecutive_login_attempts=1, lock_interval=lock_interval)
# Clear the cache by setting attempt as success
tracker.add_success_attempt()

44
frappe/tests/test_document.py
View file

@ -6,9 +6,8 @@ import os
import unittest
import frappe
from frappe.utils import cint, add_to_date, now
from frappe.utils import cint
from frappe.model.naming import revert_series_if_last, make_autoname, parse_naming_series
from frappe.exceptions import DoesNotExistError
class TestDocument(unittest.TestCase):
@ -87,13 +86,13 @@ class TestDocument(unittest.TestCase):
d.insert()
self.assertEqual(frappe.db.get_value("User", d.name), d.name)
def test_confict_validation(self):
def test_conflict_validation(self):
d1 = self.test_insert()
d2 = frappe.get_doc(d1.doctype, d1.name)
d1.save()
self.assertRaises(frappe.TimestampMismatchError, d2.save)
def test_confict_validation_single(self):
def test_conflict_validation_single(self):
d1 = frappe.get_doc("Website Settings", "Website Settings")
d1.home_page = "test-web-page-1"
@ -110,7 +109,7 @@ class TestDocument(unittest.TestCase):
def test_permission_single(self):
frappe.set_user("Guest")
d = frappe.get_doc("Website Settings", "Website Settigns")
d = frappe.get_doc("Website Settings", "Website Settings")
self.assertRaises(frappe.PermissionError, d.save)
frappe.set_user("Administrator")
@ -196,41 +195,6 @@ class TestDocument(unittest.TestCase):
self.assertTrue(xss not in d.subject)
self.assertTrue(escaped_xss in d.subject)
def test_link_count(self):
if os.environ.get('CI'):
# cannot run this test reliably in travis due to its handling
# of parallelism
return
from frappe.model.utils.link_count import update_link_count
update_link_count()
doctype, name = 'User', 'test@example.com'
d = self.test_insert()
d.append('event_participants', {"reference_doctype": doctype, "reference_docname": name})
d.save()
link_count = frappe.cache().get_value('_link_count') or {}
old_count = link_count.get((doctype, name)) or 0
frappe.db.commit()
link_count = frappe.cache().get_value('_link_count') or {}
new_count = link_count.get((doctype, name)) or 0
self.assertEqual(old_count + 1, new_count)
before_update = frappe.db.get_value(doctype, name, 'idx')
update_link_count()
after_update = frappe.db.get_value(doctype, name, 'idx')
self.assertEqual(before_update + new_count, after_update)
def test_naming_series(self):
data = ["TEST-", "TEST/17-18/.test_data./.####", "TEST.YYYY.MM.####"]

6
frappe/tests/test_email.py
View file

@ -6,11 +6,7 @@ from __future__ import unicode_literals
import unittest, frappe, re, email
from six import PY3
from frappe.test_runner import make_test_records
make_test_records("User")
make_test_records("Email Account")
test_dependencies = ['Email Account']
class TestEmail(unittest.TestCase):
def setUp(self):

32
frappe/tests/test_fmt_datetime.py
View file

@ -45,6 +45,7 @@ class TestFmtDatetime(unittest.TestCase):
frappe.db.set_default("time_format", self.pre_test_time_format)
frappe.local.user_date_format = None
frappe.local.user_time_format = None
frappe.db.rollback()
# Test utility functions
@ -97,28 +98,12 @@ class TestFmtDatetime(unittest.TestCase):
self.assertEqual(formatdate(test_date), valid_fmt)
# Test time formatters
def test_format_time_forced(self):
# Test with forced time formats
self.assertEqual(
format_time(test_time, 'ss:mm:HH'),
test_date_obj.strftime('%S:%M:%H'))
@unittest.expectedFailure
def test_format_time_forced_broken_locale(self):
# Test with forced time formats
# Currently format_time defaults to HH:mm:ss if the locale is
# broken, so this is an expected failure.
lang = frappe.local.lang
try:
# Force fallback from Babel
frappe.local.lang = 'FAKE'
self.assertEqual(
format_time(test_time, 'ss:mm:HH'),
test_date_obj.strftime('%S:%M:%H'))
finally:
frappe.local.lang = lang
def test_format_time(self):
# Test format_time with various default time formats set
for fmt, valid_fmt in test_time_formats.items():
@ -135,21 +120,6 @@ class TestFmtDatetime(unittest.TestCase):
format_datetime(test_datetime, 'dd-yyyy-MM ss:mm:HH'),
test_date_obj.strftime('%d-%Y-%m %S:%M:%H'))
@unittest.expectedFailure
def test_format_datetime_forced_broken_locale(self):
# Test with forced datetime formats
# Currently format_datetime defaults to yyyy-MM-dd HH:mm:ss
# if the locale is broken, so this is an expected failure.
lang = frappe.local.lang
# Force fallback from Babel
try:
frappe.local.lang = 'FAKE'
self.assertEqual(
format_datetime(test_datetime, 'dd-yyyy-MM ss:mm:HH'),
test_date_obj.strftime('%d-%Y-%m %S:%M:%H'))
finally:
frappe.local.lang = lang
def test_format_datetime(self):
# Test formatdate with various default date formats set
for date_fmt, valid_date in test_date_formats.items():

1
frappe/tests/test_seen.py
View file

@ -41,6 +41,7 @@ class TestSeen(unittest.TestCase):
self.assertTrue('test1@example.com' in json.loads(ev._seen))
ev.save()
ev = frappe.get_doc('Event', ev.name)
self.assertFalse('test@example.com' in json.loads(ev._seen))
self.assertTrue('test1@example.com' in json.loads(ev._seen))

12
frappe/tests/test_twofactor.py
View file

@ -8,7 +8,7 @@ from frappe.utils import cint
from frappe.utils import set_request
from frappe.auth import validate_ip_address, get_login_attempt_tracker
from frappe.twofactor import (should_run_2fa, authenticate_for_2factor, get_cached_user_pass,
two_factor_is_enabled_for_, confirm_otp_token, get_otpsecret_for_, get_verification_obj)
two_factor_is_enabled_for_, confirm_otp_token, get_otpsecret_for_, get_verification_obj, ExpiredLoginException)
from . import update_system_settings, get_system_setting
import time
@ -111,6 +111,7 @@ class TestTwoFactor(unittest.TestCase):
def test_confirm_otp_token(self):
'''Ensure otp is confirmed'''
frappe.flags.otp_expiry = 2
authenticate_for_2factor(self.user)
tmp_id = frappe.local.response['tmp_id']
otp = 'wrongotp'
@ -118,10 +119,11 @@ class TestTwoFactor(unittest.TestCase):
confirm_otp_token(self.login_manager,otp=otp,tmp_id=tmp_id)
otp = get_otp(self.user)
self.assertTrue(confirm_otp_token(self.login_manager,otp=otp,tmp_id=tmp_id))
frappe.flags.otp_expiry = None
if frappe.flags.tests_verbose:
print('Sleeping for 30secs to confirm token expires..')
time.sleep(30)
with self.assertRaises(frappe.AuthenticationError):
print('Sleeping for 2 secs to confirm token expires..')
time.sleep(2)
with self.assertRaises(ExpiredLoginException):
confirm_otp_token(self.login_manager,otp=otp,tmp_id=tmp_id)
def test_get_verification_obj(self):
@ -208,12 +210,14 @@ def enable_2fa(bypass_two_factor_auth=0, bypass_restrict_ip_check=0):
system_settings.bypass_2fa_for_retricted_ip_users = cint(bypass_two_factor_auth)
system_settings.bypass_restrict_ip_check_if_2fa_enabled = cint(bypass_restrict_ip_check)
system_settings.two_factor_method = 'OTP App'
system_settings.flags.ignore_mandatory = True
system_settings.save(ignore_permissions=True)
frappe.db.commit()
def disable_2fa():
system_settings = frappe.get_doc('System Settings')
system_settings.enable_two_factor_auth = 0
system_settings.flags.ignore_mandatory = True
system_settings.save(ignore_permissions=True)
frappe.db.commit()

4
frappe/twofactor.py
View file

@ -73,11 +73,11 @@ def cache_2fa_data(user, token, otp_secret, tmp_id):
# set increased expiry time for SMS and Email
if verification_method in ['SMS', 'Email']:
expiry_time = 300
expiry_time = frappe.flags.token_expiry or 300
frappe.cache().set(tmp_id + '_token', token)
frappe.cache().expire(tmp_id + '_token', expiry_time)
else:
expiry_time = 180
expiry_time = frappe.flags.otp_expiry or 180
for k, v in iteritems({'_usr': user, '_pwd': pwd, '_otp_secret': otp_secret}):
frappe.cache().set("{0}{1}".format(tmp_id, k), v)
frappe.cache().expire("{0}{1}".format(tmp_id, k), expiry_time)

25
frappe/utils/__init__.py
View file

@ -161,7 +161,7 @@ def validate_url(txt, throw=False, valid_schemes=None):
Parameters:
throw (`bool`): throws a validationError if URL is not valid
valid_schemes (`str` or `list`): if provided checks the given URL's scheme against this
valid_schemes (`str` or `list`): if provided checks the given URL's scheme against this
Returns:
bool: if `txt` represents a valid URL
@ -225,14 +225,17 @@ def get_gravatar(email):
return gravatar_url
def get_traceback():
def get_traceback() -> str:
"""
Returns the traceback of the Exception
"""
exc_type, exc_value, exc_tb = sys.exc_info()
if not any([exc_type, exc_value, exc_tb]):
return ""
trace_list = traceback.format_exception(exc_type, exc_value, exc_tb)
body = "".join(cstr(t) for t in trace_list)
return body
return "".join(cstr(t) for t in trace_list)
def log(event, details):
frappe.logger().info(details)
@ -425,7 +428,7 @@ def get_test_client():
return Client(application)
def get_hook_method(hook_name, fallback=None):
method = (frappe.get_hooks().get(hook_name))
method = frappe.get_hooks().get(hook_name)
if method:
method = frappe.get_attr(method[0])
return method
@ -439,6 +442,16 @@ def call_hook_method(hook, *args, **kwargs):
return out
def is_cli() -> bool:
"""Returns True if current instance is being run via a terminal
"""
invoked_from_terminal = False
try:
invoked_from_terminal = bool(os.get_terminal_size())
except Exception:
invoked_from_terminal = sys.stdin.isatty()
return invoked_from_terminal
def update_progress_bar(txt, i, l):
if os.environ.get("CI"):
if i == 0:
@ -448,7 +461,7 @@ def update_progress_bar(txt, i, l):
sys.stdout.flush()
return
if not getattr(frappe.local, 'request', None):
if not getattr(frappe.local, 'request', None) or is_cli():
lt = len(txt)
try:
col = 40 if os.get_terminal_size().columns > 80 else 20

2
frappe/utils/boilerplate.py
View file

@ -40,7 +40,7 @@ def make_boilerplate(dest, app_name):
if hook_key=="app_name" and hook_val.lower().replace(" ", "_") != hook_val:
print("App Name must be all lowercase and without spaces")
hook_val = ""
elif hook_key=="app_title" and not re.match("^(?![\W])[^\d_\s][\w -]+$", hook_val, re.UNICODE):
elif hook_key=="app_title" and not re.match(r"^(?![\W])[^\d_\s][\w -]+$", hook_val, re.UNICODE):
print("App Title should start with a letter and it can only consist of letters, numbers, spaces and underscores")
hook_val = ""

2
frappe/utils/data.py
View file

@ -1278,7 +1278,9 @@ def make_filter_dict(filters):
def sanitize_column(column_name):
from frappe import _
import sqlparse
regex = re.compile("^.*[,'();].*")
column_name = sqlparse.format(column_name, strip_comments=True, keyword_case="lower")
blacklisted_keywords = ['select', 'create', 'insert', 'delete', 'drop', 'update', 'case', 'and', 'or']
def _raise_exception():

2
frappe/utils/file_manager.py
View file

@ -418,7 +418,7 @@ def extract_images_from_html(doc, content):
return '<img src="{file_url}"'.format(file_url=file_url)
if content:
content = re.sub('<img[^>]*src\s*=\s*["\'](?=data:)(.*?)["\']', _save_file, content)
content = re.sub(r'<img[^>]*src\s*=\s*["\'](?=data:)(.*?)["\']', _save_file, content)
return content

2
frappe/utils/formatters.py
View file

@ -78,7 +78,7 @@ def format_value(value, df=None, doc=None, currency=None, translated=False):
return "{}%".format(flt(value, 2))
elif df.get("fieldtype") in ("Text", "Small Text"):
if not re.search("(\<br|\<div|\<p)", value):
if not re.search(r"(<br|<div|<p)", value):
return frappe.safe_decode(value).replace("\n", "<br>")
elif df.get("fieldtype") == "Markdown Editor":

2
frappe/utils/global_search.py
View file

@ -348,7 +348,7 @@ def get_formatted_value(value, field):
if getattr(field, 'fieldtype', None) in ["Text", "Text Editor"]:
value = unescape_html(frappe.safe_decode(value))
value = (re.subn(r'<[\s]*(script|style).*?</\1>(?s)', '', text_type(value))[0])
value = (re.subn(r'(?s)<[\s]*(script|style).*?</\1>', '', text_type(value))[0])
value = ' '.join(value.split())
return field.label + " : " + strip_html_tags(text_type(value))

2
frappe/utils/redis_wrapper.py
View file

@ -98,7 +98,7 @@ class RedisWrapper(redis.Redis):
return self.keys(key)
except redis.exceptions.ConnectionError:
regex = re.compile(cstr(key).replace("|", "\|").replace("*", "[\w]*"))
regex = re.compile(cstr(key).replace("|", r"\|").replace("*", r"[\w]*"))
return [k for k in list(frappe.local.cache) if regex.match(cstr(k))]
def delete_keys(self, key):

2
frappe/website/context.py
View file

@ -61,7 +61,7 @@ def update_controller_context(context, controller):
except (frappe.PermissionError, frappe.PageDoesNotExistError, frappe.Redirect):
raise
except:
if not frappe.flags.in_migrate:
if not any([frappe.flags.in_migrate, frappe.flags.in_website_search_build]):
frappe.errprint(frappe.utils.get_traceback())
if hasattr(module, "get_children"):

2
frappe/website/doctype/blog_post/test_blog_post.py
View file

@ -13,6 +13,8 @@ from frappe.website.doctype.blog_post.blog_post import get_blog_list
from frappe.website.website_generator import WebsiteGenerator
from frappe.custom.doctype.customize_form.customize_form import reset_customization
test_dependencies = ['Blog Post']
class TestBlogPost(unittest.TestCase):
def setUp(self):
reset_customization('Blog Post')

2
frappe/website/doctype/web_form/test_web_form.py
View file

@ -8,7 +8,7 @@ import unittest, json
from frappe.website.render import build_page
from frappe.website.doctype.web_form.web_form import accept
test_records = frappe.get_test_records('Web Form')
test_dependencies = ['Web Form']
class TestWebForm(unittest.TestCase):
def setUp(self):

2
frappe/website/doctype/web_page/test_web_page.py
View file

@ -39,7 +39,7 @@ class TestWebPage(unittest.TestCase):
published = 1,
content_type = 'Rich Text',
main_section = 'rich text',
main_section_md = '# h1\n\markdown content',
main_section_md = '# h1\nmarkdown content',
main_section_html = '<div>html content</div>'
)).insert()

2
frappe/website/router.py
View file

@ -158,7 +158,7 @@ def evaluate_dynamic_routes(rules, path):
route_map = Map(rules)
endpoint = None
if frappe.local.request:
if hasattr(frappe.local, 'request') and frappe.local.request.environ:
urls = route_map.bind_to_environ(frappe.local.request.environ)
try:
endpoint, args = urls.match("/" + path)

4
frappe/www/app.py
View file

@ -34,10 +34,10 @@ def get_context(context):
boot_json = frappe.as_json(boot)
# remove script tags from boot
boot_json = re.sub("\<script[^<]*\</script\>", "", boot_json)
boot_json = re.sub(r"\<script[^<]*\</script\>", "", boot_json)
# TODO: Find better fix
boot_json = re.sub("</script\>", "", boot_json)
boot_json = re.sub(r"</script\>", "", boot_json)
context.update({
"no_cache": 1,

2
frappe/www/printview.py
View file

@ -409,7 +409,7 @@ def get_print_style(style=None, print_format=None, for_legacy=False):
css = css + '\n' + frappe.db.get_value('Print Style', style, 'css')
# move @import to top
for at_import in list(set(re.findall("(@import url\([^\)]+\)[;]?)", css))):
for at_import in list(set(re.findall(r"(@import url\([^\)]+\)[;]?)", css))):
css = css.replace(at_import, "")
# prepend css with at_import

3
requirements.txt
View file

@ -6,7 +6,8 @@ boto3~=1.17.53
braintree~=4.8.0
chardet~=4.0.0
Click~=7.1.2
coverage~=4.5.4
colorama~=0.4.4
coverage==5.5
croniter~=1.0.11
cryptography~=3.4.7
dropbox~=11.7.0