diff --git a/.editorconfig b/.editorconfig index d76f67cd7f..a3b1ef0924 100644 --- a/.editorconfig +++ b/.editorconfig @@ -9,6 +9,7 @@ trim_trailing_whitespace = true charset = utf-8 # python, js indentation settings -[{*.py,*.js,*.vue}] +[{*.py,*.js,*.vue,*.css,*.scss,*.html}] indent_style = tab indent_size = 4 +max_line_length = 99 diff --git a/.eslintrc b/.eslintrc index 937f11586c..dd9e350b1b 100644 --- a/.eslintrc +++ b/.eslintrc @@ -5,7 +5,7 @@ "es6": true }, "parserOptions": { - "ecmaVersion": 9, + "ecmaVersion": 11, "sourceType": "module" }, "extends": "eslint:recommended", @@ -134,7 +134,6 @@ "Webcam": true, "PhotoSwipe": true, "PhotoSwipeUI_Default": true, - "fluxify": true, "io": true, "JsBarcode": true, "L": true, diff --git a/.flake8 b/.flake8 index 56c9b9a369..4b852abd7c 100644 --- a/.flake8 +++ b/.flake8 @@ -28,6 +28,10 @@ ignore = B007, B950, W191, + E124, # closing bracket, irritating while writing QB code + E131, # continuation line unaligned for hanging indent + E123, # closing bracket does not match indentation of opening bracket's line + E101, # ensured by use of black max-line-length = 200 exclude=.github/helper/semgrep_rules diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index a49668a5f4..2a39cef3a6 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -22,3 +22,12 @@ b2fc959307c7c79f5584625569d5aed04133ba13 # Format codebase and sort imports c0c5b2ebdddbe8898ce2d5e5365f4931ff73b6bf + +# update python code to use 3.10 supported features +81b37cb7d2160866afa2496873656afe53f0c145 + +# mass minified JSON schema +85e3ee940353d7b0b517b33815148672e9a8b15b + +# format JS files with pretter +40f27f908a3890c9a90d2d96794fc31fcea63c59 diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000000..5ace4600a1 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" diff --git a/.github/helper/documentation.py b/.github/helper/documentation.py index aece5f543b..eb0b373acd 100644 --- a/.github/helper/documentation.py +++ b/.github/helper/documentation.py @@ -30,7 +30,7 @@ def docs_link_exists(body): if __name__ == "__main__": pr = sys.argv[1] - response = requests.get("https://api.github.com/repos/frappe/frappe/pulls/{}".format(pr)) + response = requests.get(f"https://api.github.com/repos/frappe/frappe/pulls/{pr}") if response.ok: payload = response.json() diff --git a/.github/helper/flake8.conf b/.github/helper/flake8.conf new file mode 100644 index 0000000000..20d4b912ca --- /dev/null +++ b/.github/helper/flake8.conf @@ -0,0 +1,75 @@ +[flake8] +ignore = + B001, + B007, + B009, + B010, + B950, + E101, + E111, + E114, + E116, + E117, + E121, + E122, + E123, + E124, + E125, + E126, + E127, + E128, + E131, + E201, + E202, + E203, + E211, + E221, + E222, + E223, + E224, + E225, + E226, + E228, + E231, + E241, + E242, + E251, + E261, + E262, + E265, + E266, + E271, + E272, + E273, + E274, + E301, + E302, + E303, + E305, + E306, + E402, + E501, + E502, + E701, + E702, + E703, + E741, + F401, + F403, + F405, + W191, + W291, + W292, + W293, + W391, + W503, + W504, + E711, + E129, + F841, + E713, + E712, + + +max-line-length = 200 +exclude=.github/helper/semgrep_rules,test_*.py diff --git a/.github/helper/install.sh b/.github/helper/install.sh index 246bdbe096..1a2c62c973 100644 --- a/.github/helper/install.sh +++ b/.github/helper/install.sh @@ -1,65 +1,67 @@ #!/bin/bash - set -e - cd ~ || exit -pip install frappe-bench +echo "Setting Up Bench..." -bench init frappe-bench --skip-assets --python "$(which python)" --frappe-path "${GITHUB_WORKSPACE}" +pip install frappe-bench +bench -v init frappe-bench --skip-assets --python "$(which python)" --frappe-path "${GITHUB_WORKSPACE}" +cd ./frappe-bench || exit + +bench -v setup requirements --dev +if [ "$TYPE" == "ui" ]; then + bench -v setup requirements --node; +fi + +echo "Setting Up Sites & Database..." mkdir ~/frappe-bench/sites/test_site cp "${GITHUB_WORKSPACE}/.github/helper/consumer_db/$DB.json" ~/frappe-bench/sites/test_site/site_config.json if [ "$TYPE" == "server" ]; then - mkdir ~/frappe-bench/sites/test_site_producer; - cp "${GITHUB_WORKSPACE}/.github/helper/producer_db/$DB.json" ~/frappe-bench/sites/test_site_producer/site_config.json; + mkdir ~/frappe-bench/sites/test_site_producer; + cp "${GITHUB_WORKSPACE}/.github/helper/producer_db/$DB.json" ~/frappe-bench/sites/test_site_producer/site_config.json; fi - if [ "$DB" == "mariadb" ];then - sudo apt update && sudo apt install mariadb-client-10.3 - mysql --host 127.0.0.1 --port 3306 -u root -e "SET GLOBAL character_set_server = 'utf8mb4'"; - mysql --host 127.0.0.1 --port 3306 -u root -e "SET GLOBAL collation_server = 'utf8mb4_unicode_ci'"; + mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "SET GLOBAL character_set_server = 'utf8mb4'"; + mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "SET GLOBAL collation_server = 'utf8mb4_unicode_ci'"; - mysql --host 127.0.0.1 --port 3306 -u root -e "CREATE DATABASE test_frappe_consumer"; - mysql --host 127.0.0.1 --port 3306 -u root -e "CREATE USER 'test_frappe_consumer'@'localhost' IDENTIFIED BY 'test_frappe_consumer'"; - mysql --host 127.0.0.1 --port 3306 -u root -e "GRANT ALL PRIVILEGES ON \`test_frappe_consumer\`.* TO 'test_frappe_consumer'@'localhost'"; + mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "CREATE DATABASE test_frappe_consumer"; + mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "CREATE USER 'test_frappe_consumer'@'localhost' IDENTIFIED BY 'test_frappe_consumer'"; + mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "GRANT ALL PRIVILEGES ON \`test_frappe_consumer\`.* TO 'test_frappe_consumer'@'localhost'"; - mysql --host 127.0.0.1 --port 3306 -u root -e "CREATE DATABASE test_frappe_producer"; - mysql --host 127.0.0.1 --port 3306 -u root -e "CREATE USER 'test_frappe_producer'@'localhost' IDENTIFIED BY 'test_frappe_producer'"; - mysql --host 127.0.0.1 --port 3306 -u root -e "GRANT ALL PRIVILEGES ON \`test_frappe_producer\`.* TO 'test_frappe_producer'@'localhost'"; - - mysql --host 127.0.0.1 --port 3306 -u root -e "UPDATE mysql.user SET Password=PASSWORD('travis') WHERE User='root'"; - mysql --host 127.0.0.1 --port 3306 -u root -e "FLUSH PRIVILEGES"; - fi + mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "CREATE DATABASE test_frappe_producer"; + mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "CREATE USER 'test_frappe_producer'@'localhost' IDENTIFIED BY 'test_frappe_producer'"; + mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "GRANT ALL PRIVILEGES ON \`test_frappe_producer\`.* TO 'test_frappe_producer'@'localhost'"; + mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "FLUSH PRIVILEGES"; +fi if [ "$DB" == "postgres" ];then - echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE DATABASE test_frappe_consumer" -U postgres; - echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE USER test_frappe_consumer WITH PASSWORD 'test_frappe'" -U postgres; + echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE DATABASE test_frappe_consumer" -U postgres; + echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE USER test_frappe_consumer WITH PASSWORD 'test_frappe'" -U postgres; - echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE DATABASE test_frappe_producer" -U postgres; - echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE USER test_frappe_producer WITH PASSWORD 'test_frappe'" -U postgres; + echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE DATABASE test_frappe_producer" -U postgres; + echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE USER test_frappe_producer WITH PASSWORD 'test_frappe'" -U postgres; fi -cd ./frappe-bench || exit +echo "Setting Up Procfile..." sed -i 's/^watch:/# watch:/g' Procfile sed -i 's/^schedule:/# schedule:/g' Procfile +if [ "$TYPE" == "server" ]; then + sed -i 's/^socketio:/# socketio:/g' Procfile; + sed -i 's/^redis_socketio:/# redis_socketio:/g' Procfile; +fi +if [ "$TYPE" == "ui" ]; then + sed -i 's/^web: bench serve/web: bench serve --with-coverage/g' Procfile; +fi -if [ "$TYPE" == "server" ]; then sed -i 's/^socketio:/# socketio:/g' Procfile; fi -if [ "$TYPE" == "server" ]; then sed -i 's/^redis_socketio:/# redis_socketio:/g' Procfile; fi +echo "Starting Bench..." -if [ "$TYPE" == "ui" ]; then bench setup requirements --node; fi -bench setup requirements --dev - -if [ "$TYPE" == "ui" ]; then sed -i 's/^web: bench serve/web: bench serve --with-coverage/g' Procfile; fi - -# install node-sass which is required for website theme test -cd ./apps/frappe || exit -yarn add node-sass@4.13.1 -cd ../.. - -bench start & +bench start &> bench_start.log & bench --site test_site reinstall --yes -if [ "$TYPE" == "server" ]; then bench --site test_site_producer reinstall --yes; fi -if [ "$TYPE" == "server" ]; then CI=Yes bench build --app frappe; fi + +if [ "$TYPE" == "server" ]; then + bench --site test_site_producer reinstall --yes; + CI=Yes bench build --app frappe; +fi diff --git a/.github/helper/install_dependencies.sh b/.github/helper/install_dependencies.sh index d16f5b62ad..2306ccc94a 100644 --- a/.github/helper/install_dependencies.sh +++ b/.github/helper/install_dependencies.sh @@ -1,16 +1,14 @@ #!/bin/bash - set -e - # install wkhtmltopdf -wget -O /tmp/wkhtmltox.tar.xz https://github.com/frappe/wkhtmltopdf/raw/master/wkhtmltox-0.12.3_linux-generic-amd64.tar.xz -tar -xf /tmp/wkhtmltox.tar.xz -C /tmp -sudo mv /tmp/wkhtmltox/bin/wkhtmltopdf /usr/local/bin/wkhtmltopdf -sudo chmod o+x /usr/local/bin/wkhtmltopdf +echo "Setting Up System Dependencies..." -# install cups -sudo apt-get install libcups2-dev +install_wkhtmltopdf() { + wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.focal_amd64.deb + sudo apt install ./wkhtmltox_0.12.6-1.focal_amd64.deb +} +install_wkhtmltopdf & -# install redis -sudo apt-get install redis-server +sudo apt update +sudo apt install libcups2-dev redis-server mariadb-client-10.3 diff --git a/.github/helper/roulette.py b/.github/helper/roulette.py index 90f4608a22..554f4ae5f5 100644 --- a/.github/helper/roulette.py +++ b/.github/helper/roulette.py @@ -5,19 +5,40 @@ import shlex import subprocess import sys import urllib.request +from functools import lru_cache +@lru_cache(maxsize=None) +def fetch_pr_data(pr_number, repo, endpoint=""): + api_url = f"https://api.github.com/repos/{repo}/pulls/{pr_number}" + + if endpoint: + api_url += f"/{endpoint}" + + req = urllib.request.Request(api_url) + res = urllib.request.urlopen(req) + return json.loads(res.read().decode('utf8')) + def get_files_list(pr_number, repo="frappe/frappe"): - req = urllib.request.Request(f"https://api.github.com/repos/{repo}/pulls/{pr_number}/files") - res = urllib.request.urlopen(req) - dump = json.loads(res.read().decode('utf8')) - return [change["filename"] for change in dump] + return [change["filename"] for change in fetch_pr_data(pr_number, repo, "files")] def get_output(command, shell=True): print(command) command = shlex.split(command) return subprocess.check_output(command, shell=shell, encoding="utf8").strip() +def has_skip_ci_label(pr_number, repo="frappe/frappe"): + return has_label(pr_number, "Skip CI", repo) + +def has_run_server_tests_label(pr_number, repo="frappe/frappe"): + return has_label(pr_number, "Run Server Tests", repo) + +def has_run_ui_tests_label(pr_number, repo="frappe/frappe"): + return has_label(pr_number, "Run UI Tests", repo) + +def has_label(pr_number, label, repo="frappe/frappe"): + return any([fetched_label["name"] for fetched_label in fetch_pr_data(pr_number, repo)["labels"] if fetched_label["name"] == label]) + def is_py(file): return file.endswith("py") @@ -25,10 +46,10 @@ def is_ci(file): return ".github" in file def is_frontend_code(file): - return file.lower().endswith((".css", ".scss", ".less", ".sass", ".styl", ".js", ".ts", ".vue")) + return file.lower().endswith((".css", ".scss", ".less", ".sass", ".styl", ".js", ".ts", ".vue", ".html")) def is_docs(file): - regex = re.compile(r'\.(md|png|jpg|jpeg|csv)$|^.github|LICENSE') + regex = re.compile(r'\.(md|png|jpg|jpeg|csv|svg)$|^.github|LICENSE') return bool(regex.search(file)) @@ -56,23 +77,23 @@ if __name__ == "__main__": updated_py_file_count = len(list(filter(is_py, files_list))) only_py_changed = updated_py_file_count == len(files_list) - if ci_files_changed: + if has_skip_ci_label(pr_number, repo): + print("Found `Skip CI` label on pr, stopping build process.") + sys.exit(0) + + elif ci_files_changed: print("CI related files were updated, running all build processes.") elif only_docs_changed: print("Only docs were updated, stopping build process.") sys.exit(0) - elif only_frontend_code_changed and build_type == "server": + elif only_frontend_code_changed and build_type == "server" and not has_run_server_tests_label(pr_number, repo): print("Only Frontend code was updated; Stopping Python build process.") sys.exit(0) - elif build_type == "ui": - if only_py_changed: - print("Only Python code was updated, stopping Cypress build process.") - sys.exit(0) - elif updated_py_file_count > 0: - # both frontend and backend code were updated - os.system('echo "::set-output name=build-server::strawberry"') + elif build_type == "ui" and only_py_changed and not has_run_ui_tests_label(pr_number, repo): + print("Only Python code was updated, stopping Cypress build process.") + sys.exit(0) os.system('echo "::set-output name=build::strawberry"') diff --git a/.github/helper/translation.py b/.github/helper/translation.py index 9146b3b32b..72f661d3e1 100644 --- a/.github/helper/translation.py +++ b/.github/helper/translation.py @@ -20,19 +20,12 @@ for _file in files_to_scan: if 'frappe-lint: disable-translate' in line: continue - start_matches = start_pattern.search(line) - if start_matches: - starts_with_f = starts_with_f_pattern.search(line) - - if starts_with_f: - has_f_string = f_string_pattern.search(line) - if has_f_string: + if start_matches := start_pattern.search(line): + if starts_with_f := starts_with_f_pattern.search(line): + if has_f_string := f_string_pattern.search(line): errors_encounter += 1 print(f'\nF-strings are not supported for translations at line number {line_number}\n{line.strip()[:100]}') - continue - else: - continue - + continue match = pattern.search(line) error_found = False diff --git a/.github/labeler.yml b/.github/labeler.yml new file mode 100644 index 0000000000..42e52e553f --- /dev/null +++ b/.github/labeler.yml @@ -0,0 +1,4 @@ +# Any python files modifed but no test files modified +add-test-cases: +- any: ['frappe/**/*.py'] + all: ['!frappe/**/test*.py'] diff --git a/.github/semantic.yml b/.github/semantic.yml deleted file mode 100644 index fa15046b4a..0000000000 --- a/.github/semantic.yml +++ /dev/null @@ -1,30 +0,0 @@ -# Always validate the PR title AND all the commits -titleAndCommits: true - -# Allow use of Merge commits (eg on github: "Merge branch 'master' into feature/ride-unicorns") -# this is only relevant when using commitsOnly: true (or titleAndCommits: true) -allowMergeCommits: true - -# Allow use of Revert commits (eg on github: "Revert "feat: ride unicorns"") -# this is only relevant when using commitsOnly: true (or titleAndCommits: true) -allowRevertCommits: true - -# For allowed PR types: https://github.com/commitizen/conventional-commit-types/blob/v3.0.0/index.json -# Tool Reference: https://github.com/zeke/semantic-pull-requests - -# By default types specified in commitizen/conventional-commit-types is used. -# See: https://github.com/commitizen/conventional-commit-types/blob/v3.0.0/index.json -# You can override the valid types -types: - - BREAKING CHANGE - - feat - - fix - - docs - - style - - refactor - - perf - - test - - build - - ci - - chore - - revert diff --git a/.github/try-on-f-cloud-button.svg b/.github/try-on-f-cloud-button.svg index fe0bb2c52d..6a7119bdee 100644 --- a/.github/try-on-f-cloud-button.svg +++ b/.github/try-on-f-cloud-button.svg @@ -1,4 +1,4 @@ - + @@ -29,4 +29,4 @@ - + \ No newline at end of file diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml new file mode 100644 index 0000000000..010022b7f6 --- /dev/null +++ b/.github/workflows/create-release.yml @@ -0,0 +1,34 @@ +name: Generate Semantic Release +on: + push: + branches: + - version-14-beta +permissions: + contents: read + +jobs: + release: + name: Release + runs-on: ubuntu-latest + steps: + - name: Checkout Entire Repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 + persist-credentials: false + - name: Setup Node.js + uses: actions/setup-node@v3 + with: + node-version: 16 + - name: Setup dependencies + run: | + npm install @semantic-release/git @semantic-release/exec --no-save + - name: Create Release + env: + GH_TOKEN: ${{ secrets.RELEASE_TOKEN }} + GITHUB_TOKEN: ${{ secrets.RELEASE_TOKEN }} + GIT_AUTHOR_NAME: "Frappe PR Bot" + GIT_AUTHOR_EMAIL: "developers@frappe.io" + GIT_COMMITTER_NAME: "Frappe PR Bot" + GIT_COMMITTER_EMAIL: "developers@frappe.io" + run: npx semantic-release diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml deleted file mode 100644 index dba13f9358..0000000000 --- a/.github/workflows/docker-release.yml +++ /dev/null @@ -1,15 +0,0 @@ -name: 'Trigger Docker build on release' -on: - release: - types: [released] -jobs: - curl: - name: 'Trigger Docker build on release' - runs-on: ubuntu-latest - container: - image: alpine:latest - steps: - - name: curl - run: | - apk add curl bash - curl -X POST -H "Accept: application/vnd.github.v3+json" -H "Authorization: Bearer ${{ secrets.CI_PAT }}" https://api.github.com/repos/frappe/frappe_docker/actions/workflows/build_stable.yml/dispatches -d '{"ref":"main"}' diff --git a/.github/workflows/docs-checker.yml b/.github/workflows/docs-checker.yml deleted file mode 100644 index 5e91063698..0000000000 --- a/.github/workflows/docs-checker.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: 'Documentation Check' -on: - pull_request: - types: [ opened, synchronize, reopened, edited ] - -jobs: - docs-required: - name: 'Documentation Required' - runs-on: ubuntu-latest - - steps: - - name: 'Setup Environment' - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - - name: 'Clone repo' - uses: actions/checkout@v2 - - - name: Validate Docs - env: - PR_NUMBER: ${{ github.event.number }} - run: | - pip install requests --quiet - python $GITHUB_WORKSPACE/.github/helper/documentation.py $PR_NUMBER diff --git a/.github/workflows/labeller.yml b/.github/workflows/labeller.yml new file mode 100644 index 0000000000..97fa4a1a2c --- /dev/null +++ b/.github/workflows/labeller.yml @@ -0,0 +1,12 @@ +name: "Pull Request Labeler" +on: + pull_request_target: + types: [opened, reopened] + +jobs: + triage: + runs-on: ubuntu-latest + steps: + - uses: actions/labeler@v4 + with: + repo-token: "${{ secrets.GITHUB_TOKEN }}" diff --git a/.github/workflows/linters.yml b/.github/workflows/linters.yml index 443ee45bf7..f56c108f6b 100644 --- a/.github/workflows/linters.yml +++ b/.github/workflows/linters.yml @@ -1,31 +1,86 @@ name: Linters on: - pull_request: { } + pull_request: + workflow_dispatch: + push: + branches: [ develop ] + +permissions: + contents: read + +concurrency: + group: commitcheck-frappe-${{ github.event.number }} + cancel-in-progress: true jobs: - - linters: - name: Frappe Linter + commit-lint: + name: 'Semantic Commits' runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + steps: - - uses: actions/checkout@v2 - - - name: Set up Python 3.8 - uses: actions/setup-python@v2 + - uses: actions/checkout@v3 with: - python-version: 3.8 + fetch-depth: 200 + - uses: actions/setup-node@v3 + with: + node-version: 16 + check-latest: true - - name: Install and Run Pre-commit - uses: pre-commit/action@v2.0.3 + - name: Check commit titles + run: | + npm install @commitlint/cli @commitlint/config-conventional + npx commitlint --verbose --from ${{ github.event.pull_request.base.sha }} --to ${{ github.event.pull_request.head.sha }} + + docs-required: + name: 'Documentation Required' + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + + steps: + - name: 'Setup Environment' + uses: actions/setup-python@v4 + with: + python-version: '3.10' + - uses: actions/checkout@v3 + + - name: Validate Docs + env: + PR_NUMBER: ${{ github.event.number }} + run: | + pip install requests --quiet + python $GITHUB_WORKSPACE/.github/helper/documentation.py $PR_NUMBER + + linter: + name: 'Frappe Linter' + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + - uses: pre-commit/action@v3.0.0 - name: Download Semgrep rules run: git clone --depth 1 https://github.com/frappe/semgrep-rules.git frappe-semgrep-rules - - uses: returntocorp/semgrep-action@v1 - env: - SEMGREP_TIMEOUT: 120 + - name: Run Semgrep rules + run: | + pip install semgrep==0.97.0 + semgrep ci --config ./frappe-semgrep-rules/rules --config r/python.lang.correctness + + deps-vulnerable-check: + name: 'Vulnerable Dependency Check' + runs-on: ubuntu-latest + + steps: + - uses: actions/setup-python@v4 with: - config: >- - r/python.lang.correctness - ./frappe-semgrep-rules/rules + python-version: '3.10' + - uses: actions/checkout@v3 + - run: | + pip install pip-audit + pip-audit ${GITHUB_WORKSPACE} diff --git a/.github/workflows/publish-assets-releases.yml b/.github/workflows/on_release.yml similarity index 51% rename from .github/workflows/publish-assets-releases.yml rename to .github/workflows/on_release.yml index 2582632fa0..59e14a8c4d 100644 --- a/.github/workflows/publish-assets-releases.yml +++ b/.github/workflows/on_release.yml @@ -1,8 +1,11 @@ -name: 'Frappe Assets' +name: 'Release' on: release: - types: [ created ] + types: [released] + +permissions: + contents: read env: GITHUB_TOKEN: ${{ github.token }} @@ -13,20 +16,22 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: path: 'frappe' - - uses: actions/setup-node@v1 + + - uses: actions/setup-node@v3 with: - python-version: '12.x' - - uses: actions/setup-python@v2 + node-version: 16 + + - uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: '3.10' - name: Set up bench and build assets run: | npm install -g yarn pip3 install -U frappe-bench - bench init frappe-bench --no-procfile --no-backups --skip-assets --skip-redis-config-generation --python $(which python) --frappe-path $GITHUB_WORKSPACE/frappe + bench -v init frappe-bench --no-procfile --no-backups --skip-assets --skip-redis-config-generation --python $(which python) --frappe-path $GITHUB_WORKSPACE/frappe cd frappe-bench && bench build - name: Package assets @@ -36,7 +41,7 @@ jobs: - name: Get release id: get_release - uses: bruceadams/get-release@v1.2.0 + uses: bruceadams/get-release@v1.2.3 - name: Upload built Assets to Release uses: actions/upload-release-asset@v1.0.2 @@ -45,3 +50,16 @@ jobs: asset_path: build/assets.tar.gz asset_name: assets.tar.gz asset_content_type: application/octet-stream + + docker-release: + name: 'Trigger Docker build on release' + runs-on: ubuntu-latest + permissions: + contents: none + container: + image: alpine:latest + steps: + - name: curl + run: | + apk add curl bash + curl -X POST -H "Accept: application/vnd.github.v3+json" -H "Authorization: Bearer ${{ secrets.CI_PAT }}" https://api.github.com/repos/frappe/frappe_docker/actions/workflows/build_stable.yml/dispatches -d '{"ref":"main"}' diff --git a/.github/workflows/patch-mariadb-tests.yml b/.github/workflows/patch-mariadb-tests.yml index c8294886a0..3412fe7503 100644 --- a/.github/workflows/patch-mariadb-tests.yml +++ b/.github/workflows/patch-mariadb-tests.yml @@ -1,42 +1,42 @@ -name: Patch - -on: [pull_request, workflow_dispatch] +name: Server (MariaDB) +on: + pull_request: + workflow_dispatch: concurrency: group: patch-mariadb-develop-${{ github.event.number }} cancel-in-progress: true +permissions: + contents: read + jobs: test: + name: Patch runs-on: ubuntu-latest timeout-minutes: 60 - name: Patch Test - services: - mysql: - image: mariadb:10.3 + mariadb: + image: mariadb:10.6 env: - MYSQL_ALLOW_EMPTY_PASSWORD: YES + MARIADB_ROOT_PASSWORD: travis ports: - 3306:3306 options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=3 steps: - name: Clone - uses: actions/checkout@v2 + uses: actions/checkout@v3 - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: '3.9' - - - name: Setup Node - uses: actions/setup-node@v2 - with: - node-version: 14 - check-latest: true + - name: Check for valid Python & Merge Conflicts + run: | + python -m compileall -f "${GITHUB_WORKSPACE}" + if grep -lr --exclude-dir=node_modules "^<<<<<<< " "${GITHUB_WORKSPACE}" + then echo "Found merge conflicts" + exit 1 + fi - name: Check if build should be run id: check-build @@ -47,23 +47,36 @@ jobs: PR_NUMBER: ${{ github.event.number }} REPO_NAME: ${{ github.repository }} + - name: Setup Python + if: ${{ steps.check-build.outputs.build == 'strawberry' }} + uses: "gabrielfalcao/pyenv-action@v10" + with: + versions: 3.10:latest, 3.7:latest + + - name: Setup Node + if: ${{ steps.check-build.outputs.build == 'strawberry' }} + uses: actions/setup-node@v3 + with: + node-version: 16 + check-latest: true + - name: Add to Hosts if: ${{ steps.check-build.outputs.build == 'strawberry' }} run: echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts - name: Cache pip if: ${{ steps.check-build.outputs.build == 'strawberry' }} - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + key: ${{ runner.os }}-pip-${{ hashFiles('**/*requirements.txt', '**/pyproject.toml', '**/setup.py') }} restore-keys: | ${{ runner.os }}-pip- ${{ runner.os }}- - name: Cache node modules if: ${{ steps.check-build.outputs.build == 'strawberry' }} - uses: actions/cache@v2 + uses: actions/cache@v3 env: cache-name: cache-node-modules with: @@ -79,7 +92,7 @@ jobs: id: yarn-cache-dir-path run: echo "::set-output name=dir::$(yarn cache dir)" - - uses: actions/cache@v2 + - uses: actions/cache@v3 if: ${{ steps.check-build.outputs.build == 'strawberry' }} id: yarn-cache with: @@ -90,18 +103,16 @@ jobs: - name: Install Dependencies if: ${{ steps.check-build.outputs.build == 'strawberry' }} - run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh + run: | + bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh + pip install frappe-bench + pyenv global $(pyenv versions | grep '3.10') + bash ${GITHUB_WORKSPACE}/.github/helper/install.sh env: BEFORE: ${{ env.GITHUB_EVENT_PATH.before }} AFTER: ${{ env.GITHUB_EVENT_PATH.after }} TYPE: server - - - name: Install - if: ${{ steps.check-build.outputs.build == 'strawberry' }} - run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh - env: DB: mariadb - TYPE: server - name: Run Patch Tests if: ${{ steps.check-build.outputs.build == 'strawberry' }} @@ -114,18 +125,23 @@ jobs: cd apps/frappe/ git remote set-url upstream https://github.com/frappe/frappe.git + pyenv global $(pyenv versions | grep '3.7') for version in $(seq 12 13) do echo "Updating to v$version" branch_name="version-$version-hotfix" git fetch --depth 1 upstream $branch_name:$branch_name - git checkout -q -f $branch_name - pip install -q -r requirements.txt + pip install -U frappe-bench + + rm -rf ~/frappe-bench/env + bench -v setup env bench --site test_site migrate done echo "Updating to last commit" git checkout -q -f "$GITHUB_SHA" - bench setup requirements --python + pyenv global $(pyenv versions | grep '3.10') + rm -rf ~/frappe-bench/env + bench -v setup env bench --site test_site migrate diff --git a/.github/workflows/publish-assets-develop.yml b/.github/workflows/publish-assets-develop.yml index f56d1460b5..12bf9eca55 100644 --- a/.github/workflows/publish-assets-develop.yml +++ b/.github/workflows/publish-assets-develop.yml @@ -1,6 +1,7 @@ name: 'Frappe Assets' on: + workflow_dispatch: push: branches: [ develop ] @@ -10,20 +11,20 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: path: 'frappe' - - uses: actions/setup-node@v1 + - uses: actions/setup-node@v3 with: - node-version: 14 - - uses: actions/setup-python@v2 + node-version: 16 + - uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: '3.10' - name: Set up bench and build assets run: | npm install -g yarn pip3 install -U frappe-bench - bench init frappe-bench --no-procfile --no-backups --skip-assets --skip-redis-config-generation --python $(which python) --frappe-path $GITHUB_WORKSPACE/frappe + bench -v init frappe-bench --no-procfile --no-backups --skip-assets --skip-redis-config-generation --python $(which python) --frappe-path $GITHUB_WORKSPACE/frappe cd frappe-bench && bench build - name: Package assets diff --git a/.github/workflows/server-mariadb-tests.yml b/.github/workflows/server-mariadb-tests.yml index 4edf74ba71..c8ccfa7862 100644 --- a/.github/workflows/server-mariadb-tests.yml +++ b/.github/workflows/server-mariadb-tests.yml @@ -1,4 +1,4 @@ -name: Server +name: Server (MariaDB) on: pull_request: @@ -11,8 +11,12 @@ concurrency: cancel-in-progress: true +permissions: + contents: read + jobs: test: + name: Unit Tests runs-on: ubuntu-latest timeout-minutes: 60 @@ -21,25 +25,31 @@ jobs: matrix: container: [1, 2] - name: Python Unit Tests (MariaDB) - services: - mysql: - image: mariadb:10.3 + mariadb: + image: mariadb:10.6 env: - MYSQL_ALLOW_EMPTY_PASSWORD: YES + MARIADB_ROOT_PASSWORD: travis ports: - 3306:3306 options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=3 steps: - name: Clone - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: '3.10' + + - name: Check for valid Python & Merge Conflicts + run: | + python -m compileall -f "${GITHUB_WORKSPACE}" + if grep -lr --exclude-dir=node_modules "^<<<<<<< " "${GITHUB_WORKSPACE}" + then echo "Found merge conflicts" + exit 1 + fi - name: Check if build should be run id: check-build @@ -50,10 +60,10 @@ jobs: PR_NUMBER: ${{ github.event.number }} REPO_NAME: ${{ github.repository }} - - uses: actions/setup-node@v2 + - uses: actions/setup-node@v3 if: ${{ steps.check-build.outputs.build == 'strawberry' }} with: - node-version: 14 + node-version: 16 check-latest: true - name: Add to Hosts @@ -64,17 +74,17 @@ jobs: - name: Cache pip if: ${{ steps.check-build.outputs.build == 'strawberry' }} - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + key: ${{ runner.os }}-pip-${{ hashFiles('**/*requirements.txt', '**/pyproject.toml', '**/setup.py') }} restore-keys: | ${{ runner.os }}-pip- ${{ runner.os }}- - name: Cache node modules if: ${{ steps.check-build.outputs.build == 'strawberry' }} - uses: actions/cache@v2 + uses: actions/cache@v3 env: cache-name: cache-node-modules with: @@ -90,7 +100,7 @@ jobs: id: yarn-cache-dir-path run: echo "::set-output name=dir::$(yarn cache dir)" - - uses: actions/cache@v2 + - uses: actions/cache@v3 if: ${{ steps.check-build.outputs.build == 'strawberry' }} id: yarn-cache with: @@ -101,18 +111,14 @@ jobs: - name: Install Dependencies if: ${{ steps.check-build.outputs.build == 'strawberry' }} - run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh + run: | + bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh + bash ${GITHUB_WORKSPACE}/.github/helper/install.sh env: BEFORE: ${{ env.GITHUB_EVENT_PATH.before }} AFTER: ${{ env.GITHUB_EVENT_PATH.after }} TYPE: server - - - name: Install - if: ${{ steps.check-build.outputs.build == 'strawberry' }} - run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh - env: DB: mariadb - TYPE: server - name: Run Tests if: ${{ steps.check-build.outputs.build == 'strawberry' }} @@ -123,7 +129,7 @@ jobs: - name: Upload coverage data if: ${{ steps.check-build.outputs.build == 'strawberry' }} - uses: codecov/codecov-action@v2 + uses: codecov/codecov-action@v3 with: name: MariaDB fail_ci_if_error: true diff --git a/.github/workflows/server-postgres-tests.yml b/.github/workflows/server-postgres-tests.yml index 895af5184e..9760067197 100644 --- a/.github/workflows/server-postgres-tests.yml +++ b/.github/workflows/server-postgres-tests.yml @@ -1,4 +1,4 @@ -name: Server +name: Server (Postgres) on: pull_request: @@ -10,8 +10,12 @@ concurrency: group: server-postgres-develop-${{ github.event.number }} cancel-in-progress: true +permissions: + contents: read + jobs: test: + name: Unit Tests runs-on: ubuntu-latest timeout-minutes: 60 @@ -20,8 +24,6 @@ jobs: matrix: container: [1, 2] - name: Python Unit Tests (Postgres) - services: postgres: image: postgres:12.4 @@ -37,12 +39,20 @@ jobs: steps: - name: Clone - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: '3.10' + + - name: Check for valid Python & Merge Conflicts + run: | + python -m compileall -f "${GITHUB_WORKSPACE}" + if grep -lr --exclude-dir=node_modules "^<<<<<<< " "${GITHUB_WORKSPACE}" + then echo "Found merge conflicts" + exit 1 + fi - name: Check if build should be run id: check-build @@ -53,10 +63,10 @@ jobs: PR_NUMBER: ${{ github.event.number }} REPO_NAME: ${{ github.repository }} - - uses: actions/setup-node@v2 + - uses: actions/setup-node@v3 if: ${{ steps.check-build.outputs.build == 'strawberry' }} with: - node-version: '14' + node-version: '16' check-latest: true - name: Add to Hosts @@ -67,17 +77,17 @@ jobs: - name: Cache pip if: ${{ steps.check-build.outputs.build == 'strawberry' }} - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + key: ${{ runner.os }}-pip-${{ hashFiles('**/*requirements.txt', '**/pyproject.toml', '**/setup.py') }} restore-keys: | ${{ runner.os }}-pip- ${{ runner.os }}- - name: Cache node modules if: ${{ steps.check-build.outputs.build == 'strawberry' }} - uses: actions/cache@v2 + uses: actions/cache@v3 env: cache-name: cache-node-modules with: @@ -93,7 +103,7 @@ jobs: id: yarn-cache-dir-path run: echo "::set-output name=dir::$(yarn cache dir)" - - uses: actions/cache@v2 + - uses: actions/cache@v3 if: ${{ steps.check-build.outputs.build == 'strawberry' }} id: yarn-cache with: @@ -104,18 +114,14 @@ jobs: - name: Install Dependencies if: ${{ steps.check-build.outputs.build == 'strawberry' }} - run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh + run: | + bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh + bash ${GITHUB_WORKSPACE}/.github/helper/install.sh env: BEFORE: ${{ env.GITHUB_EVENT_PATH.before }} AFTER: ${{ env.GITHUB_EVENT_PATH.after }} TYPE: server - - - name: Install - if: ${{ steps.check-build.outputs.build == 'strawberry' }} - run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh - env: DB: postgres - TYPE: server - name: Run Tests if: ${{ steps.check-build.outputs.build == 'strawberry' }} @@ -126,7 +132,7 @@ jobs: - name: Upload coverage data if: ${{ steps.check-build.outputs.build == 'strawberry' }} - uses: codecov/codecov-action@v2 + uses: codecov/codecov-action@v3 with: name: Postgres fail_ci_if_error: true diff --git a/.github/workflows/ui-tests.yml b/.github/workflows/ui-tests.yml index fc8093444e..115b8c2b1b 100644 --- a/.github/workflows/ui-tests.yml +++ b/.github/workflows/ui-tests.yml @@ -10,6 +10,9 @@ concurrency: group: ui-develop-${{ github.event.number }} cancel-in-progress: true +permissions: + contents: read + jobs: test: runs-on: ubuntu-latest @@ -18,27 +21,35 @@ jobs: strategy: fail-fast: false matrix: - containers: [1, 2] + containers: [1, 2, 3] name: UI Tests (Cypress) services: - mysql: - image: mariadb:10.3 + mariadb: + image: mariadb:10.6 env: - MYSQL_ALLOW_EMPTY_PASSWORD: YES + MARIADB_ROOT_PASSWORD: travis ports: - 3306:3306 options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=3 steps: - name: Clone - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: '3.10' + + - name: Check for valid Python & Merge Conflicts + run: | + python -m compileall -f "${GITHUB_WORKSPACE}" + if grep -lr --exclude-dir=node_modules "^<<<<<<< " "${GITHUB_WORKSPACE}" + then echo "Found merge conflicts" + exit 1 + fi - name: Check if build should be run id: check-build @@ -49,10 +60,10 @@ jobs: PR_NUMBER: ${{ github.event.number }} REPO_NAME: ${{ github.repository }} - - uses: actions/setup-node@v2 + - uses: actions/setup-node@v3 if: ${{ steps.check-build.outputs.build == 'strawberry' }} with: - node-version: 14 + node-version: 16 check-latest: true - name: Add to Hosts @@ -63,17 +74,17 @@ jobs: - name: Cache pip if: ${{ steps.check-build.outputs.build == 'strawberry' }} - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + key: ${{ runner.os }}-pip-${{ hashFiles('**/*requirements.txt', '**/pyproject.toml', '**/setup.py') }} restore-keys: | ${{ runner.os }}-pip- ${{ runner.os }}- - name: Cache node modules if: ${{ steps.check-build.outputs.build == 'strawberry' }} - uses: actions/cache@v2 + uses: actions/cache@v3 env: cache-name: cache-node-modules with: @@ -89,7 +100,7 @@ jobs: id: yarn-cache-dir-path run: echo "::set-output name=dir::$(yarn cache dir)" - - uses: actions/cache@v2 + - uses: actions/cache@v3 if: ${{ steps.check-build.outputs.build == 'strawberry' }} id: yarn-cache with: @@ -100,7 +111,7 @@ jobs: - name: Cache cypress binary if: ${{ steps.check-build.outputs.build == 'strawberry' }} - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ~/.cache key: ${{ runner.os }}-cypress- @@ -110,18 +121,14 @@ jobs: - name: Install Dependencies if: ${{ steps.check-build.outputs.build == 'strawberry' }} - run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh + run: | + bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh + bash ${GITHUB_WORKSPACE}/.github/helper/install.sh env: BEFORE: ${{ env.GITHUB_EVENT_PATH.before }} AFTER: ${{ env.GITHUB_EVENT_PATH.after }} TYPE: ui - - - name: Install - if: ${{ steps.check-build.outputs.build == 'strawberry' }} - run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh - env: DB: mariadb - TYPE: ui - name: Instrument Source Code if: ${{ steps.check-build.outputs.build == 'strawberry' }} @@ -155,7 +162,7 @@ jobs: - name: Upload Coverage Data if: ${{ steps.check-build.outputs.build == 'strawberry' && steps.check_coverage.outputs.files_exists == 'true' }} - uses: codecov/codecov-action@v2 + uses: codecov/codecov-action@v3 with: name: Cypress fail_ci_if_error: true @@ -165,10 +172,14 @@ jobs: - name: Upload Server Coverage Data if: ${{ steps.check-build.outputs.build-server == 'strawberry' }} - uses: codecov/codecov-action@v2 + uses: codecov/codecov-action@v3 with: name: MariaDB fail_ci_if_error: true files: /home/runner/frappe-bench/sites/coverage.xml verbose: true flags: server + + - name: Show bench console if tests failed + if: ${{ failure() }} + run: cat ~/frappe-bench/bench_start.log \ No newline at end of file diff --git a/.gitignore b/.gitignore index 7e3d178630..a134417a5c 100644 --- a/.gitignore +++ b/.gitignore @@ -36,7 +36,6 @@ dist/ downloads/ eggs/ .eggs/ -lib/ lib64/ parts/ sdist/ diff --git a/.mergify.yml b/.mergify.yml index 838ce75835..d6a9272d5f 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -6,6 +6,8 @@ pull_request_rules: - author!=surajshetty3416 - author!=gavindsouza - author!=deepeshgarg007 + - author!=ankush + - author!=mergify[bot] - or: - base=version-13 - base=version-12 @@ -13,19 +15,20 @@ pull_request_rules: close: comment: message: | - @{{author}}, thanks for the contribution, but we do not accept pull requests on a stable branch. Please raise PR on an appropriate hotfix branch. + @{{author}}, thanks for the contribution, but we do not accept pull requests on a stable branch. Please raise PR on an appropriate hotfix branch. https://github.com/frappe/erpnext/wiki/Pull-Request-Checklist#which-branch - name: Automatic merge on CI success and review conditions: - status-success=Sider - - status-success=Semantic Pull Request + - status-success=Check Commit Titles - status-success=Python Unit Tests (MariaDB) (1) - status-success=Python Unit Tests (MariaDB) (2) - status-success=Python Unit Tests (Postgres) (1) - status-success=Python Unit Tests (Postgres) (2) - status-success=UI Tests (Cypress) (1) - status-success=UI Tests (Cypress) (2) + - status-success=UI Tests (Cypress) (3) - status-success=security/snyk (frappe) - label!=dont-merge - label!=squash @@ -42,6 +45,7 @@ pull_request_rules: - status-success=Python Unit Tests (Postgres) (2) - status-success=UI Tests (Cypress) (1) - status-success=UI Tests (Cypress) (2) + - status-success=UI Tests (Cypress) (3) - status-success=security/snyk (frappe) - label!=dont-merge - label=squash @@ -53,3 +57,63 @@ pull_request_rules: {{ title }} (#{{ number }}) {{ body }} + + - name: backport to develop + conditions: + - label="backport develop" + actions: + backport: + branches: + - develop + assignees: + - "{{ author }}" + + - name: backport to version-13-hotfix + conditions: + - label="backport version-13-hotfix" + actions: + backport: + branches: + - version-13-hotfix + assignees: + - "{{ author }}" + + - name: backport to version-14-hotfix + conditions: + - label="backport version-14-hotfix" + actions: + backport: + branches: + - version-14-hotfix + assignees: + - "{{ author }}" + + - name: backport to develop + conditions: + - label="backport develop" + actions: + backport: + branches: + - develop + assignees: + - "{{ author }}" + + - name: backport to version-13-pre-release + conditions: + - label="backport version-13-pre-release" + actions: + backport: + branches: + - version-13-pre-release + assignees: + - "{{ author }}" + + - name: backport to version-12-hotfix + conditions: + - label="backport version-12-hotfix" + actions: + backport: + branches: + - version-12-hotfix + assignees: + - "{{ author }}" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b39f1ca85d..27fae671c9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ fail_fast: false repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.0.1 + rev: v4.3.0 hooks: - id: trailing-whitespace files: "frappe.*" @@ -15,6 +15,16 @@ repos: args: ['--branch', 'develop'] - id: check-merge-conflict - id: check-ast + - id: check-json + - id: check-toml + - id: check-yaml + - id: debug-statements + + - repo: https://github.com/asottile/pyupgrade + rev: v2.34.0 + hooks: + - id: pyupgrade + args: ['--py310-plus'] - repo: https://github.com/adityahase/black rev: 9cb0a69f4d0030cdf687eddf314468b39ed54119 @@ -22,11 +32,34 @@ repos: - id: black additional_dependencies: ['click==8.0.4'] + - repo: https://github.com/pre-commit/mirrors-prettier + rev: v2.7.1 + hooks: + - id: prettier + types_or: [javascript] + # Ignore any files that might contain jinja / bundles + exclude: | + (?x)^( + frappe/public/dist/.*| + .*node_modules.*| + .*boilerplate.*| + frappe/www/website_script.js| + frappe/templates/includes/.*| + frappe/public/js/lib/.* + )$ + + - repo: https://github.com/timothycrosley/isort rev: 5.9.1 hooks: - id: isort - exclude: ".*setup.py$" + + - repo: https://gitlab.com/pycqa/flake8 + rev: 3.9.2 + hooks: + - id: flake8 + additional_dependencies: ['flake8-bugbear',] + args: ['--config', '.github/helper/flake8.conf'] ci: autoupdate_schedule: weekly diff --git a/.pylintrc b/.pylintrc deleted file mode 100644 index c11c0ab6a3..0000000000 --- a/.pylintrc +++ /dev/null @@ -1,2 +0,0 @@ -disable=access-member-before-definition -disable=no-member \ No newline at end of file diff --git a/.releaserc b/.releaserc new file mode 100644 index 0000000000..c9ca71bbf5 --- /dev/null +++ b/.releaserc @@ -0,0 +1,21 @@ +{ + "branches": ["develop", {"name": "version-14-beta", "channel": "beta", "prerelease": true}], + "plugins": [ + "@semantic-release/commit-analyzer", { + "preset": "angular" + }, + "@semantic-release/release-notes-generator", + [ + "@semantic-release/exec", { + "prepareCmd": 'sed -ir "s/[0-9]*\.[0-9]*\.[0-9]*/${nextRelease.version}/" frappe/__init__.py' + } + ], + [ + "@semantic-release/git", { + "assets": ["frappe/__init__.py"], + "message": "chore(release): Bumped to Version ${nextRelease.version}\n\n${nextRelease.notes}" + } + ], + "@semantic-release/github" + ] +} \ No newline at end of file diff --git a/CODEOWNERS b/CODEOWNERS index 170334a4b4..59832e8636 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -12,7 +12,7 @@ data_import* @netchampfaris core/ @surajshetty3416 database @gavindsouza model @gavindsouza -requirements.txt @gavindsouza +pyproject.toml @gavindsouza query_builder/ @gavindsouza commands/ @gavindsouza workspace @shariquerik diff --git a/Makefile b/Makefile deleted file mode 100644 index 44a7d2fd59..0000000000 --- a/Makefile +++ /dev/null @@ -1,4 +0,0 @@ -BASEDIR := $(realpath .) - -clean: - find $(BASEDIR) | grep -E "__pycache__|\.pyc" | xargs rm -rf \ No newline at end of file diff --git a/README.md b/README.md index 8c8317c8bd..4942d87e18 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,16 @@
-

-
- - - -

-

- a web framework with "batteries included" -

-
- it's pronounced - fra-pay -
+

+
+ + + +

+

+ a web framework with "batteries included" +

+
+ it's pronounced - fra-pay +
@@ -27,20 +27,24 @@ - +
- Full-stack web application framework that uses Python and MariaDB on the server side and a tightly integrated client side library. Built for [ERPNext](https://erpnext.com) -
- +
+ + + Try in PWD +
+> Login for the PWD site: (username: Administrator, password: admin) + ## Table of Contents * [Installation](#installation) * [Contributing](#contributing) @@ -52,7 +56,7 @@ Full-stack web application framework that uses Python and MariaDB on the server * [Install via Docker](https://github.com/frappe/frappe_docker) * [Install via Frappe Bench](https://github.com/frappe/bench) * [Offical Documentation](https://frappeframework.com/docs/user/en/installation) -* [Managed Hosting on Frappe Cloud](https://frappecloud.com/deploy?apps=frappe&source=frappe_readme) +* [Managed Hosting on Frappe Cloud](https://frappecloud.com/frappe/signup) ## Contributing diff --git a/attributions.md b/attributions.md index b7c72912e4..5afc9f9d46 100644 --- a/attributions.md +++ b/attributions.md @@ -1,50 +1,31 @@ -## Frappe framework includes these public works +## 3rd-Party Software Report -### Javascript / CSS +The following 3rd-party software packages may be used by or distributed with . -- Bootstrap: MIT License, (c) Twitter Inc, https://getbootstrap.com -- JQuery: MIT License, (c) JQuery Foundation, http://jquery.org/license -- JQuery UI: MIT License / GPL 2, (c) JQuery Foundation, https://jqueryui.com/about -- JQuery UI Bootstrap Theme: MIT / GPL 2, (c) Addy Osmani, http://addyosmani.github.com/jquery-ui-bootstrap -- QUnit: MIT License, (c) JQuery Foundation, http://jquery.org/license -- jquery.event.drag, MIT License, (c) 2010 Three Dub Media - http://threedubmedia.com -- JQuery Cookie Plugin, MIT / GPL 2, (c) 2011, Klaus Hartl -- JQuery Time Picker, MIT License, (c) 2013 Trent Richardson, http://trentrichardson.com/examples/timepicker -- JQuery Hotkeys Plugin, MIT License, (c) 2010, John Resig -- prettydate.js, MIT License, (c) 2011, John Resig -- JQuery Resize Event, MIT License, (c) 2010 "Cowboy" Ben Alman -- excanvas.js, Apache License Version 2.0, (c) 2006 Google Inc -- showdown.js - Javascript Markdown, BSD-style Open Source License, (c) 2007 John Fraser -- Beautify HTML - MIT License, (c) 2007-2013 Einar Lielmanis and contributors. -- JQuery Gantt - MIT License, http://taitems.github.com/jQuery.Gantt/ -- SlickGrid - MIT License, https://github.com/mleibman/SlickGrid -- MomentJS - MIT License, https://github.com/moment/moment -- JSColor - LGPL, (c) Jan Odvarko, http://jscolor.com -- FullCalendar - MIT License, (c) 2013 Adam Shaw, http://fullcalendar.io/license/ -- Sortable - MIT License (c) 2013-2015 Lebedev Konstantin http://rubaxa.github.io/Sortable/ - -### Python - -- minify.js - MIT License, (c) 2002 Douglas Crockford +- Bootstrap: MIT License, (c) Twitter Inc, +- JQuery: MIT License, (c) JQuery Foundation, +- FullCalendar - MIT License, (c) 2013 Adam Shaw, +- JSignature - MIT License, (c) 2012 Willow Systems Corp , (c) 2010 Brinley Ang +- PhotoSwipe - MIT License, (c) 2014-2015 Dmitry Semenov, +- Leaflet - (c) 2010-2016, Vladimir Agafonkin, (c) 2010-2011, CloudMade +- Leaflet.Locate - (c) 2016 Dominik Moritz +- Leaflet.draw - (c) 2012-2017, Jacob Toye, Jon West, Smartrak +- Leaflet.EasyButton - MIT License, (C) 2014 Daniel Montague ### Icon Fonts -- Font Awesome - http://fontawesome.io/ - - Font License: SIL OFL 1.1 (http://scripts.sil.org/OFL) - - Code License: MIT (http://choosealicense.com/licenses/mit/) -- Octicons (c) GitHub Inc, https://octicons.github.com/ - - Font License: SIL OFL 1.1 (http://scripts.sil.org/OFL) - - Code License: MIT (http://choosealicense.com/licenses/mit/) -- Ionicons - MIT License, http://ionicons.com/ +- Font Awesome - + - Font License: SIL OFL 1.1 () + - Code License: MIT () +- Octicons (c) GitHub Inc, + - Font License: SIL OFL 1.1 () + - Code License: MIT () +- Inter - SIL Open Font License, 1.1 (c) 2020 Rasmus Andersson () ### IP Address Database -- GeoIP: (c) 2014 MaxMind, http://dev.maxmind.com/geoip/geoip2/downloadable/ - -### Wallpaper - -- Version 5 Wallpaper: http://magdeleine.co/photo-nick-west-n-139/ (Public Domain) +- GeoIP: (c) 2014 MaxMind, --- -Last updated: 1st Jan 2015 +Last updated: 4th July 2022 diff --git a/commitlint.config.js b/commitlint.config.js new file mode 100644 index 0000000000..09de8b8272 --- /dev/null +++ b/commitlint.config.js @@ -0,0 +1,25 @@ +module.exports = { + parserPreset: "conventional-changelog-conventionalcommits", + rules: { + "subject-empty": [2, "never"], + "type-case": [2, "always", "lower-case"], + "type-empty": [2, "never"], + "type-enum": [ + 2, + "always", + [ + "build", + "chore", + "ci", + "docs", + "feat", + "fix", + "perf", + "refactor", + "revert", + "style", + "test", + ], + ], + }, +}; diff --git a/cypress/fixtures/child_table_doctype.js b/cypress/fixtures/child_table_doctype.js index f65e5d1765..88a925aca3 100644 --- a/cypress/fixtures/child_table_doctype.js +++ b/cypress/fixtures/child_table_doctype.js @@ -13,8 +13,8 @@ export default { fieldtype: "Data", in_list_view: 1, label: "Title", - unique: 1 - } + unique: 1, + }, ], links: [], istable: 1, @@ -24,7 +24,7 @@ export default { naming_rule: "By fieldname", owner: "Administrator", permissions: [], - sort_field: 'modified', - sort_order: 'ASC', - track_changes: 1 -}; \ No newline at end of file + sort_field: "modified", + sort_order: "ASC", + track_changes: 1, +}; diff --git a/cypress/fixtures/child_table_doctype_1.js b/cypress/fixtures/child_table_doctype_1.js index 4657d63e2e..abf8873bff 100644 --- a/cypress/fixtures/child_table_doctype_1.js +++ b/cypress/fixtures/child_table_doctype_1.js @@ -12,38 +12,38 @@ export default { fieldname: "data", fieldtype: "Data", in_list_view: 1, - label: "Data" + label: "Data", }, { fieldname: "barcode", fieldtype: "Barcode", in_list_view: 1, - label: "Barcode" + label: "Barcode", }, { fieldname: "check", fieldtype: "Check", in_list_view: 1, - label: "Check" + label: "Check", }, { fieldname: "rating", fieldtype: "Rating", in_list_view: 1, - label: "Rating" + label: "Rating", }, { fieldname: "duration", fieldtype: "Duration", in_list_view: 1, - label: "Duration" + label: "Duration", }, { fieldname: "date", fieldtype: "Date", in_list_view: 1, - label: "Date" - } + label: "Date", + }, ], links: [], istable: 1, @@ -53,7 +53,7 @@ export default { naming_rule: "By fieldname", owner: "Administrator", permissions: [], - sort_field: 'modified', - sort_order: 'ASC', - track_changes: 1 -}; \ No newline at end of file + sort_field: "modified", + sort_order: "ASC", + track_changes: 1, +}; diff --git a/cypress/fixtures/custom_submittable_doctype.js b/cypress/fixtures/custom_submittable_doctype.js index c88d37b373..30aa698db4 100644 --- a/cypress/fixtures/custom_submittable_doctype.js +++ b/cypress/fixtures/custom_submittable_doctype.js @@ -1,37 +1,37 @@ export default { - name: 'Custom Submittable DocType', + name: "Custom Submittable DocType", custom: 1, actions: [], is_submittable: 1, - creation: '2019-12-10 06:29:07.215072', - doctype: 'DocType', + creation: "2019-12-10 06:29:07.215072", + doctype: "DocType", editable_grid: 1, - engine: 'InnoDB', + engine: "InnoDB", fields: [ { - fieldname: 'enabled', - fieldtype: 'Check', - label: 'Enabled', + fieldname: "enabled", + fieldtype: "Check", + label: "Enabled", allow_on_submit: 1, - reqd: 1 + reqd: 1, }, { - fieldname: 'title', - fieldtype: 'Data', - label: 'title', - reqd: 1 + fieldname: "title", + fieldtype: "Data", + label: "title", + reqd: 1, }, { - fieldname: 'description', - fieldtype: 'Text Editor', - label: 'Description' - } + fieldname: "description", + fieldtype: "Text Editor", + label: "Description", + }, ], links: [], - modified: '2019-12-10 14:40:53.127615', - modified_by: 'Administrator', - module: 'Custom', - owner: 'Administrator', + modified: "2019-12-10 14:40:53.127615", + modified_by: "Administrator", + module: "Custom", + owner: "Administrator", permissions: [ { create: 1, @@ -39,15 +39,15 @@ export default { email: 1, print: 1, read: 1, - role: 'System Manager', + role: "System Manager", share: 1, write: 1, submit: 1, - cancel: 1 - } + cancel: 1, + }, ], quick_entry: 1, - sort_field: 'modified', - sort_order: 'ASC', - track_changes: 1 -}; \ No newline at end of file + sort_field: "modified", + sort_order: "ASC", + track_changes: 1, +}; diff --git a/cypress/fixtures/data_field_validation_doctype.js b/cypress/fixtures/data_field_validation_doctype.js index da091af7e5..2901630d3f 100644 --- a/cypress/fixtures/data_field_validation_doctype.js +++ b/cypress/fixtures/data_field_validation_doctype.js @@ -1,51 +1,51 @@ export default { - name: 'Validation Test', + name: "Validation Test", custom: 1, actions: [], - creation: '2019-03-15 06:29:07.215072', - doctype: 'DocType', + creation: "2019-03-15 06:29:07.215072", + doctype: "DocType", editable_grid: 1, - engine: 'InnoDB', + engine: "InnoDB", fields: [ { - fieldname: 'email', - fieldtype: 'Data', - label: 'Email', - options: 'Email' + fieldname: "email", + fieldtype: "Data", + label: "Email", + options: "Email", }, { - fieldname: 'URL', - fieldtype: 'Data', - label: 'URL', - options: 'URL' + fieldname: "URL", + fieldtype: "Data", + label: "URL", + options: "URL", }, { - fieldname: 'Phone', - fieldtype: 'Data', - label: 'Phone', - options: 'Phone' + fieldname: "Phone", + fieldtype: "Data", + label: "Phone", + options: "Phone", }, { - fieldname: 'person_name', - fieldtype: 'Data', - label: 'Person Name', - options: 'Name' + fieldname: "person_name", + fieldtype: "Data", + label: "Person Name", + options: "Name", }, { - fieldname: 'read_only_url', - fieldtype: 'Data', - label: 'Read Only URL', - options: 'URL', - read_only: '1', - default: 'https://frappe.io' - } + fieldname: "read_only_url", + fieldtype: "Data", + label: "Read Only URL", + options: "URL", + read_only: "1", + default: "https://frappe.io", + }, ], issingle: 1, links: [], - modified: '2021-04-19 14:40:53.127615', - modified_by: 'Administrator', - module: 'Custom', - owner: 'Administrator', + modified: "2021-04-19 14:40:53.127615", + modified_by: "Administrator", + module: "Custom", + owner: "Administrator", permissions: [ { create: 1, @@ -53,13 +53,13 @@ export default { email: 1, print: 1, read: 1, - role: 'System Manager', + role: "System Manager", share: 1, - write: 1 - } + write: 1, + }, ], quick_entry: 1, - sort_field: 'modified', - sort_order: 'ASC', - track_changes: 1 + sort_field: "modified", + sort_order: "ASC", + track_changes: 1, }; diff --git a/cypress/fixtures/datetime_doctype.js b/cypress/fixtures/datetime_doctype.js index b8c89ced5c..f1a77ba6bb 100644 --- a/cypress/fixtures/datetime_doctype.js +++ b/cypress/fixtures/datetime_doctype.js @@ -1,34 +1,34 @@ export default { - name: 'DateTime Test', + name: "DateTime Test", custom: 1, actions: [], - creation: '2019-03-15 06:29:07.215072', - doctype: 'DocType', + creation: "2019-03-15 06:29:07.215072", + doctype: "DocType", editable_grid: 1, - engine: 'InnoDB', + engine: "InnoDB", fields: [ { - fieldname: 'date', - fieldtype: 'Date', - label: 'Date' + fieldname: "date", + fieldtype: "Date", + label: "Date", }, { - fieldname: 'time', - fieldtype: 'Time', - label: 'Time' + fieldname: "time", + fieldtype: "Time", + label: "Time", }, { - fieldname: 'datetime', - fieldtype: 'Datetime', - label: 'Datetime' - } + fieldname: "datetime", + fieldtype: "Datetime", + label: "Datetime", + }, ], issingle: 1, links: [], - modified: '2019-12-09 14:40:53.127615', - modified_by: 'Administrator', - module: 'Custom', - owner: 'Administrator', + modified: "2019-12-09 14:40:53.127615", + modified_by: "Administrator", + module: "Custom", + owner: "Administrator", permissions: [ { create: 1, @@ -36,13 +36,13 @@ export default { email: 1, print: 1, read: 1, - role: 'System Manager', + role: "System Manager", share: 1, - write: 1 - } + write: 1, + }, ], quick_entry: 1, - sort_field: 'modified', - sort_order: 'ASC', - track_changes: 1 + sort_field: "modified", + sort_order: "ASC", + track_changes: 1, }; diff --git a/cypress/fixtures/doctype_to_link.js b/cypress/fixtures/doctype_to_link.js index f5335b1755..ff5d1b5c68 100644 --- a/cypress/fixtures/doctype_to_link.js +++ b/cypress/fixtures/doctype_to_link.js @@ -10,18 +10,18 @@ export default { engine: "InnoDB", fields: [ { - "fieldname": "title", - "fieldtype": "Data", - "label": "Title", - "unique": 1 - } + fieldname: "title", + fieldtype: "Data", + label: "Title", + unique: 1, + }, ], links: [ { - "group": "Child Doctype", - "link_doctype": "Doctype With Child Table", - "link_fieldname": "title" - } + group: "Child Doctype", + link_doctype: "Doctype With Child Table", + link_fieldname: "title", + }, ], modified: "2022-02-10 12:03:12.603763", modified_by: "Administrator", @@ -34,12 +34,12 @@ export default { email: 1, print: 1, read: 1, - role: 'System Manager', + role: "System Manager", share: 1, - write: 1 - } + write: 1, + }, ], - sort_field: 'modified', - sort_order: 'ASC', - track_changes: 1 -}; \ No newline at end of file + sort_field: "modified", + sort_order: "ASC", + track_changes: 1, +}; diff --git a/cypress/fixtures/doctype_with_child_table.js b/cypress/fixtures/doctype_with_child_table.js index 014074b0b5..7caba516cf 100644 --- a/cypress/fixtures/doctype_with_child_table.js +++ b/cypress/fixtures/doctype_with_child_table.js @@ -12,21 +12,21 @@ export default { fieldname: "title", fieldtype: "Data", label: "Title", - unique: 1 + unique: 1, }, { fieldname: "child_table", fieldtype: "Table", label: "Child Table", options: "Child Table Doctype", - reqd: 1 + reqd: 1, }, { fieldname: "child_table_1", fieldtype: "Table", label: "Child Table 1", - options: "Child Table Doctype 1" - } + options: "Child Table Doctype 1", + }, ], links: [], modified: "2022-02-10 12:03:12.603763", @@ -41,12 +41,12 @@ export default { email: 1, print: 1, read: 1, - role: 'System Manager', + role: "System Manager", share: 1, - write: 1 - } + write: 1, + }, ], - sort_field: 'modified', - sort_order: 'ASC', - track_changes: 1 + sort_field: "modified", + sort_order: "ASC", + track_changes: 1, }; diff --git a/cypress/fixtures/doctype_with_phone.js b/cypress/fixtures/doctype_with_phone.js new file mode 100644 index 0000000000..06a24a5be5 --- /dev/null +++ b/cypress/fixtures/doctype_with_phone.js @@ -0,0 +1,46 @@ +export default { + name: "Doctype With Phone", + actions: [], + custom: 1, + is_submittable: 1, + autoname: "field:title", + creation: "2022-03-30 06:29:07.215072", + doctype: "DocType", + engine: "InnoDB", + fields: [ + { + fieldname: "title", + fieldtype: "Data", + label: "title", + unique: 1, + }, + { + fieldname: "phone", + fieldtype: "Phone", + label: "Phone", + }, + ], + links: [], + modified: "2019-03-30 14:40:53.127615", + modified_by: "Administrator", + naming_rule: "By fieldname", + module: "Custom", + owner: "Administrator", + permissions: [ + { + create: 1, + delete: 1, + email: 1, + print: 1, + read: 1, + role: "System Manager", + share: 1, + write: 1, + submit: 1, + cancel: 1, + }, + ], + sort_field: "modified", + sort_order: "ASC", + track_changes: 1, +}; diff --git a/cypress/fixtures/doctype_with_tab_break.js b/cypress/fixtures/doctype_with_tab_break.js index 74e5e6abba..44d6c16682 100644 --- a/cypress/fixtures/doctype_with_tab_break.js +++ b/cypress/fixtures/doctype_with_tab_break.js @@ -1,39 +1,39 @@ export default { - name: 'Form With Tab Break', + name: "Form With Tab Break", custom: 1, actions: [], - doctype: 'DocType', - engine: 'InnoDB', + doctype: "DocType", + engine: "InnoDB", fields: [ { - fieldname: 'username', - fieldtype: 'Data', - label: 'Name', - options: 'Name' + fieldname: "username", + fieldtype: "Data", + label: "Name", + options: "Name", }, { - fieldname: 'tab', - fieldtype: 'Tab Break', - label: 'Tab 2', + fieldname: "tab", + fieldtype: "Tab Break", + label: "Tab 2", }, { - fieldname: 'Phone', - fieldtype: 'Data', - label: 'Phone', - options: 'Phone', - reqd: 1 + fieldname: "Phone", + fieldtype: "Data", + label: "Phone", + options: "Phone", + reqd: 1, }, ], links: [ { - "group": "Profile", - "link_doctype": "Contact", - "link_fieldname": "user" + group: "Profile", + link_doctype: "Contact", + link_fieldname: "user", }, ], - modified_by: 'Administrator', - module: 'Custom', - owner: 'Administrator', + modified_by: "Administrator", + module: "Custom", + owner: "Administrator", permissions: [ { create: 1, @@ -41,14 +41,14 @@ export default { email: 1, print: 1, read: 1, - role: 'System Manager', + role: "System Manager", share: 1, - write: 1 - } + write: 1, + }, ], quick_entry: 1, autoname: "format: Test-{####}", - sort_field: 'modified', - sort_order: 'ASC', - track_changes: 1 + sort_field: "modified", + sort_order: "ASC", + track_changes: 1, }; diff --git a/cypress/integration/api.js b/cypress/integration/api.js index e8c39e6e25..420cea25fd 100644 --- a/cypress/integration/api.js +++ b/cypress/integration/api.js @@ -1,42 +1,43 @@ -context('API Resources', () => { +context("API Resources", () => { before(() => { - cy.visit('/login'); + cy.visit("/login"); cy.login(); - cy.visit('/app/website'); + cy.visit("/app/website"); }); - it('Creates two Comments', () => { - cy.insert_doc('Comment', { comment_type: 'Comment', content: "hello" }); - cy.insert_doc('Comment', { comment_type: 'Comment', content: "world" }); + it("Creates two Comments", () => { + cy.insert_doc("Comment", { comment_type: "Comment", content: "hello" }); + cy.insert_doc("Comment", { comment_type: "Comment", content: "world" }); }); - it('Lists the Comments', () => { - cy.get_list('Comment') - .its('data') - .then(data => expect(data.length).to.be.at.least(2)); + it("Lists the Comments", () => { + cy.get_list("Comment") + .its("data") + .then((data) => expect(data.length).to.be.at.least(2)); - cy.get_list('Comment', ['name', 'content'], [['content', '=', 'hello']]) - .then(body => { - expect(body).to.have.property('data'); - expect(body.data).to.have.lengthOf(1); - expect(body.data[0]).to.have.property('content'); - expect(body.data[0]).to.have.property('name'); - }); + cy.get_list("Comment", ["name", "content"], [["content", "=", "hello"]]).then((body) => { + expect(body).to.have.property("data"); + expect(body.data).to.have.lengthOf(1); + expect(body.data[0]).to.have.property("content"); + expect(body.data[0]).to.have.property("name"); + }); }); - it('Gets each Comment', () => { - cy.get_list('Comment').then(body => body.data.forEach(comment => { - cy.get_doc('Comment', comment.name); - })); + it("Gets each Comment", () => { + cy.get_list("Comment").then((body) => + body.data.forEach((comment) => { + cy.get_doc("Comment", comment.name); + }) + ); }); - it('Removes the Comments', () => { - cy.get_list('Comment').then(body => { + it("Removes the Comments", () => { + cy.get_list("Comment").then((body) => { let comment_names = []; - body.data.map(comment => comment_names.push(comment.name)); + body.data.map((comment) => comment_names.push(comment.name)); comment_names = [...new Set(comment_names)]; // remove duplicates comment_names.forEach((comment_name) => { - cy.remove_doc('Comment', comment_name); + cy.remove_doc("Comment", comment_name); }); }); }); diff --git a/cypress/integration/awesome_bar.js b/cypress/integration/awesome_bar.js index 053d015366..71e5e498cf 100644 --- a/cypress/integration/awesome_bar.js +++ b/cypress/integration/awesome_bar.js @@ -1,47 +1,57 @@ -context('Awesome Bar', () => { +context("Awesome Bar", () => { before(() => { - cy.visit('/login'); + cy.visit("/login"); cy.login(); - cy.visit('/app/website'); + cy.visit("/app/website"); }); beforeEach(() => { - cy.get('.navbar .navbar-home').click(); - cy.findByPlaceholderText('Search or type a command (Ctrl + G)').clear(); + cy.get(".navbar .navbar-home").click(); + cy.findByPlaceholderText("Search or type a command (Ctrl + G)").clear(); }); - it('navigates to doctype list', () => { - cy.findByPlaceholderText('Search or type a command (Ctrl + G)').type('todo', { delay: 700 }); - cy.get('.awesomplete').findByRole('listbox').should('be.visible'); - cy.findByPlaceholderText('Search or type a command (Ctrl + G)').type('{enter}', { delay: 700 }); + it("navigates to doctype list", () => { + cy.findByPlaceholderText("Search or type a command (Ctrl + G)").type("todo", { + delay: 700, + }); + cy.get(".awesomplete").findByRole("listbox").should("be.visible"); + cy.findByPlaceholderText("Search or type a command (Ctrl + G)").type("{enter}", { + delay: 700, + }); - cy.get('.title-text').should('contain', 'To Do'); + cy.get(".title-text").should("contain", "To Do"); - cy.location('pathname').should('eq', '/app/todo'); + cy.location("pathname").should("eq", "/app/todo"); }); - it('find text in doctype list', () => { - cy.findByPlaceholderText('Search or type a command (Ctrl + G)') - .type('test in todo{enter}', { delay: 700 }); + it("find text in doctype list", () => { + cy.findByPlaceholderText("Search or type a command (Ctrl + G)").type( + "test in todo{enter}", + { delay: 700 } + ); - cy.get('.title-text').should('contain', 'To Do'); + cy.get(".title-text").should("contain", "To Do"); - cy.findByPlaceholderText('Name') - .should('have.value', '%test%'); + cy.findByPlaceholderText("ID").should("have.value", "%test%"); + cy.clear_filters(); }); - it('navigates to new form', () => { - cy.findByPlaceholderText('Search or type a command (Ctrl + G)') - .type('new blog post{enter}', { delay: 700 }); + it("navigates to new form", () => { + cy.findByPlaceholderText("Search or type a command (Ctrl + G)").type( + "new blog post{enter}", + { delay: 700 } + ); - cy.get('.title-text:visible').should('have.text', 'New Blog Post'); + cy.get(".title-text:visible").should("have.text", "New Blog Post"); }); - it('calculates math expressions', () => { - cy.findByPlaceholderText('Search or type a command (Ctrl + G)') - .type('55 + 32{downarrow}{enter}', { delay: 700 }); + it("calculates math expressions", () => { + cy.findByPlaceholderText("Search or type a command (Ctrl + G)").type( + "55 + 32{downarrow}{enter}", + { delay: 700 } + ); - cy.get('.modal-title').should('contain', 'Result'); - cy.get('.msgprint').should('contain', '55 + 32 = 87'); + cy.get(".modal-title").should("contain", "Result"); + cy.get(".msgprint").should("contain", "55 + 32 = 87"); }); }); diff --git a/cypress/integration/control_attach.js b/cypress/integration/control_attach.js index 0552780737..96b8c73b6e 100644 --- a/cypress/integration/control_attach.js +++ b/cypress/integration/control_attach.js @@ -1,90 +1,95 @@ -context('Attach Control', () => { +context("Attach Control", () => { before(() => { cy.login(); - cy.visit('/app/doctype'); - return cy.window().its('frappe').then(frappe => { - return frappe.xcall('frappe.tests.ui_test_helpers.create_doctype', { - name: 'Test Attach Control', - fields: [ - { - "label": "Attach File or Image", - "fieldname": "attach", - "fieldtype": "Attach", - "in_list_view": 1, - }, - ] + cy.visit("/app/doctype"); + return cy + .window() + .its("frappe") + .then((frappe) => { + return frappe.xcall("frappe.tests.ui_test_helpers.create_doctype", { + name: "Test Attach Control", + fields: [ + { + label: "Attach File or Image", + fieldname: "attach", + fieldtype: "Attach", + in_list_view: 1, + }, + ], + }); }); - }); }); it('Checking functionality for "Link" button in the "Attach" fieldtype', () => { //Navigating to the new form for the newly created doctype - cy.new_form('Test Attach Control'); + cy.new_form("Test Attach Control"); //Clicking on the attach button which is displayed as part of creating a doctype with "Attach" fieldtype - cy.findByRole('button', {name: 'Attach'}).click(); + cy.findByRole("button", { name: "Attach" }).click(); //Clicking on "Link" button to attach a file using the "Link" button - cy.findByRole('button', {name: 'Link'}).click(); - cy.findByPlaceholderText('Attach a web link').type('https://wallpaperplay.com/walls/full/8/2/b/72402.jpg'); - + cy.findByRole("button", { name: "Link" }).click(); + cy.findByPlaceholderText("Attach a web link").type( + "https://wallpaperplay.com/walls/full/8/2/b/72402.jpg" + ); + //Clicking on the Upload button to upload the file cy.intercept("POST", "/api/method/upload_file").as("upload_image"); - cy.get('.modal-footer').findByRole("button", {name: "Upload"}).click({delay: 500}); + cy.get(".modal-footer").findByRole("button", { name: "Upload" }).click({ delay: 500 }); cy.wait("@upload_image"); - cy.findByRole('button', {name: 'Save'}).click(); + cy.findByRole("button", { name: "Save" }).click(); //Checking if the URL of the attached image is getting displayed in the field of the newly created doctype - cy.get('.attached-file > .ellipsis > .attached-file-link') - .should('have.attr', 'href') - .and('equal', 'https://wallpaperplay.com/walls/full/8/2/b/72402.jpg'); + cy.get(".attached-file > .ellipsis > .attached-file-link") + .should("have.attr", "href") + .and("equal", "https://wallpaperplay.com/walls/full/8/2/b/72402.jpg"); //Clicking on the "Clear" button cy.get('[data-action="clear_attachment"]').click(); //Checking if clicking on the clear button clears the field of the doctype form and again displays the attach button - cy.get('.control-input > .btn-sm').should('contain', 'Attach'); + cy.get(".control-input > .btn-sm").should("contain", "Attach"); //Deleting the doc - cy.go_to_list('Test Attach Control'); - cy.get('.list-row-checkbox').eq(0).click(); - cy.get('.actions-btn-group > .btn').contains('Actions').click(); + cy.go_to_list("Test Attach Control"); + cy.get(".list-row-checkbox").eq(0).click(); + cy.get(".actions-btn-group > .btn").contains("Actions").click(); cy.get('.actions-btn-group > .dropdown-menu [data-label="Delete"]').click(); - cy.click_modal_primary_button('Yes'); + cy.click_modal_primary_button("Yes"); }); it('Checking functionality for "Library" button in the "Attach" fieldtype', () => { //Navigating to the new form for the newly created doctype - cy.new_form('Test Attach Control'); + cy.new_form("Test Attach Control"); //Clicking on the attach button which is displayed as part of creating a doctype with "Attach" fieldtype - cy.findByRole('button', {name: 'Attach'}).click(); + cy.findByRole("button", { name: "Attach" }).click(); //Clicking on "Library" button to attach a file using the "Library" button - cy.findByRole('button', {name: 'Library'}).click(); - cy.contains('72402.jpg').click(); + cy.findByRole("button", { name: "Library" }).click(); + cy.contains("72402.jpg").click(); //Clicking on the Upload button to upload the file cy.intercept("POST", "/api/method/upload_file").as("upload_image"); - cy.get('.modal-footer').findByRole("button", {name: "Upload"}).click({delay: 500}); + cy.get(".modal-footer").findByRole("button", { name: "Upload" }).click({ delay: 500 }); cy.wait("@upload_image"); - cy.findByRole('button', {name: 'Save'}).click(); + cy.findByRole("button", { name: "Save" }).click(); //Checking if the URL of the attached image is getting displayed in the field of the newly created doctype - cy.get('.attached-file > .ellipsis > .attached-file-link') - .should('have.attr', 'href') - .and('equal', 'https://wallpaperplay.com/walls/full/8/2/b/72402.jpg'); + cy.get(".attached-file > .ellipsis > .attached-file-link") + .should("have.attr", "href") + .and("equal", "https://wallpaperplay.com/walls/full/8/2/b/72402.jpg"); //Clicking on the "Clear" button cy.get('[data-action="clear_attachment"]').click(); //Checking if clicking on the clear button clears the field of the doctype form and again displays the attach button - cy.get('.control-input > .btn-sm').should('contain', 'Attach'); + cy.get(".control-input > .btn-sm").should("contain", "Attach"); //Deleting the doc - cy.go_to_list('Test Attach Control'); - cy.get('.list-row-checkbox').eq(0).click(); - cy.get('.actions-btn-group > .btn').contains('Actions').click(); + cy.go_to_list("Test Attach Control"); + cy.get(".list-row-checkbox").eq(0).click(); + cy.get(".actions-btn-group > .btn").contains("Actions").click(); cy.get('.actions-btn-group > .dropdown-menu [data-label="Delete"]').click(); - cy.click_modal_primary_button('Yes'); + cy.click_modal_primary_button("Yes"); }); -}); \ No newline at end of file +}); diff --git a/cypress/integration/control_autocomplete.js b/cypress/integration/control_autocomplete.js index 3bf3e829f9..6dc57fcf43 100644 --- a/cypress/integration/control_autocomplete.js +++ b/cypress/integration/control_autocomplete.js @@ -1,57 +1,64 @@ -context('Control Autocomplete', () => { +context("Control Autocomplete", () => { before(() => { cy.login(); - cy.visit('/app/website'); + cy.visit("/app/website"); }); function get_dialog_with_autocomplete(options) { - cy.visit('/app/website'); + cy.visit("/app/website"); return cy.dialog({ - title: 'Autocomplete', + title: "Autocomplete", fields: [ { - 'label': 'Select an option', - 'fieldname': 'autocomplete', - 'fieldtype': 'Autocomplete', - 'options': options || ['Option 1', 'Option 2', 'Option 3'], - } - ] + label: "Select an option", + fieldname: "autocomplete", + fieldtype: "Autocomplete", + options: options || ["Option 1", "Option 2", "Option 3"], + }, + ], }); } - it('should set the valid value', () => { - get_dialog_with_autocomplete().as('dialog'); + it("should set the valid value", () => { + get_dialog_with_autocomplete().as("dialog"); - cy.get('.frappe-control[data-fieldname=autocomplete] input').focus().as('input'); + cy.get(".frappe-control[data-fieldname=autocomplete] input").focus().as("input"); cy.wait(1000); - cy.get('@input').type('2', { delay: 300 }); - cy.get('.frappe-control[data-fieldname=autocomplete]').findByRole('listbox').should('be.visible'); - cy.get('.frappe-control[data-fieldname=autocomplete] input').type('{enter}', { delay: 300 }); - cy.get('.frappe-control[data-fieldname=autocomplete] input').blur(); - cy.get('@dialog').then(dialog => { - let value = dialog.get_value('autocomplete'); - expect(value).to.eq('Option 2'); + cy.get("@input").type("2", { delay: 300 }); + cy.get(".frappe-control[data-fieldname=autocomplete]") + .findByRole("listbox") + .should("be.visible"); + cy.get(".frappe-control[data-fieldname=autocomplete] input").type("{enter}", { + delay: 300, + }); + cy.get(".frappe-control[data-fieldname=autocomplete] input").blur(); + cy.get("@dialog").then((dialog) => { + let value = dialog.get_value("autocomplete"); + expect(value).to.eq("Option 2"); dialog.clear(); }); }); - it('should set the valid value with different label', () => { + it("should set the valid value with different label", () => { const options_with_label = [ { label: "Option 1", value: "option_1" }, - { label: "Option 2", value: "option_2" } + { label: "Option 2", value: "option_2" }, ]; - get_dialog_with_autocomplete(options_with_label).as('dialog'); + get_dialog_with_autocomplete(options_with_label).as("dialog"); - cy.get('.frappe-control[data-fieldname=autocomplete] input').focus().as('input'); - cy.get('.frappe-control[data-fieldname=autocomplete]').findByRole('listbox').should('be.visible'); - cy.get('@input').type('2', { delay: 300 }); - cy.get('.frappe-control[data-fieldname=autocomplete] input').type('{enter}', { delay: 300 }); - cy.get('.frappe-control[data-fieldname=autocomplete] input').blur(); - cy.get('@dialog').then(dialog => { - let value = dialog.get_value('autocomplete'); - expect(value).to.eq('option_2'); + cy.get(".frappe-control[data-fieldname=autocomplete] input").focus().as("input"); + cy.get(".frappe-control[data-fieldname=autocomplete]") + .findByRole("listbox") + .should("be.visible"); + cy.get("@input").type("2", { delay: 300 }); + cy.get(".frappe-control[data-fieldname=autocomplete] input").type("{enter}", { + delay: 300, + }); + cy.get(".frappe-control[data-fieldname=autocomplete] input").blur(); + cy.get("@dialog").then((dialog) => { + let value = dialog.get_value("autocomplete"); + expect(value).to.eq("option_2"); dialog.clear(); }); }); - }); diff --git a/cypress/integration/control_barcode.js b/cypress/integration/control_barcode.js index 03ab61fac4..96a1bb43d4 100644 --- a/cypress/integration/control_barcode.js +++ b/cypress/integration/control_barcode.js @@ -1,53 +1,57 @@ -context('Control Barcode', () => { +context("Control Barcode", () => { beforeEach(() => { cy.login(); - cy.visit('/app/website'); + cy.visit("/app/website"); }); function get_dialog_with_barcode() { return cy.dialog({ - title: 'Barcode', + title: "Barcode", fields: [ { - label: 'Barcode', - fieldname: 'barcode', - fieldtype: 'Barcode' - } - ] + label: "Barcode", + fieldname: "barcode", + fieldtype: "Barcode", + }, + ], }); } - it('should generate barcode on setting a value', () => { - get_dialog_with_barcode().as('dialog'); + it("should generate barcode on setting a value", () => { + get_dialog_with_barcode().as("dialog"); - cy.get('.frappe-control[data-fieldname=barcode]').findByRole('textbox') - .type('123456789') + cy.focused().blur(); + cy.get(".frappe-control[data-fieldname=barcode]") + .findByRole("textbox") + .type("123456789") .blur(); - cy.get('.frappe-control[data-fieldname=barcode] svg[data-barcode-value="123456789"]') - .should('exist'); + cy.get( + '.frappe-control[data-fieldname=barcode] svg[data-barcode-value="123456789"]' + ).should("exist"); - cy.get('@dialog').then(dialog => { - let value = dialog.get_value('barcode'); - expect(value).to.contain(' { + let value = dialog.get_value("barcode"); + expect(value).to.contain(" { - get_dialog_with_barcode().as('dialog'); + it("should reset when input is cleared", () => { + get_dialog_with_barcode().as("dialog"); - cy.get('.frappe-control[data-fieldname=barcode]').findByRole('textbox') - .type('123456789') + cy.focused().blur(); + cy.get(".frappe-control[data-fieldname=barcode]") + .findByRole("textbox") + .type("123456789") .blur(); - cy.get('.frappe-control[data-fieldname=barcode]').findByRole('textbox') - .clear() - .blur(); - cy.get('.frappe-control[data-fieldname=barcode] svg[data-barcode-value="123456789"]') - .should('not.exist'); + cy.get(".frappe-control[data-fieldname=barcode]").findByRole("textbox").clear().blur(); + cy.get( + '.frappe-control[data-fieldname=barcode] svg[data-barcode-value="123456789"]' + ).should("not.exist"); - cy.get('@dialog').then(dialog => { - let value = dialog.get_value('barcode'); - expect(value).to.equal(''); + cy.get("@dialog").then((dialog) => { + let value = dialog.get_value("barcode"); + expect(value).to.equal(""); }); }); }); diff --git a/cypress/integration/control_color.js b/cypress/integration/control_color.js new file mode 100644 index 0000000000..aa3a45eed8 --- /dev/null +++ b/cypress/integration/control_color.js @@ -0,0 +1,80 @@ +context("Control Color", () => { + before(() => { + cy.login(); + cy.visit("/app/website"); + }); + + function get_dialog_with_color() { + return cy.dialog({ + title: "Color", + fields: [ + { + label: "Color", + fieldname: "color", + fieldtype: "Color", + }, + ], + }); + } + + it("Verifying if the color control is selecting correct", () => { + get_dialog_with_color().as("dialog"); + cy.findByPlaceholderText("Choose a color").click(); + + ///Selecting a color from the color palette + cy.get('[style="background-color: rgb(79, 157, 217);"]').click(); + + //Checking if the css attribute is correct + cy.get(".color-map").should("have.css", "color", "rgb(79, 157, 217)"); + cy.get(".hue-map").should("have.css", "color", "rgb(0, 145, 255)"); + + //Checking if the correct color is being selected + cy.get("@dialog").then((dialog) => { + let value = dialog.get_value("color"); + expect(value).to.equal("#4F9DD9"); + }); + + //Selecting a color + cy.get('[style="background-color: rgb(203, 41, 41);"]').click(); + + //Checking if the correct css is being selected + cy.get(".color-map").should("have.css", "color", "rgb(203, 41, 41)"); + cy.get(".hue-map").should("have.css", "color", "rgb(255, 0, 0)"); + + //Checking if the correct color is being selected + cy.get("@dialog").then((dialog) => { + let value = dialog.get_value("color"); + expect(value).to.equal("#CB2929"); + }); + + //Selecting color from the palette + cy.get(".color-map > .color-selector").click(65, 87, { force: true }); + cy.get(".color-map").should("have.css", "color", "rgb(56, 0, 0)"); + + //Checking if the expected color is selected and getting displayed + cy.get("@dialog").then((dialog) => { + let value = dialog.get_value("color"); + expect(value).to.equal("#380000"); + }); + + //Selecting the color from the hue map + cy.get(".hue-map > .hue-selector").click(35, -1, { force: true }); + cy.get(".color-map").should("have.css", "color", "rgb(56, 45, 0)"); + cy.get(".hue-map").should("have.css", "color", "rgb(255, 204, 0)"); + cy.get(".color-map > .color-selector").click(55, 12, { force: true }); + cy.get(".color-map").should("have.css", "color", "rgb(46, 37, 0)"); + + //Checking if the correct color is being displayed + cy.get("@dialog").then((dialog) => { + let value = dialog.get_value("color"); + expect(value).to.equal("#2e2500"); + }); + + //Clearing the field and checking if the field contains the placeholder "Choose a color" + cy.get(".input-with-feedback").click({ force: true }); + cy.get_field("color", "Color").type("{selectall}").clear(); + cy.get_field("color", "Color") + .invoke("attr", "placeholder") + .should("contain", "Choose a color"); + }); +}); diff --git a/cypress/integration/control_data.js b/cypress/integration/control_data.js new file mode 100644 index 0000000000..d855df2919 --- /dev/null +++ b/cypress/integration/control_data.js @@ -0,0 +1,145 @@ +context("Data Control", () => { + before(() => { + cy.login(); + cy.visit("/app/doctype"); + return cy + .window() + .its("frappe") + .then((frappe) => { + return frappe.xcall("frappe.tests.ui_test_helpers.create_doctype", { + name: "Test Data Control", + fields: [ + { + label: "Name", + fieldname: "name1", + fieldtype: "Data", + options: "Name", + in_list_view: 1, + reqd: 1, + }, + { + label: "Email-ID", + fieldname: "email", + fieldtype: "Data", + options: "Email", + in_list_view: 1, + reqd: 1, + }, + { + label: "Phone No.", + fieldname: "phone", + fieldtype: "Data", + options: "Phone", + in_list_view: 1, + reqd: 1, + }, + ], + }); + }); + }); + + it("check custom formatters", () => { + cy.visit(`/app/doctype/User`); + cy.get( + '[data-fieldname="fields"] .grid-row[data-idx="2"] [data-fieldname="fieldtype"] .static-area' + ).should("have.text", "🔵 Section Break"); + }); + + it('Verifying data control by inputting different patterns for "Name" field', () => { + cy.new_form("Test Data Control"); + + //Checking the URL for the new form of the doctype + cy.location("pathname").should("eq", "/app/test-data-control/new-test-data-control-1"); + cy.get(".title-text").should("have.text", "New Test Data Control"); + cy.get('.frappe-control[data-fieldname="name1"]') + .find("label") + .should("have.class", "reqd"); + cy.get('.frappe-control[data-fieldname="email"]') + .find("label") + .should("have.class", "reqd"); + cy.get('.frappe-control[data-fieldname="phone"]') + .find("label") + .should("have.class", "reqd"); + + //Checking if the status is "Not Saved" initially + cy.get(".indicator-pill").should("have.text", "Not Saved"); + + //Inputting data in the field + cy.fill_field("name1", "@@###", "Data"); + cy.fill_field("email", "test@example.com", "Data"); + cy.fill_field("phone", "9834280031", "Data"); + + //Checking if the border color of the field changes to red + cy.get('.frappe-control[data-fieldname="name1"]').should("have.class", "has-error"); + cy.save(); + + //Checking for the error message + cy.get(".modal-title").should("have.text", "Message"); + cy.get(".msgprint").should("have.text", "@@### is not a valid Name"); + cy.hide_dialog(); + + cy.get_field("name1", "Data").clear({ force: true }); + cy.fill_field("name1", "Komal{}/!", "Data"); + cy.get('.frappe-control[data-fieldname="name1"]').should("have.class", "has-error"); + cy.save(); + cy.get(".modal-title").should("have.text", "Message"); + cy.get(".msgprint").should("have.text", "Komal{}/! is not a valid Name"); + cy.hide_dialog(); + }); + + it('Verifying data control by inputting different patterns for "Email" field', () => { + cy.get_field("name1", "Data").clear({ force: true }); + cy.fill_field("name1", "Komal", "Data"); + cy.get_field("email", "Data").clear({ force: true }); + cy.fill_field("email", "komal", "Data"); + cy.get('.frappe-control[data-fieldname="email"]').should("have.class", "has-error"); + cy.save(); + cy.get(".modal-title").should("have.text", "Message"); + cy.get(".msgprint").should("have.text", "komal is not a valid Email Address"); + cy.hide_dialog(); + cy.get_field("email", "Data").clear({ force: true }); + cy.fill_field("email", "komal@test", "Data"); + cy.get('.frappe-control[data-fieldname="email"]').should("have.class", "has-error"); + cy.save(); + cy.get(".modal-title").should("have.text", "Message"); + cy.get(".msgprint").should("have.text", "komal@test is not a valid Email Address"); + cy.hide_dialog(); + }); + + it('Verifying data control by inputting different patterns for "Phone" field', () => { + cy.get_field("email", "Data").clear({ force: true }); + cy.fill_field("email", "komal@test.com", "Data"); + cy.get_field("phone", "Data").clear({ force: true }); + cy.fill_field("phone", "komal", "Data"); + cy.get('.frappe-control[data-fieldname="phone"]').should("have.class", "has-error"); + cy.findByRole("button", { name: "Save" }).click({ force: true }); + cy.get(".modal-title").should("have.text", "Message"); + cy.get(".msgprint").should("have.text", "komal is not a valid Phone Number"); + cy.hide_dialog(); + }); + + it("Inputting correct data and saving the doc", () => { + //Inputting the data as expected and saving the document + cy.get_field("name1", "Data").clear({ force: true }); + cy.get_field("email", "Data").clear({ force: true }); + cy.get_field("phone", "Data").clear({ force: true }); + cy.fill_field("name1", "Komal", "Data"); + cy.fill_field("email", "komal@test.com", "Data"); + cy.fill_field("phone", "9432380001", "Data"); + cy.findByRole("button", { name: "Save" }).click({ force: true }); + //Checking if the fields contains the data which has been filled in + cy.location("pathname").should("not.be", "/app/test-data-control/new-test-data-control-1"); + cy.get_field("name1").should("have.value", "Komal"); + cy.get_field("email").should("have.value", "komal@test.com"); + cy.get_field("phone").should("have.value", "9432380001"); + }); + + it("Deleting the doc", () => { + //Deleting the inserted document + cy.go_to_list("Test Data Control"); + cy.get(".list-row-checkbox").eq(0).click({ force: true }); + cy.get(".actions-btn-group > .btn").contains("Actions").click(); + cy.get('.actions-btn-group > .dropdown-menu [data-label="Delete"]').click(); + cy.click_modal_primary_button("Yes"); + }); +}); diff --git a/cypress/integration/control_date.js b/cypress/integration/control_date.js index 35c585306c..408f7b819e 100644 --- a/cypress/integration/control_date.js +++ b/cypress/integration/control_date.js @@ -1,71 +1,89 @@ -context('Date Control', () => { +context("Date Control", () => { before(() => { cy.login(); - cy.visit('/app/doctype'); - return cy.window().its('frappe').then(frappe => { - return frappe.xcall('frappe.tests.ui_test_helpers.create_doctype', { - name: 'Test Date Control', - fields: [ - { - "label": "Date", - "fieldname": "date", - "fieldtype": "Date", - "in_list_view": 1 - }, - ] - }); - }); + cy.visit("/app"); }); - it('Selecting a date from the datepicker', () => { - cy.new_form('Test Date Control'); - cy.get_field('date', 'Date').click(); - cy.get('.datepicker--nav-title').click(); - cy.get('.datepicker--nav-title').click({force: true}); + function get_dialog(date_field_options) { + return cy.dialog({ + title: "Date", + fields: [ + { + label: "Date", + fieldname: "date", + fieldtype: "Date", + in_list_view: 1, + ...date_field_options, + }, + ], + }); + } + + it("Selecting a date from the datepicker", () => { + cy.clear_dialogs(); + cy.clear_datepickers(); + + get_dialog().as("dialog"); + cy.get_field("date", "Date").click(); + cy.get(".datepicker--nav-title").click(); + cy.get(".datepicker--nav-title").click({ force: true }); //Inputing values in the date field - cy.get('.datepicker--years > .datepicker--cells > .datepicker--cell[data-year=2020]').click(); - cy.get('.datepicker--months > .datepicker--cells > .datepicker--cell[data-month=0]').click(); - cy.get('.datepicker--days > .datepicker--cells > .datepicker--cell[data-date=15]').click(); + cy.get( + ".datepicker--years > .datepicker--cells > .datepicker--cell[data-year=2020]" + ).click(); + cy.get( + ".datepicker--months > .datepicker--cells > .datepicker--cell[data-month=0]" + ).click(); + cy.get(".datepicker--days > .datepicker--cells > .datepicker--cell[data-date=15]").click(); - //Verifying if the selected date is displayed in the date field - cy.get_field('date', 'Date').should('have.value', '01-15-2020'); + // Verify if the selected date is set the date field + cy.window().its("cur_dialog.fields_dict.date.value").should("be.equal", "2020-01-15"); }); - it('Checking next and previous button', () => { - cy.get_field('date', 'Date').click(); + it("Checking next and previous button", () => { + cy.clear_dialogs(); + cy.clear_datepickers(); + + get_dialog({ default: "2020-01-15" }).as("dialog"); + cy.get_field("date", "Date").click(); //Clicking on the next button in the datepicker - cy.get('.datepicker--nav-action[data-action=next]').click(); + cy.get(".datepicker--nav-action[data-action=next]").click(); //Selecting a date from the datepicker - cy.get('.datepicker--cell[data-date=15]').click({force: true}); + cy.get(".datepicker--cell[data-date=15]").click({ force: true }); //Verifying if the selected date has been displayed in the date field - cy.get_field('date', 'Date').should('have.value', '02-15-2020'); + cy.window().its("cur_dialog.fields_dict.date.value").should("be.equal", "2020-02-15"); cy.wait(500); - cy.get_field('date', 'Date').click(); + cy.get_field("date", "Date").click(); //Clicking on the previous button in the datepicker - cy.get('.datepicker--nav-action[data-action=prev]').click(); + cy.get(".datepicker--nav-action[data-action=prev]").click(); //Selecting a date from the datepicker - cy.get('.datepicker--cell[data-date=15]').click({force: true}); + cy.get(".datepicker--cell[data-date=15]").click({ force: true }); //Verifying if the selected date has been displayed in the date field - cy.get_field('date', 'Date').should('have.value', '01-15-2020'); + cy.window().its("cur_dialog.fields_dict.date.value").should("be.equal", "2020-01-15"); }); it('Clicking on "Today" button gives todays date', () => { - cy.get_field('date', 'Date').click(); + cy.clear_dialogs(); + cy.clear_datepickers(); + + get_dialog().as("dialog"); + cy.get_field("date", "Date").click(); //Clicking on "Today" button - cy.get('.datepicker--button').click(); - - //Picking up the todays date - const todays_date = Cypress.moment().format('MM-DD-YYYY'); + cy.get(".datepicker--button").click(); //Verifying if clicking on "Today" button matches today's date - cy.get_field('date', 'Date').should('have.value', todays_date); + cy.window().then((win) => { + expect(win.cur_dialog.fields_dict.date.value).to.be.equal( + win.frappe.datetime.get_today() + ); + }); }); -}); \ No newline at end of file +}); diff --git a/cypress/integration/control_date_range.js b/cypress/integration/control_date_range.js new file mode 100644 index 0000000000..f95a3825cc --- /dev/null +++ b/cypress/integration/control_date_range.js @@ -0,0 +1,48 @@ +context("Date Range Control", () => { + before(() => { + cy.login(); + cy.visit("/app"); + }); + + function get_dialog() { + return cy.dialog({ + title: "Date Range", + fields: [ + { + label: "Date Range", + fieldname: "date_range", + fieldtype: "Date Range", + }, + ], + }); + } + + it("Selecting a date range from the datepicker", () => { + cy.clear_dialogs(); + cy.clear_datepickers(); + + get_dialog().as("dialog"); + cy.get_field("date_range", "Date Range").click(); + cy.get(".datepicker--nav-title").click(); + cy.get(".datepicker--nav-title").click({ force: true }); + + //Inputing date range values in the date range field + cy.get( + ".datepicker--years > .datepicker--cells > .datepicker--cell[data-year=2020]" + ).click(); + cy.get( + ".datepicker--months > .datepicker--cells > .datepicker--cell[data-month=0]" + ).click(); + cy.get(".datepicker--cell[data-date=1]:first").click({ force: true }); + cy.get(".datepicker--cell[data-date=15]:first").click({ force: true }); + + // Verify if the selected date range values is set in the date range field + cy.window() + .its("cur_dialog") + .then((dialog) => { + let date_range = dialog.get_value("date_range"); + expect(date_range[0]).to.equal("2020-01-01"); + expect(date_range[1]).to.equal("2020-01-15"); + }); + }); +}); diff --git a/cypress/integration/control_duration.js b/cypress/integration/control_duration.js index 09629a344f..a391eec7c1 100644 --- a/cypress/integration/control_duration.js +++ b/cypress/integration/control_duration.js @@ -1,46 +1,46 @@ -context('Control Duration', () => { +context("Control Duration", () => { before(() => { cy.login(); - cy.visit('/app/website'); + cy.visit("/app/website"); }); function get_dialog_with_duration(hide_days = 0, hide_seconds = 0) { return cy.dialog({ - title: 'Duration', - fields: [{ - 'fieldname': 'duration', - 'fieldtype': 'Duration', - 'hide_days': hide_days, - 'hide_seconds': hide_seconds - }] + title: "Duration", + fields: [ + { + fieldname: "duration", + fieldtype: "Duration", + hide_days: hide_days, + hide_seconds: hide_seconds, + }, + ], }); } - it('should set duration', () => { - get_dialog_with_duration().as('dialog'); - cy.get('.frappe-control[data-fieldname=duration] input') - .first() - .click(); - cy.get('.duration-input[data-duration=days]') + it("should set duration", () => { + get_dialog_with_duration().as("dialog"); + cy.get(".frappe-control[data-fieldname=duration] input").first().click(); + cy.get(".duration-input[data-duration=days]") .type(45, { force: true }) .blur({ force: true }); - cy.get('.duration-input[data-duration=minutes]') - .type(30) - .blur({ force: true }); - cy.get('.frappe-control[data-fieldname=duration] input').first().should('have.value', '45d 30m'); - cy.get('.frappe-control[data-fieldname=duration] input').first().blur(); - cy.get('.duration-picker').should('not.be.visible'); - cy.get('@dialog').then(dialog => { - let value = dialog.get_value('duration'); + cy.get(".duration-input[data-duration=minutes]").type(30).blur({ force: true }); + cy.get(".frappe-control[data-fieldname=duration] input") + .first() + .should("have.value", "45d 30m"); + cy.get(".frappe-control[data-fieldname=duration] input").first().blur(); + cy.get(".duration-picker").should("not.be.visible"); + cy.get("@dialog").then((dialog) => { + let value = dialog.get_value("duration"); expect(value).to.equal(3889800); cy.hide_dialog(); }); }); - it('should hide days or seconds according to duration options', () => { - get_dialog_with_duration(1, 1).as('dialog'); - cy.get('.frappe-control[data-fieldname=duration] input').first(); - cy.get('.duration-input[data-duration=days]').should('not.be.visible'); - cy.get('.duration-input[data-duration=seconds]').should('not.be.visible'); + it("should hide days or seconds according to duration options", () => { + get_dialog_with_duration(1, 1).as("dialog"); + cy.get(".frappe-control[data-fieldname=duration] input").first(); + cy.get(".duration-input[data-duration=days]").should("not.be.visible"); + cy.get(".duration-input[data-duration=seconds]").should("not.be.visible"); }); -}); \ No newline at end of file +}); diff --git a/cypress/integration/control_dynamic_link.js b/cypress/integration/control_dynamic_link.js index cc1eb0b695..7f34f7ad42 100644 --- a/cypress/integration/control_dynamic_link.js +++ b/cypress/integration/control_dynamic_link.js @@ -1,128 +1,159 @@ -context('Dynamic Link', () => { +context("Dynamic Link", () => { before(() => { cy.login(); - cy.visit('/app/doctype'); - return cy.window().its('frappe').then(frappe => { - return frappe.xcall('frappe.tests.ui_test_helpers.create_doctype', { - name: 'Test Dynamic Link', - fields: [ - { - "label": "Document Type", - "fieldname": "doc_type", - "fieldtype": "Link", - "options": "DocType", - "in_list_view": 1, - "in_standard_filter": 1, - }, - { - "label": "Document ID", - "fieldname": "doc_id", - "fieldtype": "Dynamic Link", - "options": "doc_type", - "in_list_view": 1, - "in_standard_filter": 1, - }, - ] + cy.visit("/app/doctype"); + return cy + .window() + .its("frappe") + .then((frappe) => { + return frappe.xcall("frappe.tests.ui_test_helpers.create_doctype", { + name: "Test Dynamic Link", + fields: [ + { + label: "Document Type", + fieldname: "doc_type", + fieldtype: "Link", + options: "DocType", + in_list_view: 1, + in_standard_filter: 1, + }, + { + label: "Document ID", + fieldname: "doc_id", + fieldtype: "Dynamic Link", + options: "doc_type", + in_list_view: 1, + in_standard_filter: 1, + }, + ], + }); }); - }); }); - function get_dialog_with_dynamic_link() { return cy.dialog({ - title: 'Dynamic Link', - fields: [{ - "label": "Document Type", - "fieldname": "doc_type", - "fieldtype": "Link", - "options": "DocType", - "in_list_view": 1, - }, - { - "label": "Document ID", - "fieldname": "doc_id", - "fieldtype": "Dynamic Link", - "options": "doc_type", - "in_list_view": 1, - }] + title: "Dynamic Link", + fields: [ + { + label: "Document Type", + fieldname: "doc_type", + fieldtype: "Link", + options: "DocType", + in_list_view: 1, + }, + { + label: "Document ID", + fieldname: "doc_id", + fieldtype: "Dynamic Link", + options: "doc_type", + in_list_view: 1, + }, + ], }); } function get_dialog_with_dynamic_link_option() { return cy.dialog({ - title: 'Dynamic Link', - fields: [{ - "label": "Document Type", - "fieldname": "doc_type", - "fieldtype": "Link", - "options": "DocType", - "in_list_view": 1, - }, - { - "label": "Document ID", - "fieldname": "doc_id", - "fieldtype": "Dynamic Link", - "get_options": () => { - return "User"; + title: "Dynamic Link", + fields: [ + { + label: "Document Type", + fieldname: "doc_type", + fieldtype: "Link", + options: "DocType", + in_list_view: 1, }, - "in_list_view": 1, - }] + { + label: "Document ID", + fieldname: "doc_id", + fieldtype: "Dynamic Link", + get_options: () => { + return "User"; + }, + in_list_view: 1, + }, + ], }); } - it('Creating a dynamic link by passing option as function and verifying it in a dialog', () => { - get_dialog_with_dynamic_link_option().as('dialog'); - cy.get_field('doc_type').clear(); - cy.fill_field('doc_type', 'User', 'Link'); - cy.get_field('doc_id').click(); + it("Creating a dynamic link by passing option as function and verifying it in a dialog", () => { + get_dialog_with_dynamic_link_option().as("dialog"); + cy.get_field("doc_type").clear(); + cy.fill_field("doc_type", "User", "Link"); + cy.get_field("doc_id").click(); //Checking if the listbox have length greater than 0 - cy.get('[data-fieldname="doc_id"]').find('.awesomplete').find("li").its('length').should('be.gte', 0); - cy.get('.btn-modal-close').click({force: true}); + cy.get('[data-fieldname="doc_id"]') + .find(".awesomplete") + .find("li") + .its("length") + .should("be.gte", 0); + cy.get(".btn-modal-close").click({ force: true }); }); - it('Creating a dynamic link and verifying it in a dialog', () => { - get_dialog_with_dynamic_link().as('dialog'); - cy.get_field('doc_type').clear(); - cy.fill_field('doc_type', 'User', 'Link'); - cy.get_field('doc_id').click(); + it("Creating a dynamic link and verifying it in a dialog", () => { + get_dialog_with_dynamic_link().as("dialog"); + cy.get_field("doc_type").clear(); + cy.fill_field("doc_type", "User", "Link"); + cy.get_field("doc_id").click(); //Checking if the listbox have length greater than 0 - cy.get('[data-fieldname="doc_id"]').find('.awesomplete').find("li").its('length').should('be.gte', 0); - cy.get('.btn-modal-close').click({force: true, multiple: true}); + cy.get('[data-fieldname="doc_id"]') + .find(".awesomplete") + .find("li") + .its("length") + .should("be.gte", 0); + cy.get(".btn-modal-close").click({ force: true, multiple: true }); }); - it('Creating a dynamic link and verifying it', () => { - cy.visit('/app/test-dynamic-link'); + it("Creating a dynamic link and verifying it", () => { + cy.visit("/app/test-dynamic-link"); //Clicking on the Document ID field - cy.get_field('doc_type').clear(); + cy.get_field("doc_type").clear(); //Entering User in the Doctype field - cy.fill_field('doc_type', 'User', 'Link', {delay: 500}); - cy.get_field('doc_id').click(); + cy.fill_field("doc_type", "User", "Link", { delay: 500 }); + cy.get_field("doc_id").click(); //Checking if the listbox have length greater than 0 - cy.get('[data-fieldname="doc_id"]').find('.awesomplete').find("li").its('length').should('be.gte', 0); + cy.get('[data-fieldname="doc_id"]') + .find(".awesomplete") + .find("li") + .its("length") + .should("be.gte", 0); //Opening a new form for dynamic link doctype - cy.new_form('Test Dynamic Link'); - cy.get_field('doc_type').clear(); + cy.new_form("Test Dynamic Link"); + cy.get_field("doc_type").clear(); //Entering User in the Doctype field - cy.fill_field('doc_type', 'User', 'Link', {delay: 500}); - cy.get_field('doc_id').click(); + cy.fill_field("doc_type", "User", "Link", { delay: 500 }); + cy.get_field("doc_id").click(); //Checking if the listbox have length greater than 0 - cy.get('[data-fieldname="doc_id"]').find('.awesomplete').find("li").its('length').should('be.gte', 0); - cy.get_field('doc_type').clear(); + cy.get('[data-fieldname="doc_id"]') + .find(".awesomplete") + .find("li") + .its("length") + .should("be.gte", 0); + cy.get_field("doc_type").clear(); //Entering System Settings in the Doctype field - cy.fill_field('doc_type', 'System Settings', 'Link', {delay: 500}); - cy.get_field('doc_id').click(); + cy.intercept("/api/method/frappe.desk.search.search_link").as("search_query"); + cy.fill_field("doc_type", "System Settings", "Link", { delay: 500 }); + cy.wait("@search_query"); + cy.get(`[data-fieldname="doc_type"] ul:visible li:first-child`).click({ + scrollBehavior: false, + }); + + cy.get_field("doc_id").click(); //Checking if the system throws error - cy.get('.modal-title').should('have.text', 'Error'); - cy.get('.msgprint').should('have.text', 'System Settings is not a valid DocType for Dynamic Link'); + cy.get(".modal-title").should("have.text", "Error"); + cy.get(".msgprint").should( + "have.text", + "System Settings is not a valid DocType for Dynamic Link" + ); }); -}); \ No newline at end of file +}); diff --git a/cypress/integration/control_float.js b/cypress/integration/control_float.js index 670d1fe73e..c8261ad043 100644 --- a/cypress/integration/control_float.js +++ b/cypress/integration/control_float.js @@ -11,9 +11,9 @@ context("Control Float", () => { { fieldname: "float_number", fieldtype: "Float", - Label: "Float" - } - ] + Label: "Float", + }, + ], }); } @@ -21,27 +21,21 @@ context("Control Float", () => { get_dialog_with_float().as("dialog"); let data = get_data(); - data.forEach(x => { + data.forEach((x) => { cy.window() .its("frappe") - .then(frappe => { + .then((frappe) => { frappe.boot.sysdefaults.number_format = x.number_format; }); - x.values.forEach(d => { + x.values.forEach((d) => { cy.get_field("float_number", "Float").clear(); cy.fill_field("float_number", d.input, "Float").blur(); - cy.get_field("float_number", "Float").should( - "have.value", - d.blur_expected - ); + cy.get_field("float_number", "Float").should("have.value", d.blur_expected); cy.get_field("float_number", "Float").focus(); cy.get_field("float_number", "Float").blur(); cy.get_field("float_number", "Float").focus(); - cy.get_field("float_number", "Float").should( - "have.value", - d.focus_expected - ); + cy.get_field("float_number", "Float").should("have.value", d.focus_expected); }); }); }); @@ -54,19 +48,19 @@ context("Control Float", () => { { input: "364.87,334", blur_expected: "36.487,334", - focus_expected: "36487.334" + focus_expected: "36487.334", }, { input: "36487,334", blur_expected: "36.487,334", - focus_expected: "36487.334" + focus_expected: "36487.334", }, { input: "100", blur_expected: "100,000", - focus_expected: "100" - } - ] + focus_expected: "100", + }, + ], }, { number_format: "#,###.##", @@ -74,20 +68,20 @@ context("Control Float", () => { { input: "364,87.334", blur_expected: "36,487.334", - focus_expected: "36487.334" + focus_expected: "36487.334", }, { input: "36487.334", blur_expected: "36,487.334", - focus_expected: "36487.334" + focus_expected: "36487.334", }, { input: "100", blur_expected: "100.000", - focus_expected: "100" - } - ] - } + focus_expected: "100", + }, + ], + }, ]; } }); diff --git a/cypress/integration/control_icon.js b/cypress/integration/control_icon.js index d89eba8840..a965ed0f9e 100644 --- a/cypress/integration/control_icon.js +++ b/cypress/integration/control_icon.js @@ -1,50 +1,55 @@ -context('Control Icon', () => { +context("Control Icon", () => { before(() => { cy.login(); - cy.visit('/app/website'); + cy.visit("/app/website"); }); function get_dialog_with_icon() { return cy.dialog({ - title: 'Icon', - fields: [{ - label: 'Icon', - fieldname: 'icon', - fieldtype: 'Icon' - }] + title: "Icon", + fields: [ + { + label: "Icon", + fieldname: "icon", + fieldtype: "Icon", + }, + ], }); } - it('should set icon', () => { - get_dialog_with_icon().as('dialog'); - cy.get('.frappe-control[data-fieldname=icon]').findByRole('textbox').click(); + it("should set icon", () => { + get_dialog_with_icon().as("dialog"); + cy.get(".frappe-control[data-fieldname=icon]").findByRole("textbox").click(); - cy.get('.icon-picker .icon-wrapper[id=heart-active]').first().click(); - cy.get('.frappe-control[data-fieldname=icon]').findByRole('textbox').should('have.value', 'heart-active'); - cy.get('@dialog').then(dialog => { - let value = dialog.get_value('icon'); - expect(value).to.equal('heart-active'); + cy.get(".icon-picker .icon-wrapper[id=heart-active]").first().click(); + cy.get(".frappe-control[data-fieldname=icon]") + .findByRole("textbox") + .should("have.value", "heart-active"); + cy.get("@dialog").then((dialog) => { + let value = dialog.get_value("icon"); + expect(value).to.equal("heart-active"); }); - cy.get('.icon-picker .icon-wrapper[id=heart]').first().click(); - cy.get('.frappe-control[data-fieldname=icon]').findByRole('textbox').should('have.value', 'heart'); - cy.get('@dialog').then(dialog => { - let value = dialog.get_value('icon'); - expect(value).to.equal('heart'); + cy.get(".icon-picker .icon-wrapper[id=heart]").first().click(); + cy.get(".frappe-control[data-fieldname=icon]") + .findByRole("textbox") + .should("have.value", "heart"); + cy.get("@dialog").then((dialog) => { + let value = dialog.get_value("icon"); + expect(value).to.equal("heart"); }); }); - it('search for icon and clear search input', () => { - let search_text = 'ed'; - cy.get('.icon-picker').findByRole('searchbox').click().type(search_text); - cy.get('.icon-section .icon-wrapper:not(.hidden)').then(i => { - cy.get(`.icon-section .icon-wrapper[id*='${search_text}']`).then(icons => { + it("search for icon and clear search input", () => { + let search_text = "ed"; + cy.get(".icon-picker").findByRole("searchbox").click().type(search_text); + cy.get(".icon-section .icon-wrapper:not(.hidden)").then((i) => { + cy.get(`.icon-section .icon-wrapper[id*='${search_text}']`).then((icons) => { expect(i.length).to.equal(icons.length); }); }); - cy.get('.icon-picker').findByRole('searchbox').clear().blur(); - cy.get('.icon-section .icon-wrapper').should('not.have.class', 'hidden'); + cy.get(".icon-picker").findByRole("searchbox").clear().blur(); + cy.get(".icon-section .icon-wrapper").should("not.have.class", "hidden"); }); - -}); \ No newline at end of file +}); diff --git a/cypress/integration/control_link.js b/cypress/integration/control_link.js index 7a7e94d2f5..b34414e5ca 100644 --- a/cypress/integration/control_link.js +++ b/cypress/integration/control_link.js @@ -1,61 +1,101 @@ -context('Control Link', () => { +context("Control Link", () => { before(() => { cy.login(); - cy.visit('/app/website'); + cy.visit("/app/website"); }); beforeEach(() => { - cy.visit('/app/website'); + cy.visit("/app/website"); cy.create_records({ - doctype: 'ToDo', - description: 'this is a test todo for link' - }).as('todos'); + doctype: "ToDo", + description: "this is a test todo for link", + }).as("todos"); }); function get_dialog_with_link() { return cy.dialog({ - title: 'Link', + title: "Link", fields: [ { - 'label': 'Select ToDo', - 'fieldname': 'link', - 'fieldtype': 'Link', - 'options': 'ToDo' - } - ] + label: "Select ToDo", + fieldname: "link", + fieldtype: "Link", + options: "ToDo", + }, + ], }); } - it('should set the valid value', () => { - get_dialog_with_link().as('dialog'); + function get_dialog_with_user_link() { + return cy.dialog({ + title: "Link", + fields: [ + { + label: "Select User", + fieldname: "link", + fieldtype: "Link", + options: "User", + }, + ], + }); + } - cy.intercept('POST', '/api/method/frappe.desk.search.search_link').as('search_link'); + it("should set the valid value", () => { + get_dialog_with_link().as("dialog"); - cy.get('.frappe-control[data-fieldname=link] input').focus().as('input'); - cy.wait('@search_link'); - cy.get('@input').type('todo for link', { delay: 200 }); - cy.wait('@search_link'); - cy.get('.frappe-control[data-fieldname=link]').findByRole('listbox').should('be.visible'); - cy.get('.frappe-control[data-fieldname=link] input').type('{enter}', { delay: 100 }); - cy.get('.frappe-control[data-fieldname=link] input').blur(); - cy.get('@dialog').then(dialog => { - cy.get('@todos').then(todos => { - let value = dialog.get_value('link'); + cy.insert_doc( + "Property Setter", + { + doctype: "Property Setter", + doc_type: "User", + property: "translate_link_fields", + property_type: "Check", + doctype_or_field: "DocType", + value: "0", + }, + true + ); + + cy.insert_doc( + "Property Setter", + { + doctype: "Property Setter", + doc_type: "ToDo", + property: "show_title_field_in_link", + property_type: "Check", + doctype_or_field: "DocType", + value: "0", + }, + true + ); + + cy.intercept("POST", "/api/method/frappe.desk.search.search_link").as("search_link"); + + cy.get(".frappe-control[data-fieldname=link] input").focus().as("input"); + cy.wait("@search_link"); + cy.get("@input").type("todo for link", { delay: 200 }); + cy.wait("@search_link"); + cy.get(".frappe-control[data-fieldname=link]").findByRole("listbox").should("be.visible"); + cy.get(".frappe-control[data-fieldname=link] input").type("{enter}", { delay: 100 }); + cy.get(".frappe-control[data-fieldname=link] input").blur(); + cy.get("@dialog").then((dialog) => { + cy.get("@todos").then((todos) => { + let value = dialog.get_value("link"); expect(value).to.eq(todos[0]); }); }); }); - it('should unset invalid value', () => { - get_dialog_with_link().as('dialog'); + it("should unset invalid value", () => { + get_dialog_with_link().as("dialog"); - cy.intercept('POST', '/api/method/frappe.client.validate_link').as('validate_link'); + cy.intercept("POST", "/api/method/frappe.client.validate_link").as("validate_link"); - cy.get('.frappe-control[data-fieldname=link] input') - .type('invalid value', { delay: 100 }) + cy.get(".frappe-control[data-fieldname=link] input") + .type("invalid value", { delay: 100 }) .blur(); - cy.wait('@validate_link'); - cy.get('.frappe-control[data-fieldname=link] input').should('have.value', ''); + cy.wait("@validate_link"); + cy.get(".frappe-control[data-fieldname=link] input").should("have.value", ""); }); it("should be possible set empty value explicitly", () => { @@ -63,155 +103,325 @@ context('Control Link', () => { cy.intercept("POST", "/api/method/frappe.client.validate_link").as("validate_link"); - cy.get(".frappe-control[data-fieldname=link] input") - .type(" ", { delay: 100 }) - .blur(); + cy.get(".frappe-control[data-fieldname=link] input").type(" ", { delay: 100 }).blur(); cy.wait("@validate_link"); cy.get(".frappe-control[data-fieldname=link] input").should("have.value", ""); cy.window() .its("cur_dialog") .then((dialog) => { - expect(dialog.get_value("link")).to.equal(''); + expect(dialog.get_value("link")).to.equal(""); }); }); - it('should route to form on arrow click', () => { - get_dialog_with_link().as('dialog'); + it("should route to form on arrow click", () => { + get_dialog_with_link().as("dialog"); - cy.intercept('POST', '/api/method/frappe.client.validate_link').as('validate_link'); - cy.intercept('POST', '/api/method/frappe.desk.search.search_link').as('search_link'); + cy.intercept("POST", "/api/method/frappe.client.validate_link").as("validate_link"); + cy.intercept("POST", "/api/method/frappe.desk.search.search_link").as("search_link"); - cy.get('@todos').then(todos => { - cy.get('.frappe-control[data-fieldname=link] input').as('input'); - cy.get('@input').focus(); - cy.wait('@search_link'); - cy.get('@input').type(todos[0]).blur(); - cy.wait('@validate_link'); - cy.get('@input').focus(); - cy.findByTitle('Open Link') - .should('be.visible') - .click(); - cy.location('pathname').should('eq', `/app/todo/${todos[0]}`); + cy.get("@todos").then((todos) => { + cy.get(".frappe-control[data-fieldname=link] input").as("input"); + cy.get("@input").focus(); + cy.wait("@search_link"); + cy.get("@input").type(todos[0]).blur(); + cy.wait("@validate_link"); + cy.get("@input").focus(); + cy.wait(500); // wait for arrow to show + cy.get(".frappe-control[data-fieldname=link] .btn-open").should("be.visible").click(); + cy.location("pathname").should("eq", `/app/todo/${todos[0]}`); }); }); - it('show title field in link', () => { - get_dialog_with_link().as('dialog'); + it("show title field in link", () => { + cy.insert_doc( + "Property Setter", + { + doctype: "Property Setter", + doc_type: "User", + property: "translate_link_fields", + property_type: "Check", + doctype_or_field: "DocType", + value: "0", + }, + true + ); - cy.insert_doc("Property Setter", { - "doctype": "Property Setter", - "doc_type": "ToDo", - "property": "show_title_field_in_link", - "property_type": "Check", - "doctype_or_field": "DocType", - "value": "1" - }, true); + cy.insert_doc( + "Property Setter", + { + doctype: "Property Setter", + doc_type: "ToDo", + property: "show_title_field_in_link", + property_type: "Check", + doctype_or_field: "DocType", + value: "1", + }, + true + ); - cy.window().its('frappe').then(frappe => { - if (!frappe.boot) { - frappe.boot = { - link_title_doctypes: ['ToDo'] - }; - } else { - frappe.boot.link_title_doctypes = ['ToDo']; - } - }); + cy.clear_cache(); + cy.wait(500); - cy.intercept('POST', '/api/method/frappe.desk.search.search_link').as('search_link'); + get_dialog_with_link().as("dialog"); + cy.window() + .its("frappe") + .then((frappe) => { + if (!frappe.boot) { + frappe.boot = { + link_title_doctypes: ["ToDo"], + }; + } else { + frappe.boot.link_title_doctypes = ["ToDo"]; + } + }); - cy.get('.frappe-control[data-fieldname=link] input').focus().as('input'); - cy.wait('@search_link'); - cy.get('@input').type('todo for link'); - cy.wait('@search_link'); - cy.get('.frappe-control[data-fieldname=link] ul').should('be.visible'); - cy.get('.frappe-control[data-fieldname=link] input').type('{enter}', { delay: 100 }); - cy.get('.frappe-control[data-fieldname=link] input').blur(); - cy.get('@dialog').then(dialog => { - cy.get('@todos').then(todos => { - let field = dialog.get_field('link'); + cy.intercept("POST", "/api/method/frappe.desk.search.search_link").as("search_link"); + + cy.get(".frappe-control[data-fieldname=link] input").focus().as("input"); + cy.wait("@search_link"); + cy.get("@input").type("todo for link"); + cy.wait("@search_link"); + cy.get(".frappe-control[data-fieldname=link] ul").should("be.visible"); + cy.get(".frappe-control[data-fieldname=link] input").type("{enter}", { delay: 100 }); + cy.get(".frappe-control[data-fieldname=link] input").blur(); + cy.get("@dialog").then((dialog) => { + cy.get("@todos").then((todos) => { + let field = dialog.get_field("link"); let value = field.get_value(); let label = field.get_label_value(); expect(value).to.eq(todos[0]); - expect(label).to.eq('this is a test todo for link'); - - cy.remove_doc("Property Setter", "ToDo-main-show_title_field_in_link"); + expect(label).to.eq("this is a test todo for link"); }); }); }); - it('should update dependant fields (via fetch_from)', () => { - cy.get('@todos').then(todos => { + it("should update dependant fields (via fetch_from)", () => { + cy.get("@todos").then((todos) => { cy.visit(`/app/todo/${todos[0]}`); - cy.intercept('POST', '/api/method/frappe.client.validate_link').as('validate_link'); + cy.intercept("POST", "/api/method/frappe.desk.search.search_link").as("search_link"); + cy.intercept("POST", "/api/method/frappe.client.validate_link").as("validate_link"); - cy.get('.frappe-control[data-fieldname=assigned_by] input').focus().as('input'); - cy.get('@input').type('Administrator', {delay: 100}).blur(); - cy.wait('@validate_link'); - cy.get('.frappe-control[data-fieldname=assigned_by_full_name] .control-value').should( - 'contain', 'Administrator' + cy.get(".frappe-control[data-fieldname=assigned_by] input").focus().as("input"); + cy.get("@input").type("Administrator", { delay: 100 }).blur(); + cy.wait("@validate_link"); + cy.get(".frappe-control[data-fieldname=assigned_by_full_name] .control-value").should( + "contain", + "Administrator" ); - cy.window() - .its("cur_frm.doc.assigned_by") - .should("eq", "Administrator"); + cy.window().its("cur_frm.doc.assigned_by").should("eq", "Administrator"); // invalid input - cy.get('@input').clear().type('invalid input', {delay: 100}).blur(); - cy.get('.frappe-control[data-fieldname=assigned_by_full_name] .control-value').should( - 'contain', '' + cy.get("@input").clear().type("invalid input", { delay: 100 }).blur(); + cy.get(".frappe-control[data-fieldname=assigned_by_full_name] .control-value").should( + "contain", + "" ); - cy.window() - .its("cur_frm.doc.assigned_by") - .should("eq", null); + cy.window().its("cur_frm.doc.assigned_by").should("eq", null); // set valid value again - cy.get('@input').clear().type('Administrator', {delay: 100}).blur(); - cy.wait('@validate_link'); + cy.get("@input").clear().focus(); + cy.wait("@search_link"); + cy.get("@input").type("Administrator", { delay: 100 }).blur(); + cy.wait("@validate_link"); - cy.window() - .its("cur_frm.doc.assigned_by") - .should("eq", "Administrator"); + cy.window().its("cur_frm.doc.assigned_by").should("eq", "Administrator"); // clear input - cy.get('@input').clear().blur(); - cy.get('.frappe-control[data-fieldname=assigned_by_full_name] .control-value').should( - 'contain', '' + cy.get("@input").clear().blur(); + cy.get(".frappe-control[data-fieldname=assigned_by_full_name] .control-value").should( + "contain", + "" ); - cy.window() - .its("cur_frm.doc.assigned_by") - .should("eq", ""); + cy.window().its("cur_frm.doc.assigned_by").should("eq", ""); }); }); it("should set default values", () => { - cy.insert_doc("Property Setter", { - "doctype_or_field": "DocField", - "doc_type": "ToDo", - "field_name": "assigned_by", - "property": "default", - "property_type": "Text", - "value": "Administrator" - }, true); + cy.insert_doc( + "Property Setter", + { + doctype_or_field: "DocField", + doc_type: "ToDo", + field_name: "assigned_by", + property: "default", + property_type: "Text", + value: "Administrator", + }, + true + ); cy.reload(); cy.new_form("ToDo"); cy.fill_field("description", "new", "Text Editor"); cy.intercept("POST", "/api/method/frappe.desk.form.save.savedocs").as("save_form"); - cy.findByRole("button", {name: "Save"}).click(); + cy.findByRole("button", { name: "Save" }).click(); cy.wait("@save_form"); cy.get(".frappe-control[data-fieldname=assigned_by_full_name] .control-value").should( - "contain", "Administrator" + "contain", + "Administrator" ); // if user clears default value explicitly, system should not reset default again cy.get_field("assigned_by").clear().blur(); cy.intercept("POST", "/api/method/frappe.desk.form.save.savedocs").as("save_form"); - cy.findByRole("button", {name: "Save"}).click(); + cy.findByRole("button", { name: "Save" }).click(); cy.wait("@save_form"); cy.get_field("assigned_by").should("have.value", ""); cy.get(".frappe-control[data-fieldname=assigned_by_full_name] .control-value").should( - "contain", "" + "contain", + "" ); }); + + it("show translated text for link with show_title_field_in_link enabled", () => { + cy.insert_doc( + "Property Setter", + { + doctype: "Property Setter", + doc_type: "ToDo", + property: "translate_link_fields", + property_type: "Check", + doctype_or_field: "DocType", + value: "1", + }, + true + ); + + cy.insert_doc( + "Property Setter", + { + doctype: "Property Setter", + doc_type: "ToDo", + property: "show_title_field_in_link", + property_type: "Check", + doctype_or_field: "DocType", + value: "1", + }, + true + ); + + cy.window() + .its("frappe") + .then((frappe) => { + cy.insert_doc("Translation", { + doctype: "Translation", + language: frappe.boot.lang, + source_text: "this is a test todo for link", + translated_text: "this is a translated test todo for link", + }); + }); + + cy.clear_cache(); + cy.wait(500); + + cy.window() + .its("frappe") + .then((frappe) => { + if (!frappe.boot) { + frappe.boot = { + link_title_doctypes: ["ToDo"], + translatable_doctypes: ["ToDo"], + }; + } else { + frappe.boot.link_title_doctypes = ["ToDo"]; + frappe.boot.translatable_doctypes = ["ToDo"]; + } + }); + + get_dialog_with_link().as("dialog"); + cy.intercept("POST", "/api/method/frappe.desk.search.search_link").as("search_link"); + + cy.get(".frappe-control[data-fieldname=link] input").focus().as("input"); + cy.wait("@search_link"); + cy.get("@input").type("todo for link", { delay: 100 }); + cy.wait("@search_link"); + cy.get(".frappe-control[data-fieldname=link] ul").should("be.visible"); + cy.get(".frappe-control[data-fieldname=link] input").type("{enter}", { delay: 100 }); + cy.get(".frappe-control[data-fieldname=link] input").blur(); + cy.get("@dialog").then((dialog) => { + cy.get("@todos").then((todos) => { + let field = dialog.get_field("link"); + let value = field.get_value(); + let label = field.get_label_value(); + + expect(value).to.eq(todos[0]); + expect(label).to.eq("this is a translated test todo for link"); + }); + }); + }); + + it("show translated text for link with show_title_field_in_link disabled", () => { + cy.insert_doc( + "Property Setter", + { + doctype: "Property Setter", + doc_type: "User", + property: "translate_link_fields", + property_type: "Check", + doctype_or_field: "DocType", + value: "1", + }, + true + ); + + cy.insert_doc( + "Property Setter", + { + doctype: "Property Setter", + doc_type: "ToDo", + property: "show_title_field_in_link", + property_type: "Check", + doctype_or_field: "DocType", + value: "0", + }, + true + ); + + cy.window() + .its("frappe") + .then((frappe) => { + cy.insert_doc("Translation", { + doctype: "Translation", + language: frappe.boot.lang, + source_text: "test@erpnext.com", + translated_text: "translatedtest@erpnext.com", + }); + }); + + cy.clear_cache(); + cy.wait(500); + + cy.window() + .its("frappe") + .then((frappe) => { + if (!frappe.boot) { + frappe.boot = { + translatable_doctypes: ["User"], + }; + } else { + frappe.boot.translatable_doctypes = ["User"]; + } + }); + + get_dialog_with_user_link().as("dialog"); + cy.intercept("POST", "/api/method/frappe.desk.search.search_link").as("search_link"); + + cy.get(".frappe-control[data-fieldname=link] input").focus().as("input"); + cy.wait("@search_link"); + cy.get("@input").type("test@erpnext.com", { delay: 100 }); + cy.wait("@search_link"); + cy.get(".frappe-control[data-fieldname=link] ul").should("be.visible"); + cy.get(".frappe-control[data-fieldname=link] input").type("{enter}", { delay: 100 }); + cy.get(".frappe-control[data-fieldname=link] input").blur(); + cy.get("@dialog").then((dialog) => { + let field = dialog.get_field("link"); + let value = field.get_value(); + let label = field.get_label_value(); + + expect(value).to.eq("test@erpnext.com"); + expect(label).to.eq("translatedtest@erpnext.com"); + }); + }); }); diff --git a/cypress/integration/control_markdown_editor.js b/cypress/integration/control_markdown_editor.js new file mode 100644 index 0000000000..16c3dac51f --- /dev/null +++ b/cypress/integration/control_markdown_editor.js @@ -0,0 +1,19 @@ +context("Control Markdown Editor", () => { + before(() => { + cy.login(); + cy.visit("/app"); + }); + + it("should allow inserting images by drag and drop", () => { + cy.visit("/app/web-page/new"); + cy.fill_field("content_type", "Markdown", "Select"); + cy.get_field("main_section_md", "Markdown Editor").attachFile("sample_image.jpg", { + subjectType: "drag-n-drop", + }); + cy.click_modal_primary_button("Upload"); + cy.get_field("main_section_md", "Markdown Editor").should( + "contain", + "![](/private/files/sample_image" + ); + }); +}); diff --git a/cypress/integration/control_phone.js b/cypress/integration/control_phone.js new file mode 100644 index 0000000000..b56343c2d8 --- /dev/null +++ b/cypress/integration/control_phone.js @@ -0,0 +1,92 @@ +import doctype_with_phone from "../fixtures/doctype_with_phone"; + +context("Control Phone", () => { + before(() => { + cy.login(); + cy.visit("/app/website"); + }); + + function get_dialog_with_phone() { + return cy.dialog({ + title: "Phone", + fields: [ + { + fieldname: "phone", + fieldtype: "Phone", + }, + ], + }); + } + + it("should set flag and data", () => { + get_dialog_with_phone().as("dialog"); + cy.get(".selected-phone").click(); + cy.get(".phone-picker .phone-wrapper[id='afghanistan']").click(); + cy.get(".selected-phone").click(); + cy.get(".phone-picker .phone-wrapper[id='india']").click(); + cy.get(".selected-phone .country").should("have.text", "+91"); + cy.get(".selected-phone > img").should("have.attr", "src").and("include", "/in.svg"); + + let phone_number = "9312672712"; + cy.get(".selected-phone > img").click().first(); + cy.get_field("phone").first().click({ multiple: true }); + cy.get(".frappe-control[data-fieldname=phone]") + .findByRole("textbox") + .first() + .type(phone_number, { force: true }); + + cy.get_field("phone").first().should("have.value", phone_number); + cy.get_field("phone").first().blur({ force: true }); + cy.wait(100); + cy.get("@dialog").then((dialog) => { + let value = dialog.get_value("phone"); + expect(value).to.equal("+91-" + phone_number); + }); + }); + + it("case insensitive search for country and clear search", () => { + let search_text = "india"; + cy.get(".selected-phone").click().first(); + cy.get(".phone-picker").findByRole("searchbox").click().type(search_text); + cy.get(".phone-section .phone-wrapper:not(.hidden)").then((i) => { + cy.get(`.phone-section .phone-wrapper[id*="${search_text.toLowerCase()}"]`).then( + (countries) => { + expect(i.length).to.equal(countries.length); + } + ); + }); + + cy.get(".phone-picker").findByRole("searchbox").clear().blur(); + cy.get(".phone-section .phone-wrapper").should("not.have.class", "hidden"); + }); + + it("existing document should render phone field with data", () => { + cy.visit("/app/doctype"); + cy.insert_doc("DocType", doctype_with_phone, true); + cy.clear_cache(); + + // Creating custom doctype + cy.insert_doc("DocType", doctype_with_phone, true); + cy.visit("/app/doctype-with-phone"); + cy.click_listview_primary_button("Add Doctype With Phone"); + + // create a record + cy.fill_field("title", "Test Phone 1"); + cy.fill_field("phone", "+91-9823341234"); + cy.get_field("phone").should("have.value", "9823341234"); + cy.click_doc_primary_button("Save"); + cy.get_doc("Doctype With Phone", "Test Phone 1").then((doc) => { + let value = doc.data.phone; + expect(value).to.equal("+91-9823341234"); + }); + + // open the doc from list view + cy.go_to_list("Doctype With Phone"); + cy.clear_cache(); + cy.click_listview_row_item(0); + cy.title().should("eq", "Test Phone 1"); + cy.get(".selected-phone .country").should("have.text", "+91"); + cy.get(".selected-phone > img").should("have.attr", "src").and("include", "/in.svg"); + cy.get_field("phone").should("have.value", "9823341234"); + }); +}); diff --git a/cypress/integration/control_rating.js b/cypress/integration/control_rating.js index 15c11b352b..613a6e9f92 100644 --- a/cypress/integration/control_rating.js +++ b/cypress/integration/control_rating.js @@ -1,56 +1,54 @@ -context('Control Rating', () => { +context("Control Rating", () => { before(() => { cy.login(); - cy.visit('/app/website'); + cy.visit("/app/website"); }); function get_dialog_with_rating() { return cy.dialog({ - title: 'Rating', - fields: [{ - 'fieldname': 'rate', - 'fieldtype': 'Rating', - 'options': 7 - }] + title: "Rating", + fields: [ + { + fieldname: "rate", + fieldtype: "Rating", + options: 7, + }, + ], }); } - it('click on the star rating to record value', () => { - get_dialog_with_rating().as('dialog'); + it("click on the star rating to record value", () => { + get_dialog_with_rating().as("dialog"); - cy.get('div.rating') - .children('svg') - .find('.right-half') + cy.get("div.rating") + .children("svg") + .find(".right-half") .first() .click() - .should('have.class', 'star-click'); - cy.get('@dialog').then(dialog => { - var value = dialog.get_value('rate'); - expect(value).to.equal(1/7); + .should("have.class", "star-click"); + cy.get("@dialog").then((dialog) => { + var value = dialog.get_value("rate"); + expect(value).to.equal(1 / 7); dialog.hide(); }); }); - it('hover on the star', () => { + it("hover on the star", () => { get_dialog_with_rating(); - cy.get('div.rating') - .children('svg') - .find('.right-half') + cy.get("div.rating") + .children("svg") + .find(".right-half") .first() - .invoke('trigger', 'mouseenter') - .should('have.class', 'star-hover') - .invoke('trigger', 'mouseleave') - .should('not.have.class', 'star-hover'); + .invoke("trigger", "mouseenter") + .should("have.class", "star-hover") + .invoke("trigger", "mouseleave") + .should("not.have.class", "star-hover"); }); - it('check number of stars in rating', () => { + it("check number of stars in rating", () => { get_dialog_with_rating(); - cy.get('div.rating') - .first() - .children('svg') - .should('have.length', 7); + cy.get("div.rating").first().children("svg").should("have.length", 7); }); - }); diff --git a/cypress/integration/control_select.js b/cypress/integration/control_select.js index 8e18d21260..5f7a07e0c4 100644 --- a/cypress/integration/control_select.js +++ b/cypress/integration/control_select.js @@ -1,37 +1,40 @@ -context('Control Select', () => { +context("Control Select", () => { before(() => { cy.login(); - cy.visit('/app/website'); + cy.visit("/app/website"); }); function get_dialog_with_select() { return cy.dialog({ - title: 'Select', - fields: [{ - 'fieldname': 'select_control', - 'fieldtype': 'Select', - 'placeholder': 'Select an Option', - 'options': ['', 'Option 1', 'Option 2', 'Option 2'], - }] + title: "Select", + fields: [ + { + fieldname: "select_control", + fieldtype: "Select", + placeholder: "Select an Option", + options: ["", "Option 1", "Option 2", "Option 2"], + }, + ], }); } - it('toggles placholder on clicking an option', () => { - get_dialog_with_select().as('dialog'); + it("toggles placholder on clicking an option", () => { + get_dialog_with_select().as("dialog"); - cy.get('.frappe-control[data-fieldname=select_control] .control-input').as('control'); - cy.get('.frappe-control[data-fieldname=select_control] .control-input select').as('select'); - cy.get('@control').get('.select-icon').should('exist'); - cy.get('@control').get('.placeholder').should('have.css', 'display', 'block'); - cy.get('@select').select('Option 1'); - cy.findByDisplayValue('Option 1').should('exist'); - cy.get('@control').get('.placeholder').should('have.css', 'display', 'none'); - cy.get('@select').invoke('val', ''); - cy.findByDisplayValue('Option 1').should('not.exist'); - cy.get('@control').get('.placeholder').should('have.css', 'display', 'block'); + cy.get(".frappe-control[data-fieldname=select_control] .control-input").as("control"); + cy.get(".frappe-control[data-fieldname=select_control] .control-input select").as( + "select" + ); + cy.get("@control").get(".select-icon").should("exist"); + cy.get("@control").get(".placeholder").should("have.css", "display", "block"); + cy.get("@select").select("Option 1"); + cy.findByDisplayValue("Option 1").should("exist"); + cy.get("@control").get(".placeholder").should("have.css", "display", "none"); + cy.get("@select").invoke("val", ""); + cy.findByDisplayValue("Option 1").should("not.exist"); + cy.get("@control").get(".placeholder").should("have.css", "display", "block"); - - cy.get('@dialog').then(dialog => { + cy.get("@dialog").then((dialog) => { dialog.hide(); }); }); diff --git a/cypress/integration/custom_buttons.js b/cypress/integration/custom_buttons.js new file mode 100644 index 0000000000..ddbd19731a --- /dev/null +++ b/cypress/integration/custom_buttons.js @@ -0,0 +1,57 @@ +const test_button_names = [ + "Metallica", + "Pink Floyd", + "Porcupine Tree (the GOAT)", + "AC / DC", + `Electronic Dance "music"`, + "l'imperatrice", +]; + +const add_button = (label, group = "TestGroup") => { + cy.window() + .its("cur_frm") + .then((frm) => { + frm.add_custom_button(label, () => {}, group); + }); +}; + +const check_button_count = (label, group = "TestGroup") => { + // Verify main buttons + cy.findByRole("button", { name: group }).click(); + cy.get(`[data-label="${encodeURIComponent(label)}"]`) + .should("have.length", 1) + .should("be.visible"); + + // Verify dropdown buttons in mobile view + cy.viewport(420, 900); + const dropdown_btn_label = `${group} > ${label}`; + cy.get(".menu-btn-group > .btn").click(); + cy.get(`[data-label="${encodeURIComponent(dropdown_btn_label)}"]`) + .should("have.length", 1) + .should("be.visible"); + + //reset viewport + cy.viewport(Cypress.config("viewportWidth"), Cypress.config("viewportHeight")); +}; + +describe( + "Custom group button behaviour on desk", + { scrollBehavior: false }, // speeds up the test + () => { + before(() => { + cy.login(); + cy.visit(`/app/note/new`); + }); + + test_button_names.forEach((button_name) => { + it(`Custom button works with name '${button_name}'`, () => { + add_button(button_name); + check_button_count(button_name); + + // duplicate button shouldn't be added + add_button(button_name); + check_button_count(button_name); + }); + }); + } +); diff --git a/cypress/integration/customize_form.js b/cypress/integration/customize_form.js new file mode 100644 index 0000000000..cd03f7b54c --- /dev/null +++ b/cypress/integration/customize_form.js @@ -0,0 +1,23 @@ +context("Customize Form", () => { + before(() => { + cy.login(); + cy.visit("/app/customize-form"); + }); + it("Changing to naming rule should update autoname", () => { + cy.fill_field("doc_type", "ToDo", "Link").blur(); + cy.click_form_section("Naming"); + const naming_rule_default_autoname_map = { + "Set by user": "prompt", + "By fieldname": "field:", + 'By "Naming Series" field': "naming_series:", + Expression: "format:", + "Expression (old style)": "", + Random: "hash", + "By script": "", + }; + Cypress._.forOwn(naming_rule_default_autoname_map, (value, naming_rule) => { + cy.fill_field("naming_rule", naming_rule, "Select"); + cy.get_field("autoname", "Data").should("have.value", value); + }); + }); +}); diff --git a/cypress/integration/dashboard_chart.js b/cypress/integration/dashboard_chart.js index ae71fcda3a..6023a50abe 100644 --- a/cypress/integration/dashboard_chart.js +++ b/cypress/integration/dashboard_chart.js @@ -1,22 +1,22 @@ -context('Dashboard Chart', () => { +context("Dashboard Chart", () => { before(() => { cy.login(); - cy.visit('/app/website'); + cy.visit("/app/website"); }); - it('Check filter populate for child table doctype', () => { - cy.visit('/app/dashboard-chart/new-dashboard-chart-1'); - cy.get('[data-fieldname="parent_document_type"]').should('have.css', 'display', 'none'); + it("Check filter populate for child table doctype", () => { + cy.visit("/app/dashboard-chart/new-dashboard-chart-1"); + cy.get('[data-fieldname="parent_document_type"]').should("have.css", "display", "none"); - cy.get_field('document_type', 'Link'); - cy.fill_field('document_type', 'Workspace Link', 'Link').focus().blur(); - cy.get_field('document_type', 'Link').should('have.value', 'Workspace Link'); + cy.get_field("document_type", "Link"); + cy.fill_field("document_type", "Workspace Link", "Link").focus().blur(); + cy.get_field("document_type", "Link").should("have.value", "Workspace Link"); - cy.fill_field('chart_name', 'Test Chart', 'Data'); + cy.fill_field("chart_name", "Test Chart", "Data"); cy.get('[data-fieldname="filters_json"]').click().wait(200); - cy.get('.modal-body .filter-action-buttons .add-filter').click(); - cy.get('.modal-body .fieldname-select-area').click(); - cy.get('.modal-actions .btn-modal-close').click(); + cy.get(".modal-body .filter-action-buttons .add-filter").click(); + cy.get(".modal-body .fieldname-select-area").click(); + cy.get(".modal-actions .btn-modal-close").click(); }); -}); \ No newline at end of file +}); diff --git a/cypress/integration/dashboard_links.js b/cypress/integration/dashboard_links.js index 019de1991d..31572b7976 100644 --- a/cypress/integration/dashboard_links.js +++ b/cypress/integration/dashboard_links.js @@ -1,91 +1,94 @@ -import doctype_with_child_table from '../fixtures/doctype_with_child_table'; -import child_table_doctype from '../fixtures/child_table_doctype'; -import child_table_doctype_1 from '../fixtures/child_table_doctype_1'; -import doctype_to_link from '../fixtures/doctype_to_link'; +import doctype_with_child_table from "../fixtures/doctype_with_child_table"; +import child_table_doctype from "../fixtures/child_table_doctype"; +import child_table_doctype_1 from "../fixtures/child_table_doctype_1"; +import doctype_to_link from "../fixtures/doctype_to_link"; const doctype_to_link_name = doctype_to_link.name; const child_table_doctype_name = child_table_doctype.name; -context('Dashboard links', () => { +context("Dashboard links", () => { before(() => { - cy.visit('/login'); + cy.visit("/login"); cy.login(); - cy.insert_doc('DocType', child_table_doctype, true); - cy.insert_doc('DocType', child_table_doctype_1, true); - cy.insert_doc('DocType', doctype_with_child_table, true); - cy.insert_doc('DocType', doctype_to_link, true); - return cy.window().its('frappe').then(frappe => { - return frappe.xcall("frappe.tests.ui_test_helpers.update_child_table", { - name: child_table_doctype_name + cy.insert_doc("DocType", child_table_doctype, true); + cy.insert_doc("DocType", child_table_doctype_1, true); + cy.insert_doc("DocType", doctype_with_child_table, true); + cy.insert_doc("DocType", doctype_to_link, true); + return cy + .window() + .its("frappe") + .then((frappe) => { + return frappe.xcall("frappe.tests.ui_test_helpers.update_child_table", { + name: child_table_doctype_name, + }); }); - }); }); - it('Adding a new contact, checking for the counter on the dashboard and deleting the created contact', () => { - cy.visit('/app/contact'); + it("Adding a new contact, checking for the counter on the dashboard and deleting the created contact", () => { + cy.visit("/app/contact"); cy.clear_filters(); - cy.visit('/app/user'); - cy.get('.list-row-col > .level-item > .ellipsis').eq(0).click({ force: true }); + cy.visit("/app/user"); + cy.get(".list-row-col > .level-item > .ellipsis").eq(0).click({ force: true }); //To check if initially the dashboard contains only the "Contact" link and there is no counter - cy.get('[data-doctype="Contact"]').should('contain', 'Contact'); + cy.get('[data-doctype="Contact"]').should("contain", "Contact"); //Adding a new contact cy.get('.document-link-badge[data-doctype="Contact"]').click(); cy.wait(300); - cy.findByRole('button', {name: 'Add Contact'}).should('be.visible'); - cy.findByRole('button', {name: 'Add Contact'}).click(); - cy.get('[data-doctype="Contact"][data-fieldname="first_name"]').type('Admin'); - cy.findByRole('button', {name: 'Save'}).click(); - cy.visit('/app/user'); - cy.get('.list-row-col > .level-item > .ellipsis').eq(0).click({ force: true }); + cy.findByRole("button", { name: "Add Contact" }).should("be.visible"); + cy.findByRole("button", { name: "Add Contact" }).click(); + cy.get('[data-doctype="Contact"][data-fieldname="first_name"]').type("Admin"); + cy.findByRole("button", { name: "Save" }).click(); + cy.visit("/app/user"); + cy.get(".list-row-col > .level-item > .ellipsis").eq(0).click({ force: true }); //To check if the counter for contact doc is "1" after adding the contact - cy.get('[data-doctype="Contact"] > .count').should('contain', '1'); - cy.get('[data-doctype="Contact"]').contains('Contact').click(); + cy.get('[data-doctype="Contact"] > .count').should("contain", "1"); + cy.get('[data-doctype="Contact"]').contains("Contact").click(); //Deleting the newly created contact - cy.visit('/app/contact'); - cy.get('.list-subject > .select-like > .list-row-checkbox').eq(0).click({ force: true }); - cy.findByRole('button', {name: 'Actions'}).click(); + cy.visit("/app/contact"); + cy.get(".list-subject > .select-like > .list-row-checkbox").eq(0).click({ force: true }); + cy.findByRole("button", { name: "Actions" }).click(); cy.get('.actions-btn-group [data-label="Delete"]').click(); - cy.findByRole('button', {name: 'Yes'}).click({delay: 700}); - + cy.findByRole("button", { name: "Yes" }).click({ delay: 700 }); //To check if the counter from the "Contact" doc link is removed cy.wait(700); - cy.visit('/app/user'); - cy.get('.list-row-col > .level-item > .ellipsis').eq(0).click({ force: true }); - cy.get('[data-doctype="Contact"]').should('contain', 'Contact'); + cy.visit("/app/user"); + cy.get(".list-row-col > .level-item > .ellipsis").eq(0).click({ force: true }); + cy.get('[data-doctype="Contact"]').should("contain", "Contact"); }); - it('Report link in dashboard', () => { - cy.visit('/app/user'); - cy.visit('/app/user/Administrator'); - cy.get('[data-doctype="Contact"]').should('contain', 'Contact'); - cy.findByText('Connections'); + it("Report link in dashboard", () => { + cy.visit("/app/user"); + cy.visit("/app/user/Administrator"); + cy.get('[data-doctype="Contact"]').should("contain", "Contact"); + cy.findByText("Connections"); cy.window() - .its('cur_frm') - .then(cur_frm => { + .its("cur_frm") + .then((cur_frm) => { cur_frm.dashboard.data.reports = [ { - 'label': 'Reports', - 'items': ['Website Analytics'] - } + label: "Reports", + items: ["Website Analytics"], + }, ]; cur_frm.dashboard.render_report_links(); - cy.get('[data-report="Website Analytics"]').contains('Website Analytics').click(); - cy.findByText('Website Analytics'); + cy.get('[data-report="Website Analytics"]').contains("Website Analytics").click(); + cy.findByText("Website Analytics"); }); }); - it('check if child table is populated with linked field on creation from dashboard link', () => { + it("check if child table is populated with linked field on creation from dashboard link", () => { cy.new_form(doctype_to_link_name); cy.fill_field("title", "Test Linking"); - cy.findByRole("button", {name: "Save"}).click(); + cy.findByRole("button", { name: "Save" }).click(); - cy.get('.document-link .btn-new').click(); - cy.get('.frappe-control[data-fieldname="child_table"] .rows .data-row .col[data-fieldname="doctype_to_link"]') - .should('contain.text', 'Test Linking'); + cy.get(".document-link .btn-new").click(); + cy.get( + '.frappe-control[data-fieldname="child_table"] .rows .data-row .col[data-fieldname="doctype_to_link"]' + ).should("contain.text", "Test Linking"); }); }); diff --git a/cypress/integration/data_field_form_validation.js b/cypress/integration/data_field_form_validation.js index c6feea5550..49513e72fb 100644 --- a/cypress/integration/data_field_form_validation.js +++ b/cypress/integration/data_field_form_validation.js @@ -1,43 +1,45 @@ -import data_field_validation_doctype from '../fixtures/data_field_validation_doctype'; +import data_field_validation_doctype from "../fixtures/data_field_validation_doctype"; const doctype_name = data_field_validation_doctype.name; - -context('Data Field Input Validation in New Form', () => { +context("Data Field Input Validation in New Form", () => { before(() => { cy.login(); - cy.visit('/app/website'); - return cy.insert_doc('DocType', data_field_validation_doctype, true); + cy.visit("/app/website"); + return cy.insert_doc("DocType", data_field_validation_doctype, true); }); function validateField(fieldname, invalid_value, valid_value) { // Invalid, should have has-error class cy.get_field(fieldname).clear().type(invalid_value).blur(); - cy.get(`.frappe-control[data-fieldname="${fieldname}"]`).should('have.class', 'has-error'); + cy.get(`.frappe-control[data-fieldname="${fieldname}"]`).should("have.class", "has-error"); // Valid value, should not have has-error class cy.get_field(fieldname).clear().type(valid_value); - cy.get(`.frappe-control[data-fieldname="${fieldname}"]`).should('not.have.class', 'has-error'); + cy.get(`.frappe-control[data-fieldname="${fieldname}"]`).should( + "not.have.class", + "has-error" + ); } - describe('Data Field Options', () => { - it('should validate email address', () => { + describe("Data Field Options", () => { + it("should validate email address", () => { cy.new_form(doctype_name); - validateField('email', 'captian', 'hello@test.com'); + validateField("email", "captian", "hello@test.com"); }); - it('should validate URL', () => { - validateField('url', 'jkl', 'https://frappe.io'); - validateField('url', 'abcd.com', 'http://google.com/home'); - validateField('url', '&&http://google.uae', 'gopher://frappe.io'); - validateField('url', 'ftt2:://google.in?q=news', 'ftps2://frappe.io/__/#home'); - validateField('url', 'ftt2://', 'ntps://localhost'); // For intranet URLs + it("should validate URL", () => { + validateField("url", "jkl", "https://frappe.io"); + validateField("url", "abcd.com", "http://google.com/home"); + validateField("url", "&&http://google.uae", "gopher://frappe.io"); + validateField("url", "ftt2:://google.in?q=news", "ftps2://frappe.io/__/#home"); + validateField("url", "ftt2://", "ntps://localhost"); // For intranet URLs }); - it('should validate phone number', () => { - validateField('phone', 'america', '89787878'); + it("should validate phone number", () => { + validateField("phone", "america", "89787878"); }); - it('should validate name', () => { - validateField('person_name', ' 777Hello', 'James Bond'); + it("should validate name", () => { + validateField("person_name", " 777Hello", "James Bond"); }); }); -}); \ No newline at end of file +}); diff --git a/cypress/integration/datetime.js b/cypress/integration/datetime.js index 4a24faf40b..7a8a68c1d9 100644 --- a/cypress/integration/datetime.js +++ b/cypress/integration/datetime.js @@ -1,53 +1,52 @@ -import datetime_doctype from '../fixtures/datetime_doctype'; +import datetime_doctype from "../fixtures/datetime_doctype"; const doctype_name = datetime_doctype.name; -context('Control Date, Time and DateTime', () => { +context("Control Date, Time and DateTime", () => { before(() => { cy.login(); - cy.visit('/app/website'); - return cy.insert_doc('DocType', datetime_doctype, true); + cy.visit("/app/website"); + return cy.insert_doc("DocType", datetime_doctype, true); }); - describe('Date formats', () => { + describe("Date formats", () => { let date_formats = [ { - date_format: 'dd-mm-yyyy', + date_format: "dd-mm-yyyy", part: 2, length: 4, - separator: '-' + separator: "-", }, { - date_format: 'mm/dd/yyyy', + date_format: "mm/dd/yyyy", part: 0, length: 2, - separator: '/' - } + separator: "/", + }, ]; - date_formats.forEach(d => { - it('test date format ' + d.date_format, () => { - cy.set_value('System Settings', 'System Settings', { - date_format: d.date_format + date_formats.forEach((d) => { + it("test date format " + d.date_format, () => { + cy.set_value("System Settings", "System Settings", { + date_format: d.date_format, }); cy.window() - .its('frappe') - .then(frappe => { + .its("frappe") + .then((frappe) => { // update sys_defaults value to avoid a reload frappe.sys_defaults.date_format = d.date_format; }); cy.new_form(doctype_name); - cy.get('.form-control[data-fieldname=date]').focus(); - cy.get('.datepickers-container .datepicker.active') - .should('be.visible'); + cy.get(".form-control[data-fieldname=date]").focus(); + cy.get(".datepickers-container .datepicker.active").should("be.visible"); cy.get( - '.datepickers-container .datepicker.active .datepicker--cell-day.-current-' + ".datepickers-container .datepicker.active .datepicker--cell-day.-current-" ).click({ force: true }); cy.window() - .its('cur_frm') - .then(cur_frm => { - let formatted_value = cur_frm.get_field('date').input.value; + .its("cur_frm") + .then((cur_frm) => { + let formatted_value = cur_frm.get_field("date").input.value; let parts = formatted_value.split(d.separator); expect(parts[d.part].length).to.equal(d.length); }); @@ -55,74 +54,72 @@ context('Control Date, Time and DateTime', () => { }); }); - describe('Time formats', () => { + describe("Time formats", () => { let time_formats = [ { - time_format: 'HH:mm:ss', - value: ' 11:00:12', - match_value: '11:00:12' + time_format: "HH:mm:ss", + value: " 11:00:12", + match_value: "11:00:12", }, { - time_format: 'HH:mm', - value: ' 11:00:12', - match_value: '11:00' - } + time_format: "HH:mm", + value: " 11:00:12", + match_value: "11:00", + }, ]; - time_formats.forEach(d => { - it('test time format ' + d.time_format, () => { - cy.set_value('System Settings', 'System Settings', { - time_format: d.time_format + time_formats.forEach((d) => { + it("test time format " + d.time_format, () => { + cy.set_value("System Settings", "System Settings", { + time_format: d.time_format, }); cy.window() - .its('frappe') - .then(frappe => { + .its("frappe") + .then((frappe) => { frappe.sys_defaults.time_format = d.time_format; }); cy.new_form(doctype_name); - cy.fill_field('time', d.value, 'Time').blur(); - cy.get_field('time').should('have.value', d.match_value); + cy.fill_field("time", d.value, "Time").blur(); + cy.get_field("time").should("have.value", d.match_value); }); }); }); - describe('DateTime formats', () => { + describe("DateTime formats", () => { let datetime_formats = [ { - date_format: 'dd.mm.yyyy', - time_format: 'HH:mm:ss', - value: ' 02.12.2019 11:00:12', - doc_value: '2019-12-02 00:30:12', // system timezone (America/New_York) - input_value: '02.12.2019 11:00:12' // admin timezone (Asia/Kolkata) + date_format: "dd.mm.yyyy", + time_format: "HH:mm:ss", + value: " 02.12.2019 11:00:12", + doc_value: "2019-12-02 00:30:12", // system timezone (America/New_York) + input_value: "02.12.2019 11:00:12", // admin timezone (Asia/Kolkata) }, { - date_format: 'mm-dd-yyyy', - time_format: 'HH:mm', - value: ' 12-02-2019 11:00:00', - doc_value: '2019-12-02 00:30:00', // system timezone (America/New_York) - input_value: '12-02-2019 11:00' // admin timezone (Asia/Kolkata) - } + date_format: "mm-dd-yyyy", + time_format: "HH:mm", + value: " 12-02-2019 11:00:00", + doc_value: "2019-12-02 00:30:00", // system timezone (America/New_York) + input_value: "12-02-2019 11:00", // admin timezone (Asia/Kolkata) + }, ]; - datetime_formats.forEach(d => { + datetime_formats.forEach((d) => { it(`test datetime format ${d.date_format} ${d.time_format}`, () => { - cy.set_value('System Settings', 'System Settings', { + cy.set_value("System Settings", "System Settings", { date_format: d.date_format, - time_format: d.time_format + time_format: d.time_format, }); cy.window() - .its('frappe') - .then(frappe => { + .its("frappe") + .then((frappe) => { frappe.sys_defaults.date_format = d.date_format; frappe.sys_defaults.time_format = d.time_format; }); cy.new_form(doctype_name); - cy.fill_field('datetime', d.value, 'Datetime').blur(); - cy.get_field('datetime').should('have.value', d.input_value); + cy.fill_field("datetime", d.value, "Datetime").blur(); + cy.get_field("datetime").should("have.value", d.input_value); - cy.window() - .its('cur_frm.doc.datetime') - .should('eq', d.doc_value); + cy.window().its("cur_frm.doc.datetime").should("eq", d.doc_value); }); }); }); diff --git a/cypress/integration/datetime_field_form_validation.js b/cypress/integration/datetime_field_form_validation.js index ef47a0fbf7..1a549d8a1d 100644 --- a/cypress/integration/datetime_field_form_validation.js +++ b/cypress/integration/datetime_field_form_validation.js @@ -16,4 +16,4 @@ // cy.get('.indicator-pill').should('contain', 'Open').should('have.class', 'red'); // }); // }); -// }); \ No newline at end of file +// }); diff --git a/cypress/integration/depends_on.js b/cypress/integration/depends_on.js index 12f54f2b6e..6419809466 100644 --- a/cypress/integration/depends_on.js +++ b/cypress/integration/depends_on.js @@ -1,135 +1,152 @@ -context('Depends On', () => { +context("Depends On", () => { before(() => { cy.login(); - cy.visit('/app/website'); - return cy.window().its('frappe').then(frappe => { - return frappe.xcall('frappe.tests.ui_test_helpers.create_child_doctype', { - name: 'Child Test Depends On', - fields: [ - { - "label": "Child Test Field", - "fieldname": "child_test_field", - "fieldtype": "Data", - "in_list_view": 1, - }, - { - "label": "Child Dependant Field", - "fieldname": "child_dependant_field", - "fieldtype": "Data", - "in_list_view": 1, - }, - { - "label": "Child Display Dependant Field", - "fieldname": "child_display_dependant_field", - "fieldtype": "Data", - "in_list_view": 1, - }, - ] + cy.visit("/app/website"); + return cy + .window() + .its("frappe") + .then((frappe) => { + return frappe.xcall("frappe.tests.ui_test_helpers.create_child_doctype", { + name: "Child Test Depends On", + fields: [ + { + label: "Child Test Field", + fieldname: "child_test_field", + fieldtype: "Data", + in_list_view: 1, + }, + { + label: "Child Dependant Field", + fieldname: "child_dependant_field", + fieldtype: "Data", + in_list_view: 1, + }, + { + label: "Child Display Dependant Field", + fieldname: "child_display_dependant_field", + fieldtype: "Data", + in_list_view: 1, + }, + ], + }); + }) + .then((frappe) => { + return frappe.xcall("frappe.tests.ui_test_helpers.create_doctype", { + name: "Test Depends On", + fields: [ + { + label: "Test Field", + fieldname: "test_field", + fieldtype: "Data", + }, + { + label: "Dependant Field", + fieldname: "dependant_field", + fieldtype: "Data", + mandatory_depends_on: "eval:doc.test_field=='Some Value'", + read_only_depends_on: "eval:doc.test_field=='Some Other Value'", + }, + { + label: "Display Dependant Field", + fieldname: "display_dependant_field", + fieldtype: "Data", + depends_on: "eval:doc.test_field=='Value'", + }, + { + label: "Child Test Depends On Field", + fieldname: "child_test_depends_on_field", + fieldtype: "Table", + read_only_depends_on: "eval:doc.test_field=='Some Other Value'", + options: "Child Test Depends On", + }, + { + label: "Dependent Tab", + fieldname: "dependent_tab", + fieldtype: "Tab Break", + depends_on: "eval:doc.test_field=='Show Tab'", + }, + { + fieldname: "tab_section", + fieldtype: "Section Break", + }, + { + label: "Field in Tab", + fieldname: "field_in_tab", + fieldtype: "Data", + }, + ], + }); }); - }).then(frappe => { - return frappe.xcall('frappe.tests.ui_test_helpers.create_doctype', { - name: 'Test Depends On', - fields: [ - { - "label": "Test Field", - "fieldname": "test_field", - "fieldtype": "Data", - }, - { - "label": "Dependant Field", - "fieldname": "dependant_field", - "fieldtype": "Data", - "mandatory_depends_on": "eval:doc.test_field=='Some Value'", - "read_only_depends_on": "eval:doc.test_field=='Some Other Value'", - }, - { - "label": "Display Dependant Field", - "fieldname": "display_dependant_field", - "fieldtype": "Data", - 'depends_on': "eval:doc.test_field=='Value'" - }, - { - "label": "Child Test Depends On Field", - "fieldname": "child_test_depends_on_field", - "fieldtype": "Table", - 'read_only_depends_on': "eval:doc.test_field=='Some Other Value'", - 'options': "Child Test Depends On" - }, - { - "label": "Dependent Tab", - "fieldname": "dependent_tab", - "fieldtype": "Tab Break", - "depends_on": "eval:doc.test_field=='Show Tab'" - }, - { - "fieldname": "tab_section", - "fieldtype": "Section Break", - }, - { - "label": "Field in Tab", - "fieldname": "field_in_tab", - "fieldtype": "Data", - } - ] - }); - }); }); - it('should show the tab on other setting field value', () => { - cy.new_form('Test Depends On'); - cy.fill_field('test_field', 'Show Tab'); - cy.get('body').click(); - cy.findByRole("tab", {name: "Dependent Tab"}).should('be.visible'); + it("should show the tab on other setting field value", () => { + cy.new_form("Test Depends On"); + cy.fill_field("test_field", "Show Tab"); + cy.get("body").click(); + cy.findByRole("tab", { name: "Dependent Tab" }).should("be.visible"); }); - it('should set the field as mandatory depending on other fields value', () => { - cy.new_form('Test Depends On'); - cy.fill_field('test_field', 'Some Value'); - cy.findByRole('button', {name: 'Save'}).click(); - cy.get('.msgprint-dialog .modal-title').contains('Missing Fields').should('be.visible'); + it("should set the field as mandatory depending on other fields value", () => { + cy.new_form("Test Depends On"); + cy.fill_field("test_field", "Some Value"); + cy.findByRole("button", { name: "Save" }).click(); + cy.get(".msgprint-dialog .modal-title").contains("Missing Fields").should("be.visible"); cy.hide_dialog(); - cy.fill_field('test_field', 'Random value'); - cy.findByRole('button', {name: 'Save'}).click(); - cy.get('.msgprint-dialog .modal-title').contains('Missing Fields').should('not.be.visible'); + cy.fill_field("test_field", "Random value"); + cy.findByRole("button", { name: "Save" }).click(); + cy.get(".msgprint-dialog .modal-title") + .contains("Missing Fields") + .should("not.be.visible"); }); - it('should set the field as read only depending on other fields value', () => { - cy.new_form('Test Depends On'); - cy.fill_field('dependant_field', 'Some Value'); - cy.fill_field('test_field', 'Some Other Value'); - cy.get('body').click(); - cy.get('.control-input [data-fieldname="dependant_field"]').should('be.disabled'); - cy.fill_field('test_field', 'Random Value'); - cy.get('body').click(); - cy.get('.control-input [data-fieldname="dependant_field"]').should('not.be.disabled'); + it("should set the field as read only depending on other fields value", () => { + cy.new_form("Test Depends On"); + cy.fill_field("dependant_field", "Some Value"); + cy.fill_field("test_field", "Some Other Value"); + cy.get("body").click(); + cy.get('.control-input [data-fieldname="dependant_field"]').should("be.disabled"); + cy.fill_field("test_field", "Random Value"); + cy.get("body").click(); + cy.get('.control-input [data-fieldname="dependant_field"]').should("not.be.disabled"); }); - it('should set the table and its fields as read only depending on other fields value', () => { - cy.new_form('Test Depends On'); - cy.fill_field('dependant_field', 'Some Value'); + it("should set the table and its fields as read only depending on other fields value", () => { + cy.new_form("Test Depends On"); + cy.fill_field("dependant_field", "Some Value"); //cy.fill_field('test_field', 'Some Other Value'); - cy.get('.frappe-control[data-fieldname="child_test_depends_on_field"]').as('table'); - cy.get('@table').findByRole('button', {name: 'Add Row'}).click(); - cy.get('@table').find('[data-idx="1"]').as('row1'); - cy.get('@row1').find('.btn-open-row').click(); - cy.get('@row1').find('.form-in-grid').as('row1-form_in_grid'); + cy.get('.frappe-control[data-fieldname="child_test_depends_on_field"]').as("table"); + cy.get("@table").findByRole("button", { name: "Add Row" }).click(); + cy.get("@table").find('[data-idx="1"]').as("row1"); + cy.get("@row1").find(".btn-open-row").click(); + cy.get("@row1").find(".form-in-grid").as("row1-form_in_grid"); //cy.get('@row1-form_in_grid').find('') - cy.fill_table_field('child_test_depends_on_field', '1', 'child_test_field', 'Some Value'); - cy.fill_table_field('child_test_depends_on_field', '1', 'child_dependant_field', 'Some Other Value'); + cy.fill_table_field("child_test_depends_on_field", "1", "child_test_field", "Some Value"); + cy.fill_table_field( + "child_test_depends_on_field", + "1", + "child_dependant_field", + "Some Other Value" + ); - cy.get('@row1-form_in_grid').find('.grid-collapse-row').click(); + cy.get("@row1-form_in_grid").find(".grid-collapse-row").click(); // set the table to read-only - cy.fill_field('test_field', 'Some Other Value'); + cy.fill_field("test_field", "Some Other Value"); // grid row form fields should be read-only - cy.get('@row1').find('.btn-open-row').click(); + cy.get("@row1").find(".btn-open-row").click(); - cy.get('@row1-form_in_grid').find('.control-input [data-fieldname="child_test_field"]').should('be.disabled'); - cy.get('@row1-form_in_grid').find('.control-input [data-fieldname="child_dependant_field"]').should('be.disabled'); + cy.get("@row1-form_in_grid") + .find('.control-input [data-fieldname="child_test_field"]') + .should("be.disabled"); + cy.get("@row1-form_in_grid") + .find('.control-input [data-fieldname="child_dependant_field"]') + .should("be.disabled"); }); - it('should display the field depending on other fields value', () => { - cy.new_form('Test Depends On'); - cy.get('.control-input [data-fieldname="display_dependant_field"]').should('not.be.visible'); + it("should display the field depending on other fields value", () => { + cy.new_form("Test Depends On"); + cy.get('.control-input [data-fieldname="display_dependant_field"]').should( + "not.be.visible" + ); cy.get('.control-input [data-fieldname="test_field"]').clear(); - cy.fill_field('test_field', 'Value'); - cy.get('body').click(); - cy.get('.control-input [data-fieldname="display_dependant_field"]').should('be.visible'); + cy.fill_field("test_field", "Value"); + cy.get("body").click(); + cy.get('.control-input [data-fieldname="display_dependant_field"]').should("be.visible"); }); }); diff --git a/cypress/integration/discussions.js b/cypress/integration/discussions.js index caf7d6c3f9..55bcabce19 100644 --- a/cypress/integration/discussions.js +++ b/cypress/integration/discussions.js @@ -1,79 +1,101 @@ -context('Discussions', () => { +context("Discussions", () => { before(() => { cy.login(); - cy.visit('/app'); - return cy.window().its('frappe').then(frappe => { - return frappe.call('frappe.tests.ui_test_helpers.create_data_for_discussions'); - }); + cy.visit("/app"); + return cy + .window() + .its("frappe") + .then((frappe) => { + return frappe.call("frappe.tests.ui_test_helpers.create_data_for_discussions"); + }); }); const reply_through_modal = () => { - cy.visit('/test-page-discussions'); + cy.visit("/test-page-discussions"); // Open the modal - cy.get('.reply').click(); + cy.get(".reply").click(); cy.wait(500); - cy.get('.discussion-modal').should('be.visible'); + cy.get(".discussion-modal").should("be.visible"); // Enter title - cy.get('.modal .topic-title').type('Discussion from tests') - .should('have.value', 'Discussion from tests'); + cy.get(".modal .topic-title") + .type("Discussion from tests") + .should("have.value", "Discussion from tests"); // Enter comment - cy.get('.modal .comment-field') - .type('This is a discussion from the cypress ui tests.') - .should('have.value', 'This is a discussion from the cypress ui tests.'); + cy.get(".modal .comment-field") + .type("This is a discussion from the cypress ui tests.") + .should("have.value", "This is a discussion from the cypress ui tests."); // Submit - cy.get('.modal .submit-discussion').click(); + cy.get(".modal .submit-discussion").click(); cy.wait(2000); // Check if discussion is added to page and content is visible - cy.get('.sidebar-parent:first .discussion-topic-title').should('have.text', 'Discussion from tests'); - cy.get('.discussion-on-page:visible').should('have.class', 'show'); - cy.get('.discussion-on-page:visible .reply-card .reply-text') - .should('have.text', 'This is a discussion from the cypress ui tests.\n'); - + cy.get(".sidebar-parent:first .discussion-topic-title").should( + "have.text", + "Discussion from tests" + ); + cy.get(".discussion-on-page:visible").should("have.class", "show"); + cy.get(".discussion-on-page:visible .reply-card .reply-text").should( + "have.text", + "This is a discussion from the cypress ui tests.\n" + ); }; const reply_through_comment_box = () => { - cy.get('.discussion-form:visible .comment-field') - .type('This is a discussion from the cypress ui tests. \n\nThis comment was entered through the commentbox on the page.') - .should('have.value', 'This is a discussion from the cypress ui tests. \n\nThis comment was entered through the commentbox on the page.'); + cy.get(".discussion-form:visible .comment-field") + .type( + "This is a discussion from the cypress ui tests. \n\nThis comment was entered through the commentbox on the page." + ) + .should( + "have.value", + "This is a discussion from the cypress ui tests. \n\nThis comment was entered through the commentbox on the page." + ); - cy.get('.discussion-form:visible .submit-discussion').click(); + cy.get(".discussion-form:visible .submit-discussion").click(); cy.wait(3000); - cy.get('.discussion-on-page:visible').should('have.class', 'show'); - cy.get('.discussion-on-page:visible').children(".reply-card").eq(1).find(".reply-text") - .should('have.text', 'This is a discussion from the cypress ui tests. \n\nThis comment was entered through the commentbox on the page.\n'); + cy.get(".discussion-on-page:visible").should("have.class", "show"); + cy.get(".discussion-on-page:visible") + .children(".reply-card") + .eq(1) + .find(".reply-text") + .should( + "have.text", + "This is a discussion from the cypress ui tests. \n\nThis comment was entered through the commentbox on the page.\n" + ); }; const cancel_and_clear_comment_box = () => { - cy.get('.discussion-form:visible .comment-field') - .type('This is a discussion from the cypress ui tests.') - .should('have.value', 'This is a discussion from the cypress ui tests.'); + cy.get(".discussion-form:visible .comment-field") + .type("This is a discussion from the cypress ui tests.") + .should("have.value", "This is a discussion from the cypress ui tests."); - cy.get('.discussion-form:visible .cancel-comment').click(); - cy.get('.discussion-form:visible .comment-field').should('have.value', ''); + cy.get(".discussion-form:visible .cancel-comment").click(); + cy.get(".discussion-form:visible .comment-field").should("have.value", ""); }; const single_thread_discussion = () => { - cy.visit('/test-single-thread'); - cy.get('.discussions-sidebar').should('have.length', 0); - cy.get('.reply').should('have.length', 0); + cy.visit("/test-single-thread"); + cy.get(".discussions-sidebar").should("have.length", 0); + cy.get(".reply").should("have.length", 0); - cy.get('.discussion-form:visible .comment-field') - .type('This comment is being made on a single thread discussion.') - .should('have.value', 'This comment is being made on a single thread discussion.'); + cy.get(".discussion-form:visible .comment-field") + .type("This comment is being made on a single thread discussion.") + .should("have.value", "This comment is being made on a single thread discussion."); - cy.get('.discussion-form:visible .submit-discussion').click(); + cy.get(".discussion-form:visible .submit-discussion").click(); cy.wait(3000); - cy.get('.discussion-on-page').children(".reply-card").eq(-1).find(".reply-text") - .should('have.text', 'This comment is being made on a single thread discussion.\n'); + cy.get(".discussion-on-page") + .children(".reply-card") + .eq(-1) + .find(".reply-text") + .should("have.text", "This comment is being made on a single thread discussion.\n"); }; - it('reply through modal', reply_through_modal); - it('reply through comment box', reply_through_comment_box); - it('cancel and clear comment box', cancel_and_clear_comment_box); - it('single thread discussion', single_thread_discussion); + it("reply through modal", reply_through_modal); + it("reply through comment box", reply_through_comment_box); + it("cancel and clear comment box", cancel_and_clear_comment_box); + it("single thread discussion", single_thread_discussion); }); diff --git a/cypress/integration/file_uploader.js b/cypress/integration/file_uploader.js index 3d4f92df3c..669f9ba385 100644 --- a/cypress/integration/file_uploader.js +++ b/cypress/integration/file_uploader.js @@ -1,78 +1,82 @@ -context('FileUploader', () => { +context("FileUploader", () => { before(() => { cy.login(); - cy.visit('/app'); + cy.visit("/app"); }); function open_upload_dialog() { - cy.window().its('frappe').then(frappe => { - new frappe.ui.FileUploader(); - }); + cy.window() + .its("frappe") + .then((frappe) => { + new frappe.ui.FileUploader(); + }); } - it('upload dialog api works', () => { + it("upload dialog api works", () => { open_upload_dialog(); - cy.get_open_dialog().should('contain', 'Drag and drop files'); + cy.get_open_dialog().should("contain", "Drag and drop files"); cy.hide_dialog(); }); - it('should accept dropped files', () => { + it("should accept dropped files", () => { open_upload_dialog(); - cy.get_open_dialog().find('.file-upload-area').attachFile('example.json', { - subjectType: 'drag-n-drop', + cy.get_open_dialog().find(".file-upload-area").attachFile("example.json", { + subjectType: "drag-n-drop", }); - cy.get_open_dialog().find('.file-name').should('contain', 'example.json'); - cy.intercept('POST', '/api/method/upload_file').as('upload_file'); - cy.get_open_dialog().findByRole('button', {name: 'Upload'}).click(); - cy.wait('@upload_file').its('response.statusCode').should('eq', 200); - cy.get('.modal:visible').should('not.exist'); + cy.get_open_dialog().find(".file-name").should("contain", "example.json"); + cy.intercept("POST", "/api/method/upload_file").as("upload_file"); + cy.get_open_dialog().findByRole("button", { name: "Upload" }).click(); + cy.wait("@upload_file").its("response.statusCode").should("eq", 200); + cy.get(".modal:visible").should("not.exist"); }); - it('should accept uploaded files', () => { + it("should accept uploaded files", () => { open_upload_dialog(); - cy.get_open_dialog().findByRole('button', {name: 'Library'}).click(); - cy.findByPlaceholderText('Search by filename or extension').type('example.json'); - cy.get_open_dialog().findAllByText('example.json').first().click(); - cy.intercept('POST', '/api/method/upload_file').as('upload_file'); - cy.get_open_dialog().findByRole('button', {name: 'Upload'}).click(); - cy.wait('@upload_file').its('response.body.message') - .should('have.property', 'file_name', 'example.json'); - cy.get('.modal:visible').should('not.exist'); + cy.get_open_dialog().findByRole("button", { name: "Library" }).click(); + cy.findByPlaceholderText("Search by filename or extension").type("example.json"); + cy.get_open_dialog().findAllByText("example.json").first().click(); + cy.intercept("POST", "/api/method/upload_file").as("upload_file"); + cy.get_open_dialog().findByRole("button", { name: "Upload" }).click(); + cy.wait("@upload_file") + .its("response.body.message") + .should("have.property", "file_name", "example.json"); + cy.get(".modal:visible").should("not.exist"); }); - it('should accept web links', () => { + it("should accept web links", () => { open_upload_dialog(); - cy.get_open_dialog().findByRole('button', {name: 'Link'}).click(); + cy.get_open_dialog().findByRole("button", { name: "Link" }).click(); cy.get_open_dialog() - .findByPlaceholderText('Attach a web link') - .type('https://github.com', { delay: 100, force: true }); - cy.intercept('POST', '/api/method/upload_file').as('upload_file'); - cy.get_open_dialog().findByRole('button', {name: 'Upload'}).click(); - cy.wait('@upload_file').its('response.body.message') - .should('have.property', 'file_url', 'https://github.com'); - cy.get('.modal:visible').should('not.exist'); + .findByPlaceholderText("Attach a web link") + .type("https://github.com", { delay: 100, force: true }); + cy.intercept("POST", "/api/method/upload_file").as("upload_file"); + cy.get_open_dialog().findByRole("button", { name: "Upload" }).click(); + cy.wait("@upload_file") + .its("response.body.message") + .should("have.property", "file_url", "https://github.com"); + cy.get(".modal:visible").should("not.exist"); }); - it('should allow cropping and optimization for valid images', () => { + it("should allow cropping and optimization for valid images", () => { open_upload_dialog(); - cy.get_open_dialog().find('.file-upload-area').attachFile('sample_image.jpg', { - subjectType: 'drag-n-drop', + cy.get_open_dialog().find(".file-upload-area").attachFile("sample_image.jpg", { + subjectType: "drag-n-drop", }); - cy.get_open_dialog().findAllByText('sample_image.jpg').should('exist'); - cy.get_open_dialog().find('.btn-crop').first().click(); - cy.get_open_dialog().findByRole('button', {name: 'Crop'}).click(); - cy.get_open_dialog().findAllByRole('checkbox', {name: 'Optimize'}).should('exist'); - cy.get_open_dialog().findAllByLabelText('Optimize').first().click(); + cy.get_open_dialog().findAllByText("sample_image.jpg").should("exist"); + cy.get_open_dialog().find(".btn-crop").first().click(); + cy.get_open_dialog().findByRole("button", { name: "Crop" }).click(); + cy.get_open_dialog().findAllByRole("checkbox", { name: "Optimize" }).should("exist"); + cy.get_open_dialog().findAllByLabelText("Optimize").first().click(); - cy.intercept('POST', '/api/method/upload_file').as('upload_file'); - cy.get_open_dialog().findByRole('button', {name: 'Upload'}).click(); - cy.wait('@upload_file').its('response.statusCode').should('eq', 200); - cy.get('.modal:visible').should('not.exist'); + cy.intercept("POST", "/api/method/upload_file").as("upload_file"); + cy.get_open_dialog().findByRole("button", { name: "Upload" }).click(); + cy.wait("@upload_file").its("response.statusCode").should("eq", 200); + cy.get(".modal:visible").should("not.exist"); }); }); diff --git a/cypress/integration/first_day_of_the_week.js b/cypress/integration/first_day_of_the_week.js index 1e65b78990..784c068f01 100644 --- a/cypress/integration/first_day_of_the_week.js +++ b/cypress/integration/first_day_of_the_week.js @@ -4,42 +4,48 @@ context("First Day of the Week", () => { }); beforeEach(() => { - cy.visit('/app/system-settings'); - cy.findByText('Date and Number Format').click(); + cy.visit("/app/system-settings"); + cy.findByText("Date and Number Format").click(); }); it("Date control starts with same day as selected in System Settings", () => { - cy.intercept('POST', '/api/method/frappe.core.doctype.system_settings.system_settings.load').as("load_settings"); - cy.fill_field('first_day_of_the_week', 'Tuesday', 'Select'); - cy.findByRole('button', {name: 'Save'}).click(); + cy.intercept( + "POST", + "/api/method/frappe.core.doctype.system_settings.system_settings.load" + ).as("load_settings"); + cy.fill_field("first_day_of_the_week", "Tuesday", "Select"); + cy.findByRole("button", { name: "Save" }).click(); cy.wait("@load_settings"); cy.dialog({ - title: 'Date', + title: "Date", fields: [ { - label: 'Date', - fieldname: 'date', - fieldtype: 'Date' - } - ] + label: "Date", + fieldname: "date", + fieldtype: "Date", + }, + ], }); - cy.get_field('date').click(); - cy.get('.datepicker--day-name').eq(0).should('have.text', 'Tu'); + cy.get_field("date").click(); + cy.get(".datepicker--day-name").eq(0).should("have.text", "Tu"); }); it("Calendar view starts with same day as selected in System Settings", () => { - cy.intercept('POST', '/api/method/frappe.core.doctype.system_settings.system_settings.load').as("load_settings"); - cy.fill_field('first_day_of_the_week', 'Monday', 'Select'); - cy.findByRole('button', {name: 'Save'}).click(); + cy.intercept( + "POST", + "/api/method/frappe.core.doctype.system_settings.system_settings.load" + ).as("load_settings"); + cy.fill_field("first_day_of_the_week", "Monday", "Select"); + cy.findByRole("button", { name: "Save" }).click(); cy.wait("@load_settings"); cy.visit("app/todo/view/calendar/default"); - cy.get('.fc-day-header > span').eq(0).should('have.text', 'Mon'); + cy.get(".fc-day-header > span").eq(0).should("have.text", "Mon"); }); after(() => { - cy.visit('/app/system-settings'); - cy.findByText('Date and Number Format').click(); - cy.fill_field('first_day_of_the_week', 'Sunday', 'Select'); - cy.findByRole('button', {name: 'Save'}).click(); + cy.visit("/app/system-settings"); + cy.findByText("Date and Number Format").click(); + cy.fill_field("first_day_of_the_week", "Sunday", "Select"); + cy.findByRole("button", { name: "Save" }).click(); }); -}); \ No newline at end of file +}); diff --git a/cypress/integration/folder_navigation.js b/cypress/integration/folder_navigation.js index cec7edb59f..e15a354de0 100644 --- a/cypress/integration/folder_navigation.js +++ b/cypress/integration/folder_navigation.js @@ -1,79 +1,85 @@ -context('Folder Navigation', () => { +context("Folder Navigation", () => { before(() => { - cy.visit('/login'); + cy.visit("/login"); cy.login(); - cy.visit('/app/file'); + cy.visit("/app/file"); }); - it('Adding Folders', () => { + it("Adding Folders", () => { //Adding filter to go into the home folder - cy.get('.filter-selector > .btn').findByText('1 filter').click(); - cy.findByRole('button', {name: 'Clear Filters'}).click(); - cy.get('.filter-action-buttons > .text-muted').findByText('+ Add a Filter').click(); - cy.get('.fieldname-select-area > .awesomplete > .form-control').type('Fol{enter}'); - cy.get('.filter-field > .form-group > .link-field > .awesomplete > .input-with-feedback').type('Home{enter}'); - cy.get('.filter-action-buttons > div > .btn-primary').findByText('Apply Filters').click(); + cy.get(".filter-selector > .btn").findByText("1 filter").click(); + cy.findByRole("button", { name: "Clear Filters" }).click(); + cy.get(".filter-action-buttons > .text-muted").findByText("+ Add a Filter").click(); + cy.get(".fieldname-select-area > .awesomplete > .form-control").type("Fol{enter}"); + cy.get( + ".filter-field > .form-group > .link-field > .awesomplete > .input-with-feedback" + ).type("Home{enter}"); + cy.get(".filter-action-buttons > div > .btn-primary").findByText("Apply Filters").click(); //Adding folder (Test Folder) - cy.get('.menu-btn-group > .btn').click(); - cy.get('.menu-btn-group [data-label="New Folder"]').click(); - cy.get('form > [data-fieldname="value"]').type('Test Folder'); - cy.findByRole('button', {name: 'Create'}).click(); + cy.click_menu_button("New Folder"); + cy.fill_field("value", "Test Folder"); + cy.click_modal_primary_button("Create"); }); - it('Navigating the nested folders, checking if the URL formed is correct, checking if the added content in the child folder is correct', () => { + it("Navigating the nested folders, checking if the URL formed is correct, checking if the added content in the child folder is correct", () => { //Navigating inside the Attachments folder cy.get('[title="Attachments"] > span').click(); //To check if the URL formed after visiting the attachments folder is correct - cy.location('pathname').should('eq', '/app/file/view/home/Attachments'); - cy.visit('/app/file/view/home/Attachments'); + cy.location("pathname").should("eq", "/app/file/view/home/Attachments"); + cy.visit("/app/file/view/home/Attachments"); //Adding folder inside the attachments folder - cy.get('.menu-btn-group > .btn').click(); - cy.get('.menu-btn-group [data-label="New Folder"]').click(); - cy.get('form > [data-fieldname="value"]').type('Test Folder'); - cy.findByRole('button', {name: 'Create'}).click(); + cy.click_menu_button("New Folder"); + cy.fill_field("value", "Test Folder"); + cy.click_modal_primary_button("Create"); //Navigating inside the added folder in the Attachments folder cy.get('[title="Test Folder"] > span').click(); //To check if the URL is correct after visiting the Test Folder - cy.location('pathname').should('eq', '/app/file/view/home/Attachments/Test%20Folder'); - cy.visit('/app/file/view/home/Attachments/Test%20Folder'); + cy.location("pathname").should("eq", "/app/file/view/home/Attachments/Test%20Folder"); + cy.visit("/app/file/view/home/Attachments/Test%20Folder"); //Adding a file inside the Test Folder - cy.findByRole('button', {name: 'Add File'}).eq(0).click({force: true}); - cy.get('.file-uploader').findByText('Link').click(); - cy.get('.input-group > .form-control').type('https://wallpaperplay.com/walls/full/8/2/b/72402.jpg'); - cy.findByRole('button', {name: 'Upload'}).click(); + cy.findByRole("button", { name: "Add File" }).eq(0).click({ force: true }); + cy.get(".file-uploader").findByText("Link").click(); + cy.get(".input-group > .form-control").type( + "https://wallpaperplay.com/walls/full/8/2/b/72402.jpg" + ); + cy.click_modal_primary_button("Upload"); //To check if the added file is present in the Test Folder - cy.get('span.level-item > span').should('contain', 'Test Folder'); - cy.get('.list-row-container').eq(0).should('contain.text', '72402.jpg'); - cy.get('.list-row-checkbox').eq(0).click(); + cy.get("span.level-item > span").should("contain", "Test Folder"); + cy.get(".list-row-container").eq(0).should("contain.text", "72402.jpg"); + cy.get(".list-row-checkbox").eq(0).click(); + + cy.intercept({ + method: "POST", + url: "api/method/frappe.desk.reportview.delete_items", + }).as("file_deleted"); //Deleting the added file from the Test folder - cy.findByRole('button', {name: 'Actions'}).click(); - cy.get('.actions-btn-group [data-label="Delete"]').click(); - cy.wait(700); - cy.findByRole('button', {name: 'Yes'}).click(); - cy.wait(700); + cy.click_action_button("Delete"); + cy.click_modal_primary_button("Yes"); + cy.wait("@file_deleted"); //Deleting the Test Folder - cy.visit('/app/file/view/home/Attachments'); - cy.get('.list-row-checkbox').eq(0).click(); - cy.findByRole('button', {name: 'Actions'}).click(); - cy.get('.actions-btn-group [data-label="Delete"]').click(); - cy.findByRole('button', {name: 'Yes'}).click(); + cy.visit("/app/file/view/home/Attachments"); + cy.get(".list-row-checkbox").eq(0).click(); + cy.click_action_button("Delete"); + cy.click_modal_primary_button("Yes"); + cy.wait("@file_deleted"); }); - it('Deleting Test Folder from the home', () => { - //Deleting the Test Folder added in the home directory - cy.visit('/app/file/view/home'); - cy.get('.level-left > .list-subject > .file-select >.list-row-checkbox').eq(0).click({force: true, delay: 500}); - cy.findByRole('button', {name: 'Actions'}).click(); - cy.get('.actions-btn-group [data-label="Delete"]').click(); - cy.findByRole('button', {name: 'Yes'}).click(); + it("Deleting Test Folder from the home", () => { + //Deleting the Test Folder added in the home directory + cy.visit("/app/file/view/home"); + cy.get(".level-left > .list-subject > .file-select >.list-row-checkbox") + .eq(0) + .click({ force: true, delay: 500 }); + cy.click_action_button("Delete"); + cy.click_modal_primary_button("Yes"); }); -}); +}); diff --git a/cypress/integration/form.js b/cypress/integration/form.js index acaff9a191..43ab5350b7 100644 --- a/cypress/integration/form.js +++ b/cypress/integration/form.js @@ -1,81 +1,170 @@ -context('Form', () => { +context("Form", () => { before(() => { cy.login(); - cy.visit('/app/website'); - return cy.window().its('frappe').then(frappe => { - return frappe.call("frappe.tests.ui_test_helpers.create_contact_records"); - }); + cy.visit("/app/website"); + return cy + .window() + .its("frappe") + .then((frappe) => { + return frappe.call("frappe.tests.ui_test_helpers.create_contact_records"); + }); }); - it('create a new form', () => { - cy.visit('/app/todo/new'); - cy.get_field('description', 'Text Editor').type('this is a test todo', {force: true}).wait(200); - cy.get('.page-title').should('contain', 'Not Saved'); + + it("create a new form", () => { + cy.visit("/app/todo/new"); + cy.get_field("description", "Text Editor") + .type("this is a test todo", { force: true }) + .wait(200); + cy.get(".page-title").should("contain", "Not Saved"); cy.intercept({ - method: 'POST', - url: 'api/method/frappe.desk.form.save.savedocs' - }).as('form_save'); - cy.get('.primary-action').click(); - cy.wait('@form_save').its('response.statusCode').should('eq', 200); + method: "POST", + url: "api/method/frappe.desk.form.save.savedocs", + }).as("form_save"); + cy.get(".primary-action").click(); + cy.wait("@form_save").its("response.statusCode").should("eq", 200); - cy.visit('/app/todo'); - cy.get('.page-head').findByTitle('To Do').should('exist'); - cy.get('.list-row').should('contain', 'this is a test todo'); + cy.go_to_list("ToDo"); + cy.clear_filters(); + cy.get(".page-head").findByTitle("To Do").should("exist"); + cy.get(".list-row").should("contain", "this is a test todo"); }); - it('navigates between documents with child table list filters applied', () => { - cy.visit('/app/contact'); + it("navigates between documents with child table list filters applied", () => { + cy.visit("/app/contact"); cy.clear_filters(); - cy.get('.standard-filter-section [data-fieldname="name"] input').type('Test Form Contact 3').blur(); - cy.click_listview_row_item(0); + cy.get('.standard-filter-section [data-fieldname="name"] input') + .type("Test Form Contact 3") + .blur(); + cy.click_listview_row_item_with_text("Test Form Contact 3"); - cy.get('#page-Contact .page-head').findByTitle('Test Form Contact 3').should('exist'); - cy.get('.prev-doc').should('be.visible').click(); - cy.get('.msgprint-dialog .modal-body').contains('No further records').should('be.visible'); + cy.get("#page-Contact .page-head").findByTitle("Test Form Contact 3").should("exist"); + cy.get(".prev-doc").should("be.visible").click(); + cy.get(".msgprint-dialog .modal-body").contains("No further records").should("be.visible"); cy.hide_dialog(); - cy.get('#page-Contact .page-head').findByTitle('Test Form Contact 3').should('exist'); - cy.get('.next-doc').should('be.visible').click(); - cy.get('.msgprint-dialog .modal-body').contains('No further records').should('be.visible'); + cy.get("#page-Contact .page-head").findByTitle("Test Form Contact 3").should("exist"); + cy.get(".next-doc").should("be.visible").click(); + cy.get(".msgprint-dialog .modal-body").contains("No further records").should("be.visible"); cy.hide_dialog(); - cy.get('#page-Contact .page-head').findByTitle('Test Form Contact 3').should('exist'); + cy.get("#page-Contact .page-head").findByTitle("Test Form Contact 3").should("exist"); // clear filters - cy.visit('/app/contact'); + cy.visit("/app/contact"); cy.clear_filters(); }); - it('validates behaviour of Data options validations in child table', () => { + it("validates behaviour of Data options validations in child table", () => { // test email validations for set_invalid controller - let website_input = 'website.in'; - let valid_email = 'user@email.com'; - let expectBackgroundColor = 'rgb(255, 245, 245)'; + let website_input = "website.in"; + let valid_email = "user@email.com"; + let expectBackgroundColor = "rgb(255, 245, 245)"; - cy.visit('/app/contact/new'); - cy.get('.frappe-control[data-fieldname="email_ids"]').as('table'); - cy.get('@table').find('button.grid-add-row').click(); - cy.get('@table').find('button.grid-add-row').click(); - cy.get('@table').find('[data-idx="1"]').as('row1'); - cy.get('@table').find('[data-idx="2"]').as('row2'); - cy.get('@row1').click(); - cy.get('@row1').find('input.input-with-feedback.form-control').as('email_input1'); + cy.visit("/app/contact/new"); + cy.get('.frappe-control[data-fieldname="email_ids"]').as("table"); + cy.get("@table").find("button.grid-add-row").click(); + cy.get("@table").find("button.grid-add-row").click(); + cy.get("@table").find('[data-idx="1"]').as("row1"); + cy.get("@table").find('[data-idx="2"]').as("row2"); + cy.get("@row1").click(); + cy.get("@row1").find("input.input-with-feedback.form-control").as("email_input1"); - cy.get('@email_input1').type(website_input, { waitForAnimations: false }); - cy.fill_field('company_name', 'Test Company'); + cy.get("@email_input1").type(website_input, { waitForAnimations: false }); + cy.fill_field("company_name", "Test Company"); - cy.get('@row2').click(); - cy.get('@row2').find('input.input-with-feedback.form-control').as('email_input2'); - cy.get('@email_input2').type(valid_email, { waitForAnimations: false }); + cy.get("@row2").click(); + cy.get("@row2").find("input.input-with-feedback.form-control").as("email_input2"); + cy.get("@email_input2").type(valid_email, { waitForAnimations: false }); - cy.get('@row1').click(); - cy.get('@email_input1').should($div => { + cy.get("@row1").click(); + cy.get("@email_input1").should(($div) => { const style = window.getComputedStyle($div[0]); expect(style.backgroundColor).to.equal(expectBackgroundColor); }); - cy.get('@email_input1').should('have.class', 'invalid'); + cy.get("@email_input1").should("have.class", "invalid"); - cy.get('@row2').click(); - cy.get('@email_input2').should('not.have.class', 'invalid'); + cy.get("@row2").click(); + cy.get("@email_input2").should("not.have.class", "invalid"); + }); + + it("Shows version conflict warning", { scrollBehavior: false }, () => { + cy.visit("/app/todo"); + + cy.insert_doc("ToDo", { description: "old" }).then((doc) => { + cy.visit(`/app/todo/${doc.name}`); + // make form dirty + cy.fill_field("status", "Cancelled", "Select"); + + // update doc using api - simulating parallel change by another user + cy.update_doc("ToDo", doc.name, { status: "Closed" }).then(() => { + cy.findByRole("button", { name: "Refresh" }).click(); + cy.get_field("status", "Select").should("have.value", "Closed"); + }); + }); + }); + + it("let user undo/redo field value changes", { scrollBehavior: false }, () => { + const jump_to_field = (field_label) => { + cy.get("body") + .type("{esc}") // lose focus if any + .type("{ctrl+j}") // jump to field + .type(field_label) + .wait(500) + .type("{enter}") + .wait(200) + .type("{enter}") + .wait(500); + }; + + const type_value = (value) => { + cy.focused().clear().type(value).type("{esc}"); + }; + + const undo = () => cy.get("body").type("{esc}").type("{ctrl+z}").wait(500); + const redo = () => cy.get("body").type("{esc}").type("{ctrl+y}").wait(500); + + cy.new_form("User"); + + jump_to_field("Email"); + type_value("admin@example.com"); + + jump_to_field("Username"); + type_value("admin42"); + + jump_to_field("Birth Date"); + type_value("12-31-01"); + + jump_to_field("Send Welcome Email"); + cy.focused().uncheck(); + + // make a mistake + jump_to_field("Username"); + type_value("admin24"); + + // undo behaviour + undo(); + cy.get_field("username").should("have.value", "admin42"); + + // redo behaviour + redo(); + cy.get_field("username").should("have.value", "admin24"); + + // undo everything & redo everything, ensure same values at the end + undo(); + undo(); + undo(); + undo(); + undo(); + redo(); + redo(); + redo(); + redo(); + redo(); + + cy.get_field("username").should("have.value", "admin24"); + cy.get_field("email").should("have.value", "admin@example.com"); + cy.get_field("birth_date").should("have.value", "12-31-2001"); // parsed value + cy.get_field("send_welcome_email").should("not.be.checked"); }); }); diff --git a/cypress/integration/form_tab_break.js b/cypress/integration/form_tab_break.js index 45c3c92084..91695cb143 100644 --- a/cypress/integration/form_tab_break.js +++ b/cypress/integration/form_tab_break.js @@ -1,31 +1,30 @@ -import doctype_with_tab_break from '../fixtures/doctype_with_tab_break'; +import doctype_with_tab_break from "../fixtures/doctype_with_tab_break"; const doctype_name = doctype_with_tab_break.name; context("Form Tab Break", () => { before(() => { cy.login(); - cy.visit('/app/website'); - return cy.insert_doc('DocType', doctype_with_tab_break, true); + cy.visit("/app/website"); + return cy.insert_doc("DocType", doctype_with_tab_break, true); }); it("Should switch tab and open correct tabs on validation error", () => { cy.new_form(doctype_name); // test tab switch - cy.findByRole("tab", {name: "Tab 2"}).click(); + cy.findByRole("tab", { name: "Tab 2" }).click(); cy.findByText("Phone"); - cy.findByRole("tab", {name: "Details"}).click(); + cy.findByRole("tab", { name: "Details" }).click(); cy.findByText("Name"); // form should switch to the tab with un-filled mandatory field cy.fill_field("username", "Test"); - cy.findByRole("button", {name: "Save"}).click(); + cy.findByRole("button", { name: "Save" }).click(); cy.findByText("Missing Fields"); cy.hide_dialog(); cy.findByText("Phone"); cy.fill_field("phone", "12345678"); - cy.findByRole("button", {name: "Save"}).click(); + cy.findByRole("button", { name: "Save" }).click(); // After save, first tab should have dashboard cy.get(".form-tabs > .nav-item").eq(0).click(); cy.findByText("Connections"); - }); -}); \ No newline at end of file +}); diff --git a/cypress/integration/form_tour.js b/cypress/integration/form_tour.js index ab7ada9034..f4ae0dbb6d 100644 --- a/cypress/integration/form_tour.js +++ b/cypress/integration/form_tour.js @@ -1,88 +1,94 @@ -context('Form Tour', () => { +context.skip("Form Tour", () => { before(() => { cy.login(); - cy.visit('/app/form-tour'); - return cy.window().its('frappe').then(frappe => { - return frappe.call("frappe.tests.ui_test_helpers.create_form_tour"); - }); + cy.visit("/app"); + return cy + .window() + .its("frappe") + .then((frappe) => { + return frappe.call("frappe.tests.ui_test_helpers.create_form_tour"); + }); }); const open_test_form_tour = () => { - cy.visit('/app/form-tour/Test Form Tour'); - cy.findByRole('button', {name: 'Show Tour'}).should('be.visible').as('show_tour'); - cy.get('@show_tour').click(); + cy.visit("/app/form-tour/Test Form Tour"); + cy.findByRole("button", { name: "Show Tour" }).should("be.visible").as("show_tour"); + cy.get("@show_tour").click(); cy.wait(500); - cy.url().should('include', '/app/contact'); + cy.url().should("include", "/app/contact"); }; - it('jump to a form tour', open_test_form_tour); + it("jump to a form tour", open_test_form_tour); - it('navigates a form tour', () => { + it("navigates a form tour", () => { open_test_form_tour(); - cy.get('.frappe-driver').should('be.visible'); - cy.get('.frappe-control[data-fieldname="first_name"]').as('first_name'); - cy.get('@first_name').should('have.class', 'driver-highlighted-element'); - cy.get('.frappe-driver').findByRole('button', {name: 'Next'}).as('next_btn'); + cy.get(".frappe-driver").should("be.visible"); + cy.get('.frappe-control[data-fieldname="first_name"]').as("first_name"); + cy.get("@first_name").should("have.class", "driver-highlighted-element"); + cy.get(".frappe-driver").findByRole("button", { name: "Next" }).as("next_btn"); // next btn shouldn't move to next step, if first name is not entered - cy.get('@next_btn').click(); + cy.get("@next_btn").click(); cy.wait(500); - cy.get('@first_name').should('have.class', 'driver-highlighted-element'); + cy.get("@first_name").should("have.class", "driver-highlighted-element"); // after filling the field, next step should be highlighted - cy.fill_field('first_name', 'Test Name', 'Data'); + cy.fill_field("first_name", "Test Name", "Data"); cy.wait(500); - cy.get('@next_btn').click(); + cy.get("@next_btn").click(); cy.wait(500); // assert field is highlighted - cy.get('.frappe-control[data-fieldname="last_name"]').as('last_name'); - cy.get('@last_name').should('have.class', 'driver-highlighted-element'); + cy.get('.frappe-control[data-fieldname="last_name"]').as("last_name"); + cy.get("@last_name").should("have.class", "driver-highlighted-element"); // after filling the field, next step should be highlighted - cy.fill_field('last_name', 'Test Last Name', 'Data'); + cy.fill_field("last_name", "Test Last Name", "Data"); cy.wait(500); - cy.get('@next_btn').click(); + cy.get("@next_btn").click(); cy.wait(500); // assert field is highlighted - cy.get('.frappe-control[data-fieldname="phone_nos"]').as('phone_nos'); - cy.get('@phone_nos').should('have.class', 'driver-highlighted-element'); + cy.get('.frappe-control[data-fieldname="phone_nos"]').as("phone_nos"); + cy.get("@phone_nos").should("have.class", "driver-highlighted-element"); // move to next step cy.wait(500); - cy.get('@next_btn').click(); + cy.get("@next_btn").click(); cy.wait(500); // assert add row btn is highlighted - cy.get('@phone_nos').find('.grid-add-row').as('add_row'); - cy.get('@add_row').should('have.class', 'driver-highlighted-element'); + cy.get("@phone_nos").find(".grid-add-row").as("add_row"); + cy.get("@add_row").should("have.class", "driver-highlighted-element"); // add a row & move to next step cy.wait(500); - cy.get('@add_row').click(); + cy.get("@add_row").click(); cy.wait(500); // assert table field is highlighted - cy.get('.grid-row-open .frappe-control[data-fieldname="phone"]').as('phone'); - cy.get('@phone').should('have.class', 'driver-highlighted-element'); + cy.get('.grid-row-open .frappe-control[data-fieldname="phone"]').as("phone"); + cy.get("@phone").should("have.class", "driver-highlighted-element"); // enter value in a table field - let field = cy.fill_table_field('phone_nos', '1', 'phone', '1234567890'); + let field = cy.fill_table_field("phone_nos", "1", "phone", "1234567890"); field.blur(); // move to collapse row step cy.wait(500); - cy.get('.driver-popover-title').contains('Test Title 4').siblings().get('@next_btn').click(); + cy.get(".driver-popover-title") + .contains("Test Title 4") + .siblings() + .get("@next_btn") + .click(); cy.wait(500); // collapse row - cy.get('.grid-row-open .grid-collapse-row').click(); + cy.get(".grid-row-open .grid-collapse-row").click(); cy.wait(500); // assert save btn is highlighted - cy.get('.primary-action').should('have.class', 'driver-highlighted-element'); + cy.get(".primary-action").should("have.class", "driver-highlighted-element"); cy.wait(500); - cy.get('.frappe-driver').findByRole('button', {name: 'Save'}).should('be.visible'); - + cy.get(".frappe-driver").findByRole("button", { name: "Save" }).should("be.visible"); }); }); diff --git a/cypress/integration/grid.js b/cypress/integration/grid.js index 4fa52712cf..6cf9e8cdc7 100644 --- a/cypress/integration/grid.js +++ b/cypress/integration/grid.js @@ -1,92 +1,114 @@ -context('Grid', () => { +context("Grid", () => { beforeEach(() => { cy.login(); - cy.visit('/app/website'); + cy.visit("/app/website"); }); before(() => { cy.login(); - cy.visit('/app/website'); - return cy.window().its('frappe').then(frappe => { - return frappe.call("frappe.tests.ui_test_helpers.create_contact_phone_nos_records"); - }); + cy.visit("/app/website"); + return cy + .window() + .its("frappe") + .then((frappe) => { + return frappe.call( + "frappe.tests.ui_test_helpers.create_contact_phone_nos_records" + ); + }); }); - it('update docfield property using update_docfield_property', () => { - cy.visit('/app/contact/Test Contact'); - cy.window().its("cur_frm").then(frm => { - cy.get('.frappe-control[data-fieldname="phone_nos"]').as('table'); - let field = frm.get_field("phone_nos"); - field.grid.update_docfield_property("is_primary_phone", "hidden", true); + it("update docfield property using update_docfield_property", () => { + cy.visit("/app/contact/Test Contact"); + cy.window() + .its("cur_frm") + .then((frm) => { + cy.get('.frappe-control[data-fieldname="phone_nos"]').as("table"); + let field = frm.get_field("phone_nos"); + field.grid.update_docfield_property("is_primary_phone", "hidden", true); - cy.get('@table').find('[data-idx="1"] .edit-grid-row').click(); - cy.get('.grid-row-open').as('table-form'); - cy.get('@table-form').find('.frappe-control[data-fieldname="is_primary_phone"]').should("be.hidden"); - cy.get('@table-form').find('.grid-footer-toolbar').click(); + cy.get("@table").find('[data-idx="1"] .edit-grid-row').click(); + cy.get(".grid-row-open").as("table-form"); + cy.get("@table-form") + .find('.frappe-control[data-fieldname="is_primary_phone"]') + .should("be.hidden"); + cy.get("@table-form").find(".grid-footer-toolbar").click(); - cy.get('@table').find('[data-idx="2"] .edit-grid-row').click(); - cy.get('.grid-row-open').as('table-form'); - cy.get('@table-form').find('.frappe-control[data-fieldname="is_primary_phone"]').should("be.hidden"); - cy.get('@table-form').find('.grid-footer-toolbar').click(); - }); + cy.get("@table").find('[data-idx="2"] .edit-grid-row').click(); + cy.get(".grid-row-open").as("table-form"); + cy.get("@table-form") + .find('.frappe-control[data-fieldname="is_primary_phone"]') + .should("be.hidden"); + cy.get("@table-form").find(".grid-footer-toolbar").click(); + }); }); - it('update docfield property using toggle_display', () => { - cy.visit('/app/contact/Test Contact'); - cy.window().its("cur_frm").then(frm => { - cy.get('.frappe-control[data-fieldname="phone_nos"]').as('table'); - let field = frm.get_field("phone_nos"); - field.grid.toggle_display("is_primary_mobile_no", false); + it("update docfield property using toggle_display", () => { + cy.visit("/app/contact/Test Contact"); + cy.window() + .its("cur_frm") + .then((frm) => { + cy.get('.frappe-control[data-fieldname="phone_nos"]').as("table"); + let field = frm.get_field("phone_nos"); + field.grid.toggle_display("is_primary_mobile_no", false); - cy.get('@table').find('[data-idx="1"] .edit-grid-row').click(); - cy.get('.grid-row-open').as('table-form'); - cy.get('@table-form').find('.frappe-control[data-fieldname="is_primary_mobile_no"]').should("be.hidden"); - cy.get('@table-form').find('.grid-footer-toolbar').click(); + cy.get("@table").find('[data-idx="1"] .edit-grid-row').click(); + cy.get(".grid-row-open").as("table-form"); + cy.get("@table-form") + .find('.frappe-control[data-fieldname="is_primary_mobile_no"]') + .should("be.hidden"); + cy.get("@table-form").find(".grid-footer-toolbar").click(); - cy.get('@table').find('[data-idx="2"] .edit-grid-row').click(); - cy.get('.grid-row-open').as('table-form'); - cy.get('@table-form').find('.frappe-control[data-fieldname="is_primary_mobile_no"]').should("be.hidden"); - cy.get('@table-form').find('.grid-footer-toolbar').click(); - }); + cy.get("@table").find('[data-idx="2"] .edit-grid-row').click(); + cy.get(".grid-row-open").as("table-form"); + cy.get("@table-form") + .find('.frappe-control[data-fieldname="is_primary_mobile_no"]') + .should("be.hidden"); + cy.get("@table-form").find(".grid-footer-toolbar").click(); + }); }); - it('update docfield property using toggle_enable', () => { - cy.visit('/app/contact/Test Contact'); - cy.window().its("cur_frm").then(frm => { - cy.get('.frappe-control[data-fieldname="phone_nos"]').as('table'); - let field = frm.get_field("phone_nos"); - field.grid.toggle_enable("phone", false); + it("update docfield property using toggle_enable", () => { + cy.visit("/app/contact/Test Contact"); + cy.window() + .its("cur_frm") + .then((frm) => { + cy.get('.frappe-control[data-fieldname="phone_nos"]').as("table"); + let field = frm.get_field("phone_nos"); + field.grid.toggle_enable("phone", false); + cy.get("@table").find('[data-idx="1"] .edit-grid-row').click(); + cy.get(".grid-row-open").as("table-form"); + cy.get("@table-form") + .find('.frappe-control[data-fieldname="phone"] .control-value') + .should("have.class", "like-disabled-input"); + cy.get("@table-form").find(".grid-footer-toolbar").click(); - cy.get('@table').find('[data-idx="1"] .edit-grid-row').click(); - cy.get('.grid-row-open').as('table-form'); - cy.get('@table-form').find('.frappe-control[data-fieldname="phone"] .control-value').should('have.class', 'like-disabled-input'); - cy.get('@table-form').find('.grid-footer-toolbar').click(); - - cy.get('@table').find('[data-idx="2"] .edit-grid-row').click(); - cy.get('.grid-row-open').as('table-form'); - cy.get('@table-form').find('.frappe-control[data-fieldname="phone"] .control-value').should('have.class', 'like-disabled-input'); - cy.get('@table-form').find('.grid-footer-toolbar').click(); - }); + cy.get("@table").find('[data-idx="2"] .edit-grid-row').click(); + cy.get(".grid-row-open").as("table-form"); + cy.get("@table-form") + .find('.frappe-control[data-fieldname="phone"] .control-value') + .should("have.class", "like-disabled-input"); + cy.get("@table-form").find(".grid-footer-toolbar").click(); + }); }); - it('update docfield property using toggle_reqd', () => { - cy.visit('/app/contact/Test Contact'); - cy.window().its("cur_frm").then(frm => { - cy.get('.frappe-control[data-fieldname="phone_nos"]').as('table'); - let field = frm.get_field("phone_nos"); - field.grid.toggle_reqd("phone", false); + it("update docfield property using toggle_reqd", () => { + cy.visit("/app/contact/Test Contact"); + cy.window() + .its("cur_frm") + .then((frm) => { + cy.get('.frappe-control[data-fieldname="phone_nos"]').as("table"); + let field = frm.get_field("phone_nos"); + field.grid.toggle_reqd("phone", false); - cy.get('@table').find('[data-idx="1"] .edit-grid-row').click(); - cy.get('.grid-row-open').as('table-form'); - cy.get_field("phone").as('phone-field'); - cy.get('@phone-field').focus().clear().wait(500).blur(); - cy.get('@phone-field').should("not.have.class", "has-error"); - cy.get('@table-form').find('.grid-footer-toolbar').click(); + cy.get("@table").find('[data-idx="1"] .edit-grid-row').click(); + cy.get(".grid-row-open").as("table-form"); + cy.get_field("phone").as("phone-field"); + cy.get("@phone-field").focus().clear().wait(500).blur(); + cy.get("@phone-field").should("not.have.class", "has-error"); + cy.get("@table-form").find(".grid-footer-toolbar").click(); - cy.get('@table').find('[data-idx="2"] .edit-grid-row').click(); - cy.get('.grid-row-open').as('table-form'); - cy.get_field("phone").as('phone-field'); - cy.get('@phone-field').focus().clear().wait(500).blur(); - cy.get('@phone-field').should("not.have.class", "has-error"); - cy.get('@table-form').find('.grid-footer-toolbar').click(); - - }); + cy.get("@table").find('[data-idx="2"] .edit-grid-row').click(); + cy.get(".grid-row-open").as("table-form"); + cy.get_field("phone").as("phone-field"); + cy.get("@phone-field").focus().clear().wait(500).blur(); + cy.get("@phone-field").should("not.have.class", "has-error"); + cy.get("@table-form").find(".grid-footer-toolbar").click(); + }); }); }); - diff --git a/cypress/integration/grid_configuration.js b/cypress/integration/grid_configuration.js index 7193d804c2..9112d7023e 100644 --- a/cypress/integration/grid_configuration.js +++ b/cypress/integration/grid_configuration.js @@ -1,23 +1,23 @@ -context('Grid Configuration', () => { +context("Grid Configuration", () => { beforeEach(() => { cy.login(); - cy.visit('/app/doctype/User'); + cy.visit("/app/doctype/User"); }); - it('Set user wise grid settings', () => { + it("Set user wise grid settings", () => { cy.wait(100); - cy.get('.frappe-control[data-fieldname="fields"]').as('table'); - cy.get('@table').find('.icon-sm').click(); + cy.get('.frappe-control[data-fieldname="fields"]').as("table"); + cy.get("@table").find(".icon-sm").click(); cy.wait(100); - cy.get('.frappe-control[data-fieldname="fields_html"]').as('modal'); - cy.get('@modal').find('.add-new-fields').click(); + cy.get('.frappe-control[data-fieldname="fields_html"]').as("modal"); + cy.get("@modal").find(".add-new-fields").click(); cy.wait(100); cy.get('[type="checkbox"][data-unit="read_only"]').check(); - cy.findByRole('button', {name: 'Add'}).click(); + cy.findByRole("button", { name: "Add" }).click(); cy.wait(100); - cy.get('[data-fieldname="options"]').invoke('attr', 'value', '1'); - cy.get('.form-control.column-width[data-fieldname="options"]').trigger('change'); - cy.findByRole('button', {name: 'Update'}).click(); + cy.get('[data-fieldname="options"]').invoke("attr", "value", "1"); + cy.get('.form-control.column-width[data-fieldname="options"]').trigger("change"); + cy.findByRole("button", { name: "Update" }).click(); cy.wait(200); - cy.get('[title="Read Only"').should('be.visible'); + cy.get('[title="Read Only"').should("be.visible"); }); -}); \ No newline at end of file +}); diff --git a/cypress/integration/grid_keyboard_shortcut.js b/cypress/integration/grid_keyboard_shortcut.js index 9cf39165ad..414e822516 100644 --- a/cypress/integration/grid_keyboard_shortcut.js +++ b/cypress/integration/grid_keyboard_shortcut.js @@ -1,40 +1,47 @@ -context('Grid Keyboard Shortcut', () => { +context("Grid Keyboard Shortcut", () => { let total_count = 0; before(() => { cy.login(); }); beforeEach(() => { cy.reload(); - cy.visit('/app/contact/new-contact-1'); + cy.visit("/app/contact/new-contact-1"); cy.get('.frappe-control[data-fieldname="email_ids"]').find(".grid-add-row").click(); }); - it('Insert new row at the end', () => { - cy.add_new_row_in_grid('{ctrl}{shift}{downarrow}', (cy, total_count) => { - cy.get('[data-name="new-contact-email-1"]').should('have.attr', 'data-idx', `${total_count+1}`); - }, total_count); + it("Insert new row at the end", () => { + cy.add_new_row_in_grid( + "{ctrl}{shift}{downarrow}", + (cy, total_count) => { + cy.get('[data-name="new-contact-email-1"]').should( + "have.attr", + "data-idx", + `${total_count + 1}` + ); + }, + total_count + ); }); - it('Insert new row at the top', () => { - cy.add_new_row_in_grid('{ctrl}{shift}{uparrow}', (cy) => { - cy.get('[data-name="new-contact-email-1"]').should('have.attr', 'data-idx', '2'); + it("Insert new row at the top", () => { + cy.add_new_row_in_grid("{ctrl}{shift}{uparrow}", (cy) => { + cy.get('[data-name="new-contact-email-1"]').should("have.attr", "data-idx", "2"); }); }); - it('Insert new row below', () => { - cy.add_new_row_in_grid('{ctrl}{downarrow}', (cy) => { - cy.get('[data-name="new-contact-email-1"]').should('have.attr', 'data-idx', '1'); + it("Insert new row below", () => { + cy.add_new_row_in_grid("{ctrl}{downarrow}", (cy) => { + cy.get('[data-name="new-contact-email-1"]').should("have.attr", "data-idx", "1"); }); }); - it('Insert new row above', () => { - cy.add_new_row_in_grid('{ctrl}{uparrow}', (cy) => { - cy.get('[data-name="new-contact-email-1"]').should('have.attr', 'data-idx', '2'); + it("Insert new row above", () => { + cy.add_new_row_in_grid("{ctrl}{uparrow}", (cy) => { + cy.get('[data-name="new-contact-email-1"]').should("have.attr", "data-idx", "2"); }); }); }); -Cypress.Commands.add('add_new_row_in_grid', (shortcut_keys, callbackFn, total_count) => { - cy.get('.frappe-control[data-fieldname="email_ids"]').as('table'); - cy.get('@table').find('.grid-body [data-fieldname="email_id"]').first().click(); - cy.get('@table').find('.grid-body [data-fieldname="email_id"]') - .first().type(shortcut_keys); +Cypress.Commands.add("add_new_row_in_grid", (shortcut_keys, callbackFn, total_count) => { + cy.get('.frappe-control[data-fieldname="email_ids"]').as("table"); + cy.get("@table").find('.grid-body [data-fieldname="email_id"]').first().click(); + cy.get("@table").find('.grid-body [data-fieldname="email_id"]').first().type(shortcut_keys); callbackFn(cy, total_count); -}); \ No newline at end of file +}); diff --git a/cypress/integration/grid_pagination.js b/cypress/integration/grid_pagination.js index 84b3320282..097f2a5cdc 100644 --- a/cypress/integration/grid_pagination.js +++ b/cypress/integration/grid_pagination.js @@ -1,65 +1,73 @@ -context('Grid Pagination', () => { +context("Grid Pagination", () => { beforeEach(() => { cy.login(); - cy.visit('/app/website'); + cy.visit("/app/website"); }); before(() => { cy.login(); - cy.visit('/app/website'); - return cy.window().its('frappe').then(frappe => { - return frappe.call("frappe.tests.ui_test_helpers.create_contact_phone_nos_records"); - }); + cy.visit("/app/website"); + return cy + .window() + .its("frappe") + .then((frappe) => { + return frappe.call( + "frappe.tests.ui_test_helpers.create_contact_phone_nos_records" + ); + }); }); - it('creates pages for child table', () => { - cy.visit('/app/contact/Test Contact'); - cy.get('.frappe-control[data-fieldname="phone_nos"]').as('table'); - cy.get('@table').find('.current-page-number').should('have.value', '1'); - cy.get('@table').find('.total-page-number').should('contain', '20'); - cy.get('@table').find('.grid-body .grid-row').should('have.length', 50); + it("creates pages for child table", () => { + cy.visit("/app/contact/Test Contact"); + cy.get('.frappe-control[data-fieldname="phone_nos"]').as("table"); + cy.get("@table").find(".current-page-number").should("have.value", "1"); + cy.get("@table").find(".total-page-number").should("contain", "20"); + cy.get("@table").find(".grid-body .grid-row").should("have.length", 50); }); - it('goes to the next and previous page', () => { - cy.visit('/app/contact/Test Contact'); - cy.get('.frappe-control[data-fieldname="phone_nos"]').as('table'); - cy.get('@table').find('.next-page').click(); - cy.get('@table').find('.current-page-number').should('have.value', '2'); - cy.get('@table').find('.grid-body .grid-row').first().should('have.attr', 'data-idx', '51'); - cy.get('@table').find('.prev-page').click(); - cy.get('@table').find('.current-page-number').should('have.value', '1'); - cy.get('@table').find('.grid-body .grid-row').first().should('have.attr', 'data-idx', '1'); + it("goes to the next and previous page", () => { + cy.visit("/app/contact/Test Contact"); + cy.get('.frappe-control[data-fieldname="phone_nos"]').as("table"); + cy.get("@table").find(".next-page").click(); + cy.get("@table").find(".current-page-number").should("have.value", "2"); + cy.get("@table") + .find(".grid-body .grid-row") + .first() + .should("have.attr", "data-idx", "51"); + cy.get("@table").find(".prev-page").click(); + cy.get("@table").find(".current-page-number").should("have.value", "1"); + cy.get("@table").find(".grid-body .grid-row").first().should("have.attr", "data-idx", "1"); }); - it('adds and deletes rows and changes page', () => { - cy.visit('/app/contact/Test Contact'); - cy.get('.frappe-control[data-fieldname="phone_nos"]').as('table'); - cy.get('@table').findByRole('button', {name: 'Add Row'}).click(); - cy.get('@table').find('.grid-body .row-index').should('contain', 1001); - cy.get('@table').find('.current-page-number').should('have.value', '21'); - cy.get('@table').find('.total-page-number').should('contain', '21'); - cy.get('@table').find('.grid-body .grid-row .grid-row-check').click({ force: true }); - cy.get('@table').findByRole('button', {name: 'Delete'}).click(); - cy.get('@table').find('.grid-body .row-index').last().should('contain', 1000); - cy.get('@table').find('.current-page-number').should('have.value', '20'); - cy.get('@table').find('.total-page-number').should('contain', '20'); + it("adds and deletes rows and changes page", () => { + cy.visit("/app/contact/Test Contact"); + cy.get('.frappe-control[data-fieldname="phone_nos"]').as("table"); + cy.get("@table").findByRole("button", { name: "Add Row" }).click(); + cy.get("@table").find(".grid-body .row-index").should("contain", 1001); + cy.get("@table").find(".current-page-number").should("have.value", "21"); + cy.get("@table").find(".total-page-number").should("contain", "21"); + cy.get("@table").find(".grid-body .grid-row .grid-row-check").click({ force: true }); + cy.get("@table").findByRole("button", { name: "Delete" }).click(); + cy.get("@table").find(".grid-body .row-index").last().should("contain", 1000); + cy.get("@table").find(".current-page-number").should("have.value", "20"); + cy.get("@table").find(".total-page-number").should("contain", "20"); }); - it('go to specific page, use up and down arrow, type characters, 0 page and more than existing page', () => { - cy.visit('/app/contact/Test Contact'); - cy.get('.frappe-control[data-fieldname="phone_nos"]').as('table'); - cy.get('@table').find('.current-page-number').focus().clear().type('17').blur(); - cy.get('@table').find('.grid-body .row-index').should('contain', 801); + it("go to specific page, use up and down arrow, type characters, 0 page and more than existing page", () => { + cy.visit("/app/contact/Test Contact"); + cy.get('.frappe-control[data-fieldname="phone_nos"]').as("table"); + cy.get("@table").find(".current-page-number").focus().clear().type("17").blur(); + cy.get("@table").find(".grid-body .row-index").should("contain", 801); - cy.get('@table').find('.current-page-number').focus().type('{uparrow}{uparrow}'); - cy.get('@table').find('.current-page-number').should('have.value', '19'); + cy.get("@table").find(".current-page-number").focus().type("{uparrow}{uparrow}"); + cy.get("@table").find(".current-page-number").should("have.value", "19"); - cy.get('@table').find('.current-page-number').focus().type('{downarrow}{downarrow}'); - cy.get('@table').find('.current-page-number').should('have.value', '17'); + cy.get("@table").find(".current-page-number").focus().type("{downarrow}{downarrow}"); + cy.get("@table").find(".current-page-number").should("have.value", "17"); - cy.get('@table').find('.current-page-number').focus().clear().type('700').blur(); - cy.get('@table').find('.current-page-number').should('have.value', '20'); + cy.get("@table").find(".current-page-number").focus().clear().type("700").blur(); + cy.get("@table").find(".current-page-number").should("have.value", "20"); - cy.get('@table').find('.current-page-number').focus().clear().type('0').blur(); - cy.get('@table').find('.current-page-number').should('have.value', '1'); + cy.get("@table").find(".current-page-number").focus().clear().type("0").blur(); + cy.get("@table").find(".current-page-number").should("have.value", "1"); - cy.get('@table').find('.current-page-number').focus().clear().type('abc').blur(); - cy.get('@table').find('.current-page-number').should('have.value', '1'); + cy.get("@table").find(".current-page-number").focus().clear().type("abc").blur(); + cy.get("@table").find(".current-page-number").should("have.value", "1"); }); // it('deletes all rows', ()=> { // cy.visit('/app/contact/Test Contact'); @@ -69,4 +77,4 @@ context('Grid Pagination', () => { // cy.get('.modal-dialog .btn-primary').contains('Yes').click(); // cy.get('@table').find('.grid-body .grid-row').should('have.length', 0); // }); -}); \ No newline at end of file +}); diff --git a/cypress/integration/grid_search.js b/cypress/integration/grid_search.js index d30545a2e1..3d43412313 100644 --- a/cypress/integration/grid_search.js +++ b/cypress/integration/grid_search.js @@ -1,107 +1,133 @@ -import doctype_with_child_table from '../fixtures/doctype_with_child_table'; -import child_table_doctype from '../fixtures/child_table_doctype'; -import child_table_doctype_1 from '../fixtures/child_table_doctype_1'; +import doctype_with_child_table from "../fixtures/doctype_with_child_table"; +import child_table_doctype from "../fixtures/child_table_doctype"; +import child_table_doctype_1 from "../fixtures/child_table_doctype_1"; const doctype_with_child_table_name = doctype_with_child_table.name; -context('Grid Search', () => { +context("Grid Search", () => { before(() => { - cy.visit('/login'); + cy.visit("/login"); cy.login(); - cy.visit('/app/website'); - cy.insert_doc('DocType', child_table_doctype, true); - cy.insert_doc('DocType', child_table_doctype_1, true); - cy.insert_doc('DocType', doctype_with_child_table, true); - return cy.window().its('frappe').then(frappe => { - return frappe.xcall("frappe.tests.ui_test_helpers.insert_doctype_with_child_table_record", { - name: doctype_with_child_table_name + cy.visit("/app/website"); + cy.insert_doc("DocType", child_table_doctype, true); + cy.insert_doc("DocType", child_table_doctype_1, true); + cy.insert_doc("DocType", doctype_with_child_table, true); + return cy + .window() + .its("frappe") + .then((frappe) => { + return frappe.xcall( + "frappe.tests.ui_test_helpers.insert_doctype_with_child_table_record", + { + name: doctype_with_child_table_name, + } + ); }); - }); }); - it('Test search row visibility', () => { - cy.window().its('frappe').then(frappe => { - frappe.model.user_settings.save('Doctype With Child Table', 'GridView', { - 'Child Table Doctype 1': [ - {'fieldname': 'data', 'columns': 2}, - {'fieldname': 'barcode', 'columns': 1}, - {'fieldname': 'check', 'columns': 1}, - {'fieldname': 'rating', 'columns': 2}, - {'fieldname': 'duration', 'columns': 2}, - {'fieldname': 'date', 'columns': 2} - ] + it("Test search row visibility", () => { + cy.window() + .its("frappe") + .then((frappe) => { + frappe.model.user_settings.save("Doctype With Child Table", "GridView", { + "Child Table Doctype 1": [ + { fieldname: "data", columns: 2 }, + { fieldname: "barcode", columns: 1 }, + { fieldname: "check", columns: 1 }, + { fieldname: "rating", columns: 2 }, + { fieldname: "duration", columns: 2 }, + { fieldname: "date", columns: 2 }, + ], + }); }); - }); cy.visit(`/app/doctype-with-child-table/Test Grid Search`); - cy.get('.frappe-control[data-fieldname="child_table_1"]').as('table'); - cy.get('@table').find('.grid-row-check:last').click(); - cy.get('@table').find('.grid-footer').contains('Delete').click(); - cy.get('.grid-heading-row .grid-row .search').should('not.exist'); + cy.get('.frappe-control[data-fieldname="child_table_1"]').as("table"); + cy.get("@table").find(".grid-row-check:last").click(); + cy.get("@table").find(".grid-footer").contains("Delete").click(); + cy.get(".grid-heading-row .grid-row .search").should("not.exist"); }); - it('test search field for different fieldtypes', () => { + it("test search field for different fieldtypes", () => { cy.visit(`/app/doctype-with-child-table/Test Grid Search`); - cy.get('.frappe-control[data-fieldname="child_table_1"]').as('table'); + cy.get('.frappe-control[data-fieldname="child_table_1"]').as("table"); // Index Column - cy.get('@table').find('.grid-heading-row .row-index.search input').type('3'); - cy.get('@table').find('.grid-body .rows .grid-row').should('have.length', 2); - cy.get('@table').find('.grid-heading-row .row-index.search input').clear(); + cy.get("@table").find(".grid-heading-row .row-index.search input").type("3"); + cy.get("@table").find(".grid-body .rows .grid-row").should("have.length", 2); + cy.get("@table").find(".grid-heading-row .row-index.search input").clear(); // Data Column - cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Data"]').type('Data'); - cy.get('@table').find('.grid-body .rows .grid-row').should('have.length', 1); - cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Data"]').clear(); + cy.get("@table") + .find('.grid-heading-row .search input[data-fieldtype="Data"]') + .type("Data"); + cy.get("@table").find(".grid-body .rows .grid-row").should("have.length", 1); + cy.get("@table").find('.grid-heading-row .search input[data-fieldtype="Data"]').clear(); // Barcode Column - cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Barcode"]').type('092'); - cy.get('@table').find('.grid-body .rows .grid-row').should('have.length', 4); - cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Barcode"]').clear(); + cy.get("@table") + .find('.grid-heading-row .search input[data-fieldtype="Barcode"]') + .type("092"); + cy.get("@table").find(".grid-body .rows .grid-row").should("have.length", 4); + cy.get("@table").find('.grid-heading-row .search input[data-fieldtype="Barcode"]').clear(); // Check Column - cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Check"]').type('1'); - cy.get('@table').find('.grid-body .rows .grid-row').should('have.length', 9); - cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Check"]').clear(); + cy.get("@table").find('.grid-heading-row .search input[data-fieldtype="Check"]').type("1"); + cy.get("@table").find(".grid-body .rows .grid-row").should("have.length", 9); + cy.get("@table").find('.grid-heading-row .search input[data-fieldtype="Check"]').clear(); - cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Check"]').type('0'); - cy.get('@table').find('.grid-body .rows .grid-row').should('have.length', 11); - cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Check"]').clear(); + cy.get("@table").find('.grid-heading-row .search input[data-fieldtype="Check"]').type("0"); + cy.get("@table").find(".grid-body .rows .grid-row").should("have.length", 11); + cy.get("@table").find('.grid-heading-row .search input[data-fieldtype="Check"]').clear(); // Rating Column - cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Rating"]').type('3'); - cy.get('@table').find('.grid-body .rows .grid-row').should('have.length', 3); - cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Rating"]').clear(); + cy.get("@table") + .find('.grid-heading-row .search input[data-fieldtype="Rating"]') + .type("3"); + cy.get("@table").find(".grid-body .rows .grid-row").should("have.length", 3); + cy.get("@table").find('.grid-heading-row .search input[data-fieldtype="Rating"]').clear(); // Duration Column - cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Duration"]').type('3d'); - cy.get('@table').find('.grid-body .rows .grid-row').should('have.length', 3); - cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Duration"]').clear(); + cy.get("@table") + .find('.grid-heading-row .search input[data-fieldtype="Duration"]') + .type("3d"); + cy.get("@table").find(".grid-body .rows .grid-row").should("have.length", 3); + cy.get("@table") + .find('.grid-heading-row .search input[data-fieldtype="Duration"]') + .clear(); // Date Column - cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Date"]').type('2022'); - cy.get('@table').find('.grid-body .rows .grid-row').should('have.length', 4); - cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Date"]').clear(); + cy.get("@table") + .find('.grid-heading-row .search input[data-fieldtype="Date"]') + .type("2022"); + cy.get("@table").find(".grid-body .rows .grid-row").should("have.length", 4); + cy.get("@table").find('.grid-heading-row .search input[data-fieldtype="Date"]').clear(); }); - it('test with multiple filter', () => { - cy.get('.frappe-control[data-fieldname="child_table_1"]').as('table'); + it("test with multiple filter", () => { + cy.get('.frappe-control[data-fieldname="child_table_1"]').as("table"); // Data Column - cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Data"]').type('a'); - cy.get('@table').find('.grid-body .rows .grid-row').should('have.length', 10); + cy.get("@table").find('.grid-heading-row .search input[data-fieldtype="Data"]').type("a"); + cy.get("@table").find(".grid-body .rows .grid-row").should("have.length", 10); // Barcode Column - cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Barcode"]').type('0'); - cy.get('@table').find('.grid-body .rows .grid-row').should('have.length', 8); + cy.get("@table") + .find('.grid-heading-row .search input[data-fieldtype="Barcode"]') + .type("0"); + cy.get("@table").find(".grid-body .rows .grid-row").should("have.length", 8); // Duration Column - cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Duration"]').type('d'); - cy.get('@table').find('.grid-body .rows .grid-row').should('have.length', 5); + cy.get("@table") + .find('.grid-heading-row .search input[data-fieldtype="Duration"]') + .type("d"); + cy.get("@table").find(".grid-body .rows .grid-row").should("have.length", 5); // Date Column - cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Date"]').type('02-'); - cy.get('@table').find('.grid-body .rows .grid-row').should('have.length', 2); + cy.get("@table") + .find('.grid-heading-row .search input[data-fieldtype="Date"]') + .type("02-"); + cy.get("@table").find(".grid-body .rows .grid-row").should("have.length", 2); }); -}); \ No newline at end of file +}); diff --git a/cypress/integration/kanban.js b/cypress/integration/kanban.js new file mode 100644 index 0000000000..7296a12666 --- /dev/null +++ b/cypress/integration/kanban.js @@ -0,0 +1,112 @@ +context("Kanban Board", () => { + before(() => { + cy.login(); + cy.visit("/app"); + }); + + it("Create ToDo Kanban", () => { + cy.visit("/app/todo"); + + cy.get(".page-actions .custom-btn-group button").click(); + cy.get(".page-actions .custom-btn-group ul.dropdown-menu li").contains("Kanban").click(); + + cy.focused().blur(); + cy.fill_field("board_name", "ToDo Kanban", "Data"); + cy.fill_field("field_name", "Status", "Select"); + cy.click_modal_primary_button("Save"); + + cy.get(".title-text").should("contain", "ToDo Kanban"); + }); + + it("Create ToDo from kanban", () => { + cy.intercept({ + method: "POST", + url: "api/method/frappe.client.save", + }).as("save-todo"); + + cy.click_listview_primary_button("Add ToDo"); + + cy.fill_field("description", "Test Kanban ToDo", "Text Editor").wait(300); + cy.get(".modal-footer .btn-primary").last().click(); + + cy.wait("@save-todo"); + }); + + it("Add and Remove fields", () => { + cy.visit("/app/todo/view/kanban/ToDo Kanban"); + + cy.intercept( + "POST", + "/api/method/frappe.desk.doctype.kanban_board.kanban_board.save_settings" + ).as("save-kanban"); + cy.intercept( + "POST", + "/api/method/frappe.desk.doctype.kanban_board.kanban_board.update_order" + ).as("update-order"); + + cy.get(".page-actions .menu-btn-group > .btn").click(); + cy.get(".page-actions .menu-btn-group .dropdown-menu li") + .contains("Kanban Settings") + .click(); + cy.get(".add-new-fields").click(); + + cy.get(".checkbox-options .checkbox").contains("ID").click(); + cy.get(".checkbox-options .checkbox").contains("Status").first().click(); + cy.get(".checkbox-options .checkbox").contains("Priority").click(); + + cy.get(".modal-footer .btn-primary").last().click(); + + cy.get(".frappe-control .label-area").contains("Show Labels").click(); + cy.click_modal_primary_button("Save"); + + cy.wait("@save-kanban"); + + cy.get('.kanban-column[data-column-value="Open"] .kanban-cards').as("open-cards"); + cy.get("@open-cards") + .find(".kanban-card .kanban-card-doc") + .first() + .should("contain", "ID:"); + cy.get("@open-cards") + .find(".kanban-card .kanban-card-doc") + .first() + .should("contain", "Status:"); + cy.get("@open-cards") + .find(".kanban-card .kanban-card-doc") + .first() + .should("contain", "Priority:"); + + cy.get(".page-actions .menu-btn-group > .btn").click(); + cy.get(".page-actions .menu-btn-group .dropdown-menu li") + .contains("Kanban Settings") + .click(); + cy.get_open_dialog() + .find( + '.frappe-control[data-fieldname="fields_html"] div[data-label="ID"] .remove-field' + ) + .click(); + + cy.wait("@update-order"); + cy.get_open_dialog().find(".frappe-control .label-area").contains("Show Labels").click(); + cy.get(".modal-footer .btn-primary").last().click(); + + cy.wait("@save-kanban"); + + cy.get("@open-cards") + .find(".kanban-card .kanban-card-doc") + .first() + .should("not.contain", "ID:"); + }); + + // it('Drag todo', () => { + // cy.intercept({ + // method: 'POST', + // url: 'api/method/frappe.desk.doctype.kanban_board.kanban_board.update_order_for_single_card' + // }).as('drag-completed'); + + // cy.get('.kanban-card-body') + // .contains('Test Kanban ToDo').first() + // .drag('[data-column-value="Closed"] .kanban-cards', { force: true }); + + // cy.wait('@drag-completed'); + // }); +}); diff --git a/cypress/integration/list_paging.js b/cypress/integration/list_paging.js index 4a59024a7b..3071950260 100644 --- a/cypress/integration/list_paging.js +++ b/cypress/integration/list_paging.js @@ -1,38 +1,42 @@ -context('List Paging', () => { +context("List Paging", () => { before(() => { cy.login(); - cy.visit('/app/website'); - return cy.window().its('frappe').then(frappe => { - return frappe.call("frappe.tests.ui_test_helpers.create_multiple_todo_records"); - }); + cy.visit("/app/website"); + return cy + .window() + .its("frappe") + .then((frappe) => { + return frappe.call("frappe.tests.ui_test_helpers.create_multiple_todo_records"); + }); }); - it('test load more with count selection buttons', () => { - cy.visit('/app/todo/view/report'); + it("test load more with count selection buttons", () => { + cy.visit("/app/todo/view/report"); + cy.clear_filters(); - cy.get('.list-paging-area .list-count').should('contain.text', '20 of'); - cy.get('.list-paging-area .btn-more').click(); - cy.get('.list-paging-area .list-count').should('contain.text', '40 of'); - cy.get('.list-paging-area .btn-more').click(); - cy.get('.list-paging-area .list-count').should('contain.text', '60 of'); + cy.get(".list-paging-area .list-count").should("contain.text", "20 of"); + cy.get(".list-paging-area .btn-more").click(); + cy.get(".list-paging-area .list-count").should("contain.text", "40 of"); + cy.get(".list-paging-area .btn-more").click(); + cy.get(".list-paging-area .list-count").should("contain.text", "60 of"); cy.get('.list-paging-area .btn-group .btn-paging[data-value="100"]').click(); - cy.get('.list-paging-area .list-count').should('contain.text', '100 of'); - cy.get('.list-paging-area .btn-more').click(); - cy.get('.list-paging-area .list-count').should('contain.text', '200 of'); - cy.get('.list-paging-area .btn-more').click(); - cy.get('.list-paging-area .list-count').should('contain.text', '300 of'); + cy.get(".list-paging-area .list-count").should("contain.text", "100 of"); + cy.get(".list-paging-area .btn-more").click(); + cy.get(".list-paging-area .list-count").should("contain.text", "200 of"); + cy.get(".list-paging-area .btn-more").click(); + cy.get(".list-paging-area .list-count").should("contain.text", "300 of"); // check if refresh works after load more cy.get('.page-head .standard-actions [data-original-title="Refresh"]').click(); - cy.get('.list-paging-area .list-count').should('contain.text', '300 of'); + cy.get(".list-paging-area .list-count").should("contain.text", "300 of"); cy.get('.list-paging-area .btn-group .btn-paging[data-value="500"]').click(); - cy.get('.list-paging-area .list-count').should('contain.text', '500 of'); - cy.get('.list-paging-area .btn-more').click(); + cy.get(".list-paging-area .list-count").should("contain.text", "500 of"); + cy.get(".list-paging-area .btn-more").click(); - cy.get('.list-paging-area .list-count').should('contain.text', '1000 of'); + cy.get(".list-paging-area .list-count").should("contain.text", "1000 of"); }); }); diff --git a/cypress/integration/list_view.js b/cypress/integration/list_view.js index 3e0d1c9d50..7fb0ef445c 100644 --- a/cypress/integration/list_view.js +++ b/cypress/integration/list_view.js @@ -1,46 +1,67 @@ -context('List View', () => { +context("List View", () => { before(() => { cy.login(); - cy.visit('/app/website'); - return cy.window().its('frappe').then(frappe => { - return frappe.xcall("frappe.tests.ui_test_helpers.setup_workflow"); - }); + cy.visit("/app/website"); + return cy + .window() + .its("frappe") + .then((frappe) => { + return frappe.xcall("frappe.tests.ui_test_helpers.setup_workflow"); + }); }); - it('Keep checkbox checked after Refresh', () => { - cy.go_to_list('ToDo'); - cy.get('.list-row-container .list-row-checkbox').click({ multiple: true, force: true }); - cy.get('.actions-btn-group button').contains('Actions').should('be.visible'); - cy.intercept('/api/method/frappe.desk.reportview.get').as('list-refresh'); + it("Keep checkbox checked after Refresh", () => { + cy.go_to_list("ToDo"); + cy.clear_filters(); + cy.get(".list-row-container .list-row-checkbox").click({ multiple: true, force: true }); + cy.get(".actions-btn-group button").contains("Actions").should("be.visible"); + cy.intercept("/api/method/frappe.desk.reportview.get").as("list-refresh"); cy.wait(3000); // wait before you hit another refresh cy.get('button[data-original-title="Refresh"]').click(); - cy.wait('@list-refresh'); - cy.get('.list-row-container .list-row-checkbox:checked').should('be.visible'); + cy.wait("@list-refresh"); + cy.get(".list-row-container .list-row-checkbox:checked").should("be.visible"); }); it('enables "Actions" button', () => { - const actions = ['Approve', 'Reject', 'Edit', 'Export', 'Assign To', 'Apply Assignment Rule', 'Add Tags', 'Print', 'Delete']; - cy.go_to_list('ToDo'); - cy.get('.list-row-container:contains("Pending") .list-row-checkbox').click({ multiple: true, force: true }); - cy.get('.actions-btn-group button').contains('Actions').should('be.visible').click(); - cy.get('.dropdown-menu li:visible .dropdown-item').should('have.length', 9).each((el, index) => { - cy.wrap(el).contains(actions[index]); - }).then((elements) => { - cy.intercept({ - method: 'POST', - url: 'api/method/frappe.model.workflow.bulk_workflow_approval' - }).as('bulk-approval'); - cy.intercept({ - method: 'POST', - url: 'api/method/frappe.desk.reportview.get' - }).as('real-time-update'); - cy.wrap(elements).contains('Approve').click(); - cy.wait(['@bulk-approval', '@real-time-update']); - cy.wait(300); - cy.get_open_dialog().find('.btn-modal-close').click(); - cy.reload(); - cy.clear_filters(); - cy.get('.list-row-container:visible').should('contain', 'Approved'); + const actions = [ + "Approve", + "Reject", + "Edit", + "Export", + "Assign To", + "Apply Assignment Rule", + "Add Tags", + "Print", + "Delete", + ]; + cy.go_to_list("ToDo"); + cy.clear_filters(); + cy.get('.list-row-container:contains("Pending") .list-row-checkbox').click({ + multiple: true, + force: true, }); + cy.get(".actions-btn-group button").contains("Actions").should("be.visible").click(); + cy.get(".dropdown-menu li:visible .dropdown-item") + .should("have.length", 9) + .each((el, index) => { + cy.wrap(el).contains(actions[index]); + }) + .then((elements) => { + cy.intercept({ + method: "POST", + url: "api/method/frappe.model.workflow.bulk_workflow_approval", + }).as("bulk-approval"); + cy.intercept({ + method: "POST", + url: "api/method/frappe.desk.reportview.get", + }).as("real-time-update"); + cy.wrap(elements).contains("Approve").click(); + cy.wait(["@bulk-approval", "@real-time-update"]); + cy.wait(300); + cy.get_open_dialog().find(".btn-modal-close").click(); + cy.reload(); + cy.clear_filters(); + cy.get(".list-row-container:visible").should("contain", "Approved"); + }); }); }); diff --git a/cypress/integration/list_view_settings.js b/cypress/integration/list_view_settings.js index 61d4b8aae5..5e66ee43f5 100644 --- a/cypress/integration/list_view_settings.js +++ b/cypress/integration/list_view_settings.js @@ -1,36 +1,36 @@ -context('List View Settings', () => { +context("List View Settings", () => { beforeEach(() => { cy.login(); - cy.visit('/app/website'); + cy.visit("/app/website"); }); - it('Default settings', () => { - cy.visit('/app/List/DocType/List'); - cy.get('.list-count').should('contain', "20 of"); - cy.get('.list-stats').should('contain', "Tags"); + it("Default settings", () => { + cy.visit("/app/List/DocType/List"); + cy.get(".list-count").should("contain", "20 of"); + cy.get(".list-stats").should("contain", "Tags"); }); - it('disable count and sidebar stats then verify', () => { + it("disable count and sidebar stats then verify", () => { cy.wait(300); - cy.visit('/app/List/DocType/List'); + cy.visit("/app/List/DocType/List"); cy.wait(300); - cy.get('.list-count').should('contain', "20 of"); - cy.get('.menu-btn-group button').click(); - cy.get('.dropdown-menu li').filter(':visible').contains('List Settings').click(); - cy.get('.modal-dialog').should('contain', 'DocType Settings'); + cy.get(".list-count").should("contain", "20 of"); + cy.get(".menu-btn-group button").click(); + cy.get(".dropdown-menu li").filter(":visible").contains("List Settings").click(); + cy.get(".modal-dialog").should("contain", "DocType Settings"); - cy.findByLabelText('Disable Count').check({ force: true }); - cy.findByLabelText('Disable Sidebar Stats').check({ force: true }); - cy.findByRole('button', {name: 'Save'}).click(); + cy.findByLabelText("Disable Count").check({ force: true }); + cy.findByLabelText("Disable Sidebar Stats").check({ force: true }); + cy.findByRole("button", { name: "Save" }).click(); cy.reload({ force: true }); - cy.get('.list-count').should('be.empty'); - cy.get('.list-sidebar .list-tags').should('not.exist'); + cy.get(".list-count").should("be.empty"); + cy.get(".list-sidebar .list-tags").should("not.exist"); - cy.get('.menu-btn-group button').click({ force: true }); - cy.get('.dropdown-menu li').filter(':visible').contains('List Settings').click(); - cy.get('.modal-dialog').should('contain', 'DocType Settings'); - cy.findByLabelText('Disable Count').uncheck({ force: true }); - cy.findByLabelText('Disable Sidebar Stats').uncheck({ force: true }); - cy.findByRole('button', {name: 'Save'}).click(); + cy.get(".menu-btn-group button").click({ force: true }); + cy.get(".dropdown-menu li").filter(":visible").contains("List Settings").click(); + cy.get(".modal-dialog").should("contain", "DocType Settings"); + cy.findByLabelText("Disable Count").uncheck({ force: true }); + cy.findByLabelText("Disable Sidebar Stats").uncheck({ force: true }); + cy.findByRole("button", { name: "Save" }).click(); }); }); diff --git a/cypress/integration/login.js b/cypress/integration/login.js index 98739bb4c9..2db4b1fdcd 100644 --- a/cypress/integration/login.js +++ b/cypress/integration/login.js @@ -1,68 +1,66 @@ -context('Login', () => { +context("Login", () => { beforeEach(() => { - cy.request('/api/method/logout'); - cy.visit('/login'); - cy.location('pathname').should('eq', '/login'); + cy.request("/api/method/logout"); + cy.visit("/login"); + cy.location("pathname").should("eq", "/login"); }); - it('greets with login screen', () => { - cy.get('.page-card-head').contains('Login'); + it("greets with login screen", () => { + cy.get(".page-card-head").contains("Login"); }); - it('validates password', () => { - cy.get('#login_email').type('Administrator'); - cy.findByRole('button', {name: 'Login'}).click(); - cy.location('pathname').should('eq', '/login'); + it("validates password", () => { + cy.get("#login_email").type("Administrator"); + cy.findByRole("button", { name: "Login" }).click(); + cy.location("pathname").should("eq", "/login"); }); - it('validates email', () => { - cy.get('#login_password').type('qwe'); - cy.findByRole('button', {name: 'Login'}).click(); - cy.location('pathname').should('eq', '/login'); + it("validates email", () => { + cy.get("#login_password").type("qwe"); + cy.findByRole("button", { name: "Login" }).click(); + cy.location("pathname").should("eq", "/login"); }); - it('shows invalid login if incorrect credentials', () => { - cy.get('#login_email').type('Administrator'); - cy.get('#login_password').type('qwer'); + it("shows invalid login if incorrect credentials", () => { + cy.get("#login_email").type("Administrator"); + cy.get("#login_password").type("qwer"); - cy.findByRole('button', {name: 'Login'}).click(); - cy.findByRole('button', {name: 'Invalid Login. Try again.'}).should('exist'); - cy.location('pathname').should('eq', '/login'); + cy.findByRole("button", { name: "Login" }).click(); + cy.findByRole("button", { name: "Invalid Login. Try again." }).should("exist"); + cy.location("pathname").should("eq", "/login"); }); - it('logs in using correct credentials', () => { - cy.get('#login_email').type('Administrator'); - cy.get('#login_password').type(Cypress.config('adminPassword')); + it("logs in using correct credentials", () => { + cy.get("#login_email").type("Administrator"); + cy.get("#login_password").type(Cypress.config("adminPassword")); - cy.findByRole('button', {name: 'Login'}).click(); - cy.location('pathname').should('eq', '/app'); - cy.window().its('frappe.session.user').should('eq', 'Administrator'); + cy.findByRole("button", { name: "Login" }).click(); + cy.location("pathname").should("eq", "/app"); + cy.window().its("frappe.session.user").should("eq", "Administrator"); }); - it('check redirect after login', () => { - + it("check redirect after login", () => { // mock for OAuth 2.0 client_id, redirect_uri, scope and state const payload = new URLSearchParams({ - uuid: '6fed1519-cfd8-4a2d-84a6-9a1799c7c741', - encoded_string: 'hello all', - encoded_url: 'http://test.localhost/callback', - base64_string: 'aGVsbG8gYWxs' + uuid: "6fed1519-cfd8-4a2d-84a6-9a1799c7c741", + encoded_string: "hello all", + encoded_url: "http://test.localhost/callback", + base64_string: "aGVsbG8gYWxs", }); - cy.request('/api/method/logout'); + cy.request("/api/method/logout"); // redirect-to /me page with params to mock OAuth 2.0 like request cy.visit( - '/login?redirect-to=/me?' + - encodeURIComponent(payload.toString().replace("+", " ")) + "/login?redirect-to=/me?" + encodeURIComponent(payload.toString().replace("+", " ")) ); - cy.get('#login_email').type('Administrator'); - cy.get('#login_password').type(Cypress.config('adminPassword')); + cy.get("#login_email").type("Administrator"); + cy.get("#login_password").type(Cypress.config("adminPassword")); - cy.findByRole('button', {name: 'Login'}).click(); + cy.findByRole("button", { name: "Login" }).click(); // verify redirected location and url params after login - cy.url().should('include', '/me?' + payload.toString().replace('+', '%20')); + cy.url().should("include", "/me?" + payload.toString().replace("+", "%20")); }); }); diff --git a/cypress/integration/multi_select_dialog.js b/cypress/integration/multi_select_dialog.js index 607db506c7..1be56d3b3d 100644 --- a/cypress/integration/multi_select_dialog.js +++ b/cypress/integration/multi_select_dialog.js @@ -1,99 +1,102 @@ -context('MultiSelectDialog', () => { +context("MultiSelectDialog", () => { before(() => { cy.login(); - cy.visit('/app'); + cy.visit("/app"); const contact_template = { - "doctype": "Contact", - "first_name": "Test", - "status": "Passive", - "email_ids": [ + doctype: "Contact", + first_name: "Test", + status: "Passive", + email_ids: [ { - "doctype": "Contact Email", - "email_id": "test@example.com", - "is_primary": 0 - } - ] + doctype: "Contact Email", + email_id: "test@example.com", + is_primary: 0, + }, + ], }; - const promises = Array.from({length: 25}) - .map(() => cy.insert_doc('Contact', contact_template, true)); + const promises = Array.from({ length: 25 }).map(() => + cy.insert_doc("Contact", contact_template, true) + ); Promise.all(promises); }); function open_multi_select_dialog() { - cy.window().its('frappe').then(frappe => { - new frappe.ui.form.MultiSelectDialog({ - doctype: "Contact", - target: {}, - setters: { - status: null, - gender: null - }, - add_filters_group: 1, - allow_child_item_selection: 1, - child_fieldname: "email_ids", - child_columns: ["email_id", "is_primary"] + cy.window() + .its("frappe") + .then((frappe) => { + new frappe.ui.form.MultiSelectDialog({ + doctype: "Contact", + target: {}, + setters: { + status: null, + gender: null, + }, + add_filters_group: 1, + allow_child_item_selection: 1, + child_fieldname: "email_ids", + child_columns: ["email_id", "is_primary"], + }); }); - }); } - it('checks multi select dialog api works', () => { + it("checks multi select dialog api works", () => { open_multi_select_dialog(); - cy.get_open_dialog().should('contain', 'Select Contacts'); + cy.get_open_dialog().should("contain", "Select Contacts"); }); - it('checks for filters', () => { - ['search_term', 'status', 'gender'].forEach(fieldname => { - cy.get_open_dialog().get(`.frappe-control[data-fieldname="${fieldname}"]`).should('exist'); + it("checks for filters", () => { + ["search_term", "status", "gender"].forEach((fieldname) => { + cy.get_open_dialog() + .get(`.frappe-control[data-fieldname="${fieldname}"]`) + .should("exist"); }); // add_filters_group: 1 should add a filter group - cy.get_open_dialog().get(`.frappe-control[data-fieldname="filter_area"]`).should('exist'); - + cy.get_open_dialog().get(`.frappe-control[data-fieldname="filter_area"]`).should("exist"); }); - it('checks for child item selection', () => { - cy.get_open_dialog() - .get(`.dt-row-header`).should('not.exist'); + it("checks for child item selection", () => { + cy.get_open_dialog().get(`.dt-row-header`).should("not.exist"); cy.get_open_dialog() .get(`.frappe-control[data-fieldname="allow_child_item_selection"]`) .find('input[data-fieldname="allow_child_item_selection"]') - .should('exist') - .click({force: true}); + .should("exist") + .click({ force: true }); cy.get_open_dialog() .get(`.frappe-control[data-fieldname="child_selection_area"]`) - .should('exist'); + .should("exist"); - cy.get_open_dialog() - .get(`.dt-row-header`).should('contain', 'Contact'); + cy.get_open_dialog().get(`.dt-row-header`).should("contain", "Contact"); - cy.get_open_dialog() - .get(`.dt-row-header`).should('contain', 'Email Id'); + cy.get_open_dialog().get(`.dt-row-header`).should("contain", "Email Id"); - cy.get_open_dialog() - .get(`.dt-row-header`).should('contain', 'Is Primary'); + cy.get_open_dialog().get(`.dt-row-header`).should("contain", "Is Primary"); }); - it('tests more button', () => { + it("tests more button", () => { cy.get_open_dialog() .get(`.frappe-control[data-fieldname="more_child_btn"]`) - .should('exist') - .as('more-btn'); - - cy.get_open_dialog().get('.datatable .dt-scrollable .dt-row').should(($rows) => { - expect($rows).to.have.length(20); - }); + .should("exist") + .as("more-btn"); - cy.intercept('POST', 'api/method/frappe.client.get_list').as('get-more-records'); - cy.get('@more-btn').find('button').click({force: true}); - cy.wait('@get-more-records'); + cy.get_open_dialog() + .get(".datatable .dt-scrollable .dt-row") + .should(($rows) => { + expect($rows).to.have.length(20); + }); - cy.get_open_dialog().get('.datatable .dt-scrollable .dt-row').should(($rows) => { - if ($rows.length <= 20) { - throw new Error("More button doesn't work"); - } - }); + cy.intercept("POST", "api/method/frappe.client.get_list").as("get-more-records"); + cy.get("@more-btn").find("button").click({ force: true }); + cy.wait("@get-more-records"); + cy.get_open_dialog() + .get(".datatable .dt-scrollable .dt-row") + .should(($rows) => { + if ($rows.length <= 20) { + throw new Error("More button doesn't work"); + } + }); }); -}); \ No newline at end of file +}); diff --git a/cypress/integration/navigation.js b/cypress/integration/navigation.js index b4e023c53e..2302296f23 100644 --- a/cypress/integration/navigation.js +++ b/cypress/integration/navigation.js @@ -1,25 +1,29 @@ -context('Navigation', () => { +context("Navigation", () => { before(() => { cy.login(); }); - it('Navigate to route with hash in document name', () => { - cy.insert_doc('ToDo', {'__newname': 'ABC#123', 'description': 'Test this', 'ignore_duplicate': true}); - cy.visit('/app/todo/ABC#123'); - cy.title().should('eq', 'Test this - ABC#123'); - cy.get_field('description', 'Text Editor').contains('Test this'); - cy.go('back'); - cy.title().should('eq', 'Website'); + it("Navigate to route with hash in document name", () => { + cy.insert_doc("ToDo", { + __newname: "ABC#123", + description: "Test this", + ignore_duplicate: true, + }); + cy.visit("/app/todo/ABC#123"); + cy.title().should("eq", "Test this - ABC#123"); + cy.get_field("description", "Text Editor").contains("Test this"); + cy.go("back"); + cy.title().should("eq", "Website"); }); - it.only('Navigate to previous page after login', () => { - cy.visit('/app/todo'); - cy.get('.page-head').findByTitle('To Do').should('be.visible'); - cy.request('/api/method/logout'); - cy.reload().as('reload'); - cy.get('@reload').get('.page-card .btn-primary').contains('Login').click(); - cy.location('pathname').should('eq', '/login'); + it.only("Navigate to previous page after login", () => { + cy.visit("/app/todo"); + cy.get(".page-head").findByTitle("To Do").should("be.visible"); + cy.request("/api/method/logout"); + cy.reload().as("reload"); + cy.get("@reload").get(".page-card .btn-primary").contains("Login").click(); + cy.location("pathname").should("eq", "/login"); cy.login(); - cy.visit('/app'); - cy.location('pathname').should('eq', '/app/todo'); + cy.visit("/app"); + cy.location("pathname").should("eq", "/app/todo"); }); }); diff --git a/cypress/integration/number_card.js b/cypress/integration/number_card.js index a01ff1152d..eb0f19be26 100644 --- a/cypress/integration/number_card.js +++ b/cypress/integration/number_card.js @@ -1,22 +1,22 @@ -context('Number Card', () => { +context("Number Card", () => { before(() => { cy.login(); - cy.visit('/app/website'); + cy.visit("/app/website"); }); - it('Check filter populate for child table doctype', () => { - cy.visit('/app/number-card/new-number-card-1'); - cy.get('[data-fieldname="parent_document_type"]').should('have.css', 'display', 'none'); + it("Check filter populate for child table doctype", () => { + cy.visit("/app/number-card/new-number-card-1"); + cy.get('[data-fieldname="parent_document_type"]').should("have.css", "display", "none"); - cy.get_field('document_type', 'Link'); - cy.fill_field('document_type', 'Workspace Link', 'Link').focus().blur(); - cy.get_field('document_type', 'Link').should('have.value', 'Workspace Link'); + cy.get_field("document_type", "Link"); + cy.fill_field("document_type", "Workspace Link", "Link").focus().blur(); + cy.get_field("document_type", "Link").should("have.value", "Workspace Link"); - cy.fill_field('label', 'Test Number Card', 'Data'); + cy.fill_field("label", "Test Number Card", "Data"); cy.get('[data-fieldname="filters_json"]').click().wait(200); - cy.get('.modal-body .filter-action-buttons .add-filter').click(); - cy.get('.modal-body .fieldname-select-area').click(); - cy.get('.modal-actions .btn-modal-close').click(); + cy.get(".modal-body .filter-action-buttons .add-filter").click(); + cy.get(".modal-body .fieldname-select-area").click(); + cy.get(".modal-actions .btn-modal-close").click(); }); -}); \ No newline at end of file +}); diff --git a/cypress/integration/query_report.js b/cypress/integration/query_report.js index 43f26f8b50..5dd0ab2d53 100644 --- a/cypress/integration/query_report.js +++ b/cypress/integration/query_report.js @@ -1,63 +1,91 @@ -context('Query Report', () => { +context("Query Report", () => { before(() => { cy.login(); - cy.visit('/app/website'); - cy.insert_doc('Report', { - 'report_name': 'Test ToDo Report', - 'ref_doctype': 'ToDo', - 'report_type': 'Query Report', - 'query': 'select * from tabToDo' - }, true).as('doc'); + cy.visit("/app/website"); + cy.insert_doc( + "Report", + { + report_name: "Test ToDo Report", + ref_doctype: "ToDo", + report_type: "Query Report", + query: "select * from tabToDo", + }, + true + ).as("doc"); cy.create_records({ - doctype: 'ToDo', - description: 'this is a test todo for query report' - }).as('todos'); + doctype: "ToDo", + description: "this is a test todo for query report", + }).as("todos"); }); - it('add custom column in report', () => { - cy.visit('/app/query-report/Permitted Documents For User'); + it("add custom column in report", () => { + cy.visit("/app/query-report/Permitted Documents For User"); - cy.get('.page-form.flex', { timeout: 60000 }).should('have.length', 1).then(() => { - cy.get('#page-query-report input[data-fieldname="user"]').as('input-user'); - cy.get('@input-user').focus().type('test@erpnext.com', { delay: 100 }).blur(); - cy.wait(300); - cy.get('#page-query-report input[data-fieldname="doctype"]').as('input-role'); - cy.get('@input-role').focus().type('Role', { delay: 100 }).blur(); + cy.get(".page-form.flex", { timeout: 60000 }) + .should("have.length", 1) + .then(() => { + cy.get('#page-query-report input[data-fieldname="user"]').as("input-user"); + cy.get("@input-user").focus().type("test@erpnext.com", { delay: 100 }).blur(); + cy.wait(300); + cy.get('#page-query-report input[data-fieldname="doctype"]').as("input-role"); + cy.get("@input-role").focus().type("Role", { delay: 100 }).blur(); - cy.get('.datatable').should('exist'); - cy.get('#page-query-report .page-actions .menu-btn-group button').click({ force: true }); - cy.get('#page-query-report .menu-btn-group .dropdown-menu').contains('Add Column').click({ force: true }); - cy.get_open_dialog().get('.modal-title').should('contain', 'Add Column'); - cy.get('select[data-fieldname="doctype"]').select("Role", { force: true }); - cy.get('select[data-fieldname="field"]').select("Role Name", { force: true }); - cy.get('select[data-fieldname="insert_after"]').select("Name", { force: true }); - cy.get_open_dialog().findByRole('button', {name: 'Submit'}).click({ force: true }); - cy.get('#page-query-report .page-actions .menu-btn-group button').click({ force: true }); - cy.get('#page-query-report .menu-btn-group .dropdown-menu').contains('Save').click({ timeout: 100, force: true }); - cy.get_open_dialog().get('.modal-title').should('contain', 'Save Report'); + cy.get(".datatable").should("exist"); + cy.get("#page-query-report .page-actions .menu-btn-group button").click({ + force: true, + }); + cy.get("#page-query-report .menu-btn-group .dropdown-menu") + .contains("Add Column") + .click({ force: true }); + cy.get_open_dialog().get(".modal-title").should("contain", "Add Column"); + cy.get('select[data-fieldname="doctype"]').select("Role", { force: true }); + cy.get('select[data-fieldname="field"]').select("Role Name", { force: true }); + cy.get('select[data-fieldname="insert_after"]').select("Name", { force: true }); + cy.get_open_dialog() + .findByRole("button", { name: "Submit" }) + .click({ force: true }); + cy.get("#page-query-report .page-actions .menu-btn-group button").click({ + force: true, + }); + cy.get("#page-query-report .menu-btn-group .dropdown-menu") + .contains("Save") + .click({ timeout: 100, force: true }); + cy.get_open_dialog().get(".modal-title").should("contain", "Save Report"); - cy.get('input[data-fieldname="report_name"]').type("Test Report", { delay: 100, force: true }); - cy.get_open_dialog().findByRole('button', {name: 'Submit'}).click({ timeout: 1000, force: true }); - }); + cy.get('input[data-fieldname="report_name"]').type("Test Report", { + delay: 100, + force: true, + }); + cy.get_open_dialog() + .findByRole("button", { name: "Submit" }) + .click({ timeout: 1000, force: true }); + }); }); let save_report_and_open = (report, update_name) => { - cy.get('#page-query-report .page-actions .menu-btn-group button').click({ force: true }); - cy.get('#page-query-report .menu-btn-group .dropdown-menu').contains('Save').click({ timeout: 100, force: true }); - cy.get_open_dialog().get('.modal-title').should('contain', 'Save Report'); + cy.get("#page-query-report .page-actions .menu-btn-group button").click({ force: true }); + cy.get("#page-query-report .menu-btn-group .dropdown-menu") + .contains("Save") + .click({ timeout: 100, force: true }); + cy.get_open_dialog().get(".modal-title").should("contain", "Save Report"); - cy.get('input[data-fieldname="report_name"]').type(update_name, { delay: 100, force: true }); - cy.get_open_dialog().findByRole('button', {name: 'Submit'}).click({ timeout: 1000, force: true }); + cy.get('input[data-fieldname="report_name"]').type(update_name, { + delay: 100, + force: true, + }); + cy.get_open_dialog() + .findByRole("button", { name: "Submit" }) + .click({ timeout: 1000, force: true }); - cy.visit('/app/query-report/'+report); - cy.get('.datatable').should('exist'); + cy.visit("/app/query-report/" + report); + cy.get(".datatable").should("exist"); }; - it('test multi level query report', () => { - cy.visit('/app/query-report/Test ToDo Report'); - cy.get('.datatable').should('exist'); + it("test multi level query report", () => { + cy.visit("/app/query-report/Test ToDo Report"); + cy.get(".datatable").should("exist"); - save_report_and_open('Test ToDo Report 1', ' 1'); - save_report_and_open('Test ToDo Report 11', '1'); + save_report_and_open("Test ToDo Report 1", " 1"); + save_report_and_open("Test ToDo Report 11", "1"); }); -}); \ No newline at end of file +}); diff --git a/cypress/integration/recorder.js b/cypress/integration/recorder.js index 7d4c83abf5..de95a852fc 100644 --- a/cypress/integration/recorder.js +++ b/cypress/integration/recorder.js @@ -1,66 +1,72 @@ -context('Recorder', () => { +context.skip("Recorder", () => { before(() => { cy.login(); }); beforeEach(() => { - cy.visit('/app/recorder'); - return cy.window().its('frappe').then(frappe => { - // reset recorder - return frappe.xcall("frappe.recorder.stop").then(() => { - return frappe.xcall("frappe.recorder.delete"); + cy.visit("/app/recorder"); + return cy + .window() + .its("frappe") + .then((frappe) => { + // reset recorder + return frappe.xcall("frappe.recorder.stop").then(() => { + return frappe.xcall("frappe.recorder.delete"); + }); }); - }); }); - it('Recorder Empty State', () => { - cy.get('.page-head').findByTitle('Recorder').should('exist'); + it("Recorder Empty State", () => { + cy.get(".page-head").findByTitle("Recorder").should("exist"); - cy.get('.indicator-pill').should('contain', 'Inactive').should('have.class', 'red'); + cy.get(".indicator-pill").should("contain", "Inactive").should("have.class", "red"); - cy.get('.page-actions').findByRole('button', {name: 'Start'}).should('exist'); - cy.get('.page-actions').findByRole('button', {name: 'Clear'}).should('exist'); + cy.get(".page-actions").findByRole("button", { name: "Start" }).should("exist"); + cy.get(".page-actions").findByRole("button", { name: "Clear" }).should("exist"); - cy.get('.msg-box').should('contain', 'Recorder is Inactive'); - cy.get('.msg-box').findByRole('button', {name: 'Start Recording'}).should('exist'); + cy.get(".msg-box").should("contain", "Recorder is Inactive"); + cy.get(".msg-box").findByRole("button", { name: "Start Recording" }).should("exist"); }); - it('Recorder Start', () => { - cy.get('.page-actions').findByRole('button', {name: 'Start'}).click(); - cy.get('.indicator-pill').should('contain', 'Active').should('have.class', 'green'); + it("Recorder Start", () => { + cy.get(".page-actions").findByRole("button", { name: "Start" }).click(); + cy.get(".indicator-pill").should("contain", "Active").should("have.class", "green"); - cy.get('.msg-box').should('contain', 'No Requests found'); + cy.get(".msg-box").should("contain", "No Requests found"); - cy.visit('/app/List/DocType/List'); - cy.intercept('POST', '/api/method/frappe.desk.reportview.get').as('list_refresh'); - cy.wait('@list_refresh'); + cy.visit("/app/List/DocType/List"); + cy.intercept("POST", "/api/method/frappe.desk.reportview.get").as("list_refresh"); + cy.wait("@list_refresh"); - cy.get('.page-head').findByTitle('DocType').should('exist'); - cy.get('.list-count').should('contain', '20 of '); + cy.get(".page-head").findByTitle("DocType").should("exist"); + cy.get(".list-count").should("contain", "20 of "); - cy.visit('/app/recorder'); - cy.get('.page-head').findByTitle('Recorder').should('exist'); - cy.get('.frappe-list .result-list').should('contain', '/api/method/frappe.desk.reportview.get'); + cy.visit("/app/recorder"); + cy.get(".page-head").findByTitle("Recorder").should("exist"); + cy.get(".frappe-list .result-list").should( + "contain", + "/api/method/frappe.desk.reportview.get" + ); }); - it('Recorder View Request', () => { - cy.get('.page-actions').findByRole('button', {name: 'Start'}).click(); + it("Recorder View Request", () => { + cy.get(".page-actions").findByRole("button", { name: "Start" }).click(); - cy.visit('/app/List/DocType/List'); - cy.intercept('POST', '/api/method/frappe.desk.reportview.get').as('list_refresh'); - cy.wait('@list_refresh'); + cy.visit("/app/List/DocType/List"); + cy.intercept("POST", "/api/method/frappe.desk.reportview.get").as("list_refresh"); + cy.wait("@list_refresh"); - cy.get('.page-head').findByTitle('DocType').should('exist'); - cy.get('.list-count').should('contain', '20 of '); + cy.get(".page-head").findByTitle("DocType").should("exist"); + cy.get(".list-count").should("contain", "20 of "); - cy.visit('/app/recorder'); + cy.visit("/app/recorder"); - cy.get('.frappe-list .list-row-container span') - .contains('/api/method/frappe') - .should('be.visible') - .click({force: true}); + cy.get(".frappe-list .list-row-container span") + .contains("/api/method/frappe") + .should("be.visible") + .click({ force: true }); - cy.url().should('include', '/recorder/request'); - cy.get('form').should('contain', '/api/method/frappe'); + cy.url().should("include", "/recorder/request"); + cy.get("form").should("contain", "/api/method/frappe"); }); }); diff --git a/cypress/integration/report_view.js b/cypress/integration/report_view.js index bacbf9c172..27fe840450 100644 --- a/cypress/integration/report_view.js +++ b/cypress/integration/report_view.js @@ -1,42 +1,46 @@ -import custom_submittable_doctype from '../fixtures/custom_submittable_doctype'; +import custom_submittable_doctype from "../fixtures/custom_submittable_doctype"; const doctype_name = custom_submittable_doctype.name; -context('Report View', () => { +context("Report View", () => { before(() => { cy.login(); - cy.visit('/app/website'); - cy.insert_doc('DocType', custom_submittable_doctype, true); + cy.visit("/app/website"); + cy.insert_doc("DocType", custom_submittable_doctype, true); cy.clear_cache(); - cy.insert_doc(doctype_name, { - 'title': 'Doc 1', - 'description': 'Random Text', - 'enabled': 0, - 'docstatus': 1 // submit document - }, true); + cy.insert_doc( + doctype_name, + { + title: "Doc 1", + description: "Random Text", + enabled: 0, + docstatus: 1, // submit document + }, + true + ); }); - it('Field with enabled allow_on_submit should be editable.', () => { - cy.intercept('POST', 'api/method/frappe.client.set_value').as('value-update'); + it("Field with enabled allow_on_submit should be editable.", () => { + cy.intercept("POST", "api/method/frappe.client.set_value").as("value-update"); cy.visit(`/app/List/${doctype_name}/Report`); // check status column added from docstatus - cy.get('.dt-row-0 > .dt-cell--col-3').should('contain', 'Submitted'); - let cell = cy.get('.dt-row-0 > .dt-cell--col-4'); + cy.get(".dt-row-0 > .dt-cell--col-3").should("contain", "Submitted"); + let cell = cy.get(".dt-row-0 > .dt-cell--col-4"); // select the cell cell.dblclick(); - cell.get('.dt-cell__edit--col-4').findByRole('checkbox').check({ force: true }); - cy.get('.dt-row-0 > .dt-cell--col-3').click(); // click outside + cell.get(".dt-cell__edit--col-4").findByRole("checkbox").check({ force: true }); + cy.get(".dt-row-0 > .dt-cell--col-3").click(); // click outside - cy.wait('@value-update'); + cy.wait("@value-update"); - cy.call('frappe.client.get_value', { + cy.call("frappe.client.get_value", { doctype: doctype_name, filters: { - title: 'Doc 1', + title: "Doc 1", }, - fieldname: 'enabled' - }).then(r => { + fieldname: "enabled", + }).then((r) => { expect(r.message.enabled).to.equals(1); }); }); diff --git a/cypress/integration/routing.js b/cypress/integration/routing.js new file mode 100644 index 0000000000..0822dd9b7d --- /dev/null +++ b/cypress/integration/routing.js @@ -0,0 +1,40 @@ +const list_view = "/app/todo"; + +// test round trip with filter types + +const test_queries = [ + "?status=Open", + `?date=%5B"Between"%2C%5B"2022-06-01"%2C"2022-06-30"%5D%5D`, + `?date=%5B">"%2C"2022-06-01"%5D`, + `?name=%5B"like"%2C"%2542%25"%5D`, + `?status=%5B"not%20in"%2C%5B"Open"%2C"Closed"%5D%5D`, +]; + +describe("SPA Routing", { scrollBehavior: false }, () => { + before(() => { + cy.login(); + cy.go_to_list("ToDo"); + }); + + after(() => { + cy.clear_filters(); // avoid flake in future tests + }); + + it("should apply filter on list view from route", () => { + test_queries.forEach((query) => { + const full_url = `${list_view}${query}`; + cy.visit(full_url); + cy.findByTitle("To Do").should("exist"); + + const expected = new URLSearchParams(query); + cy.location().then((loc) => { + const actual = new URLSearchParams(loc.search); + // This might appear like a dumb test checking visited URL to itself + // but it's actually doing a round trip + // URL with params -> parsed filters -> new URL + // if it's same that means everything worked in between. + expect(actual.toString()).to.eq(expected.toString()); + }); + }); + }); +}); diff --git a/cypress/integration/sidebar.js b/cypress/integration/sidebar.js index 2831c9bad5..6ed2d4022c 100644 --- a/cypress/integration/sidebar.js +++ b/cypress/integration/sidebar.js @@ -1,55 +1,64 @@ -context('Sidebar', () => { +context("Sidebar", () => { before(() => { - cy.visit('/login'); + cy.visit("/login"); cy.login(); - cy.visit('/app/doctype'); + cy.visit("/app/doctype"); }); it('Test for checking "Assigned To" counter value, adding filter and adding & removing an assignment', () => { cy.click_sidebar_button("Assigned To"); //To check if no filter is available in "Assigned To" dropdown - cy.get('.empty-state').should('contain', 'No filters found'); + cy.get(".empty-state").should("contain", "No filters found"); cy.click_sidebar_button("Created By"); //To check if "Created By" dropdown contains filter - cy.get('.group-by-item > .dropdown-item').should('contain', 'Me'); + cy.get(".group-by-item > .dropdown-item").should("contain", "Me"); //Assigning a doctype to a user - cy.visit('/app/doctype/ToDo'); - cy.get('.form-assignments > .flex > .text-muted').click(); - cy.get_field('assign_to_me', 'Check').click(); - cy.get('.modal-footer > .standard-actions > .btn-primary').click(); - cy.visit('/app/doctype'); + cy.visit("/app/doctype/ToDo"); + cy.get(".form-assignments > .flex > .text-muted").click(); + cy.get_field("assign_to_me", "Check").click(); + cy.get(".modal-footer > .standard-actions > .btn-primary").click(); + cy.visit("/app/doctype"); cy.click_sidebar_button("Assigned To"); //To check if filter is added in "Assigned To" dropdown after assignment - cy.get('.group-by-field.show > .dropdown-menu > .group-by-item > .dropdown-item').should('contain', '1'); + cy.get(".group-by-field.show > .dropdown-menu > .group-by-item > .dropdown-item").should( + "contain", + "1" + ); //To check if there is no filter added to the listview - cy.get('.filter-selector > .btn').should('contain', 'Filter'); + cy.get(".filter-selector > .btn").should("contain", "Filter"); //To add a filter to display data into the listview - cy.get('.group-by-field.show > .dropdown-menu > .group-by-item > .dropdown-item').click(); + cy.get(".group-by-field.show > .dropdown-menu > .group-by-item > .dropdown-item").click(); //To check if filter is applied - cy.click_filter_button().should('contain', '1 filter'); - cy.get('.fieldname-select-area > .awesomplete > .form-control').should('have.value', 'Assigned To'); - cy.get('.condition').should('have.value', 'like'); - cy.get('.filter-field > .form-group > .input-with-feedback').should('have.value', '%Administrator%'); + cy.click_filter_button().should("contain", "1 filter"); + cy.get(".fieldname-select-area > .awesomplete > .form-control").should( + "have.value", + "Assigned To" + ); + cy.get(".condition").should("have.value", "like"); + cy.get(".filter-field > .form-group > .input-with-feedback").should( + "have.value", + "%Administrator%" + ); cy.click_filter_button(); //To remove the applied filter cy.clear_filters(); //To remove the assignment - cy.visit('/app/doctype/ToDo'); - cy.get('.assignments > .avatar-group > .avatar > .avatar-frame').click(); - cy.get('.remove-btn').click({force: true}); + cy.visit("/app/doctype/ToDo"); + cy.get(".assignments > .avatar-group > .avatar > .avatar-frame").click(); + cy.get(".remove-btn").click({ force: true }); cy.hide_dialog(); - cy.visit('/app/doctype'); + cy.visit("/app/doctype"); cy.click_sidebar_button("Assigned To"); - cy.get('.empty-state').should('contain', 'No filters found'); + cy.get(".empty-state").should("contain", "No filters found"); }); }); diff --git a/cypress/integration/table_multiselect.js b/cypress/integration/table_multiselect.js index f873461efb..133af44d51 100644 --- a/cypress/integration/table_multiselect.js +++ b/cypress/integration/table_multiselect.js @@ -1,51 +1,57 @@ -context('Table MultiSelect', () => { +context("Table MultiSelect", () => { before(() => { cy.login(); }); - let name = 'table multiselect' + Math.random().toString().slice(2, 8); + let name = "table multiselect" + Math.random().toString().slice(2, 8); - it('select value from multiselect dropdown', () => { - cy.new_form('Assignment Rule'); - cy.fill_field('__newname', name); - cy.fill_field('document_type', 'Blog Post'); - cy.get('.section-head').contains('Assignment Rules').scrollIntoView(); - cy.fill_field('assign_condition', 'status=="Open"', 'Code'); - cy.get('input[data-fieldname="users"]').focus().as('input'); - cy.get('input[data-fieldname="users"] + ul').should('be.visible'); - cy.get('@input').type('test{enter}', { delay: 100 }); - cy.get('.frappe-control[data-fieldname="users"] .form-control .tb-selected-value .btn-link-to-form') - .as('selected-value'); - cy.get('@selected-value').should('contain', 'test@erpnext.com'); + it("select value from multiselect dropdown", () => { + cy.new_form("Assignment Rule"); + cy.fill_field("__newname", name); + cy.fill_field("document_type", "Blog Post"); + cy.get(".section-head").contains("Assignment Rules").scrollIntoView(); + cy.fill_field("assign_condition", 'status=="Open"', "Code"); + cy.get('input[data-fieldname="users"]').focus().as("input"); + cy.get('input[data-fieldname="users"] + ul').should("be.visible"); + cy.get("@input").type("test{enter}", { delay: 100 }); + cy.get( + '.frappe-control[data-fieldname="users"] .form-control .tb-selected-value .btn-link-to-form' + ).as("selected-value"); + cy.get("@selected-value").should("contain", "test@erpnext.com"); - cy.intercept('POST', '/api/method/frappe.desk.form.save.savedocs').as('save_form'); + cy.intercept("POST", "/api/method/frappe.desk.form.save.savedocs").as("save_form"); // trigger save - cy.get('.primary-action').click(); - cy.wait('@save_form').its('response.statusCode').should('eq', 200); - cy.get('@selected-value').should('contain', 'test@erpnext.com'); + cy.get(".primary-action").click(); + cy.wait("@save_form").its("response.statusCode").should("eq", 200); + cy.get("@selected-value").should("contain", "test@erpnext.com"); }); - it('delete value using backspace', () => { - cy.go_to_list('Assignment Rule'); - cy.get(`.list-subject:contains("table multiselect")`).last().find('a').click(); - cy.get('input[data-fieldname="users"]').focus().type('{backspace}'); - cy.get('.frappe-control[data-fieldname="users"] .form-control .tb-selected-value') - .should('not.exist'); + it("delete value using backspace", () => { + cy.go_to_list("Assignment Rule"); + cy.get(`.list-subject:contains("table multiselect")`).last().find("a").click(); + cy.get('input[data-fieldname="users"]').focus().type("{backspace}"); + cy.get('.frappe-control[data-fieldname="users"] .form-control .tb-selected-value').should( + "not.exist" + ); }); - it('delete value using x', () => { - cy.go_to_list('Assignment Rule'); - cy.get(`.list-subject:contains("table multiselect")`).last().find('a').click(); - cy.get('.frappe-control[data-fieldname="users"] .form-control .tb-selected-value').as('existing_value'); - cy.get('@existing_value').find('.btn-remove').click(); - cy.get('@existing_value').should('not.exist'); + it("delete value using x", () => { + cy.go_to_list("Assignment Rule"); + cy.get(`.list-subject:contains("table multiselect")`).last().find("a").click(); + cy.get('.frappe-control[data-fieldname="users"] .form-control .tb-selected-value').as( + "existing_value" + ); + cy.get("@existing_value").find(".btn-remove").click(); + cy.get("@existing_value").should("not.exist"); }); - it('navigate to selected value', () => { - cy.go_to_list('Assignment Rule'); - cy.get(`.list-subject:contains("table multiselect")`).last().find('a').click(); - cy.get('.frappe-control[data-fieldname="users"] .form-control .tb-selected-value').as('existing_value'); - cy.get('@existing_value').find('.btn-link-to-form').click(); - cy.location('pathname').should('contain', '/user/test@erpnext.com'); + it("navigate to selected value", () => { + cy.go_to_list("Assignment Rule"); + cy.get(`.list-subject:contains("table multiselect")`).last().find("a").click(); + cy.get('.frappe-control[data-fieldname="users"] .form-control .tb-selected-value').as( + "existing_value" + ); + cy.get("@existing_value").find(".btn-link-to-form").click(); + cy.location("pathname").should("contain", "/user/test@erpnext.com"); }); }); diff --git a/cypress/integration/theme_switcher_dialog.js b/cypress/integration/theme_switcher_dialog.js index b4297e5674..158ff3e244 100644 --- a/cypress/integration/theme_switcher_dialog.js +++ b/cypress/integration/theme_switcher_dialog.js @@ -1,30 +1,29 @@ -context('Theme Switcher Shortcut', () => { +context("Theme Switcher Shortcut", () => { before(() => { cy.login(); - cy.visit('/app'); + cy.visit("/app"); }); beforeEach(() => { cy.reload(); }); - it('Check Toggle', () => { - cy.open_theme_dialog('{ctrl+shift+g}'); - cy.get('.modal-backdrop').should('exist'); - cy.get('.theme-grid > div').first().click(); - cy.close_theme('{ctrl+shift+g}'); - cy.get('.modal-backdrop').should('not.exist'); + it("Check Toggle", () => { + cy.open_theme_dialog("{ctrl+shift+g}"); + cy.get(".modal-backdrop").should("exist"); + cy.get(".theme-grid > div").first().click(); + cy.close_theme("{ctrl+shift+g}"); + cy.get(".modal-backdrop").should("not.exist"); }); - it('Check Enter', () => { - cy.open_theme_dialog('{ctrl+shift+g}'); - cy.get('.theme-grid > div').first().click(); - cy.close_theme('{enter}'); - cy.get('.modal-backdrop').should('not.exist'); + it("Check Enter", () => { + cy.open_theme_dialog("{ctrl+shift+g}"); + cy.get(".theme-grid > div").first().click(); + cy.close_theme("{enter}"); + cy.get(".modal-backdrop").should("not.exist"); }); - }); -Cypress.Commands.add('open_theme_dialog', (shortcut_keys) => { - cy.get('body').type(shortcut_keys); +Cypress.Commands.add("open_theme_dialog", (shortcut_keys) => { + cy.get("body").type(shortcut_keys); +}); +Cypress.Commands.add("close_theme", (shortcut_keys) => { + cy.get(".modal-header").type(shortcut_keys); }); -Cypress.Commands.add('close_theme', (shortcut_keys) => { - cy.get('.modal-header').type(shortcut_keys); -}); \ No newline at end of file diff --git a/cypress/integration/timeline.js b/cypress/integration/timeline.js index 6c4733400d..5841891af6 100644 --- a/cypress/integration/timeline.js +++ b/cypress/integration/timeline.js @@ -1,89 +1,91 @@ -import custom_submittable_doctype from '../fixtures/custom_submittable_doctype'; +import custom_submittable_doctype from "../fixtures/custom_submittable_doctype"; -context('Timeline', () => { +context("Timeline", () => { before(() => { - cy.visit('/login'); + cy.visit("/login"); cy.login(); }); - it('Adding new ToDo, adding new comment, verifying comment addition & deletion and deleting ToDo', () => { + it("Adding new ToDo, adding new comment, verifying comment addition & deletion and deleting ToDo", () => { //Adding new ToDo - cy.visit('/app/todo/new-todo-1'); - cy.get('[data-fieldname="description"] .ql-editor.ql-blank').type('Test ToDo', {force: true}).wait(200); - cy.get('.page-head .page-actions').findByRole('button', {name: 'Save'}).click(); + cy.visit("/app/todo/new-todo-1"); + cy.get('[data-fieldname="description"] .ql-editor.ql-blank') + .type("Test ToDo", { force: true }) + .wait(200); + cy.get(".page-head .page-actions").findByRole("button", { name: "Save" }).click(); - cy.visit('/app/todo'); + cy.go_to_list("ToDo"); + cy.clear_filters(); cy.click_listview_row_item(0); //To check if the comment box is initially empty and tying some text into it - cy.get('[data-fieldname="comment"] .ql-editor').should('contain', '').type('Testing Timeline'); + cy.get('[data-fieldname="comment"] .ql-editor') + .should("contain", "") + .type("Testing Timeline"); //Adding new comment - cy.get('.comment-box').findByRole('button', {name: 'Comment'}).click(); + cy.get(".comment-box").findByRole("button", { name: "Comment" }).click(); //To check if the commented text is visible in the timeline content - cy.get('.timeline-content').should('contain', 'Testing Timeline'); + cy.get(".timeline-content").should("contain", "Testing Timeline"); //Editing comment cy.click_timeline_action_btn("Edit"); - cy.get('.timeline-content [data-fieldname="comment"] .ql-editor').first().type(' 123'); + cy.get('.timeline-content [data-fieldname="comment"] .ql-editor').first().type(" 123"); cy.click_timeline_action_btn("Save"); //To check if the edited comment text is visible in timeline content - cy.get('.timeline-content').should('contain', 'Testing Timeline 123'); + cy.get(".timeline-content").should("contain", "Testing Timeline 123"); //Discarding comment cy.click_timeline_action_btn("Edit"); cy.click_timeline_action_btn("Dismiss"); //To check if after discarding the timeline content is same as previous - cy.get('.timeline-content').should('contain', 'Testing Timeline 123'); + cy.get(".timeline-content").should("contain", "Testing Timeline 123"); //Deleting the added comment - cy.get('.timeline-message-box .more-actions > .action-btn').click(); //Menu button in timeline item - cy.get('.timeline-message-box .more-actions .dropdown-item').contains('Delete').click({ force: true }); - cy.get_open_dialog().findByRole('button', {name: 'Yes'}).click({ force: true }); + cy.get(".timeline-message-box .more-actions > .action-btn").click(); //Menu button in timeline item + cy.get(".timeline-message-box .more-actions .dropdown-item") + .contains("Delete") + .click({ force: true }); + cy.get_open_dialog().findByRole("button", { name: "Yes" }).click({ force: true }); - cy.get('.timeline-content').should('not.contain', 'Testing Timeline 123'); + cy.get(".timeline-content").should("not.contain", "Testing Timeline 123"); }); - it('Timeline should have submit and cancel activity information', () => { - cy.visit('/app/doctype'); + it("Timeline should have submit and cancel activity information", () => { + cy.visit("/app/doctype"); //Creating custom doctype - cy.insert_doc('DocType', custom_submittable_doctype, true); + cy.insert_doc("DocType", custom_submittable_doctype, true); - cy.visit('/app/custom-submittable-doctype'); - cy.click_listview_primary_button('Add Custom Submittable DocType'); + cy.visit("/app/custom-submittable-doctype"); + cy.click_listview_primary_button("Add Custom Submittable DocType"); //Adding a new entry for the created custom doctype - cy.fill_field('title', 'Test'); - cy.click_modal_primary_button('Save'); - cy.click_modal_primary_button('Submit'); + cy.fill_field("title", "Test"); + cy.click_modal_primary_button("Save"); + cy.click_modal_primary_button("Submit"); - cy.visit('/app/custom-submittable-doctype'); + cy.visit("/app/custom-submittable-doctype"); cy.click_listview_row_item(0); //To check if the submission of the documemt is visible in the timeline content - cy.get('.timeline-content').should('contain', 'Administrator submitted this document'); - cy.get('[id="page-Custom Submittable DocType"] .page-actions').findByRole('button', {name: 'Cancel'}).click(); - cy.get_open_dialog().findByRole('button', {name: 'Yes'}).click(); + cy.get(".timeline-content").should("contain", "Administrator submitted this document"); + cy.get('[id="page-Custom Submittable DocType"] .page-actions') + .findByRole("button", { name: "Cancel" }) + .click(); + cy.get_open_dialog().findByRole("button", { name: "Yes" }).click(); //To check if the cancellation of the documemt is visible in the timeline content - cy.get('.timeline-content').should('contain', 'Administrator cancelled this document'); + cy.get(".timeline-content").should("contain", "Administrator cancelled this document"); //Deleting the document - cy.visit('/app/custom-submittable-doctype'); + cy.visit("/app/custom-submittable-doctype"); cy.select_listview_row_checkbox(0); - cy.get('.page-actions').findByRole('button', {name: 'Actions'}).click(); + cy.get(".page-actions").findByRole("button", { name: "Actions" }).click(); cy.get('.page-actions .actions-btn-group [data-label="Delete"]').click(); - cy.click_modal_primary_button('Yes'); - - //Deleting the custom doctype - cy.visit('/app/doctype'); - cy.select_listview_row_checkbox(0); - cy.get('.page-actions').findByRole('button', {name: 'Actions'}).click(); - cy.get('.page-actions .actions-btn-group [data-label="Delete"]').click(); - cy.click_modal_primary_button('Yes'); + cy.click_modal_primary_button("Yes"); }); -}); \ No newline at end of file +}); diff --git a/cypress/integration/timeline_email.js b/cypress/integration/timeline_email.js index 5808bd52ef..3a22f49bfa 100644 --- a/cypress/integration/timeline_email.js +++ b/cypress/integration/timeline_email.js @@ -1,76 +1,93 @@ -context('Timeline Email', () => { +context("Timeline Email", () => { before(() => { - cy.visit('/login'); + cy.visit("/login"); cy.login(); - cy.visit('/app/todo'); + cy.visit("/app/todo"); }); - it('Adding new ToDo', () => { - cy.click_listview_primary_button('Add ToDo'); - cy.get('.custom-actions:visible > .btn').contains("Edit in full page").click({delay: 500}); - cy.fill_field("description", "Test ToDo", "Text Editor"); + it("Adding new ToDo", () => { + cy.click_listview_primary_button("Add ToDo"); + cy.get(".custom-actions:visible > .btn").contains("Edit Full Form").click({ delay: 500 }); + cy.fill_field("description", "Test ToDo", "Text Editor"); cy.wait(500); - cy.get('.primary-action').contains('Save').click({force: true}); + cy.get(".primary-action").contains("Save").click({ force: true }); cy.wait(700); }); - it('Adding email and verifying timeline content for email attachment', () => { - cy.visit('/app/todo'); - cy.get('.list-row > .level-left > .list-subject').eq(0).click(); + it("Adding email and verifying timeline content for email attachment", () => { + cy.visit("/app/todo"); + cy.click_listview_row_item_with_text("Test ToDo"); //Creating a new email - cy.get('.timeline-actions > .timeline-item > .action-buttons > .action-btn').click(); - cy.fill_field('recipients', 'test@example.com', 'MultiSelect'); - cy.get('.modal.show > .modal-dialog > .modal-content > .modal-body > :nth-child(1) > .form-layout > .form-page > :nth-child(3) > .section-body > .form-column > form > [data-fieldtype="Text Editor"] > .form-group > .control-input-wrapper > .control-input > .ql-container > .ql-editor').type('Test Mail'); + cy.get(".timeline-actions > .timeline-item > .action-buttons > .action-btn").click(); + cy.fill_field("recipients", "test@example.com", "MultiSelect"); + cy.get( + '.modal.show > .modal-dialog > .modal-content > .modal-body > :nth-child(1) > .form-layout > .form-page > :nth-child(3) > .section-body > .form-column > form > [data-fieldtype="Text Editor"] > .form-group > .control-input-wrapper > .control-input > .ql-container > .ql-editor' + ).type("Test Mail"); //Adding attachment to the email - cy.get('.add-more-attachments > .btn').click(); - cy.get('.mt-2 > .btn > .mt-1').eq(2).click(); - cy.get('.input-group > .form-control').type('https://wallpaperplay.com/walls/full/8/2/b/72402.jpg'); - cy.get('.btn-primary').contains('Upload').click(); + cy.get(".add-more-attachments > .btn").click(); + cy.get(".mt-2 > .btn > .mt-1").eq(2).click(); + cy.get(".input-group > .form-control").type( + "https://wallpaperplay.com/walls/full/8/2/b/72402.jpg" + ); + cy.get(".btn-primary").contains("Upload").click(); //Sending the email - cy.click_modal_primary_button('Send', {delay: 500}); + cy.click_modal_primary_button("Send", { delay: 500 }); //To check if the sent mail content is shown in the timeline content - cy.get('[data-doctype="Communication"] > .timeline-content').should('contain', 'Test Mail'); + cy.get('[data-doctype="Communication"] > .timeline-content').should( + "contain", + "Test Mail" + ); //To check if the attachment of email is shown in the timeline content - cy.get('.timeline-content').should('contain', 'Added 72402.jpg'); + cy.get(".timeline-content").should("contain", "Added 72402.jpg"); //Deleting the sent email - cy.get('[title="Open Communication"] > .icon').first().click({force: true}); - cy.get('#page-Communication > .page-head > .container > .row > .col > .standard-actions > .menu-btn-group > .btn').click(); - cy.get('#page-Communication > .page-head > .container > .row > .col > .standard-actions > .menu-btn-group > .dropdown-menu > li > .grey-link').eq(9).click(); - cy.get('.modal.show > .modal-dialog > .modal-content > .modal-footer > .standard-actions > .btn-primary').click(); + cy.get('[title="Open Communication"] > .icon').first().click({ force: true }); + cy.get( + "#page-Communication > .page-head > .container > .row > .col > .standard-actions > .menu-btn-group > .btn" + ).click(); + cy.get( + "#page-Communication > .page-head > .container > .row > .col > .standard-actions > .menu-btn-group > .dropdown-menu > li > .grey-link" + ) + .eq(9) + .click(); + cy.get( + ".modal.show > .modal-dialog > .modal-content > .modal-footer > .standard-actions > .btn-primary" + ).click(); }); - it('Deleting attachment and ToDo', () => { - cy.visit('/app/todo'); - cy.get('.list-row > .level-left > .list-subject > .level-item.ellipsis > .ellipsis').eq(0).click(); + it("Deleting attachment and ToDo", () => { + cy.visit("/app/todo"); + cy.click_listview_row_item_with_text("Test ToDo"); //Removing the added attachment - cy.get('.attachment-row > .data-pill > .remove-btn > .icon').click(); + cy.get(".attachment-row > .data-pill > .remove-btn > .icon").click(); cy.wait(500); - cy.get('.modal-footer:visible > .standard-actions > .btn-primary').contains('Yes').click(); + cy.get(".modal-footer:visible > .standard-actions > .btn-primary").contains("Yes").click(); //To check if the removed attachment is shown in the timeline content - cy.get('.timeline-content').should('contain', 'Removed 72402.jpg'); + cy.get(".timeline-content").should("contain", "Removed 72402.jpg"); cy.wait(500); //To check if the discard button functionality in email is working correctly - cy.get('.timeline-actions > .timeline-item > .action-buttons > .action-btn').click(); - cy.fill_field('recipients', 'test@example.com', 'MultiSelect'); - cy.get('.modal-footer > .standard-actions > .btn-secondary').contains('Discard').click(); + cy.get(".timeline-actions > .timeline-item > .action-buttons > .action-btn").click(); + cy.fill_field("recipients", "test@example.com", "MultiSelect"); + cy.get(".modal-footer > .standard-actions > .btn-secondary").contains("Discard").click(); cy.wait(500); - cy.get('.timeline-actions > .timeline-item > .action-buttons > .action-btn').click(); + cy.get(".timeline-actions > .timeline-item > .action-buttons > .action-btn").click(); cy.wait(500); - cy.get_field('recipients', 'MultiSelect').should('have.text', ''); - cy.get('.modal-header:visible > .modal-actions > .btn-modal-close > .icon').click(); + cy.get_field("recipients", "MultiSelect").should("have.text", ""); + cy.get(".modal-header:visible > .modal-actions > .btn-modal-close > .icon").click(); //Deleting the added ToDo - cy.get('.menu-btn-group:visible > .btn').click(); - cy.get('.menu-btn-group:visible > .dropdown-menu > li > .dropdown-item').contains('Delete').click(); - cy.get('.modal-footer:visible > .standard-actions > .btn-primary').click(); + cy.get(".menu-btn-group:visible > .btn").click(); + cy.get(".menu-btn-group:visible > .dropdown-menu > li > .dropdown-item") + .contains("Delete") + .click(); + cy.get(".modal-footer:visible > .standard-actions > .btn-primary").click(); }); }); diff --git a/cypress/integration/url_data_field.js b/cypress/integration/url_data_field.js index cf22c62363..c74bc8b1a2 100644 --- a/cypress/integration/url_data_field.js +++ b/cypress/integration/url_data_field.js @@ -1,43 +1,42 @@ -import data_field_validation_doctype from '../fixtures/data_field_validation_doctype'; +import data_field_validation_doctype from "../fixtures/data_field_validation_doctype"; const doctype_name = data_field_validation_doctype.name; -context('URL Data Field Input', () => { +context("URL Data Field Input", () => { before(() => { cy.login(); - cy.visit('/app/website'); - return cy.insert_doc('DocType', data_field_validation_doctype, true); + cy.visit("/app/website"); + return cy.insert_doc("DocType", data_field_validation_doctype, true); }); - - describe('URL Data Field Input ', () => { - it('should not show URL link button without focus', () => { + describe("URL Data Field Input ", () => { + it("should not show URL link button without focus", () => { cy.new_form(doctype_name); - cy.get_field('url').clear().type('https://frappe.io'); - cy.get_field('url').blur().wait(500); - cy.get('.link-btn').should('not.be.visible'); + cy.get_field("url").clear().type("https://frappe.io"); + cy.get_field("url").blur().wait(500); + cy.get(".link-btn").should("not.be.visible"); }); - it('should show URL link button on focus', () => { - cy.get_field('url').focus().wait(500); - cy.get('.link-btn').should('be.visible'); + it("should show URL link button on focus", () => { + cy.get_field("url").focus().wait(500); + cy.get(".link-btn").should("be.visible"); }); - it('should not show URL link button for invalid URL', () => { - cy.get_field('url').clear().type('fuzzbuzz'); - cy.get('.link-btn').should('not.be.visible'); + it("should not show URL link button for invalid URL", () => { + cy.get_field("url").clear().type("fuzzbuzz"); + cy.get(".link-btn").should("not.be.visible"); }); - it('should have valid URL link with target _blank', () => { - cy.get_field('url').clear().type('https://frappe.io'); - cy.get('.link-btn .btn-open').should('have.attr', 'href', 'https://frappe.io'); - cy.get('.link-btn .btn-open').should('have.attr', 'target', '_blank'); + it("should have valid URL link with target _blank", () => { + cy.get_field("url").clear().type("https://frappe.io"); + cy.get(".link-btn .btn-open").should("have.attr", "href", "https://frappe.io"); + cy.get(".link-btn .btn-open").should("have.attr", "target", "_blank"); }); - it('should inject anchor tag in read-only URL data field', () => { + it("should inject anchor tag in read-only URL data field", () => { cy.get('[data-fieldname="read_only_url"]') - .find('a') - .should('have.attr', 'target', '_blank'); + .find("a") + .should("have.attr", "target", "_blank"); }); }); -}); \ No newline at end of file +}); diff --git a/cypress/integration/web_form.js b/cypress/integration/web_form.js index bd1c7e147e..66975ce19b 100644 --- a/cypress/integration/web_form.js +++ b/cypress/integration/web_form.js @@ -1,27 +1,264 @@ -context('Web Form', () => { +context("Web Form", () => { before(() => { cy.login(); }); - it('Navigate and Submit a WebForm', () => { - cy.visit('/update-profile'); - cy.get_field('last_name', 'Data').type('_Test User', {force: true}).wait(200); - cy.get('.web-form-actions .btn-primary').click(); - cy.wait(5000); - cy.url().should('include', '/me'); + it("Create Web Form", () => { + cy.visit("/app/web-form/new"); + + cy.intercept("POST", "/api/method/frappe.desk.form.save.savedocs").as("save_form"); + + cy.fill_field("title", "Note"); + cy.fill_field("doc_type", "Note", "Link"); + cy.fill_field("module", "Website", "Link"); + cy.click_custom_action_button("Get Fields"); + cy.click_custom_action_button("Publish"); + + cy.wait("@save_form"); + + cy.get_field("route").should("have.value", "note"); + cy.get(".title-area .indicator-pill").contains("Published"); }); - it('Navigate and Submit a MultiStep WebForm', () => { - cy.call('frappe.tests.ui_test_helpers.update_webform_to_multistep').then(() => { - cy.visit('/update-profile-duplicate'); - cy.get_field('last_name', 'Data').type('_Test User', {force: true}).wait(200); - cy.get('.btn-next').should('be.visible'); - cy.get('.btn-next').click(); - cy.get('.btn-previous').should('be.visible'); - cy.get('.btn-next').should('not.be.visible'); - cy.get('.web-form-actions .btn-primary').click(); - cy.wait(5000); - cy.url().should('include', '/me'); + it("Open Web Form", () => { + cy.visit("/note"); + cy.fill_field("title", "Note 1"); + cy.get(".web-form-actions button").contains("Save").click(); + + cy.url().should("include", "/note/new"); + + cy.request("/api/method/logout"); + cy.visit("/note"); + + cy.url().should("include", "/note/new"); + + cy.fill_field("title", "Guest Note 1"); + cy.get(".web-form-actions button").contains("Save").click(); + + cy.url().should("include", "/note/new"); + + cy.visit("/note"); + cy.url().should("include", "/note/new"); + }); + + it("Login Required", () => { + cy.login(); + cy.visit("/app/web-form/note"); + + cy.findByRole("tab", { name: "Form Settings" }).click(); + cy.get('input[data-fieldname="login_required"]').check({ force: true }); + + cy.save(); + + cy.visit("/note"); + cy.url().should("include", "/note/Note%201"); + + cy.call("logout"); + + cy.visit("/note"); + cy.get_open_dialog() + .get(".modal-message") + .contains("You are not permitted to access this page without login."); + }); + + it("Show List", () => { + cy.login(); + cy.visit("/app/web-form/note"); + + cy.findByRole("tab", { name: "List Settings" }).click(); + cy.get('input[data-fieldname="show_list"]').check(); + + cy.save(); + + cy.visit("/note"); + cy.url().should("include", "/note/list"); + cy.get(".web-list-table").should("be.visible"); + }); + + it("Show Custom List Title", () => { + cy.visit("/app/web-form/note"); + + cy.findByRole("tab", { name: "List Settings" }).click(); + cy.fill_field("list_title", "Note List"); + + cy.save(); + + cy.visit("/note"); + cy.url().should("include", "/note/list"); + cy.get(".web-list-header h1").should("contain.text", "Note List"); + }); + + it("Show Custom List Columns", () => { + cy.visit("/note"); + cy.url().should("include", "/note/list"); + + cy.get(".web-list-table thead th").contains("Name"); + cy.get(".web-list-table thead th").contains("Title"); + + cy.visit("/app/web-form/note"); + + cy.findByRole("tab", { name: "List Settings" }).click(); + + cy.get('[data-fieldname="list_columns"] .grid-footer button') + .contains("Add Row") + .as("add-row"); + + cy.get("@add-row").click(); + cy.get('[data-fieldname="list_columns"] .grid-body .rows').as("grid-rows"); + cy.get("@grid-rows").find('.grid-row:first [data-fieldname="fieldname"]').click(); + cy.get("@grid-rows") + .find('.grid-row:first select[data-fieldname="fieldname"]') + .select("Title (Data)"); + + cy.get("@add-row").click(); + cy.get("@grid-rows").find('.grid-row[data-idx="2"] [data-fieldname="fieldname"]').click(); + cy.get("@grid-rows") + .find('.grid-row[data-idx="2"] select[data-fieldname="fieldname"]') + .select("Public (Check)"); + + cy.get("@add-row").click(); + cy.get("@grid-rows").find('.grid-row:last [data-fieldname="fieldname"]').click(); + cy.get("@grid-rows") + .find('.grid-row:last select[data-fieldname="fieldname"]') + .select("Content (Text Editor)"); + + cy.save(); + + cy.visit("/note"); + cy.url().should("include", "/note/list"); + cy.get(".web-list-table thead th").contains("Title"); + cy.get(".web-list-table thead th").contains("Public"); + cy.get(".web-list-table thead th").contains("Content"); + }); + + it("Breadcrumbs", () => { + cy.visit("/note/Note 1"); + cy.get(".breadcrumb-container .breadcrumb .breadcrumb-item:first a") + .should("contain.text", "Note") + .click(); + cy.url().should("include", "/note/list"); + }); + + it("Custom Breadcrumbs", () => { + cy.visit("/app/web-form/note"); + + cy.findByRole("tab", { name: "Form Settings" }).click(); + cy.get(".form-section .section-head").contains("Customization").click(); + cy.fill_field("breadcrumbs", '[{"label": _("Notes"), "route":"note"}]', "Code"); + cy.get(".form-section .section-head").contains("Customization").click(); + cy.save(); + + cy.visit("/note/Note 1"); + cy.get(".breadcrumb-container .breadcrumb .breadcrumb-item:first a").should( + "contain.text", + "Notes" + ); + }); + + it("Read Only", () => { + cy.login(); + cy.visit("/note"); + cy.url().should("include", "/note/list"); + + // Read Only Field + cy.get('.web-list-table tbody tr[id="Note 1"]').click(); + cy.get('.frappe-control[data-fieldname="title"] .control-input').should( + "have.css", + "display", + "none" + ); + }); + + it("Edit Mode", () => { + cy.visit("/app/web-form/note"); + + cy.findByRole("tab", { name: "Form Settings" }).click(); + cy.get('input[data-fieldname="allow_edit"]').check(); + + cy.save(); + + cy.visit("/note/Note 1"); + cy.url().should("include", "/note/Note%201"); + + cy.get(".web-form-actions a").contains("Edit").click(); + cy.url().should("include", "/note/Note%201/edit"); + + // Editable Field + cy.get_field("title").should("have.value", "Note 1"); + + cy.fill_field("title", " Edited"); + cy.get(".web-form-actions button").contains("Save").click(); + cy.get_field("title").should("have.value", "Note 1 Edited"); + }); + + it("Allow Multiple Response", () => { + cy.visit("/app/web-form/note"); + + cy.findByRole("tab", { name: "Form Settings" }).click(); + cy.get('input[data-fieldname="allow_multiple"]').check(); + + cy.save(); + + cy.visit("/note"); + cy.url().should("include", "/note/list"); + + cy.get(".web-list-actions a:visible").contains("New").click(); + cy.url().should("include", "/note/new"); + + cy.fill_field("title", "Note 2"); + cy.get(".web-form-actions button").contains("Save").click(); + }); + + it("Allow Delete", () => { + cy.visit("/app/web-form/note"); + + cy.findByRole("tab", { name: "Form Settings" }).click(); + cy.get('input[data-fieldname="allow_delete"]').check(); + + cy.save(); + + cy.visit("/note"); + cy.url().should("include", "/note/list"); + + cy.get('.web-list-table tbody tr[id="Note 1"] .list-col-checkbox input').click(); + cy.get('.web-list-table tbody tr[id="Note 2"] .list-col-checkbox input').click(); + cy.get(".web-list-actions button:visible").contains("Delete").click({ force: true }); + + cy.get(".web-list-actions button").contains("Delete").should("not.be.visible"); + + cy.visit("/note"); + cy.get('.web-list-table tbody tr[id="Note 1"]').should("not.exist"); + cy.get('.web-list-table tbody tr[id="Note 2"]').should("not.exist"); + cy.get('.web-list-table tbody tr[id="Guest Note 1"]').should("exist"); + }); + + it("Navigate and Submit a WebForm", () => { + cy.visit("/update-profile"); + + cy.get(".web-form-actions a").contains("Edit").click(); + + cy.fill_field("last_name", "_Test User"); + + cy.get(".web-form-actions .btn-primary").click(); + cy.url().should("include", "/me"); + }); + + it("Navigate and Submit a MultiStep WebForm", () => { + cy.call("frappe.tests.ui_test_helpers.update_webform_to_multistep").then(() => { + cy.visit("/update-profile-duplicate"); + + cy.get(".web-form-actions a").contains("Edit").click(); + + cy.fill_field("last_name", "_Test User"); + + cy.get(".btn-next").should("be.visible"); + cy.get(".btn-next").click(); + + cy.get(".btn-previous").should("be.visible"); + cy.get(".btn-next").should("not.be.visible"); + + cy.get(".web-form-actions .btn-primary").click(); + cy.url().should("include", "/me"); }); }); }); diff --git a/cypress/integration/workspace.js b/cypress/integration/workspace.js index fbff451305..e7d97c705b 100644 --- a/cypress/integration/workspace.js +++ b/cypress/integration/workspace.js @@ -1,88 +1,214 @@ -context('Workspace 2.0', () => { +context("Workspace 2.0", () => { before(() => { - cy.visit('/login'); + cy.visit("/login"); cy.login(); - cy.visit('/app/website'); }); - it('Navigate to page from sidebar', () => { - cy.visit('/app/build'); - cy.get('.codex-editor__redactor .ce-block'); + it("Navigate to page from sidebar", () => { + cy.visit("/app/build"); + cy.get(".codex-editor__redactor .ce-block"); cy.get('.sidebar-item-container[item-name="Settings"]').first().click(); - cy.location('pathname').should('eq', '/app/settings'); + cy.location("pathname").should("eq", "/app/settings"); }); - it('Create Private Page', () => { - cy.get('.codex-editor__redactor .ce-block'); + it("Create Private Page", () => { + cy.intercept({ + method: "POST", + url: "api/method/frappe.desk.doctype.workspace.workspace.new_page", + }).as("new_page"); + + cy.get(".codex-editor__redactor .ce-block"); cy.get('.custom-actions button[data-label="Create%20Workspace"]').click(); - cy.fill_field('title', 'Test Private Page', 'Data'); - cy.fill_field('icon', 'edit', 'Icon'); - cy.get_open_dialog().find('.modal-header').click(); - cy.get_open_dialog().find('.btn-primary').click(); + cy.fill_field("title", "Test Private Page", "Data"); + cy.fill_field("icon", "edit", "Icon"); + cy.get_open_dialog().find(".modal-header").click(); + cy.get_open_dialog().find(".btn-primary").click(); // check if sidebar item is added in pubic section - cy.get('.sidebar-item-container[item-name="Test Private Page"]').should('have.attr', 'item-public', '0'); + cy.get('.sidebar-item-container[item-name="Test Private Page"]').should( + "have.attr", + "item-public", + "0" + ); cy.get('.standard-actions .btn-primary[data-label="Save"]').click(); cy.wait(300); - cy.get('.sidebar-item-container[item-name="Test Private Page"]').should('have.attr', 'item-public', '0'); + cy.get('.sidebar-item-container[item-name="Test Private Page"]').should( + "have.attr", + "item-public", + "0" + ); - cy.wait(500); - cy.get('.codex-editor__redactor .ce-block'); - cy.get('.standard-actions .btn-secondary[data-label=Edit]').click(); + cy.wait("@new_page"); }); - it('Add New Block', () => { - cy.get('.ce-block').click().type('{enter}'); - cy.get('.block-list-container .block-list-item').contains('Heading').click(); - cy.get(":focus").type('Header'); - cy.get(".ce-block:last").find('.ce-header').should('exist'); + it("Create Child Page", () => { + cy.intercept({ + method: "POST", + url: "api/method/frappe.desk.doctype.workspace.workspace.new_page", + }).as("new_page"); - cy.get('.ce-block:last').click().type('{enter}'); - cy.get('.block-list-container .block-list-item').contains('Text').click(); - cy.get(":focus").type('Paragraph text'); - cy.get(".ce-block:last").find('.ce-paragraph').should('exist'); - }); + cy.get(".codex-editor__redactor .ce-block"); + cy.get('.custom-actions button[data-label="Create%20Workspace"]').click(); + cy.fill_field("title", "Test Child Page", "Data"); + cy.fill_field("parent", "Test Private Page", "Select"); + cy.fill_field("icon", "edit", "Icon"); + cy.get_open_dialog().find(".modal-header").click(); + cy.get_open_dialog().find(".btn-primary").click(); - it('Delete A Block', () => { - cy.get(":focus").click(); - cy.get('.paragraph-control .setting-btn').click(); - cy.get('.paragraph-control .dropdown-item').contains('Delete').click(); - cy.get(".ce-block:last").find('.ce-paragraph').should('not.exist'); - }); - - it('Shrink and Expand A Block', () => { - cy.get(":focus").click(); - cy.get('.ce-block:last .setting-btn').click(); - cy.get('.ce-block:last .dropdown-item').contains('Shrink').click(); - cy.get(".ce-block:last").should('have.class', 'col-xs-11'); - cy.get('.ce-block:last .dropdown-item').contains('Shrink').click(); - cy.get(".ce-block:last").should('have.class', 'col-xs-10'); - cy.get('.ce-block:last .dropdown-item').contains('Shrink').click(); - cy.get(".ce-block:last").should('have.class', 'col-xs-9'); - cy.get('.ce-block:last .dropdown-item').contains('Expand').click(); - cy.get(".ce-block:last").should('have.class', 'col-xs-10'); - cy.get('.ce-block:last .dropdown-item').contains('Expand').click(); - cy.get(".ce-block:last").should('have.class', 'col-xs-11'); - cy.get('.ce-block:last .dropdown-item').contains('Expand').click(); - cy.get(".ce-block:last").should('have.class', 'col-xs-12'); + // check if sidebar item is added in pubic section + cy.get('.sidebar-item-container[item-name="Test Child Page"]').should( + "have.attr", + "item-public", + "0" + ); cy.get('.standard-actions .btn-primary[data-label="Save"]').click(); - }); - - it('Delete Private Page', () => { - cy.get('.codex-editor__redactor .ce-block'); - cy.get('.standard-actions .btn-secondary[data-label=Edit]').click(); - - cy.get('.sidebar-item-container[item-name="Test Private Page"]') - .find('.sidebar-item-control .setting-btn').click(); - cy.get('.sidebar-item-container[item-name="Test Private Page"]') - .find('.dropdown-item[title="Delete Workspace"]').click({force: true}); cy.wait(300); - cy.get('.modal-footer > .standard-actions > .btn-modal-primary:visible').first().click(); - cy.get('.standard-actions .btn-primary[data-label="Save"]').click(); - cy.get('.codex-editor__redactor .ce-block'); - cy.get('.sidebar-item-container[item-name="Test Private Page"]').should('not.exist'); + cy.get('.sidebar-item-container[item-name="Test Child Page"]').should( + "have.attr", + "item-public", + "0" + ); + + cy.wait("@new_page"); }); -}); \ No newline at end of file + it("Duplicate Page", () => { + cy.intercept({ + method: "POST", + url: "api/method/frappe.desk.doctype.workspace.workspace.duplicate_page", + }).as("page_duplicated"); + + cy.get(".codex-editor__redactor .ce-block"); + cy.get(".standard-actions .btn-secondary[data-label=Edit]").click(); + + cy.get('.sidebar-item-container[item-name="Test Private Page"]').as("sidebar-item"); + + cy.get("@sidebar-item").find(".standard-sidebar-item").first().click(); + cy.get("@sidebar-item").find(".dropdown-btn").first().click(); + cy.get("@sidebar-item") + .find(".dropdown-list .dropdown-item") + .contains("Duplicate") + .first() + .click({ force: true }); + + cy.get_open_dialog().fill_field("title", "Duplicate Page", "Data"); + cy.click_modal_primary_button("Duplicate"); + + cy.wait("@page_duplicated"); + }); + + it("Drag Sidebar Item", () => { + cy.intercept({ + method: "POST", + url: "api/method/frappe.desk.doctype.workspace.workspace.sort_pages", + }).as("page_sorted"); + + cy.get('.sidebar-item-container[item-name="Duplicate Page"]').as("sidebar-item"); + + cy.get("@sidebar-item").find(".standard-sidebar-item").first().click(); + cy.get("@sidebar-item").find(".drag-handle").first().move({ deltaX: 0, deltaY: 100 }); + + cy.get('.sidebar-item-container[item-name="Build"]').as("sidebar-item"); + + cy.get("@sidebar-item").find(".standard-sidebar-item").first().click(); + cy.get("@sidebar-item").find(".drag-handle").first().move({ deltaX: 0, deltaY: 100 }); + + cy.wait("@page_sorted"); + }); + + it("Edit Page Detail", () => { + cy.intercept({ + method: "POST", + url: "api/method/frappe.desk.doctype.workspace.workspace.update_page", + }).as("page_updated"); + + cy.get('.sidebar-item-container[item-name="Test Private Page"]').as("sidebar-item"); + + cy.get("@sidebar-item").find(".standard-sidebar-item").first().click(); + cy.get("@sidebar-item").find(".dropdown-btn").first().click(); + cy.get("@sidebar-item") + .find(".dropdown-list .dropdown-item") + .contains("Edit") + .first() + .click({ force: true }); + + cy.get_open_dialog().fill_field("title", " 1", "Data"); + cy.get_open_dialog().find('input[data-fieldname="is_public"]').check(); + cy.click_modal_primary_button("Update"); + + cy.get( + '.standard-sidebar-section:first .sidebar-item-container[item-name="Test Private Page"]' + ).should("not.exist"); + cy.get( + '.standard-sidebar-section:last .sidebar-item-container[item-name="Test Private Page 1"]' + ).should("exist"); + + cy.wait("@page_updated"); + }); + + it("Add New Block", () => { + cy.get('.sidebar-item-container[item-name="Duplicate Page"]').as("sidebar-item"); + + cy.get("@sidebar-item").find(".standard-sidebar-item").first().click(); + + cy.get(".ce-block").click().type("{enter}"); + cy.get(".block-list-container .block-list-item").contains("Heading").click(); + cy.get(":focus").type("Header"); + cy.get(".ce-block:last").find(".ce-header").should("exist"); + + cy.get(".ce-block:last").click().type("{enter}"); + cy.get(".block-list-container .block-list-item").contains("Text").click(); + cy.get(":focus").type("Paragraph text"); + cy.get(".ce-block:last").find(".ce-paragraph").should("exist"); + }); + + it("Delete A Block", () => { + cy.get(":focus").click(); + cy.get(".paragraph-control .setting-btn").click(); + cy.get(".paragraph-control .dropdown-item").contains("Delete").click(); + cy.get(".ce-block:last").find(".ce-paragraph").should("not.exist"); + }); + + it("Shrink and Expand A Block", () => { + cy.get(":focus").click(); + cy.get(".ce-block:last .setting-btn").click(); + cy.get(".ce-block:last .dropdown-item").contains("Shrink").click(); + cy.get(".ce-block:last").should("have.class", "col-xs-11"); + cy.get(".ce-block:last .dropdown-item").contains("Shrink").click(); + cy.get(".ce-block:last").should("have.class", "col-xs-10"); + cy.get(".ce-block:last .dropdown-item").contains("Shrink").click(); + cy.get(".ce-block:last").should("have.class", "col-xs-9"); + cy.get(".ce-block:last .dropdown-item").contains("Expand").click(); + cy.get(".ce-block:last").should("have.class", "col-xs-10"); + cy.get(".ce-block:last .dropdown-item").contains("Expand").click(); + cy.get(".ce-block:last").should("have.class", "col-xs-11"); + cy.get(".ce-block:last .dropdown-item").contains("Expand").click(); + cy.get(".ce-block:last").should("have.class", "col-xs-12"); + + cy.get('.standard-actions .btn-primary[data-label="Save"]').click(); + }); + + it("Delete Duplicate Page", () => { + cy.intercept({ + method: "POST", + url: "api/method/frappe.desk.doctype.workspace.workspace.delete_page", + }).as("page_deleted"); + + cy.get(".codex-editor__redactor .ce-block"); + cy.get(".standard-actions .btn-secondary[data-label=Edit]").click(); + + cy.get('.sidebar-item-container[item-name="Duplicate Page"]') + .find(".sidebar-item-control .setting-btn") + .click(); + cy.get('.sidebar-item-container[item-name="Duplicate Page"]') + .find('.dropdown-item[title="Delete Workspace"]') + .click({ force: true }); + cy.wait(300); + cy.get(".modal-footer > .standard-actions > .btn-modal-primary:visible").first().click(); + cy.get('.sidebar-item-container[item-name="Duplicate Page"]').should("not.exist"); + + cy.wait("@page_deleted"); + }); +}); diff --git a/cypress/integration/workspace_blocks.js b/cypress/integration/workspace_blocks.js new file mode 100644 index 0000000000..5b3167b3ac --- /dev/null +++ b/cypress/integration/workspace_blocks.js @@ -0,0 +1,152 @@ +context("Workspace Blocks", () => { + before(() => { + cy.login(); + cy.visit("/app"); + return cy + .window() + .its("frappe") + .then((frappe) => { + return frappe.xcall("frappe.tests.ui_test_helpers.setup_workflow"); + }); + }); + + it("Create Test Page", () => { + cy.intercept({ + method: "POST", + url: "api/method/frappe.desk.doctype.workspace.workspace.new_page", + }).as("new_page"); + + cy.visit("/app/website"); + cy.get(".codex-editor__redactor .ce-block"); + cy.get('.custom-actions button[data-label="Create%20Workspace"]').click(); + cy.fill_field("title", "Test Block Page", "Data"); + cy.fill_field("icon", "edit", "Icon"); + cy.get_open_dialog().find(".modal-header").click(); + cy.get_open_dialog().find(".btn-primary").click(); + + // check if sidebar item is added in private section + cy.get('.sidebar-item-container[item-name="Test Block Page"]').should( + "have.attr", + "item-public", + "0" + ); + + cy.get('.standard-actions .btn-primary[data-label="Save"]').click(); + cy.wait(300); + cy.get('.sidebar-item-container[item-name="Test Block Page"]').should( + "have.attr", + "item-public", + "0" + ); + + cy.wait("@new_page"); + }); + + it("Quick List Block", () => { + cy.create_records([ + { + doctype: "ToDo", + description: "Quick List ToDo 1", + status: "Open", + }, + { + doctype: "ToDo", + description: "Quick List ToDo 2", + status: "Open", + }, + { + doctype: "ToDo", + description: "Quick List ToDo 3", + status: "Open", + }, + { + doctype: "ToDo", + description: "Quick List ToDo 4", + status: "Open", + }, + ]); + + cy.intercept({ + method: "GET", + url: "api/method/frappe.desk.form.load.getdoctype", + }).as("get_doctype"); + + cy.get(".codex-editor__redactor .ce-block"); + cy.get(".standard-actions .btn-secondary[data-label=Edit]").click(); + + // test quick list creation + cy.get(".ce-block").first().click({ force: true }).type("{enter}"); + cy.get(".block-list-container .block-list-item").contains("Quick List").click(); + + cy.get_open_dialog().find(".modal-header").click(); + + cy.fill_field("document_type", "ToDo", "Link").blur(); + cy.fill_field("label", "ToDo", "Data").blur(); + cy.wait("@get_doctype"); + + cy.get_open_dialog().find(".filter-edit-area").should("contain", "No filters selected"); + cy.get_open_dialog().find(".filter-area .add-filter").click(); + + cy.get_open_dialog() + .find(".fieldname-select-area input") + .type("Workflow State{enter}") + .blur(); + cy.get_open_dialog().find(".filter-field .input-with-feedback").type("Pending"); + + cy.get_open_dialog().find(".modal-header").click(); + cy.get_open_dialog().find(".btn-primary").click(); + + cy.get('.standard-actions .btn-primary[data-label="Save"]').click(); + + cy.get(".codex-editor__redactor .ce-block"); + + cy.get(".ce-block .quick-list-widget-box").first().as("todo-quick-list"); + + cy.get("@todo-quick-list").find(".quick-list-item .status").should("contain", "Pending"); + + // test quick-list-item + cy.get("@todo-quick-list") + .find(".quick-list-item .title") + .first() + .invoke("attr", "title") + .then((title) => { + cy.get("@todo-quick-list").find(".quick-list-item").contains(title).click(); + cy.get_field("description", "Text Editor").should("contain", title); + cy.click_action_button("Approve"); + }); + cy.go("back"); + + // test filter-list + cy.get("@todo-quick-list").realHover().find(".widget-control .filter-list").click(); + + cy.get_open_dialog() + .find(".filter-field .input-with-feedback") + .type("{selectall}Approved"); + cy.get_open_dialog().find(".modal-header").click(); + cy.get_open_dialog().find(".btn-primary").click(); + + cy.get("@todo-quick-list").find(".quick-list-item .status").should("contain", "Approved"); + + // test refresh-list + cy.intercept({ + method: "POST", + url: "api/method/frappe.desk.reportview.get", + }).as("refresh-list"); + + cy.get("@todo-quick-list").realHover().find(".widget-control .refresh-list").click(); + cy.wait("@refresh-list"); + + // test add-new + cy.get("@todo-quick-list").realHover().find(".widget-control .add-new").click(); + cy.url().should("include", `/todo/new-todo-1`); + cy.go("back"); + + // test see-all + cy.get("@todo-quick-list").find(".widget-footer .see-all").click(); + cy.open_list_filter(); + cy.get('.filter-field input[data-fieldname="workflow_state"]') + .invoke("val") + .should("eq", "Pending"); + cy.go("back"); + }); +}); diff --git a/cypress/plugins/index.js b/cypress/plugins/index.js index 9720faa666..b13275373c 100644 --- a/cypress/plugins/index.js +++ b/cypress/plugins/index.js @@ -12,6 +12,6 @@ // the project's config changing) module.exports = (on, config) => { - require('@cypress/code-coverage/task')(on, config); + require("@cypress/code-coverage/task")(on, config); return config; -}; \ No newline at end of file +}; diff --git a/cypress/support/commands.js b/cypress/support/commands.js index 37134f0cbc..cbb88cb8cb 100644 --- a/cypress/support/commands.js +++ b/cypress/support/commands.js @@ -1,5 +1,7 @@ -import 'cypress-file-upload'; -import '@testing-library/cypress/add-commands'; +import "cypress-file-upload"; +import "@testing-library/cypress/add-commands"; +import "@4tw/cypress-drag-drop"; +import "cypress-real-events/support"; // *********************************************** // This example commands.js shows you how to // create various custom commands and overwrite @@ -25,315 +27,435 @@ import '@testing-library/cypress/add-commands'; // // -- This is will overwrite an existing command -- // Cypress.Commands.overwrite("visit", (originalFn, url, options) => { ... }); -Cypress.Commands.add('login', (email, password) => { + +Cypress.Commands.add("login", (email, password) => { if (!email) { - email = 'Administrator'; + email = "Administrator"; } if (!password) { - password = Cypress.env('adminPassword'); + password = Cypress.env("adminPassword"); } cy.request({ - url: '/api/method/login', - method: 'POST', + url: "/api/method/login", + method: "POST", body: { usr: email, - pwd: password - } + pwd: password, + }, }); }); -Cypress.Commands.add('call', (method, args) => { +Cypress.Commands.add("call", (method, args) => { return cy .window() - .its('frappe.csrf_token') - .then(csrf_token => { + .its("frappe.csrf_token") + .then((csrf_token) => { return cy .request({ url: `/api/method/${method}`, - method: 'POST', + method: "POST", body: args, headers: { - Accept: 'application/json', - 'Content-Type': 'application/json', - 'X-Frappe-CSRF-Token': csrf_token - } + Accept: "application/json", + "Content-Type": "application/json", + "X-Frappe-CSRF-Token": csrf_token, + }, }) - .then(res => { + .then((res) => { expect(res.status).eq(200); return res.body; }); }); }); -Cypress.Commands.add('get_list', (doctype, fields = [], filters = []) => { +Cypress.Commands.add("get_list", (doctype, fields = [], filters = []) => { filters = JSON.stringify(filters); fields = JSON.stringify(fields); let url = `/api/resource/${doctype}?fields=${fields}&filters=${filters}`; return cy .window() - .its('frappe.csrf_token') - .then(csrf_token => { + .its("frappe.csrf_token") + .then((csrf_token) => { return cy .request({ - method: 'GET', + method: "GET", url, headers: { - Accept: 'application/json', - 'X-Frappe-CSRF-Token': csrf_token - } + Accept: "application/json", + "X-Frappe-CSRF-Token": csrf_token, + }, }) - .then(res => { + .then((res) => { expect(res.status).eq(200); return res.body; }); }); }); -Cypress.Commands.add('get_doc', (doctype, name) => { +Cypress.Commands.add("get_doc", (doctype, name) => { return cy .window() - .its('frappe.csrf_token') - .then(csrf_token => { + .its("frappe.csrf_token") + .then((csrf_token) => { return cy .request({ - method: 'GET', + method: "GET", url: `/api/resource/${doctype}/${name}`, headers: { - Accept: 'application/json', - 'X-Frappe-CSRF-Token': csrf_token - } + Accept: "application/json", + "X-Frappe-CSRF-Token": csrf_token, + }, }) - .then(res => { + .then((res) => { expect(res.status).eq(200); return res.body; }); }); }); -Cypress.Commands.add('remove_doc', (doctype, name) => { +Cypress.Commands.add("remove_doc", (doctype, name) => { return cy .window() - .its('frappe.csrf_token') - .then(csrf_token => { + .its("frappe.csrf_token") + .then((csrf_token) => { return cy .request({ - method: 'DELETE', + method: "DELETE", url: `/api/resource/${doctype}/${name}`, headers: { - Accept: 'application/json', - 'X-Frappe-CSRF-Token': csrf_token - } + Accept: "application/json", + "X-Frappe-CSRF-Token": csrf_token, + }, }) - .then(res => { + .then((res) => { expect(res.status).eq(202); return res.body; }); }); }); -Cypress.Commands.add('create_records', doc => { +Cypress.Commands.add("create_records", (doc) => { return cy - .call('frappe.tests.ui_test_helpers.create_if_not_exists', {doc: JSON.stringify(doc)}) - .then(r => r.message); + .call("frappe.tests.ui_test_helpers.create_if_not_exists", { doc: JSON.stringify(doc) }) + .then((r) => r.message); }); -Cypress.Commands.add('set_value', (doctype, name, obj) => { - return cy.call('frappe.client.set_value', { +Cypress.Commands.add("set_value", (doctype, name, obj) => { + return cy.call("frappe.client.set_value", { doctype, name, - fieldname: obj + fieldname: obj, }); }); -Cypress.Commands.add('fill_field', (fieldname, value, fieldtype = 'Data') => { - cy.get_field(fieldname, fieldtype).as('input'); +Cypress.Commands.add("fill_field", (fieldname, value, fieldtype = "Data") => { + cy.get_field(fieldname, fieldtype).as("input"); - if (['Date', 'Time', 'Datetime'].includes(fieldtype)) { - cy.get('@input').click().wait(200); - cy.get('.datepickers-container .datepicker.active').should('exist'); + if (["Date", "Time", "Datetime"].includes(fieldtype)) { + cy.get("@input").click().wait(200); + cy.get(".datepickers-container .datepicker.active").should("exist"); } - if (fieldtype === 'Time') { - cy.get('@input').clear().wait(200); + if (fieldtype === "Time") { + cy.get("@input").clear().wait(200); } - if (fieldtype === 'Select') { - cy.get('@input').select(value); + if (fieldtype === "Select") { + cy.get("@input").select(value); } else { - cy.get('@input').type(value, {waitForAnimations: false, force: true, delay: 100}); + cy.get("@input").type(value, { + waitForAnimations: false, + parseSpecialCharSequences: false, + force: true, + delay: 100, + }); } - return cy.get('@input'); + return cy.get("@input"); }); -Cypress.Commands.add('get_field', (fieldname, fieldtype = 'Data') => { - let field_element = fieldtype === 'Select' ? 'select': 'input'; +Cypress.Commands.add("get_field", (fieldname, fieldtype = "Data") => { + let field_element = fieldtype === "Select" ? "select" : "input"; let selector = `[data-fieldname="${fieldname}"] ${field_element}:visible`; - if (fieldtype === 'Text Editor') { + if (fieldtype === "Text Editor") { selector = `[data-fieldname="${fieldname}"] .ql-editor[contenteditable=true]:visible`; } - if (fieldtype === 'Code') { + if (fieldtype === "Code") { selector = `[data-fieldname="${fieldname}"] .ace_text-input`; } + if (fieldtype === "Markdown Editor") { + selector = `[data-fieldname="${fieldname}"] .ace-editor-target`; + } return cy.get(selector).first(); }); -Cypress.Commands.add('fill_table_field', (tablefieldname, row_idx, fieldname, value, fieldtype = 'Data') => { - cy.get_table_field(tablefieldname, row_idx, fieldname, fieldtype).as('input'); +Cypress.Commands.add( + "fill_table_field", + (tablefieldname, row_idx, fieldname, value, fieldtype = "Data") => { + cy.get_table_field(tablefieldname, row_idx, fieldname, fieldtype).as("input"); - if (['Date', 'Time', 'Datetime'].includes(fieldtype)) { - cy.get('@input').click().wait(200); - cy.get('.datepickers-container .datepicker.active').should('exist'); - } - if (fieldtype === 'Time') { - cy.get('@input').clear().wait(200); - } + if (["Date", "Time", "Datetime"].includes(fieldtype)) { + cy.get("@input").click().wait(200); + cy.get(".datepickers-container .datepicker.active").should("exist"); + } + if (fieldtype === "Time") { + cy.get("@input").clear().wait(200); + } - if (fieldtype === 'Select') { - cy.get('@input').select(value); - } else { - cy.get('@input').type(value, {waitForAnimations: false, force: true}); + if (fieldtype === "Select") { + cy.get("@input").select(value); + } else { + cy.get("@input").type(value, { waitForAnimations: false, force: true }); + } + return cy.get("@input"); } - return cy.get('@input'); +); + +Cypress.Commands.add( + "get_table_field", + (tablefieldname, row_idx, fieldname, fieldtype = "Data") => { + let selector = `.frappe-control[data-fieldname="${tablefieldname}"]`; + selector += ` [data-idx="${row_idx}"]`; + + if (fieldtype === "Text Editor") { + selector += ` [data-fieldname="${fieldname}"] .ql-editor[contenteditable=true]`; + } else if (fieldtype === "Code") { + selector += ` [data-fieldname="${fieldname}"] .ace_text-input`; + } else { + selector += ` [data-fieldname="${fieldname}"]`; + return cy.get(selector).find(".form-control:visible, .static-area:visible").first(); + } + return cy.get(selector); + } +); + +Cypress.Commands.add("awesomebar", (text) => { + cy.get("#navbar-search").type(`${text}{downarrow}{enter}`, { delay: 700 }); }); -Cypress.Commands.add('get_table_field', (tablefieldname, row_idx, fieldname, fieldtype = 'Data') => { - let selector = `.frappe-control[data-fieldname="${tablefieldname}"]`; - selector += ` [data-idx="${row_idx}"]`; - - if (fieldtype === 'Text Editor') { - selector += ` [data-fieldname="${fieldname}"] .ql-editor[contenteditable=true]`; - } else if (fieldtype === 'Code') { - selector += ` [data-fieldname="${fieldname}"] .ace_text-input`; - } else { - selector += ` [data-fieldname="${fieldname}"]`; - return cy.get(selector).find('.form-control:visible, .static-area:visible').first(); - } - return cy.get(selector); -}); - -Cypress.Commands.add('awesomebar', text => { - cy.get('#navbar-search').type(`${text}{downarrow}{enter}`, {delay: 700}); -}); - -Cypress.Commands.add('new_form', doctype => { - let dt_in_route = doctype.toLowerCase().replace(/ /g, '-'); +Cypress.Commands.add("new_form", (doctype) => { + let dt_in_route = doctype.toLowerCase().replace(/ /g, "-"); cy.visit(`/app/${dt_in_route}/new`); - cy.get('body').should('have.attr', 'data-route', `Form/${doctype}/new-${dt_in_route}-1`); - cy.get('body').should('have.attr', 'data-ajax-state', 'complete'); + cy.get("body").should("have.attr", "data-route", `Form/${doctype}/new-${dt_in_route}-1`); + cy.get("body").should("have.attr", "data-ajax-state", "complete"); }); -Cypress.Commands.add('go_to_list', doctype => { - let dt_in_route = doctype.toLowerCase().replace(/ /g, '-'); +Cypress.Commands.add("go_to_list", (doctype) => { + let dt_in_route = doctype.toLowerCase().replace(/ /g, "-"); cy.visit(`/app/${dt_in_route}`); }); -Cypress.Commands.add('clear_cache', () => { +Cypress.Commands.add("clear_cache", () => { cy.window() - .its('frappe') - .then(frappe => { + .its("frappe") + .then((frappe) => { frappe.ui.toolbar.clear_cache(); }); }); -Cypress.Commands.add('dialog', opts => { - return cy.window().then(win => { - var d = new win.frappe.ui.Dialog(opts); - d.show(); - return d; - }); +Cypress.Commands.add("dialog", (opts) => { + return cy + .window({ log: false }) + .its("frappe", { log: false }) + .then((frappe) => { + Cypress.log({ + name: "dialog", + displayName: "dialog", + message: "frappe.ui.Dialog", + consoleProps: () => { + return { + options: opts, + dialog: d, + }; + }, + }); + + var d = new frappe.ui.Dialog(opts); + d.show(); + return d; + }); }); -Cypress.Commands.add('get_open_dialog', () => { - return cy.get('.modal:visible').last(); +Cypress.Commands.add("get_open_dialog", () => { + return cy.get(".modal:visible").last(); }); -Cypress.Commands.add('hide_dialog', () => { +Cypress.Commands.add("save", () => { + cy.intercept("/api").as("api"); + cy.get(`button[data-label="Save"]:visible`).click({ scrollBehavior: false, force: true }); + cy.wait("@api"); +}); +Cypress.Commands.add("hide_dialog", () => { cy.wait(300); - cy.get_open_dialog().find('.btn-modal-close').click(); - cy.get('.modal:visible').should('not.exist'); + cy.get_open_dialog().focus().find(".btn-modal-close").click(); + cy.get(".modal:visible").should("not.exist"); }); -Cypress.Commands.add('insert_doc', (doctype, args, ignore_duplicate) => { +Cypress.Commands.add("clear_dialogs", () => { + cy.window().then((win) => { + win.$(".modal, .modal-backdrop").remove(); + }); + cy.get(".modal").should("not.exist"); +}); + +Cypress.Commands.add("clear_datepickers", () => { + cy.window().then((win) => { + win.$(".datepicker").remove(); + }); + cy.get(".datepicker").should("not.exist"); +}); + +Cypress.Commands.add("insert_doc", (doctype, args, ignore_duplicate) => { + if (!args.doctype) { + args.doctype = doctype; + } return cy .window() - .its('frappe.csrf_token') - .then(csrf_token => { + .its("frappe.csrf_token") + .then((csrf_token) => { return cy .request({ - method: 'POST', + method: "POST", url: `/api/resource/${doctype}`, body: args, headers: { - Accept: 'application/json', - 'Content-Type': 'application/json', - 'X-Frappe-CSRF-Token': csrf_token + Accept: "application/json", + "Content-Type": "application/json", + "X-Frappe-CSRF-Token": csrf_token, }, - failOnStatusCode: !ignore_duplicate + failOnStatusCode: !ignore_duplicate, }) - .then(res => { + .then((res) => { let status_codes = [200]; if (ignore_duplicate) { status_codes.push(409); } - expect(res.status).to.be.oneOf(status_codes); + + let message = null; + if (ignore_duplicate && !status_codes.includes(res.status)) { + message = `Document insert failed, response: ${JSON.stringify( + res, + null, + "\t" + )}`; + } + expect(res.status).to.be.oneOf(status_codes, message); return res.body.data; }); }); }); -Cypress.Commands.add('add_filter', () => { - cy.get('.filter-section .filter-button').click(); - cy.wait(300); - cy.get('.filter-popover').should('exist'); +Cypress.Commands.add("update_doc", (doctype, docname, args) => { + return cy + .window() + .its("frappe.csrf_token") + .then((csrf_token) => { + return cy + .request({ + method: "PUT", + url: `/api/resource/${doctype}/${docname}`, + body: args, + headers: { + Accept: "application/json", + "Content-Type": "application/json", + "X-Frappe-CSRF-Token": csrf_token, + }, + }) + .then((res) => { + expect(res.status).to.eq(200); + return res.body.data; + }); + }); }); -Cypress.Commands.add('clear_filters', () => { +Cypress.Commands.add("open_list_filter", () => { + cy.get(".filter-section .filter-button").click(); + cy.wait(300); + cy.get(".filter-popover").should("exist"); +}); + +Cypress.Commands.add("click_custom_action_button", (name) => { + cy.get(`.custom-actions [data-label="${encodeURIComponent(name)}"]`).click(); +}); + +Cypress.Commands.add("click_action_button", (name) => { + cy.findByRole("button", { name: "Actions" }).click(); + cy.get(`.actions-btn-group [data-label="${encodeURIComponent(name)}"]`).click(); +}); + +Cypress.Commands.add("click_menu_button", (name) => { + cy.get(".standard-actions .menu-btn-group > .btn").click(); + cy.get(`.menu-btn-group [data-label="${encodeURIComponent(name)}"]`).click(); +}); + +Cypress.Commands.add("clear_filters", () => { let has_filter = false; cy.intercept({ - method: 'POST', - url: 'api/method/frappe.model.utils.user_settings.save' - }).as('filter-saved'); - cy.get('.filter-section .filter-button').click({force: true}); + method: "POST", + url: "api/method/frappe.model.utils.user_settings.save", + }).as("filter-saved"); + cy.get(".filter-section .filter-button").click({ force: true }); cy.wait(300); - cy.get('.filter-popover').should('exist'); - cy.get('.filter-popover').then(popover => { - if (popover.find('input.input-with-feedback')[0].value != '') { + cy.get(".filter-popover").should("exist"); + cy.get(".filter-popover").then((popover) => { + if (popover.find("input.input-with-feedback")[0].value != "") { has_filter = true; } }); - cy.get('.filter-popover').find('.clear-filters').click(); - cy.get('.filter-section .filter-button').click(); - cy.window().its('cur_list').then(cur_list => { - cur_list && cur_list.filter_area && cur_list.filter_area.clear(); - has_filter && cy.wait('@filter-saved'); - }); + cy.get(".filter-popover").find(".clear-filters").click(); + cy.get(".filter-section .filter-button").click(); + cy.window() + .its("cur_list") + .then((cur_list) => { + cur_list && cur_list.filter_area && cur_list.filter_area.clear(); + has_filter && cy.wait("@filter-saved"); + }); }); -Cypress.Commands.add('click_modal_primary_button', (btn_name) => { - cy.get('.modal-footer > .standard-actions > .btn-primary').contains(btn_name).trigger('click', {force: true}); +Cypress.Commands.add("click_modal_primary_button", (btn_name) => { + cy.wait(400); + cy.get(".modal-footer > .standard-actions > .btn-primary") + .contains(btn_name) + .click({ force: true }); }); -Cypress.Commands.add('click_sidebar_button', (btn_name) => { - cy.get('.list-group-by-fields .list-link > a').contains(btn_name).click({force: true}); +Cypress.Commands.add("click_sidebar_button", (btn_name) => { + cy.get(".list-group-by-fields .list-link > a").contains(btn_name).click({ force: true }); }); -Cypress.Commands.add('click_listview_row_item', (row_no) => { - cy.get('.list-row > .level-left > .list-subject > .level-item > .ellipsis').eq(row_no).click({force: true}); +Cypress.Commands.add("click_listview_row_item", (row_no) => { + cy.get(".list-row > .level-left > .list-subject > .level-item > .ellipsis") + .eq(row_no) + .click({ force: true }); }); -Cypress.Commands.add('click_filter_button', () => { - cy.get('.filter-selector > .btn').click(); +Cypress.Commands.add("click_listview_row_item_with_text", (text) => { + cy.get(".list-row > .level-left > .list-subject > .level-item > .ellipsis") + .contains(text) + .first() + .click({ force: true }); }); -Cypress.Commands.add('click_listview_primary_button', (btn_name) => { - cy.get('.primary-action').contains(btn_name).click({force: true}); +Cypress.Commands.add("click_filter_button", () => { + cy.get(".filter-selector > .btn").click(); }); -Cypress.Commands.add('click_timeline_action_btn', (btn_name) => { - cy.get('.timeline-message-box .actions .action-btn').contains(btn_name).click(); +Cypress.Commands.add("click_listview_primary_button", (btn_name) => { + cy.get(".primary-action").contains(btn_name).click({ force: true }); }); -Cypress.Commands.add('select_listview_row_checkbox', (row_no) => { - cy.get('.frappe-list .select-like > .list-row-checkbox').eq(row_no).click(); +Cypress.Commands.add("click_doc_primary_button", (btn_name) => { + cy.get(".primary-action").contains(btn_name).click({ force: true }); +}); + +Cypress.Commands.add("click_timeline_action_btn", (btn_name) => { + cy.get(".timeline-message-box .actions .action-btn").contains(btn_name).click(); +}); + +Cypress.Commands.add("select_listview_row_checkbox", (row_no) => { + cy.get(".frappe-list .select-like > .list-row-checkbox").eq(row_no).click(); +}); + +Cypress.Commands.add("click_form_section", (section_name) => { + cy.get(".section-head").contains(section_name).click(); }); diff --git a/cypress/support/index.js b/cypress/support/index.js index 5980e96677..8ce8317a2f 100644 --- a/cypress/support/index.js +++ b/cypress/support/index.js @@ -14,10 +14,10 @@ // *********************************************************** // Import commands.js using ES2015 syntax: -import './commands'; -import '@cypress/code-coverage/support'; +import "./commands"; +import "@cypress/code-coverage/support"; -Cypress.on('uncaught:exception', (err, runnable) => { +Cypress.on("uncaught:exception", (err, runnable) => { return false; }); @@ -25,5 +25,5 @@ Cypress.on('uncaught:exception', (err, runnable) => { // require('./commands') Cypress.Cookies.defaults({ - preserve: 'sid' -}); \ No newline at end of file + preserve: "sid", +}); diff --git a/dev-requirements.txt b/dev-requirements.txt deleted file mode 100644 index f4045c6bed..0000000000 --- a/dev-requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -coverage==5.5 -Faker~=8.1.0 -pyngrok~=5.0.5 -unittest-xml-reporting~=3.0.4 diff --git a/esbuild/build-cleanup.js b/esbuild/build-cleanup.js index cf03606a34..023fce08c5 100644 --- a/esbuild/build-cleanup.js +++ b/esbuild/build-cleanup.js @@ -4,9 +4,9 @@ const fs = require("fs"); const glob = require("fast-glob"); module.exports = { - name: 'build_cleanup', + name: "build_cleanup", setup(build) { - build.onEnd(result => { + build.onEnd((result) => { if (result.errors.length) return; clean_dist_files(Object.keys(result.metafile.outputs)); }); @@ -14,25 +14,18 @@ module.exports = { }; function clean_dist_files(new_files) { - new_files.forEach( - file => { - if (file.endsWith(".map")) return; + new_files.forEach((file) => { + if (file.endsWith(".map")) return; - const pattern = file.split(".").slice(0, -2).join(".") + "*"; - glob.sync(pattern).forEach( - file_to_delete => { - if (file_to_delete.startsWith(file)) return; + const pattern = file.split(".").slice(0, -2).join(".") + "*"; + glob.sync(pattern).forEach((file_to_delete) => { + if (file_to_delete.startsWith(file)) return; - fs.unlink(path.resolve(file_to_delete), err => { - if (!err) return; + fs.unlink(path.resolve(file_to_delete), (err) => { + if (!err) return; - console.error( - `Error deleting ${file.split(path.sep).pop()}` - ); - }); - } - - ); - } - ); -} \ No newline at end of file + console.error(`Error deleting ${file.split(path.sep).pop()}`); + }); + }); + }); +} diff --git a/esbuild/esbuild.js b/esbuild/esbuild.js index 4aa1ebc824..56910cbcac 100644 --- a/esbuild/esbuild.js +++ b/esbuild/esbuild.js @@ -8,7 +8,7 @@ const yargs = require("yargs"); const cliui = require("cliui")(); const chalk = require("chalk"); const html_plugin = require("./frappe-html"); -const rtlcss = require('rtlcss'); +const rtlcss = require("rtlcss"); const postCssPlugin = require("@frappe/esbuild-plugin-postcss2").default; const ignore_assets = require("./ignore-assets"); const sass_options = require("./sass_options"); @@ -25,44 +25,41 @@ const { log_warn, log_error, bench_path, - get_redis_subscriber + get_redis_subscriber, } = require("./utils"); const argv = yargs .usage("Usage: node esbuild [options]") .option("apps", { type: "string", - description: "Run build for specific apps" + description: "Run build for specific apps", }) .option("skip_frappe", { type: "boolean", - description: "Skip building frappe assets" + description: "Skip building frappe assets", }) .option("files", { type: "string", - description: "Run build for specified bundles" + description: "Run build for specified bundles", }) .option("watch", { type: "boolean", - description: "Run in watch mode and rebuild on file changes" + description: "Run in watch mode and rebuild on file changes", }) .option("live-reload", { type: "boolean", description: `Automatically reload Desk when assets are rebuilt. - Can only be used with the --watch flag.` + Can only be used with the --watch flag.`, }) .option("production", { type: "boolean", - description: "Run build in production mode" + description: "Run build in production mode", }) .option("run-build-command", { type: "boolean", - description: "Run build command for apps" + description: "Run build command for apps", }) - .example( - "node esbuild --apps frappe,erpnext", - "Run build only for frappe and erpnext" - ) + .example("node esbuild --apps frappe,erpnext", "Run build only for frappe and erpnext") .example( "node esbuild --files frappe/website.bundle.js,frappe/desk.bundle.js", "Run build only for specified bundles" @@ -70,7 +67,7 @@ const argv = yargs .version(false).argv; const APPS = (!argv.apps ? app_list : argv.apps.split(",")).filter( - app => !(argv.skip_frappe && app == "frappe") + (app) => !(argv.skip_frappe && app == "frappe") ); const FILES_TO_BUILD = argv.files ? argv.files.split(",") : []; const WATCH_MODE = Boolean(argv.watch); @@ -81,17 +78,15 @@ const TOTAL_BUILD_TIME = `${chalk.black.bgGreen(" DONE ")} Total Build Time`; const NODE_PATHS = [].concat( // node_modules of apps directly importable app_list - .map(app => path.resolve(get_app_path(app), "../node_modules")) + .map((app) => path.resolve(get_app_path(app), "../node_modules")) .filter(fs.existsSync), // import js file of any app if you provide the full path - app_list - .map(app => path.resolve(get_app_path(app), "..")) - .filter(fs.existsSync) + app_list.map((app) => path.resolve(get_app_path(app), "..")).filter(fs.existsSync) ); execute() .then(() => RUN_BUILD_COMMAND && run_build_command_for_apps(APPS)) - .catch(e => console.error(e)); + .catch((e) => console.error(e)); if (WATCH_MODE) { // listen for open files in editor event @@ -131,7 +126,7 @@ function build_assets_for_apps(apps, files) { ? get_files_to_build(files) : get_all_files_to_build(apps); - return glob(include_patterns, { ignore: ignore_patterns }).then(files => { + return glob(include_patterns, { ignore: ignore_patterns }).then((files) => { let output_path = assets_path; let file_map = {}; @@ -143,39 +138,38 @@ function build_assets_for_apps(apps, files) { let extension = path.extname(file); let output_name = path.basename(file, extension); - if ( - [".css", ".scss", ".less", ".sass", ".styl"].includes(extension) - ) { + if ([".css", ".scss", ".less", ".sass", ".styl"].includes(extension)) { output_name = path.join("css", output_name); } else if ([".js", ".ts"].includes(extension)) { output_name = path.join("js", output_name); } output_name = path.join(app, "dist", output_name); - if (Object.keys(file_map).includes(output_name) || Object.keys(style_file_map).includes(output_name)) { - log_warn( - `Duplicate output file ${output_name} generated from ${file}` - ); + if ( + Object.keys(file_map).includes(output_name) || + Object.keys(style_file_map).includes(output_name) + ) { + log_warn(`Duplicate output file ${output_name} generated from ${file}`); } if ([".css", ".scss", ".less", ".sass", ".styl"].includes(extension)) { style_file_map[output_name] = file; - rtl_style_file_map[output_name.replace('/css/', '/css-rtl/')] = file; + rtl_style_file_map[output_name.replace("/css/", "/css-rtl/")] = file; } else { file_map[output_name] = file; } } let build = build_files({ files: file_map, - outdir: output_path + outdir: output_path, }); let style_build = build_style_files({ files: style_file_map, - outdir: output_path + outdir: output_path, }); let rtl_style_build = build_style_files({ files: rtl_style_file_map, outdir: output_path, - rtl_style: true + rtl_style: true, }); return Promise.all([build, style_build, rtl_style_build]); }); @@ -188,11 +182,7 @@ function get_all_files_to_build(apps) { for (let app of apps) { let public_path = get_public_path(app); include_patterns.push( - path.resolve( - public_path, - "**", - "*.bundle.{js,ts,css,sass,scss,less,styl}" - ) + path.resolve(public_path, "**", "*.bundle.{js,ts,css,sass,scss,less,styl}") ); ignore_patterns.push( path.resolve(public_path, "node_modules"), @@ -202,7 +192,7 @@ function get_all_files_to_build(apps) { return { include_patterns, - ignore_patterns + ignore_patterns, }; } @@ -223,16 +213,12 @@ function get_files_to_build(files) { return { include_patterns, - ignore_patterns + ignore_patterns, }; } function build_files({ files, outdir }) { - let build_plugins = [ - html_plugin, - build_cleanup_plugin, - vue(), - ]; + let build_plugins = [html_plugin, build_cleanup_plugin, vue()]; return esbuild.build(get_build_options(files, outdir, build_plugins)); } @@ -247,8 +233,8 @@ function build_style_files({ files, outdir, rtl_style = false }) { build_cleanup_plugin, postCssPlugin({ plugins: plugins, - sassOptions: sass_options - }) + sassOptions: sass_options, + }), ]; plugins.push(require("autoprefixer")); @@ -259,7 +245,7 @@ function get_build_options(files, outdir, plugins) { return { entryPoints: files, entryNames: "[dir]/[name].[hash]", - target: ['es2017'], + target: ["es2017"], outdir, sourcemap: true, bundle: true, @@ -267,12 +253,10 @@ function get_build_options(files, outdir, plugins) { minify: PRODUCTION, nodePaths: NODE_PATHS, define: { - "process.env.NODE_ENV": JSON.stringify( - PRODUCTION ? "production" : "development" - ) + "process.env.NODE_ENV": JSON.stringify(PRODUCTION ? "production" : "development"), }, plugins: plugins, - watch: get_watch_config() + watch: get_watch_config(), }; } @@ -286,17 +270,13 @@ function get_watch_config() { log(chalk.dim(error.stack)); notify_redis({ error }); } else { - let { - new_assets_json, - prev_assets_json - } = await write_assets_json(result.metafile); + let { new_assets_json, prev_assets_json } = await write_assets_json( + result.metafile + ); let changed_files; if (prev_assets_json) { - changed_files = get_rebuilt_assets( - prev_assets_json, - new_assets_json - ); + changed_files = get_rebuilt_assets(prev_assets_json, new_assets_json); let timestamp = new Date().toLocaleTimeString(); let message = `${timestamp}: Compiled ${changed_files.length} files...`; @@ -309,7 +289,7 @@ function get_watch_config() { } notify_redis({ success: true, changed_files }); } - } + }, }; } return null; @@ -324,11 +304,11 @@ function log_built_assets(results) { cliui.div( { text: chalk.cyan.bold("File"), - width: column_widths[0] + width: column_widths[0], }, { text: chalk.cyan.bold("Size"), - width: column_widths[1] + width: column_widths[1], } ); cliui.div(""); @@ -344,7 +324,7 @@ function log_built_assets(results) { output_by_dist_path[dist_path] = output_by_dist_path[dist_path] || []; output_by_dist_path[dist_path].push({ name: filename, - size: (data.bytes / 1000).toFixed(2) + " Kb" + size: (data.bytes / 1000).toFixed(2) + " Kb", }); } @@ -352,7 +332,7 @@ function log_built_assets(results) { let files = output_by_dist_path[dist_path]; cliui.div({ text: dist_path, - width: column_widths[0] + width: column_widths[0], }); for (let i in files) { @@ -367,11 +347,11 @@ function log_built_assets(results) { cliui.div( { text: branch + chalk[color]("" + file.name), - width: column_widths[0] + width: column_widths[0], }, { text: file.size, - width: column_widths[1] + width: column_widths[1], } ); } @@ -393,7 +373,7 @@ async function write_assets_json(metafile) { let asset_path = "/" + path.relative(sites_path, output); if (info.entryPoint) { let key = path.basename(info.entryPoint); - if (key.endsWith('.css') && asset_path.includes('/css-rtl/')) { + if (key.endsWith(".css") && asset_path.includes("/css-rtl/")) { rtl = true; key = `rtl_${key}`; } @@ -401,7 +381,7 @@ async function write_assets_json(metafile) { } } - let assets_json_path = path.resolve(assets_path, `assets${rtl?'-rtl':''}.json`); + let assets_json_path = path.resolve(assets_path, `assets${rtl ? "-rtl" : ""}.json`); let assets_json; try { assets_json = await fs.promises.readFile(assets_json_path, "utf-8"); @@ -413,26 +393,23 @@ async function write_assets_json(metafile) { let new_assets_json = Object.assign({}, assets_json, out); curr_assets_json = new_assets_json; - await fs.promises.writeFile( - assets_json_path, - JSON.stringify(new_assets_json, null, 4) - ); + await fs.promises.writeFile(assets_json_path, JSON.stringify(new_assets_json, null, 4)); await update_assets_json_in_cache(); return { new_assets_json, - prev_assets_json + prev_assets_json, }; } function update_assets_json_in_cache() { // update assets_json cache in redis, so that it can be read directly by python - return new Promise(resolve => { + return new Promise((resolve) => { let client = get_redis_subscriber("redis_cache"); // handle error event to avoid printing stack traces - client.on("error", _ => { + client.on("error", (_) => { log_warn("Cannot connect to redis_cache to update assets_json"); }); - client.del("assets_json", err => { + client.del("assets_json", (err) => { client.unref(); resolve(); }); @@ -464,7 +441,7 @@ function run_build_command_for_apps(apps) { async function notify_redis({ error, success, changed_files }) { // notify redis which in turns tells socketio to publish this to browser let subscriber = get_redis_subscriber("redis_socketio"); - subscriber.on("error", _ => { + subscriber.on("error", (_) => { log_warn("Cannot connect to redis_socketio for browser events"); }); @@ -472,20 +449,20 @@ async function notify_redis({ error, success, changed_files }) { if (error) { let formatted = await esbuild.formatMessages(error.errors, { kind: "error", - terminalWidth: 100 + terminalWidth: 100, }); let stack = error.stack.replace(new RegExp(bench_path, "g"), ""); payload = { error, formatted, - stack + stack, }; } if (success) { payload = { success: true, changed_files, - live_reload: argv["live-reload"] + live_reload: argv["live-reload"], }; } @@ -493,14 +470,14 @@ async function notify_redis({ error, success, changed_files }) { "events", JSON.stringify({ event: "build_event", - message: payload + message: payload, }) ); } function open_in_editor() { let subscriber = get_redis_subscriber("redis_socketio"); - subscriber.on("error", _ => { + subscriber.on("error", (_) => { log_warn("Cannot connect to redis_socketio for open_in_editor events"); }); subscriber.on("message", (event, file) => { diff --git a/esbuild/frappe-html.js b/esbuild/frappe-html.js index 9a7edb144d..d38a0c23cb 100644 --- a/esbuild/frappe-html.js +++ b/esbuild/frappe-html.js @@ -4,24 +4,24 @@ module.exports = { let path = require("path"); let fs = require("fs/promises"); - build.onResolve({ filter: /\.html$/ }, args => { + build.onResolve({ filter: /\.html$/ }, (args) => { return { path: path.join(args.resolveDir, args.path), - namespace: "frappe-html" + namespace: "frappe-html", }; }); - build.onLoad({ filter: /.*/, namespace: "frappe-html" }, args => { + build.onLoad({ filter: /.*/, namespace: "frappe-html" }, (args) => { let filepath = args.path; let filename = path.basename(filepath).split(".")[0]; return fs .readFile(filepath, "utf-8") - .then(content => { + .then((content) => { content = scrub_html_template(content); return { contents: `\n\tfrappe.templates['${filename}'] = \`${content}\`;\n`, - watchFiles: [filepath] + watchFiles: [filepath], }; }) .catch(() => { @@ -29,13 +29,13 @@ module.exports = { contents: "", warnings: [ { - text: `There was an error importing ${filepath}` - } - ] + text: `There was an error importing ${filepath}`, + }, + ], }; }); }); - } + }, }; function scrub_html_template(content) { diff --git a/esbuild/ignore-assets.js b/esbuild/ignore-assets.js index 5edfef2110..fad7a95e0d 100644 --- a/esbuild/ignore-assets.js +++ b/esbuild/ignore-assets.js @@ -1,11 +1,11 @@ module.exports = { name: "frappe-ignore-asset", setup(build) { - build.onResolve({ filter: /^\/assets\// }, args => { + build.onResolve({ filter: /^\/assets\// }, (args) => { return { path: args.path, - external: true + external: true, }; }); - } + }, }; diff --git a/esbuild/sass_options.js b/esbuild/sass_options.js index fcc7e04ccd..92b691cb46 100644 --- a/esbuild/sass_options.js +++ b/esbuild/sass_options.js @@ -1,18 +1,13 @@ let path = require("path"); let { get_app_path, app_list } = require("./utils"); -let node_modules_path = path.resolve( - get_app_path("frappe"), - "..", - "node_modules" -); -let app_paths = app_list - .map(get_app_path) - .map(app_path => path.resolve(app_path, "..")); +let node_modules_path = path.resolve(get_app_path("frappe"), "..", "node_modules"); +let app_paths = app_list.map(get_app_path).map((app_path) => path.resolve(app_path, "..")); module.exports = { includePaths: [node_modules_path, ...app_paths], - importer: function(url) { + quietDeps: true, + importer: function (url) { if (url.startsWith("~")) { // strip ~ so that it can resolve from node_modules url = url.slice(1); @@ -23,7 +18,7 @@ module.exports = { } // normal file, let it go return { - file: url + file: url, }; - } + }, }; diff --git a/esbuild/utils.js b/esbuild/utils.js index 82490adb36..db58b89e8b 100644 --- a/esbuild/utils.js +++ b/esbuild/utils.js @@ -26,24 +26,20 @@ const bundle_map = app_list.reduce((out, app) => { const public_js_path = public_js_paths[app]; if (fs.existsSync(public_js_path)) { const all_files = fs.readdirSync(public_js_path); - const js_files = all_files.filter(file => file.endsWith(".js")); + const js_files = all_files.filter((file) => file.endsWith(".js")); for (let js_file of js_files) { const filename = path.basename(js_file).split(".")[0]; - out[path.join(app, "js", filename)] = path.resolve( - public_js_path, - js_file - ); + out[path.join(app, "js", filename)] = path.resolve(public_js_path, js_file); } } return out; }, {}); -const get_public_path = app => public_paths[app]; +const get_public_path = (app) => public_paths[app]; -const get_build_json_path = app => - path.resolve(get_public_path(app), "build.json"); +const get_build_json_path = (app) => path.resolve(get_public_path(app), "build.json"); function get_build_json(app) { try { @@ -62,7 +58,7 @@ function delete_file(path) { function run_serially(tasks) { let result = Promise.resolve(); - tasks.forEach(task => { + tasks.forEach((task) => { if (task) { result = result.then ? result.then(task) : Promise.resolve(); } @@ -70,12 +66,12 @@ function run_serially(tasks) { return result; } -const get_app_path = app => app_paths[app]; +const get_app_path = (app) => app_paths[app]; function get_apps_list() { return fs .readFileSync(path.resolve(sites_path, "apps.txt"), { - encoding: "utf-8" + encoding: "utf-8", }) .split("\n") .filter(Boolean); @@ -112,16 +108,21 @@ function log(...args) { function get_redis_subscriber(kind) { // get redis subscriber that aborts after 10 connection attempts - let { get_redis_subscriber: get_redis } = require("../node_utils"); - return get_redis(kind, { - retry_strategy: function(options) { + let retry_strategy; + let { get_redis_subscriber: get_redis, get_conf } = require("../node_utils"); + + if (process.env.CI == 1 || get_conf().developer_mode == 1) { + retry_strategy = () => {}; + } else { + retry_strategy = function (options) { // abort after 10 connection attempts if (options.attempt > 10) { return undefined; } return Math.min(options.attempt * 100, 2000); - } - }); + }; + } + return get_redis(kind, { retry_strategy }); } module.exports = { @@ -141,5 +142,5 @@ module.exports = { log, log_warn, log_error, - get_redis_subscriber + get_redis_subscriber, }; diff --git a/frappe/__init__.py b/frappe/__init__.py index d1b574f500..95e5e9c6c2 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -10,28 +10,26 @@ be used to build database driven apps. Read the documentation: https://frappeframework.com/docs """ -import os -import warnings - -STANDARD_USERS = ("Guest", "Administrator") - -_dev_server = os.environ.get("DEV_SERVER", False) - -if _dev_server: - warnings.simplefilter("always", DeprecationWarning) - warnings.simplefilter("always", PendingDeprecationWarning) - +import functools import importlib import inspect import json -import sys -from typing import TYPE_CHECKING, Dict, List, Optional, Union +import os +import re +import warnings +from typing import TYPE_CHECKING, Any, Callable, Literal, Optional, overload import click from werkzeug.local import Local, release_local -from frappe.query_builder import get_query_builder, patch_query_aggregation, patch_query_execute -from frappe.utils.data import cstr +from frappe.query_builder import ( + get_qb_engine, + get_query_builder, + patch_query_aggregation, + patch_query_execute, +) +from frappe.utils.caching import request_cache +from frappe.utils.data import cstr, sbool # Local application imports from .exceptions import * @@ -45,11 +43,22 @@ from .utils.jinja import ( from .utils.lazy_loader import lazy_import __version__ = "14.0.0-dev" - __title__ = "Frappe Framework" -local = Local() controllers = {} +local = Local() +STANDARD_USERS = ("Guest", "Administrator") + +_dev_server = int(sbool(os.environ.get("DEV_SERVER", False))) +_qb_patched = {} +re._MAXCACHE = ( + 50 # reduced from default 512 given we are already maintaining this on parent worker +) + + +if _dev_server: + warnings.simplefilter("always", DeprecationWarning) + warnings.simplefilter("always", PendingDeprecationWarning) class _dict(dict): @@ -74,7 +83,7 @@ class _dict(dict): return _dict(self) -def _(msg: str, lang: Optional[str] = None, context: str = None) -> str: +def _(msg: str, lang: str | None = None, context: str | None = None) -> str: """Returns translated string in current lang, if exists. Usage: _('Change') @@ -99,7 +108,7 @@ def _(msg: str, lang: Optional[str] = None, context: str = None) -> str: translated_string = "" if context: - string_key = "{msg}:{context}".format(msg=msg, context=context) + string_key = f"{msg}:{context}" translated_string = get_full_dict(lang).get(string_key) if not translated_string: @@ -109,7 +118,7 @@ def _(msg: str, lang: Optional[str] = None, context: str = None) -> str: return translated_string or non_translated_string -def as_unicode(text, encoding="utf-8"): +def as_unicode(text: str, encoding: str = "utf-8") -> str: """Convert to unicode if required""" if isinstance(text, str): return text @@ -121,7 +130,7 @@ def as_unicode(text, encoding="utf-8"): return str(text) -def get_lang_dict(fortype, name=None): +def get_lang_dict(fortype: str, name: str | None = None) -> dict[str, str]: """Returns the translated language dict for the given type and name. :param fortype: must be one of `doctype`, `page`, `report`, `include`, `jsfile`, `boot` @@ -131,11 +140,11 @@ def get_lang_dict(fortype, name=None): return get_dict(fortype, name) -def set_user_lang(user, user_language=None): +def set_user_lang(user: str, user_language: str | None = None) -> None: """Guess and set user language for the session. `frappe.local.lang`""" from frappe.translate import get_user_lang - local.lang = get_user_lang(user) + local.lang = get_user_lang(user) or user_language # local-globals @@ -161,24 +170,22 @@ lang = local("lang") if TYPE_CHECKING: from frappe.database.mariadb.database import MariaDBDatabase from frappe.database.postgres.database import PostgresDatabase + from frappe.model.document import Document from frappe.query_builder.builder import MariaDB, Postgres from frappe.utils.redis_wrapper import RedisWrapper - db: Union[MariaDBDatabase, PostgresDatabase] - qb: Union[MariaDB, Postgres] + db: MariaDBDatabase | PostgresDatabase + qb: MariaDB | Postgres # end: static analysis hack -def init(site, sites_path=None, new_site=False): +def init(site: str, sites_path: str = ".", new_site: bool = False) -> None: """Initialize frappe for the current site. Reset thread locals `frappe.local`""" if getattr(local, "initialised", None): return - if not sites_path: - sites_path = "." - local.error_log = [] local.message_log = [] local.debug_log = [] @@ -199,6 +206,7 @@ def init(site, sites_path=None, new_site=False): } ) local.rollback_observers = [] + local.locked_documents = [] local.before_commit = [] local.test_objects = {} @@ -217,7 +225,6 @@ def init(site, sites_path=None, new_site=False): local.module_app = None local.app_modules = None - local.system_settings = _dict() local.user = None local.user_perms = None @@ -232,20 +239,23 @@ def init(site, sites_path=None, new_site=False): local.cache = {} local.document_cache = {} local.meta_cache = {} - local.autoincremented_status_map = {site: -1} local.form_dict = _dict() local.session = _dict() local.dev_server = _dev_server local.qb = get_query_builder(local.conf.db_type or "mariadb") - + local.qb.engine = get_qb_engine() setup_module_map() - patch_query_execute() - patch_query_aggregation() + + if not _qb_patched.get(local.conf.db_type): + patch_query_execute() + patch_query_aggregation() local.initialised = True -def connect(site=None, db_name=None, set_admin_as_user=True): +def connect( + site: str | None = None, db_name: str | None = None, set_admin_as_user: bool = True +) -> None: """Connect to site database instance. :param site: If site is given, calls `frappe.init`. @@ -273,14 +283,16 @@ def connect_replica(): user = local.conf.replica_db_name password = local.conf.replica_db_password - local.replica_db = get_db(host=local.conf.replica_host, user=user, password=password, port=port) + local.replica_db = get_db( + host=local.conf.replica_host, user=user, password=password, port=port, read_only=True + ) # swap db connections local.primary_db = local.db local.db = local.replica_db -def get_site_config(sites_path=None, site_path=None): +def get_site_config(sites_path: str | None = None, site_path: str | None = None) -> dict[str, Any]: """Returns `site_config.json` combined with `sites/common_site_config.json`. `site_config` is a set of site wide settings like database name, password, email etc.""" config = {} @@ -303,15 +315,15 @@ def get_site_config(sites_path=None, site_path=None): try: config.update(get_file_json(site_config)) except Exception as error: - click.secho("{0}/site_config.json is invalid".format(local.site), fg="red") + click.secho(f"{local.site}/site_config.json is invalid", fg="red") print(error) elif local.site and not local.flags.new_site: - raise IncorrectSitePath("{0} does not exist".format(local.site)) + raise IncorrectSitePath(f"{local.site} does not exist") return _dict(config) -def get_conf(site: Optional[str] = None) -> _dict: +def get_conf(site: str | None = None) -> dict[str, Any]: if hasattr(local, "conf"): return local.conf @@ -354,14 +366,14 @@ def cache() -> "RedisWrapper": return redis_server -def get_traceback(): +def get_traceback(with_context: bool = False) -> str: """Returns error traceback.""" from frappe.utils import get_traceback - return get_traceback() + return get_traceback(with_context=with_context) -def errprint(msg): +def errprint(msg: str) -> None: """Log error. This is sent back as `exc` in response. :param msg: Message.""" @@ -372,11 +384,11 @@ def errprint(msg): error_log.append({"exc": msg}) -def print_sql(enable=True): +def print_sql(enable: bool = True) -> None: return cache().set_value("flag_print_sql", enable) -def log(msg): +def log(msg: str) -> None: """Add to `debug_log`. :param msg: Message.""" @@ -388,17 +400,17 @@ def log(msg): def msgprint( - msg, - title=None, - raise_exception=0, - as_table=False, - as_list=False, - indicator=None, - alert=False, - primary_action=None, - is_minimizable=None, - wide=None, -): + msg: str, + title: str | None = None, + raise_exception: bool | type[Exception] = False, + as_table: bool = False, + as_list: bool = False, + indicator: Literal["blue", "green", "orange", "red", "yellow"] | None = None, + alert: bool = False, + primary_action: str = None, + is_minimizable: bool = False, + wide: bool = False, +) -> None: """Print a message to the user (via HTTP response). Messages are sent in the `__server_messages` property in the response JSON and shown in a pop-up / modal. @@ -412,17 +424,20 @@ def msgprint( :param is_minimizable: [optional] Allow users to minimize the modal :param wide: [optional] Show wide modal """ + import inspect + import sys + from frappe.utils import strip_html_tags msg = safe_decode(msg) out = _dict(message=msg) + @functools.lru_cache(maxsize=1024) + def _strip_html_tags(message): + return strip_html_tags(message) + def _raise_exception(): if raise_exception: - if flags.rollback_on_exception: - db.rollback() - import inspect - if inspect.isclass(raise_exception) and issubclass(raise_exception, Exception): raise raise_exception(msg) else: @@ -435,11 +450,14 @@ def msgprint( if as_table and type(msg) in (list, tuple): out.as_table = 1 - if as_list and type(msg) in (list, tuple) and len(msg) > 1: + if as_list and type(msg) in (list, tuple): out.as_list = 1 + if sys.stdin.isatty(): + msg = _strip_html_tags(out.message) + if flags.print_messages and out.message: - print(f"Message: {strip_html_tags(out.message)}") + print(f"Message: {_strip_html_tags(out.message)}") out.title = title or _("Message", context="Default title of the message dialog") @@ -489,7 +507,14 @@ def clear_last_message(): local.message_log = local.message_log[:-1] -def throw(msg, exc=ValidationError, title=None, is_minimizable=None, wide=None, as_list=False): +def throw( + msg: str, + exc: type[Exception] = ValidationError, + title: str | None = None, + is_minimizable: bool = False, + wide: bool = False, + as_list: bool = False, +) -> None: """Throw execption and show message (`msgprint`). :param msg: Message. @@ -505,12 +530,6 @@ def throw(msg, exc=ValidationError, title=None, is_minimizable=None, wide=None, ) -def emit_js(js, user=False, **kwargs): - if user is False: - user = session.user - publish_realtime("eval_js", js, user=user, **kwargs) - - def create_folder(path, with_init=False): """Create a folder in the given path and add an `__init__.py` file (optional). @@ -525,7 +544,7 @@ def create_folder(path, with_init=False): touch_file(os.path.join(path, "__init__.py")) -def set_user(username): +def set_user(username: str): """Set current user. :param username: **User** name to set as current user.""" @@ -548,7 +567,7 @@ def get_user(): return local.user_perms -def get_roles(username=None): +def get_roles(username=None) -> list[str]: """Returns roles of current user.""" if not local.session: return ["Guest"] @@ -798,23 +817,30 @@ def write_only(): return innfn -def only_for(roles, message=False): - """Raise `frappe.PermissionError` if the user does not have any of the given **Roles**. +def only_for(roles: list[str] | tuple[str] | str, message=False): + """ + Raises `frappe.PermissionError` if the user does not have any of the permitted roles. - :param roles: List of roles to check.""" - if local.flags.in_test: + :param roles: Permitted role(s) + """ + + if local.flags.in_test or local.session.user == "Administrator": return - if not isinstance(roles, (tuple, list)): + if isinstance(roles, str): roles = (roles,) - roles = set(roles) - myroles = set(get_roles()) - if not roles.intersection(myroles): - if message: - msgprint( - _("This action is only allowed for {}").format(bold(", ".join(roles))), _("Not Permitted") - ) - raise PermissionError + + if not set(roles).intersection(get_roles()): + if not message: + raise PermissionError + + throw( + _("This action is only allowed for {}").format( + ", ".join(bold(_(role)) for role in roles), + ), + PermissionError, + _("Not Permitted"), + ) def get_domain_data(module): @@ -831,12 +857,13 @@ def get_domain_data(module): raise -def clear_cache(user=None, doctype=None): +def clear_cache(user: str | None = None, doctype: str | None = None): """Clear **User**, **DocType** or global cache. :param user: If user is given, only user cache is cleared. :param doctype: If doctype is given, only DocType cache is cleared.""" import frappe.cache_manager + import frappe.utils.caching if doctype: frappe.cache_manager.clear_doctype_cache(doctype) @@ -856,7 +883,14 @@ def clear_cache(user=None, doctype=None): for fn in get_hooks("clear_cache"): get_attr(fn)() + frappe.utils.caching._SITE_CACHE.clear() local.role_permissions = {} + if hasattr(local, "request_cache"): + local.request_cache.clear() + if hasattr(local, "system_settings"): + del local.system_settings + if hasattr(local, "website_settings"): + del local.website_settings def only_has_select_perm(doctype, user=None, ignore_permissions=False): @@ -903,7 +937,7 @@ def has_permission( if throw and not out: # mimics frappe.throw - document_label = f"{doc.doctype} {doc.name}" if doc else doctype + document_label = f"{_(doc.doctype)} {doc.name}" if doc else _(doctype) msgprint( _("No permission for {0}").format(document_label), raise_exception=ValidationError, @@ -956,7 +990,7 @@ def has_website_permission(doc=None, ptype="read", user=None, verbose=False, doc return False -def is_table(doctype): +def is_table(doctype: str) -> bool: """Returns True if `istable` property (indicating child Table) is set for given DocType.""" def get_tables(): @@ -966,14 +1000,16 @@ def is_table(doctype): return doctype in tables -def get_precision(doctype, fieldname, currency=None, doc=None): +def get_precision( + doctype: str, fieldname: str, currency: str | None = None, doc: Optional["Document"] = None +) -> int: """Get precision for a given field""" from frappe.model.meta import get_field_precision return get_field_precision(get_meta(doctype).get_field(fieldname), doc, currency) -def generate_hash(txt=None, length=None): +def generate_hash(txt: str | None = None, length: int | None = None) -> str: """Generates random hash for given text + current timestamp + random string.""" import hashlib import time @@ -995,7 +1031,12 @@ def reset_metadata_version(): return v -def new_doc(doctype, parent_doc=None, parentfield=None, as_dict=False): +def new_doc( + doctype: str, + parent_doc: Optional["Document"] = None, + parentfield: str | None = None, + as_dict: bool = False, +) -> "Document": """Returns a new document of the given DocType with defaults set. :param doctype: DocType of the new document. @@ -1013,6 +1054,16 @@ def set_value(doctype, docname, fieldname, value=None): return frappe.client.set_value(doctype, docname, fieldname, value) +@overload +def get_cached_doc(doctype, docname, _allow_dict=True) -> dict: + ... + + +@overload +def get_cached_doc(*args, **kwargs) -> "Document": + ... + + def get_cached_doc(*args, **kwargs): allow_dict = kwargs.pop("_allow_dict", False) @@ -1026,7 +1077,7 @@ def get_cached_doc(*args, **kwargs): return doc if key := can_cache_doc(args): - # local cache + # local cache - has "ready" `Document` objects if doc := local.document_cache.get(key): return _respond(doc) @@ -1034,13 +1085,26 @@ def get_cached_doc(*args, **kwargs): if doc := cache().hget("document_cache", key): return _respond(doc, True) - # database + # Not found in local/redis, fetch from DB doc = get_doc(*args, **kwargs) + # Store in cache + if not key: + key = get_document_cache_key(doc.doctype, doc.name) + + local.document_cache[key] = doc + + # Avoid setting in local.cache since we're already using local.document_cache above + # Try pickling the doc object as-is first, else fallback to doc.as_dict() + try: + cache().hset("document_cache", key, doc, cache_locally=False) + except Exception: + cache().hset("document_cache", key, doc.as_dict(), cache_locally=False) + return doc -def can_cache_doc(args): +def can_cache_doc(args) -> str | None: """ Determine if document should be cached based on get_doc params. Returns cache key if doc can be cached, None otherwise. @@ -1057,7 +1121,7 @@ def can_cache_doc(args): return get_document_cache_key(doctype, name) -def get_document_cache_key(doctype, name): +def get_document_cache_key(doctype: str, name: str): return f"{doctype}::{name}" @@ -1067,9 +1131,15 @@ def clear_document_cache(doctype, name): if key in local.document_cache: del local.document_cache[key] cache().hdel("document_cache", key) + if doctype == "System Settings" and hasattr(local, "system_settings"): + delattr(local, "system_settings") + if doctype == "Website Settings" and hasattr(local, "website_settings"): + delattr(local, "website_settings") -def get_cached_value(doctype, name, fieldname="name", as_dict=False): +def get_cached_value( + doctype: str, name: str, fieldname: str = "name", as_dict: bool = False +) -> Any: try: doc = get_cached_doc(doctype, name, _allow_dict=True) except DoesNotExistError: @@ -1087,7 +1157,7 @@ def get_cached_value(doctype, name, fieldname="name", as_dict=False): return values -def get_doc(*args, **kwargs): +def get_doc(*args, **kwargs) -> "Document": """Return a `frappe.model.document.Document` object of the given type and name. :param arg1: DocType name as string **or** document JSON. @@ -1107,19 +1177,22 @@ def get_doc(*args, **kwargs): doc = frappe.model.document.get_doc(*args, **kwargs) - # set in cache + # Replace cache if key := can_cache_doc(args): - local.document_cache[key] = doc - cache().hset("document_cache", key, doc.as_dict()) + if key in local.document_cache: + local.document_cache[key] = doc + + if cache().hexists("document_cache", key): + cache().hset("document_cache", key, doc.as_dict()) return doc -def get_last_doc(doctype, filters=None, order_by="creation desc"): +def get_last_doc(doctype, filters=None, order_by="creation desc", *, for_update=False): """Get last created document of this type.""" d = get_all(doctype, filters=filters, limit_page_length=1, order_by=order_by, pluck="name") if d: - return get_doc(doctype, d[0]) + return get_doc(doctype, d[0], for_update=for_update) else: raise DoesNotExistError @@ -1143,16 +1216,16 @@ def get_meta_module(doctype): def delete_doc( - doctype=None, - name=None, - force=0, - ignore_doctypes=None, - for_reload=False, - ignore_permissions=False, - flags=None, - ignore_on_trash=False, - ignore_missing=True, - delete_permanently=False, + doctype: str | None = None, + name: str | None = None, + force: bool = False, + ignore_doctypes: list[str] | None = None, + for_reload: bool = False, + ignore_permissions: bool = False, + flags: None = None, + ignore_on_trash: bool = False, + ignore_missing: bool = True, + delete_permanently: bool = False, ): """Delete a document. Calls `frappe.model.delete_doc.delete_doc`. @@ -1165,7 +1238,7 @@ def delete_doc( :param delete_permanently: Do not create a Deleted Document for the document.""" import frappe.model.delete_doc - frappe.model.delete_doc.delete_doc( + return frappe.model.delete_doc.delete_doc( doctype, name, force, @@ -1195,7 +1268,13 @@ def reload_doctype(doctype, force=False, reset_permissions=False): ) -def reload_doc(module, dt=None, dn=None, force=False, reset_permissions=False): +def reload_doc( + module: str, + dt: str | None = None, + dn: str | None = None, + force: bool = False, + reset_permissions: bool = False, +): """Reload Document from model (`[module]/[doctype]/[name]/[name].json`) files. :param module: Module name. @@ -1210,18 +1289,35 @@ def reload_doc(module, dt=None, dn=None, force=False, reset_permissions=False): @whitelist() -def rename_doc(*args, **kwargs): +def rename_doc( + doctype: str, + old: str, + new: str, + force: bool = False, + merge: bool = False, + *, + ignore_if_exists: bool = False, + show_alert: bool = True, + rebuild_search: bool = True, +) -> str: """ Renames a doc(dt, old) to doc(dt, new) and updates all linked fields of type "Link" Calls `frappe.model.rename_doc.rename_doc` """ - kwargs.pop("ignore_permissions", None) - kwargs.pop("cmd", None) from frappe.model.rename_doc import rename_doc - return rename_doc(*args, **kwargs) + return rename_doc( + doctype=doctype, + old=old, + new=new, + force=force, + merge=merge, + ignore_if_exists=ignore_if_exists, + show_alert=show_alert, + rebuild_search=rebuild_search, + ) def get_module(modulename): @@ -1229,12 +1325,12 @@ def get_module(modulename): return importlib.import_module(modulename) -def scrub(txt): +def scrub(txt: str) -> str: """Returns sluggified string. e.g. `Sales Order` becomes `sales_order`.""" return cstr(txt).replace(" ", "_").replace("-", "_").lower() -def unscrub(txt): +def unscrub(txt: str) -> str: """Returns titlified string. e.g. `sales_order` becomes `Sales Order`.""" return txt.replace("_", " ").replace("-", " ").title() @@ -1244,8 +1340,10 @@ def get_module_path(module, *joins): :param module: Module name. :param *joins: Join additional path elements using `os.path.join`.""" - module = scrub(module) - return get_pymodule_path(local.module_app[module] + "." + module, *joins) + from frappe.modules.utils import get_module_app + + app = get_module_app(module) + return get_pymodule_path(app + "." + scrub(module), *joins) def get_app_path(app_name, *joins): @@ -1297,6 +1395,7 @@ def get_all_apps(with_internal_apps=True, sites_path=None): return apps +@request_cache def get_installed_apps(sort=False, frappe_last=False): """Get list of installed apps in current site.""" if getattr(flags, "in_install_db", True): @@ -1338,47 +1437,49 @@ def get_doc_hooks(): return local.doc_events_hooks -def get_hooks(hook=None, default=None, app_name=None): +@request_cache +def _load_app_hooks(app_name: str | None = None): + hooks = {} + apps = [app_name] if app_name else get_installed_apps(sort=True) + + for app in apps: + try: + app_hooks = get_module(f"{app}.hooks") + except ImportError: + if local.flags.in_install_app: + # if app is not installed while restoring + # ignore it + pass + print(f'Could not find app "{app}"') + if not request: + raise SystemExit + raise + for key in dir(app_hooks): + if not key.startswith("_"): + append_hook(hooks, key, getattr(app_hooks, key)) + return hooks + + +def get_hooks( + hook: str = None, default: Any | None = "_KEEP_DEFAULT_LIST", app_name: str = None +) -> _dict: """Get hooks via `app/hooks.py` :param hook: Name of the hook. Will gather all hooks for this name and return as a list. :param default: Default if no hook found. :param app_name: Filter by app.""" - def load_app_hooks(app_name=None): - hooks = {} - for app in [app_name] if app_name else get_installed_apps(sort=True): - app = "frappe" if app == "webnotes" else app - try: - app_hooks = get_module(app + ".hooks") - except ImportError: - if local.flags.in_install_app: - # if app is not installed while restoring - # ignore it - pass - print('Could not find app "{0}"'.format(app_name)) - if not request: - sys.exit(1) - raise - for key in dir(app_hooks): - if not key.startswith("_"): - append_hook(hooks, key, getattr(app_hooks, key)) - return hooks - - no_cache = conf.developer_mode or False - if app_name: - hooks = _dict(load_app_hooks(app_name)) + hooks = _dict(_load_app_hooks(app_name)) else: - if no_cache: - hooks = _dict(load_app_hooks()) + if conf.developer_mode: + hooks = _dict(_load_app_hooks()) else: - hooks = _dict(cache().get_value("app_hooks", load_app_hooks)) + hooks = _dict(cache().get_value("app_hooks", _load_app_hooks)) if hook: - return hooks.get(hook) or (default if default is not None else []) - else: - return hooks + return hooks.get(hook, ([] if default == "_KEEP_DEFAULT_LIST" else default)) + return hooks def append_hook(target, key, value): @@ -1443,7 +1544,7 @@ def get_file_items(path, raise_not_found=False, ignore_empty_lines=True): def get_file_json(path): """Read a file and return parsed JSON object.""" - with open(path, "r") as f: + with open(path) as f: return json.load(f) @@ -1453,15 +1554,15 @@ def read_file(path, raise_not_found=False): path = path.encode("utf-8") if os.path.exists(path): - with open(path, "r") as f: + with open(path) as f: return as_unicode(f.read()) elif raise_not_found: - raise IOError("{} Not Found".format(path)) + raise OSError(f"{path} Not Found") else: return None -def get_attr(method_string): +def get_attr(method_string: str) -> Any: """Get python method object from its name.""" app_name = method_string.split(".")[0] if ( @@ -1476,7 +1577,7 @@ def get_attr(method_string): return getattr(get_module(modulename), methodname) -def call(fn, *args, **kwargs): +def call(fn: str | Callable, *args, **kwargs): """Call a function and match arguments.""" if isinstance(fn, str): fn = get_attr(fn) @@ -1486,18 +1587,35 @@ def call(fn, *args, **kwargs): return fn(*args, **newargs) -def get_newargs(fn, kwargs): +def get_newargs(fn: Callable, kwargs: dict[str, Any]) -> dict[str, Any]: + """Remove any kwargs that are not supported by the function. + + Example: + >>> def fn(a=1, b=2): pass + + >>> get_newargs(fn, {"a": 2, "c": 1}) + {"a": 2} + """ + + # if function has any **kwargs parameter that capture arbitrary keyword arguments + # Ref: https://docs.python.org/3/library/inspect.html#inspect.Parameter.kind + varkw_exist = False + if hasattr(fn, "fnargs"): fnargs = fn.fnargs else: - fullargspec = inspect.getfullargspec(fn) - fnargs = fullargspec.args - fnargs.extend(fullargspec.kwonlyargs) - varkw = fullargspec.varkw + signature = inspect.signature(fn) + fnargs = list(signature.parameters) + + for param_name, parameter in signature.parameters.items(): + if parameter.kind == inspect.Parameter.VAR_KEYWORD: + varkw_exist = True + fnargs.remove(param_name) + break newargs = {} for a in kwargs: - if (a in fnargs) or varkw: + if (a in fnargs) or varkw_exist: newargs[a] = kwargs.get(a) newargs.pop("ignore_permissions", None) @@ -1568,7 +1686,7 @@ def import_doc(path): import_doc(path) -def copy_doc(doc, ignore_no_copy=True): +def copy_doc(doc: "Document", ignore_no_copy: bool = True) -> "Document": """No_copy fields also get copied.""" import copy @@ -1686,6 +1804,14 @@ def respond_as_web_page( local.response["context"] = context +def redirect(url): + """Raise a 301 redirect to url""" + from frappe.exceptions import Redirect + + flags.redirect_location = url + raise Redirect + + def redirect_to_message(title, html, http_status_code=None, context=None, indicator_color=None): """Redirects to /message?id=random Similar to respond_as_web_page, but used to 'redirect' and show message pages like success, failure, etc. with a detailed message @@ -1706,8 +1832,8 @@ def redirect_to_message(title, html, http_status_code=None, context=None, indica if indicator_color: message["context"].update({"indicator_color": indicator_color}) - cache().set_value("message_id:{0}".format(message_id), message, expires_in_sec=60) - location = "/message?id={0}".format(message_id) + cache().set_value(f"message_id:{message_id}", message, expires_in_sec=60) + location = f"/message?id={message_id}" if not getattr(local, "is_ajax", False): local.response["type"] = "redirect" @@ -1793,18 +1919,21 @@ def get_value(*args, **kwargs): return db.get_value(*args, **kwargs) -def as_json(obj: Union[Dict, List], indent=1) -> str: +def as_json(obj: dict | list, indent=1, separators=None) -> str: from frappe.utils.response import json_handler + if separators is None: + separators = (",", ": ") + try: return json.dumps( - obj, indent=indent, sort_keys=True, default=json_handler, separators=(",", ": ") + obj, indent=indent, sort_keys=True, default=json_handler, separators=separators ) except TypeError: # this would break in case the keys are not all os "str" type - as defined in the JSON # adding this to ensure keys are sorted (expected behaviour) sorted_obj = dict(sorted(obj.items(), key=lambda kv: str(kv[0]))) - return json.dumps(sorted_obj, indent=indent, default=json_handler, separators=(",", ": ")) + return json.dumps(sorted_obj, indent=indent, default=json_handler, separators=separators) def are_emails_muted(): @@ -1821,7 +1950,7 @@ def get_test_records(doctype): get_module_path(get_doctype_module(doctype)), "doctype", scrub(doctype), "test_records.json" ) if os.path.exists(path): - with open(path, "r") as f: + with open(path) as f: return json.loads(f.read()) else: return [] @@ -1907,7 +2036,7 @@ def attach_print( if not file_name: file_name = name - file_name = file_name.replace(" ", "").replace("/", "-") + file_name = cstr(file_name).replace(" ", "").replace("/", "-") print_settings = db.get_singles_dict("Print Settings") @@ -2067,25 +2196,30 @@ def logger( ) -def log_error(message=None, title=_("Error")): +def log_error(title=None, message=None, reference_doctype=None, reference_name=None): """Log error to Error Log""" - - # AI ALERT: + # Parameter ALERT: # the title and message may be swapped # the better API for this is log_error(title, message), and used in many cases this way # this hack tries to be smart about whats a title (single line ;-)) and fixes it + traceback = None if message: - if "\n" in title: - error, title = title, message + if "\n" in title: # traceback sent as title + traceback, title = title, message else: - error = message - else: - error = get_traceback() + traceback = message - return get_doc(dict(doctype="Error Log", error=as_unicode(error), method=title)).insert( - ignore_permissions=True - ) + title = title or "Error" + traceback = as_unicode(traceback or get_traceback(with_context=True)) + + return get_doc( + doctype="Error Log", + error=traceback, + method=title, + reference_doctype=reference_doctype, + reference_name=reference_name, + ).insert(ignore_permissions=True) def get_desk_link(doctype, name): @@ -2096,7 +2230,7 @@ def get_desk_link(doctype, name): def bold(text): - return "{0}".format(text) + return f"{text}" def safe_eval(code, eval_globals=None, eval_locals=None): @@ -2124,10 +2258,10 @@ def safe_eval(code, eval_globals=None, eval_locals=None): for attribute in UNSAFE_ATTRIBUTES: if attribute in code: - throw('Illegal rule {0}. Cannot use "{1}"'.format(bold(code), attribute)) + throw(f'Illegal rule {bold(code)}. Cannot use "{attribute}"') if "__" in code: - throw('Illegal rule {0}. Cannot use "__"'.format(bold(code))) + throw(f'Illegal rule {bold(code)}. Cannot use "__"') if not eval_globals: eval_globals = {} @@ -2137,9 +2271,17 @@ def safe_eval(code, eval_globals=None, eval_locals=None): return eval(code, eval_globals, eval_locals) +def get_website_settings(key): + if not hasattr(local, "website_settings"): + local.website_settings = db.get_singles_dict("Website Settings", cast=True) + + return local.website_settings.get(key) + + def get_system_settings(key): - if key not in local.system_settings: - local.system_settings.update({key: db.get_single_value("System Settings", key)}) + if not hasattr(local, "system_settings"): + local.system_settings = db.get_singles_dict("System Settings", cast=True) + return local.system_settings.get(key) @@ -2243,7 +2385,4 @@ def mock(type, size=1, locale="en"): return squashify(results) -def validate_and_sanitize_search_inputs(fn): - from frappe.desk.search import validate_and_sanitize_search_inputs as func - - return func(fn) +from frappe.desk.search import validate_and_sanitize_search_inputs # noqa diff --git a/frappe/api.py b/frappe/api.py index 32e19a1b43..1048468077 100644 --- a/frappe/api.py +++ b/frappe/api.py @@ -167,7 +167,7 @@ def validate_auth(): """ Authenticate and sets user for the request. """ - authorization_header = frappe.get_request_header("Authorization", str()).split(" ") + authorization_header = frappe.get_request_header("Authorization", "").split(" ") if len(authorization_header) == 2: validate_oauth(authorization_header) diff --git a/frappe/app.py b/frappe/app.py index e6df29fbd9..298d94b06c 100644 --- a/frappe/app.py +++ b/frappe/app.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE @@ -26,13 +25,15 @@ from frappe.utils import get_site_name, sanitize_html from frappe.utils.error import make_error_snapshot from frappe.website.serve import get_response -local_manager = LocalManager([frappe.local]) +local_manager = LocalManager(frappe.local) _site = None _sites_path = os.environ.get("SITES_PATH", ".") +SAFE_HTTP_METHODS = ("GET", "HEAD", "OPTIONS") +UNSAFE_HTTP_METHODS = ("POST", "PUT", "DELETE", "PATCH") -class RequestContext(object): +class RequestContext: def __init__(self, environ): self.request = Request(environ) @@ -43,6 +44,7 @@ class RequestContext(object): frappe.destroy() +@local_manager.middleware @Request.application def application(request): response = None @@ -221,10 +223,6 @@ def handle_exception(e): or (frappe.local.request.path.startswith("/api/") and not accept_header.startswith("text")) ) - if frappe.conf.get("developer_mode"): - # don't fail silently - print(frappe.get_traceback()) - if respond_as_json: # handle ajax responses first # if the request is ajax, send back the trace or error message @@ -288,11 +286,18 @@ def handle_exception(e): if return_as_message: response = get_response("message", http_status_code=http_status_code) + if frappe.conf.get("developer_mode") and not respond_as_json: + # don't fail silently for non-json response errors + print(frappe.get_traceback()) + return response def after_request(rollback): - if (frappe.local.request.method in ("POST", "PUT") or frappe.local.flags.commit) and frappe.db: + # if HTTP method would change server state, commit if necessary + if frappe.db and ( + frappe.local.flags.commit or frappe.local.request.method in UNSAFE_HTTP_METHODS + ): if frappe.db.transaction_writes: frappe.db.commit() rollback = False @@ -309,9 +314,6 @@ def after_request(rollback): return rollback -application = local_manager.make_middleware(application) - - def serve( port=8000, profile=False, no_reload=False, no_threading=False, site=None, sites_path="." ): @@ -326,12 +328,10 @@ def serve( if not os.environ.get("NO_STATICS"): application = SharedDataMiddleware( - application, {str("/assets"): str(os.path.join(sites_path, "assets"))} + application, {"/assets": str(os.path.join(sites_path, "assets"))} ) - application = StaticDataMiddleware( - application, {str("/files"): str(os.path.abspath(sites_path))} - ) + application = StaticDataMiddleware(application, {"/files": str(os.path.abspath(sites_path))}) application.debug = True application.config = {"SERVER_NAME": "localhost:8000"} diff --git a/frappe/auth.py b/frappe/auth.py index dc53c20f28..455e9ee0c5 100644 --- a/frappe/auth.py +++ b/frappe/auth.py @@ -109,6 +109,9 @@ class HTTPRequest: class LoginManager: + + __slots__ = ("user", "info", "full_name", "user_type", "resume") + def __init__(self): self.user = None self.info = None @@ -165,7 +168,7 @@ class LoginManager: self.set_user_info() def get_user_info(self): - self.info = frappe.db.get_value( + self.info = frappe.get_cached_value( "User", self.user, ["user_type", "first_name", "last_name", "user_image"], as_dict=1 ) @@ -412,10 +415,16 @@ def clear_cookies(): def validate_ip_address(user): """check if IP Address is valid""" - user = ( - frappe.get_cached_doc("User", user) if not frappe.flags.in_test else frappe.get_doc("User", user) + from frappe.core.doctype.user.user import get_restricted_ip_list + + # Only fetch required fields - for perf + user_fields = ["restrict_ip", "bypass_restrict_ip_check_if_2fa_enabled"] + user_info = ( + frappe.get_cached_value("User", user, user_fields, as_dict=True) + if not frappe.flags.in_test + else frappe.db.get_value("User", user, user_fields, as_dict=True) ) - ip_list = user.get_restricted_ip_list() + ip_list = get_restricted_ip_list(user_info) if not ip_list: return @@ -430,7 +439,7 @@ def validate_ip_address(user): # check if two factor auth is enabled if system_settings.enable_two_factor_auth and not bypass_restrict_ip_check: # check if bypass restrict ip is enabled for login user - bypass_restrict_ip_check = user.bypass_restrict_ip_check_if_2fa_enabled + bypass_restrict_ip_check = user_info.bypass_restrict_ip_check_if_2fa_enabled for ip in ip_list: if frappe.local.request_ip.startswith(ip) or bypass_restrict_ip_check: @@ -465,7 +474,7 @@ def get_login_attempt_tracker(user_name: str, raise_locked_exception: bool = Tru return tracker -class LoginAttemptTracker(object): +class LoginAttemptTracker: """Track login attemts of a user. Lock the account for s number of seconds if there have been n consecutive unsuccessful attempts to log in. diff --git a/frappe/automation/doctype/assignment_rule/assignment_rule.js b/frappe/automation/doctype/assignment_rule/assignment_rule.js index 97bed4f8f3..3e029e8444 100644 --- a/frappe/automation/doctype/assignment_rule/assignment_rule.js +++ b/frappe/automation/doctype/assignment_rule/assignment_rule.js @@ -1,82 +1,77 @@ // Copyright (c) 2019, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Assignment Rule', { - refresh: function(frm) { - frm.trigger('setup_assignment_days_buttons'); - frm.trigger('set_options'); +frappe.ui.form.on("Assignment Rule", { + refresh: function (frm) { + frm.trigger("setup_assignment_days_buttons"); + frm.trigger("set_options"); // refresh description frm.events.rule(frm); }, - setup: function(frm) { + setup: function (frm) { frm.set_query("document_type", () => { return { filters: { - name: ["!=", "ToDo"] - } + name: ["!=", "ToDo"], + }, }; }); }, - document_type: function(frm) { - frm.trigger('set_options'); + document_type: function (frm) { + frm.trigger("set_options"); }, - setup_assignment_days_buttons: function(frm) { - const labels = ['Weekends', 'Weekdays', 'All Days']; + setup_assignment_days_buttons: function (frm) { + const labels = ["Weekends", "Weekdays", "All Days"]; let get_days = (label) => { - const weekdays = ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday']; - const weekends = ['Saturday', 'Sunday']; + const weekdays = ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday"]; + const weekends = ["Saturday", "Sunday"]; return { - 'All Days': weekdays.concat(weekends), - 'Weekdays': weekdays, - 'Weekends': weekends, + "All Days": weekdays.concat(weekends), + Weekdays: weekdays, + Weekends: weekends, }[label]; }; let set_days = (e) => { - frm.clear_table('assignment_days'); + frm.clear_table("assignment_days"); const label = $(e.currentTarget).text(); - get_days(label).forEach((day) => - frm.add_child('assignment_days', { day: day }) - ); - frm.refresh_field('assignment_days'); + get_days(label).forEach((day) => frm.add_child("assignment_days", { day: day })); + frm.refresh_field("assignment_days"); }; - labels.forEach(label => - frm.fields_dict['assignment_days'].grid.add_custom_button( - label, - set_days, - 'top' - ) + labels.forEach((label) => + frm.fields_dict["assignment_days"].grid.add_custom_button(label, set_days, "top") ); }, - rule: function(frm) { + rule: function (frm) { const description_map = { - 'Round Robin': __('Assign one by one, in sequence'), - 'Load Balancing': __('Assign to the one who has the least assignments'), - 'Based on Field': __('Assign to the user set in this field'), + "Round Robin": __("Assign one by one, in sequence"), + "Load Balancing": __("Assign to the one who has the least assignments"), + "Based on Field": __("Assign to the user set in this field"), }; - frm.get_field('rule').set_description(description_map[frm.doc.rule]); + frm.get_field("rule").set_description(description_map[frm.doc.rule]); }, set_options(frm) { const doctype = frm.doc.document_type; frm.set_fields_as_options( - 'field', + "field", doctype, - (df) => ['Dynamic Link', 'Data'].includes(df.fieldtype) - || (df.fieldtype == 'Link' && df.options == 'User'), - [{ label: 'Owner', value: 'owner' }] + (df) => + ["Dynamic Link", "Data"].includes(df.fieldtype) || + (df.fieldtype == "Link" && df.options == "User"), + [{ label: "Owner", value: "owner" }] ); if (doctype) { - frm.set_fields_as_options( - 'due_date_based_on', - doctype, - (df) => ['Date', 'Datetime'].includes(df.fieldtype) - ).then(options => frm.set_df_property('due_date_based_on', 'hidden', !options.length)); + frm.set_fields_as_options("due_date_based_on", doctype, (df) => + ["Date", "Datetime"].includes(df.fieldtype) + ).then((options) => + frm.set_df_property("due_date_based_on", "hidden", !options.length) + ); } }, }); diff --git a/frappe/automation/doctype/assignment_rule/assignment_rule.py b/frappe/automation/doctype/assignment_rule/assignment_rule.py index f3dfa4cf0a..508ed317c6 100644 --- a/frappe/automation/doctype/assignment_rule/assignment_rule.py +++ b/frappe/automation/doctype/assignment_rule/assignment_rule.py @@ -1,7 +1,7 @@ # Copyright (c) 2022, Frappe Technologies and contributors # License: MIT. See LICENSE -from typing import Dict, Iterable, List +from collections.abc import Iterable import frappe from frappe import _ @@ -157,7 +157,7 @@ class AssignmentRule(Document): return assignment_days and today not in assignment_days -def get_assignments(doc) -> List[Dict]: +def get_assignments(doc) -> list[dict]: return frappe.get_all( "ToDo", fields=["name", "assignment_rule"], @@ -228,7 +228,7 @@ def apply(doc=None, method=None, doctype=None, name=None): ) # multiple auto assigns - assignment_rule_docs: List[AssignmentRule] = [ + assignment_rule_docs: list[AssignmentRule] = [ frappe.get_cached_doc("Assignment Rule", d.get("name")) for d in assignment_rules ] @@ -298,8 +298,6 @@ def apply(doc=None, method=None, doctype=None, name=None): if reopened: break - # print(f"Rule:{assignment_rule}\nDoc: {doc}\nReOpened: {reopened}") - assignment_rule.close_assignments(doc) @@ -358,11 +356,11 @@ def update_due_date(doc, state=None): todo_doc.save(ignore_permissions=True) -def get_assignment_rules() -> List[str]: +def get_assignment_rules() -> list[str]: return frappe.get_all("Assignment Rule", filters={"disabled": 0}, pluck="document_type") -def get_repeated(values: Iterable) -> List: +def get_repeated(values: Iterable) -> list: unique = set() repeated = set() diff --git a/frappe/automation/doctype/assignment_rule_day/assignment_rule_day.py b/frappe/automation/doctype/assignment_rule_day/assignment_rule_day.py index 2a7e6dd66f..5a1af94696 100644 --- a/frappe/automation/doctype/assignment_rule_day/assignment_rule_day.py +++ b/frappe/automation/doctype/assignment_rule_day/assignment_rule_day.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and contributors # License: MIT. See LICENSE diff --git a/frappe/automation/doctype/assignment_rule_user/assignment_rule_user.py b/frappe/automation/doctype/assignment_rule_user/assignment_rule_user.py index 3f47f3c866..8b848589c3 100644 --- a/frappe/automation/doctype/assignment_rule_user/assignment_rule_user.py +++ b/frappe/automation/doctype/assignment_rule_user/assignment_rule_user.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and contributors # License: MIT. See LICENSE diff --git a/frappe/automation/doctype/auto_repeat/auto_repeat.js b/frappe/automation/doctype/auto_repeat/auto_repeat.js index 80f2255f47..c0fa2696be 100644 --- a/frappe/automation/doctype/auto_repeat/auto_repeat.js +++ b/frappe/automation/doctype/auto_repeat/auto_repeat.js @@ -2,41 +2,45 @@ // For license information, please see license.txt frappe.provide("frappe.auto_repeat"); -frappe.ui.form.on('Auto Repeat', { - setup: function(frm) { - frm.fields_dict['reference_doctype'].get_query = function() { +frappe.ui.form.on("Auto Repeat", { + setup: function (frm) { + frm.fields_dict["reference_doctype"].get_query = function () { return { - query: "frappe.automation.doctype.auto_repeat.auto_repeat.get_auto_repeat_doctypes" + query: "frappe.automation.doctype.auto_repeat.auto_repeat.get_auto_repeat_doctypes", }; }; - frm.fields_dict['reference_document'].get_query = function() { + frm.fields_dict["reference_document"].get_query = function () { return { filters: { - "auto_repeat": '' - } + auto_repeat: "", + }, }; }; - frm.fields_dict['print_format'].get_query = function() { + frm.fields_dict["print_format"].get_query = function () { return { filters: { - "doc_type": frm.doc.reference_doctype - } + doc_type: frm.doc.reference_doctype, + }, }; }; }, - refresh: function(frm) { + refresh: function (frm) { // auto repeat message if (frm.is_new()) { - let customize_form_link = `${__('Customize Form')}`; - frm.dashboard.set_headline(__('To configure Auto Repeat, enable "Allow Auto Repeat" from {0}.', [customize_form_link])); + let customize_form_link = `${__("Customize Form")}`; + frm.dashboard.set_headline( + __('To configure Auto Repeat, enable "Allow Auto Repeat" from {0}.', [ + customize_form_link, + ]) + ); } // view document button if (!frm.is_dirty()) { - let label = __('View {0}', [__(frm.doc.reference_doctype)]); + let label = __("View {0}", [__(frm.doc.reference_doctype)]); frm.add_custom_button(label, () => frappe.set_route("List", frm.doc.reference_doctype, { auto_repeat: frm.doc.name }) ); @@ -45,24 +49,24 @@ frappe.ui.form.on('Auto Repeat', { // auto repeat schedule frappe.auto_repeat.render_schedule(frm); - frm.trigger('toggle_submit_on_creation'); + frm.trigger("toggle_submit_on_creation"); }, - reference_doctype: function(frm) { - frm.trigger('toggle_submit_on_creation'); + reference_doctype: function (frm) { + frm.trigger("toggle_submit_on_creation"); }, - toggle_submit_on_creation: function(frm) { + toggle_submit_on_creation: function (frm) { // submit on creation checkbox if (frm.doc.reference_doctype) { frappe.model.with_doctype(frm.doc.reference_doctype, () => { let meta = frappe.get_meta(frm.doc.reference_doctype); - frm.toggle_display('submit_on_creation', meta.is_submittable); + frm.toggle_display("submit_on_creation", meta.is_submittable); }); } }, - template: function(frm) { + template: function (frm) { if (frm.doc.template) { frappe.model.with_doc("Email Template", frm.doc.template, () => { let email_template = frappe.get_doc("Email Template", frm.doc.template); @@ -74,11 +78,11 @@ frappe.ui.form.on('Auto Repeat', { } }, - get_contacts: function(frm) { - frm.call('fetch_linked_contacts'); + get_contacts: function (frm) { + frm.call("fetch_linked_contacts"); }, - preview_message: function(frm) { + preview_message: function (frm) { if (frm.doc.message) { frappe.call({ method: "frappe.automation.doctype.auto_repeat.auto_repeat.generate_message_preview", @@ -86,29 +90,29 @@ frappe.ui.form.on('Auto Repeat', { reference_dt: frm.doc.reference_doctype, reference_doc: frm.doc.reference_document, subject: frm.doc.subject, - message: frm.doc.message + message: frm.doc.message, }, - callback: function(r) { + callback: function (r) { if (r.message) { - frappe.msgprint(r.message.message, r.message.subject) + frappe.msgprint(r.message.message, r.message.subject); } - } + }, }); } else { - frappe.msgprint(__("Please setup a message first"), __("Message not setup")) + frappe.msgprint(__("Please setup a message first"), __("Message not setup")); } - } + }, }); -frappe.auto_repeat.render_schedule = function(frm) { - if (!frm.is_dirty() && frm.doc.status !== 'Disabled') { - frm.call("get_auto_repeat_schedule").then(r => { +frappe.auto_repeat.render_schedule = function (frm) { + if (!frm.is_dirty() && frm.doc.status !== "Disabled") { + frm.call("get_auto_repeat_schedule").then((r) => { frm.dashboard.reset(); frm.dashboard.add_section( frappe.render_template("auto_repeat_schedule", { - schedule_details: r.message || [] + schedule_details: r.message || [], }), - __('Auto Repeat Schedule') + __("Auto Repeat Schedule") ); frm.dashboard.show(); }); diff --git a/frappe/automation/doctype/auto_repeat/auto_repeat.py b/frappe/automation/doctype/auto_repeat/auto_repeat.py index 5ff9e9f3ab..0442be0976 100644 --- a/frappe/automation/doctype/auto_repeat/auto_repeat.py +++ b/frappe/automation/doctype/auto_repeat/auto_repeat.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2018, Frappe Technologies Pvt. Ltd. and contributors # License: MIT. See LICENSE @@ -189,7 +188,7 @@ class AutoRepeat(Document): if self.notify_by_email and self.recipients: self.send_notification(new_doc) except Exception: - error_log = frappe.log_error(frappe.get_traceback(), _("Auto Repeat Document Creation Failure")) + error_log = self.log_error("Auto repeat failed") self.disable_auto_repeat() diff --git a/frappe/automation/doctype/auto_repeat/auto_repeat_list.js b/frappe/automation/doctype/auto_repeat/auto_repeat_list.js index f906580f7e..f970341fa3 100644 --- a/frappe/automation/doctype/auto_repeat/auto_repeat_list.js +++ b/frappe/automation/doctype/auto_repeat/auto_repeat_list.js @@ -1,11 +1,11 @@ -frappe.listview_settings['Auto Repeat'] = { +frappe.listview_settings["Auto Repeat"] = { add_fields: ["next_schedule_date"], - get_indicator: function(doc) { + get_indicator: function (doc) { var colors = { - "Active": "green", - "Disabled": "red", - "Completed": "blue", + Active: "green", + Disabled: "red", + Completed: "blue", }; return [__(doc.status), colors[doc.status], "status,=," + doc.status]; - } + }, }; diff --git a/frappe/automation/doctype/auto_repeat/test_auto_repeat.py b/frappe/automation/doctype/auto_repeat/test_auto_repeat.py index e1db7ca6d1..ee0addf847 100644 --- a/frappe/automation/doctype/auto_repeat/test_auto_repeat.py +++ b/frappe/automation/doctype/auto_repeat/test_auto_repeat.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2018, Frappe Technologies and Contributors # License: MIT. See LICENSE import unittest @@ -200,7 +199,7 @@ class TestAutoRepeat(unittest.TestCase): # next_schedule_date is set as on or after current date # it should not be a previous month's date - self.assertTrue((doc.next_schedule_date >= current_date)) + self.assertTrue(doc.next_schedule_date >= current_date) todo = frappe.get_doc( dict( diff --git a/frappe/automation/doctype/auto_repeat_day/auto_repeat_day.py b/frappe/automation/doctype/auto_repeat_day/auto_repeat_day.py index 95d75bf9da..6453f2e80d 100644 --- a/frappe/automation/doctype/auto_repeat_day/auto_repeat_day.py +++ b/frappe/automation/doctype/auto_repeat_day/auto_repeat_day.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and contributors # License: MIT. See LICENSE diff --git a/frappe/automation/doctype/milestone/milestone.js b/frappe/automation/doctype/milestone/milestone.js index 9a1cf577ff..2a5ab04135 100644 --- a/frappe/automation/doctype/milestone/milestone.js +++ b/frappe/automation/doctype/milestone/milestone.js @@ -1,8 +1,7 @@ // Copyright (c) 2019, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Milestone', { +frappe.ui.form.on("Milestone", { // refresh: function(frm) { - // } }); diff --git a/frappe/automation/doctype/milestone/milestone.json b/frappe/automation/doctype/milestone/milestone.json index 8360ce7bf4..aa2dd35891 100644 --- a/frappe/automation/doctype/milestone/milestone.json +++ b/frappe/automation/doctype/milestone/milestone.json @@ -1,230 +1,81 @@ { - "allow_copy": 0, - "allow_events_in_timeline": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "autoname": "", - "beta": 0, + "actions": [], "creation": "2019-04-17 09:39:15.647817", - "custom": 0, - "docstatus": 0, "doctype": "DocType", - "document_type": "", - "editable_grid": 0, "engine": "InnoDB", + "field_order": [ + "reference_type", + "reference_name", + "track_field", + "value", + "milestone_tracker" + ], "fields": [ { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fetch_if_empty": 0, "fieldname": "reference_type", "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, "in_list_view": 1, - "in_standard_filter": 0, "label": "Document Type", - "length": 0, - "no_copy": 0, "options": "DocType", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, "reqd": 1, - "search_index": 1, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "search_index": 1 }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fetch_if_empty": 0, "fieldname": "reference_name", "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, "in_list_view": 1, - "in_standard_filter": 0, "label": "Document", - "length": 0, - "no_copy": 0, - "options": "", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "reqd": 1 }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fetch_if_empty": 0, "fieldname": "track_field", "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, "label": "Track Field", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "reqd": 1 }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fetch_if_empty": 0, "fieldname": "value", "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, "in_list_view": 1, - "in_standard_filter": 0, "label": "Value", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "reqd": 1 }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fetch_if_empty": 0, "fieldname": "milestone_tracker", "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, "label": "Milestone Tracker", - "length": 0, - "no_copy": 0, - "options": "Milestone Tracker", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "options": "Milestone Tracker" } ], - "has_web_view": 0, - "hide_toolbar": 0, - "idx": 0, "in_create": 1, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2019-04-17 16:01:21.430344", + "links": [], + "modified": "2022-08-03 12:20:55.076769", "modified_by": "Administrator", "module": "Automation", "name": "Milestone", - "name_case": "", "owner": "Administrator", "permissions": [ { - "amend": 0, - "cancel": 0, "create": 1, "delete": 1, "email": 1, "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, "print": 1, "read": 1, "report": 1, "role": "System Manager", - "set_user_permissions": 0, "share": 1, - "submit": 0, "write": 1 } ], "quick_entry": 1, - "read_only": 0, - "show_name_in_global_search": 0, "sort_field": "modified", "sort_order": "ASC", + "states": [], "title_field": "reference_type", - "track_changes": 1, - "track_seen": 0, - "track_views": 0 + "track_changes": 1 } \ No newline at end of file diff --git a/frappe/automation/doctype/milestone/milestone.py b/frappe/automation/doctype/milestone/milestone.py index 4059a2eb73..40d6fae989 100644 --- a/frappe/automation/doctype/milestone/milestone.py +++ b/frappe/automation/doctype/milestone/milestone.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and contributors # License: MIT. See LICENSE diff --git a/frappe/automation/doctype/milestone/test_milestone.py b/frappe/automation/doctype/milestone/test_milestone.py index 1824220497..5ac0754e5a 100644 --- a/frappe/automation/doctype/milestone/test_milestone.py +++ b/frappe/automation/doctype/milestone/test_milestone.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # License: MIT. See LICENSE # import frappe diff --git a/frappe/automation/doctype/milestone_tracker/milestone_tracker.js b/frappe/automation/doctype/milestone_tracker/milestone_tracker.js index 2a74bfb070..bf5be880a0 100644 --- a/frappe/automation/doctype/milestone_tracker/milestone_tracker.js +++ b/frappe/automation/doctype/milestone_tracker/milestone_tracker.js @@ -1,14 +1,14 @@ // Copyright (c) 2019, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Milestone Tracker', { - refresh: function(frm) { - frm.trigger('update_options'); +frappe.ui.form.on("Milestone Tracker", { + refresh: function (frm) { + frm.trigger("update_options"); }, - document_type: function(frm) { - frm.trigger('update_options'); + document_type: function (frm) { + frm.trigger("update_options"); }, - update_options: function(frm) { + update_options: function (frm) { // update select options for `track_field` let doctype = frm.doc.document_type; let track_fields = []; @@ -16,18 +16,16 @@ frappe.ui.form.on('Milestone Tracker', { if (doctype) { frappe.model.with_doctype(doctype, () => { // get all date and datetime fields - frappe.get_meta(doctype).fields.map(df => { - if (['Link', 'Select'].includes(df.fieldtype)) { - track_fields.push({label: df.label, value: df.fieldname}); + frappe.get_meta(doctype).fields.map((df) => { + if (["Link", "Select"].includes(df.fieldtype)) { + track_fields.push({ label: df.label, value: df.fieldname }); } }); - frm.set_df_property('track_field', 'options', track_fields); + frm.set_df_property("track_field", "options", track_fields); }); } else { // update select options - frm.set_df_property('track_field', 'options', []); + frm.set_df_property("track_field", "options", []); } - }, - }); diff --git a/frappe/automation/doctype/milestone_tracker/milestone_tracker.json b/frappe/automation/doctype/milestone_tracker/milestone_tracker.json index 8e22e3e199..8d4ed94dcd 100644 --- a/frappe/automation/doctype/milestone_tracker/milestone_tracker.json +++ b/frappe/automation/doctype/milestone_tracker/milestone_tracker.json @@ -1,162 +1,61 @@ { - "allow_copy": 0, - "allow_events_in_timeline": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, + "actions": [], "autoname": "format:{document_type}-{track_field}", - "beta": 0, "creation": "2019-04-17 09:36:41.774774", - "custom": 0, "description": "Track milestones for any document", - "docstatus": 0, "doctype": "DocType", - "document_type": "", - "editable_grid": 0, "engine": "InnoDB", + "field_order": [ + "document_type", + "track_field", + "disabled" + ], "fields": [ { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fetch_if_empty": 0, "fieldname": "document_type", "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, "in_list_view": 1, - "in_standard_filter": 0, "label": "Document Type to Track", - "length": 0, - "no_copy": 0, "options": "DocType", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, "unique": 1 }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fetch_if_empty": 0, "fieldname": "track_field", "fieldtype": "Select", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, "in_list_view": 1, - "in_standard_filter": 0, "label": "Field to Track", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "reqd": 1 }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fetch_if_empty": 0, + "default": "0", "fieldname": "disabled", "fieldtype": "Check", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Disabled", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "label": "Disabled" } ], - "has_web_view": 0, - "hide_toolbar": 0, - "idx": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2019-04-22 16:03:32.848937", + "links": [], + "modified": "2022-08-03 12:20:54.955953", "modified_by": "Administrator", "module": "Automation", "name": "Milestone Tracker", - "name_case": "", "owner": "Administrator", "permissions": [ { - "amend": 0, - "cancel": 0, "create": 1, "delete": 1, "email": 1, "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, "print": 1, "read": 1, "report": 1, "role": "System Manager", - "set_user_permissions": 0, "share": 1, - "submit": 0, "write": 1 } ], - "quick_entry": 0, - "read_only": 0, - "show_name_in_global_search": 0, "sort_field": "modified", "sort_order": "ASC", - "track_changes": 1, - "track_seen": 0, - "track_views": 0 + "states": [], + "track_changes": 1 } \ No newline at end of file diff --git a/frappe/automation/doctype/milestone_tracker/milestone_tracker.py b/frappe/automation/doctype/milestone_tracker/milestone_tracker.py index 16b2fe9204..5388797b80 100644 --- a/frappe/automation/doctype/milestone_tracker/milestone_tracker.py +++ b/frappe/automation/doctype/milestone_tracker/milestone_tracker.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and contributors # License: MIT. See LICENSE diff --git a/frappe/automation/doctype/milestone_tracker/test_milestone_tracker.py b/frappe/automation/doctype/milestone_tracker/test_milestone_tracker.py index 4e53072348..2b48a76805 100644 --- a/frappe/automation/doctype/milestone_tracker/test_milestone_tracker.py +++ b/frappe/automation/doctype/milestone_tracker/test_milestone_tracker.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # License: MIT. See LICENSE import unittest diff --git a/frappe/boot.py b/frappe/boot.py index 1b8e471e00..98d1e59a57 100644 --- a/frappe/boot.py +++ b/frappe/boot.py @@ -14,13 +14,13 @@ from frappe.email.inbox import get_email_accounts from frappe.model.base_document import get_controller from frappe.query_builder import DocType from frappe.query_builder.functions import Count -from frappe.query_builder.terms import subqry +from frappe.query_builder.terms import ParameterizedValueWrapper, SubQuery from frappe.social.doctype.energy_point_log.energy_point_log import get_energy_points from frappe.social.doctype.energy_point_settings.energy_point_settings import ( is_energy_point_enabled, ) -from frappe.translate import get_lang_dict -from frappe.utils import add_user_info, get_time_zone +from frappe.translate import get_lang_dict, get_messages_for_boot +from frappe.utils import add_user_info, cstr, get_time_zone from frappe.utils.change_log import get_versions from frappe.website.doctype.web_page_view.web_page_view import is_tracking_enabled @@ -75,6 +75,8 @@ def get_bootinfo(): # add docs bootinfo.docs = doclist + load_country_doc(bootinfo) + load_currency_docs(bootinfo) for method in hooks.boot_session or []: frappe.get_attr(method)(bootinfo) @@ -98,6 +100,7 @@ def get_bootinfo(): bootinfo.desk_settings = get_desk_settings() bootinfo.app_logo_url = get_app_logo() bootinfo.link_title_doctypes = get_link_title_doctypes() + bootinfo.translatable_doctypes = get_translatable_doctypes() return bootinfo @@ -137,6 +140,10 @@ def get_allowed_reports(cache=False): return get_user_pages_or_reports("Report", cache=cache) +def get_allowed_report_names(cache=False) -> set[str]: + return {cstr(report) for report in get_allowed_reports(cache).keys() if report} + + def get_user_pages_or_reports(parent, cache=False): _cache = frappe.cache() @@ -208,7 +215,7 @@ def get_user_pages_or_reports(parent, cache=False): if parent == "Report": has_role[p.name].update({"ref_doctype": p.ref_doctype}) - no_of_roles = ( + no_of_roles = SubQuery( frappe.qb.from_(hasRole).select(Count("*")).where(hasRole.parent == parentTable.name) ) @@ -218,7 +225,7 @@ def get_user_pages_or_reports(parent, cache=False): pages_with_no_roles = ( frappe.qb.from_(parentTable) .select(parentTable.name, parentTable.modified, *columns) - .where(subqry(no_of_roles) == 0) + .where(no_of_roles == 0) ).run(as_dict=True) for p in pages_with_no_roles: @@ -241,18 +248,8 @@ def get_user_pages_or_reports(parent, cache=False): def load_translations(bootinfo): - messages = frappe.get_lang_dict("boot") - bootinfo["lang"] = frappe.lang - - # load translated report names - for name in bootinfo.user.all_reports: - messages[name] = frappe._(name) - - # only untranslated - messages = {k: v for k, v in messages.items() if k != v} - - bootinfo["__messages"] = messages + bootinfo["__messages"] = get_messages_for_boot() def get_user_info(): @@ -321,11 +318,11 @@ def get_unseen_notes(): frappe.qb.from_(note) .select(note.name, note.title, note.content, note.notify_on_every_login) .where( - (note.notify_on_every_login == 1) + (note.notify_on_login == 1) & (note.expire_notification_on > frappe.utils.now()) & ( - subqry(frappe.qb.from_(nsb).select(nsb.user).where(nsb.parent == note.name)).notin( - [frappe.session.user] + ParameterizedValueWrapper(frappe.session.user).notin( + SubQuery(frappe.qb.from_(nsb).select(nsb.user).where(nsb.parent == note.name)) ) ) ) @@ -384,7 +381,6 @@ def get_notification_settings(): return frappe.get_cached_doc("Notification Settings", frappe.session.user) -@frappe.whitelist() def get_link_title_doctypes(): dts = frappe.get_all("DocType", {"show_title_field_in_link": 1}) custom_dts = frappe.get_all( @@ -401,3 +397,42 @@ def set_time_zone(bootinfo): "user": bootinfo.get("user_info", {}).get(frappe.session.user, {}).get("time_zone", None) or get_time_zone(), } + + +def get_translatable_doctypes(): + dts = frappe.get_all("DocType", {"translate_link_fields": 1}, pluck="name") + custom_dts = frappe.get_all( + "Property Setter", {"property": "translate_link_fields", "value": "1"}, pluck="doc_type" + ) + return dts + custom_dts + + +def load_country_doc(bootinfo): + country = frappe.db.get_default("country") + if not country: + return + try: + bootinfo.docs.append(frappe.get_cached_doc("Country", country)) + except Exception: + pass + + +def load_currency_docs(bootinfo): + currency = frappe.qb.DocType("Currency") + + currency_docs = ( + frappe.qb.from_(currency) + .select( + currency.name, + currency.fraction, + currency.fraction_units, + currency.number_format, + currency.smallest_currency_fraction_value, + currency.symbol, + currency.symbol_on_right, + ) + .where(currency.enabled == 1) + .run(as_dict=1, update={"doctype": ":Currency"}) + ) + + bootinfo.docs += currency_docs diff --git a/frappe/build.py b/frappe/build.py index e20ee0d698..e66da4bd79 100644 --- a/frappe/build.py +++ b/frappe/build.py @@ -20,6 +20,8 @@ import frappe timestamps = {} app_paths = None sites_path = os.path.abspath(os.getcwd()) +WHITESPACE_PATTERN = re.compile(r"\s+") +HTML_COMMENT_PATTERN = re.compile(r"()") class AssetsNotDownloadedError(Exception): @@ -198,12 +200,12 @@ def symlink(target, link_name, overwrite=False): try: # Pre-empt os.replace on a directory with a nicer message if os.path.isdir(link_name): - raise IsADirectoryError("Cannot symlink over existing directory: '{}'".format(link_name)) + raise IsADirectoryError(f"Cannot symlink over existing directory: '{link_name}'") try: os.replace(temp_link_name, link_name) except AttributeError: os.renames(temp_link_name, link_name) - except: + except Exception: if os.path.islink(temp_link_name): os.remove(temp_link_name) raise @@ -237,10 +239,10 @@ def bundle( make_asset_dirs(hard_link=hard_link) mode = "production" if mode == "production" else "build" - command = "yarn run {mode}".format(mode=mode) + command = f"yarn run {mode}" if apps: - command += " --apps {apps}".format(apps=apps) + command += f" --apps {apps}" if skip_frappe: command += " --skip_frappe" @@ -261,7 +263,7 @@ def watch(apps=None): command = "yarn run watch" if apps: - command += " --apps {apps}".format(apps=apps) + command += f" --apps {apps}" live_reload = frappe.utils.cint(os.environ.get("LIVE_RELOAD", frappe.conf.live_reload)) @@ -406,10 +408,10 @@ def link_assets_dir(source, target, hard_link=False): def scrub_html_template(content): """Returns HTML content with removed whitespace and comments""" # remove whitespace to a single space - content = re.sub(r"\s+", " ", content) + content = WHITESPACE_PATTERN.sub(" ", content) # strip comments - content = re.sub(r"()", "", content) + content = HTML_COMMENT_PATTERN.sub("", content) return content.replace("'", "'") diff --git a/frappe/cache_manager.py b/frappe/cache_manager.py index b15f8f2234..01ccc03753 100644 --- a/frappe/cache_manager.py +++ b/frappe/cache_manager.py @@ -5,7 +5,6 @@ import json import frappe from frappe.desk.notifications import clear_notifications, delete_notification_count_for -from frappe.model.document import Document common_default_keys = ["__default", "__global"] diff --git a/frappe/client.py b/frappe/client.py index e970a64802..129c73a0cf 100644 --- a/frappe/client.py +++ b/frappe/client.py @@ -2,6 +2,7 @@ # License: MIT. See LICENSE import json import os +from typing import TYPE_CHECKING import frappe import frappe.model @@ -11,6 +12,9 @@ from frappe.desk.reportview import validate_args from frappe.model.db_query import check_parent_permission from frappe.utils import get_safe_filters +if TYPE_CHECKING: + from frappe.model.document import Document + """ Handle RESTful requests that are mapped to the `/api/resource` route. @@ -74,16 +78,9 @@ def get(doctype, name=None, filters=None, parent=None): if frappe.is_table(doctype): check_parent_permission(parent, doctype) - if filters and not name: - name = frappe.db.get_value(doctype, json.loads(filters)) - if not name: - frappe.throw(_("No document found for given filters")) - - doc = frappe.get_doc(doctype, name) - if not doc.has_permission("read"): - raise frappe.PermissionError - - return frappe.get_doc(doctype, name).as_dict() + doc = frappe.get_doc(doctype, name or frappe.parse_json(filters)) + doc.check_permission() + return doc.as_dict() @frappe.whitelist() @@ -96,8 +93,8 @@ def get_value(doctype, fieldname, filters=None, as_dict=True, debug=False, paren if frappe.is_table(doctype): check_parent_permission(parent, doctype) - if not frappe.has_permission(doctype): - frappe.throw(_("No permission for {0}").format(doctype), frappe.PermissionError) + if not frappe.has_permission(doctype, parent_doctype=parent): + frappe.throw(_("No permission for {0}").format(_(doctype)), frappe.PermissionError) filters = get_safe_filters(filters) if isinstance(filters, str): @@ -139,9 +136,9 @@ def get_value(doctype, fieldname, filters=None, as_dict=True, debug=False, paren @frappe.whitelist() def get_single_value(doctype, field): if not frappe.has_permission(doctype): - frappe.throw(_("No permission for {0}").format(doctype), frappe.PermissionError) - value = frappe.db.get_single_value(doctype, field) - return value + frappe.throw(_("No permission for {0}").format(_(doctype)), frappe.PermissionError) + + return frappe.db.get_single_value(doctype, field) @frappe.whitelist(methods=["POST", "PUT"]) @@ -189,15 +186,7 @@ def insert(doc=None): if isinstance(doc, str): doc = json.loads(doc) - if doc.get("parenttype"): - # inserting a child record - parent = frappe.get_doc(doc.parenttype, doc.parent) - parent.append(doc.parentfield, doc) - parent.save() - return parent.as_dict() - else: - doc = frappe.get_doc(doc).insert() - return doc.as_dict() + return insert_doc(doc).as_dict() @frappe.whitelist(methods=["POST", "PUT"]) @@ -208,21 +197,12 @@ def insert_many(docs=None): if isinstance(docs, str): docs = json.loads(docs) - out = [] - if len(docs) > 200: frappe.throw(_("Only 200 inserts allowed in one request")) + out = set() for doc in docs: - if doc.get("parenttype"): - # inserting a child record - parent = frappe.get_doc(doc.parenttype, doc.parent) - parent.append(doc.parentfield, doc) - parent.save() - out.append(parent.name) - else: - doc = frappe.get_doc(doc).insert() - out.append(doc.name) + out.add(insert_doc(doc).name) return out @@ -287,30 +267,6 @@ def delete(doctype, name): frappe.delete_doc(doctype, name, ignore_missing=False) -@frappe.whitelist(methods=["POST", "PUT"]) -def set_default(key, value, parent=None): - """set a user default value""" - frappe.db.set_default(key, value, parent or frappe.session.user) - frappe.clear_cache(user=frappe.session.user) - - -@frappe.whitelist() -def get_default(key, parent=None): - """set a user default value""" - return frappe.db.get_default(key, parent) - - -@frappe.whitelist(methods=["POST", "PUT"]) -def make_width_property_setter(doc): - """Set width Property Setter - - :param doc: Property Setter document with `width` property""" - if isinstance(doc, str): - doc = json.loads(doc) - if doc["doctype"] == "Property Setter" and doc["property"] == "width": - frappe.get_doc(doc).insert(ignore_permissions=True) - - @frappe.whitelist(methods=["POST", "PUT"]) def bulk_update(docs): """Bulk update documents @@ -368,13 +324,13 @@ def get_js(items): frappe.throw(_("Invalid file path: {0}").format("/".join(src))) contentpath = os.path.join(frappe.local.sites_path, *src) - with open(contentpath, "r") as srcfile: + with open(contentpath) as srcfile: code = frappe.utils.cstr(srcfile.read()) if frappe.local.lang != "en": messages = frappe.get_lang_dict("jsfile", contentpath) messages = json.dumps(messages) - code += "\n\n$.extend(frappe._messages, {})".format(messages) + code += f"\n\n$.extend(frappe._messages, {messages})" out.append(code) @@ -398,7 +354,7 @@ def attach_file( is_private=None, docfield=None, ): - """Attach a file to Document (POST) + """Attach a file to Document :param filename: filename e.g. test-file.txt :param filedata: base64 encode filedata which must be urlencoded @@ -409,17 +365,10 @@ def attach_file( :param is_private: Attach file as private file (1 or 0) :param docfield: file to attach to (optional)""" - request_method = frappe.local.request.environ.get("REQUEST_METHOD") - - if request_method.upper() != "POST": - frappe.throw(_("Invalid Request")) - doc = frappe.get_doc(doctype, docname) + doc.check_permission() - if not doc.has_permission(): - frappe.throw(_("Not permitted"), frappe.PermissionError) - - _file = frappe.get_doc( + file = frappe.get_doc( { "doctype": "File", "file_name": filename, @@ -431,19 +380,13 @@ def attach_file( "content": filedata, "decode": decode_base64, } - ) - _file.save() + ).save() if docfield and doctype: - doc.set(docfield, _file.file_url) + doc.set(docfield, file.file_url) doc.save() - return _file.as_dict() - - -@frappe.whitelist() -def get_hooks(hook, app_name=None): - return frappe.get_hooks(hook, app_name) + return file @frappe.whitelist() @@ -493,3 +436,23 @@ def validate_link(doctype: str, docname: str, fields=None): ) return values + + +def insert_doc(doc) -> "Document": + """Inserts document and returns parent document object with appended child document + if `doc` is child document else returns the inserted document object + + :param doc: doc to insert (dict)""" + + doc = frappe._dict(doc) + if frappe.is_table(doc.doctype): + if not (doc.parenttype and doc.parent and doc.parentfield): + frappe.throw(_("Parenttype, Parent and Parentfield are required to insert a child record")) + + # inserting a child record + parent = frappe.get_doc(doc.parenttype, doc.parent) + parent.append(doc.parentfield, doc) + parent.save() + return parent + + return frappe.get_doc(doc).insert() diff --git a/frappe/commands/scheduler.py b/frappe/commands/scheduler.py index ed6a0dea57..e481676088 100755 --- a/frappe/commands/scheduler.py +++ b/frappe/commands/scheduler.py @@ -8,21 +8,6 @@ from frappe.exceptions import SiteNotSpecifiedError from frappe.utils import cint -def _is_scheduler_enabled(): - enable_scheduler = False - try: - frappe.connect() - enable_scheduler = ( - cint(frappe.db.get_single_value("System Settings", "enable_scheduler")) and True or False - ) - except: - pass - finally: - frappe.db.close() - - return enable_scheduler - - @click.command("trigger-scheduler-event", help="Trigger a scheduler event") @click.argument("event") @pass_context @@ -114,7 +99,7 @@ def scheduler(context, state, site=None): frappe.utils.scheduler.enable_scheduler() frappe.db.commit() - print("Scheduler {0}d for site {1}".format(state, site)) + print(f"Scheduler {state}d for site {site}") finally: frappe.destroy() @@ -182,7 +167,7 @@ def purge_jobs(site=None, queue=None, event=None): frappe.init(site or "") count = purge_pending_jobs(event=event, site=site, queue=queue) - print("Purged {} jobs".format(count)) + print(f"Purged {count} jobs") @click.command("schedule") @@ -218,11 +203,11 @@ def ready_for_migration(context, site=None): pending_jobs = get_pending_jobs(site=site) if pending_jobs: - print("NOT READY for migration: site {0} has pending background jobs".format(site)) + print(f"NOT READY for migration: site {site} has pending background jobs") sys.exit(1) else: - print("READY for migration: site {0} does not have any background jobs".format(site)) + print(f"READY for migration: site {site} does not have any background jobs") return 0 finally: diff --git a/frappe/commands/site.py b/frappe/commands/site.py index fa9ab4be59..ab599be121 100644 --- a/frappe/commands/site.py +++ b/frappe/commands/site.py @@ -9,6 +9,7 @@ import click # imports - module imports import frappe from frappe.commands import get_site, pass_context +from frappe.core.doctype.log_settings.log_settings import LOG_DOCTYPES from frappe.exceptions import SiteNotSpecifiedError @@ -54,7 +55,6 @@ def new_site( db_root_password=None, admin_password=None, verbose=False, - install_apps=None, source_sql=None, force=None, no_mariadb_socket=False, @@ -86,7 +86,6 @@ def new_site( db_type=db_type, db_host=db_host, db_port=db_port, - new_site=True, ) if set_default: @@ -143,11 +142,7 @@ def restore( is_partial, validate_database_sql, ) - from frappe.utils.backups import Backup - - if not os.path.exists(sql_file_path): - print("Invalid path", sql_file_path) - sys.exit(1) + from frappe.utils.backups import Backup, get_or_generate_backup_encryption_key _backup = Backup(sql_file_path) @@ -176,7 +171,7 @@ def restore( else: click.secho("Encrypted backup file detected. Decrypting using site config.", fg="yellow") - encryption_key = frappe.get_site_config().encryption_key + encryption_key = get_or_generate_backup_encryption_key() _backup.backup_decryption(encryption_key) # Rollback on unsuccessful decryrption @@ -260,7 +255,7 @@ def restore( os.remove(private) _backup.decryption_rollback() - success_message = "Site {0} has been restored{1}".format( + success_message = "Site {} has been restored{}".format( site, " with files" if (with_public_files or with_private_files) else "" ) click.secho(success_message, fg="green") @@ -273,7 +268,7 @@ def restore( @pass_context def partial_restore(context, sql_file_path, verbose, encryption_key=None): from frappe.installer import extract_sql_from_archive, partial_restore - from frappe.utils.backups import Backup + from frappe.utils.backups import Backup, get_or_generate_backup_encryption_key if not os.path.exists(sql_file_path): print("Invalid path", sql_file_path) @@ -309,7 +304,7 @@ def partial_restore(context, sql_file_path, verbose, encryption_key=None): else: click.secho("Encrypted backup file detected. Decrypting using site config.", fg="yellow") - key = frappe.get_site_config().encryption_key + key = get_or_generate_backup_encryption_key() _backup.backup_decryption(key) @@ -398,8 +393,9 @@ def _reinstall( @click.command("install-app") @click.argument("apps", nargs=-1) +@click.option("--force", is_flag=True, default=False) @pass_context -def install_app(context, apps): +def install_app(context, apps, force=False): "Install a new app to site, supports multiple apps" from frappe.installer import install_app as _install_app @@ -414,14 +410,14 @@ def install_app(context, apps): for app in apps: try: - _install_app(app, verbose=context.verbose) + _install_app(app, verbose=context.verbose, force=force) except frappe.IncompatibleApp as err: - err_msg = ":\n{}".format(err) if str(err) else "" - print("App {} is Incompatible with Site {}{}".format(app, site, err_msg)) + err_msg = f":\n{err}" if str(err) else "" + print(f"App {app} is Incompatible with Site {site}{err_msg}") exit_code = 1 except Exception as err: - err_msg = ": {}\n{}".format(str(err), frappe.get_traceback()) - print("An error occurred while installing {}{}".format(app, err_msg)) + err_msg = f": {str(err)}\n{frappe.get_traceback()}" + print(f"An error occurred while installing {app}{err_msg}") exit_code = 1 frappe.destroy() @@ -451,8 +447,8 @@ def list_apps(context, format): apps = frappe.get_single("Installed Applications").installed_applications if apps: - name_len, ver_len = [max([len(x.get(y)) for x in apps]) for y in ["app_name", "app_version"]] - template = "{{0:{0}}} {{1:{1}}} {{2}}".format(name_len, ver_len) + name_len, ver_len = (max(len(x.get(y)) for x in apps) for y in ["app_name", "app_version"]) + template = f"{{0:{name_len}}} {{1:{ver_len}}} {{2}}" installed_applications = [ template.format(app.app_name, app.app_version, app.git_branch) for app in apps @@ -610,7 +606,7 @@ def reload_doctype(context, doctype): def add_to_hosts(context): "Add site to hosts" for site in context.sites: - frappe.commands.popen("echo 127.0.0.1\t{0} | sudo tee -a /etc/hosts".format(site)) + frappe.commands.popen(f"echo 127.0.0.1\t{site} | sudo tee -a /etc/hosts") if not context.sites: raise SiteNotSpecifiedError @@ -626,9 +622,9 @@ def use(site, sites_path="."): if os.path.exists(os.path.join(sites_path, site)): with open(os.path.join(sites_path, "currentsite.txt"), "w") as sitefile: sitefile.write(site) - print("Current Site set to {}".format(site)) + print(f"Current Site set to {site}") else: - print("Site {} does not exist".format(site)) + print(f"Site {site} does not exist") @click.command("backup") @@ -702,7 +698,7 @@ def backup( ) except Exception: click.secho( - "Backup failed for Site {0}. Database or site_config.json may be corrupted".format(site), + f"Backup failed for Site {site}. Database or site_config.json may be corrupted", fg="red", ) if verbose: @@ -716,7 +712,7 @@ def backup( odb.print_summary() click.secho( - "Backup for Site {0} has been successfully completed{1}".format( + "Backup for Site {} has been successfully completed{}".format( site, " with files" if with_files else "" ), fg="green", @@ -825,7 +821,7 @@ def _drop_site( try: if not no_backup: click.secho(f"Taking backup of {site}", fg="green") - odb = scheduled_backup(ignore_files=False, force=True, verbose=True) + odb = scheduled_backup(ignore_files=False, ignore_conf=True, force=True, verbose=True) odb.print_summary() except Exception as err: if force: @@ -833,8 +829,8 @@ def _drop_site( else: messages = [ "=" * 80, - "Error: The operation has stopped because backup of {0}'s database failed.".format(site), - "Reason: {0}\n".format(str(err)), + f"Error: The operation has stopped because backup of {site}'s database failed.", + f"Reason: {str(err)}\n", "Fix the issue and try again.", "Hint: Use 'bench drop-site {0} --force' to force the removal of {0}".format(site), ] @@ -847,9 +843,10 @@ def _drop_site( archived_sites_path = archived_sites_path or os.path.join( frappe.get_app_path("frappe"), "..", "..", "..", "archived", "sites" ) + archived_sites_path = os.path.realpath(archived_sites_path) + click.secho(f"Moving site to archive under {archived_sites_path}", fg="green") os.makedirs(archived_sites_path, exist_ok=True) - move(archived_sites_path, site) @@ -923,7 +920,6 @@ def set_user_password(site, user, password, logout_all_sessions=False): update_password(user=user, pwd=password, logout_all_sessions=logout_all_sessions) frappe.db.commit() - password = None finally: frappe.destroy() @@ -1084,7 +1080,7 @@ def build_search_index(context): if not site: raise SiteNotSpecifiedError - print("Building search index for {}".format(site)) + print(f"Building search index for {site}") frappe.init(site=site) frappe.connect() try: @@ -1093,6 +1089,51 @@ def build_search_index(context): frappe.destroy() +@click.command("clear-log-table") +@click.option("--doctype", default="text", type=click.Choice(LOG_DOCTYPES), help="Log DocType") +@click.option("--days", type=int, help="Keep records for days") +@click.option("--no-backup", is_flag=True, default=False, help="Do not backup the table") +@pass_context +def clear_log_table(context, doctype, days, no_backup): + """If any logtype table grows too large then clearing it with DELETE query + is not feasible in reasonable time. This command copies recent data to new + table and replaces current table with new smaller table. + + + ref: https://mariadb.com/kb/en/big-deletes/#deleting-more-than-half-a-table + """ + from frappe.core.doctype.log_settings.log_settings import clear_log_table as clear_logs + from frappe.utils.backups import scheduled_backup + + if not context.sites: + raise SiteNotSpecifiedError + + if doctype not in LOG_DOCTYPES: + raise frappe.ValidationError(f"Unsupported logging DocType: {doctype}") + + for site in context.sites: + frappe.init(site=site) + frappe.connect() + + if not no_backup: + scheduled_backup( + ignore_conf=False, + include_doctypes=doctype, + ignore_files=True, + force=True, + ) + click.echo(f"Backed up {doctype}") + + try: + click.echo(f"Copying {doctype} records from last {days} days to temporary table.") + clear_logs(doctype, days=days) + except Exception as e: + click.echo(f"Log cleanup for {doctype} failed:\n{e}") + sys.exit(1) + else: + click.secho(f"Cleared {doctype} records older than {days} days", fg="green") + + @click.command("trim-database") @click.option("--dry-run", is_flag=True, default=False, help="Show what would be deleted") @click.option( @@ -1265,4 +1306,5 @@ commands = [ partial_restore, trim_tables, trim_database, + clear_log_table, ] diff --git a/frappe/commands/translate.py b/frappe/commands/translate.py index 0b14e03002..69970d8d97 100644 --- a/frappe/commands/translate.py +++ b/frappe/commands/translate.py @@ -48,11 +48,12 @@ def new_language(context, lang_code, app): @click.command("get-untranslated") +@click.option("--app", default="_ALL_APPS") @click.argument("lang") @click.argument("untranslated_file") @click.option("--all", default=False, is_flag=True, help="Get all message strings") @pass_context -def get_untranslated(context, lang, untranslated_file, all=None): +def get_untranslated(context, lang, untranslated_file, app="_ALL_APPS", all=None): "Get untranslated strings for language" import frappe.translate @@ -60,17 +61,18 @@ def get_untranslated(context, lang, untranslated_file, all=None): try: frappe.init(site=site) frappe.connect() - frappe.translate.get_untranslated(lang, untranslated_file, get_all=all) + frappe.translate.get_untranslated(lang, untranslated_file, get_all=all, app=app) finally: frappe.destroy() @click.command("update-translations") +@click.option("--app", default="_ALL_APPS") @click.argument("lang") @click.argument("untranslated_file") @click.argument("translated-file") @pass_context -def update_translations(context, lang, untranslated_file, translated_file): +def update_translations(context, lang, untranslated_file, translated_file, app="_ALL_APPS"): "Update translated strings" import frappe.translate @@ -78,7 +80,7 @@ def update_translations(context, lang, untranslated_file, translated_file): try: frappe.init(site=site) frappe.connect() - frappe.translate.update_translations(lang, untranslated_file, translated_file) + frappe.translate.update_translations(lang, untranslated_file, translated_file, app=app) finally: frappe.destroy() diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py index 2e27b8d6fe..3658a35992 100644 --- a/frappe/commands/utils.py +++ b/frappe/commands/utils.py @@ -387,7 +387,7 @@ def import_doc(context, path, force=False): if not os.path.exists(path): path = os.path.join("..", path) if not os.path.exists(path): - print("Invalid path {0}".format(path)) + print(f"Invalid path {path}") sys.exit(1) for site in context.sites: @@ -471,7 +471,7 @@ def bulk_rename(context, doctype, path): site = get_site(context) - with open(path, "r") as csvfile: + with open(path) as csvfile: rows = read_csv_content(csvfile.read()) frappe.init(site=site) @@ -523,22 +523,24 @@ def postgres(context): def _mariadb(): + from frappe.database.mariadb.database import MariaDBDatabase + mysql = find_executable("mysql") - os.execv( + command = [ mysql, - [ - mysql, - "-u", - frappe.conf.db_name, - "-p" + frappe.conf.db_password, - frappe.conf.db_name, - "-h", - frappe.conf.db_host or "localhost", - "--pager=less -SFX", - "--safe-updates", - "-A", - ], - ) + "--port", + frappe.conf.db_port or MariaDBDatabase.default_port, + "-u", + frappe.conf.db_name, + f"-p{frappe.conf.db_password}", + frappe.conf.db_name, + "-h", + frappe.conf.db_host or "localhost", + "--pager=less -SFX", + "--safe-updates", + "-A", + ] + os.execv(mysql, command) def _psql(): @@ -566,7 +568,7 @@ def jupyter(context): try: os.stat(jupyter_notebooks_path) except OSError: - print("Creating folder to keep jupyter notebooks at {}".format(jupyter_notebooks_path)) + print(f"Creating folder to keep jupyter notebooks at {jupyter_notebooks_path}") os.mkdir(jupyter_notebooks_path) bin_path = os.path.abspath("../env/bin") print( @@ -585,9 +587,9 @@ frappe.db.connect() ) ) os.execv( - "{0}/jupyter".format(bin_path), + f"{bin_path}/jupyter", [ - "{0}/jupyter".format(bin_path), + f"{bin_path}/jupyter", "notebook", jupyter_notebooks_path, ], @@ -730,6 +732,7 @@ def transform_database(context, table, engine, row_format, failfast): @click.command("run-tests") @click.option("--app", help="For App") @click.option("--doctype", help="For DocType") +@click.option("--module-def", help="For all Doctypes in Module Def") @click.option("--case", help="Select particular TestCase") @click.option( "--doctype-list-path", @@ -754,6 +757,7 @@ def run_tests( app=None, module=None, doctype=None, + module_def=None, test=(), profile=False, coverage=False, @@ -778,7 +782,7 @@ def run_tests( if not (allow_tests or os.environ.get("CI")): click.secho("Testing is disabled for the site!", bold=True) click.secho("You can enable tests by entering following command:") - click.secho("bench --site {0} set-config allow_tests true".format(site), fg="green") + click.secho(f"bench --site {site} set-config allow_tests true", fg="green") return frappe.init(site=site) @@ -790,6 +794,7 @@ def run_tests( app, module, doctype, + module_def, context.verbose, tests=tests, force=context.force, @@ -856,6 +861,8 @@ def run_ui_tests( node_bin = subprocess.getoutput("npm bin") cypress_path = f"{node_bin}/cypress" plugin_path = f"{node_bin}/../cypress-file-upload" + drag_drop_plugin_path = f"{node_bin}/../@4tw/cypress-drag-drop" + real_events_plugin_path = f"{node_bin}/../cypress-real-events" testing_library_path = f"{node_bin}/../@testing-library" coverage_plugin_path = f"{node_bin}/../@cypress/code-coverage" @@ -863,15 +870,24 @@ def run_ui_tests( if not ( os.path.exists(cypress_path) and os.path.exists(plugin_path) + and os.path.exists(drag_drop_plugin_path) + and os.path.exists(real_events_plugin_path) and os.path.exists(testing_library_path) and os.path.exists(coverage_plugin_path) - and cint(subprocess.getoutput("npm view cypress version")[:1]) >= 6 ): - # install cypress + # install cypress & dependent plugins click.secho("Installing Cypress...", fg="yellow") - frappe.commands.popen( - "yarn add cypress@^6 cypress-file-upload@^5 @testing-library/cypress@^8 @cypress/code-coverage@^3 --no-lockfile" + packages = " ".join( + [ + "cypress@^6", + "cypress-file-upload@^5", + "@4tw/cypress-drag-drop@^2", + "cypress-real-events", + "@testing-library/cypress@^8", + "@cypress/code-coverage@^3", + ] ) + frappe.commands.popen(f"yarn add {packages} --no-lockfile") # run for headless mode run_or_open = "run --browser chrome --record" if headless else "open" @@ -948,7 +964,7 @@ def request(context, args=None, path=None): if args.startswith("/api/method"): frappe.local.form_dict.cmd = args.split("?")[0].split("/")[-1] elif path: - with open(os.path.join("..", path), "r") as f: + with open(os.path.join("..", path)) as f: args = json.loads(f.read()) frappe.local.form_dict = frappe._dict(args) @@ -1024,6 +1040,7 @@ def set_config(context, key, value, global_=False, parse=False, as_dict=False): def get_version(output): """Show the versions of all the installed apps.""" from git import Repo + from git.exc import InvalidGitRepositoryError from frappe.utils.change_log import get_app_branch from frappe.utils.commands import render_table @@ -1034,12 +1051,16 @@ def get_version(output): for app in sorted(frappe.get_all_apps()): module = frappe.get_module(app) app_hooks = frappe.get_module(app + ".hooks") - repo = Repo(frappe.get_app_path(app, "..")) app_info = frappe._dict() + + try: + app_info.commit = Repo(frappe.get_app_path(app, "..")).head.object.hexsha[:7] + except InvalidGitRepositoryError: + app_info.commit = "" + app_info.app = app app_info.branch = get_app_branch(app) - app_info.commit = repo.head.object.hexsha[:7] app_info.version = getattr(app_hooks, f"{app_info.branch}_version", None) or module.__version__ data.append(app_info) diff --git a/frappe/contacts/address_and_contact.py b/frappe/contacts/address_and_contact.py index 1eb9f1cf33..4df32c6705 100644 --- a/frappe/contacts/address_and_contact.py +++ b/frappe/contacts/address_and_contact.py @@ -17,7 +17,7 @@ def load_address_and_contact(doc, key=None): ["Dynamic Link", "link_name", "=", doc.name], ["Dynamic Link", "parenttype", "=", "Address"], ] - address_list = frappe.get_list("Address", filters=filters, fields=["*"]) + address_list = frappe.get_list("Address", filters=filters, fields=["*"], order_by="creation asc") address_list = [a.update({"display": get_address_display(a)}) for a in address_list] @@ -116,9 +116,7 @@ def get_permission_query_conditions(doctype): # when everything is not permitted for df in links.get("not_permitted_links"): # like ifnull(customer, '')='' and ifnull(supplier, '')='' - conditions.append( - "ifnull(`tab{doctype}`.`{fieldname}`, '')=''".format(doctype=doctype, fieldname=df.fieldname) - ) + conditions.append(f"ifnull(`tab{doctype}`.`{df.fieldname}`, '')=''") return "( " + " and ".join(conditions) + " )" @@ -127,9 +125,7 @@ def get_permission_query_conditions(doctype): for df in links.get("permitted_links"): # like ifnull(customer, '')!='' or ifnull(supplier, '')!='' - conditions.append( - "ifnull(`tab{doctype}`.`{fieldname}`, '')!=''".format(doctype=doctype, fieldname=df.fieldname) - ) + conditions.append(f"ifnull(`tab{doctype}`.`{df.fieldname}`, '')!=''") return "( " + " or ".join(conditions) + " )" @@ -169,29 +165,35 @@ def delete_contact_and_address(doctype, docname): @frappe.whitelist() @frappe.validate_and_sanitize_search_inputs -def filter_dynamic_link_doctypes(doctype, txt, searchfield, start, page_len, filters): - if not txt: - txt = "" +def filter_dynamic_link_doctypes( + doctype, txt: str, searchfield, start, page_len, filters: dict +) -> list[list[str]]: + from frappe.permissions import get_doctypes_with_read - doctypes = frappe.db.get_all( - "DocField", filters=filters, fields=["parent"], distinct=True, as_list=True + txt = txt or "" + filters = filters or {} + + _doctypes_from_df = frappe.get_all( + "DocField", + filters=filters, + pluck="parent", + distinct=True, + order_by=None, ) + doctypes_from_df = {d for d in _doctypes_from_df if txt.lower() in _(d).lower()} - doctypes = tuple(d for d in doctypes if re.search(txt + ".*", _(d[0]), re.IGNORECASE)) + filters.update({"dt": ("not in", doctypes_from_df)}) + _doctypes_from_cdf = frappe.get_all( + "Custom Field", filters=filters, pluck="dt", distinct=True, order_by=None + ) + doctypes_from_cdf = {d for d in _doctypes_from_cdf if txt.lower() in _(d).lower()} - filters.update({"dt": ("not in", [d[0] for d in doctypes])}) + all_doctypes = doctypes_from_df.union(doctypes_from_cdf) + allowed_doctypes = set(get_doctypes_with_read()) - _doctypes = frappe.db.get_all("Custom Field", filters=filters, fields=["dt"], as_list=True) + valid_doctypes = sorted(all_doctypes.intersection(allowed_doctypes)) - _doctypes = tuple([d for d in _doctypes if re.search(txt + ".*", _(d[0]), re.IGNORECASE)]) - - all_doctypes = [d[0] for d in doctypes + _doctypes] - allowed_doctypes = frappe.permissions.get_doctypes_with_read() - - valid_doctypes = sorted(set(all_doctypes).intersection(set(allowed_doctypes))) - valid_doctypes = [[doctype] for doctype in valid_doctypes] - - return valid_doctypes + return [[doctype] for doctype in valid_doctypes] def set_link_title(doc): diff --git a/frappe/contacts/doctype/address/address.js b/frappe/contacts/doctype/address/address.js index 63574622c0..548dd40060 100644 --- a/frappe/contacts/doctype/address/address.js +++ b/frappe/contacts/doctype/address/address.js @@ -2,60 +2,74 @@ // For license information, please see license.txt frappe.ui.form.on("Address", { - refresh: function(frm) { - if(frm.doc.__islocal) { + refresh: function (frm) { + if (frm.doc.__islocal) { const last_doc = frappe.contacts.get_last_doc(frm); - if(frappe.dynamic_link && frappe.dynamic_link.doc - && frappe.dynamic_link.doc.name == last_doc.docname) { - frm.set_value('links', ''); - frm.add_child('links', { + if ( + frappe.dynamic_link && + frappe.dynamic_link.doc && + frappe.dynamic_link.doc.name == last_doc.docname + ) { + frm.set_value("links", ""); + frm.add_child("links", { link_doctype: frappe.dynamic_link.doctype, - link_name: frappe.dynamic_link.doc[frappe.dynamic_link.fieldname] + link_name: frappe.dynamic_link.doc[frappe.dynamic_link.fieldname], }); } } - frm.set_query('link_doctype', "links", function() { + frm.set_query("link_doctype", "links", function () { return { query: "frappe.contacts.address_and_contact.filter_dynamic_link_doctypes", filters: { fieldtype: "HTML", fieldname: "address_html", - } - } + }, + }; }); frm.refresh_field("links"); if (frm.doc.links) { for (let i in frm.doc.links) { let link = frm.doc.links[i]; - frm.add_custom_button(__("{0}: {1}", [__(link.link_doctype), __(link.link_name)]), function() { - frappe.set_route("Form", link.link_doctype, link.link_name); - }, __("Links")); + frm.add_custom_button( + __("{0}: {1}", [__(link.link_doctype), __(link.link_name)]), + function () { + frappe.set_route("Form", link.link_doctype, link.link_name); + }, + __("Links") + ); } } }, - validate: function(frm) { + validate: function (frm) { // clear linked customer / supplier / sales partner on saving... - if(frm.doc.links) { - frm.doc.links.forEach(function(d) { + if (frm.doc.links) { + frm.doc.links.forEach(function (d) { frappe.model.remove_from_locals(d.link_doctype, d.link_name); }); } }, - after_save: function(frm) { + after_save: function (frm) { frappe.run_serially([ () => frappe.timeout(1), () => { const last_doc = frappe.contacts.get_last_doc(frm); - if (frappe.dynamic_link && frappe.dynamic_link.doc && frappe.dynamic_link.doc.name == last_doc.docname) { + if ( + frappe.dynamic_link && + frappe.dynamic_link.doc && + frappe.dynamic_link.doc.name == last_doc.docname + ) { for (let i in frm.doc.links) { let link = frm.doc.links[i]; - if (last_doc.doctype == link.link_doctype && last_doc.docname == link.link_name) { - frappe.set_route('Form', last_doc.doctype, last_doc.docname); + if ( + last_doc.doctype == link.link_doctype && + last_doc.docname == link.link_name + ) { + frappe.set_route("Form", last_doc.doctype, last_doc.docname); } } } - } + }, ]); - } + }, }); diff --git a/frappe/contacts/doctype/address/address.py b/frappe/contacts/doctype/address/address.py index d9ba31d474..42dbdd6177 100644 --- a/frappe/contacts/doctype/address/address.py +++ b/frappe/contacts/doctype/address/address.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and contributors # License: MIT. See LICENSE @@ -236,7 +235,7 @@ def address_query(doctype, txt, searchfield, start, page_len, filters): meta = frappe.get_meta("Address") for fieldname, value in filters.items(): if meta.get_field(fieldname) or fieldname in frappe.db.DEFAULT_COLUMNS: - condition += " and {field}={value}".format(field=fieldname, value=frappe.db.escape(value)) + condition += f" and {fieldname}={frappe.db.escape(value)}" searchfields = meta.get_search_fields() @@ -246,9 +245,9 @@ def address_query(doctype, txt, searchfield, start, page_len, filters): search_condition = "" for field in searchfields: if search_condition == "": - search_condition += "`tabAddress`.`{field}` like %(txt)s".format(field=field) + search_condition += f"`tabAddress`.`{field}` like %(txt)s" else: - search_condition += " or `tabAddress`.`{field}` like %(txt)s".format(field=field) + search_condition += f" or `tabAddress`.`{field}` like %(txt)s" return frappe.db.sql( """select @@ -268,7 +267,6 @@ def address_query(doctype, txt, searchfield, start, page_len, filters): `tabAddress`.idx desc, `tabAddress`.name limit %(start)s, %(page_len)s """.format( mcond=get_match_cond(doctype), - key=searchfield, search_condition=search_condition, condition=condition or "", ), diff --git a/frappe/contacts/doctype/address/test_address.py b/frappe/contacts/doctype/address/test_address.py index 4a6e6e53f7..edcf87f5bc 100644 --- a/frappe/contacts/doctype/address/test_address.py +++ b/frappe/contacts/doctype/address/test_address.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and Contributors # License: MIT. See LICENSE import unittest diff --git a/frappe/contacts/doctype/address_template/address_template.js b/frappe/contacts/doctype/address_template/address_template.js index 502d02e7f9..bfe139bce8 100644 --- a/frappe/contacts/doctype/address_template/address_template.js +++ b/frappe/contacts/doctype/address_template/address_template.js @@ -1,16 +1,16 @@ // Copyright (c) 2016, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Address Template', { - refresh: function(frm) { - if(frm.is_new() && !frm.doc.template) { +frappe.ui.form.on("Address Template", { + refresh: function (frm) { + if (frm.is_new() && !frm.doc.template) { // set default template via js so that it is translated frappe.call({ - method: 'frappe.contacts.doctype.address_template.address_template.get_default_address_template', - callback: function(r) { - frm.set_value('template', r.message); - } + method: "frappe.contacts.doctype.address_template.address_template.get_default_address_template", + callback: function (r) { + frm.set_value("template", r.message); + }, }); } - } + }, }); diff --git a/frappe/contacts/doctype/address_template/address_template.json b/frappe/contacts/doctype/address_template/address_template.json index e27d97daad..48eacc0fc7 100644 --- a/frappe/contacts/doctype/address_template/address_template.json +++ b/frappe/contacts/doctype/address_template/address_template.json @@ -1,152 +1,65 @@ { - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 1, - "autoname": "field:country", - "beta": 0, - "creation": "2014-06-05 02:22:36.029850", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "Setup", - "editable_grid": 0, - "engine": "InnoDB", + "actions": [], + "allow_rename": 1, + "autoname": "field:country", + "creation": "2014-06-05 02:22:36.029850", + "doctype": "DocType", + "document_type": "Setup", + "engine": "InnoDB", + "field_order": [ + "country", + "is_default", + "template" + ], "fields": [ { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "country", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 1, - "label": "Country", - "length": 0, - "no_copy": 0, - "options": "Country", - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 1, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "country", + "fieldtype": "Link", + "in_list_view": 1, + "in_standard_filter": 1, + "label": "Country", + "options": "Country", + "reqd": 1, + "search_index": 1, + "unique": 1 + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "description": "This format is used if country specific format is not found", - "fieldname": "is_default", - "fieldtype": "Check", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Is Default", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "default": "0", + "description": "This format is used if country specific format is not found", + "fieldname": "is_default", + "fieldtype": "Check", + "in_list_view": 1, + "label": "Is Default" + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "default": "", - "description": "

Default Template

\n

Uses Jinja Templating and all the fields of Address (including Custom Fields if any) will be available

\n
{{ address_line1 }}<br>\n{% if address_line2 %}{{ address_line2 }}<br>{% endif -%}\n{{ city }}<br>\n{% if state %}{{ state }}<br>{% endif -%}\n{% if pincode %} PIN:  {{ pincode }}<br>{% endif -%}\n{{ country }}<br>\n{% if phone %}Phone: {{ phone }}<br>{% endif -%}\n{% if fax %}Fax: {{ fax }}<br>{% endif -%}\n{% if email_id %}Email: {{ email_id }}<br>{% endif -%}\n
", - "fieldname": "template", - "fieldtype": "Code", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Template", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "description": "

Default Template

\n

Uses Jinja Templating and all the fields of Address (including Custom Fields if any) will be available

\n
{{ address_line1 }}<br>\n{% if address_line2 %}{{ address_line2 }}<br>{% endif -%}\n{{ city }}<br>\n{% if state %}{{ state }}<br>{% endif -%}\n{% if pincode %} PIN:  {{ pincode }}<br>{% endif -%}\n{{ country }}<br>\n{% if phone %}Phone: {{ phone }}<br>{% endif -%}\n{% if fax %}Fax: {{ fax }}<br>{% endif -%}\n{% if email_id %}Email: {{ email_id }}<br>{% endif -%}\n
", + "fieldname": "template", + "fieldtype": "Code", + "label": "Template" } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "icon": "fa fa-map-marker", - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2017-04-10 13:09:53.761009", - "modified_by": "Administrator", - "module": "Contacts", - "name": "Address Template", - "name_case": "", - "owner": "Administrator", + ], + "icon": "fa fa-map-marker", + "links": [], + "modified": "2022-08-03 12:20:49.095228", + "modified_by": "Administrator", + "module": "Contacts", + "name": "Address Template", + "owner": "Administrator", "permissions": [ { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 1, - "delete": 1, - "email": 0, - "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 0, - "read": 1, - "report": 1, - "role": "System Manager", - "set_user_permissions": 1, - "share": 1, - "submit": 0, + "create": 1, + "delete": 1, + "export": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "set_user_permissions": 1, + "share": 1, "write": 1 } - ], - "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 0, - "track_seen": 0 + ], + "quick_entry": 1, + "sort_field": "modified", + "sort_order": "DESC", + "states": [] } \ No newline at end of file diff --git a/frappe/contacts/doctype/address_template/address_template.py b/frappe/contacts/doctype/address_template/address_template.py index 85e9a986ef..a8806b336b 100644 --- a/frappe/contacts/doctype/address_template/address_template.py +++ b/frappe/contacts/doctype/address_template/address_template.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and contributors # License: MIT. See LICENSE diff --git a/frappe/contacts/doctype/address_template/test_address_template.py b/frappe/contacts/doctype/address_template/test_address_template.py index 699de5ada0..8045313c69 100644 --- a/frappe/contacts/doctype/address_template/test_address_template.py +++ b/frappe/contacts/doctype/address_template/test_address_template.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and Contributors # License: MIT. See LICENSE import unittest diff --git a/frappe/contacts/doctype/contact/contact.js b/frappe/contacts/doctype/contact/contact.js index fae6e6515e..d4ae9379fa 100644 --- a/frappe/contacts/doctype/contact/contact.js +++ b/frappe/contacts/doctype/contact/contact.js @@ -5,49 +5,52 @@ frappe.ui.form.on("Contact", { onload(frm) { frm.email_field = "email_id"; }, - refresh: function(frm) { - if(frm.doc.__islocal) { + refresh: function (frm) { + if (frm.doc.__islocal) { const last_doc = frappe.contacts.get_last_doc(frm); - if(frappe.dynamic_link && frappe.dynamic_link.doc - && frappe.dynamic_link.doc.name == last_doc.docname) { - frm.set_value('links', ''); - frm.add_child('links', { + if ( + frappe.dynamic_link && + frappe.dynamic_link.doc && + frappe.dynamic_link.doc.name == last_doc.docname + ) { + frm.set_value("links", ""); + frm.add_child("links", { link_doctype: frappe.dynamic_link.doctype, - link_name: frappe.dynamic_link.doc[frappe.dynamic_link.fieldname] + link_name: frappe.dynamic_link.doc[frappe.dynamic_link.fieldname], }); } } - if(!frm.doc.user && !frm.is_new() && frm.perm[0].write) { - frm.add_custom_button(__("Invite as User"), function() { + if (!frm.doc.user && !frm.is_new() && frm.perm[0].write) { + frm.add_custom_button(__("Invite as User"), function () { return frappe.call({ method: "frappe.contacts.doctype.contact.contact.invite_user", args: { - contact: frm.doc.name + contact: frm.doc.name, }, - callback: function(r) { + callback: function (r) { frm.set_value("user", r.message); - } + }, }); }); } - frm.set_query('link_doctype', "links", function() { + frm.set_query("link_doctype", "links", function () { return { query: "frappe.contacts.address_and_contact.filter_dynamic_link_doctypes", filters: { fieldtype: "HTML", fieldname: "contact_html", - } - } + }, + }; }); frm.refresh_field("links"); let numbers = frm.doc.phone_nos; if (numbers && numbers.length && frappe.phone_call.handler) { - frm.add_custom_button(__('Call'), () => { + frm.add_custom_button(__("Call"), () => { numbers = frm.doc.phone_nos .sort((prev, next) => next.is_primary_mobile_no - prev.is_primary_mobile_no) - .map(d => d.phone); + .map((d) => d.phone); frappe.phone_call.handler(numbers); }); } @@ -55,73 +58,94 @@ frappe.ui.form.on("Contact", { if (frm.doc.links) { frappe.call({ method: "frappe.contacts.doctype.contact.contact.address_query", - args: {links: frm.doc.links}, - callback: function(r) { + args: { links: frm.doc.links }, + callback: function (r) { if (r && r.message) { frm.set_query("address", function () { return { filters: { name: ["in", r.message], - } - } + }, + }; }); } - } + }, }); for (let i in frm.doc.links) { let link = frm.doc.links[i]; - frm.add_custom_button(__("{0}: {1}", [__(link.link_doctype), __(link.link_name)]), function() { - frappe.set_route("Form", link.link_doctype, link.link_name); - }, __("Links")); + frm.add_custom_button( + __("{0}: {1}", [__(link.link_doctype), __(link.link_name)]), + function () { + frappe.set_route("Form", link.link_doctype, link.link_name); + }, + __("Links") + ); } } }, - validate: function(frm) { + validate: function (frm) { // clear linked customer / supplier / sales partner on saving... - if(frm.doc.links) { - frm.doc.links.forEach(function(d) { + if (frm.doc.links) { + frm.doc.links.forEach(function (d) { frappe.model.remove_from_locals(d.link_doctype, d.link_name); }); } }, - after_save: function(frm) { + after_save: function (frm) { frappe.run_serially([ () => frappe.timeout(1), () => { const last_doc = frappe.contacts.get_last_doc(frm); - if (frappe.dynamic_link && frappe.dynamic_link.doc && frappe.dynamic_link.doc.name == last_doc.docname) { + if ( + frappe.dynamic_link && + frappe.dynamic_link.doc && + frappe.dynamic_link.doc.name == last_doc.docname + ) { for (let i in frm.doc.links) { let link = frm.doc.links[i]; - if (last_doc.doctype == link.link_doctype && last_doc.docname == link.link_name) { - frappe.set_route('Form', last_doc.doctype, last_doc.docname); + if ( + last_doc.doctype == link.link_doctype && + last_doc.docname == link.link_name + ) { + frappe.set_route("Form", last_doc.doctype, last_doc.docname); } } } - } + }, ]); }, - sync_with_google_contacts: function(frm) { + sync_with_google_contacts: function (frm) { if (frm.doc.sync_with_google_contacts) { - frappe.db.get_value("Google Contacts", {"email_id": frappe.session.user}, "name", (r) => { - if (r && r.name) { - frm.set_value("google_contacts", r.name); + frappe.db.get_value( + "Google Contacts", + { email_id: frappe.session.user }, + "name", + (r) => { + if (r && r.name) { + frm.set_value("google_contacts", r.name); + } } - }) + ); } - } + }, }); frappe.ui.form.on("Dynamic Link", { - link_name:function(frm, cdt, cdn){ + link_name: function (frm, cdt, cdn) { var child = locals[cdt][cdn]; - if(child.link_name) { + if (child.link_name) { frappe.model.with_doctype(child.link_doctype, function () { - var title_field = frappe.get_meta(child.link_doctype).title_field || "name" - frappe.model.get_value(child.link_doctype, child.link_name, title_field, function (r) { - frappe.model.set_value(cdt, cdn, "link_title", r[title_field]) - }) - }) + var title_field = frappe.get_meta(child.link_doctype).title_field || "name"; + frappe.model.get_value( + child.link_doctype, + child.link_name, + title_field, + function (r) { + frappe.model.set_value(cdt, cdn, "link_title", r[title_field]); + } + ); + }); } - } -}) + }, +}); diff --git a/frappe/contacts/doctype/contact/contact.py b/frappe/contacts/doctype/contact/contact.py index 4036cda853..a17f46216b 100644 --- a/frappe/contacts/doctype/contact/contact.py +++ b/frappe/contacts/doctype/contact/contact.py @@ -290,7 +290,7 @@ def get_contact_with_phone_number(number): return contacts = frappe.get_all( - "Contact Phone", filters=[["phone", "like", "%{0}".format(number)]], fields=["parent"], limit=1 + "Contact Phone", filters=[["phone", "like", f"%{number}"]], fields=["parent"], limit=1 ) return contacts[0].parent if contacts else None diff --git a/frappe/contacts/doctype/contact/contact_list.js b/frappe/contacts/doctype/contact/contact_list.js index a93b3f0d73..2b3cd8a062 100644 --- a/frappe/contacts/doctype/contact/contact_list.js +++ b/frappe/contacts/doctype/contact/contact_list.js @@ -1,3 +1,3 @@ -frappe.listview_settings['Contact'] = { +frappe.listview_settings["Contact"] = { add_fields: ["image"], -}; \ No newline at end of file +}; diff --git a/frappe/contacts/doctype/contact/test_contact.py b/frappe/contacts/doctype/contact/test_contact.py index 7ca47476e8..bf0d1037db 100644 --- a/frappe/contacts/doctype/contact/test_contact.py +++ b/frappe/contacts/doctype/contact/test_contact.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2017, Frappe Technologies and Contributors # License: MIT. See LICENSE import unittest diff --git a/frappe/contacts/doctype/contact_email/contact_email.py b/frappe/contacts/doctype/contact_email/contact_email.py index ed794ac06c..b6be852e57 100644 --- a/frappe/contacts/doctype/contact_email/contact_email.py +++ b/frappe/contacts/doctype/contact_email/contact_email.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and contributors # License: MIT. See LICENSE diff --git a/frappe/contacts/doctype/contact_phone/contact_phone.py b/frappe/contacts/doctype/contact_phone/contact_phone.py index 2a842c9c9e..ba6cd89834 100644 --- a/frappe/contacts/doctype/contact_phone/contact_phone.py +++ b/frappe/contacts/doctype/contact_phone/contact_phone.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and contributors # License: MIT. See LICENSE diff --git a/frappe/contacts/doctype/gender/gender.js b/frappe/contacts/doctype/gender/gender.js index e2fd2f18eb..3b34b1584e 100644 --- a/frappe/contacts/doctype/gender/gender.js +++ b/frappe/contacts/doctype/gender/gender.js @@ -1,8 +1,6 @@ // Copyright (c) 2017, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Gender', { - refresh: function() { - - } +frappe.ui.form.on("Gender", { + refresh: function () {}, }); diff --git a/frappe/contacts/doctype/gender/gender.json b/frappe/contacts/doctype/gender/gender.json index 86a066cf0f..20d0210f05 100644 --- a/frappe/contacts/doctype/gender/gender.json +++ b/frappe/contacts/doctype/gender/gender.json @@ -1,113 +1,47 @@ { - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "autoname": "field:gender", - "beta": 0, - "creation": "2017-04-10 12:11:36.526508", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", + "actions": [], + "autoname": "field:gender", + "creation": "2017-04-10 12:11:36.526508", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "gender" + ], "fields": [ { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "gender", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Gender", - "length": 0, - "no_copy": 0, - "options": "", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "fieldname": "gender", + "fieldtype": "Data", + "label": "Gender", + "unique": 1 } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2017-04-10 12:17:04.848338", - "modified_by": "Administrator", - "module": "Contacts", - "name": "Gender", - "name_case": "", - "owner": "Administrator", + ], + "links": [], + "modified": "2022-08-03 12:20:48.408685", + "modified_by": "Administrator", + "module": "Contacts", + "name": "Gender", + "owner": "Administrator", "permissions": [ { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "set_user_permissions": 0, - "share": 1, - "submit": 0, + "create": 1, + "delete": 1, + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1, "write": 1 - }, + }, { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 0, - "delete": 0, - "email": 0, - "export": 0, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 0, - "read": 1, - "report": 0, - "role": "All", - "set_user_permissions": 0, - "share": 0, - "submit": 0, - "write": 0 + "read": 1, + "role": "All" } - ], - "quick_entry": 0, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1, - "track_seen": 0 + ], + "sort_field": "modified", + "sort_order": "DESC", + "states": [], + "track_changes": 1 } \ No newline at end of file diff --git a/frappe/contacts/doctype/gender/gender.py b/frappe/contacts/doctype/gender/gender.py index 8e9951eaf9..fa38a5a6b0 100644 --- a/frappe/contacts/doctype/gender/gender.py +++ b/frappe/contacts/doctype/gender/gender.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2017, Frappe Technologies and contributors # License: MIT. See LICENSE diff --git a/frappe/contacts/doctype/gender/test_gender.py b/frappe/contacts/doctype/gender/test_gender.py index 6b795749ee..c8df3b566d 100644 --- a/frappe/contacts/doctype/gender/test_gender.py +++ b/frappe/contacts/doctype/gender/test_gender.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2017, Frappe Technologies and Contributors # License: MIT. See LICENSE import unittest diff --git a/frappe/contacts/doctype/salutation/salutation.js b/frappe/contacts/doctype/salutation/salutation.js index 856b72e04c..e7da1f389b 100644 --- a/frappe/contacts/doctype/salutation/salutation.js +++ b/frappe/contacts/doctype/salutation/salutation.js @@ -1,8 +1,6 @@ // Copyright (c) 2017, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Salutation', { - refresh: function() { - - } +frappe.ui.form.on("Salutation", { + refresh: function () {}, }); diff --git a/frappe/contacts/doctype/salutation/salutation.json b/frappe/contacts/doctype/salutation/salutation.json index 579f176aa7..c80faf1cda 100644 --- a/frappe/contacts/doctype/salutation/salutation.json +++ b/frappe/contacts/doctype/salutation/salutation.json @@ -1,132 +1,60 @@ { - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 1, - "autoname": "field:salutation", - "beta": 0, - "creation": "2017-04-10 12:17:58.071915", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", + "actions": [], + "allow_rename": 1, + "autoname": "field:salutation", + "creation": "2017-04-10 12:17:58.071915", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "salutation" + ], "fields": [ { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "salutation", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Salutation", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "fieldname": "salutation", + "fieldtype": "Data", + "label": "Salutation", + "unique": 1 } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2020-09-14 12:55:18.855578", - "modified_by": "Administrator", - "module": "Contacts", - "name": "Salutation", - "name_case": "", - "owner": "Administrator", + ], + "links": [], + "modified": "2022-08-03 12:20:48.954912", + "modified_by": "Administrator", + "module": "Contacts", + "name": "Salutation", + "owner": "Administrator", "permissions": [ { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "set_user_permissions": 0, - "share": 1, - "submit": 0, + "create": 1, + "delete": 1, + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1, "write": 1 - }, + }, { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 0, - "delete": 0, - "email": 0, - "export": 0, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 0, - "read": 1, - "report": 0, - "role": "All", - "set_user_permissions": 0, - "share": 0, - "submit": 0, - "write": 0 - }, + "read": 1, + "role": "All" + }, { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 1, - "role": "Administrator", - "set_user_permissions": 0, - "share": 1, - "submit": 0, + "create": 1, + "delete": 1, + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "Administrator", + "share": 1, "write": 1 } - ], - "quick_entry": 0, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1, - "track_seen": 0 -} + ], + "sort_field": "modified", + "sort_order": "DESC", + "states": [], + "track_changes": 1 +} \ No newline at end of file diff --git a/frappe/contacts/doctype/salutation/salutation.py b/frappe/contacts/doctype/salutation/salutation.py index 57fb2d6abc..66469c4700 100644 --- a/frappe/contacts/doctype/salutation/salutation.py +++ b/frappe/contacts/doctype/salutation/salutation.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2017, Frappe Technologies and contributors # License: MIT. See LICENSE diff --git a/frappe/contacts/doctype/salutation/test_salutation.py b/frappe/contacts/doctype/salutation/test_salutation.py index 5149ced3dd..2c35e5bd2b 100644 --- a/frappe/contacts/doctype/salutation/test_salutation.py +++ b/frappe/contacts/doctype/salutation/test_salutation.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2017, Frappe Technologies and Contributors # License: MIT. See LICENSE import unittest diff --git a/frappe/contacts/report/addresses_and_contacts/addresses_and_contacts.js b/frappe/contacts/report/addresses_and_contacts/addresses_and_contacts.js index 10137e80d5..9870ee611b 100644 --- a/frappe/contacts/report/addresses_and_contacts/addresses_and_contacts.js +++ b/frappe/contacts/report/addresses_and_contacts/addresses_and_contacts.js @@ -2,32 +2,32 @@ // For license information, please see license.txt frappe.query_reports["Addresses And Contacts"] = { - "filters": [ + filters: [ { - "reqd": 1, - "fieldname":"reference_doctype", - "label": __("Entity Type"), - "fieldtype": "Link", - "options": "DocType", - "get_query": function() { + reqd: 1, + fieldname: "reference_doctype", + label: __("Entity Type"), + fieldtype: "Link", + options: "DocType", + get_query: function () { return { - "filters": { - "name": ["in", "Contact, Address"], - } - } - } + filters: { + name: ["in", "Contact, Address"], + }, + }; + }, }, { - "fieldname":"reference_name", - "label": __("Entity Name"), - "fieldtype": "Dynamic Link", - "get_options": function() { - let reference_doctype = frappe.query_report.get_filter_value('reference_doctype'); - if(!reference_doctype) { + fieldname: "reference_name", + label: __("Entity Name"), + fieldtype: "Dynamic Link", + get_options: function () { + let reference_doctype = frappe.query_report.get_filter_value("reference_doctype"); + if (!reference_doctype) { frappe.throw(__("Please select Entity Type first")); } return reference_doctype; - } - } - ] -} + }, + }, + ], +}; diff --git a/frappe/core/doctype/feedback/__init__.py b/frappe/core/api/__init__.py similarity index 100% rename from frappe/core/doctype/feedback/__init__.py rename to frappe/core/api/__init__.py diff --git a/frappe/core/api/file.py b/frappe/core/api/file.py new file mode 100644 index 0000000000..ec305aff4f --- /dev/null +++ b/frappe/core/api/file.py @@ -0,0 +1,121 @@ +import json + +import frappe +from frappe.core.doctype.file.file import File, setup_folder_path +from frappe.utils import cint, cstr + + +@frappe.whitelist() +def unzip_file(name: str): + """Unzip the given file and make file records for each of the extracted files""" + file: File = frappe.get_doc("File", name) + return file.unzip() + + +@frappe.whitelist() +def get_attached_images(doctype: str, names: list[str]) -> frappe._dict: + """get list of image urls attached in form + returns {name: ['image.jpg', 'image.png']}""" + + if isinstance(names, str): + names = json.loads(names) + + img_urls = frappe.db.get_list( + "File", + filters={ + "attached_to_doctype": doctype, + "attached_to_name": ("in", names), + "is_folder": 0, + }, + fields=["file_url", "attached_to_name as docname"], + ) + + out = frappe._dict() + for i in img_urls: + out[i.docname] = out.get(i.docname, []) + out[i.docname].append(i.file_url) + + return out + + +@frappe.whitelist() +def get_files_in_folder(folder: str, start: int = 0, page_length: int = 20) -> dict: + start = cint(start) + page_length = cint(page_length) + + attachment_folder = frappe.db.get_value( + "File", + "Home/Attachments", + ["name", "file_name", "file_url", "is_folder", "modified"], + as_dict=1, + ) + + files = frappe.get_list( + "File", + {"folder": folder}, + ["name", "file_name", "file_url", "is_folder", "modified"], + start=start, + page_length=page_length + 1, + ) + + if folder == "Home" and attachment_folder not in files: + files.insert(0, attachment_folder) + + return {"files": files[:page_length], "has_more": len(files) > page_length} + + +@frappe.whitelist() +def get_files_by_search_text(text: str) -> list[dict]: + if not text: + return [] + + text = "%" + cstr(text).lower() + "%" + return frappe.get_list( + "File", + fields=["name", "file_name", "file_url", "is_folder", "modified"], + filters={"is_folder": False}, + or_filters={ + "file_name": ("like", text), + "file_url": text, + "name": ("like", text), + }, + order_by="modified desc", + limit=20, + ) + + +@frappe.whitelist(allow_guest=True) +def get_max_file_size() -> int: + return cint(frappe.conf.get("max_file_size")) or 10485760 + + +@frappe.whitelist() +def create_new_folder(file_name: str, folder: str) -> File: + """create new folder under current parent folder""" + file = frappe.new_doc("File") + file.file_name = file_name + file.is_folder = 1 + file.folder = folder + file.insert(ignore_if_duplicate=True) + return file + + +@frappe.whitelist() +def move_file(file_list: list[File], new_parent: str, old_parent: str) -> None: + if isinstance(file_list, str): + file_list = json.loads(file_list) + + for file_obj in file_list: + setup_folder_path(file_obj.get("name"), new_parent) + + # recalculate sizes + frappe.get_doc("File", old_parent).save() + frappe.get_doc("File", new_parent).save() + + +@frappe.whitelist() +def zip_files(files: str): + files = frappe.parse_json(files) + frappe.response["filename"] = "files.zip" + frappe.response["filecontent"] = File.zip_files(files) + frappe.response["type"] = "download" diff --git a/frappe/core/doctype/access_log/access_log.js b/frappe/core/doctype/access_log/access_log.js index d36d10768b..94f1bf732d 100644 --- a/frappe/core/doctype/access_log/access_log.js +++ b/frappe/core/doctype/access_log/access_log.js @@ -1,17 +1,17 @@ // Copyright (c) 2019, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Access Log', { +frappe.ui.form.on("Access Log", { show_document: function (frm) { - frappe.set_route('Form', frm.doc.export_from, frm.doc.reference_document); + frappe.set_route("Form", frm.doc.export_from, frm.doc.reference_document); }, show_report: function (frm) { - if (frm.doc.report_name.includes('/')) { + if (frm.doc.report_name.includes("/")) { frappe.set_route(frm.doc.report_name); } else { let filters = frm.doc.filters ? JSON.parse(frm.doc.filters) : {}; - frappe.set_route('query-report', frm.doc.report_name, filters); + frappe.set_route("query-report", frm.doc.report_name, filters); } - } + }, }); diff --git a/frappe/core/doctype/access_log/access_log.json b/frappe/core/doctype/access_log/access_log.json index 6a3028303b..69803ef05a 100644 --- a/frappe/core/doctype/access_log/access_log.json +++ b/frappe/core/doctype/access_log/access_log.json @@ -1,5 +1,6 @@ { - "autoname": "format:AL-{#####}", + "actions": [], + "autoname": "hash", "creation": "2019-07-25 15:44:44.955496", "doctype": "DocType", "editable_grid": 1, @@ -35,6 +36,7 @@ "fieldname": "user", "fieldtype": "Link", "in_list_view": 1, + "in_standard_filter": 1, "label": "User ", "options": "User", "read_only": 1 @@ -50,6 +52,7 @@ "fieldname": "reference_document", "fieldtype": "Data", "in_list_view": 1, + "in_standard_filter": 1, "label": "Reference Document", "read_only": 1 }, @@ -127,10 +130,12 @@ "read_only": 1 } ], - "modified": "2019-08-05 19:00:13.839471", + "links": [], + "modified": "2022-06-13 05:59:26.866004", "modified_by": "Administrator", "module": "Core", "name": "Access Log", + "naming_rule": "Random", "owner": "Administrator", "permissions": [ { @@ -146,5 +151,6 @@ ], "sort_field": "modified", "sort_order": "DESC", + "states": [], "track_seen": 1 } \ No newline at end of file diff --git a/frappe/core/doctype/access_log/access_log.py b/frappe/core/doctype/access_log/access_log.py index b7a6d77206..ca2909b970 100644 --- a/frappe/core/doctype/access_log/access_log.py +++ b/frappe/core/doctype/access_log/access_log.py @@ -35,7 +35,11 @@ def make_access_log( @frappe.write_only() -@retry(stop=stop_after_attempt(3), retry=retry_if_exception_type(frappe.DuplicateEntryError)) +@retry( + stop=stop_after_attempt(3), + retry=retry_if_exception_type(frappe.DuplicateEntryError), + reraise=True, +) def _make_access_log( doctype=None, document=None, diff --git a/frappe/core/doctype/access_log/test_access_log.py b/frappe/core/doctype/access_log/test_access_log.py index 983e3cb5e4..ee0422e11a 100644 --- a/frappe/core/doctype/access_log/test_access_log.py +++ b/frappe/core/doctype/access_log/test_access_log.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # License: MIT. See LICENSE @@ -28,7 +27,7 @@ class TestAccessLog(unittest.TestCase): "User", frappe.session.user, fieldname="api_secret" ) api_key = frappe.db.get_value("User", "Administrator", "api_key") - self.header = {"Authorization": "token {}:{}".format(api_key, generated_secret)} + self.header = {"Authorization": f"token {api_key}:{generated_secret}"} self.test_html_template = """ diff --git a/frappe/core/doctype/activity_log/activity_log.js b/frappe/core/doctype/activity_log/activity_log.js index 97e49e4b34..7df644a86a 100644 --- a/frappe/core/doctype/activity_log/activity_log.js +++ b/frappe/core/doctype/activity_log/activity_log.js @@ -1,8 +1,6 @@ // Copyright (c) 2017, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Activity Log', { - refresh: function() { - - } +frappe.ui.form.on("Activity Log", { + refresh: function () {}, }); diff --git a/frappe/core/doctype/activity_log/activity_log.py b/frappe/core/doctype/activity_log/activity_log.py index 84c908d0ae..468b7f4473 100644 --- a/frappe/core/doctype/activity_log/activity_log.py +++ b/frappe/core/doctype/activity_log/activity_log.py @@ -2,7 +2,6 @@ # License: MIT. See LICENSE import frappe -from frappe import _ from frappe.core.utils import set_timeline_doc from frappe.model.document import Document from frappe.query_builder import DocType, Interval @@ -26,6 +25,13 @@ class ActivityLog(Document): if self.reference_doctype and self.reference_name: self.status = "Linked" + @staticmethod + def clear_old_logs(days=None): + if not days: + days = 90 + doctype = DocType("Activity Log") + frappe.db.delete(doctype, filters=(doctype.modified < (Now() - Interval(days=days)))) + def on_doctype_update(): """Add indexes in `tabActivity Log`""" @@ -44,12 +50,3 @@ def add_authentication_log(subject, user, operation="Login", status="Success"): "operation": operation, } ).insert(ignore_permissions=True, ignore_links=True) - - -def clear_activity_logs(days=None): - """clear 90 day old authentication logs or configured in log settings""" - - if not days: - days = 90 - doctype = DocType("Activity Log") - frappe.db.delete(doctype, filters=(doctype.creation < (Now() - Interval(days=days)))) diff --git a/frappe/core/doctype/activity_log/activity_log_list.js b/frappe/core/doctype/activity_log/activity_log_list.js index 111a230827..53758a7b62 100644 --- a/frappe/core/doctype/activity_log/activity_log_list.js +++ b/frappe/core/doctype/activity_log/activity_log_list.js @@ -1,8 +1,12 @@ -frappe.listview_settings['Activity Log'] = { - get_indicator: function(doc) { - if(doc.operation == "Login" && doc.status == "Success") - return [__(doc.status), "green"]; - else if(doc.operation == "Login" && doc.status == "Failed") +frappe.listview_settings["Activity Log"] = { + get_indicator: function (doc) { + if (doc.operation == "Login" && doc.status == "Success") return [__(doc.status), "green"]; + else if (doc.operation == "Login" && doc.status == "Failed") return [__(doc.status), "red"]; - } -}; \ No newline at end of file + }, + onload: function (listview) { + frappe.require("logtypes.bundle.js", () => { + frappe.utils.logtypes.show_log_retention_message(cur_list.doctype); + }); + }, +}; diff --git a/frappe/core/doctype/activity_log/feed.py b/frappe/core/doctype/activity_log/feed.py index 3e29154242..eb040927e1 100644 --- a/frappe/core/doctype/activity_log/feed.py +++ b/frappe/core/doctype/activity_log/feed.py @@ -74,9 +74,7 @@ def get_feed_match_conditions(user=None, doctype="Comment"): user_permissions = frappe.permissions.get_user_permissions(user) can_read = frappe.get_user().get_can_read() - can_read_doctypes = [ - "'{}'".format(dt) for dt in list(set(can_read) - set(list(user_permissions))) - ] + can_read_doctypes = [f"'{dt}'" for dt in list(set(can_read) - set(list(user_permissions)))] if can_read_doctypes: conditions += [ diff --git a/frappe/core/doctype/activity_log/test_activity_log.py b/frappe/core/doctype/activity_log/test_activity_log.py index f5f94dc44b..c362fca521 100644 --- a/frappe/core/doctype/activity_log/test_activity_log.py +++ b/frappe/core/doctype/activity_log/test_activity_log.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and Contributors # License: MIT. See LICENSE import time diff --git a/frappe/core/doctype/block_module/block_module.json b/frappe/core/doctype/block_module/block_module.json index 64deff66ee..9711aaa001 100644 --- a/frappe/core/doctype/block_module/block_module.json +++ b/frappe/core/doctype/block_module/block_module.json @@ -1,71 +1,31 @@ { - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, - "creation": "2015-03-24 14:28:15.882903", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "Other", - "editable_grid": 1, + "actions": [], + "creation": "2015-03-24 14:28:15.882903", + "doctype": "DocType", + "document_type": "Other", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "module" + ], "fields": [ { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "module", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Module", - "length": 0, - "no_copy": 0, - "options": "", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "fieldname": "module", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Module", + "reqd": 1 } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 1, - "max_attachments": 0, - "modified": "2017-10-31 19:36:18.586834", - "modified_by": "Administrator", - "module": "Core", - "name": "Block Module", - "name_case": "", - "owner": "Administrator", - "permissions": [], - "quick_entry": 0, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 0, - "track_seen": 0 + ], + "istable": 1, + "links": [], + "modified": "2022-08-03 12:20:52.738977", + "modified_by": "Administrator", + "module": "Core", + "name": "Block Module", + "owner": "Administrator", + "permissions": [], + "sort_field": "modified", + "sort_order": "DESC", + "states": [] } \ No newline at end of file diff --git a/frappe/core/doctype/block_module/block_module.py b/frappe/core/doctype/block_module/block_module.py index 3158e3e6a5..b9de5b325e 100644 --- a/frappe/core/doctype/block_module/block_module.py +++ b/frappe/core/doctype/block_module/block_module.py @@ -1,8 +1,6 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors # License: MIT. See LICENSE -import frappe from frappe.model.document import Document diff --git a/frappe/core/doctype/comment/comment.js b/frappe/core/doctype/comment/comment.js index a793f766cb..4d227f6f5f 100644 --- a/frappe/core/doctype/comment/comment.js +++ b/frappe/core/doctype/comment/comment.js @@ -1,8 +1,7 @@ // Copyright (c) 2019, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Comment', { +frappe.ui.form.on("Comment", { // refresh: function(frm) { - // } }); diff --git a/frappe/core/doctype/comment/comment.json b/frappe/core/doctype/comment/comment.json index fe465f46bd..9f27e7e7be 100644 --- a/frappe/core/doctype/comment/comment.json +++ b/frappe/core/doctype/comment/comment.json @@ -1,4 +1,5 @@ { + "actions": [], "creation": "2019-02-07 10:10:46.845678", "doctype": "DocType", "editable_grid": 1, @@ -17,7 +18,8 @@ "link_name", "reference_owner", "section_break_10", - "content" + "content", + "ip_address" ], "fields": [ { @@ -102,9 +104,16 @@ "ignore_xss_filter": 1, "in_list_view": 1, "label": "Content" + }, + { + "fieldname": "ip_address", + "fieldtype": "Data", + "hidden": 1, + "label": "IP Address" } ], - "modified": "2019-09-02 21:00:10.784787", + "links": [], + "modified": "2022-07-12 17:35:31.774137", "modified_by": "Administrator", "module": "Core", "name": "Comment", @@ -138,6 +147,7 @@ "quick_entry": 1, "sort_field": "modified", "sort_order": "DESC", + "states": [], "title_field": "comment_type", "track_changes": 1 } \ No newline at end of file diff --git a/frappe/core/doctype/comment/comment.py b/frappe/core/doctype/comment/comment.py index 1456f1ec93..02a5d2d58e 100644 --- a/frappe/core/doctype/comment/comment.py +++ b/frappe/core/doctype/comment/comment.py @@ -1,26 +1,19 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and contributors # License: MIT. See LICENSE import json import frappe -from frappe import _ -from frappe.core.doctype.user.user import extract_mentions from frappe.database.schema import add_column -from frappe.desk.doctype.notification_log.notification_log import ( - enqueue_create_notification, - get_title, - get_title_html, -) +from frappe.desk.notifications import notify_mentions from frappe.exceptions import ImplicitCommitError from frappe.model.document import Document -from frappe.utils import get_fullname +from frappe.model.utils import is_virtual_doctype from frappe.website.utils import clear_cache class Comment(Document): def after_insert(self): - self.notify_mentions() + notify_mentions(self.reference_doctype, self.reference_name, self.content) self.notify_change("add") def validate(self): @@ -51,7 +44,7 @@ class Comment(Document): return frappe.publish_realtime( - "update_docinfo_for_{}_{}".format(self.reference_doctype, self.reference_name), + f"update_docinfo_for_{self.reference_doctype}_{self.reference_name}", {"doc": self.as_dict(), "key": key, "action": action}, after_commit=True, ) @@ -64,40 +57,6 @@ class Comment(Document): update_comments_in_parent(self.reference_doctype, self.reference_name, _comments) - def notify_mentions(self): - if self.reference_doctype and self.reference_name and self.content: - mentions = extract_mentions(self.content) - - if not mentions: - return - - sender_fullname = get_fullname(frappe.session.user) - title = get_title(self.reference_doctype, self.reference_name) - - recipients = [ - frappe.db.get_value( - "User", - {"enabled": 1, "name": name, "user_type": "System User", "allowed_in_mentions": 1}, - "email", - ) - for name in mentions - ] - - notification_message = _("""{0} mentioned you in a comment in {1} {2}""").format( - frappe.bold(sender_fullname), frappe.bold(self.reference_doctype), get_title_html(title) - ) - - notification_doc = { - "type": "Mention", - "document_type": self.reference_doctype, - "document_name": self.reference_name, - "subject": notification_message, - "from_user": frappe.session.user, - "email_content": self.content, - } - - enqueue_create_notification(recipients, notification_doc) - def on_doctype_update(): frappe.db.add_index("Comment", ["reference_doctype", "reference_name"]) @@ -153,7 +112,10 @@ def get_comments_from_parent(doc): `_comments` """ try: - _comments = frappe.db.get_value(doc.reference_doctype, doc.reference_name, "_comments") or "[]" + if is_virtual_doctype(doc.reference_doctype): + _comments = "[]" + else: + _comments = frappe.db.get_value(doc.reference_doctype, doc.reference_name, "_comments") or "[]" except Exception as e: if frappe.db.is_missing_table_or_column(e): @@ -176,14 +138,14 @@ def update_comments_in_parent(reference_doctype, reference_name, _comments): not reference_doctype or not reference_name or frappe.db.get_value("DocType", reference_doctype, "issingle") - or frappe.db.get_value("DocType", reference_doctype, "is_virtual") + or is_virtual_doctype(reference_doctype) ): return try: # use sql, so that we do not mess with the timestamp frappe.db.sql( - """update `tab{0}` set `_comments`=%s where name=%s""".format(reference_doctype), # nosec + f"""update `tab{reference_doctype}` set `_comments`=%s where name=%s""", # nosec (json.dumps(_comments[-100:]), reference_name), ) diff --git a/frappe/core/doctype/comment/test_comment.py b/frappe/core/doctype/comment/test_comment.py index 3072f8b5b9..bedcea6e7e 100644 --- a/frappe/core/doctype/comment/test_comment.py +++ b/frappe/core/doctype/comment/test_comment.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # License: MIT. See LICENSE import json diff --git a/frappe/core/doctype/communication/communication.js b/frappe/core/doctype/communication/communication.js index 07674d16ae..a36af705a7 100644 --- a/frappe/core/doctype/communication/communication.js +++ b/frappe/core/doctype/communication/communication.js @@ -1,120 +1,158 @@ frappe.ui.form.on("Communication", { - onload: function(frm) { - if(frm.doc.content) { + onload: function (frm) { + if (frm.doc.content) { frm.doc.content = frappe.dom.remove_script_and_style(frm.doc.content); } - frm.set_query("reference_doctype", function() { + frm.set_query("reference_doctype", function () { return { filters: { - "issingle": 0, - "istable": 0 - } - } + issingle: 0, + istable: 0, + }, + }; }); }, - refresh: function(frm) { - if(frm.is_new()) return; + refresh: function (frm) { + if (frm.is_new()) return; frm.convert_to_click && frm.set_convert_button(); frm.subject_field = "subject"; // content field contains weird table html that does not render well in Quill // this field is not to be edited directly anyway, so setting it as read only - frm.set_df_property('content', 'read_only', 1); + frm.set_df_property("content", "read_only", 1); - if(frm.doc.reference_doctype && frm.doc.reference_name) { - frm.add_custom_button(__(frm.doc.reference_name), function() { + if (frm.doc.reference_doctype && frm.doc.reference_name) { + frm.add_custom_button(__(frm.doc.reference_name), function () { frappe.set_route("Form", frm.doc.reference_doctype, frm.doc.reference_name); }); } else { // if an unlinked communication, set email field - if (frm.doc.sent_or_received==="Received") { + if (frm.doc.sent_or_received === "Received") { frm.email_field = "sender"; } else { frm.email_field = "recipients"; } } - if(frm.doc.status==="Open") { - frm.add_custom_button(__("Close"), function() { - frm.trigger('mark_as_closed_open'); + if (frm.doc.status === "Open") { + frm.add_custom_button(__("Close"), function () { + frm.trigger("mark_as_closed_open"); }); } else if (frm.doc.status !== "Linked") { - frm.add_custom_button(__("Reopen"), function() { - frm.trigger('mark_as_closed_open'); + frm.add_custom_button(__("Reopen"), function () { + frm.trigger("mark_as_closed_open"); }); } - frm.add_custom_button(__("Relink"), function() { - frm.trigger('show_relink_dialog'); + frm.add_custom_button(__("Relink"), function () { + frm.trigger("show_relink_dialog"); }); - if(frm.doc.communication_type=="Communication" - && frm.doc.communication_medium == "Email" - && frm.doc.sent_or_received == "Received") { - - frm.add_custom_button(__("Reply"), function() { - frm.trigger('reply'); + if ( + frm.doc.communication_type == "Communication" && + frm.doc.communication_medium == "Email" && + frm.doc.sent_or_received == "Received" + ) { + frm.add_custom_button(__("Reply"), function () { + frm.trigger("reply"); }); - frm.add_custom_button(__("Reply All"), function() { - frm.trigger('reply_all'); - }, __("Actions")); + frm.add_custom_button( + __("Reply All"), + function () { + frm.trigger("reply_all"); + }, + __("Actions") + ); - frm.add_custom_button(__("Forward"), function() { - frm.trigger('forward_mail'); - }, __("Actions")); + frm.add_custom_button( + __("Forward"), + function () { + frm.trigger("forward_mail"); + }, + __("Actions") + ); - frm.add_custom_button(frm.doc.seen ? __("Mark as Unread") : __("Mark as Read"), function() { - frm.trigger('mark_as_read_unread'); - }, __("Actions")); + frm.add_custom_button( + frm.doc.seen ? __("Mark as Unread") : __("Mark as Read"), + function () { + frm.trigger("mark_as_read_unread"); + }, + __("Actions") + ); - frm.add_custom_button(__("Move"), function() { - frm.trigger('show_move_dialog'); - }, __("Actions")); + frm.add_custom_button( + __("Move"), + function () { + frm.trigger("show_move_dialog"); + }, + __("Actions") + ); - if(frm.doc.email_status != "Spam") - frm.add_custom_button(__("Mark as Spam"), function() { - frm.trigger('mark_as_spam'); - }, __("Actions")); + if (frm.doc.email_status != "Spam") + frm.add_custom_button( + __("Mark as Spam"), + function () { + frm.trigger("mark_as_spam"); + }, + __("Actions") + ); - if(frm.doc.email_status != "Trash") { - frm.add_custom_button(__("Move To Trash"), function() { - frm.trigger('move_to_trash'); - }, __("Actions")); + if (frm.doc.email_status != "Trash") { + frm.add_custom_button( + __("Move To Trash"), + function () { + frm.trigger("move_to_trash"); + }, + __("Actions") + ); } - frm.add_custom_button(__("Contact"), function() { - frm.trigger('add_to_contact'); - }, __('Create')); + frm.add_custom_button( + __("Contact"), + function () { + frm.trigger("add_to_contact"); + }, + __("Create") + ); } - if(frm.doc.communication_type=="Communication" - && frm.doc.communication_medium == "Phone" - && frm.doc.sent_or_received == "Received"){ - - frm.add_custom_button(__("Add Contact"), function() { - frm.trigger('add_to_contact'); - }, __("Actions")); + if ( + frm.doc.communication_type == "Communication" && + frm.doc.communication_medium == "Phone" && + frm.doc.sent_or_received == "Received" + ) { + frm.add_custom_button( + __("Add Contact"), + function () { + frm.trigger("add_to_contact"); + }, + __("Actions") + ); } }, - show_relink_dialog: function(frm) { - var d = new frappe.ui.Dialog ({ + show_relink_dialog: function (frm) { + var d = new frappe.ui.Dialog({ title: __("Relink Communication"), - fields: [{ - "fieldtype": "Link", - "options": "DocType", - "label": __("Reference Doctype"), - "fieldname": "reference_doctype", - "get_query": function() {return {"query": "frappe.email.get_communication_doctype"}} - }, - { - "fieldtype": "Dynamic Link", - "options": "reference_doctype", - "label": __("Reference Name"), - "fieldname": "reference_name" - }] + fields: [ + { + fieldtype: "Link", + options: "DocType", + label: __("Reference Doctype"), + fieldname: "reference_doctype", + get_query: function () { + return { query: "frappe.email.get_communication_doctype" }; + }, + }, + { + fieldtype: "Dynamic Link", + options: "reference_doctype", + label: __("Reference Name"), + fieldname: "reference_name", + }, + ], }); d.set_value("reference_doctype", frm.doc.reference_doctype); d.set_value("reference_name", frm.doc.reference_name); @@ -122,24 +160,27 @@ frappe.ui.form.on("Communication", { var values = d.get_values(); if (values) { frappe.confirm( - __('Are you sure you want to relink this communication to {0}?', [values["reference_name"]]), + __("Are you sure you want to relink this communication to {0}?", [ + values["reference_name"], + ]), function () { d.hide(); frappe.call({ method: "frappe.email.relink", args: { - "name": frm.doc.name, - "reference_doctype": values["reference_doctype"], - "reference_name": values["reference_name"] + name: frm.doc.name, + reference_doctype: values["reference_doctype"], + reference_name: values["reference_name"], }, callback: function () { frm.refresh(); - } + }, }); }, - function() { + function () { frappe.show_alert({ - message: __('Document not Relinked'), 'indicator': 'info' + message: __("Document not Relinked"), + indicator: "info", }); } ); @@ -148,24 +189,26 @@ frappe.ui.form.on("Communication", { d.show(); }, - show_move_dialog: function(frm) { - var d = new frappe.ui.Dialog ({ + show_move_dialog: function (frm) { + var d = new frappe.ui.Dialog({ title: __("Move"), - fields: [{ - "fieldtype": "Link", - "options": "Email Account", - "label": __("Email Account"), - "fieldname": "email_account", - "reqd": 1, - "get_query": function() { - return { - "filters": { - "name": ["!=", frm.doc.email_account], - "enable_incoming": ["=", 1] - } - }; - } - }], + fields: [ + { + fieldtype: "Link", + options: "Email Account", + label: __("Email Account"), + fieldname: "email_account", + reqd: 1, + get_query: function () { + return { + filters: { + name: ["!=", frm.doc.email_account], + enable_incoming: ["=", 1], + }, + }; + }, + }, + ], primary_action_label: __("Move"), primary_action(values) { d.hide(); @@ -173,88 +216,88 @@ frappe.ui.form.on("Communication", { method: "frappe.email.inbox.move_email", args: { communication: frm.doc.name, - email_account: values.email_account + email_account: values.email_account, }, freeze: true, - callback: function() { + callback: function () { window.history.back(); - } + }, }); - } + }, }); d.show(); }, - mark_as_read_unread: function(frm) { - var action = frm.doc.seen? "Unread": "Read"; + mark_as_read_unread: function (frm) { + var action = frm.doc.seen ? "Unread" : "Read"; var flag = "(\\SEEN)"; return frappe.call({ method: "frappe.email.inbox.create_email_flag_queue", args: { - 'names': [frm.doc.name], - 'action': action, - 'flag': flag + names: [frm.doc.name], + action: action, + flag: flag, }, freeze: true, - callback: function() { + callback: function () { frm.reload_doc(); - } + }, }); }, - mark_as_closed_open: function(frm) { + mark_as_closed_open: function (frm) { var status = frm.doc.status == "Open" ? "Closed" : "Open"; return frappe.call({ method: "frappe.email.inbox.mark_as_closed_open", args: { communication: frm.doc.name, - status: status + status: status, }, freeze: true, - callback: function() { + callback: function () { frm.reload_doc(); - } + }, }); }, - reply: function(frm) { + reply: function (frm) { var args = frm.events.get_mail_args(frm); $.extend(args, { subject: __("Re: {0}", [frm.doc.subject]), - recipients: frm.doc.sender - }) + recipients: frm.doc.sender, + }); new frappe.views.CommunicationComposer(args); }, - reply_all: function(frm) { - var args = frm.events.get_mail_args(frm) + reply_all: function (frm) { + var args = frm.events.get_mail_args(frm); $.extend(args, { subject: __("Res: {0}", [frm.doc.subject]), recipients: frm.doc.sender, - cc: frm.doc.cc - }) + cc: frm.doc.cc, + }); new frappe.views.CommunicationComposer(args); }, - forward_mail: function(frm) { - var args = frm.events.get_mail_args(frm) + forward_mail: function (frm) { + var args = frm.events.get_mail_args(frm); $.extend(args, { forward: true, subject: __("Fw: {0}", [frm.doc.subject]), - }) + }); new frappe.views.CommunicationComposer(args); }, - get_mail_args: function(frm) { - var sender_email_id = "" - $.each(frappe.boot.email_accounts, function(idx, account) { - if(account.email_account == frm.doc.email_account) { - sender_email_id = account.email_id - return + get_mail_args: function (frm) { + var sender_email_id = ""; + $.each(frappe.boot.email_accounts, function (idx, account) { + if (account.email_account == frm.doc.email_account) { + sender_email_id = account.email_id; + return; } }); @@ -263,51 +306,51 @@ frappe.ui.form.on("Communication", { doc: frm.doc, last_email: frm.doc, sender: sender_email_id, - attachments: frm.doc.attachments - } + attachments: frm.doc.attachments, + }; }, - add_to_contact: function(frm) { + add_to_contact: function (frm) { var me = this; - var fullname = frm.doc.sender_full_name || "" + var fullname = frm.doc.sender_full_name || ""; - var names = fullname.split(" ") - var first_name = names[0] - var last_name = names.length >= 2? names[names.length - 1]: "" + var names = fullname.split(" "); + var first_name = names[0]; + var last_name = names.length >= 2 ? names[names.length - 1] : ""; frappe.route_options = { - "email_id": frm.doc.sender || "", - "first_name": first_name, - "last_name": last_name, - "mobile_no": frm.doc.phone_no || "" - } - frappe.new_doc("Contact") + email_id: frm.doc.sender || "", + first_name: first_name, + last_name: last_name, + mobile_no: frm.doc.phone_no || "", + }; + frappe.new_doc("Contact"); }, - mark_as_spam: function(frm) { + mark_as_spam: function (frm) { frappe.call({ method: "frappe.email.inbox.mark_as_spam", args: { communication: frm.doc.name, - sender: frm.doc.sender + sender: frm.doc.sender, }, freeze: true, - callback: function(r) { - frappe.msgprint(__("Email has been marked as spam")) - } - }) + callback: function (r) { + frappe.msgprint(__("Email has been marked as spam")); + }, + }); }, - move_to_trash: function(frm) { + move_to_trash: function (frm) { frappe.call({ method: "frappe.email.inbox.mark_as_trash", args: { - communication: frm.doc.name + communication: frm.doc.name, }, freeze: true, - callback: function(r) { - frappe.msgprint(__("Email has been moved to trash")) - } - }) - } + callback: function (r) { + frappe.msgprint(__("Email has been moved to trash")); + }, + }); + }, }); diff --git a/frappe/core/doctype/communication/communication.py b/frappe/core/doctype/communication/communication.py index 409c4c0956..c049ccff45 100644 --- a/frappe/core/doctype/communication/communication.py +++ b/frappe/core/doctype/communication/communication.py @@ -3,10 +3,9 @@ from collections import Counter from email.utils import getaddresses -from typing import List from urllib.parse import unquote -from parse import compile +from bs4 import BeautifulSoup import frappe from frappe import _ @@ -20,7 +19,6 @@ from frappe.core.doctype.communication.mixins import CommunicationEmailMixin from frappe.core.utils import get_parent_doc from frappe.model.document import Document from frappe.utils import ( - cstr, parse_addr, split_emails, strip_html, @@ -146,14 +144,12 @@ class Communication(Document, CommunicationEmailMixin): if not self.content: return - quill_parser = compile('
{}
') - email_body = quill_parser.parse(self.content) + soup = BeautifulSoup(self.content, "html.parser") + email_body = soup.find("div", {"class": "ql-editor read-mode"}) if not email_body: return - email_body = email_body[0] - user_email_signature = ( frappe.db.get_value( "User", @@ -173,7 +169,11 @@ class Communication(Document, CommunicationEmailMixin): if not signature: return - _signature = quill_parser.parse(signature)[0] if "ql-editor" in signature else None + soup = BeautifulSoup(signature, "html.parser") + html_signature = soup.find("div", {"class": "ql-editor read-mode"}) + _signature = None + if html_signature: + _signature = html_signature.renderContents() if (_signature or signature) not in self.content: self.content = f'{self.content}


{signature}' @@ -207,7 +207,7 @@ class Communication(Document, CommunicationEmailMixin): """ emails = split_emails(emails) if isinstance(emails, str) else (emails or []) if exclude_displayname: - return [email.lower() for email in set([parse_addr(email)[1] for email in emails]) if email] + return [email.lower() for email in {parse_addr(email)[1] for email in emails} if email] return [email.lower() for email in set(emails) if email] def to_list(self, exclude_displayname=True): @@ -232,7 +232,7 @@ class Communication(Document, CommunicationEmailMixin): def notify_change(self, action): frappe.publish_realtime( - "update_docinfo_for_{}_{}".format(self.reference_doctype, self.reference_name), + f"update_docinfo_for_{self.reference_doctype}_{self.reference_name}", {"doc": self.as_dict(), "key": "communications", "action": action}, after_commit=True, ) @@ -428,7 +428,7 @@ def get_permission_query_conditions_for_communication(user): ) -def get_contacts(email_strings: List[str], auto_create_contact=False) -> List[str]: +def get_contacts(email_strings: list[str], auto_create_contact=False) -> list[str]: email_addrs = get_emails(email_strings) contacts = [] for email in email_addrs: @@ -440,9 +440,7 @@ def get_contacts(email_strings: List[str], auto_create_contact=False) -> List[st first_name = frappe.unscrub(email_parts[0]) try: - contact_name = ( - "{0}-{1}".format(first_name, email_parts[1]) if first_name == "Contact" else first_name - ) + contact_name = f"{first_name}-{email_parts[1]}" if first_name == "Contact" else first_name contact = frappe.get_doc( {"doctype": "Contact", "first_name": contact_name, "name": contact_name} ) @@ -450,8 +448,7 @@ def get_contacts(email_strings: List[str], auto_create_contact=False) -> List[st contact.insert(ignore_permissions=True) contact_name = contact.name except Exception: - traceback = frappe.get_traceback() - frappe.log_error(traceback) + contact.log_error("Unable to add contact") if contact_name: contacts.append(contact_name) @@ -459,7 +456,7 @@ def get_contacts(email_strings: List[str], auto_create_contact=False) -> List[st return contacts -def get_emails(email_strings: List[str]) -> List[str]: +def get_emails(email_strings: list[str]) -> list[str]: email_addrs = [] for email_string in email_strings: @@ -526,7 +523,7 @@ def get_email_without_link(email): except IndexError: return email - return "{0}@{1}".format(email_id, email_host) + return f"{email_id}@{email_host}" def update_parent_document_on_communication(doc): diff --git a/frappe/core/doctype/communication/communication_list.js b/frappe/core/doctype/communication/communication_list.js index 315b74a39c..4ef3a384ff 100644 --- a/frappe/core/doctype/communication/communication_list.js +++ b/frappe/core/doctype/communication/communication_list.js @@ -1,25 +1,32 @@ -frappe.listview_settings['Communication'] = { +frappe.listview_settings["Communication"] = { add_fields: [ - "sent_or_received","recipients", "subject", - "communication_medium", "communication_type", - "sender", "seen", "reference_doctype", "reference_name", - "has_attachment", "communication_date" + "sent_or_received", + "recipients", + "subject", + "communication_medium", + "communication_type", + "sender", + "seen", + "reference_doctype", + "reference_name", + "has_attachment", + "communication_date", ], filters: [["status", "=", "Open"]], - onload: function(list_view) { - let method = "frappe.email.inbox.create_email_flag_queue" + onload: function (list_view) { + let method = "frappe.email.inbox.create_email_flag_queue"; - list_view.page.add_menu_item(__("Mark as Read"), function() { + list_view.page.add_menu_item(__("Mark as Read"), function () { list_view.call_for_selected_items(method, { action: "Read" }); }); - list_view.page.add_menu_item(__("Mark as Unread"), function() { + list_view.page.add_menu_item(__("Mark as Unread"), function () { list_view.call_for_selected_items(method, { action: "Unread" }); }); }, - primary_action: function() { + primary_action: function () { new frappe.views.CommunicationComposer(); - } + }, }; diff --git a/frappe/core/doctype/communication/email.py b/frappe/core/doctype/communication/email.py index 5737572194..c5585fd463 100755 --- a/frappe/core/doctype/communication/email.py +++ b/frappe/core/doctype/communication/email.py @@ -2,7 +2,7 @@ # License: MIT. See LICENSE import json -from typing import TYPE_CHECKING, Dict +from typing import TYPE_CHECKING import frappe import frappe.email.smtp @@ -12,6 +12,7 @@ from frappe.utils import ( cint, get_datetime, get_formatted_email, + get_string_between, list_to_str, split_emails, validate_email_address, @@ -21,14 +22,6 @@ if TYPE_CHECKING: from frappe.core.doctype.communication.communication import Communication -OUTGOING_EMAIL_ACCOUNT_MISSING = _( - """ - Unable to send mail because of a missing email account. - Please setup default Email Account from Setup > Email > Email Account -""" -) - - @frappe.whitelist() def make( doctype=None, @@ -52,7 +45,7 @@ def make( email_template=None, communication_type=None, **kwargs, -) -> Dict[str, str]: +) -> dict[str, str]: """Make a new communication. Checks for email permissions for specified Document. :param doctype: Reference DocType. @@ -129,7 +122,7 @@ def _make( email_template=None, communication_type=None, add_signature=True, -) -> Dict[str, str]: +) -> dict[str, str]: """Internal method to make a new communication that ignores Permission checks.""" sender = sender or get_formatted_email(frappe.session.user) @@ -152,7 +145,7 @@ def _make( "reference_doctype": doctype, "reference_name": name, "email_template": email_template, - "message_id": get_message_id().strip(" <>"), + "message_id": get_string_between("<", get_message_id(), ">"), "read_receipt": read_receipt, "has_attachment": 1 if attachments else 0, "communication_type": communication_type, @@ -169,7 +162,12 @@ def _make( if cint(send_email): if not comm.get_outgoing_email_account(): - frappe.throw(msg=OUTGOING_EMAIL_ACCOUNT_MISSING, exc=frappe.OutgoingEmailError) + frappe.throw( + _( + "Unable to send mail because of a missing email account. Please setup default Email Account from Setup > Email > Email Account" + ), + exc=frappe.OutgoingEmailError, + ) comm.send_email( print_html=print_html, @@ -248,7 +246,7 @@ def mark_email_as_seen(name: str = None): frappe.db.commit() # nosemgrep: this will be called in a GET request except Exception: - frappe.log_error(frappe.get_traceback()) + frappe.log_error("Unable to mark as seen", None, "Communication", name) finally: frappe.response.update( diff --git a/frappe/core/doctype/communication/mixins.py b/frappe/core/doctype/communication/mixins.py index 68abba3c13..bfadaf4f6c 100644 --- a/frappe/core/doctype/communication/mixins.py +++ b/frappe/core/doctype/communication/mixins.py @@ -1,5 +1,3 @@ -from typing import List - import frappe from frappe import _ from frappe.core.utils import get_parent_doc @@ -75,7 +73,8 @@ class CommunicationEmailMixin: if include_sender: cc.append(self.sender_mailid) if is_inbound_mail_communcation: - cc.append(self.get_owner()) + if (doc_owner := self.get_owner()) not in frappe.STANDARD_USERS: + cc.append(doc_owner) cc = set(cc) - {self.sender_mailid} cc.update(self.get_assignees()) @@ -94,7 +93,7 @@ class CommunicationEmailMixin: cc_list = self.mail_cc( is_inbound_mail_communcation=is_inbound_mail_communcation, include_sender=include_sender ) - return [self.get_email_with_displayname(email) for email in cc_list] + return [self.get_email_with_displayname(email) for email in cc_list if email] def mail_bcc(self, is_inbound_mail_communcation=False): """ @@ -122,7 +121,7 @@ class CommunicationEmailMixin: def get_mail_bcc_with_displayname(self, is_inbound_mail_communcation=False): bcc_list = self.mail_bcc(is_inbound_mail_communcation=is_inbound_mail_communcation) - return [self.get_email_with_displayname(email) for email in bcc_list] + return [self.get_email_with_displayname(email) for email in bcc_list if email] def mail_sender(self): email_account = self.get_outgoing_email_account() @@ -152,7 +151,7 @@ class CommunicationEmailMixin: "doctype": self.reference_doctype, "name": self.reference_name, "print_format": print_format, - "key": get_parent_doc(self).get_signature(), + "key": get_parent_doc(self).get_document_share_key(), } ) @@ -201,7 +200,7 @@ class CommunicationEmailMixin: return _("Leave this conversation") return "" - def exclude_emails_list(self, is_inbound_mail_communcation=False, include_sender=False) -> List: + def exclude_emails_list(self, is_inbound_mail_communcation=False, include_sender=False) -> list: """List of mail id's excluded while sending mail.""" all_ids = self.get_all_email_addresses(exclude_displayname=True) @@ -249,7 +248,7 @@ class CommunicationEmailMixin: send_me_a_copy=None, print_letterhead=None, is_inbound_mail_communcation=None, - ): + ) -> dict: outgoing_email_account = self.get_outgoing_email_account() if not outgoing_email_account: @@ -299,13 +298,11 @@ class CommunicationEmailMixin: print_letterhead=None, is_inbound_mail_communcation=None, ): - input_dict = self.sendmail_input_dict( + if input_dict := self.sendmail_input_dict( print_html=print_html, print_format=print_format, send_me_a_copy=send_me_a_copy, print_letterhead=print_letterhead, is_inbound_mail_communcation=is_inbound_mail_communcation, - ) - - if input_dict: + ): frappe.sendmail(**input_dict) diff --git a/frappe/core/doctype/communication/test_communication.py b/frappe/core/doctype/communication/test_communication.py index a338295374..085dd3fe60 100644 --- a/frappe/core/doctype/communication/test_communication.py +++ b/frappe/core/doctype/communication/test_communication.py @@ -1,16 +1,22 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors +# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE import unittest +from typing import TYPE_CHECKING from urllib.parse import quote import frappe -from frappe.core.doctype.communication.communication import get_emails +from frappe.core.doctype.communication.communication import Communication, get_emails from frappe.email.doctype.email_queue.email_queue import EmailQueue +from frappe.tests.utils import FrappeTestCase + +if TYPE_CHECKING: + from frappe.contacts.doctype.contact.contact import Contact + from frappe.email.doctype.email_account.email_account import EmailAccount test_records = frappe.get_test_records("Communication") -class TestCommunication(unittest.TestCase): +class TestCommunication(FrappeTestCase): def test_email(self): valid_email_list = [ "Full Name ", @@ -32,11 +38,13 @@ class TestCommunication(unittest.TestCase): "[invalid!email].com", ] - for x in valid_email_list: - self.assertTrue(frappe.utils.parse_addr(x)[1]) + for i, x in enumerate(valid_email_list): + with self.subTest(i=i, x=x): + self.assertTrue(frappe.utils.parse_addr(x)[1]) - for x in invalid_email_list: - self.assertFalse(frappe.utils.parse_addr(x)[0]) + for i, x in enumerate(invalid_email_list): + with self.subTest(i=i, x=x): + self.assertFalse(frappe.utils.parse_addr(x)[0]) def test_name(self): valid_email_list = [ @@ -129,7 +137,7 @@ class TestCommunication(unittest.TestCase): self.assertNotEqual(2, len(comm.timeline_links)) def test_contacts_attached(self): - contact_sender = frappe.get_doc( + contact_sender: "Contact" = frappe.get_doc( { "doctype": "Contact", "first_name": "contact_sender", @@ -138,7 +146,7 @@ class TestCommunication(unittest.TestCase): contact_sender.add_email("comm_sender@example.com") contact_sender.insert(ignore_permissions=True) - contact_recipient = frappe.get_doc( + contact_recipient: "Contact" = frappe.get_doc( { "doctype": "Contact", "first_name": "contact_recipient", @@ -147,7 +155,7 @@ class TestCommunication(unittest.TestCase): contact_recipient.add_email("comm_recipient@example.com") contact_recipient.insert(ignore_permissions=True) - contact_cc = frappe.get_doc( + contact_cc: "Contact" = frappe.get_doc( { "doctype": "Contact", "first_name": "contact_cc", @@ -156,7 +164,7 @@ class TestCommunication(unittest.TestCase): contact_cc.add_email("comm_cc@example.com") contact_cc.insert(ignore_permissions=True) - comm = frappe.get_doc( + comm: Communication = frappe.get_doc( { "doctype": "Communication", "communication_medium": "Email", @@ -168,10 +176,7 @@ class TestCommunication(unittest.TestCase): ).insert(ignore_permissions=True) comm = frappe.get_doc("Communication", comm.name) - - contact_links = [] - for timeline_link in comm.timeline_links: - contact_links.append(timeline_link.link_name) + contact_links = [x.link_name for x in comm.timeline_links] self.assertIn(contact_sender.name, contact_links) self.assertIn(contact_recipient.name, contact_links) @@ -210,10 +215,7 @@ class TestCommunication(unittest.TestCase): comms = get_communication_data("Note", note.name, as_dict=True) - data = [] - for comm in comms: - data.append(comm.name) - + data = [comm.name for comm in comms] self.assertIn(comm_note_1.name, data) self.assertIn(comm_note_2.name, data) @@ -236,14 +238,13 @@ class TestCommunication(unittest.TestCase): "communication_medium": "Email", "subject": "Document Link in Email", "sender": "comm_sender@example.com", - "recipients": "comm_recipient+{0}+{1}@example.com".format(quote("Note"), quote(note.name)), + "recipients": f'comm_recipient+{quote("Note")}+{quote(note.name)}@example.com', } ).insert(ignore_permissions=True) - doc_links = [] - for timeline_link in comm.timeline_links: - doc_links.append((timeline_link.link_doctype, timeline_link.link_name)) - + doc_links = [ + (timeline_link.link_doctype, timeline_link.link_name) for timeline_link in comm.timeline_links + ] self.assertIn(("Note", note.name), doc_links) def test_parse_emails(self): @@ -259,14 +260,46 @@ class TestCommunication(unittest.TestCase): self.assertEqual(emails[1], "first.lastname@email.com") self.assertEqual(emails[2], "test@user.com") + def test_signature_in_email_content(self): + email_account = create_email_account() + signature = email_account.signature + base_communication = { + "doctype": "Communication", + "communication_medium": "Email", + "subject": "Document Link in Email", + "sender": "comm_sender@example.com", + } + comm_with_signature = frappe.get_doc( + base_communication + | { + "content": f"""

+ Hi, + How are you? +


{signature}

""", + } + ).insert(ignore_permissions=True) + comm_without_signature = frappe.get_doc( + base_communication + | { + "content": """
+ Hi, + How are you? +
""" + } + ).insert(ignore_permissions=True) -class TestCommunicationEmailMixin(unittest.TestCase): - def new_communication(self, recipients=None, cc=None, bcc=None): + self.assertEqual(comm_with_signature.content, comm_without_signature.content) + self.assertEqual(comm_with_signature.content.count(signature), 1) + self.assertEqual(comm_without_signature.content.count(signature), 1) + + +class TestCommunicationEmailMixin(FrappeTestCase): + def new_communication(self, recipients=None, cc=None, bcc=None) -> Communication: recipients = ", ".join(recipients or []) cc = ", ".join(cc or []) bcc = ", ".join(bcc or []) - comm = frappe.get_doc( + return frappe.get_doc( { "doctype": "Communication", "communication_type": "Communication", @@ -277,7 +310,6 @@ class TestCommunicationEmailMixin(unittest.TestCase): "bcc": bcc, } ).insert(ignore_permissions=True) - return comm def new_user(self, email, **user_data): user_data.setdefault("first_name", "first_name") @@ -330,13 +362,13 @@ class TestCommunicationEmailMixin(unittest.TestCase): comm.delete() -def create_email_account(): +def create_email_account() -> "EmailAccount": frappe.delete_doc_if_exists("Email Account", "_Test Comm Account 1") frappe.flags.mute_emails = False frappe.flags.sent_mail = None - email_account = frappe.get_doc( + return frappe.get_doc( { "is_default": 1, "is_global": 1, @@ -345,6 +377,7 @@ def create_email_account(): "append_to": "ToDo", "email_account_name": "_Test Comm Account 1", "enable_outgoing": 1, + "default_outgoing": 1, "smtp_server": "test.example.com", "email_id": "test_comm@example.com", "password": "password", @@ -358,9 +391,6 @@ def create_email_account(): "send_notification_to": "test_comm@example.com", "pop3_server": "pop.test.example.com", "imap_folder": [{"folder_name": "INBOX", "append_to": "ToDo"}], - "no_remaining": "0", "enable_automatic_linking": 1, } ).insert(ignore_permissions=True) - - return email_account diff --git a/frappe/core/doctype/communication_link/communication_link.py b/frappe/core/doctype/communication_link/communication_link.py index 21b6a7828a..07633ad174 100644 --- a/frappe/core/doctype/communication_link/communication_link.py +++ b/frappe/core/doctype/communication_link/communication_link.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and contributors # License: MIT. See LICENSE diff --git a/frappe/core/doctype/custom_docperm/custom_docperm.js b/frappe/core/doctype/custom_docperm/custom_docperm.js index 1f04a638a1..0da50217d2 100644 --- a/frappe/core/doctype/custom_docperm/custom_docperm.js +++ b/frappe/core/doctype/custom_docperm/custom_docperm.js @@ -1,8 +1,6 @@ // Copyright (c) 2016, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Custom DocPerm', { - refresh: function(frm) { - - } +frappe.ui.form.on("Custom DocPerm", { + refresh: function (frm) {}, }); diff --git a/frappe/core/doctype/custom_docperm/custom_docperm.py b/frappe/core/doctype/custom_docperm/custom_docperm.py index 23815e9bf6..e24c765df4 100644 --- a/frappe/core/doctype/custom_docperm/custom_docperm.py +++ b/frappe/core/doctype/custom_docperm/custom_docperm.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and contributors # License: MIT. See LICENSE diff --git a/frappe/core/doctype/custom_docperm/test_custom_docperm.py b/frappe/core/doctype/custom_docperm/test_custom_docperm.py index e3a831bb8d..4aa04f0223 100644 --- a/frappe/core/doctype/custom_docperm/test_custom_docperm.py +++ b/frappe/core/doctype/custom_docperm/test_custom_docperm.py @@ -1,10 +1,7 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and Contributors # License: MIT. See LICENSE import unittest -import frappe - # test_records = frappe.get_test_records('Custom DocPerm') diff --git a/frappe/core/doctype/custom_role/custom_role.js b/frappe/core/doctype/custom_role/custom_role.js index 85302a48b7..16b86485ed 100644 --- a/frappe/core/doctype/custom_role/custom_role.js +++ b/frappe/core/doctype/custom_role/custom_role.js @@ -1,8 +1,6 @@ // Copyright (c) 2016, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Custom Role', { - refresh: function(frm) { - - } +frappe.ui.form.on("Custom Role", { + refresh: function (frm) {}, }); diff --git a/frappe/core/doctype/custom_role/custom_role.json b/frappe/core/doctype/custom_role/custom_role.json index 55af8e2acd..7504882caf 100644 --- a/frappe/core/doctype/custom_role/custom_role.json +++ b/frappe/core/doctype/custom_role/custom_role.json @@ -1,240 +1,76 @@ { - "allow_copy": 0, - "allow_guest_to_view": 0, + "actions": [], "allow_import": 1, - "allow_rename": 0, "autoname": "hash", - "beta": 0, "creation": "2017-02-13 14:53:36.240122", - "custom": 0, - "docstatus": 0, "doctype": "DocType", - "document_type": "", "editable_grid": 1, "engine": "InnoDB", + "field_order": [ + "page", + "report", + "permission_rules", + "roles", + "response", + "ref_doctype" + ], "fields": [ { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "page", "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, "label": "Page", - "length": 0, - "no_copy": 0, - "options": "Page", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "options": "Page" }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "report", "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, "label": "Report", - "length": 0, - "no_copy": 0, - "options": "Report", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "options": "Report" }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "permission_rules", "fieldtype": "Section Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Permission Rules", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "label": "Permission Rules" }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "roles", "fieldtype": "Table", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, "label": "Role", - "length": 0, - "no_copy": 0, - "options": "Has Role", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "options": "Has Role" }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "response", "fieldtype": "HTML", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "response", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "label": "response" }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "ref_doctype", "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Reference Document Type", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "label": "Reference Document Type" } ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2019-09-05 14:22:27.664645", + "links": [], + "modified": "2022-08-03 12:20:52.985554", "modified_by": "Administrator", "module": "Core", "name": "Custom Role", - "name_case": "", "owner": "Administrator", "permissions": [ { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, "create": 1, "delete": 1, "email": 1, "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, "print": 1, "read": 1, "report": 1, "role": "System Manager", - "set_user_permissions": 0, "share": 1, - "submit": 0, "write": 1 } ], - "quick_entry": 0, "read_only": 1, - "read_only_onload": 0, - "show_name_in_global_search": 0, "sort_field": "modified", "sort_order": "DESC", - "track_changes": 0, - "track_seen": 0 + "states": [] } \ No newline at end of file diff --git a/frappe/core/doctype/custom_role/custom_role.py b/frappe/core/doctype/custom_role/custom_role.py index dd215dea17..4d96def662 100644 --- a/frappe/core/doctype/custom_role/custom_role.py +++ b/frappe/core/doctype/custom_role/custom_role.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and contributors # License: MIT. See LICENSE diff --git a/frappe/core/doctype/custom_role/test_custom_role.py b/frappe/core/doctype/custom_role/test_custom_role.py index 01956ceda3..79c66255c0 100644 --- a/frappe/core/doctype/custom_role/test_custom_role.py +++ b/frappe/core/doctype/custom_role/test_custom_role.py @@ -1,10 +1,7 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and Contributors # License: MIT. See LICENSE import unittest -import frappe - # test_records = frappe.get_test_records('Custom Role') diff --git a/frappe/core/doctype/data_export/data_export.js b/frappe/core/doctype/data_export/data_export.js index f195e79119..8d65a209b5 100644 --- a/frappe/core/doctype/data_export/data_export.js +++ b/frappe/core/doctype/data_export/data_export.js @@ -1,64 +1,65 @@ // Copyright (c) 2018, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Data Export', { - refresh: frm => { +frappe.ui.form.on("Data Export", { + refresh: (frm) => { frm.disable_save(); - frm.page.set_primary_action('Export', () => { + frm.page.set_primary_action("Export", () => { can_export(frm) ? export_data(frm) : null; }); }, onload: (frm) => { frm.set_query("reference_doctype", () => { return { - "filters": { - "issingle": 0, - "istable": 0, - "name": ['in', frappe.boot.user.can_export] - } + filters: { + issingle: 0, + istable: 0, + name: ["in", frappe.boot.user.can_export], + }, }; }); }, - reference_doctype: frm => { + reference_doctype: (frm) => { const doctype = frm.doc.reference_doctype; if (doctype) { frappe.model.with_doctype(doctype, () => set_field_options(frm)); } else { reset_filter_and_field(frm); } - } + }, }); -const can_export = frm => { +const can_export = (frm) => { const doctype = frm.doc.reference_doctype; - const parent_multicheck_options = frm.fields_multicheck[doctype] ? - frm.fields_multicheck[doctype].get_checked_options() : []; + const parent_multicheck_options = frm.fields_multicheck[doctype] + ? frm.fields_multicheck[doctype].get_checked_options() + : []; let is_valid_form = false; if (!doctype) { - frappe.msgprint(__('Please select the Document Type.')); + frappe.msgprint(__("Please select the Document Type.")); } else if (!parent_multicheck_options.length) { - frappe.msgprint(__('Atleast one field of Parent Document Type is mandatory')); + frappe.msgprint(__("Atleast one field of Parent Document Type is mandatory")); } else { is_valid_form = true; } return is_valid_form; }; -const export_data = frm => { - let get_template_url = '/api/method/frappe.core.doctype.data_export.exporter.export_data'; +const export_data = (frm) => { + let get_template_url = "/api/method/frappe.core.doctype.data_export.exporter.export_data"; var export_params = () => { let columns = {}; - Object.keys(frm.fields_multicheck).forEach(dt => { + Object.keys(frm.fields_multicheck).forEach((dt) => { const options = frm.fields_multicheck[dt].get_checked_options(); columns[dt] = options; }); return { doctype: frm.doc.reference_doctype, select_columns: JSON.stringify(columns), - filters: frm.filter_list.get_filters().map(filter => filter.slice(1, 4)), + filters: frm.filter_list.get_filters().map((filter) => filter.slice(1, 4)), file_type: frm.doc.file_type, template: true, - with_data: 1 + with_data: 1, }; }; @@ -86,26 +87,24 @@ const set_field_options = (frm) => { frm.filter_list = new frappe.ui.FilterGroup({ parent: filter_wrapper, doctype: doctype, - on_change: () => { }, + on_change: () => {}, }); // Add 'Select All' and 'Unselect All' button make_multiselect_buttons(parent_wrapper); frm.fields_multicheck = {}; - related_doctypes.forEach(dt => { + related_doctypes.forEach((dt) => { frm.fields_multicheck[dt] = add_doctype_field_multicheck_control(dt, parent_wrapper); }); frm.refresh(); }; -const make_multiselect_buttons = parent_wrapper => { - const button_container = $(parent_wrapper) - .append('
') - .find('.flex'); +const make_multiselect_buttons = (parent_wrapper) => { + const button_container = $(parent_wrapper).append('
').find(".flex"); - ["Select All", "Unselect All"].map(d => { + ["Select All", "Unselect All"].map((d) => { frappe.ui.form.make_control({ parent: $(button_container), df: { @@ -113,59 +112,59 @@ const make_multiselect_buttons = parent_wrapper => { fieldname: frappe.scrub(d), fieldtype: "Button", click: () => { - checkbox_toggle(d !== 'Select All'); - } + checkbox_toggle(d !== "Select All"); + }, }, - render_input: true + render_input: true, }); }); - $(button_container).find('.frappe-control').map((index, button) => { - $(button).css({"margin-right": "1em"}); - }); + $(button_container) + .find(".frappe-control") + .map((index, button) => { + $(button).css({ "margin-right": "1em" }); + }); function checkbox_toggle(checked) { - $(parent_wrapper).find('[data-fieldtype="MultiCheck"]').map((index, element) => { - $(element).find(`:checkbox`).prop("checked", checked).trigger('click'); - }); + $(parent_wrapper) + .find('[data-fieldtype="MultiCheck"]') + .map((index, element) => { + $(element).find(`:checkbox`).prop("checked", checked).trigger("click"); + }); } - }; -const get_doctypes = parentdt => { - return [parentdt].concat( - frappe.meta.get_table_fields(parentdt).map(df => df.options) - ); +const get_doctypes = (parentdt) => { + return [parentdt].concat(frappe.meta.get_table_fields(parentdt).map((df) => df.options)); }; const add_doctype_field_multicheck_control = (doctype, parent_wrapper) => { const fields = get_fields(doctype); - const options = fields - .map(df => { - return { - label: df.label, - value: df.fieldname, - danger: df.reqd, - checked: 1 - }; - }); + const options = fields.map((df) => { + return { + label: df.label, + value: df.fieldname, + danger: df.reqd, + checked: 1, + }; + }); const multicheck_control = frappe.ui.form.make_control({ parent: parent_wrapper, df: { - "label": doctype, - "fieldname": doctype + '_fields', - "fieldtype": "MultiCheck", - "options": options, - "columns": 3, + label: doctype, + fieldname: doctype + "_fields", + fieldtype: "MultiCheck", + options: options, + columns: 3, }, - render_input: true + render_input: true, }); multicheck_control.refresh_input(); return multicheck_control; }; -const filter_fields = df => frappe.model.is_value_type(df) && !df.hidden; -const get_fields = dt => frappe.meta.get_docfields(dt).filter(filter_fields); \ No newline at end of file +const filter_fields = (df) => frappe.model.is_value_type(df) && !df.hidden; +const get_fields = (dt) => frappe.meta.get_docfields(dt).filter(filter_fields); diff --git a/frappe/core/doctype/data_export/data_export.json b/frappe/core/doctype/data_export/data_export.json index 8304430fdb..01a680503d 100644 --- a/frappe/core/doctype/data_export/data_export.json +++ b/frappe/core/doctype/data_export/data_export.json @@ -1,250 +1,76 @@ { - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, - "creation": "2018-03-07 10:09:49.794764", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", + "actions": [], + "creation": "2018-03-07 10:09:49.794764", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "reference_doctype", + "column_break_2", + "file_type", + "section_break", + "filter_list", + "fields_multicheck" + ], "fields": [ { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "reference_doctype", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Select Doctype", - "length": 0, - "no_copy": 0, - "options": "DocType", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "column_break_2", - "fieldtype": "Column Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "default": "CSV", - "fieldname": "file_type", - "fieldtype": "Select", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "File Type", - "length": 0, - "no_copy": 0, - "options": "Excel\nCSV", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "depends_on": "reference_doctype", - "fieldname": "section_break", - "fieldtype": "Section Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "filter_list", - "fieldtype": "HTML", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Filter List", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "fieldname": "reference_doctype", + "fieldtype": "Link", + "label": "Select Doctype", + "options": "DocType", + "reqd": 1 }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "fields_multicheck", - "fieldtype": "HTML", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Fields Multicheck", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "fieldname": "column_break_2", + "fieldtype": "Column Break" + }, + { + "default": "CSV", + "fieldname": "file_type", + "fieldtype": "Select", + "in_list_view": 1, + "label": "File Type", + "options": "Excel\nCSV", + "reqd": 1 + }, + { + "depends_on": "reference_doctype", + "fieldname": "section_break", + "fieldtype": "Section Break" + }, + { + "fieldname": "filter_list", + "fieldtype": "HTML", + "label": "Filter List" + }, + { + "fieldname": "fields_multicheck", + "fieldtype": "HTML", + "label": "Fields Multicheck" } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 1, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 1, - "istable": 0, - "max_attachments": 0, - "modified": "2018-03-21 13:23:05.623052", - "modified_by": "Administrator", - "module": "Core", - "name": "Data Export", - "name_case": "", - "owner": "Administrator", + ], + "hide_toolbar": 1, + "issingle": 1, + "links": [], + "modified": "2022-08-03 12:20:53.658574", + "modified_by": "Administrator", + "module": "Core", + "name": "Data Export", + "owner": "Administrator", "permissions": [ { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 1, - "delete": 1, - "email": 1, - "export": 0, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 0, - "role": "System Manager", - "set_user_permissions": 0, - "share": 1, - "submit": 0, + "create": 1, + "delete": 1, + "email": 1, + "print": 1, + "read": 1, + "role": "System Manager", + "share": 1, "write": 1 } - ], - "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 0, - "track_seen": 0 + ], + "quick_entry": 1, + "sort_field": "modified", + "sort_order": "DESC", + "states": [] } \ No newline at end of file diff --git a/frappe/core/doctype/data_export/data_export.py b/frappe/core/doctype/data_export/data_export.py index 268182fbd4..b27966aa3d 100644 --- a/frappe/core/doctype/data_export/data_export.py +++ b/frappe/core/doctype/data_export/data_export.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors # License: MIT. See LICENSE diff --git a/frappe/core/doctype/data_export/exporter.py b/frappe/core/doctype/data_export/exporter.py index 514be16694..b7f69ab43d 100644 --- a/frappe/core/doctype/data_export/exporter.py +++ b/frappe/core/doctype/data_export/exporter.py @@ -331,7 +331,7 @@ class DataExporter: order_by = None table_columns = frappe.db.get_table_columns(self.parent_doctype) if "lft" in table_columns and "rgt" in table_columns: - order_by = "`tab{doctype}`.`lft` asc".format(doctype=self.parent_doctype) + order_by = f"`tab{self.parent_doctype}`.`lft` asc" # get permitted data only self.data = frappe.get_list( self.doctype, fields=["*"], filters=self.filters, limit_page_length=None, order_by=order_by diff --git a/frappe/core/doctype/data_export/test_data_exporter.py b/frappe/core/doctype/data_export/test_data_exporter.py index 7b7dfc0069..812f65aaad 100644 --- a/frappe/core/doctype/data_export/test_data_exporter.py +++ b/frappe/core/doctype/data_export/test_data_exporter.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # License: MIT. See LICENSE import unittest diff --git a/frappe/core/doctype/data_import/data_import.js b/frappe/core/doctype/data_import/data_import.js index dfc560a98a..7db3aa9629 100644 --- a/frappe/core/doctype/data_import/data_import.js +++ b/frappe/core/doctype/data_import/data_import.js @@ -1,17 +1,17 @@ // Copyright (c) 2019, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Data Import', { +frappe.ui.form.on("Data Import", { setup(frm) { - frappe.realtime.on('data_import_refresh', ({ data_import }) => { + frappe.realtime.on("data_import_refresh", ({ data_import }) => { frm.import_in_progress = false; if (data_import !== frm.doc.name) return; - frappe.model.clear_doc('Data Import', frm.doc.name); - frappe.model.with_doc('Data Import', frm.doc.name).then(() => { + frappe.model.clear_doc("Data Import", frm.doc.name); + frappe.model.with_doc("Data Import", frm.doc.name).then(() => { frm.refresh(); }); }); - frappe.realtime.on('data_import_progress', data => { + frappe.realtime.on("data_import_progress", (data) => { frm.import_in_progress = true; if (data.data_import !== frm.doc.name) { return; @@ -31,20 +31,16 @@ frappe.ui.form.on('Data Import', { if (data.success) { let message_args = [data.current, data.total, eta_message]; message = - frm.doc.import_type === 'Insert New Records' - ? __('Importing {0} of {1}, {2}', message_args) - : __('Updating {0} of {1}, {2}', message_args); + frm.doc.import_type === "Insert New Records" + ? __("Importing {0} of {1}, {2}", message_args) + : __("Updating {0} of {1}, {2}", message_args); } if (data.skipping) { - message = __('Skipping {0} of {1}, {2}', [ - data.current, - data.total, - eta_message - ]); + message = __("Skipping {0} of {1}, {2}", [data.current, data.total, eta_message]); } - frm.dashboard.show_progress(__('Import Progress'), percent, message); - frm.page.set_indicator(__('In Progress'), 'orange'); - frm.trigger('update_primary_action'); + frm.dashboard.show_progress(__("Import Progress"), percent, message); + frm.page.set_indicator(__("In Progress"), "orange"); + frm.trigger("update_primary_action"); // hide progress when complete if (data.current === data.total) { @@ -55,18 +51,18 @@ frappe.ui.form.on('Data Import', { } }); - frm.set_query('reference_doctype', () => { + frm.set_query("reference_doctype", () => { return { filters: { - name: ['in', frappe.boot.user.can_import] - } + name: ["in", frappe.boot.user.can_import], + }, }; }); - frm.get_field('import_file').df.options = { + frm.get_field("import_file").df.options = { restrictions: { - allowed_file_types: ['.csv', '.xls', '.xlsx'] - } + allowed_file_types: [".csv", ".xls", ".xlsx"], + }, }; frm.has_import_file = () => { @@ -76,33 +72,31 @@ frappe.ui.form.on('Data Import', { refresh(frm) { frm.page.hide_icon_group(); - frm.trigger('update_indicators'); - frm.trigger('import_file'); - frm.trigger('show_import_log'); - frm.trigger('show_import_warnings'); - frm.trigger('toggle_submit_after_import'); + frm.trigger("update_indicators"); + frm.trigger("import_file"); + frm.trigger("show_import_log"); + frm.trigger("show_import_warnings"); + frm.trigger("toggle_submit_after_import"); - if (frm.doc.status != 'Pending') - frm.trigger('show_import_status'); + if (frm.doc.status != "Pending") frm.trigger("show_import_status"); - frm.trigger('show_report_error_button'); + frm.trigger("show_report_error_button"); - if (frm.doc.status === 'Partial Success') { - frm.add_custom_button(__('Export Errored Rows'), () => - frm.trigger('export_errored_rows') + if (frm.doc.status === "Partial Success") { + frm.add_custom_button(__("Export Errored Rows"), () => + frm.trigger("export_errored_rows") ); } - if (frm.doc.status.includes('Success')) { - frm.add_custom_button( - __('Go to {0} List', [__(frm.doc.reference_doctype)]), - () => frappe.set_route('List', frm.doc.reference_doctype) + if (frm.doc.status.includes("Success")) { + frm.add_custom_button(__("Go to {0} List", [__(frm.doc.reference_doctype)]), () => + frappe.set_route("List", frm.doc.reference_doctype) ); } }, onload_post_render(frm) { - frm.trigger('update_primary_action'); + frm.trigger("update_primary_action"); }, update_primary_action(frm) { @@ -111,13 +105,12 @@ frappe.ui.form.on('Data Import', { return; } frm.disable_save(); - if (frm.doc.status !== 'Success') { - if (!frm.is_new() && (frm.has_import_file())) { - let label = - frm.doc.status === 'Pending' ? __('Start Import') : __('Retry'); + if (frm.doc.status !== "Success") { + if (!frm.is_new() && frm.has_import_file()) { + let label = frm.doc.status === "Pending" ? __("Start Import") : __("Retry"); frm.page.set_primary_action(label, () => frm.events.start_import(frm)); } else { - frm.page.set_primary_action(__('Save'), () => frm.save()); + frm.page.set_primary_action(__("Save"), () => frm.save()); } } }, @@ -133,11 +126,11 @@ frappe.ui.form.on('Data Import', { show_import_status(frm) { frappe.call({ - 'method': 'frappe.core.doctype.data_import.data_import.get_import_status', - 'args': { - 'data_import_name': frm.doc.name + method: "frappe.core.doctype.data_import.data_import.get_import_status", + args: { + data_import_name: frm.doc.name, }, - 'callback': function(r) { + callback: function (r) { let successful_records = cint(r.message.success); let failed_records = cint(r.message.failed); let total_records = cint(r.message.total_records); @@ -147,52 +140,64 @@ frappe.ui.form.on('Data Import', { let message; if (failed_records === 0) { let message_args = [successful_records]; - if (frm.doc.import_type === 'Insert New Records') { + if (frm.doc.import_type === "Insert New Records") { message = successful_records > 1 - ? __('Successfully imported {0} records.', message_args) - : __('Successfully imported {0} record.', message_args); + ? __("Successfully imported {0} records.", message_args) + : __("Successfully imported {0} record.", message_args); } else { message = successful_records > 1 - ? __('Successfully updated {0} records.', message_args) - : __('Successfully updated {0} record.', message_args); + ? __("Successfully updated {0} records.", message_args) + : __("Successfully updated {0} record.", message_args); } } else { let message_args = [successful_records, total_records]; - if (frm.doc.import_type === 'Insert New Records') { + if (frm.doc.import_type === "Insert New Records") { message = successful_records > 1 - ? __('Successfully imported {0} records out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args) - : __('Successfully imported {0} record out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args); + ? __( + "Successfully imported {0} records out of {1}. Click on Export Errored Rows, fix the errors and import again.", + message_args + ) + : __( + "Successfully imported {0} record out of {1}. Click on Export Errored Rows, fix the errors and import again.", + message_args + ); } else { message = successful_records > 1 - ? __('Successfully updated {0} records out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args) - : __('Successfully updated {0} record out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args); + ? __( + "Successfully updated {0} records out of {1}. Click on Export Errored Rows, fix the errors and import again.", + message_args + ) + : __( + "Successfully updated {0} record out of {1}. Click on Export Errored Rows, fix the errors and import again.", + message_args + ); } } frm.dashboard.set_headline(message); - } + }, }); }, show_report_error_button(frm) { - if (frm.doc.status === 'Error') { + if (frm.doc.status === "Error") { frappe.db - .get_list('Error Log', { + .get_list("Error Log", { filters: { method: frm.doc.name }, - fields: ['method', 'error'], - order_by: 'creation desc', - limit: 1 + fields: ["method", "error"], + order_by: "creation desc", + limit: 1, }) - .then(result => { + .then((result) => { if (result.length > 0) { - frm.add_custom_button('Report Error', () => { + frm.add_custom_button("Report Error", () => { let fake_xhr = { responseText: JSON.stringify({ - exc: result[0].error - }) + exc: result[0].error, + }), }; frappe.request.report_error(fake_xhr, {}); }); @@ -202,21 +207,19 @@ frappe.ui.form.on('Data Import', { }, start_import(frm) { - frm - .call({ - method: 'form_start_import', - args: { data_import: frm.doc.name }, - btn: frm.page.btn_primary - }) - .then(r => { - if (r.message === true) { - frm.disable_save(); - } - }); + frm.call({ + method: "form_start_import", + args: { data_import: frm.doc.name }, + btn: frm.page.btn_primary, + }).then((r) => { + if (r.message === true) { + frm.disable_save(); + } + }); }, download_template(frm) { - frappe.require('data_import_tools.bundle.js', () => { + frappe.require("data_import_tools.bundle.js", () => { frm.data_exporter = new frappe.data_import.DataExporter( frm.doc.reference_doctype, frm.doc.import_type @@ -225,127 +228,123 @@ frappe.ui.form.on('Data Import', { }, reference_doctype(frm) { - frm.trigger('toggle_submit_after_import'); + frm.trigger("toggle_submit_after_import"); }, toggle_submit_after_import(frm) { - frm.toggle_display('submit_after_import', false); + frm.toggle_display("submit_after_import", false); let doctype = frm.doc.reference_doctype; if (doctype) { frappe.model.with_doctype(doctype, () => { let meta = frappe.get_meta(doctype); - frm.toggle_display('submit_after_import', meta.is_submittable); + frm.toggle_display("submit_after_import", meta.is_submittable); }); } }, google_sheets_url(frm) { if (!frm.is_dirty()) { - frm.trigger('import_file'); + frm.trigger("import_file"); } else { - frm.trigger('update_primary_action'); + frm.trigger("update_primary_action"); } }, refresh_google_sheet(frm) { - frm.trigger('import_file'); + frm.trigger("import_file"); }, import_file(frm) { - frm.toggle_display('section_import_preview', frm.has_import_file()); + frm.toggle_display("section_import_preview", frm.has_import_file()); if (!frm.has_import_file()) { - frm.get_field('import_preview').$wrapper.empty(); + frm.get_field("import_preview").$wrapper.empty(); return; } else { - frm.trigger('update_primary_action'); + frm.trigger("update_primary_action"); } // load import preview - frm.get_field('import_preview').$wrapper.empty(); + frm.get_field("import_preview").$wrapper.empty(); $('') - .html(__('Loading import file...')) - .appendTo(frm.get_field('import_preview').$wrapper); + .html(__("Loading import file...")) + .appendTo(frm.get_field("import_preview").$wrapper); - frm - .call({ - method: 'get_preview_from_template', - args: { - data_import: frm.doc.name, - import_file: frm.doc.import_file, - google_sheets_url: frm.doc.google_sheets_url + frm.call({ + method: "get_preview_from_template", + args: { + data_import: frm.doc.name, + import_file: frm.doc.import_file, + google_sheets_url: frm.doc.google_sheets_url, + }, + error_handlers: { + TimestampMismatchError() { + // ignore this error }, - error_handlers: { - TimestampMismatchError() { - // ignore this error - } - } - }) - .then(r => { - let preview_data = r.message; - frm.events.show_import_preview(frm, preview_data); - frm.events.show_import_warnings(frm, preview_data); - }); + }, + }).then((r) => { + let preview_data = r.message; + frm.events.show_import_preview(frm, preview_data); + frm.events.show_import_warnings(frm, preview_data); + }); }, show_import_preview(frm, preview_data) { let import_log = preview_data.import_log; - if ( - frm.import_preview && - frm.import_preview.doctype === frm.doc.reference_doctype - ) { + if (frm.import_preview && frm.import_preview.doctype === frm.doc.reference_doctype) { frm.import_preview.preview_data = preview_data; frm.import_preview.import_log = import_log; frm.import_preview.refresh(); return; } - frappe.require('data_import_tools.bundle.js', () => { + frappe.require("data_import_tools.bundle.js", () => { frm.import_preview = new frappe.data_import.ImportPreview({ - wrapper: frm.get_field('import_preview').$wrapper, + wrapper: frm.get_field("import_preview").$wrapper, doctype: frm.doc.reference_doctype, preview_data, import_log, frm, events: { remap_column(changed_map) { - let template_options = JSON.parse(frm.doc.template_options || '{}'); - template_options.column_to_field_map = template_options.column_to_field_map || {}; + let template_options = JSON.parse(frm.doc.template_options || "{}"); + template_options.column_to_field_map = + template_options.column_to_field_map || {}; Object.assign(template_options.column_to_field_map, changed_map); - frm.set_value('template_options', JSON.stringify(template_options)); - frm.save().then(() => frm.trigger('import_file')); - } - } + frm.set_value("template_options", JSON.stringify(template_options)); + frm.save().then(() => frm.trigger("import_file")); + }, + }, }); }); }, export_errored_rows(frm) { open_url_post( - '/api/method/frappe.core.doctype.data_import.data_import.download_errored_template', + "/api/method/frappe.core.doctype.data_import.data_import.download_errored_template", { - data_import_name: frm.doc.name + data_import_name: frm.doc.name, } ); }, export_import_log(frm) { open_url_post( - '/api/method/frappe.core.doctype.data_import.data_import.download_import_log', + "/api/method/frappe.core.doctype.data_import.data_import.download_import_log", { - data_import_name: frm.doc.name + data_import_name: frm.doc.name, } ); }, show_import_warnings(frm, preview_data) { let columns = preview_data.columns; - let warnings = JSON.parse(frm.doc.template_warnings || '[]'); + let warnings = JSON.parse(frm.doc.template_warnings || "[]"); warnings = warnings.concat(preview_data.warnings || []); - frm.toggle_display('import_warnings_section', warnings.length > 0); + frm.toggle_display("import_warnings_section", warnings.length > 0); if (warnings.length === 0) { - frm.get_field('import_warnings').$wrapper.html(''); + frm.get_field("import_warnings").$wrapper.html(""); return; } @@ -361,36 +360,38 @@ frappe.ui.form.on('Data Import', { } } - let html = ''; + let html = ""; html += Object.keys(warnings_by_row) - .map(row_number => { + .map((row_number) => { let message = warnings_by_row[row_number] - .map(w => { + .map((w) => { if (w.field) { let label = w.field.label + (w.field.parent !== frm.doc.reference_doctype ? ` (${w.field.parent})` - : ''); + : ""); return `
  • ${label}: ${w.message}
    • `; } return `
  • ${w.message}
    • `; }) - .join(''); + .join(""); return `
    -
    ${__('Row {0}', [row_number])}
    +
    ${__("Row {0}", [row_number])}
      ${message}
    `; }) - .join(''); + .join(""); html += other_warnings - .map(warning => { - let header = ''; + .map((warning) => { + let header = ""; if (warning.col) { - let column_number = `${__('Column {0}', [warning.col])}`; + let column_number = `${__("Column {0}", [ + warning.col, + ])}`; let column_header = columns[warning.col].header_title; header = `${column_number} (${column_header})`; } @@ -401,8 +402,8 @@ frappe.ui.form.on('Data Import', {
    `; }) - .join(''); - frm.get_field('import_warnings').$wrapper.html(` + .join(""); + frm.get_field("import_warnings").$wrapper.html(`
    ${html}
    @@ -410,62 +411,62 @@ frappe.ui.form.on('Data Import', { }, show_failed_logs(frm) { - frm.trigger('show_import_log'); + frm.trigger("show_import_log"); }, render_import_log(frm) { frappe.call({ - 'method': 'frappe.client.get_list', - 'args': { - 'doctype': 'Data Import Log', - 'filters': { - 'data_import': frm.doc.name + method: "frappe.client.get_list", + args: { + doctype: "Data Import Log", + filters: { + data_import: frm.doc.name, }, - 'fields': ['success', 'docname', 'messages', 'exception', 'row_indexes'], - 'limit_page_length': 5000, - 'order_by': 'log_index' + fields: ["success", "docname", "messages", "exception", "row_indexes"], + limit_page_length: 5000, + order_by: "log_index", }, - callback: function(r) { + callback: function (r) { let logs = r.message; if (logs.length === 0) return; - frm.toggle_display('import_log_section', true); + frm.toggle_display("import_log_section", true); let rows = logs - .map(log => { - let html = ''; + .map((log) => { + let html = ""; if (log.success) { - if (frm.doc.import_type === 'Insert New Records') { - html = __('Successfully imported {0}', [ + if (frm.doc.import_type === "Insert New Records") { + html = __("Successfully imported {0}", [ `${frappe.utils.get_form_link( frm.doc.reference_doctype, log.docname, true - )}` + )}`, ]); } else { - html = __('Successfully updated {0}', [ + html = __("Successfully updated {0}", [ `${frappe.utils.get_form_link( frm.doc.reference_doctype, log.docname, true - )}` + )}`, ]); } } else { - let messages = (JSON.parse(log.messages || '[]')) + let messages = JSON.parse(log.messages || "[]") .map(JSON.parse) - .map(m => { - let title = m.title ? `${m.title}` : ''; - let message = m.message ? `
    ${m.message}
    ` : ''; + .map((m) => { + let title = m.title ? `${m.title}` : ""; + let message = m.message ? `
    ${m.message}
    ` : ""; return title + message; }) - .join(''); + .join(""); let id = frappe.dom.get_unique_id(); html = `${messages}
    @@ -473,15 +474,15 @@ frappe.ui.form.on('Data Import', {
    `; } - let indicator_color = log.success ? 'green' : 'red'; - let title = log.success ? __('Success') : __('Failure'); + let indicator_color = log.success ? "green" : "red"; + let title = log.success ? __("Success") : __("Failure"); if (frm.doc.show_failed_logs && log.success) { - return ''; + return ""; } return ` - ${JSON.parse(log.row_indexes).join(', ')} + ${JSON.parse(log.row_indexes).join(", ")}
    ${title}
    @@ -490,54 +491,54 @@ frappe.ui.form.on('Data Import', { `; }) - .join(''); + .join(""); if (!rows && frm.doc.show_failed_logs) { rows = ` - ${__('No failed logs')} + ${__("No failed logs")} `; } - frm.get_field('import_log_preview').$wrapper.html(` + frm.get_field("import_log_preview").$wrapper.html(` - - - + + + ${rows}
    ${__('Row Number')}${__('Status')}${__('Message')}${__("Row Number")}${__("Status")}${__("Message")}
    `); - } + }, }); }, show_import_log(frm) { - frm.toggle_display('import_log_section', false); + frm.toggle_display("import_log_section", false); if (frm.import_in_progress) { return; } frappe.call({ - 'method': 'frappe.client.get_count', - 'args': { - 'doctype': 'Data Import Log', - 'filters': { - 'data_import': frm.doc.name - } + method: "frappe.client.get_count", + args: { + doctype: "Data Import Log", + filters: { + data_import: frm.doc.name, + }, }, - 'callback': function(r) { + callback: function (r) { let count = r.message; if (count < 5000) { - frm.trigger('render_import_log'); + frm.trigger("render_import_log"); } else { - frm.toggle_display('import_log_section', false); - frm.add_custom_button(__('Export Import Log'), () => - frm.trigger('export_import_log') + frm.toggle_display("import_log_section", false); + frm.add_custom_button(__("Export Import Log"), () => + frm.trigger("export_import_log") ); } - } + }, }); }, }); diff --git a/frappe/core/doctype/data_import/data_import.py b/frappe/core/doctype/data_import/data_import.py index 295f7e79ba..5ad603e416 100644 --- a/frappe/core/doctype/data_import/data_import.py +++ b/frappe/core/doctype/data_import/data_import.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and contributors # License: MIT. See LICENSE @@ -113,7 +112,7 @@ def start_import(data_import): except Exception: frappe.db.rollback() data_import.db_set("status", "Error") - frappe.log_error(title=data_import.name) + data_import.log_error("Data import failed") finally: frappe.flags.in_import = False diff --git a/frappe/core/doctype/data_import/data_import_list.js b/frappe/core/doctype/data_import/data_import_list.js index 6ab750ba25..c054655e62 100644 --- a/frappe/core/doctype/data_import/data_import_list.js +++ b/frappe/core/doctype/data_import/data_import_list.js @@ -1,46 +1,44 @@ let imports_in_progress = []; -frappe.listview_settings['Data Import'] = { +frappe.listview_settings["Data Import"] = { onload(listview) { - frappe.realtime.on('data_import_progress', data => { + frappe.realtime.on("data_import_progress", (data) => { if (!imports_in_progress.includes(data.data_import)) { imports_in_progress.push(data.data_import); } }); - frappe.realtime.on('data_import_refresh', data => { - imports_in_progress = imports_in_progress.filter( - d => d !== data.data_import - ); + frappe.realtime.on("data_import_refresh", (data) => { + imports_in_progress = imports_in_progress.filter((d) => d !== data.data_import); listview.refresh(); }); }, - get_indicator: function(doc) { + get_indicator: function (doc) { var colors = { - 'Pending': 'orange', - 'Not Started': 'orange', - 'Partial Success': 'orange', - 'Success': 'green', - 'In Progress': 'orange', - 'Error': 'red' + Pending: "orange", + "Not Started": "orange", + "Partial Success": "orange", + Success: "green", + "In Progress": "orange", + Error: "red", }; let status = doc.status; if (imports_in_progress.includes(doc.name)) { - status = 'In Progress'; + status = "In Progress"; } - if (status == 'Pending') { - status = 'Not Started'; + if (status == "Pending") { + status = "Not Started"; } - return [__(status), colors[status], 'status,=,' + doc.status]; + return [__(status), colors[status], "status,=," + doc.status]; }, formatters: { import_type(value) { return { - 'Insert New Records': __('Insert'), - 'Update Existing Records': __('Update') + "Insert New Records": __("Insert"), + "Update Existing Records": __("Update"), }[value]; - } + }, }, - hide_name_column: true + hide_name_column: true, }; diff --git a/frappe/core/doctype/data_import/exporter.py b/frappe/core/doctype/data_import/exporter.py index b647bdcb62..8c73391bd0 100644 --- a/frappe/core/doctype/data_import/exporter.py +++ b/frappe/core/doctype/data_import/exporter.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE @@ -132,8 +131,7 @@ class Exporter: child_doctype = table_df.options rows = self.add_data_row(child_doctype, child_row.parentfield, child_row, rows, i) - for row in rows: - yield row + yield from rows def add_data_row(self, doctype, parentfield, doc, rows, row_idx): if len(rows) < row_idx + 1: @@ -156,14 +154,14 @@ class Exporter: def get_data_as_docs(self): def format_column_name(df): - return "`tab{0}`.`{1}`".format(df.parent, df.fieldname) + return f"`tab{df.parent}`.`{df.fieldname}`" filters = self.export_filters if self.meta.is_nested_set(): - order_by = "`tab{0}`.`lft` ASC".format(self.doctype) + order_by = f"`tab{self.doctype}`.`lft` ASC" else: - order_by = "`tab{0}`.`creation` DESC".format(self.doctype) + order_by = f"`tab{self.doctype}`.`creation` DESC" parent_fields = [format_column_name(df) for df in self.fields if df.parent == self.doctype] parent_data = frappe.db.get_list( @@ -183,7 +181,7 @@ class Exporter: child_table_df = self.meta.get_field(key) child_table_doctype = child_table_df.options child_fields = ["name", "idx", "parent", "parentfield"] + list( - set([format_column_name(df) for df in self.fields if df.parent == child_table_doctype]) + {format_column_name(df) for df in self.fields if df.parent == child_table_doctype} ) data = frappe.db.get_all( child_table_doctype, @@ -211,16 +209,16 @@ class Exporter: if is_parent: label = _(df.label) else: - label = "{0} ({1})".format(_(df.label), _(df.child_table_df.label)) + label = f"{_(df.label)} ({_(df.child_table_df.label)})" if label in header: # this label is already in the header, # which means two fields with the same label # add the fieldname to avoid clash if is_parent: - label = "{0}".format(df.fieldname) + label = f"{df.fieldname}" else: - label = "{0}.{1}".format(df.child_table_df.fieldname, df.fieldname) + label = f"{df.child_table_df.fieldname}.{df.fieldname}" header.append(label) @@ -253,5 +251,5 @@ class Exporter: def build_xlsx_response(self): build_xlsx_response(self.get_csv_array_for_export(), _(self.doctype)) - def group_children_data_by_parent(self, children_data: typing.Dict[str, list]): + def group_children_data_by_parent(self, children_data: dict[str, list]): return groupby_metric(children_data, key="parent") diff --git a/frappe/core/doctype/data_import/importer.py b/frappe/core/doctype/data_import/importer.py index 9a801cfc19..25136db74b 100644 --- a/frappe/core/doctype/data_import/importer.py +++ b/frappe/core/doctype/data_import/importer.py @@ -4,14 +4,14 @@ import io import json import os +import re import timeit -from datetime import date, datetime +from datetime import date, datetime, time import frappe from frappe import _ from frappe.core.doctype.version.version import get_diff from frappe.model import no_value_fields -from frappe.model import table_fields as table_fieldtypes from frappe.utils import cint, cstr, duration_to_seconds, flt, update_progress_bar from frappe.utils.csvutils import get_csv_content_from_google_sheets, read_csv_content from frappe.utils.xlsxutils import ( @@ -23,6 +23,7 @@ INVALID_VALUES = ("", None) MAX_ROWS_IN_PREVIEW = 10 INSERT = "Insert New Records" UPDATE = "Update Existing Records" +DURATION_PATTERN = re.compile(r"^(?:(\d+d)?((^|\s)\d+h)?((^|\s)\d+m)?((^|\s)\d+s)?)$") class Importer: @@ -149,7 +150,7 @@ class Importer: if self.console: update_progress_bar( - "Importing {0} records".format(total_payload_count), + f"Importing {total_payload_count} records", current_index, total_payload_count, ) @@ -341,7 +342,7 @@ class Importer: row_number = json.loads(log.get("row_indexes"))[0] status = "Success" if log.get("success") else "Failure" message = ( - "Successfully Imported {0}".format(log.get("docname")) + "Successfully Imported {}".format(log.get("docname")) if log.get("success") else log.get("messages") ) @@ -356,19 +357,17 @@ class Importer: if successful_records: print() - print( - "Successfully imported {0} records out of {1}".format(len(successful_records), len(import_log)) - ) + print(f"Successfully imported {len(successful_records)} records out of {len(import_log)}") if failed_records: - print("Failed to import {0} records".format(len(failed_records))) - file_name = "{0}_import_on_{1}.txt".format(self.doctype, frappe.utils.now()) - print("Check {0} for errors".format(os.path.join("sites", file_name))) + print(f"Failed to import {len(failed_records)} records") + file_name = f"{self.doctype}_import_on_{frappe.utils.now()}.txt" + print("Check {} for errors".format(os.path.join("sites", file_name))) text = "" for w in failed_records: - text += "Row Indexes: {0}\n".format(str(w.get("row_indexes", []))) - text += "Messages:\n{0}\n".format("\n".join(w.get("messages", []))) - text += "Traceback:\n{0}\n\n".format(w.get("exception")) + text += "Row Indexes: {}\n".format(str(w.get("row_indexes", []))) + text += "Messages:\n{}\n".format("\n".join(w.get("messages", []))) + text += "Traceback:\n{}\n\n".format(w.get("exception")) with open(file_name, "w") as f: f.write(text) @@ -383,7 +382,7 @@ class Importer: other_warnings.append(w) for row_number, warnings in warnings_by_row.items(): - print("Row {0}".format(row_number)) + print(f"Row {row_number}") for w in warnings: print(w.get("message")) @@ -574,10 +573,10 @@ class ImportFile: ###### def read_file(self, file_path): - extn = file_path.split(".")[1] + extn = os.path.splitext(file_path)[1][1:] file_content = None - with io.open(file_path, mode="rb") as f: + with open(file_path, mode="rb") as f: file_content = f.read() return file_content, extn @@ -726,10 +725,7 @@ class Row: ) return elif df.fieldtype == "Duration": - import re - - is_valid_duration = re.match(r"^(?:(\d+d)?((^|\s)\d+h)?((^|\s)\d+m)?((^|\s)\d+s)?)$", value) - if not is_valid_duration: + if not DURATION_PATTERN.match(value): self.warnings.append( { "row": self.row_number, @@ -941,11 +937,13 @@ class Column: """ def guess_date_format(d): - if isinstance(d, (datetime, date)): + if isinstance(d, (datetime, date, time)): if self.df.fieldtype == "Date": return "%Y-%m-%d" if self.df.fieldtype == "Datetime": return "%Y-%m-%d %H:%M:%S" + if self.df.fieldtype == "Time": + return "%H:%M:%S" if isinstance(d, str): return frappe.utils.guess_date_format(d) @@ -993,23 +991,27 @@ class Column: self.warnings.append( { "col": self.column_number, - "message": ( - "The following values do not exist for {}: {}".format(self.df.options, missing_values) - ), + "message": (f"The following values do not exist for {self.df.options}: {missing_values}"), "type": "warning", } ) elif self.df.fieldtype in ("Date", "Time", "Datetime"): - # guess date format + # guess date/time format self.date_format = self.guess_date_format_for_column() if not self.date_format: - self.date_format = "%Y-%m-%d" + if self.df.fieldtype == "Time": + self.date_format = "%H:%M:%S" + format = "HH:mm:ss" + else: + self.date_format = "%Y-%m-%d" + format = "yyyy-mm-dd" + self.warnings.append( { "col": self.column_number, "message": _( - "Date format could not be determined from the values in this column. Defaulting to yyyy-mm-dd." - ), + "{0} format could not be determined from the values in this column. Defaulting to {1}." + ).format(self.df.fieldtype, format), "type": "info", } ) @@ -1025,8 +1027,8 @@ class Column: { "col": self.column_number, "message": ( - "The following values are invalid: {0}. Values must be" - " one of {1}".format(invalid_values, valid_values) + "The following values are invalid: {}. Values must be" + " one of {}".format(invalid_values, valid_values) ), } ) @@ -1112,9 +1114,9 @@ def build_fields_dict_for_column_matching(parent_doctype): ) else: name_headers = ( - "{0}.name".format(table_df.fieldname), # fieldname - "ID ({0})".format(table_df.label), # label - "{0} ({1})".format(_("ID"), translated_table_label), # translated label + f"{table_df.fieldname}.name", # fieldname + f"ID ({table_df.label})", # label + "{} ({})".format(_("ID"), translated_table_label), # translated label ) name_df.is_child_table_field = True @@ -1166,11 +1168,11 @@ def build_fields_dict_for_column_matching(parent_doctype): for header in ( # fieldname - "{0}.{1}".format(table_df.fieldname, df.fieldname), + f"{table_df.fieldname}.{df.fieldname}", # label - "{0} ({1})".format(label, table_df.label), + f"{label} ({table_df.label})", # translated label - "{0} ({1})".format(translated_label, translated_table_label), + f"{translated_label} ({translated_table_label})", ): out[header] = new_df @@ -1179,8 +1181,8 @@ def build_fields_dict_for_column_matching(parent_doctype): autoname_field = get_autoname_field(parent_doctype) if autoname_field: for header in ( - "ID ({})".format(autoname_field.label), # label - "{0} ({1})".format(_("ID"), _(autoname_field.label)), # translated label + f"ID ({autoname_field.label})", # label + "{} ({})".format(_("ID"), _(autoname_field.label)), # translated label # ID field should also map to the autoname field "ID", _("ID"), diff --git a/frappe/core/doctype/data_import/test_data_import.py b/frappe/core/doctype/data_import/test_data_import.py index fb15b3ad52..253882383c 100644 --- a/frappe/core/doctype/data_import/test_data_import.py +++ b/frappe/core/doctype/data_import/test_data_import.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and Contributors # License: MIT. See LICENSE # import frappe diff --git a/frappe/core/doctype/data_import/test_exporter.py b/frappe/core/doctype/data_import/test_exporter.py index ed01a2648c..ceeac90e36 100644 --- a/frappe/core/doctype/data_import/test_exporter.py +++ b/frappe/core/doctype/data_import/test_exporter.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # License: MIT. See LICENSE import unittest diff --git a/frappe/core/doctype/data_import/test_importer.py b/frappe/core/doctype/data_import/test_importer.py index 46b3c352ca..2c250a4e87 100644 --- a/frappe/core/doctype/data_import/test_importer.py +++ b/frappe/core/doctype/data_import/test_importer.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # License: MIT. See LICENSE import unittest diff --git a/frappe/core/doctype/data_import_log/data_import_log.js b/frappe/core/doctype/data_import_log/data_import_log.js index c376edeec9..19ba0eb727 100644 --- a/frappe/core/doctype/data_import_log/data_import_log.js +++ b/frappe/core/doctype/data_import_log/data_import_log.js @@ -1,8 +1,7 @@ // Copyright (c) 2021, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Data Import Log', { +frappe.ui.form.on("Data Import Log", { // refresh: function(frm) { - // } }); diff --git a/frappe/core/doctype/defaultvalue/defaultvalue.json b/frappe/core/doctype/defaultvalue/defaultvalue.json index 35b08c2dca..22e2583774 100644 --- a/frappe/core/doctype/defaultvalue/defaultvalue.json +++ b/frappe/core/doctype/defaultvalue/defaultvalue.json @@ -1,90 +1,47 @@ { - "allow_copy": 0, - "allow_import": 0, - "allow_rename": 0, - "autoname": "hash", - "beta": 0, - "creation": "2013-02-22 01:27:32", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "editable_grid": 1, + "actions": [], + "autoname": "hash", + "creation": "2013-02-22 01:27:32", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "defkey", + "defvalue" + ], "fields": [ { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "fieldname": "defkey", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 1, - "label": "Key", - "length": 0, - "no_copy": 0, - "oldfieldname": "defkey", - "oldfieldtype": "Data", - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "print_width": "200px", - "read_only": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0, + "fieldname": "defkey", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Key", + "oldfieldname": "defkey", + "oldfieldtype": "Data", + "print_width": "200px", + "reqd": 1, "width": "200px" - }, + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "fieldname": "defvalue", - "fieldtype": "Text", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 1, - "label": "Value", - "length": 0, - "no_copy": 0, - "oldfieldname": "defvalue", - "oldfieldtype": "Text", - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "print_width": "200px", - "read_only": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0, + "fieldname": "defvalue", + "fieldtype": "Text", + "in_list_view": 1, + "label": "Value", + "oldfieldname": "defvalue", + "oldfieldtype": "Text", + "print_width": "200px", "width": "200px" } - ], - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 1, - "image_view": 0, - "in_create": 0, - - "is_submittable": 0, - "issingle": 0, - "istable": 1, - "max_attachments": 0, - "modified": "2016-07-11 03:27:59.126216", - "modified_by": "Administrator", - "module": "Core", - "name": "DefaultValue", - "owner": "Administrator", - "permissions": [], - "quick_entry": 0, - "read_only": 0, - "read_only_onload": 0, - "track_seen": 0 + ], + "idx": 1, + "istable": 1, + "links": [], + "modified": "2022-08-03 12:20:54.832785", + "modified_by": "Administrator", + "module": "Core", + "name": "DefaultValue", + "owner": "Administrator", + "permissions": [], + "sort_field": "modified", + "sort_order": "DESC", + "states": [] } \ No newline at end of file diff --git a/frappe/core/doctype/deleted_document/deleted_document.js b/frappe/core/doctype/deleted_document/deleted_document.js index 3125cb2f1c..34e13c01ab 100644 --- a/frappe/core/doctype/deleted_document/deleted_document.js +++ b/frappe/core/doctype/deleted_document/deleted_document.js @@ -1,22 +1,22 @@ // Copyright (c) 2016, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Deleted Document', { - refresh: function(frm) { - if(frm.doc.restored) { - frm.add_custom_button(__('Open'), function() { - frappe.set_route('Form', frm.doc.deleted_doctype, frm.doc.new_name); +frappe.ui.form.on("Deleted Document", { + refresh: function (frm) { + if (frm.doc.restored) { + frm.add_custom_button(__("Open"), function () { + frappe.set_route("Form", frm.doc.deleted_doctype, frm.doc.new_name); }); } else { - frm.add_custom_button(__('Restore'), function() { + frm.add_custom_button(__("Restore"), function () { frappe.call({ - method: 'frappe.core.doctype.deleted_document.deleted_document.restore', - args: {name: frm.doc.name}, - callback: function(r) { + method: "frappe.core.doctype.deleted_document.deleted_document.restore", + args: { name: frm.doc.name }, + callback: function (r) { frm.reload_doc(); - } + }, }); }); } - } + }, }); diff --git a/frappe/core/doctype/deleted_document/deleted_document.json b/frappe/core/doctype/deleted_document/deleted_document.json index 1a612c7411..6b95a523c1 100644 --- a/frappe/core/doctype/deleted_document/deleted_document.json +++ b/frappe/core/doctype/deleted_document/deleted_document.json @@ -1,256 +1,81 @@ { - "allow_copy": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, - "creation": "2016-12-29 12:59:48.638970", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", + "actions": [], + "creation": "2016-12-29 12:59:48.638970", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "deleted_name", + "deleted_doctype", + "column_break_3", + "restored", + "new_name", + "section_break_6", + "data" + ], "fields": [ { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "deleted_name", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Deleted Name", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "deleted_name", + "fieldtype": "Data", + "in_list_view": 1, + "in_standard_filter": 1, + "label": "Deleted Name", + "read_only": 1 + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "deleted_doctype", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Deleted DocType", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "deleted_doctype", + "fieldtype": "Data", + "in_list_view": 1, + "in_standard_filter": 1, + "label": "Deleted DocType", + "read_only": 1 + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "column_break_3", - "fieldtype": "Column Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "column_break_3", + "fieldtype": "Column Break" + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "restored", - "fieldtype": "Check", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Restored", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "default": "0", + "fieldname": "restored", + "fieldtype": "Check", + "in_list_view": 1, + "label": "Restored", + "read_only": 1 + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "new_name", - "fieldtype": "Read Only", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "New Name", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "new_name", + "fieldtype": "Read Only", + "label": "New Name" + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "section_break_6", - "fieldtype": "Section Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "section_break_6", + "fieldtype": "Section Break" + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "data", - "fieldtype": "Code", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Data", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "fieldname": "data", + "fieldtype": "Code", + "label": "Data", + "read_only": 1 } - ], - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 1, - - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2016-12-29 14:39:45.724494", - "modified_by": "Administrator", - "module": "Core", - "name": "Deleted Document", - "name_case": "", - "owner": "Administrator", + ], + "in_create": 1, + "links": [], + "modified": "2022-06-13 05:50:58.314908", + "modified_by": "Administrator", + "module": "Core", + "name": "Deleted Document", + "owner": "Administrator", "permissions": [ { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 0, - "delete": 1, - "email": 0, - "export": 1, - "if_owner": 0, - "import": 0, - "is_custom": 0, - "permlevel": 0, - "print": 0, - "read": 1, - "report": 0, - "role": "System Manager", - "set_user_permissions": 0, - "share": 0, - "submit": 0, - "write": 0 + "delete": 1, + "export": 1, + "read": 1, + "role": "System Manager" } - ], - "quick_entry": 0, - "read_only": 0, - "read_only_onload": 0, - "sort_field": "modified", - "sort_order": "DESC", - "title_field": "deleted_name", - "track_changes": 1, - "track_seen": 0 + ], + "sort_field": "modified", + "sort_order": "DESC", + "states": [], + "title_field": "deleted_name", + "track_changes": 1 } \ No newline at end of file diff --git a/frappe/core/doctype/deleted_document/deleted_document.py b/frappe/core/doctype/deleted_document/deleted_document.py index f2c98a41b8..14b9bb5c11 100644 --- a/frappe/core/doctype/deleted_document/deleted_document.py +++ b/frappe/core/doctype/deleted_document/deleted_document.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and contributors # License: MIT. See LICENSE diff --git a/frappe/core/doctype/deleted_document/deleted_document_list.js b/frappe/core/doctype/deleted_document/deleted_document_list.js index 92413bfdf4..6a271f5ae9 100644 --- a/frappe/core/doctype/deleted_document/deleted_document_list.js +++ b/frappe/core/doctype/deleted_document/deleted_document_list.js @@ -3,27 +3,36 @@ frappe.listview_settings["Deleted Document"] = { const action = () => { const selected_docs = doclist.get_checked_items(); if (selected_docs.length > 0) { - let docnames = selected_docs.map(doc => doc.name); + let docnames = selected_docs.map((doc) => doc.name); frappe.call({ method: "frappe.core.doctype.deleted_document.deleted_document.bulk_restore", args: { docnames }, callback: function (r) { if (r.message) { let body = (docnames) => { - const html = docnames.map(docname => { + const html = docnames.map((docname) => { return `
  • ${docname}
    • `; }); return "
      " + html.join(""); }; let message = (title, docnames) => { - return (docnames.length > 0) ? title + body(docnames) + "
    ": ""; + return docnames.length > 0 ? title + body(docnames) + "" : ""; }; const { restored, invalid, failed } = r.message; - const restored_summary = message(__("Documents restored successfully"), restored); - const invalid_summary = message(__("Documents that were already restored"), invalid); - const failed_summary = message(__("Documents that failed to restore"), failed); + const restored_summary = message( + __("Documents restored successfully"), + restored + ); + const invalid_summary = message( + __("Documents that were already restored"), + invalid + ); + const failed_summary = message( + __("Documents that failed to restore"), + failed + ); const summary = restored_summary + invalid_summary + failed_summary; frappe.msgprint(summary, __("Document Restoration Summary"), true); diff --git a/frappe/core/doctype/deleted_document/test_deleted_document.py b/frappe/core/doctype/deleted_document/test_deleted_document.py index 0c8e88a32f..f696689664 100644 --- a/frappe/core/doctype/deleted_document/test_deleted_document.py +++ b/frappe/core/doctype/deleted_document/test_deleted_document.py @@ -1,10 +1,7 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and Contributors # License: MIT. See LICENSE import unittest -import frappe - # test_records = frappe.get_test_records('Deleted Document') diff --git a/frappe/core/doctype/docfield/docfield.json b/frappe/core/doctype/docfield/docfield.json index 9e9aaf489b..803ad3c140 100644 --- a/frappe/core/doctype/docfield/docfield.json +++ b/frappe/core/doctype/docfield/docfield.json @@ -99,7 +99,7 @@ "label": "Type", "oldfieldname": "fieldtype", "oldfieldtype": "Select", - "options": "Autocomplete\nAttach\nAttach Image\nBarcode\nButton\nCheck\nCode\nColor\nColumn Break\nCurrency\nData\nDate\nDatetime\nDuration\nDynamic Link\nFloat\nFold\nGeolocation\nHeading\nHTML\nHTML Editor\nIcon\nImage\nInt\nJSON\nLink\nLong Text\nMarkdown Editor\nPassword\nPercent\nRead Only\nRating\nSection Break\nSelect\nSignature\nSmall Text\nTab Break\nTable\nTable MultiSelect\nText\nText Editor\nTime", + "options": "Autocomplete\nAttach\nAttach Image\nBarcode\nButton\nCheck\nCode\nColor\nColumn Break\nCurrency\nData\nDate\nDatetime\nDuration\nDynamic Link\nFloat\nFold\nGeolocation\nHeading\nHTML\nHTML Editor\nIcon\nImage\nInt\nJSON\nLink\nLong Text\nMarkdown Editor\nPassword\nPercent\nPhone\nRead Only\nRating\nSection Break\nSelect\nSignature\nSmall Text\nTab Break\nTable\nTable MultiSelect\nText\nText Editor\nTime", "reqd": 1, "search_index": 1 }, @@ -347,7 +347,7 @@ }, { "default": "0", - "description": "Don't HTML Encode HTML tags like <script> or just characters like < or >, as they could be intentionally used in this field", + "description": "Don't encode HTML tags like <script> or just characters like < or >, as they could be intentionally used in this field", "fieldname": "ignore_xss_filter", "fieldtype": "Check", "label": "Ignore XSS Filter" @@ -547,7 +547,7 @@ "index_web_pages_for_search": 1, "istable": 1, "links": [], - "modified": "2022-03-02 17:07:32.117897", + "modified": "2022-04-19 12:27:28.641580", "modified_by": "Administrator", "module": "Core", "name": "DocField", @@ -557,4 +557,4 @@ "sort_field": "modified", "sort_order": "ASC", "states": [] -} \ No newline at end of file +} diff --git a/frappe/core/doctype/docperm/docperm.py b/frappe/core/doctype/docperm/docperm.py index 68ef21e770..87d6457c3c 100644 --- a/frappe/core/doctype/docperm/docperm.py +++ b/frappe/core/doctype/docperm/docperm.py @@ -1,7 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe from frappe.model.document import Document diff --git a/frappe/core/doctype/docshare/docshare.js b/frappe/core/doctype/docshare/docshare.js index 48db47a8cc..4d68c65cff 100644 --- a/frappe/core/doctype/docshare/docshare.js +++ b/frappe/core/doctype/docshare/docshare.js @@ -1,8 +1,6 @@ // Copyright (c) 2016, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('DocShare', { - refresh: function(frm) { - - } +frappe.ui.form.on("DocShare", { + refresh: function (frm) {}, }); diff --git a/frappe/core/doctype/doctype/doctype.js b/frappe/core/doctype/doctype/doctype.js index 88cc5577a6..e91a05e17d 100644 --- a/frappe/core/doctype/doctype/doctype.js +++ b/frappe/core/doctype/doctype/doctype.js @@ -1,19 +1,19 @@ // Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors // MIT License. See license.txt -frappe.ui.form.on('DocType', { - refresh: function(frm) { - frm.set_query('role', 'permissions', function(doc) { - if (doc.custom && frappe.session.user != 'Administrator') { +frappe.ui.form.on("DocType", { + refresh: function (frm) { + frm.set_query("role", "permissions", function (doc) { + if (doc.custom && frappe.session.user != "Administrator") { return { query: "frappe.core.doctype.role.role.role_query", - filters: [['Role', 'name', '!=', 'All']] + filters: [["Role", "name", "!=", "All"]], }; } }); - if(frappe.session.user !== "Administrator" || !frappe.boot.developer_mode) { - if(frm.is_new()) { + if (frappe.session.user !== "Administrator" || !frappe.boot.developer_mode) { + if (frm.is_new()) { frm.set_value("custom", 1); } frm.toggle_enable("custom", 0); @@ -23,123 +23,64 @@ frappe.ui.form.on('DocType', { if (!frm.is_new() && !frm.doc.istable) { if (frm.doc.issingle) { - frm.add_custom_button(__('Go to {0}', [__(frm.doc.name)]), () => { + frm.add_custom_button(__("Go to {0}", [__(frm.doc.name)]), () => { window.open(`/app/${frappe.router.slug(frm.doc.name)}`); }); } else { - frm.add_custom_button(__('Go to {0} List', [__(frm.doc.name)]), () => { + frm.add_custom_button(__("Go to {0} List", [__(frm.doc.name)]), () => { window.open(`/app/${frappe.router.slug(frm.doc.name)}`); }); } } const customize_form_link = "Customize Form"; - if(!frappe.boot.developer_mode && !frm.doc.custom) { + if (!frappe.boot.developer_mode && !frm.doc.custom) { // make the document read-only frm.set_read_only(); - frm.dashboard.add_comment(__("DocTypes can not be modified, please use {0} instead", [customize_form_link]), "blue", true); + frm.dashboard.add_comment( + __("DocTypes can not be modified, please use {0} instead", [customize_form_link]), + "blue", + true + ); } else if (frappe.boot.developer_mode) { - let msg = __("This site is running in developer mode. Any change made here will be updated in code."); + let msg = __( + "This site is running in developer mode. Any change made here will be updated in code." + ); msg += "
    "; - msg += __("If you just want to customize for your site, use {0} instead.", [customize_form_link]); + msg += __("If you just want to customize for your site, use {0} instead.", [ + customize_form_link, + ]); frm.dashboard.add_comment(msg, "yellow"); } - if(frm.is_new()) { - if (!(frm.doc.permissions && frm.doc.permissions.length)) { - frm.add_child('permissions', {role: 'System Manager'}); - } + if (frm.is_new()) { + frm.events.set_default_permission(frm); } else { frm.toggle_enable("engine", 0); } // set label for "In List View" for child tables - frm.get_docfield('fields', 'in_list_view').label = frm.doc.istable ? - __('In Grid View') : __('In List View'); + frm.get_docfield("fields", "in_list_view").label = frm.doc.istable + ? __("In Grid View") + : __("In List View"); - frm.events.autoname(frm); - frm.events.set_naming_rule_description(frm); + frm.cscript.autoname(frm); + frm.cscript.set_naming_rule_description(frm); }, istable: (frm) => { if (frm.doc.istable && frm.is_new()) { - frm.set_value('autoname', 'autoincrement'); - frm.set_value('allow_rename', 0); + frm.set_value("autoname", "autoincrement"); + frm.set_value("allow_rename", 0); + } else if (!frm.doc.istable && !frm.is_new()) { + frm.events.set_default_permission(frm); } }, - naming_rule: function(frm) { - // set the "autoname" property based on naming_rule - if (frm.doc.naming_rule && !frm.__from_autoname) { - - // flag to avoid recursion - frm.__from_naming_rule = true; - - if (frm.doc.naming_rule=='Set by user') { - frm.set_value('autoname', 'Prompt'); - } else if (frm.doc.naming_rule === 'Autoincrement') { - frm.set_value('autoname', 'autoincrement'); - // set allow rename to be false when using autoincrement - frm.set_value('allow_rename', 0); - } else if (frm.doc.naming_rule=='By fieldname') { - frm.set_value('autoname', 'field:'); - } else if (frm.doc.naming_rule=='By "Naming Series" field') { - frm.set_value('autoname', 'naming_series:'); - } else if (frm.doc.naming_rule=='Expression') { - frm.set_value('autoname', 'format:'); - } else if (frm.doc.naming_rule=='Expression (old style)') { - // pass - } else if (frm.doc.naming_rule=='Random') { - frm.set_value('autoname', 'hash'); - } - setTimeout(() =>frm.__from_naming_rule = false, 500); - - frm.events.set_naming_rule_description(frm); + set_default_permission: (frm) => { + if (!(frm.doc.permissions && frm.doc.permissions.length)) { + frm.add_child("permissions", { role: "System Manager" }); } - - }, - - set_naming_rule_description(frm) { - let naming_rule_description = { - 'Set by user': '', - 'Autoincrement': 'Uses Auto Increment feature of database.
    WARNING: After using this option, any other naming option will not be accessible.', - 'By fieldname': 'Format: field:[fieldname]. Valid fieldname must exist', - 'By "Naming Series" field': 'Format: naming_series:[fieldname]. Fieldname called naming_series must exist', - 'Expression': 'Format: format:EXAMPLE-{MM}morewords{fieldname1}-{fieldname2}-{#####} - Replace all braced words (fieldnames, date words (DD, MM, YY), series) with their value. Outside braces, any characters can be used.', - 'Expression (old style)': 'Format: EXAMPLE-.##### Series by prefix (separated by a dot)', - 'Random': '', - 'By script': '' - }; - - if (frm.doc.naming_rule) { - frm.get_field('autoname').set_description(naming_rule_description[frm.doc.naming_rule]); - } - }, - - autoname: function(frm) { - // set naming_rule based on autoname (for old doctypes where its not been set) - if (frm.doc.autoname && !frm.doc.naming_rule && !frm.__from_naming_rule) { - // flag to avoid recursion - frm.__from_autoname = true; - if (frm.doc.autoname.toLowerCase() === 'prompt') { - frm.set_value('naming_rule', 'Set by user'); - } else if (frm.doc.autoname.toLowerCase() === 'autoincrement') { - frm.set_value('naming_rule', 'Autoincrement'); - } else if (frm.doc.autoname.startsWith('field:')) { - frm.set_value('naming_rule', 'By fieldname'); - } else if (frm.doc.autoname.startsWith('naming_series:')) { - frm.set_value('naming_rule', 'By "Naming Series" field'); - } else if (frm.doc.autoname.startsWith('format:')) { - frm.set_value('naming_rule', 'Expression'); - } else if (frm.doc.autoname.toLowerCase() === 'hash') { - frm.set_value('naming_rule', 'Random'); - } else { - frm.set_value('naming_rule', 'Expression (old style)'); - } - setTimeout(() => frm.__from_autoname = false, 500); - } - - frm.set_df_property('fields', 'reqd', frm.doc.autoname !== 'Prompt'); }, }); @@ -166,15 +107,15 @@ frappe.ui.form.on("DocField", { } let doctypes = frm.doc.fields - .filter(df => df.fieldtype == "Link") - .filter(df => df.options && df.fieldname != row.fieldname) - .map(df => ({ + .filter((df) => df.fieldtype == "Link") + .filter((df) => df.options && df.fieldname != row.fieldname) + .map((df) => ({ label: `${df.options} (${df.fieldname})`, - value: df.fieldname + value: df.fieldname, })); $doctype_select.add_options([ { label: __("Select DocType"), value: "", selected: true }, - ...doctypes + ...doctypes, ]); $doctype_select.on("change", () => { @@ -188,27 +129,25 @@ frappe.ui.form.on("DocField", { let link_fieldname = $doctype_select.val(); if (!link_fieldname) return; - let link_field = frm.doc.fields.find( - df => df.fieldname === link_fieldname - ); + let link_field = frm.doc.fields.find((df) => df.fieldname === link_fieldname); let link_doctype = link_field.options; frappe.model.with_doctype(link_doctype, () => { let fields = frappe.meta .get_docfields(link_doctype, null, { - fieldtype: ["not in", frappe.model.no_value_type] + fieldtype: ["not in", frappe.model.no_value_type], }) - .map(df => ({ + .map((df) => ({ label: `${df.label} (${df.fieldtype})`, - value: df.fieldname + value: df.fieldname, })); $field_select.add_options([ { label: __("Select Field"), value: "", selected: true, - disabled: true + disabled: true, }, - ...fields + ...fields, ]); if (curr_value.fieldname) { @@ -229,9 +168,9 @@ frappe.ui.form.on("DocField", { } }, - fieldtype: function(frm) { + fieldtype: function (frm) { frm.trigger("max_attachments"); - } + }, }); -extend_cscript(cur_frm.cscript, new frappe.model.DocTypeController({frm: cur_frm})); +extend_cscript(cur_frm.cscript, new frappe.model.DocTypeController({ frm: cur_frm })); diff --git a/frappe/core/doctype/doctype/doctype.json b/frappe/core/doctype/doctype/doctype.json index 8169a59566..4e110202d2 100644 --- a/frappe/core/doctype/doctype/doctype.json +++ b/frappe/core/doctype/doctype/doctype.json @@ -47,6 +47,7 @@ "view_settings", "title_field", "show_title_field_in_link", + "translate_link_fields", "search_fields", "default_print_format", "sort_field", @@ -208,7 +209,7 @@ "label": "Naming" }, { - "description": "Naming Options:\n
    1. field:[fieldname] - By Field
    2. autoincrement - Uses Databases' Auto Increment feature
    3. naming_series: - By Naming Series (field called naming_series must be present
    4. Prompt - Prompt user for a name
    5. [series] - Series by prefix (separated by a dot); for example PRE.#####
      6. \n
    6. format:EXAMPLE-{MM}morewords{fieldname1}-{fieldname2}-{#####} - Replace all braced words (fieldnames, date words (DD, MM, YY), series) with their value. Outside braces, any characters can be used.
    ", + "description": "Naming Options:\n
    1. field:[fieldname] - By Field
    2. autoincrement - Uses Databases' Auto Increment feature
    3. naming_series: - By Naming Series (field called naming_series must be present)
    4. Prompt - Prompt user for a name
    5. [series] - Series by prefix (separated by a dot); for example PRE.#####
      6. \n
    6. format:EXAMPLE-{MM}morewords{fieldname1}-{fieldname2}-{#####} - Replace all braced words (fieldnames, date words (DD, MM, YY), series) with their value. Outside braces, any characters can be used.
    ", "fieldname": "autoname", "fieldtype": "Data", "label": "Auto Name", @@ -591,6 +592,12 @@ "fieldname": "show_title_field_in_link", "fieldtype": "Check", "label": "Show Title in Link Fields" + }, + { + "default": "0", + "fieldname": "translate_link_fields", + "fieldtype": "Check", + "label": "Translate Link Fields" } ], "icon": "fa fa-bolt", @@ -673,7 +680,7 @@ "link_fieldname": "reference_doctype" } ], - "modified": "2022-02-15 21:47:16.467217", + "modified": "2022-02-28 21:56:52.116915", "modified_by": "Administrator", "module": "Core", "name": "DocType", @@ -708,5 +715,6 @@ "sort_field": "modified", "sort_order": "DESC", "states": [], - "track_changes": 1 + "track_changes": 1, + "translate_link_fields": 1 } \ No newline at end of file diff --git a/frappe/core/doctype/doctype/doctype.py b/frappe/core/doctype/doctype/doctype.py index 1d02f09820..9a8a976b9f 100644 --- a/frappe/core/doctype/doctype/doctype.py +++ b/frappe/core/doctype/doctype/doctype.py @@ -8,6 +8,7 @@ import os # imports - standard imports import re import shutil +from typing import TYPE_CHECKING, Union # imports - module imports import frappe @@ -16,7 +17,7 @@ from frappe.cache_manager import clear_controller_cache, clear_user_cache from frappe.custom.doctype.custom_field.custom_field import create_custom_field from frappe.custom.doctype.property_setter.property_setter import make_property_setter from frappe.database.schema import validate_column_length, validate_column_name -from frappe.desk.notifications import delete_notification_count_for +from frappe.desk.notifications import delete_notification_count_for, get_filters_for from frappe.desk.utils import validate_route_conflict from frappe.model import ( child_table_fields, @@ -32,9 +33,18 @@ from frappe.model.meta import Meta from frappe.modules import get_doc_path, make_boilerplate from frappe.modules.import_file import get_file_path from frappe.query_builder.functions import Concat -from frappe.utils import cint, now +from frappe.utils import cint from frappe.website.utils import clear_cache +if TYPE_CHECKING: + from frappe.custom.doctype.customize_form.customize_form import CustomizeForm + +DEPENDS_ON_PATTERN = re.compile(r'[\w\.:_]+\s*={1}\s*[\w\.@\'"]+') +ILLEGAL_FIELDNAME_PATTERN = re.compile("""['",./%@()<>{}]""") +WHITESPACE_PADDING_PATTERN = re.compile(r"^[ \t\n\r]+|[ \t\n\r]+$", flags=re.ASCII) +START_WITH_LETTERS_PATTERN = re.compile(r"^(?![\W])[^\d_\s][\w -]+$", flags=re.ASCII) +FIELD_PATTERN = re.compile("{(.*?)}", flags=re.UNICODE) + class InvalidFieldNameError(frappe.ValidationError): pass @@ -92,14 +102,15 @@ class DocType(Document): self.check_developer_mode() - self.validate_autoname() self.validate_name() self.set_defaults_for_single_and_table() + self.set_defaults_for_autoincremented() self.scrub_field_names() self.set_default_in_list_view() self.set_default_translatable() validate_series(self) + self.set("can_change_name_type", validate_autoincrement_autoname(self)) self.validate_document_type() validate_fields(self) @@ -160,7 +171,7 @@ class DocType(Document): if docfield.fieldname in method_set: conflict_type = "controller method" - if docfield.fieldname in property_set: + if docfield.fieldname in property_set and not docfield.is_virtual: conflict_type = "class property" if conflict_type: @@ -170,10 +181,6 @@ class DocType(Document): ) ) - def after_insert(self): - # clear user cache so that on the next reload this doctype is included in boot - clear_user_cache(frappe.session.user) - def set_defaults_for_single_and_table(self): if self.issingle: self.allow_import = 0 @@ -184,6 +191,10 @@ class DocType(Document): self.allow_import = 0 self.permissions = [] + def set_defaults_for_autoincremented(self): + if self.autoname and self.autoname == "autoincrement": + self.allow_rename = 0 + def set_default_in_list_view(self): """Set default in-list-view for first 4 mandatory fields""" if not [d.fieldname for d in self.fields if d.in_list_view]: @@ -233,7 +244,7 @@ class DocType(Document): self.flags.update_fields_to_fetch_queries = [] - if set(old_fields_to_fetch) != set(df.fieldname for df in new_meta.get_fields_to_fetch()): + if set(old_fields_to_fetch) != {df.fieldname for df in new_meta.get_fields_to_fetch()}: for df in new_meta.get_fields_to_fetch(): if df.fieldname not in old_fields_to_fetch: link_fieldname, source_fieldname = df.fetch_from.split(".", 1) @@ -352,8 +363,7 @@ class DocType(Document): else: if d.fieldname in restricted: frappe.throw(_("Fieldname {0} is restricted").format(d.fieldname), InvalidFieldNameError) - - d.fieldname = re.sub("""['",./%@()<>{}]""", "", d.fieldname) + d.fieldname = ILLEGAL_FIELDNAME_PATTERN.sub("", d.fieldname) # fieldnames should be lowercase d.fieldname = d.fieldname.lower() @@ -364,10 +374,14 @@ class DocType(Document): def on_update(self): """Update database schema, make controller templates if `custom` is not set and clear cache.""" + + if self.get("can_change_name_type"): + self.setup_autoincrement_and_sequence() + try: frappe.db.updatedb(self.name, Meta(self)) except Exception as e: - print("\n\nThere was an issue while migrating the DocType: {}\n".format(self.name)) + print(f"\n\nThere was an issue while migrating the DocType: {self.name}\n") raise e self.change_modified_of_parent() @@ -394,6 +408,9 @@ class DocType(Document): delete_notification_count_for(doctype=self.name) frappe.clear_cache(doctype=self.name) + # clear user cache so that on the next reload this doctype is included in boot + clear_user_cache(frappe.session.user) + if not frappe.flags.in_install and hasattr(self, "before_update"): self.sync_global_search() @@ -403,6 +420,17 @@ class DocType(Document): clear_linked_doctype_cache() + def setup_autoincrement_and_sequence(self): + """Changes name type and makes sequence on change (if required)""" + + name_type = f"varchar({frappe.db.VARCHAR_LEN})" + + if self.autoname == "autoincrement": + name_type = "bigint" + frappe.db.create_sequence(self.name, check_not_exists=True, cache=frappe.db.SEQUENCE_CACHE) + + change_name_column_type(self.name, name_type) + def sync_global_search(self): """If global search settings are changed, rebuild search properties for this table""" global_search_fields_before_update = [ @@ -523,7 +551,7 @@ class DocType(Document): for fname in ("{}.js", "{}.py", "{}_list.js", "{}_calendar.js", "test_{}.py", "test_{}.js"): fname = os.path.join(new_path, fname.format(frappe.scrub(new))) if os.path.exists(fname): - with open(fname, "r") as f: + with open(fname) as f: code = f.read() with open(fname, "w") as f: if fname.endswith(".js"): @@ -540,7 +568,7 @@ class DocType(Document): f.write(file_content) # updating json file with new name - doctype_json_path = os.path.join(new_path, "{}.json".format(frappe.scrub(new))) + doctype_json_path = os.path.join(new_path, f"{frappe.scrub(new)}.json") current_data = frappe.get_file_json(doctype_json_path) current_data["name"] = new @@ -614,7 +642,7 @@ class DocType(Document): path = get_file_path(self.module, "DocType", self.name) if os.path.exists(path): try: - with open(path, "r") as txtfile: + with open(path) as txtfile: olddoc = json.loads(txtfile.read()) old_field_names = [f["fieldname"] for f in olddoc.get("fields", [])] @@ -623,14 +651,14 @@ class DocType(Document): remaining_field_names = [f.fieldname for f in self.fields] for fieldname in old_field_names: - field_dict = list(filter(lambda d: d["fieldname"] == fieldname, docdict["fields"])) + field_dict = [f for f in docdict["fields"] if f["fieldname"] == fieldname] if field_dict: new_field_dicts.append(field_dict[0]) if fieldname in remaining_field_names: remaining_field_names.remove(fieldname) for fieldname in remaining_field_names: - field_dict = list(filter(lambda d: d["fieldname"] == fieldname, docdict["fields"])) + field_dict = [f for f in docdict["fields"] if f["fieldname"] == fieldname] new_field_dicts.append(field_dict[0]) docdict["fields"] = new_field_dicts @@ -645,14 +673,14 @@ class DocType(Document): remaining_field_names = [f["fieldname"] for f in docdict.get("fields", [])] for fieldname in docdict.get("field_order"): - field_dict = list(filter(lambda d: d["fieldname"] == fieldname, docdict.get("fields", []))) + field_dict = [f for f in docdict.get("fields", []) if f["fieldname"] == fieldname] if field_dict: new_field_dicts.append(field_dict[0]) if fieldname in remaining_field_names: remaining_field_names.remove(fieldname) for fieldname in remaining_field_names: - field_dict = list(filter(lambda d: d["fieldname"] == fieldname, docdict.get("fields", []))) + field_dict = [f for f in docdict.get("fields", []) if f["fieldname"] == fieldname] new_field_dicts.append(field_dict[0]) docdict["fields"] = new_field_dicts @@ -775,7 +803,7 @@ class DocType(Document): {"label": "Old Parent", "fieldtype": "Link", "options": self.name, "fieldname": "old_parent"}, ) - parent_field_label = "Parent {}".format(self.name) + parent_field_label = f"Parent {self.name}" parent_field_name = frappe.scrub(parent_field_label) self.append( "fields", @@ -789,7 +817,7 @@ class DocType(Document): self.nsm_parent_field = parent_field_name def validate_child_table(self): - if not self.get("istable") or self.is_new(): + if not self.get("istable") or self.is_new() or self.get("is_virtual"): # if the doctype is not a child table then return # if the doctype is a new doctype and also a child table then # don't move forward as it will be handled via schema @@ -809,19 +837,6 @@ class DocType(Document): max_idx = frappe.db.sql("""select max(idx) from `tabDocField` where parent = %s""", self.name) return max_idx and max_idx[0][0] or 0 - def validate_autoname(self): - if not self.is_new(): - doc_before_save = self.get_doc_before_save() - if doc_before_save: - if (self.autoname == "autoincrement" and doc_before_save.autoname != "autoincrement") or ( - self.autoname != "autoincrement" and doc_before_save.autoname == "autoincrement" - ): - frappe.throw(_("Cannot change to/from Autoincrement naming rule")) - - else: - if self.autoname == "autoincrement": - self.allow_rename = 0 - def validate_name(self, name=None): if not name: name = self.name @@ -835,15 +850,13 @@ class DocType(Document): _("Doctype name is limited to {0} characters ({1})").format(max_length, name), frappe.NameError ) - flags = {"flags": re.ASCII} - # a DocType name should not start or end with an empty space - if re.search(r"^[ \t\n\r]+|[ \t\n\r]+$", name, **flags): + if WHITESPACE_PADDING_PATTERN.search(name): frappe.throw(_("DocType's name should not start or end with whitespace"), frappe.NameError) # a DocType's name should not start with a number or underscore # and should only contain letters, numbers, underscore, and hyphen - if not re.match(r"^(?![\W])[^\d_\s][\w -]+$", name, **flags): + if not START_WITH_LETTERS_PATTERN.match(name): frappe.throw( _( "A DocType's name should start with a letter and can only " @@ -865,8 +878,13 @@ def validate_series(dt, autoname=None, name=None): if not autoname and dt.get("fields", {"fieldname": "naming_series"}): dt.autoname = "naming_series:" - elif dt.autoname == "naming_series:" and not dt.get("fields", {"fieldname": "naming_series"}): - frappe.throw(_("Invalid fieldname '{0}' in autoname").format(dt.autoname)) + elif dt.autoname and dt.autoname.startswith("naming_series:"): + fieldname = dt.autoname.split("naming_series:")[0] or "naming_series" + if not dt.get("fields", {"fieldname": fieldname}): + frappe.throw( + _("Fieldname called {0} must exist to enable autonaming").format(frappe.bold(fieldname)), + title=_("Field Missing"), + ) # validate field name if autoname field:fieldname is used # Create unique index on autoname field automatically. @@ -884,7 +902,7 @@ def validate_series(dt, autoname=None, name=None): autoname and (not autoname.startswith("field:")) and (not autoname.startswith("eval:")) - and (not autoname.lower() in ("prompt", "hash")) + and (autoname.lower() not in ("prompt", "hash")) and (not autoname.startswith("naming_series:")) and (not autoname.startswith("format:")) ): @@ -901,6 +919,62 @@ def validate_series(dt, autoname=None, name=None): frappe.throw(_("Series {0} already used in {1}").format(prefix, used_in[0][0])) +def validate_autoincrement_autoname(dt: Union[DocType, "CustomizeForm"]) -> bool: + """Checks if can doctype can change to/from autoincrement autoname""" + + def get_autoname_before_save(dt: Union[DocType, "CustomizeForm"]) -> str: + if dt.doctype == "Customize Form": + property_value = frappe.db.get_value( + "Property Setter", {"doc_type": dt.doc_type, "property": "autoname"}, "value" + ) + # initially no property setter is set, + # hence getting autoname value from the doctype itself + if not property_value: + return frappe.db.get_value("DocType", dt.doc_type, "autoname") or "" + + return property_value + + return getattr(dt.get_doc_before_save(), "autoname", "") + + if not dt.is_new(): + autoname_before_save = get_autoname_before_save(dt) + is_autoname_autoincrement = dt.autoname == "autoincrement" + + if ( + is_autoname_autoincrement + and autoname_before_save != "autoincrement" + or (not is_autoname_autoincrement and autoname_before_save == "autoincrement") + ): + + if dt.doctype == "Customize Form": + frappe.throw(_("Cannot change to/from autoincrement autoname in Customize Form")) + + if frappe.get_meta(dt.name).issingle: + return False + + if not frappe.get_all(dt.name, limit=1): + # allow changing the column type if there is no data + return True + + frappe.throw( + _("Can only change to/from Autoincrement naming rule when there is no data in the doctype") + ) + + return False + + +def change_name_column_type(doctype_name: str, type: str) -> None: + """Changes name column type""" + + args = ( + (doctype_name, "name", type, False, True) + if (frappe.db.db_type == "postgres") + else (doctype_name, "name", type, True) + ) + + frappe.db.change_column_type(*args) + + def validate_links_table_fieldnames(meta): """Validate fieldnames in Links table""" if not meta.links or frappe.flags.in_patch or frappe.flags.in_fixtures: @@ -908,11 +982,7 @@ def validate_links_table_fieldnames(meta): fieldnames = tuple(field.fieldname for field in meta.fields) for index, link in enumerate(meta.links, 1): - if not frappe.get_meta(link.link_doctype).has_field(link.link_fieldname): - message = _("Document Links Row #{0}: Could not find field {1} in {2} DocType").format( - index, frappe.bold(link.link_fieldname), frappe.bold(link.link_doctype) - ) - frappe.throw(message, InvalidFieldNameError, _("Invalid Fieldname")) + _test_connection_query(doctype=link.link_doctype, field=link.link_fieldname, idx=index) if not link.is_child_table: continue @@ -941,6 +1011,25 @@ def validate_links_table_fieldnames(meta): frappe.throw(message, frappe.ValidationError, _("Invalid Table Fieldname")) +def _test_connection_query(doctype, field, idx): + """Make sure that connection can be queried. + + This function executes query similar to one that would be executed for + finding count on dashboard and hence validates if fieldname/doctype are + correct. + """ + filters = get_filters_for(doctype) or {} + filters[field] = "" + + try: + frappe.get_all(doctype, filters=filters, limit=1, distinct=True, ignore_ifnull=True) + except Exception as e: + frappe.clear_last_message() + msg = _("Document Links Row #{0}: Invalid doctype or fieldname.").format(idx) + msg += "
    " + str(e) + frappe.throw(msg, InvalidFieldNameError) + + def validate_fields_for_doctype(doctype): meta = frappe.get_meta(doctype, cached=False) validate_links_table_fieldnames(meta) @@ -971,12 +1060,13 @@ def validate_fields(meta): validate_column_name(fieldname) def check_invalid_fieldnames(docname, fieldname): - invalid_fields = ("doctype",) - if fieldname in invalid_fields: + if fieldname in Document._reserved_keywords: frappe.throw( - _("{0}: Fieldname cannot be one of {1}").format( - docname, ", ".join(frappe.bold(d) for d in invalid_fields) - ) + _("{0}: fieldname cannot be set to reserved keyword {1}").format( + frappe.bold(docname), + frappe.bold(fieldname), + ), + title=_("Invalid Fieldname"), ) def check_unique_fieldname(docname, fieldname): @@ -1185,7 +1275,7 @@ def validate_fields(meta): if not pattern: return - for fieldname in re.findall("{(.*?)}", pattern, re.UNICODE): + for fieldname in FIELD_PATTERN.findall(pattern): if fieldname.startswith("{"): # edge case when double curlies are used for escape continue @@ -1222,7 +1312,7 @@ def validate_fields(meta): frappe.throw(_("Is Published Field must be a valid fieldname"), InvalidFieldNameError) def check_website_search_field(meta): - if not meta.website_search_field: + if not meta.get("website_search_field"): return if meta.website_search_field not in fieldname_list: @@ -1267,9 +1357,7 @@ def validate_fields(meta): ] for field in depends_on_fields: depends_on = docfield.get(field, None) - if ( - depends_on and ("=" in depends_on) and re.match(r'[\w\.:_]+\s*={1}\s*[\w\.@\'"]+', depends_on) - ): + if depends_on and ("=" in depends_on) and DEPENDS_ON_PATTERN.match(depends_on): frappe.throw(_("Invalid {0} condition").format(frappe.unscrub(field)), frappe.ValidationError) def check_table_multiselect_option(docfield): @@ -1343,7 +1431,7 @@ def validate_fields(meta): def check_max_height(docfield): if getattr(docfield, "max_height", None) and (docfield.max_height[-2:] not in ("px", "em")): - frappe.throw("Max for {} height must be in px, em, rem".format(frappe.bold(docfield.fieldname))) + frappe.throw(f"Max for {frappe.bold(docfield.fieldname)} height must be in px, em, rem") def check_no_of_ratings(docfield): if docfield.fieldtype == "Rating": diff --git a/frappe/core/doctype/doctype/test_doctype.py b/frappe/core/doctype/doctype/test_doctype.py index 7b4806da59..a083939c94 100644 --- a/frappe/core/doctype/doctype/test_doctype.py +++ b/frappe/core/doctype/doctype/test_doctype.py @@ -1,10 +1,12 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +import random +import string import unittest -from typing import Dict, List, Optional +from unittest.mock import patch import frappe +from frappe.cache_manager import clear_doctype_cache from frappe.core.doctype.doctype.doctype import ( CannotIndexedError, DoctypeLinkError, @@ -15,8 +17,8 @@ from frappe.core.doctype.doctype.doctype import ( WrongOptionsDoctypeLinkError, validate_links_table_fieldnames, ) - -# test_records = frappe.get_test_records('DocType') +from frappe.custom.doctype.custom_field.custom_field import create_custom_fields +from frappe.desk.form.load import getdoc class TestDocType(unittest.TestCase): @@ -38,6 +40,52 @@ class TestDocType(unittest.TestCase): doc = new_doctype(name).insert() doc.delete() + def test_making_sequence_on_change(self): + frappe.delete_doc_if_exists("DocType", self._testMethodName) + dt = new_doctype(self._testMethodName).insert(ignore_permissions=True) + autoname = dt.autoname + + # change autoname + dt.autoname = "autoincrement" + dt.save() + + # check if name type has been changed + self.assertEqual( + frappe.db.sql( + f"""select data_type FROM information_schema.columns + where column_name = 'name' and table_name = 'tab{self._testMethodName}'""" + )[0][0], + "bigint", + ) + + if frappe.db.db_type == "mariadb": + table_name = "information_schema.tables" + conditions = f"table_type = 'sequence' and table_name = '{self._testMethodName}_id_seq'" + else: + table_name = "information_schema.sequences" + conditions = f"sequence_name = '{self._testMethodName}_id_seq'" + + # check if sequence table is created + self.assertTrue( + frappe.db.sql( + f"""select * from {table_name} + where {conditions}""" + ) + ) + + # change the autoname/naming rule back to original + dt.autoname = autoname + dt.save() + + # check if name type has changed + self.assertEqual( + frappe.db.sql( + f"""select data_type FROM information_schema.columns + where column_name = 'name' and table_name = 'tab{self._testMethodName}'""" + )[0][0], + "varchar" if frappe.db.db_type == "mariadb" else "character varying", + ) + def test_doctype_unique_constraint_dropped(self): if frappe.db.exists("DocType", "With_Unique"): frappe.delete_doc("DocType", "With_Unique") @@ -137,7 +185,7 @@ class TestDocType(unittest.TestCase): "module": "Core", "custom": 1, "fields": [ - {"fieldname": "{0}_field".format(field_option), "fieldtype": "Data", "options": field_option} + {"fieldname": f"{field_option}_field", "fieldtype": "Data", "options": field_option} ], } ) @@ -268,7 +316,7 @@ class TestDocType(unittest.TestCase): self.assertListEqual( test_doctype_json["field_order"], ["field_4", "field_5", "field_1", "field_2"] ) - except: + except Exception: raise finally: frappe.flags.allow_doctype_export = 0 @@ -495,7 +543,7 @@ class TestDocType(unittest.TestCase): # check invalid doctype doc.append("links", {"link_doctype": "User2", "link_fieldname": "first_name"}) - self.assertRaises(frappe.DoesNotExistError, validate_links_table_fieldnames, doc) + self.assertRaises(InvalidFieldNameError, validate_links_table_fieldnames, doc) doc.links = [] # reset links table # check invalid fieldname @@ -514,6 +562,46 @@ class TestDocType(unittest.TestCase): self.assertEqual(doc.is_virtual, 1) self.assertFalse(frappe.db.table_exists("Test Virtual Doctype")) + def test_create_virtual_doctype_as_child_table(self): + """Test virtual DocType as Child Table below a normal DocType.""" + frappe.delete_doc_if_exists("DocType", "Test Parent Virtual DocType", force=1) + frappe.delete_doc_if_exists("DocType", "Test Virtual DocType as Child Table", force=1) + + virtual_doc = new_doctype("Test Virtual DocType as Child Table") + virtual_doc.is_virtual = 1 + virtual_doc.istable = 1 + virtual_doc.insert(ignore_permissions=True) + + doc = frappe.get_doc("DocType", "Test Virtual DocType as Child Table") + + self.assertEqual(doc.is_virtual, 1) + self.assertEqual(doc.istable, 1) + self.assertFalse(frappe.db.table_exists("Test Virtual DocType as Child Table")) + + parent_doc = new_doctype("Test Parent Virtual DocType") + parent_doc.append( + "fields", + { + "fieldname": "virtual_child_table", + "fieldtype": "Table", + "options": "Test Virtual DocType as Child Table", + }, + ) + parent_doc.insert(ignore_permissions=True) + + # create entry for parent doctype + parent_doc_entry = frappe.get_doc( + {"doctype": "Test Parent Virtual DocType", "some_fieldname": "Test"} + ) + parent_doc_entry.insert(ignore_permissions=True) + + # update the parent doc (should not abort because of any DB query to a virtual child table, as there is none) + parent_doc_entry.some_fieldname = "Test update" + parent_doc_entry.save(ignore_permissions=True) + + # delete the parent doc (should not abort because of any DB query to a virtual child table, as there is none) + parent_doc_entry.delete() + def test_default_fieldname(self): fields = [ {"label": "title", "fieldname": "title", "fieldtype": "Data", "default": "{some_fieldname}"} @@ -524,18 +612,33 @@ class TestDocType(unittest.TestCase): dt.delete() def test_autoincremented_doctype_transition(self): - frappe.delete_doc("testy_autoinc_dt") + frappe.delete_doc_if_exists("DocType", "testy_autoinc_dt") dt = new_doctype("testy_autoinc_dt", autoname="autoincrement").insert(ignore_permissions=True) dt.autoname = "hash" + dt.save(ignore_permissions=True) + + dt_data = frappe.get_doc({"doctype": dt.name, "some_fieldname": "test data"}).insert( + ignore_permissions=True + ) + + dt.autoname = "autoincrement" + try: dt.save(ignore_permissions=True) except frappe.ValidationError as e: - self.assertEqual(e.args[0], "Cannot change to/from Autoincrement naming rule") + self.assertEqual( + e.args[0], + "Can only change to/from Autoincrement naming rule when there is no data in the doctype", + ) else: - self.fail("Shouldnt be possible to transition autoincremented doctype to any other naming rule") + self.fail( + """Shouldn't be possible to transition to/from autoincremented doctype + when data is present in doctype""" + ) finally: # cleanup + dt_data.delete(ignore_permissions=True) dt.delete(ignore_permissions=True) def test_json_field(self): @@ -567,10 +670,55 @@ class TestDocType(unittest.TestCase): self.assertEqual(test_json.test_json_field["hello"], "world") + @patch.dict(frappe.conf, {"developer_mode": 1}) + def test_delete_doctype_with_customization(self): + from frappe.custom.doctype.property_setter.property_setter import make_property_setter + + custom_field = "customfield" + + doctype = new_doctype(custom=0).insert().name + + # Create property setter and custom field + field = "some_fieldname" + make_property_setter(doctype, field, "default", "DELETETHIS", "Data") + create_custom_fields({doctype: [{"fieldname": custom_field, "fieldtype": "Data"}]}) + + # Create 1 record + original_doc = frappe.get_doc(doctype=doctype, custom_field_name="wat").insert() + self.assertEqual(original_doc.some_fieldname, "DELETETHIS") + + # delete doctype + frappe.delete_doc("DocType", doctype) + clear_doctype_cache(doctype) + + # "restore" doctype by inserting doctype with same schema again + new_doctype(doctype, custom=0).insert() + + # Ensure basically same doctype getting "restored" + restored_doc = frappe.get_last_doc(doctype) + verify_fields = ["doctype", field, custom_field] + for f in verify_fields: + self.assertEqual(original_doc.get(f), restored_doc.get(f)) + + # Check form load of restored doctype + getdoc(doctype, restored_doc.name) + + # ensure meta - property setter + self.assertEqual(frappe.get_meta(doctype).get_field(field).default, "DELETETHIS") + frappe.delete_doc("DocType", doctype) + def new_doctype( - name, unique: bool = False, depends_on: str = "", fields: Optional[List[Dict]] = None, **kwargs + name: str | None = None, + unique: bool = False, + depends_on: str = "", + fields: list[dict] | None = None, + **kwargs, ): + if not name: + # Test prefix is required to avoid coverage + name = "Test " + "".join(random.sample(string.ascii_lowercase, 10)) + doc = frappe.get_doc( { "doctype": "DocType", diff --git a/frappe/core/doctype/doctype_action/doctype_action.py b/frappe/core/doctype/doctype_action/doctype_action.py index 49cb21f99f..1c5be9ae01 100644 --- a/frappe/core/doctype/doctype_action/doctype_action.py +++ b/frappe/core/doctype/doctype_action/doctype_action.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and contributors # License: MIT. See LICENSE diff --git a/frappe/core/doctype/doctype_link/doctype_link.py b/frappe/core/doctype/doctype_link/doctype_link.py index f534cd1780..0658b1fb28 100644 --- a/frappe/core/doctype/doctype_link/doctype_link.py +++ b/frappe/core/doctype/doctype_link/doctype_link.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and contributors # License: MIT. See LICENSE diff --git a/frappe/core/doctype/document_naming_rule/document_naming_rule.js b/frappe/core/doctype/document_naming_rule/document_naming_rule.js index 097a4e9a6e..70d95673e6 100644 --- a/frappe/core/doctype/document_naming_rule/document_naming_rule.js +++ b/frappe/core/doctype/document_naming_rule/document_naming_rule.js @@ -1,64 +1,70 @@ // Copyright (c) 2020, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Document Naming Rule', { - refresh: function(frm) { - frm.trigger('document_type'); +frappe.ui.form.on("Document Naming Rule", { + refresh: function (frm) { + frm.trigger("document_type"); if (!frm.doc.__islocal) frm.trigger("add_update_counter_button"); }, document_type: (frm) => { // update the select field options with fieldnames if (frm.doc.document_type) { frappe.model.with_doctype(frm.doc.document_type, () => { - let fieldnames = frappe.get_meta(frm.doc.document_type).fields - .filter((d) => { + let fieldnames = frappe + .get_meta(frm.doc.document_type) + .fields.filter((d) => { return frappe.model.no_value_type.indexOf(d.fieldtype) === -1; - }).map((d) => { - return {label: `${d.label} (${d.fieldname})`, value: d.fieldname}; + }) + .map((d) => { + return { label: `${d.label} (${d.fieldname})`, value: d.fieldname }; }); frm.fields_dict.conditions.grid.update_docfield_property( - 'field', 'options', fieldnames + "field", + "options", + fieldnames ); }); } }, add_update_counter_button: (frm) => { - frm.add_custom_button(__('Update Counter'), function() { + frm.add_custom_button(__("Update Counter"), function () { + const fields = [ + { + fieldtype: "Data", + fieldname: "new_counter", + label: __("New Counter"), + default: frm.doc.counter, + reqd: 1, + description: __( + "Warning: Updating counter may lead to document name conflicts if not done properly" + ), + }, + ]; - const fields = [{ - fieldtype: 'Data', - fieldname: 'new_counter', - label: __('New Counter'), - default: frm.doc.counter, - reqd: 1, - description: __('Warning: Updating counter may lead to document name conflicts if not done properly') - }]; - - let primary_action_label = __('Save'); + let primary_action_label = __("Save"); let primary_action = (fields) => { frappe.call({ - method: 'frappe.core.doctype.document_naming_rule.document_naming_rule.update_current', + method: "frappe.core.doctype.document_naming_rule.document_naming_rule.update_current", args: { name: frm.doc.name, - new_counter: fields.new_counter + new_counter: fields.new_counter, }, - callback: function() { + callback: function () { frm.set_value("counter", fields.new_counter); dialog.hide(); - } + }, }); }; const dialog = new frappe.ui.Dialog({ - title: __('Update Counter Value for Prefix: {0}', [frm.doc.prefix]), + title: __("Update Counter Value for Prefix: {0}", [frm.doc.prefix]), fields, primary_action_label, - primary_action + primary_action, }); dialog.show(); - }); - } + }, }); diff --git a/frappe/core/doctype/document_naming_rule/document_naming_rule.py b/frappe/core/doctype/document_naming_rule/document_naming_rule.py index 41aa3d7aff..3fecf26ade 100644 --- a/frappe/core/doctype/document_naming_rule/document_naming_rule.py +++ b/frappe/core/doctype/document_naming_rule/document_naming_rule.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and contributors # License: MIT. See LICENSE diff --git a/frappe/core/doctype/document_naming_rule/test_document_naming_rule.py b/frappe/core/doctype/document_naming_rule/test_document_naming_rule.py index 459b17da8b..cc406ed5cd 100644 --- a/frappe/core/doctype/document_naming_rule/test_document_naming_rule.py +++ b/frappe/core/doctype/document_naming_rule/test_document_naming_rule.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and Contributors # License: MIT. See LICENSE import unittest diff --git a/frappe/core/doctype/document_naming_rule_condition/document_naming_rule_condition.js b/frappe/core/doctype/document_naming_rule_condition/document_naming_rule_condition.js index 8ef39c7b70..fdf46e82e0 100644 --- a/frappe/core/doctype/document_naming_rule_condition/document_naming_rule_condition.js +++ b/frappe/core/doctype/document_naming_rule_condition/document_naming_rule_condition.js @@ -1,8 +1,7 @@ // Copyright (c) 2020, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Document Naming Rule Condition', { +frappe.ui.form.on("Document Naming Rule Condition", { // refresh: function(frm) { - // } }); diff --git a/frappe/core/doctype/document_naming_rule_condition/document_naming_rule_condition.py b/frappe/core/doctype/document_naming_rule_condition/document_naming_rule_condition.py index dc45798c34..7ecc0162a5 100644 --- a/frappe/core/doctype/document_naming_rule_condition/document_naming_rule_condition.py +++ b/frappe/core/doctype/document_naming_rule_condition/document_naming_rule_condition.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and contributors # License: MIT. See LICENSE diff --git a/frappe/core/doctype/document_naming_rule_condition/test_document_naming_rule_condition.py b/frappe/core/doctype/document_naming_rule_condition/test_document_naming_rule_condition.py index d88335758a..68f0677f2c 100644 --- a/frappe/core/doctype/document_naming_rule_condition/test_document_naming_rule_condition.py +++ b/frappe/core/doctype/document_naming_rule_condition/test_document_naming_rule_condition.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and Contributors # License: MIT. See LICENSE # import frappe diff --git a/frappe/core/doctype/payment_gateway/__init__.py b/frappe/core/doctype/document_naming_settings/__init__.py similarity index 100% rename from frappe/core/doctype/payment_gateway/__init__.py rename to frappe/core/doctype/document_naming_settings/__init__.py diff --git a/frappe/core/doctype/document_naming_settings/document_naming_settings.js b/frappe/core/doctype/document_naming_settings/document_naming_settings.js new file mode 100644 index 0000000000..2a9ec4aae5 --- /dev/null +++ b/frappe/core/doctype/document_naming_settings/document_naming_settings.js @@ -0,0 +1,70 @@ +// Copyright (c) 2022, Frappe Technologies and contributors +// For license information, please see license.txt + +frappe.ui.form.on("Document Naming Settings", { + refresh: function (frm) { + frm.trigger("setup_transaction_autocomplete"); + frm.disable_save(); + }, + + setup_transaction_autocomplete: function (frm) { + frappe.call({ + method: "get_transactions_and_prefixes", + doc: frm.doc, + callback: function (r) { + frm.fields_dict.transaction_type.set_data(r.message.transactions); + frm.fields_dict.prefix.set_data(r.message.prefixes); + }, + }); + }, + + transaction_type: function (frm) { + frm.set_value("user_must_always_select", 0); + frappe.call({ + method: "get_options", + doc: frm.doc, + callback: function (r) { + frm.set_value("naming_series_options", r.message); + if (r.message && r.message.split("\n")[0] == "") + frm.set_value("user_must_always_select", 1); + }, + }); + }, + + prefix: function (frm) { + frappe.call({ + method: "get_current", + doc: frm.doc, + callback: function (r) { + frm.refresh_field("current_value"); + }, + }); + }, + + update: function (frm) { + frappe.call({ + method: "update_series", + doc: frm.doc, + freeze: true, + freeze_msg: __("Updating naming series options"), + callback: function (r) { + frm.trigger("setup_transaction_autocomplete"); + frm.trigger("transaction_type"); + }, + }); + }, + + try_naming_series(frm) { + frappe.call({ + method: "preview_series", + doc: frm.doc, + callback: function (r) { + if (!r.exc) { + frm.set_value("series_preview", r.message); + } else { + frm.set_value("series_preview", __("Failed to generate preview of series")); + } + }, + }); + }, +}); diff --git a/frappe/core/doctype/document_naming_settings/document_naming_settings.json b/frappe/core/doctype/document_naming_settings/document_naming_settings.json new file mode 100644 index 0000000000..4c86b2ec1d --- /dev/null +++ b/frappe/core/doctype/document_naming_settings/document_naming_settings.json @@ -0,0 +1,133 @@ +{ + "actions": [], + "creation": "2022-05-30 07:24:07.736646", + "description": "Configure various aspects of how document naming works like naming series, current counter.", + "doctype": "DocType", + "engine": "InnoDB", + "field_order": [ + "naming_series_tab", + "setup_series", + "transaction_type", + "naming_series_options", + "user_must_always_select", + "update", + "column_break_9", + "try_naming_series", + "series_preview", + "help_html", + "update_series", + "prefix", + "current_value", + "update_series_start" + ], + "fields": [ + { + "collapsible": 1, + "description": "Set Naming Series options on your transactions.", + "fieldname": "setup_series", + "fieldtype": "Section Break", + "label": "Setup Series for transactions" + }, + { + "depends_on": "transaction_type", + "fieldname": "help_html", + "fieldtype": "HTML", + "label": "Help HTML", + "options": "
    \n Edit list of Series in the box. Rules:\n
      \n
    • Each Series Prefix on a new line.
      • \n
    • Allowed special characters are \"/\" and \"-\"
      • \n
    • \n Optionally, set the number of digits in the series using dot (.)\n followed by hashes (#). For example, \".####\" means that the series\n will have four digits. Default is five digits.\n
      • \n
    • \n You can also use variables in the series name by putting them\n between (.) dots\n
      \n Supported Variables:\n
        \n
      • .YYYY. - Year in 4 digits
        • \n
      • .YY. - Year in 2 digits
        • \n
      • .MM. - Month
        • \n
      • .DD. - Day of month
        • \n
      • .WW. - Week of the year
        • \n
      • .FY. - Fiscal Year
        • \n
      • \n .{fieldname}. - fieldname on the document e.g.\n branch\n
        • \n
      \n
      • \n
    \n Examples:\n
      \n
    • INV-
      • \n
    • INV-10-
      • \n
    • INVK-
      • \n
    • INV-.YYYY.-.{branch}.-.MM.-.####
      • \n
    \n
    \n
    \n" + }, + { + "default": "0", + "depends_on": "transaction_type", + "description": "Check this if you want to force the user to select a series before saving. There will be no default if you check this.", + "fieldname": "user_must_always_select", + "fieldtype": "Check", + "label": "User must always select" + }, + { + "depends_on": "transaction_type", + "fieldname": "update", + "fieldtype": "Button", + "label": "Update" + }, + { + "collapsible": 1, + "description": "Change the starting / current sequence number of an existing series.
    \n\nWarning: Incorrectly updating counters can prevent documents from getting created. ", + "fieldname": "update_series", + "fieldtype": "Section Break", + "label": "Update Series Counter" + }, + { + "fieldname": "prefix", + "fieldtype": "Autocomplete", + "label": "Prefix" + }, + { + "description": "This is the number of the last created transaction with this prefix", + "fieldname": "current_value", + "fieldtype": "Int", + "label": "Current Value" + }, + { + "fieldname": "update_series_start", + "fieldtype": "Button", + "label": "Update Series Number", + "options": "update_series_start" + }, + { + "depends_on": "transaction_type", + "fieldname": "naming_series_options", + "fieldtype": "Text", + "label": "Series List for this Transaction" + }, + { + "depends_on": "transaction_type", + "description": "Generate 3 preview of names generate by any valid series.", + "fieldname": "try_naming_series", + "fieldtype": "Data", + "label": "Try a naming Series" + }, + { + "fieldname": "transaction_type", + "fieldtype": "Autocomplete", + "label": "Select Transaction" + }, + { + "fieldname": "column_break_9", + "fieldtype": "Column Break" + }, + { + "fieldname": "naming_series_tab", + "fieldtype": "Tab Break", + "label": "Naming Series" + }, + { + "fieldname": "series_preview", + "fieldtype": "Text", + "label": "Preview of generated names", + "read_only": 1 + } + ], + "hide_toolbar": 1, + "icon": "fa fa-sort-by-order", + "issingle": 1, + "links": [], + "modified": "2022-05-30 23:51:36.136535", + "modified_by": "Administrator", + "module": "Core", + "name": "Document Naming Settings", + "owner": "Administrator", + "permissions": [ + { + "create": 1, + "email": 1, + "print": 1, + "read": 1, + "role": "System Manager", + "share": 1, + "write": 1 + } + ], + "sort_field": "modified", + "sort_order": "DESC", + "states": [] +} diff --git a/frappe/core/doctype/document_naming_settings/document_naming_settings.py b/frappe/core/doctype/document_naming_settings/document_naming_settings.py new file mode 100644 index 0000000000..6fc4d9b23e --- /dev/null +++ b/frappe/core/doctype/document_naming_settings/document_naming_settings.py @@ -0,0 +1,220 @@ +# Copyright (c) 2022, Frappe Technologies and contributors +# For license information, please see license.txt + + +import frappe +from frappe import _ +from frappe.core.doctype.doctype.doctype import validate_series +from frappe.model.document import Document +from frappe.model.naming import NamingSeries +from frappe.permissions import get_doctypes_with_read + + +class NamingSeriesNotSetError(frappe.ValidationError): + pass + + +class DocumentNamingSettings(Document): + @frappe.whitelist() + def get_transactions_and_prefixes(self): + + transactions = self._get_transactions() + prefixes = self._get_prefixes(transactions) + + return {"transactions": transactions, "prefixes": prefixes} + + def _get_transactions(self) -> list[str]: + + readable_doctypes = set(get_doctypes_with_read()) + + standard = frappe.get_all("DocField", {"fieldname": "naming_series"}, "parent", pluck="parent") + custom = frappe.get_all("Custom Field", {"fieldname": "naming_series"}, "dt", pluck="dt") + + return sorted(readable_doctypes.intersection(standard + custom)) + + def _get_prefixes(self, doctypes) -> list[str]: + """Get all prefixes for naming series. + + - For all templates prefix is evaluated considering today's date + - All existing prefix in DB are shared as is. + """ + series_templates = set() + for d in doctypes: + try: + options = frappe.get_meta(d).get_naming_series_options() + series_templates.update(options) + except frappe.DoesNotExistError: + frappe.msgprint(_("Unable to find DocType {0}").format(d)) + continue + + custom_templates = frappe.get_all( + "DocType", + fields=["autoname"], + filters={ + "name": ("not in", doctypes), + "autoname": ("like", "%.#%"), + "module": ("not in", ["Core"]), + }, + ) + if custom_templates: + series_templates.update([d.autoname.rsplit(".", 1)[0] for d in custom_templates]) + + return self._evaluate_and_clean_templates(series_templates) + + def _evaluate_and_clean_templates(self, series_templates: set[str]) -> list[str]: + evalauted_prefix = set() + + series = frappe.qb.DocType("Series") + prefixes_from_db = frappe.qb.from_(series).select(series.name).run(pluck=True) + evalauted_prefix.update(prefixes_from_db) + + for series_template in series_templates: + try: + prefix = NamingSeries(series_template).get_prefix() + if "{" in prefix: + # fieldnames can't be evalauted, rely on data in DB instead + continue + evalauted_prefix.add(prefix) + except Exception: + frappe.clear_last_message() + frappe.log_error(f"Invalid naming series {series_template}") + + return sorted(evalauted_prefix) + + def get_options_list(self, options: str) -> list[str]: + return [op.strip() for op in options.split("\n") if op.strip()] + + @frappe.whitelist() + def update_series(self): + """update series list""" + self.validate_set_series() + self.check_duplicate() + self.set_series_options_in_meta(self.transaction_type, self.naming_series_options) + + frappe.msgprint( + _("Series Updated for {}").format(self.transaction_type), alert=True, indicator="green" + ) + + def validate_set_series(self): + if self.transaction_type and not self.naming_series_options: + frappe.throw(_("Please set the series to be used.")) + + def set_series_options_in_meta(self, doctype: str, options: str) -> None: + options = self.get_options_list(options) + + # validate names + for series in options: + self.validate_series_name(series) + + if options and self.user_must_always_select: + options = [""] + options + + default = options[0] if options else "" + + option_string = "\n".join(options) + + self.update_naming_series_property_setter(doctype, "options", option_string) + self.update_naming_series_property_setter(doctype, "default", default) + + self.naming_series_options = option_string + + frappe.clear_cache(doctype=doctype) + + def update_naming_series_property_setter(self, doctype, property, value): + from frappe.custom.doctype.property_setter.property_setter import make_property_setter + + make_property_setter(doctype, "naming_series", property, value, "Text") + + def check_duplicate(self): + def stripped_series(s: str) -> str: + return s.strip().rstrip("#") + + standard = frappe.get_all("DocField", {"fieldname": "naming_series"}, "parent", pluck="parent") + custom = frappe.get_all("Custom Field", {"fieldname": "naming_series"}, "dt", pluck="dt") + + all_doctypes_with_naming_series = set(standard + custom) + all_doctypes_with_naming_series.remove(self.transaction_type) + + existing_series = {} + for doctype in all_doctypes_with_naming_series: + for series in frappe.get_meta(doctype).get_naming_series_options(): + existing_series[stripped_series(series)] = doctype + + dt = frappe.get_doc("DocType", self.transaction_type) + + options = self.get_options_list(self.naming_series_options) + for series in options: + if stripped_series(series) in existing_series: + frappe.throw(_("Series {0} already used in {1}").format(series, existing_series[series])) + validate_series(dt, series) + + def validate_series_name(self, series): + NamingSeries(series).validate() + + @frappe.whitelist() + def get_options(self, doctype=None): + doctype = doctype or self.transaction_type + if not doctype: + return + + if frappe.get_meta(doctype or self.transaction_type).get_field("naming_series"): + return frappe.get_meta(doctype or self.transaction_type).get_field("naming_series").options + + @frappe.whitelist() + def get_current(self): + """get series current""" + if self.prefix: + self.current_value = NamingSeries(self.prefix).get_current_value() + return self.current_value + + @frappe.whitelist() + def update_series_start(self): + frappe.only_for("System Manager") + + if not self.prefix: + frappe.throw(_("Please select prefix first")) + + naming_series = NamingSeries(self.prefix) + previous_value = naming_series.get_current_value() + naming_series.update_counter(self.current_value) + + self.create_version_log_for_change( + naming_series.get_prefix(), previous_value, self.current_value + ) + + frappe.msgprint( + _("Series counter for {} updated to {} successfully").format(self.prefix, self.current_value), + alert=True, + indicator="green", + ) + + def create_version_log_for_change(self, series, old, new): + version = frappe.new_doc("Version") + version.ref_doctype = "Series" + version.docname = series + version.data = frappe.as_json({"changed": [["current", old, new]]}) + version.flags.ignore_links = True # series is not a "real" doctype + version.flags.ignore_permissions = True + version.insert() + + @frappe.whitelist() + def preview_series(self) -> str: + """Preview what the naming series will generate.""" + + series = self.try_naming_series + if not series: + return "" + try: + doc = self._fetch_last_doc_if_available() + return "\n".join(NamingSeries(series).get_preview(doc=doc)) + except Exception as e: + if frappe.message_log: + frappe.message_log.pop() + return _("Failed to generate names from the series") + f"\n{str(e)}" + + def _fetch_last_doc_if_available(self): + """Fetch last doc for evaluating naming series with fields.""" + try: + return frappe.get_last_doc(self.transaction_type) + except Exception: + return None diff --git a/frappe/core/doctype/document_naming_settings/test_document_naming_settings.py b/frappe/core/doctype/document_naming_settings/test_document_naming_settings.py new file mode 100644 index 0000000000..98ce9e738b --- /dev/null +++ b/frappe/core/doctype/document_naming_settings/test_document_naming_settings.py @@ -0,0 +1,65 @@ +# Copyright (c) 2022, Frappe Technologies and Contributors +# See license.txt + +import frappe +from frappe.core.doctype.document_naming_settings.document_naming_settings import ( + DocumentNamingSettings, +) +from frappe.model.naming import NamingSeries, get_default_naming_series +from frappe.tests.utils import FrappeTestCase +from frappe.utils import cint + + +class TestNamingSeries(FrappeTestCase): + def setUp(self): + self.dns: DocumentNamingSettings = frappe.get_doc("Document Naming Settings") + + def tearDown(self): + frappe.db.rollback() + + def get_valid_serieses(self): + VALID_SERIES = ["SINV-", "SI-.{field}.", "SI-#.###", ""] + exisiting_series = self.dns.get_transactions_and_prefixes()["prefixes"] + return VALID_SERIES + exisiting_series + + def test_naming_preview(self): + self.dns.transaction_type = "Webhook" + + self.dns.try_naming_series = "AXBZ.####" + serieses = self.dns.preview_series().split("\n") + self.assertEqual(["AXBZ0001", "AXBZ0002", "AXBZ0003"], serieses) + + self.dns.try_naming_series = "AXBZ-.{currency}.-" + serieses = self.dns.preview_series().split("\n") + + def test_get_transactions(self): + + naming_info = self.dns.get_transactions_and_prefixes() + self.assertIn("Webhook", naming_info["transactions"]) + + existing_naming_series = frappe.get_meta("Webhook").get_field("naming_series").options + + for series in existing_naming_series.split("\n"): + self.assertIn(NamingSeries(series).get_prefix(), naming_info["prefixes"]) + + def test_default_naming_series(self): + self.assertIn("HOOK", get_default_naming_series("Webhook")) + self.assertIsNone(get_default_naming_series("DocType")) + + def test_updates_naming_options(self): + self.dns.transaction_type = "Webhook" + test_series = "KOOHBEW.###" + self.dns.naming_series_options = self.dns.get_options() + "\n" + test_series + self.dns.update_series() + self.assertIn(test_series, frappe.get_meta("Webhook").get_naming_series_options()) + + def test_update_series_counter(self): + for series in self.get_valid_serieses(): + if not series: + continue + self.dns.prefix = series + current_count = cint(self.dns.get_current()) + new_count = self.dns.current_value = current_count + 1 + self.dns.update_series_start() + + self.assertEqual(self.dns.get_current(), new_count, f"Incorrect update for {series}") diff --git a/frappe/core/doctype/test/__init__.py b/frappe/core/doctype/document_share_key/__init__.py similarity index 100% rename from frappe/core/doctype/test/__init__.py rename to frappe/core/doctype/document_share_key/__init__.py diff --git a/frappe/core/doctype/test/test.js b/frappe/core/doctype/document_share_key/document_share_key.js similarity index 78% rename from frappe/core/doctype/test/test.js rename to frappe/core/doctype/document_share_key/document_share_key.js index e423c58686..7e1712beff 100644 --- a/frappe/core/doctype/test/test.js +++ b/frappe/core/doctype/document_share_key/document_share_key.js @@ -1,8 +1,7 @@ // Copyright (c) 2021, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('test', { +frappe.ui.form.on("Document Share Key", { // refresh: function(frm) { - // } }); diff --git a/frappe/core/doctype/feedback/feedback.json b/frappe/core/doctype/document_share_key/document_share_key.json similarity index 54% rename from frappe/core/doctype/feedback/feedback.json rename to frappe/core/doctype/document_share_key/document_share_key.json index f8380cfda6..b96fe09f0b 100644 --- a/frappe/core/doctype/feedback/feedback.json +++ b/frappe/core/doctype/document_share_key/document_share_key.json @@ -1,56 +1,56 @@ { "actions": [], - "creation": "2021-06-03 19:02:55.328423", + "allow_rename": 1, + "autoname": "hash", + "creation": "2022-01-14 13:40:49.487646", "doctype": "DocType", "editable_grid": 1, "engine": "InnoDB", "field_order": [ "reference_doctype", - "reference_name", - "column_break_3", - "like", - "ip_address" + "reference_docname", + "key", + "expires_on" ], "fields": [ - { - "fieldname": "column_break_3", - "fieldtype": "Column Break" - }, { "fieldname": "reference_doctype", - "fieldtype": "Select", - "in_list_view": 1, + "fieldtype": "Link", "label": "Reference Document Type", - "options": "\nBlog Post" + "options": "DocType", + "read_only": 1, + "search_index": 1 }, { - "fieldname": "reference_name", + "fieldname": "reference_docname", "fieldtype": "Dynamic Link", - "in_list_view": 1, - "label": "Reference Name", + "label": "Reference Document Name", "options": "reference_doctype", - "reqd": 1 + "read_only": 1, + "search_index": 1 }, { - "fieldname": "ip_address", + "fieldname": "key", "fieldtype": "Data", - "hidden": 1, - "label": "IP Address", + "label": "Key", "read_only": 1 }, { - "default": "0", - "fieldname": "like", - "fieldtype": "Check", - "label": "Like" + "fieldname": "expires_on", + "fieldtype": "Date", + "in_list_view": 1, + "label": "Expires On", + "read_only": 1 } ], + "in_create": 1, "index_web_pages_for_search": 1, "links": [], - "modified": "2021-11-10 20:53:21.255593", + "modified": "2022-01-14 13:57:28.050678", "modified_by": "Administrator", "module": "Core", - "name": "Feedback", + "name": "Document Share Key", + "naming_rule": "Expression", "owner": "Administrator", "permissions": [ { @@ -66,8 +66,8 @@ "write": 1 } ], + "read_only": 1, "sort_field": "modified", "sort_order": "DESC", - "title_field": "reference_name", - "track_changes": 1 + "states": [] } \ No newline at end of file diff --git a/frappe/core/doctype/document_share_key/document_share_key.py b/frappe/core/doctype/document_share_key/document_share_key.py new file mode 100644 index 0000000000..88608b992c --- /dev/null +++ b/frappe/core/doctype/document_share_key/document_share_key.py @@ -0,0 +1,20 @@ +# Copyright (c) 2021, Frappe Technologies and contributors +# For license information, please see license.txt + +from random import randrange + +import frappe +from frappe.model.document import Document + + +class DocumentShareKey(Document): + def before_insert(self): + self.key = frappe.generate_hash(length=randrange(25, 35)) + if not self.expires_on and not self.flags.no_expiry: + self.expires_on = frappe.utils.add_days( + None, days=frappe.get_system_settings("document_share_key_expiry") or 90 + ) + + +def is_expired(expires_on): + return expires_on and expires_on < frappe.utils.getdate() diff --git a/frappe/core/doctype/test/test_test.py b/frappe/core/doctype/document_share_key/test_document_share_key.py similarity index 53% rename from frappe/core/doctype/test/test_test.py rename to frappe/core/doctype/document_share_key/test_document_share_key.py index e4ee3de5dd..10499fcc5d 100644 --- a/frappe/core/doctype/test/test_test.py +++ b/frappe/core/doctype/document_share_key/test_document_share_key.py @@ -1,9 +1,9 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2021, Frappe Technologies and Contributors -# License: MIT. See LICENSE +# See license.txt + # import frappe import unittest -class Testtest(unittest.TestCase): +class TestDocumentShareKey(unittest.TestCase): pass diff --git a/frappe/core/doctype/domain/domain.js b/frappe/core/doctype/domain/domain.js index 397ed4b19c..9b51c10d77 100644 --- a/frappe/core/doctype/domain/domain.js +++ b/frappe/core/doctype/domain/domain.js @@ -1,8 +1,6 @@ // Copyright (c) 2017, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Domain', { - refresh: function(frm) { - - } +frappe.ui.form.on("Domain", { + refresh: function (frm) {}, }); diff --git a/frappe/core/doctype/domain/domain.py b/frappe/core/doctype/domain/domain.py index 897f7ee655..e0b0e80982 100644 --- a/frappe/core/doctype/domain/domain.py +++ b/frappe/core/doctype/domain/domain.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2017, Frappe Technologies and contributors # License: MIT. See LICENSE @@ -113,8 +112,8 @@ class Domain(Document): # enable frappe.db.sql( """update `tabPortal Menu Item` set enabled=1 - where route in ({0})""".format( - ", ".join('"{0}"'.format(d) for d in self.data.allow_sidebar_items) + where route in ({})""".format( + ", ".join(f'"{d}"' for d in self.data.allow_sidebar_items) ) ) @@ -125,7 +124,7 @@ class Domain(Document): # enable frappe.db.sql( """update `tabPortal Menu Item` set enabled=0 - where route in ({0})""".format( - ", ".join('"{0}"'.format(d) for d in self.data.remove_sidebar_items) + where route in ({})""".format( + ", ".join(f'"{d}"' for d in self.data.remove_sidebar_items) ) ) diff --git a/frappe/core/doctype/domain/test_domain.py b/frappe/core/doctype/domain/test_domain.py index 85f613a6bd..32592705b4 100644 --- a/frappe/core/doctype/domain/test_domain.py +++ b/frappe/core/doctype/domain/test_domain.py @@ -1,10 +1,7 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2017, Frappe Technologies and Contributors # License: MIT. See LICENSE import unittest -import frappe - class TestDomain(unittest.TestCase): pass diff --git a/frappe/core/doctype/domain_settings/domain_settings.js b/frappe/core/doctype/domain_settings/domain_settings.js index 7178cb4cd6..87386ce9bd 100644 --- a/frappe/core/doctype/domain_settings/domain_settings.js +++ b/frappe/core/doctype/domain_settings/domain_settings.js @@ -1,66 +1,69 @@ // Copyright (c) 2017, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Domain Settings', { - before_load: function(frm) { - if(!frm.domains_multicheck) { +frappe.ui.form.on("Domain Settings", { + before_load: function (frm) { + if (!frm.domains_multicheck) { frm.domains_multicheck = frappe.ui.form.make_control({ parent: frm.fields_dict.domains_html.$wrapper, df: { fieldname: "domains_multicheck", fieldtype: "MultiCheck", get_data: () => { - let active_domains = (frm.doc.active_domains || []).map(row => row.domain); - return frappe.boot.all_domains.map(domain => { + let active_domains = (frm.doc.active_domains || []).map( + (row) => row.domain + ); + return frappe.boot.all_domains.map((domain) => { return { label: domain, value: domain, - checked: active_domains.includes(domain) + checked: active_domains.includes(domain), }; }); }, on_change: () => { frm.dirty(); - } + }, }, - render_input: true + render_input: true, }); frm.domains_multicheck.refresh_input(); } }, - validate: function(frm) { - frm.trigger('set_options_in_table'); + validate: function (frm) { + frm.trigger("set_options_in_table"); }, - set_options_in_table: function(frm) { + set_options_in_table: function (frm) { let selected_options = frm.domains_multicheck.get_value(); let unselected_options = frm.domains_multicheck.options - .map(option => option.value) - .filter(value => { + .map((option) => option.value) + .filter((value) => { return !selected_options.includes(value); }); - let map = {}, list = []; - (frm.doc.active_domains || []).map(row => { + let map = {}, + list = []; + (frm.doc.active_domains || []).map((row) => { map[row.domain] = row.name; list.push(row.domain); }); - unselected_options.map(option => { - if(list.includes(option)) { + unselected_options.map((option) => { + if (list.includes(option)) { frappe.model.clear_doc("Has Domain", map[option]); } }); - selected_options.map(option => { - if(!list.includes(option)) { + selected_options.map((option) => { + if (!list.includes(option)) { frappe.model.clear_doc("Has Domain", map[option]); let row = frappe.model.add_child(frm.doc, "Has Domain", "active_domains"); row.domain = option; } }); - refresh_field('active_domains'); - } + refresh_field("active_domains"); + }, }); diff --git a/frappe/core/doctype/domain_settings/domain_settings.json b/frappe/core/doctype/domain_settings/domain_settings.json index 8efd296da6..c363529cbd 100644 --- a/frappe/core/doctype/domain_settings/domain_settings.json +++ b/frappe/core/doctype/domain_settings/domain_settings.json @@ -1,153 +1,56 @@ { - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, - "creation": "2017-05-03 16:28:11.295095", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", + "actions": [], + "creation": "2017-05-03 16:28:11.295095", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "active_domains_sb", + "domains_html", + "active_domains" + ], "fields": [ { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "active_domains_sb", - "fieldtype": "Section Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Active Domains", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "active_domains_sb", + "fieldtype": "Section Break", + "label": "Active Domains" + }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "domains_html", - "fieldtype": "HTML", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Domains HTML", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "domains_html", + "fieldtype": "HTML", + "label": "Domains HTML" + }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "active_domains", - "fieldtype": "Table", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Active Domains", - "length": 0, - "no_copy": 0, - "options": "Has Domain", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "fieldname": "active_domains", + "fieldtype": "Table", + "hidden": 1, + "label": "Active Domains", + "options": "Has Domain", + "read_only": 1 } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 1, - "istable": 0, - "max_attachments": 0, - "modified": "2017-12-05 17:36:46.842134", - "modified_by": "Administrator", - "module": "Core", - "name": "Domain Settings", - "name_case": "", - "owner": "Administrator", + ], + "issingle": 1, + "links": [], + "modified": "2022-08-03 12:20:53.256607", + "modified_by": "Administrator", + "module": "Core", + "name": "Domain Settings", + "owner": "Administrator", "permissions": [ { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 1, - "delete": 1, - "email": 1, - "export": 0, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 0, - "role": "System Manager", - "set_user_permissions": 0, - "share": 1, - "submit": 0, + "create": 1, + "delete": 1, + "email": 1, + "print": 1, + "read": 1, + "role": "System Manager", + "share": 1, "write": 1 } - ], - "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1, - "track_seen": 0 + ], + "quick_entry": 1, + "sort_field": "modified", + "sort_order": "DESC", + "states": [], + "track_changes": 1 } \ No newline at end of file diff --git a/frappe/core/doctype/domain_settings/domain_settings.py b/frappe/core/doctype/domain_settings/domain_settings.py index ba7f397c51..85b26f53dd 100644 --- a/frappe/core/doctype/domain_settings/domain_settings.py +++ b/frappe/core/doctype/domain_settings/domain_settings.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2017, Frappe Technologies and contributors # License: MIT. See LICENSE @@ -32,7 +31,7 @@ class DomainSettings(Document): def restrict_roles_and_modules(self): """Disable all restricted roles and set `restrict_to_domain` property in Module Def""" active_domains = frappe.get_active_domains() - all_domains = list((frappe.get_hooks("domains") or {})) + all_domains = list(frappe.get_hooks("domains") or {}) def remove_role(role): frappe.db.delete("Has Role", {"role": role}) diff --git a/frappe/core/doctype/dynamic_link/dynamic_link.py b/frappe/core/doctype/dynamic_link/dynamic_link.py index e253147167..53eb750b5c 100644 --- a/frappe/core/doctype/dynamic_link/dynamic_link.py +++ b/frappe/core/doctype/dynamic_link/dynamic_link.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and contributors # License: MIT. See LICENSE diff --git a/frappe/core/doctype/error_log/error_log.js b/frappe/core/doctype/error_log/error_log.js index 4fe8fde5d6..85b1c8b60a 100644 --- a/frappe/core/doctype/error_log/error_log.js +++ b/frappe/core/doctype/error_log/error_log.js @@ -1,8 +1,17 @@ -// Copyright (c) 2016, Frappe Technologies and contributors +// Copyright (c) 2022, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Error Log', { - refresh: function(frm) { +frappe.ui.form.on("Error Log", { + refresh: function (frm) { + frm.disable_save(); - } + if (frm.doc.reference_doctype && frm.doc.reference_name) { + frm.add_custom_button(__("Show Related Errors"), function () { + frappe.set_route("List", "Error Log", { + reference_doctype: frm.doc.reference_doctype, + reference_name: frm.doc.reference_name, + }); + }); + } + }, }); diff --git a/frappe/core/doctype/error_log/error_log.json b/frappe/core/doctype/error_log/error_log.json index 35ca3ceeef..2ee86bd118 100644 --- a/frappe/core/doctype/error_log/error_log.json +++ b/frappe/core/doctype/error_log/error_log.json @@ -1,148 +1,88 @@ { - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "autoname": "", - "beta": 0, - "creation": "2013-01-16 13:09:40", - "custom": 0, - "description": "Log of Scheduler Errors", - "docstatus": 0, - "doctype": "DocType", - "document_type": "System", - "editable_grid": 0, - "engine": "MyISAM", + "actions": [], + "creation": "2013-01-16 13:09:40", + "doctype": "DocType", + "document_type": "System", + "engine": "MyISAM", + "field_order": [ + "seen", + "reference_doctype", + "column_break_3", + "reference_name", + "section_break_5", + "method", + "error" + ], "fields": [ { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "default": "0", - "fieldname": "seen", - "fieldtype": "Check", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Seen", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "default": "0", + "fieldname": "seen", + "fieldtype": "Check", + "hidden": 1, + "label": "Seen" + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "method", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Title", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "method", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Title", + "read_only": 1 + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "error", - "fieldtype": "Code", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Error", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "fieldname": "error", + "fieldtype": "Code", + "label": "Error", + "read_only": 1 + }, + { + "fieldname": "reference_doctype", + "fieldtype": "Link", + "in_list_view": 1, + "in_standard_filter": 1, + "label": "Reference DocType", + "options": "DocType", + "read_only": 1, + "search_index": 1 + }, + { + "fieldname": "reference_name", + "fieldtype": "Data", + "label": "Reference Name", + "read_only": 1 + }, + { + "fieldname": "column_break_3", + "fieldtype": "Column Break" + }, + { + "fieldname": "section_break_5", + "fieldtype": "Section Break" } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "icon": "fa fa-warning-sign", - "idx": 1, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2021-10-25 12:21:44.292471", - "modified_by": "Administrator", - "module": "Core", - "name": "Error Log", - "owner": "Administrator", + ], + "icon": "fa fa-warning-sign", + "idx": 1, + "in_create": 1, + "links": [], + "modified": "2022-06-13 06:34:05.158606", + "modified_by": "Administrator", + "module": "Core", + "name": "Error Log", + "owner": "Administrator", "permissions": [ { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 1, - "delete": 1, - "email": 1, - "export": 0, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "set_user_permissions": 0, - "share": 1, - "submit": 0, + "create": 1, + "delete": 1, + "email": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1, "write": 1 } - ], - "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_order": "ASC", - "track_seen": 0 -} + ], + "sort_field": "modified", + "sort_order": "ASC", + "states": [], + "title_field": "method" +} \ No newline at end of file diff --git a/frappe/core/doctype/error_log/error_log.py b/frappe/core/doctype/error_log/error_log.py index d93029179c..c7ab98e034 100644 --- a/frappe/core/doctype/error_log/error_log.py +++ b/frappe/core/doctype/error_log/error_log.py @@ -1,9 +1,10 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and contributors # License: MIT. See LICENSE import frappe from frappe.model.document import Document +from frappe.query_builder import Interval +from frappe.query_builder.functions import Now class ErrorLog(Document): @@ -12,13 +13,10 @@ class ErrorLog(Document): self.db_set("seen", 1, update_modified=0) frappe.db.commit() - -def set_old_logs_as_seen(): - # set logs as seen - frappe.db.sql( - """UPDATE `tabError Log` SET `seen`=1 - WHERE `seen`=0 AND `creation` < (NOW() - INTERVAL '7' DAY)""" - ) + @staticmethod + def clear_old_logs(days=30): + table = frappe.qb.DocType("Error Log") + frappe.db.delete(table, filters=(table.modified < (Now() - Interval(days=days)))) @frappe.whitelist() diff --git a/frappe/core/doctype/error_log/error_log_list.js b/frappe/core/doctype/error_log/error_log_list.js index 91e69452ff..dabe95d6d7 100644 --- a/frappe/core/doctype/error_log/error_log_list.js +++ b/frappe/core/doctype/error_log/error_log_list.js @@ -1,21 +1,25 @@ -frappe.listview_settings['Error Log'] = { +frappe.listview_settings["Error Log"] = { add_fields: ["seen"], - get_indicator: function(doc) { - if(cint(doc.seen)) { + get_indicator: function (doc) { + if (cint(doc.seen)) { return [__("Seen"), "green", "seen,=,1"]; } else { return [__("Not Seen"), "red", "seen,=,0"]; } }, order_by: "seen asc, modified desc", - onload: function(listview) { - listview.page.add_menu_item(__("Clear Error Logs"), function() { + onload: function (listview) { + listview.page.add_menu_item(__("Clear Error Logs"), function () { frappe.call({ - method:'frappe.core.doctype.error_log.error_log.clear_error_logs', - callback: function() { + method: "frappe.core.doctype.error_log.error_log.clear_error_logs", + callback: function () { listview.refresh(); - } + }, }); }); - } + + frappe.require("logtypes.bundle.js", () => { + frappe.utils.logtypes.show_log_retention_message(cur_list.doctype); + }); + }, }; diff --git a/frappe/core/doctype/error_log/test_error_log.py b/frappe/core/doctype/error_log/test_error_log.py index e20ac92650..c7cf26a0cf 100644 --- a/frappe/core/doctype/error_log/test_error_log.py +++ b/frappe/core/doctype/error_log/test_error_log.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and Contributors # License: MIT. See LICENSE import unittest @@ -9,4 +8,8 @@ import frappe class TestErrorLog(unittest.TestCase): - pass + def test_error_log(self): + """let's do an error log on error log?""" + doc = frappe.new_doc("Error Log") + error = doc.log_error("This is an error") + self.assertEqual(error.doctype, "Error Log") diff --git a/frappe/core/doctype/error_snapshot/error_snapshot.js b/frappe/core/doctype/error_snapshot/error_snapshot.js index c1b2d996a1..f8a7e3ded5 100644 --- a/frappe/core/doctype/error_snapshot/error_snapshot.js +++ b/frappe/core/doctype/error_snapshot/error_snapshot.js @@ -1,14 +1,18 @@ -frappe.ui.form.on("Error Snapshot", "load", function(frm){ +frappe.ui.form.on("Error Snapshot", "load", function (frm) { frm.set_read_only(true); }); -frappe.ui.form.on("Error Snapshot", "refresh", function(frm){ - frm.set_df_property("view", "options", frappe.render_template("error_snapshot", {"doc": frm.doc})); +frappe.ui.form.on("Error Snapshot", "refresh", function (frm) { + frm.set_df_property( + "view", + "options", + frappe.render_template("error_snapshot", { doc: frm.doc }) + ); if (frm.doc.relapses) { - frm.add_custom_button(__('Show Relapses'), function() { + frm.add_custom_button(__("Show Relapses"), function () { frappe.route_options = { - parent_error_snapshot: frm.doc.name + parent_error_snapshot: frm.doc.name, }; frappe.set_route("List", "Error Snapshot"); }); diff --git a/frappe/core/doctype/error_snapshot/error_snapshot.json b/frappe/core/doctype/error_snapshot/error_snapshot.json index 1333fe0d5b..b92db8f99a 100644 --- a/frappe/core/doctype/error_snapshot/error_snapshot.json +++ b/frappe/core/doctype/error_snapshot/error_snapshot.json @@ -1,398 +1,130 @@ { - "allow_copy": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, - "creation": "2015-11-28 00:57:39.766888", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "System", - "editable_grid": 0, + "actions": [], + "creation": "2015-11-28 00:57:39.766888", + "doctype": "DocType", + "document_type": "System", + "engine": "InnoDB", + "field_order": [ + "view", + "seen", + "evalue", + "timestamp", + "relapses", + "etype", + "traceback", + "parent_error_snapshot", + "pyver", + "exception", + "locals", + "frames" + ], "fields": [ { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "view", - "fieldtype": "HTML", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Snapshot View", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "view", + "fieldtype": "HTML", + "label": "Snapshot View" + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "seen", - "fieldtype": "Check", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 1, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Seen", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "default": "0", + "fieldname": "seen", + "fieldtype": "Check", + "hidden": 1, + "in_filter": 1, + "label": "Seen" + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "evalue", - "fieldtype": "Code", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Friendly Title", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "evalue", + "fieldtype": "Code", + "hidden": 1, + "in_list_view": 1, + "label": "Friendly Title", + "read_only": 1 + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "timestamp", - "fieldtype": "Datetime", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Timestamp", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "timestamp", + "fieldtype": "Datetime", + "hidden": 1, + "label": "Timestamp", + "read_only": 1 + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "default": "1", - "fieldname": "relapses", - "fieldtype": "Int", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Relapses", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "default": "1", + "fieldname": "relapses", + "fieldtype": "Int", + "hidden": 1, + "in_list_view": 1, + "label": "Relapses", + "read_only": 1 + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "etype", - "fieldtype": "Data", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Exception Type", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "etype", + "fieldtype": "Data", + "hidden": 1, + "label": "Exception Type", + "read_only": 1 + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "traceback", - "fieldtype": "Code", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Traceback", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "traceback", + "fieldtype": "Code", + "hidden": 1, + "label": "Traceback", + "read_only": 1 + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "parent_error_snapshot", - "fieldtype": "Data", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Parent Error Snapshot", - "length": 0, - "no_copy": 0, - "options": "", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "parent_error_snapshot", + "fieldtype": "Data", + "hidden": 1, + "label": "Parent Error Snapshot" + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "pyver", - "fieldtype": "Code", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Pyver", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "pyver", + "fieldtype": "Code", + "hidden": 1, + "label": "Pyver", + "read_only": 1 + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "exception", - "fieldtype": "Code", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Exception", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "exception", + "fieldtype": "Code", + "hidden": 1, + "label": "Exception" + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "locals", - "fieldtype": "Code", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Locals", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "locals", + "fieldtype": "Code", + "hidden": 1, + "label": "Locals" + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "frames", - "fieldtype": "Code", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Frames", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "fieldname": "frames", + "fieldtype": "Code", + "hidden": 1, + "label": "Frames" } - ], - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 1, - - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2021-10-25 14:40:38.619106", - "modified_by": "Administrator", - "module": "Core", - "name": "Error Snapshot", - "name_case": "", - "owner": "Administrator", + ], + "in_create": 1, + "links": [], + "modified": "2022-08-03 12:20:53.504160", + "modified_by": "Administrator", + "module": "Core", + "name": "Error Snapshot", + "owner": "Administrator", "permissions": [ { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "if_owner": 0, - "import": 0, - "is_custom": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 1, - "role": "Administrator", - "set_user_permissions": 0, - "share": 1, - "submit": 0, + "create": 1, + "delete": 1, + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "Administrator", + "share": 1, "write": 1 } - ], - "quick_entry": 0, - "read_only": 0, - "read_only_onload": 0, - "sort_field": "timestamp", - "sort_order": "DESC", - "title_field": "evalue", - "track_seen": 0 -} + ], + "sort_field": "timestamp", + "sort_order": "DESC", + "states": [], + "title_field": "evalue" +} \ No newline at end of file diff --git a/frappe/core/doctype/error_snapshot/error_snapshot.py b/frappe/core/doctype/error_snapshot/error_snapshot.py index 82f189217f..6966cf0aca 100644 --- a/frappe/core/doctype/error_snapshot/error_snapshot.py +++ b/frappe/core/doctype/error_snapshot/error_snapshot.py @@ -1,9 +1,10 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors # License: MIT. See LICENSE import frappe from frappe.model.document import Document +from frappe.query_builder import Interval +from frappe.query_builder.functions import Now class ErrorSnapshot(Document): @@ -32,3 +33,8 @@ class ErrorSnapshot(Document): frappe.db.set_value("Error Snapshot", parent["name"], "relapses", parent["relapses"] + 1) if parent["seen"]: frappe.db.set_value("Error Snapshot", parent["name"], "seen", False) + + @staticmethod + def clear_old_logs(days=30): + table = frappe.qb.DocType("Error Snapshot") + frappe.db.delete(table, filters=(table.modified < (Now() - Interval(days=days)))) diff --git a/frappe/core/doctype/error_snapshot/error_snapshot_list.js b/frappe/core/doctype/error_snapshot/error_snapshot_list.js index 1ba3e344ae..b331788852 100644 --- a/frappe/core/doctype/error_snapshot/error_snapshot_list.js +++ b/frappe/core/doctype/error_snapshot/error_snapshot_list.js @@ -1,14 +1,19 @@ frappe.listview_settings["Error Snapshot"] = { add_fields: ["parent_error_snapshot", "relapses", "seen"], - filters:[ - ["parent_error_snapshot","=",null], - ["seen", "=", false] + filters: [ + ["parent_error_snapshot", "=", null], + ["seen", "=", false], ], - get_indicator: function(doc){ - if (doc.parent_error_snapshot && doc.parent_error_snapshot.length){ + get_indicator: function (doc) { + if (doc.parent_error_snapshot && doc.parent_error_snapshot.length) { return [__("Relapsed"), !doc.seen ? "orange" : "blue", "parent_error_snapshot,!=,"]; } else { return [__("First Level"), !doc.seen ? "red" : "green", "parent_error_snapshot,=,"]; } - } -} + }, + onload: function (listview) { + frappe.require("logtypes.bundle.js", () => { + frappe.utils.logtypes.show_log_retention_message(cur_list.doctype); + }); + }, +}; diff --git a/frappe/core/doctype/error_snapshot/test_error_snapshot.py b/frappe/core/doctype/error_snapshot/test_error_snapshot.py index 40596b3d22..0c1f861b43 100644 --- a/frappe/core/doctype/error_snapshot/test_error_snapshot.py +++ b/frappe/core/doctype/error_snapshot/test_error_snapshot.py @@ -1,10 +1,7 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE import unittest -import frappe - # test_records = frappe.get_test_records('Error Snapshot') diff --git a/frappe/core/doctype/feedback/feedback.js b/frappe/core/doctype/feedback/feedback.js deleted file mode 100644 index 131f0e19d8..0000000000 --- a/frappe/core/doctype/feedback/feedback.js +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright (c) 2021, Frappe Technologies and contributors -// For license information, please see license.txt - -frappe.ui.form.on('Feedback', { - // refresh: function(frm) { - - // } -}); diff --git a/frappe/core/doctype/feedback/feedback.py b/frappe/core/doctype/feedback/feedback.py deleted file mode 100644 index c616787e4b..0000000000 --- a/frappe/core/doctype/feedback/feedback.py +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright (c) 2021, Frappe Technologies and contributors -# License: MIT. See LICENSE - -# import frappe -from frappe.model.document import Document - - -class Feedback(Document): - pass diff --git a/frappe/core/doctype/feedback/test_feedback.py b/frappe/core/doctype/feedback/test_feedback.py deleted file mode 100644 index e8e29e75ae..0000000000 --- a/frappe/core/doctype/feedback/test_feedback.py +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright (c) 2021, Frappe Technologies and Contributors -# License: MIT. See LICENSE - -import unittest - -import frappe - - -class TestFeedback(unittest.TestCase): - def tearDown(self): - frappe.form_dict.reference_doctype = None - frappe.form_dict.reference_name = None - frappe.form_dict.like = None - frappe.local.request_ip = None - - def test_feedback_creation_updation(self): - from frappe.website.doctype.blog_post.test_blog_post import make_test_blog - - test_blog = make_test_blog() - - frappe.db.delete("Feedback", {"reference_doctype": "Blog Post"}) - - from frappe.templates.includes.feedback.feedback import give_feedback - - frappe.form_dict.reference_doctype = "Blog Post" - frappe.form_dict.reference_name = test_blog.name - frappe.form_dict.like = True - frappe.local.request_ip = "127.0.0.1" - - feedback = give_feedback() - - self.assertEqual(feedback.like, True) - - frappe.form_dict.like = False - - updated_feedback = give_feedback() - - self.assertEqual(updated_feedback.like, False) - - frappe.db.delete("Feedback", {"reference_doctype": "Blog Post"}) - - test_blog.delete() diff --git a/frappe/core/doctype/file/__init__.py b/frappe/core/doctype/file/__init__.py index e69de29bb2..ad28c17e36 100644 --- a/frappe/core/doctype/file/__init__.py +++ b/frappe/core/doctype/file/__init__.py @@ -0,0 +1,2 @@ +from .exceptions import * +from .utils import * diff --git a/frappe/core/doctype/file/exceptions.py b/frappe/core/doctype/file/exceptions.py new file mode 100644 index 0000000000..d8939b69da --- /dev/null +++ b/frappe/core/doctype/file/exceptions.py @@ -0,0 +1,12 @@ +import frappe + + +class MaxFileSizeReachedError(frappe.ValidationError): + pass + + +class FolderNotEmpty(frappe.ValidationError): + pass + + +from frappe.exceptions import * diff --git a/frappe/core/doctype/file/file.js b/frappe/core/doctype/file/file.js index d40328d3cd..f9051dc896 100644 --- a/frappe/core/doctype/file/file.js +++ b/frappe/core/doctype/file/file.js @@ -1,33 +1,27 @@ -frappe.ui.form.on("File", "refresh", function(frm) { - if(!frm.doc.is_folder) { - frm.add_custom_button(__('Download'), function() { - var file_url = frm.doc.file_url; - if (frm.doc.file_name) { - file_url = file_url.replace(/#/g, '%23'); - } - window.open(file_url); - }, "fa fa-download"); +frappe.ui.form.on("File", "refresh", function (frm) { + if (!frm.doc.is_folder) { + frm.add_custom_button( + __("Download"), + function () { + var file_url = frm.doc.file_url; + if (frm.doc.file_name) { + file_url = file_url.replace(/#/g, "%23"); + } + window.open(file_url); + }, + "fa fa-download" + ); } - var wrapper = frm.get_field("preview_html").$wrapper; - var is_viewable = frappe.utils.is_image_file(frm.doc.file_url); + frm.get_field("preview_html").$wrapper.html(`
    + +
    `); - frm.toggle_display("preview", is_viewable); - frm.toggle_display("preview_html", is_viewable); - - if(is_viewable){ - wrapper.html('
    \ - \ -
    '); - } else { - wrapper.empty(); - } - - var is_raster_image = (/\.(gif|jpg|jpeg|tiff|png)$/i).test(frm.doc.file_url); + var is_raster_image = /\.(gif|jpg|jpeg|tiff|png)$/i.test(frm.doc.file_url); var is_optimizable = !frm.doc.is_folder && is_raster_image && frm.doc.file_size > 0; if (is_optimizable) { - frm.add_custom_button(__("Optimize"), function() { + frm.add_custom_button(__("Optimize"), function () { frappe.show_alert(__("Optimizing image...")); frm.call("optimize_file").then(() => { frappe.show_alert(__("Image optimized")); @@ -35,16 +29,16 @@ frappe.ui.form.on("File", "refresh", function(frm) { }); } - if(frm.doc.file_name && frm.doc.file_name.split('.').splice(-1)[0]==='zip') { - frm.add_custom_button(__('Unzip'), function() { + if (frm.doc.file_name && frm.doc.file_name.split(".").splice(-1)[0] === "zip") { + frm.add_custom_button(__("Unzip"), function () { frappe.call({ - method: "frappe.core.doctype.file.file.unzip_file", + method: "frappe.core.api.file.unzip_file", args: { name: frm.doc.name, }, - callback: function() { - frappe.set_route('List', 'File'); - } + callback: function () { + frappe.set_route("List", "File"); + }, }); }); } diff --git a/frappe/core/doctype/file/file.py b/frappe/core/doctype/file/file.py index d8b45cf043..e1faf331d6 100755 --- a/frappe/core/doctype/file/file.py +++ b/frappe/core/doctype/file/file.py @@ -1,78 +1,48 @@ # Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -""" -record of files - -naming for same name files: file.gif, file-1.gif, file-2.gif etc -""" - -import hashlib -import imghdr import io -import json import mimetypes import os import re import shutil import zipfile -from io import BytesIO -from typing import TYPE_CHECKING, Tuple from urllib.parse import quote, unquote -import requests from PIL import Image, ImageFile, ImageOps from requests.exceptions import HTTPError, SSLError import frappe -from frappe import _, conf, safe_decode +from frappe import _ from frappe.model.document import Document -from frappe.utils import ( - call_hook_method, - cint, - cstr, - encode, - get_files_path, - get_hook_method, - random_string, - strip, -) -from frappe.utils.file_manager import safe_b64decode +from frappe.utils import call_hook_method, cint, get_files_path, get_hook_method, get_url +from frappe.utils.file_manager import is_safe_path from frappe.utils.image import optimize_image, strip_exif_data -if TYPE_CHECKING: - from PIL.ImageFile import ImageFile - from requests.models import Response - - -class MaxFileSizeReachedError(frappe.ValidationError): - pass - - -class FolderNotEmpty(frappe.ValidationError): - pass - +from .exceptions import AttachmentLimitReached, FolderNotEmpty, MaxFileSizeReachedError +from .utils import * exclude_from_linked_with = True ImageFile.LOAD_TRUNCATED_IMAGES = True +URL_PREFIXES = ("http://", "https://") class File(Document): no_feed_on_delete = True - def before_insert(self): - frappe.local.rollback_observers.append(self) - self.set_folder_name() - if self.file_name: - self.file_name = re.sub(r"/", "", self.file_name) - self.content = self.get("content", None) - self.decode = self.get("decode", False) - if self.content: - self.save_file(content=self.content, decode=self.decode) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # if content is set, file_url will be generated + # decode comes in the picture if content passed has to be decoded before writing to disk - def get_name_based_on_parent_folder(self): - if self.folder: - return "/".join([self.folder, self.file_name]) + self.content = self.get("content") or b"" + self.decode = self.get("decode", False) + + @property + def is_remote_file(self): + if self.file_url: + return self.file_url.startswith(URL_PREFIXES) + return not self.content def autoname(self): """Set name for folder""" @@ -83,85 +53,159 @@ class File(Document): # home self.name = self.file_name else: - self.name = frappe.generate_hash("", 10) + self.name = frappe.generate_hash(length=10) + + def before_insert(self): + self.set_folder_name() + self.set_file_name() + self.validate_attachment_limit() + + if self.is_folder: + return + + if self.is_remote_file: + self.validate_remote_file() + else: + self.save_file(content=self.get_content()) + self.flags.new_file = True + frappe.local.rollback_observers.append(self) def after_insert(self): if not self.is_folder: - self.add_comment_in_reference_doc( - "Attachment", - _("Added {0}").format( - "{file_name}{icon}".format( - **{ - "icon": ' ' if self.is_private else "", - "file_url": quote(frappe.safe_encode(self.file_url)) if self.file_url else self.file_name, - "file_name": self.file_name or self.file_url, - } - ) - ), - ) - - def after_rename(self, olddn, newdn, merge=False): - for successor in self.get_successor(): - setup_folder_path(successor[0], self.name) - - def get_successor(self): - return frappe.db.get_values(doctype="File", filters={"folder": self.name}, fieldname="name") + self.create_attachment_record() + self.set_is_private() + self.set_file_name() + self.validate_duplicate_entry() def validate(self): - if self.is_new(): - self.set_is_private() - self.set_file_name() - self.validate_duplicate_entry() - self.validate_attachment_limit() + # Ensure correct formatting and type + self.file_url = unquote(self.file_url) if self.file_url else "" - self.validate_folder() + # when dict is passed to get_doc for creation of new_doc, is_new returns None + # this case is handled inside handle_is_private_changed + if not self.is_new() and self.has_value_changed("is_private"): + self.handle_is_private_changed() - if self.is_folder: - self.file_url = "" - else: - self.validate_url() + if not self.is_folder: + self.validate_file_path() + self.validate_file_url() + self.validate_file_on_disk() self.file_size = frappe.form_dict.file_size or self.file_size - def validate_url(self): - if not self.file_url or self.file_url.startswith(("http://", "https://")): - if not self.flags.ignore_file_validate: - self.validate_file() + def after_rename(self, *args, **kwargs): + for successor in self.get_successors(): + setup_folder_path(successor, self.name) + def on_trash(self): + if self.is_home_folder or self.is_attachments_folder: + frappe.throw(_("Cannot delete Home and Attachments folders")) + self.validate_empty_folder() + self._delete_file_on_disk() + if not self.is_folder: + self.add_comment_in_reference_doc("Attachment Removed", _("Removed {0}").format(self.file_name)) + + def on_rollback(self): + # following condition is only executed when an insert has been rolledback + if self.flags.new_file: + self._delete_file_on_disk() + self.flags.pop("new_file") return - # Probably an invalid web URL - if not self.file_url.startswith(("/files/", "/private/files/")): - frappe.throw(_("URL must start with http:// or https://"), title=_("Invalid URL")) + # if original_content flag is set, this rollback should revert the file to its original state + if self.flags.original_content: + file_path = self.get_full_path() - # Ensure correct formatting and type - self.file_url = unquote(self.file_url) - self.is_private = cint(self.is_private) + if isinstance(self.flags.original_content, bytes): + mode = "wb+" + elif isinstance(self.flags.original_content, str): + mode = "w+" - self.handle_is_private_changed() + with open(file_path, mode) as f: + f.write(self.flags.original_content) + os.fsync(f.fileno()) + self.flags.pop("original_content") + + # used in case file path (File.file_url) has been changed + if self.flags.original_path: + target = self.flags.original_path["old"] + source = self.flags.original_path["new"] + shutil.move(source, target) + self.flags.pop("original_path") + + def get_name_based_on_parent_folder(self) -> str | None: + if self.folder: + return os.path.join(self.folder, self.file_name) + + def get_successors(self): + return frappe.get_all("File", filters={"folder": self.name}, pluck="name") + + def validate_file_path(self): + if self.is_remote_file: + return base_path = os.path.realpath(get_files_path(is_private=self.is_private)) if not os.path.realpath(self.get_full_path()).startswith(base_path): - frappe.throw(_("The File URL you've entered is incorrect"), title=_("Invalid File URL")) + frappe.throw( + _("The File URL you've entered is incorrect"), + title=_("Invalid File URL"), + ) - def handle_is_private_changed(self): - if not frappe.db.exists("File", {"name": self.name, "is_private": cint(not self.is_private)}): + def validate_file_url(self): + if self.is_remote_file or not self.file_url: return + if not self.file_url.startswith(("/files/", "/private/files/")): + # Probably an invalid URL since it doesn't start with http either + frappe.throw( + _("URL must start with http:// or https://"), + title=_("Invalid URL"), + ) + + def handle_is_private_changed(self): + if self.is_remote_file: + return + + from pathlib import Path + old_file_url = self.file_url - file_name = self.file_url.split("/")[-1] - private_file_path = frappe.get_site_path("private", "files", file_name) - public_file_path = frappe.get_site_path("public", "files", file_name) + private_file_path = Path(frappe.get_site_path("private", "files", file_name)) + public_file_path = Path(frappe.get_site_path("public", "files", file_name)) - if self.is_private: - shutil.move(public_file_path, private_file_path) + if cint(self.is_private): + source = public_file_path + target = private_file_path url_starts_with = "/private/files/" else: - shutil.move(private_file_path, public_file_path) + source = private_file_path + target = public_file_path url_starts_with = "/files/" + updated_file_url = f"{url_starts_with}{file_name}" - self.file_url = "{0}{1}".format(url_starts_with, file_name) + # if a file document is created by passing dict throught get_doc and __local is not set, + # handle_is_private_changed would be executed; we're checking if updated_file_url is same + # as old_file_url to avoid a FileNotFoundError for this case. + if updated_file_url == old_file_url: + return + + if not source.exists(): + frappe.throw( + _("Cannot find file {} on disk").format(source), + exc=FileNotFoundError, + ) + if target.exists(): + frappe.throw( + _("A file with same name {} already exists").format(target), + exc=FileExistsError, + ) + + # Uses os.rename which is an atomic operation + shutil.move(source, target) + self.flags.original_path = {"old": source, "new": target} + frappe.local.rollback_observers.append(self) + + self.file_url = updated_file_url update_existing_file_docs(self) if ( @@ -172,7 +216,10 @@ class File(Document): return frappe.db.set_value( - self.attached_to_doctype, self.attached_to_name, self.attached_to_field, self.file_url + self.attached_to_doctype, + self.attached_to_name, + self.attached_to_field, + self.file_url, ) def fetch_attached_to_field(self, old_file_url): @@ -208,26 +255,32 @@ class File(Document): _("Maximum Attachment Limit of {0} has been reached for {1} {2}.").format( frappe.bold(attachment_limit), self.attached_to_doctype, self.attached_to_name ), - exc=frappe.exceptions.AttachmentLimitReached, + exc=AttachmentLimitReached, title=_("Attachment Limit Reached"), ) + def validate_remote_file(self): + """Validates if file uploaded using URL already exist""" + site_url = get_url() + if self.file_url and "/files/" in self.file_url and self.file_url.startswith(site_url): + self.file_url = self.file_url.split(site_url, 1)[1] + def set_folder_name(self): """Make parent folders if not exists based on reference doctype and name""" - if self.attached_to_doctype and not self.folder: + if self.folder: + return + + if self.attached_to_doctype: self.folder = frappe.db.get_value("File", {"is_attachments_folder": 1}) - def validate_folder(self): - if not self.is_home_folder and not self.folder and not self.flags.ignore_folder_validate: + elif not self.is_home_folder: self.folder = "Home" - def validate_file(self): - """Validates existence of public file - TODO: validate for private file - """ + def validate_file_on_disk(self): + """Validates existence file""" full_path = self.get_full_path() - if full_path.startswith("http"): + if full_path.startswith(URL_PREFIXES): return True if not os.path.exists(full_path): @@ -247,7 +300,10 @@ class File(Document): } if self.attached_to_doctype and self.attached_to_name: filters.update( - {"attached_to_doctype": self.attached_to_doctype, "attached_to_name": self.attached_to_name} + { + "attached_to_doctype": self.attached_to_doctype, + "attached_to_name": self.attached_to_name, + } ) duplicate_file = frappe.db.get_value("File", filters, ["name", "file_url"], as_dict=1) @@ -264,89 +320,73 @@ class File(Document): self.file_name = re.sub(r"/", "", self.file_name) def generate_content_hash(self): - if self.content_hash or not self.file_url or self.file_url.startswith("http"): + if self.content_hash or not self.file_url or self.is_remote_file: return file_name = self.file_url.split("/")[-1] try: file_path = get_files_path(file_name, is_private=self.is_private) with open(file_path, "rb") as f: self.content_hash = get_content_hash(f.read()) - except IOError: + except OSError: frappe.throw(_("File {0} does not exist").format(file_path)) - def on_trash(self): - if self.is_home_folder or self.is_attachments_folder: - frappe.throw(_("Cannot delete Home and Attachments folders")) - self.check_folder_is_empty() - self.call_delete_file() - if not self.is_folder: - self.add_comment_in_reference_doc("Attachment Removed", _("Removed {0}").format(self.file_name)) - def make_thumbnail( - self, set_as_thumbnail=True, width=300, height=300, suffix="small", crop=False - ): - if self.file_url: - try: - if self.file_url.startswith(("/files", "/private/files")): - image, filename, extn = get_local_image(self.file_url) - else: - image, filename, extn = get_web_image(self.file_url) - except (HTTPError, SSLError, IOError, TypeError): - return + self, + set_as_thumbnail: bool = True, + width: int = 300, + height: int = 300, + suffix: str = "small", + crop: bool = False, + ) -> str: + if not self.file_url: + return - size = width, height - if crop: - image = ImageOps.fit(image, size, Image.ANTIALIAS) + try: + if self.file_url.startswith(("/files", "/private/files")): + image, filename, extn = get_local_image(self.file_url) else: - image.thumbnail(size, Image.ANTIALIAS) + image, filename, extn = get_web_image(self.file_url) + except (HTTPError, SSLError, OSError, TypeError): + return - thumbnail_url = filename + "_" + suffix + "." + extn - path = os.path.abspath(frappe.get_site_path("public", thumbnail_url.lstrip("/"))) + size = width, height + if crop: + image = ImageOps.fit(image, size, Image.Resampling.LANCZOS) + else: + image.thumbnail(size, Image.Resampling.LANCZOS) - try: - image.save(path) - if set_as_thumbnail: - self.db_set("thumbnail_url", thumbnail_url) + thumbnail_url = f"{filename}_{suffix}.{extn}" + path = os.path.abspath(frappe.get_site_path("public", thumbnail_url.lstrip("/"))) - except IOError: - frappe.msgprint(_("Unable to write file format for {0}").format(path)) - return + try: + image.save(path) + if set_as_thumbnail: + self.db_set("thumbnail_url", thumbnail_url) - return thumbnail_url + except OSError: + frappe.msgprint(_("Unable to write file format for {0}").format(path)) + return - def check_folder_is_empty(self): + return thumbnail_url + + def validate_empty_folder(self): """Throw exception if folder is not empty""" - files = frappe.get_all("File", filters={"folder": self.name}, fields=("name", "file_name")) - - if self.is_folder and files: + if self.is_folder and frappe.get_all("File", filters={"folder": self.name}, limit=1): frappe.throw(_("Folder {0} is not empty").format(self.name), FolderNotEmpty) - def call_delete_file(self): + def _delete_file_on_disk(self): """If file not attached to any other record, delete it""" - if ( - self.file_name - and self.content_hash - and ( - not frappe.db.count("File", {"content_hash": self.content_hash, "name": ["!=", self.name]}) - ) - ): + on_disk_file_not_shared = self.content_hash and not frappe.get_all( + "File", + filters={"content_hash": self.content_hash, "name": ["!=", self.name]}, + limit=1, + ) + if on_disk_file_not_shared: self.delete_file_data_content() - elif self.file_url: + else: self.delete_file_data_content(only_thumbnail=True) - def on_rollback(self): - # if original_content flag is set, this rollback should revert the file to its original state - if self.flags.original_content: - file_path = self.get_full_path() - with open(file_path, "wb+") as f: - f.write(self.flags.original_content) - - # following condition is only executed when an insert has been rolledback - else: - self.flags.on_rollback = True - self.on_trash() - - def unzip(self): + def unzip(self) -> list["File"]: """Unzip current file and replace it by its children""" if not self.file_url.endswith(".zip"): frappe.throw(_("{0} is not a zip file").format(self.file_name)) @@ -379,39 +419,51 @@ class File(Document): return files def exists_on_disk(self): - exists = os.path.exists(self.get_full_path()) - return exists + return os.path.exists(self.get_full_path()) - def get_content(self): + def get_content(self) -> bytes: """Returns [`file_name`, `content`] for given file name `fname`""" if self.is_folder: frappe.throw(_("Cannot get file contents of a Folder")) if self.get("content"): - return self.content + self._content = self.content + if self.decode: + self._content = decode_file_content(self._content) + self.decode = False + # self.content = None # TODO: This needs to happen; make it happen somehow + return self._content - self.validate_url() + if self.file_url: + self.validate_file_url() file_path = self.get_full_path() # read the file - with io.open(encode(file_path), mode="rb") as f: - content = f.read() + with open(file_path, mode="rb") as f: + self._content = f.read() try: # for plain text files - content = content.decode() + self._content = self._content.decode() except UnicodeDecodeError: # for .png, .jpg, etc pass - return content + return self._content def get_full_path(self): """Returns file path from given file name""" file_path = self.file_url or self.file_name + site_url = get_url() + if "/files/" in file_path and file_path.startswith(site_url): + file_path = file_path.split(site_url, 1)[1] + if "/" not in file_path: - file_path = "/files/" + file_path + if self.is_private: + file_path = f"/private/files/{file_path}" + else: + file_path = f"/files/{file_path}" if file_path.startswith("/private/files/"): file_path = get_files_path(*file_path.split("/private/files/", 1)[1].split("/"), is_private=1) @@ -419,61 +471,75 @@ class File(Document): elif file_path.startswith("/files/"): file_path = get_files_path(*file_path.split("/files/", 1)[1].split("/")) - elif file_path.startswith("http"): + elif file_path.startswith(URL_PREFIXES): pass elif not self.file_url: frappe.throw(_("There is some problem with the file url: {0}").format(file_path)) - return file_path - - def write_file(self): - """write file to disk with a random name (to compare)""" - file_path = get_files_path(is_private=self.is_private) + if not is_safe_path(file_path): + frappe.throw(_("Cannot access file path {0}").format(file_path)) if os.path.sep in self.file_name: frappe.throw(_("File name cannot have {0}").format(os.path.sep)) - # create directory (if not exists) - frappe.create_folder(file_path) - # write the file - self.content = self.get_content() - if isinstance(self.content, str): - self.content = self.content.encode() - with open(os.path.join(file_path.encode("utf-8"), self.file_name.encode("utf-8")), "wb+") as f: - f.write(self.content) + return file_path - return get_files_path(self.file_name, is_private=self.is_private) + def write_file(self): + """write file to disk with a random name (to compare)""" + if self.is_remote_file: + return + + file_path = self.get_full_path() + + if isinstance(self._content, str): + self._content = self._content.encode() + + with open(file_path, "wb+") as f: + f.write(self._content) + os.fsync(f.fileno()) + + frappe.local.rollback_observers.append(self) + + return file_path + + def save_file( + self, + content: bytes | str | None = None, + decode=False, + ignore_existing_file_check=False, + overwrite=False, + ): + if self.is_remote_file: + return + + if not self.flags.new_file: + self.flags.original_content = self.get_content() + + if content: + self.content = content + self.decode = decode + self.get_content() + + if not self._content: + return - def save_file(self, content=None, decode=False, ignore_existing_file_check=False): file_exists = False - self.content = content - - if decode: - if isinstance(content, str): - self.content = content.encode("utf-8") - - if b"," in self.content: - self.content = self.content.split(b",")[1] - self.content = safe_b64decode(self.content) - - if not self.is_private: - self.is_private = 0 + duplicate_file = None + self.is_private = cint(self.is_private) self.content_type = mimetypes.guess_type(self.file_name)[0] - self.file_size = self.check_max_file_size() - + # transform file content based on site settings if ( self.content_type and self.content_type == "image/jpeg" and frappe.get_system_settings("strip_exif_metadata_from_uploaded_images") ): - self.content = strip_exif_data(self.content, self.content_type) + self._content = strip_exif_data(self._content, self.content_type) - self.content_hash = get_content_hash(self.content) - - duplicate_file = None + self.file_size = self.check_max_file_size() + self.content_hash = get_content_hash(self._content) # check if a file exists with the same content hash and is also in the same folder (public or private) if not ignore_existing_file_check: @@ -485,15 +551,18 @@ class File(Document): ) if duplicate_file: - file_doc = frappe.get_cached_doc("File", duplicate_file.name) + file_doc: "File" = frappe.get_cached_doc("File", duplicate_file.name) if file_doc.exists_on_disk(): self.file_url = duplicate_file.file_url file_exists = True - if os.path.exists(encode(get_files_path(self.file_name, is_private=self.is_private))): - self.file_name = get_file_name(self.file_name, self.content_hash[-6:]) - if not file_exists: + if not overwrite: + self.file_name = generate_file_name( + name=self.file_name, + suffix=self.content_hash[-6:], + is_private=self.is_private, + ) call_hook_method("before_write_file", file_size=self.file_size) write_file_method = get_hook_method("write_file") if write_file_method: @@ -501,23 +570,25 @@ class File(Document): return self.save_file_on_filesystem() def save_file_on_filesystem(self): - fpath = self.write_file() - if self.is_private: - self.file_url = "/private/files/{0}".format(self.file_name) + self.file_url = f"/private/files/{self.file_name}" else: - self.file_url = "/files/{0}".format(self.file_name) + self.file_url = f"/files/{self.file_name}" + + fpath = self.write_file() return {"file_name": os.path.basename(fpath), "file_url": self.file_url} def check_max_file_size(self): + from frappe.core.api.file import get_max_file_size + max_file_size = get_max_file_size() - file_size = len(self.content) + file_size = len(self._content or b"") if file_size > max_file_size: - frappe.msgprint( + frappe.throw( _("File size exceeded the maximum allowed size of {0} MB").format(max_file_size / 1048576), - raise_exception=MaxFileSizeReachedError, + exc=MaxFileSizeReachedError, ) return file_size @@ -544,6 +615,16 @@ class File(Document): """returns split filename and extension""" return os.path.splitext(self.file_name) + def create_attachment_record(self): + icon = ' ' if self.is_private else "" + file_url = quote(frappe.safe_encode(self.file_url)) if self.file_url else self.file_name + file_name = self.file_name or self.file_url + + self.add_comment_in_reference_doc( + "Attachment", + _("Added {0}").format(f"{file_name}{icon}"), + ) + def add_comment_in_reference_doc(self, comment_type, text): if self.attached_to_doctype and self.attached_to_name: try: @@ -571,18 +652,13 @@ class File(Document): if is_svg: raise TypeError("Optimization of SVG images is not supported") - content = self.get_content() - file_path = self.get_full_path() - optimized_content = optimize_image(content, content_type) + original_content = self.get_content() + optimized_content = optimize_image( + content=original_content, + content_type=content_type, + ) - with open(file_path, "wb+") as f: - f.write(optimized_content) - - self.file_size = len(optimized_content) - self.content_hash = get_content_hash(optimized_content) - # if rolledback, revert back to original - self.flags.original_content = content - frappe.local.rollback_observers.append(self) + self.save_file(content=optimized_content, overwrite=True) self.save() @staticmethod @@ -605,182 +681,6 @@ def on_doctype_update(): frappe.db.add_index("File", ["attached_to_doctype", "attached_to_name"]) -def make_home_folder(): - home = frappe.get_doc( - {"doctype": "File", "is_folder": 1, "is_home_folder": 1, "file_name": _("Home")} - ).insert() - - frappe.get_doc( - { - "doctype": "File", - "folder": home.name, - "is_folder": 1, - "is_attachments_folder": 1, - "file_name": _("Attachments"), - } - ).insert() - - -@frappe.whitelist() -def create_new_folder(file_name, folder): - """create new folder under current parent folder""" - file = frappe.new_doc("File") - file.file_name = file_name - file.is_folder = 1 - file.folder = folder - file.insert(ignore_if_duplicate=True) - return file - - -@frappe.whitelist() -def move_file(file_list, new_parent, old_parent): - - if isinstance(file_list, str): - file_list = json.loads(file_list) - - for file_obj in file_list: - setup_folder_path(file_obj.get("name"), new_parent) - - # recalculate sizes - frappe.get_doc("File", old_parent).save() - frappe.get_doc("File", new_parent).save() - - -@frappe.whitelist() -def zip_files(files): - files = frappe.parse_json(files) - zipped_files = File.zip_files(files) - frappe.response["filename"] = "files.zip" - frappe.response["filecontent"] = zipped_files - frappe.response["type"] = "download" - - -def setup_folder_path(filename, new_parent): - file = frappe.get_doc("File", filename) - file.folder = new_parent - file.save() - - if file.is_folder: - from frappe.model.rename_doc import rename_doc - - rename_doc("File", file.name, file.get_name_based_on_parent_folder(), ignore_permissions=True) - - -def get_extension(filename, extn, content: bytes = None, response: "Response" = None) -> str: - mimetype = None - - if response: - content_type = response.headers.get("Content-Type") - - if content_type: - _extn = mimetypes.guess_extension(content_type) - if _extn: - return _extn[1:] - - if extn: - # remove '?' char and parameters from extn if present - if "?" in extn: - extn = extn.split("?", 1)[0] - - mimetype = mimetypes.guess_type(filename + "." + extn)[0] - - if mimetype is None or not mimetype.startswith("image/") and content: - # detect file extension by reading image header properties - extn = imghdr.what(filename + "." + (extn or ""), h=content) - - return extn - - -def get_local_image(file_url): - if file_url.startswith("/private"): - file_url_path = (file_url.lstrip("/"),) - else: - file_url_path = ("public", file_url.lstrip("/")) - - file_path = frappe.get_site_path(*file_url_path) - - try: - image = Image.open(file_path) - except IOError: - frappe.throw(_("Unable to read file format for {0}").format(file_url)) - - content = None - - try: - filename, extn = file_url.rsplit(".", 1) - except ValueError: - # no extn - with open(file_path, "r") as f: - content = f.read() - - filename = file_url - extn = None - - extn = get_extension(filename, extn, content) - - return image, filename, extn - - -def get_web_image(file_url: str) -> Tuple["ImageFile", str, str]: - # download - file_url = frappe.utils.get_url(file_url) - r = requests.get(file_url, stream=True) - try: - r.raise_for_status() - except HTTPError: - if r.status_code == 404: - frappe.msgprint(_("File '{0}' not found").format(file_url)) - else: - frappe.msgprint(_("Unable to read file format for {0}").format(file_url)) - raise - - try: - image = Image.open(BytesIO(r.content)) - except Exception as e: - frappe.msgprint(_("Image link '{0}' is not valid").format(file_url), raise_exception=e) - - try: - filename, extn = file_url.rsplit("/", 1)[1].rsplit(".", 1) - except ValueError: - # the case when the file url doesn't have filename or extension - # but is fetched due to a query string. example: https://encrypted-tbn3.gstatic.com/images?q=something - filename = get_random_filename() - extn = None - - extn = get_extension(filename, extn, response=r) - if extn == "bin": - extn = get_extension(filename, extn, content=r.content) or "png" - - filename = "/files/" + strip(unquote(filename)) - - return image, filename, extn - - -def delete_file(path): - """Delete file from `public folder`""" - if path: - if ".." in path.split("/"): - frappe.throw( - _("It is risky to delete this file: {0}. Please contact your System Manager.").format(path) - ) - - parts = os.path.split(path.strip("/")) - if parts[0] == "files": - path = frappe.utils.get_site_path("public", "files", parts[-1]) - - else: - path = frappe.utils.get_site_path("private", "files", parts[-1]) - - path = encode(path) - if os.path.exists(path): - os.remove(path) - - -@frappe.whitelist() -def get_max_file_size(): - return cint(conf.get("max_file_size")) or 10485760 - - def has_permission(doc, ptype=None, user=None): has_access = False user = user or frappe.session.user @@ -818,238 +718,5 @@ def has_permission(doc, ptype=None, user=None): return has_access -def remove_file_by_url(file_url, doctype=None, name=None): - if doctype and name: - fid = frappe.db.get_value( - "File", {"file_url": file_url, "attached_to_doctype": doctype, "attached_to_name": name} - ) - else: - fid = frappe.db.get_value("File", {"file_url": file_url}) - - if fid: - from frappe.utils.file_manager import remove_file - - return remove_file(fid=fid) - - -def get_content_hash(content): - if isinstance(content, str): - content = content.encode() - return hashlib.md5(content).hexdigest() # nosec - - -def get_file_name(fname, optional_suffix): - # convert to unicode - fname = cstr(fname) - - f = fname.rsplit(".", 1) - if len(f) == 1: - partial, extn = f[0], "" - else: - partial, extn = f[0], "." + f[1] - return "{partial}{suffix}{extn}".format(partial=partial, extn=extn, suffix=optional_suffix) - - -@frappe.whitelist() -def download_file(file_url): - """ - Download file using token and REST API. Valid session or - token is required to download private files. - - Method : GET - Endpoint : frappe.core.doctype.file.file.download_file - URL Params : file_name = /path/to/file relative to site path - """ - file_doc = frappe.get_doc("File", {"file_url": file_url}) - file_doc.check_permission("read") - - frappe.local.response.filename = os.path.basename(file_url) - frappe.local.response.filecontent = file_doc.get_content() - frappe.local.response.type = "download" - - -def extract_images_from_doc(doc, fieldname): - content = doc.get(fieldname) - content = extract_images_from_html(doc, content) - if frappe.flags.has_dataurl: - doc.set(fieldname, content) - - -def extract_images_from_html(doc, content, is_private=False): - frappe.flags.has_dataurl = False - - def _save_file(match): - data = match.group(1) - data = data.split("data:")[1] - headers, content = data.split(",") - mtype = headers.split(";")[0] - - if isinstance(content, str): - content = content.encode("utf-8") - if b"," in content: - content = content.split(b",")[1] - content = safe_b64decode(content) - - content = optimize_image(content, mtype) - - if "filename=" in headers: - filename = headers.split("filename=")[-1] - filename = safe_decode(filename).split(";")[0] - - else: - filename = get_random_filename(content_type=mtype) - - # attaching a file to a child table doc, attaches it to the parent doc - doctype = doc.parenttype if doc.get("parent") else doc.doctype - name = doc.get("parent") or doc.name - - _file = frappe.get_doc( - { - "doctype": "File", - "file_name": filename, - "attached_to_doctype": doctype, - "attached_to_name": name, - "content": content, - "decode": False, - "is_private": is_private, - } - ) - _file.save(ignore_permissions=True) - file_url = _file.file_url - if not frappe.flags.has_dataurl: - frappe.flags.has_dataurl = True - - return ']*src\s*=\s*["\'](?=data:)(.*?)["\']', _save_file, content) - - return content - - -def get_random_filename(content_type=None): - extn = None - if content_type: - extn = mimetypes.guess_extension(content_type) - - return random_string(7) + (extn or "") - - -@frappe.whitelist() -def unzip_file(name): - """Unzip the given file and make file records for each of the extracted files""" - file_obj = frappe.get_doc("File", name) - files = file_obj.unzip() - return files - - -@frappe.whitelist() -def get_attached_images(doctype, names): - """get list of image urls attached in form - returns {name: ['image.jpg', 'image.png']}""" - - if isinstance(names, str): - names = json.loads(names) - - img_urls = frappe.db.get_list( - "File", - filters={"attached_to_doctype": doctype, "attached_to_name": ("in", names), "is_folder": 0}, - fields=["file_url", "attached_to_name as docname"], - ) - - out = frappe._dict() - for i in img_urls: - out[i.docname] = out.get(i.docname, []) - out[i.docname].append(i.file_url) - - return out - - -@frappe.whitelist() -def get_files_in_folder(folder, start=0, page_length=20): - start = cint(start) - page_length = cint(page_length) - - attachment_folder = frappe.db.get_value( - "File", "Home/Attachments", ["name", "file_name", "file_url", "is_folder", "modified"], as_dict=1 - ) - - files = frappe.db.get_list( - "File", - {"folder": folder}, - ["name", "file_name", "file_url", "is_folder", "modified"], - start=start, - page_length=page_length + 1, - ) - - if folder == "Home" and attachment_folder not in files: - files.insert(0, attachment_folder) - - return {"files": files[:page_length], "has_more": len(files) > page_length} - - -@frappe.whitelist() -def get_files_by_search_text(text): - if not text: - return [] - - text = "%" + cstr(text).lower() + "%" - return frappe.db.get_all( - "File", - fields=["name", "file_name", "file_url", "is_folder", "modified"], - filters={"is_folder": False}, - or_filters={"file_name": ("like", text), "file_url": text, "name": ("like", text)}, - order_by="modified desc", - limit=20, - ) - - -def update_existing_file_docs(doc): - # Update is private and file url of all file docs that point to the same file - file_doctype = frappe.qb.DocType("File") - ( - frappe.qb.update(file_doctype) - .set(file_doctype.file_url, doc.file_url) - .set(file_doctype.is_private, doc.is_private) - .where(file_doctype.content_hash == doc.content_hash) - .where(file_doctype.name != doc.name) - ).run() - - -def attach_files_to_document(doc, event): - """Runs on on_update hook of all documents. - Goes through every Attach and Attach Image field and attaches - the file url to the document if it is not already attached. - """ - - attach_fields = doc.meta.get("fields", {"fieldtype": ["in", ["Attach", "Attach Image"]]}) - - for df in attach_fields: - # this method runs in on_update hook of all documents - # we dont want the update to fail if file cannot be attached for some reason - try: - value = doc.get(df.fieldname) - if not (value or "").startswith(("/files", "/private/files")): - return - - if frappe.db.exists( - "File", - { - "file_url": value, - "attached_to_name": doc.name, - "attached_to_doctype": doc.doctype, - "attached_to_field": df.fieldname, - }, - ): - return - - frappe.get_doc( - doctype="File", - file_url=value, - attached_to_name=doc.name, - attached_to_doctype=doc.doctype, - attached_to_field=df.fieldname, - folder="Home/Attachments", - ).insert() - except Exception: - frappe.log_error(title=_("Error Attaching File")) +# Note: kept at the end to not cause circular, partial imports & maintain backwards compatibility +from frappe.core.api.file import * diff --git a/frappe/core/doctype/file/test_file.py b/frappe/core/doctype/file/test_file.py index b02bb581ab..d849a2de4d 100644 --- a/frappe/core/doctype/file/test_file.py +++ b/frappe/core/doctype/file/test_file.py @@ -4,18 +4,25 @@ import base64 import json import os import unittest +from contextlib import contextmanager +from typing import TYPE_CHECKING import frappe from frappe import _ -from frappe.core.doctype.file.file import ( - File, +from frappe.core.api.file import ( + create_new_folder, get_attached_images, get_files_in_folder, move_file, unzip_file, ) +from frappe.exceptions import ValidationError +from frappe.tests.utils import FrappeTestCase from frappe.utils import get_files_path +if TYPE_CHECKING: + from frappe.core.doctype.file.file import File + test_content1 = "Hello" test_content2 = "Hello World" @@ -28,7 +35,25 @@ def make_test_doc(): return d.doctype, d.name -class TestSimpleFile(unittest.TestCase): +@contextmanager +def make_test_image_file(): + file_path = frappe.get_app_path("frappe", "tests/data/sample_image_for_optimization.jpg") + with open(file_path, "rb") as f: + file_content = f.read() + + test_file = frappe.get_doc( + {"doctype": "File", "file_name": "sample_image_for_optimization.jpg", "content": file_content} + ).insert() + # remove those flags + _test_file: "File" = frappe.get_doc("File", test_file.name) + + try: + yield _test_file + finally: + _test_file.delete() + + +class TestSimpleFile(FrappeTestCase): def setUp(self): self.attached_to_doctype, self.attached_to_docname = make_test_doc() self.test_content = test_content1 @@ -50,11 +75,11 @@ class TestSimpleFile(unittest.TestCase): self.assertEqual(content, self.test_content) -class TestBase64File(unittest.TestCase): +class TestBase64File(FrappeTestCase): def setUp(self): self.attached_to_doctype, self.attached_to_docname = make_test_doc() self.test_content = base64.b64encode(test_content1.encode("utf-8")) - _file = frappe.get_doc( + _file: "File" = frappe.get_doc( { "doctype": "File", "file_name": "test_base64.txt", @@ -73,7 +98,7 @@ class TestBase64File(unittest.TestCase): self.assertEqual(content, test_content1) -class TestSameFileName(unittest.TestCase): +class TestSameFileName(FrappeTestCase): def test_saved_content(self): self.attached_to_doctype, self.attached_to_docname = make_test_doc() self.test_content1 = test_content1 @@ -133,7 +158,7 @@ class TestSameFileName(unittest.TestCase): self.assertEqual(_file.get_content(), test_content2) -class TestSameContent(unittest.TestCase): +class TestSameContent(FrappeTestCase): def setUp(self): self.attached_to_doctype1, self.attached_to_docname1 = make_test_doc() self.attached_to_doctype2, self.attached_to_docname2 = make_test_doc() @@ -201,7 +226,7 @@ class TestSameContent(unittest.TestCase): frappe.clear_cache(doctype="ToDo") -class TestFile(unittest.TestCase): +class TestFile(FrappeTestCase): def setUp(self): frappe.set_user("Administrator") self.delete_test_data() @@ -327,7 +352,7 @@ class TestFile(unittest.TestCase): _file.save() folder = frappe.get_doc("File", "Home/Test Folder 1/Test Folder 3") - self.assertRaises(frappe.ValidationError, folder.delete) + self.assertRaises(ValidationError, folder.delete) def test_same_file_url_update(self): attached_to_doctype1, attached_to_docname1 = make_test_doc() @@ -373,38 +398,35 @@ class TestFile(unittest.TestCase): { "doctype": "File", "file_name": "parent_dir.txt", - "attached_to_doctype": "", - "attached_to_name": "", "is_private": 1, "content": test_content1, } ).insert() file1.file_url = "/private/files/../test.txt" - self.assertRaises(frappe.exceptions.ValidationError, file1.save) + self.assertRaises(ValidationError, file1.save) # No validation to see if file exists file1.reload() file1.file_url = "/private/files/parent_dir2.txt" - file1.save() + self.assertRaises(OSError, file1.save) def test_file_url_validation(self): - test_file = frappe.get_doc( - {"doctype": "File", "file_name": "logo", "file_url": "https://frappe.io/files/frappe.png"} - ) + test_file: "File" = frappe.new_doc("File") + test_file.update({"file_name": "logo", "file_url": "https://frappe.io/files/frappe.png"}) self.assertIsNone(test_file.validate()) # bad path test_file.file_url = "/usr/bin/man" self.assertRaisesRegex( - frappe.exceptions.ValidationError, "URL must start with http:// or https://", test_file.validate + ValidationError, f"Cannot access file path {test_file.file_url}", test_file.validate ) test_file.file_url = None test_file.file_name = "/usr/bin/man" self.assertRaisesRegex( - frappe.exceptions.ValidationError, "There is some problem with the file url", test_file.validate + ValidationError, "There is some problem with the file url", test_file.validate ) test_file.file_url = None @@ -413,11 +435,11 @@ class TestFile(unittest.TestCase): test_file.file_url = None test_file.file_name = "/private/files/_file" - self.assertRaisesRegex(IOError, "does not exist", test_file.validate) + self.assertRaisesRegex(ValidationError, "File name cannot have", test_file.validate) def test_make_thumbnail(self): # test web image - test_file: File = frappe.get_doc( + test_file: "File" = frappe.get_doc( { "doctype": "File", "file_name": "logo", @@ -486,37 +508,46 @@ class TestFile(unittest.TestCase): "file_url": frappe.utils.get_url("/_test/assets/image.jpg"), } ).insert(ignore_permissions=True) - self.assertRaisesRegex(frappe.exceptions.ValidationError, "not a zip file", test_file.unzip) + self.assertRaisesRegex(ValidationError, "not a zip file", test_file.unzip) + + def test_create_file_without_file_url(self): + test_file = frappe.get_doc( + { + "doctype": "File", + "file_name": "logo", + "content": "frappe", + } + ).insert() + assert test_file is not None class TestAttachment(unittest.TestCase): test_doctype = "Test For Attachment" - def setUp(self): - if frappe.db.exists("DocType", self.test_doctype): - return - + @classmethod + def setUpClass(cls): frappe.get_doc( doctype="DocType", - name=self.test_doctype, + name=cls.test_doctype, module="Custom", custom=1, fields=[ {"label": "Title", "fieldname": "title", "fieldtype": "Data"}, {"label": "Attachment", "fieldname": "attachment", "fieldtype": "Attach"}, ], - ).insert() + ).insert(ignore_if_duplicate=True) - def tearDown(self): - frappe.delete_doc("DocType", self.test_doctype) + @classmethod + def tearDownClass(cls): + frappe.db.rollback() + frappe.delete_doc("DocType", cls.test_doctype) def test_file_attachment_on_update(self): doc = frappe.get_doc(doctype=self.test_doctype, title="test for attachment on update").insert() file = frappe.get_doc( {"doctype": "File", "file_name": "test_attach.txt", "content": "Test Content"} - ) - file.save() + ).save() doc.attachment = file.file_url doc.save() @@ -535,9 +566,11 @@ class TestAttachment(unittest.TestCase): self.assertTrue(exists) -class TestAttachmentsAccess(unittest.TestCase): - def test_attachments_access(self): +class TestAttachmentsAccess(FrappeTestCase): + def setUp(self) -> None: + frappe.db.delete("File", {"is_folder": 0}) + def test_attachments_access(self): frappe.set_user("test4@example.com") self.attached_to_doctype, self.attached_to_docname = make_test_doc() @@ -600,11 +633,12 @@ class TestAttachmentsAccess(unittest.TestCase): self.assertIn("test_user.txt", system_manager_attachments_files) self.assertIn("test_user.txt", user_attachments_files) + def tearDown(self) -> None: frappe.set_user("Administrator") frappe.db.rollback() -class TestFileUtils(unittest.TestCase): +class TestFileUtils(FrappeTestCase): def test_extract_images_from_doc(self): # with filename in data URI todo = frappe.get_doc( @@ -628,30 +662,22 @@ class TestFileUtils(unittest.TestCase): self.assertIn(f' None: + home = frappe.get_doc( + {"doctype": "File", "is_folder": 1, "is_home_folder": 1, "file_name": _("Home")} + ).insert(ignore_if_duplicate=True) + + frappe.get_doc( + { + "doctype": "File", + "folder": home.name, + "is_folder": 1, + "is_attachments_folder": 1, + "file_name": _("Attachments"), + } + ).insert(ignore_if_duplicate=True) + + +def setup_folder_path(filename: str, new_parent: str) -> None: + file: "File" = frappe.get_doc("File", filename) + file.folder = new_parent + file.save() + + if file.is_folder: + from frappe.model.rename_doc import rename_doc + + rename_doc("File", file.name, file.get_name_based_on_parent_folder(), ignore_permissions=True) + + +def get_extension( + filename, + extn: str | None = None, + content: bytes | None = None, + response: Optional["Response"] = None, +) -> str: + mimetype = None + + if response: + content_type = response.headers.get("Content-Type") + + if content_type: + _extn = mimetypes.guess_extension(content_type) + if _extn: + return _extn[1:] + + if extn: + # remove '?' char and parameters from extn if present + if "?" in extn: + extn = extn.split("?", 1)[0] + + mimetype = mimetypes.guess_type(filename + "." + extn)[0] + + if mimetype is None or not mimetype.startswith("image/") and content: + # detect file extension by reading image header properties + extn = imghdr.what(filename + "." + (extn or ""), h=content) + + return extn + + +def get_local_image(file_url: str) -> tuple["ImageFile", str, str]: + if file_url.startswith("/private"): + file_url_path = (file_url.lstrip("/"),) + else: + file_url_path = ("public", file_url.lstrip("/")) + + file_path = frappe.get_site_path(*file_url_path) + + try: + image = Image.open(file_path) + except OSError: + frappe.throw(_("Unable to read file format for {0}").format(file_url)) + + content = None + + try: + filename, extn = file_url.rsplit(".", 1) + except ValueError: + # no extn + with open(file_path) as f: + content = f.read() + + filename = file_url + extn = None + + extn = get_extension(filename, extn, content) + + return image, filename, extn + + +def get_web_image(file_url: str) -> tuple["ImageFile", str, str]: + # download + file_url = frappe.utils.get_url(file_url) + r = requests.get(file_url, stream=True) + try: + r.raise_for_status() + except requests.exceptions.HTTPError as e: + if "404" in e.args[0]: + frappe.msgprint(_("File '{0}' not found").format(file_url)) + else: + frappe.msgprint(_("Unable to read file format for {0}").format(file_url)) + raise + + try: + image = Image.open(BytesIO(r.content)) + except Exception as e: + frappe.msgprint(_("Image link '{0}' is not valid").format(file_url), raise_exception=e) + + try: + filename, extn = file_url.rsplit("/", 1)[1].rsplit(".", 1) + except ValueError: + # the case when the file url doesn't have filename or extension + # but is fetched due to a query string. example: https://encrypted-tbn3.gstatic.com/images?q=something + filename = get_random_filename() + extn = None + + extn = get_extension(filename, extn, r.content) + filename = "/files/" + strip(unquote(filename)) + + return image, filename, extn + + +def delete_file(path: str) -> None: + """Delete file from `public folder`""" + if path: + if ".." in path.split("/"): + frappe.throw( + _("It is risky to delete this file: {0}. Please contact your System Manager.").format(path) + ) + + parts = os.path.split(path.strip("/")) + if parts[0] == "files": + path = frappe.utils.get_site_path("public", "files", parts[-1]) + + else: + path = frappe.utils.get_site_path("private", "files", parts[-1]) + + path = encode(path) + if os.path.exists(path): + os.remove(path) + + +def remove_file_by_url(file_url: str, doctype: str = None, name: str = None) -> "Document": + if doctype and name: + fid = frappe.db.get_value( + "File", {"file_url": file_url, "attached_to_doctype": doctype, "attached_to_name": name} + ) + else: + fid = frappe.db.get_value("File", {"file_url": file_url}) + + if fid: + from frappe.utils.file_manager import remove_file + + return remove_file(fid=fid) + + +def get_content_hash(content: bytes | str) -> str: + if isinstance(content, str): + content = content.encode() + return hashlib.md5(content).hexdigest() # nosec + + +def generate_file_name(name: str, suffix: str | None = None, is_private: bool = False) -> str: + """Generate conflict-free file name. Suffix will be ignored if name available. If the + provided suffix doesn't result in an available path, a random suffix will be picked. + """ + + def path_exists(name, is_private): + return os.path.exists(encode(get_files_path(name, is_private=is_private))) + + if not path_exists(name, is_private): + return name + + candidate_path = get_file_name(name, suffix) + + if path_exists(candidate_path, is_private): + return generate_file_name(name, is_private=is_private) + return candidate_path + + +def get_file_name(fname: str, optional_suffix: str | None = None) -> str: + # convert to unicode + fname = cstr(fname) + partial, extn = os.path.splitext(fname) + suffix = optional_suffix or frappe.generate_hash(length=6) + + return f"{partial}{suffix}{extn}" + + +def extract_images_from_doc(doc: "Document", fieldname: str): + content = doc.get(fieldname) + content = extract_images_from_html(doc, content) + if frappe.flags.has_dataurl: + doc.set(fieldname, content) + + +def extract_images_from_html(doc: "Document", content: str, is_private: bool = False): + frappe.flags.has_dataurl = False + + def _save_file(match): + data = match.group(1).split("data:")[1] + headers, content = data.split(",") + mtype = headers.split(";")[0] + + if isinstance(content, str): + content = content.encode("utf-8") + if b"," in content: + content = content.split(b",")[1] + content = safe_b64decode(content) + + content = optimize_image(content, mtype) + + if "filename=" in headers: + filename = headers.split("filename=")[-1] + filename = safe_decode(filename).split(";")[0] + + else: + filename = get_random_filename(content_type=mtype) + + if doc.meta.istable: + doctype = doc.parenttype + name = doc.parent + else: + doctype = doc.doctype + name = doc.name + + _file = frappe.get_doc( + { + "doctype": "File", + "file_name": filename, + "attached_to_doctype": doctype, + "attached_to_name": name, + "content": content, + "decode": False, + "is_private": is_private, + } + ) + _file.save(ignore_permissions=True) + file_url = _file.file_url + frappe.flags.has_dataurl = True + + return f']*src\s*=\s*["\'](?=data:)(.*?)["\']', _save_file, content) + + return content + + +def get_random_filename(content_type: str = None) -> str: + extn = None + if content_type: + extn = mimetypes.guess_extension(content_type) + + return random_string(7) + (extn or "") + + +def update_existing_file_docs(doc: "File") -> None: + # Update is private and file url of all file docs that point to the same file + file_doctype = frappe.qb.DocType("File") + ( + frappe.qb.update(file_doctype) + .set(file_doctype.file_url, doc.file_url) + .set(file_doctype.is_private, doc.is_private) + .where(file_doctype.content_hash == doc.content_hash) + .where(file_doctype.name != doc.name) + ).run() + + +def attach_files_to_document(doc: "File", event) -> None: + """Runs on on_update hook of all documents. + Goes through every Attach and Attach Image field and attaches + the file url to the document if it is not already attached. + """ + + attach_fields = doc.meta.get("fields", {"fieldtype": ["in", ["Attach", "Attach Image"]]}) + + for df in attach_fields: + # this method runs in on_update hook of all documents + # we dont want the update to fail if file cannot be attached for some reason + value = doc.get(df.fieldname) + if not (value or "").startswith(("/files", "/private/files")): + return + + if frappe.db.exists( + "File", + { + "file_url": value, + "attached_to_name": doc.name, + "attached_to_doctype": doc.doctype, + "attached_to_field": df.fieldname, + }, + ): + return + + file: "File" = frappe.get_doc( + doctype="File", + file_url=value, + attached_to_name=doc.name, + attached_to_doctype=doc.doctype, + attached_to_field=df.fieldname, + folder="Home/Attachments", + ) + try: + file.insert(ignore_permissions=True) + except Exception: + doc.log_error("Error Attaching File") + + +def decode_file_content(content: bytes) -> bytes: + if isinstance(content, str): + content = content.encode("utf-8") + if b"," in content: + content = content.split(b",")[1] + return safe_b64decode(content) diff --git a/frappe/core/doctype/has_domain/has_domain.json b/frappe/core/doctype/has_domain/has_domain.json index e2b646b457..c34626b269 100644 --- a/frappe/core/doctype/has_domain/has_domain.json +++ b/frappe/core/doctype/has_domain/has_domain.json @@ -1,72 +1,32 @@ { - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, - "creation": "2017-05-03 15:20:22.326623", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", + "actions": [], + "creation": "2017-05-03 15:20:22.326623", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "domain" + ], "fields": [ { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "domain", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Domain", - "length": 0, - "no_copy": 0, - "options": "Domain", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "fieldname": "domain", + "fieldtype": "Link", + "in_list_view": 1, + "label": "Domain", + "options": "Domain" } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 1, - "max_attachments": 0, - "modified": "2020-09-18 17:26:09.703215", - "modified_by": "Administrator", - "module": "Core", - "name": "Has Domain", - "name_case": "", - "owner": "Administrator", - "permissions": [], - "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1, - "track_seen": 0 + ], + "istable": 1, + "links": [], + "modified": "2022-08-03 12:20:53.381248", + "modified_by": "Administrator", + "module": "Core", + "name": "Has Domain", + "owner": "Administrator", + "permissions": [], + "quick_entry": 1, + "sort_field": "modified", + "sort_order": "DESC", + "states": [], + "track_changes": 1 } \ No newline at end of file diff --git a/frappe/core/doctype/has_domain/has_domain.py b/frappe/core/doctype/has_domain/has_domain.py index 15dc596365..0d8257fc33 100644 --- a/frappe/core/doctype/has_domain/has_domain.py +++ b/frappe/core/doctype/has_domain/has_domain.py @@ -1,8 +1,6 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2017, Frappe Technologies and contributors # License: MIT. See LICENSE -import frappe from frappe.model.document import Document diff --git a/frappe/core/doctype/has_role/has_role.json b/frappe/core/doctype/has_role/has_role.json index e0759dcd7e..689e80480e 100644 --- a/frappe/core/doctype/has_role/has_role.json +++ b/frappe/core/doctype/has_role/has_role.json @@ -1,64 +1,34 @@ { - "allow_copy": 0, - "allow_import": 0, - "allow_rename": 0, - "autoname": "hash", - "beta": 0, - "creation": "2013-02-22 01:27:34", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "editable_grid": 1, + "actions": [], + "autoname": "hash", + "creation": "2013-02-22 01:27:34", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "role" + ], "fields": [ { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "role", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Role", - "length": 0, - "no_copy": 0, - "oldfieldname": "role", - "oldfieldtype": "Link", - "options": "Role", - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "fieldname": "role", + "fieldtype": "Link", + "in_list_view": 1, + "label": "Role", + "oldfieldname": "role", + "oldfieldtype": "Link", + "options": "Role" } - ], - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 1, - "image_view": 0, - "in_create": 0, - - "is_submittable": 0, - "issingle": 0, - "istable": 1, - "max_attachments": 0, - "modified": "2017-02-13 14:00:08.116312", - "modified_by": "Administrator", - "module": "Core", - "name": "Has Role", - "owner": "Administrator", - "permissions": [], - "quick_entry": 0, - "read_only": 0, - "read_only_onload": 0, - "track_changes": 0, - "track_seen": 0 + ], + "idx": 1, + "istable": 1, + "links": [], + "modified": "2022-08-03 12:20:54.382064", + "modified_by": "Administrator", + "module": "Core", + "name": "Has Role", + "owner": "Administrator", + "permissions": [], + "sort_field": "modified", + "sort_order": "DESC", + "states": [] } \ No newline at end of file diff --git a/frappe/core/doctype/has_role/has_role.py b/frappe/core/doctype/has_role/has_role.py index 83cf90aac6..8798adf031 100644 --- a/frappe/core/doctype/has_role/has_role.py +++ b/frappe/core/doctype/has_role/has_role.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and contributors # License: MIT. See LICENSE diff --git a/frappe/core/doctype/installed_application/installed_application.py b/frappe/core/doctype/installed_application/installed_application.py index 07b6c105af..0a1dab8f2e 100644 --- a/frappe/core/doctype/installed_application/installed_application.py +++ b/frappe/core/doctype/installed_application/installed_application.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and contributors # License: MIT. See LICENSE diff --git a/frappe/core/doctype/installed_applications/installed_applications.js b/frappe/core/doctype/installed_applications/installed_applications.js index 9a1fd5ac18..223c028e7a 100644 --- a/frappe/core/doctype/installed_applications/installed_applications.js +++ b/frappe/core/doctype/installed_applications/installed_applications.js @@ -1,8 +1,7 @@ // Copyright (c) 2020, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Installed Applications', { +frappe.ui.form.on("Installed Applications", { // refresh: function(frm) { - // } }); diff --git a/frappe/core/doctype/installed_applications/installed_applications.py b/frappe/core/doctype/installed_applications/installed_applications.py index d13118e169..07b20db153 100644 --- a/frappe/core/doctype/installed_applications/installed_applications.py +++ b/frappe/core/doctype/installed_applications/installed_applications.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and contributors # License: MIT. See LICENSE diff --git a/frappe/core/doctype/installed_applications/test_installed_applications.py b/frappe/core/doctype/installed_applications/test_installed_applications.py index 24d3a9a49a..b67cc4c8c7 100644 --- a/frappe/core/doctype/installed_applications/test_installed_applications.py +++ b/frappe/core/doctype/installed_applications/test_installed_applications.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and Contributors # License: MIT. See LICENSE # import frappe diff --git a/frappe/core/doctype/language/language.js b/frappe/core/doctype/language/language.js index e60282ebbf..9c7852f9e0 100644 --- a/frappe/core/doctype/language/language.js +++ b/frappe/core/doctype/language/language.js @@ -1,8 +1,6 @@ // Copyright (c) 2016, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Language', { - refresh: function(frm) { - - } +frappe.ui.form.on("Language", { + refresh: function (frm) {}, }); diff --git a/frappe/core/doctype/language/language.py b/frappe/core/doctype/language/language.py index 948810eb39..efac7b0d77 100644 --- a/frappe/core/doctype/language/language.py +++ b/frappe/core/doctype/language/language.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and contributors # License: MIT. See LICENSE @@ -42,7 +41,7 @@ def export_languages_json(): def sync_languages(): """Sync frappe/geo/languages.json with Language""" - with open(frappe.get_app_path("frappe", "geo", "languages.json"), "r") as f: + with open(frappe.get_app_path("frappe", "geo", "languages.json")) as f: data = json.loads(f.read()) for l in data: @@ -59,7 +58,7 @@ def sync_languages(): def update_language_names(): """Update frappe/geo/languages.json names (for use via patch)""" - with open(frappe.get_app_path("frappe", "geo", "languages.json"), "r") as f: + with open(frappe.get_app_path("frappe", "geo", "languages.json")) as f: data = json.loads(f.read()) for l in data: diff --git a/frappe/core/doctype/language/test_language.py b/frappe/core/doctype/language/test_language.py index 7262cf444e..1f9c96a5d8 100644 --- a/frappe/core/doctype/language/test_language.py +++ b/frappe/core/doctype/language/test_language.py @@ -1,10 +1,7 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and Contributors # License: MIT. See LICENSE import unittest -import frappe - # test_records = frappe.get_test_records('Language') diff --git a/frappe/core/doctype/log_setting_user/log_setting_user.js b/frappe/core/doctype/log_setting_user/log_setting_user.js index a1eb824e22..61f3aa67c5 100644 --- a/frappe/core/doctype/log_setting_user/log_setting_user.js +++ b/frappe/core/doctype/log_setting_user/log_setting_user.js @@ -1,8 +1,7 @@ // Copyright (c) 2020, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Log Setting User', { +frappe.ui.form.on("Log Setting User", { // refresh: function(frm) { - // } }); diff --git a/frappe/core/doctype/log_setting_user/log_setting_user.py b/frappe/core/doctype/log_setting_user/log_setting_user.py index cf66da31b1..830d29e3be 100644 --- a/frappe/core/doctype/log_setting_user/log_setting_user.py +++ b/frappe/core/doctype/log_setting_user/log_setting_user.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and contributors # License: MIT. See LICENSE diff --git a/frappe/core/doctype/log_setting_user/test_log_setting_user.py b/frappe/core/doctype/log_setting_user/test_log_setting_user.py index dc70677079..9ea56e8ec4 100644 --- a/frappe/core/doctype/log_setting_user/test_log_setting_user.py +++ b/frappe/core/doctype/log_setting_user/test_log_setting_user.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and Contributors # License: MIT. See LICENSE # import frappe diff --git a/frappe/core/doctype/log_settings/log_settings.js b/frappe/core/doctype/log_settings/log_settings.js index 09a2086a1d..b72cd5285a 100644 --- a/frappe/core/doctype/log_settings/log_settings.js +++ b/frappe/core/doctype/log_settings/log_settings.js @@ -1,8 +1,14 @@ // Copyright (c) 2020, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Log Settings', { - // refresh: function(frm) { - - // } +frappe.ui.form.on("Log Settings", { + refresh: (frm) => { + frm.set_query("ref_doctype", "logs_to_clear", () => { + const added_doctypes = frm.doc.logs_to_clear.map((r) => r.ref_doctype); + return { + query: "frappe.core.doctype.log_settings.log_settings.get_log_doctypes", + filters: [["name", "not in", added_doctypes]], + }; + }); + }, }); diff --git a/frappe/core/doctype/log_settings/log_settings.json b/frappe/core/doctype/log_settings/log_settings.json index f06d14f16b..5a9dd159cc 100644 --- a/frappe/core/doctype/log_settings/log_settings.json +++ b/frappe/core/doctype/log_settings/log_settings.json @@ -5,61 +5,20 @@ "editable_grid": 1, "engine": "InnoDB", "field_order": [ - "error_log_notification_section", - "users_to_notify", - "log_cleanup_section", - "clear_error_log_after", - "clear_activity_log_after", - "column_break_4", - "clear_email_queue_after" + "logs_to_clear" ], "fields": [ { - "fieldname": "log_cleanup_section", - "fieldtype": "Section Break", - "label": "Log Cleanup" - }, - { - "fieldname": "column_break_4", - "fieldtype": "Column Break" - }, - { - "fieldname": "error_log_notification_section", - "fieldtype": "Section Break", - "label": "Error Log Notification" - }, - { - "fieldname": "users_to_notify", - "fieldtype": "Table MultiSelect", - "label": "Users To Notify", - "options": "Log Setting User" - }, - { - "default": "90", - "description": "In Days", - "fieldname": "clear_error_log_after", - "fieldtype": "Int", - "label": "Clear Error log After" - }, - { - "default": "90", - "description": "In Days", - "fieldname": "clear_activity_log_after", - "fieldtype": "Int", - "label": "Clear Activity Log After" - }, - { - "default": "30", - "description": "In Days", - "fieldname": "clear_email_queue_after", - "fieldtype": "Int", - "label": "Clear Email Queue After" + "fieldname": "logs_to_clear", + "fieldtype": "Table", + "label": "Logs to Clear", + "options": "Logs To Clear" } ], "index_web_pages_for_search": 1, "issingle": 1, "links": [], - "modified": "2020-10-13 12:18:48.649038", + "modified": "2022-06-11 02:17:30.803721", "modified_by": "Administrator", "module": "Core", "name": "Log Settings", @@ -79,5 +38,6 @@ "quick_entry": 1, "sort_field": "modified", "sort_order": "DESC", + "states": [], "track_changes": 1 -} +} \ No newline at end of file diff --git a/frappe/core/doctype/log_settings/log_settings.py b/frappe/core/doctype/log_settings/log_settings.py index 58d5fddcd3..8009324e70 100644 --- a/frappe/core/doctype/log_settings/log_settings.py +++ b/frappe/core/doctype/log_settings/log_settings.py @@ -1,45 +1,119 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and contributors # License: MIT. See LICENSE +from typing import Protocol, runtime_checkable + import frappe from frappe import _ +from frappe.model.base_document import get_controller from frappe.model.document import Document -from frappe.query_builder import DocType, Interval -from frappe.query_builder.functions import Now +from frappe.utils import cint +from frappe.utils.caching import site_cache + +DEFAULT_LOGTYPES_RETENTION = { + "Error Log": 30, + "Activity Log": 90, + "Email Queue": 30, + "Error Snapshot": 30, + "Scheduled Job Log": 90, +} + + +@runtime_checkable +class LogType(Protocol): + """Interface requirement for doctypes that can be cleared using log settings.""" + + @staticmethod + def clear_old_logs(days: int) -> None: + ... + + +@site_cache +def _supports_log_clearing(doctype: str) -> bool: + try: + controller = get_controller(doctype) + return issubclass(controller, LogType) + except Exception: + return False class LogSettings(Document): + def validate(self): + self.validate_supported_doctypes() + self.validate_duplicates() + self.add_default_logtypes() + + def validate_supported_doctypes(self): + for entry in self.logs_to_clear: + if _supports_log_clearing(entry.ref_doctype): + continue + + msg = _("{} does not support automated log clearing.").format(frappe.bold(entry.ref_doctype)) + if frappe.conf.developer_mode: + msg += "
    " + _("Implement `clear_old_logs` method to enable auto error clearing.") + frappe.throw(msg, title=_("DocType not supported by Log Settings.")) + + def validate_duplicates(self): + seen = set() + for entry in self.logs_to_clear: + if entry.ref_doctype in seen: + frappe.throw( + _("{} appears more than once in configured log doctypes.").format(entry.ref_doctype) + ) + seen.add(entry.ref_doctype) + + def add_default_logtypes(self): + existing_logtypes = {d.ref_doctype for d in self.logs_to_clear} + added_logtypes = set() + for logtype, retention in DEFAULT_LOGTYPES_RETENTION.items(): + if logtype not in existing_logtypes and _supports_log_clearing(logtype): + self.append("logs_to_clear", {"ref_doctype": logtype, "days": cint(retention)}) + added_logtypes.add(logtype) + + if added_logtypes: + frappe.msgprint( + _("Added default log doctypes: {}").format(",".join(added_logtypes)), alert=True + ) + def clear_logs(self): - self.clear_error_logs() - self.clear_activity_logs() - self.clear_email_queue() + """ + Log settings can clear any log type that's registered to it and provides a method to delete old logs. - def clear_error_logs(self): - table = DocType("Error Log") - frappe.db.delete( - table, filters=(table.creation < (Now() - Interval(days=self.clear_error_log_after))) - ) + Check `LogDoctype` above for interface that doctypes need to implement. + """ - def clear_activity_logs(self): - from frappe.core.doctype.activity_log.activity_log import clear_activity_logs + for entry in self.logs_to_clear: + controller: LogType = get_controller(entry.ref_doctype) + func = controller.clear_old_logs - clear_activity_logs(days=self.clear_activity_log_after) + # Only pass what the method can handle, this is considering any + # future addition that might happen to the required interface. + kwargs = frappe.get_newargs(func, {"days": entry.days}) + func(**kwargs) + frappe.db.commit() - def clear_email_queue(self): - from frappe.email.queue import clear_outbox + def register_doctype(self, doctype: str, days=30): + existing_logtypes = {d.ref_doctype for d in self.logs_to_clear} - clear_outbox(days=self.clear_email_queue_after) + if doctype not in existing_logtypes and _supports_log_clearing(doctype): + self.append("logs_to_clear", {"ref_doctype": doctype, "days": cint(days)}) + else: + for entry in self.logs_to_clear: + if entry.ref_doctype == doctype: + entry.days = days + break def run_log_clean_up(): doc = frappe.get_doc("Log Settings") + doc.add_default_logtypes() + doc.save() doc.clear_logs() @frappe.whitelist() -def has_unseen_error_log(user): - def _get_response(show_alert=True): +def has_unseen_error_log(): + if frappe.get_all("Error Log", filters={"seen": 0}, limit=1): return { "show_alert": True, "message": _("You have unseen {0}").format( @@ -47,13 +121,67 @@ def has_unseen_error_log(user): ), } - if frappe.get_all("Error Log", filters={"seen": 0}, limit=1): - log_settings = frappe.get_cached_doc("Log Settings") - if log_settings.users_to_notify: - if user in [u.user for u in log_settings.users_to_notify]: - return _get_response() - else: - return _get_response(show_alert=False) - else: - return _get_response() +@frappe.whitelist() +@frappe.validate_and_sanitize_search_inputs +def get_log_doctypes(doctype, txt, searchfield, start, page_len, filters): + + filters = filters or {} + + filters.extend( + [ + ["istable", "=", 0], + ["issingle", "=", 0], + ["name", "like", f"%%{txt}%%"], + ] + ) + doctypes = frappe.get_list("DocType", filters=filters, pluck="name") + + supported_doctypes = [(d,) for d in doctypes if _supports_log_clearing(d)] + + return supported_doctypes[start:page_len] + + +LOG_DOCTYPES = [ + "Scheduled Job Log", + "Activity Log", + "Route History", + "Email Queue", + "Email Queue Recipient", + "Error Snapshot", + "Error Log", +] + + +def clear_log_table(doctype, days=90): + """If any logtype table grows too large then clearing it with DELETE query + is not feasible in reasonable time. This command copies recent data to new + table and replaces current table with new smaller table. + + ref: https://mariadb.com/kb/en/big-deletes/#deleting-more-than-half-a-table + """ + from frappe.utils import get_table_name + + if doctype not in LOG_DOCTYPES: + raise frappe.ValidationError(f"Unsupported logging DocType: {doctype}") + + original = get_table_name(doctype) + temporary = f"{original} temp_table" + backup = f"{original} backup_table" + + try: + frappe.db.sql_ddl(f"CREATE TABLE `{temporary}` LIKE `{original}`") + + # Copy all recent data to new table + frappe.db.sql( + f"""INSERT INTO `{temporary}` + SELECT * FROM `{original}` + WHERE `{original}`.`modified` > NOW() - INTERVAL '{days}' DAY""" + ) + frappe.db.sql_ddl(f"RENAME TABLE `{original}` TO `{backup}`, `{temporary}` TO `{original}`") + except Exception: + frappe.db.rollback() + frappe.db.sql_ddl(f"DROP TABLE IF EXISTS `{temporary}`") + raise + else: + frappe.db.sql_ddl(f"DROP TABLE `{backup}`") diff --git a/frappe/core/doctype/log_settings/test_log_settings.py b/frappe/core/doctype/log_settings/test_log_settings.py index 1b78745103..d7f43a181d 100644 --- a/frappe/core/doctype/log_settings/test_log_settings.py +++ b/frappe/core/doctype/log_settings/test_log_settings.py @@ -4,7 +4,7 @@ from datetime import datetime import frappe -from frappe.core.doctype.log_settings.log_settings import run_log_clean_up +from frappe.core.doctype.log_settings.log_settings import _supports_log_clearing, run_log_clean_up from frappe.tests.utils import FrappeTestCase from frappe.utils import add_to_date, now_datetime @@ -56,6 +56,23 @@ class TestLogSettings(FrappeTestCase): self.assertEqual(error_log_count, 0) self.assertEqual(email_queue_count, 0) + def test_logtype_identification(self): + supported_types = [ + "Error Log", + "Activity Log", + "Email Queue", + "Route History", + "Error Snapshot", + "Scheduled Job Log", + ] + + for lt in supported_types: + self.assertTrue(_supports_log_clearing(lt), f"{lt} should be recognized as log type") + + unsupported_types = ["DocType", "User", "Non Existing dt"] + for dt in unsupported_types: + self.assertFalse(_supports_log_clearing(dt), f"{dt} shouldn't be recognized as log type") + def setup_test_logs(past: datetime) -> None: activity_log = frappe.get_doc( diff --git a/frappe/custom/doctype/test_rename_new/__init__.py b/frappe/core/doctype/logs_to_clear/__init__.py similarity index 100% rename from frappe/custom/doctype/test_rename_new/__init__.py rename to frappe/core/doctype/logs_to_clear/__init__.py diff --git a/frappe/core/doctype/logs_to_clear/logs_to_clear.json b/frappe/core/doctype/logs_to_clear/logs_to_clear.json new file mode 100644 index 0000000000..df19ccd9e7 --- /dev/null +++ b/frappe/core/doctype/logs_to_clear/logs_to_clear.json @@ -0,0 +1,43 @@ +{ + "actions": [], + "autoname": "hash", + "creation": "2022-06-11 02:02:39.472511", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "ref_doctype", + "days" + ], + "fields": [ + { + "fieldname": "ref_doctype", + "fieldtype": "Link", + "in_list_view": 1, + "label": "Log DocType", + "options": "DocType", + "reqd": 1 + }, + { + "default": "30", + "fieldname": "days", + "fieldtype": "Int", + "in_list_view": 1, + "label": "Clear Logs After (days)", + "non_negative": 1, + "reqd": 1 + } + ], + "index_web_pages_for_search": 1, + "istable": 1, + "links": [], + "modified": "2022-06-13 02:51:36.857786", + "modified_by": "Administrator", + "module": "Core", + "name": "Logs To Clear", + "owner": "Administrator", + "permissions": [], + "sort_field": "modified", + "sort_order": "DESC", + "states": [] +} diff --git a/frappe/core/doctype/logs_to_clear/logs_to_clear.py b/frappe/core/doctype/logs_to_clear/logs_to_clear.py new file mode 100644 index 0000000000..3fb4f8e72a --- /dev/null +++ b/frappe/core/doctype/logs_to_clear/logs_to_clear.py @@ -0,0 +1,9 @@ +# Copyright (c) 2022, Frappe Technologies and contributors +# For license information, please see license.txt + +# import frappe +from frappe.model.document import Document + + +class LogsToClear(Document): + pass diff --git a/frappe/core/doctype/module_def/module_def.js b/frappe/core/doctype/module_def/module_def.js index 73d2d6562c..8d542e620d 100644 --- a/frappe/core/doctype/module_def/module_def.js +++ b/frappe/core/doctype/module_def/module_def.js @@ -1,13 +1,13 @@ // Copyright (c) 2016, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Module Def', { - refresh: function(frm) { - frappe.xcall('frappe.core.doctype.module_def.module_def.get_installed_apps').then(r => { - frm.set_df_property('app_name', 'options', JSON.parse(r)); +frappe.ui.form.on("Module Def", { + refresh: function (frm) { + frappe.xcall("frappe.core.doctype.module_def.module_def.get_installed_apps").then((r) => { + frm.set_df_property("app_name", "options", JSON.parse(r)); if (!frm.doc.app_name) { - frm.set_value('app_name', 'frappe'); + frm.set_value("app_name", "frappe"); } }); - } + }, }); diff --git a/frappe/core/doctype/module_def/module_def.py b/frappe/core/doctype/module_def/module_def.py index 8f80ffd4ee..c9dac11b69 100644 --- a/frappe/core/doctype/module_def/module_def.py +++ b/frappe/core/doctype/module_def/module_def.py @@ -29,7 +29,7 @@ class ModuleDef(Document): """Adds to `[app]/modules.txt`""" modules = None if not frappe.local.module_app.get(frappe.scrub(self.name)): - with open(frappe.get_app_path(self.app_name, "modules.txt"), "r") as f: + with open(frappe.get_app_path(self.app_name, "modules.txt")) as f: content = f.read() if not self.name in content.splitlines(): modules = list(filter(None, content.splitlines())) @@ -50,7 +50,7 @@ class ModuleDef(Document): modules = None if frappe.local.module_app.get(frappe.scrub(self.name)): - with open(frappe.get_app_path(self.app_name, "modules.txt"), "r") as f: + with open(frappe.get_app_path(self.app_name, "modules.txt")) as f: content = f.read() if self.name in content.splitlines(): modules = list(filter(None, content.splitlines())) diff --git a/frappe/core/doctype/module_def/test_module_def.py b/frappe/core/doctype/module_def/test_module_def.py index 3d269e4734..914ba07949 100644 --- a/frappe/core/doctype/module_def/test_module_def.py +++ b/frappe/core/doctype/module_def/test_module_def.py @@ -1,10 +1,7 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE import unittest -import frappe - # test_records = frappe.get_test_records('Module Def') diff --git a/frappe/core/doctype/module_profile/module_profile.js b/frappe/core/doctype/module_profile/module_profile.js index 3714d31ade..7860577a6c 100644 --- a/frappe/core/doctype/module_profile/module_profile.js +++ b/frappe/core/doctype/module_profile/module_profile.js @@ -19,5 +19,5 @@ frappe.ui.form.on("Module Profile", { if (frm.module_editor) { frm.module_editor.set_modules_in_table(); } - } + }, }); diff --git a/frappe/core/doctype/module_profile/module_profile.py b/frappe/core/doctype/module_profile/module_profile.py index 7c5f896ba8..46c09827f3 100644 --- a/frappe/core/doctype/module_profile/module_profile.py +++ b/frappe/core/doctype/module_profile/module_profile.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and contributors # License: MIT. See LICENSE diff --git a/frappe/core/doctype/module_profile/test_module_profile.py b/frappe/core/doctype/module_profile/test_module_profile.py index e15a70d93f..099d1371fb 100644 --- a/frappe/core/doctype/module_profile/test_module_profile.py +++ b/frappe/core/doctype/module_profile/test_module_profile.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and Contributors # License: MIT. See LICENSE import unittest diff --git a/frappe/core/doctype/navbar_item/navbar_item.js b/frappe/core/doctype/navbar_item/navbar_item.js index bd4274db49..b14d0a5670 100644 --- a/frappe/core/doctype/navbar_item/navbar_item.js +++ b/frappe/core/doctype/navbar_item/navbar_item.js @@ -1,8 +1,7 @@ // Copyright (c) 2020, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Navbar Item', { +frappe.ui.form.on("Navbar Item", { // refresh: function(frm) { - // } }); diff --git a/frappe/core/doctype/navbar_item/navbar_item.py b/frappe/core/doctype/navbar_item/navbar_item.py index 27aa339c93..60be154e75 100644 --- a/frappe/core/doctype/navbar_item/navbar_item.py +++ b/frappe/core/doctype/navbar_item/navbar_item.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and contributors # License: MIT. See LICENSE diff --git a/frappe/core/doctype/navbar_item/test_navbar_item.py b/frappe/core/doctype/navbar_item/test_navbar_item.py index 913e84704b..7eeb4f642b 100644 --- a/frappe/core/doctype/navbar_item/test_navbar_item.py +++ b/frappe/core/doctype/navbar_item/test_navbar_item.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and Contributors # License: MIT. See LICENSE # import frappe diff --git a/frappe/core/doctype/navbar_settings/navbar_settings.js b/frappe/core/doctype/navbar_settings/navbar_settings.js index e2c157fe6a..c0e1113087 100644 --- a/frappe/core/doctype/navbar_settings/navbar_settings.js +++ b/frappe/core/doctype/navbar_settings/navbar_settings.js @@ -1,8 +1,7 @@ // Copyright (c) 2020, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Navbar Settings', { +frappe.ui.form.on("Navbar Settings", { // refresh: function(frm) { - // } }); diff --git a/frappe/core/doctype/navbar_settings/navbar_settings.py b/frappe/core/doctype/navbar_settings/navbar_settings.py index 6243107d63..1eba0f8fa7 100644 --- a/frappe/core/doctype/navbar_settings/navbar_settings.py +++ b/frappe/core/doctype/navbar_settings/navbar_settings.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and contributors # License: MIT. See LICENSE diff --git a/frappe/core/doctype/navbar_settings/test_navbar_settings.py b/frappe/core/doctype/navbar_settings/test_navbar_settings.py index f63e361e8b..76fb3d298a 100644 --- a/frappe/core/doctype/navbar_settings/test_navbar_settings.py +++ b/frappe/core/doctype/navbar_settings/test_navbar_settings.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and Contributors # License: MIT. See LICENSE # import frappe diff --git a/frappe/core/doctype/package/package.js b/frappe/core/doctype/package/package.js index 90e2eed1e3..97d9c32c85 100644 --- a/frappe/core/doctype/package/package.js +++ b/frappe/core/doctype/package/package.js @@ -1,17 +1,20 @@ // Copyright (c) 2021, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Package', { - validate: function(frm) { +frappe.ui.form.on("Package", { + validate: function (frm) { if (!frm.doc.package_name) { - frm.set_value('package_name', frm.doc.name.toLowerCase().replace(' ', '-')); + frm.set_value("package_name", frm.doc.name.toLowerCase().replace(" ", "-")); } }, - license_type: function(frm) { - frappe.call('frappe.core.doctype.package.package.get_license_text', - {'license_type': frm.doc.license_type}).then(r => { - frm.set_value('license', r.message); - }); - } + license_type: function (frm) { + frappe + .call("frappe.core.doctype.package.package.get_license_text", { + license_type: frm.doc.license_type, + }) + .then((r) => { + frm.set_value("license", r.message); + }); + }, }); diff --git a/frappe/core/doctype/package/package.py b/frappe/core/doctype/package/package.py index a32f1bc534..c1cb482832 100644 --- a/frappe/core/doctype/package/package.py +++ b/frappe/core/doctype/package/package.py @@ -15,7 +15,5 @@ class Package(Document): @frappe.whitelist() def get_license_text(license_type): - with open( - os.path.join(os.path.dirname(__file__), "licenses", license_type + ".md"), "r" - ) as textfile: + with open(os.path.join(os.path.dirname(__file__), "licenses", license_type + ".md")) as textfile: return textfile.read() diff --git a/frappe/core/doctype/package_import/package_import.js b/frappe/core/doctype/package_import/package_import.js index c01a6266cc..72f5bbf681 100644 --- a/frappe/core/doctype/package_import/package_import.js +++ b/frappe/core/doctype/package_import/package_import.js @@ -1,8 +1,7 @@ // Copyright (c) 2021, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Package Import', { +frappe.ui.form.on("Package Import", { // refresh: function(frm) { - // } }); diff --git a/frappe/core/doctype/package_import/package_import.py b/frappe/core/doctype/package_import/package_import.py index 659017c498..19762eae4a 100644 --- a/frappe/core/doctype/package_import/package_import.py +++ b/frappe/core/doctype/package_import/package_import.py @@ -44,7 +44,7 @@ class PackageImport(Document): package_path = frappe.get_site_path("packages", package_name) # import Package - with open(os.path.join(package_path, package_name + ".json"), "r") as packagefile: + with open(os.path.join(package_path, package_name + ".json")) as packagefile: doc_dict = json.loads(packagefile.read()) frappe.flags.package = import_doc(doc_dict) @@ -55,11 +55,11 @@ class PackageImport(Document): for module in os.listdir(package_path): module_path = os.path.join(package_path, module) if os.path.isdir(module_path): - get_doc_files(files, module_path) + files = get_doc_files(files, module_path) # import files for file in files: - import_file_by_path(file, force=self.force, ignore_version=True, for_sync=True) - log.append("Imported {}".format(file)) + import_file_by_path(file, force=self.force, ignore_version=True) + log.append(f"Imported {file}") self.log = "\n".join(log) diff --git a/frappe/core/doctype/package_release/package_release.js b/frappe/core/doctype/package_release/package_release.js index 9eabe36839..af482fc4a0 100644 --- a/frappe/core/doctype/package_release/package_release.js +++ b/frappe/core/doctype/package_release/package_release.js @@ -1,8 +1,7 @@ // Copyright (c) 2021, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Package Release', { +frappe.ui.form.on("Package Release", { // refresh: function(frm) { - // } }); diff --git a/frappe/core/doctype/package_release/package_release.py b/frappe/core/doctype/package_release/package_release.py index 05277dcf2e..58fdc2ab86 100644 --- a/frappe/core/doctype/package_release/package_release.py +++ b/frappe/core/doctype/package_release/package_release.py @@ -87,7 +87,7 @@ class PackageRelease(Document): def make_tarfile(self, package): # make tarfile - filename = "{}.tar.gz".format(self.name) + filename = f"{self.name}.tar.gz" subprocess.check_output( ["tar", "czf", filename, package.package_name], cwd=frappe.get_site_path("packages") ) diff --git a/frappe/core/doctype/page/page.js b/frappe/core/doctype/page/page.js index d1d9600e59..295dca1aae 100644 --- a/frappe/core/doctype/page/page.js +++ b/frappe/core/doctype/page/page.js @@ -1,16 +1,16 @@ // Copyright (c) 2016, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Page', { - refresh: function(frm) { - if (!frappe.boot.developer_mode && frappe.session.user != 'Administrator') { +frappe.ui.form.on("Page", { + refresh: function (frm) { + if (!frappe.boot.developer_mode && frappe.session.user != "Administrator") { // make the document read-only frm.set_read_only(); } if (!frm.is_new() && !frm.doc.istable) { - frm.add_custom_button(__('Go to {0} Page', [frm.doc.title || frm.doc.name]), () => { + frm.add_custom_button(__("Go to {0} Page", [frm.doc.title || frm.doc.name]), () => { frappe.set_route(frm.doc.name); }); } - } + }, }); diff --git a/frappe/core/doctype/page/page.json b/frappe/core/doctype/page/page.json index 0c586643d4..e913f126af 100644 --- a/frappe/core/doctype/page/page.json +++ b/frappe/core/doctype/page/page.json @@ -1,415 +1,133 @@ { - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 1, - "autoname": "field:page_name", - "beta": 0, - "creation": "2012-12-20 17:16:49", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "System", - "editable_grid": 0, - "engine": "InnoDB", + "actions": [], + "allow_rename": 1, + "autoname": "field:page_name", + "creation": "2012-12-20 17:16:49", + "doctype": "DocType", + "document_type": "System", + "engine": "InnoDB", + "field_order": [ + "system_page", + "page_html", + "page_name", + "title", + "icon", + "column_break0", + "module", + "restrict_to_domain", + "standard", + "section_break0", + "roles" + ], "fields": [ { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "system_page", - "fieldtype": "Check", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "System Page", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "default": "0", + "fieldname": "system_page", + "fieldtype": "Check", + "label": "System Page" + }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "page_html", - "fieldtype": "Section Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Page HTML", - "length": 0, - "no_copy": 0, - "oldfieldtype": "Section Break", - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "page_html", + "fieldtype": "Section Break", + "label": "Page HTML", + "oldfieldtype": "Section Break" + }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "page_name", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Page Name", - "length": 0, - "no_copy": 0, - "oldfieldname": "page_name", - "oldfieldtype": "Data", - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "page_name", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Page Name", + "oldfieldname": "page_name", + "oldfieldtype": "Data", + "reqd": 1, + "unique": 1 + }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "title", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Title", - "length": 0, - "no_copy": 1, - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "title", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Title", + "no_copy": 1 + }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "icon", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "icon", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "icon", + "fieldtype": "Data", + "in_list_view": 1, + "label": "icon" + }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "column_break0", - "fieldtype": "Column Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "column_break0", + "fieldtype": "Column Break" + }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "module", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 1, - "label": "Module", - "length": 0, - "no_copy": 0, - "oldfieldname": "module", - "oldfieldtype": "Select", - "options": "Module Def", - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "module", + "fieldtype": "Link", + "in_standard_filter": 1, + "label": "Module", + "oldfieldname": "module", + "oldfieldtype": "Select", + "options": "Module Def", + "reqd": 1 + }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "restrict_to_domain", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Restrict To Domain", - "length": 0, - "no_copy": 0, - "options": "Domain", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "restrict_to_domain", + "fieldtype": "Link", + "label": "Restrict To Domain", + "options": "Domain" + }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "standard", - "fieldtype": "Select", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Standard", - "length": 0, - "no_copy": 0, - "oldfieldname": "standard", - "oldfieldtype": "Select", - "options": "Yes\nNo", - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 1, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "standard", + "fieldtype": "Select", + "label": "Standard", + "oldfieldname": "standard", + "oldfieldtype": "Select", + "options": "Yes\nNo", + "reqd": 1, + "search_index": 1 + }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "section_break0", - "fieldtype": "Section Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "section_break0", + "fieldtype": "Section Break" + }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "depends_on": "eval:doc.standard == 'Yes'", - "fieldname": "roles", - "fieldtype": "Table", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Roles", - "length": 0, - "no_copy": 0, - "oldfieldname": "roles", - "oldfieldtype": "Table", - "options": "Has Role", - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "depends_on": "eval:doc.standard == 'Yes'", + "fieldname": "roles", + "fieldtype": "Table", + "label": "Roles", + "oldfieldname": "roles", + "oldfieldtype": "Table", + "options": "Has Role" } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "icon": "fa fa-file", - "idx": 1, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2017-11-13 16:37:04.422547", - "modified_by": "Administrator", - "module": "Core", - "name": "Page", - "owner": "Administrator", + ], + "icon": "fa fa-file", + "idx": 1, + "links": [], + "modified": "2022-08-03 12:20:54.219236", + "modified_by": "Administrator", + "module": "Core", + "name": "Page", + "owner": "Administrator", "permissions": [ { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 1, - "delete": 0, - "email": 1, - "export": 0, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 0, - "role": "Administrator", - "set_user_permissions": 0, - "share": 1, - "submit": 0, + "create": 1, + "email": 1, + "print": 1, + "read": 1, + "role": "Administrator", + "share": 1, "write": 1 - }, + }, { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 0, - "delete": 0, - "email": 1, - "export": 0, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 0, - "role": "System Manager", - "set_user_permissions": 0, - "share": 1, - "submit": 0, + "email": 1, + "print": 1, + "read": 1, + "role": "System Manager", + "share": 1, "write": 1 } - ], - "quick_entry": 0, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_order": "ASC", - "track_changes": 1, - "track_seen": 0 + ], + "sort_field": "modified", + "sort_order": "ASC", + "states": [], + "track_changes": 1 } \ No newline at end of file diff --git a/frappe/core/doctype/page/page.py b/frappe/core/doctype/page/page.py index 7185a25e01..8210875b3a 100644 --- a/frappe/core/doctype/page/page.py +++ b/frappe/core/doctype/page/page.py @@ -79,7 +79,7 @@ class Page(Document): ) def as_dict(self, no_nulls=False): - d = super(Page, self).as_dict(no_nulls=no_nulls) + d = super().as_dict(no_nulls=no_nulls) for key in ("script", "style", "content"): d[key] = self.get(key) return d @@ -120,20 +120,20 @@ class Page(Document): # script fpath = os.path.join(path, page_name + ".js") if os.path.exists(fpath): - with open(fpath, "r") as f: + with open(fpath) as f: self.script = render_include(f.read()) self.script += f"\n\n//# sourceURL={page_name}.js" # css fpath = os.path.join(path, page_name + ".css") if os.path.exists(fpath): - with open(fpath, "r") as f: + with open(fpath) as f: self.style = safe_decode(f.read()) # html as js template for fname in os.listdir(path): if fname.endswith(".html"): - with open(os.path.join(path, fname), "r") as f: + with open(os.path.join(path, fname)) as f: template = f.read() if "" in template: context = frappe._dict({}) diff --git a/frappe/core/doctype/patch_log/patch_log.js b/frappe/core/doctype/patch_log/patch_log.js index 0080584a29..171a1d3a0f 100644 --- a/frappe/core/doctype/patch_log/patch_log.js +++ b/frappe/core/doctype/patch_log/patch_log.js @@ -1,8 +1,8 @@ // Copyright (c) 2016, Frappe Technologies Pvt. Ltd. and contributors // For license information, please see license.txt -frappe.ui.form.on('Patch Log', { - refresh: function(frm) { - - } +frappe.ui.form.on("Patch Log", { + refresh: function (frm) { + frm.disable_save(); + }, }); diff --git a/frappe/core/doctype/patch_log/patch_log.json b/frappe/core/doctype/patch_log/patch_log.json index aa054f1360..9750c51279 100644 --- a/frappe/core/doctype/patch_log/patch_log.json +++ b/frappe/core/doctype/patch_log/patch_log.json @@ -1,87 +1,44 @@ { - "allow_copy": 0, - "allow_import": 0, - "allow_rename": 0, - "autoname": "PATCHLOG.#####", - "beta": 0, - "creation": "2013-01-17 11:36:45", - "custom": 0, - "description": "List of patches executed", - "docstatus": 0, - "doctype": "DocType", - "document_type": "System", - "editable_grid": 0, + "actions": [], + "autoname": "PATCHLOG.#####", + "creation": "2013-01-17 11:36:45", + "description": "List of patches executed", + "doctype": "DocType", + "document_type": "System", + "engine": "InnoDB", + "field_order": [ + "patch" + ], "fields": [ { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "patch", - "fieldtype": "Code", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Patch", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "fieldname": "patch", + "fieldtype": "Code", + "label": "Patch", + "read_only": 1 } - ], - "hide_heading": 0, - "hide_toolbar": 0, - "icon": "fa fa-cog", - "idx": 1, - "image_view": 0, - "in_create": 0, - - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2016-12-29 14:40:35.048570", - "modified_by": "Administrator", - "module": "Core", - "name": "Patch Log", - "owner": "Administrator", + ], + "icon": "fa fa-cog", + "idx": 1, + "links": [], + "modified": "2022-06-13 05:34:37.845368", + "modified_by": "Administrator", + "module": "Core", + "name": "Patch Log", + "naming_rule": "Expression (old style)", + "owner": "Administrator", "permissions": [ { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 0, - "delete": 0, - "email": 1, - "export": 0, - "if_owner": 0, - "import": 0, - "is_custom": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 1, - "role": "Administrator", - "set_user_permissions": 0, - "share": 0, - "submit": 0, - "write": 0 + "email": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "Administrator" } - ], - "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "track_changes": 1, - "track_seen": 0 + ], + "quick_entry": 1, + "sort_field": "modified", + "sort_order": "DESC", + "states": [], + "title_field": "patch", + "track_changes": 1 } \ No newline at end of file diff --git a/frappe/core/doctype/patch_log/patch_log.py b/frappe/core/doctype/patch_log/patch_log.py index dbedbebdeb..eee57af4c2 100644 --- a/frappe/core/doctype/patch_log/patch_log.py +++ b/frappe/core/doctype/patch_log/patch_log.py @@ -3,7 +3,6 @@ # License: MIT. See LICENSE -import frappe from frappe.model.document import Document diff --git a/frappe/core/doctype/patch_log/test_patch_log.py b/frappe/core/doctype/patch_log/test_patch_log.py index 521eaf5e41..0c8a2ae4d4 100644 --- a/frappe/core/doctype/patch_log/test_patch_log.py +++ b/frappe/core/doctype/patch_log/test_patch_log.py @@ -1,10 +1,7 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE import unittest -import frappe - # test_records = frappe.get_test_records('Patch Log') diff --git a/frappe/core/doctype/payment_gateway/payment_gateway.js b/frappe/core/doctype/payment_gateway/payment_gateway.js deleted file mode 100644 index 0eff5a5608..0000000000 --- a/frappe/core/doctype/payment_gateway/payment_gateway.js +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright (c) 2016, Frappe Technologies Pvt. Ltd. and contributors -// For license information, please see license.txt - -frappe.ui.form.on('Payment Gateway', { - refresh: function(frm) { - - } -}); diff --git a/frappe/core/doctype/payment_gateway/payment_gateway.json b/frappe/core/doctype/payment_gateway/payment_gateway.json deleted file mode 100644 index 7195b3949e..0000000000 --- a/frappe/core/doctype/payment_gateway/payment_gateway.json +++ /dev/null @@ -1,55 +0,0 @@ -{ - "actions": [], - "autoname": "field:gateway", - "creation": "2022-01-24 21:09:47.229371", - "doctype": "DocType", - "editable_grid": 1, - "engine": "InnoDB", - "field_order": [ - "gateway", - "gateway_settings", - "gateway_controller" - ], - "fields": [ - { - "fieldname": "gateway", - "fieldtype": "Data", - "in_list_view": 1, - "label": "Gateway", - "reqd": 1, - "unique": 1 - }, - { - "fieldname": "gateway_settings", - "fieldtype": "Link", - "label": "Gateway Settings", - "options": "DocType" - }, - { - "fieldname": "gateway_controller", - "fieldtype": "Dynamic Link", - "label": "Gateway Controller", - "options": "gateway_settings" - } - ], - "links": [], - "modified": "2022-01-24 21:17:03.864719", - "modified_by": "Administrator", - "module": "Core", - "name": "Payment Gateway", - "naming_rule": "By fieldname", - "owner": "Administrator", - "permissions": [ - { - "create": 1, - "delete": 1, - "read": 1, - "role": "System Manager", - "write": 1 - } - ], - "quick_entry": 1, - "sort_field": "modified", - "sort_order": "DESC", - "states": [] -} \ No newline at end of file diff --git a/frappe/core/doctype/payment_gateway/payment_gateway.py b/frappe/core/doctype/payment_gateway/payment_gateway.py deleted file mode 100644 index c48fd340cd..0000000000 --- a/frappe/core/doctype/payment_gateway/payment_gateway.py +++ /dev/null @@ -1,10 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors -# License: MIT. See LICENSE - -import frappe -from frappe.model.document import Document - - -class PaymentGateway(Document): - pass diff --git a/frappe/core/doctype/payment_gateway/test_payment_gateway.py b/frappe/core/doctype/payment_gateway/test_payment_gateway.py deleted file mode 100644 index d40c7bbece..0000000000 --- a/frappe/core/doctype/payment_gateway/test_payment_gateway.py +++ /dev/null @@ -1,12 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors -# License: MIT. See LICENSE -import unittest - -import frappe - -# test_records = frappe.get_test_records('Payment Gateway') - - -class TestPaymentGateway(unittest.TestCase): - pass diff --git a/frappe/core/doctype/prepared_report/prepared_report.js b/frappe/core/doctype/prepared_report/prepared_report.js index 6a7cf2728c..58f4c1957f 100644 --- a/frappe/core/doctype/prepared_report/prepared_report.js +++ b/frappe/core/doctype/prepared_report/prepared_report.js @@ -1,15 +1,15 @@ // Copyright (c) 2018, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Prepared Report', { - onload: function(frm) { +frappe.ui.form.on("Prepared Report", { + onload: function (frm) { var wrapper = $(frm.fields_dict["filter_values"].wrapper).empty(); let filter_table = $(` - - + + @@ -17,29 +17,29 @@ frappe.ui.form.on('Prepared Report', { const filters = JSON.parse(frm.doc.filters); - Object.keys(filters).forEach(key => { + Object.keys(filters).forEach((key) => { const filter_row = $(``); - filter_table.find('tbody').append(filter_row); + filter_table.find("tbody").append(filter_row); }); wrapper.append(filter_table); }, - refresh: function(frm) { + refresh: function (frm) { frm.disable_save(); - if (frm.doc.status == 'Completed') { + if (frm.doc.status == "Completed") { frm.page.set_primary_action(__("Show Report"), () => { frappe.set_route( "query-report", frm.doc.report_name, frappe.utils.make_query_string({ - prepared_report_name: frm.doc.name + prepared_report_name: frm.doc.name, }) ); }); } - } + }, }); diff --git a/frappe/core/doctype/prepared_report/prepared_report.json b/frappe/core/doctype/prepared_report/prepared_report.json index 4663dcb463..cafe323519 100644 --- a/frappe/core/doctype/prepared_report/prepared_report.json +++ b/frappe/core/doctype/prepared_report/prepared_report.json @@ -23,15 +23,14 @@ { "fieldname": "report_name", "fieldtype": "Data", - "hidden": 1, "label": "Report Name", "read_only": 1 }, { "fieldname": "ref_report_doctype", "fieldtype": "Link", - "hidden": 1, - "label": "Ref Report DocType", + "in_standard_filter": 1, + "label": "Report Type", "options": "Report", "read_only": 1 }, @@ -41,6 +40,7 @@ "fieldtype": "Select", "hidden": 1, "in_list_view": 1, + "in_standard_filter": 1, "label": "Status", "options": "Error\nQueued\nCompleted", "read_only": 1 @@ -103,10 +103,11 @@ ], "in_create": 1, "links": [], - "modified": "2020-03-05 10:52:56.598365", + "modified": "2022-06-13 06:20:34.496412", "modified_by": "Administrator", "module": "Core", "name": "Prepared Report", + "naming_rule": "Expression (old style)", "owner": "Administrator", "permissions": [ { @@ -131,9 +132,9 @@ "share": 1 } ], - "quick_entry": 1, "sort_field": "modified", "sort_order": "DESC", - "title_field": "report_name", + "states": [], + "title_field": "ref_report_doctype", "track_changes": 1 } \ No newline at end of file diff --git a/frappe/core/doctype/prepared_report/prepared_report.py b/frappe/core/doctype/prepared_report/prepared_report.py index c3122fe52f..0ff4ce3070 100644 --- a/frappe/core/doctype/prepared_report/prepared_report.py +++ b/frappe/core/doctype/prepared_report/prepared_report.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2018, Frappe Technologies and contributors # License: MIT. See LICENSE @@ -47,7 +46,7 @@ def run_background(prepared_report): instance.save(ignore_permissions=True) except Exception: - frappe.log_error(frappe.get_traceback()) + report.log_error("Prepared report failed") instance = frappe.get_doc("Prepared Report", prepared_report) instance.status = "Error" instance.error_message = frappe.get_traceback() @@ -103,7 +102,7 @@ def delete_prepared_reports(reports): def create_json_gz_file(data, dt, dn): # Storing data in CSV file causes information loss # Reports like P&L Statement were completely unsuable because of this - json_filename = "{0}.json.gz".format( + json_filename = "{}.json.gz".format( frappe.utils.data.format_datetime(frappe.utils.now(), "Y-m-d-H:M") ) encoded_content = frappe.safe_encode(frappe.as_json(data)) diff --git a/frappe/core/doctype/prepared_report/prepared_report_list.js b/frappe/core/doctype/prepared_report/prepared_report_list.js index 8acb3bc75a..1414dd7580 100644 --- a/frappe/core/doctype/prepared_report/prepared_report_list.js +++ b/frappe/core/doctype/prepared_report/prepared_report_list.js @@ -1,12 +1,12 @@ -frappe.listview_settings['Prepared Report'] = { +frappe.listview_settings["Prepared Report"] = { add_fields: ["status"], - get_indicator: function(doc) { - if(doc.status==="Completed"){ + get_indicator: function (doc) { + if (doc.status === "Completed") { return [__("Completed"), "green", "status,=,Completed"]; - } else if(doc.status ==="Error"){ + } else if (doc.status === "Error") { return [__("Error"), "red", "status,=,Error"]; - } else if(doc.status ==="Queued"){ + } else if (doc.status === "Queued") { return [__("Queued"), "orange", "status,=,Queued"]; } - } -}; \ No newline at end of file + }, +}; diff --git a/frappe/core/doctype/prepared_report/test_prepared_report.py b/frappe/core/doctype/prepared_report/test_prepared_report.py index 86793cb802..6d0c809a01 100644 --- a/frappe/core/doctype/prepared_report/test_prepared_report.py +++ b/frappe/core/doctype/prepared_report/test_prepared_report.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2018, Frappe Technologies and Contributors # License: MIT. See LICENSE import json diff --git a/frappe/core/doctype/report/report.js b/frappe/core/doctype/report/report.js index 71ed0dac64..c912e217a6 100644 --- a/frappe/core/doctype/report/report.js +++ b/frappe/core/doctype/report/report.js @@ -1,5 +1,5 @@ -frappe.ui.form.on('Report', { - refresh: function(frm) { +frappe.ui.form.on("Report", { + refresh: function (frm) { if (frm.doc.is_standard === "Yes" && !frappe.boot.developer_mode) { // make the document read-only frm.disable_form(); @@ -8,43 +8,51 @@ frappe.ui.form.on('Report', { } let doc = frm.doc; - frm.add_custom_button(__("Show Report"), function() { - switch(doc.report_type) { - case "Report Builder": - frappe.set_route('List', doc.ref_doctype, 'Report', doc.name); - break; - case "Query Report": - frappe.set_route("query-report", doc.name); - break; - case "Script Report": - frappe.set_route("query-report", doc.name); - break; - case "Custom Report": - frappe.set_route("query-report", doc.name); - break; - } - }, "fa fa-table"); + frm.add_custom_button( + __("Show Report"), + function () { + switch (doc.report_type) { + case "Report Builder": + frappe.set_route("List", doc.ref_doctype, "Report", doc.name); + break; + case "Query Report": + frappe.set_route("query-report", doc.name); + break; + case "Script Report": + frappe.set_route("query-report", doc.name); + break; + case "Custom Report": + frappe.set_route("query-report", doc.name); + break; + } + }, + "fa fa-table" + ); if (doc.is_standard === "Yes" && frm.perm[0].write) { - frm.add_custom_button(doc.disabled ? __("Enable Report") : __("Disable Report"), function() { - frm.call('toggle_disable', { - disable: doc.disabled ? 0 : 1 - }).then(() => { - frm.reload_doc(); - }); - }, doc.disabled ? "fa fa-check" : "fa fa-off"); + frm.add_custom_button( + doc.disabled ? __("Enable Report") : __("Disable Report"), + function () { + frm.call("toggle_disable", { + disable: doc.disabled ? 0 : 1, + }).then(() => { + frm.reload_doc(); + }); + }, + doc.disabled ? "fa fa-check" : "fa fa-off" + ); } }, - ref_doctype: function(frm) { - if(frm.doc.ref_doctype) { + ref_doctype: function (frm) { + if (frm.doc.ref_doctype) { frm.trigger("set_doctype_roles"); } }, - set_doctype_roles: function(frm) { - return frm.call('set_doctype_roles').then(() => { - frm.refresh_field('roles'); + set_doctype_roles: function (frm) { + return frm.call("set_doctype_roles").then(() => { + frm.refresh_field("roles"); }); - } -}) + }, +}); diff --git a/frappe/core/doctype/report/report.py b/frappe/core/doctype/report/report.py index 9e6cc73f11..7fe3cadf9c 100644 --- a/frappe/core/doctype/report/report.py +++ b/frappe/core/doctype/report/report.py @@ -89,7 +89,9 @@ class Report(Document): ] custom_roles = get_custom_allowed_roles("report", self.name) - allowed.extend(custom_roles) + + if custom_roles: + allowed = custom_roles if not allowed: return True @@ -241,7 +243,7 @@ class Report(Document): @staticmethod def _format(parts): # sort by is saved as DocType.fieldname, covert it to sql - return "`tab{0}`.`{1}`".format(*parts) + return "`tab{}`.`{}`".format(*parts) def get_standard_report_columns(self, params): if params.get("fields"): @@ -363,9 +365,7 @@ def get_group_by_field(args, doctype): if args["aggregate_function"] == "count": group_by_field = "count(*) as _aggregate_column" else: - group_by_field = "{0}({1}) as _aggregate_column".format( - args.aggregate_function, args.aggregate_on - ) + group_by_field = f"{args.aggregate_function}({args.aggregate_on}) as _aggregate_column" return group_by_field diff --git a/frappe/core/doctype/report/test_report.py b/frappe/core/doctype/report/test_report.py index 7b17a5a8d5..0e1ed80eda 100644 --- a/frappe/core/doctype/report/test_report.py +++ b/frappe/core/doctype/report/test_report.py @@ -22,7 +22,7 @@ class TestReport(FrappeTestCase): if frappe.db.exists("Report", "User Activity Report"): frappe.delete_doc("Report", "User Activity Report") - with open(os.path.join(os.path.dirname(__file__), "user_activity_report.json"), "r") as f: + with open(os.path.join(os.path.dirname(__file__), "user_activity_report.json")) as f: frappe.get_doc(json.loads(f.read())).insert() report = frappe.get_doc("Report", "User Activity Report") @@ -186,12 +186,44 @@ class TestReport(FrappeTestCase): self.assertNotEqual(report.is_permitted(), True) frappe.set_user("Administrator") + def test_report_custom_permissions(self): + frappe.set_user("test@example.com") + frappe.db.delete("Custom Role", {"report": "Test Custom Role Report"}) + frappe.db.commit() # nosemgrep + if not frappe.db.exists("Report", "Test Custom Role Report"): + report = frappe.get_doc( + { + "doctype": "Report", + "ref_doctype": "User", + "report_name": "Test Custom Role Report", + "report_type": "Query Report", + "is_standard": "No", + "roles": [{"role": "_Test Role"}, {"role": "System Manager"}], + } + ).insert(ignore_permissions=True) + else: + report = frappe.get_doc("Report", "Test Custom Role Report") + + self.assertEqual(report.is_permitted(), True) + + frappe.get_doc( + { + "doctype": "Custom Role", + "report": "Test Custom Role Report", + "roles": [{"role": "_Test Role 2"}], + "ref_doctype": "User", + } + ).insert(ignore_permissions=True) + + self.assertNotEqual(report.is_permitted(), True) + frappe.set_user("Administrator") + # test for the `_format` method if report data doesn't have sort_by parameter def test_format_method(self): if frappe.db.exists("Report", "User Activity Report Without Sort"): frappe.delete_doc("Report", "User Activity Report Without Sort") with open( - os.path.join(os.path.dirname(__file__), "user_activity_report_without_sort.json"), "r" + os.path.join(os.path.dirname(__file__), "user_activity_report_without_sort.json") ) as f: frappe.get_doc(json.loads(f.read())).insert() diff --git a/frappe/core/doctype/report_column/report_column.py b/frappe/core/doctype/report_column/report_column.py index c0984a5ca8..0d6045d121 100644 --- a/frappe/core/doctype/report_column/report_column.py +++ b/frappe/core/doctype/report_column/report_column.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and contributors # License: MIT. See LICENSE diff --git a/frappe/core/doctype/report_filter/report_filter.py b/frappe/core/doctype/report_filter/report_filter.py index e35d7064d2..f3a9607e20 100644 --- a/frappe/core/doctype/report_filter/report_filter.py +++ b/frappe/core/doctype/report_filter/report_filter.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and contributors # License: MIT. See LICENSE diff --git a/frappe/core/doctype/role/role.js b/frappe/core/doctype/role/role.js index f436c8c166..c0a65bcf58 100644 --- a/frappe/core/doctype/role/role.js +++ b/frappe/core/doctype/role/role.js @@ -1,17 +1,24 @@ -// Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors -// MIT License. See license.txt +// Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors +// MIT License. See LICENSE -frappe.ui.form.on('Role', { - refresh: function(frm) { - frm.set_df_property('is_custom', 'read_only', frappe.session.user !== 'Administrator'); +frappe.ui.form.on("Role", { + refresh: function (frm) { + if (frm.doc.name === "All") { + frm.dashboard.add_comment( + __("Role 'All' will be given to all System Users."), + "yellow" + ); + } - frm.add_custom_button("Role Permissions Manager", function() { - frappe.route_options = {"role": frm.doc.name}; + frm.set_df_property("is_custom", "read_only", frappe.session.user !== "Administrator"); + + frm.add_custom_button("Role Permissions Manager", function () { + frappe.route_options = { role: frm.doc.name }; frappe.set_route("permission-manager"); }); - frm.add_custom_button("Show Users", function() { - frappe.route_options = {"role": frm.doc.name}; + frm.add_custom_button("Show Users", function () { + frappe.route_options = { role: frm.doc.name }; frappe.set_route("List", "User", "Report"); }); - } + }, }); diff --git a/frappe/core/doctype/role/role.py b/frappe/core/doctype/role/role.py index 7092004eaf..97a0e9b581 100644 --- a/frappe/core/doctype/role/role.py +++ b/frappe/core/doctype/role/role.py @@ -67,7 +67,10 @@ class Role(Document): def get_info_based_on_role(role, field="email"): """Get information of all users that have been assigned this role""" users = frappe.get_list( - "Has Role", filters={"role": role}, parent_doctype="User", fields=["parent as user_name"] + "Has Role", + filters={"role": role, "parenttype": "User"}, + parent_doctype="User", + fields=["parent as user_name"], ) return get_user_info(users, field) @@ -96,7 +99,7 @@ def get_users(role): @frappe.whitelist() @frappe.validate_and_sanitize_search_inputs def role_query(doctype, txt, searchfield, start, page_len, filters): - report_filters = [["Role", "name", "like", "%{}%".format(txt)], ["Role", "is_custom", "=", 0]] + report_filters = [["Role", "name", "like", f"%{txt}%"], ["Role", "is_custom", "=", 0]] if filters and isinstance(filters, list): report_filters.extend(filters) diff --git a/frappe/core/doctype/role/test_role.py b/frappe/core/doctype/role/test_role.py index a94796436d..44b9b1cdee 100644 --- a/frappe/core/doctype/role/test_role.py +++ b/frappe/core/doctype/role/test_role.py @@ -3,6 +3,7 @@ import unittest import frappe +from frappe.core.doctype.role.role import get_info_based_on_role test_records = frappe.get_test_records("Role") @@ -43,3 +44,11 @@ class TestUser(unittest.TestCase): role.save() user.reload() self.assertTrue(user.user_type == "Website User") + + def test_get_users_by_role(self): + + role = "System Manager" + sys_managers = get_info_based_on_role(role, field="name") + + for user in sys_managers: + self.assertIn(role, frappe.get_roles(user)) diff --git a/frappe/core/doctype/role_permission_for_page_and_report/role_permission_for_page_and_report.js b/frappe/core/doctype/role_permission_for_page_and_report/role_permission_for_page_and_report.js index 5048d24077..86d09bef27 100644 --- a/frappe/core/doctype/role_permission_for_page_and_report/role_permission_for_page_and_report.js +++ b/frappe/core/doctype/role_permission_for_page_and_report/role_permission_for_page_and_report.js @@ -1,22 +1,22 @@ // Copyright (c) 2016, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Role Permission for Page and Report', { - setup: function(frm) { +frappe.ui.form.on("Role Permission for Page and Report", { + setup: function (frm) { frm.trigger("set_queries"); }, - refresh: function(frm) { + refresh: function (frm) { frm.disable_save(); frm.role_area.hide(); frm.events.setup_buttons(frm); }, - setup_buttons: function(frm) { + setup_buttons: function (frm) { frm.clear_custom_buttons(); frm.page.clear_actions(); if (frm.doc.set_role_for && frm.doc[frappe.model.scrub(frm.doc.set_role_for)]) { - frm.add_custom_button(__("Reset to defaults"), function() { + frm.add_custom_button(__("Reset to defaults"), function () { frm.trigger("reset_roles"); }); @@ -26,34 +26,34 @@ frappe.ui.form.on('Role Permission for Page and Report', { } }, - onload: function(frm) { + onload: function (frm) { if (!frm.roles_editor) { frm.role_area = $(frm.fields_dict.roles_html.wrapper); frm.roles_editor = new frappe.RoleEditor(frm.role_area, frm); } }, - set_queries: function(frm) { - frm.set_query("page", function() { + set_queries: function (frm) { + frm.set_query("page", function () { return { filters: { - system_page: 0 - } - } + system_page: 0, + }, + }; }); }, - set_role_for: function(frm) { - frm.trigger("clear_fields") - frm.toggle_display('roles_html', false) + set_role_for: function (frm) { + frm.trigger("clear_fields"); + frm.toggle_display("roles_html", false); }, - clear_fields: function(frm) { - var field = (frm.doc.set_role_for == 'Report') ? 'page' : 'report'; - frm.set_value(field, ''); + clear_fields: function (frm) { + var field = frm.doc.set_role_for == "Report" ? "page" : "report"; + frm.set_value(field, ""); }, - page: function(frm) { + page: function (frm) { frm.events.setup_buttons(frm); if (frm.doc.page) { frm.trigger("set_report_page_data"); @@ -62,7 +62,7 @@ frappe.ui.form.on('Role Permission for Page and Report', { } }, - report: function(frm) { + report: function (frm) { frm.events.setup_buttons(frm); if (frm.doc.report) { frm.trigger("set_report_page_data"); @@ -71,57 +71,57 @@ frappe.ui.form.on('Role Permission for Page and Report', { } }, - set_report_page_data: function(frm) { - frm.toggle_display('roles_html', true) + set_report_page_data: function (frm) { + frm.toggle_display("roles_html", true); frm.role_area.show(); return frm.call({ - method:"set_report_page_data", + method: "set_report_page_data", doc: frm.doc, - callback: function(r) { - refresh_field('roles') - frm.roles_editor.show() - } - }) + callback: function (r) { + refresh_field("roles"); + frm.roles_editor.show(); + }, + }); }, - update_report_page_data: function(frm) { - frm.trigger("validate_mandatory_fields") - if(frm.roles_editor) { - frm.roles_editor.set_roles_in_table() + update_report_page_data: function (frm) { + frm.trigger("validate_mandatory_fields"); + if (frm.roles_editor) { + frm.roles_editor.set_roles_in_table(); } return frm.call({ - method:"update_report_page_data", + method: "update_report_page_data", doc: frm.doc, - callback: function(r) { - refresh_field('roles') - frm.roles_editor.show() - frappe.msgprint(__("Successfully Updated")) - } - }) + callback: function (r) { + refresh_field("roles"); + frm.roles_editor.show(); + frappe.msgprint(__("Successfully Updated")); + }, + }); }, - reset_roles: function(frm) { - frm.trigger("validate_mandatory_fields") + reset_roles: function (frm) { + frm.trigger("validate_mandatory_fields"); return frm.call({ - method:"reset_roles", + method: "reset_roles", doc: frm.doc, - callback: function(r) { - refresh_field('roles') - frm.roles_editor.show() - frappe.msgprint(__("Successfully Updated")) - } - }) + callback: function (r) { + refresh_field("roles"); + frm.roles_editor.show(); + frappe.msgprint(__("Successfully Updated")); + }, + }); }, - validate_mandatory_fields: function(frm) { - if(!frm.doc.set_role_for){ - frappe.throw(__("Mandatory field: set role for")) + validate_mandatory_fields: function (frm) { + if (!frm.doc.set_role_for) { + frappe.throw(__("Mandatory field: set role for")); } - if(frm.doc.set_role_for && !frm.doc[frm.doc.set_role_for.toLocaleLowerCase()]) { - frappe.throw(__("Mandatory field: {0}", [frm.doc.set_role_for])) + if (frm.doc.set_role_for && !frm.doc[frm.doc.set_role_for.toLocaleLowerCase()]) { + frappe.throw(__("Mandatory field: {0}", [frm.doc.set_role_for])); } - } + }, }); diff --git a/frappe/core/doctype/role_permission_for_page_and_report/role_permission_for_page_and_report.json b/frappe/core/doctype/role_permission_for_page_and_report/role_permission_for_page_and_report.json index 8a5393b872..09982cf639 100644 --- a/frappe/core/doctype/role_permission_for_page_and_report/role_permission_for_page_and_report.json +++ b/frappe/core/doctype/role_permission_for_page_and_report/role_permission_for_page_and_report.json @@ -1,327 +1,95 @@ { - "allow_copy": 1, - "allow_events_in_timeline": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, - "creation": "2017-02-13 17:33:25.157332", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", + "actions": [], + "allow_copy": 1, + "creation": "2017-02-13 17:33:25.157332", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "set_role_for", + "page", + "report", + "column_break_4", + "disable_prepared_report", + "roles_permission", + "roles_html", + "roles" + ], "fields": [ { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "set_role_for", - "fieldtype": "Select", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Set Role For", - "length": 0, - "no_copy": 0, - "options": "\nPage\nReport", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "set_role_for", + "fieldtype": "Select", + "in_list_view": 1, + "label": "Set Role For", + "options": "\nPage\nReport", + "reqd": 1 + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "depends_on": "eval:doc.set_role_for == 'Page'", - "fieldname": "page", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Page", - "length": 0, - "no_copy": 0, - "options": "Page", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "depends_on": "eval:doc.set_role_for == 'Page'", + "fieldname": "page", + "fieldtype": "Link", + "label": "Page", + "options": "Page" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "depends_on": "eval:doc.set_role_for == 'Report'", - "fieldname": "report", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Report", - "length": 0, - "no_copy": 0, - "options": "Report", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "depends_on": "eval:doc.set_role_for == 'Report'", + "fieldname": "report", + "fieldtype": "Link", + "label": "Report", + "options": "Report" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "column_break_4", - "fieldtype": "Column Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "column_break_4", + "fieldtype": "Column Break" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "depends_on": "report", - "fetch_from": "", - "fieldname": "disable_prepared_report", - "fieldtype": "Check", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Disable Prepared Report", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "default": "0", + "depends_on": "report", + "fieldname": "disable_prepared_report", + "fieldtype": "Check", + "label": "Disable Prepared Report" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "roles_permission", - "fieldtype": "Section Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Allow Roles", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "roles_permission", + "fieldtype": "Section Break", + "label": "Allow Roles" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "depends_on": "", - "fieldname": "roles_html", - "fieldtype": "HTML", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Roles Html", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "roles_html", + "fieldtype": "HTML", + "label": "Roles Html" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "roles", - "fieldtype": "Table", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Roles", - "length": 0, - "no_copy": 0, - "options": "Has Role", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "fieldname": "roles", + "fieldtype": "Table", + "hidden": 1, + "label": "Roles", + "options": "Has Role", + "read_only": 1 } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 1, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 1, - "istable": 0, - "max_attachments": 0, - "modified": "2019-01-25 12:08:57.250719", - "modified_by": "Administrator", - "module": "Core", - "name": "Role Permission for Page and Report", - "name_case": "", - "owner": "Administrator", + ], + "hide_toolbar": 1, + "issingle": 1, + "links": [], + "modified": "2022-08-03 12:20:54.079809", + "modified_by": "Administrator", + "module": "Core", + "name": "Role Permission for Page and Report", + "owner": "Administrator", "permissions": [ { - "amend": 0, - "cancel": 0, - "create": 1, - "delete": 1, - "email": 1, - "export": 0, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 0, - "role": "System Manager", - "set_user_permissions": 0, - "share": 1, - "submit": 0, + "create": 1, + "delete": 1, + "email": 1, + "print": 1, + "read": 1, + "role": "System Manager", + "share": 1, "write": 1 } - ], - "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 0, - "track_seen": 0, - "track_views": 0 + ], + "quick_entry": 1, + "sort_field": "modified", + "sort_order": "DESC", + "states": [] } \ No newline at end of file diff --git a/frappe/core/doctype/role_permission_for_page_and_report/role_permission_for_page_and_report.py b/frappe/core/doctype/role_permission_for_page_and_report/role_permission_for_page_and_report.py index d89131b0d7..bd61995ba3 100644 --- a/frappe/core/doctype/role_permission_for_page_and_report/role_permission_for_page_and_report.py +++ b/frappe/core/doctype/role_permission_for_page_and_report/role_permission_for_page_and_report.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and contributors # License: MIT. See LICENSE diff --git a/frappe/core/doctype/role_profile/role_profile.js b/frappe/core/doctype/role_profile/role_profile.js index e43980770a..1a5ed95287 100644 --- a/frappe/core/doctype/role_profile/role_profile.js +++ b/frappe/core/doctype/role_profile/role_profile.js @@ -1,21 +1,20 @@ // Copyright (c) 2017, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Role Profile', { - refresh: function(frm) { +frappe.ui.form.on("Role Profile", { + refresh: function (frm) { if (has_common(frappe.user_roles, ["Administrator", "System Manager"])) { if (!frm.roles_editor) { const role_area = $(frm.fields_dict.roles_html.wrapper); frm.roles_editor = new frappe.RoleEditor(role_area, frm); } frm.roles_editor.show(); - } }, - validate: function(frm) { + validate: function (frm) { if (frm.roles_editor) { frm.roles_editor.set_roles_in_table(); } - } + }, }); diff --git a/frappe/core/doctype/role_profile/role_profile.py b/frappe/core/doctype/role_profile/role_profile.py index ab4660d7c9..a8abd6d1c1 100644 --- a/frappe/core/doctype/role_profile/role_profile.py +++ b/frappe/core/doctype/role_profile/role_profile.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2017, Frappe Technologies and contributors # License: MIT. See LICENSE diff --git a/frappe/core/doctype/role_profile/test_role_profile.py b/frappe/core/doctype/role_profile/test_role_profile.py index 19239a81cd..726a5fc83e 100644 --- a/frappe/core/doctype/role_profile/test_role_profile.py +++ b/frappe/core/doctype/role_profile/test_role_profile.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2017, Frappe Technologies and Contributors # License: MIT. See LICENSE import unittest diff --git a/frappe/core/doctype/scheduled_job_log/scheduled_job_log.js b/frappe/core/doctype/scheduled_job_log/scheduled_job_log.js index d43160c658..dd9691854d 100644 --- a/frappe/core/doctype/scheduled_job_log/scheduled_job_log.js +++ b/frappe/core/doctype/scheduled_job_log/scheduled_job_log.js @@ -1,8 +1,7 @@ // Copyright (c) 2019, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Scheduled Job Log', { +frappe.ui.form.on("Scheduled Job Log", { // refresh: function(frm) { - // } }); diff --git a/frappe/core/doctype/scheduled_job_log/scheduled_job_log.json b/frappe/core/doctype/scheduled_job_log/scheduled_job_log.json index 396b32bdf9..451c4108a0 100644 --- a/frappe/core/doctype/scheduled_job_log/scheduled_job_log.json +++ b/frappe/core/doctype/scheduled_job_log/scheduled_job_log.json @@ -38,7 +38,7 @@ } ], "links": [], - "modified": "2021-10-25 00:00:00.000000", + "modified": "2022-06-13 05:41:21.090972", "modified_by": "Administrator", "module": "Core", "name": "Scheduled Job Log", @@ -59,5 +59,7 @@ ], "quick_entry": 1, "sort_field": "modified", - "sort_order": "DESC" -} + "sort_order": "DESC", + "states": [], + "title_field": "scheduled_job_type" +} \ No newline at end of file diff --git a/frappe/core/doctype/scheduled_job_log/scheduled_job_log.py b/frappe/core/doctype/scheduled_job_log/scheduled_job_log.py index bead463ba5..01fa837af0 100644 --- a/frappe/core/doctype/scheduled_job_log/scheduled_job_log.py +++ b/frappe/core/doctype/scheduled_job_log/scheduled_job_log.py @@ -1,10 +1,14 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and contributors # License: MIT. See LICENSE -# import frappe +import frappe from frappe.model.document import Document +from frappe.query_builder import Interval +from frappe.query_builder.functions import Now class ScheduledJobLog(Document): - pass + @staticmethod + def clear_old_logs(days=90): + table = frappe.qb.DocType("Scheduled Job Log") + frappe.db.delete(table, filters=(table.modified < (Now() - Interval(days=days)))) diff --git a/frappe/core/doctype/scheduled_job_log/scheduled_job_log_list.js b/frappe/core/doctype/scheduled_job_log/scheduled_job_log_list.js new file mode 100644 index 0000000000..1edd718651 --- /dev/null +++ b/frappe/core/doctype/scheduled_job_log/scheduled_job_log_list.js @@ -0,0 +1,7 @@ +frappe.listview_settings["Scheduled Job Log"] = { + onload: function (listview) { + frappe.require("logtypes.bundle.js", () => { + frappe.utils.logtypes.show_log_retention_message(cur_list.doctype); + }); + }, +}; diff --git a/frappe/core/doctype/scheduled_job_log/test_scheduled_job_log.py b/frappe/core/doctype/scheduled_job_log/test_scheduled_job_log.py index 3c99bb5cb8..11d60e35d8 100644 --- a/frappe/core/doctype/scheduled_job_log/test_scheduled_job_log.py +++ b/frappe/core/doctype/scheduled_job_log/test_scheduled_job_log.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # License: MIT. See LICENSE # import frappe diff --git a/frappe/core/doctype/scheduled_job_type/scheduled_job_type.js b/frappe/core/doctype/scheduled_job_type/scheduled_job_type.js index 55907b17fc..238754277b 100644 --- a/frappe/core/doctype/scheduled_job_type/scheduled_job_type.js +++ b/frappe/core/doctype/scheduled_job_type/scheduled_job_type.js @@ -1,8 +1,7 @@ // Copyright (c) 2019, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Scheduled Job Type', { +frappe.ui.form.on("Scheduled Job Type", { // refresh: function(frm) { - // } }); diff --git a/frappe/core/doctype/scheduled_job_type/scheduled_job_type.json b/frappe/core/doctype/scheduled_job_type/scheduled_job_type.json index d4d79b21fb..cc2a0e870a 100644 --- a/frappe/core/doctype/scheduled_job_type/scheduled_job_type.json +++ b/frappe/core/doctype/scheduled_job_type/scheduled_job_type.json @@ -16,8 +16,11 @@ "server_script", "frequency", "cron_format", + "create_log", + "status_section", "last_execution", - "create_log" + "column_break_9", + "next_execution" ], "fields": [ { @@ -72,6 +75,22 @@ "options": "Server Script", "read_only": 1, "search_index": 1 + }, + { + "fieldname": "next_execution", + "fieldtype": "Datetime", + "is_virtual": 1, + "label": "Next Execution", + "read_only": 1 + }, + { + "fieldname": "status_section", + "fieldtype": "Section Break", + "label": "Status" + }, + { + "fieldname": "column_break_9", + "fieldtype": "Column Break" } ], "in_create": 1, @@ -81,7 +100,7 @@ "link_fieldname": "scheduled_job_type" } ], - "modified": "2020-10-07 10:39:24.519460", + "modified": "2022-06-28 02:55:12.470915", "modified_by": "Administrator", "module": "Core", "name": "Scheduled Job Type", @@ -103,5 +122,7 @@ "quick_entry": 1, "sort_field": "modified", "sort_order": "DESC", + "states": [], + "title_field": "method", "track_changes": 1 } \ No newline at end of file diff --git a/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py b/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py index 9665a20843..1c178fcee2 100644 --- a/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py +++ b/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py @@ -3,8 +3,8 @@ import json from datetime import datetime -from typing import Dict, List +import click from croniter import croniter import frappe @@ -50,6 +50,10 @@ class ScheduledJobType(Document): queued_jobs = get_jobs(site=frappe.local.site, key="job_type")[frappe.local.site] return self.method in queued_jobs + @property + def next_execution(self): + return self.get_next_execution() + def get_next_execution(self): CRON_MAP = { "Yearly": "0 0 1 1 *", @@ -134,14 +138,14 @@ def run_scheduled_job(job_type: str): print(frappe.get_traceback()) -def sync_jobs(hooks: Dict = None): +def sync_jobs(hooks: dict = None): frappe.reload_doc("core", "doctype", "scheduled_job_type") scheduler_events = hooks or frappe.get_hooks("scheduler_events") all_events = insert_events(scheduler_events) clear_events(all_events) -def insert_events(scheduler_events: Dict) -> List: +def insert_events(scheduler_events: dict) -> list: cron_jobs, event_jobs = [], [] for event_type in scheduler_events: events = scheduler_events.get(event_type) @@ -153,7 +157,7 @@ def insert_events(scheduler_events: Dict) -> List: return cron_jobs + event_jobs -def insert_cron_jobs(events: Dict) -> List: +def insert_cron_jobs(events: dict) -> list: cron_jobs = [] for cron_format in events: for event in events.get(cron_format): @@ -162,7 +166,7 @@ def insert_cron_jobs(events: Dict) -> List: return cron_jobs -def insert_event_jobs(events: List, event_type: str) -> List: +def insert_event_jobs(events: list, event_type: str) -> list: event_jobs = [] for event in events: event_jobs.append(event) @@ -173,6 +177,12 @@ def insert_event_jobs(events: List, event_type: str) -> List: def insert_single_event(frequency: str, event: str, cron_format: str = None): cron_expr = {"cron_format": cron_format} if cron_format else {} + + try: + frappe.get_attr(event) + except Exception as e: + click.secho(f"{event} is not a valid method: {e}", fg="yellow") + doc = frappe.get_doc( { "doctype": "Scheduled Job Type", @@ -185,14 +195,17 @@ def insert_single_event(frequency: str, event: str, cron_format: str = None): if not frappe.db.exists( "Scheduled Job Type", {"method": event, "frequency": frequency, **cron_expr} ): + savepoint = "scheduled_job_type_creation" try: + frappe.db.savepoint(savepoint) doc.insert() except frappe.DuplicateEntryError: + frappe.db.rollback(save_point=savepoint) doc.delete() doc.insert() -def clear_events(all_events: List): +def clear_events(all_events: list): for event in frappe.get_all("Scheduled Job Type", fields=["name", "method", "server_script"]): is_server_script = event.server_script is_defined_in_hooks = event.method in all_events diff --git a/frappe/core/doctype/scheduled_job_type/test_scheduled_job_type.py b/frappe/core/doctype/scheduled_job_type/test_scheduled_job_type.py index 3e63985692..5448bda91f 100644 --- a/frappe/core/doctype/scheduled_job_type/test_scheduled_job_type.py +++ b/frappe/core/doctype/scheduled_job_type/test_scheduled_job_type.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # License: MIT. See LICENSE import unittest diff --git a/frappe/core/doctype/server_script/server_script.js b/frappe/core/doctype/server_script/server_script.js index ca34af11ab..ca5b8d721b 100644 --- a/frappe/core/doctype/server_script/server_script.js +++ b/frappe/core/doctype/server_script/server_script.js @@ -1,31 +1,30 @@ // Copyright (c) 2019, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Server Script', { - setup: function(frm) { - frm.trigger('setup_help'); +frappe.ui.form.on("Server Script", { + setup: function (frm) { + frm.trigger("setup_help"); }, - refresh: function(frm) { - if (frm.doc.script_type != 'Scheduler Event') { + refresh: function (frm) { + if (frm.doc.script_type != "Scheduler Event") { frm.dashboard.hide(); } if (!frm.is_new()) { - frm.add_custom_button(__('Compare Versions'), () => { + frm.add_custom_button(__("Compare Versions"), () => { new frappe.ui.DiffView("Server Script", "script", frm.doc.name); }); } - - frm.call('get_autocompletion_items') - .then(r => r.message) - .then(items => { - frm.set_df_property('script', 'autocompletions', items); + frm.call("get_autocompletion_items") + .then((r) => r.message) + .then((items) => { + frm.set_df_property("script", "autocompletions", items); }); }, setup_help(frm) { - frm.get_field('help_html').html(` + frm.get_field("help_html").html(`

    DocType Event

    Add logic for standard doctype events like Before Insert, After Submit, etc.

    @@ -77,6 +76,5 @@ where tenant_id = 2
 order by creation desc
    `); - } - + }, }); diff --git a/frappe/core/doctype/server_script/server_script.json b/frappe/core/doctype/server_script/server_script.json index 548d21bb60..5446cc1a39 100644 --- a/frappe/core/doctype/server_script/server_script.json +++ b/frappe/core/doctype/server_script/server_script.json @@ -25,6 +25,7 @@ "fieldname": "script_type", "fieldtype": "Select", "in_list_view": 1, + "in_standard_filter": 1, "label": "Script Type", "options": "DocType Event\nScheduler Event\nPermission Query\nAPI", "reqd": 1 @@ -41,6 +42,7 @@ "fieldname": "reference_doctype", "fieldtype": "Link", "in_list_view": 1, + "in_standard_filter": 1, "label": "Reference Document Type", "options": "DocType" }, @@ -49,7 +51,7 @@ "fieldname": "doctype_event", "fieldtype": "Select", "label": "DocType Event", - "options": "Before Insert\nBefore Validate\nBefore Save\nAfter Insert\nAfter Save\nBefore Submit\nAfter Submit\nBefore Cancel\nAfter Cancel\nBefore Delete\nAfter Delete\nBefore Save (Submitted Document)\nAfter Save (Submitted Document)" + "options": "Before Insert\nBefore Validate\nBefore Save\nAfter Insert\nAfter Save\nBefore Submit\nAfter Submit\nBefore Cancel\nAfter Cancel\nBefore Delete\nAfter Delete\nBefore Save (Submitted Document)\nAfter Save (Submitted Document)\nOn Payment Authorization" }, { "depends_on": "eval:doc.script_type==='API'", @@ -109,7 +111,7 @@ "link_fieldname": "server_script" } ], - "modified": "2022-04-07 19:41:23.178772", + "modified": "2022-06-13 06:04:20.937969", "modified_by": "Administrator", "module": "Core", "name": "Server Script", diff --git a/frappe/core/doctype/server_script/server_script.py b/frappe/core/doctype/server_script/server_script.py index 0e2eac16ba..fda5ca8591 100644 --- a/frappe/core/doctype/server_script/server_script.py +++ b/frappe/core/doctype/server_script/server_script.py @@ -1,10 +1,7 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and contributors # License: MIT. See LICENSE -import ast from types import FunctionType, MethodType, ModuleType -from typing import Dict, List import frappe from frappe import _ @@ -17,6 +14,7 @@ class ServerScript(Document): frappe.only_for("Script Manager", True) self.sync_scheduled_jobs() self.clear_scheduled_events() + self.check_if_compilable_in_restricted_context() def on_update(self): frappe.cache().delete_value("server_script_map") @@ -31,7 +29,7 @@ class ServerScript(Document): return {"script": "py"} @property - def scheduled_jobs(self) -> List[Dict[str, str]]: + def scheduled_jobs(self) -> list[dict[str, str]]: return frappe.get_all( "Scheduled Job Type", filters={"server_script": self.name}, @@ -60,7 +58,16 @@ class ServerScript(Document): for scheduled_job in self.scheduled_jobs: frappe.delete_doc("Scheduled Job Type", scheduled_job.name) - def execute_method(self) -> Dict: + def check_if_compilable_in_restricted_context(self): + """Check compilation errors and send them back as warnings.""" + from RestrictedPython import compile_restricted + + try: + compile_restricted(self.script) + except Exception as e: + frappe.msgprint(str(e), title=_("Compilation warning")) + + def execute_method(self) -> dict: """Specific to API endpoint Server Scripts Raises: @@ -101,7 +108,7 @@ class ServerScript(Document): safe_exec(self.script) - def get_permission_query_conditions(self, user: str) -> List[str]: + def get_permission_query_conditions(self, user: str) -> list[str]: """Specific to Permission Query Server Scripts Args: diff --git a/frappe/core/doctype/server_script/server_script_utils.py b/frappe/core/doctype/server_script/server_script_utils.py index 5300baa199..b807b43d10 100644 --- a/frappe/core/doctype/server_script/server_script_utils.py +++ b/frappe/core/doctype/server_script/server_script_utils.py @@ -17,6 +17,7 @@ EVENT_MAP = { "after_delete": "After Delete", "before_update_after_submit": "Before Save (Submitted Document)", "on_update_after_submit": "After Save (Submitted Document)", + "on_payment_authorized": "On Payment Authorization", } diff --git a/frappe/core/doctype/server_script/test_server_script.py b/frappe/core/doctype/server_script/test_server_script.py index 2685367695..4c1c12b7f2 100644 --- a/frappe/core/doctype/server_script/test_server_script.py +++ b/frappe/core/doctype/server_script/test_server_script.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # License: MIT. See LICENSE import unittest @@ -72,6 +71,16 @@ frappe.method_that_doesnt_exist("do some magic") disabled=1, script=""" frappe.db.commit() +""", + ), + dict( + name="test_add_index", + script_type="DocType Event", + doctype_event="Before Save", + reference_doctype="ToDo", + disabled=1, + script=""" +frappe.db.add_index("Todo", ["color", "date"]) """, ), ] @@ -153,6 +162,18 @@ class TestServerScript(unittest.TestCase): server_script.disabled = 1 server_script.save() + def test_add_index_in_doctype_event(self): + server_script = frappe.get_doc("Server Script", "test_add_index") + server_script.disabled = 0 + server_script.save() + + self.assertRaises( + AttributeError, frappe.get_doc(dict(doctype="ToDo", description="test me")).insert + ) + + server_script.disabled = 1 + server_script.save() + def test_restricted_qb(self): todo = frappe.get_doc(doctype="ToDo", description="QbScriptTestNote") todo.insert() diff --git a/frappe/core/doctype/session_default/session_default.py b/frappe/core/doctype/session_default/session_default.py index df261f4a39..3fada1b5e0 100644 --- a/frappe/core/doctype/session_default/session_default.py +++ b/frappe/core/doctype/session_default/session_default.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and contributors # License: MIT. See LICENSE diff --git a/frappe/core/doctype/session_default_settings/session_default_settings.js b/frappe/core/doctype/session_default_settings/session_default_settings.js index f7cce14809..af333e29a3 100644 --- a/frappe/core/doctype/session_default_settings/session_default_settings.js +++ b/frappe/core/doctype/session_default_settings/session_default_settings.js @@ -1,15 +1,15 @@ // Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors // MIT License. See license.txt -frappe.ui.form.on('Session Default Settings', { - refresh: function(frm) { - frm.set_query('ref_doctype', 'session_defaults', function() { +frappe.ui.form.on("Session Default Settings", { + refresh: function (frm) { + frm.set_query("ref_doctype", "session_defaults", function () { return { filters: { issingle: 0, - istable: 0 - } + istable: 0, + }, }; }); - } + }, }); diff --git a/frappe/core/doctype/session_default_settings/session_default_settings.py b/frappe/core/doctype/session_default_settings/session_default_settings.py index 4ac9b61553..2b0d731920 100644 --- a/frappe/core/doctype/session_default_settings/session_default_settings.py +++ b/frappe/core/doctype/session_default_settings/session_default_settings.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and contributors # License: MIT. See LICENSE diff --git a/frappe/core/doctype/session_default_settings/test_session_default_settings.py b/frappe/core/doctype/session_default_settings/test_session_default_settings.py index f763f90a1d..aa60085ce9 100644 --- a/frappe/core/doctype/session_default_settings/test_session_default_settings.py +++ b/frappe/core/doctype/session_default_settings/test_session_default_settings.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # License: MIT. See LICENSE import unittest diff --git a/frappe/core/doctype/sms_parameter/__init__.py b/frappe/core/doctype/sms_parameter/__init__.py index 8b13789179..e69de29bb2 100755 --- a/frappe/core/doctype/sms_parameter/__init__.py +++ b/frappe/core/doctype/sms_parameter/__init__.py @@ -1 +0,0 @@ - diff --git a/frappe/core/doctype/sms_parameter/sms_parameter.json b/frappe/core/doctype/sms_parameter/sms_parameter.json index 43b93ed182..98972f9e7d 100755 --- a/frappe/core/doctype/sms_parameter/sms_parameter.json +++ b/frappe/core/doctype/sms_parameter/sms_parameter.json @@ -1,128 +1,51 @@ { - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, - "creation": "2013-02-22 01:27:58", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "editable_grid": 1, + "actions": [], + "creation": "2013-02-22 01:27:58", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "parameter", + "value", + "header" + ], "fields": [ { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "parameter", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Parameter", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "print_width": "150px", - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0, + "fieldname": "parameter", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Parameter", + "print_width": "150px", + "reqd": 1, "width": "150px" - }, + }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "value", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Value", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "print_width": "150px", - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0, + "fieldname": "value", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Value", + "print_width": "150px", + "reqd": 1, "width": "150px" - }, + }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "header", - "fieldtype": "Check", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Header", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "default": "0", + "fieldname": "header", + "fieldtype": "Check", + "in_list_view": 1, + "label": "Header" } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 1, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 1, - "max_attachments": 0, - "modified": "2017-10-13 16:48:00.518463", - "modified_by": "Administrator", - "module": "Core", - "name": "SMS Parameter", - "owner": "Administrator", - "permissions": [], - "quick_entry": 0, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "track_changes": 0, - "track_seen": 0 + ], + "idx": 1, + "istable": 1, + "links": [], + "modified": "2022-08-03 12:20:53.129765", + "modified_by": "Administrator", + "module": "Core", + "name": "SMS Parameter", + "owner": "Administrator", + "permissions": [], + "sort_field": "modified", + "sort_order": "DESC", + "states": [] } \ No newline at end of file diff --git a/frappe/core/doctype/sms_parameter/sms_parameter.py b/frappe/core/doctype/sms_parameter/sms_parameter.py index d67e905234..d2c5be6c50 100644 --- a/frappe/core/doctype/sms_parameter/sms_parameter.py +++ b/frappe/core/doctype/sms_parameter/sms_parameter.py @@ -1,7 +1,6 @@ # Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe from frappe.model.document import Document diff --git a/frappe/core/doctype/sms_settings/__init__.py b/frappe/core/doctype/sms_settings/__init__.py index 8b13789179..e69de29bb2 100755 --- a/frappe/core/doctype/sms_settings/__init__.py +++ b/frappe/core/doctype/sms_settings/__init__.py @@ -1 +0,0 @@ - diff --git a/frappe/core/doctype/sms_settings/sms_settings.py b/frappe/core/doctype/sms_settings/sms_settings.py index e1da200ee5..0a5536eb9b 100644 --- a/frappe/core/doctype/sms_settings/sms_settings.py +++ b/frappe/core/doctype/sms_settings/sms_settings.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE @@ -64,7 +63,7 @@ def send_sms(receiver_list, msg, sender_name="", success_msg=True): "success_msg": success_msg, } - if frappe.db.get_value("SMS Settings", None, "sms_gateway_url"): + if frappe.db.get_single_value("SMS Settings", "sms_gateway_url"): send_via_gateway(arg) else: msgprint(_("Please Update SMS Settings")) diff --git a/frappe/core/doctype/sms_settings/test_sms_settings.py b/frappe/core/doctype/sms_settings/test_sms_settings.py index a7ec761b82..61be20ff66 100644 --- a/frappe/core/doctype/sms_settings/test_sms_settings.py +++ b/frappe/core/doctype/sms_settings/test_sms_settings.py @@ -1,10 +1,7 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2017, Frappe Technologies and Contributors # License: MIT. See LICENSE import unittest -import frappe - class TestSMSSettings(unittest.TestCase): pass diff --git a/frappe/core/doctype/success_action/success_action.js b/frappe/core/doctype/success_action/success_action.js index 50ddb3b66a..993f6eabf4 100644 --- a/frappe/core/doctype/success_action/success_action.js +++ b/frappe/core/doctype/success_action/success_action.js @@ -1,28 +1,28 @@ // Copyright (c) 2018, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Success Action', { +frappe.ui.form.on("Success Action", { on_load: (frm) => { if (!frm.action_multicheck) { - frm.trigger('set_next_action_multicheck'); + frm.trigger("set_next_action_multicheck"); } }, refresh: (frm) => { if (!frm.action_multicheck) { - frm.trigger('set_next_action_multicheck'); + frm.trigger("set_next_action_multicheck"); } }, validate: (frm) => { const checked_actions = frm.action_multicheck.get_checked_options(); if (checked_actions.length < 2) { - frappe.msgprint(__('Select atleast 2 actions')); + frappe.msgprint(__("Select atleast 2 actions")); } else { return true; } }, before_save: (frm) => { const checked_actions = frm.action_multicheck.get_checked_options(); - frm.doc.next_actions = checked_actions.join('\n'); + frm.doc.next_actions = checked_actions.join("\n"); }, after_save: (frm) => { frappe.boot.success_action.push(frm.doc); @@ -30,31 +30,31 @@ frappe.ui.form.on('Success Action', { }, set_next_action_multicheck: (frm) => { const next_actions_wrapper = frm.fields_dict.next_actions_html.$wrapper; - const checked_actions = frm.doc.next_actions ? frm.doc.next_actions.split('\n') : []; - const action_multicheck_options = get_default_next_actions().map(action => { + const checked_actions = frm.doc.next_actions ? frm.doc.next_actions.split("\n") : []; + const action_multicheck_options = get_default_next_actions().map((action) => { return { label: action.label, value: action.value, - checked: checked_actions.length ? checked_actions.includes(action.value) : 1 + checked: checked_actions.length ? checked_actions.includes(action.value) : 1, }; }); frm.action_multicheck = frappe.ui.form.make_control({ parent: next_actions_wrapper, df: { - 'label': 'Next Actions', - 'fieldname': 'next_actions_multicheck', - 'fieldtype': 'MultiCheck', - 'options': action_multicheck_options, + label: "Next Actions", + fieldname: "next_actions_multicheck", + fieldtype: "MultiCheck", + options: action_multicheck_options, }, }); - } + }, }); const get_default_next_actions = () => { return [ - { label: __('New'), value: 'new' }, - { label: __('Print'), value: 'print' }, - { label: __('Email'), value: 'email' }, - { label: __('View All'), value: 'list' } + { label: __("New"), value: "new" }, + { label: __("Print"), value: "print" }, + { label: __("Email"), value: "email" }, + { label: __("View All"), value: "list" }, ]; -}; \ No newline at end of file +}; diff --git a/frappe/core/doctype/success_action/success_action.json b/frappe/core/doctype/success_action/success_action.json index 25c8e79a05..749fa6764f 100644 --- a/frappe/core/doctype/success_action/success_action.json +++ b/frappe/core/doctype/success_action/success_action.json @@ -1,259 +1,84 @@ { - "allow_copy": 0, - "allow_events_in_timeline": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, + "actions": [], "autoname": "field:ref_doctype", - "beta": 0, "creation": "2018-04-15 18:07:35.316870", - "custom": 0, - "docstatus": 0, "doctype": "DocType", - "document_type": "", "editable_grid": 1, "engine": "InnoDB", + "field_order": [ + "ref_doctype", + "first_success_message", + "message", + "next_actions_html", + "next_actions", + "action_timeout" + ], "fields": [ { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "ref_doctype", "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, "in_list_view": 1, - "in_standard_filter": 0, "label": "Reference Document Type", - "length": 0, - "no_copy": 0, "options": "DocType", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "unique": 1 }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "default": "Congratulations on first creations", "fieldname": "first_success_message", "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, "in_list_view": 1, - "in_standard_filter": 0, "label": "First Success Message", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "reqd": 1 }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "default": "Successfully created", "fieldname": "message", "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, "in_list_view": 1, - "in_standard_filter": 0, "label": "Message", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "reqd": 1 }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "next_actions_html", "fieldtype": "HTML", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Next Actions HTML", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "label": "Next Actions HTML" }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "next_actions", "fieldtype": "Data", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "hidden": 1 }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, + "default": "7", "fieldname": "action_timeout", "fieldtype": "Int", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Action Timeout (Seconds)", - "default": 7, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "label": "Action Timeout (Seconds)" } ], - "has_web_view": 0, - "hide_heading": 0, "hide_toolbar": 1, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2019-09-05 14:22:27.664645", + "links": [], + "modified": "2022-08-03 12:20:54.532708", "modified_by": "Administrator", "module": "Core", "name": "Success Action", - "name_case": "", "owner": "Administrator", "permissions": [ { - "amend": 0, - "cancel": 0, "create": 1, "delete": 1, "email": 1, "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, "print": 1, "read": 1, "report": 1, "role": "System Manager", - "set_user_permissions": 0, "share": 1, - "submit": 0, "write": 1 } ], - "quick_entry": 0, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, "sort_field": "modified", "sort_order": "DESC", - "track_changes": 1, - "track_seen": 0, - "track_views": 0 + "states": [], + "track_changes": 1 } \ No newline at end of file diff --git a/frappe/core/doctype/success_action/success_action.py b/frappe/core/doctype/success_action/success_action.py index 95a81ee0fb..e3db646a2e 100644 --- a/frappe/core/doctype/success_action/success_action.py +++ b/frappe/core/doctype/success_action/success_action.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2018, Frappe Technologies and contributors # License: MIT. See LICENSE diff --git a/frappe/core/doctype/system_settings/system_settings.js b/frappe/core/doctype/system_settings/system_settings.js index 5128ae24cb..f7c43045d2 100644 --- a/frappe/core/doctype/system_settings/system_settings.js +++ b/frappe/core/doctype/system_settings/system_settings.js @@ -1,12 +1,12 @@ frappe.ui.form.on("System Settings", { - refresh: function(frm) { + refresh: function (frm) { frappe.call({ method: "frappe.core.doctype.system_settings.system_settings.load", - callback: function(data) { + callback: function (data) { frappe.all_timezones = data.message.timezones; frm.set_df_property("time_zone", "options", frappe.all_timezones); - $.each(data.message.defaults, function(key, val) { + $.each(data.message.defaults, function (key, val) { frm.set_value(key, val); frappe.sys_defaults[key] = val; }); @@ -14,30 +14,30 @@ frappe.ui.form.on("System Settings", { frappe.app.setup_moment(); delete frm.re_setup_moment; } - } + }, }); }, - enable_password_policy: function(frm) { + enable_password_policy: function (frm) { if (frm.doc.enable_password_policy == 0) { frm.set_value("minimum_password_score", ""); } else { frm.set_value("minimum_password_score", "2"); } }, - enable_two_factor_auth: function(frm) { + enable_two_factor_auth: function (frm) { if (frm.doc.enable_two_factor_auth == 0) { frm.set_value("bypass_2fa_for_retricted_ip_users", 0); frm.set_value("bypass_restrict_ip_check_if_2fa_enabled", 0); } }, - enable_prepared_report_auto_deletion: function(frm) { + enable_prepared_report_auto_deletion: function (frm) { if (frm.doc.enable_prepared_report_auto_deletion) { if (!frm.doc.prepared_report_expiry_period) { - frm.set_value('prepared_report_expiry_period', 7); + frm.set_value("prepared_report_expiry_period", 7); } } }, - on_update: function(frm) { + on_update: function (frm) { if (frappe.boot.time_zone && frappe.boot.time_zone.system !== frm.doc.time_zone) { // Clear cache after saving to refresh the values of boot. frappe.ui.toolbar.clear_cache(); diff --git a/frappe/core/doctype/system_settings/system_settings.json b/frappe/core/doctype/system_settings/system_settings.json index 61410fb1a8..a444062b5a 100644 --- a/frappe/core/doctype/system_settings/system_settings.json +++ b/frappe/core/doctype/system_settings/system_settings.json @@ -1,6 +1,6 @@ { "actions": [], - "creation": "2014-04-17 16:53:52.640856", + "creation": "2022-01-06 03:18:16.326761", "doctype": "DocType", "document_type": "System", "engine": "InnoDB", @@ -11,7 +11,6 @@ "language", "column_break_3", "time_zone", - "is_first_startup", "enable_onboarding", "setup_complete", "date_and_number_format", @@ -35,15 +34,18 @@ "security", "session_expiry", "session_expiry_mobile", + "document_share_key_expiry", "column_break_13", "deny_multiple_sessions", "allow_login_using_mobile_number", "allow_login_using_user_name", "allow_error_traceback", "strip_exif_metadata_from_uploaded_images", + "allow_older_web_view_links", "password_settings", "logout_on_password_reset", "force_user_to_reset_password", + "reset_password_link_expiry_duration", "password_reset_limit", "column_break_31", "enable_password_policy", @@ -61,6 +63,7 @@ "otp_issuer_name", "email", "email_footer_address", + "email_retry_limit", "column_break_18", "disable_standard_email_footer", "hide_footer_in_auto_email_reports", @@ -68,8 +71,11 @@ "prepared_report_section", "enable_prepared_report_auto_deletion", "prepared_report_expiry_period", + "column_break_64", + "max_auto_email_report_per_user", "system_updates_section", - "disable_system_update_notification" + "disable_system_update_notification", + "disable_change_log_notification" ], "fields": [ { @@ -101,14 +107,6 @@ "read_only": 1, "reqd": 1 }, - { - "default": "0", - "fieldname": "is_first_startup", - "fieldtype": "Check", - "hidden": 1, - "label": "Is First Startup", - "read_only": 1 - }, { "default": "0", "fieldname": "setup_complete", @@ -445,7 +443,7 @@ "collapsible": 1, "fieldname": "prepared_report_section", "fieldtype": "Section Break", - "label": "Prepared Report" + "label": "Reports" }, { "default": "Frappe", @@ -485,12 +483,54 @@ "fieldtype": "Select", "label": "First Day of the Week", "options": "Sunday\nMonday\nTuesday\nWednesday\nThursday\nFriday\nSaturday" + }, + { + "default": "30", + "description": "Number of days after which the document Web View link shared on email will be expired", + "fieldname": "document_share_key_expiry", + "fieldtype": "Int", + "label": "Document Share Key Expiry (in Days)" + }, + { + "default": "0", + "fieldname": "allow_older_web_view_links", + "fieldtype": "Check", + "label": "Allow Older Web View Links (Insecure)" + }, + { + "fieldname": "column_break_64", + "fieldtype": "Column Break" + }, + { + "default": "20", + "fieldname": "max_auto_email_report_per_user", + "fieldtype": "Int", + "label": "Max auto email report per user" + }, + { + "default": "0", + "fieldname": "disable_change_log_notification", + "fieldtype": "Check", + "label": "Disable Change Log Notification" + }, + { + "default": "1200", + "fieldname": "reset_password_link_expiry_duration", + "fieldtype": "Duration", + "label": "Reset Password Link Expiry Duration", + "non_negative": 1 + }, + { + "default": "3", + "fieldname": "email_retry_limit", + "fieldtype": "Int", + "label": "Email Retry Limit" } ], "icon": "fa fa-cog", "issingle": 1, "links": [], - "modified": "2022-01-04 11:28:34.881192", + "modified": "2022-06-21 13:55:04.796152", "modified_by": "Administrator", "module": "Core", "name": "System Settings", diff --git a/frappe/core/doctype/system_settings/system_settings.py b/frappe/core/doctype/system_settings/system_settings.py index 3d01015087..4bd41be974 100644 --- a/frappe/core/doctype/system_settings/system_settings.py +++ b/frappe/core/doctype/system_settings/system_settings.py @@ -28,7 +28,7 @@ class SystemSettings(Document): if self.enable_two_factor_auth: if self.two_factor_method == "SMS": - if not frappe.db.get_value("SMS Settings", None, "sms_gateway_url"): + if not frappe.db.get_single_value("SMS Settings", "sms_gateway_url"): frappe.throw( _("Please setup SMS before setting it as an authentication method, via SMS Settings") ) @@ -44,6 +44,15 @@ class SystemSettings(Document): frappe.flags.update_last_reset_password_date = True def on_update(self): + self.set_defaults() + + frappe.cache().delete_value("system_settings") + frappe.cache().delete_value("time_zone") + + if frappe.flags.update_last_reset_password_date: + update_last_reset_password_date() + + def set_defaults(self): for df in self.meta.get("fields"): if df.fieldtype not in no_value_fields and self.has_value_changed(df.fieldname): frappe.db.set_default(df.fieldname, self.get(df.fieldname)) @@ -51,13 +60,6 @@ class SystemSettings(Document): if self.language: set_default_language(self.language) - frappe.cache().delete_value("system_settings") - frappe.cache().delete_value("time_zone") - frappe.local.system_settings = {} - - if frappe.flags.update_last_reset_password_date: - update_last_reset_password_date() - def update_last_reset_password_date(): frappe.db.sql( diff --git a/frappe/core/doctype/system_settings/test_system_settings.py b/frappe/core/doctype/system_settings/test_system_settings.py index 955f4193f0..b126976eeb 100644 --- a/frappe/core/doctype/system_settings/test_system_settings.py +++ b/frappe/core/doctype/system_settings/test_system_settings.py @@ -1,10 +1,7 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2017, Frappe Technologies and Contributors # License: MIT. See LICENSE import unittest -import frappe - class TestSystemSettings(unittest.TestCase): pass diff --git a/frappe/core/doctype/test/test.json b/frappe/core/doctype/test/test.json deleted file mode 100644 index 31a57c9964..0000000000 --- a/frappe/core/doctype/test/test.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "actions": [], - "creation": "2021-03-31 10:06:57.919697", - "doctype": "DocType", - "editable_grid": 1, - "engine": "InnoDB", - "field_order": [ - "test" - ], - "fields": [ - { - "fieldname": "test", - "fieldtype": "Data", - "label": "Test" - } - ], - "index_web_pages_for_search": 1, - "is_virtual": 1, - "links": [], - "modified": "2021-03-31 10:06:57.919697", - "modified_by": "Administrator", - "module": "Core", - "name": "test", - "owner": "Administrator", - "permissions": [ - { - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "share": 1, - "write": 1 - } - ], - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1 -} \ No newline at end of file diff --git a/frappe/core/doctype/test/test.py b/frappe/core/doctype/test/test.py deleted file mode 100644 index 8f3cf7111d..0000000000 --- a/frappe/core/doctype/test/test.py +++ /dev/null @@ -1,44 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2021, Frappe Technologies and contributors -# License: MIT. See LICENSE - -import json - -# import frappe -from frappe.model.document import Document - - -class test(Document): - def db_insert(self): - d = self.get_valid_dict(convert_dates_to_str=True) - with open("data_file.json", "w+") as read_file: - json.dump(d, read_file) - - def load_from_db(self): - with open("data_file.json", "r") as read_file: - d = json.load(read_file) - super(Document, self).__init__(d) - - def db_update(self): - d = self.get_valid_dict(convert_dates_to_str=True) - with open("data_file.json", "w+") as read_file: - json.dump(d, read_file) - - def get_list(self, args): - with open("data_file.json", "r") as read_file: - return [json.load(read_file)] - - def get_value(self, fields, filters, **kwargs): - # return [] - with open("data_file.json", "r") as read_file: - return [json.load(read_file)] - - def get_count(self, args): - # return [] - with open("data_file.json", "r") as read_file: - return [json.load(read_file)] - - def get_stats(self, args): - # return [] - with open("data_file.json", "r") as read_file: - return [json.load(read_file)] diff --git a/frappe/core/doctype/transaction_log/test_transaction_log.py b/frappe/core/doctype/transaction_log/test_transaction_log.py index a4bb066eea..8b179f8d85 100644 --- a/frappe/core/doctype/transaction_log/test_transaction_log.py +++ b/frappe/core/doctype/transaction_log/test_transaction_log.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2018, Frappe Technologies and Contributors # License: MIT. See LICENSE import hashlib diff --git a/frappe/core/doctype/transaction_log/transaction_log.js b/frappe/core/doctype/transaction_log/transaction_log.js index 569cd9bf61..8f22b859f7 100644 --- a/frappe/core/doctype/transaction_log/transaction_log.js +++ b/frappe/core/doctype/transaction_log/transaction_log.js @@ -1,6 +1,4 @@ // Copyright (c) 2018, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Transaction Log', { - -}); +frappe.ui.form.on("Transaction Log", {}); diff --git a/frappe/core/doctype/transaction_log/transaction_log.json b/frappe/core/doctype/transaction_log/transaction_log.json index 5c6aa5bc8b..2135976add 100644 --- a/frappe/core/doctype/transaction_log/transaction_log.json +++ b/frappe/core/doctype/transaction_log/transaction_log.json @@ -1,476 +1,124 @@ { - "allow_copy": 0, - "allow_events_in_timeline": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, + "actions": [], "creation": "2018-02-06 11:48:51.270524", - "custom": 0, - "docstatus": 0, "doctype": "DocType", - "document_type": "", "editable_grid": 1, "engine": "InnoDB", + "field_order": [ + "row_index", + "section_break_2", + "reference_doctype", + "document_name", + "column_break_5", + "timestamp", + "checksum_version", + "section_break_8", + "previous_hash", + "transaction_hash", + "chaining_hash", + "data", + "amended_from" + ], "fields": [ { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "row_index", "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, "label": "Row Index", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "read_only": 1 }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "section_break_2", - "fieldtype": "Section Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "fieldtype": "Section Break" }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "reference_doctype", "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, "label": "Reference Document Type", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "read_only": 1 }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "document_name", "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, "label": "Document Name", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "read_only": 1 }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "column_break_5", - "fieldtype": "Column Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "fieldtype": "Column Break" }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "timestamp", "fieldtype": "Datetime", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, "label": "Timestamp", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "read_only": 1 }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "checksum_version", "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, "label": "Checksum Version", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "read_only": 1 }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "section_break_8", - "fieldtype": "Section Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "fieldtype": "Section Break" }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "previous_hash", "fieldtype": "Small Text", "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, "label": "Previous Hash", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "read_only": 1 }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "transaction_hash", "fieldtype": "Small Text", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, "label": "Transaction Hash", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "read_only": 1 }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "chaining_hash", "fieldtype": "Small Text", "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, "label": "Chaining Hash", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "read_only": 1 }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "data", "fieldtype": "Long Text", "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, "label": "Data", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "read_only": 1 }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "amended_from", "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, "label": "Amended From", - "length": 0, "no_copy": 1, "options": "Transaction Log", - "permlevel": 0, "print_hide": 1, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "read_only": 1 } ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, "in_create": 1, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2019-09-05 14:22:27.664645", + "links": [], + "modified": "2022-08-03 12:20:54.684305", "modified_by": "Administrator", "module": "Core", "name": "Transaction Log", - "name_case": "", "owner": "Administrator", "permissions": [ { - "amend": 0, - "cancel": 0, - "create": 0, - "delete": 0, "email": 1, "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, "print": 1, "read": 1, "report": 1, "role": "Administrator", - "set_user_permissions": 0, - "share": 1, - "submit": 0, - "write": 0 + "share": 1 } ], - "quick_entry": 0, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, "sort_field": "modified", "sort_order": "DESC", - "track_changes": 1, - "track_seen": 0, - "track_views": 0 + "states": [], + "track_changes": 1 } \ No newline at end of file diff --git a/frappe/core/doctype/transaction_log/transaction_log.py b/frappe/core/doctype/transaction_log/transaction_log.py index d0c0342f6f..efe9bbddc2 100644 --- a/frappe/core/doctype/transaction_log/transaction_log.py +++ b/frappe/core/doctype/transaction_log/transaction_log.py @@ -4,7 +4,6 @@ import hashlib import frappe -from frappe import _ from frappe.model.document import Document from frappe.query_builder import DocType from frappe.utils import cint, now_datetime diff --git a/frappe/core/doctype/translation/test_translation.py b/frappe/core/doctype/translation/test_translation.py index df5ae3767a..c9f4e85086 100644 --- a/frappe/core/doctype/translation/test_translation.py +++ b/frappe/core/doctype/translation/test_translation.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and Contributors # License: MIT. See LICENSE import unittest diff --git a/frappe/core/doctype/translation/translation.js b/frappe/core/doctype/translation/translation.js index 454f2564ba..a1b5b182b8 100644 --- a/frappe/core/doctype/translation/translation.js +++ b/frappe/core/doctype/translation/translation.js @@ -1,9 +1,8 @@ // Copyright (c) 2016, Frappe Technologies and contributors // For license information, please see license.txt - -frappe.ui.form.on('Translation', { - refresh: function() { +frappe.ui.form.on("Translation", { + refresh: function () { // - } + }, }); diff --git a/frappe/core/doctype/translation/translation.json b/frappe/core/doctype/translation/translation.json index 560f3b2ce2..68b83ed5d9 100644 --- a/frappe/core/doctype/translation/translation.json +++ b/frappe/core/doctype/translation/translation.json @@ -1,6 +1,4 @@ { - "_comments": "[]", - "_liked_by": "[]", "actions": [], "allow_import": 1, "autoname": "hash", @@ -26,6 +24,7 @@ "fieldtype": "Link", "label": "Language", "options": "Language", + "reqd": 1, "search_index": 1 }, { @@ -72,20 +71,23 @@ "description": "If your data is in HTML, please copy paste the exact HTML code with the tags.", "fieldname": "source_text", "fieldtype": "Code", - "label": "Source Text" + "label": "Source Text", + "reqd": 1 }, { "fieldname": "translated_text", "fieldtype": "Code", "in_list_view": 1, - "label": "Translated Text" + "label": "Translated Text", + "reqd": 1 } ], "links": [], - "modified": "2021-12-31 10:19:52.541055", + "modified": "2022-07-04 06:53:54.997004", "modified_by": "Administrator", "module": "Core", "name": "Translation", + "naming_rule": "Random", "owner": "Administrator", "permissions": [ { @@ -103,6 +105,7 @@ ], "sort_field": "modified", "sort_order": "DESC", + "states": [], "title_field": "source_text", "track_changes": 1 } \ No newline at end of file diff --git a/frappe/core/doctype/translation/translation.py b/frappe/core/doctype/translation/translation.py index 90ea4d1523..b08198eb13 100644 --- a/frappe/core/doctype/translation/translation.py +++ b/frappe/core/doctype/translation/translation.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and contributors # License: MIT. See LICENSE diff --git a/frappe/core/doctype/user/test_records.json b/frappe/core/doctype/user/test_records.json index 21fe3ff69d..9d1bf0efd4 100644 --- a/frappe/core/doctype/user/test_records.json +++ b/frappe/core/doctype/user/test_records.json @@ -45,6 +45,13 @@ "new_password": "Eastern_43A1W", "enabled": 1 }, + { + "doctype": "User", + "email": "test'5@example.com", + "first_name": "_Test'5", + "new_password": "Eastern_43A1W", + "enabled": 1 + }, { "doctype": "User", "email": "testperm@example.com", diff --git a/frappe/core/doctype/user/test_user.py b/frappe/core/doctype/user/test_user.py index 151dd40308..7582954175 100644 --- a/frappe/core/doctype/user/test_user.py +++ b/frappe/core/doctype/user/test_user.py @@ -1,19 +1,20 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE import json +import time import unittest from unittest.mock import patch import frappe import frappe.exceptions from frappe.core.doctype.user.user import ( - extract_mentions, reset_password, sign_up, test_password_strength, update_password, verify_password, ) +from frappe.desk.notifications import extract_mentions from frappe.frappeclient import FrappeClient from frappe.model.delete_doc import delete_doc from frappe.utils import get_url @@ -256,7 +257,8 @@ class TestUser(unittest.TestCase): @Team and - + @Unknown Team please check @@ -365,7 +367,7 @@ class TestUser(unittest.TestCase): self.assertEqual(update_password(new_password, key=test_user.reset_password_key), "/app") self.assertEqual( update_password(new_password, key="wrong_key"), - "The Link specified has either been used before or Invalid", + "The reset password link has either been used before or is invalid", ) # password verification should fail with old password @@ -374,7 +376,6 @@ class TestUser(unittest.TestCase): # reset password update_password(old_password, old_password=new_password) - self.assertRaisesRegex( frappe.exceptions.ValidationError, "Invalid key type", update_password, "test", 1, ["like", "%"] ) @@ -434,6 +435,21 @@ class TestUser(unittest.TestCase): [m.get("module_name") for m in get_modules_from_all_apps()], ) + def test_reset_password_link_expiry(self): + new_password = "new_password" + # set the reset password expiry to 1 second + frappe.db.set_value( + "System Settings", "System Settings", "reset_password_link_expiry_duration", 1 + ) + frappe.set_user("testpassword@example.com") + test_user = frappe.get_doc("User", "testpassword@example.com") + test_user.reset_password() + time.sleep(1) # sleep for 1 sec to expire the reset link + self.assertEqual( + update_password(new_password, key=test_user.reset_password_key), + "The reset password link has been expired", + ) + def delete_contact(user): frappe.db.delete("Contact", {"email_id": user}) diff --git a/frappe/core/doctype/user/user.js b/frappe/core/doctype/user/user.js index 77c199cdd4..24f9eb2cea 100644 --- a/frappe/core/doctype/user/user.js +++ b/frappe/core/doctype/user/user.js @@ -1,74 +1,85 @@ -frappe.ui.form.on('User', { - before_load: function(frm) { - var update_tz_select = function(user_language) { +frappe.ui.form.on("User", { + before_load: function (frm) { + var update_tz_select = function (user_language) { frm.set_df_property("time_zone", "options", [""].concat(frappe.all_timezones)); }; - if(!frappe.all_timezones) { + if (!frappe.all_timezones) { frappe.call({ method: "frappe.core.doctype.user.user.get_timezones", - callback: function(r) { + callback: function (r) { frappe.all_timezones = r.message.timezones; update_tz_select(); - } + }, }); } else { update_tz_select(); } - }, - role_profile_name: function(frm) { - if(frm.doc.role_profile_name) { + role_profile_name: function (frm) { + if (frm.doc.role_profile_name) { frappe.call({ - "method": "frappe.core.doctype.user.user.get_role_profile", + method: "frappe.core.doctype.user.user.get_role_profile", args: { - role_profile: frm.doc.role_profile_name + role_profile: frm.doc.role_profile_name, }, - callback: function(data) { + callback: function (data) { frm.set_value("roles", []); - $.each(data.message || [], function(i, v) { + $.each(data.message || [], function (i, v) { var d = frm.add_child("roles"); d.role = v.role; }); frm.roles_editor.show(); - } + }, }); } }, - module_profile: function(frm) { + module_profile: function (frm) { if (frm.doc.module_profile) { frappe.call({ - "method": "frappe.core.doctype.user.user.get_module_profile", + method: "frappe.core.doctype.user.user.get_module_profile", args: { - module_profile: frm.doc.module_profile + module_profile: frm.doc.module_profile, }, - callback: function(data) { + callback: function (data) { frm.set_value("block_modules", []); - $.each(data.message || [], function(i, v) { + $.each(data.message || [], function (i, v) { let d = frm.add_child("block_modules"); d.module = v.module; }); frm.module_editor && frm.module_editor.show(); - } + }, }); } }, - onload: function(frm) { + onload: function (frm) { frm.can_edit_roles = has_access_to_edit_user(); - if (frm.can_edit_roles && !frm.is_new() && in_list(['System User', 'Website User'], frm.doc.user_type)) { + if (frm.is_new() && frm.roles_editor) { + frm.roles_editor.reset(); + } + + if ( + frm.can_edit_roles && + !frm.is_new() && + in_list(["System User", "Website User"], frm.doc.user_type) + ) { if (!frm.roles_editor) { - const role_area = $('
    ') - .appendTo(frm.fields_dict.roles_html.wrapper); + const role_area = $('
    ').appendTo( + frm.fields_dict.roles_html.wrapper + ); - frm.roles_editor = new frappe.RoleEditor(role_area, frm, frm.doc.role_profile_name ? 1 : 0); + frm.roles_editor = new frappe.RoleEditor( + role_area, + frm, + frm.doc.role_profile_name ? 1 : 0 + ); - if (frm.doc.user_type == 'System User') { - var module_area = $('
    ') - .appendTo(frm.fields_dict.modules_html.wrapper); + if (frm.doc.user_type == "System User") { + var module_area = $("
    ").appendTo(frm.fields_dict.modules_html.wrapper); frm.module_editor = new frappe.ModuleEditor(frm, module_area); } } else { @@ -76,109 +87,140 @@ frappe.ui.form.on('User', { } } }, - refresh: function(frm) { + refresh: function (frm) { let doc = frm.doc; if (frm.is_new()) { frm.set_value("time_zone", frappe.sys_defaults.time_zone); } - if (in_list(['System User', 'Website User'], frm.doc.user_type) - && !frm.is_new() && !frm.roles_editor && frm.can_edit_roles) { + if ( + in_list(["System User", "Website User"], frm.doc.user_type) && + !frm.is_new() && + !frm.roles_editor && + frm.can_edit_roles + ) { frm.reload_doc(); return; } - if(doc.name===frappe.session.user && !doc.__unsaved - && frappe.all_timezones - && (doc.language || frappe.boot.user.language) - && doc.language !== frappe.boot.user.language) { + if ( + doc.name === frappe.session.user && + !doc.__unsaved && + frappe.all_timezones && + (doc.language || frappe.boot.user.language) && + doc.language !== frappe.boot.user.language + ) { frappe.msgprint(__("Refreshing...")); window.location.reload(); } - frm.toggle_display(['sb1', 'sb3', 'modules_access'], false); + frm.toggle_display(["sb1", "sb3", "modules_access"], false); - if(!frm.is_new()) { - if(has_access_to_edit_user()) { + if (!frm.is_new()) { + if (has_access_to_edit_user()) { + frm.add_custom_button( + __("Set User Permissions"), + function () { + frappe.route_options = { + user: doc.name, + }; + frappe.set_route("List", "User Permission"); + }, + __("Permissions") + ); - frm.add_custom_button(__("Set User Permissions"), function() { - frappe.route_options = { - "user": doc.name - }; - frappe.set_route('List', 'User Permission'); - }, __("Permissions")); + frm.add_custom_button( + __("View Permitted Documents"), + () => + frappe.set_route("query-report", "Permitted Documents For User", { + user: frm.doc.name, + }), + __("Permissions") + ); - frm.add_custom_button(__('View Permitted Documents'), - () => frappe.set_route('query-report', 'Permitted Documents For User', - {user: frm.doc.name}), __("Permissions")); - - frm.toggle_display(['sb1', 'sb3', 'modules_access'], true); + frm.toggle_display(["sb1", "sb3", "modules_access"], true); } - frm.add_custom_button(__("Reset Password"), function() { - frappe.call({ - method: "frappe.core.doctype.user.user.reset_password", - args: { - "user": frm.doc.name - } - }); - }, __("Password")); + frm.add_custom_button( + __("Reset Password"), + function () { + frappe.call({ + method: "frappe.core.doctype.user.user.reset_password", + args: { + user: frm.doc.name, + }, + }); + }, + __("Password") + ); if (frappe.user.has_role("System Manager")) { frappe.db.get_single_value("LDAP Settings", "enabled").then((value) => { if (value === 1 && frm.doc.name != "Administrator") { - frm.add_custom_button(__("Reset LDAP Password"), function() { - const d = new frappe.ui.Dialog({ - title: __("Reset LDAP Password"), - fields: [ - { - label: __("New Password"), - fieldtype: "Password", - fieldname: "new_password", - reqd: 1 + frm.add_custom_button( + __("Reset LDAP Password"), + function () { + const d = new frappe.ui.Dialog({ + title: __("Reset LDAP Password"), + fields: [ + { + label: __("New Password"), + fieldtype: "Password", + fieldname: "new_password", + reqd: 1, + }, + { + label: __("Confirm New Password"), + fieldtype: "Password", + fieldname: "confirm_password", + reqd: 1, + }, + { + label: __("Logout All Sessions"), + fieldtype: "Check", + fieldname: "logout_sessions", + }, + ], + primary_action: (values) => { + d.hide(); + if (values.new_password !== values.confirm_password) { + frappe.throw(__("Passwords do not match!")); + } + frappe.call( + "frappe.integrations.doctype.ldap_settings.ldap_settings.reset_password", + { + user: frm.doc.email, + password: values.new_password, + logout: values.logout_sessions, + } + ); }, - { - label: __("Confirm New Password"), - fieldtype: "Password", - fieldname: "confirm_password", - reqd: 1 - }, - { - label: __("Logout All Sessions"), - fieldtype: "Check", - fieldname: "logout_sessions" - } - ], - primary_action: (values) => { - d.hide(); - if (values.new_password !== values.confirm_password) { - frappe.throw(__("Passwords do not match!")); - } - frappe.call( - "frappe.integrations.doctype.ldap_settings.ldap_settings.reset_password", { - user: frm.doc.email, - password: values.new_password, - logout: values.logout_sessions - }); - } - }); - d.show(); - }, __("Password")); + }); + d.show(); + }, + __("Password") + ); } }); } - frm.add_custom_button(__("Reset OTP Secret"), function() { - frappe.call({ - method: "frappe.twofactor.reset_otp_secret", - args: { - "user": frm.doc.name - } - }); - }, __("Password")); + if (frappe.session.user == doc.name || frappe.user.has_role("System Manager")) { + frm.add_custom_button( + __("Reset OTP Secret"), + function () { + frappe.call({ + method: "frappe.twofactor.reset_otp_secret", + args: { + user: frm.doc.name, + }, + }); + }, + __("Password") + ); + } - frm.trigger('enabled'); + frm.trigger("enabled"); if (frm.roles_editor && frm.can_edit_roles) { frm.roles_editor.disable = frm.doc.role_profile_name ? 1 : 0; @@ -187,66 +229,67 @@ frappe.ui.form.on('User', { frm.module_editor && frm.module_editor.show(); - if(frappe.session.user==doc.name) { + if (frappe.session.user == doc.name) { // update display settings - if(doc.user_image) { - frappe.boot.user_info[frappe.session.user].image = frappe.utils.get_file_link(doc.user_image); + if (doc.user_image) { + frappe.boot.user_info[frappe.session.user].image = frappe.utils.get_file_link( + doc.user_image + ); } } } - if (frm.doc.user_emails){ - var found =0; - for (var i = 0;i { + child_row.used_oauth = value.auth_method === "OAuth"; + frm.refresh_field("user_emails", cdn, "used_oauth"); + } + ); + }, }); function has_access_to_edit_user() { @@ -286,7 +344,10 @@ function has_access_to_edit_user() { } function get_roles_for_editing_user() { - return frappe.get_meta('User').permissions - .filter(perm => perm.permlevel >= 1 && perm.write) - .map(perm => perm.role) || ['System Manager']; + return ( + frappe + .get_meta("User") + .permissions.filter((perm) => perm.permlevel >= 1 && perm.write) + .map((perm) => perm.role) || ["System Manager"] + ); } diff --git a/frappe/core/doctype/user/user.json b/frappe/core/doctype/user/user.json index 642a392a58..82e3fa71f3 100644 --- a/frappe/core/doctype/user/user.json +++ b/frappe/core/doctype/user/user.json @@ -43,6 +43,7 @@ "new_password", "logout_all_sessions", "reset_password_key", + "last_reset_password_key_generated_on", "last_password_reset_date", "redirect_url", "document_follow_notifications_section", @@ -613,6 +614,14 @@ "label": "Module Profile", "options": "Module Profile" }, + { + "description": "Stores the datetime when the last reset password key was generated.", + "fieldname": "last_reset_password_key_generated_on", + "fieldtype": "Datetime", + "hidden": 1, + "label": "Last Reset Password Key Generated On", + "read_only": 1 + }, { "fieldname": "column_break_75", "fieldtype": "Column Break" @@ -713,7 +722,7 @@ "link_fieldname": "user" } ], - "modified": "2022-03-09 01:47:56.745069", + "modified": "2022-05-25 01:00:51.345319", "modified_by": "Administrator", "module": "Core", "name": "User", @@ -738,6 +747,10 @@ "read": 1, "role": "System Manager", "write": 1 + }, + { + "role": "All", + "select": 1 } ], "quick_entry": 1, diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py index c90cbf1fce..232e915435 100644 --- a/frappe/core/doctype/user/user.py +++ b/frappe/core/doctype/user/user.py @@ -1,6 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -from bs4 import BeautifulSoup +from datetime import timedelta import frappe import frappe.defaults @@ -132,11 +132,11 @@ class User(Document): if self.time_zone: frappe.defaults.set_default("time_zone", self.time_zone, self.name) - if self.has_value_changed("allow_in_mentions") or self.has_value_changed("user_type"): - frappe.cache().delete_key("users_for_mentions") - if self.has_value_changed("enabled"): + frappe.cache().delete_key("users_for_mentions") frappe.cache().delete_key("enabled_users") + elif self.has_value_changed("allow_in_mentions") or self.has_value_changed("user_type"): + frappe.cache().delete_key("users_for_mentions") def has_website_permission(self, ptype, user, verbose=False): """Returns true if current user is the session user""" @@ -161,6 +161,9 @@ class User(Document): toggle_notifications(self.name, enable=cint(self.enabled)) def add_system_manager_role(self): + if self.is_system_manager_disabled(): + return + # if adding system manager, do nothing if not cint(self.enabled) or ( "System Manager" in [user_role.role for user_role in self.get("roles")] @@ -187,6 +190,9 @@ class User(Document): ], ) + def is_system_manager_disabled(self): + return frappe.db.get_value("Role", {"name": "System Manager"}, ["disabled"]) + def email_new_password(self, new_password=None): if new_password and not self.flags.in_insert: _update_password(user=self.name, pwd=new_password, logout_all_sessions=self.logout_all_sessions) @@ -265,7 +271,7 @@ class User(Document): except frappe.OutgoingEmailError: # email server not set, don't send email - frappe.log_error(frappe.get_traceback()) + self.log_error("Unable to send new password notification") @Document.hook def validate_reset_password(self): @@ -276,6 +282,7 @@ class User(Document): key = random_string(32) self.db_set("reset_password_key", key) + self.db_set("last_reset_password_key_generated_on", now_datetime()) url = "/update-password?key=" + key if password_expired: @@ -369,6 +376,9 @@ class User(Document): ) def a_system_manager_should_exist(self): + if self.is_system_manager_disabled(): + return + if not self.get_other_system_managers(): throw(_("There should remain at least one System Manager")) @@ -421,6 +431,9 @@ class User(Document): frappe.cache().delete_key("enabled_users") + # delete user permissions + frappe.db.delete("User Permission", {"user": self.name}) + def before_rename(self, old_name, new_name, merge=False): frappe.clear_cache(user=old_name) self.validate_rename(old_name, new_name) @@ -474,7 +487,7 @@ class User(Document): self.save() def remove_roles(self, *roles): - existing_roles = dict((d.role, d) for d in self.get("roles")) + existing_roles = {d.role: d for d in self.get("roles")} for role in roles: if role in existing_roles: self.get("roles").remove(existing_roles[role]) @@ -483,7 +496,7 @@ class User(Document): def remove_all_roles_for_guest(self): if self.name == "Guest": - self.set("roles", list(set(d for d in self.get("roles") if d.role == "Guest"))) + self.set("roles", list({d for d in self.get("roles") if d.role == "Guest"})) def remove_disabled_roles(self): disabled_roles = [d.name for d in frappe.get_all("Role", filters={"disabled": 1})] @@ -542,7 +555,7 @@ class User(Document): if not username: # @firstname_last_name username = _check_suggestion( - frappe.scrub("{0} {1}".format(self.first_name, self.last_name or "")) + frappe.scrub("{} {}".format(self.first_name, self.last_name or "")) ) if username: @@ -571,7 +584,7 @@ class User(Document): for p in self.social_logins: if p.provider == provider: return p.userid - except: + except Exception: return None def set_social_login_userid(self, provider, userid, username=None): @@ -583,10 +596,7 @@ class User(Document): self.append("social_logins", social_logins) def get_restricted_ip_list(self): - if not self.restrict_ip: - return - - return [i.strip() for i in self.restrict_ip.split(",")] + return get_restricted_ip_list(self) @classmethod def find_by_credentials(cls, user_name: str, password: str, validate_password: bool = True): @@ -599,10 +609,10 @@ class User(Document): """ login_with_mobile = cint( - frappe.db.get_value("System Settings", "System Settings", "allow_login_using_mobile_number") + frappe.db.get_single_value("System Settings", "allow_login_using_mobile_number") ) login_with_username = cint( - frappe.db.get_value("System Settings", "System Settings", "allow_login_using_user_name") + frappe.db.get_single_value("System Settings", "allow_login_using_user_name") ) or_filters = [{"name": user_name}] @@ -751,19 +761,11 @@ def has_email_account(email): @frappe.whitelist(allow_guest=False) def get_email_awaiting(user): - waiting = frappe.get_all( + return frappe.get_all( "User Email", fields=["email_account", "email_id"], - filters={"awaiting_password": 1, "parent": user}, + filters={"awaiting_password": 1, "parent": user, "used_oauth": 0}, ) - if waiting: - return waiting - else: - user_email_table = DocType("User Email") - frappe.qb.update(user_email_table).set(user_email_table.user_email_table, 0).where( - user_email_table.parent == user - ).run() - return False def ask_pass_update(): @@ -771,7 +773,7 @@ def ask_pass_update(): from frappe.utils import set_default password_list = frappe.get_all( - "User Email", filters={"awaiting_password": True}, pluck="parent", distinct=True + "User Email", filters={"awaiting_password": 1, "used_oauth": 0}, pluck="parent", distinct=True ) set_default("email_user_password", ",".join(password_list)) @@ -780,16 +782,27 @@ def _get_user_for_update_password(key, old_password): # verify old password result = frappe._dict() if key: - result.user = frappe.db.get_value("User", {"reset_password_key": key}) - if not result.user: - result.message = _("The Link specified has either been used before or Invalid") - + user = frappe.db.get_value( + "User", {"reset_password_key": key}, ["name", "last_reset_password_key_generated_on"] + ) + result.user, last_reset_password_key_generated_on = user or (None, None) + if result.user: + reset_password_link_expiry = cint( + frappe.db.get_single_value("System Settings", "reset_password_link_expiry_duration") + ) + if ( + reset_password_link_expiry + and now_datetime() + > last_reset_password_key_generated_on + timedelta(seconds=reset_password_link_expiry) + ): + result.message = _("The reset password link has been expired") + else: + result.message = _("The reset password link has either been used before or is invalid") elif old_password: # verify old password frappe.local.login_manager.check_password(frappe.session.user, old_password) user = frappe.session.user result.user = user - return result @@ -846,7 +859,7 @@ def sign_up(email, full_name, redirect_to): user.insert() # set default signup role as per Portal Settings - default_role = frappe.db.get_value("Portal Settings", None, "default_role") + default_role = frappe.db.get_single_value("Portal Settings", "default_role") if default_role: user.add_roles(default_role) @@ -895,7 +908,7 @@ def user_query(doctype, txt, searchfield, start, page_len, filters): user_type_condition = "" filters.pop("ignore_user_type") - txt = "%{}%".format(txt) + txt = f"%{txt}%" return frappe.db.sql( """SELECT `name`, CONCAT_WS(' ', first_name, middle_name, last_name) FROM `tabUser` @@ -947,7 +960,7 @@ def get_system_users(exclude_users=None, limit=None): limit_cond = "" if limit: - limit_cond = "limit {0}".format(limit) + limit_cond = f"limit {limit}" exclude_users += list(STANDARD_USERS) @@ -1013,7 +1026,7 @@ def notify_admin_access_to_system_manager(login_manager=None): ): site = '{0}'.format(frappe.local.request.host_url) - date_and_time = "{0}".format(format_datetime(now_datetime(), format_string="medium")) + date_and_time = "{}".format(format_datetime(now_datetime(), format_string="medium")) ip_address = frappe.local.request_ip access_message = _("Administrator accessed {0} on {1} via IP Address {2}.").format( @@ -1029,24 +1042,6 @@ def notify_admin_access_to_system_manager(login_manager=None): ) -def extract_mentions(txt): - """Find all instances of @mentions in the html.""" - soup = BeautifulSoup(txt, "html.parser") - emails = [] - for mention in soup.find_all(class_="mention"): - if mention.get("data-is-group") == "true": - try: - user_group = frappe.get_cached_doc("User Group", mention["data-id"]) - emails += [d.user for d in user_group.user_group_members] - except frappe.DoesNotExistError: - pass - continue - email = mention["data-id"] - emails.append(email) - - return emails - - def handle_password_test_fail(result): suggestions = result["feedback"]["suggestions"][0] if result["feedback"]["suggestions"] else "" warning = result["feedback"]["warning"] if "warning" in result["feedback"] else "" @@ -1142,6 +1137,13 @@ def create_contact(user, ignore_links=False, ignore_mandatory=False): contact.save(ignore_permissions=True) +def get_restricted_ip_list(user): + if not user.restrict_ip: + return + + return [i.strip() for i in user.restrict_ip.split(",")] + + @frappe.whitelist() def generate_keys(user): """ diff --git a/frappe/core/doctype/user/user_list.js b/frappe/core/doctype/user/user_list.js index 5632edf0cc..334ed0b370 100644 --- a/frappe/core/doctype/user/user_list.js +++ b/frappe/core/doctype/user/user_list.js @@ -1,19 +1,19 @@ // Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors // MIT License. See license.txt -frappe.listview_settings['User'] = { +frappe.listview_settings["User"] = { add_fields: ["enabled", "user_type", "user_image"], - filters: [["enabled","=",1]], - prepare_data: function(data) { + filters: [["enabled", "=", 1]], + prepare_data: function (data) { data["user_for_avatar"] = data["name"]; }, - get_indicator: function(doc) { - if(doc.enabled) { + get_indicator: function (doc) { + if (doc.enabled) { return [__("Active"), "green", "enabled,=,1"]; } else { return [__("Disabled"), "grey", "enabled,=,0"]; } - } + }, }; frappe.help.youtube_id["User"] = "8Slw1hsTmUI"; diff --git a/frappe/core/doctype/user_document_type/user_document_type.py b/frappe/core/doctype/user_document_type/user_document_type.py index 1e7eab4bd4..731acf582b 100644 --- a/frappe/core/doctype/user_document_type/user_document_type.py +++ b/frappe/core/doctype/user_document_type/user_document_type.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2021, Frappe Technologies and contributors # License: MIT. See LICENSE diff --git a/frappe/core/doctype/user_email/user_email.json b/frappe/core/doctype/user_email/user_email.json index b106ed4a19..6e3f813035 100644 --- a/frappe/core/doctype/user_email/user_email.json +++ b/frappe/core/doctype/user_email/user_email.json @@ -9,6 +9,7 @@ "email_id", "column_break_3", "awaiting_password", + "used_oauth", "enable_outgoing" ], "fields": [ @@ -48,16 +49,25 @@ "fieldtype": "Check", "label": "Enable Outgoing", "read_only": 1 + }, + { + "default": "0", + "fieldname": "used_oauth", + "fieldtype": "Check", + "in_list_view": 1, + "label": "Used OAuth", + "read_only": 1 } ], "istable": 1, "links": [], - "modified": "2020-04-06 19:19:12.130246", + "modified": "2022-06-03 14:25:46.944733", "modified_by": "Administrator", "module": "Core", "name": "User Email", "owner": "Administrator", "permissions": [], "sort_field": "modified", - "sort_order": "DESC" + "sort_order": "DESC", + "states": [] } \ No newline at end of file diff --git a/frappe/core/doctype/user_email/user_email.py b/frappe/core/doctype/user_email/user_email.py index 4e125f5308..ebca480f47 100644 --- a/frappe/core/doctype/user_email/user_email.py +++ b/frappe/core/doctype/user_email/user_email.py @@ -1,8 +1,6 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and contributors # License: MIT. See LICENSE -import frappe from frappe.model.document import Document diff --git a/frappe/core/doctype/user_group/test_user_group.py b/frappe/core/doctype/user_group/test_user_group.py index 546d4fd54c..368f4eaef2 100644 --- a/frappe/core/doctype/user_group/test_user_group.py +++ b/frappe/core/doctype/user_group/test_user_group.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2021, Frappe Technologies and Contributors # License: MIT. See LICENSE # import frappe diff --git a/frappe/core/doctype/user_group/user_group.js b/frappe/core/doctype/user_group/user_group.js index 2aa9b68658..cab1f5dff1 100644 --- a/frappe/core/doctype/user_group/user_group.js +++ b/frappe/core/doctype/user_group/user_group.js @@ -1,8 +1,7 @@ // Copyright (c) 2021, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('User Group', { +frappe.ui.form.on("User Group", { // refresh: function(frm) { - // } }); diff --git a/frappe/core/doctype/user_group/user_group.py b/frappe/core/doctype/user_group/user_group.py index a59117426f..812f230f7a 100644 --- a/frappe/core/doctype/user_group/user_group.py +++ b/frappe/core/doctype/user_group/user_group.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2021, Frappe Technologies and contributors # License: MIT. See LICENSE diff --git a/frappe/core/doctype/user_group_member/test_user_group_member.py b/frappe/core/doctype/user_group_member/test_user_group_member.py index f1bdc41cff..5d709d0bec 100644 --- a/frappe/core/doctype/user_group_member/test_user_group_member.py +++ b/frappe/core/doctype/user_group_member/test_user_group_member.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2021, Frappe Technologies and Contributors # License: MIT. See LICENSE # import frappe diff --git a/frappe/core/doctype/user_group_member/user_group_member.js b/frappe/core/doctype/user_group_member/user_group_member.js index 0b2dbe0d46..4c4011c8b4 100644 --- a/frappe/core/doctype/user_group_member/user_group_member.js +++ b/frappe/core/doctype/user_group_member/user_group_member.js @@ -1,8 +1,7 @@ // Copyright (c) 2021, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('User Group Member', { +frappe.ui.form.on("User Group Member", { // refresh: function(frm) { - // } }); diff --git a/frappe/core/doctype/user_group_member/user_group_member.py b/frappe/core/doctype/user_group_member/user_group_member.py index 6b948d797f..e9722a07ad 100644 --- a/frappe/core/doctype/user_group_member/user_group_member.py +++ b/frappe/core/doctype/user_group_member/user_group_member.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2021, Frappe Technologies and contributors # License: MIT. See LICENSE diff --git a/frappe/core/doctype/user_permission/user_permission.js b/frappe/core/doctype/user_permission/user_permission.js index f6989db5d8..39ee4348b9 100644 --- a/frappe/core/doctype/user_permission/user_permission.js +++ b/frappe/core/doctype/user_permission/user_permission.js @@ -1,59 +1,58 @@ // Copyright (c) 2017, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('User Permission', { - setup: frm => { +frappe.ui.form.on("User Permission", { + setup: (frm) => { frm.set_query("allow", () => { return { - "filters": { + filters: { issingle: 0, - istable: 0 - } + istable: 0, + }, }; }); - frm.set_query('applicable_for', () => { + frm.set_query("applicable_for", () => { return { - 'query': 'frappe.core.doctype.user_permission.user_permission.get_applicable_for_doctype_list', - 'doctype': frm.doc.allow + query: "frappe.core.doctype.user_permission.user_permission.get_applicable_for_doctype_list", + doctype: frm.doc.allow, }; }); - }, - refresh: frm => { - frm.add_custom_button(__('View Permitted Documents'), - () => frappe.set_route('query-report', 'Permitted Documents For User', - { user: frm.doc.user })); - frm.trigger('set_applicable_for_constraint'); - frm.trigger('toggle_hide_descendants'); + refresh: (frm) => { + frm.add_custom_button(__("View Permitted Documents"), () => + frappe.set_route("query-report", "Permitted Documents For User", { + user: frm.doc.user, + }) + ); + frm.trigger("set_applicable_for_constraint"); + frm.trigger("toggle_hide_descendants"); }, - allow: frm => { + allow: (frm) => { if (frm.doc.allow) { if (frm.doc.for_value) { - frm.set_value('for_value', null); + frm.set_value("for_value", null); } - frm.trigger('toggle_hide_descendants'); + frm.trigger("toggle_hide_descendants"); } }, - apply_to_all_doctypes: frm => { - frm.trigger('set_applicable_for_constraint'); + apply_to_all_doctypes: (frm) => { + frm.trigger("set_applicable_for_constraint"); }, - set_applicable_for_constraint: frm => { - frm.toggle_reqd('applicable_for', !frm.doc.apply_to_all_doctypes); + set_applicable_for_constraint: (frm) => { + frm.toggle_reqd("applicable_for", !frm.doc.apply_to_all_doctypes); if (frm.doc.apply_to_all_doctypes && frm.doc.applicable_for) { - frm.set_value('applicable_for', null, null, true); + frm.set_value("applicable_for", null, null, true); } }, - toggle_hide_descendants: frm => { + toggle_hide_descendants: (frm) => { let show = frappe.boot.nested_set_doctypes.includes(frm.doc.allow); - frm.toggle_display('hide_descendants', show); - } - - + frm.toggle_display("hide_descendants", show); + }, }); diff --git a/frappe/core/doctype/user_permission/user_permission.py b/frappe/core/doctype/user_permission/user_permission.py index e43a288744..2dfd7863b1 100644 --- a/frappe/core/doctype/user_permission/user_permission.py +++ b/frappe/core/doctype/user_permission/user_permission.py @@ -8,7 +8,6 @@ from frappe import _ from frappe.core.utils import find from frappe.desk.form.linked_with import get_linked_doctypes from frappe.model.document import Document -from frappe.permissions import get_valid_perms, update_permission_property from frappe.utils import cstr diff --git a/frappe/core/doctype/user_permission/user_permission_list.js b/frappe/core/doctype/user_permission/user_permission_list.js index 0ce66fa8e3..ce5e624403 100644 --- a/frappe/core/doctype/user_permission/user_permission_list.js +++ b/frappe/core/doctype/user_permission/user_permission_list.js @@ -1,18 +1,17 @@ -frappe.listview_settings['User Permission'] = { - - onload: function(list_view) { +frappe.listview_settings["User Permission"] = { + onload: function (list_view) { var me = this; - list_view.page.add_inner_button( __("Add / Update"), function() { - let dialog =new frappe.ui.Dialog({ - title : __('Add User Permissions'), + list_view.page.add_inner_button(__("Add / Update"), function () { + let dialog = new frappe.ui.Dialog({ + title: __("Add User Permissions"), fields: [ { - fieldname: 'user', - label: __('For User'), - fieldtype: 'Link', - options: 'User', + fieldname: "user", + label: __("For User"), + fieldtype: "Link", + options: "User", reqd: 1, - onchange: function() { + onchange: function () { dialog.fields_dict.doctype.set_input(undefined); dialog.fields_dict.docname.set_input(undefined); dialog.set_df_property("docname", "hidden", 1); @@ -20,77 +19,87 @@ frappe.listview_settings['User Permission'] = { dialog.set_df_property("apply_to_all_doctypes", "hidden", 1); dialog.set_df_property("applicable_doctypes", "hidden", 1); dialog.set_df_property("hide_descendants", "hidden", 1); - } + }, }, { - fieldname: 'doctype', - label: __('Document Type'), - fieldtype: 'Link', - options: 'DocType', + fieldname: "doctype", + label: __("Document Type"), + fieldtype: "Link", + options: "DocType", reqd: 1, - onchange: function() { + onchange: function () { me.on_doctype_change(dialog); - } + }, }, { - fieldname: 'docname', - label: __('Document Name'), - fieldtype: 'Dynamic Link', - options: 'doctype', + fieldname: "docname", + label: __("Document Name"), + fieldtype: "Dynamic Link", + options: "doctype", hidden: 1, - onchange: function() { + onchange: function () { let field = dialog.fields_dict["docname"]; - if(field.value != field.last_value) { - if(dialog.fields_dict.doctype.value && dialog.fields_dict.docname.value && dialog.fields_dict.user.value){ - me.get_applicable_doctype(dialog).then(applicable => { - me.get_multi_select_options(dialog, applicable).then(options => { - me.applicable_options = options; - me.on_docname_change(dialog, options, applicable); - if(options.length > 5){ - dialog.fields_dict.applicable_doctypes.setup_select_all(); + if (field.value != field.last_value) { + if ( + dialog.fields_dict.doctype.value && + dialog.fields_dict.docname.value && + dialog.fields_dict.user.value + ) { + me.get_applicable_doctype(dialog).then((applicable) => { + me.get_multi_select_options(dialog, applicable).then( + (options) => { + me.applicable_options = options; + me.on_docname_change(dialog, options, applicable); + if (options.length > 5) { + dialog.fields_dict.applicable_doctypes.setup_select_all(); + } } - }); + ); }); } } - } + }, }, { fieldtype: "Section Break", - hide_border: 1 + hide_border: 1, }, { - fieldname: 'is_default', - label: __('Is Default'), - fieldtype: 'Check', - hidden: 1 - }, - { - fieldname: 'apply_to_all_doctypes', - label: __('Apply to all Documents Types'), - fieldtype: 'Check', + fieldname: "is_default", + label: __("Is Default"), + fieldtype: "Check", hidden: 1, - onchange: function() { - if(dialog.fields_dict.doctype.value && dialog.fields_dict.docname.value && dialog.fields_dict.user.value){ + }, + { + fieldname: "apply_to_all_doctypes", + label: __("Apply to all Documents Types"), + fieldtype: "Check", + hidden: 1, + onchange: function () { + if ( + dialog.fields_dict.doctype.value && + dialog.fields_dict.docname.value && + dialog.fields_dict.user.value + ) { me.on_apply_to_all_doctypes_change(dialog, me.applicable_options); - if(me.applicable_options.length > 5){ + if (me.applicable_options.length > 5) { dialog.fields_dict.applicable_doctypes.setup_select_all(); } } - } + }, }, { - fieldtype: "Column Break" + fieldtype: "Column Break", }, { - fieldname: 'hide_descendants', - label: __('Hide Descendants'), - fieldtype: 'Check', - hidden: 1 + fieldname: "hide_descendants", + label: __("Hide Descendants"), + fieldtype: "Check", + hidden: 1, }, { fieldtype: "Section Break", - hide_border: 1 + hide_border: 1, }, { label: __("Applicable Document Types"), @@ -98,7 +107,7 @@ frappe.listview_settings['User Permission'] = { fieldtype: "MultiCheck", options: [], columns: 2, - hidden: 1 + hidden: 1, }, ], primary_action: (data) => { @@ -107,126 +116,137 @@ frappe.listview_settings['User Permission'] = { async: false, method: "frappe.core.doctype.user_permission.user_permission.add_user_permissions", args: { - data : data + data: data, }, - callback: function(r) { - if(r.message === 1) { - frappe.show_alert({message:__("User Permissions created sucessfully"), indicator:'blue'}); + callback: function (r) { + if (r.message === 1) { + frappe.show_alert({ + message: __("User Permissions created sucessfully"), + indicator: "blue", + }); } else { - frappe.show_alert({message:__("Nothing to update"), indicator:'red'}); - + frappe.show_alert({ + message: __("Nothing to update"), + indicator: "red", + }); } - } + }, }); dialog.hide(); list_view.refresh(); }, - primary_action_label: __('Submit') + primary_action_label: __("Submit"), }); dialog.show(); }); - list_view.page.add_inner_button( __("Bulk Delete"), function() { + list_view.page.add_inner_button(__("Bulk Delete"), function () { const dialog = new frappe.ui.Dialog({ - title: __('Clear User Permissions'), + title: __("Clear User Permissions"), fields: [ { - fieldname: 'user', - label: __('For User'), - fieldtype: 'Link', - options: 'User', - reqd: 1 + fieldname: "user", + label: __("For User"), + fieldtype: "Link", + options: "User", + reqd: 1, }, { - fieldname: 'for_doctype', - label: __('For Document Type'), - fieldtype: 'Link', - options: 'DocType', - reqd: 1 + fieldname: "for_doctype", + label: __("For Document Type"), + fieldtype: "Link", + options: "DocType", + reqd: 1, }, ], primary_action: (data) => { // mandatory not filled if (!data) return; - frappe.confirm(__('Are you sure?'), () => { + frappe.confirm(__("Are you sure?"), () => { frappe - .xcall('frappe.core.doctype.user_permission.user_permission.clear_user_permissions', data) - .then(data => { + .xcall( + "frappe.core.doctype.user_permission.user_permission.clear_user_permissions", + data + ) + .then((data) => { dialog.hide(); - let message = ''; + let message = ""; if (data === 0) { - message = __('No records deleted'); - } else if(data === 1) { - message = __('{0} record deleted', [data]); + message = __("No records deleted"); + } else if (data === 1) { + message = __("{0} record deleted", [data]); } else { - message = __('{0} records deleted', [data]); + message = __("{0} records deleted", [data]); } frappe.show_alert({ message, - indicator: 'info' + indicator: "info", }); list_view.refresh(); }); }); - }, - primary_action_label: __('Delete') + primary_action_label: __("Delete"), }); dialog.show(); }); }, - validate: function(dialog, data) { - if(dialog.fields_dict.applicable_doctypes.get_unchecked_options().length == 0) { + validate: function (dialog, data) { + if (dialog.fields_dict.applicable_doctypes.get_unchecked_options().length == 0) { data.apply_to_all_doctypes = 1; data.applicable_doctypes = []; return data; } - if(data.apply_to_all_doctypes == 0 && !("applicable_doctypes" in data)) { + if (data.apply_to_all_doctypes == 0 && !("applicable_doctypes" in data)) { frappe.throw(__("Please select applicable Doctypes")); } return data; }, - get_applicable_doctype: function(dialog) { - return new Promise(resolve => { - frappe.call({ - method: 'frappe.core.doctype.user_permission.user_permission.check_applicable_doc_perm', - async: false, - args:{ - user: dialog.fields_dict.user.value, - doctype: dialog.fields_dict.doctype.value, - docname: dialog.fields_dict.docname.value - } - }).then(r => { - resolve(r.message); - }); + get_applicable_doctype: function (dialog) { + return new Promise((resolve) => { + frappe + .call({ + method: "frappe.core.doctype.user_permission.user_permission.check_applicable_doc_perm", + async: false, + args: { + user: dialog.fields_dict.user.value, + doctype: dialog.fields_dict.doctype.value, + docname: dialog.fields_dict.docname.value, + }, + }) + .then((r) => { + resolve(r.message); + }); }); }, - get_multi_select_options: function(dialog, applicable){ - return new Promise(resolve => { - frappe.call({ - method: 'frappe.desk.form.linked_with.get_linked_doctypes', - async: false, - args:{ - user: dialog.fields_dict.user.value, - doctype: dialog.fields_dict.doctype.value, - docname: dialog.fields_dict.docname.value - } - }).then(r => { - var options = []; - for(var d in r.message){ - var checked = ($.inArray(d, applicable) != -1) ? 1 : 0; - options.push({ "label":d, "value": d , "checked": checked}); - } - resolve(options); - }); + get_multi_select_options: function (dialog, applicable) { + return new Promise((resolve) => { + frappe + .call({ + method: "frappe.desk.form.linked_with.get_linked_doctypes", + async: false, + args: { + user: dialog.fields_dict.user.value, + doctype: dialog.fields_dict.doctype.value, + docname: dialog.fields_dict.docname.value, + }, + }) + .then((r) => { + var options = []; + for (var d in r.message) { + var checked = $.inArray(d, applicable) != -1 ? 1 : 0; + options.push({ label: d, value: d, checked: checked }); + } + resolve(options); + }); }); }, - on_doctype_change: function(dialog) { + on_doctype_change: function (dialog) { dialog.set_df_property("docname", "hidden", 0); dialog.set_df_property("docname", "reqd", 1); dialog.set_df_property("is_default", "hidden", 0); @@ -237,12 +257,15 @@ frappe.listview_settings['User Permission'] = { dialog.refresh(); }, - on_docname_change: function(dialog, options, applicable) { - if(applicable.length != 0 ) { + on_docname_change: function (dialog, options, applicable) { + if (applicable.length != 0) { dialog.set_primary_action("Update"); dialog.set_title("Update User Permissions"); dialog.set_df_property("applicable_doctypes", "options", options); - if(dialog.fields_dict.applicable_doctypes.get_checked_options().length == options.length) { + if ( + dialog.fields_dict.applicable_doctypes.get_checked_options().length == + options.length + ) { dialog.set_df_property("applicable_doctypes", "hidden", 1); } else { dialog.set_df_property("applicable_doctypes", "hidden", 0); @@ -257,8 +280,8 @@ frappe.listview_settings['User Permission'] = { dialog.refresh(); }, - on_apply_to_all_doctypes_change: function(dialog, options) { - if(dialog.fields_dict.apply_to_all_doctypes.get_value() == 0) { + on_apply_to_all_doctypes_change: function (dialog, options) { + if (dialog.fields_dict.apply_to_all_doctypes.get_value() == 0) { dialog.set_df_property("applicable_doctypes", "hidden", 0); dialog.set_df_property("applicable_doctypes", "options", options); } else { @@ -266,5 +289,5 @@ frappe.listview_settings['User Permission'] = { dialog.set_df_property("applicable_doctypes", "hidden", 1); } dialog.refresh_sections(); - } + }, }; diff --git a/frappe/core/doctype/user_select_document_type/user_select_document_type.py b/frappe/core/doctype/user_select_document_type/user_select_document_type.py index 07b8123a13..9cf2e4856d 100644 --- a/frappe/core/doctype/user_select_document_type/user_select_document_type.py +++ b/frappe/core/doctype/user_select_document_type/user_select_document_type.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2021, Frappe Technologies and contributors # License: MIT. See LICENSE diff --git a/frappe/core/doctype/user_social_login/user_social_login.json b/frappe/core/doctype/user_social_login/user_social_login.json index 3cac838016..6b4b1822d1 100644 --- a/frappe/core/doctype/user_social_login/user_social_login.json +++ b/frappe/core/doctype/user_social_login/user_social_login.json @@ -1,189 +1,58 @@ { - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, - "creation": "2017-12-02 13:01:20.507112", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", + "actions": [], + "creation": "2017-12-02 13:01:20.507112", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "provider", + "section_break_0", + "username", + "column_break_0", + "userid" + ], "fields": [ { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "provider", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Provider", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "provider", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Provider", + "read_only": 1 + }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "section_break_0", - "fieldtype": "Section Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "section_break_0", + "fieldtype": "Section Break" + }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "username", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Username", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "username", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Username", + "read_only": 1 + }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "column_break_0", - "fieldtype": "Column Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "column_break_0", + "fieldtype": "Column Break" + }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "userid", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "User ID", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "fieldname": "userid", + "fieldtype": "Data", + "in_list_view": 1, + "label": "User ID", + "read_only": 1 } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 1, - "max_attachments": 0, - "modified": "2017-12-02 15:37:58.397062", - "modified_by": "Administrator", - "module": "Core", - "name": "User Social Login", - "name_case": "", - "owner": "Administrator", - "permissions": [], - "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1, - "track_seen": 0 + ], + "istable": 1, + "links": [], + "modified": "2022-08-03 12:20:53.800689", + "modified_by": "Administrator", + "module": "Core", + "name": "User Social Login", + "owner": "Administrator", + "permissions": [], + "quick_entry": 1, + "sort_field": "modified", + "sort_order": "DESC", + "states": [], + "track_changes": 1 } \ No newline at end of file diff --git a/frappe/core/doctype/user_social_login/user_social_login.py b/frappe/core/doctype/user_social_login/user_social_login.py index d12b5823d1..4cf3f720cd 100644 --- a/frappe/core/doctype/user_social_login/user_social_login.py +++ b/frappe/core/doctype/user_social_login/user_social_login.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2017, Frappe Technologies and contributors # License: MIT. See LICENSE diff --git a/frappe/core/doctype/user_type/test_user_type.py b/frappe/core/doctype/user_type/test_user_type.py index 53999ed3df..235881517a 100644 --- a/frappe/core/doctype/user_type/test_user_type.py +++ b/frappe/core/doctype/user_type/test_user_type.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2021, Frappe Technologies and Contributors # License: MIT. See LICENSE import unittest diff --git a/frappe/core/doctype/user_type/user_type.js b/frappe/core/doctype/user_type/user_type.js index c8bd499b58..5cf0dbb25f 100644 --- a/frappe/core/doctype/user_type/user_type.js +++ b/frappe/core/doctype/user_type/user_type.js @@ -1,77 +1,71 @@ // Copyright (c) 2021, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('User Type', { - refresh: function(frm) { - frm.toggle_display('is_standard', frappe.boot.developer_mode); - frm.set_df_property('is_standard', 'read_only', !frappe.boot.developer_mode); +frappe.ui.form.on("User Type", { + refresh: function (frm) { + if (frm.is_new() && !frappe.boot.developer_mode) frm.set_value("is_standard", 1); - const fields = ['role', 'apply_user_permission_on', 'user_id_field', - 'user_doctypes', 'user_type_modules']; - - frm.toggle_display(fields, !frm.doc.is_standard); - - frm.set_query('document_type', 'user_doctypes', function() { + frm.set_query("document_type", "user_doctypes", function () { return { filters: { - istable: 0 - } + istable: 0, + }, }; }); - frm.set_query('document_type', 'select_doctypes', function() { + frm.set_query("document_type", "select_doctypes", function () { return { filters: { - istable: 0 - } + istable: 0, + }, }; }); - frm.set_query('document_type', 'custom_select_doctypes', function() { + frm.set_query("document_type", "custom_select_doctypes", function () { return { filters: { - istable: 0 - } + istable: 0, + }, }; }); - frm.set_query('role', function() { + frm.set_query("role", function () { return { filters: { is_custom: 1, disabled: 0, - desk_access: 1 - } + desk_access: 1, + }, }; }); - frm.set_query('apply_user_permission_on', function() { + frm.set_query("apply_user_permission_on", function () { return { - query: "frappe.core.doctype.user_type.user_type.get_user_linked_doctypes" + query: "frappe.core.doctype.user_type.user_type.get_user_linked_doctypes", }; }); }, - onload: function(frm) { - frm.trigger('get_user_id_fields'); + onload: function (frm) { + frm.trigger("get_user_id_fields"); }, - apply_user_permission_on: function(frm) { - frm.set_value('user_id_field', ''); - frm.trigger('get_user_id_fields'); + apply_user_permission_on: function (frm) { + frm.set_value("user_id_field", ""); + frm.trigger("get_user_id_fields"); }, - get_user_id_fields: function(frm) { + get_user_id_fields: function (frm) { if (frm.doc.apply_user_permission_on) { frappe.call({ - method: 'frappe.core.doctype.user_type.user_type.get_user_id', + method: "frappe.core.doctype.user_type.user_type.get_user_id", args: { - parent: frm.doc.apply_user_permission_on + parent: frm.doc.apply_user_permission_on, + }, + callback: function (r) { + set_field_options("user_id_field", [""].concat(r.message)); }, - callback: function(r) { - set_field_options('user_id_field', [""].concat(r.message)); - } }); } - } + }, }); diff --git a/frappe/core/doctype/user_type/user_type.json b/frappe/core/doctype/user_type/user_type.json index 9ea5d5be71..3d6b470af5 100644 --- a/frappe/core/doctype/user_type/user_type.json +++ b/frappe/core/doctype/user_type/user_type.json @@ -22,9 +22,11 @@ "fields": [ { "default": "0", + "depends_on": "eval: frappe.boot.developer_mode", "fieldname": "is_standard", "fieldtype": "Check", - "label": "Is Standard" + "label": "Is Standard", + "read_only_depends_on": "eval: !frappe.boot.developer_mode" }, { "depends_on": "eval: !doc.is_standard", @@ -33,21 +35,21 @@ "label": "Document Types and Permissions" }, { + "depends_on": "eval: !doc.is_standard", "fieldname": "user_doctypes", "fieldtype": "Table", "label": "Document Types", "mandatory_depends_on": "eval: !doc.is_standard", - "options": "User Document Type", - "read_only": 1 + "options": "User Document Type" }, { + "depends_on": "eval: !doc.is_standard", "fieldname": "role", "fieldtype": "Link", "in_list_view": 1, "label": "Role", "mandatory_depends_on": "eval: !doc.is_standard", - "options": "Role", - "read_only": 1 + "options": "Role" }, { "fieldname": "select_doctypes", @@ -62,13 +64,13 @@ "fieldtype": "Column Break" }, { + "depends_on": "eval: !doc.is_standard", "description": "Can only list down the document types which has been linked to the User document type.", "fieldname": "apply_user_permission_on", "fieldtype": "Link", "label": "Apply User Permission On", "mandatory_depends_on": "eval: !doc.is_standard", - "options": "DocType", - "read_only": 1 + "options": "DocType" }, { "depends_on": "eval: !doc.is_standard", @@ -81,8 +83,7 @@ "fieldname": "user_id_field", "fieldtype": "Select", "label": "User Id Field", - "mandatory_depends_on": "eval: !doc.is_standard", - "read_only": 1 + "mandatory_depends_on": "eval: !doc.is_standard" }, { "depends_on": "eval: !doc.is_standard", @@ -93,6 +94,7 @@ { "fieldname": "user_type_modules", "fieldtype": "Table", + "label": "User Type Module", "no_copy": 1, "options": "User Type Module", "print_hide": 1, @@ -107,10 +109,11 @@ ], "index_web_pages_for_search": 1, "links": [], - "modified": "2021-03-12 16:25:18.639050", + "modified": "2022-06-09 14:00:36.820306", "modified_by": "Administrator", "module": "Core", "name": "User Type", + "naming_rule": "Set by user", "owner": "Administrator", "permissions": [ { @@ -137,5 +140,6 @@ ], "sort_field": "modified", "sort_order": "DESC", + "states": [], "track_changes": 1 } \ No newline at end of file diff --git a/frappe/core/doctype/user_type/user_type.py b/frappe/core/doctype/user_type/user_type.py index 7efb66f569..369e70bf56 100644 --- a/frappe/core/doctype/user_type/user_type.py +++ b/frappe/core/doctype/user_type/user_type.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2021, Frappe Technologies and contributors # License: MIT. See LICENSE @@ -211,7 +210,7 @@ def get_user_linked_doctypes(doctype, txt, searchfield, start, page_len, filters ["DocType", "issingle", "=", 0], ["DocType", "module", "not in", modules], ["DocType", "read_only", "=", 0], - ["DocType", "name", "like", "%{0}%".format(txt)], + ["DocType", "name", "like", f"%{txt}%"], ] doctypes = frappe.get_all( @@ -225,7 +224,7 @@ def get_user_linked_doctypes(doctype, txt, searchfield, start, page_len, filters ) custom_dt_filters = [ - ["Custom Field", "dt", "like", "%{0}%".format(txt)], + ["Custom Field", "dt", "like", f"%{txt}%"], ["Custom Field", "options", "=", "User"], ["Custom Field", "fieldtype", "=", "Link"], ] diff --git a/frappe/core/doctype/user_type/user_type_list.js b/frappe/core/doctype/user_type/user_type_list.js index 9a9ef417ac..856fe8985e 100644 --- a/frappe/core/doctype/user_type/user_type_list.js +++ b/frappe/core/doctype/user_type/user_type_list.js @@ -1,4 +1,4 @@ -frappe.listview_settings['User Type'] = { +frappe.listview_settings["User Type"] = { add_fields: ["is_standard"], get_indicator: function (doc) { if (doc.is_standard) { @@ -6,5 +6,5 @@ frappe.listview_settings['User Type'] = { } else { return [__("Custom"), "blue", "is_standard,=,0"]; } - } + }, }; diff --git a/frappe/core/doctype/user_type_module/user_type_module.py b/frappe/core/doctype/user_type_module/user_type_module.py index 83662bfcaf..1dc7c849e8 100644 --- a/frappe/core/doctype/user_type_module/user_type_module.py +++ b/frappe/core/doctype/user_type_module/user_type_module.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2021, Frappe Technologies and contributors # License: MIT. See LICENSE diff --git a/frappe/core/doctype/version/test_version.py b/frappe/core/doctype/version/test_version.py index c35430b17b..3e82f30f06 100644 --- a/frappe/core/doctype/version/test_version.py +++ b/frappe/core/doctype/version/test_version.py @@ -32,6 +32,19 @@ class TestVersion(unittest.TestCase): self.assertEqual(get_old_values(diff)[1], "01-01-2014 00:00:00") self.assertEqual(get_new_values(diff)[1], "07-20-2017 00:00:00") + def test_no_version_on_new_doc(self): + from frappe.desk.form.load import get_versions + + t = frappe.get_doc(doctype="ToDo", description="something") + t.save(ignore_version=False) + + self.assertFalse(get_versions(t)) + + t = frappe.get_doc(t.doctype, t.name) + t.description = "changed" + t.save(ignore_version=False) + self.assertTrue(get_versions(t)) + def get_fieldnames(change_array): return [d[0] for d in change_array] diff --git a/frappe/core/doctype/version/version.js b/frappe/core/doctype/version/version.js index d39d2eac03..1e26e5f748 100644 --- a/frappe/core/doctype/version/version.js +++ b/frappe/core/doctype/version/version.js @@ -1,9 +1,12 @@ -frappe.ui.form.on("Version", "refresh", function(frm) { - $(frappe.render_template('version_view', {doc:frm.doc, data:JSON.parse(frm.doc.data)})) - .appendTo(frm.fields_dict.table_html.$wrapper.empty()); +frappe.ui.form.on("Version", "refresh", function (frm) { + $( + frappe.render_template("version_view", { doc: frm.doc, data: JSON.parse(frm.doc.data) }) + ).appendTo(frm.fields_dict.table_html.$wrapper.empty()); - frm.add_custom_button(__('Show all Versions'), function() { - frappe.set_route('List', 'Version', - {ref_doctype: frm.doc.ref_doctype, docname: frm.doc.docname}); + frm.add_custom_button(__("Show all Versions"), function () { + frappe.set_route("List", "Version", { + ref_doctype: frm.doc.ref_doctype, + docname: frm.doc.docname, + }); }); }); diff --git a/frappe/core/doctype/version/version.json b/frappe/core/doctype/version/version.json index 463a7d3cba..13c82fa2b2 100644 --- a/frappe/core/doctype/version/version.json +++ b/frappe/core/doctype/version/version.json @@ -1,247 +1,81 @@ { - "allow_copy": 0, - "allow_import": 0, - "allow_rename": 0, - "autoname": "hash", - "beta": 0, - "creation": "2014-02-20 17:22:37", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "Setup", - "editable_grid": 0, - "engine": "InnoDB", + "actions": [], + "autoname": "hash", + "creation": "2014-02-20 17:22:37", + "doctype": "DocType", + "document_type": "Setup", + "engine": "InnoDB", + "field_order": [ + "ref_doctype", + "column_break_3", + "docname", + "data", + "section_break_4", + "table_html" + ], "fields": [ { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "ref_doctype", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 1, - "in_standard_filter": 1, - "label": "DocType", - "length": 0, - "no_copy": 0, - "options": "DocType", - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "ref_doctype", + "fieldtype": "Link", + "in_list_view": 1, + "in_standard_filter": 1, + "label": "DocType", + "options": "DocType", + "reqd": 1 + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "column_break_3", - "fieldtype": "Column Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "column_break_3", + "fieldtype": "Column Break" + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "docname", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Document Name", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "docname", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Document Name", + "reqd": 1 + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "data", - "fieldtype": "Code", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Data", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "data", + "fieldtype": "Code", + "hidden": 1, + "label": "Data" + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "section_break_4", - "fieldtype": "Section Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "section_break_4", + "fieldtype": "Section Break" + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "table_html", - "fieldtype": "HTML", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Table HTML", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "fieldname": "table_html", + "fieldtype": "HTML", + "label": "Table HTML" } - ], - "hide_heading": 0, - "hide_toolbar": 0, - "icon": "fa fa-copy", - "idx": 1, - "image_view": 0, - "in_create": 1, - - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2018-04-10 14:39:45.926836", - "modified_by": "Administrator", - "module": "Core", - "name": "Version", - "owner": "Administrator", + ], + "icon": "fa fa-copy", + "idx": 1, + "in_create": 1, + "links": [], + "modified": "2022-08-03 12:20:53.929691", + "modified_by": "Administrator", + "module": "Core", + "name": "Version", + "owner": "Administrator", "permissions": [ { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 0, - "delete": 0, - "email": 0, - "export": 1, - "if_owner": 0, - "import": 0, - "is_custom": 0, - "permlevel": 0, - "print": 0, - "read": 1, - "report": 1, - "role": "System Manager", - "set_user_permissions": 0, - "share": 0, - "submit": 0, - "write": 0 - }, + "export": 1, + "read": 1, + "report": 1, + "role": "System Manager" + }, { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 0, - "delete": 1, - "email": 0, - "export": 0, - "if_owner": 0, - "import": 0, - "is_custom": 0, - "permlevel": 0, - "print": 0, - "read": 1, - "report": 0, - "role": "Administrator", - "set_user_permissions": 0, - "share": 0, - "submit": 0, - "write": 0 + "delete": 1, + "read": 1, + "role": "Administrator" } - ], - "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "sort_order": "ASC", - "title_field": "docname", - "track_changes": 1, - "track_seen": 0 + ], + "quick_entry": 1, + "sort_field": "modified", + "sort_order": "ASC", + "states": [], + "title_field": "docname", + "track_changes": 1 } \ No newline at end of file diff --git a/frappe/core/doctype/version/version.py b/frappe/core/doctype/version/version.py index 540f8c7a02..99eeb8c2b0 100644 --- a/frappe/core/doctype/version/version.py +++ b/frappe/core/doctype/version/version.py @@ -9,19 +9,30 @@ from frappe.model.document import Document class Version(Document): - def set_diff(self, old, new): + def update_version_info(self, old: Document | None, new: Document) -> bool: + """Update changed info and return true if change contains useful data.""" + if not old: + # Check if doc has some information about creation source like data import + return self.for_insert(new) + else: + return self.set_diff(old, new) + + def set_diff(self, old: Document, new: Document) -> bool: """Set the data property with the diff of the docs if present""" diff = get_diff(old, new) if diff: self.ref_doctype = new.doctype self.docname = new.name - self.data = frappe.as_json(diff) + self.data = frappe.as_json(diff, indent=None, separators=(",", ":")) return True else: return False - def for_insert(self, doc): + def for_insert(self, doc: Document) -> bool: updater_reference = doc.flags.updater_reference + if not updater_reference: + return False + data = { "creation": doc.creation, "updater_reference": updater_reference, @@ -29,7 +40,8 @@ class Version(Document): } self.ref_doctype = doc.doctype self.docname = doc.name - self.data = frappe.as_json(data) + self.data = frappe.as_json(data, indent=None, separators=(",", ":")) + return True def get_data(self): return json.loads(self.data) diff --git a/frappe/core/doctype/version/version_view.html b/frappe/core/doctype/version/version_view.html index 67f005ed4c..a17460ccc7 100644 --- a/frappe/core/doctype/version/version_view.html +++ b/frappe/core/doctype/version/version_view.html @@ -18,8 +18,8 @@ {% for item in data.changed %}
    - - + + {% endfor %} @@ -43,8 +43,7 @@ {% for item in values %} - + filters.forEach((filter) => { + const filter_row = $(``); - table.find('tbody').append(filter_row); + table.find("tbody").append(filter_row); filters_set = true; }); } } else if (frm.chart_filters.length) { - fields = frm.chart_filters.filter(f => f.fieldname); + fields = frm.chart_filters.filter((f) => f.fieldname); - fields.map(f => { + fields.map((f) => { if (filters[f.fieldname]) { - let condition = '='; - const filter_row = - $(` + let condition = "="; + const filter_row = $(``); - table.find('tbody').append(filter_row); + table.find("tbody").append(filter_row); filters_set = true; } }); @@ -340,39 +367,39 @@ frappe.ui.form.on('Dashboard Chart', { if (!filters_set) { const filter_row = $(``); - table.find('tbody').append(filter_row); + table.find("tbody").append(filter_row); } - table.on('click', () => { - frm.is_disabled && frappe.throw(__('Cannot edit filters for standard charts')); + table.on("click", () => { + frm.is_disabled && frappe.throw(__("Cannot edit filters for standard charts")); let dialog = new frappe.ui.Dialog({ - title: __('Set Filters'), - fields: fields.filter(f => !is_dynamic_filter(f)), - primary_action: function() { + title: __("Set Filters"), + fields: fields.filter((f) => !is_dynamic_filter(f)), + primary_action: function () { let values = this.get_values(); if (values) { this.hide(); if (is_document_type) { let filters = frm.filter_group.get_filters(); - frm.set_value('filters_json', JSON.stringify(filters)); + frm.set_value("filters_json", JSON.stringify(filters)); } else { - frm.set_value('filters_json', JSON.stringify(values)); + frm.set_value("filters_json", JSON.stringify(values)); } - frm.trigger('show_filters'); - if (frm.doc.chart_type == 'Report') { - frm.trigger('set_chart_report_filters'); + frm.trigger("show_filters"); + if (frm.doc.chart_type == "Report") { + frm.trigger("set_chart_report_filters"); } } }, - primary_action_label: "Set" + primary_action_label: "Set", }); frappe.dashboards.filters_dialog = dialog; if (is_document_type) { frm.filter_group = new frappe.ui.FilterGroup({ - parent: dialog.get_field('filter_area').$wrapper, + parent: dialog.get_field("filter_area").$wrapper, doctype: frm.doc.document_type, parent_doctype: frm.doc.parent_document_type, on_change: () => {}, @@ -383,12 +410,14 @@ frappe.ui.form.on('Dashboard Chart', { dialog.show(); - if (frm.doc.chart_type == 'Report') { + if (frm.doc.chart_type == "Report") { //Set query report object so that it can be used while fetching filter values in the report - frappe.query_report = new frappe.views.QueryReport({'filters': dialog.fields_list}); - frappe.query_reports[frm.doc.report_name] - && frappe.query_reports[frm.doc.report_name].onload - && frappe.query_reports[frm.doc.report_name].onload(frappe.query_report); + frappe.query_report = new frappe.views.QueryReport({ + filters: dialog.fields_list, + }); + frappe.query_reports[frm.doc.report_name] && + frappe.query_reports[frm.doc.report_name].onload && + frappe.query_reports[frm.doc.report_name].onload(frappe.query_report); } dialog.set_values(filters); @@ -398,37 +427,40 @@ frappe.ui.form.on('Dashboard Chart', { render_dynamic_filters_table(frm) { frm.set_df_property("dynamic_filters_section", "hidden", 0); - let is_document_type = frm.doc.chart_type !== 'Report' - && frm.doc.chart_type !== 'Custom'; + let is_document_type = frm.doc.chart_type !== "Report" && frm.doc.chart_type !== "Custom"; - let wrapper = $(frm.get_field('dynamic_filters_json').wrapper).empty(); + let wrapper = $(frm.get_field("dynamic_filters_json").wrapper).empty(); - frm.dynamic_filter_table = $(`
    ${ __("Filter") }${ __("Value") }${__("Filter")}${__("Value")}
    ${frappe.model.unscrub(key)} ${filters[key]}
    {{ frappe.meta.get_label(doc.ref_doctype, item[0]) }}{{ item[1] }}{{ item[2] }}{{ item[1] }}{{ item[2] }}
    {{ frappe.meta.get_label(doc.ref_doctype, item[0]) }} + {% var item_keys = Object.keys(item[1]).sort(); %} @@ -86,8 +85,8 @@ - - + + {% endfor %} {% endfor %} diff --git a/frappe/core/doctype/view_log/test_view_log.py b/frappe/core/doctype/view_log/test_view_log.py index 04a17cc526..5a88269028 100644 --- a/frappe/core/doctype/view_log/test_view_log.py +++ b/frappe/core/doctype/view_log/test_view_log.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2018, Frappe Technologies and Contributors # License: MIT. See LICENSE import unittest diff --git a/frappe/core/doctype/view_log/view_log.js b/frappe/core/doctype/view_log/view_log.js index a8c95b01e8..06d23802be 100644 --- a/frappe/core/doctype/view_log/view_log.js +++ b/frappe/core/doctype/view_log/view_log.js @@ -1,8 +1,6 @@ // Copyright (c) 2018, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('View Log', { - refresh: function(frm) { - - } +frappe.ui.form.on("View Log", { + refresh: function (frm) {}, }); diff --git a/frappe/core/doctype/view_log/view_log.json b/frappe/core/doctype/view_log/view_log.json index 3c4486c944..6b19cdd507 100644 --- a/frappe/core/doctype/view_log/view_log.json +++ b/frappe/core/doctype/view_log/view_log.json @@ -1,163 +1,58 @@ { - "allow_copy": 0, - "allow_events_in_timeline": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, + "actions": [], "creation": "2018-05-27 02:20:11.193944", - "custom": 0, - "docstatus": 0, "doctype": "DocType", - "document_type": "", "editable_grid": 1, "engine": "InnoDB", + "field_order": [ + "viewed_by", + "reference_doctype", + "reference_name" + ], "fields": [ { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fetch_if_empty": 0, "fieldname": "viewed_by", "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, "label": "Viewed By", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, "search_index": 1, - "set_only_once": 1, - "translatable": 0, - "unique": 0 + "set_only_once": 1 }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fetch_if_empty": 0, "fieldname": "reference_doctype", "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, "label": "Reference Document Type", - "length": 0, - "no_copy": 0, "options": "DocType", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, "search_index": 1, - "set_only_once": 1, - "translatable": 0, - "unique": 0 + "set_only_once": 1 }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fetch_if_empty": 0, "fieldname": "reference_name", "fieldtype": "Dynamic Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, "label": "Reference name", - "length": 0, - "no_copy": 0, "options": "reference_doctype", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, "search_index": 1, - "set_only_once": 1, - "translatable": 0, - "unique": 0 + "set_only_once": 1 } ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2021-10-25 14:22:27.664645", + "links": [], + "modified": "2022-08-03 12:20:52.857103", "modified_by": "Administrator", "module": "Core", "name": "View Log", - "name_case": "", "owner": "Administrator", "permissions": [ { - "amend": 0, - "cancel": 0, - "create": 0, - "delete": 0, "email": 1, "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, "print": 1, "read": 1, "report": 1, "role": "System Manager", - "set_user_permissions": 0, - "share": 1, - "submit": 0, - "write": 0 + "share": 1 } ], "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, "sort_field": "modified", "sort_order": "DESC", - "track_seen": 0, - "track_views": 0 -} + "states": [] +} \ No newline at end of file diff --git a/frappe/core/doctype/view_log/view_log.py b/frappe/core/doctype/view_log/view_log.py index 4a3ba0e83c..8383af818e 100644 --- a/frappe/core/doctype/view_log/view_log.py +++ b/frappe/core/doctype/view_log/view_log.py @@ -1,8 +1,6 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2018, Frappe Technologies and contributors # License: MIT. See LICENSE -import frappe from frappe.model.document import Document diff --git a/frappe/core/notifications.py b/frappe/core/notifications.py index a10f8ec5ae..9a127e567e 100644 --- a/frappe/core/notifications.py +++ b/frappe/core/notifications.py @@ -2,8 +2,6 @@ # License: MIT. See LICENSE import frappe -from frappe.query_builder import DocType, Interval -from frappe.query_builder.functions import Now def get_notification_config(): diff --git a/frappe/core/page/background_jobs/background_jobs.js b/frappe/core/page/background_jobs/background_jobs.js index 7334bfd5dd..94c7bbf3bc 100644 --- a/frappe/core/page/background_jobs/background_jobs.js +++ b/frappe/core/page/background_jobs/background_jobs.js @@ -1,4 +1,4 @@ -frappe.pages["background_jobs"].on_page_load = wrapper => { +frappe.pages["background_jobs"].on_page_load = (wrapper) => { const background_job = new BackgroundJobs(wrapper); $(wrapper).bind("show", () => { @@ -13,20 +13,15 @@ class BackgroundJobs { this.page = frappe.ui.make_app_page({ parent: wrapper, title: __("Background Jobs"), - single_column: true + single_column: true, }); this.page.add_inner_button(__("Remove Failed Jobs"), () => { - frappe.confirm( - __("Are you sure you want to remove all failed jobs?"), - () => { - frappe - .call( - "frappe.core.page.background_jobs.background_jobs.remove_failed_jobs" - ) - .then(() => this.refresh_jobs()); - } - ); + frappe.confirm(__("Are you sure you want to remove all failed jobs?"), () => { + frappe + .call("frappe.core.page.background_jobs.background_jobs.remove_failed_jobs") + .then(() => this.refresh_jobs()); + }); }); this.page.main.addClass("frappe-card"); @@ -34,10 +29,7 @@ class BackgroundJobs { this.$content = $(this.page.body).find(".table-area"); this.make_filters(); - this.refresh_jobs = frappe.utils.throttle( - this.refresh_jobs.bind(this), - 1000 - ); + this.refresh_jobs = frappe.utils.throttle(this.refresh_jobs.bind(this), 1000); } make_filters() { @@ -50,7 +42,7 @@ class BackgroundJobs { change: () => { this.queue_timeout.toggle(this.view.get_value() === "Jobs"); this.job_status.toggle(this.view.get_value() === "Jobs"); - } + }, }); this.queue_timeout = this.page.add_field({ label: __("Queue"), @@ -60,9 +52,9 @@ class BackgroundJobs { { label: "All Queues", value: "all" }, { label: "Default", value: "default" }, { label: "Short", value: "short" }, - { label: "Long", value: "long" } + { label: "Long", value: "long" }, ], - default: "all" + default: "all", }); this.job_status = this.page.add_field({ label: __("Job Status"), @@ -74,9 +66,9 @@ class BackgroundJobs { { label: "Deferred", value: "deferred" }, { label: "Started", value: "started" }, { label: "Finished", value: "finished" }, - { label: "Failed", value: "failed" } + { label: "Failed", value: "failed" }, ], - default: "all" + default: "all", }); this.auto_refresh = this.page.add_field({ label: __("Auto Refresh"), @@ -87,7 +79,7 @@ class BackgroundJobs { if (this.auto_refresh.get_value()) { this.refresh_jobs(); } - } + }, }); } @@ -98,16 +90,15 @@ class BackgroundJobs { update_scheduler_status() { frappe.call({ - method: - "frappe.core.page.background_jobs.background_jobs.get_scheduler_status", - callback: r => { + method: "frappe.core.page.background_jobs.background_jobs.get_scheduler_status", + callback: (r) => { let { status } = r.message; if (status === "active") { this.page.set_indicator(__("Scheduler: Active"), "green"); } else { this.page.set_indicator(__("Scheduler: Inactive"), "red"); } - } + }, }); } @@ -125,25 +116,21 @@ class BackgroundJobs { frappe.call({ method: "frappe.core.page.background_jobs.background_jobs.get_info", args, - callback: res => { + callback: (res) => { this.page.add_inner_message(""); - let template = - view === "Jobs" ? "background_jobs" : "background_workers"; + let template = view === "Jobs" ? "background_jobs" : "background_workers"; this.$content.html( frappe.render_template(template, { - jobs: res.message || [] + jobs: res.message || [], }) ); let auto_refresh = this.auto_refresh.get_value(); - if ( - frappe.get_route()[0] === "background_jobs" && - auto_refresh - ) { + if (frappe.get_route()[0] === "background_jobs" && auto_refresh) { setTimeout(() => this.refresh_jobs(), 2000); } - } + }, }); } } diff --git a/frappe/core/page/background_jobs/background_jobs.py b/frappe/core/page/background_jobs/background_jobs.py index d3f5e3d32f..8ef15b65eb 100644 --- a/frappe/core/page/background_jobs/background_jobs.py +++ b/frappe/core/page/background_jobs/background_jobs.py @@ -1,13 +1,9 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import json -from typing import TYPE_CHECKING, Dict, List - -from rq import Worker +from typing import TYPE_CHECKING import frappe -from frappe import _ from frappe.utils import convert_utc_to_user_timezone from frappe.utils.background_jobs import get_queues, get_workers from frappe.utils.scheduler import is_scheduler_inactive @@ -19,14 +15,10 @@ JOB_COLORS = {"queued": "orange", "failed": "red", "started": "blue", "finished" @frappe.whitelist() -def get_info(view=None, queue_timeout=None, job_status=None) -> List[Dict]: +def get_info(view=None, queue_timeout=None, job_status=None) -> list[dict]: jobs = [] - def add_job(job: "Job", name: str) -> None: - if job_status != "all" and job.get_status() != job_status: - return - if queue_timeout != "all" and not name.endswith(f":{queue_timeout}"): - return + def add_job(job: "Job", queue: str) -> None: if job.kwargs.get("site") == frappe.local.site: job_info = { @@ -34,7 +26,7 @@ def get_info(view=None, queue_timeout=None, job_status=None) -> List[Dict]: or job.kwargs.get("kwargs", {}).get("job_type") or str(job.kwargs.get("job_name")), "status": job.get_status(), - "queue": name, + "queue": queue, "creation": convert_utc_to_user_timezone(job.created_at), "color": JOB_COLORS[job.get_status()], } @@ -48,14 +40,21 @@ def get_info(view=None, queue_timeout=None, job_status=None) -> List[Dict]: queues = get_queues() for queue in queues: for job in queue.jobs: + if job_status != "all" and job.get_status() != job_status: + return + if queue_timeout != "all" and not queue.name.endswith(f":{queue_timeout}"): + return add_job(job, queue.name) elif view == "Workers": workers = get_workers() for worker in workers: current_job = worker.get_current_job() - if current_job and current_job.kwargs.get("site") == frappe.local.site: - add_job(current_job, job.origin) + if current_job: + if hasattr(current_job, "kwargs") and current_job.kwargs.get("site") == frappe.local.site: + add_job(current_job, current_job.origin) + else: + jobs.append({"queue": worker.name, "job_name": "busy", "status": "", "creation": ""}) else: jobs.append({"queue": worker.name, "job_name": "idle", "status": "", "creation": ""}) diff --git a/frappe/core/page/dashboard_view/dashboard_view.js b/frappe/core/page/dashboard_view/dashboard_view.js index bf9fb2a286..8f2c56910c 100644 --- a/frappe/core/page/dashboard_view/dashboard_view.js +++ b/frappe/core/page/dashboard_view/dashboard_view.js @@ -1,19 +1,18 @@ // Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and Contributors // MIT License. See license.txt -frappe.provide('frappe.dashboards'); -frappe.provide('frappe.dashboards.chart_sources'); +frappe.provide("frappe.dashboards"); +frappe.provide("frappe.dashboards.chart_sources"); - -frappe.pages['dashboard-view'].on_page_load = function(wrapper) { +frappe.pages["dashboard-view"].on_page_load = function (wrapper) { frappe.ui.make_app_page({ parent: wrapper, title: __("Dashboard"), - single_column: true + single_column: true, }); frappe.dashboard = new Dashboard(wrapper); - $(wrapper).bind('show', function() { + $(wrapper).bind("show", function () { frappe.dashboard.show(); }); }; @@ -37,20 +36,20 @@ class Dashboard { } else { // last opened if (frappe.last_dashboard) { - frappe.set_re_route('dashboard-view', frappe.last_dashboard); + frappe.set_re_route("dashboard-view", frappe.last_dashboard); } else { // default dashboard - frappe.db.get_list('Dashboard', {filters: {is_default: 1}}).then(data => { + frappe.db.get_list("Dashboard", { filters: { is_default: 1 } }).then((data) => { if (data && data.length) { - frappe.set_re_route('dashboard-view', data[0].name); + frappe.set_re_route("dashboard-view", data[0].name); } else { // no default, get the latest one - frappe.db.get_list('Dashboard', {limit: 1}).then(data => { + frappe.db.get_list("Dashboard", { limit: 1 }).then((data) => { if (data && data.length) { - frappe.set_re_route('dashboard-view', data[0].name); + frappe.set_re_route("dashboard-view", data[0].name); } else { // create a new dashboard! - frappe.new_doc('Dashboard'); + frappe.new_doc("Dashboard"); } }); } @@ -63,9 +62,9 @@ class Dashboard { if (this.dashboard_name !== current_dashboard_name) { this.dashboard_name = current_dashboard_name; let title = this.dashboard_name; - if (!this.dashboard_name.toLowerCase().includes(__('dashboard'))) { + if (!this.dashboard_name.toLowerCase().includes(__("dashboard"))) { // ensure dashboard title has "dashboard" - title = __('{0} Dashboard', [title]); + title = __("{0} Dashboard", [title]); } this.page.set_title(title); this.set_dropdown(); @@ -81,31 +80,30 @@ class Dashboard { } refresh() { - frappe.run_serially([ - () => this.render_cards(), - () => this.render_charts() - ]); + frappe.run_serially([() => this.render_cards(), () => this.render_charts()]); } render_charts() { return this.get_permitted_items( - 'frappe.desk.doctype.dashboard.dashboard.get_permitted_charts' - ).then(charts => { + "frappe.desk.doctype.dashboard.dashboard.get_permitted_charts" + ).then((charts) => { if (!charts.length) { - frappe.msgprint(__('No Permitted Charts on this Dashboard'), __('No Permitted Charts')) + frappe.msgprint( + __("No Permitted Charts on this Dashboard"), + __("No Permitted Charts") + ); } frappe.dashboard_utils.get_dashboard_settings().then((settings) => { - let chart_config = settings.chart_config? JSON.parse(settings.chart_config): {}; - this.charts = - charts.map(chart => { - return { - chart_name: chart.chart, - label: chart.chart, - chart_settings: chart_config[chart.chart] || {}, - ...chart - } - }); + let chart_config = settings.chart_config ? JSON.parse(settings.chart_config) : {}; + this.charts = charts.map((chart) => { + return { + chart_name: chart.chart, + label: chart.chart, + chart_settings: chart_config[chart.chart] || {}, + ...chart, + }; + }); this.chart_group = new frappe.widget.WidgetGroup({ title: null, @@ -121,24 +119,23 @@ class Dashboard { }, widgets: this.charts, }); - }) + }); }); } render_cards() { return this.get_permitted_items( - 'frappe.desk.doctype.dashboard.dashboard.get_permitted_cards' - ).then(cards => { + "frappe.desk.doctype.dashboard.dashboard.get_permitted_cards" + ).then((cards) => { if (!cards.length) { return; } - this.number_cards = - cards.map(card => { - return { - name: card.card, - }; - }); + this.number_cards = cards.map((card) => { + return { + name: card.card, + }; + }); this.number_card_group = new frappe.widget.WidgetGroup({ container: this.container, @@ -157,41 +154,43 @@ class Dashboard { } get_permitted_items(method) { - return frappe.xcall( - method, - { - dashboard_name: this.dashboard_name - } - ).then(items => { - return items; - }); + return frappe + .xcall(method, { + dashboard_name: this.dashboard_name, + }) + .then((items) => { + return items; + }); } set_dropdown() { this.page.clear_menu(); - this.page.add_menu_item(__('Edit'), () => { - frappe.set_route('Form', 'Dashboard', frappe.dashboard.dashboard_name); + this.page.add_menu_item(__("Edit"), () => { + frappe.set_route("Form", "Dashboard", frappe.dashboard.dashboard_name); }); - this.page.add_menu_item(__('New'), () => { - frappe.new_doc('Dashboard'); + this.page.add_menu_item(__("New"), () => { + frappe.new_doc("Dashboard"); }); - this.page.add_menu_item(__('Refresh All'), () => { - this.chart_group && - this.chart_group.widgets_list.forEach(chart => chart.refresh()); + this.page.add_menu_item(__("Refresh All"), () => { + this.chart_group && this.chart_group.widgets_list.forEach((chart) => chart.refresh()); this.number_card_group && - this.number_card_group.widgets_list.forEach(card => card.render_card()); + this.number_card_group.widgets_list.forEach((card) => card.render_card()); }); - frappe.db.get_list('Dashboard').then(dashboards => { - dashboards.map(dashboard => { + frappe.db.get_list("Dashboard").then((dashboards) => { + dashboards.map((dashboard) => { let name = dashboard.name; if (name != this.dashboard_name) { - this.page.add_menu_item(name, () => frappe.set_route("dashboard-view", name), 1); + this.page.add_menu_item( + name, + () => frappe.set_route("dashboard-view", name), + 1 + ); } }); }); } -} \ No newline at end of file +} diff --git a/frappe/core/page/permission_manager/permission_manager.js b/frappe/core/page/permission_manager/permission_manager.js index cb218b2eae..f29df0d3e5 100644 --- a/frappe/core/page/permission_manager/permission_manager.js +++ b/frappe/core/page/permission_manager/permission_manager.js @@ -1,20 +1,21 @@ -frappe.pages['permission-manager'].on_page_load = (wrapper) => { +frappe.pages["permission-manager"].on_page_load = (wrapper) => { let page = frappe.ui.make_app_page({ parent: wrapper, - title: __('Role Permissions Manager'), + title: __("Role Permissions Manager"), card_layout: true, - single_column: true + single_column: true, }); frappe.breadcrumbs.add("Setup"); - $("
    ").appendTo(page.main); + $("
    ").appendTo( + page.main + ); $(frappe.render_template("permission_manager_help", {})).appendTo(page.main); wrapper.permission_engine = new frappe.PermissionEngine(wrapper); - }; -frappe.pages['permission-manager'].refresh = function (wrapper) { +frappe.pages["permission-manager"].refresh = function (wrapper) { wrapper.permission_engine.set_from_route(); }; @@ -30,33 +31,38 @@ frappe.PermissionEngine = class PermissionEngine { make() { this.make_reset_button(); - frappe.call({ - module: "frappe.core", - page: "permission_manager", - method: "get_roles_and_doctypes" - }).then((res) => { - this.options = res.message; - this.setup_page(); - }); + frappe + .call({ + module: "frappe.core", + page: "permission_manager", + method: "get_roles_and_doctypes", + }) + .then((res) => { + this.options = res.message; + this.setup_page(); + }); } setup_page() { - this.doctype_select - = this.wrapper.page.add_select(__("Document Type"), - [{ value: "", label: __("Select Document Type") + "..." }].concat(this.options.doctypes)) - .change(function () { - frappe.set_route("permission-manager", $(this).val()); - }); + this.doctype_select = this.wrapper.page + .add_select( + __("Document Type"), + [{ value: "", label: __("Select Document Type") + "..." }].concat( + this.options.doctypes + ) + ) + .change(function () { + frappe.set_route("permission-manager", $(this).val()); + }); - this.role_select - = this.wrapper.page.add_select(__("Roles"), - [__("Select Role") + "..."].concat(this.options.roles)) - .change(() => { - this.refresh(); - }); + this.role_select = this.wrapper.page + .add_select(__("Roles"), [__("Select Role") + "..."].concat(this.options.roles)) + .change(() => { + this.refresh(); + }); - this.page.add_inner_button(__('Set User Permissions'), () => { - return frappe.set_route('List', 'User Permission'); + this.page.add_inner_button(__("Set User Permissions"), () => { + return frappe.set_route("List", "User Permission"); }); this.set_from_route(); } @@ -91,7 +97,7 @@ frappe.PermissionEngine = class PermissionEngine { page: "permission_manager", method: "get_standard_permissions", args: { doctype: doctype }, - callback: callback + callback: callback, }); } return false; @@ -100,18 +106,22 @@ frappe.PermissionEngine = class PermissionEngine { reset_std_permissions(data) { let doctype = this.get_doctype(); let d = frappe.confirm(__("Reset Permissions for {0}?", [doctype]), () => { - return frappe.call({ - module: "frappe.core", - page: "permission_manager", - method: "reset", - args: { doctype } - }).then(() => { - this.refresh(); - }); + return frappe + .call({ + module: "frappe.core", + page: "permission_manager", + method: "reset", + args: { doctype }, + }) + .then(() => { + this.refresh(); + }); }); // show standard permissions - let $d = $(d.wrapper).find(".frappe-confirm-message").append("
    Standard Permissions:

    "); + let $d = $(d.wrapper) + .find(".frappe-confirm-message") + .append("
    Standard Permissions:

    "); let $wrapper = $("

    ").appendTo($d); data.message.forEach((d) => { let rights = this.rights @@ -164,14 +174,16 @@ frappe.PermissionEngine = class PermissionEngine { } // get permissions - frappe.call({ - module: "frappe.core", - page: "permission_manager", - method: "get_permissions", - args: { doctype, role } - }).then((r) => { - this.render(r.message); - }); + frappe + .call({ + module: "frappe.core", + page: "permission_manager", + method: "get_permissions", + args: { doctype, role }, + }) + .then((r) => { + this.render(r.message); + }); } render(perm_list) { @@ -187,19 +199,21 @@ frappe.PermissionEngine = class PermissionEngine { } show_permission_table(perm_list) { - this.table = $("
    \ + this.table = $( + "
    \
    {{ frappe.meta.get_label(doc.ref_doctype, table_info[0]) }} {{ table_info[1] }} {{ item[0] }}{{ item[1] }}{{ item[2] }}{{ item[1] }}{{ item[2] }}
    \ \ \
    \ - ").appendTo(this.body); + " + ).appendTo(this.body); const table_columns = [ [__("Document Type"), 150], [__("Role"), 170], [__("Level"), 40], [__("Permissions"), 350], - ["", 40] + ["", 40], ]; table_columns.forEach((col) => { @@ -236,9 +250,9 @@ frappe.PermissionEngine = class PermissionEngine { let perm_cell = this.add_cell(row, d, "permissions"); let perm_container = $("
    ").appendTo(perm_cell); - this.rights.forEach(r => { - if (!d.is_submittable && ['submit', 'cancel', 'amend'].includes(r)) return; - if (d.in_create && ['create', 'write', 'delete'].includes(r)) return; + this.rights.forEach((r) => { + if (!d.is_submittable && ["submit", "cancel", "amend"].includes(r)) return; + if (d.in_create && ["create", "write", "delete"].includes(r)) return; this.add_check(perm_container, d, r); }); @@ -248,7 +262,8 @@ frappe.PermissionEngine = class PermissionEngine { } add_cell(row, d, fieldname) { - return $("    		").appendTo(row) + return $("") + .appendTo(row) .attr("data-fieldname", fieldname) .addClass("pt-4") .html(__(d[fieldname])); @@ -266,32 +281,48 @@ frappe.PermissionEngine = class PermissionEngine {

    ${__(description)}

    - `) + ` + ) .appendTo(cell) .attr("data-fieldname", fieldname); - checkbox.find("input") + checkbox + .find("input") .prop("checked", d[fieldname] ? true : false) .attr("data-ptype", fieldname) .attr("data-role", d.role) .attr("data-permlevel", d.permlevel) .attr("data-doctype", d.parent); - checkbox.find("label") - .css("text-transform", "capitalize"); + checkbox.find("label").css("text-transform", "capitalize"); return checkbox; } setup_if_owner(d, role_cell) { - this.add_check(role_cell, d, "if_owner", "Only If Creator") + this.add_check(role_cell, d, "if_owner", "Only if Creator") .removeClass("col-md-4") .css({ "margin-top": "15px" }); } get rights() { - return ["select", "read", "write", "create", "delete", "submit", "cancel", "amend", - "print", "email", "report", "import", "export", "set_user_permissions", "share"]; + return [ + "select", + "read", + "write", + "create", + "delete", + "submit", + "cancel", + "amend", + "print", + "email", + "report", + "import", + "export", + "set_user_permissions", + "share", + ]; } set_show_users(cell, role) { @@ -305,22 +336,29 @@ frappe.PermissionEngine = class PermissionEngine { page: "permission_manager", method: "get_users_with_role", args: { - role: role + role: role, }, callback: function (r) { r.message = $.map(r.message, function (p) { return $.format('{1}', [p, p]); }); - frappe.msgprint(__("Users with role {0}:", [__(role)]) - + "
    " + r.message.join("
    ")); - } + frappe.msgprint( + __("Users with role {0}:", [__(role)]) + + "
    " + + r.message.join("
    ") + ); + }, }); return false; }); } add_delete_button(row, d) { - $(``) + $( + `` + ) .appendTo($(`    		`).appendTo(row)) .attr("data-doctype", d.parent) .attr("data-role", d.role) @@ -333,7 +371,7 @@ frappe.PermissionEngine = class PermissionEngine { args: { doctype: d.parent, role: d.role, - permlevel: d.permlevel + permlevel: d.permlevel, }, callback: (r) => { if (r.exc) { @@ -341,7 +379,7 @@ frappe.PermissionEngine = class PermissionEngine { } else { this.refresh(); } - } + }, }); }); } @@ -350,7 +388,7 @@ frappe.PermissionEngine = class PermissionEngine { let me = this; this.body.on("click", ".show-user-permissions", () => { frappe.route_options = { allow: this.get_doctype() || "" }; - frappe.set_route('List', 'User Permission'); + frappe.set_route("List", "User Permission"); }); this.body.on("click", "input[type='checkbox']", function () { @@ -361,7 +399,7 @@ frappe.PermissionEngine = class PermissionEngine { permlevel: chk.attr("data-permlevel"), doctype: chk.attr("data-doctype"), ptype: chk.attr("data-ptype"), - value: chk.prop("checked") ? 1 : 0 + value: chk.prop("checked") ? 1 : 0, }; return frappe.call({ module: "frappe.core", @@ -376,7 +414,7 @@ frappe.PermissionEngine = class PermissionEngine { } else { me.get_perm(args.role)[args.ptype] = args.value; } - } + }, }); }); } @@ -389,19 +427,30 @@ frappe.PermissionEngine = class PermissionEngine { title: __("Add New Permission Rule"), fields: [ { - fieldtype: "Select", label: __("Document Type"), - options: this.options.doctypes, reqd: 1, fieldname: "parent" + fieldtype: "Select", + label: __("Document Type"), + options: this.options.doctypes, + reqd: 1, + fieldname: "parent", }, { - fieldtype: "Select", label: __("Role"), - options: this.options.roles, reqd: 1, fieldname: "role" + fieldtype: "Select", + label: __("Role"), + options: this.options.roles, + reqd: 1, + fieldname: "role", }, { - fieldtype: "Select", label: __("Permission Level"), - options: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], reqd: 1, fieldname: "permlevel", - description: __("Level 0 is for document level permissions, higher levels for field level permissions.") - } - ] + fieldtype: "Select", + label: __("Permission Level"), + options: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], + reqd: 1, + fieldname: "permlevel", + description: __( + "Level 0 is for document level permissions, higher levels for field level permissions." + ), + }, + ], }); if (this.get_doctype()) { d.set_value("parent", this.get_doctype()); @@ -412,7 +461,7 @@ frappe.PermissionEngine = class PermissionEngine { d.get_input("role").prop("disabled", true); } d.set_value("permlevel", "0"); - d.set_primary_action(__('Add'), () => { + d.set_primary_action(__("Add"), () => { let args = d.get_values(); if (!args) { return; @@ -428,7 +477,7 @@ frappe.PermissionEngine = class PermissionEngine { } else { this.refresh(); } - } + }, }); d.hide(); }); @@ -439,13 +488,11 @@ frappe.PermissionEngine = class PermissionEngine { } make_reset_button() { - this.page.set_secondary_action( - __("Restore Original Permissions"), - () => { - this.get_standard_permissions((data) => { - this.reset_std_permissions(data); - }); + this.page.set_secondary_action(__("Restore Original Permissions"), () => { + this.get_standard_permissions((data) => { + this.reset_std_permissions(data); }); + }); } get_perm(role) { @@ -455,7 +502,9 @@ frappe.PermissionEngine = class PermissionEngine { } get_link_fields(doctype) { - return frappe.get_children("DocType", doctype, "fields", - { fieldtype: "Link", options: ["not in", ["User", '[Select]']] }); + return frappe.get_children("DocType", doctype, "fields", { + fieldtype: "Link", + options: ["not in", ["User", "[Select]"]], + }); } }; diff --git a/frappe/core/page/permission_manager/permission_manager.py b/frappe/core/page/permission_manager/permission_manager.py index ad12e0fd4c..46c9e0aca2 100644 --- a/frappe/core/page/permission_manager/permission_manager.py +++ b/frappe/core/page/permission_manager/permission_manager.py @@ -1,6 +1,7 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE + import frappe import frappe.defaults from frappe import _ @@ -8,6 +9,7 @@ from frappe.core.doctype.doctype.doctype import ( clear_permissions_cache, validate_permissions_for_doctype, ) +from frappe.exceptions import DoesNotExistError from frappe.modules.import_file import get_file_path, read_doc_from_file from frappe.permissions import ( add_permission, @@ -68,17 +70,19 @@ def get_roles_and_doctypes(): @frappe.whitelist() -def get_permissions(doctype=None, role=None): +def get_permissions(doctype: str | None = None, role: str | None = None): frappe.only_for("System Manager") + if role: out = get_all_perms(role) if doctype: out = [p for p in out if p.parent == doctype] + else: - filters = dict(parent=doctype) + filters = {"parent": doctype} if frappe.session.user != "Administrator": - custom_roles = frappe.get_all("Role", filters={"is_custom": 1}) - filters["role"] = ["not in", [row.name for row in custom_roles]] + custom_roles = frappe.get_all("Role", filters={"is_custom": 1}, pluck="name") + filters["role"] = ["not in", custom_roles] out = frappe.get_all("Custom DocPerm", fields="*", filters=filters, order_by="permlevel") if not out: @@ -86,11 +90,15 @@ def get_permissions(doctype=None, role=None): linked_doctypes = {} for d in out: - if not d.parent in linked_doctypes: - linked_doctypes[d.parent] = get_linked_doctypes(d.parent) + if d.parent not in linked_doctypes: + try: + linked_doctypes[d.parent] = get_linked_doctypes(d.parent) + except DoesNotExistError: + # exclude & continue if linked doctype is not found + frappe.clear_last_message() + continue d.linked_doctypes = linked_doctypes[d.parent] - meta = frappe.get_meta(d.parent) - if meta: + if meta := frappe.get_meta(d.parent): d.is_submittable = meta.is_submittable d.in_create = meta.in_create diff --git a/frappe/core/page/recorder/recorder.js b/frappe/core/page/recorder/recorder.js index f1f74daf71..83b8d1a636 100644 --- a/frappe/core/page/recorder/recorder.js +++ b/frappe/core/page/recorder/recorder.js @@ -1,28 +1,28 @@ -frappe.pages['recorder'].on_page_load = function(wrapper) { +frappe.pages["recorder"].on_page_load = function (wrapper) { frappe.ui.make_app_page({ parent: wrapper, - title: __('Recorder'), + title: __("Recorder"), single_column: true, - card_layout: true + card_layout: true, }); frappe.recorder = new Recorder(wrapper); - $(wrapper).bind('show', function() { + $(wrapper).bind("show", function () { frappe.recorder.show(); }); - frappe.require('recorder.bundle.js'); + frappe.require("recorder.bundle.js"); }; class Recorder { constructor(wrapper) { this.wrapper = $(wrapper); - this.container = this.wrapper.find('.layout-main-section'); + this.container = this.wrapper.find(".layout-main-section"); this.container.append($('
    ')); } show() { if (!this.view || this.view.$route.name == "recorder-detail") return; - this.view.$router.replace({name: "recorder-detail"}); + this.view.$router.replace({ name: "recorder-detail" }); } } diff --git a/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.js b/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.js index 195f25f533..f840a49c92 100644 --- a/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.js +++ b/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.js @@ -2,33 +2,33 @@ // MIT License. See license.txt frappe.query_reports["Permitted Documents For User"] = { - "filters": [ + filters: [ { - "fieldname": "user", - "label": __("User"), - "fieldtype": "Link", - "options": "User", - "reqd": 1 + fieldname: "user", + label: __("User"), + fieldtype: "Link", + options: "User", + reqd: 1, }, { - "fieldname": "doctype", - "label": __("DocType"), - "fieldtype": "Link", - "options": "DocType", - "reqd": 1, - "get_query": function () { + fieldname: "doctype", + label: __("DocType"), + fieldtype: "Link", + options: "DocType", + reqd: 1, + get_query: function () { return { - "query": "frappe.core.report.permitted_documents_for_user.permitted_documents_for_user.query_doctypes", - "filters": { - "user": frappe.query_report.get_filter_value('user') - } - } - } + query: "frappe.core.report.permitted_documents_for_user.permitted_documents_for_user.query_doctypes", + filters: { + user: frappe.query_report.get_filter_value("user"), + }, + }; + }, }, { - "fieldname": "show_permissions", - "label": __("Show Permissions"), - "fieldtype": "Check" - } - ] -} + fieldname: "show_permissions", + label: __("Show Permissions"), + fieldtype: "Check", + }, + ], +}; diff --git a/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.py b/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.py index 32d8bbe18f..a7eff77ed0 100644 --- a/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.py +++ b/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.py @@ -3,21 +3,19 @@ import frappe import frappe.utils.user -from frappe import _, throw from frappe.model import data_fieldtypes -from frappe.permissions import check_admin_or_system_manager, rights +from frappe.permissions import rights def execute(filters=None): + frappe.only_for("System Manager") + user, doctype, show_permissions = ( filters.get("user"), filters.get("doctype"), filters.get("show_permissions"), ) - if not validate(user, doctype): - return [], [] - columns, fields = get_columns_and_fields(doctype) data = frappe.get_list(doctype, fields=fields, as_list=True, user=user) @@ -31,19 +29,13 @@ def execute(filters=None): return columns, data -def validate(user, doctype): - # check if current user is System Manager - check_admin_or_system_manager() - return user and doctype - - def get_columns_and_fields(doctype): - columns = ["Name:Link/{}:200".format(doctype)] + columns = [f"Name:Link/{doctype}:200"] fields = ["`name`"] for df in frappe.get_meta(doctype).fields: if df.in_list_view and df.fieldtype in data_fieldtypes: - fields.append("`{0}`".format(df.fieldname)) - fieldtype = "Link/{}".format(df.options) if df.fieldtype == "Link" else df.fieldtype + fields.append(f"`{df.fieldname}`") + fieldtype = f"Link/{df.options}" if df.fieldtype == "Link" else df.fieldtype columns.append( "{label}:{fieldtype}:{width}".format( label=df.label, fieldtype=fieldtype, width=df.width or 100 diff --git a/frappe/core/report/transaction_log_report/transaction_log_report.js b/frappe/core/report/transaction_log_report/transaction_log_report.js index 54ecf3fcf1..3c7261306d 100644 --- a/frappe/core/report/transaction_log_report/transaction_log_report.js +++ b/frappe/core/report/transaction_log_report/transaction_log_report.js @@ -3,9 +3,9 @@ /* eslint-disable */ frappe.query_reports["Transaction Log Report"] = { - onload: function(query_report) { - query_report.add_make_chart_button = function() { + onload: function (query_report) { + query_report.add_make_chart_button = function () { // }; - } -} + }, +}; diff --git a/frappe/core/utils.py b/frappe/core/utils.py index 8581f30f89..b445257b7d 100644 --- a/frappe/core/utils.py +++ b/frappe/core/utils.py @@ -1,6 +1,8 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +from markdownify import markdownify as md + import frappe @@ -86,3 +88,8 @@ def ljust_list(_list, length, fill_word=None): _list.extend([fill_word] * fill_length) return _list + + +def html2text(html, strip_links=False, wrap=True): + strip = ["a"] if strip_links else None + return md(html, heading_style="ATX", strip=strip, wrap=wrap) diff --git a/frappe/core/web_form/edit_profile/edit_profile.js b/frappe/core/web_form/edit_profile/edit_profile.js index 699703c579..8f56ebb353 100644 --- a/frappe/core/web_form/edit_profile/edit_profile.js +++ b/frappe/core/web_form/edit_profile/edit_profile.js @@ -1,3 +1,3 @@ -frappe.ready(function() { +frappe.ready(function () { // bind events here -}) \ No newline at end of file +}); diff --git a/frappe/core/web_form/edit_profile/edit_profile.json b/frappe/core/web_form/edit_profile/edit_profile.json index c04e705820..8abb2164f9 100644 --- a/frappe/core/web_form/edit_profile/edit_profile.json +++ b/frappe/core/web_form/edit_profile/edit_profile.json @@ -1,13 +1,10 @@ { - "accept_payment": 0, "allow_comments": 0, "allow_delete": 0, "allow_edit": 1, "allow_incomplete": 0, "allow_multiple": 0, "allow_print": 0, - "amount": 0.0, - "amount_based_on_field": 0, "apply_document_permissions": 0, "breadcrumbs": "[{\"title\": _(\"My Account\"), \"route\": \"me\"}]", "creation": "2016-09-19 05:16:59.242754", @@ -18,9 +15,10 @@ "introduction_text": "", "is_multi_step_form": 0, "is_standard": 1, + "list_columns": [], "login_required": 1, "max_attachment_size": 0, - "modified": "2022-03-22 15:00:43.456738", + "modified": "2022-07-18 16:51:19.796411", "modified_by": "Administrator", "module": "Core", "name": "edit-profile", @@ -29,9 +27,8 @@ "route": "update-profile", "route_to_success_link": 0, "show_attachments": 0, - "show_in_grid": 0, + "show_list": 0, "show_sidebar": 0, - "sidebar_items": [], "success_message": "Profile updated successfully.", "success_url": "/me", "title": "Update Profile", diff --git a/frappe/core/web_form/edit_profile/edit_profile.py b/frappe/core/web_form/edit_profile/edit_profile.py index 80b7b87352..02e3e93333 100644 --- a/frappe/core/web_form/edit_profile/edit_profile.py +++ b/frappe/core/web_form/edit_profile/edit_profile.py @@ -1,6 +1,3 @@ -import frappe - - def get_context(context): # do your magic here pass diff --git a/frappe/custom/doctype/client_script/client_script.js b/frappe/custom/doctype/client_script/client_script.js index 18786c62cf..67bb0083c8 100644 --- a/frappe/custom/doctype/client_script/client_script.js +++ b/frappe/custom/doctype/client_script/client_script.js @@ -1,46 +1,49 @@ // Copyright (c) 2016, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Client Script', { +frappe.ui.form.on("Client Script", { setup(frm) { frm.get_field("sample").html(SAMPLE_HTML); }, refresh(frm) { if (frm.doc.dt && frm.doc.script) { - frm.add_custom_button(__('Go to {0}', [frm.doc.dt]), - () => frappe.set_route('List', frm.doc.dt, 'List')); + frm.add_custom_button(__("Go to {0}", [frm.doc.dt]), () => + frappe.set_route("List", frm.doc.dt, "List") + ); } - if (frm.doc.view == 'Form') { - frm.add_custom_button(__('Add script for Child Table'), () => { + if (frm.doc.view == "Form") { + frm.add_custom_button(__("Add script for Child Table"), () => { frappe.model.with_doctype(frm.doc.dt, () => { - const child_tables = frappe.meta.get_docfields(frm.doc.dt, null, { - fieldtype: 'Table' - }).map(df => df.options); + const child_tables = frappe.meta + .get_docfields(frm.doc.dt, null, { + fieldtype: "Table", + }) + .map((df) => df.options); const d = new frappe.ui.Dialog({ - title: __('Select Child Table'), + title: __("Select Child Table"), fields: [ { - label: __('Select Child Table'), - fieldtype: 'Link', - fieldname: 'cdt', - options: 'DocType', + label: __("Select Child Table"), + fieldtype: "Link", + fieldname: "cdt", + options: "DocType", get_query: () => { return { filters: { istable: 1, - name: ['in', child_tables] - } + name: ["in", child_tables], + }, }; - } - } + }, + }, ], primary_action: ({ cdt }) => { - cdt = d.get_field('cdt').value; + cdt = d.get_field("cdt").value; frm.events.add_script_for_doctype(frm, cdt); d.hide(); - } + }, }); d.show(); @@ -48,39 +51,39 @@ frappe.ui.form.on('Client Script', { }); if (!frm.is_new()) { - frm.add_custom_button(__('Compare Versions'), () => { + frm.add_custom_button(__("Compare Versions"), () => { new frappe.ui.DiffView("Client Script", "script", frm.doc.name); }); } } - frm.set_query('dt', { + frm.set_query("dt", { filters: { - istable: 0 - } + istable: 0, + }, }); }, dt(frm) { - frm.toggle_display('view', !frappe.boot.single_types.includes(frm.doc.dt)); + frm.toggle_display("view", !frappe.boot.single_types.includes(frm.doc.dt)); if (!frm.doc.script) { frm.events.add_script_for_doctype(frm, frm.doc.dt); } if (frm.doc.script && !frm.doc.script.includes(frm.doc.dt)) { - frm.doc.script = ''; + frm.doc.script = ""; frm.events.add_script_for_doctype(frm, frm.doc.dt); } }, view(frm) { - let has_form_boilerplate = frm.doc.script.includes('frappe.ui.form.on') - if (frm.doc.view === 'List' && has_form_boilerplate) { - frm.set_value('script', ''); + let has_form_boilerplate = frm.doc.script.includes("frappe.ui.form.on"); + if (frm.doc.view === "List" && has_form_boilerplate) { + frm.set_value("script", ""); } - if (frm.doc.view === 'Form' && !has_form_boilerplate) { - frm.trigger('dt'); + if (frm.doc.view === "Form" && !has_form_boilerplate) { + frm.trigger("dt"); } }, @@ -93,12 +96,12 @@ frappe.ui.form.on('${doctype}', { } }) `.trim(); - let script = (frm.doc.script || ''); + let script = frm.doc.script || ""; if (script) { - script += '\n\n'; + script += "\n\n"; } - frm.set_value('script', script + boilerplate); - } + frm.set_value("script", script + boilerplate); + }, }); const SAMPLE_HTML = `

    Client Script Help

    diff --git a/frappe/custom/doctype/client_script/client_script.json b/frappe/custom/doctype/client_script/client_script.json index eca84b4dec..1db4dfe160 100644 --- a/frappe/custom/doctype/client_script/client_script.json +++ b/frappe/custom/doctype/client_script/client_script.json @@ -1,6 +1,7 @@ { "actions": [], "allow_import": 1, + "autoname": "Prompt", "creation": "2013-01-10 16:34:01", "description": "Adds a custom client script to a DocType", "doctype": "DocType", @@ -52,6 +53,7 @@ "default": "Form", "fieldname": "view", "fieldtype": "Select", + "in_list_view": 1, "label": "Apply To", "options": "List\nForm", "set_only_once": 1 @@ -75,10 +77,11 @@ "idx": 1, "index_web_pages_for_search": 1, "links": [], - "modified": "2022-02-18 00:43:33.941466", + "modified": "2022-04-12 12:48:15.717985", "modified_by": "Administrator", "module": "Custom", "name": "Client Script", + "naming_rule": "Set by user", "owner": "Administrator", "permissions": [ { diff --git a/frappe/custom/doctype/client_script/client_script.py b/frappe/custom/doctype/client_script/client_script.py index 3039e0a4a5..e18ad4be5d 100644 --- a/frappe/custom/doctype/client_script/client_script.py +++ b/frappe/custom/doctype/client_script/client_script.py @@ -1,25 +1,10 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE import frappe -from frappe import _ from frappe.model.document import Document class ClientScript(Document): - def autoname(self): - self.name = f"{self.dt}-{self.view}" - - def validate(self): - if not self.is_new(): - return - - exists = frappe.db.exists("Client Script", {"dt": self.dt, "view": self.view}) - if exists: - frappe.throw( - _("Client Script for {0} {1} already exists").format(frappe.bold(self.dt), self.view), - frappe.DuplicateEntryError, - ) - def on_update(self): frappe.clear_cache(doctype=self.dt) diff --git a/frappe/custom/doctype/client_script/test_client_script.py b/frappe/custom/doctype/client_script/test_client_script.py index 2538fdf515..c93df04c98 100644 --- a/frappe/custom/doctype/client_script/test_client_script.py +++ b/frappe/custom/doctype/client_script/test_client_script.py @@ -1,10 +1,7 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE import unittest -import frappe - # test_records = frappe.get_test_records('Client Script') diff --git a/frappe/custom/doctype/client_script/ui_test_client_script.js b/frappe/custom/doctype/client_script/ui_test_client_script.js new file mode 100644 index 0000000000..0d202d697c --- /dev/null +++ b/frappe/custom/doctype/client_script/ui_test_client_script.js @@ -0,0 +1,98 @@ +context("Client Script", () => { + before(() => { + cy.login(); + cy.visit("/app"); + }); + + it("should run form script in doctype form", () => { + cy.insert_doc( + "Client Script", + { + name: "Todo form script", + dt: "ToDo", + view: "Form", + enabled: 1, + script: `console.log('todo form script')`, + }, + true + ); + cy.visit("/app/todo/new", { + onBeforeLoad(win) { + cy.spy(win.console, "log").as("consoleLog"); + }, + }); + cy.get("@consoleLog").should("be.calledWith", "todo form script"); + }); + + it("should run list script in doctype list view", () => { + cy.insert_doc( + "Client Script", + { + name: "Todo list script", + dt: "ToDo", + view: "List", + enabled: 1, + script: `console.log('todo list script')`, + }, + true + ); + cy.visit("/app/todo", { + onBeforeLoad(win) { + cy.spy(win.console, "log").as("consoleLog"); + }, + }); + cy.get("@consoleLog").should("be.calledWith", "todo list script"); + }); + + it("should not run disabled scripts", () => { + cy.insert_doc( + "Client Script", + { + name: "Todo disabled list", + dt: "ToDo", + view: "List", + enabled: 0, + script: `console.log('todo disabled script')`, + }, + true + ); + cy.visit("/app/todo", { + onBeforeLoad(win) { + cy.spy(win.console, "log").as("consoleLog"); + }, + }); + cy.get("@consoleLog").should("not.be.calledWith", "todo disabled script"); + }); + + it("should run multiple scripts", () => { + cy.insert_doc( + "Client Script", + { + name: "Todo form script 1", + dt: "ToDo", + view: "Form", + enabled: 1, + script: `console.log('todo form script 1')`, + }, + true + ); + cy.insert_doc( + "Client Script", + { + name: "Todo form script 2", + dt: "ToDo", + view: "Form", + enabled: 1, + script: `console.log('todo form script 2')`, + }, + true + ); + cy.visit("/app/todo/new", { + onBeforeLoad(win) { + cy.spy(win.console, "log").as("consoleLog"); + }, + }); + cy.get("@consoleLog").should("be.calledWith", "todo form script 1"); + cy.get("@consoleLog").should("be.calledWith", "todo form script 2"); + }); +}); diff --git a/frappe/custom/doctype/custom_field/custom_field.js b/frappe/custom/doctype/custom_field/custom_field.js index c59fabeaa6..fba19ca45e 100644 --- a/frappe/custom/doctype/custom_field/custom_field.js +++ b/frappe/custom/doctype/custom_field/custom_field.js @@ -4,88 +4,99 @@ // Refresh // -------- -frappe.ui.form.on('Custom Field', { - setup: function(frm) { - frm.set_query('dt', function(doc) { +frappe.ui.form.on("Custom Field", { + setup: function (frm) { + frm.set_query("dt", function (doc) { var filters = [ - ['DocType', 'issingle', '=', 0], - ['DocType', 'custom', '=', 0], - ['DocType', 'name', 'not in', frappe.model.core_doctypes_list], - ['DocType', 'restrict_to_domain', 'in', frappe.boot.active_domains] + ["DocType", "issingle", "=", 0], + ["DocType", "custom", "=", 0], + ["DocType", "name", "not in", frappe.model.core_doctypes_list], + ["DocType", "restrict_to_domain", "in", frappe.boot.active_domains], ]; - if(frappe.session.user!=="Administrator") { - filters.push(['DocType', 'module', 'not in', ['Core', 'Custom']]) + if (frappe.session.user !== "Administrator") { + filters.push(["DocType", "module", "not in", ["Core", "Custom"]]); } return { - "filters": filters - } + filters: filters, + }; }); }, - refresh: function(frm) { - frm.toggle_enable('dt', frm.doc.__islocal); - frm.trigger('dt'); - frm.toggle_reqd('label', !frm.doc.fieldname); + refresh: function (frm) { + frm.toggle_enable("dt", frm.doc.__islocal); + frm.trigger("dt"); + frm.toggle_reqd("label", !frm.doc.fieldname); }, - dt: function(frm) { - if(!frm.doc.dt) { - set_field_options('insert_after', ''); + dt: function (frm) { + if (!frm.doc.dt) { + set_field_options("insert_after", ""); return; } var insert_after = frm.doc.insert_after || null; return frappe.call({ - method: 'frappe.custom.doctype.custom_field.custom_field.get_fields_label', + method: "frappe.custom.doctype.custom_field.custom_field.get_fields_label", args: { doctype: frm.doc.dt, fieldname: frm.doc.fieldname }, - callback: function(r) { - if(r) { - if(r._server_messages && r._server_messages.length) { + callback: function (r) { + if (r) { + if (r._server_messages && r._server_messages.length) { frm.set_value("dt", ""); } else { - set_field_options('insert_after', r.message); - var fieldnames = $.map(r.message, function(v) { return v.value; }); + set_field_options("insert_after", r.message); + var fieldnames = $.map(r.message, function (v) { + return v.value; + }); - if(insert_after==null || !in_list(fieldnames, insert_after)) { + if (insert_after == null || !in_list(fieldnames, insert_after)) { insert_after = fieldnames[-1]; } - frm.set_value('insert_after', insert_after); + frm.set_value("insert_after", insert_after); } } - } + }, }); - }, - label: function(frm) { - if(frm.doc.label && frappe.utils.has_special_chars(frm.doc.label)) { - frm.fields_dict['label_help'].disp_area.innerHTML = - ''+__('Special Characters are not allowed')+''; - frm.set_value('label', ''); + label: function (frm) { + if (frm.doc.label && frappe.utils.has_special_chars(frm.doc.label)) { + frm.fields_dict["label_help"].disp_area.innerHTML = + '' + __("Special Characters are not allowed") + ""; + frm.set_value("label", ""); } else { - frm.fields_dict['label_help'].disp_area.innerHTML = ''; + frm.fields_dict["label_help"].disp_area.innerHTML = ""; } }, - fieldtype: function(frm) { - if(frm.doc.fieldtype == 'Link') { - frm.fields_dict['options_help'].disp_area.innerHTML = - __('Name of the Document Type (DocType) you want this field to be linked to. e.g. Customer'); - } else if(frm.doc.fieldtype == 'Select') { - frm.fields_dict['options_help'].disp_area.innerHTML = - __('Options for select. Each option on a new line.')+' '+__('e.g.:')+'
    '+__('Option 1')+'
    '+__('Option 2')+'
    '+__('Option 3')+'
    '; - } else if(frm.doc.fieldtype == 'Dynamic Link') { - frm.fields_dict['options_help'].disp_area.innerHTML = - __('Fieldname which will be the DocType for this link field.'); + fieldtype: function (frm) { + if (frm.doc.fieldtype == "Link") { + frm.fields_dict["options_help"].disp_area.innerHTML = __( + "Name of the Document Type (DocType) you want this field to be linked to. e.g. Customer" + ); + } else if (frm.doc.fieldtype == "Select") { + frm.fields_dict["options_help"].disp_area.innerHTML = + __("Options for select. Each option on a new line.") + + " " + + __("e.g.:") + + "
    " + + __("Option 1") + + "
    " + + __("Option 2") + + "
    " + + __("Option 3") + + "
    "; + } else if (frm.doc.fieldtype == "Dynamic Link") { + frm.fields_dict["options_help"].disp_area.innerHTML = __( + "Fieldname which will be the DocType for this link field." + ); } else { - frm.fields_dict['options_help'].disp_area.innerHTML = ''; + frm.fields_dict["options_help"].disp_area.innerHTML = ""; } - } + }, }); - -frappe.utils.has_special_chars = function(t) { - var iChars = "!@#$%^&*()+=-[]\\\';,./{}|\":<>?"; +frappe.utils.has_special_chars = function (t) { + var iChars = "!@#$%^&*()+=-[]\\';,./{}|\":<>?"; for (var i = 0; i < t.length; i++) { if (iChars.indexOf(t.charAt(i)) != -1) { return true; } } return false; -} +}; diff --git a/frappe/custom/doctype/custom_field/custom_field.json b/frappe/custom/doctype/custom_field/custom_field.json index add6cbb828..63be70c644 100644 --- a/frappe/custom/doctype/custom_field/custom_field.json +++ b/frappe/custom/doctype/custom_field/custom_field.json @@ -67,7 +67,8 @@ "fieldtype": "Link", "in_filter": 1, "in_list_view": 1, - "label": "Document", + "in_standard_filter": 1, + "label": "DocType", "oldfieldname": "dt", "oldfieldtype": "Link", "options": "DocType", @@ -94,6 +95,7 @@ "fieldname": "fieldname", "fieldtype": "Data", "in_list_view": 1, + "in_standard_filter": 1, "label": "Fieldname", "no_copy": 1, "oldfieldname": "fieldname", @@ -123,7 +125,7 @@ "label": "Field Type", "oldfieldname": "fieldtype", "oldfieldtype": "Select", - "options": "Autocomplete\nAttach\nAttach Image\nBarcode\nButton\nCheck\nCode\nColor\nColumn Break\nCurrency\nData\nDate\nDatetime\nDuration\nDynamic Link\nFloat\nFold\nGeolocation\nHeading\nHTML\nHTML Editor\nIcon\nImage\nInt\nJSON\nLink\nLong Text\nMarkdown Editor\nPassword\nPercent\nRead Only\nRating\nSection Break\nSelect\nSignature\nSmall Text\nTab Break\nTable\nTable MultiSelect\nText\nText Editor\nTime", + "options": "Autocomplete\nAttach\nAttach Image\nBarcode\nButton\nCheck\nCode\nColor\nColumn Break\nCurrency\nData\nDate\nDatetime\nDuration\nDynamic Link\nFloat\nFold\nGeolocation\nHeading\nHTML\nHTML Editor\nIcon\nImage\nInt\nJSON\nLink\nLong Text\nMarkdown Editor\nPassword\nPercent\nPhone\nRead Only\nRating\nSection Break\nSelect\nSignature\nSmall Text\nTab Break\nTable\nTable MultiSelect\nText\nText Editor\nTime", "reqd": 1 }, { @@ -439,7 +441,7 @@ "idx": 1, "index_web_pages_for_search": 1, "links": [], - "modified": "2022-04-14 09:46:58.849765", + "modified": "2022-06-13 06:39:03.319667", "modified_by": "Administrator", "module": "Custom", "name": "Custom Field", diff --git a/frappe/custom/doctype/custom_field/test_custom_field.py b/frappe/custom/doctype/custom_field/test_custom_field.py index 519ea7f2b4..34223315c5 100644 --- a/frappe/custom/doctype/custom_field/test_custom_field.py +++ b/frappe/custom/doctype/custom_field/test_custom_field.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE diff --git a/frappe/custom/doctype/customize_form/customize_form.js b/frappe/custom/doctype/customize_form/customize_form.js index 4c2d207df9..a35db2ca18 100644 --- a/frappe/custom/doctype/customize_form/customize_form.js +++ b/frappe/custom/doctype/customize_form/customize_form.js @@ -4,7 +4,7 @@ frappe.provide("frappe.customize_form"); frappe.ui.form.on("Customize Form", { - setup: function(frm) { + setup: function (frm) { // save the last setting if refreshing window.addEventListener("beforeunload", () => { if (frm.doc.doc_type && frm.doc.doc_type != "undefined") { @@ -13,68 +13,59 @@ frappe.ui.form.on("Customize Form", { }); }, - onload: function(frm) { - frm.set_query("doc_type", function() { + onload: function (frm) { + frm.set_query("doc_type", function () { return { - translate_values: false, filters: [ ["DocType", "issingle", "=", 0], ["DocType", "custom", "=", 0], - [ - "DocType", - "name", - "not in", - frappe.model.core_doctypes_list - ], - [ - "DocType", - "restrict_to_domain", - "in", - frappe.boot.active_domains - ] - ] + ["DocType", "name", "not in", frappe.model.core_doctypes_list], + ["DocType", "restrict_to_domain", "in", frappe.boot.active_domains], + ], }; }); - frm.set_query("default_print_format", function() { + frm.set_query("default_print_format", function () { return { filters: { print_format_type: ["!=", "JS"], - doc_type: ["=", frm.doc.doc_type] - } + doc_type: ["=", frm.doc.doc_type], + }, }; }); - $(frm.wrapper).on("grid-row-render", function(e, grid_row) { + $(frm.wrapper).on("grid-row-render", function (e, grid_row) { if (grid_row.doc && grid_row.doc.fieldtype == "Section Break") { $(grid_row.row).css({ "font-weight": "bold" }); } grid_row.row.removeClass("highlight"); - if (grid_row.doc.is_custom_field && - !grid_row.row.hasClass('highlight') && - !grid_row.doc.is_system_generated) { + if ( + grid_row.doc.is_custom_field && + !grid_row.row.hasClass("highlight") && + !grid_row.doc.is_system_generated + ) { grid_row.row.addClass("highlight"); } }); - $(frm.wrapper).on("grid-make-sortable", function(e, frm) { + $(frm.wrapper).on("grid-make-sortable", function (e, frm) { frm.trigger("setup_sortable"); }); - $(frm.wrapper).on("grid-move-row", function(e, frm) { + $(frm.wrapper).on("grid-move-row", function (e, frm) { frm.trigger("setup_sortable"); }); }, - doc_type: function(frm) { + doc_type: function (frm) { if (frm.doc.doc_type) { return frm.call({ method: "fetch_to_customize", doc: frm.doc, freeze: true, - callback: function(r) { + callback: function (r) { if (r) { if (r._server_messages && r._server_messages.length) { frm.set_value("doc_type", ""); @@ -84,15 +75,15 @@ frappe.ui.form.on("Customize Form", { } } localStorage["customize_doctype"] = frm.doc.doc_type; - } + }, }); } else { frm.refresh(); } }, - setup_sortable: function(frm) { - frm.doc.fields.forEach(function(f, i) { + setup_sortable: function (frm) { + frm.doc.fields.forEach(function (f, i) { if (!f.is_custom_field) { f._sortable = false; } @@ -100,7 +91,7 @@ frappe.ui.form.on("Customize Form", { if (f.fieldtype == "Table") { frm.add_custom_button( f.options, - function() { + function () { frm.set_value("doc_type", f.options); }, __("Customize Child Table") @@ -110,17 +101,17 @@ frappe.ui.form.on("Customize Form", { frm.fields_dict.fields.grid.refresh(); }, - refresh: function(frm) { + refresh: function (frm) { frm.disable_save(true); frm.page.clear_icons(); if (frm.doc.doc_type) { - frm.page.set_title(__('Customize Form - {0}', [frm.doc.doc_type])); + frm.page.set_title(__("Customize Form - {0}", [frm.doc.doc_type])); frappe.customize_form.set_primary_action(frm); frm.add_custom_button( __("Go to {0} List", [__(frm.doc.doc_type)]), - function() { + function () { frappe.set_route("List", frm.doc.doc_type); }, __("Actions") @@ -128,7 +119,7 @@ frappe.ui.form.on("Customize Form", { frm.add_custom_button( __("Reload"), - function() { + function () { frm.script_manager.trigger("doc_type"); }, __("Actions") @@ -136,22 +127,23 @@ frappe.ui.form.on("Customize Form", { frm.add_custom_button( __("Reset to defaults"), - function() { - frappe.customize_form.confirm( - __("Remove all customizations?"), - frm - ); + function () { + frappe.customize_form.confirm(__("Remove all customizations?"), frm); }, __("Actions") ); frm.add_custom_button( __("Set Permissions"), - function() { + function () { frappe.set_route("permission-manager", frm.doc.doc_type); }, __("Actions") ); + + const is_autoname_autoincrement = frm.doc.autoname === "autoincrement"; + frm.set_df_property("naming_rule", "hidden", is_autoname_autoincrement); + frm.set_df_property("autoname", "read_only", is_autoname_autoincrement); } frm.events.setup_export(frm); @@ -178,37 +170,37 @@ frappe.ui.form.on("Customize Form", { if (frappe.boot.developer_mode) { frm.add_custom_button( __("Export Customizations"), - function() { + function () { frappe.prompt( [ { fieldtype: "Link", fieldname: "module", options: "Module Def", - label: __("Module to Export") + label: __("Module to Export"), }, { fieldtype: "Check", fieldname: "sync_on_migrate", label: __("Sync on Migrate"), - default: 1 + default: 1, }, { fieldtype: "Check", fieldname: "with_permissions", label: __("Export Custom Permissions"), - default: 1 - } + default: 1, + }, ], - function(data) { + function (data) { frappe.call({ method: "frappe.modules.utils.export_customizations", args: { doctype: frm.doc.doc_type, module: data.module, sync_on_migrate: data.sync_on_migrate, - with_permissions: data.with_permissions - } + with_permissions: data.with_permissions, + }, }); }, __("Select Module") @@ -222,127 +214,131 @@ frappe.ui.form.on("Customize Form", { setup_sort_order(frm) { // sort order select if (frm.doc.doc_type) { - var fields = $.map(frm.doc.fields, function(df) { - return frappe.model.is_value_type(df.fieldtype) - ? df.fieldname - : null; + var fields = $.map(frm.doc.fields, function (df) { + return frappe.model.is_value_type(df.fieldtype) ? df.fieldname : null; }); fields = ["", "name", "modified"].concat(fields); frm.set_df_property("sort_field", "options", fields); } - } + }, }); // can't delete standard fields frappe.ui.form.on("Customize Form Field", { - before_fields_remove: function(frm, doctype, name) { + before_fields_remove: function (frm, doctype, name) { var row = frappe.get_doc(doctype, name); if (!(row.is_custom_field || row.__islocal)) { frappe.msgprint(__("Cannot delete standard field. You can hide it if you want")); throw "cannot delete standard field"; } }, - fields_add: function(frm, cdt, cdn) { + fields_add: function (frm, cdt, cdn) { var f = frappe.model.get_doc(cdt, cdn); f.is_system_generated = false; f.is_custom_field = true; - } + }, }); // can't delete standard links frappe.ui.form.on("DocType Link", { - before_links_remove: function(frm, doctype, name) { + before_links_remove: function (frm, doctype, name) { let row = frappe.get_doc(doctype, name); if (!(row.custom || row.__islocal)) { frappe.msgprint(__("Cannot delete standard link. You can hide it if you want")); throw "cannot delete standard link"; } }, - links_add: function(frm, cdt, cdn) { + links_add: function (frm, cdt, cdn) { let f = frappe.model.get_doc(cdt, cdn); f.custom = 1; - } + }, }); // can't delete standard actions frappe.ui.form.on("DocType Action", { - before_actions_remove: function(frm, doctype, name) { + before_actions_remove: function (frm, doctype, name) { let row = frappe.get_doc(doctype, name); if (!(row.custom || row.__islocal)) { frappe.msgprint(__("Cannot delete standard action. You can hide it if you want")); throw "cannot delete standard action"; } }, - actions_add: function(frm, cdt, cdn) { + actions_add: function (frm, cdt, cdn) { let f = frappe.model.get_doc(cdt, cdn); f.custom = 1; - } + }, }); // can't delete standard states frappe.ui.form.on("DocType State", { - before_states_remove: function(frm, doctype, name) { + before_states_remove: function (frm, doctype, name) { let row = frappe.get_doc(doctype, name); if (!(row.custom || row.__islocal)) { frappe.msgprint(__("Cannot delete standard document state.")); throw "cannot delete standard document state"; } }, - states_add: function(frm, cdt, cdn) { + states_add: function (frm, cdt, cdn) { let f = frappe.model.get_doc(cdt, cdn); f.custom = 1; - } + }, }); -frappe.customize_form.set_primary_action = function(frm) { - frm.page.set_primary_action(__("Update"), function() { +frappe.customize_form.set_primary_action = function (frm) { + frm.page.set_primary_action(__("Update"), function () { if (frm.doc.doc_type) { return frm.call({ doc: frm.doc, freeze: true, btn: frm.page.btn_primary, method: "save_customization", - callback: function(r) { + callback: function (r) { if (!r.exc) { frappe.customize_form.clear_locals_and_refresh(frm); frm.script_manager.trigger("doc_type"); } - } + }, }); } }); }; -frappe.customize_form.confirm = function(msg, frm) { +frappe.customize_form.confirm = function (msg, frm) { if (!frm.doc.doc_type) return; var d = new frappe.ui.Dialog({ - title: 'Reset To Defaults', + title: "Reset To Defaults", fields: [ - {fieldtype:"HTML", options:__("All customizations will be removed. Please confirm.")}, + { + fieldtype: "HTML", + options: __("All customizations will be removed. Please confirm."), + }, ], - primary_action: function() { + primary_action: function () { return frm.call({ doc: frm.doc, method: "reset_to_defaults", - callback: function(r) { + callback: function (r) { if (r.exc) { frappe.msgprint(r.exc); } else { d.hide(); - frappe.show_alert({message:__('Customizations Reset'), indicator:'green'}); + frappe.show_alert({ + message: __("Customizations Reset"), + indicator: "green", + }); frappe.customize_form.clear_locals_and_refresh(frm); } - } + }, }); - } + }, }); frappe.customize_form.confirm.dialog = d; d.show(); -} +}; -frappe.customize_form.clear_locals_and_refresh = function(frm) { +frappe.customize_form.clear_locals_and_refresh = function (frm) { delete frm.doc.__unsaved; // clear doctype from locals frappe.model.clear_doc("DocType", frm.doc.doc_type); @@ -350,4 +346,4 @@ frappe.customize_form.clear_locals_and_refresh = function(frm) { frm.refresh(); }; -extend_cscript(cur_frm.cscript, new frappe.model.DocTypeController({frm: cur_frm})); +extend_cscript(cur_frm.cscript, new frappe.model.DocTypeController({ frm: cur_frm })); diff --git a/frappe/custom/doctype/customize_form/customize_form.json b/frappe/custom/doctype/customize_form/customize_form.json index 1ee9d4a02a..0011f51af4 100644 --- a/frappe/custom/doctype/customize_form/customize_form.json +++ b/frappe/custom/doctype/customize_form/customize_form.json @@ -24,10 +24,12 @@ "fields_section_break", "fields", "naming_section", + "naming_rule", "autoname", "view_settings_section", "title_field", "show_title_field_in_link", + "translate_link_fields", "image_field", "default_print_format", "column_break_29", @@ -50,6 +52,13 @@ "sort_order" ], "fields": [ + { + "fieldname": "naming_rule", + "fieldtype": "Select", + "label": "Naming Rule", + "length": 40, + "options": "\nSet by user\nBy fieldname\nBy \"Naming Series\" field\nExpression\nExpression (old style)\nRandom\nBy script" + }, { "fieldname": "doc_type", "fieldtype": "Link", @@ -279,7 +288,7 @@ "label": "Naming" }, { - "description": "Naming Options:\n
    1. field:[fieldname] - By Field
    2. naming_series: - By Naming Series (field called naming_series must be present
    3. Prompt - Prompt user for a name
    4. [series] - Series by prefix (separated by a dot); for example PRE.#####
      5. \n
    5. format:EXAMPLE-{MM}morewords{fieldname1}-{fieldname2}-{#####} - Replace all braced words (fieldnames, date words (DD, MM, YY), series) with their value. Outside braces, any characters can be used.
    ", + "description": "Naming Options:\n
    1. field:[fieldname] - By Field
    2. naming_series: - By Naming Series (field called naming_series must be present)
    3. Prompt - Prompt user for a name
    4. [series] - Series by prefix (separated by a dot); for example PRE.#####
      5. \n
    5. format:EXAMPLE-{MM}morewords{fieldname1}-{fieldname2}-{#####} - Replace all braced words (fieldnames, date words (DD, MM, YY), series) with their value. Outside braces, any characters can be used.
    ", "fieldname": "autoname", "fieldtype": "Data", "label": "Auto Name" @@ -303,6 +312,12 @@ "fieldname": "show_title_field_in_link", "fieldtype": "Check", "label": "Show Title in Link Fields" + }, + { + "default": "0", + "fieldname": "translate_link_fields", + "fieldtype": "Check", + "label": "Translate Link Fields" } ], "hide_toolbar": 1, @@ -311,7 +326,7 @@ "index_web_pages_for_search": 1, "issingle": 1, "links": [], - "modified": "2022-01-07 16:07:06.196534", + "modified": "2022-05-13 15:36:16.772277", "modified_by": "Administrator", "module": "Custom", "name": "Customize Form", diff --git a/frappe/custom/doctype/customize_form/customize_form.py b/frappe/custom/doctype/customize_form/customize_form.py index b4ccb21167..dc625d1a58 100644 --- a/frappe/custom/doctype/customize_form/customize_form.py +++ b/frappe/custom/doctype/customize_form/customize_form.py @@ -12,6 +12,7 @@ import frappe.translate from frappe import _ from frappe.core.doctype.doctype.doctype import ( check_email_append_to, + validate_autoincrement_autoname, validate_fields_for_doctype, validate_series, ) @@ -159,7 +160,9 @@ class CustomizeForm(Document): def save_customization(self): if not self.doc_type: return + validate_series(self, self.autoname, self.doc_type) + validate_autoincrement_autoname(self) self.flags.update_db = False self.flags.rebuild_doctype_for_global_search = False self.set_property_setters() @@ -327,7 +330,7 @@ class CustomizeForm(Document): We need to maintain the order of the link/actions if the user has shuffled them. So we create a new property (ex `links_order`) to keep a list of items. """ - property_name = "{}_order".format(fieldname) + property_name = f"{fieldname}_order" if has_custom: # save the order of the actions and links self.make_property_setter( @@ -371,6 +374,9 @@ class CustomizeForm(Document): d.insert() df.fieldname = d.fieldname + if df.get("in_global_search"): + self.flags.rebuild_doctype_for_global_search = True + def update_in_custom_field(self, df, i): meta = frappe.get_meta(self.doc_type) meta_df = meta.get("fields", {"fieldname": df.fieldname}) @@ -384,6 +390,8 @@ class CustomizeForm(Document): if df.get(prop) != custom_field.get(prop): if prop == "fieldtype": self.validate_fieldtype_change(df, meta_df[0].get(prop), df.get(prop)) + if prop == "in_global_search": + self.flags.rebuild_doctype_for_global_search = True custom_field.set(prop, df.get(prop)) changed = True @@ -521,7 +529,10 @@ class CustomizeForm(Document): """allow type change, if both old_type and new_type are in same field group. field groups are defined in ALLOWED_FIELDTYPE_CHANGE variables. """ - in_field_group = lambda group: (old_type in group) and (new_type in group) + + def in_field_group(group): + return (old_type in group) and (new_type in group) + return any(map(in_field_group, ALLOWED_FIELDTYPE_CHANGE)) @@ -571,8 +582,10 @@ doctype_properties = { "email_append_to": "Check", "subject_field": "Data", "sender_field": "Data", + "naming_rule": "Data", "autoname": "Data", "show_title_field_in_link": "Check", + "translate_link_fields": "Check", } docfield_properties = { @@ -596,6 +609,7 @@ docfield_properties = { "in_preview": "Check", "bold": "Check", "no_copy": "Check", + "ignore_xss_filter": "Check", "hidden": "Check", "collapsible": "Check", "collapsible_depends_on": "Data", diff --git a/frappe/custom/doctype/customize_form/test_customize_form.py b/frappe/custom/doctype/customize_form/test_customize_form.py index 5a1f629beb..b00f45f5d2 100644 --- a/frappe/custom/doctype/customize_form/test_customize_form.py +++ b/frappe/custom/doctype/customize_form/test_customize_form.py @@ -396,3 +396,10 @@ class TestCustomizeForm(unittest.TestCase): d.label = "" d.run_method("save_customization") self.assertEqual(d.label, "") + + def test_change_to_autoincrement_autoname(self): + d = self.get_customize_form("Event") + d.autoname = "autoincrement" + + with self.assertRaises(frappe.ValidationError): + d.run_method("save_customization") diff --git a/frappe/custom/doctype/customize_form_field/customize_form_field.json b/frappe/custom/doctype/customize_form_field/customize_form_field.json index cc446e321e..8fa054894f 100644 --- a/frappe/custom/doctype/customize_form_field/customize_form_field.json +++ b/frappe/custom/doctype/customize_form_field/customize_form_field.json @@ -46,6 +46,7 @@ "report_hide", "remember_last_selected_value", "hide_border", + "ignore_xss_filter", "property_depends_on_section", "mandatory_depends_on", "column_break_33", @@ -86,7 +87,7 @@ "label": "Type", "oldfieldname": "fieldtype", "oldfieldtype": "Select", - "options": "Autocomplete\nAttach\nAttach Image\nBarcode\nButton\nCheck\nCode\nColor\nColumn Break\nCurrency\nData\nDate\nDatetime\nDuration\nDynamic Link\nFloat\nFold\nGeolocation\nHeading\nHTML\nHTML Editor\nIcon\nImage\nInt\nLink\nLong Text\nMarkdown Editor\nPassword\nPercent\nRating\nRead Only\nSection Break\nSelect\nSignature\nSmall Text\nTab Break\nTable\nTable MultiSelect\nText\nText Editor\nTime", + "options": "Autocomplete\nAttach\nAttach Image\nBarcode\nButton\nCheck\nCode\nColor\nColumn Break\nCurrency\nData\nDate\nDatetime\nDuration\nDynamic Link\nFloat\nFold\nGeolocation\nHeading\nHTML\nHTML Editor\nIcon\nImage\nInt\nLink\nLong Text\nMarkdown Editor\nPassword\nPercent\nPhone\nRating\nRead Only\nSection Break\nSelect\nSignature\nSmall Text\nTab Break\nTable\nTable MultiSelect\nText\nText Editor\nTime", "reqd": 1, "search_index": 1 }, @@ -453,13 +454,20 @@ "hidden": 1, "label": "Is System Generated", "read_only": 1 + }, + { + "default": "0", + "description": "Don't encode HTML tags like <script> or just characters like < or >, as they could be intentionally used in this field", + "fieldname": "ignore_xss_filter", + "fieldtype": "Check", + "label": "Ignore XSS Filter" } ], "idx": 1, "index_web_pages_for_search": 1, "istable": 1, "links": [], - "modified": "2022-03-31 12:05:11.799654", + "modified": "2022-04-13 22:31:14.162661", "modified_by": "Administrator", "module": "Custom", "name": "Customize Form Field", @@ -469,4 +477,4 @@ "sort_field": "modified", "sort_order": "ASC", "states": [] -} \ No newline at end of file +} diff --git a/frappe/custom/doctype/customize_form_field/customize_form_field.py b/frappe/custom/doctype/customize_form_field/customize_form_field.py index 0e030ce812..468496ca7a 100644 --- a/frappe/custom/doctype/customize_form_field/customize_form_field.py +++ b/frappe/custom/doctype/customize_form_field/customize_form_field.py @@ -1,7 +1,6 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import frappe from frappe.model.document import Document diff --git a/frappe/custom/doctype/doctype_layout/doctype_layout.js b/frappe/custom/doctype/doctype_layout/doctype_layout.js index 533efea9b8..f91f04f762 100644 --- a/frappe/custom/doctype/doctype_layout/doctype_layout.js +++ b/frappe/custom/doctype/doctype_layout/doctype_layout.js @@ -1,30 +1,32 @@ // Copyright (c) 2020, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('DocType Layout', { - refresh: function(frm) { - frm.trigger('document_type'); +frappe.ui.form.on("DocType Layout", { + refresh: function (frm) { + frm.trigger("document_type"); frm.events.set_button(frm); }, document_type(frm) { - frm.set_fields_as_options('fields', frm.doc.document_type, null, [], 'fieldname').then(() => { - // child table empty? then show all fields as default - if (frm.doc.document_type) { - if (!(frm.doc.fields || []).length) { - for (let f of frappe.get_doc('DocType', frm.doc.document_type).fields) { - frm.add_child('fields', { fieldname: f.fieldname, label: f.label }); + frm.set_fields_as_options("fields", frm.doc.document_type, null, [], "fieldname").then( + () => { + // child table empty? then show all fields as default + if (frm.doc.document_type) { + if (!(frm.doc.fields || []).length) { + for (let f of frappe.get_doc("DocType", frm.doc.document_type).fields) { + frm.add_child("fields", { fieldname: f.fieldname, label: f.label }); + } } } } - }); + ); }, set_button(frm) { if (!frm.is_new()) { - frm.add_custom_button(__('Go to {0} List', [frm.doc.name]), () => { + frm.add_custom_button(__("Go to {0} List", [frm.doc.name]), () => { window.open(`/app/${frappe.router.slug(frm.doc.name)}`); }); } - } + }, }); diff --git a/frappe/custom/doctype/doctype_layout/doctype_layout.py b/frappe/custom/doctype/doctype_layout/doctype_layout.py index f5d37d6f60..ea8e9acc99 100644 --- a/frappe/custom/doctype/doctype_layout/doctype_layout.py +++ b/frappe/custom/doctype/doctype_layout/doctype_layout.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and contributors # License: MIT. See LICENSE diff --git a/frappe/custom/doctype/doctype_layout/test_doctype_layout.py b/frappe/custom/doctype/doctype_layout/test_doctype_layout.py index 1373b4a53a..0e64a9e727 100644 --- a/frappe/custom/doctype/doctype_layout/test_doctype_layout.py +++ b/frappe/custom/doctype/doctype_layout/test_doctype_layout.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and Contributors # License: MIT. See LICENSE # import frappe diff --git a/frappe/custom/doctype/doctype_layout_field/doctype_layout_field.py b/frappe/custom/doctype/doctype_layout_field/doctype_layout_field.py index 66fc111d32..f2b8c2b40b 100644 --- a/frappe/custom/doctype/doctype_layout_field/doctype_layout_field.py +++ b/frappe/custom/doctype/doctype_layout_field/doctype_layout_field.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and contributors # License: MIT. See LICENSE diff --git a/frappe/custom/doctype/property_setter/property_setter.js b/frappe/custom/doctype/property_setter/property_setter.js index bff5ad0e63..955e01c33e 100644 --- a/frappe/custom/doctype/property_setter/property_setter.js +++ b/frappe/custom/doctype/property_setter/property_setter.js @@ -1,10 +1,10 @@ // Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors // MIT License. See license.txt -frappe.ui.form.on('Property Setter', { - validate: function(frm) { - if(frm.doc.property_type=='Check' && !in_list(['0','1'], frm.doc.value)) { - frappe.throw(__('Value for a check field can be either 0 or 1')); +frappe.ui.form.on("Property Setter", { + validate: function (frm) { + if (frm.doc.property_type == "Check" && !in_list(["0", "1"], frm.doc.value)) { + frappe.throw(__("Value for a check field can be either 0 or 1")); } - } + }, }); diff --git a/frappe/custom/doctype/property_setter/test_property_setter.py b/frappe/custom/doctype/property_setter/test_property_setter.py index 5b877ab18c..1fa2d2cefb 100644 --- a/frappe/custom/doctype/property_setter/test_property_setter.py +++ b/frappe/custom/doctype/property_setter/test_property_setter.py @@ -1,10 +1,7 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE import unittest -import frappe - # test_records = frappe.get_test_records('Property Setter') diff --git a/frappe/custom/doctype/test_rename_new/test_rename_new.json b/frappe/custom/doctype/test_rename_new/test_rename_new.json deleted file mode 100644 index 0b089091a1..0000000000 --- a/frappe/custom/doctype/test_rename_new/test_rename_new.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "actions": [], - "creation": "2021-01-13 12:47:03.572640", - "doctype": "DocType", - "editable_grid": 1, - "engine": "InnoDB", - "field_order": [ - "random" - ], - "fields": [ - { - "fieldname": "random", - "fieldtype": "Data", - "label": "random" - } - ], - "index_web_pages_for_search": 1, - "links": [], - "modified": "2021-01-13 12:47:03.572640", - "modified_by": "Administrator", - "module": "Custom", - "name": "Test rename new", - "owner": "Administrator", - "permissions": [ - { - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "share": 1, - "write": 1 - } - ], - "route": "test-rename", - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1 -} \ No newline at end of file diff --git a/frappe/custom/doctype/test_rename_new/test_rename_new.py b/frappe/custom/doctype/test_rename_new/test_rename_new.py deleted file mode 100644 index e79cb60bbe..0000000000 --- a/frappe/custom/doctype/test_rename_new/test_rename_new.py +++ /dev/null @@ -1,10 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2021, Frappe Technologies and contributors -# License: MIT. See LICENSE - -# import frappe -from frappe.model.document import Document - - -class Testrenamenew(Document): - pass diff --git a/frappe/custom/doctype/test_rename_new/test_test_rename_new.py b/frappe/custom/doctype/test_rename_new/test_test_rename_new.py deleted file mode 100644 index 513a9286a3..0000000000 --- a/frappe/custom/doctype/test_rename_new/test_test_rename_new.py +++ /dev/null @@ -1,9 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2021, Frappe Technologies and Contributors -# License: MIT. See LICENSE -# import frappe -import unittest - - -class Testrenamenew(unittest.TestCase): - pass diff --git a/frappe/data_migration/doctype/data_migration_connector/connectors/__init__.py b/frappe/data_migration/doctype/data_migration_connector/connectors/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/data_migration/doctype/data_migration_connector/connectors/base.py b/frappe/data_migration/doctype/data_migration_connector/connectors/base.py deleted file mode 100644 index 7d2b320c59..0000000000 --- a/frappe/data_migration/doctype/data_migration_connector/connectors/base.py +++ /dev/null @@ -1,24 +0,0 @@ -from abc import ABCMeta, abstractmethod - -from frappe.utils.password import get_decrypted_password - - -class BaseConnection(metaclass=ABCMeta): - @abstractmethod - def get(self, remote_objectname, fields=None, filters=None, start=0, page_length=10): - pass - - @abstractmethod - def insert(self, doctype, doc): - pass - - @abstractmethod - def update(self, doctype, doc, migration_id): - pass - - @abstractmethod - def delete(self, doctype, migration_id): - pass - - def get_password(self): - return get_decrypted_password("Data Migration Connector", self.connector.name) diff --git a/frappe/data_migration/doctype/data_migration_connector/connectors/frappe_connection.py b/frappe/data_migration/doctype/data_migration_connector/connectors/frappe_connection.py deleted file mode 100644 index 8228529562..0000000000 --- a/frappe/data_migration/doctype/data_migration_connector/connectors/frappe_connection.py +++ /dev/null @@ -1,32 +0,0 @@ -import frappe -from frappe.frappeclient import FrappeClient - -from .base import BaseConnection - - -class FrappeConnection(BaseConnection): - def __init__(self, connector): - self.connector = connector - self.connection = FrappeClient( - self.connector.hostname, self.connector.username, self.get_password() - ) - self.name_field = "name" - - def insert(self, doctype, doc): - doc = frappe._dict(doc) - doc.doctype = doctype - return self.connection.insert(doc) - - def update(self, doctype, doc, migration_id): - doc = frappe._dict(doc) - doc.doctype = doctype - doc.name = migration_id - return self.connection.update(doc) - - def delete(self, doctype, migration_id): - return self.connection.delete(doctype, migration_id) - - def get(self, doctype, fields='"*"', filters=None, start=0, page_length=20): - return self.connection.get_list( - doctype, fields=fields, filters=filters, limit_start=start, limit_page_length=page_length - ) diff --git a/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.js b/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.js deleted file mode 100644 index 0898fcf4e7..0000000000 --- a/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.js +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright (c) 2017, Frappe Technologies and contributors -// For license information, please see license.txt - -frappe.ui.form.on('Data Migration Connector', { - onload(frm) { - if(frappe.boot.developer_mode) { - frm.add_custom_button(__('New Connection'), () => frm.events.new_connection(frm)); - } - }, - new_connection(frm) { - const d = new frappe.ui.Dialog({ - title: __('New Connection'), - fields: [ - { label: __('Module'), fieldtype: 'Link', options: 'Module Def', reqd: 1 }, - { label: __('Connection Name'), fieldtype: 'Data', description: 'For e.g: Shopify Connection', reqd: 1 }, - ], - primary_action_label: __('Create'), - primary_action: (values) => { - let { module, connection_name } = values; - - frm.events.create_new_connection(module, connection_name) - .then(r => { - if (r.message) { - const connector_name = connection_name - .replace('connection', 'Connector') - .replace('Connection', 'Connector') - .trim(); - - frm.set_value('connector_name', connector_name); - frm.set_value('connector_type', 'Custom'); - frm.set_value('python_module', r.message); - frm.save(); - frappe.show_alert(__("New module created {0}", [r.message])); - d.hide(); - } - }); - } - }); - - d.show(); - }, - create_new_connection(module, connection_name) { - return frappe.call('frappe.data_migration.doctype.data_migration_connector.data_migration_connector.create_new_connection', { - module, connection_name - }); - } -}); diff --git a/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.json b/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.json deleted file mode 100644 index 338d59aadd..0000000000 --- a/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.json +++ /dev/null @@ -1,307 +0,0 @@ -{ - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 1, - "autoname": "field:connector_name", - "beta": 1, - "creation": "2017-08-11 05:03:27.091416", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", - "fields": [ - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "connector_name", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Connector Name", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "depends_on": "eval:!doc.is_custom", - "fieldname": "connector_type", - "fieldtype": "Select", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Connector Type", - "length": 0, - "no_copy": 0, - "options": "\nFrappe\nCustom", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "depends_on": "eval:doc.connector_type == 'Custom'", - "fieldname": "python_module", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Python Module", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "authentication_credentials", - "fieldtype": "Section Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Authentication Credentials", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "default": "", - "fieldname": "hostname", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Hostname", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "database_name", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Database Name", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "username", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Username", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "password", - "fieldtype": "Password", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Password", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2017-12-01 13:38:55.992499", - "modified_by": "Administrator", - "module": "Data Migration", - "name": "Data Migration Connector", - "name_case": "", - "owner": "Administrator", - "permissions": [ - { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "set_user_permissions": 0, - "share": 1, - "submit": 0, - "write": 1 - } - ], - "quick_entry": 0, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1, - "track_seen": 0 -} \ No newline at end of file diff --git a/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.py b/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.py deleted file mode 100644 index 9db7fc2445..0000000000 --- a/frappe/data_migration/doctype/data_migration_connector/data_migration_connector.py +++ /dev/null @@ -1,107 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Frappe Technologies and contributors -# License: MIT. See LICENSE - -import os - -import frappe -from frappe import _ -from frappe.model.document import Document -from frappe.modules.export_file import create_init_py - -from .connectors.base import BaseConnection -from .connectors.frappe_connection import FrappeConnection - - -class DataMigrationConnector(Document): - def validate(self): - if not (self.python_module or self.connector_type): - frappe.throw(_("Enter python module or select connector type")) - - if self.python_module: - try: - get_connection_class(self.python_module) - except: - frappe.throw(frappe._("Invalid module path")) - - def get_connection(self): - if self.python_module: - _class = get_connection_class(self.python_module) - return _class(self) - else: - self.connection = FrappeConnection(self) - - return self.connection - - -@frappe.whitelist() -def create_new_connection(module, connection_name): - if not frappe.conf.get("developer_mode"): - frappe.msgprint(_("Please enable developer mode to create new connection")) - return - # create folder - module_path = frappe.get_module_path(module) - connectors_folder = os.path.join(module_path, "connectors") - frappe.create_folder(connectors_folder) - - # create init py - create_init_py(module_path, "connectors", "") - - connection_class = connection_name.replace(" ", "") - file_name = frappe.scrub(connection_name) + ".py" - file_path = os.path.join(module_path, "connectors", file_name) - - # create boilerplate file - with open(file_path, "w") as f: - f.write(connection_boilerplate.format(connection_class=connection_class)) - - # get python module string from file_path - app_name = frappe.db.get_value("Module Def", module, "app_name") - python_module = os.path.relpath(file_path, "../apps/{0}".format(app_name)).replace( - os.path.sep, "." - )[:-3] - - return python_module - - -def get_connection_class(python_module): - filename = python_module.rsplit(".", 1)[-1] - classname = frappe.unscrub(filename).replace(" ", "") - module = frappe.get_module(python_module) - - raise_error = False - if hasattr(module, classname): - _class = getattr(module, classname) - if not issubclass(_class, BaseConnection): - raise_error = True - else: - raise_error = True - - if raise_error: - raise ImportError(filename) - - return _class - - -connection_boilerplate = """from frappe.data_migration.doctype.data_migration_connector.connectors.base import BaseConnection - -class {connection_class}(BaseConnection): - def __init__(self, connector): - # self.connector = connector - # self.connection = YourModule(self.connector.username, self.get_password()) - # self.name_field = 'id' - pass - - def get(self, remote_objectname, fields=None, filters=None, start=0, page_length=10): - pass - - def insert(self, doctype, doc): - pass - - def update(self, doctype, doc, migration_id): - pass - - def delete(self, doctype, migration_id): - pass - -""" diff --git a/frappe/data_migration/doctype/data_migration_connector/test_data_migration_connector.py b/frappe/data_migration/doctype/data_migration_connector/test_data_migration_connector.py deleted file mode 100644 index c4090796ab..0000000000 --- a/frappe/data_migration/doctype/data_migration_connector/test_data_migration_connector.py +++ /dev/null @@ -1,8 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Frappe Technologies and Contributors -# License: MIT. See LICENSE -import unittest - - -class TestDataMigrationConnector(unittest.TestCase): - pass diff --git a/frappe/data_migration/doctype/data_migration_mapping/__init__.py b/frappe/data_migration/doctype/data_migration_mapping/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.js b/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.js deleted file mode 100644 index 6c99b9a54d..0000000000 --- a/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.js +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright (c) 2017, Frappe Technologies and contributors -// For license information, please see license.txt - -frappe.ui.form.on('Data Migration Mapping', { - refresh: function() { - - } -}); diff --git a/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.json b/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.json deleted file mode 100644 index 998abdf6ca..0000000000 --- a/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.json +++ /dev/null @@ -1,456 +0,0 @@ -{ - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 1, - "autoname": "field:mapping_name", - "beta": 1, - "creation": "2017-08-11 05:11:49.975801", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", - "fields": [ - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "mapping_name", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Mapping Name", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "remote_objectname", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Remote Objectname", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "remote_primary_key", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Remote Primary Key", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "local_doctype", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Local DocType", - "length": 0, - "no_copy": 0, - "options": "DocType", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "local_primary_key", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Local Primary Key", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "column_break_5", - "fieldtype": "Column Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "mapping_type", - "fieldtype": "Select", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Mapping Type", - "length": 0, - "no_copy": 0, - "options": "Push\nPull\nSync", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "default": "10", - "fieldname": "page_length", - "fieldtype": "Int", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Page Length", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "migration_id_field", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Migration ID Field", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "mapping", - "fieldtype": "Section Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Mapping", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "fields", - "fieldtype": "Table", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Field Maps", - "length": 0, - "no_copy": 0, - "options": "Data Migration Mapping Detail", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 1, - "columns": 0, - "fieldname": "condition_detail", - "fieldtype": "Section Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Condition Detail", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "condition", - "fieldtype": "Code", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Condition", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2017-09-27 18:06:43.275207", - "modified_by": "Administrator", - "module": "Data Migration", - "name": "Data Migration Mapping", - "name_case": "", - "owner": "Administrator", - "permissions": [ - { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "set_user_permissions": 0, - "share": 1, - "submit": 0, - "write": 1 - } - ], - "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1, - "track_seen": 0 -} \ No newline at end of file diff --git a/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.py b/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.py deleted file mode 100644 index 49af65e99b..0000000000 --- a/frappe/data_migration/doctype/data_migration_mapping/data_migration_mapping.py +++ /dev/null @@ -1,83 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Frappe Technologies and contributors -# License: MIT. See LICENSE - -import frappe -from frappe.model.document import Document -from frappe.utils.safe_exec import get_safe_globals - - -class DataMigrationMapping(Document): - def get_filters(self): - if self.condition: - return frappe.safe_eval(self.condition, get_safe_globals()) - - def get_fields(self): - fields = [] - for f in self.fields: - if not (f.local_fieldname[0] in ('"', "'") or f.local_fieldname.startswith("eval:")): - fields.append(f.local_fieldname) - - if frappe.db.has_column(self.local_doctype, self.migration_id_field): - fields.append(self.migration_id_field) - - if "name" not in fields: - fields.append("name") - - return fields - - def get_mapped_record(self, doc): - """Build a mapped record using information from the fields table""" - mapped = frappe._dict() - - key_fieldname = "remote_fieldname" - value_fieldname = "local_fieldname" - - if self.mapping_type == "Pull": - key_fieldname, value_fieldname = value_fieldname, key_fieldname - - for field_map in self.fields: - key = get_source_value(field_map, key_fieldname) - - if not field_map.is_child_table: - # field to field mapping - value = get_value_from_fieldname(field_map, value_fieldname, doc) - else: - # child table mapping - mapping_name = field_map.child_table_mapping - value = get_mapped_child_records( - mapping_name, doc.get(get_source_value(field_map, value_fieldname)) - ) - - mapped[key] = value - - return mapped - - -def get_mapped_child_records(mapping_name, child_docs): - mapped_child_docs = [] - mapping = frappe.get_doc("Data Migration Mapping", mapping_name) - for child_doc in child_docs: - mapped_child_docs.append(mapping.get_mapped_record(child_doc)) - - return mapped_child_docs - - -def get_value_from_fieldname(field_map, fieldname_field, doc): - field_name = get_source_value(field_map, fieldname_field) - - if field_name.startswith("eval:"): - value = frappe.safe_eval(field_name[5:], get_safe_globals()) - elif field_name[0] in ('"', "'"): - value = field_name[1:-1] - else: - value = get_source_value(doc, field_name) - return value - - -def get_source_value(source, key): - """Get value from source (object or dict) based on key""" - if isinstance(source, dict): - return source.get(key) - else: - return getattr(source, key) diff --git a/frappe/data_migration/doctype/data_migration_mapping/test_data_migration_mapping.py b/frappe/data_migration/doctype/data_migration_mapping/test_data_migration_mapping.py deleted file mode 100644 index 30d2a6bcfe..0000000000 --- a/frappe/data_migration/doctype/data_migration_mapping/test_data_migration_mapping.py +++ /dev/null @@ -1,8 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Frappe Technologies and Contributors -# License: MIT. See LICENSE -import unittest - - -class TestDataMigrationMapping(unittest.TestCase): - pass diff --git a/frappe/data_migration/doctype/data_migration_mapping_detail/__init__.py b/frappe/data_migration/doctype/data_migration_mapping_detail/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.json b/frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.json deleted file mode 100644 index ede9213f14..0000000000 --- a/frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.json +++ /dev/null @@ -1,163 +0,0 @@ -{ - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, - "creation": "2017-08-11 05:09:10.900237", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", - "fields": [ - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "remote_fieldname", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Remote Fieldname", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "local_fieldname", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Local Fieldname", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "is_child_table", - "fieldtype": "Check", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Is Child Table", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "depends_on": "is_child_table", - "fieldname": "child_table_mapping", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Child Table Mapping", - "length": 0, - "no_copy": 0, - "options": "Data Migration Mapping", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 1, - "max_attachments": 0, - "modified": "2017-09-28 17:13:31.337005", - "modified_by": "Administrator", - "module": "Data Migration", - "name": "Data Migration Mapping Detail", - "name_case": "", - "owner": "Administrator", - "permissions": [], - "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1, - "track_seen": 0 -} \ No newline at end of file diff --git a/frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.py b/frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.py deleted file mode 100644 index abd6348a26..0000000000 --- a/frappe/data_migration/doctype/data_migration_mapping_detail/data_migration_mapping_detail.py +++ /dev/null @@ -1,9 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Frappe Technologies and contributors -# License: MIT. See LICENSE - -from frappe.model.document import Document - - -class DataMigrationMappingDetail(Document): - pass diff --git a/frappe/data_migration/doctype/data_migration_plan/__init__.py b/frappe/data_migration/doctype/data_migration_plan/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.js b/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.js deleted file mode 100644 index 357ef2972f..0000000000 --- a/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.js +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright (c) 2017, Frappe Technologies and contributors -// For license information, please see license.txt - -frappe.ui.form.on('Data Migration Plan', { - onload(frm) { - frm.add_custom_button(__('Run'), () => frappe.new_doc('Data Migration Run', { - data_migration_plan: frm.doc.name - })); - } -}); diff --git a/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.json b/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.json deleted file mode 100644 index 2cfc2e3bd7..0000000000 --- a/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.json +++ /dev/null @@ -1,224 +0,0 @@ -{ - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "autoname": "field:plan_name", - "beta": 0, - "creation": "2017-08-11 05:15:51.482165", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", - "fields": [ - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "plan_name", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Plan Name", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 1 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "module", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Module", - "length": 0, - "no_copy": 0, - "options": "Module Def", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "mappings", - "fieldtype": "Table", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Mappings", - "length": 0, - "no_copy": 0, - "options": "Data Migration Plan Mapping", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "preprocess_method", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Preprocess Method", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "postprocess_method", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Postprocess Method", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2020-09-18 17:26:09.703215", - "modified_by": "Administrator", - "module": "Data Migration", - "name": "Data Migration Plan", - "name_case": "", - "owner": "Administrator", - "permissions": [ - { - "amend": 0, - "cancel": 0, - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "set_user_permissions": 0, - "share": 1, - "submit": 0, - "write": 1 - } - ], - "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1, - "track_seen": 0 -} \ No newline at end of file diff --git a/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.py b/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.py deleted file mode 100644 index 4118e8e7fe..0000000000 --- a/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.py +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright (c) 2021, Frappe Technologies and contributors -# License: MIT. See LICENSE - -import frappe -from frappe.custom.doctype.custom_field.custom_field import create_custom_field -from frappe.model.document import Document -from frappe.modules import get_module_path, scrub_dt_dn -from frappe.modules.export_file import create_init_py, export_to_files - - -def get_mapping_module(module, mapping_name): - app_name = frappe.db.get_value("Module Def", module, "app_name") - mapping_name = frappe.scrub(mapping_name) - module = frappe.scrub(module) - - try: - return frappe.get_module(f"{app_name}.{module}.data_migration_mapping.{mapping_name}") - except ImportError: - return None - - -class DataMigrationPlan(Document): - def on_update(self): - # update custom fields in mappings - self.make_custom_fields_for_mappings() - - if frappe.flags.in_import or frappe.flags.in_test: - return - - if frappe.local.conf.get("developer_mode"): - record_list = [["Data Migration Plan", self.name]] - - for m in self.mappings: - record_list.append(["Data Migration Mapping", m.mapping]) - - export_to_files(record_list=record_list, record_module=self.module) - - for m in self.mappings: - dt, dn = scrub_dt_dn("Data Migration Mapping", m.mapping) - create_init_py(get_module_path(self.module), dt, dn) - - def make_custom_fields_for_mappings(self): - frappe.flags.ignore_in_install = True - label = self.name + " ID" - fieldname = frappe.scrub(label) - - df = { - "label": label, - "fieldname": fieldname, - "fieldtype": "Data", - "hidden": 1, - "read_only": 1, - "unique": 1, - "no_copy": 1, - } - - for m in self.mappings: - mapping = frappe.get_doc("Data Migration Mapping", m.mapping) - create_custom_field(mapping.local_doctype, df) - mapping.migration_id_field = fieldname - mapping.save() - - # Create custom field in Deleted Document - create_custom_field("Deleted Document", df) - frappe.flags.ignore_in_install = False - - def pre_process_doc(self, mapping_name, doc): - module = get_mapping_module(self.module, mapping_name) - - if module and hasattr(module, "pre_process"): - return module.pre_process(doc) - return doc - - def post_process_doc(self, mapping_name, local_doc=None, remote_doc=None): - module = get_mapping_module(self.module, mapping_name) - - if module and hasattr(module, "post_process"): - return module.post_process(local_doc=local_doc, remote_doc=remote_doc) diff --git a/frappe/data_migration/doctype/data_migration_plan/test_data_migration_plan.py b/frappe/data_migration/doctype/data_migration_plan/test_data_migration_plan.py deleted file mode 100644 index ef3bfa3a70..0000000000 --- a/frappe/data_migration/doctype/data_migration_plan/test_data_migration_plan.py +++ /dev/null @@ -1,8 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Frappe Technologies and Contributors -# License: MIT. See LICENSE -import unittest - - -class TestDataMigrationPlan(unittest.TestCase): - pass diff --git a/frappe/data_migration/doctype/data_migration_plan_mapping/__init__.py b/frappe/data_migration/doctype/data_migration_plan_mapping/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.json b/frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.json deleted file mode 100644 index 5acf014715..0000000000 --- a/frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.json +++ /dev/null @@ -1,103 +0,0 @@ -{ - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 1, - "creation": "2017-08-11 05:15:38.390831", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", - "fields": [ - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "mapping", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Mapping", - "length": 0, - "no_copy": 0, - "options": "Data Migration Mapping", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "default": "1", - "fieldname": "enabled", - "fieldtype": "Check", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Enabled", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 1, - "max_attachments": 0, - "modified": "2017-09-20 21:43:04.908650", - "modified_by": "Administrator", - "module": "Data Migration", - "name": "Data Migration Plan Mapping", - "name_case": "", - "owner": "Administrator", - "permissions": [], - "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1, - "track_seen": 0 -} \ No newline at end of file diff --git a/frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.py b/frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.py deleted file mode 100644 index 0650f4b2c7..0000000000 --- a/frappe/data_migration/doctype/data_migration_plan_mapping/data_migration_plan_mapping.py +++ /dev/null @@ -1,9 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Frappe Technologies and contributors -# License: MIT. See LICENSE - -from frappe.model.document import Document - - -class DataMigrationPlanMapping(Document): - pass diff --git a/frappe/data_migration/doctype/data_migration_run/__init__.py b/frappe/data_migration/doctype/data_migration_run/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/data_migration/doctype/data_migration_run/data_migration_run.js b/frappe/data_migration/doctype/data_migration_run/data_migration_run.js deleted file mode 100644 index 82323c62f1..0000000000 --- a/frappe/data_migration/doctype/data_migration_run/data_migration_run.js +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright (c) 2017, Frappe Technologies and contributors -// For license information, please see license.txt - -frappe.ui.form.on('Data Migration Run', { - refresh: function(frm) { - if (frm.doc.status !== 'Success') { - frm.add_custom_button(__('Run'), () => frm.call('run')); - } - if (frm.doc.status === 'Started') { - frm.dashboard.add_progress(__('Percent Complete'), frm.doc.percent_complete, - __('Currently updating {0}', [frm.doc.current_mapping])); - } - } -}); diff --git a/frappe/data_migration/doctype/data_migration_run/data_migration_run.json b/frappe/data_migration/doctype/data_migration_run/data_migration_run.json deleted file mode 100644 index db77997928..0000000000 --- a/frappe/data_migration/doctype/data_migration_run/data_migration_run.json +++ /dev/null @@ -1,838 +0,0 @@ -{ - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, - "creation": "2017-09-11 12:55:27.597728", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", - "fields": [ - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "data_migration_plan", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Data Migration Plan", - "length": 0, - "no_copy": 0, - "options": "Data Migration Plan", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "data_migration_connector", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Data Migration Connector", - "length": 0, - "no_copy": 0, - "options": "Data Migration Connector", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "default": "Pending", - "fieldname": "status", - "fieldtype": "Select", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Status", - "length": 0, - "no_copy": 1, - "options": "Pending\nStarted\nPartial Success\nSuccess\nFail\nError", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "start_time", - "fieldtype": "Datetime", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Start Time", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "end_time", - "fieldtype": "Datetime", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "End Time", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "remote_id", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Remote ID", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "current_mapping", - "fieldtype": "Data", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Current Mapping", - "length": 0, - "no_copy": 1, - "options": "", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "current_mapping_start", - "fieldtype": "Int", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Current Mapping Start", - "length": 0, - "no_copy": 1, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "current_mapping_delete_start", - "fieldtype": "Int", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Current Mapping Delete Start", - "length": 0, - "no_copy": 1, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "current_mapping_type", - "fieldtype": "Select", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Current Mapping Type", - "length": 0, - "no_copy": 0, - "options": "Push\nPull", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "depends_on": "eval:(doc.status !== 'Pending')", - "fieldname": "current_mapping_action", - "fieldtype": "Select", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Current Mapping Action", - "length": 0, - "no_copy": 1, - "options": "Insert\nDelete", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "total_pages", - "fieldtype": "Int", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Total Pages", - "length": 0, - "no_copy": 1, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "percent_complete", - "fieldtype": "Percent", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Percent Complete", - "length": 0, - "no_copy": 1, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "trigger_name", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Trigger Name", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "depends_on": "eval:(doc.status !== 'Pending')", - "fieldname": "logs_sb", - "fieldtype": "Section Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Logs", - "length": 0, - "no_copy": 1, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "push_insert", - "fieldtype": "Int", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Push Insert", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "push_update", - "fieldtype": "Int", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Push Update", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "push_delete", - "fieldtype": "Int", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Push Delete", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "push_failed", - "fieldtype": "Code", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Push Failed", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "column_break_16", - "fieldtype": "Column Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "pull_insert", - "fieldtype": "Int", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Pull Insert", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "pull_update", - "fieldtype": "Int", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Pull Update", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "pull_failed", - "fieldtype": "Code", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Pull Failed", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "depends_on": "eval:doc.failed_log !== '[]'", - "fieldname": "log", - "fieldtype": "Code", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Log", - "length": 0, - "no_copy": 1, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2020-09-18 17:26:09.703215", - "modified_by": "Administrator", - "module": "Data Migration", - "name": "Data Migration Run", - "name_case": "", - "owner": "Administrator", - "permissions": [ - { - "amend": 0, - "cancel": 0, - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "set_user_permissions": 0, - "share": 1, - "submit": 0, - "write": 1 - } - ], - "quick_entry": 0, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1, - "track_seen": 0 -} \ No newline at end of file diff --git a/frappe/data_migration/doctype/data_migration_run/data_migration_run.py b/frappe/data_migration/doctype/data_migration_run/data_migration_run.py deleted file mode 100644 index c734cb105b..0000000000 --- a/frappe/data_migration/doctype/data_migration_run/data_migration_run.py +++ /dev/null @@ -1,514 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Frappe Technologies and contributors -# License: MIT. See LICENSE - -import json -import math - -import frappe -from frappe import _ -from frappe.data_migration.doctype.data_migration_mapping.data_migration_mapping import ( - get_source_value, -) -from frappe.model.document import Document -from frappe.utils import cstr - - -class DataMigrationRun(Document): - @frappe.whitelist() - def run(self): - self.begin() - if self.total_pages > 0: - self.enqueue_next_mapping() - else: - self.complete() - - def enqueue_next_mapping(self): - next_mapping_name = self.get_next_mapping_name() - if next_mapping_name: - next_mapping = self.get_mapping(next_mapping_name) - self.db_set( - dict( - current_mapping=next_mapping.name, - current_mapping_start=0, - current_mapping_delete_start=0, - current_mapping_action="Insert", - ), - notify=True, - commit=True, - ) - frappe.enqueue_doc(self.doctype, self.name, "run_current_mapping", now=frappe.flags.in_test) - else: - self.complete() - - def enqueue_next_page(self): - mapping = self.get_mapping(self.current_mapping) - percent_complete = self.percent_complete + (100.0 / self.total_pages) - fields = dict(percent_complete=percent_complete) - if self.current_mapping_action == "Insert": - start = self.current_mapping_start + mapping.page_length - fields["current_mapping_start"] = start - elif self.current_mapping_action == "Delete": - delete_start = self.current_mapping_delete_start + mapping.page_length - fields["current_mapping_delete_start"] = delete_start - - self.db_set(fields, notify=True, commit=True) - - if percent_complete < 100: - frappe.publish_realtime( - self.trigger_name, {"progress_percent": percent_complete}, user=frappe.session.user - ) - - frappe.enqueue_doc(self.doctype, self.name, "run_current_mapping", now=frappe.flags.in_test) - - def run_current_mapping(self): - try: - mapping = self.get_mapping(self.current_mapping) - - if mapping.mapping_type == "Push": - done = self.push() - elif mapping.mapping_type == "Pull": - done = self.pull() - - if done: - self.enqueue_next_mapping() - else: - self.enqueue_next_page() - - except Exception as e: - self.db_set("status", "Error", notify=True, commit=True) - print("Data Migration Run failed") - print(frappe.get_traceback()) - self.execute_postprocess("Error") - raise e - - def get_last_modified_condition(self): - last_run_timestamp = frappe.db.get_value( - "Data Migration Run", - dict( - data_migration_plan=self.data_migration_plan, - data_migration_connector=self.data_migration_connector, - name=("!=", self.name), - ), - "modified", - ) - if last_run_timestamp: - condition = dict(modified=(">", last_run_timestamp)) - else: - condition = {} - return condition - - def begin(self): - plan_active_mappings = [m for m in self.get_plan().mappings if m.enabled] - self.mappings = [ - frappe.get_doc("Data Migration Mapping", m.mapping) for m in plan_active_mappings - ] - - total_pages = 0 - for m in [mapping for mapping in self.mappings]: - if m.mapping_type == "Push": - count = float(self.get_count(m)) - page_count = math.ceil(count / m.page_length) - total_pages += page_count - if m.mapping_type == "Pull": - total_pages += 10 - - self.db_set( - dict( - status="Started", - current_mapping=None, - current_mapping_start=0, - current_mapping_delete_start=0, - percent_complete=0, - current_mapping_action="Insert", - total_pages=total_pages, - ), - notify=True, - commit=True, - ) - - def complete(self): - fields = dict() - - push_failed = self.get_log("push_failed", []) - pull_failed = self.get_log("pull_failed", []) - - status = "Partial Success" - - if not push_failed and not pull_failed: - status = "Success" - fields["percent_complete"] = 100 - - fields["status"] = status - - self.db_set(fields, notify=True, commit=True) - - self.execute_postprocess(status) - - frappe.publish_realtime(self.trigger_name, {"progress_percent": 100}, user=frappe.session.user) - - def execute_postprocess(self, status): - # Execute post process - postprocess_method_path = self.get_plan().postprocess_method - - if postprocess_method_path: - frappe.get_attr(postprocess_method_path)( - { - "status": status, - "stats": { - "push_insert": self.push_insert, - "push_update": self.push_update, - "push_delete": self.push_delete, - "pull_insert": self.pull_insert, - "pull_update": self.pull_update, - }, - } - ) - - def get_plan(self): - if not hasattr(self, "plan"): - self.plan = frappe.get_doc("Data Migration Plan", self.data_migration_plan) - return self.plan - - def get_mapping(self, mapping_name): - if hasattr(self, "mappings"): - for m in self.mappings: - if m.name == mapping_name: - return m - return frappe.get_doc("Data Migration Mapping", mapping_name) - - def get_next_mapping_name(self): - mappings = [m for m in self.get_plan().mappings if m.enabled] - if not self.current_mapping: - # first - return mappings[0].mapping - for i, d in enumerate(mappings): - if i == len(mappings) - 1: - # last - return None - if d.mapping == self.current_mapping: - return mappings[i + 1].mapping - - raise frappe.ValidationError("Mapping Broken") - - def get_data(self, filters): - mapping = self.get_mapping(self.current_mapping) - or_filters = self.get_or_filters(mapping) - start = self.current_mapping_start - - data = [] - doclist = frappe.get_all( - mapping.local_doctype, - filters=filters, - or_filters=or_filters, - start=start, - page_length=mapping.page_length, - ) - - for d in doclist: - doc = frappe.get_doc(mapping.local_doctype, d["name"]) - data.append(doc) - return data - - def get_new_local_data(self): - """Fetch newly inserted local data using `frappe.get_all`. Used during Push""" - mapping = self.get_mapping(self.current_mapping) - filters = mapping.get_filters() or {} - - # new docs dont have migration field set - filters.update({mapping.migration_id_field: ""}) - - return self.get_data(filters) - - def get_updated_local_data(self): - """Fetch local updated data using `frappe.get_all`. Used during Push""" - mapping = self.get_mapping(self.current_mapping) - filters = mapping.get_filters() or {} - - # existing docs must have migration field set - filters.update({mapping.migration_id_field: ("!=", "")}) - - return self.get_data(filters) - - def get_deleted_local_data(self): - """Fetch local deleted data using `frappe.get_all`. Used during Push""" - mapping = self.get_mapping(self.current_mapping) - filters = self.get_last_modified_condition() - filters.update({"deleted_doctype": mapping.local_doctype}) - - data = frappe.get_all("Deleted Document", fields=["name", "data"], filters=filters) - - _data = [] - for d in data: - doc = json.loads(d.data) - if doc.get(mapping.migration_id_field): - doc["_deleted_document_name"] = d["name"] - _data.append(doc) - - return _data - - def get_remote_data(self): - """Fetch data from remote using `connection.get`. Used during Pull""" - mapping = self.get_mapping(self.current_mapping) - start = self.current_mapping_start - filters = mapping.get_filters() or {} - connection = self.get_connection() - - return connection.get( - mapping.remote_objectname, - fields=["*"], - filters=filters, - start=start, - page_length=mapping.page_length, - ) - - def get_count(self, mapping): - filters = mapping.get_filters() or {} - or_filters = self.get_or_filters(mapping) - - to_insert = frappe.get_all( - mapping.local_doctype, ["count(name) as total"], filters=filters, or_filters=or_filters - )[0].total - - to_delete = frappe.get_all( - "Deleted Document", - ["count(name) as total"], - filters={"deleted_doctype": mapping.local_doctype}, - or_filters=or_filters, - )[0].total - - return to_insert + to_delete - - def get_or_filters(self, mapping): - or_filters = self.get_last_modified_condition() - - # docs whose migration_id_field is not set - # failed in the previous run, include those too - or_filters.update({mapping.migration_id_field: ("=", "")}) - - return or_filters - - def get_connection(self): - if not hasattr(self, "connection"): - self.connection = frappe.get_doc( - "Data Migration Connector", self.data_migration_connector - ).get_connection() - - return self.connection - - def push(self): - self.db_set("current_mapping_type", "Push") - done = True - - if self.current_mapping_action == "Insert": - done = self._push_insert() - - elif self.current_mapping_action == "Update": - done = self._push_update() - - elif self.current_mapping_action == "Delete": - done = self._push_delete() - - return done - - def _push_insert(self): - """Inserts new local docs on remote""" - mapping = self.get_mapping(self.current_mapping) - connection = self.get_connection() - data = self.get_new_local_data() - - for d in data: - # pre process before insert - doc = self.pre_process_doc(d) - doc = mapping.get_mapped_record(doc) - - try: - response_doc = connection.insert(mapping.remote_objectname, doc) - frappe.db.set_value( - mapping.local_doctype, - d.name, - mapping.migration_id_field, - response_doc[connection.name_field], - update_modified=False, - ) - frappe.db.commit() - self.update_log("push_insert", 1) - # post process after insert - self.post_process_doc(local_doc=d, remote_doc=response_doc) - except Exception as e: - self.update_log("push_failed", {d.name: cstr(e)}) - - # update page_start - self.db_set("current_mapping_start", self.current_mapping_start + mapping.page_length) - - if len(data) < mapping.page_length: - # done, no more new data to insert - self.db_set({"current_mapping_action": "Update", "current_mapping_start": 0}) - # not done with this mapping - return False - - def _push_update(self): - """Updates local modified docs on remote""" - mapping = self.get_mapping(self.current_mapping) - connection = self.get_connection() - data = self.get_updated_local_data() - - for d in data: - migration_id_value = d.get(mapping.migration_id_field) - # pre process before update - doc = self.pre_process_doc(d) - doc = mapping.get_mapped_record(doc) - try: - response_doc = connection.update(mapping.remote_objectname, doc, migration_id_value) - self.update_log("push_update", 1) - # post process after update - self.post_process_doc(local_doc=d, remote_doc=response_doc) - except Exception as e: - self.update_log("push_failed", {d.name: cstr(e)}) - - # update page_start - self.db_set("current_mapping_start", self.current_mapping_start + mapping.page_length) - - if len(data) < mapping.page_length: - # done, no more data to update - self.db_set({"current_mapping_action": "Delete", "current_mapping_start": 0}) - # not done with this mapping - return False - - def _push_delete(self): - """Deletes docs deleted from local on remote""" - mapping = self.get_mapping(self.current_mapping) - connection = self.get_connection() - data = self.get_deleted_local_data() - - for d in data: - # Deleted Document also has a custom field for migration_id - migration_id_value = d.get(mapping.migration_id_field) - # pre process before update - self.pre_process_doc(d) - try: - response_doc = connection.delete(mapping.remote_objectname, migration_id_value) - self.update_log("push_delete", 1) - # post process only when action is success - self.post_process_doc(local_doc=d, remote_doc=response_doc) - except Exception as e: - self.update_log("push_failed", {d.name: cstr(e)}) - - # update page_start - self.db_set("current_mapping_start", self.current_mapping_start + mapping.page_length) - - if len(data) < mapping.page_length: - # done, no more new data to delete - # done with this mapping - return True - - def pull(self): - self.db_set("current_mapping_type", "Pull") - - connection = self.get_connection() - mapping = self.get_mapping(self.current_mapping) - data = self.get_remote_data() - - for d in data: - migration_id_value = get_source_value(d, connection.name_field) - doc = self.pre_process_doc(d) - doc = mapping.get_mapped_record(doc) - - if migration_id_value: - try: - if not local_doc_exists(mapping, migration_id_value): - # insert new local doc - local_doc = insert_local_doc(mapping, doc) - - self.update_log("pull_insert", 1) - # set migration id - frappe.db.set_value( - mapping.local_doctype, - local_doc.name, - mapping.migration_id_field, - migration_id_value, - update_modified=False, - ) - frappe.db.commit() - else: - # update doc - local_doc = update_local_doc(mapping, doc, migration_id_value) - self.update_log("pull_update", 1) - # post process doc after success - self.post_process_doc(remote_doc=d, local_doc=local_doc) - except Exception as e: - # failed, append to log - self.update_log("pull_failed", {migration_id_value: cstr(e)}) - - if len(data) < mapping.page_length: - # last page, done with pull - return True - - def pre_process_doc(self, doc): - plan = self.get_plan() - doc = plan.pre_process_doc(self.current_mapping, doc) - return doc - - def post_process_doc(self, local_doc=None, remote_doc=None): - plan = self.get_plan() - doc = plan.post_process_doc(self.current_mapping, local_doc=local_doc, remote_doc=remote_doc) - return doc - - def set_log(self, key, value): - value = json.dumps(value) if "_failed" in key else value - self.db_set(key, value) - - def update_log(self, key, value=None): - """ - Helper for updating logs, - push_failed and pull_failed are stored as json, - other keys are stored as int - """ - if "_failed" in key: - # json - self.set_log(key, self.get_log(key, []) + [value]) - else: - # int - self.set_log(key, self.get_log(key, 0) + (value or 1)) - - def get_log(self, key, default=None): - value = self.db_get(key) - if "_failed" in key: - if not value: - value = json.dumps(default) - value = json.loads(value) - return value or default - - -def insert_local_doc(mapping, doc): - try: - # insert new doc - if not doc.doctype: - doc.doctype = mapping.local_doctype - doc = frappe.get_doc(doc).insert() - return doc - except Exception: - print("Data Migration Run failed: Error in Pull insert") - print(frappe.get_traceback()) - return None - - -def update_local_doc(mapping, remote_doc, migration_id_value): - try: - # migration id value is set in migration_id_field in mapping.local_doctype - docname = frappe.db.get_value( - mapping.local_doctype, filters={mapping.migration_id_field: migration_id_value} - ) - - doc = frappe.get_doc(mapping.local_doctype, docname) - doc.update(remote_doc) - doc.save() - return doc - except Exception: - print("Data Migration Run failed: Error in Pull update") - print(frappe.get_traceback()) - return None - - -def local_doc_exists(mapping, migration_id_value): - return frappe.db.exists(mapping.local_doctype, {mapping.migration_id_field: migration_id_value}) diff --git a/frappe/data_migration/doctype/data_migration_run/test_data_migration_run.py b/frappe/data_migration/doctype/data_migration_run/test_data_migration_run.py deleted file mode 100644 index 0357b1e0f5..0000000000 --- a/frappe/data_migration/doctype/data_migration_run/test_data_migration_run.py +++ /dev/null @@ -1,128 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Frappe Technologies and Contributors -# License: MIT. See LICENSE -import unittest - -import frappe - - -class TestDataMigrationRun(unittest.TestCase): - def test_run(self): - create_plan() - - description = "data migration todo" - new_todo = frappe.get_doc({"doctype": "ToDo", "description": description}).insert() - - event_subject = "data migration event" - frappe.get_doc( - dict( - doctype="Event", - subject=event_subject, - repeat_on="Monthly", - starts_on=frappe.utils.now_datetime(), - ) - ).insert() - - run = frappe.get_doc( - { - "doctype": "Data Migration Run", - "data_migration_plan": "ToDo Sync", - "data_migration_connector": "Local Connector", - } - ).insert() - - run.run() - self.assertEqual(run.db_get("status"), "Success") - - self.assertEqual(run.db_get("push_insert"), 1) - self.assertEqual(run.db_get("pull_insert"), 1) - - todo = frappe.get_doc("ToDo", new_todo.name) - self.assertTrue(todo.todo_sync_id) - - # Pushed Event - event = frappe.get_doc("Event", todo.todo_sync_id) - self.assertEqual(event.subject, description) - - # Pulled ToDo - created_todo = frappe.get_doc("ToDo", {"description": event_subject}) - self.assertEqual(created_todo.description, event_subject) - - todo_list = frappe.get_list( - "ToDo", filters={"description": "data migration todo"}, fields=["name"] - ) - todo_name = todo_list[0].name - - todo = frappe.get_doc("ToDo", todo_name) - todo.description = "data migration todo updated" - todo.save() - - run = frappe.get_doc( - { - "doctype": "Data Migration Run", - "data_migration_plan": "ToDo Sync", - "data_migration_connector": "Local Connector", - } - ).insert() - - run.run() - - # Update - self.assertEqual(run.db_get("status"), "Success") - self.assertEqual(run.db_get("pull_update"), 1) - - -def create_plan(): - frappe.get_doc( - { - "doctype": "Data Migration Mapping", - "mapping_name": "Todo to Event", - "remote_objectname": "Event", - "remote_primary_key": "name", - "mapping_type": "Push", - "local_doctype": "ToDo", - "fields": [ - {"remote_fieldname": "subject", "local_fieldname": "description"}, - { - "remote_fieldname": "starts_on", - "local_fieldname": "eval:frappe.utils.get_datetime_str(frappe.utils.get_datetime())", - }, - ], - "condition": '{"description": "data migration todo" }', - } - ).insert(ignore_if_duplicate=True) - - frappe.get_doc( - { - "doctype": "Data Migration Mapping", - "mapping_name": "Event to ToDo", - "remote_objectname": "Event", - "remote_primary_key": "name", - "local_doctype": "ToDo", - "local_primary_key": "name", - "mapping_type": "Pull", - "condition": '{"subject": "data migration event" }', - "fields": [{"remote_fieldname": "subject", "local_fieldname": "description"}], - } - ).insert(ignore_if_duplicate=True) - - frappe.get_doc( - { - "doctype": "Data Migration Plan", - "plan_name": "ToDo Sync", - "module": "Core", - "mappings": [{"mapping": "Todo to Event"}, {"mapping": "Event to ToDo"}], - } - ).insert(ignore_if_duplicate=True) - - frappe.get_doc( - { - "doctype": "Data Migration Connector", - "connector_name": "Local Connector", - "connector_type": "Frappe", - # connect to same host. - "hostname": frappe.conf.host_name or frappe.utils.get_site_url(frappe.local.site), - "username": "Administrator", - "password": frappe.conf.get("admin_password") or "admin", - } - ).insert(ignore_if_duplicate=True) diff --git a/frappe/database/__init__.py b/frappe/database/__init__.py index 7de3fabf01..423442d344 100644 --- a/frappe/database/__init__.py +++ b/frappe/database/__init__.py @@ -39,17 +39,21 @@ def drop_user_and_database(db_name, root_login=None, root_password=None): ) -def get_db(host=None, user=None, password=None, port=None): +def get_db(host=None, user=None, password=None, port=None, read_only=False): import frappe if frappe.conf.db_type == "postgres": import frappe.database.postgres.database - return frappe.database.postgres.database.PostgresDatabase(host, user, password, port=port) + return frappe.database.postgres.database.PostgresDatabase( + host, user, password, port=port, read_only=read_only + ) else: import frappe.database.mariadb.database - return frappe.database.mariadb.database.MariaDBDatabase(host, user, password, port=port) + return frappe.database.mariadb.database.MariaDBDatabase( + host, user, password, port=port, read_only=read_only + ) def setup_help_database(help_db_name): diff --git a/frappe/database/database.py b/frappe/database/database.py index 424bcbbc63..c55704eb64 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -1,32 +1,43 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors +# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -# Database Module -# -------------------- - import datetime +import json import random import re import string +import traceback from contextlib import contextmanager from time import time -from typing import Dict, List, Optional, Tuple, Union -from pypika.terms import Criterion, NullValue, PseudoColumn +from pypika.terms import Criterion, NullValue import frappe import frappe.defaults import frappe.model.meta from frappe import _ +from frappe.database.utils import ( + EmptyQueryValues, + FallBackDateTimeStr, + LazyMogrify, + Query, + QueryValues, + is_query_type, +) +from frappe.exceptions import DoesNotExistError from frappe.model.utils.link_count import flush_local_link_count from frappe.query_builder.functions import Count from frappe.query_builder.utils import DocType -from frappe.utils import cast, get_datetime, getdate, now, sbool +from frappe.utils import cast as cast_fieldtype +from frappe.utils import get_datetime, get_table_name, getdate, now, sbool -from .query import Query +IFNULL_PATTERN = re.compile(r"ifnull\(", flags=re.IGNORECASE) +INDEX_PATTERN = re.compile(r"\s*\([^)]+\)\s*") +SINGLE_WORD_PATTERN = re.compile(r'([`"]?)(tab([A-Z]\w+))\1') +MULTI_WORD_PATTERN = re.compile(r'([`"])(tab([A-Z]\w+)( [A-Z]\w+)+)\1') -class Database(object): +class Database: """ Open a database connection with the given parmeters, if use_default is True, use the login details from `conf.py`. This is called by the request handler and is accessible using @@ -43,15 +54,42 @@ class Database(object): CHILD_TABLE_COLUMNS = ("parent", "parenttype", "parentfield") MAX_WRITES_PER_TRANSACTION = 200_000 + # NOTE: + # FOR MARIADB - using no cache - as during backup, if the sequence was used in anyform, + # it drops the cache and uses the next non cached value in setval query and + # puts that in the backup file, which will start the counter + # from that value when inserting any new record in the doctype. + # By default the cache is 1000 which will mess up the sequence when + # using the system after a restore. + # + # Another case could be if the cached values expire then also there is a chance of + # the cache being skipped. + # + # FOR POSTGRES - The sequence cache for postgres is per connection. + # Since we're opening and closing connections for every request this results in skipping the cache + # to the next non-cached value hence not using cache in postgres. + # ref: https://stackoverflow.com/questions/21356375/postgres-9-0-4-sequence-skipping-numbers + SEQUENCE_CACHE = 0 + class InvalidColumnName(frappe.ValidationError): pass - def __init__(self, host=None, user=None, password=None, ac_name=None, use_default=0, port=None): + def __init__( + self, + host=None, + user=None, + password=None, + ac_name=None, + use_default=0, + port=None, + read_only=False, + ): self.setup_type_map() self.host = host or frappe.conf.db_host or "127.0.0.1" self.port = port or frappe.conf.db_port or "" self.user = user or frappe.conf.db_name self.db_name = frappe.conf.db_name + self.read_only = read_only # Uses READ ONLY connection if set self._conn = None if ac_name: @@ -65,7 +103,8 @@ class Database(object): self.password = password or frappe.conf.db_password self.value_cache = {} - self.query = Query() + # self.db_type: str + # self.last_query (lazy) attribute of last sql query executed def setup_type_map(self): pass @@ -82,15 +121,22 @@ class Database(object): self._conn.select_db(db_name) def get_connection(self): - pass + """Returns a Database connection object that conforms with https://peps.python.org/pep-0249/#connection-objects""" + raise NotImplementedError def get_database_size(self): - pass + raise NotImplementedError + + def _transform_query(self, query: Query, values: QueryValues) -> tuple: + return query, values + + def _transform_result(self, result: list[tuple]) -> list[tuple]: + return result def sql( self, - query, - values=(), + query: Query, + values: QueryValues = EmptyQueryValues, as_dict=0, as_list=0, formatted=0, @@ -106,7 +152,7 @@ class Database(object): """Execute a SQL query and fetch all rows. :param query: SQL query. - :param values: List / dict of values to be escaped and substituted in the query. + :param values: Tuple / List / Dict of values to be escaped and substituted in the query. :param as_dict: Return as a dictionary. :param as_list: Always return as a list. :param formatted: Format values like date etc. @@ -137,55 +183,33 @@ class Database(object): # remove whitespace / indentation from start and end of query query = query.strip() - if re.search(r"ifnull\(", query, flags=re.IGNORECASE): - # replaces ifnull in query with coalesce - query = re.sub(r"ifnull\(", "coalesce(", query, flags=re.IGNORECASE) + # replaces ifnull in query with coalesce + query = IFNULL_PATTERN.sub("coalesce(", query) if not self._conn: self.connect() # in transaction validations self.check_transaction_status(query) - self.clear_db_table_cache(query) - # autocommit if auto_commit: self.commit() - # execute + if debug: + time_start = time() + + if values == EmptyQueryValues: + values = None + elif not isinstance(values, (tuple, dict, list)): + values = (values,) + query, values = self._transform_query(query, values) + try: - if debug: - time_start = time() - - self.log_query(query, values, debug, explain) - - if values != (): - - # MySQL-python==1.2.5 hack! - if not isinstance(values, (dict, tuple, list)): - values = (values,) - - self._cursor.execute(query, values) - - if frappe.flags.in_migrate: - self.log_touched_tables(query, values) - - else: - self._cursor.execute(query) - - if frappe.flags.in_migrate: - self.log_touched_tables(query) - - if debug: - time_end = time() - frappe.errprint(("Execution time: {0} sec").format(round(time_end - time_start, 2))) - + self._cursor.execute(query, values) except Exception as e: if self.is_syntax_error(e): - # only for mariadb - frappe.errprint("Syntax error in query:") - frappe.errprint(query) + frappe.errprint(f"Syntax error in query:\n{query} {values}") elif self.is_deadlocked(e): raise frappe.QueryDeadlockError(e) @@ -193,26 +217,39 @@ class Database(object): elif self.is_timedout(e): raise frappe.QueryTimeoutError(e) - elif frappe.conf.db_type == "postgres": - # TODO: added temporarily - print(e) + # TODO: added temporarily + elif self.db_type == "postgres": + traceback.print_stack() + frappe.errprint(f"Error in query:\n{e}") raise - if ignore_ddl and ( - self.is_missing_column(e) or self.is_table_missing(e) or self.cant_drop_field_or_key(e) - ): - pass - else: + elif isinstance(e, self.ProgrammingError): + traceback.print_stack() + frappe.errprint(f"Error in query:\n{query, values}") raise + if not ( + ignore_ddl + and (self.is_missing_column(e) or self.is_table_missing(e) or self.cant_drop_field_or_key(e)) + ): + raise + + if debug: + time_end = time() + frappe.errprint(f"Execution time: {time_end - time_start:.2f} sec") + + self.log_query(query, values, debug, explain) + if auto_commit: self.commit() if not self._cursor.description: return () + self.last_result = self._transform_result(self._cursor.fetchall()) + if pluck: - return [r[0] for r in self._cursor.fetchall()] + return [r[0] for r in self.last_result] # scrub output if required if as_dict: @@ -222,53 +259,72 @@ class Database(object): r.update(update) return ret elif as_list: - return self.convert_to_lists(self._cursor.fetchall(), formatted, as_utf8) + return self.convert_to_lists(self.last_result, formatted, as_utf8) elif as_utf8: - return self.convert_to_lists(self._cursor.fetchall(), formatted, as_utf8) + return self.convert_to_lists(self.last_result, formatted, as_utf8) else: - return self._cursor.fetchall() + return self.last_result - def log_query(self, query, values, debug, explain): - # for debugging in tests - if frappe.conf.get("allow_tests") and frappe.cache().get_value("flag_print_sql"): - print(self.mogrify(query, values)) + def _log_query(self, mogrified_query: str, debug: bool = False, explain: bool = False) -> None: + """Takes the query and logs it to various interfaces according to the settings.""" + _query = None + + if frappe.conf.allow_tests and frappe.cache().get_value("flag_print_sql"): + _query = _query or str(mogrified_query) + print(_query) - # debug if debug: - if explain and query.strip().lower().startswith("select"): - self.explain_query(query, values) - frappe.errprint(self.mogrify(query, values)) + _query = _query or str(mogrified_query) + if explain and is_query_type(_query, "select"): + self.explain_query(_query) + frappe.errprint(_query) - # info - if (frappe.conf.get("logging") or False) == 2: - frappe.log("<<<< query") - frappe.log(self.mogrify(query, values)) - frappe.log(">>>>") + if frappe.conf.logging == 2: + _query = _query or str(mogrified_query) + frappe.log(f"<<<< query\n{_query}\n>>>>") - def mogrify(self, query, values): + if frappe.flags.in_migrate: + _query = _query or str(mogrified_query) + self.log_touched_tables(_query) + + def log_query( + self, query: str, values: QueryValues = None, debug: bool = False, explain: bool = False + ) -> str: + # TODO: Use mogrify until MariaDB Connector/C 1.1 is released and we can fetch something + # like cursor._transformed_statement from the cursor object. We can also avoid setting + # mogrified_query if we don't need to log it. + mogrified_query = self.lazy_mogrify(query, values) + self._log_query(mogrified_query, debug, explain) + return mogrified_query + + def mogrify(self, query: Query, values: QueryValues): """build the query string with values""" if not values: return query - else: - try: - return self._cursor.mogrify(query, values) - except: # noqa: E722 - return (query, values) + + try: + return self._cursor.mogrify(query, values) + except AttributeError: + if isinstance(values, dict): + return query % {k: frappe.db.escape(v) if isinstance(v, str) else v for k, v in values.items()} + elif isinstance(values, (list, tuple)): + return query % tuple(frappe.db.escape(v) if isinstance(v, str) else v for v in values) + return query, values + + def lazy_mogrify(self, query: Query, values: QueryValues) -> LazyMogrify: + """Wrap the object with str to generate mogrified query.""" + return LazyMogrify(query, values) def explain_query(self, query, values=None): """Print `EXPLAIN` in error log.""" + frappe.errprint("--- query explain ---") try: - frappe.errprint("--- query explain ---") - if values is None: - self._cursor.execute("explain " + query) - else: - self._cursor.execute("explain " + query, values) - import json - + self._cursor.execute(f"EXPLAIN {query}", values) + except Exception as e: + frappe.errprint(f"error in query explain: {e}") + else: frappe.errprint(json.dumps(self.fetch_as_dict(), indent=1)) frappe.errprint("--- query explain end ---") - except Exception: - frappe.errprint("error in query explain") def sql_list(self, query, values=(), debug=False, **kwargs): """Return data as list of single elements (first column). @@ -278,9 +334,9 @@ class Database(object): # doctypes = ["DocType", "DocField", "User", ...] doctypes = frappe.db.sql_list("select name from DocType") """ - return [r[0] for r in self.sql(query, values, **kwargs, debug=debug)] + return self.sql(query, values, **kwargs, debug=debug, pluck=True) - def sql_ddl(self, query, values=(), debug=False): + def sql_ddl(self, query, debug=False): """Commit and execute a query. DDL (Data Definition Language) queries that alter schema autocommit in MariaDB.""" self.commit() @@ -292,7 +348,7 @@ class Database(object): could cause the system to hang.""" self.check_implicit_commit(query) - if query and query.strip().lower() in ("commit", "rollback"): + if query and is_query_type(query, ("commit", "rollback")): self.transaction_writes = 0 if query[:6].lower() in ("update", "insert", "delete"): @@ -309,14 +365,13 @@ class Database(object): if ( self.transaction_writes and query - and query.strip().split()[0].lower() - in ["start", "alter", "drop", "create", "begin", "truncate"] + and is_query_type(query, ("start", "alter", "drop", "create", "begin", "truncate")) ): raise Exception("This statement can cause implicit commit") def fetch_as_dict(self, formatted=0, as_utf8=0): """Internal. Converts results to dict.""" - result = self._cursor.fetchall() + result = self.last_result ret = [] if result: keys = [column[0] for column in self._cursor.description] @@ -333,7 +388,7 @@ class Database(object): @staticmethod def clear_db_table_cache(query): - if query and query.strip().split()[0].lower() in {"drop", "create"}: + if query and is_query_type(query, ("drop", "create")): frappe.cache().delete_key("db_tables") @staticmethod @@ -582,7 +637,7 @@ class Database(object): return [map(values.get, fields)] else: - r = self.query.get_sql( + r = frappe.qb.engine.get_query( "Singles", filters={"field": ("in", tuple(fields)), "doctype": doctype}, fields=["field", "value"], @@ -602,24 +657,44 @@ class Database(object): else: return r and [[i[1] for i in r]] or [] - def get_singles_dict(self, doctype, debug=False, *, for_update=False): + def get_singles_dict(self, doctype, debug=False, *, for_update=False, cast=False): """Get Single DocType as dict. :param doctype: DocType of the single object whose value is requested + :param debug: Execute query in debug mode - print to STDOUT + :param for_update: Take `FOR UPDATE` lock on the records + :param cast: Cast values to Python data types based on field type Example: # Get coulmn and value of the single doctype Accounts Settings account_settings = frappe.db.get_singles_dict("Accounts Settings") """ - result = self.query.get_sql( + queried_result = frappe.qb.engine.get_query( "Singles", filters={"doctype": doctype}, fields=["field", "value"], for_update=for_update, - ).run() + ).run(debug=debug) - return frappe._dict(result) + if not cast: + return frappe._dict(queried_result) + + try: + meta = frappe.get_meta(doctype) + except DoesNotExistError: + return frappe._dict(queried_result) + + return_value = frappe._dict() + + for fieldname, value in queried_result: + if df := meta.get_field(fieldname): + casted_value = cast_fieldtype(df.fieldtype, value) + else: + casted_value = value + return_value[fieldname] = casted_value + + return return_value @staticmethod def get_all(*args, **kwargs): @@ -632,8 +707,8 @@ class Database(object): def set_single_value( self, doctype: str, - fieldname: Union[str, Dict], - value: Optional[Union[str, int]] = None, + fieldname: str | dict, + value: str | int | None = None, *args, **kwargs, ): @@ -668,7 +743,7 @@ class Database(object): if cache and fieldname in self.value_cache[doctype]: return self.value_cache[doctype][fieldname] - val = self.query.get_sql( + val = frappe.qb.engine.get_query( table="Singles", filters={"doctype": doctype, "field": fieldname}, fields="value", @@ -682,7 +757,7 @@ class Database(object): _("Invalid field name: {0}").format(frappe.bold(fieldname)), self.InvalidColumnName ) - val = cast(df.fieldtype, val) + val = cast_fieldtype(df.fieldtype, val) self.value_cache[doctype][fieldname] = val @@ -710,14 +785,7 @@ class Database(object): ): field_objects = [] - if not isinstance(fields, Criterion): - for field in fields: - if "(" in str(field) or " as " in str(field): - field_objects.append(PseudoColumn(field)) - else: - field_objects.append(field) - - query = self.query.get_sql( + query = frappe.qb.engine.get_query( table=doctype, filters=filters, orderby=order_by, @@ -827,7 +895,7 @@ class Database(object): frappe.clear_document_cache(dt, docname) else: - query = self.query.build_conditions(table=dt, filters=dn, update=True) + query = frappe.qb.engine.build_conditions(table=dt, filters=dn, update=True) # TODO: Fix this; doesn't work rn - gavin@frappe.io # frappe.cache().hdel_keys(dt, "document_cache") # Workaround: clear all document caches @@ -849,13 +917,8 @@ class Database(object): def touch(self, doctype, docname): """Update the modified timestamp of this document.""" modified = now() - self.sql( - """update `tab{doctype}` set `modified`=%s - where name=%s""".format( - doctype=doctype - ), - (modified, docname), - ) + DocType = frappe.qb.DocType(doctype) + frappe.qb.update(DocType).set(DocType.modified, modified).where(DocType.name == docname).run() return modified @staticmethod @@ -896,14 +959,14 @@ class Database(object): @staticmethod def get_defaults(key=None, parent="__default"): """Get all defaults""" - if key: - defaults = frappe.defaults.get_defaults(parent) - d = defaults.get(key, None) - if not d and key != frappe.scrub(key): - d = defaults.get(frappe.scrub(key), None) - return d - else: - return frappe.defaults.get_defaults(parent) + defaults = frappe.defaults.get_defaults_for(parent) + if not key: + return defaults + + if key in defaults: + return defaults[key] + + return defaults.get(frappe.scrub(key)) def begin(self): self.sql("START TRANSACTION") @@ -914,6 +977,9 @@ class Database(object): frappe.call(method[0], *(method[1] or []), **(method[2] or {})) self.sql("commit") + if self.db_type == "postgres": + # Postgres requires explicitly starting new transaction + self.begin() frappe.local.rollback_observers = [] self.flush_realtime_log() @@ -950,7 +1016,7 @@ class Database(object): else: self.sql("rollback") self.begin() - for obj in frappe.local.rollback_observers: + for obj in dict.fromkeys(frappe.local.rollback_observers): if hasattr(obj, "on_rollback"): obj.on_rollback() frappe.local.rollback_observers = [] @@ -967,22 +1033,11 @@ class Database(object): return self.table_exists(doctype) def get_tables(self, cached=True): - tables = frappe.cache().get_value("db_tables") - if not tables or not cached: - table_rows = self.sql( - """ - SELECT table_name - FROM information_schema.tables - WHERE table_schema NOT IN ('pg_catalog', 'information_schema') - """ - ) - tables = {d[0] for d in table_rows} - frappe.cache().set_value("db_tables", tables) - return tables + raise NotImplementedError def a_row_exists(self, doctype): """Returns True if atleast one row exists.""" - return self.sql("select name from `tab{doctype}` limit 1".format(doctype=doctype)) + return frappe.get_all(doctype, limit=1, order_by=None, as_list=True) def exists(self, dt, dn=None, cache=False): """Return the document name of a matching document, or None. @@ -1019,21 +1074,19 @@ class Database(object): return self.get_value(dt, dn, ignore=True, cache=cache) - def count(self, dt, filters=None, debug=False, cache=False): + def count(self, dt, filters=None, debug=False, cache=False, distinct: bool = True): """Returns `COUNT(*)` for given DocType and filters.""" if cache and not filters: - cache_count = frappe.cache().get_value("doctype:count:{}".format(dt)) + cache_count = frappe.cache().get_value(f"doctype:count:{dt}") if cache_count is not None: return cache_count - query = self.query.get_sql(table=dt, filters=filters, fields=Count("*")) - if filters: - count = self.sql(query, debug=debug)[0][0] - return count - else: - count = self.sql(query, debug=debug)[0][0] - if cache: - frappe.cache().set_value("doctype:count:{}".format(dt), count, expires_in_sec=86400) - return count + query = frappe.qb.engine.get_query( + table=dt, filters=filters, fields=Count("*"), distinct=distinct + ) + count = self.sql(query, debug=debug)[0][0] + if not filters and cache: + frappe.cache().set_value(f"doctype:count:{dt}", count, expires_in_sec=86400) + return count @staticmethod def format_date(date): @@ -1042,7 +1095,7 @@ class Database(object): @staticmethod def format_datetime(datetime): if not datetime: - return "0001-01-01 00:00:00.000000" + return FallBackDateTimeStr if isinstance(datetime, str): if ":" not in datetime: @@ -1058,28 +1111,27 @@ class Database(object): from frappe.utils import now_datetime - return self.sql( - """select count(name) from `tab{doctype}` - where creation >= %s""".format( - doctype=doctype - ), - now_datetime() - relativedelta(minutes=minutes), - )[0][0] + Table = frappe.qb.DocType(doctype) - def get_db_table_columns(self, table): + return ( + frappe.qb.from_(Table) + .select(Count(Table.name)) + .where(Table.creation >= now_datetime() - relativedelta(minutes=minutes)) + .run()[0][0] + ) + + def get_db_table_columns(self, table) -> list[str]: """Returns list of column names from given table.""" columns = frappe.cache().hget("table_columns", table) if columns is None: - columns = [ - r[0] - for r in self.sql( - """ - select column_name - from information_schema.columns - where table_name = %s """, - table, - ) - ] + information_schema = frappe.qb.Schema("information_schema") + + columns = ( + frappe.qb.from_(information_schema.columns) + .select(information_schema.columns.column_name) + .where(information_schema.columns.table_name == table) + .run(pluck=True) + ) if columns: frappe.cache().hset("table_columns", table, columns) @@ -1098,12 +1150,19 @@ class Database(object): return column in self.get_table_columns(doctype) def get_column_type(self, doctype, column): - return self.sql( - """SELECT column_type FROM INFORMATION_SCHEMA.COLUMNS - WHERE table_name = 'tab{0}' AND column_name = '{1}' """.format( - doctype, column + """Returns column type from database.""" + information_schema = frappe.qb.Schema("information_schema") + table = get_table_name(doctype) + + return ( + frappe.qb.from_(information_schema.columns) + .select(information_schema.columns.column_type) + .where( + (information_schema.columns.table_name == table) + & (information_schema.columns.column_name == column) ) - )[0][0] + .run(pluck=True)[0] + ) def has_index(self, table_name, index_name): raise NotImplementedError @@ -1118,19 +1177,14 @@ class Database(object): def get_index_name(fields): index_name = "_".join(fields) + "_index" # remove index length if present e.g. (10) from index name - index_name = re.sub(r"\s*\([^)]+\)\s*", r"", index_name) - return index_name + return INDEX_PATTERN.sub(r"", index_name) def get_system_setting(self, key): - def _load_system_settings(): - return self.get_singles_dict("System Settings") - - return frappe.cache().get_value("system_settings", _load_system_settings).get(key) + return frappe.get_system_settings(key) def close(self): """Close database connection.""" if self._conn: - # self._cursor.close() self._conn.close() self._cursor = None self._conn = None @@ -1146,40 +1200,34 @@ class Database(object): return frappe.db.is_missing_column(e) def get_descendants(self, doctype, name): - """Return descendants of the current record""" - node_location_indexes = self.get_value(doctype, name, ("lft", "rgt")) - if node_location_indexes: - lft, rgt = node_location_indexes - return self.sql_list( - """select name from `tab{doctype}` - where lft > {lft} and rgt < {rgt}""".format( - doctype=doctype, lft=lft, rgt=rgt - ) - ) - else: - # when document does not exist + """Return descendants of the group node in tree""" + from frappe.utils.nestedset import get_descendants_of + + try: + return get_descendants_of(doctype, name, ignore_permissions=True) + except Exception: + # Can only happen if document doesn't exists - kept for backward compatibility return [] def is_missing_table_or_column(self, e): return self.is_missing_column(e) or self.is_table_missing(e) def multisql(self, sql_dict, values=(), **kwargs): - current_dialect = frappe.db.db_type or "mariadb" + current_dialect = self.db_type or "mariadb" query = sql_dict.get(current_dialect) return self.sql(query, values, **kwargs) - def delete(self, doctype: str, filters: Union[Dict, List] = None, debug=False, **kwargs): + def delete(self, doctype: str, filters: dict | list = None, debug=False, **kwargs): """Delete rows from a table in site which match the passed filters. This does trigger DocType hooks. Simply runs a DELETE query in the database. Doctype name can be passed directly, it will be pre-pended with `tab`. """ - values = () filters = filters or kwargs.get("conditions") - query = self.query.build_conditions(table=doctype, filters=filters).delete() + query = frappe.qb.engine.build_conditions(table=doctype, filters=filters).delete() if "debug" not in kwargs: kwargs["debug"] = debug - return self.sql(query, values, **kwargs) + return query.run(**kwargs) def truncate(self, doctype: str): """Truncate a table in the database. This runs a DDL command `TRUNCATE TABLE`. @@ -1187,8 +1235,7 @@ class Database(object): Doctype name can be passed directly, it will be pre-pended with `tab`. """ - table = doctype if doctype.startswith("__") else f"tab{doctype}" - return self.sql_ddl(f"truncate `{table}`") + return self.sql_ddl(f"truncate `{get_table_name(doctype)}`") def clear_table(self, doctype): return self.truncate(doctype) @@ -1200,10 +1247,8 @@ class Database(object): else: return None - def log_touched_tables(self, query, values=None): - if values: - query = frappe.safe_decode(self._cursor.mogrify(query, values)) - if query.strip().lower().split()[0] in ("insert", "delete", "update", "alter", "drop", "rename"): + def log_touched_tables(self, query): + if is_query_type(query, ("insert", "delete", "update", "alter", "drop", "rename")): # single_word_regex is designed to match following patterns # `tabXxx`, tabXxx and "tabXxx" @@ -1218,17 +1263,15 @@ class Database(object): # and are continued with multiple words that start with a captital letter # e.g. 'tabXxx' or 'tabXxx Xxx' or 'tabXxx Xxx Xxx' and so on - single_word_regex = r'([`"]?)(tab([A-Z]\w+))\1' - multi_word_regex = r'([`"])(tab([A-Z]\w+)( [A-Z]\w+)+)\1' tables = [] - for regex in (single_word_regex, multi_word_regex): - tables += [groups[1] for groups in re.findall(regex, query)] + for regex in (SINGLE_WORD_PATTERN, MULTI_WORD_PATTERN): + tables += [groups[1] for groups in regex.findall(query)] if frappe.flags.touched_tables is None: frappe.flags.touched_tables = set() frappe.flags.touched_tables.update(tables) - def bulk_insert(self, doctype, fields, values, ignore_duplicates=False): + def bulk_insert(self, doctype, fields, values, ignore_duplicates=False, *, chunk_size=10_000): """ Insert multiple records at a time @@ -1236,22 +1279,35 @@ class Database(object): :param fields: list of fields :params values: list of list of values """ - insert_list = [] - fields = ", ".join("`" + field + "`" for field in fields) + values = list(values) + table = frappe.qb.DocType(doctype) - for idx, value in enumerate(values): - insert_list.append(tuple(value)) - if idx and (idx % 10000 == 0 or idx < len(values) - 1): - self.sql( - """INSERT {ignore_duplicates} INTO `tab{doctype}` ({fields}) VALUES {values}""".format( - ignore_duplicates="IGNORE" if ignore_duplicates else "", - doctype=doctype, - fields=fields, - values=", ".join(["%s"] * len(insert_list)), - ), - tuple(insert_list), - ) - insert_list = [] + for start_index in range(0, len(values), chunk_size): + query = frappe.qb.into(table) + if ignore_duplicates: + # Pypika does not have same api for ignoring duplicates + if self.db_type == "mariadb": + query = query.ignore() + elif self.db_type == "postgres": + query = query.on_conflict().do_nothing() + + values_to_insert = values[start_index : start_index + chunk_size] + query.columns(fields).insert(*values_to_insert).run() + + def create_sequence(self, *args, **kwargs): + from frappe.database.sequence import create_sequence + + return create_sequence(*args, **kwargs) + + def set_next_sequence_val(self, *args, **kwargs): + from frappe.database.sequence import set_next_val + + set_next_val(*args, **kwargs) + + def get_next_sequence_val(self, *args, **kwargs): + from frappe.database.sequence import get_next_val + + return get_next_val(*args, **kwargs) def enqueue_jobs_after_commit(): @@ -1265,7 +1321,7 @@ def enqueue_jobs_after_commit(): @contextmanager -def savepoint(catch: Union[type, Tuple[type, ...]] = Exception): +def savepoint(catch: type | tuple[type, ...] = Exception): """Wrapper for wrapping blocks of DB operations in a savepoint. as contextmanager: diff --git a/frappe/database/db_manager.py b/frappe/database/db_manager.py index 8f810fe54b..3dddb7f862 100644 --- a/frappe/database/db_manager.py +++ b/frappe/database/db_manager.py @@ -1,5 +1,3 @@ -import os - import frappe @@ -15,69 +13,57 @@ class DbManager: return self.db.sql("select user()")[0][0].split("@")[1] def create_user(self, user, password, host=None): - # Create user if it doesn't exist. - if not host: - host = self.get_current_host() - - if password: - self.db.sql("CREATE USER '%s'@'%s' IDENTIFIED BY '%s';" % (user, host, password)) - else: - self.db.sql("CREATE USER '%s'@'%s';" % (user, host)) + host = host or self.get_current_host() + password_predicate = f" IDENTIFIED BY '{password}'" if password else "" + self.db.sql(f"CREATE USER '{user}'@'{host}'{password_predicate}") def delete_user(self, target, host=None): - if not host: - host = self.get_current_host() - try: - self.db.sql("DROP USER '%s'@'%s';" % (target, host)) - except Exception as e: - if e.args[0] == 1396: - pass - else: - raise + host = host or self.get_current_host() + self.db.sql(f"DROP USER IF EXISTS '{target}'@'{host}'") def create_database(self, target): if target in self.get_database_list(): self.drop_database(target) - - self.db.sql("CREATE DATABASE `%s` ;" % target) + self.db.sql(f"CREATE DATABASE `{target}`") def drop_database(self, target): - self.db.sql("DROP DATABASE IF EXISTS `%s`;" % target) + self.db.sql_ddl(f"DROP DATABASE IF EXISTS `{target}`") def grant_all_privileges(self, target, user, host=None): - if not host: - host = self.get_current_host() - - if frappe.conf.get("rds_db", 0) == 1: - self.db.sql( - "GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, INDEX, ALTER, CREATE TEMPORARY TABLES, CREATE VIEW, EVENT, TRIGGER, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EXECUTE, LOCK TABLES ON `%s`.* TO '%s'@'%s';" - % (target, user, host) + host = host or self.get_current_host() + permissions = ( + ( + "SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, INDEX, ALTER, " + "CREATE TEMPORARY TABLES, CREATE VIEW, EVENT, TRIGGER, SHOW VIEW, " + "CREATE ROUTINE, ALTER ROUTINE, EXECUTE, LOCK TABLES" ) - else: - self.db.sql("GRANT ALL PRIVILEGES ON `%s`.* TO '%s'@'%s';" % (target, user, host)) + if frappe.conf.rds_db + else "ALL PRIVILEGES" + ) + self.db.sql(f"GRANT {permissions} ON `{target}`.* TO '{user}'@'{host}'") def flush_privileges(self): self.db.sql("FLUSH PRIVILEGES") def get_database_list(self): - """get list of databases""" - return [d[0] for d in self.db.sql("SHOW DATABASES")] + return self.db.sql("SHOW DATABASES", pluck=True) @staticmethod def restore_database(target, source, user, password): + import os + from distutils.spawn import find_executable + from frappe.utils import make_esc esc = make_esc("$ ") - - from distutils.spawn import find_executable - pv = find_executable("pv") + if pv: - pipe = "{pv} {source} |".format(pv=pv, source=source) + pipe = f"{pv} {source} |" source = "" else: pipe = "" - source = "< {source}".format(source=source) + source = f"< {source}" if pipe: print("Restoring Database file...") diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index 1ae3fd8a61..303098049a 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -1,4 +1,4 @@ -from typing import List, Tuple, Union +import re import pymysql from pymysql.constants import ER, FIELD_TYPE @@ -9,15 +9,132 @@ from frappe.database.database import Database from frappe.database.mariadb.schema import MariaDBTable from frappe.utils import UnicodeWithAttrs, cstr, get_datetime, get_table_name +_PARAM_COMP = re.compile(r"%\([\w]*\)s") -class MariaDBDatabase(Database): - ProgrammingError = pymysql.err.ProgrammingError - TableMissingError = pymysql.err.ProgrammingError - OperationalError = pymysql.err.OperationalError - InternalError = pymysql.err.InternalError - SQLError = pymysql.err.ProgrammingError - DataError = pymysql.err.DataError + +class MariaDBExceptionUtil: + ProgrammingError = pymysql.ProgrammingError + TableMissingError = pymysql.ProgrammingError + OperationalError = pymysql.OperationalError + InternalError = pymysql.InternalError + SQLError = pymysql.ProgrammingError + DataError = pymysql.DataError + + # match ER_SEQUENCE_RUN_OUT - https://mariadb.com/kb/en/mariadb-error-codes/ + SequenceGeneratorLimitExceeded = pymysql.OperationalError + SequenceGeneratorLimitExceeded.errno = 4084 + + @staticmethod + def is_deadlocked(e: pymysql.Error) -> bool: + return e.args[0] == ER.LOCK_DEADLOCK + + @staticmethod + def is_timedout(e: pymysql.Error) -> bool: + return e.args[0] == ER.LOCK_WAIT_TIMEOUT + + @staticmethod + def is_table_missing(e: pymysql.Error) -> bool: + return e.args[0] == ER.NO_SUCH_TABLE + + @staticmethod + def is_missing_table(e: pymysql.Error) -> bool: + return MariaDBDatabase.is_table_missing(e) + + @staticmethod + def is_missing_column(e: pymysql.Error) -> bool: + return e.args[0] == ER.BAD_FIELD_ERROR + + @staticmethod + def is_duplicate_fieldname(e: pymysql.Error) -> bool: + return e.args[0] == ER.DUP_FIELDNAME + + @staticmethod + def is_duplicate_entry(e: pymysql.Error) -> bool: + return e.args[0] == ER.DUP_ENTRY + + @staticmethod + def is_access_denied(e: pymysql.Error) -> bool: + return e.args[0] == ER.ACCESS_DENIED_ERROR + + @staticmethod + def cant_drop_field_or_key(e: pymysql.Error) -> bool: + return e.args[0] == ER.CANT_DROP_FIELD_OR_KEY + + @staticmethod + def is_syntax_error(e: pymysql.Error) -> bool: + return e.args[0] == ER.PARSE_ERROR + + @staticmethod + def is_data_too_long(e: pymysql.Error) -> bool: + return e.args[0] == ER.DATA_TOO_LONG + + @staticmethod + def is_primary_key_violation(e: pymysql.Error) -> bool: + return ( + MariaDBDatabase.is_duplicate_entry(e) + and "PRIMARY" in cstr(e.args[1]) + and isinstance(e, pymysql.IntegrityError) + ) + + @staticmethod + def is_unique_key_violation(e: pymysql.Error) -> bool: + return ( + MariaDBDatabase.is_duplicate_entry(e) + and "Duplicate" in cstr(e.args[1]) + and isinstance(e, pymysql.IntegrityError) + ) + + +class MariaDBConnectionUtil: + def get_connection(self): + conn = self._get_connection() + conn.auto_reconnect = True + return conn + + def _get_connection(self): + """Return MariaDB connection object.""" + return self.create_connection() + + def create_connection(self): + return pymysql.connect(**self.get_connection_settings()) + + def get_connection_settings(self) -> dict: + conn_settings = { + "host": self.host, + "user": self.user, + "password": self.password, + "conv": self.CONVERSION_MAP, + "charset": "utf8mb4", + "use_unicode": True, + } + + if self.user != "root": + conn_settings["database"] = self.user + + if self.port: + conn_settings["port"] = int(self.port) + + if frappe.conf.local_infile: + conn_settings["local_infile"] = frappe.conf.local_infile + + if frappe.conf.db_ssl_ca and frappe.conf.db_ssl_cert and frappe.conf.db_ssl_key: + ssl_params = { + "ca": frappe.conf.db_ssl_ca, + "cert": frappe.conf.db_ssl_cert, + "key": frappe.conf.db_ssl_key, + } + conn_settings |= ssl_params + return conn_settings + + +class MariaDBDatabase(MariaDBConnectionUtil, MariaDBExceptionUtil, Database): REGEX_CHARACTER = "regexp" + CONVERSION_MAP = conversions | { + FIELD_TYPE.NEWDECIMAL: float, + FIELD_TYPE.DATETIME: get_datetime, + UnicodeWithAttrs: escape_string, + } + default_port = "3306" def setup_type_map(self): self.db_type = "mariadb" @@ -53,48 +170,11 @@ class MariaDBDatabase(Database): "Geolocation": ("longtext", ""), "Duration": ("decimal", "21,9"), "Icon": ("varchar", self.VARCHAR_LEN), + "Phone": ("varchar", self.VARCHAR_LEN), "Autocomplete": ("varchar", self.VARCHAR_LEN), "JSON": ("json", ""), } - def get_connection(self): - usessl = 0 - if frappe.conf.db_ssl_ca and frappe.conf.db_ssl_cert and frappe.conf.db_ssl_key: - usessl = 1 - ssl_params = { - "ca": frappe.conf.db_ssl_ca, - "cert": frappe.conf.db_ssl_cert, - "key": frappe.conf.db_ssl_key, - } - - conversions.update( - { - FIELD_TYPE.NEWDECIMAL: float, - FIELD_TYPE.DATETIME: get_datetime, - UnicodeWithAttrs: conversions[str], - } - ) - - conn = pymysql.connect( - user=self.user or "", - password=self.password or "", - host=self.host, - port=self.port, - charset="utf8mb4", - use_unicode=True, - ssl=ssl_params if usessl else None, - conv=conversions, - local_infile=frappe.conf.local_infile, - ) - - # MYSQL_OPTION_MULTI_STATEMENTS_OFF = 1 - # # self._conn.set_server_option(MYSQL_OPTION_MULTI_STATEMENTS_OFF) - - if self.user != "root": - conn.select_db(self.user) - - return conn - def get_database_size(self): """'Returns database size in MB""" db_size = self.sql( @@ -109,9 +189,18 @@ class MariaDBDatabase(Database): return db_size[0].get("database_size") + def log_query(self, query, values, debug, explain): + self.last_query = self._cursor._last_executed + self._log_query(query, debug, explain) + return self.last_query + @staticmethod def escape(s, percent=True): """Excape quotes and percent in given string.""" + # Update: We've scrapped PyMySQL in favour of MariaDB's official Python client + # Also, given we're promoting use of the PyPika builder via frappe.qb, the use + # of this method should be limited. + # pymysql expects unicode argument to escape_string with Python 3 s = frappe.as_unicode(escape_string(frappe.as_unicode(s)), "utf-8").replace("`", "\\`") @@ -132,74 +221,23 @@ class MariaDBDatabase(Database): @staticmethod def is_type_datetime(code): - return code in (pymysql.DATE, pymysql.DATETIME) + return code == pymysql.DATETIME - def rename_table(self, old_name: str, new_name: str) -> Union[List, Tuple]: + def rename_table(self, old_name: str, new_name: str) -> list | tuple: old_name = get_table_name(old_name) new_name = get_table_name(new_name) return self.sql(f"RENAME TABLE `{old_name}` TO `{new_name}`") - def describe(self, doctype: str) -> Union[List, Tuple]: + def describe(self, doctype: str) -> list | tuple: table_name = get_table_name(doctype) return self.sql(f"DESC `{table_name}`") def change_column_type( self, doctype: str, column: str, type: str, nullable: bool = False - ) -> Union[List, Tuple]: + ) -> list | tuple: table_name = get_table_name(doctype) null_constraint = "NOT NULL" if not nullable else "" - return self.sql(f"ALTER TABLE `{table_name}` MODIFY `{column}` {type} {null_constraint}") - - # exception types - @staticmethod - def is_deadlocked(e): - return e.args[0] == ER.LOCK_DEADLOCK - - @staticmethod - def is_timedout(e): - return e.args[0] == ER.LOCK_WAIT_TIMEOUT - - @staticmethod - def is_table_missing(e): - return e.args[0] == ER.NO_SUCH_TABLE - - @staticmethod - def is_missing_table(e): - return MariaDBDatabase.is_table_missing(e) - - @staticmethod - def is_missing_column(e): - return e.args[0] == ER.BAD_FIELD_ERROR - - @staticmethod - def is_duplicate_fieldname(e): - return e.args[0] == ER.DUP_FIELDNAME - - @staticmethod - def is_duplicate_entry(e): - return e.args[0] == ER.DUP_ENTRY - - @staticmethod - def is_access_denied(e): - return e.args[0] == ER.ACCESS_DENIED_ERROR - - @staticmethod - def cant_drop_field_or_key(e): - return e.args[0] == ER.CANT_DROP_FIELD_OR_KEY - - @staticmethod - def is_syntax_error(e): - return e.args[0] == ER.PARSE_ERROR - - @staticmethod - def is_data_too_long(e): - return e.args[0] == ER.DATA_TOO_LONG - - def is_primary_key_violation(self, e): - return self.is_duplicate_entry(e) and "PRIMARY" in cstr(e.args[1]) - - def is_unique_key_violation(self, e): - return self.is_duplicate_entry(e) and "Duplicate" in cstr(e.args[1]) + return self.sql_ddl(f"ALTER TABLE `{table_name}` MODIFY `{column}` {type} {null_constraint}") def create_auth_table(self): self.sql_ddl( @@ -242,22 +280,6 @@ class MariaDBDatabase(Database): ) ENGINE=InnoDB DEFAULT CHARSET=utf8""" ) - def create_help_table(self): - self.sql( - """create table help( - path varchar(255), - content text, - title text, - intro text, - full_path text, - fulltext(title), - fulltext(content), - index (path)) - COLLATE=utf8mb4_unicode_ci - ENGINE=MyISAM - CHARACTER SET=utf8mb4""" - ) - @staticmethod def get_on_duplicate_update(key=None): return "ON DUPLICATE key UPDATE " @@ -293,7 +315,7 @@ class MariaDBDatabase(Database): ) ) - def add_index(self, doctype: str, fields: List, index_name: str = None): + def add_index(self, doctype: str, fields: list, index_name: str = None): """Creates an index with given fields if not already created. Index name will be `fieldname1_fieldname2_index`""" index_name = index_name or self.get_index_name(fields) @@ -333,7 +355,7 @@ class MariaDBDatabase(Database): """ res = self.sql("select issingle from `tabDocType` where name=%s", (doctype,)) if not res: - raise Exception("Wrong doctype {0} in updatedb".format(doctype)) + raise Exception(f"Wrong doctype {doctype} in updatedb") if not res[0][0]: db_table = MariaDBTable(doctype, meta) @@ -343,5 +365,26 @@ class MariaDBDatabase(Database): db_table.sync() self.begin() - def get_database_list(self, target): - return [d[0] for d in self.sql("SHOW DATABASES;")] + def get_database_list(self): + return self.sql("SHOW DATABASES", pluck=True) + + def get_tables(self, cached=True): + """Returns list of tables""" + to_query = not cached + + if cached: + tables = frappe.cache().get_value("db_tables") + to_query = not tables + + if to_query: + information_schema = frappe.qb.Schema("information_schema") + + tables = ( + frappe.qb.from_(information_schema.tables) + .select(information_schema.tables.table_name) + .where(information_schema.tables.table_schema != "information_schema") + .run(pluck=True) + ) + frappe.cache().set_value("db_tables", tables) + + return tables diff --git a/frappe/database/mariadb/framework_mariadb.sql b/frappe/database/mariadb/framework_mariadb.sql index f2a1206c7c..dc91873a82 100644 --- a/frappe/database/mariadb/framework_mariadb.sql +++ b/frappe/database/mariadb/framework_mariadb.sql @@ -226,6 +226,7 @@ CREATE TABLE `tabDocType` ( `sender_field` varchar(255) DEFAULT NULL, `show_title_field_in_link` int(1) NOT NULL DEFAULT 0, `migration_hash` varchar(255) DEFAULT NULL, + `translate_link_fields` int(1) NOT NULL DEFAULT 0, PRIMARY KEY (`name`) ) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; diff --git a/frappe/database/mariadb/schema.py b/frappe/database/mariadb/schema.py index 7c95e9ffcb..99297fbab2 100644 --- a/frappe/database/mariadb/schema.py +++ b/frappe/database/mariadb/schema.py @@ -1,7 +1,6 @@ import frappe from frappe import _ from frappe.database.schema import DBTable -from frappe.database.sequence import create_sequence from frappe.model import log_types @@ -41,18 +40,11 @@ class MariaDBTable(DBTable): not self.meta.issingle and self.meta.autoname == "autoincrement" ) or self.doctype in log_types: - # NOTE: using a very small cache - as during backup, if the sequence was used in anyform, - # it drops the cache and uses the next non cached value in setval func and - # puts that in the backup file, which will start the counter - # from that value when inserting any new record in the doctype. - # By default the cache is 1000 which will mess up the sequence when - # using the system after a restore. - # issue link: https://jira.mariadb.org/browse/MDEV-21786 - create_sequence(self.doctype, check_not_exists=True, cache=50) + frappe.db.create_sequence(self.doctype, check_not_exists=True, cache=frappe.db.SEQUENCE_CACHE) # NOTE: not used nextval func as default as the ability to restore # database with sequences has bugs in mariadb and gives a scary error. - # issue link: https://jira.mariadb.org/browse/MDEV-21786 + # issue link: https://jira.mariadb.org/browse/MDEV-20070 name_column = "name bigint primary key" # create table @@ -85,15 +77,15 @@ class MariaDBTable(DBTable): columns_to_modify = set(self.change_type + self.add_unique + self.set_default) for col in self.add_column: - add_column_query.append("ADD COLUMN `{}` {}".format(col.fieldname, col.get_definition())) + add_column_query.append(f"ADD COLUMN `{col.fieldname}` {col.get_definition()}") for col in columns_to_modify: - modify_column_query.append("MODIFY `{}` {}".format(col.fieldname, col.get_definition())) + modify_column_query.append(f"MODIFY `{col.fieldname}` {col.get_definition()}") for col in self.add_index: # if index key does not exists if not frappe.db.has_index(self.table_name, col.fieldname + "_index"): - add_index_query.append("ADD INDEX `{}_index`(`{}`)".format(col.fieldname, col.fieldname)) + add_index_query.append(f"ADD INDEX `{col.fieldname}_index`(`{col.fieldname}`)") for col in self.drop_index + self.drop_unique: if col.fieldname != "name": # primary key @@ -103,7 +95,7 @@ class MariaDBTable(DBTable): # nosemgrep unique_index_record = frappe.db.sql( """ - SHOW INDEX FROM `{0}` + SHOW INDEX FROM `{}` WHERE Key_name=%s AND Non_unique=0 """.format( @@ -113,14 +105,14 @@ class MariaDBTable(DBTable): as_dict=1, ) if unique_index_record: - drop_index_query.append("DROP INDEX `{}`".format(unique_index_record[0].Key_name)) + drop_index_query.append(f"DROP INDEX `{unique_index_record[0].Key_name}`") index_constraint_changed = current_column.index != col.set_index # if index key exists if index_constraint_changed and not col.set_index: # nosemgrep index_record = frappe.db.sql( """ - SHOW INDEX FROM `{0}` + SHOW INDEX FROM `{}` WHERE Key_name=%s AND Non_unique=1 """.format( @@ -130,13 +122,13 @@ class MariaDBTable(DBTable): as_dict=1, ) if index_record: - drop_index_query.append("DROP INDEX `{}`".format(index_record[0].Key_name)) + drop_index_query.append(f"DROP INDEX `{index_record[0].Key_name}`") try: for query_parts in [add_column_query, modify_column_query, add_index_query, drop_index_query]: if query_parts: query_body = ", ".join(query_parts) - query = "ALTER TABLE `{}` {}".format(self.table_name, query_body) + query = f"ALTER TABLE `{self.table_name}` {query_body}" frappe.db.sql(query) except Exception as e: diff --git a/frappe/database/mariadb/setup_db.py b/frappe/database/mariadb/setup_db.py index 4399ccfa6a..ef246712b1 100644 --- a/frappe/database/mariadb/setup_db.py +++ b/frappe/database/mariadb/setup_db.py @@ -42,7 +42,7 @@ def setup_database(force, source_sql, verbose, no_mariadb_socket=False): dbman.delete_user(db_name, **dbman_kwargs) dbman.drop_database(db_name) else: - raise Exception("Database %s already exists" % (db_name,)) + raise Exception(f"Database {db_name} already exists") dbman.create_user(db_name, frappe.conf.db_password, **dbman_kwargs) if verbose: @@ -55,7 +55,7 @@ def setup_database(force, source_sql, verbose, no_mariadb_socket=False): dbman.grant_all_privileges(db_name, db_name, **dbman_kwargs) dbman.flush_privileges() if verbose: - print("Granted privileges to user %s and database %s" % (db_name, db_name)) + print(f"Granted privileges to user {db_name} and database {db_name}") # close root connection root_conn.close() @@ -83,9 +83,9 @@ def setup_help_database(help_db_name): def drop_user_and_database(db_name, root_login, root_password): frappe.local.db = get_root_connection(root_login, root_password) dbman = DbManager(frappe.local.db) + dbman.drop_database(db_name) dbman.delete_user(db_name, host="%") dbman.delete_user(db_name) - dbman.drop_database(db_name) def bootstrap_database(db_name, verbose, source_sql=None): @@ -131,7 +131,7 @@ def check_database_settings(): else: expected_variables = expected_settings_10_3_later - mariadb_variables = frappe._dict(frappe.db.sql("""show variables""")) + mariadb_variables = frappe._dict(frappe.db.sql("show variables")) # Check each expected value vs. actuals: result = True for key, expected_value in expected_variables.items(): @@ -142,16 +142,19 @@ def check_database_settings(): ) result = False if not result: - site = frappe.local.site - msg = ( - "Creation of your site - {x} failed because MariaDB is not properly {sep}" - "configured. If using version 10.2.x or earlier, make sure you use the {sep}" - "the Barracuda storage engine. {sep}{sep}" - "Please verify the settings above in MariaDB's my.cnf. Restart MariaDB. And {sep}" - "then run `bench new-site {x}` again.{sep2}" - "" - ).format(x=site, sep2="\n" * 2, sep="\n") - print_db_config(msg) + print( + ( + "=" * 80 + "\n" + "Creation of your site - {x} failed because MariaDB is not properly {sep}" + "configured. If using version 10.2.x or earlier, make sure you use the {sep}" + "the Barracuda storage engine. {sep}{sep}" + "Please verify the settings above in MariaDB's my.cnf. Restart MariaDB. And {sep}" + "then run `bench new-site {x}` again.{sep2}" + "" + "=" * 80 + ).format(x=frappe.local.site, sep2="\n" * 2, sep="\n") + ) + return result @@ -173,9 +176,3 @@ def get_root_connection(root_login, root_password): ) return frappe.local.flags.root_connection - - -def print_db_config(explanation): - print("=" * 80) - print(explanation) - print("=" * 80) diff --git a/frappe/database/postgres/database.py b/frappe/database/postgres/database.py index 228d0f48be..cb566736ad 100644 --- a/frappe/database/postgres/database.py +++ b/frappe/database/postgres/database.py @@ -1,14 +1,24 @@ import re -from typing import List, Tuple, Union import psycopg2 import psycopg2.extensions -from psycopg2.errorcodes import STRING_DATA_RIGHT_TRUNCATION +from psycopg2.errorcodes import ( + CLASS_INTEGRITY_CONSTRAINT_VIOLATION, + DEADLOCK_DETECTED, + DUPLICATE_COLUMN, + INSUFFICIENT_PRIVILEGE, + STRING_DATA_RIGHT_TRUNCATION, + UNDEFINED_COLUMN, + UNDEFINED_TABLE, + UNIQUE_VIOLATION, +) +from psycopg2.errors import SequenceGeneratorLimitExceeded, SyntaxError from psycopg2.extensions import ISOLATION_LEVEL_REPEATABLE_READ import frappe from frappe.database.database import Database from frappe.database.postgres.schema import PostgresTable +from frappe.database.utils import EmptyQueryValues, LazyDecode from frappe.utils import cstr, get_table_name # cast decimals as floats @@ -20,8 +30,13 @@ DEC2FLOAT = psycopg2.extensions.new_type( psycopg2.extensions.register_type(DEC2FLOAT) +LOCATE_SUB_PATTERN = re.compile(r"locate\(([^,]+),([^)]+)(\)?)\)", flags=re.IGNORECASE) +LOCATE_QUERY_PATTERN = re.compile(r"locate\(", flags=re.IGNORECASE) +PG_TRANSFORM_PATTERN = re.compile(r"([=><]+)\s*([+-]?\d+)(\.0)?(?![a-zA-Z\.\d])") +FROM_TAB_PATTERN = re.compile(r"from tab([\w-]*)", flags=re.IGNORECASE) -class PostgresDatabase(Database): + +class PostgresExceptionUtil: ProgrammingError = psycopg2.ProgrammingError TableMissingError = psycopg2.ProgrammingError OperationalError = psycopg2.OperationalError @@ -29,7 +44,65 @@ class PostgresDatabase(Database): SQLError = psycopg2.ProgrammingError DataError = psycopg2.DataError InterfaceError = psycopg2.InterfaceError + SequenceGeneratorLimitExceeded = SequenceGeneratorLimitExceeded + + @staticmethod + def is_deadlocked(e): + return getattr(e, "pgcode", None) == DEADLOCK_DETECTED + + @staticmethod + def is_timedout(e): + # http://initd.org/psycopg/docs/extensions.html?highlight=datatype#psycopg2.extensions.QueryCanceledError + return isinstance(e, psycopg2.extensions.QueryCanceledError) + + @staticmethod + def is_syntax_error(e): + return isinstance(e, SyntaxError) + + @staticmethod + def is_table_missing(e): + return getattr(e, "pgcode", None) == UNDEFINED_TABLE + + @staticmethod + def is_missing_table(e): + return PostgresDatabase.is_table_missing(e) + + @staticmethod + def is_missing_column(e): + return getattr(e, "pgcode", None) == UNDEFINED_COLUMN + + @staticmethod + def is_access_denied(e): + return getattr(e, "pgcode", None) == INSUFFICIENT_PRIVILEGE + + @staticmethod + def cant_drop_field_or_key(e): + return getattr(e, "pgcode", None) == CLASS_INTEGRITY_CONSTRAINT_VIOLATION + + @staticmethod + def is_duplicate_entry(e): + return getattr(e, "pgcode", None) == UNIQUE_VIOLATION + + @staticmethod + def is_primary_key_violation(e): + return getattr(e, "pgcode", None) == UNIQUE_VIOLATION and "_pkey" in cstr(e.args[0]) + + @staticmethod + def is_unique_key_violation(e): + return getattr(e, "pgcode", None) == UNIQUE_VIOLATION and "_key" in cstr(e.args[0]) + + @staticmethod + def is_duplicate_fieldname(e): + return getattr(e, "pgcode", None) == DUPLICATE_COLUMN + + @staticmethod + def is_data_too_long(e): + return getattr(e, "pgcode", None) == STRING_DATA_RIGHT_TRUNCATION + + +class PostgresDatabase(PostgresExceptionUtil, Database): REGEX_CHARACTER = "~" + default_port = "5432" def setup_type_map(self): self.db_type = "postgres" @@ -65,10 +138,15 @@ class PostgresDatabase(Database): "Geolocation": ("text", ""), "Duration": ("decimal", "21,9"), "Icon": ("varchar", self.VARCHAR_LEN), + "Phone": ("varchar", self.VARCHAR_LEN), "Autocomplete": ("varchar", self.VARCHAR_LEN), "JSON": ("json", ""), } + @property + def last_query(self): + return LazyDecode(self._cursor.query) + def get_connection(self): conn = psycopg2.connect( "host='{}' dbname='{}' user='{}' password='{}' port={}".format( @@ -105,10 +183,11 @@ class PostgresDatabase(Database): return db_size[0].get("database_size") # pylint: disable=W0221 - def sql(self, query, values=(), *args, **kwargs): - return super(PostgresDatabase, self).sql( - modify_query(query), modify_values(values), *args, **kwargs - ) + def sql(self, query, values=EmptyQueryValues, *args, **kwargs): + return super().sql(modify_query(query), modify_values(values), *args, **kwargs) + + def lazy_mogrify(self, *args, **kwargs) -> str: + return self.last_query def get_tables(self, cached=True): return [ @@ -116,9 +195,9 @@ class PostgresDatabase(Database): for d in self.sql( """select table_name from information_schema.tables - where table_catalog='{0}' + where table_catalog='{}' and table_type = 'BASE TABLE' - and table_schema='{1}'""".format( + and table_schema='{}'""".format( frappe.conf.db_name, frappe.conf.get("db_schema", "public") ) ) @@ -142,80 +221,31 @@ class PostgresDatabase(Database): def is_type_datetime(code): return code == psycopg2.DATETIME - # exception type - @staticmethod - def is_deadlocked(e): - return e.pgcode == "40P01" - - @staticmethod - def is_timedout(e): - # http://initd.org/psycopg/docs/extensions.html?highlight=datatype#psycopg2.extensions.QueryCanceledError - return isinstance(e, psycopg2.extensions.QueryCanceledError) - - @staticmethod - def is_syntax_error(e): - return isinstance(e, psycopg2.errors.SyntaxError) - - @staticmethod - def is_table_missing(e): - return getattr(e, "pgcode", None) == "42P01" - - @staticmethod - def is_missing_table(e): - return PostgresDatabase.is_table_missing(e) - - @staticmethod - def is_missing_column(e): - return getattr(e, "pgcode", None) == "42703" - - @staticmethod - def is_access_denied(e): - return e.pgcode == "42501" - - @staticmethod - def cant_drop_field_or_key(e): - return e.pgcode.startswith("23") - - @staticmethod - def is_duplicate_entry(e): - return e.pgcode == "23505" - - @staticmethod - def is_primary_key_violation(e): - return getattr(e, "pgcode", None) == "23505" and "_pkey" in cstr(e.args[0]) - - @staticmethod - def is_unique_key_violation(e): - return getattr(e, "pgcode", None) == "23505" and "_key" in cstr(e.args[0]) - - @staticmethod - def is_duplicate_fieldname(e): - return e.pgcode == "42701" - - @staticmethod - def is_data_too_long(e): - return e.pgcode == STRING_DATA_RIGHT_TRUNCATION - - def rename_table(self, old_name: str, new_name: str) -> Union[List, Tuple]: + def rename_table(self, old_name: str, new_name: str) -> list | tuple: old_name = get_table_name(old_name) new_name = get_table_name(new_name) return self.sql(f"ALTER TABLE `{old_name}` RENAME TO `{new_name}`") - def describe(self, doctype: str) -> Union[List, Tuple]: + def describe(self, doctype: str) -> list | tuple: table_name = get_table_name(doctype) return self.sql( f"SELECT COLUMN_NAME FROM information_schema.COLUMNS WHERE TABLE_NAME = '{table_name}'" ) def change_column_type( - self, doctype: str, column: str, type: str, nullable: bool = False - ) -> Union[List, Tuple]: + self, doctype: str, column: str, type: str, nullable: bool = False, use_cast: bool = False + ) -> list | tuple: table_name = get_table_name(doctype) null_constraint = "SET NOT NULL" if not nullable else "DROP NOT NULL" - return self.sql( + using_cast = f'using "{column}"::{type}' if use_cast else "" + + # postgres allows ddl in transactions but since we've currently made + # things same as mariadb (raising exception on ddl commands if the transaction has any writes), + # hence using sql_ddl here for committing and then moving forward. + return self.sql_ddl( f"""ALTER TABLE "{table_name}" - ALTER COLUMN "{column}" TYPE {type}, - ALTER COLUMN "{column}" {null_constraint}""" + ALTER COLUMN "{column}" TYPE {type} {using_cast}, + ALTER COLUMN "{column}" {null_constraint}""" ) def create_auth_table(self): @@ -255,17 +285,6 @@ class PostgresDatabase(Database): )""" ) - def create_help_table(self): - self.sql( - """CREATE TABLE "help"( - "path" varchar(255), - "content" text, - "title" text, - "intro" text, - "full_path" text)""" - ) - self.sql("""CREATE INDEX IF NOT EXISTS "help_index" ON "help" ("path")""") - def updatedb(self, doctype, meta=None): """ Syncs a `DocType` to the table @@ -273,9 +292,9 @@ class PostgresDatabase(Database): * updates columns * updates indices """ - res = self.sql("select issingle from `tabDocType` where name='{}'".format(doctype)) + res = self.sql(f"select issingle from `tabDocType` where name='{doctype}'") if not res: - raise Exception("Wrong doctype {0} in updatedb".format(doctype)) + raise Exception(f"Wrong doctype {doctype} in updatedb") if not res[0][0]: db_table = PostgresTable(doctype, meta) @@ -289,7 +308,7 @@ class PostgresDatabase(Database): def get_on_duplicate_update(key="name"): if isinstance(key, list): key = '", "'.join(key) - return 'ON CONFLICT ("{key}") DO UPDATE SET '.format(key=key) + return f'ON CONFLICT ("{key}") DO UPDATE SET ' def check_implicit_commit(self, query): pass # postgres can run DDL in transactions without implicit commits @@ -302,7 +321,7 @@ class PostgresDatabase(Database): ) ) - def add_index(self, doctype: str, fields: List, index_name: str = None): + def add_index(self, doctype: str, fields: list, index_name: str = None): """Creates an index with given fields if not already created. Index name will be `fieldname1_fieldname2_index`""" table_name = get_table_name(doctype) @@ -363,65 +382,58 @@ class PostgresDatabase(Database): as_dict=1, ) - def get_database_list(self, target): - return [d[0] for d in self.sql("SELECT datname FROM pg_database;")] + def get_database_list(self): + return self.sql("SELECT datname FROM pg_database", pluck=True) def modify_query(query): """ "Modifies query according to the requirements of postgres""" # replace ` with " for definitions - query = str(query) - query = query.replace("`", '"') + query = str(query).replace("`", '"') query = replace_locate_with_strpos(query) # select from requires "" - if re.search("from tab", query, flags=re.IGNORECASE): - query = re.sub(r"from tab([\w-]*)", r'from "tab\1"', query, flags=re.IGNORECASE) + query = FROM_TAB_PATTERN.sub(r'from "tab\1"', query) # only find int (with/without signs), ignore decimals (with/without signs), ignore hashes (which start with numbers), # drop .0 from decimals and add quotes around them # # >>> query = "c='abcd' , a >= 45, b = -45.0, c = 40, d=4500.0, e=3500.53, f=40psdfsd, g=9092094312, h=12.00023" - # >>> re.sub(r"([=><]+)\s*(?!\d+[a-zA-Z])(?![+-]?\d+\.\d\d+)([+-]?\d+)(\.0)?", r"\1 '\2'", query) + # >>> re.sub(r"([=><]+)\s*([+-]?\d+)(\.0)?(?![a-zA-Z\.\d])", r"\1 '\2'", query) # "c='abcd' , a >= '45', b = '-45', c = '40', d= '4500', e=3500.53, f=40psdfsd, g= '9092094312', h=12.00023 - query = re.sub( - r"([=><]+)\s*(?!\d+[a-zA-Z])(?![+-]?\d+\.\d\d+)([+-]?\d+)(\.0)?", r"\1 '\2'", query - ) - return query + return PG_TRANSFORM_PATTERN.sub(r"\1 '\2'", query) def modify_values(values): - def stringify_value(value): - if isinstance(value, int): + def modify_value(value): + if isinstance(value, (list, tuple)): + value = tuple(modify_values(value)) + + elif isinstance(value, int): value = str(value) - elif isinstance(value, float): - truncated_float = int(value) - if value == truncated_float: - value = str(truncated_float) return value - if not values: + if not values or values == EmptyQueryValues: return values if isinstance(values, dict): for k, v in values.items(): - values[k] = stringify_value(v) + values[k] = modify_value(v) elif isinstance(values, (tuple, list)): new_values = [] for val in values: - new_values.append(stringify_value(val)) + new_values.append(modify_value(val)) + values = new_values else: - values = stringify_value(values) + values = modify_value(values) return values def replace_locate_with_strpos(query): # strpos is the locate equivalent in postgres - if re.search(r"locate\(", query, flags=re.IGNORECASE): - query = re.sub( - r"locate\(([^,]+),([^)]+)(\)?)\)", r"strpos(\2\3, \1)", query, flags=re.IGNORECASE - ) + if LOCATE_QUERY_PATTERN.search(query): + query = LOCATE_SUB_PATTERN.sub(r"strpos(\2\3, \1)", query) return query diff --git a/frappe/database/postgres/framework_postgres.sql b/frappe/database/postgres/framework_postgres.sql index 1e79bf67d8..99e94a226f 100644 --- a/frappe/database/postgres/framework_postgres.sql +++ b/frappe/database/postgres/framework_postgres.sql @@ -231,6 +231,7 @@ CREATE TABLE "tabDocType" ( "sender_field" varchar(255) DEFAULT NULL, "show_title_field_in_link" smallint NOT NULL DEFAULT 0, "migration_hash" varchar(255) DEFAULT NULL, + "translate_link_fields" smallint NOT NULL DEFAULT 0, PRIMARY KEY ("name") ) ; @@ -240,7 +241,7 @@ CREATE TABLE "tabDocType" ( DROP TABLE IF EXISTS "tabSeries"; CREATE TABLE "tabSeries" ( - "name" varchar(100) DEFAULT NULL, + "name" varchar(100), "current" bigint NOT NULL DEFAULT 0, PRIMARY KEY ("name") ) ; diff --git a/frappe/database/postgres/schema.py b/frappe/database/postgres/schema.py index 3432c8b548..5e28c81455 100644 --- a/frappe/database/postgres/schema.py +++ b/frappe/database/postgres/schema.py @@ -1,7 +1,6 @@ import frappe from frappe import _ from frappe.database.schema import DBTable, get_definition -from frappe.database.sequence import create_sequence from frappe.model import log_types from frappe.utils import cint, flt @@ -35,11 +34,7 @@ class PostgresTable(DBTable): not self.meta.issingle and self.meta.autoname == "autoincrement" ) or self.doctype in log_types: - # The sequence cache is per connection. - # Since we're opening and closing connections for every transaction this results in skipping the cache - # to the next non-cached value hence not using cache in postgres. - # ref: https://stackoverflow.com/questions/21356375/postgres-9-0-4-sequence-skipping-numbers - create_sequence(self.doctype, check_not_exists=True) + frappe.db.create_sequence(self.doctype, check_not_exists=True, cache=frappe.db.SEQUENCE_CACHE) name_column = "name bigint primary key" # TODO: set docstatus length @@ -84,7 +79,7 @@ class PostgresTable(DBTable): query = [] for col in self.add_column: - query.append("ADD COLUMN `{}` {}".format(col.fieldname, col.get_definition())) + query.append(f"ADD COLUMN `{col.fieldname}` {col.get_definition()}") for col in self.change_type: using_clause = "" @@ -92,12 +87,12 @@ class PostgresTable(DBTable): # The USING option of SET DATA TYPE can actually specify any expression # involving the old values of the row # read more https://www.postgresql.org/docs/9.1/sql-altertable.html - using_clause = "USING {}::timestamp without time zone".format(col.fieldname) + using_clause = f"USING {col.fieldname}::timestamp without time zone" elif col.fieldtype in ("Check"): - using_clause = "USING {}::smallint".format(col.fieldname) + using_clause = f"USING {col.fieldname}::smallint" query.append( - "ALTER COLUMN `{0}` TYPE {1} {2}".format( + "ALTER COLUMN `{}` TYPE {} {}".format( col.fieldname, get_definition(col.fieldtype, precision=col.precision, length=col.length), using_clause, @@ -118,9 +113,9 @@ class PostgresTable(DBTable): col_default = "NULL" else: - col_default = "{}".format(frappe.db.escape(col.default)) + col_default = f"{frappe.db.escape(col.default)}" - query.append("ALTER COLUMN `{}` SET DEFAULT {}".format(col.fieldname, col_default)) + query.append(f"ALTER COLUMN `{col.fieldname}` SET DEFAULT {col_default}") create_contraint_query = "" for col in self.add_index: @@ -144,13 +139,13 @@ class PostgresTable(DBTable): # primary key if col.fieldname != "name": # if index key exists - drop_contraint_query += 'DROP INDEX IF EXISTS "{}" ;'.format(col.fieldname) + drop_contraint_query += f'DROP INDEX IF EXISTS "{col.fieldname}" ;' for col in self.drop_unique: # primary key if col.fieldname != "name": # if index key exists - drop_contraint_query += 'DROP INDEX IF EXISTS "unique_{}" ;'.format(col.fieldname) + drop_contraint_query += f'DROP INDEX IF EXISTS "unique_{col.fieldname}" ;' try: if query: final_alter_query = "ALTER TABLE `{}` {}".format(self.table_name, ", ".join(query)) diff --git a/frappe/database/postgres/setup_db.py b/frappe/database/postgres/setup_db.py index 90d5f72c16..7eee8081c0 100644 --- a/frappe/database/postgres/setup_db.py +++ b/frappe/database/postgres/setup_db.py @@ -6,12 +6,11 @@ import frappe def setup_database(force, source_sql=None, verbose=False): root_conn = get_root_connection(frappe.flags.root_login, frappe.flags.root_password) root_conn.commit() - root_conn.sql("DROP DATABASE IF EXISTS `{0}`".format(frappe.conf.db_name)) - root_conn.sql("DROP USER IF EXISTS {0}".format(frappe.conf.db_name)) - root_conn.sql("CREATE DATABASE `{0}`".format(frappe.conf.db_name)) - root_conn.sql( - "CREATE user {0} password '{1}'".format(frappe.conf.db_name, frappe.conf.db_password) - ) + root_conn.sql("end") + root_conn.sql(f"DROP DATABASE IF EXISTS `{frappe.conf.db_name}`") + root_conn.sql(f"DROP USER IF EXISTS {frappe.conf.db_name}") + root_conn.sql(f"CREATE DATABASE `{frappe.conf.db_name}`") + root_conn.sql(f"CREATE user {frappe.conf.db_name} password '{frappe.conf.db_password}'") root_conn.sql("GRANT ALL PRIVILEGES ON DATABASE `{0}` TO {0}".format(frappe.conf.db_name)) root_conn.close() @@ -78,10 +77,10 @@ def import_db_from_sql(source_sql=None, verbose=False): def setup_help_database(help_db_name): root_conn = get_root_connection(frappe.flags.root_login, frappe.flags.root_password) - root_conn.sql("DROP DATABASE IF EXISTS `{0}`".format(help_db_name)) - root_conn.sql("DROP USER IF EXISTS {0}".format(help_db_name)) - root_conn.sql("CREATE DATABASE `{0}`".format(help_db_name)) - root_conn.sql("CREATE user {0} password '{1}'".format(help_db_name, help_db_name)) + root_conn.sql(f"DROP DATABASE IF EXISTS `{help_db_name}`") + root_conn.sql(f"DROP USER IF EXISTS {help_db_name}") + root_conn.sql(f"CREATE DATABASE `{help_db_name}`") + root_conn.sql(f"CREATE user {help_db_name} password '{help_db_name}'") root_conn.sql("GRANT ALL PRIVILEGES ON DATABASE `{0}` TO {0}".format(help_db_name)) @@ -114,8 +113,9 @@ def drop_user_and_database(db_name, root_login, root_password): ) root_conn.commit() root_conn.sql( - f"SELECT pg_terminate_backend (pg_stat_activity.pid) FROM pg_stat_activity WHERE pg_stat_activity.datname = %s", + "SELECT pg_terminate_backend (pg_stat_activity.pid) FROM pg_stat_activity WHERE pg_stat_activity.datname = %s", (db_name,), ) + root_conn.sql("end") root_conn.sql(f"DROP DATABASE IF EXISTS {db_name}") root_conn.sql(f"DROP USER IF EXISTS {db_name}") diff --git a/frappe/database/query.py b/frappe/database/query.py index 8d8a767370..8dbb564edc 100644 --- a/frappe/database/query.py +++ b/frappe/database/query.py @@ -1,13 +1,24 @@ import operator import re -from typing import Any, Dict, List, Tuple, Union +from ast import literal_eval +from functools import cached_property +from types import BuiltinFunctionType +from typing import TYPE_CHECKING, Any, Callable import frappe from frappe import _ -from frappe.query_builder import Criterion, Field, Order +from frappe.model.db_query import get_timespan_date_range +from frappe.query_builder import Criterion, Field, Order, Table, functions +from frappe.query_builder.functions import Function, SqlFunctions + +TAB_PATTERN = re.compile("^tab") +WORDS_PATTERN = re.compile(r"\w+") +BRACKETS_PATTERN = re.compile(r"\(.*?\)|$") +SQL_FUNCTIONS = [sql_function.value for sql_function in SqlFunctions] +COMMA_PATTERN = re.compile(r",\s*(?![^()]*\))") -def like(key: str, value: str) -> frappe.qb: +def like(key: Field, value: str) -> frappe.qb: """Wrapper method for `LIKE` Args: @@ -17,10 +28,10 @@ def like(key: str, value: str) -> frappe.qb: Returns: frappe.qb: `frappe.qb object with `LIKE` """ - return Field(key).like(value) + return key.like(value) -def func_in(key: str, value: Union[List, Tuple]) -> frappe.qb: +def func_in(key: Field, value: list | tuple) -> frappe.qb: """Wrapper method for `IN` Args: @@ -30,10 +41,10 @@ def func_in(key: str, value: Union[List, Tuple]) -> frappe.qb: Returns: frappe.qb: `frappe.qb object with `IN` """ - return Field(key).isin(value) + return key.isin(value) -def not_like(key: str, value: str) -> frappe.qb: +def not_like(key: Field, value: str) -> frappe.qb: """Wrapper method for `NOT LIKE` Args: @@ -43,10 +54,10 @@ def not_like(key: str, value: str) -> frappe.qb: Returns: frappe.qb: `frappe.qb object with `NOT LIKE` """ - return Field(key).not_like(value) + return key.not_like(value) -def func_not_in(key: str, value: Union[List, Tuple]): +def func_not_in(key: Field, value: list | tuple): """Wrapper method for `NOT IN` Args: @@ -56,10 +67,10 @@ def func_not_in(key: str, value: Union[List, Tuple]): Returns: frappe.qb: `frappe.qb object with `NOT IN` """ - return Field(key).notin(value) + return key.notin(value) -def func_regex(key: str, value: str) -> frappe.qb: +def func_regex(key: Field, value: str) -> frappe.qb: """Wrapper method for `REGEX` Args: @@ -69,10 +80,10 @@ def func_regex(key: str, value: str) -> frappe.qb: Returns: frappe.qb: `frappe.qb object with `REGEX` """ - return Field(key).regex(value) + return key.regex(value) -def func_between(key: str, value: Union[List, Tuple]) -> frappe.qb: +def func_between(key: Field, value: list | tuple) -> frappe.qb: """Wrapper method for `BETWEEN` Args: @@ -82,10 +93,29 @@ def func_between(key: str, value: Union[List, Tuple]) -> frappe.qb: Returns: frappe.qb: `frappe.qb object with `BETWEEN` """ - return Field(key)[slice(*value)] + return key[slice(*value)] -def make_function(key: Any, value: Union[int, str]): +def func_is(key, value): + "Wrapper for IS" + return key.isnotnull() if value.lower() == "set" else key.isnull() + + +def func_timespan(key: Field, value: str) -> frappe.qb: + """Wrapper method for `TIMESPAN` + + Args: + key (str): field + value (str): criterion + + Returns: + frappe.qb: `frappe.qb object with `TIMESPAN` + """ + + return func_between(key, get_timespan_date_range(value)) + + +def make_function(key: Any, value: int | str): """returns fucntion query Args: @@ -95,7 +125,7 @@ def make_function(key: Any, value: Union[int, str]): Returns: frappe.qb: frappe.qb object """ - return OPERATOR_MAP[value[0]](key, value[1]) + return OPERATOR_MAP[value[0].casefold()](key, value[1]) def change_orderby(order: str): @@ -108,14 +138,25 @@ def change_orderby(order: str): tuple: field, order """ order = order.split() - if order[1].lower() == "asc": - orderby, order = order[0], Order.asc - return orderby, order - orderby, order = order[0], Order.desc - return orderby, order + + try: + if order[1].lower() == "asc": + return order[0], Order.asc + except IndexError: + pass + + return order[0], Order.desc -OPERATOR_MAP = { +def literal_eval_(literal): + try: + return literal_eval(literal) + except (ValueError, SyntaxError): + return literal + + +# default operators +OPERATOR_MAP: dict[str, Callable] = { "+": operator.add, "=": operator.eq, "-": operator.sub, @@ -126,17 +167,41 @@ OPERATOR_MAP = { "=<": operator.le, ">=": operator.ge, "=>": operator.ge, + "/": operator.truediv, + "*": operator.mul, "in": func_in, "not in": func_not_in, "like": like, "not like": not_like, "regex": func_regex, "between": func_between, + "is": func_is, + "timespan": func_timespan, + # TODO: Add support for nested set + # TODO: Add support for custom operators (WIP) - via filters_config hooks } -class Query: - def get_condition(self, table: str, **kwargs) -> frappe.qb: +class Engine: + tables: dict[str, str] = {} + + @cached_property + def OPERATOR_MAP(self): + # default operators + all_operators = OPERATOR_MAP.copy() + + # TODO: update with site-specific custom operators / removed previous buggy implementation + if frappe.get_hooks("filters_config"): + from frappe.utils.commands import warn + + warn( + "The 'filters_config' hook used to add custom operators is not yet implemented" + " in frappe.db.query engine. Use db_query (frappe.get_list) instead." + ) + + return all_operators + + def get_condition(self, table: str | Table, **kwargs) -> frappe.qb: """Get initial table object Args: @@ -145,11 +210,20 @@ class Query: Returns: frappe.qb: DocType with initial condition """ + table_object = self.get_table(table) if kwargs.get("update"): - return frappe.qb.update(table) + return frappe.qb.update(table_object) if kwargs.get("into"): - return frappe.qb.into(table) - return frappe.qb.from_(table) + return frappe.qb.into(table_object) + return frappe.qb.from_(table_object) + + def get_table(self, table_name: str | Table) -> Table: + if isinstance(table_name, Table): + return table_name + table_name = table_name.strip('"').strip("'") + if table_name not in self.tables: + self.tables[table_name] = frappe.qb.DocType(table_name) + return self.tables[table_name] def criterion_query(self, table: str, criterion: Criterion, **kwargs) -> frappe.qb: """Generate filters from Criterion objects @@ -173,15 +247,19 @@ class Query: Returns: conditions (frappe.qb): frappe.qb object """ - if kwargs.get("orderby"): + if kwargs.get("orderby") and kwargs.get("orderby") != "KEEP_DEFAULT_ORDERING": orderby = kwargs.get("orderby") - order = kwargs.get("order") if kwargs.get("order") else Order.desc if isinstance(orderby, str) and len(orderby.split()) > 1: - orderby, order = change_orderby(orderby) - conditions = conditions.orderby(orderby, order=order) + for ordby in orderby.split(","): + if ordby := ordby.strip(): + orderby, order = change_orderby(ordby) + conditions = conditions.orderby(orderby, order=order) + else: + conditions = conditions.orderby(orderby, order=kwargs.get("order") or Order.desc) if kwargs.get("limit"): conditions = conditions.limit(kwargs.get("limit")) + conditions = conditions.offset(kwargs.get("offset", 0)) if kwargs.get("distinct"): conditions = conditions.distinct() @@ -189,9 +267,12 @@ class Query: if kwargs.get("for_update"): conditions = conditions.for_update() + if kwargs.get("groupby"): + conditions = conditions.groupby(kwargs.get("groupby")) + return conditions - def misc_query(self, table: str, filters: Union[List, Tuple] = None, **kwargs): + def misc_query(self, table: str, filters: list | tuple = None, **kwargs): """Build conditions using the given Lists or Tuple filters Args: @@ -203,22 +284,27 @@ class Query: return conditions if isinstance(filters, list): for f in filters: - if not isinstance(f, (list, tuple)): - _operator = OPERATOR_MAP[filters[1]] + if isinstance(f, (list, tuple)): + _operator = self.OPERATOR_MAP[f[-2].casefold()] + if len(f) == 4: + table_object = self.get_table(f[0]) + _field = table_object[f[1]] + else: + _field = Field(f[0]) + conditions = conditions.where(_operator(_field, f[-1])) + elif isinstance(f, dict): + conditions = self.dict_query(table, f, **kwargs) + else: + _operator = self.OPERATOR_MAP[filters[1].casefold()] if not isinstance(filters[0], str): conditions = make_function(filters[0], filters[2]) break conditions = conditions.where(_operator(Field(filters[0]), filters[2])) break - else: - _operator = OPERATOR_MAP[f[1]] - conditions = conditions.where(_operator(Field(f[0]), f[2])) return self.add_conditions(conditions, **kwargs) - def dict_query( - self, table: str, filters: Dict[str, Union[str, int]] = None, **kwargs - ) -> frappe.qb: + def dict_query(self, table: str, filters: dict[str, str | int] = None, **kwargs) -> frappe.qb: """Build conditions using the given dictionary filters Args: @@ -233,20 +319,21 @@ class Query: conditions = self.add_conditions(conditions, **kwargs) return conditions + for key, value in filters.items(): + if isinstance(value, bool): + filters.update({key: str(int(value))}) + for key in filters: value = filters.get(key) - _operator = OPERATOR_MAP["="] + _operator = self.OPERATOR_MAP["="] if not isinstance(key, str): conditions = conditions.where(make_function(key, value)) continue if isinstance(value, (list, tuple)): - if isinstance(value[1], (list, tuple)) or value[0] in list(OPERATOR_MAP.keys())[-4:]: - _operator = OPERATOR_MAP[value[0]] - conditions = conditions.where(_operator(key, value[1])) - else: - _operator = OPERATOR_MAP[value[0]] - conditions = conditions.where(_operator(Field(key), value[1])) + _operator = self.OPERATOR_MAP[value[0].casefold()] + _value = value[1] if value[1] else ("",) + conditions = conditions.where(_operator(Field(key), _value)) else: if value is not None: conditions = conditions.where(_operator(Field(key), value)) @@ -258,7 +345,7 @@ class Query: return self.add_conditions(conditions, **kwargs) def build_conditions( - self, table: str, filters: Union[Dict[str, Union[str, int]], str, int] = None, **kwargs + self, table: str, filters: dict[str, str | int] | str | int = None, **kwargs ) -> frappe.qb: """Build conditions for sql query @@ -283,16 +370,161 @@ class Query: return criterion - def get_sql( + def get_function_object(self, field: str) -> "Function": + """Expects field to look like 'SUM(*)' or 'name' or something similar. Returns PyPika Function object""" + func = field.split("(", maxsplit=1)[0].capitalize() + args_start, args_end = len(func) + 1, field.index(")") + args = field[args_start:args_end].split(",") + + _, alias = field.split(" as ") if " as " in field else (None, None) + + to_cast = "*" not in args + _args = [] + + for arg in args: + initial_fields = literal_eval_(arg.strip()) + if to_cast: + has_primitive_operator = False + for _operator in OPERATOR_MAP.keys(): + if _operator in initial_fields: + operator_mapping = OPERATOR_MAP[_operator] + # Only perform this if operator is of primitive type. + if isinstance(operator_mapping, BuiltinFunctionType): + has_primitive_operator = True + field = operator_mapping( + *map(lambda field: Field(field.strip()), arg.split(_operator)), + ) + + field = Field(initial_fields) if not has_primitive_operator else field + else: + field = initial_fields + + _args.append(field) + try: + return getattr(functions, func)(*_args, alias=alias or None) + except AttributeError: + # Fall back for functions not present in `SqlFunctions`` + return Function(func, *_args, alias=alias or None) + + def function_objects_from_string(self, fields): + fields = list(map(lambda str: str.strip(), COMMA_PATTERN.split(fields))) + return self.function_objects_from_list(fields=fields) + + def function_objects_from_list(self, fields): + functions = [] + for field in fields: + field = field.casefold() if isinstance(field, str) else field + if not issubclass(type(field), Criterion): + if any([f"{func}(" in field for func in SQL_FUNCTIONS]) or "(" in field: + functions.append(field) + + return [self.get_function_object(function) for function in functions] + + def remove_string_functions(self, fields, function_objects): + """Remove string functions from fields which have already been converted to function objects""" + for function in function_objects: + if isinstance(fields, str): + if function.alias: + fields = fields.replace(" as " + function.alias.casefold(), "") + fields = BRACKETS_PATTERN.sub("", fields.replace(function.name.casefold(), "")) + # Check if only comma is left in fields after stripping functions. + if "," in fields and (len(fields.strip()) == 1): + fields = "" + else: + updated_fields = [] + for field in fields: + if isinstance(field, str): + if function.alias: + field = field.replace(" as " + function.alias.casefold(), "") + field = ( + BRACKETS_PATTERN.sub("", field).strip().casefold().replace(function.name.casefold(), "") + ) + updated_fields.append(field) + + fields = [field for field in updated_fields if field] + + return fields + + def set_fields(self, fields, **kwargs): + fields = kwargs.get("pluck") if kwargs.get("pluck") else fields or "name" + if isinstance(fields, list) and None in fields and Field not in fields: + return None + + function_objects = [] + + is_list = isinstance(fields, (list, tuple, set)) + if is_list and len(fields) == 1: + fields = fields[0] + is_list = False + + if is_list: + function_objects += self.function_objects_from_list(fields=fields) + + is_str = isinstance(fields, str) + if is_str: + fields = fields.casefold() + function_objects += self.function_objects_from_string(fields=fields) + + fields = self.remove_string_functions(fields, function_objects) + + if is_str and "," in fields: + fields = [field.replace(" ", "") if "as" not in field else field for field in fields.split(",")] + is_list, is_str = True, False + + if is_str: + if fields == "*": + return fields + if " as " in fields: + fields, reference = fields.split(" as ") + fields = Field(fields).as_(reference) + + if not is_str and fields: + if issubclass(type(fields), Criterion): + return fields + updated_fields = [] + if "*" in fields: + return fields + for field in fields: + if not isinstance(field, Criterion) and field: + if " as " in field: + field, reference = field.split(" as ") + updated_fields.append(Field(field.strip()).as_(reference)) + else: + updated_fields.append(Field(field)) + + fields = updated_fields + + # Need to check instance again since fields modified. + if not isinstance(fields, (list, tuple, set)): + fields = [fields] if fields else [] + + fields.extend(function_objects) + return fields + + def get_query( self, table: str, - fields: Union[List, Tuple], - filters: Union[Dict[str, Union[str, int]], str, int] = None, - **kwargs + fields: list | tuple, + filters: dict[str, str | int] | str | int | list[list | str | int] = None, + **kwargs, ): + # Clean up state before each query + self.tables = {} criterion = self.build_conditions(table, filters, **kwargs) + fields = self.set_fields(kwargs.get("field_objects") or fields, **kwargs) + + join = kwargs.get("join").replace(" ", "_") if kwargs.get("join") else "left_join" + + if len(self.tables) > 1: + primary_table = self.tables[table] + del self.tables[table] + for table_object in self.tables.values(): + criterion = getattr(criterion, join)(table_object).on( + table_object.parent == primary_table.name + ) + if isinstance(fields, (list, tuple)): - query = criterion.select(*kwargs.get("field_objects", fields)) + query = criterion.select(*fields) elif isinstance(fields, Criterion): query = criterion.select(fields) @@ -314,7 +546,7 @@ class Permission: doctype = [doctype] for dt in doctype: - dt = re.sub("^tab", "", dt) + dt = TAB_PATTERN.sub("", dt) if not frappe.has_permission( dt, "select", @@ -330,4 +562,4 @@ class Permission: @staticmethod def get_tables_from_query(query: str): - return [table for table in re.findall(r"\w+", query) if table.startswith("tab")] + return [table for table in WORDS_PATTERN.findall(query) if table.startswith("tab")] diff --git a/frappe/database/schema.py b/frappe/database/schema.py index 19af447aae..5920d14c3d 100644 --- a/frappe/database/schema.py +++ b/frappe/database/schema.py @@ -4,6 +4,9 @@ import frappe from frappe import _ from frappe.utils import cint, cstr, flt +SPECIAL_CHAR_PATTERN = re.compile(r"[\W]", flags=re.UNICODE) +VARCHAR_CAST_PATTERN = re.compile(r"varchar\(([\d]+)\)") + class InvalidColumnName(frappe.ValidationError): pass @@ -12,7 +15,7 @@ class InvalidColumnName(frappe.ValidationError): class DBTable: def __init__(self, doctype, meta=None): self.doctype = doctype - self.table_name = "tab{}".format(doctype) + self.table_name = f"tab{doctype}" self.meta = meta or frappe.get_meta(doctype, False) self.columns = {} self.current_columns = {} @@ -130,7 +133,7 @@ class DBTable: if not current_col: continue current_type = self.current_columns[col.fieldname]["type"] - current_length = re.findall(r"varchar\(([\d]+)\)", current_type) + current_length = VARCHAR_CAST_PATTERN.findall(current_type) if not current_length: # case when the field is no longer a varchar continue @@ -192,11 +195,11 @@ class DbColumn: if self.fieldtype in ("Check", "Int"): default_value = cint(self.default) or 0 - column_def += " not null default {0}".format(default_value) + column_def += f" not null default {default_value}" elif self.fieldtype in ("Currency", "Float", "Percent"): default_value = flt(self.default) or 0 - column_def += " not null default {0}".format(default_value) + column_def += f" not null default {default_value}" elif ( self.default @@ -204,7 +207,7 @@ class DbColumn: and not cstr(self.default).startswith(":") and column_def not in ("text", "longtext") ): - column_def += " default {}".format(frappe.db.escape(self.default)) + column_def += f" default {frappe.db.escape(self.default)}" if self.unique and (column_def not in ("text", "longtext")): column_def += " unique" @@ -304,9 +307,8 @@ class DbColumn: def validate_column_name(n): - special_characters = re.findall(r"[\W]", n, re.UNICODE) - if special_characters: - special_characters = ", ".join('"{0}"'.format(c) for c in special_characters) + if special_characters := SPECIAL_CHAR_PATTERN.findall(n): + special_characters = ", ".join(f'"{c}"' for c in special_characters) frappe.throw( _("Fieldname {0} cannot have special characters like {1}").format( frappe.bold(cstr(n)), special_characters @@ -350,7 +352,7 @@ def get_definition(fieldtype, precision=None, length=None): size = length if size is not None: - coltype = "{coltype}({size})".format(coltype=coltype, size=size) + coltype = f"{coltype}({size})" return coltype @@ -364,7 +366,7 @@ def add_column( frappe.db.commit() - query = "alter table `tab%s` add column %s %s" % ( + query = "alter table `tab{}` add column {} {}".format( doctype, column_name, get_definition(fieldtype, precision, length), diff --git a/frappe/database/sequence.py b/frappe/database/sequence.py index c4789dbdaf..54362a5895 100644 --- a/frappe/database/sequence.py +++ b/frappe/database/sequence.py @@ -5,6 +5,7 @@ def create_sequence( doctype_name: str, *, slug: str = "_id_seq", + temporary: bool = False, check_not_exists: bool = False, cycle: bool = False, cache: int = 0, @@ -14,7 +15,7 @@ def create_sequence( max_value: int = 0, ) -> str: - query = "create sequence" + query = "create sequence" if not temporary else "create temporary sequence" sequence_name = scrub(doctype_name + slug) if check_not_exists: @@ -22,55 +23,62 @@ def create_sequence( query += f" {sequence_name}" - if cache: - query += f" cache {cache}" - else: - # in postgres, the default is cache 1 - if db.db_type == "mariadb": - query += " nocache" - - if start_value: - # default is 1 - query += f" start with {start_value}" - if increment_by: # default is 1 query += f" increment by {increment_by}" if min_value: # default is 1 - query += f" min value {min_value}" + query += f" minvalue {min_value}" if max_value: - query += f" max value {max_value}" + query += f" maxvalue {max_value}" + + if start_value: + # default is 1 + query += f" start {start_value}" + + # in postgres, the default is cache 1 / no cache + if cache: + query += f" cache {cache}" + elif db.db_type == "mariadb": + query += " nocache" if not cycle: + # in postgres, default is no cycle if db.db_type == "mariadb": query += " nocycle" else: query += " cycle" - db.sql(query) + db.sql_ddl(query) return sequence_name def get_next_val(doctype_name: str, slug: str = "_id_seq") -> int: + sequence_name = scrub(f"{doctype_name}{slug}") + if db.db_type == "postgres": - return db.sql(f"select nextval('\"{scrub(doctype_name + slug)}\"')")[0][0] - return db.sql(f"select nextval(`{scrub(doctype_name + slug)}`)")[0][0] + sequence_name = f"'\"{sequence_name}\"'" + elif db.db_type == "mariadb": + sequence_name = f"`{sequence_name}`" + + try: + return db.sql(f"SELECT nextval({sequence_name})")[0][0] + except IndexError: + raise db.SequenceGeneratorLimitExceeded def set_next_val( doctype_name: str, next_val: int, *, slug: str = "_id_seq", is_val_used: bool = False ) -> None: - if not is_val_used: - is_val_used = 0 if db.db_type == "mariadb" else "f" - else: - is_val_used = 1 if db.db_type == "mariadb" else "t" + is_val_used = "false" if not is_val_used else "true" - if db.db_type == "postgres": - db.sql(f"SELECT SETVAL('\"{scrub(doctype_name + slug)}\"', {next_val}, '{is_val_used}')") - else: - db.sql(f"SELECT SETVAL(`{scrub(doctype_name + slug)}`, {next_val}, {is_val_used})") + db.multisql( + { + "postgres": f"SELECT SETVAL('\"{scrub(doctype_name + slug)}\"', {next_val}, {is_val_used})", + "mariadb": f"SELECT SETVAL(`{scrub(doctype_name + slug)}`, {next_val}, {is_val_used})", + } + ) diff --git a/frappe/database/utils.py b/frappe/database/utils.py new file mode 100644 index 0000000000..c4d8cb4953 --- /dev/null +++ b/frappe/database/utils.py @@ -0,0 +1,54 @@ +# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors +# License: MIT. See LICENSE + +from functools import cached_property +from types import NoneType + +import frappe +from frappe.query_builder.builder import MariaDB, Postgres + +Query = str | MariaDB | Postgres +QueryValues = tuple | list | dict | NoneType + +EmptyQueryValues = object() +FallBackDateTimeStr = "0001-01-01 00:00:00.000000" + + +def is_query_type(query: str, query_type: str | tuple[str]) -> bool: + return query.lstrip().split(maxsplit=1)[0].lower().startswith(query_type) + + +class LazyString: + def _setup(self) -> None: + raise NotImplementedError + + @cached_property + def value(self) -> str: + return self._setup() + + def __str__(self) -> str: + return self.value + + def __repr__(self) -> str: + return f"'{self.value}'" + + +class LazyDecode(LazyString): + __slots__ = () + + def __init__(self, value: str) -> None: + self._value = value + + def _setup(self) -> None: + return self._value.decode() + + +class LazyMogrify(LazyString): + __slots__ = () + + def __init__(self, query, values) -> None: + self.query = query + self.values = values + + def _setup(self) -> str: + return frappe.db.mogrify(self.query, self.values) diff --git a/frappe/defaults.py b/frappe/defaults.py index 4bbdcf25c6..02076b1fda 100644 --- a/frappe/defaults.py +++ b/frappe/defaults.py @@ -85,18 +85,19 @@ def get_user_permissions(user=None): def get_defaults(user=None): - globald = get_defaults_for() + global_defaults = get_defaults_for() if not user: user = frappe.session.user if frappe.session else "Guest" - if user: - userd = {} - userd.update(get_defaults_for(user)) - userd.update({"user": user, "owner": user}) - globald.update(userd) + if not user: + return global_defaults - return globald + defaults = global_defaults.copy() + defaults.update(get_defaults_for(user)) + defaults.update(user=user, owner=user) + + return defaults def clear_user_default(key, user=None): @@ -222,7 +223,7 @@ def get_defaults_for(parent="__default"): .run(as_dict=True) ) - defaults = frappe._dict({}) + defaults = frappe._dict() for d in res: if d.defkey in defaults: # listify @@ -241,8 +242,4 @@ def get_defaults_for(parent="__default"): def _clear_cache(parent): - if parent in common_default_keys: - frappe.clear_cache() - else: - clear_notifications(user=parent) - frappe.clear_cache(user=parent) + frappe.clear_cache(user=parent if parent not in common_default_keys else None) diff --git a/frappe/deferred_insert.py b/frappe/deferred_insert.py index 28c77002f8..328d8dd555 100644 --- a/frappe/deferred_insert.py +++ b/frappe/deferred_insert.py @@ -1,5 +1,5 @@ import json -from typing import TYPE_CHECKING, Dict, List, Union +from typing import TYPE_CHECKING, Union import redis @@ -12,7 +12,7 @@ if TYPE_CHECKING: queue_prefix = "insert_queue_for_" -def deferred_insert(doctype: str, records: Union[List[Union[Dict, "Document"]], str]): +def deferred_insert(doctype: str, records: list[Union[dict, "Document"]] | str): if isinstance(records, (dict, list)): _records = json.dumps(records) else: @@ -42,12 +42,10 @@ def save_to_db(): record_count += 1 insert_record(record, doctype) - frappe.db.commit() - -def insert_record(record: Union[Dict, "Document"], doctype: str): - setattr(record, "doctype", doctype) +def insert_record(record: Union[dict, "Document"], doctype: str): try: + record.update({"doctype": doctype}) frappe.get_doc(record).insert() except Exception as e: frappe.logger().error(f"Error while inserting deferred {doctype} record: {e}") diff --git a/frappe/desk/desktop.py b/frappe/desk/desktop.py index 385151f754..a90950f411 100644 --- a/frappe/desk/desktop.py +++ b/frappe/desk/desktop.py @@ -40,7 +40,6 @@ class Workspace: self.allowed_modules = self.get_cached("user_allowed_modules", self.get_allowed_modules) self.doc = frappe.get_cached_doc("Workspace", self.page_name) - if ( self.doc and self.doc.module @@ -166,6 +165,8 @@ class Workspace: self.onboardings = {"items": self.get_onboardings()} + self.quick_lists = {"items": self.get_quick_lists()} + def _doctype_contains_a_record(self, name): exists = self.table_counts.get(name, False) @@ -284,6 +285,21 @@ class Workspace: return items + @handle_not_exist + def get_quick_lists(self): + items = [] + quick_lists = self.doc.quick_lists + + for item in quick_lists: + new_item = item.as_dict().copy() + + # Translate label + new_item["label"] = _(item.label) if item.label else _(item.document_type) + + items.append(new_item) + + return items + @handle_not_exist def get_onboardings(self): if self.onboarding_list: @@ -336,9 +352,10 @@ def get_desktop_page(page): "shortcuts": workspace.shortcuts, "cards": workspace.cards, "onboardings": workspace.onboardings, + "quick_lists": workspace.quick_lists, } except DoesNotExistError: - frappe.log_error(frappe.get_traceback()) + frappe.log_error("Workspace Missing") return {} @@ -452,6 +469,8 @@ def save_new_widget(doc, page, blocks, new_widgets): doc.charts.extend(new_widget(widgets.chart, "Workspace Chart", "charts")) if widgets.shortcut: doc.shortcuts.extend(new_widget(widgets.shortcut, "Workspace Shortcut", "shortcuts")) + if widgets.quick_list: + doc.quick_lists.extend(new_widget(widgets.quick_list, "Workspace Quick List", "quick_lists")) if widgets.card: doc.build_links_table_from_card(widgets.card) @@ -466,13 +485,13 @@ def save_new_widget(doc, page, blocks, new_widgets): # Error log body log = """ - page: {0} - config: {1} - exception: {2} + page: {} + config: {} + exception: {} """.format( page, json_config, e ) - frappe.log_error(log, _("Could not save customization")) + doc.log_error("Could not save customization", log) return False return True @@ -481,12 +500,12 @@ def save_new_widget(doc, page, blocks, new_widgets): def clean_up(original_page, blocks): page_widgets = {} - for wid in ["shortcut", "card", "chart"]: + for wid in ["shortcut", "card", "chart", "quick_list"]: # get list of widget's name from blocks page_widgets[wid] = [x["data"][wid + "_name"] for x in loads(blocks) if x["type"] == wid] - # shortcut & chart cleanup - for wid in ["shortcut", "chart"]: + # shortcut, chart & quick_list cleanup + for wid in ["shortcut", "chart", "quick_list"]: updated_widgets = [] original_page.get(wid + "s").reverse() diff --git a/frappe/desk/doctype/bulk_update/bulk_update.js b/frappe/desk/doctype/bulk_update/bulk_update.js index bb9cf2af51..017eee1480 100644 --- a/frappe/desk/doctype/bulk_update/bulk_update.js +++ b/frappe/desk/doctype/bulk_update/bulk_update.js @@ -1,65 +1,69 @@ // Copyright (c) 2016, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Bulk Update', { - refresh: function(frm) { - frm.set_query("document_type", function() { +frappe.ui.form.on("Bulk Update", { + refresh: function (frm) { + frm.set_query("document_type", function () { return { filters: [ - ['DocType', 'issingle', '=', 0], - ['DocType', 'name', 'not in', frappe.model.core_doctypes_list] - ] + ["DocType", "issingle", "=", 0], + ["DocType", "name", "not in", frappe.model.core_doctypes_list], + ], }; }); - frm.page.set_primary_action(__('Update'), function() { + frm.page.set_primary_action(__("Update"), function () { if (!frm.doc.update_value) { frappe.throw(__('Field "value" is mandatory. Please specify value to be updated')); } else { - frappe.call({ - method: 'frappe.desk.doctype.bulk_update.bulk_update.update', - args: { - doctype: frm.doc.document_type, - field: frm.doc.field, - value: frm.doc.update_value, - condition: frm.doc.condition, - limit: frm.doc.limit - }, - }).then(r => { - let failed = r.message; - if (!failed) failed = []; + frappe + .call({ + method: "frappe.desk.doctype.bulk_update.bulk_update.update", + args: { + doctype: frm.doc.document_type, + field: frm.doc.field, + value: frm.doc.update_value, + condition: frm.doc.condition, + limit: frm.doc.limit, + }, + }) + .then((r) => { + let failed = r.message; + if (!failed) failed = []; - if (failed.length && !r._server_messages) { - frappe.throw(__('Cannot update {0}', [failed.map(f => f.bold ? f.bold(): f).join(', ')])); - } else { - frappe.msgprint({ - title: __('Success'), - message: __('Updated Successfully'), - indicator: 'green' - }); - } + if (failed.length && !r._server_messages) { + frappe.throw( + __("Cannot update {0}", [ + failed.map((f) => (f.bold ? f.bold() : f)).join(", "), + ]) + ); + } else { + frappe.msgprint({ + title: __("Success"), + message: __("Updated Successfully"), + indicator: "green", + }); + } - frappe.hide_progress(); - frm.save(); - }); + frappe.hide_progress(); + frm.save(); + }); } }); }, - document_type: function(frm) { + document_type: function (frm) { // set field options - if(!frm.doc.document_type) return; + if (!frm.doc.document_type) return; - frappe.model.with_doctype(frm.doc.document_type, function() { - var options = $.map(frappe.get_meta(frm.doc.document_type).fields, - function(d) { - if(d.fieldname && frappe.model.no_value_type.indexOf(d.fieldtype)===-1) { - return d.fieldname; - } - return null; + frappe.model.with_doctype(frm.doc.document_type, function () { + var options = $.map(frappe.get_meta(frm.doc.document_type).fields, function (d) { + if (d.fieldname && frappe.model.no_value_type.indexOf(d.fieldtype) === -1) { + return d.fieldname; } - ); - frm.set_df_property('field', 'options', options); + return null; + }); + frm.set_df_property("field", "options", options); }); - } + }, }); diff --git a/frappe/desk/doctype/bulk_update/bulk_update.json b/frappe/desk/doctype/bulk_update/bulk_update.json index 0ec29a0dda..93458516fd 100644 --- a/frappe/desk/doctype/bulk_update/bulk_update.json +++ b/frappe/desk/doctype/bulk_update/bulk_update.json @@ -1,204 +1,77 @@ { - "allow_copy": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, - "creation": "2016-07-15 05:51:29.224123", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, + "actions": [], + "creation": "2016-07-15 05:51:29.224123", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "document_type", + "field", + "update_value", + "condition", + "limit" + ], "fields": [ { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "document_type", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Document Type", - "length": 0, - "no_copy": 0, - "options": "DocType", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "document_type", + "fieldtype": "Link", + "in_list_view": 1, + "label": "Document Type", + "options": "DocType", + "reqd": 1 + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "field", - "fieldtype": "Select", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Field", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "field", + "fieldtype": "Select", + "in_list_view": 1, + "label": "Field", + "reqd": 1 + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "update_value", - "fieldtype": "Small Text", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Update Value", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "update_value", + "fieldtype": "Small Text", + "in_list_view": 1, + "label": "Update Value", + "reqd": 1 + }, { - "allow_on_submit": 0, - "bold": 1, - "collapsible": 0, - "columns": 0, - "description": "SQL Conditions. Example: status=\"Open\"", - "fieldname": "condition", - "fieldtype": "Small Text", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Condition", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "bold": 1, + "description": "SQL Conditions. Example: status=\"Open\"", + "fieldname": "condition", + "fieldtype": "Small Text", + "label": "Condition" + }, { - "allow_on_submit": 0, - "bold": 1, - "collapsible": 0, - "columns": 0, - "default": "500", - "description": "Max 500 records at a time", - "fieldname": "limit", - "fieldtype": "Int", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Limit", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "bold": 1, + "default": "500", + "description": "Max 500 records at a time", + "fieldname": "limit", + "fieldtype": "Int", + "label": "Limit" } - ], - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - - "is_submittable": 0, - "issingle": 1, - "istable": 0, - "max_attachments": 0, - "modified": "2016-12-29 14:40:31.929701", - "modified_by": "Administrator", - "module": "Desk", - "name": "Bulk Update", - "name_case": "", - "owner": "Administrator", + ], + "issingle": 1, + "links": [], + "modified": "2022-08-03 12:20:50.742376", + "modified_by": "Administrator", + "module": "Desk", + "name": "Bulk Update", + "owner": "Administrator", "permissions": [ { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 1, - "delete": 1, - "email": 1, - "export": 0, - "if_owner": 0, - "import": 0, - "is_custom": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 0, - "role": "System Manager", - "set_user_permissions": 0, - "share": 1, - "submit": 0, + "create": 1, + "delete": 1, + "email": 1, + "print": 1, + "read": 1, + "role": "System Manager", + "share": 1, "write": 1 } - ], - "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1, - "track_seen": 0 + ], + "quick_entry": 1, + "sort_field": "modified", + "sort_order": "DESC", + "states": [], + "track_changes": 1 } \ No newline at end of file diff --git a/frappe/desk/doctype/bulk_update/bulk_update.py b/frappe/desk/doctype/bulk_update/bulk_update.py index 4d4e83a242..1e515bbc47 100644 --- a/frappe/desk/doctype/bulk_update/bulk_update.py +++ b/frappe/desk/doctype/bulk_update/bulk_update.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and contributors # License: MIT. See LICENSE @@ -24,7 +23,7 @@ def update(doctype, field, value, condition="", limit=500): frappe.throw(_("; not allowed in condition")) docnames = frappe.db.sql_list( - """select name from `tab{0}`{1} limit {2} offset 0""".format(doctype, condition, limit) + f"""select name from `tab{doctype}`{condition} limit {limit} offset 0""" ) data = {} data[field] = value @@ -46,7 +45,7 @@ def submit_cancel_or_update_docs(doctype, docnames, action="submit", data=None): message = "" if action == "submit" and doc.docstatus.is_draft(): doc.submit() - message = _("Submiting {0}").format(doctype) + message = _("Submitting {0}").format(doctype) elif action == "cancel" and doc.docstatus.is_submitted(): doc.cancel() message = _("Cancelling {0}").format(doctype) diff --git a/frappe/desk/doctype/calendar_view/calendar_view.js b/frappe/desk/doctype/calendar_view/calendar_view.js index a58a9555db..c302c1a4d8 100644 --- a/frappe/desk/doctype/calendar_view/calendar_view.js +++ b/frappe/desk/doctype/calendar_view/calendar_view.js @@ -1,35 +1,36 @@ // Copyright (c) 2017, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Calendar View', { - onload: function(frm) { - frm.trigger('reference_doctype'); +frappe.ui.form.on("Calendar View", { + onload: function (frm) { + frm.trigger("reference_doctype"); }, - refresh: function(frm) { + refresh: function (frm) { if (!frm.is_new()) { - frm.add_custom_button(__('Show Calendar'), - () => frappe.set_route('List', frm.doc.reference_doctype, 'Calendar', frm.doc.name)); + frm.add_custom_button(__("Show Calendar"), () => + frappe.set_route("List", frm.doc.reference_doctype, "Calendar", frm.doc.name) + ); } }, - reference_doctype: function(frm) { + reference_doctype: function (frm) { const { reference_doctype } = frm.doc; if (!reference_doctype) return; frappe.model.with_doctype(reference_doctype, () => { const meta = frappe.get_meta(reference_doctype); - const subject_options = meta.fields.filter( - df => !frappe.model.no_value_type.includes(df.fieldtype) - ).map(df => df.fieldname); + const subject_options = meta.fields + .filter((df) => !frappe.model.no_value_type.includes(df.fieldtype)) + .map((df) => df.fieldname); - const date_options = meta.fields.filter( - df => ['Date', 'Datetime'].includes(df.fieldtype) - ).map(df => df.fieldname); + const date_options = meta.fields + .filter((df) => ["Date", "Datetime"].includes(df.fieldtype)) + .map((df) => df.fieldname); - frm.set_df_property('subject_field', 'options', subject_options); - frm.set_df_property('start_date_field', 'options', date_options); - frm.set_df_property('end_date_field', 'options', date_options); + frm.set_df_property("subject_field", "options", subject_options); + frm.set_df_property("start_date_field", "options", date_options); + frm.set_df_property("end_date_field", "options", date_options); frm.refresh(); }); - } + }, }); diff --git a/frappe/desk/doctype/calendar_view/calendar_view.py b/frappe/desk/doctype/calendar_view/calendar_view.py index 01968e835d..1e187682ec 100644 --- a/frappe/desk/doctype/calendar_view/calendar_view.py +++ b/frappe/desk/doctype/calendar_view/calendar_view.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2017, Frappe Technologies and contributors # License: MIT. See LICENSE diff --git a/frappe/desk/doctype/console_log/console_log.js b/frappe/desk/doctype/console_log/console_log.js index 1ef4fdce59..9a980667ac 100644 --- a/frappe/desk/doctype/console_log/console_log.js +++ b/frappe/desk/doctype/console_log/console_log.js @@ -1,8 +1,7 @@ // Copyright (c) 2020, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Console Log', { +frappe.ui.form.on("Console Log", { // refresh: function(frm) { - // } }); diff --git a/frappe/desk/doctype/console_log/console_log.py b/frappe/desk/doctype/console_log/console_log.py index ebe93f535d..7e20afb22f 100644 --- a/frappe/desk/doctype/console_log/console_log.py +++ b/frappe/desk/doctype/console_log/console_log.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and contributors # License: MIT. See LICENSE diff --git a/frappe/desk/doctype/console_log/test_console_log.py b/frappe/desk/doctype/console_log/test_console_log.py index 409ac88299..0579098382 100644 --- a/frappe/desk/doctype/console_log/test_console_log.py +++ b/frappe/desk/doctype/console_log/test_console_log.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and Contributors # License: MIT. See LICENSE # import frappe diff --git a/frappe/desk/doctype/dashboard/dashboard.js b/frappe/desk/doctype/dashboard/dashboard.js index c640259cf2..9f584ca552 100644 --- a/frappe/desk/doctype/dashboard/dashboard.js +++ b/frappe/desk/doctype/dashboard/dashboard.js @@ -1,30 +1,30 @@ // Copyright (c) 2019, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Dashboard', { - refresh: function(frm) { - frm.add_custom_button(__("Show Dashboard"), - () => frappe.set_route('dashboard-view', frm.doc.name) +frappe.ui.form.on("Dashboard", { + refresh: function (frm) { + frm.add_custom_button(__("Show Dashboard"), () => + frappe.set_route("dashboard-view", frm.doc.name) ); if (!frappe.boot.developer_mode && frm.doc.is_standard) { frm.disable_form(); } - frm.set_query("chart", "charts", function() { + frm.set_query("chart", "charts", function () { return { filters: { is_public: 1, - } + }, }; }); - frm.set_query("card", "cards", function() { + frm.set_query("card", "cards", function () { return { filters: { is_public: 1, - } + }, }; }); - } + }, }); diff --git a/frappe/desk/doctype/dashboard/dashboard.py b/frappe/desk/doctype/dashboard/dashboard.py index 61e997836c..960431d220 100644 --- a/frappe/desk/doctype/dashboard/dashboard.py +++ b/frappe/desk/doctype/dashboard/dashboard.py @@ -115,7 +115,7 @@ def get_non_standard_warning_message(non_standard_docs_map): message = _("""Please set the following documents in this Dashboard as standard first.""") def get_html(docs, doctype): - html = "

    {}

    ".format(frappe.bold(doctype)) + html = f"

    {frappe.bold(doctype)}

    " for doc in docs: html += ''.format( doctype=doctype, doc=doc diff --git a/frappe/desk/doctype/dashboard/dashboard_list.js b/frappe/desk/doctype/dashboard/dashboard_list.js index d60a324048..80ebbd4355 100644 --- a/frappe/desk/doctype/dashboard/dashboard_list.js +++ b/frappe/desk/doctype/dashboard/dashboard_list.js @@ -1,4 +1,4 @@ -frappe.listview_settings['Dashboard'] = { +frappe.listview_settings["Dashboard"] = { button: { show(doc) { return doc.name; @@ -7,10 +7,10 @@ frappe.listview_settings['Dashboard'] = { return frappe.utils.icon("dashboard-list", "sm"); }, get_description(doc) { - return __('View {0}', [`${doc.name}`]); + return __("View {0}", [`${doc.name}`]); }, action(doc) { - frappe.set_route('dashboard-view', doc.name); - } + frappe.set_route("dashboard-view", doc.name); + }, }, -}; \ No newline at end of file +}; diff --git a/frappe/desk/doctype/dashboard/test_dashboard.py b/frappe/desk/doctype/dashboard/test_dashboard.py index ee3d1848e2..d2ba871509 100644 --- a/frappe/desk/doctype/dashboard/test_dashboard.py +++ b/frappe/desk/doctype/dashboard/test_dashboard.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # License: MIT. See LICENSE import unittest diff --git a/frappe/desk/doctype/dashboard_chart/dashboard_chart.js b/frappe/desk/doctype/dashboard_chart/dashboard_chart.js index 0b93786e8e..b1c23eba28 100644 --- a/frappe/desk/doctype/dashboard_chart/dashboard_chart.js +++ b/frappe/desk/doctype/dashboard_chart/dashboard_chart.js @@ -1,42 +1,44 @@ // Copyright (c) 2019, Frappe Technologies and contributors // For license information, please see license.txt -frappe.provide('frappe.dashboards.chart_sources'); +frappe.provide("frappe.dashboards.chart_sources"); -frappe.ui.form.on('Dashboard Chart', { - setup: function(frm) { +frappe.ui.form.on("Dashboard Chart", { + setup: function (frm) { // fetch timeseries from source - frm.add_fetch('source', 'timeseries', 'timeseries'); + frm.add_fetch("source", "timeseries", "timeseries"); }, - before_save: function(frm) { - let dynamic_filters = JSON.parse(frm.doc.dynamic_filters_json || 'null'); - let static_filters = JSON.parse(frm.doc.filters_json || 'null'); - static_filters = - frappe.dashboard_utils.remove_common_static_filter_values(static_filters, dynamic_filters); + before_save: function (frm) { + let dynamic_filters = JSON.parse(frm.doc.dynamic_filters_json || "null"); + let static_filters = JSON.parse(frm.doc.filters_json || "null"); + static_filters = frappe.dashboard_utils.remove_common_static_filter_values( + static_filters, + dynamic_filters + ); - frm.set_value('filters_json', JSON.stringify(static_filters)); - frm.trigger('show_filters'); + frm.set_value("filters_json", JSON.stringify(static_filters)); + frm.trigger("show_filters"); }, - refresh: function(frm) { + refresh: function (frm) { frm.chart_filters = null; frm.is_disabled = !frappe.boot.developer_mode && frm.doc.is_standard; if (frm.is_disabled) { - !frm.doc.custom_options && frm.set_df_property('chart_options_section', 'hidden', 1); + !frm.doc.custom_options && frm.set_df_property("chart_options_section", "hidden", 1); frm.disable_form(); } - frm.add_custom_button('Add Chart to Dashboard', () => { + frm.add_custom_button("Add Chart to Dashboard", () => { const dialog = frappe.dashboard_utils.get_add_to_dashboard_dialog( frm.doc.name, - 'Dashboard Chart', - 'frappe.desk.doctype.dashboard_chart.dashboard_chart.add_chart_to_dashboard' + "Dashboard Chart", + "frappe.desk.doctype.dashboard_chart.dashboard_chart.add_chart_to_dashboard" ); if (!frm.doc.chart_name) { - frappe.msgprint(__('Please create chart first')); + frappe.msgprint(__("Please create chart first")); } else { dialog.show(); } @@ -45,199 +47,227 @@ frappe.ui.form.on('Dashboard Chart', { frm.set_df_property("filters_section", "hidden", 1); frm.set_df_property("dynamic_filters_section", "hidden", 1); - frm.trigger('set_parent_document_type'); - frm.trigger('set_time_series'); - frm.set_query('document_type', function() { + frm.trigger("set_parent_document_type"); + frm.trigger("set_time_series"); + frm.set_query("document_type", function () { return { filters: { - 'issingle': false - } - } + issingle: false, + }, + }; }); - frm.trigger('update_options'); - frm.trigger('set_heatmap_year_options'); + frm.trigger("update_options"); + frm.trigger("set_heatmap_year_options"); if (frm.doc.report_name) { - frm.trigger('set_chart_report_filters'); + frm.trigger("set_chart_report_filters"); } }, - is_standard: function(frm) { + is_standard: function (frm) { if (frappe.boot.developer_mode && frm.doc.is_standard) { - frm.trigger('render_dynamic_filters_table'); + frm.trigger("render_dynamic_filters_table"); } else { frm.set_df_property("dynamic_filters_section", "hidden", 1); } }, - source: function(frm) { + source: function (frm) { frm.trigger("show_filters"); }, - set_heatmap_year_options: function(frm) { - if (frm.doc.type == 'Heatmap') { - frappe.db.get_doc('System Settings').then(doc => { + set_heatmap_year_options: function (frm) { + if (frm.doc.type == "Heatmap") { + frappe.db.get_doc("System Settings").then((doc) => { const creation_date = doc.creation; - frm.set_df_property('heatmap_year', 'options', frappe.dashboard_utils.get_years_since_creation(creation_date)); + frm.set_df_property( + "heatmap_year", + "options", + frappe.dashboard_utils.get_years_since_creation(creation_date) + ); }); } }, - chart_type: function(frm) { - frm.trigger('set_time_series'); - if (frm.doc.chart_type == 'Report') { - frm.set_query('report_name', () => { + chart_type: function (frm) { + frm.trigger("set_time_series"); + if (frm.doc.chart_type == "Report") { + frm.set_query("report_name", () => { return { filters: { - 'report_type': ['!=', 'Report Builder'] - } - } + report_type: ["!=", "Report Builder"], + }, + }; }); } else { - frm.set_value('document_type', ''); + frm.set_value("document_type", ""); } }, - set_time_series: function(frm) { + set_time_series: function (frm) { // set timeseries based on chart type - if (['Count', 'Average', 'Sum'].includes(frm.doc.chart_type)) { - frm.set_value('timeseries', 1); + if (["Count", "Average", "Sum"].includes(frm.doc.chart_type)) { + frm.set_value("timeseries", 1); } else { - frm.set_value('timeseries', 0); + frm.set_value("timeseries", 0); } }, - document_type: function(frm) { + document_type: function (frm) { // update `based_on` options based on date / datetime fields - frm.set_value('source', ''); - frm.set_value('based_on', ''); - frm.set_value('value_based_on', ''); - frm.set_value('parent_document_type', ''); - frm.set_value('filters_json', '[]'); - frm.set_value('dynamic_filters_json', '[]'); - frm.trigger('update_options'); - frm.trigger('set_parent_document_type'); + frm.set_value("source", ""); + frm.set_value("based_on", ""); + frm.set_value("value_based_on", ""); + frm.set_value("parent_document_type", ""); + frm.set_value("filters_json", "[]"); + frm.set_value("dynamic_filters_json", "[]"); + frm.trigger("update_options"); + frm.trigger("set_parent_document_type"); }, - report_name: function(frm) { - frm.set_value('x_field', ''); - frm.set_value('y_axis', []); - frm.set_df_property('x_field', 'options', []); - frm.set_value('filters_json', '{}'); - frm.set_value('dynamic_filters_json', '{}'); - frm.set_value('use_report_chart', 0); - frm.trigger('set_chart_report_filters'); + report_name: function (frm) { + frm.set_value("x_field", ""); + frm.set_value("y_axis", []); + frm.set_df_property("x_field", "options", []); + frm.set_value("filters_json", "{}"); + frm.set_value("dynamic_filters_json", "{}"); + frm.set_value("use_report_chart", 0); + frm.trigger("set_chart_report_filters"); }, - set_chart_report_filters: function(frm) { + set_chart_report_filters: function (frm) { let report_name = frm.doc.report_name; if (report_name) { if (frm.doc.filters_json.length > 2) { - frm.trigger('show_filters'); - frm.trigger('set_chart_field_options'); + frm.trigger("show_filters"); + frm.trigger("set_chart_field_options"); } else { - frappe.report_utils.get_report_filters(report_name).then(filters => { + frappe.report_utils.get_report_filters(report_name).then((filters) => { if (filters) { frm.chart_filters = filters; let filter_values = frappe.report_utils.get_filter_values(filters); - frm.set_value('filters_json', JSON.stringify(filter_values)); + frm.set_value("filters_json", JSON.stringify(filter_values)); } - frm.trigger('show_filters'); - frm.trigger('set_chart_field_options'); + frm.trigger("show_filters"); + frm.trigger("set_chart_field_options"); }); } - } }, - use_report_chart: function(frm) { - !frm.doc.use_report_chart && frm.trigger('set_chart_field_options'); + use_report_chart: function (frm) { + !frm.doc.use_report_chart && frm.trigger("set_chart_field_options"); }, - set_chart_field_options: function(frm) { + set_chart_field_options: function (frm) { let filters = frm.doc.filters_json.length > 2 ? JSON.parse(frm.doc.filters_json) : null; if (frm.doc.dynamic_filters_json && frm.doc.dynamic_filters_json.length > 2) { filters = frappe.dashboard_utils.get_all_filters(frm.doc); } - frappe.xcall( - 'frappe.desk.query_report.run', - { + frappe + .xcall("frappe.desk.query_report.run", { report_name: frm.doc.report_name, filters: filters, - ignore_prepared_report: 1 - } - ).then(data => { - frm.report_data = data; - let report_has_chart = Boolean(data.chart); + ignore_prepared_report: 1, + }) + .then((data) => { + frm.report_data = data; + let report_has_chart = Boolean(data.chart); - frm.set_df_property('use_report_chart', 'hidden', !report_has_chart); + frm.set_df_property("use_report_chart", "hidden", !report_has_chart); - if (!frm.doc.use_report_chart) { - if (data.result.length) { - frm.field_options = frappe.report_utils.get_field_options_from_report(data.columns, data); - frm.set_df_property('x_field', 'options', frm.field_options.non_numeric_fields); - if (!frm.field_options.numeric_fields.length) { - frappe.msgprint(__("Report has no numeric fields, please change the Report Name")); + if (!frm.doc.use_report_chart) { + if (data.result.length) { + frm.field_options = frappe.report_utils.get_field_options_from_report( + data.columns, + data + ); + frm.set_df_property( + "x_field", + "options", + frm.field_options.non_numeric_fields + ); + if (!frm.field_options.numeric_fields.length) { + frappe.msgprint( + __("Report has no numeric fields, please change the Report Name") + ); + } else { + let y_field_df = frappe.meta.get_docfield( + "Dashboard Chart Field", + "y_field", + frm.doc.name + ); + y_field_df.options = frm.field_options.numeric_fields; + } } else { - let y_field_df = frappe.meta.get_docfield('Dashboard Chart Field', 'y_field', frm.doc.name); - y_field_df.options = frm.field_options.numeric_fields; + frappe.msgprint( + __( + "Report has no data, please modify the filters or change the Report Name" + ) + ); } } else { - frappe.msgprint(__('Report has no data, please modify the filters or change the Report Name')); + frm.set_value("use_report_chart", 1); + frm.set_df_property("use_report_chart", "hidden", false); } - } else { - frm.set_value('use_report_chart', 1); - frm.set_df_property('use_report_chart', 'hidden', false); - } - }); + }); }, - timespan: function(frm) { + timespan: function (frm) { const time_interval_options = { "Select Date Range": ["Quarterly", "Monthly", "Weekly", "Daily"], "All Time": ["Yearly", "Monthly"], "Last Year": ["Quarterly", "Monthly", "Weekly", "Daily"], "Last Quarter": ["Monthly", "Weekly", "Daily"], "Last Month": ["Weekly", "Daily"], - "Last Week": ["Daily"] + "Last Week": ["Daily"], }; if (frm.doc.timespan) { - frm.set_df_property('time_interval', 'options', time_interval_options[frm.doc.timespan]); + frm.set_df_property( + "time_interval", + "options", + time_interval_options[frm.doc.timespan] + ); } }, - update_options: function(frm) { + update_options: function (frm) { let doctype = frm.doc.document_type; let date_fields = [ - {label: __('Created On'), value: 'creation'}, - {label: __('Last Modified On'), value: 'modified'} + { label: __("Created On"), value: "creation" }, + { label: __("Last Modified On"), value: "modified" }, ]; let value_fields = []; - let group_by_fields = [{label: 'Created By', value: 'owner'}]; + let group_by_fields = [{ label: "Created By", value: "owner" }]; let aggregate_function_fields = []; - let update_form = function() { + let update_form = function () { // update select options - frm.set_df_property('based_on', 'options', date_fields); - frm.set_df_property('value_based_on', 'options', value_fields); - frm.set_df_property('group_by_based_on', 'options', group_by_fields); - frm.set_df_property('aggregate_function_based_on', 'options', aggregate_function_fields); + frm.set_df_property("based_on", "options", date_fields); + frm.set_df_property("value_based_on", "options", value_fields); + frm.set_df_property("group_by_based_on", "options", group_by_fields); + frm.set_df_property( + "aggregate_function_based_on", + "options", + aggregate_function_fields + ); frm.trigger("show_filters"); - } - + }; if (doctype) { frappe.model.with_doctype(doctype, () => { // get all date and datetime fields - frappe.get_meta(doctype).fields.map(df => { - if (['Date', 'Datetime'].includes(df.fieldtype)) { - date_fields.push({label: df.label, value: df.fieldname}); + frappe.get_meta(doctype).fields.map((df) => { + if (["Date", "Datetime"].includes(df.fieldtype)) { + date_fields.push({ label: df.label, value: df.fieldname }); } - if (['Int', 'Float', 'Currency', 'Percent', 'Duration'].includes(df.fieldtype)) { - value_fields.push({label: df.label, value: df.fieldname}); - aggregate_function_fields.push({label: df.label, value: df.fieldname}); + if ( + ["Int", "Float", "Currency", "Percent", "Duration"].includes(df.fieldtype) + ) { + value_fields.push({ label: df.label, value: df.fieldname }); + aggregate_function_fields.push({ label: df.label, value: df.fieldname }); } - if (['Link', 'Select'].includes(df.fieldtype)) { - group_by_fields.push({label: df.label, value: df.fieldname}); + if (["Link", "Select"].includes(df.fieldtype)) { + group_by_fields.push({ label: df.label, value: df.fieldname }); } }); update_form(); @@ -246,92 +276,89 @@ frappe.ui.form.on('Dashboard Chart', { // update select options update_form(); } - }, - show_filters: function(frm) { + show_filters: function (frm) { frm.chart_filters = []; - frappe.dashboard_utils.get_filters_for_chart_type(frm.doc).then(filters => { + frappe.dashboard_utils.get_filters_for_chart_type(frm.doc).then((filters) => { if (filters) { frm.chart_filters = filters; } - frm.trigger('render_filters_table'); + frm.trigger("render_filters_table"); if (frappe.boot.developer_mode && frm.doc.is_standard) { - frm.trigger('render_dynamic_filters_table'); + frm.trigger("render_dynamic_filters_table"); } }); }, - render_filters_table: function(frm) { + render_filters_table: function (frm) { frm.set_df_property("filters_section", "hidden", 0); - let is_document_type = frm.doc.chart_type!== 'Report' && frm.doc.chart_type!=='Custom'; - let is_dynamic_filter = f => ['Date', 'DateRange'].includes(f.fieldtype) && f.default; + let is_document_type = frm.doc.chart_type !== "Report" && frm.doc.chart_type !== "Custom"; + let is_dynamic_filter = (f) => ["Date", "DateRange"].includes(f.fieldtype) && f.default; - let wrapper = $(frm.get_field('filters_json').wrapper).empty(); + let wrapper = $(frm.get_field("filters_json").wrapper).empty(); let table = $(` - - - + + +
    ${__('Filter')}${__('Condition')}${__('Value')}${__("Filter")}${__("Condition")}${__("Value")}
    `).appendTo(wrapper); $(`

    ${__("Click table to edit")}

    `).appendTo(wrapper); - let filters = JSON.parse(frm.doc.filters_json || '[]'); + let filters = JSON.parse(frm.doc.filters_json || "[]"); var filters_set = false; // Set dynamic filters for reports - if (frm.doc.chart_type == 'Report') { + if (frm.doc.chart_type == "Report") { let set_filters = false; - frm.chart_filters.forEach(f => { + frm.chart_filters.forEach((f) => { if (is_dynamic_filter(f)) { filters[f.fieldname] = f.default; set_filters = true; } }); - set_filters && frm.set_value('filters_json', JSON.stringify(filters)); + set_filters && frm.set_value("filters_json", JSON.stringify(filters)); } let fields = []; if (is_document_type) { fields = [ { - fieldtype: 'HTML', - fieldname: 'filter_area', - } + fieldtype: "HTML", + fieldname: "filter_area", + }, ]; if (filters.length > 0) { - filters.forEach( filter => { - const filter_row = - $(`
    ${filter[1]} ${filter[2] || ""} ${filter[3]}
    ${f.label} ${condition} ${filters[f.fieldname] || ""}
    ${__("Click to Set Filters")}
    + frm.dynamic_filter_table = + $(`
    - - - + + +
    ${__('Filter')}${__('Condition')}${__('Value')}${__("Filter")}${__("Condition")}${__("Value")}
    `).appendTo(wrapper); - frm.dynamic_filters = frm.doc.dynamic_filters_json && frm.doc.dynamic_filters_json.length > 2 - ? JSON.parse(frm.doc.dynamic_filters_json) - : null; + frm.dynamic_filters = + frm.doc.dynamic_filters_json && frm.doc.dynamic_filters_json.length > 2 + ? JSON.parse(frm.doc.dynamic_filters_json) + : null; - frm.trigger('set_dynamic_filters_in_table'); + frm.trigger("set_dynamic_filters_in_table"); - let filters = JSON.parse(frm.doc.filters_json || '[]'); + let filters = JSON.parse(frm.doc.filters_json || "[]"); let fields = frappe.dashboard_utils.get_fields_for_dynamic_filter_dialog( - is_document_type, filters, frm.dynamic_filters + is_document_type, + filters, + frm.dynamic_filters ); - frm.dynamic_filter_table.on('click', () => { + frm.dynamic_filter_table.on("click", () => { let dialog = new frappe.ui.Dialog({ - title: __('Set Dynamic Filters'), + title: __("Set Dynamic Filters"), fields: fields, primary_action: () => { let values = dialog.get_values(); @@ -436,19 +468,19 @@ frappe.ui.form.on('Dashboard Chart', { let dynamic_filters = []; for (let key of Object.keys(values)) { if (is_document_type) { - let [doctype, fieldname] = key.split(':'); - dynamic_filters.push([doctype, fieldname, '=', values[key]]); + let [doctype, fieldname] = key.split(":"); + dynamic_filters.push([doctype, fieldname, "=", values[key]]); } } if (is_document_type) { - frm.set_value('dynamic_filters_json', JSON.stringify(dynamic_filters)); + frm.set_value("dynamic_filters_json", JSON.stringify(dynamic_filters)); } else { - frm.set_value('dynamic_filters_json', JSON.stringify(values)); + frm.set_value("dynamic_filters_json", JSON.stringify(values)); } - frm.trigger('set_dynamic_filters_in_table'); + frm.trigger("set_dynamic_filters_in_table"); }, - primary_action_label: "Set" + primary_action_label: "Set", }); dialog.show(); @@ -456,71 +488,70 @@ frappe.ui.form.on('Dashboard Chart', { }); }, - set_dynamic_filters_in_table: function(frm) { - frm.dynamic_filters = frm.doc.dynamic_filters_json && frm.doc.dynamic_filters_json.length > 2 - ? JSON.parse(frm.doc.dynamic_filters_json) - : null; + set_dynamic_filters_in_table: function (frm) { + frm.dynamic_filters = + frm.doc.dynamic_filters_json && frm.doc.dynamic_filters_json.length > 2 + ? JSON.parse(frm.doc.dynamic_filters_json) + : null; if (!frm.dynamic_filters) { const filter_row = $(` ${__("Click to Set Dynamic Filters")}`); - frm.dynamic_filter_table.find('tbody').html(filter_row); + frm.dynamic_filter_table.find("tbody").html(filter_row); } else { - let filter_rows = ''; + let filter_rows = ""; if ($.isArray(frm.dynamic_filters)) { - frm.dynamic_filters.forEach(filter => { - filter_rows += - ` + frm.dynamic_filters.forEach((filter) => { + filter_rows += ` ${filter[1]} ${filter[2] || ""} ${filter[3]} `; }); } else { - let condition = '='; + let condition = "="; for (let [key, val] of Object.entries(frm.dynamic_filters)) { - filter_rows += - ` + filter_rows += ` ${key} ${condition} ${val || ""} - ` - ; + `; } } - frm.dynamic_filter_table.find('tbody').html(filter_rows); + frm.dynamic_filter_table.find("tbody").html(filter_rows); } }, - set_parent_document_type: async function(frm) { + set_parent_document_type: async function (frm) { let document_type = frm.doc.document_type; - let doc_is_table = document_type && - (await frappe.db.get_value('DocType', document_type, 'istable')).message.istable; + let doc_is_table = + document_type && + (await frappe.db.get_value("DocType", document_type, "istable")).message.istable; - frm.set_df_property('parent_document_type', 'hidden', !doc_is_table); + frm.set_df_property("parent_document_type", "hidden", !doc_is_table); if (document_type && doc_is_table) { - let parent = await frappe.db.get_list('DocField', { + let parent = await frappe.db.get_list("DocField", { filters: { - 'fieldtype': 'Table', - 'options': document_type + fieldtype: "Table", + options: document_type, }, - fields: ['parent'] + fields: ["parent"], }); - parent && frm.set_query('parent_document_type', function() { - return { - filters: { - "name": ['in', parent.map(({ parent }) => parent)] - } - }; - }); + parent && + frm.set_query("parent_document_type", function () { + return { + filters: { + name: ["in", parent.map(({ parent }) => parent)], + }, + }; + }); if (parent.length === 1) { - frm.set_value('parent_document_type', parent[0].parent); + frm.set_value("parent_document_type", parent[0].parent); } } - } - + }, }); diff --git a/frappe/desk/doctype/dashboard_chart/dashboard_chart.json b/frappe/desk/doctype/dashboard_chart/dashboard_chart.json index a5d30c10e5..a5aa6cc20a 100644 --- a/frappe/desk/doctype/dashboard_chart/dashboard_chart.json +++ b/frappe/desk/doctype/dashboard_chart/dashboard_chart.json @@ -42,7 +42,8 @@ "column_break_2", "color", "section_break_10", - "last_synced_on" + "last_synced_on", + "roles" ], "fields": [ { @@ -277,13 +278,20 @@ "fieldtype": "Link", "label": "Parent Document Type", "options": "DocType" + }, + { + "fieldname": "roles", + "fieldtype": "Table", + "label": "Roles", + "options": "Has Role" } ], "links": [], - "modified": "2021-11-09 17:18:11.456145", + "modified": "2022-07-27 11:09:09.203236", "modified_by": "Administrator", "module": "Desk", "name": "Dashboard Chart", + "naming_rule": "By fieldname", "owner": "Administrator", "permissions": [ { @@ -323,5 +331,6 @@ ], "sort_field": "modified", "sort_order": "DESC", + "states": [], "track_changes": 1 } \ No newline at end of file diff --git a/frappe/desk/doctype/dashboard_chart/dashboard_chart.py b/frappe/desk/doctype/dashboard_chart/dashboard_chart.py index 246c9ad4cd..aa76932050 100644 --- a/frappe/desk/doctype/dashboard_chart/dashboard_chart.py +++ b/frappe/desk/doctype/dashboard_chart/dashboard_chart.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and contributors # License: MIT. See LICENSE @@ -7,13 +6,14 @@ import json import frappe from frappe import _ -from frappe.boot import get_allowed_reports +from frappe.boot import get_allowed_report_names from frappe.config import get_modules_from_all_apps_for_user from frappe.model.document import Document from frappe.model.naming import append_number_if_name_exists from frappe.modules.export_file import export_to_files -from frappe.utils import cint, get_datetime, getdate, now_datetime, nowdate +from frappe.utils import cint, get_datetime, getdate, has_common, now_datetime, nowdate from frappe.utils.dashboard import cache_source +from frappe.utils.data import format_date from frappe.utils.dateutils import ( get_dates_from_timegrain, get_from_date_from_timespan, @@ -40,10 +40,7 @@ def get_permission_query_conditions(user): allowed_doctypes = [ frappe.db.escape(doctype) for doctype in frappe.permissions.get_doctypes_with_read() ] - allowed_reports = [ - frappe.db.escape(key) if type(key) == str else key.encode("UTF8") - for key in get_allowed_reports() - ] + allowed_reports = [frappe.db.escape(report) for report in get_allowed_report_names()] allowed_modules = [ frappe.db.escape(module.get("module_name")) for module in get_modules_from_all_apps_for_user() ] @@ -83,16 +80,18 @@ def has_permission(doc, ptype, user): return True if doc.chart_type == "Report": - allowed_reports = [ - key if type(key) == str else key.encode("UTF8") for key in get_allowed_reports() - ] - if doc.report_name in allowed_reports: + if doc.report_name in get_allowed_report_names(): return True else: allowed_doctypes = frappe.permissions.get_doctypes_with_read() if doc.document_type in allowed_doctypes: return True + if doc.roles: + allowed = [d.role for d in doc.roles] + if has_common(roles, allowed): + return True + return False @@ -211,7 +210,7 @@ def get_chart_config(chart, filters, timespan, timegrain, from_date, to_date): data = frappe.db.get_list( doctype, - fields=["{} as _unit".format(datefield), "SUM({})".format(value_field), "COUNT(*)"], + fields=[f"{datefield} as _unit", f"SUM({value_field})", "COUNT(*)"], filters=filters, group_by="_unit", order_by="_unit asc", @@ -221,13 +220,16 @@ def get_chart_config(chart, filters, timespan, timegrain, from_date, to_date): result = get_result(data, timegrain, from_date, to_date, chart.chart_type) - chart_config = { - "labels": [get_period(r[0], timegrain) for r in result], + return { + "labels": [ + format_date(get_period(r[0], timegrain)) + if timegrain in ("Daily", "Weekly") + else get_period(r[0], timegrain) + for r in result + ], "datasets": [{"name": chart.name, "values": [r[1] for r in result]}], } - return chart_config - def get_heatmap_chart_config(chart, filters, heatmap_year): aggregate_function = get_aggregate_function(chart.chart_type) @@ -238,13 +240,13 @@ def get_heatmap_chart_config(chart, filters, heatmap_year): year_start_date = datetime.date(year, 1, 1).strftime("%Y-%m-%d") next_year_start_date = datetime.date(year + 1, 1, 1).strftime("%Y-%m-%d") - filters.append([doctype, datefield, ">", "{date}".format(date=year_start_date), False]) - filters.append([doctype, datefield, "<", "{date}".format(date=next_year_start_date), False]) + filters.append([doctype, datefield, ">", f"{year_start_date}", False]) + filters.append([doctype, datefield, "<", f"{next_year_start_date}", False]) if frappe.db.db_type == "mariadb": - timestamp_field = "unix_timestamp({datefield})".format(datefield=datefield) + timestamp_field = f"unix_timestamp({datefield})" else: - timestamp_field = "extract(epoch from timestamp {datefield})".format(datefield=datefield) + timestamp_field = f"extract(epoch from timestamp {datefield})" data = dict( frappe.db.get_all( @@ -256,9 +258,9 @@ def get_heatmap_chart_config(chart, filters, heatmap_year): ), ], filters=filters, - group_by="date({datefield})".format(datefield=datefield), + group_by=f"date({datefield})", as_list=1, - order_by="{datefield} asc".format(datefield=datefield), + order_by=f"{datefield} asc", ignore_ifnull=True, ) ) @@ -280,7 +282,7 @@ def get_group_by_chart_config(chart, filters): data = frappe.db.get_list( doctype, fields=[ - "{} as name".format(group_by_field), + f"{group_by_field} as name", "{aggregate_function}({value_field}) as count".format( aggregate_function=aggregate_function, value_field=value_field ), @@ -347,7 +349,7 @@ def get_charts_for_user(doctype, txt, searchfield, start, page_len, filters): class DashboardChart(Document): def on_update(self): - frappe.cache().delete_key("chart-data:{}".format(self.name)) + frappe.cache().delete_key(f"chart-data:{self.name}") if frappe.conf.developer_mode and self.is_standard: export_to_files(record_list=[["Dashboard Chart", self.name]], record_module=self.module) diff --git a/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py b/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py index 94ea1af35c..820f3c0555 100644 --- a/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py +++ b/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py @@ -1,7 +1,6 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # License: MIT. See LICENSE -import unittest + from datetime import datetime from unittest.mock import patch @@ -9,11 +8,12 @@ from dateutil.relativedelta import relativedelta import frappe from frappe.desk.doctype.dashboard_chart.dashboard_chart import get +from frappe.tests.utils import FrappeTestCase from frappe.utils import formatdate, get_last_day, getdate from frappe.utils.dateutils import get_period, get_period_ending -class TestDashboardChart(unittest.TestCase): +class TestDashboardChart(FrappeTestCase): def test_period_ending(self): self.assertEqual(get_period_ending("2019-04-10", "Daily"), getdate("2019-04-10")) @@ -57,8 +57,6 @@ class TestDashboardChart(unittest.TestCase): self.assertEqual(result.get("labels")[idx], get_period(month)) cur_date += relativedelta(months=1) - frappe.db.rollback() - def test_empty_dashboard_chart(self): if frappe.db.exists("Dashboard Chart", "Test Empty Dashboard Chart"): frappe.delete_doc("Dashboard Chart", "Test Empty Dashboard Chart") @@ -89,8 +87,6 @@ class TestDashboardChart(unittest.TestCase): self.assertEqual(result.get("labels")[idx], get_period(month)) cur_date += relativedelta(months=1) - frappe.db.rollback() - def test_chart_wih_one_value(self): if frappe.db.exists("Dashboard Chart", "Test Empty Dashboard Chart 2"): frappe.delete_doc("Dashboard Chart", "Test Empty Dashboard Chart 2") @@ -127,8 +123,6 @@ class TestDashboardChart(unittest.TestCase): # only 1 data point with value self.assertEqual(result.get("datasets")[0].get("values")[2], 0) - frappe.db.rollback() - def test_group_by_chart_type(self): if frappe.db.exists("Dashboard Chart", "Test Group By Dashboard Chart"): frappe.delete_doc("Dashboard Chart", "Test Group By Dashboard Chart") @@ -151,8 +145,6 @@ class TestDashboardChart(unittest.TestCase): self.assertEqual(result.get("datasets")[0].get("values")[0], todo_status_count) - frappe.db.rollback() - def test_daily_dashboard_chart(self): insert_test_records() @@ -180,11 +172,10 @@ class TestDashboardChart(unittest.TestCase): self.assertEqual(result.get("datasets")[0].get("values"), [200.0, 400.0, 300.0, 0.0, 100.0, 0.0]) self.assertEqual( - result.get("labels"), ["06-01-19", "07-01-19", "08-01-19", "09-01-19", "10-01-19", "11-01-19"] + result.get("labels"), + ["06-01-2019", "07-01-2019", "08-01-2019", "09-01-2019", "10-01-2019", "11-01-2019"], ) - frappe.db.rollback() - def test_weekly_dashboard_chart(self): insert_test_records() @@ -212,9 +203,7 @@ class TestDashboardChart(unittest.TestCase): result = get(chart_name="Test Weekly Dashboard Chart", refresh=1) self.assertEqual(result.get("datasets")[0].get("values"), [50.0, 300.0, 800.0, 0.0]) - self.assertEqual(result.get("labels"), ["30-12-18", "06-01-19", "13-01-19", "20-01-19"]) - - frappe.db.rollback() + self.assertEqual(result.get("labels"), ["12-30-2018", "06-01-2019", "01-13-2019", "01-20-2019"]) def test_avg_dashboard_chart(self): insert_test_records() @@ -241,10 +230,39 @@ class TestDashboardChart(unittest.TestCase): with patch.object(frappe.utils.data, "get_first_day_of_the_week", return_value="Monday"): result = get(chart_name="Test Average Dashboard Chart", refresh=1) - self.assertEqual(result.get("labels"), ["30-12-18", "06-01-19", "13-01-19", "20-01-19"]) + self.assertEqual(result.get("labels"), ["12-30-2018", "06-01-2019", "01-13-2019", "01-20-2019"]) self.assertEqual(result.get("datasets")[0].get("values"), [50.0, 150.0, 266.6666666666667, 0.0]) - frappe.db.rollback() + def test_user_date_label_dashboard_chart(self): + frappe.delete_doc_if_exists("Dashboard Chart", "Test Dashboard Chart Date Label") + + frappe.get_doc( + dict( + doctype="Dashboard Chart", + chart_name="Test Dashboard Chart Date Label", + chart_type="Count", + document_type="DocType", + based_on="creation", + timespan="Select Date Range", + time_interval="Weekly", + from_date=datetime(2018, 12, 30), + to_date=datetime(2019, 1, 15), + filters_json="[]", + timeseries=1, + ) + ).insert() + + with patch.object(frappe.utils.data, "get_user_date_format", return_value="dd.mm.yyyy"): + result = get(chart_name="Test Dashboard Chart Date Label") + self.assertEqual( + sorted(result.get("labels")), sorted(["01.05.2019", "01.12.2019", "19.01.2019"]) + ) + + with patch.object(frappe.utils.data, "get_user_date_format", return_value="mm-dd-yyyy"): + result = get(chart_name="Test Dashboard Chart Date Label") + self.assertEqual( + sorted(result.get("labels")), sorted(["01-19-2019", "05-01-2019", "12-01-2019"]) + ) def insert_test_records(): diff --git a/frappe/desk/doctype/dashboard_chart_field/dashboard_chart_field.py b/frappe/desk/doctype/dashboard_chart_field/dashboard_chart_field.py index 41f35d2ee4..adc03663a2 100644 --- a/frappe/desk/doctype/dashboard_chart_field/dashboard_chart_field.py +++ b/frappe/desk/doctype/dashboard_chart_field/dashboard_chart_field.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and contributors # License: MIT. See LICENSE diff --git a/frappe/desk/doctype/dashboard_chart_link/dashboard_chart_link.py b/frappe/desk/doctype/dashboard_chart_link/dashboard_chart_link.py index b2a7caefeb..4d98b69458 100644 --- a/frappe/desk/doctype/dashboard_chart_link/dashboard_chart_link.py +++ b/frappe/desk/doctype/dashboard_chart_link/dashboard_chart_link.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and contributors # License: MIT. See LICENSE diff --git a/frappe/desk/doctype/dashboard_chart_source/dashboard_chart_source.js b/frappe/desk/doctype/dashboard_chart_source/dashboard_chart_source.js index 96dd40d3c1..6f1fa36ffd 100644 --- a/frappe/desk/doctype/dashboard_chart_source/dashboard_chart_source.js +++ b/frappe/desk/doctype/dashboard_chart_source/dashboard_chart_source.js @@ -1,5 +1,4 @@ // Copyright (c) 2019, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Dashboard Chart Source', { -}); +frappe.ui.form.on("Dashboard Chart Source", {}); diff --git a/frappe/desk/doctype/dashboard_chart_source/dashboard_chart_source.py b/frappe/desk/doctype/dashboard_chart_source/dashboard_chart_source.py index 29c1b6ee7d..5519ad9097 100644 --- a/frappe/desk/doctype/dashboard_chart_source/dashboard_chart_source.py +++ b/frappe/desk/doctype/dashboard_chart_source/dashboard_chart_source.py @@ -1,11 +1,9 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and contributors # License: MIT. See LICENSE import os import frappe -from frappe import _ from frappe.model.document import Document from frappe.modules import get_module_path, scrub from frappe.modules.export_file import export_to_files @@ -18,7 +16,6 @@ def get_config(name): os.path.join( get_module_path(doc.module), "dashboard_chart_source", scrub(doc.name), scrub(doc.name) + ".js" ), - "r", ) as f: return f.read() diff --git a/frappe/desk/doctype/dashboard_chart_source/test_dashboard_chart_source.py b/frappe/desk/doctype/dashboard_chart_source/test_dashboard_chart_source.py index 0c219c08cc..457487bb6d 100644 --- a/frappe/desk/doctype/dashboard_chart_source/test_dashboard_chart_source.py +++ b/frappe/desk/doctype/dashboard_chart_source/test_dashboard_chart_source.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # License: MIT. See LICENSE import unittest diff --git a/frappe/desk/doctype/dashboard_settings/dashboard_settings.js b/frappe/desk/doctype/dashboard_settings/dashboard_settings.js index 8e7966366d..aa5be2b1a5 100644 --- a/frappe/desk/doctype/dashboard_settings/dashboard_settings.js +++ b/frappe/desk/doctype/dashboard_settings/dashboard_settings.js @@ -1,8 +1,7 @@ // Copyright (c) 2020, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Dashboard Settings', { +frappe.ui.form.on("Dashboard Settings", { // refresh: function(frm) { - // } }); diff --git a/frappe/desk/doctype/dashboard_settings/dashboard_settings.py b/frappe/desk/doctype/dashboard_settings/dashboard_settings.py index d43476ad7d..489beda0bf 100644 --- a/frappe/desk/doctype/dashboard_settings/dashboard_settings.py +++ b/frappe/desk/doctype/dashboard_settings/dashboard_settings.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and contributors # License: MIT. See LICENSE @@ -28,7 +27,7 @@ def get_permission_query_conditions(user): if not user: user = frappe.session.user - return """(`tabDashboard Settings`.name = '{user}')""".format(user=user) + return f"""(`tabDashboard Settings`.name = {frappe.db.escape(user)})""" @frappe.whitelist() diff --git a/frappe/desk/doctype/desktop_icon/desktop_icon.js b/frappe/desk/doctype/desktop_icon/desktop_icon.js index 58ea09e732..72ef1f7a12 100644 --- a/frappe/desk/doctype/desktop_icon/desktop_icon.js +++ b/frappe/desk/doctype/desktop_icon/desktop_icon.js @@ -1,8 +1,6 @@ // Copyright (c) 2016, Frappe Technologies Pvt. Ltd. and contributors // For license information, please see license.txt -frappe.ui.form.on('Desktop Icon', { - refresh: function(frm) { - - } +frappe.ui.form.on("Desktop Icon", { + refresh: function (frm) {}, }); diff --git a/frappe/desk/doctype/desktop_icon/desktop_icon.json b/frappe/desk/doctype/desktop_icon/desktop_icon.json index 59c95953ad..ef88346f53 100644 --- a/frappe/desk/doctype/desktop_icon/desktop_icon.json +++ b/frappe/desk/doctype/desktop_icon/desktop_icon.json @@ -1,736 +1,175 @@ { - "allow_copy": 0, - "allow_events_in_timeline": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, - "creation": "2016-02-22 03:47:45.387068", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 0, - "engine": "InnoDB", + "actions": [], + "creation": "2016-02-22 03:47:45.387068", + "doctype": "DocType", + "engine": "InnoDB", + "field_order": [ + "module_name", + "label", + "standard", + "custom", + "column_break_3", + "app", + "description", + "category", + "hidden", + "blocked", + "force_show", + "section_break_7", + "type", + "_doctype", + "_report", + "link", + "column_break_10", + "color", + "icon", + "reverse", + "idx" + ], "fields": [ { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "module_name", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Module Name", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "module_name", + "fieldtype": "Data", + "label": "Module Name" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "label", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Label", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "label", + "fieldtype": "Data", + "label": "Label" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "standard", - "fieldtype": "Check", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Standard", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "default": "0", + "fieldname": "standard", + "fieldtype": "Check", + "in_list_view": 1, + "label": "Standard" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "custom", - "fieldtype": "Check", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Custom", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "default": "0", + "fieldname": "custom", + "fieldtype": "Check", + "label": "Custom", + "read_only": 1 + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "column_break_3", - "fieldtype": "Column Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "column_break_3", + "fieldtype": "Column Break" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "app", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "App", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "app", + "fieldtype": "Data", + "label": "App", + "read_only": 1 + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "description", - "fieldtype": "Small Text", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Description", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "description", + "fieldtype": "Small Text", + "label": "Description" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "category", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Category", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "category", + "fieldtype": "Data", + "label": "Category" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "hidden", - "fieldtype": "Check", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Hidden", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "default": "0", + "fieldname": "hidden", + "fieldtype": "Check", + "label": "Hidden" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "blocked", - "fieldtype": "Check", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Blocked", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "default": "0", + "fieldname": "blocked", + "fieldtype": "Check", + "label": "Blocked" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "force_show", - "fieldtype": "Check", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Force Show", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "default": "0", + "fieldname": "force_show", + "fieldtype": "Check", + "label": "Force Show", + "read_only": 1 + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "section_break_7", - "fieldtype": "Section Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "section_break_7", + "fieldtype": "Section Break" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "type", - "fieldtype": "Select", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 1, - "label": "Type", - "length": 0, - "no_copy": 0, - "options": "module\nlist\nlink\npage\nquery-report", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "type", + "fieldtype": "Select", + "in_list_view": 1, + "in_standard_filter": 1, + "label": "Type", + "options": "module\nlist\nlink\npage\nquery-report" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "_doctype", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "_doctype", - "length": 0, - "no_copy": 0, - "options": "DocType", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "_doctype", + "fieldtype": "Link", + "label": "_doctype", + "options": "DocType" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "_report", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "_report", - "length": 0, - "no_copy": 0, - "options": "Report", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "_report", + "fieldtype": "Link", + "label": "_report", + "options": "Report" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "link", - "fieldtype": "Small Text", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Link", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "link", + "fieldtype": "Small Text", + "label": "Link" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "column_break_10", - "fieldtype": "Column Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "column_break_10", + "fieldtype": "Column Break" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "color", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Color", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "color", + "fieldtype": "Data", + "label": "Color" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "icon", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Icon", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "icon", + "fieldtype": "Data", + "label": "Icon" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "reverse", - "fieldtype": "Check", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Reverse Icon Color", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "default": "0", + "fieldname": "reverse", + "fieldtype": "Check", + "label": "Reverse Icon Color" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "idx", - "fieldtype": "Int", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Idx", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "fieldname": "idx", + "fieldtype": "Int", + "label": "Idx" } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 1, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2019-01-24 04:58:58.720618", - "modified_by": "Administrator", - "module": "Desk", - "name": "Desktop Icon", - "name_case": "", - "owner": "Administrator", + ], + "in_create": 1, + "links": [], + "modified": "2022-08-03 12:20:50.577580", + "modified_by": "Administrator", + "module": "Desk", + "name": "Desktop Icon", + "owner": "Administrator", "permissions": [ { - "amend": 0, - "cancel": 0, - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "set_user_permissions": 0, - "share": 1, - "submit": 0, + "create": 1, + "delete": 1, + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1, "write": 1 } - ], - "quick_entry": 0, - "read_only": 1, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "title_field": "module_name", - "track_changes": 1, - "track_seen": 0, - "track_views": 0 + ], + "read_only": 1, + "sort_field": "modified", + "sort_order": "DESC", + "states": [], + "title_field": "module_name", + "track_changes": 1 } \ No newline at end of file diff --git a/frappe/desk/doctype/desktop_icon/desktop_icon.py b/frappe/desk/doctype/desktop_icon/desktop_icon.py index 29de1f56d9..5602f4da24 100644 --- a/frappe/desk/doctype/desktop_icon/desktop_icon.py +++ b/frappe/desk/doctype/desktop_icon/desktop_icon.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors # License: MIT. See LICENSE @@ -133,7 +132,7 @@ def add_user_icon(_doctype, _report=None, label=None, link=None, type="link", st if not label: label = _doctype or _report if not link: - link = "List/{0}".format(_doctype) + link = f"List/{_doctype}" # find if a standard icon exists icon_name = frappe.db.exists( diff --git a/frappe/desk/doctype/event/event.js b/frappe/desk/doctype/event/event.js index 87d78bae94..61bf66f5e5 100644 --- a/frappe/desk/doctype/event/event.js +++ b/frappe/desk/doctype/event/event.js @@ -3,70 +3,79 @@ frappe.provide("frappe.desk"); frappe.ui.form.on("Event", { - onload: function(frm) { - frm.set_query('reference_doctype', "event_participants", function() { - return { - "filters": { - "issingle": 0, - } - }; - }); - frm.set_query('google_calendar', function() { + onload: function (frm) { + frm.set_query("reference_doctype", "event_participants", function () { return { filters: { - "owner": frappe.session.user - } + issingle: 0, + }, + }; + }); + frm.set_query("google_calendar", function () { + return { + filters: { + owner: frappe.session.user, + }, }; }); }, - refresh: function(frm) { - if(frm.doc.event_participants) { - frm.doc.event_participants.forEach(value => { - frm.add_custom_button(__(value.reference_docname), function() { - frappe.set_route("Form", value.reference_doctype, value.reference_docname); - }, __("Participants")); - }) + refresh: function (frm) { + if (frm.doc.event_participants) { + frm.doc.event_participants.forEach((value) => { + frm.add_custom_button( + __(value.reference_docname), + function () { + frappe.set_route("Form", value.reference_doctype, value.reference_docname); + }, + __("Participants") + ); + }); } frm.page.set_inner_btn_group_as_primary(__("Add Participants")); - frm.add_custom_button(__('Add Contacts'), function() { - new frappe.desk.eventParticipants(frm, "Contact"); - }, __("Add Participants")); + frm.add_custom_button( + __("Add Contacts"), + function () { + new frappe.desk.eventParticipants(frm, "Contact"); + }, + __("Add Participants") + ); }, - repeat_on: function(frm) { - if(frm.doc.repeat_on==="Every Day") { - ["monday", "tuesday", "wednesday", "thursday", - "friday", "saturday", "sunday"].map(function(v) { + repeat_on: function (frm) { + if (frm.doc.repeat_on === "Every Day") { + ["monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday"].map( + function (v) { frm.set_value(v, 1); - }); + } + ); } - } + }, }); frappe.ui.form.on("Event Participants", { - event_participants_remove: function(frm, cdt, cdn) { - if (cdt&&!cdn.includes("New Event Participants")){ + event_participants_remove: function (frm, cdt, cdn) { + if (cdt && !cdn.includes("New Event Participants")) { frappe.call({ type: "POST", method: "frappe.desk.doctype.event.event.delete_communication", args: { - "event": frm.doc, - "reference_doctype": cdt, - "reference_docname": cdn + event: frm.doc, + reference_doctype: cdt, + reference_docname: cdn, }, freeze: true, - callback: function(r) { - if(r.exc) { + callback: function (r) { + if (r.exc) { frappe.show_alert({ message: __("{0}", [r.exc]), - indicator: 'orange' + indicator: "orange", }); } - } + }, }); } - } + }, }); frappe.desk.eventParticipants = class eventParticipants { @@ -86,7 +95,7 @@ frappe.desk.eventParticipants = class eventParticipants { dynamic_link_reference: me.doctype, fieldname: "reference_docname", target: table, - txt: "" + txt: "", }); } }; diff --git a/frappe/desk/doctype/event/event.json b/frappe/desk/doctype/event/event.json index 2f67c36fc0..bce3b1e65a 100644 --- a/frappe/desk/doctype/event/event.json +++ b/frappe/desk/doctype/event/event.json @@ -277,7 +277,7 @@ "icon": "fa fa-calendar", "idx": 1, "links": [], - "modified": "2021-11-18 05:06:24.881742", + "modified": "2022-05-12 05:43:27.935510", "modified_by": "Administrator", "module": "Desk", "name": "Event", @@ -312,6 +312,7 @@ "sender_field": "sender", "sort_field": "modified", "sort_order": "DESC", + "states": [], "subject_field": "subject", "title_field": "subject", "track_changes": 1, diff --git a/frappe/desk/doctype/event/event.py b/frappe/desk/doctype/event/event.py index 531cc69c57..e9104ef897 100644 --- a/frappe/desk/doctype/event/event.py +++ b/frappe/desk/doctype/event/event.py @@ -18,7 +18,6 @@ from frappe.utils import ( cstr, date_diff, format_datetime, - get_datetime, get_datetime_str, getdate, now_datetime, @@ -162,9 +161,9 @@ def delete_communication(event, reference_doctype, reference_docname): def get_permission_query_conditions(user): if not user: user = frappe.session.user - return """(`tabEvent`.`event_type`='Public' or `tabEvent`.`owner`=%(user)s)""" % { - "user": frappe.db.escape(user), - } + return """(`tabEvent`.`event_type`='Public' or `tabEvent`.`owner`={user})""".format( + user=frappe.db.escape(user), + ) def has_permission(doc, user): diff --git a/frappe/desk/doctype/event/event_calendar.js b/frappe/desk/doctype/event/event_calendar.js index df474a0258..bfdd09b864 100644 --- a/frappe/desk/doctype/event/event_calendar.js +++ b/frappe/desk/doctype/event/event_calendar.js @@ -1,16 +1,16 @@ frappe.views.calendar["Event"] = { field_map: { - "start": "starts_on", - "end": "ends_on", - "id": "name", - "allDay": "all_day", - "title": "subject", - "status": "event_type", - "color": "color" + start: "starts_on", + end: "ends_on", + id: "name", + allDay: "all_day", + title: "subject", + status: "event_type", + color: "color", }, style_map: { - "Public": "success", - "Private": "info" + Public: "success", + Private: "info", }, - get_events_method: "frappe.desk.doctype.event.event.get_events" -} \ No newline at end of file + get_events_method: "frappe.desk.doctype.event.event.get_events", +}; diff --git a/frappe/desk/doctype/event/event_list.js b/frappe/desk/doctype/event/event_list.js index 5d73d9dd1a..f6460288d8 100644 --- a/frappe/desk/doctype/event/event_list.js +++ b/frappe/desk/doctype/event/event_list.js @@ -1,8 +1,8 @@ -frappe.listview_settings['Event'] = { +frappe.listview_settings["Event"] = { add_fields: ["starts_on", "ends_on"], - onload: function() { + onload: function () { frappe.route_options = { - "status": "Open" + status: "Open", }; - } -} \ No newline at end of file + }, +}; diff --git a/frappe/desk/doctype/event/test_event.py b/frappe/desk/doctype/event/test_event.py index 041bda643e..efbd54fb09 100644 --- a/frappe/desk/doctype/event/test_event.py +++ b/frappe/desk/doctype/event/test_event.py @@ -96,7 +96,7 @@ class TestEvent(unittest.TestCase): ev = frappe.get_doc("Event", ev.name) - self.assertEqual(set(json.loads(ev._assign)), set(["test@example.com", self.test_user])) + self.assertEqual(set(json.loads(ev._assign)), {"test@example.com", self.test_user}) # Remove an assignment todo = frappe.get_doc( diff --git a/frappe/desk/doctype/event_participants/event_participants.json b/frappe/desk/doctype/event_participants/event_participants.json index 86cf2670c9..1b40e7042b 100644 --- a/frappe/desk/doctype/event_participants/event_participants.json +++ b/frappe/desk/doctype/event_participants/event_participants.json @@ -1,108 +1,42 @@ { - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, - "creation": "2018-09-21 15:44:58.836156", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", - "fields": [ - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "reference_doctype", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Reference Document Type", - "length": 0, - "no_copy": 0, - "options": "DocType", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "reference_docname", - "fieldtype": "Dynamic Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Reference Name", - "length": 0, - "no_copy": 0, - "options": "reference_doctype", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 1, - "max_attachments": 0, - "modified": "2019-09-05 14:22:27.664645", - "modified_by": "Administrator", - "module": "Desk", - "name": "Event Participants", - "name_case": "", - "owner": "Administrator", - "permissions": [], - "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1, - "track_seen": 0, - "track_views": 0 - } \ No newline at end of file + "actions": [], + "creation": "2018-09-21 15:44:58.836156", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "reference_doctype", + "reference_docname" + ], + "fields": [ + { + "fieldname": "reference_doctype", + "fieldtype": "Link", + "in_list_view": 1, + "label": "Reference Document Type", + "options": "DocType", + "reqd": 1 + }, + { + "fieldname": "reference_docname", + "fieldtype": "Dynamic Link", + "in_list_view": 1, + "label": "Reference Name", + "options": "reference_doctype", + "reqd": 1 + } + ], + "istable": 1, + "links": [], + "modified": "2022-08-03 12:20:50.466370", + "modified_by": "Administrator", + "module": "Desk", + "name": "Event Participants", + "owner": "Administrator", + "permissions": [], + "quick_entry": 1, + "sort_field": "modified", + "sort_order": "DESC", + "states": [], + "track_changes": 1 +} \ No newline at end of file diff --git a/frappe/desk/doctype/event_participants/event_participants.py b/frappe/desk/doctype/event_participants/event_participants.py index fdb834b285..869ae4092b 100644 --- a/frappe/desk/doctype/event_participants/event_participants.py +++ b/frappe/desk/doctype/event_participants/event_participants.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2018, Frappe Technologies and contributors # License: MIT. See LICENSE from frappe.model.document import Document diff --git a/frappe/desk/doctype/form_tour/form_tour.js b/frappe/desk/doctype/form_tour/form_tour.js index 3f3fc0ff8a..1e67e10779 100644 --- a/frappe/desk/doctype/form_tour/form_tour.js +++ b/frappe/desk/doctype/form_tour/form_tour.js @@ -1,10 +1,10 @@ // Copyright (c) 2021, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Form Tour', { - setup: function(frm) { +frappe.ui.form.on("Form Tour", { + setup: function (frm) { if (!frm.doc.is_standard || frappe.boot.developer_mode) { - frm.trigger('setup_queries'); + frm.trigger("setup_queries"); } }, @@ -13,28 +13,26 @@ frappe.ui.form.on('Form Tour', { frm.trigger("disable_form"); } - frm.add_custom_button(__('Show Tour'), async () => { + frm.add_custom_button(__("Show Tour"), async () => { const issingle = await check_if_single(frm.doc.reference_doctype); let route_changed = null; if (issingle) { - route_changed = frappe.set_route('Form', frm.doc.reference_doctype); + route_changed = frappe.set_route("Form", frm.doc.reference_doctype); } else if (frm.doc.first_document) { const name = await get_first_document(frm.doc.reference_doctype); - route_changed = frappe.set_route('Form', frm.doc.reference_doctype, name); + route_changed = frappe.set_route("Form", frm.doc.reference_doctype, name); } else { - route_changed = frappe.set_route('Form', frm.doc.reference_doctype, 'new'); + route_changed = frappe.set_route("Form", frm.doc.reference_doctype, "new"); } route_changed.then(() => { const tour_name = frm.doc.name; - cur_frm.tour - .init({ tour_name }) - .then(() => cur_frm.tour.start()); + cur_frm.tour.init({ tour_name }).then(() => cur_frm.tour.start()); }); }); }, - disable_form: function(frm) { + disable_form: function (frm) { frm.set_read_only(); frm.fields .filter((field) => field.has_input) @@ -45,51 +43,48 @@ frappe.ui.form.on('Form Tour', { }, setup_queries(frm) { - frm.set_query("reference_doctype", function() { + frm.set_query("reference_doctype", function () { return { filters: { - istable: 0 - } + istable: 0, + }, }; }); - frm.trigger('reference_doctype'); + frm.trigger("reference_doctype"); }, reference_doctype(frm) { if (!frm.doc.reference_doctype) return; - frm.set_fields_as_options( - "fieldname", - frm.doc.reference_doctype, - df => !df.hidden - ).then(options => { - frm.fields_dict.steps.grid.update_docfield_property( - "fieldname", - "options", - [""].concat(options) - ); - }); + frm.set_fields_as_options("fieldname", frm.doc.reference_doctype, (df) => !df.hidden).then( + (options) => { + frm.fields_dict.steps.grid.update_docfield_property( + "fieldname", + "options", + [""].concat(options) + ); + } + ); frm.set_fields_as_options( - 'parent_fieldname', + "parent_fieldname", frm.doc.reference_doctype, - (df) => df.fieldtype == "Table" && !df.hidden, - ).then(options => { + (df) => df.fieldtype == "Table" && !df.hidden + ).then((options) => { frm.fields_dict.steps.grid.update_docfield_property( "parent_fieldname", "options", [""].concat(options) ); }); - - } + }, }); -frappe.ui.form.on('Form Tour Step', { +frappe.ui.form.on("Form Tour Step", { form_render(frm, cdt, cdn) { if (locals[cdt][cdn].is_table_field) { - frm.trigger('parent_fieldname', cdt, cdn); + frm.trigger("parent_fieldname", cdt, cdn); } }, parent_fieldname(frm, cdt, cdn) { @@ -97,37 +92,36 @@ frappe.ui.form.on('Form Tour Step', { const parent_fieldname_df = frappe .get_meta(frm.doc.reference_doctype) - .fields.find(df => df.fieldname == child_row.parent_fieldname); + .fields.find((df) => df.fieldname == child_row.parent_fieldname); frm.set_fields_as_options( - 'fieldname', + "fieldname", parent_fieldname_df.options, - (df) => !df.hidden, - ).then(options => { + (df) => !df.hidden + ).then((options) => { frm.fields_dict.steps.grid.update_docfield_property( "fieldname", "options", [""].concat(options) ); if (child_row.fieldname) { - frappe.model.set_value(cdt, cdn, 'fieldname', child_row.fieldname); + frappe.model.set_value(cdt, cdn, "fieldname", child_row.fieldname); } }); - } + }, }); async function check_if_single(doctype) { - const { message } = await frappe.db.get_value('DocType', doctype, 'issingle'); + const { message } = await frappe.db.get_value("DocType", doctype, "issingle"); return message.issingle || 0; } async function get_first_document(doctype) { let docname; - await frappe.db.get_list(doctype, { order_by: "creation" }).then(res => { - if (Array.isArray(res) && res.length) - docname = res[0].name; + await frappe.db.get_list(doctype, { order_by: "creation" }).then((res) => { + if (Array.isArray(res) && res.length) docname = res[0].name; }); - return docname || 'new'; + return docname || "new"; } diff --git a/frappe/desk/doctype/global_search_doctype/global_search_doctype.py b/frappe/desk/doctype/global_search_doctype/global_search_doctype.py index 8bdc05cd71..48fdb3d4d1 100644 --- a/frappe/desk/doctype/global_search_doctype/global_search_doctype.py +++ b/frappe/desk/doctype/global_search_doctype/global_search_doctype.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and contributors # License: MIT. See LICENSE diff --git a/frappe/desk/doctype/global_search_settings/global_search_settings.js b/frappe/desk/doctype/global_search_settings/global_search_settings.js index c333f83585..147a72eef1 100644 --- a/frappe/desk/doctype/global_search_settings/global_search_settings.js +++ b/frappe/desk/doctype/global_search_settings/global_search_settings.js @@ -1,14 +1,17 @@ // Copyright (c) 2019, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Global Search Settings', { - refresh: function(frm) { - - frappe.realtime.on('global_search_settings', (data) => { +frappe.ui.form.on("Global Search Settings", { + refresh: function (frm) { + frappe.realtime.on("global_search_settings", (data) => { if (data.progress) { - frm.dashboard.show_progress('Setting up Global Search', data.progress / data.total * 100, data.msg); + frm.dashboard.show_progress( + "Setting up Global Search", + (data.progress / data.total) * 100, + data.msg + ); if (data.progress === data.total) { - frm.dashboard.hide_progress('Setting up Global Search'); + frm.dashboard.hide_progress("Setting up Global Search"); } } }); @@ -16,14 +19,14 @@ frappe.ui.form.on('Global Search Settings', { frm.add_custom_button(__("Reset"), function () { frappe.call({ method: "frappe.desk.doctype.global_search_settings.global_search_settings.reset_global_search_settings_doctypes", - callback: function() { + callback: function () { frappe.show_alert({ message: __("Global Search Document Types Reset."), - indicator: "green" + indicator: "green", }); frm.refresh(); - } + }, }); }); - } + }, }); diff --git a/frappe/desk/doctype/global_search_settings/global_search_settings.py b/frappe/desk/doctype/global_search_settings/global_search_settings.py index b7ffd7faf7..4e2b1e85f9 100644 --- a/frappe/desk/doctype/global_search_settings/global_search_settings.py +++ b/frappe/desk/doctype/global_search_settings/global_search_settings.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and contributors # License: MIT. See LICENSE diff --git a/frappe/desk/doctype/kanban_board/kanban_board.js b/frappe/desk/doctype/kanban_board/kanban_board.js index ff80a58fa0..3b815fef0e 100644 --- a/frappe/desk/doctype/kanban_board/kanban_board.js +++ b/frappe/desk/doctype/kanban_board/kanban_board.js @@ -1,43 +1,45 @@ // Copyright (c) 2016, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Kanban Board', { - onload: function(frm) { - frm.trigger('reference_doctype'); +frappe.ui.form.on("Kanban Board", { + onload: function (frm) { + frm.trigger("reference_doctype"); }, - refresh: function(frm) { - if(frm.is_new()) return; - frm.add_custom_button("Show Board", function() { + refresh: function (frm) { + if (frm.is_new()) return; + frm.add_custom_button("Show Board", function () { frappe.set_route("List", frm.doc.reference_doctype, "Kanban", frm.doc.name); }); }, - reference_doctype: function(frm) { - + reference_doctype: function (frm) { // set field options - if(!frm.doc.reference_doctype) return; + if (!frm.doc.reference_doctype) return; - frappe.model.with_doctype(frm.doc.reference_doctype, function() { - var options = $.map(frappe.get_meta(frm.doc.reference_doctype).fields, - function(d) { - if(d.fieldname && d.fieldtype === 'Select' && - frappe.model.no_value_type.indexOf(d.fieldtype)===-1) { - return d.fieldname; - } - return null; - }); - frm.set_df_property('field_name', 'options', options); - frm.get_field('field_name').refresh(); + frappe.model.with_doctype(frm.doc.reference_doctype, function () { + var options = $.map(frappe.get_meta(frm.doc.reference_doctype).fields, function (d) { + if ( + d.fieldname && + d.fieldtype === "Select" && + frappe.model.no_value_type.indexOf(d.fieldtype) === -1 + ) { + return d.fieldname; + } + return null; + }); + frm.set_df_property("field_name", "options", options); + frm.get_field("field_name").refresh(); }); }, - field_name: function(frm) { + field_name: function (frm) { var field = frappe.meta.get_field(frm.doc.reference_doctype, frm.doc.field_name); frm.doc.columns = []; - field.options && field.options.split('\n').forEach(function(o) { - o = o.trim(); - if(!o) return; - var d = frm.add_child('columns'); - d.column_name = o; - }); + field.options && + field.options.split("\n").forEach(function (o) { + o = o.trim(); + if (!o) return; + var d = frm.add_child("columns"); + d.column_name = o; + }); frm.refresh(); - } + }, }); diff --git a/frappe/desk/doctype/kanban_board/kanban_board.json b/frappe/desk/doctype/kanban_board/kanban_board.json index f2e1a78d40..b1f120687c 100644 --- a/frappe/desk/doctype/kanban_board/kanban_board.json +++ b/frappe/desk/doctype/kanban_board/kanban_board.json @@ -1,267 +1,124 @@ { - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, + "actions": [], "allow_rename": 1, "autoname": "field:kanban_board_name", - "beta": 0, "creation": "2016-10-19 12:26:04.809812", - "custom": 0, - "docstatus": 0, "doctype": "DocType", - "document_type": "", "editable_grid": 1, "engine": "InnoDB", + "field_order": [ + "kanban_board_name", + "reference_doctype", + "field_name", + "column_break_4", + "private", + "show_labels", + "section_break_3", + "columns", + "filters", + "fields" + ], "fields": [ { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "kanban_board_name", "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, "label": "Kanban Board Name", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, "reqd": 1, - "search_index": 0, - "set_only_once": 0, "unique": 1 }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "reference_doctype", "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, "in_list_view": 1, - "in_standard_filter": 0, "label": "Reference Document Type", - "length": 0, - "no_copy": 0, "options": "DocType", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "reqd": 1 }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "field_name", "fieldtype": "Select", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, "label": "Field Name", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "reqd": 1 }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "section_break_3", - "fieldtype": "Section Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "fieldtype": "Section Break" }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "columns", "fieldtype": "Table", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, "label": "Columns", - "length": 0, - "no_copy": 0, - "options": "Kanban Board Column", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "options": "Kanban Board Column" }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "filters", - "fieldtype": "Text", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, + "fieldtype": "Code", "label": "Filters", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "options": "JSON", + "read_only": 1 }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, + "default": "0", "fieldname": "private", "fieldtype": "Check", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, "label": "Private", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "read_only": 1 + }, + { + "fieldname": "fields", + "fieldtype": "Code", + "label": "Fields", + "options": "JSON", + "read_only": 1 + }, + { + "fieldname": "column_break_4", + "fieldtype": "Column Break" + }, + { + "default": "0", + "fieldname": "show_labels", + "fieldtype": "Check", + "label": "Show Labels", + "read_only": 1 } ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2019-09-05 14:22:27.664645", + "links": [], + "modified": "2022-04-13 12:10:20.284367", "modified_by": "Administrator", "module": "Desk", "name": "Kanban Board", - "name_case": "", + "naming_rule": "By fieldname", "owner": "Administrator", "permissions": [ { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, + "read": 1, + "role": "All" + }, + { + "create": 1, + "delete": 1, + "if_owner": 1, + "read": 1, + "role": "All", + "write": 1 + }, + { "create": 1, "delete": 1, "email": 1, "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, "print": 1, "read": 1, "report": 1, - "role": "All", - "set_user_permissions": 0, + "role": "System Manager", "share": 1, - "submit": 0, "write": 1 } ], - "quick_entry": 0, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, + "read_only": 1, "sort_field": "modified", "sort_order": "DESC", - "track_changes": 0, - "track_seen": 0 + "states": [], + "track_changes": 1 } \ No newline at end of file diff --git a/frappe/desk/doctype/kanban_board/kanban_board.py b/frappe/desk/doctype/kanban_board/kanban_board.py index e864f68728..83f0f46df0 100644 --- a/frappe/desk/doctype/kanban_board/kanban_board.py +++ b/frappe/desk/doctype/kanban_board/kanban_board.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and contributors # License: MIT. See LICENSE @@ -24,7 +23,7 @@ class KanbanBoard(Document): def validate_column_name(self): for column in self.columns: if not column.column_name: - frappe.msgprint(frappe._("Column Name cannot be empty"), raise_exception=True) + frappe.msgprint(_("Column Name cannot be empty"), raise_exception=True) def get_permission_query_conditions(user): @@ -34,7 +33,9 @@ def get_permission_query_conditions(user): if user == "Administrator": return "" - return """(`tabKanban Board`.private=0 or `tabKanban Board`.owner='{user}')""".format(user=user) + return """(`tabKanban Board`.private=0 or `tabKanban Board`.owner={user})""".format( + user=frappe.db.escape(user) + ) def has_permission(doc, ptype, user): @@ -92,7 +93,6 @@ def update_order(board_name, order): updated_cards = [] for col_name, cards in order_dict.items(): - order_list = [] for card in cards: column = frappe.get_value(doctype, {"name": card}, fieldname) if column != col_name: @@ -125,7 +125,7 @@ def update_order_for_single_card( if from_colname == to_colname: from_col_order = to_col_order - to_col_order.insert(new_index, from_col_order.pop((old_index))) + to_col_order.insert(new_index, from_col_order.pop(old_index)) # save updated order board.columns[from_col_idx].order = frappe.as_json(from_col_order) @@ -172,7 +172,7 @@ def quick_kanban_board(doctype, board_name, field_name, project=None): doc.field_name = field_name if project: - doc.filters = '[["Task","project","=","{0}"]]'.format(project) + doc.filters = f'[["Task","project","=","{project}"]]' options = "" for field in meta.fields: @@ -246,3 +246,22 @@ def set_indicator(board_name, column_name, indicator): board.save() return board + + +@frappe.whitelist() +def save_settings(board_name: str, settings: str) -> Document: + settings = json.loads(settings) + doc = frappe.get_doc("Kanban Board", board_name) + + fields = settings["fields"] + if not isinstance(fields, str): + fields = json.dumps(fields) + + doc.fields = fields + doc.show_labels = settings["show_labels"] + doc.save() + + resp = doc.as_dict() + resp["fields"] = frappe.parse_json(resp["fields"]) + + return resp diff --git a/frappe/desk/doctype/kanban_board/test_kanban_board.py b/frappe/desk/doctype/kanban_board/test_kanban_board.py index 179e6c71e5..73f566b906 100644 --- a/frappe/desk/doctype/kanban_board/test_kanban_board.py +++ b/frappe/desk/doctype/kanban_board/test_kanban_board.py @@ -1,10 +1,7 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and Contributors # License: MIT. See LICENSE import unittest -import frappe - # test_records = frappe.get_test_records('Kanban Board') diff --git a/frappe/desk/doctype/kanban_board_column/kanban_board_column.py b/frappe/desk/doctype/kanban_board_column/kanban_board_column.py index 8a1f839c98..e57d92857e 100644 --- a/frappe/desk/doctype/kanban_board_column/kanban_board_column.py +++ b/frappe/desk/doctype/kanban_board_column/kanban_board_column.py @@ -1,8 +1,6 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and contributors # License: MIT. See LICENSE -import frappe from frappe.model.document import Document diff --git a/frappe/desk/doctype/list_filter/list_filter.json b/frappe/desk/doctype/list_filter/list_filter.json index dad62bf8d6..257bbc6d45 100644 --- a/frappe/desk/doctype/list_filter/list_filter.json +++ b/frappe/desk/doctype/list_filter/list_filter.json @@ -1,188 +1,62 @@ { - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, + "actions": [], "creation": "2018-02-22 15:10:24.401801", - "custom": 0, - "docstatus": 0, "doctype": "DocType", - "document_type": "", "editable_grid": 1, "engine": "InnoDB", + "field_order": [ + "filter_name", + "reference_doctype", + "for_user", + "filters" + ], "fields": [ { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "filter_name", "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Filter Name", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "label": "Filter Name" }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "reference_doctype", "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, "label": "Reference Document Type", - "length": 0, - "no_copy": 0, - "options": "DocType", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "options": "DocType" }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "for_user", "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, "label": "For User", - "length": 0, - "no_copy": 0, - "options": "User", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "options": "User" }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "filters", "fieldtype": "Long Text", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Filters", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "label": "Filters" } ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, "in_create": 1, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2019-09-05 14:22:27.664645", + "links": [], + "modified": "2022-08-03 12:20:50.889979", "modified_by": "Administrator", "module": "Desk", "name": "List Filter", - "name_case": "", "owner": "Administrator", "permissions": [ { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, "create": 1, "delete": 1, "email": 1, "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, "print": 1, "read": 1, "report": 1, "role": "All", - "set_user_permissions": 0, "share": 1, - "submit": 0, "write": 1 } ], - "quick_entry": 0, "read_only": 1, - "read_only_onload": 0, - "show_name_in_global_search": 0, "sort_field": "modified", "sort_order": "DESC", - "track_changes": 0, - "track_seen": 0 + "states": [] } \ No newline at end of file diff --git a/frappe/desk/doctype/list_filter/list_filter.py b/frappe/desk/doctype/list_filter/list_filter.py index ac434a760a..e4c59ee9e4 100644 --- a/frappe/desk/doctype/list_filter/list_filter.py +++ b/frappe/desk/doctype/list_filter/list_filter.py @@ -1,10 +1,6 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2018, Frappe Technologies and contributors # License: MIT. See LICENSE -import json - -import frappe from frappe.model.document import Document diff --git a/frappe/desk/doctype/list_view_settings/list_view_settings.js b/frappe/desk/doctype/list_view_settings/list_view_settings.js index db33f71675..007a242dd3 100644 --- a/frappe/desk/doctype/list_view_settings/list_view_settings.js +++ b/frappe/desk/doctype/list_view_settings/list_view_settings.js @@ -1,8 +1,7 @@ // Copyright (c) 2020, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('List View Settings', { +frappe.ui.form.on("List View Settings", { // refresh: function(frm) { - // } -}); \ No newline at end of file +}); diff --git a/frappe/desk/doctype/list_view_settings/list_view_settings.py b/frappe/desk/doctype/list_view_settings/list_view_settings.py index 7d25f57acf..36ebce34d5 100644 --- a/frappe/desk/doctype/list_view_settings/list_view_settings.py +++ b/frappe/desk/doctype/list_view_settings/list_view_settings.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and contributors # License: MIT. See LICENSE diff --git a/frappe/desk/doctype/list_view_settings/test_list_view_settings.py b/frappe/desk/doctype/list_view_settings/test_list_view_settings.py index 0b6a0773e3..0eab9cd7a6 100644 --- a/frappe/desk/doctype/list_view_settings/test_list_view_settings.py +++ b/frappe/desk/doctype/list_view_settings/test_list_view_settings.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # License: MIT. See LICENSE # import frappe diff --git a/frappe/desk/doctype/module_onboarding/module_onboarding.js b/frappe/desk/doctype/module_onboarding/module_onboarding.js index d95920e2ca..0e312025bf 100644 --- a/frappe/desk/doctype/module_onboarding/module_onboarding.js +++ b/frappe/desk/doctype/module_onboarding/module_onboarding.js @@ -2,7 +2,7 @@ // For license information, please see license.txt frappe.ui.form.on("Module Onboarding", { - refresh: function(frm) { + refresh: function (frm) { frappe.boot.developer_mode && frm.set_intro( __( @@ -15,7 +15,7 @@ frappe.ui.form.on("Module Onboarding", { } }, - disable_form: function(frm) { + disable_form: function (frm) { frm.set_read_only(); frm.fields .filter((field) => field.has_input) diff --git a/frappe/desk/doctype/module_onboarding/module_onboarding.py b/frappe/desk/doctype/module_onboarding/module_onboarding.py index 7a12328ee0..ea02f5911d 100644 --- a/frappe/desk/doctype/module_onboarding/module_onboarding.py +++ b/frappe/desk/doctype/module_onboarding/module_onboarding.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and contributors # License: MIT. See LICENSE diff --git a/frappe/desk/doctype/module_onboarding/test_module_onboarding.py b/frappe/desk/doctype/module_onboarding/test_module_onboarding.py index 8def3ac40e..fa19794c1e 100644 --- a/frappe/desk/doctype/module_onboarding/test_module_onboarding.py +++ b/frappe/desk/doctype/module_onboarding/test_module_onboarding.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2020, Frappe Technologies and Contributors # License: MIT. See LICENSE # import frappe diff --git a/frappe/desk/doctype/note/note.js b/frappe/desk/doctype/note/note.js index 5718180b70..dd556a3969 100644 --- a/frappe/desk/doctype/note/note.js +++ b/frappe/desk/doctype/note/note.js @@ -1,5 +1,5 @@ frappe.ui.form.on("Note", { - refresh: function(frm) { + refresh: function (frm) { if (frm.doc.__islocal) { frm.events.set_editable(frm, true); } else { @@ -8,13 +8,13 @@ frappe.ui.form.on("Note", { } // toggle edit - frm.add_custom_button("Edit", function() { + frm.add_custom_button("Edit", function () { frm.events.set_editable(frm, !frm.is_note_editable); }); frm.events.set_editable(frm, false); } }, - set_editable: function(frm, editable) { + set_editable: function (frm, editable) { // hide all fields other than content // no permission @@ -24,7 +24,7 @@ frappe.ui.form.on("Note", { frm.set_df_property("content", "read_only", editable ? 0 : 1); // hide all other fields - $.each(frm.fields_dict, function(fieldname) { + $.each(frm.fields_dict, function (fieldname) { if (fieldname !== "content") { frm.set_df_property(fieldname, "hidden", editable ? 0 : 1); } @@ -36,10 +36,10 @@ frappe.ui.form.on("Note", { // set flag for toggle frm.is_note_editable = editable; - } + }, }); -frappe.tour['Note'] = [ +frappe.tour["Note"] = [ { fieldname: "title", title: "Title of the Note", @@ -48,6 +48,7 @@ frappe.tour['Note'] = [ { fieldname: "public", title: "Sets the Note to Public", - description: "You can change the visibility of the note with this, setting it to public will allow other users to view it.", + description: + "You can change the visibility of the note with this, setting it to public will allow other users to view it.", }, -]; \ No newline at end of file +]; diff --git a/frappe/desk/doctype/note/note.py b/frappe/desk/doctype/note/note.py index de019d9898..c0a37d5f44 100644 --- a/frappe/desk/doctype/note/note.py +++ b/frappe/desk/doctype/note/note.py @@ -1,16 +1,18 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +import re + import frappe from frappe.model.document import Document +NAME_PATTERN = re.compile("[%'\"#*?`]") + class Note(Document): def autoname(self): # replace forbidden characters - import re - - self.name = re.sub("[%'\"#*?`]", "", self.title.strip()) + self.name = NAME_PATTERN.sub("", self.title.strip()) def validate(self): if self.notify_on_login and not self.expire_notification_on: @@ -38,7 +40,7 @@ def get_permission_query_conditions(user): if user == "Administrator": return "" - return """(`tabNote`.public=1 or `tabNote`.owner="{user}")""".format(user=user) + return f"""(`tabNote`.public=1 or `tabNote`.owner={frappe.db.escape(user)})""" def has_permission(doc, ptype, user): diff --git a/frappe/desk/doctype/note/note_list.js b/frappe/desk/doctype/note/note_list.js index f7f8d37dcf..1e0ed40880 100644 --- a/frappe/desk/doctype/note/note_list.js +++ b/frappe/desk/doctype/note/note_list.js @@ -1,13 +1,13 @@ -frappe.listview_settings['Note'] = { - onload: function(me) { +frappe.listview_settings["Note"] = { + onload: function (me) { me.page.set_title(__("Notes")); }, add_fields: ["title", "public"], - get_indicator: function(doc) { - if(doc.public) { + get_indicator: function (doc) { + if (doc.public) { return [__("Public"), "green", "public,=,Yes"]; } else { return [__("Private"), "gray", "public,=,No"]; } - } -} + }, +}; diff --git a/frappe/desk/doctype/note_seen_by/note_seen_by.json b/frappe/desk/doctype/note_seen_by/note_seen_by.json index 7ee423e347..f54559d2f5 100644 --- a/frappe/desk/doctype/note_seen_by/note_seen_by.json +++ b/frappe/desk/doctype/note_seen_by/note_seen_by.json @@ -1,64 +1,31 @@ { - "allow_copy": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, - "creation": "2016-08-29 05:29:16.726172", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, + "actions": [], + "creation": "2016-08-29 05:29:16.726172", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "user" + ], "fields": [ { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "user", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 1, - "label": "User", - "length": 0, - "no_copy": 0, - "options": "User", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "fieldname": "user", + "fieldtype": "Link", + "in_list_view": 1, + "label": "User", + "options": "User" } - ], - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - - "is_submittable": 0, - "issingle": 0, - "istable": 1, - "max_attachments": 0, - "modified": "2016-08-29 06:02:41.531341", - "modified_by": "Administrator", - "module": "Desk", - "name": "Note Seen By", - "name_case": "", - "owner": "Administrator", - "permissions": [], - "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_seen": 0 + ], + "istable": 1, + "links": [], + "modified": "2022-08-03 12:20:51.030908", + "modified_by": "Administrator", + "module": "Desk", + "name": "Note Seen By", + "owner": "Administrator", + "permissions": [], + "quick_entry": 1, + "sort_field": "modified", + "sort_order": "DESC", + "states": [] } \ No newline at end of file diff --git a/frappe/desk/doctype/note_seen_by/note_seen_by.py b/frappe/desk/doctype/note_seen_by/note_seen_by.py index 7b87cf13b2..5acdca222e 100644 --- a/frappe/desk/doctype/note_seen_by/note_seen_by.py +++ b/frappe/desk/doctype/note_seen_by/note_seen_by.py @@ -1,8 +1,6 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2015, Frappe Technologies and contributors # License: MIT. See LICENSE -import frappe from frappe.model.document import Document diff --git a/frappe/desk/doctype/notification_log/notification_log.js b/frappe/desk/doctype/notification_log/notification_log.js index 1f381d115b..ea5fdc6400 100644 --- a/frappe/desk/doctype/notification_log/notification_log.js +++ b/frappe/desk/doctype/notification_log/notification_log.js @@ -1,25 +1,25 @@ // Copyright (c) 2019, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Notification Log', { - refresh: function(frm) { +frappe.ui.form.on("Notification Log", { + refresh: function (frm) { if (frm.doc.attached_file) { - frm.trigger('set_attachment'); + frm.trigger("set_attachment"); } else { - frm.get_field('attachment_link').$wrapper.empty(); + frm.get_field("attachment_link").$wrapper.empty(); } }, - open_reference_document: function(frm) { + open_reference_document: function (frm) { const dt = frm.doc.document_type; const dn = frm.doc.document_name; - frappe.set_route('Form', dt, dn); + frappe.set_route("Form", dt, dn); }, - set_attachment: function(frm) { + set_attachment: function (frm) { const attachment = JSON.parse(frm.doc.attached_file); - const $wrapper = frm.get_field('attachment_link').$wrapper; + const $wrapper = frm.get_field("attachment_link").$wrapper; $wrapper.html(`
    @@ -41,5 +41,5 @@ frappe.ui.form.on('Notification Log', { frappe.msgprint(__("Please enable pop-ups")); } }); - } + }, }); diff --git a/frappe/desk/doctype/notification_log/notification_log.py b/frappe/desk/doctype/notification_log/notification_log.py index 1a466ea78b..482f404e65 100644 --- a/frappe/desk/doctype/notification_log/notification_log.py +++ b/frappe/desk/doctype/notification_log/notification_log.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and contributors # License: MIT. See LICENSE @@ -7,7 +6,6 @@ from frappe import _ from frappe.desk.doctype.notification_settings.notification_settings import ( is_email_notifications_enabled_for_type, is_notifications_enabled, - set_seen_value, ) from frappe.model.document import Document @@ -20,7 +18,7 @@ class NotificationLog(Document): try: send_notification_email(self) except frappe.OutgoingEmailError: - frappe.log_error(message=frappe.get_traceback(), title=_("Failed to send notification email")) + self.log_error(_("Failed to send notification email")) def get_permission_query_conditions(for_user): @@ -30,7 +28,7 @@ def get_permission_query_conditions(for_user): if for_user == "Administrator": return - return """(`tabNotification Log`.for_user = '{user}')""".format(user=for_user) + return f"""(`tabNotification Log`.for_user = {frappe.db.escape(for_user)})""" def get_title(doctype, docname, title_field=None): @@ -41,15 +39,19 @@ def get_title(doctype, docname, title_field=None): def get_title_html(title): - return '{0}'.format(title) + return f'{title}' -def enqueue_create_notification(users, doc): - """ - During installation of new site, enqueue_create_notification tries to connect to Redis. - This breaks new site creation if Redis server is not running. - We do not need any notifications in fresh installation +def enqueue_create_notification(users: list[str] | str, doc: dict): + """Send notification to users. + + users: list of user emails or string of users with comma separated emails + doc: contents of `Notification` doc """ + + # During installation of new site, enqueue_create_notification tries to connect to Redis. + # This breaks new site creation if Redis server is not running. + # We do not need any notifications in fresh installation if frappe.flags.in_install: return @@ -68,21 +70,23 @@ def enqueue_create_notification(users, doc): def make_notification_logs(doc, users): - from frappe.social.doctype.energy_point_settings.energy_point_settings import ( - is_energy_point_enabled, + for user in _get_user_ids(users): + notification = frappe.new_doc("Notification Log") + notification.update(doc) + notification.for_user = user + if ( + notification.for_user != notification.from_user + or doc.type == "Energy Point" + or doc.type == "Alert" + ): + notification.insert(ignore_permissions=True) + + +def _get_user_ids(user_emails): + user_names = frappe.db.get_values( + "User", {"enabled": 1, "email": ("in", user_emails)}, "name", pluck=True ) - - for user in users: - if frappe.db.exists("User", {"email": user, "enabled": 1}): - if is_notifications_enabled(user): - if doc.type == "Energy Point" and not is_energy_point_enabled(): - return - - _doc = frappe.new_doc("Notification Log") - _doc.update(doc) - _doc.for_user = user - if _doc.for_user != _doc.from_user or doc.type == "Energy Point" or doc.type == "Alert": - _doc.insert(ignore_permissions=True) + return [user for user in user_names if is_notifications_enabled(user)] def send_notification_email(doc): @@ -92,12 +96,16 @@ def send_notification_email(doc): from frappe.utils import get_url_to_form, strip_html + email = frappe.db.get_value("User", doc.for_user, "email") + if not email: + return + doc_link = get_url_to_form(doc.document_type, doc.document_name) header = get_email_header(doc) email_subject = strip_html(doc.subject) frappe.sendmail( - recipients=doc.for_user, + recipients=email, subject=email_subject, template="new_notification", args={ @@ -125,6 +133,22 @@ def get_email_header(doc): return header_map[doc.type or "Default"] +@frappe.whitelist() +def get_notification_logs(limit=20): + notification_logs = frappe.db.get_list( + "Notification Log", fields=["*"], limit=limit, order_by="creation desc" + ) + + users = [log.from_user for log in notification_logs] + users = [*set(users)] # remove duplicates + user_info = frappe._dict() + + for user in users: + frappe.utils.add_user_info(user, user_info) + + return {"notification_logs": notification_logs, "user_info": user_info} + + @frappe.whitelist() def mark_all_as_read(): unread_docs_list = frappe.db.get_all( @@ -149,6 +173,6 @@ def trigger_indicator_hide(): def set_notifications_as_unseen(user): try: - frappe.db.set_value("Notification Settings", user, "seen", 0) + frappe.db.set_value("Notification Settings", user, "seen", 0, update_modified=False) except frappe.DoesNotExistError: return diff --git a/frappe/desk/doctype/notification_log/test_notification_log.py b/frappe/desk/doctype/notification_log/test_notification_log.py index 44b1b53ead..532f05ab57 100644 --- a/frappe/desk/doctype/notification_log/test_notification_log.py +++ b/frappe/desk/doctype/notification_log/test_notification_log.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and Contributors # License: MIT. See LICENSE import unittest diff --git a/frappe/desk/doctype/notification_settings/notification_settings.js b/frappe/desk/doctype/notification_settings/notification_settings.js index cc2fd95204..ba72369273 100644 --- a/frappe/desk/doctype/notification_settings/notification_settings.js +++ b/frappe/desk/doctype/notification_settings/notification_settings.js @@ -1,28 +1,27 @@ // Copyright (c) 2019, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Notification Settings', { +frappe.ui.form.on("Notification Settings", { onload: (frm) => { frappe.breadcrumbs.add({ - label: __('Settings'), - route: '#modules/Settings', - type: 'Custom' + label: __("Settings"), + route: "#modules/Settings", + type: "Custom", }); - frm.set_query('subscribed_documents', () => { + frm.set_query("subscribed_documents", () => { return { filters: { - istable: 0 - } + istable: 0, + }, }; }); }, refresh: (frm) => { - if (frappe.user.has_role('System Manager')) { - frm.add_custom_button(__('Go to Notification Settings List'), () => { - frappe.set_route('List', 'Notification Settings'); + if (frappe.user.has_role("System Manager")) { + frm.add_custom_button(__("Go to Notification Settings List"), () => { + frappe.set_route("List", "Notification Settings"); }); } - } - + }, }); diff --git a/frappe/desk/doctype/notification_settings/notification_settings.py b/frappe/desk/doctype/notification_settings/notification_settings.py index bbb4a62154..801d512fe7 100644 --- a/frappe/desk/doctype/notification_settings/notification_settings.py +++ b/frappe/desk/doctype/notification_settings/notification_settings.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and contributors # License: MIT. See LICENSE @@ -48,9 +47,16 @@ def create_notification_settings(user): _doc.insert(ignore_permissions=True) -def toggle_notifications(user, enable=False): - if frappe.db.exists("Notification Settings", user): - frappe.db.set_value("Notification Settings", user, "enabled", enable) +def toggle_notifications(user: str, enable: bool = False): + try: + settings = frappe.get_doc("Notification Settings", user) + except frappe.DoesNotExistError: + frappe.clear_last_message() + return + + if settings.enabled != enable: + settings.enabled = enable + settings.save() @frappe.whitelist() @@ -81,7 +87,7 @@ def get_permission_query_conditions(user): if "System Manager" in roles: return """(`tabNotification Settings`.name != 'Administrator')""" - return """(`tabNotification Settings`.name = '{user}')""".format(user=user) + return f"""(`tabNotification Settings`.name = {frappe.db.escape(user)})""" @frappe.whitelist() diff --git a/frappe/desk/doctype/notification_subscribed_document/notification_subscribed_document.py b/frappe/desk/doctype/notification_subscribed_document/notification_subscribed_document.py index b72f827cd7..551ee6dc85 100644 --- a/frappe/desk/doctype/notification_subscribed_document/notification_subscribed_document.py +++ b/frappe/desk/doctype/notification_subscribed_document/notification_subscribed_document.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and contributors # License: MIT. See LICENSE diff --git a/frappe/desk/doctype/number_card/number_card.js b/frappe/desk/doctype/number_card/number_card.js index f548388a99..77ab2b4ef8 100644 --- a/frappe/desk/doctype/number_card/number_card.js +++ b/frappe/desk/doctype/number_card/number_card.js @@ -1,70 +1,75 @@ // Copyright (c) 2020, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Number Card', { - refresh: function(frm) { +frappe.ui.form.on("Number Card", { + refresh: function (frm) { if (!frappe.boot.developer_mode && frm.doc.is_standard) { frm.disable_form(); } frm.set_df_property("filters_section", "hidden", 1); frm.set_df_property("dynamic_filters_section", "hidden", 1); - frm.trigger('set_options'); + frm.trigger("set_options"); if (!frm.doc.type) { - frm.set_value('type', 'Document Type'); + frm.set_value("type", "Document Type"); } - if (frm.doc.type == 'Report' && frm.doc.report_name) { - frm.trigger('set_report_filters'); + if (frm.doc.type == "Report" && frm.doc.report_name) { + frm.trigger("set_report_filters"); } - if (frm.doc.type == 'Custom') { + if (frm.doc.type == "Custom") { if (!frappe.boot.developer_mode) { frm.disable_form(); } frm.filters = eval(frm.doc.filters_config); - frm.trigger('set_filters_description'); - frm.trigger('set_method_description'); - frm.trigger('render_filters_table'); + frm.trigger("set_filters_description"); + frm.trigger("set_method_description"); + frm.trigger("render_filters_table"); + } + frm.trigger("set_parent_document_type"); + + if (!frm.is_new()) { + frm.trigger("create_add_to_dashboard_button"); } - frm.trigger('create_add_to_dashboard_button'); - frm.trigger('set_parent_document_type'); }, - create_add_to_dashboard_button: function(frm) { - frm.add_custom_button('Add Card to Dashboard', () => { + create_add_to_dashboard_button: function (frm) { + frm.add_custom_button("Add Card to Dashboard", () => { const dialog = frappe.dashboard_utils.get_add_to_dashboard_dialog( frm.doc.name, - 'Number Card', - 'frappe.desk.doctype.number_card.number_card.add_card_to_dashboard' + "Number Card", + "frappe.desk.doctype.number_card.number_card.add_card_to_dashboard" ); if (!frm.doc.name) { - frappe.msgprint(__('Please create Card first')); + frappe.msgprint(__("Please create Card first")); } else { dialog.show(); } }); }, - before_save: function(frm) { - let dynamic_filters = JSON.parse(frm.doc.dynamic_filters_json || 'null'); - let static_filters = JSON.parse(frm.doc.filters_json || 'null'); - static_filters = - frappe.dashboard_utils.remove_common_static_filter_values(static_filters, dynamic_filters); + before_save: function (frm) { + let dynamic_filters = JSON.parse(frm.doc.dynamic_filters_json || "null"); + let static_filters = JSON.parse(frm.doc.filters_json || "null"); + static_filters = frappe.dashboard_utils.remove_common_static_filter_values( + static_filters, + dynamic_filters + ); - frm.set_value('filters_json', JSON.stringify(static_filters)); - frm.trigger('render_filters_table'); - frm.trigger('render_dynamic_filters_table'); + frm.set_value("filters_json", JSON.stringify(static_filters)); + frm.trigger("render_filters_table"); + frm.trigger("render_dynamic_filters_table"); }, - is_standard: function(frm) { - frm.trigger('render_dynamic_filters_table'); + is_standard: function (frm) { + frm.trigger("render_dynamic_filters_table"); frm.set_df_property("dynamic_filters_section", "hidden", 1); }, - set_filters_description: function(frm) { - if (frm.doc.type == 'Custom') { + set_filters_description: function (frm) { + if (frm.doc.type == "Custom") { frm.fields_dict.filters_config.set_description(` Set the filters here. For example: 
    @@ -88,8 +93,8 @@ frappe.ui.form.on('Number Card', {
 		}
 	},
 
-	set_method_description: function(frm) {
-		if (frm.doc.type == 'Custom') {
+	set_method_description: function (frm) {
+		if (frm.doc.type == "Custom") {
 			frm.fields_dict.method.set_description(`
 		Set the path to a whitelisted function that will return the number on the card in the format:
 
    @@ -102,53 +107,52 @@ frappe.ui.form.on('Number Card', {
 		}
 	},
 
-	type: function(frm) {
-		frm.trigger('set_filters_description');
-		if (frm.doc.type == 'Report') {
-			frm.set_query('report_name', () => {
+	type: function (frm) {
+		frm.trigger("set_filters_description");
+		if (frm.doc.type == "Report") {
+			frm.set_query("report_name", () => {
 				return {
 					filters: {
-						'report_type': ['!=', 'Report Builder']
-					}
+						report_type: ["!=", "Report Builder"],
+					},
 				};
 			});
 		}
-
 	},
 
-	report_name: function(frm) {
+	report_name: function (frm) {
 		frm.filters = [];
-		frm.set_value('filters_json', '{}');
-		frm.set_value('dynamic_filters_json', '{}');
-		frm.set_df_property('report_field', 'options', []);
-		frm.trigger('set_report_filters');
+		frm.set_value("filters_json", "{}");
+		frm.set_value("dynamic_filters_json", "{}");
+		frm.set_df_property("report_field", "options", []);
+		frm.trigger("set_report_filters");
 	},
 
-	filters_config: function(frm) {
+	filters_config: function (frm) {
 		frm.filters = eval(frm.doc.filters_config);
 		const filter_values = frappe.report_utils.get_filter_values(frm.filters);
-		frm.set_value('filters_json', JSON.stringify(filter_values));
-		frm.trigger('render_filters_table');
+		frm.set_value("filters_json", JSON.stringify(filter_values));
+		frm.trigger("render_filters_table");
 	},
 
-	document_type: function(frm) {
-		frm.set_query('document_type', function() {
+	document_type: function (frm) {
+		frm.set_query("document_type", function () {
 			return {
 				filters: {
-					'issingle': false
-				}
+					issingle: false,
+				},
 			};
 		});
-		frm.set_value('filters_json', '[]');
-		frm.set_value('dynamic_filters_json', '[]');
-		frm.set_value('aggregate_function_based_on', '');
-		frm.set_value('parent_document_type', '');
-		frm.trigger('set_options');
-		frm.trigger('set_parent_document_type');
+		frm.set_value("filters_json", "[]");
+		frm.set_value("dynamic_filters_json", "[]");
+		frm.set_value("aggregate_function_based_on", "");
+		frm.set_value("parent_document_type", "");
+		frm.trigger("set_options");
+		frm.trigger("set_parent_document_type");
 	},
 
-	set_options: function(frm) {
-		if (frm.doc.type !== 'Document Type') {
+	set_options: function (frm) {
+		if (frm.doc.type !== "Document Type") {
 			return;
 		}
 
@@ -157,134 +161,148 @@ frappe.ui.form.on('Number Card', {
 
 		if (doctype) {
 			frappe.model.with_doctype(doctype, () => {
-				frappe.get_meta(doctype).fields.map(df => {
+				frappe.get_meta(doctype).fields.map((df) => {
 					if (frappe.model.numeric_fieldtypes.includes(df.fieldtype)) {
-						if (df.fieldtype == 'Currency') {
-							if (!df.options || df.options !== 'Company:company:default_currency') {
+						if (df.fieldtype == "Currency") {
+							if (!df.options || df.options !== "Company:company:default_currency") {
 								return;
 							}
 						}
-						aggregate_based_on_fields.push({label: df.label, value: df.fieldname});
+						aggregate_based_on_fields.push({ label: df.label, value: df.fieldname });
 					}
 				});
 
-				frm.set_df_property('aggregate_function_based_on', 'options', aggregate_based_on_fields);
+				frm.set_df_property(
+					"aggregate_function_based_on",
+					"options",
+					aggregate_based_on_fields
+				);
 			});
-			frm.trigger('render_filters_table');
-			frm.trigger('render_dynamic_filters_table');
+			frm.trigger("render_filters_table");
+			frm.trigger("render_dynamic_filters_table");
 		}
 	},
 
-	set_report_filters: function(frm) {
+	set_report_filters: function (frm) {
 		const report_name = frm.doc.report_name;
 		if (report_name) {
-			frappe.report_utils.get_report_filters(report_name).then(filters => {
+			frappe.report_utils.get_report_filters(report_name).then((filters) => {
 				if (filters) {
 					frm.filters = filters;
 					const filter_values = frappe.report_utils.get_filter_values(filters);
 					if (frm.doc.filters_json.length <= 2) {
-						frm.set_value('filters_json', JSON.stringify(filter_values));
+						frm.set_value("filters_json", JSON.stringify(filter_values));
 					}
 				}
-				frm.trigger('render_filters_table');
-				frm.trigger('set_report_field_options');
-				frm.trigger('render_dynamic_filters_table');
+				frm.trigger("render_filters_table");
+				frm.trigger("set_report_field_options");
+				frm.trigger("render_dynamic_filters_table");
 			});
 		}
 	},
 
-	set_report_field_options: function(frm) {
+	set_report_field_options: function (frm) {
 		let filters = frm.doc.filters_json.length > 2 ? JSON.parse(frm.doc.filters_json) : null;
 		if (frm.doc.dynamic_filters_json && frm.doc.dynamic_filters_json.length > 2) {
 			filters = frappe.dashboard_utils.get_all_filters(frm.doc);
 		}
-		frappe.xcall(
-			'frappe.desk.query_report.run',
-			{
+		frappe
+			.xcall("frappe.desk.query_report.run", {
 				report_name: frm.doc.report_name,
 				filters: filters,
-				ignore_prepared_report: 1
-			}
-		).then(data => {
-			if (data.result.length) {
-				frm.field_options = frappe.report_utils.get_field_options_from_report(data.columns, data);
-				frm.set_df_property('report_field', 'options', frm.field_options.numeric_fields);
-				if (!frm.field_options.numeric_fields.length) {
-					frappe.msgprint(__("Report has no numeric fields, please change the Report Name"));
+				ignore_prepared_report: 1,
+			})
+			.then((data) => {
+				if (data.result.length) {
+					frm.field_options = frappe.report_utils.get_field_options_from_report(
+						data.columns,
+						data
+					);
+					frm.set_df_property(
+						"report_field",
+						"options",
+						frm.field_options.numeric_fields
+					);
+					if (!frm.field_options.numeric_fields.length) {
+						frappe.msgprint(
+							__("Report has no numeric fields, please change the Report Name")
+						);
+					}
+				} else {
+					frappe.msgprint(
+						__(
+							"Report has no data, please modify the filters or change the Report Name"
+						)
+					);
 				}
-			} else {
-				frappe.msgprint(__('Report has no data, please modify the filters or change the Report Name'));
-			}
-		});
+			});
 	},
 
-	render_filters_table: function(frm) {
+	render_filters_table: function (frm) {
 		frm.set_df_property("filters_section", "hidden", 0);
-		let is_document_type = frm.doc.type == 'Document Type';
-		let is_dynamic_filter = f => ['Date', 'DateRange'].includes(f.fieldtype) && f.default;
+		let is_document_type = frm.doc.type == "Document Type";
+		let is_dynamic_filter = (f) => ["Date", "DateRange"].includes(f.fieldtype) && f.default;
 
-		let wrapper = $(frm.get_field('filters_json').wrapper).empty();
+		let wrapper = $(frm.get_field("filters_json").wrapper).empty();
 		let table = $(`
-					
-					
-					
+					
+					
+					
    
 			
 				
    ${__('Filter')}${__('Condition')}${__('Value')}${__("Filter")}${__("Condition")}${__("Value")}
 				
    
 			    
 			
 		
    `).appendTo(wrapper);
 		$(`
    ${__("Click table to edit")}
    `).appendTo(wrapper);
 
-		let filters = JSON.parse(frm.doc.filters_json || '[]');
+		let filters = JSON.parse(frm.doc.filters_json || "[]");
 		let filters_set = false;
 
 		// Set dynamic filters for reports
-		if (frm.doc.type == 'Report') {
+		if (frm.doc.type == "Report") {
 			let set_filters = false;
-			frm.filters.forEach(f => {
+			frm.filters.forEach((f) => {
 				if (is_dynamic_filter(f)) {
 					filters[f.fieldname] = f.default;
 					set_filters = true;
 				}
 			});
-			set_filters && frm.set_value('filters_json', JSON.stringify(filters));
+			set_filters && frm.set_value("filters_json", JSON.stringify(filters));
 		}
 
 		let fields = [];
 		if (is_document_type) {
 			fields = [
 				{
-					fieldtype: 'HTML',
-					fieldname: 'filter_area',
-				}
+					fieldtype: "HTML",
+					fieldname: "filter_area",
+				},
 			];
 
 			if (filters.length) {
-				filters.forEach(filter => {
-					const filter_row =
-						$(`
+				filters.forEach((filter) => {
+					const filter_row = $(`
 							${filter[1]}
 							${filter[2] || ""}
 							${filter[3]}
 						`);
 
-					table.find('tbody').append(filter_row);
+					table.find("tbody").append(filter_row);
 				});
 				filters_set = true;
 			}
 		} else if (frm.filters.length) {
-			fields = frm.filters.filter(f => f.fieldname);
-			fields.map(f => {
+			fields = frm.filters.filter((f) => f.fieldname);
+			fields.map((f) => {
 				if (filters[f.fieldname]) {
-					let condition = '=';
-					const filter_row =
-						$(`
+					let condition = "=";
+					const filter_row = $(`
 							${f.label}
 							${condition}
 							${filters[f.fieldname] || ""}
 						`);
-					table.find('tbody').append(filter_row);
+					table.find("tbody").append(filter_row);
 					if (!filters_set) filters_set = true;
 				}
 			});
@@ -293,32 +311,32 @@ frappe.ui.form.on('Number Card', {
 		if (!filters_set) {
 			const filter_row = $(`
 				${__("Click to Set Filters")}`);
-			table.find('tbody').append(filter_row);
+			table.find("tbody").append(filter_row);
 		}
 
-		table.on('click', () => {
+		table.on("click", () => {
 			let dialog = new frappe.ui.Dialog({
-				title: __('Set Filters'),
-				fields: fields.filter(f => !is_dynamic_filter(f)),
-				primary_action: function() {
+				title: __("Set Filters"),
+				fields: fields.filter((f) => !is_dynamic_filter(f)),
+				primary_action: function () {
 					let values = this.get_values();
 					if (values) {
 						this.hide();
 						if (is_document_type) {
 							let filters = frm.filter_group.get_filters();
-							frm.set_value('filters_json', JSON.stringify(filters));
+							frm.set_value("filters_json", JSON.stringify(filters));
 						} else {
-							frm.set_value('filters_json', JSON.stringify(values));
+							frm.set_value("filters_json", JSON.stringify(values));
 						}
-						frm.trigger('render_filters_table');
+						frm.trigger("render_filters_table");
 					}
 				},
-				primary_action_label: "Set"
+				primary_action_label: "Set",
 			});
 
 			if (is_document_type) {
 				frm.filter_group = new frappe.ui.FilterGroup({
-					parent: dialog.get_field('filter_area').$wrapper,
+					parent: dialog.get_field("filter_area").$wrapper,
 					doctype: frm.doc.document_type,
 					parent_doctype: frm.doc.parent_document_type,
 					on_change: () => {},
@@ -328,56 +346,61 @@ frappe.ui.form.on('Number Card', {
 
 			dialog.show();
 
-			if (frm.doc.type == 'Report') {
+			if (frm.doc.type == "Report") {
 				//Set query report object so that it can be used while fetching filter values in the report
-				frappe.query_report = new frappe.views.QueryReport({'filters': dialog.fields_list});
-				frappe.query_reports[frm.doc.report_name]
-					&& frappe.query_reports[frm.doc.report_name].onload
-						&& frappe.query_reports[frm.doc.report_name].onload(frappe.query_report);
+				frappe.query_report = new frappe.views.QueryReport({
+					filters: dialog.fields_list,
+				});
+				frappe.query_reports[frm.doc.report_name] &&
+					frappe.query_reports[frm.doc.report_name].onload &&
+					frappe.query_reports[frm.doc.report_name].onload(frappe.query_report);
 			}
 
 			dialog.set_values(filters);
 		});
-
 	},
 
 	render_dynamic_filters_table(frm) {
-		if (!frappe.boot.developer_mode || !frm.doc.is_standard || frm.doc.type == 'Custom') {
+		if (!frappe.boot.developer_mode || !frm.doc.is_standard || frm.doc.type == "Custom") {
 			return;
 		}
 
 		frm.set_df_property("dynamic_filters_section", "hidden", 0);
 
-		let is_document_type = frm.doc.type == 'Document Type';
+		let is_document_type = frm.doc.type == "Document Type";
 
-		let wrapper = $(frm.get_field('dynamic_filters_json').wrapper).empty();
+		let wrapper = $(frm.get_field("dynamic_filters_json").wrapper).empty();
 
-		frm.dynamic_filter_table = $(`
+		frm.dynamic_filter_table =
+			$(`
    
-					
-					
-					
+					
+					
+					
    
 			
 				
    ${__('Filter')}${__('Condition')}${__('Value')}${__("Filter")}${__("Condition")}${__("Value")}
 				
    
 			    
 			
 		
    `).appendTo(wrapper);
 
-		frm.dynamic_filters = frm.doc.dynamic_filters_json && frm.doc.dynamic_filters_json.length > 2
-			? JSON.parse(frm.doc.dynamic_filters_json)
-			: null;
+		frm.dynamic_filters =
+			frm.doc.dynamic_filters_json && frm.doc.dynamic_filters_json.length > 2
+				? JSON.parse(frm.doc.dynamic_filters_json)
+				: null;
 
-		frm.trigger('set_dynamic_filters_in_table');
+		frm.trigger("set_dynamic_filters_in_table");
 
-		let filters = JSON.parse(frm.doc.filters_json || '[]');
+		let filters = JSON.parse(frm.doc.filters_json || "[]");
 
 		let fields = frappe.dashboard_utils.get_fields_for_dynamic_filter_dialog(
-			is_document_type, filters, frm.dynamic_filters
+			is_document_type,
+			filters,
+			frm.dynamic_filters
 		);
 
-		frm.dynamic_filter_table.on('click', () => {
+		frm.dynamic_filter_table.on("click", () => {
 			let dialog = new frappe.ui.Dialog({
-				title: __('Set Dynamic Filters'),
+				title: __("Set Dynamic Filters"),
 				fields: fields,
 				primary_action: () => {
 					let values = dialog.get_values();
@@ -385,19 +408,19 @@ frappe.ui.form.on('Number Card', {
 					let dynamic_filters = [];
 					for (let key of Object.keys(values)) {
 						if (is_document_type) {
-							let [doctype, fieldname] = key.split(':');
-							dynamic_filters.push([doctype, fieldname, '=', values[key]]);
+							let [doctype, fieldname] = key.split(":");
+							dynamic_filters.push([doctype, fieldname, "=", values[key]]);
 						}
 					}
 
 					if (is_document_type) {
-						frm.set_value('dynamic_filters_json', JSON.stringify(dynamic_filters));
+						frm.set_value("dynamic_filters_json", JSON.stringify(dynamic_filters));
 					} else {
-						frm.set_value('dynamic_filters_json', JSON.stringify(values));
+						frm.set_value("dynamic_filters_json", JSON.stringify(values));
 					}
-					frm.trigger('set_dynamic_filters_in_table');
+					frm.trigger("set_dynamic_filters_in_table");
 				},
-				primary_action_label: "Set"
+				primary_action_label: "Set",
 			});
 
 			dialog.show();
@@ -405,71 +428,70 @@ frappe.ui.form.on('Number Card', {
 		});
 	},
 
-	set_dynamic_filters_in_table: function(frm) {
-		frm.dynamic_filters =  frm.doc.dynamic_filters_json && frm.doc.dynamic_filters_json.length > 2
-			? JSON.parse(frm.doc.dynamic_filters_json)
-			: null;
+	set_dynamic_filters_in_table: function (frm) {
+		frm.dynamic_filters =
+			frm.doc.dynamic_filters_json && frm.doc.dynamic_filters_json.length > 2
+				? JSON.parse(frm.doc.dynamic_filters_json)
+				: null;
 
 		if (!frm.dynamic_filters) {
 			const filter_row = $(`
 				${__("Click to Set Dynamic Filters")}`);
-			frm.dynamic_filter_table.find('tbody').html(filter_row);
+			frm.dynamic_filter_table.find("tbody").html(filter_row);
 		} else {
-			let filter_rows = '';
+			let filter_rows = "";
 			if ($.isArray(frm.dynamic_filters)) {
-				frm.dynamic_filters.forEach(filter => {
-					filter_rows +=
-						`
+				frm.dynamic_filters.forEach((filter) => {
+					filter_rows += `
 							${filter[1]}
 							${filter[2] || ""}
 							${filter[3]}
 						`;
 				});
 			} else {
-				let condition = '=';
+				let condition = "=";
 				for (let [key, val] of Object.entries(frm.dynamic_filters)) {
-					filter_rows +=
-						`
+					filter_rows += `
 							${key}
 							${condition}
 							${val || ""}
-						`
-					;
+						`;
 				}
 			}
 
-			frm.dynamic_filter_table.find('tbody').html(filter_rows);
+			frm.dynamic_filter_table.find("tbody").html(filter_rows);
 		}
 	},
 
-	set_parent_document_type: async function(frm) {
+	set_parent_document_type: async function (frm) {
 		let document_type = frm.doc.document_type;
-		let doc_is_table = document_type &&
-			(await frappe.db.get_value('DocType', document_type, 'istable')).message.istable;
+		let doc_is_table =
+			document_type &&
+			(await frappe.db.get_value("DocType", document_type, "istable")).message.istable;
 
-		frm.set_df_property('parent_document_type', 'hidden', !doc_is_table);
+		frm.set_df_property("parent_document_type", "hidden", !doc_is_table);
 
 		if (document_type && doc_is_table) {
-			let parent = await frappe.db.get_list('DocField', {
+			let parent = await frappe.db.get_list("DocField", {
 				filters: {
-					'fieldtype': 'Table',
-					'options': document_type
+					fieldtype: "Table",
+					options: document_type,
 				},
-				fields: ['parent']
+				fields: ["parent"],
 			});
 
-			parent && frm.set_query('parent_document_type', function() {
-				return {
-					filters: {
-						"name": ['in', parent.map(({ parent }) => parent)]
-					}
-				};
-			});
+			parent &&
+				frm.set_query("parent_document_type", function () {
+					return {
+						filters: {
+							name: ["in", parent.map(({ parent }) => parent)],
+						},
+					};
+				});
 
 			if (parent.length === 1) {
-				frm.set_value('parent_document_type', parent[0].parent);
+				frm.set_value("parent_document_type", parent[0].parent);
 			}
 		}
-	}
-
+	},
 });
diff --git a/frappe/desk/doctype/number_card/number_card.json b/frappe/desk/doctype/number_card/number_card.json
index 7975d878ba..ab33715d12 100644
--- a/frappe/desk/doctype/number_card/number_card.json
+++ b/frappe/desk/doctype/number_card/number_card.json
@@ -51,7 +51,7 @@
    "options": "Count\nSum\nAverage\nMinimum\nMaximum"
   },
   {
-   "depends_on": "eval: doc.function !== 'Count'",
+   "depends_on": "eval: doc.type === 'Document Type' && doc.function !== 'Count'",
    "fieldname": "aggregate_function_based_on",
    "fieldtype": "Select",
    "label": "Aggregate Function Based On",
@@ -192,6 +192,7 @@
   },
   {
    "description": "The document type selected is a child table, so the parent document type is required.",
+   "depends_on": "eval: doc.type === 'Document Type'",
    "fieldname": "parent_document_type",
    "fieldtype": "Link",
    "label": "Parent Document Type",
@@ -199,7 +200,7 @@
   }
  ],
  "links": [],
- "modified": "2022-03-10 15:34:38.210910",
+ "modified": "2022-06-12 15:34:38.210910",
  "modified_by": "Administrator",
  "module": "Desk",
  "name": "Number Card",
diff --git a/frappe/desk/doctype/number_card/number_card.py b/frappe/desk/doctype/number_card/number_card.py
index 370b187ffe..8e808ff635 100644
--- a/frappe/desk/doctype/number_card/number_card.py
+++ b/frappe/desk/doctype/number_card/number_card.py
@@ -1,13 +1,15 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2020, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
 import frappe
 from frappe import _
+from frappe.boot import get_allowed_report_names
 from frappe.config import get_modules_from_all_apps_for_user
 from frappe.model.document import Document
 from frappe.model.naming import append_number_if_name_exists
 from frappe.modules.export_file import export_to_files
+from frappe.query_builder import Criterion
+from frappe.query_builder.utils import DocType
 from frappe.utils import cint
 
 
@@ -20,15 +22,23 @@ class NumberCard(Document):
 			self.name = append_number_if_name_exists("Number Card", self.name)
 
 	def validate(self):
-		if not self.document_type:
-			frappe.throw(_("Document type is required to create a number card"))
+		if self.type == "Document Type":
+			if not (self.document_type and self.function):
+				frappe.throw(_("Document Type and Function are required to create a number card"))
 
-		if (
-			self.document_type
-			and frappe.get_meta(self.document_type).istable
-			and not self.parent_document_type
-		):
-			frappe.throw(_("Parent document type is required to create a number card"))
+			if self.function != "Count" and not self.aggregate_function_based_on:
+				frappe.throw(_("Aggregate Field is required to create a number card"))
+
+			if frappe.get_meta(self.document_type).istable and not self.parent_document_type:
+				frappe.throw(_("Parent Document Type is required to create a number card"))
+
+		elif self.type == "Report":
+			if not (self.report_name and self.report_field and self.function):
+				frappe.throw(_("Report Name, Report Field and Fucntion are required to create a number card"))
+
+		elif self.type == "Custom":
+			if not self.method:
+				frappe.throw(_("Method is required to create a number card"))
 
 	def on_update(self):
 		if frappe.conf.developer_mode and self.is_standard:
@@ -80,9 +90,13 @@ def has_permission(doc, ptype, user):
 	if "System Manager" in roles:
 		return True
 
-	allowed_doctypes = tuple(frappe.permissions.get_doctypes_with_read())
-	if doc.document_type in allowed_doctypes:
-		return True
+	if doc.type == "Report":
+		if doc.report_name in get_allowed_report_names():
+			return True
+	else:
+		allowed_doctypes = tuple(frappe.permissions.get_doctypes_with_read())
+		if doc.document_type in allowed_doctypes:
+			return True
 
 	return False
 
@@ -102,7 +116,7 @@ def get_result(doc, filters, to_date=None):
 	function = sql_function_map[doc.function]
 
 	if function == "count":
-		fields = ["{function}(*) as result".format(function=function)]
+		fields = [f"{function}(*) as result"]
 	else:
 		fields = [
 			"{function}({based_on}) as result".format(
@@ -181,36 +195,18 @@ def get_cards_for_user(doctype, txt, searchfield, start, page_len, filters):
 	if not frappe.db.exists("DocType", doctype):
 		return
 
+	numberCard = DocType("Number Card")
+
 	if txt:
-		for field in searchfields:
-			search_conditions.append(
-				"`tab{doctype}`.`{field}` like %(txt)s".format(field=field, doctype=doctype, txt=txt)
-			)
+		search_conditions = [numberCard[field].like(f"%{txt}%") for field in searchfields]
 
-		search_conditions = " or ".join(search_conditions)
+	condition_query = frappe.qb.engine.build_conditions(doctype, filters)
 
-	search_conditions = "and (" + search_conditions + ")" if search_conditions else ""
-	conditions, values = frappe.db.build_conditions(filters)
-	values["txt"] = "%" + txt + "%"
-
-	return frappe.db.sql(
-		"""select
-			`tabNumber Card`.name, `tabNumber Card`.label, `tabNumber Card`.document_type
-		from
-			`tabNumber Card`
-		where
-			{conditions} and
-			(`tabNumber Card`.owner = '{user}' or
-			`tabNumber Card`.is_public = 1)
-			{search_conditions}
-	""".format(
-			filters=filters,
-			user=frappe.session.user,
-			search_conditions=search_conditions,
-			conditions=conditions,
-		),
-		values,
-	)
+	return (
+		condition_query.select(numberCard.name, numberCard.label, numberCard.document_type)
+		.where((numberCard.owner == frappe.session.user) | (numberCard.is_public == 1))
+		.where(Criterion.any(search_conditions))
+	).run()
 
 
 @frappe.whitelist()
diff --git a/frappe/desk/doctype/number_card/test_number_card.py b/frappe/desk/doctype/number_card/test_number_card.py
index 817ea2fad4..c0dda40104 100644
--- a/frappe/desk/doctype/number_card/test_number_card.py
+++ b/frappe/desk/doctype/number_card/test_number_card.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2020, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
 # import frappe
diff --git a/frappe/desk/doctype/number_card_link/number_card_link.py b/frappe/desk/doctype/number_card_link/number_card_link.py
index b630d7caa7..16cc7ba4e3 100644
--- a/frappe/desk/doctype/number_card_link/number_card_link.py
+++ b/frappe/desk/doctype/number_card_link/number_card_link.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2020, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
diff --git a/frappe/desk/doctype/onboarding_permission/onboarding_permission.js b/frappe/desk/doctype/onboarding_permission/onboarding_permission.js
index 752b8a02cc..ec2c8a03b0 100644
--- a/frappe/desk/doctype/onboarding_permission/onboarding_permission.js
+++ b/frappe/desk/doctype/onboarding_permission/onboarding_permission.js
@@ -1,8 +1,7 @@
 // Copyright (c) 2020, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Onboarding Permission', {
+frappe.ui.form.on("Onboarding Permission", {
 	// refresh: function(frm) {
-
 	// }
 });
diff --git a/frappe/desk/doctype/onboarding_permission/onboarding_permission.py b/frappe/desk/doctype/onboarding_permission/onboarding_permission.py
index a0e87c3067..d7db13762a 100644
--- a/frappe/desk/doctype/onboarding_permission/onboarding_permission.py
+++ b/frappe/desk/doctype/onboarding_permission/onboarding_permission.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2020, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
diff --git a/frappe/desk/doctype/onboarding_permission/test_onboarding_permission.py b/frappe/desk/doctype/onboarding_permission/test_onboarding_permission.py
index 9a12b0aab9..cdfe0d7890 100644
--- a/frappe/desk/doctype/onboarding_permission/test_onboarding_permission.py
+++ b/frappe/desk/doctype/onboarding_permission/test_onboarding_permission.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2020, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
 # import frappe
diff --git a/frappe/desk/doctype/onboarding_step/onboarding_step.js b/frappe/desk/doctype/onboarding_step/onboarding_step.js
index 3c9bbab9ac..67b2ed0501 100644
--- a/frappe/desk/doctype/onboarding_step/onboarding_step.js
+++ b/frappe/desk/doctype/onboarding_step/onboarding_step.js
@@ -2,18 +2,17 @@
 // For license information, please see license.txt
 
 frappe.ui.form.on("Onboarding Step", {
-
-	setup: function(frm) {
-		frm.set_query("form_tour", function() {
+	setup: function (frm) {
+		frm.set_query("form_tour", function () {
 			return {
 				filters: {
-					reference_doctype: frm.doc.reference_document
-				}
+					reference_doctype: frm.doc.reference_document,
+				},
 			};
 		});
 	},
 
-	refresh: function(frm) {
+	refresh: function (frm) {
 		frappe.boot.developer_mode &&
 			frm.set_intro(
 				__(
@@ -30,15 +29,16 @@ frappe.ui.form.on("Onboarding Step", {
 		}
 	},
 
-	reference_document: function(frm) {
+	reference_document: function (frm) {
 		if (frm.doc.reference_document && frm.doc.action == "Update Settings") {
 			setup_fields(frm);
 		}
 	},
 
-	action: function(frm) {
+	action: function (frm) {
 		if (frm.doc.action == "Show Form Tour") {
-			frm.fields_dict.reference_document.set_description(`You need to add the steps in the contoller JS file. For example: note.js
+			frm.fields_dict.reference_document
+				.set_description(`You need to add the steps in the contoller JS file. For example: note.js
 
    
 frappe.tour['Note'] = [
 	{
@@ -54,7 +54,7 @@ frappe.tour['Note'] = [
 		}
 	},
 
-	disable_form: function(frm) {
+	disable_form: function (frm) {
 		frm.set_read_only();
 		frm.fields
 			.filter((field) => field.has_input)
@@ -71,9 +71,7 @@ function setup_fields(frm) {
 			let fields = frappe
 				.get_meta(frm.doc.reference_document)
 				.fields.filter((df) => {
-					return ["Data", "Check", "Int", "Link", "Select"].includes(
-						df.fieldtype
-					);
+					return ["Data", "Check", "Int", "Link", "Select"].includes(df.fieldtype);
 				})
 				.map((df) => {
 					return {
diff --git a/frappe/desk/doctype/onboarding_step/onboarding_step.py b/frappe/desk/doctype/onboarding_step/onboarding_step.py
index 4a4d487cc8..b6807b62bd 100644
--- a/frappe/desk/doctype/onboarding_step/onboarding_step.py
+++ b/frappe/desk/doctype/onboarding_step/onboarding_step.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2020, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
diff --git a/frappe/desk/doctype/onboarding_step/test_onboarding_step.py b/frappe/desk/doctype/onboarding_step/test_onboarding_step.py
index 2342656a72..d8bf55584c 100644
--- a/frappe/desk/doctype/onboarding_step/test_onboarding_step.py
+++ b/frappe/desk/doctype/onboarding_step/test_onboarding_step.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2020, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
 # import frappe
diff --git a/frappe/desk/doctype/onboarding_step_map/onboarding_step_map.py b/frappe/desk/doctype/onboarding_step_map/onboarding_step_map.py
index 7c20e220db..8844316e68 100644
--- a/frappe/desk/doctype/onboarding_step_map/onboarding_step_map.py
+++ b/frappe/desk/doctype/onboarding_step_map/onboarding_step_map.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2020, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
diff --git a/frappe/desk/doctype/route_history/route_history.js b/frappe/desk/doctype/route_history/route_history.js
index 19689e406b..c68d4e2b54 100644
--- a/frappe/desk/doctype/route_history/route_history.js
+++ b/frappe/desk/doctype/route_history/route_history.js
@@ -1,8 +1,6 @@
 // Copyright (c) 2018, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Route History', {
-	refresh: function() {
-
-	}
+frappe.ui.form.on("Route History", {
+	refresh: function () {},
 });
diff --git a/frappe/desk/doctype/route_history/route_history.json b/frappe/desk/doctype/route_history/route_history.json
index 09db2320ca..a5d73fc360 100644
--- a/frappe/desk/doctype/route_history/route_history.json
+++ b/frappe/desk/doctype/route_history/route_history.json
@@ -1,126 +1,52 @@
 {
- "allow_copy": 0,
- "allow_guest_to_view": 0,
- "allow_import": 0,
- "allow_rename": 0,
- "beta": 0,
+ "actions": [],
  "creation": "2018-10-05 11:26:04.601113",
- "custom": 0,
- "docstatus": 0,
  "doctype": "DocType",
- "document_type": "",
  "editable_grid": 1,
  "engine": "InnoDB",
+ "field_order": [
+  "route",
+  "user"
+ ],
  "fields": [
   {
-   "allow_bulk_edit": 0,
-   "allow_in_quick_entry": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
    "fieldname": "route",
    "fieldtype": "Data",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
    "in_list_view": 1,
-   "in_standard_filter": 0,
-   "label": "Route",
-   "length": 0,
-   "no_copy": 0,
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "translatable": 0,
-   "unique": 0
+   "label": "Route"
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_in_quick_entry": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
    "fieldname": "user",
    "fieldtype": "Link",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
    "in_list_view": 1,
-   "in_standard_filter": 0,
+   "in_standard_filter": 1,
    "label": "User",
-   "length": 0,
-   "no_copy": 0,
-   "options": "User",
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "translatable": 0,
-   "unique": 0
+   "options": "User"
   }
  ],
- "has_web_view": 0,
- "hide_heading": 0,
- "hide_toolbar": 0,
- "idx": 0,
- "image_view": 0,
- "in_create": 0,
- "is_submittable": 0,
- "issingle": 0,
- "istable": 0,
- "max_attachments": 0,
- "modified": "2021-10-25 13:26:03.106050",
+ "links": [],
+ "modified": "2022-06-13 05:48:56.967244",
  "modified_by": "Administrator",
  "module": "Desk",
  "name": "Route History",
- "name_case": "",
  "owner": "Administrator",
  "permissions": [
   {
-   "amend": 0,
-   "cancel": 0,
    "create": 1,
    "delete": 1,
    "email": 1,
    "export": 1,
-   "if_owner": 0,
-   "import": 0,
-   "permlevel": 0,
    "print": 1,
    "read": 1,
    "report": 1,
    "role": "System Manager",
-   "set_user_permissions": 0,
    "share": 1,
-   "submit": 0,
    "write": 1
   }
  ],
  "quick_entry": 1,
- "read_only": 0,
- "read_only_onload": 0,
- "show_name_in_global_search": 0,
  "sort_field": "modified",
  "sort_order": "DESC",
- "track_seen": 0,
- "track_views": 0
-}
+ "states": [],
+ "title_field": "route"
+}
\ No newline at end of file
diff --git a/frappe/desk/doctype/route_history/route_history.py b/frappe/desk/doctype/route_history/route_history.py
index e712a5bb11..a576ac73f5 100644
--- a/frappe/desk/doctype/route_history/route_history.py
+++ b/frappe/desk/doctype/route_history/route_history.py
@@ -4,40 +4,16 @@
 import frappe
 from frappe.deferred_insert import deferred_insert as _deferred_insert
 from frappe.model.document import Document
-from frappe.query_builder import DocType
-from frappe.query_builder.functions import Count
 
 
 class RouteHistory(Document):
-	pass
+	@staticmethod
+	def clear_old_logs(days=30):
+		from frappe.query_builder import Interval
+		from frappe.query_builder.functions import Now
 
-
-def flush_old_route_records():
-	"""Deletes all route records except last 500 records per user"""
-	records_to_keep_limit = 500
-	RouteHistory = DocType("Route History")
-
-	users = (
-		frappe.qb.from_(RouteHistory)
-		.select(RouteHistory.user)
-		.groupby(RouteHistory.user)
-		.having(Count(RouteHistory.name) > records_to_keep_limit)
-	).run(pluck=True)
-
-	for user in users:
-		last_record_to_keep = frappe.get_all(
-			"Route History",
-			filters={"user": user},
-			limit_start=500,
-			fields=["modified"],
-			order_by="modified desc",
-			limit=1,
-		)
-
-		frappe.db.delete(
-			"Route History",
-			{"modified": ("<=", last_record_to_keep[0].modified), "user": user},
-		)
+		table = frappe.qb.DocType("Route History")
+		frappe.db.delete(table, filters=(table.modified < (Now() - Interval(days=days))))
 
 
 @frappe.whitelist()
diff --git a/frappe/desk/doctype/route_history/route_history_list.js b/frappe/desk/doctype/route_history/route_history_list.js
new file mode 100644
index 0000000000..03bf86b9fd
--- /dev/null
+++ b/frappe/desk/doctype/route_history/route_history_list.js
@@ -0,0 +1,7 @@
+frappe.listview_settings["Route History"] = {
+	onload: function (listview) {
+		frappe.require("logtypes.bundle.js", () => {
+			frappe.utils.logtypes.show_log_retention_message(cur_list.doctype);
+		});
+	},
+};
diff --git a/frappe/desk/doctype/system_console/system_console.js b/frappe/desk/doctype/system_console/system_console.js
index 7751ffe860..dc73f33b67 100644
--- a/frappe/desk/doctype/system_console/system_console.js
+++ b/frappe/desk/doctype/system_console/system_console.js
@@ -1,21 +1,21 @@
 // Copyright (c) 2020, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('System Console', {
-	onload: function(frm) {
+frappe.ui.form.on("System Console", {
+	onload: function (frm) {
 		frappe.ui.keys.add_shortcut({
-			shortcut: 'shift+enter',
-			action: () => frm.page.btn_primary.trigger('click'),
+			shortcut: "shift+enter",
+			action: () => frm.page.btn_primary.trigger("click"),
 			page: frm.page,
-			description: __('Execute Console script'),
+			description: __("Execute Console script"),
 			ignore_inputs: true,
 		});
 		frm.set_value("type", "Python");
 	},
 
-	refresh: function(frm) {
+	refresh: function (frm) {
 		frm.disable_save();
-		frm.page.set_primary_action(__("Execute"), $btn => {
+		frm.page.set_primary_action(__("Execute"), ($btn) => {
 			$btn.text(__("Executing..."));
 			return frm
 				.execute_action("Execute")
@@ -24,7 +24,7 @@ frappe.ui.form.on('System Console', {
 		});
 	},
 
-	type: function(frm) {
+	type: function (frm) {
 		if (frm.doc.type == "Python") {
 			frm.set_value("output", "");
 			if (frm.sql_output) {
@@ -34,7 +34,7 @@ frappe.ui.form.on('System Console', {
 		}
 	},
 
-	render_sql_output: function(frm) {
+	render_sql_output: function (frm) {
 		if (frm.doc.type !== "SQL") return;
 		if (frm.sql_output) {
 			frm.sql_output.destroy();
@@ -46,50 +46,51 @@ frappe.ui.form.on('System Console', {
 		}
 
 		let result = JSON.parse(frm.doc.output);
-		frm.set_value("output", `${result.length} ${result.length == 1 ? 'row' : 'rows'}`);
+		frm.set_value("output", `${result.length} ${result.length == 1 ? "row" : "rows"}`);
 
 		if (result.length) {
 			let columns = Object.keys(result[0]);
-			frm.sql_output = new DataTable(
-				frm.get_field("sql_output").$wrapper.get(0),
-				{
-					columns,
-					data: result
-				}
-			);
+			frm.sql_output = new DataTable(frm.get_field("sql_output").$wrapper.get(0), {
+				columns,
+				data: result,
+			});
 		}
 	},
 
-	show_processlist: function(frm) {
+	show_processlist: function (frm) {
 		if (frm.doc.show_processlist) {
 			// keep refreshing every 5 seconds
 			frm.events.refresh_processlist(frm);
-			frm.processlist_interval = setInterval(() => frm.events.refresh_processlist(frm), 5000);
+			frm.processlist_interval = setInterval(
+				() => frm.events.refresh_processlist(frm),
+				5000
+			);
 		} else {
 			if (frm.processlist_interval) {
-
 				// end it
 				clearInterval(frm.processlist_interval);
-				frm.get_field("processlist").html('');
+				frm.get_field("processlist").html("");
 			}
 		}
 	},
 
-	refresh_processlist: function(frm) {
+	refresh_processlist: function (frm) {
 		let timestamp = new Date();
-		frappe.call('frappe.desk.doctype.system_console.system_console.show_processlist').then(r => {
-			let rows = '';
-			for (let row of r.message) {
-				rows += `
+		frappe
+			.call("frappe.desk.doctype.system_console.system_console.show_processlist")
+			.then((r) => {
+				let rows = "";
+				for (let row of r.message) {
+					rows += `
 					${row.Id}
 					${row.Time}
 					${row.State}
 					${row.Info}
 					${row.Progress}
-				`
-			}
+				`;
+				}
 
-			frm.get_field('processlist').html(`
+				frm.get_field("processlist").html(`
 				
    Requested on: ${timestamp}
    
 				
@@ -100,6 +101,6 @@ frappe.ui.form.on('System Console', {
 					${rows}`);
-		});
+			});
 	},
 });
diff --git a/frappe/desk/doctype/system_console/system_console.json b/frappe/desk/doctype/system_console/system_console.json
index c92b2005ed..a851831909 100644
--- a/frappe/desk/doctype/system_console/system_console.json
+++ b/frappe/desk/doctype/system_console/system_console.json
@@ -1,7 +1,7 @@
 {
  "actions": [
   {
-   "action": "app/console-log",
+   "action": "/app/console-log",
    "action_type": "Route",
    "label": "Logs"
   },
@@ -86,7 +86,7 @@
  "index_web_pages_for_search": 1,
  "issingle": 1,
  "links": [],
- "modified": "2022-04-09 16:35:32.345542",
+ "modified": "2022-04-15 14:15:58.398590",
  "modified_by": "Administrator",
  "module": "Desk",
  "name": "System Console",
@@ -106,4 +106,4 @@
  "sort_order": "DESC",
  "states": [],
  "track_changes": 1
-}
\ No newline at end of file
+}
diff --git a/frappe/desk/doctype/system_console/system_console.py b/frappe/desk/doctype/system_console/system_console.py
index f1324403c3..993af6e753 100644
--- a/frappe/desk/doctype/system_console/system_console.py
+++ b/frappe/desk/doctype/system_console/system_console.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2020, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
@@ -19,7 +18,7 @@ class SystemConsole(Document):
 				self.output = "\n".join(frappe.debug_log)
 			elif self.type == "SQL":
 				self.output = frappe.as_json(read_sql(self.console, as_dict=1))
-		except:  # noqa: E722
+		except Exception:
 			self.output = frappe.get_traceback()
 
 		if self.commit:
diff --git a/frappe/desk/doctype/system_console/test_system_console.py b/frappe/desk/doctype/system_console/test_system_console.py
index 372cbbc1f4..96bf555f59 100644
--- a/frappe/desk/doctype/system_console/test_system_console.py
+++ b/frappe/desk/doctype/system_console/test_system_console.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2020, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
 import unittest
diff --git a/frappe/desk/doctype/tag/tag.js b/frappe/desk/doctype/tag/tag.js
index f55f98c3d0..1c60f417e0 100644
--- a/frappe/desk/doctype/tag/tag.js
+++ b/frappe/desk/doctype/tag/tag.js
@@ -1,8 +1,7 @@
 // Copyright (c) 2019, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Tag', {
+frappe.ui.form.on("Tag", {
 	// refresh: function(frm) {
-
 	// }
 });
diff --git a/frappe/desk/doctype/tag/tag.py b/frappe/desk/doctype/tag/tag.py
index aabf0351a5..ca167c148e 100644
--- a/frappe/desk/doctype/tag/tag.py
+++ b/frappe/desk/doctype/tag/tag.py
@@ -56,7 +56,7 @@ def get_tagged_docs(doctype, tag):
 
 @frappe.whitelist()
 def get_tags(doctype, txt):
-	tag = frappe.get_list("Tag", filters=[["name", "like", "%{}%".format(txt)]])
+	tag = frappe.get_list("Tag", filters=[["name", "like", f"%{txt}%"]])
 	tags = [t.name for t in tag]
 
 	return sorted(filter(lambda t: t and txt.lower() in t.lower(), list(set(tags))))
@@ -104,7 +104,7 @@ class DocTags:
 			tags = "," + ",".join(tl)
 		try:
 			frappe.db.sql(
-				"update `tab%s` set _user_tags=%s where name=%s" % (self.dt, "%s", "%s"), (tags, dn)
+				"update `tab{}` set _user_tags={} where name={}".format(self.dt, "%s", "%s"), (tags, dn)
 			)
 			doc = frappe.get_doc(self.dt, dn)
 			update_tags(doc, tags)
diff --git a/frappe/desk/doctype/tag_link/tag_link.js b/frappe/desk/doctype/tag_link/tag_link.js
index d85655bb90..e2cb4fcd7f 100644
--- a/frappe/desk/doctype/tag_link/tag_link.js
+++ b/frappe/desk/doctype/tag_link/tag_link.js
@@ -1,8 +1,7 @@
 // Copyright (c) 2019, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Tag Link', {
+frappe.ui.form.on("Tag Link", {
 	// refresh: function(frm) {
-
 	// }
 });
diff --git a/frappe/desk/doctype/tag_link/tag_link.py b/frappe/desk/doctype/tag_link/tag_link.py
index ec816352ca..a67e6a62d3 100644
--- a/frappe/desk/doctype/tag_link/tag_link.py
+++ b/frappe/desk/doctype/tag_link/tag_link.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2019, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
diff --git a/frappe/desk/doctype/tag_link/test_tag_link.py b/frappe/desk/doctype/tag_link/test_tag_link.py
index d4d1dd61fa..59d7bcd2bc 100644
--- a/frappe/desk/doctype/tag_link/test_tag_link.py
+++ b/frappe/desk/doctype/tag_link/test_tag_link.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2019, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
 # import frappe
diff --git a/frappe/desk/doctype/todo/test_todo.py b/frappe/desk/doctype/todo/test_todo.py
index 5c54889e00..56ca1f30e7 100644
--- a/frappe/desk/doctype/todo/test_todo.py
+++ b/frappe/desk/doctype/todo/test_todo.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
 import unittest
diff --git a/frappe/desk/doctype/todo/todo.js b/frappe/desk/doctype/todo/todo.js
index 0317281371..6c1af67d2b 100644
--- a/frappe/desk/doctype/todo/todo.js
+++ b/frappe/desk/doctype/todo/todo.js
@@ -1,40 +1,55 @@
 // bind events
 
 frappe.ui.form.on("ToDo", {
-	onload: function(frm) {
-		frm.set_query("reference_type", function(txt) {
+	onload: function (frm) {
+		frm.set_query("reference_type", function (txt) {
 			return {
-				"filters": {
-					"issingle": 0,
-				}
+				filters: {
+					issingle: 0,
+				},
 			};
 		});
 	},
-	refresh: function(frm) {
-		if(frm.doc.reference_type && frm.doc.reference_name) {
-			frm.add_custom_button(__(frm.doc.reference_name), function() {
+	refresh: function (frm) {
+		if (frm.doc.reference_type && frm.doc.reference_name) {
+			frm.add_custom_button(__(frm.doc.reference_name), function () {
 				frappe.set_route("Form", frm.doc.reference_type, frm.doc.reference_name);
 			});
 		}
 
 		if (!frm.doc.__islocal) {
-			if(frm.doc.status!=="Closed") {
-				frm.add_custom_button(__("Close"), function() {
-					frm.set_value("status", "Closed");
-					frm.save(null, function() {
-						// back to list
-						frappe.set_route("List", "ToDo");
-					});
-				}, "fa fa-check", "btn-success");
+			if (frm.doc.status !== "Closed") {
+				frm.add_custom_button(
+					__("Close"),
+					function () {
+						frm.set_value("status", "Closed");
+						frm.save(null, function () {
+							// back to list
+							frappe.set_route("List", "ToDo");
+						});
+					},
+					"fa fa-check",
+					"btn-success"
+				);
 			} else {
-				frm.add_custom_button(__("Reopen"), function() {
-					frm.set_value("status", "Open");
-					frm.save();
-				}, null, "btn-default");
+				frm.add_custom_button(
+					__("Reopen"),
+					function () {
+						frm.set_value("status", "Open");
+						frm.save();
+					},
+					null,
+					"btn-default"
+				);
 			}
-			frm.add_custom_button(__("New"), function() {
-				frappe.new_doc("ToDo")
-			}, null, "btn-default");
+			frm.add_custom_button(
+				__("New"),
+				function () {
+					frappe.new_doc("ToDo");
+				},
+				null,
+				"btn-default"
+			);
 		}
-	}
+	},
 });
diff --git a/frappe/desk/doctype/todo/todo_calendar.js b/frappe/desk/doctype/todo/todo_calendar.js
index 8ba020fac1..f79243a86e 100644
--- a/frappe/desk/doctype/todo/todo_calendar.js
+++ b/frappe/desk/doctype/todo/todo_calendar.js
@@ -3,29 +3,27 @@
 
 frappe.views.calendar["ToDo"] = {
 	field_map: {
-		"start": "date",
-		"end": "date",
-		"id": "name",
-		"title": "description",
-		"allDay": "allDay",
-		"progress": "progress"
+		start: "date",
+		end: "date",
+		id: "name",
+		title: "description",
+		allDay: "allDay",
+		progress: "progress",
 	},
 	gantt: true,
 	filters: [
 		{
-			"fieldtype": "Link",
-			"fieldname": "reference_type",
-			"options": "Task",
-			"label": __("Task")
+			fieldtype: "Link",
+			fieldname: "reference_type",
+			options: "Task",
+			label: __("Task"),
 		},
 		{
-			"fieldtype": "Dynamic Link",
-			"fieldname": "reference_name",
-			"options": "reference_type",
-			"label": __("Task")
-		}
-
+			fieldtype: "Dynamic Link",
+			fieldname: "reference_name",
+			options: "reference_type",
+			label: __("Task"),
+		},
 	],
-	get_events_method: "frappe.desk.calendar.get_events"
+	get_events_method: "frappe.desk.calendar.get_events",
 };
-
diff --git a/frappe/desk/doctype/todo/todo_list.js b/frappe/desk/doctype/todo/todo_list.js
index 53564cc017..2e4534e05c 100644
--- a/frappe/desk/doctype/todo/todo_list.js
+++ b/frappe/desk/doctype/todo/todo_list.js
@@ -1,40 +1,44 @@
-frappe.listview_settings['ToDo'] = {
+frappe.listview_settings["ToDo"] = {
 	hide_name_column: true,
 	add_fields: ["reference_type", "reference_name"],
 
-	onload: function(me) {
+	onload: function (me) {
 		if (!frappe.route_options) {
 			frappe.route_options = {
-				"owner": frappe.session.user,
-				"status": "Open"
+				owner: frappe.session.user,
+				status: "Open",
 			};
 		}
 		me.page.set_title(__("To Do"));
 	},
 
 	button: {
-		show: function(doc) {
+		show: function (doc) {
 			return doc.reference_name;
 		},
-		get_label: function() {
-			return __('Open');
+		get_label: function () {
+			return __("Open");
 		},
-		get_description: function(doc) {
-			return __('Open {0}', [`${doc.reference_type} ${doc.reference_name}`])
+		get_description: function (doc) {
+			return __("Open {0}", [`${doc.reference_type} ${doc.reference_name}`]);
+		},
+		action: function (doc) {
+			frappe.set_route("Form", doc.reference_type, doc.reference_name);
 		},
-		action: function(doc) {
-			frappe.set_route('Form', doc.reference_type, doc.reference_name);
-		}
 	},
 
-	refresh: function(me) {
+	refresh: function (me) {
 		if (me.todo_sidebar_setup) return;
 
 		// add assigned by me
-		me.page.add_sidebar_item(__("Assigned By Me"), function() {
-			me.filter_area.add([[me.doctype, "assigned_by", '=', frappe.session.user]]);
-		}, ('.list-link[data-view="Kanban"]'));
+		me.page.add_sidebar_item(
+			__("Assigned By Me"),
+			function () {
+				me.filter_area.add([[me.doctype, "assigned_by", "=", frappe.session.user]]);
+			},
+			'.list-link[data-view="Kanban"]'
+		);
 
 		me.todo_sidebar_setup = true;
 	},
-}
\ No newline at end of file
+};
diff --git a/frappe/desk/doctype/workspace/test_workspace.py b/frappe/desk/doctype/workspace/test_workspace.py
index 9281240e08..d0b0eba9e2 100644
--- a/frappe/desk/doctype/workspace/test_workspace.py
+++ b/frappe/desk/doctype/workspace/test_workspace.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2020, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
 import unittest
diff --git a/frappe/desk/doctype/workspace/workspace.js b/frappe/desk/doctype/workspace/workspace.js
index 3f912127fc..25721f9ae2 100644
--- a/frappe/desk/doctype/workspace/workspace.js
+++ b/frappe/desk/doctype/workspace/workspace.js
@@ -1,26 +1,30 @@
 // Copyright (c) 2020, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Workspace', {
-	setup: function() {
-		frappe.meta.get_field('Workspace Link', 'only_for').no_default = true;
+frappe.ui.form.on("Workspace", {
+	setup: function () {
+		frappe.meta.get_field("Workspace Link", "only_for").no_default = true;
 	},
 
-	refresh: function(frm) {
+	refresh: function (frm) {
 		frm.enable_save();
 
-		if (frm.doc.for_user || (frm.doc.public && !frm.has_perm('write') &&
-			!frappe.user.has_role('Workspace Manager'))) {
-			frm.trigger('disable_form');
+		if (
+			frm.doc.for_user ||
+			(frm.doc.public &&
+				!frm.has_perm("write") &&
+				!frappe.user.has_role("Workspace Manager"))
+		) {
+			frm.trigger("disable_form");
 		}
 	},
 
-	disable_form: function(frm) {
+	disable_form: function (frm) {
 		frm.fields
-			.filter(field => field.has_input)
-			.forEach(field => {
+			.filter((field) => field.has_input)
+			.forEach((field) => {
 				frm.set_df_property(field.df.fieldname, "read_only", "1");
 			});
 		frm.disable_save();
-	}
-});
\ No newline at end of file
+	},
+});
diff --git a/frappe/desk/doctype/workspace/workspace.json b/frappe/desk/doctype/workspace/workspace.json
index fa8b81f5fd..032de9de4e 100644
--- a/frappe/desk/doctype/workspace/workspace.json
+++ b/frappe/desk/doctype/workspace/workspace.json
@@ -26,6 +26,8 @@
   "shortcuts",
   "tab_break_18",
   "links",
+  "quick_lists_tab",
+  "quick_lists",
   "roles_tab",
   "roles"
  ],
@@ -155,11 +157,22 @@
    "fieldname": "roles_tab",
    "fieldtype": "Tab Break",
    "label": "Roles"
+  },
+  {
+   "fieldname": "quick_lists_tab",
+   "fieldtype": "Tab Break",
+   "label": "Quick Lists"
+  },
+  {
+   "fieldname": "quick_lists",
+   "fieldtype": "Table",
+   "label": "Quick Lists",
+   "options": "Workspace Quick List"
   }
  ],
  "in_create": 1,
  "links": [],
- "modified": "2022-01-27 12:06:13.111743",
+ "modified": "2022-05-12 13:00:03.925605",
  "modified_by": "Administrator",
  "module": "Desk",
  "name": "Workspace",
@@ -189,5 +202,6 @@
   }
  ],
  "sort_field": "modified",
- "sort_order": "DESC"
+ "sort_order": "DESC",
+ "states": []
 }
\ No newline at end of file
diff --git a/frappe/desk/doctype/workspace/workspace.py b/frappe/desk/doctype/workspace/workspace.py
index a2dbcbfbe2..9fa99884fb 100644
--- a/frappe/desk/doctype/workspace/workspace.py
+++ b/frappe/desk/doctype/workspace/workspace.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2020, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
@@ -80,7 +79,14 @@ class Workspace(Document):
 
 			# remove duplicate before adding
 			for idx, link in enumerate(self.links):
-				if link.label == card.get("label") and link.type == "Card Break":
+				if link.get("label") == card.get("label") and link.get("type") == "Card Break":
+					# count and set number of links for the card if link_count is 0
+					if link.link_count == 0:
+						for count, card_link in enumerate(self.links[idx + 1 :]):
+							if card_link.get("type") == "Card Break":
+								break
+							link.link_count = count + 1
+
 					del self.links[idx : idx + link.link_count + 1]
 
 			self.append(
@@ -199,21 +205,29 @@ def update_page(name, title, icon, parent, public):
 			doc.sequence_id = frappe.db.count("Workspace", {"public": public}, cache=True)
 			doc.public = public
 		doc.for_user = "" if public else doc.for_user or frappe.session.user
-		doc.label = "{0}-{1}".format(title, doc.for_user) if doc.for_user else title
+		doc.label = new_name = f"{title}-{doc.for_user}" if doc.for_user else title
 		doc.save(ignore_permissions=True)
 
-		if name != doc.label:
-			rename_doc("Workspace", name, doc.label, force=True, ignore_permissions=True)
+		if name != new_name:
+			rename_doc("Workspace", name, new_name, force=True, ignore_permissions=True)
 
 		# update new name and public in child pages
 		if child_docs:
 			for child in child_docs:
 				child_doc = frappe.get_doc("Workspace", child.name)
 				child_doc.parent_page = doc.title
-				child_doc.public = doc.public
+				if child_doc.public != public:
+					child_doc.public = public
+				child_doc.for_user = "" if public else child_doc.for_user or frappe.session.user
+				child_doc.label = new_child_name = (
+					f"{child_doc.title}-{child_doc.for_user}" if child_doc.for_user else child_doc.title
+				)
 				child_doc.save(ignore_permissions=True)
 
-	return {"name": doc.title, "public": doc.public, "label": doc.label}
+				if child.name != new_child_name:
+					rename_doc("Workspace", child.name, new_child_name, force=True, ignore_permissions=True)
+
+	return {"name": title, "public": public, "label": new_name}
 
 
 @frappe.whitelist()
@@ -236,7 +250,7 @@ def duplicate_page(page_name, new_page):
 	doc.label = doc.title
 	if not doc.public:
 		doc.for_user = doc.for_user or frappe.session.user
-		doc.label = "{0}-{1}".format(doc.title, doc.for_user)
+		doc.label = f"{doc.title}-{doc.for_user}"
 	doc.name = doc.label
 	if old_doc.public == doc.public:
 		doc.sequence_id += 0.1
diff --git a/frappe/desk/doctype/workspace_chart/workspace_chart.py b/frappe/desk/doctype/workspace_chart/workspace_chart.py
index e02cf06ee0..45f4229401 100644
--- a/frappe/desk/doctype/workspace_chart/workspace_chart.py
+++ b/frappe/desk/doctype/workspace_chart/workspace_chart.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2021, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
diff --git a/frappe/desk/doctype/workspace_link/workspace_link.py b/frappe/desk/doctype/workspace_link/workspace_link.py
index 5e55a7c2bd..5756846f38 100644
--- a/frappe/desk/doctype/workspace_link/workspace_link.py
+++ b/frappe/desk/doctype/workspace_link/workspace_link.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2021, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
diff --git a/frappe/data_migration/__init__.py b/frappe/desk/doctype/workspace_quick_list/__init__.py
similarity index 100%
rename from frappe/data_migration/__init__.py
rename to frappe/desk/doctype/workspace_quick_list/__init__.py
diff --git a/frappe/desk/doctype/workspace_quick_list/workspace_quick_list.json b/frappe/desk/doctype/workspace_quick_list/workspace_quick_list.json
new file mode 100644
index 0000000000..1542ebe03c
--- /dev/null
+++ b/frappe/desk/doctype/workspace_quick_list/workspace_quick_list.json
@@ -0,0 +1,60 @@
+{
+ "actions": [],
+ "creation": "2022-05-12 12:58:41.824496",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "document_type",
+  "column_break_1",
+  "label",
+  "section_break_4",
+  "quick_list_filter"
+ ],
+ "fields": [
+  {
+   "fieldname": "document_type",
+   "fieldtype": "Link",
+   "in_list_view": 1,
+   "label": "DocType",
+   "options": "DocType",
+   "reqd": 1
+  },
+  {
+   "fieldname": "column_break_1",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "label",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Label",
+   "reqd": 1
+  },
+  {
+   "fieldname": "section_break_4",
+   "fieldtype": "Section Break"
+  },
+  {
+   "fieldname": "quick_list_filter",
+   "fieldtype": "Code",
+   "in_list_view": 1,
+   "label": "Quick List Filter",
+   "options": "JSON"
+  }
+ ],
+ "index_web_pages_for_search": 1,
+ "istable": 1,
+ "links": [],
+ "modified": "2022-05-12 13:48:40.617623",
+ "modified_by": "Administrator",
+ "module": "Desk",
+ "name": "Workspace Quick List",
+ "owner": "Administrator",
+ "permissions": [],
+ "quick_entry": 1,
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "states": [],
+ "track_changes": 1
+}
\ No newline at end of file
diff --git a/frappe/desk/doctype/workspace_quick_list/workspace_quick_list.py b/frappe/desk/doctype/workspace_quick_list/workspace_quick_list.py
new file mode 100644
index 0000000000..9f26424115
--- /dev/null
+++ b/frappe/desk/doctype/workspace_quick_list/workspace_quick_list.py
@@ -0,0 +1,9 @@
+# Copyright (c) 2022, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+# import frappe
+from frappe.model.document import Document
+
+
+class WorkspaceQuickList(Document):
+	pass
diff --git a/frappe/desk/doctype/workspace_shortcut/workspace_shortcut.py b/frappe/desk/doctype/workspace_shortcut/workspace_shortcut.py
index 4ca86c8146..49ba37854c 100644
--- a/frappe/desk/doctype/workspace_shortcut/workspace_shortcut.py
+++ b/frappe/desk/doctype/workspace_shortcut/workspace_shortcut.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2021, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
diff --git a/frappe/desk/form/assign_to.py b/frappe/desk/form/assign_to.py
index 4107f95827..7853e807b8 100644
--- a/frappe/desk/form/assign_to.py
+++ b/frappe/desk/form/assign_to.py
@@ -217,7 +217,7 @@ def notify_assignment(
 	# Search for email address in description -- i.e. assignee
 	user_name = frappe.get_cached_value("User", frappe.session.user, "full_name")
 	title = get_title(doc_type, doc_name)
-	description_html = "
    {0}
    ".format(description) if description else None
+	description_html = f"
    {description}
    " if description else None
 
 	if action == "CLOSE":
 		subject = _("Your assignment on {0} {1} has been removed by {2}").format(
diff --git a/frappe/desk/form/document_follow.py b/frappe/desk/form/document_follow.py
index 527b9da036..f12e44fe61 100644
--- a/frappe/desk/form/document_follow.py
+++ b/frappe/desk/form/document_follow.py
@@ -161,9 +161,14 @@ def get_document_followed_by_user(user):
 
 def get_version(doctype, doc_name, frequency, user):
 	timeline = []
-	filters = get_filters("docname", doc_name, frequency, user)
 	version = frappe.get_all(
-		"Version", filters=filters, fields=["ref_doctype", "data", "modified", "modified", "modified_by"]
+		"Version",
+		filters=[
+			["ref_doctype", "=", doctype],
+			["docname", "=", doc_name],
+			*_get_filters(frequency, user),
+		],
+		fields=["data", "modified", "modified_by"],
 	)
 	if version:
 		for v in version:
@@ -183,18 +188,23 @@ def get_version(doctype, doc_name, frequency, user):
 
 
 def get_comments(doctype, doc_name, frequency, user):
-	from html2text import html2text
+	from frappe.core.utils import html2text
 
 	timeline = []
-	filters = get_filters("reference_name", doc_name, frequency, user)
 	comments = frappe.get_all(
-		"Comment", filters=filters, fields=["content", "modified", "modified_by", "comment_type"]
+		"Comment",
+		filters=[
+			["reference_doctype", "=", doctype],
+			["reference_name", "=", doc_name],
+			*_get_filters(frequency, user),
+		],
+		fields=["content", "modified", "modified_by", "comment_type"],
 	)
 	for comment in comments:
 		if comment.comment_type == "Like":
-			by = """ By : {0}""".format(comment.modified_by)
+			by = f""" By : {comment.modified_by}"""
 		elif comment.comment_type == "Comment":
-			by = """Commented by : {0}""".format(comment.modified_by)
+			by = f"""Commented by : {comment.modified_by}"""
 		else:
 			by = ""
 
@@ -225,7 +235,7 @@ def get_follow_users(doctype, doc_name):
 
 
 def get_row_changed(row_changed, time, doctype, doc_name, v):
-	from html2text import html2text
+	from frappe.core.utils import html2text
 
 	items = []
 	for d in row_changed:
@@ -269,7 +279,7 @@ def get_added_row(added, time, doctype, doc_name, v):
 
 
 def get_field_changed(changed, time, doctype, doc_name, v):
-	from html2text import html2text
+	from frappe.core.utils import html2text
 
 	items = []
 	for d in changed:
@@ -306,29 +316,27 @@ def send_weekly_updates():
 	send_document_follow_mails("Weekly")
 
 
-def get_filters(search_by, name, frequency, user):
-	filters = []
+def _get_filters(frequency, user):
+	filters = [
+		["modified_by", "!=", user],
+	]
 
 	if frequency == "Weekly":
-		filters = [
-			[search_by, "=", name],
+		filters += [
 			["modified", ">", frappe.utils.add_days(frappe.utils.nowdate(), -7)],
 			["modified", "<", frappe.utils.nowdate()],
-			["modified_by", "!=", user],
 		]
+
 	elif frequency == "Daily":
-		filters = [
-			[search_by, "=", name],
+		filters += [
 			["modified", ">", frappe.utils.add_days(frappe.utils.nowdate(), -1)],
 			["modified", "<", frappe.utils.nowdate()],
-			["modified_by", "!=", user],
 		]
+
 	elif frequency == "Hourly":
-		filters = [
-			[search_by, "=", name],
+		filters += [
 			["modified", ">", frappe.utils.add_to_date(frappe.utils.now_datetime(), hours=-1)],
 			["modified", "<", frappe.utils.now_datetime()],
-			["modified_by", "!=", user],
 		]
 
 	return filters
diff --git a/frappe/desk/form/linked_with.py b/frappe/desk/form/linked_with.py
index 7a53c8b65a..b60c11774f 100644
--- a/frappe/desk/form/linked_with.py
+++ b/frappe/desk/form/linked_with.py
@@ -4,7 +4,6 @@
 import itertools
 import json
 from collections import defaultdict
-from typing import Dict, List, Optional
 
 import frappe
 import frappe.desk.form.load
@@ -15,7 +14,7 @@ from frappe.modules import load_doctype_module
 
 
 @frappe.whitelist()
-def get_submitted_linked_docs(doctype: str, name: str) -> List[tuple]:
+def get_submitted_linked_docs(doctype: str, name: str) -> list[tuple]:
 	"""Get all the nested submitted documents those are present in referencing tables (dependent tables).
 
 	:param doctype: Document type
@@ -134,14 +133,14 @@ class SubmittableDocumentTree:
 		"""limit doctype links to these doctypes."""
 		return list(set(self.get_submittable_doctypes()) - set(get_exempted_doctypes() or []))
 
-	def get_submittable_doctypes(self) -> List[str]:
+	def get_submittable_doctypes(self) -> list[str]:
 		"""Returns list of submittable doctypes."""
 		if not self._submittable_doctypes:
 			self._submittable_doctypes = frappe.db.get_all("DocType", {"is_submittable": 1}, pluck="name")
 		return self._submittable_doctypes
 
 
-def get_child_tables_of_doctypes(doctypes: List[str] = None):
+def get_child_tables_of_doctypes(doctypes: list[str] = None):
 	"""Returns child tables by doctype."""
 	filters = [["fieldtype", "=", "Table"]]
 	filters_for_docfield = filters
@@ -174,8 +173,8 @@ def get_child_tables_of_doctypes(doctypes: List[str] = None):
 
 
 def get_references_across_doctypes(
-	to_doctypes: List[str] = None, limit_link_doctypes: List[str] = None
-) -> List:
+	to_doctypes: list[str] = None, limit_link_doctypes: list[str] = None
+) -> list:
 	"""Find doctype wise foreign key references.
 
 	:param to_doctypes: Get links of these doctypes.
@@ -213,7 +212,7 @@ def get_references_across_doctypes(
 
 
 def get_references_across_doctypes_by_link_field(
-	to_doctypes: List[str] = None, limit_link_doctypes: List[str] = None
+	to_doctypes: list[str] = None, limit_link_doctypes: list[str] = None
 ):
 	"""Find doctype wise foreign key references based on link fields.
 
@@ -253,7 +252,7 @@ def get_references_across_doctypes_by_link_field(
 
 
 def get_references_across_doctypes_by_dynamic_link_field(
-	to_doctypes: List[str] = None, limit_link_doctypes: List[str] = None
+	to_doctypes: list[str] = None, limit_link_doctypes: list[str] = None
 ):
 	"""Find doctype wise foreign key references based on dynamic link fields.
 
@@ -304,10 +303,10 @@ def get_references_across_doctypes_by_dynamic_link_field(
 
 def get_referencing_documents(
 	reference_doctype: str,
-	reference_names: List[str],
+	reference_names: list[str],
 	link_info: dict,
 	get_parent_if_child_table_doc: bool = True,
-	parent_filters: List[list] = None,
+	parent_filters: list[list] = None,
 	child_filters=None,
 	allowed_parents=None,
 ):
@@ -340,7 +339,7 @@ def get_referencing_documents(
 	for parent, rows in itertools.groupby(res, key=lambda row: row["parenttype"]):
 		if allowed_parents and parent not in allowed_parents:
 			continue
-		filters = (parent_filters or []) + [["name", "in", tuple([row.parent for row in rows])]]
+		filters = (parent_filters or []) + [["name", "in", tuple(row.parent for row in rows)]]
 		documents[parent].extend(frappe.db.get_all(parent, filters=filters, pluck="name") or [])
 	return documents
 
@@ -407,7 +406,7 @@ def get_exempted_doctypes():
 
 
 @frappe.whitelist()
-def get_linked_docs(doctype: str, name: str, linkinfo: Optional[Dict] = None) -> Dict[str, List]:
+def get_linked_docs(doctype: str, name: str, linkinfo: dict | None = None) -> dict[str, list]:
 	if isinstance(linkinfo, str):
 		# additional fields are added in linkinfo
 		linkinfo = json.loads(linkinfo)
@@ -447,9 +446,7 @@ def get_linked_docs(doctype: str, name: str, linkinfo: Optional[Dict] = None) ->
 			if link.get("add_fields"):
 				fields += link["add_fields"]
 
-			fields = [
-				"`tab{dt}`.`{fn}`".format(dt=dt, fn=sf.strip()) for sf in fields if sf and "`tab" not in sf
-			]
+			fields = [f"`tab{dt}`.`{sf.strip()}`" for sf in fields if sf and "`tab" not in sf]
 
 			try:
 				if link.get("filters"):
diff --git a/frappe/desk/form/load.py b/frappe/desk/form/load.py
index 75cd403aac..d68aab927a 100644
--- a/frappe/desk/form/load.py
+++ b/frappe/desk/form/load.py
@@ -2,7 +2,6 @@
 # License: MIT. See LICENSE
 
 import json
-from typing import Dict, List, Union
 from urllib.parse import quote
 
 import frappe
@@ -12,6 +11,7 @@ import frappe.share
 import frappe.utils
 from frappe import _, _dict
 from frappe.desk.form.document_follow import is_document_followed
+from frappe.model.utils import is_virtual_doctype
 from frappe.model.utils.user_settings import get_user_settings
 from frappe.permissions import get_doc_permissions
 from frappe.utils.data import cstr
@@ -31,28 +31,23 @@ def getdoc(doctype, name, user=None):
 	if not name:
 		name = doctype
 
-	if not frappe.db.exists(doctype, name):
+	if not is_virtual_doctype(doctype) and not frappe.db.exists(doctype, name):
 		return []
 
-	try:
-		doc = frappe.get_doc(doctype, name)
-		run_onload(doc)
+	doc = frappe.get_doc(doctype, name)
+	run_onload(doc)
 
-		if not doc.has_permission("read"):
-			frappe.flags.error_message = _("Insufficient Permission for {0}").format(
-				frappe.bold(doctype + " " + name)
-			)
-			raise frappe.PermissionError(("read", doctype, name))
+	if not doc.has_permission("read"):
+		frappe.flags.error_message = _("Insufficient Permission for {0}").format(
+			frappe.bold(doctype + " " + name)
+		)
+		raise frappe.PermissionError(("read", doctype, name))
 
-		doc.apply_fieldlevel_read_permissions()
+	doc.apply_fieldlevel_read_permissions()
 
-		# add file list
-		doc.add_viewed()
-		get_docinfo(doc)
-
-	except Exception:
-		frappe.errprint(frappe.utils.get_traceback())
-		raise
+	# add file list
+	doc.add_viewed()
+	get_docinfo(doc)
 
 	doc.add_seen()
 	set_link_titles(doc)
@@ -218,8 +213,8 @@ def get_communications(doctype, name, start=0, limit=20):
 
 
 def get_comments(
-	doctype: str, name: str, comment_type: Union[str, List[str]] = "Comment"
-) -> List[frappe._dict]:
+	doctype: str, name: str, comment_type: str | list[str] = "Comment"
+) -> list[frappe._dict]:
 	if isinstance(comment_type, list):
 		comment_types = comment_type
 
@@ -294,7 +289,7 @@ def get_communication_data(
 	if after:
 		# find after a particular date
 		conditions += """
-			AND C.creation > {0}
+			AND C.creation > {}
 		""".format(
 			after
 		)
@@ -411,7 +406,7 @@ def get_document_email(doctype, name):
 		return None
 
 	email = email.split("@")
-	return "{0}+{1}+{2}@{3}".format(email[0], quote(doctype), quote(cstr(name)), email[1])
+	return f"{email[0]}+{quote(doctype)}+{quote(cstr(name))}@{email[1]}"
 
 
 def get_automatic_email_link():
diff --git a/frappe/desk/form/meta.py b/frappe/desk/form/meta.py
index ba19377c48..5a426b4c63 100644
--- a/frappe/desk/form/meta.py
+++ b/frappe/desk/form/meta.py
@@ -52,15 +52,9 @@ def get_meta(doctype, cached=True):
 
 class FormMeta(Meta):
 	def __init__(self, doctype):
-		super(FormMeta, self).__init__(doctype)
+		super().__init__(doctype)
 		self.load_assets()
 
-	def set(self, key, value, *args, **kwargs):
-		if key in ASSET_KEYS:
-			self.__dict__[key] = value
-		else:
-			super(FormMeta, self).set(key, value, *args, **kwargs)
-
 	def load_assets(self):
 		if self.get("__assets_loaded", False):
 			return
@@ -80,7 +74,7 @@ class FormMeta(Meta):
 		self.set("__assets_loaded", True)
 
 	def as_dict(self, no_nulls=False):
-		d = super(FormMeta, self).as_dict(no_nulls=no_nulls)
+		d = super().as_dict(no_nulls=no_nulls)
 
 		for k in ASSET_KEYS:
 			d[k] = self.get(k)
@@ -139,7 +133,7 @@ class FormMeta(Meta):
 		templates = dict()
 		for fname in os.listdir(path):
 			if fname.endswith(".html"):
-				with io.open(os.path.join(path, fname), "r", encoding="utf-8") as f:
+				with open(os.path.join(path, fname), encoding="utf-8") as f:
 					templates[fname.split(".")[0]] = scrub_html_template(f.read())
 
 		self.set("__templates", templates or None)
@@ -155,7 +149,7 @@ class FormMeta(Meta):
 			frappe.db.get_all(
 				"Client Script",
 				filters={"dt": self.name, "enabled": 1},
-				fields=["script", "view"],
+				fields=["name", "script", "view"],
 				order_by="creation asc",
 			)
 			or ""
@@ -165,10 +159,18 @@ class FormMeta(Meta):
 		form_script = ""
 		for script in client_scripts:
 			if script.view == "List":
-				list_script += script.script
+				list_script += f"""
+// {script.name}
+{script.script}
+
+"""
 
 			if script.view == "Form":
-				form_script += script.script
+				form_script += f"""
+// {script.name}
+{script.script}
+
+"""
 
 		file = scrub(self.name)
 		form_script += f"\n\n//# sourceURL={file}__custom_js"
diff --git a/frappe/desk/form/save.py b/frappe/desk/form/save.py
index cc3865bc60..f3e7b6294f 100644
--- a/frappe/desk/form/save.py
+++ b/frappe/desk/form/save.py
@@ -10,42 +10,33 @@ from frappe.desk.form.load import run_onload
 @frappe.whitelist()
 def savedocs(doc, action):
 	"""save / submit / update doclist"""
-	try:
-		doc = frappe.get_doc(json.loads(doc))
-		set_local_name(doc)
+	doc = frappe.get_doc(json.loads(doc))
+	set_local_name(doc)
 
-		# action
-		doc.docstatus = {"Save": 0, "Submit": 1, "Update": 1, "Cancel": 2}[action]
+	# action
+	doc.docstatus = {"Save": 0, "Submit": 1, "Update": 1, "Cancel": 2}[action]
 
-		if doc.docstatus == 1:
-			doc.submit()
-		else:
-			doc.save()
+	if doc.docstatus == 1:
+		doc.submit()
+	else:
+		doc.save()
 
-		# update recent documents
-		run_onload(doc)
-		send_updated_docs(doc)
+	# update recent documents
+	run_onload(doc)
+	send_updated_docs(doc)
 
-		frappe.msgprint(frappe._("Saved"), indicator="green", alert=True)
-	except Exception:
-		frappe.errprint(frappe.utils.get_traceback())
-		raise
+	frappe.msgprint(frappe._("Saved"), indicator="green", alert=True)
 
 
 @frappe.whitelist()
 def cancel(doctype=None, name=None, workflow_state_fieldname=None, workflow_state=None):
 	"""cancel a doclist"""
-	try:
-		doc = frappe.get_doc(doctype, name)
-		if workflow_state_fieldname and workflow_state:
-			doc.set(workflow_state_fieldname, workflow_state)
-		doc.cancel()
-		send_updated_docs(doc)
-		frappe.msgprint(frappe._("Cancelled"), indicator="red", alert=True)
-
-	except Exception:
-		frappe.errprint(frappe.utils.get_traceback())
-		raise
+	doc = frappe.get_doc(doctype, name)
+	if workflow_state_fieldname and workflow_state:
+		doc.set(workflow_state_fieldname, workflow_state)
+	doc.cancel()
+	send_updated_docs(doc)
+	frappe.msgprint(frappe._("Cancelled"), indicator="red", alert=True)
 
 
 def send_updated_docs(doc):
diff --git a/frappe/desk/form/utils.py b/frappe/desk/form/utils.py
index 2738d1f74a..9e10ced912 100644
--- a/frappe/desk/form/utils.py
+++ b/frappe/desk/form/utils.py
@@ -2,42 +2,51 @@
 # License: MIT. See LICENSE
 
 import json
+from typing import TYPE_CHECKING
 
 import frappe
 import frappe.desk.form.load
 import frappe.desk.form.meta
 from frappe import _
-from frappe.core.doctype.file.file import extract_images_from_html
+from frappe.core.doctype.file.utils import extract_images_from_html
 from frappe.desk.form.document_follow import follow_document
 
+if TYPE_CHECKING:
+	from frappe.core.doctype.comment.comment import Comment
 
-@frappe.whitelist()
+
+@frappe.whitelist(methods=["DELETE", "POST"])
 def remove_attach():
 	"""remove attachment"""
 	fid = frappe.form_dict.get("fid")
-	file_name = frappe.form_dict.get("file_name")
 	frappe.delete_doc("File", fid)
 
 
-@frappe.whitelist()
-def add_comment(reference_doctype, reference_name, content, comment_email, comment_by):
-	"""allow any logged user to post a comment"""
-	doc = frappe.get_doc(
-		dict(
-			doctype="Comment",
-			reference_doctype=reference_doctype,
-			reference_name=reference_name,
-			comment_email=comment_email,
-			comment_type="Comment",
-			comment_by=comment_by,
-		)
-	)
+@frappe.whitelist(methods=["POST", "PUT"])
+def add_comment(
+	reference_doctype: str, reference_name: str, content: str, comment_email: str, comment_by: str
+) -> "Comment":
+	"""Allow logged user with permission to read document to add a comment"""
 	reference_doc = frappe.get_doc(reference_doctype, reference_name)
-	doc.content = extract_images_from_html(reference_doc, content, is_private=True)
-	doc.insert(ignore_permissions=True)
+	reference_doc.check_permission()
+
+	comment = frappe.new_doc("Comment")
+	comment.update(
+		{
+			"comment_type": "Comment",
+			"reference_doctype": reference_doctype,
+			"reference_name": reference_name,
+			"comment_email": comment_email,
+			"comment_by": comment_by,
+			"content": extract_images_from_html(reference_doc, content, is_private=True),
+		}
+	)
+	comment.insert(ignore_permissions=True)
+
 	if frappe.get_cached_value("User", frappe.session.user, "follow_commented_documents"):
-		follow_document(doc.reference_doctype, doc.reference_name, frappe.session.user)
-	return doc.as_dict()
+		follow_document(comment.reference_doctype, comment.reference_name, frappe.session.user)
+
+	return comment
 
 
 @frappe.whitelist()
@@ -76,7 +85,7 @@ def get_next(doctype, value, prev, filters=None, sort_order="desc", sort_field="
 		doctype,
 		fields=["name"],
 		filters=filters,
-		order_by="`tab{0}`.{1}".format(doctype, sort_field) + " " + sort_order,
+		order_by=f"`tab{doctype}`.{sort_field}" + " " + sort_order,
 		limit_start=0,
 		limit_page_length=1,
 		as_list=True,
diff --git a/frappe/desk/like.py b/frappe/desk/like.py
index 9e97cb269c..0f297455e7 100644
--- a/frappe/desk/like.py
+++ b/frappe/desk/like.py
@@ -90,7 +90,7 @@ def add_comment(doctype, name):
 		link = get_link_to_form(
 			doc.reference_doctype,
 			doc.reference_name,
-			"{0} {1}".format(_(doc.reference_doctype), doc.reference_name),
+			f"{_(doc.reference_doctype)} {doc.reference_name}",
 		)
 
 		doc.add_comment(
diff --git a/frappe/desk/link_preview.py b/frappe/desk/link_preview.py
index 374a151505..7778d9e373 100644
--- a/frappe/desk/link_preview.py
+++ b/frappe/desk/link_preview.py
@@ -1,5 +1,3 @@
-import json
-
 import frappe
 from frappe.model import no_value_fields, table_fields
 
diff --git a/frappe/desk/listview.py b/frappe/desk/listview.py
index 88216d3998..ea6eb6259c 100644
--- a/frappe/desk/listview.py
+++ b/frappe/desk/listview.py
@@ -1,6 +1,11 @@
-# Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and Contributors
+# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
+
 import frappe
+from frappe.query_builder import Order
+from frappe.query_builder.functions import Count
+from frappe.query_builder.terms import SubQuery
+from frappe.query_builder.utils import DocType
 
 
 @frappe.whitelist()
@@ -24,37 +29,36 @@ def set_list_settings(doctype, values):
 
 
 @frappe.whitelist()
-def get_group_by_count(doctype, current_filters, field):
+def get_group_by_count(doctype: str, current_filters: str, field: str) -> list[dict]:
 	current_filters = frappe.parse_json(current_filters)
-	subquery_condition = ""
 
-	subquery = frappe.get_all(doctype, filters=current_filters, run=False)
 	if field == "assigned_to":
-		subquery_condition = " and `tabToDo`.reference_name in ({subquery})".format(subquery=subquery)
-		return frappe.db.sql(
-			"""select `tabToDo`.allocated_to as name, count(*) as count
-			from
-				`tabToDo`, `tabUser`
-			where
-				`tabToDo`.status!='Cancelled' and
-				`tabToDo`.allocated_to = `tabUser`.name and
-				`tabUser`.user_type = 'System User'
-				{subquery_condition}
-			group by
-				`tabToDo`.allocated_to
-			order by
-				count desc
-			limit 50""".format(
-				subquery_condition=subquery_condition
-			),
-			as_dict=True,
-		)
-	else:
-		return frappe.db.get_list(
-			doctype,
-			filters=current_filters,
-			group_by="`tab{0}`.{1}".format(doctype, field),
-			fields=["count(*) as count", "`{}` as name".format(field)],
-			order_by="count desc",
-			limit=50,
+		ToDo = DocType("ToDo")
+		User = DocType("User")
+		count = Count("*").as_("count")
+		filtered_records = frappe.qb.engine.build_conditions(doctype, current_filters).select("name")
+
+		return (
+			frappe.qb.from_(ToDo)
+			.from_(User)
+			.select(ToDo.allocated_to.as_("name"), count)
+			.where(
+				(ToDo.status != "Cancelled")
+				& (ToDo.allocated_to == User.name)
+				& (User.user_type == "System User")
+				& (ToDo.reference_name.isin(SubQuery(filtered_records)))
+			)
+			.groupby(ToDo.allocated_to)
+			.orderby(count, order=Order.desc)
+			.limit(50)
+			.run(as_dict=True)
 		)
+
+	return frappe.get_list(
+		doctype,
+		filters=current_filters,
+		group_by=f"`tab{doctype}`.{field}",
+		fields=["count(*) as count", f"`{field}` as name"],
+		order_by="count desc",
+		limit=50,
+	)
diff --git a/frappe/desk/moduleview.py b/frappe/desk/moduleview.py
index a4fc2ccd1e..913b3406e3 100644
--- a/frappe/desk/moduleview.py
+++ b/frappe/desk/moduleview.py
@@ -253,13 +253,13 @@ def apply_permissions(data):
 
 def get_disabled_reports():
 	if not hasattr(frappe.local, "disabled_reports"):
-		frappe.local.disabled_reports = set(r.name for r in frappe.get_all("Report", {"disabled": 1}))
+		frappe.local.disabled_reports = {r.name for r in frappe.get_all("Report", {"disabled": 1})}
 	return frappe.local.disabled_reports
 
 
 def get_config(app, module):
 	"""Load module info from `[app].config.[module]`."""
-	config = frappe.get_module("{app}.config.{module}".format(app=app, module=module))
+	config = frappe.get_module(f"{app}.config.{module}")
 	config = config.get_data()
 
 	sections = [s for s in config if s.get("condition", True)]
@@ -283,7 +283,7 @@ def get_config(app, module):
 
 def config_exists(app, module):
 	try:
-		frappe.get_module("{app}.config.{module}".format(app=app, module=module))
+		frappe.get_module(f"{app}.config.{module}")
 		return True
 	except ImportError:
 		return False
diff --git a/frappe/desk/notifications.py b/frappe/desk/notifications.py
index e92a7492ce..77d40f44d2 100644
--- a/frappe/desk/notifications.py
+++ b/frappe/desk/notifications.py
@@ -3,10 +3,19 @@
 
 import json
 
+from bs4 import BeautifulSoup
+
 import frappe
+from frappe import _
+from frappe.desk.doctype.notification_log.notification_log import (
+	enqueue_create_notification,
+	get_title,
+	get_title_html,
+)
 from frappe.desk.doctype.notification_settings.notification_settings import (
 	get_subscribed_documents,
 )
+from frappe.utils import get_fullname
 
 
 @frappe.whitelist()
@@ -249,9 +258,9 @@ def get_open_count(doctype, name, items=None):
 	if frappe.flags.in_migrate or frappe.flags.in_install:
 		return {"count": []}
 
-	frappe.has_permission(doc=frappe.get_doc(doctype, name), throw=True)
-
-	meta = frappe.get_meta(doctype)
+	doc = frappe.get_doc(doctype, name)
+	doc.check_permission()
+	meta = doc.meta
 	links = meta.get_dashboard_data()
 
 	# compile all items in a list
@@ -266,7 +275,6 @@ def get_open_count(doctype, name, items=None):
 	out = []
 	for d in items:
 		if d in links.get("internal_links", {}):
-			# internal link
 			continue
 
 		filters = get_filters_for(d)
@@ -299,3 +307,56 @@ def get_open_count(doctype, name, items=None):
 			out["timeline_data"] = module.get_timeline_data(doctype, name)
 
 	return out
+
+
+def notify_mentions(ref_doctype, ref_name, content):
+	if ref_doctype and ref_name and content:
+		mentions = extract_mentions(content)
+
+		if not mentions:
+			return
+
+		sender_fullname = get_fullname(frappe.session.user)
+		title = get_title(ref_doctype, ref_name)
+
+		recipients = [
+			frappe.db.get_value(
+				"User",
+				{"enabled": 1, "name": name, "user_type": "System User", "allowed_in_mentions": 1},
+				"email",
+			)
+			for name in mentions
+		]
+
+		notification_message = _("""{0} mentioned you in a comment in {1} {2}""").format(
+			frappe.bold(sender_fullname), frappe.bold(ref_doctype), get_title_html(title)
+		)
+
+		notification_doc = {
+			"type": "Mention",
+			"document_type": ref_doctype,
+			"document_name": ref_name,
+			"subject": notification_message,
+			"from_user": frappe.session.user,
+			"email_content": content,
+		}
+
+		enqueue_create_notification(recipients, notification_doc)
+
+
+def extract_mentions(txt):
+	"""Find all instances of @mentions in the html."""
+	soup = BeautifulSoup(txt, "html.parser")
+	emails = []
+	for mention in soup.find_all(class_="mention"):
+		if mention.get("data-is-group") == "true":
+			try:
+				user_group = frappe.get_cached_doc("User Group", mention["data-id"])
+				emails += [d.user for d in user_group.user_group_members]
+			except frappe.DoesNotExistError:
+				pass
+			continue
+		email = mention["data-id"]
+		emails.append(email)
+
+	return emails
diff --git a/frappe/desk/page/activity/__init__.py b/frappe/desk/page/activity/__init__.py
index 8b13789179..e69de29bb2 100644
--- a/frappe/desk/page/activity/__init__.py
+++ b/frappe/desk/page/activity/__init__.py
@@ -1 +0,0 @@
-
diff --git a/frappe/desk/page/activity/activity.js b/frappe/desk/page/activity/activity.js
index 7b4e8ddc1a..0291edb225 100644
--- a/frappe/desk/page/activity/activity.js
+++ b/frappe/desk/page/activity/activity.js
@@ -3,12 +3,12 @@
 
 frappe.provide("frappe.activity");
 
-frappe.pages['activity'].on_page_load = function (wrapper) {
+frappe.pages["activity"].on_page_load = function (wrapper) {
 	var me = this;
 
 	frappe.ui.make_app_page({
 		parent: wrapper,
-		single_column: true
+		single_column: true,
 	});
 
 	me.page = wrapper.page;
@@ -16,8 +16,8 @@ frappe.pages['activity'].on_page_load = function (wrapper) {
 
 	frappe.model.with_doctype("Communication", function () {
 		me.page.list = new frappe.views.Activity({
-			doctype: 'Communication',
-			parent: wrapper
+			doctype: "Communication",
+			parent: wrapper,
 		});
 	});
 
@@ -29,17 +29,21 @@ frappe.pages['activity'].on_page_load = function (wrapper) {
 			doctype = $(this).attr("data-doctype"),
 			docname = $(this).attr("data-docname");
 
-		[link_doctype, link_name, doctype, docname] =
-			[link_doctype, link_name, doctype, docname].map(decodeURIComponent);
+		[link_doctype, link_name, doctype, docname] = [
+			link_doctype,
+			link_name,
+			doctype,
+			docname,
+		].map(decodeURIComponent);
 
-		link_doctype = link_doctype && link_doctype !== 'null' ? link_doctype : null;
-		link_name = link_name && link_name !== 'null' ? link_name : null;
+		link_doctype = link_doctype && link_doctype !== "null" ? link_doctype : null;
+		link_name = link_name && link_name !== "null" ? link_name : null;
 
 		if (doctype && docname) {
 			if (link_doctype && link_name) {
 				frappe.route_options = {
-					scroll_to: { "doctype": doctype, "name": docname }
-				}
+					scroll_to: { doctype: doctype, name: docname },
+				};
 			}
 
 			frappe.set_route(["Form", link_doctype || doctype, link_name || docname]);
@@ -48,37 +52,46 @@ frappe.pages['activity'].on_page_load = function (wrapper) {
 
 	// Build Report Button
 	if (frappe.boot.user.can_get_report.indexOf("Feed") != -1) {
-		this.page.add_menu_item(__('Build Report'), function () {
-			frappe.set_route("List", "Feed", "Report");
-		}, 'fa fa-th')
+		this.page.add_menu_item(
+			__("Build Report"),
+			function () {
+				frappe.set_route("List", "Feed", "Report");
+			},
+			"fa fa-th"
+		);
 	}
 
-	this.page.add_menu_item(__('Activity Log'), function () {
-		frappe.route_options = {
-			"user": frappe.session.user
-		}
+	this.page.add_menu_item(
+		__("Activity Log"),
+		function () {
+			frappe.route_options = {
+				user: frappe.session.user,
+			};
 
-		frappe.set_route("List", "Activity Log", "Report");
-	}, 'fa fa-th');
+			frappe.set_route("List", "Activity Log", "Report");
+		},
+		"fa fa-th"
+	);
 };
 
-frappe.pages['activity'].on_page_show = function () {
+frappe.pages["activity"].on_page_show = function () {
 	frappe.breadcrumbs.add("Desk");
-}
+};
 
 frappe.activity.last_feed_date = false;
 frappe.activity.Feed = class Feed {
 	constructor(row, data) {
 		this.scrub_data(data);
 		this.add_date_separator(row, data);
-		if (!data.add_class)
-			data.add_class = "label-default";
+		if (!data.add_class) data.add_class = "label-default";
 
 		data.link = "";
 		if (data.link_doctype && data.link_name) {
-			data.link = frappe.format(data.link_name, { fieldtype: "Link", options: data.link_doctype },
-				{ label: __(data.link_doctype) + " " + __(data.link_name) });
-
+			data.link = frappe.format(
+				data.link_name,
+				{ fieldtype: "Link", options: data.link_doctype },
+				{ label: __(data.link_doctype) + " " + __(data.link_name) }
+			);
 		} else if (data.feed_type === "Comment" && data.comment_type === "Comment") {
 			// hack for backward compatiblity
 			data.link_doctype = data.reference_doctype;
@@ -86,17 +99,20 @@ frappe.activity.Feed = class Feed {
 			data.reference_doctype = "Communication";
 			data.reference_name = data.name;
 
-			data.link = frappe.format(data.link_name, { fieldtype: "Link", options: data.link_doctype },
-				{ label: __(data.link_doctype) + " " + __(data.link_name) });
-
+			data.link = frappe.format(
+				data.link_name,
+				{ fieldtype: "Link", options: data.link_doctype },
+				{ label: __(data.link_doctype) + " " + __(data.link_name) }
+			);
 		} else if (data.reference_doctype && data.reference_name) {
-			data.link = frappe.format(data.reference_name, { fieldtype: "Link", options: data.reference_doctype },
-				{ label: __(data.reference_doctype) + " " + __(data.reference_name) });
+			data.link = frappe.format(
+				data.reference_name,
+				{ fieldtype: "Link", options: data.reference_doctype },
+				{ label: __(data.reference_doctype) + " " + __(data.reference_name) }
+			);
 		}
 
-		$(row)
-			.append(frappe.render_template("activity_row", data))
-			.find("a").addClass("grey");
+		$(row).append(frappe.render_template("activity_row", data)).find("a").addClass("grey");
 	}
 
 	scrub_data(data) {
@@ -106,11 +122,12 @@ frappe.activity.Feed = class Feed {
 		data.icon = "fa fa-flag";
 
 		// color for comment
-		data.add_class = {
-			"Comment": "label-danger",
-			"Assignment": "label-warning",
-			"Login": "label-default"
-		}[data.comment_type || data.communication_medium] || "label-info"
+		data.add_class =
+			{
+				Comment: "label-danger",
+				Assignment: "label-warning",
+				Login: "label-default",
+			}[data.comment_type || data.communication_medium] || "label-info";
 
 		data.when = comment_when(data.creation);
 		data.feed_type = data.comment_type || data.communication_medium;
@@ -120,18 +137,24 @@ frappe.activity.Feed = class Feed {
 		var date = frappe.datetime.str_to_obj(data.creation);
 		var last = frappe.activity.last_feed_date;
 
-		if ((last && frappe.datetime.obj_to_str(last) != frappe.datetime.obj_to_str(date)) || (!last)) {
-			var diff = frappe.datetime.get_day_diff(frappe.datetime.get_today(), frappe.datetime.obj_to_str(date));
+		if (
+			(last && frappe.datetime.obj_to_str(last) != frappe.datetime.obj_to_str(date)) ||
+			!last
+		) {
+			var diff = frappe.datetime.get_day_diff(
+				frappe.datetime.get_today(),
+				frappe.datetime.obj_to_str(date)
+			);
 			var pdate;
 			if (diff < 1) {
-				pdate = 'Today';
+				pdate = "Today";
 			} else if (diff < 2) {
-				pdate = 'Yesterday';
+				pdate = "Yesterday";
 			} else {
 				pdate = frappe.datetime.global_date_format(date);
 			}
 			data.date_sep = pdate;
-			data.date_class = pdate == 'Today' ? "date-indicator blue" : "date-indicator";
+			data.date_class = pdate == "Today" ? "date-indicator blue" : "date-indicator";
 		} else {
 			data.date_sep = null;
 			data.date_class = "";
@@ -141,26 +164,28 @@ frappe.activity.Feed = class Feed {
 };
 
 frappe.activity.render_heatmap = function (page) {
-	$('
    \
+	$(
+		'
    \
 		
    \
-		
    ').prependTo(page.main);
+		
    '
+	).prependTo(page.main);
 
 	frappe.call({
 		method: "frappe.desk.page.activity.activity.get_heatmap_data",
 		callback: function (r) {
 			if (r.message) {
 				new frappe.Chart(".heatmap", {
-					type: 'heatmap',
-					start: new Date(moment().subtract(1, 'year').toDate()),
+					type: "heatmap",
+					start: new Date(moment().subtract(1, "year").toDate()),
 					countLabel: "actions",
 					discreteDomains: 1,
 					radius: 3, // default 0
 					data: {
-						'dataPoints': r.message
-					}
+						dataPoints: r.message,
+					},
 				});
 			}
-		}
+		},
 	});
 };
 
@@ -173,10 +198,9 @@ frappe.views.Activity = class Activity extends frappe.views.BaseList {
 	setup_defaults() {
 		super.setup_defaults();
 
-		this.page_title = __('Activity');
-		this.doctype = 'Communication';
-		this.method = 'frappe.desk.page.activity.activity.get_feed';
-
+		this.page_title = __("Activity");
+		this.doctype = "Communication";
+		this.method = "frappe.desk.page.activity.activity.get_feed";
 	}
 
 	setup_filter_area() {
@@ -187,18 +211,14 @@ frappe.views.Activity = class Activity extends frappe.views.BaseList {
 		//
 	}
 
-	setup_sort_selector() {
+	setup_sort_selector() {}
 
-	}
-
-	setup_side_bar() {
-
-	}
+	setup_side_bar() {}
 
 	get_args() {
 		return {
 			start: this.start,
-			page_length: this.page_length
+			page_length: this.page_length,
 		};
 	}
 
@@ -213,8 +233,11 @@ frappe.views.Activity = class Activity extends frappe.views.BaseList {
 	}
 
 	render() {
-		this.data.map(value => {
-			const row = $('
    ').data("data", value).appendTo(this.$result).get(0);
+		this.data.map((value) => {
+			const row = $('
    ')
+				.data("data", value)
+				.appendTo(this.$result)
+				.get(0);
 			new frappe.activity.Feed(row, value);
 		});
 	}
diff --git a/frappe/desk/page/backups/backups.js b/frappe/desk/page/backups/backups.js
index d6cab750f0..08289cab2d 100644
--- a/frappe/desk/page/backups/backups.js
+++ b/frappe/desk/page/backups/backups.js
@@ -1,18 +1,18 @@
-frappe.pages['backups'].on_page_load = function (wrapper) {
+frappe.pages["backups"].on_page_load = function (wrapper) {
 	var page = frappe.ui.make_app_page({
 		parent: wrapper,
-		title: __('Download Backups'),
-		single_column: true
+		title: __("Download Backups"),
+		single_column: true,
 	});
 
 	page.add_inner_button(__("Set Number of Backups"), function () {
-		frappe.set_route('Form', 'System Settings');
+		frappe.set_route("Form", "System Settings");
 	});
 
 	page.add_inner_button(__("Download Files Backup"), function () {
 		frappe.call({
 			method: "frappe.desk.page.backups.backups.schedule_files_backup",
-			args: { "user_email": frappe.session.user_email }
+			args: { user_email: frappe.session.user_email },
 		});
 	});
 
@@ -23,18 +23,18 @@ frappe.pages['backups'].on_page_load = function (wrapper) {
 					method: "frappe.utils.backups.get_backup_encryption_key",
 					callback: function (r) {
 						frappe.msgprint({
-							title: __('Backup Encryption Key'),
+							title: __("Backup Encryption Key"),
 							message: __(r.message),
-							indicator: 'blue'
+							indicator: "blue",
 						});
-					}
+					},
 				});
 			});
 		} else {
 			frappe.msgprint({
-				title: __('Error'),
-				message: __('System Manager privileges required.'),
-				indicator: 'red'
+				title: __("Error"),
+				message: __("System Manager privileges required."),
+				indicator: "red",
 			});
 		}
 	});
diff --git a/frappe/desk/page/backups/backups.py b/frappe/desk/page/backups/backups.py
index ec5bea3c4b..2ef09df900 100644
--- a/frappe/desk/page/backups/backups.py
+++ b/frappe/desk/page/backups/backups.py
@@ -21,9 +21,9 @@ def get_context(context):
 	def get_size(path):
 		size = os.path.getsize(path)
 		if size > 1048576:
-			return "{0:.1f}M".format(float(size) / 1048576)
+			return f"{float(size) / 1048576:.1f}M"
 		else:
-			return "{0:.1f}K".format(float(size) / 1024)
+			return f"{float(size) / 1024:.1f}K"
 
 	path = get_site_path("private", "backups")
 	files = [x for x in os.listdir(path) if os.path.isfile(os.path.join(path, x))]
diff --git a/frappe/desk/page/leaderboard/leaderboard.js b/frappe/desk/page/leaderboard/leaderboard.js
index aa1678af37..845dac0d63 100644
--- a/frappe/desk/page/leaderboard/leaderboard.js
+++ b/frappe/desk/page/leaderboard/leaderboard.js
@@ -1,7 +1,7 @@
 frappe.pages["leaderboard"].on_page_load = (wrapper) => {
 	frappe.leaderboard = new Leaderboard(wrapper);
 
-	$(wrapper).bind('show', ()=> {
+	$(wrapper).bind("show", () => {
 		// Get which leaderboard to show
 		let doctype = frappe.get_route()[1];
 		frappe.leaderboard.show_leaderboard(doctype);
@@ -9,7 +9,6 @@ frappe.pages["leaderboard"].on_page_load = (wrapper) => {
 };
 
 class Leaderboard {
-
 	constructor(parent) {
 		frappe.ui.make_app_page({
 			parent: parent,
@@ -20,11 +19,12 @@ class Leaderboard {
 
 		this.parent = parent;
 		this.page = this.parent.page;
-		this.page.sidebar.html(`
      `);
-		this.$sidebar_list = this.page.sidebar.find('ul');
+		this.page.sidebar.html(
+			`
        `
+		);
+		this.$sidebar_list = this.page.sidebar.find("ul");
 
 		this.get_leaderboard_config();
-
 	}
 
 	get_leaderboard_config() {
@@ -32,47 +32,57 @@ class Leaderboard {
 		this.filters = {};
 		this.leaderboard_limit = 20;
 
-		frappe.xcall("frappe.desk.page.leaderboard.leaderboard.get_leaderboard_config").then(config => {
-			this.leaderboard_config = config;
-			for (let doctype in this.leaderboard_config) {
-				this.doctypes.push(doctype);
-				this.filters[doctype] = this.leaderboard_config[doctype].fields.map(field => {
-					if (typeof field ==='object') {
-						return field.label || field.fieldname;
-					}
-					return field;
-				});
-			}
+		frappe
+			.xcall("frappe.desk.page.leaderboard.leaderboard.get_leaderboard_config")
+			.then((config) => {
+				this.leaderboard_config = config;
+				for (let doctype in this.leaderboard_config) {
+					this.doctypes.push(doctype);
+					this.filters[doctype] = this.leaderboard_config[doctype].fields.map(
+						(field) => {
+							if (typeof field === "object") {
+								return field.label || field.fieldname;
+							}
+							return field;
+						}
+					);
+				}
 
-			// For translation. Do not remove this
-			// __("This Week"), __("This Month"), __("This Quarter"), __("This Year"),
-			//	__("Last Week"), __("Last Month"), __("Last Quarter"), __("Last Year"),
-			//	__("All Time"), __("Select From Date")
-			this.timespans = [
-				"This Week", "This Month", "This Quarter", "This Year",
-				"Last Week", "Last Month", "Last Quarter", "Last Year",
-				"All Time", "Select Date Range"
-			];
+				// For translation. Do not remove this
+				// __("This Week"), __("This Month"), __("This Quarter"), __("This Year"),
+				//	__("Last Week"), __("Last Month"), __("Last Quarter"), __("Last Year"),
+				//	__("All Time"), __("Select From Date")
+				this.timespans = [
+					"This Week",
+					"This Month",
+					"This Quarter",
+					"This Year",
+					"Last Week",
+					"Last Month",
+					"Last Quarter",
+					"Last Year",
+					"All Time",
+					"Select Date Range",
+				];
 
-			// for saving current selected filters
-			const _initial_doctype = frappe.get_route()[1] || this.doctypes[0];
-			const _initial_timespan = this.timespans[0];
-			const _initial_filter = this.filters[_initial_doctype];
+				// for saving current selected filters
+				const _initial_doctype = frappe.get_route()[1] || this.doctypes[0];
+				const _initial_timespan = this.timespans[0];
+				const _initial_filter = this.filters[_initial_doctype];
 
-			this.options = {
-				selected_doctype: _initial_doctype,
-				selected_filter: _initial_filter,
-				selected_filter_item: _initial_filter[0],
-				selected_timespan: _initial_timespan,
-			};
+				this.options = {
+					selected_doctype: _initial_doctype,
+					selected_filter: _initial_filter,
+					selected_filter_item: _initial_filter[0],
+					selected_timespan: _initial_timespan,
+				};
 
-			this.message = null;
-			this.make();
-		});
+				this.message = null;
+				this.make();
+			});
 	}
 
 	make() {
-
 		this.$container = $(`
        
 			
        
 			
        
@@ -80,7 +90,7 @@ class Leaderboard {
 
 		this.$graph_area = this.$container.find(".leaderboard-graph");
 
-		this.doctypes.map(doctype => {
+		this.doctypes.map((doctype) => {
 			const icon = this.leaderboard_config[doctype].icon;
 			this.get_sidebar_item(doctype, icon).appendTo(this.$sidebar_list);
 		});
@@ -94,7 +104,6 @@ class Leaderboard {
 		// Get which leaderboard to show
 		let doctype = frappe.get_route()[1];
 		this.show_leaderboard(doctype);
-
 	}
 
 	setup_leaderboard_fields() {
@@ -108,25 +117,27 @@ class Leaderboard {
 			change: (e) => {
 				this.options.selected_company = e.currentTarget.value;
 				this.make_request();
-			}
+			},
 		});
 
-		this.timespan_select = this.page.add_select(__("Timespan"),
-			this.timespans.map(d => {
-				return {"label": __(d), value: d };
+		this.timespan_select = this.page.add_select(
+			__("Timespan"),
+			this.timespans.map((d) => {
+				return { label: __(d), value: d };
 			})
 		);
 		this.create_date_range_field();
 
-		this.type_select = this.page.add_select(__("Field"),
-			this.options.selected_filter.map(d => {
-				return {"label": __(frappe.model.unscrub(d)), value: d };
+		this.type_select = this.page.add_select(
+			__("Field"),
+			this.options.selected_filter.map((d) => {
+				return { label: __(frappe.model.unscrub(d)), value: d };
 			})
 		);
 
 		this.timespan_select.on("change", (e) => {
 			this.options.selected_timespan = e.currentTarget.value;
-			if (this.options.selected_timespan === 'Select Date Range') {
+			if (this.options.selected_timespan === "Select Date Range") {
 				this.date_range_field.show();
 			} else {
 				this.date_range_field.hide();
@@ -141,30 +152,33 @@ class Leaderboard {
 	}
 
 	create_date_range_field() {
-		let timespan_field = $(this.parent).find(`.frappe-control[data-original-title="${__('Timespan')}"]`);
-		this.date_range_field = $(`
        `).insertAfter(timespan_field).hide();
+		let timespan_field = $(this.parent).find(
+			`.frappe-control[data-original-title="${__("Timespan")}"]`
+		);
+		this.date_range_field = $(`
        `)
+			.insertAfter(timespan_field)
+			.hide();
 
 		let date_field = frappe.ui.form.make_control({
 			df: {
-				fieldtype: 'DateRange',
-				fieldname: 'selected_date_range',
+				fieldtype: "DateRange",
+				fieldname: "selected_date_range",
 				placeholder: __("Date Range"),
 				default: [frappe.datetime.month_start(), frappe.datetime.now_date()],
-				input_class: 'input-xs',
+				input_class: "input-xs",
 				reqd: 1,
 				change: () => {
 					this.selected_date_range = date_field.get_value();
 					if (this.selected_date_range) this.make_request();
-				}
+				},
 			},
-			parent: $(this.parent).find('.from-date-field'),
-			render_input: 1
+			parent: $(this.parent).find(".from-date-field"),
+			render_input: 1,
 		});
 	}
 
 	render_selected_doctype() {
-
-		this.$sidebar_list.on("click", "li", (e)=> {
+		this.$sidebar_list.on("click", "li", (e) => {
 			let $li = $(e.currentTarget);
 			let doctype = $li.find(".doctype-text").attr("doctype-value");
 
@@ -174,8 +188,8 @@ class Leaderboard {
 			this.options.selected_filter_item = this.filters[doctype][0];
 
 			this.type_select.empty().add_options(
-				this.options.selected_filter.map(d => {
-					return {"label": __(frappe.model.unscrub(d)), value: d };
+				this.options.selected_filter.map((d) => {
+					return { label: __(frappe.model.unscrub(d)), value: d };
 				})
 			);
 			if (this.leaderboard_config[this.options.selected_doctype].company_disabled) {
@@ -193,10 +207,10 @@ class Leaderboard {
 	}
 
 	render_search_box() {
-
-		this.$search_box =
-			$(`
        
-				
+		this.$search_box = $(`
        
+				
 			
        `);
 
 		$(this.parent).find(".page-form").append(this.$search_box);
@@ -206,7 +220,9 @@ class Leaderboard {
 		if (this.doctypes.length) {
 			if (this.doctypes.includes(doctype)) {
 				this.options.selected_doctype = doctype;
-				this.$sidebar_list.find(`[doctype-value = "${this.options.selected_doctype}"]`).trigger("click");
+				this.$sidebar_list
+					.find(`[doctype-value = "${this.options.selected_doctype}"]`)
+					.trigger("click");
 			}
 
 			this.$search_box.find(".leaderboard-search-input").val("");
@@ -215,8 +231,7 @@ class Leaderboard {
 	}
 
 	make_request() {
-
-		frappe.model.with_doctype(this.options.selected_doctype, ()=> {
+		frappe.model.with_doctype(this.options.selected_doctype, () => {
 			this.get_leaderboard(this.get_leaderboard_data);
 		});
 	}
@@ -225,33 +240,32 @@ class Leaderboard {
 		if (!this.options.selected_company) {
 			frappe.throw(__("Please select Company"));
 		}
-		frappe.call(
-			this.leaderboard_config[this.options.selected_doctype].method,
-			{
-				'date_range': this.get_date_range(),
-				'company': this.options.selected_company,
-				'field': this.options.selected_filter_item,
-				'limit': this.leaderboard_limit,
-			}
-		).then(r => {
-			let results = r.message || [];
+		frappe
+			.call(this.leaderboard_config[this.options.selected_doctype].method, {
+				date_range: this.get_date_range(),
+				company: this.options.selected_company,
+				field: this.options.selected_filter_item,
+				limit: this.leaderboard_limit,
+			})
+			.then((r) => {
+				let results = r.message || [];
 
-			let graph_items = results.slice(0, 10);
+				let graph_items = results.slice(0, 10);
 
-			this.$graph_area.show().empty();
+				this.$graph_area.show().empty();
 
-			const custom_options = {
-				data: {
-					datasets: [{ values: graph_items.map(d => d.value) }],
-					labels: graph_items.map(d => d.name)
-				},
-				format_tooltip_x: d => d[this.options.selected_filter_item],
-				height: 140
-			};
-			frappe.utils.make_chart('.leaderboard-graph', custom_options);
+				const custom_options = {
+					data: {
+						datasets: [{ values: graph_items.map((d) => d.value) }],
+						labels: graph_items.map((d) => d.name),
+					},
+					format_tooltip_x: (d) => d[this.options.selected_filter_item],
+					height: 140,
+				};
+				frappe.utils.make_chart(".leaderboard-graph", custom_options);
 
-			notify(this, r);
-		});
+				notify(this, r);
+			});
 	}
 
 	get_leaderboard_data(me, res) {
@@ -267,9 +281,7 @@ class Leaderboard {
 	}
 
 	render_list_view(items = []) {
-
-		var html =
-			`${this.render_message()}
+		var html = `${this.render_message()}
 			
        
 				${this.render_result(items)}
 			
        `;
@@ -278,47 +290,43 @@ class Leaderboard {
 	}
 
 	render_result(items) {
-
-		var html =
-			`${this.render_list_header()}
+		var html = `${this.render_list_header()}
 			${this.render_list_result(items)}`;
 		return html;
 	}
 
 	render_list_header() {
-		const _selected_filter = this.options.selected_filter
-			.map(i => frappe.model.unscrub(i));
+		const _selected_filter = this.options.selected_filter.map((i) => frappe.model.unscrub(i));
 		const fields = ["rank", "name", this.options.selected_filter_item];
-		const filters = fields.map(filter => {
-			const col = __(frappe.model.unscrub(filter));
-			return (
-				`
        
+					${col && _selected_filter.indexOf(col) !== -1 ? "text-right" : ""}">
 					
 						${col}
 					
-				
        `
-			);
-		}).join("");
+				
        `;
+			})
+			.join("");
 
-		const html =
-			`
        
+		const html = `
        
 				
        ${filters}
        
 			
        `;
 		return html;
 	}
 
 	render_list_result(items) {
+		let _html = items
+			.map((item, index) => {
+				const $value = $(this.get_item_html(item, index + 1));
+				const $item_container = $(`
        `).append($value);
+				return $item_container[0].outerHTML;
+			})
+			.join("");
 
-		let _html = items.map((item, index) => {
-			const $value = $(this.get_item_html(item, index+1));
-			const $item_container = $(`
        `).append($value);
-			return $item_container[0].outerHTML;
-		}).join("");
-
-		let html =
-			`
        
+		let html = `
        
 				
        
 					${_html}
 				
        
@@ -328,7 +336,7 @@ class Leaderboard {
 	}
 
 	render_message() {
-		const display_class = this.message ? '' : 'hide';
+		const display_class = this.message ? "" : "hide";
 		let html = `
        
 			
        
 				 {
-			let fieldname = field.fieldname || field;
-			return fieldname === this.options.selected_filter_item;
-		}));
+		const value = frappe.format(
+			item.value,
+			fields.find((field) => {
+				let fieldname = field.fieldname || field;
+				return fieldname === this.options.selected_filter_item;
+			})
+		);
 
 		const link = `/app/${frappe.router.slug(this.options.selected_doctype)}/${item.name}`;
-		const name_html = item.formatted_name ?
-			`${item.formatted_name}`
+		const name_html = item.formatted_name
+			? `${item.formatted_name}`
 			: ` ${item.name} `;
-		const html =
-			`
        
+		const html = `
        
 				
        
 					${index}
 				
        
@@ -369,11 +379,11 @@ class Leaderboard {
 	}
 
 	get_sidebar_item(item, icon) {
-		let icon_html = icon ? frappe.utils.icon(icon, 'md') : '';
+		let icon_html = icon ? frappe.utils.icon(icon, "md") : "";
 		return $(`
      • 
 			${icon_html}
 			
-				${ __(item) }
+				${__(item)}
 			
 		
        • `);
 	}
@@ -391,9 +401,11 @@ class Leaderboard {
 			"last quarter": [frappe.datetime.add_months(current_date, -3), current_date],
 			"last year": [frappe.datetime.add_months(current_date, -12), current_date],
 			"all time": null,
-			"select date range": this.selected_date_range || [frappe.datetime.month_start(), current_date]
-		}
+			"select date range": this.selected_date_range || [
+				frappe.datetime.month_start(),
+				current_date,
+			],
+		};
 		return date_range_map[timespan];
 	}
-
 }
diff --git a/frappe/desk/page/setup_wizard/install_fixtures.py b/frappe/desk/page/setup_wizard/install_fixtures.py
index 18a519f87f..91ea386948 100644
--- a/frappe/desk/page/setup_wizard/install_fixtures.py
+++ b/frappe/desk/page/setup_wizard/install_fixtures.py
@@ -2,7 +2,6 @@
 # License: MIT. See LICENSE
 
 import frappe
-from frappe import _
 from frappe.desk.doctype.global_search_settings.global_search_settings import (
 	update_global_search_doctypes,
 )
diff --git a/frappe/desk/page/setup_wizard/setup_wizard.js b/frappe/desk/page/setup_wizard/setup_wizard.js
index cc91a16345..1cfceb29b0 100644
--- a/frappe/desk/page/setup_wizard/setup_wizard.js
+++ b/frappe/desk/page/setup_wizard/setup_wizard.js
@@ -27,11 +27,11 @@ frappe.setup = {
 		$.each(frappe.setup.events[event] || [], function (i, fn) {
 			fn();
 		});
-	}
-}
+	},
+};
 
-frappe.pages['setup-wizard'].on_page_load = function (wrapper) {
-	let requires = (frappe.boot.setup_wizard_requires || []);
+frappe.pages["setup-wizard"].on_page_load = function (wrapper) {
+	let requires = frappe.boot.setup_wizard_requires || [];
 	frappe.require(requires, function () {
 		frappe.call({
 			method: "frappe.desk.page.setup_wizard.setup_wizard.load_languages",
@@ -46,20 +46,19 @@ frappe.pages['setup-wizard'].on_page_load = function (wrapper) {
 					slide_class: frappe.setup.SetupWizardSlide,
 					unidirectional: 1,
 					done_state: 1,
-				}
+				};
 				frappe.wizard = new frappe.setup.SetupWizard(wizard_settings);
 				frappe.setup.run_event("after_load");
-				// frappe.wizard.values = test_values_edu;
 				let route = frappe.get_route();
 				if (route) {
 					frappe.wizard.show_slide(route[1]);
 				}
-			}
+			},
 		});
 	});
 };
 
-frappe.pages['setup-wizard'].on_page_show = function () {
+frappe.pages["setup-wizard"].on_page_show = function () {
 	if (frappe.get_route()[1]) {
 		frappe.wizard && frappe.wizard.show_slide(frappe.get_route()[1]);
 	}
@@ -68,7 +67,7 @@ frappe.pages['setup-wizard'].on_page_show = function () {
 frappe.setup.on("before_load", function () {
 	// load slides
 	frappe.setup.slides_settings.forEach((s) => {
-		if (!(s.name === 'user' && frappe.boot.developer_mode)) {
+		if (!(s.name === "user" && frappe.boot.developer_mode)) {
 			// if not user slide with developer mode
 			frappe.setup.add_slide(s);
 		}
@@ -88,26 +87,26 @@ frappe.setup.SetupWizard = class SetupWizard extends frappe.ui.Slides {
 	make() {
 		super.make();
 		this.container.addClass("container setup-wizard-slide with-form");
-		this.$next_btn.addClass('action');
-		this.$complete_btn.addClass('action');
+		this.$next_btn.addClass("action");
+		this.$complete_btn.addClass("action");
 		this.setup_keyboard_nav();
 	}
 
 	setup_keyboard_nav() {
-		$('body').on('keydown', this.handle_enter_press.bind(this));
+		$("body").on("keydown", this.handle_enter_press.bind(this));
 	}
 
 	disable_keyboard_nav() {
-		$('body').off('keydown', this.handle_enter_press.bind(this));
+		$("body").off("keydown", this.handle_enter_press.bind(this));
 	}
 
 	handle_enter_press(e) {
 		if (e.which === frappe.ui.keyCode.ENTER) {
 			var $target = $(e.target);
-			if ($target.hasClass('prev-btn')) {
-				$target.trigger('click');
+			if ($target.hasClass("prev-btn")) {
+				$target.trigger("click");
 			} else {
-				this.container.find('.next-btn').trigger('click');
+				this.container.find(".next-btn").trigger("click");
 				e.preventDefault();
 			}
 		}
@@ -135,8 +134,10 @@ frappe.setup.SetupWizard = class SetupWizard extends frappe.ui.Slides {
 		super.show_hide_prev_next(id);
 		if (id + 1 === this.slides.length) {
 			this.$next_btn.removeClass("btn-primary").hide();
-			this.$complete_btn.addClass("btn-primary").show()
-				.on('click', () => this.action_on_complete());
+			this.$complete_btn
+				.addClass("btn-primary")
+				.show()
+				.on("click", () => this.action_on_complete());
 		} else {
 			this.$next_btn.addClass("btn-primary").show();
 			this.$complete_btn.removeClass("btn-primary").hide();
@@ -145,7 +146,7 @@ frappe.setup.SetupWizard = class SetupWizard extends frappe.ui.Slides {
 
 	refresh_slides() {
 		// For Translations, etc.
-		if (this.in_refresh_slides || !this.current_slide.set_values()) {
+		if (this.in_refresh_slides || !this.current_slide.set_values(true)) {
 			return;
 		}
 		this.in_refresh_slides = true;
@@ -171,7 +172,7 @@ frappe.setup.SetupWizard = class SetupWizard extends frappe.ui.Slides {
 		this.show_slide(this.current_id);
 		this.refresh(this.current_id);
 		setTimeout(() => {
-			this.container.find('.form-control').first().focus();
+			this.container.find(".form-control").first().focus();
 		}, 200);
 		this.in_refresh_slides = false;
 	}
@@ -187,15 +188,15 @@ frappe.setup.SetupWizard = class SetupWizard extends frappe.ui.Slides {
 			method: "frappe.desk.page.setup_wizard.setup_wizard.setup_complete",
 			args: { args: this.values },
 			callback: (r) => {
-				if (r.message.status === 'ok') {
+				if (r.message.status === "ok") {
 					this.post_setup_success();
-				} else if (r.message.status === 'registered') {
+				} else if (r.message.status === "registered") {
 					this.update_setup_message(__("starting the setup..."));
 				} else if (r.message.fail !== undefined) {
 					this.abort_setup(r.message.fail);
 				}
 			},
-			error: () => this.abort_setup("Error in setup")
+			error: () => this.abort_setup("Error in setup"),
 		});
 	}
 
@@ -206,17 +207,17 @@ frappe.setup.SetupWizard = class SetupWizard extends frappe.ui.Slides {
 		}
 		setTimeout(function () {
 			// Reload
-			window.location.href = '/app';
+			window.location.href = "/app";
 		}, 2000);
 	}
 
 	abort_setup(fail_msg) {
-		this.$working_state.find('.state-icon-container').html('');
+		this.$working_state.find(".state-icon-container").html("");
 		fail_msg = fail_msg ? fail_msg : __("Failed to complete setup");
 
-		this.update_setup_message('Could not start up: ' + fail_msg);
+		this.update_setup_message("Could not start up: " + fail_msg);
 
-		this.$working_state.find('.title').html('Setup failed');
+		this.$working_state.find(".title").html("Setup failed");
 
 		this.$abort_btn.show();
 	}
@@ -227,19 +228,19 @@ frappe.setup.SetupWizard = class SetupWizard extends frappe.ui.Slides {
 			if (data.stage_status) {
 				// .html('Process '+ data.progress[0] + ' of ' + data.progress[1] + ': ' + data.stage_status);
 				this.update_setup_message(data.stage_status);
-				this.set_setup_load_percent((data.progress[0] + 1) / data.progress[1] * 100);
+				this.set_setup_load_percent(((data.progress[0] + 1) / data.progress[1]) * 100);
 			}
 			if (data.fail_msg) {
 				this.abort_setup(data.fail_msg);
 			}
-			if (data.status === 'ok') {
+			if (data.status === "ok") {
 				this.post_setup_success();
 			}
-		})
+		});
 	}
 
 	update_setup_message(message) {
-		this.$working_state.find('.setup-message').html(message);
+		this.$working_state.find(".setup-message").html(message);
 	}
 
 	get_setup_slides_filtered_by_domain() {
@@ -247,14 +248,16 @@ frappe.setup.SetupWizard = class SetupWizard extends frappe.ui.Slides {
 		frappe.setup.slides.forEach(function (slide) {
 			if (frappe.setup.domains) {
 				let active_domains = frappe.setup.domains;
-				if (!slide.domains ||
-					slide.domains.filter(d => active_domains.includes(d)).length > 0) {
+				if (
+					!slide.domains ||
+					slide.domains.filter((d) => active_domains.includes(d)).length > 0
+				) {
 					filtered_slides.push(slide);
 				}
 			} else {
 				filtered_slides.push(slide);
 			}
-		})
+		});
 		return filtered_slides;
 	}
 
@@ -264,7 +267,8 @@ frappe.setup.SetupWizard = class SetupWizard extends frappe.ui.Slides {
 
 		this.$working_state = this.get_message(
 			__("Setting up your system"),
-			__("Starting Frappe ...")).appendTo(this.parent);
+			__("Starting Frappe ...")
+		).appendTo(this.parent);
 
 		this.attach_abort_button();
 
@@ -273,11 +277,13 @@ frappe.setup.SetupWizard = class SetupWizard extends frappe.ui.Slides {
 	}
 
 	attach_abort_button() {
-		this.$abort_btn = $(``);
-		this.$working_state.find('.content').append(this.$abort_btn);
+		this.$abort_btn = $(
+			``
+		);
+		this.$working_state.find(".content").append(this.$abort_btn);
 
-		this.$abort_btn.on('click', () => {
-			$(this.parent).find('.setup-in-progress').remove();
+		this.$abort_btn.on("click", () => {
+			$(this.parent).find(".setup-in-progress").remove();
 			this.container.show();
 			frappe.set_route(this.page_name, this.slides.length - 1);
 		});
@@ -302,12 +308,12 @@ frappe.setup.SetupWizard = class SetupWizard extends frappe.ui.Slides {
 	}
 
 	set_setup_complete_message(title, message) {
-		this.$working_state.find('.title').html(title);
-		this.$working_state.find('.setup-message').html(message);
+		this.$working_state.find(".title").html(title);
+		this.$working_state.find(".setup-message").html(message);
 	}
 
 	set_setup_load_percent(percent) {
-		this.$working_state.find('.progress-bar').css({ "width": percent + "%" });
+		this.$working_state.find(".progress-bar").css({ width: percent + "%" });
 	}
 };
 
@@ -334,7 +340,6 @@ frappe.setup.SetupWizardSlide = class SetupWizardSlide extends frappe.ui.Slide {
 			});
 		}
 	}
-
 };
 
 // Frappe slides settings
@@ -344,64 +349,41 @@ frappe.setup.slides_settings = [
 		// Welcome (language) slide
 		name: "welcome",
 		title: __("Hello!"),
-		icon: "fa fa-world",
-		// help: __("Let's prepare the system for first use."),
 
 		fields: [
 			{
-				fieldname: "language", label: __("Your Language"),
-				fieldtype: "Select", reqd: 1
-			}
-		],
-
-		onload: function (slide) {
-			this.setup_fields(slide);
-			let browser_language = frappe.setup.utils.get_language_name_from_code(navigator.language);
-			let language_field = slide.get_field("language");
-
-			language_field.set_input(browser_language || "English");
-
-			if (!frappe.setup._from_load_messages) {
-				language_field.$input.trigger("change");
-			}
-			delete frappe.setup._from_load_messages;
-			moment.locale("en");
-		},
-
-		setup_fields: function (slide) {
-			frappe.setup.utils.setup_language_field(slide);
-			frappe.setup.utils.bind_language_events(slide);
-		},
-	},
-
-	{
-		// Region slide
-		name: 'region',
-		title: __("Select Your Region"),
-		icon: "fa fa-flag",
-		// help: __("Select your Country, Time Zone and Currency"),
-		fields: [
-			{
-				fieldname: "country", label: __("Your Country"), reqd: 1,
+				fieldname: "language",
+				label: __("Your Language"),
 				fieldtype: "Autocomplete",
-				placeholder: __('Select Country')
+				placeholder: __("Select Language"),
+				default: "English",
+				reqd: 1,
+			},
+			{
+				fieldname: "country",
+				label: __("Your Country"),
+				fieldtype: "Autocomplete",
+				placeholder: __("Select Country"),
+				reqd: 1,
+			},
+			{
+				fieldtype: "Section Break",
 			},
-			{ fieldtype: "Section Break" },
 			{
 				fieldname: "timezone",
 				label: __("Time Zone"),
-				placeholder: __('Select Time Zone'),
-				reqd: 1,
+				placeholder: __("Select Time Zone"),
 				fieldtype: "Select",
+				reqd: 1,
 			},
 			{ fieldtype: "Column Break" },
 			{
 				fieldname: "currency",
 				label: __("Currency"),
-				placeholder: __('Select Currency'),
-				reqd: 1,
+				placeholder: __("Select Currency"),
 				fieldtype: "Select",
-			}
+				reqd: 1,
+			},
 		],
 
 		onload: function (slide) {
@@ -410,35 +392,57 @@ frappe.setup.slides_settings = [
 			} else {
 				frappe.setup.utils.load_regional_data(slide, this.setup_fields);
 			}
+			if (!slide.get_value("language")) {
+				let session_language =
+					frappe.setup.utils.get_language_name_from_code(
+						frappe.boot.lang || navigator.language
+					) || "English";
+				let language_field = slide.get_field("language");
+
+				language_field.set_input(session_language);
+				if (!frappe.setup._from_load_messages) {
+					language_field.$input.trigger("change");
+				}
+				delete frappe.setup._from_load_messages;
+				moment.locale("en");
+			}
+			frappe.setup.utils.bind_region_events(slide);
+			frappe.setup.utils.bind_language_events(slide);
 		},
 
 		setup_fields: function (slide) {
 			frappe.setup.utils.setup_region_fields(slide);
-			frappe.setup.utils.bind_region_events(slide);
-		}
+			frappe.setup.utils.setup_language_field(slide);
+		},
 	},
-
 	{
 		// Profile slide
-		name: 'user',
+		name: "user",
 		title: __("The First User: You"),
 		icon: "fa fa-user",
 		fields: [
 			{
-				"fieldtype": "Attach Image", "fieldname": "attach_user_image",
-				label: __("Attach Your Picture"), is_private: 0, align: 'center'
+				fieldtype: "Attach Image",
+				fieldname: "attach_user_image",
+				label: __("Attach Your Picture"),
+				is_private: 0,
+				align: "center",
 			},
 			{
-				"fieldname": "full_name", "label": __("Full Name"), "fieldtype": "Data",
-				reqd: 1
+				fieldname: "full_name",
+				label: __("Full Name"),
+				fieldtype: "Data",
+				reqd: 1,
 			},
 			{
-				"fieldname": "email", "label": __("Email Address") + ' (' + __("Will be your login ID") + ')',
-				"fieldtype": "Data", "options": "Email"
+				fieldname: "email",
+				label: __("Email Address") + " (" + __("Will be your login ID") + ")",
+				fieldtype: "Data",
+				options: "Email",
 			},
-			{ "fieldname": "password", "label": __("Password"), "fieldtype": "Password" }
+			{ fieldname: "password", label: __("Password"), fieldtype: "Password" },
 		],
-		// help: __('The first user will become the System Manager (you can change this later).'),
+
 		onload: function (slide) {
 			if (frappe.session.user !== "Administrator") {
 				slide.form.fields_dict.email.$wrapper.toggle(false);
@@ -449,7 +453,8 @@ frappe.setup.slides_settings = [
 
 				if (frappe.boot.user.first_name || frappe.boot.user.last_name) {
 					slide.form.fields_dict.full_name.set_input(
-						[frappe.boot.user.first_name, frappe.boot.user.last_name].join(' ').trim());
+						[frappe.boot.user.first_name, frappe.boot.user.last_name].join(" ").trim()
+					);
 				}
 
 				var user_image = frappe.get_cookie("user_image");
@@ -461,7 +466,6 @@ frappe.setup.slides_settings = [
 					$attach_user_image.find(".img-container").toggle(true);
 				}
 				delete slide.form.fields_dict.email;
-
 			} else {
 				slide.form.fields_dict.email.df.reqd = 1;
 				slide.form.fields_dict.email.refresh();
@@ -487,7 +491,7 @@ frappe.setup.slides_settings = [
 				}
 			}
 		},
-	}
+	},
 ];
 
 frappe.setup.utils = {
@@ -497,7 +501,7 @@ frappe.setup.utils = {
 			callback: function (data) {
 				frappe.setup.data.regional_data = data.message;
 				callback(slide);
-			}
+			},
 		});
 	},
 
@@ -509,14 +513,14 @@ frappe.setup.utils = {
 				frappe.setup.data.full_name = r.message.full_name;
 				frappe.setup.data.email = r.message.email;
 				callback(slide);
-			}
+			},
 		});
 	},
 
 	setup_language_field: function (slide) {
 		var language_field = slide.get_field("language");
 		language_field.df.options = frappe.setup.data.lang.languages;
-		language_field.refresh();
+		language_field.set_options();
 	},
 
 	setup_region_fields: function (slide) {
@@ -524,28 +528,28 @@ frappe.setup.utils = {
 			Set a slide's country, timezone and currency fields
 		*/
 		let data = frappe.setup.data.regional_data;
-		let country_field = slide.get_field('country');
+		let country_field = slide.get_field("country");
 		let translated_countries = [];
 
-		Object.keys(data.country_info).sort().forEach(country => {
-			translated_countries.push({
-				label: __(country),
-				value: country
+		Object.keys(data.country_info)
+			.sort()
+			.forEach((country) => {
+				translated_countries.push({
+					label: __(country),
+					value: country,
+				});
 			});
-		});
 
 		country_field.set_data(translated_countries);
 
-		slide.get_input("currency")
+		slide
+			.get_input("currency")
 			.empty()
 			.add_options(
-				frappe.utils.unique(
-					$.map(data.country_info, opts => opts.currency).sort()
-				)
+				frappe.utils.unique($.map(data.country_info, (opts) => opts.currency).sort())
 			);
 
-		slide.get_input("timezone").empty()
-			.add_options(data.all_timezones);
+		slide.get_input("timezone").empty().add_options(data.all_timezones);
 
 		// set values if present
 		if (frappe.wizard.values.country) {
@@ -555,30 +559,31 @@ frappe.setup.utils = {
 		}
 
 		slide.get_field("currency").set_input(frappe.wizard.values.currency);
-
 		slide.get_field("timezone").set_input(frappe.wizard.values.timezone);
-
 	},
 
 	bind_language_events: function (slide) {
-		slide.get_input("language").unbind("change").on("change", function () {
-			clearTimeout(slide.language_call_timeout);
-			slide.language_call_timeout = setTimeout(() => {
-				var lang = $(this).val() || "English";
-				frappe._messages = {};
-				frappe.call({
-					method: "frappe.desk.page.setup_wizard.setup_wizard.load_messages",
-					freeze: true,
-					args: {
-						language: lang
-					},
-					callback: function () {
-						frappe.setup._from_load_messages = true;
-						frappe.wizard.refresh_slides();
-					}
-				});
-			}, 500);
-		});
+		slide
+			.get_input("language")
+			.unbind("change")
+			.on("change", function () {
+				clearTimeout(slide.language_call_timeout);
+				slide.language_call_timeout = setTimeout(() => {
+					var lang = $(this).val() || "English";
+					frappe._messages = {};
+					frappe.call({
+						method: "frappe.desk.page.setup_wizard.setup_wizard.load_messages",
+						freeze: true,
+						args: {
+							language: lang,
+						},
+						callback: function () {
+							frappe.setup._from_load_messages = true;
+							frappe.wizard.refresh_slides();
+						},
+					});
+				}, 500);
+			});
 	},
 
 	get_language_name_from_code: function (language_code) {
@@ -608,8 +613,8 @@ frappe.setup.utils = {
 			slide.get_field("timezone").set_input($timezone.val());
 
 			// temporarily set date format
-			frappe.boot.sysdefaults.date_format = (data.country_info[country].date_format
-				|| "dd-mm-yyyy");
+			frappe.boot.sysdefaults.date_format =
+				data.country_info[country].date_format || "dd-mm-yyyy";
 		});
 
 		slide.get_input("currency").on("change", function () {
diff --git a/frappe/desk/page/setup_wizard/setup_wizard.py b/frappe/desk/page/setup_wizard/setup_wizard.py
index f85d24704f..3422602720 100755
--- a/frappe/desk/page/setup_wizard/setup_wizard.py
+++ b/frappe/desk/page/setup_wizard/setup_wizard.py
@@ -269,7 +269,6 @@ def add_all_roles_to(name):
 def disable_future_access():
 	frappe.db.set_default("desktop:home_page", "workspace")
 	frappe.db.set_value("System Settings", "System Settings", "setup_complete", 1)
-	frappe.db.set_value("System Settings", "System Settings", "is_first_startup", 1)
 
 	# Enable onboarding after install
 	frappe.db.set_value("System Settings", "System Settings", "enable_onboarding", 1)
@@ -334,22 +333,17 @@ def load_user_details():
 	}
 
 
-@frappe.whitelist()
-def reset_is_first_startup():
-	frappe.db.set_value("System Settings", "System Settings", "is_first_startup", 0)
-
-
 def prettify_args(args):
 	# remove attachments
 	for key, val in args.items():
 		if isinstance(val, str) and "data:image" in val:
 			filename = val.split("data:image", 1)[0].strip(", ")
 			size = round((len(val) * 3 / 4) / 1048576.0, 2)
-			args[key] = "Image Attached: '{0}' of size {1} MB".format(filename, size)
+			args[key] = f"Image Attached: '{filename}' of size {size} MB"
 
 	pretty_args = []
 	for key in sorted(args):
-		pretty_args.append("{} = {}".format(key, args[key]))
+		pretty_args.append(f"{key} = {args[key]}")
 	return pretty_args
 
 
@@ -392,7 +386,7 @@ def email_setup_wizard_exception(traceback, args):
 	frappe.sendmail(
 		recipients=frappe.conf.setup_wizard_exception_email,
 		sender=frappe.session.user,
-		subject="Setup failed: {}".format(frappe.local.site),
+		subject=f"Setup failed: {frappe.local.site}",
 		message=message,
 		delayed=False,
 	)
@@ -437,22 +431,13 @@ def make_records(records, debug=False):
 		if doc.meta.get_field(parent_link_field) and not doc.get(parent_link_field):
 			doc.flags.ignore_mandatory = True
 
+		savepoint = "setup_fixtures_creation"
 		try:
-			doc.insert(ignore_permissions=True)
-			frappe.db.commit()
-
-		except frappe.DuplicateEntryError as e:
-			# print("Failed to insert duplicate {0} {1}".format(doctype, doc.name))
-
-			# pass DuplicateEntryError and continue
-			if e.args and e.args[0] == doc.doctype and e.args[1] == doc.name:
-				# make sure DuplicateEntryError is for the exact same doc and not a related doc
-				frappe.clear_messages()
-			else:
-				raise
-
+			frappe.db.savepoint(savepoint)
+			doc.insert(ignore_permissions=True, ignore_if_duplicate=True)
 		except Exception as e:
-			frappe.db.rollback()
+			frappe.clear_last_message()
+			frappe.db.rollback(save_point=savepoint)
 			exception = record.get("__exception")
 			if exception:
 				config = _dict(exception)
@@ -467,3 +452,4 @@ def make_records(records, debug=False):
 def show_document_insert_error():
 	print("Document Insert Error")
 	print(frappe.get_traceback())
+	frappe.log_error("Exception during Setup")
diff --git a/frappe/desk/page/translation_tool/translation_tool.js b/frappe/desk/page/translation_tool/translation_tool.js
index 13f68e647a..5739eddfc7 100644
--- a/frappe/desk/page/translation_tool/translation_tool.js
+++ b/frappe/desk/page/translation_tool/translation_tool.js
@@ -1,7 +1,7 @@
-frappe.pages['translation-tool'].on_page_load = function(wrapper) {
+frappe.pages["translation-tool"].on_page_load = function (wrapper) {
 	var page = frappe.ui.make_app_page({
 		parent: wrapper,
-		title: __('Translation Tool'),
+		title: __("Translation Tool"),
 		single_column: true,
 		card_layout: true,
 	});
@@ -13,77 +13,74 @@ class TranslationTool {
 	constructor(page) {
 		this.page = page;
 		this.wrapper = $(page.body);
-		this.wrapper.append(frappe.render_template('translation_tool'));
+		this.wrapper.append(frappe.render_template("translation_tool"));
 		frappe.utils.bind_actions_with_object(this.wrapper, this);
 		this.active_translation = null;
 		this.edited_translations = {};
 		this.setup_search_box();
 		this.setup_language_filter();
 		this.page.set_primary_action(
-			__('Contribute Translations'),
+			__("Contribute Translations"),
 			this.show_confirmation_dialog.bind(this)
 		);
-		this.page.set_secondary_action(
-			__('Refresh'),
-			this.fetch_messages_then_render.bind(this)
-		);
+		this.page.set_secondary_action(__("Refresh"), this.fetch_messages_then_render.bind(this));
 		this.update_header();
 	}
 
 	setup_language_filter() {
-		let languages = Object.keys(frappe.boot.lang_dict).map(language_label => {
+		let languages = Object.keys(frappe.boot.lang_dict).map((language_label) => {
 			let value = frappe.boot.lang_dict[language_label];
 			return {
 				label: `${language_label} (${value})`,
-				value: value
+				value: value,
 			};
 		});
 
 		let language_selector = this.page.add_field({
-			fieldname: 'language',
-			fieldtype: 'Select',
+			fieldname: "language",
+			fieldtype: "Select",
 			options: languages,
 			change: () => {
 				let language = language_selector.get_value();
-				localStorage.setItem('translation_language', language);
+				localStorage.setItem("translation_language", language);
 				this.language = language;
 				this.fetch_messages_then_render();
-			}
+			},
 		});
-		let translation_language = localStorage.getItem('translation_language');
-		if (translation_language || frappe.boot.lang !== 'en') {
+		let translation_language = localStorage.getItem("translation_language");
+		if (translation_language || frappe.boot.lang !== "en") {
 			language_selector.set_value(translation_language || frappe.boot.lang);
 		} else {
 			frappe.prompt(
 				{
-					label: __('Please select target language for translation'),
-					fieldname: 'language',
-					fieldtype: 'Select',
+					label: __("Please select target language for translation"),
+					fieldname: "language",
+					fieldtype: "Select",
 					options: languages,
-					reqd: 1
+					reqd: 1,
 				},
-				values => {
+				(values) => {
 					language_selector.set_value(values.language);
 				},
-				__('Select Language')
+				__("Select Language")
 			);
 		}
 	}
 
 	setup_search_box() {
 		let search_box = this.page.add_field({
-			fieldname: 'search',
-			fieldtype: 'Data',
-			label: __('Search Source Text'),
+			fieldname: "search",
+			fieldtype: "Data",
+			label: __("Search Source Text"),
 			change: () => {
 				this.search_text = search_box.get_value();
 				this.fetch_messages_then_render();
-			}
+			},
 		});
 	}
 
 	fetch_messages_then_render() {
-		this.fetch_messages().then(messages => {
+		this.fetch_messages().then((messages) => {
 			this.messages = messages;
 			this.render_messages(messages);
 		});
@@ -91,13 +88,13 @@ class TranslationTool {
 	}
 
 	fetch_messages() {
-		frappe.dom.freeze(__('Fetching...'));
+		frappe.dom.freeze(__("Fetching..."));
 		return frappe
-			.xcall('frappe.translate.get_messages', {
+			.xcall("frappe.translate.get_messages", {
 				language: this.language,
-				search_text: this.search_text
+				search_text: this.search_text,
 			})
-			.then(messages => {
+			.then((messages) => {
 				return messages;
 			})
 			.finally(() => {
@@ -106,7 +103,7 @@ class TranslationTool {
 	}
 
 	render_messages(messages) {
-		let template = message => `
+		let template = (message) => `
 			
        
 		`;
 
-		let html = messages.map(template).join('');
-		this.wrapper.find('.translation-item-container').html(html);
+		let html = messages.map(template).join("");
+		this.wrapper.find(".translation-item-container").html(html);
 	}
 
 	on_translation_click(e, $el) {
-		let message_id = decodeURIComponent($el.data('message-id'));
-		this.wrapper.find('.translation-item').removeClass('active');
-		$el.addClass('active');
-		this.active_translation = this.messages.find(m => m.id === message_id);
+		let message_id = decodeURIComponent($el.data("message-id"));
+		this.wrapper.find(".translation-item").removeClass("active");
+		$el.addClass("active");
+		this.active_translation = this.messages.find((m) => m.id === message_id);
 		this.edit_translation(this.active_translation);
 	}
 
@@ -135,17 +132,19 @@ class TranslationTool {
 		if (this.form) {
 			this.form.set_values({});
 		}
-		this.get_additional_info(translation.id).then(data => {
+		this.get_additional_info(translation.id).then((data) => {
 			this.make_edit_form(translation, data);
 		});
 	}
 
 	get_additional_info(source_id) {
-		frappe.dom.freeze('Fetching...');
-		return frappe.xcall('frappe.translate.get_source_additional_info', {
-			source: source_id,
-			language: this.page.fields_dict['language'].get_value()
-		}).finally(frappe.dom.unfreeze);
+		frappe.dom.freeze("Fetching...");
+		return frappe
+			.xcall("frappe.translate.get_source_additional_info", {
+				source: source_id,
+				language: this.page.fields_dict["language"].get_value(),
+			})
+			.finally(frappe.dom.unfreeze);
 	}
 
 	make_edit_form(translation, { contributions, positions }) {
@@ -153,80 +152,77 @@ class TranslationTool {
 			this.form = new frappe.ui.FieldGroup({
 				fields: [
 					{
-						fieldtype: 'HTML',
-						fieldname: 'header',
-						read_only: 1
-					},
-					{
-						fieldtype: 'Data',
-						fieldname: 'id',
-						hidden: 1
-					},
-					{
-						label: 'Source Text',
-						fieldtype: 'Code',
-						fieldname: 'source_text',
+						fieldtype: "HTML",
+						fieldname: "header",
 						read_only: 1,
-						enable_copy_button: 1
 					},
 					{
-						label: 'Context',
-						fieldtype: 'Code',
-						fieldname: 'context',
-						read_only: 1
+						fieldtype: "Data",
+						fieldname: "id",
+						hidden: 1,
 					},
 					{
-						label: 'DocType',
-						fieldtype: 'Data',
-						fieldname: 'doctype',
-						read_only: 1
+						label: "Source Text",
+						fieldtype: "Code",
+						fieldname: "source_text",
+						read_only: 1,
+						enable_copy_button: 1,
 					},
 					{
-						label: 'Translated Text',
-						fieldtype: 'Small Text',
-						fieldname: 'translated_text',
+						label: "Context",
+						fieldtype: "Code",
+						fieldname: "context",
+						read_only: 1,
 					},
 					{
-						label: 'Suggest',
-						fieldtype: 'Button',
+						label: "DocType",
+						fieldtype: "Data",
+						fieldname: "doctype",
+						read_only: 1,
+					},
+					{
+						label: "Translated Text",
+						fieldtype: "Small Text",
+						fieldname: "translated_text",
+					},
+					{
+						label: "Suggest",
+						fieldtype: "Button",
 						click: () => {
 							let { id, translated_text, source_text } = this.form.get_values();
 							let existing_value = this.form.translation_dict.translated_text;
-							if (
-								is_null(translated_text) ||
-								existing_value === translated_text
-							) {
+							if (is_null(translated_text) || existing_value === translated_text) {
 								delete this.edited_translations[id];
 							} else if (existing_value !== translated_text) {
 								this.edited_translations[id] = {
 									id,
 									translated_text,
-									source_text
+									source_text,
 								};
 							}
 							this.update_header();
-						}
+						},
 					},
 					{
-						fieldtype: 'Section Break',
-						fieldname: 'contributed_translations_section',
-						label: 'Contributed Translations'
+						fieldtype: "Section Break",
+						fieldname: "contributed_translations_section",
+						label: "Contributed Translations",
 					},
 					{
-						fieldtype: 'HTML',
-						fieldname: 'contributed_translations'
+						fieldtype: "HTML",
+						fieldname: "contributed_translations",
 					},
 					{
-						fieldtype: 'Section Break',
+						fieldtype: "Section Break",
 						collapsible: 1,
-						label: 'Occurences in source code'
+						label: "Occurences in source code",
 					},
 					{
-						fieldtype: 'HTML',
-						fieldname: 'positions'
+						fieldtype: "HTML",
+						fieldname: "positions",
 					},
 				],
-				body: this.wrapper.find('.translation-edit-form')
+				body: this.wrapper.find(".translation-edit-form"),
 			});
 
 			this.form.make();
@@ -235,8 +231,8 @@ class TranslationTool {
 
 		this.form.set_values(translation);
 		this.form.translation_dict = translation;
-		this.form.set_df_property('doctype', 'hidden', !translation.doctype);
-		this.form.set_df_property('context', 'hidden', !translation.context);
+		this.form.set_df_property("doctype", "hidden", !translation.doctype);
+		this.form.set_df_property("context", "hidden", !translation.context);
 		this.set_status(translation);
 
 		this.setup_contributions(contributions);
@@ -244,13 +240,13 @@ class TranslationTool {
 	}
 
 	setup_header() {
-		this.form.get_field('header').$wrapper.html(`
        
+		this.form.get_field("header").$wrapper.html(`
        
 			
 		
        `);
 	}
 
 	set_status(translation) {
-		this.form.get_field('header').$wrapper.find('.translation-status').html(`
+		this.form.get_field("header").$wrapper.find(".translation-status").html(`
 			
 				${this.get_indicator_status_text(translation)}
 			
@@ -258,15 +254,16 @@ class TranslationTool {
 	}
 
 	setup_positions(positions) {
-		let position_dom = '';
+		let position_dom = "";
 		if (positions && positions.length) {
-			position_dom = positions.map(position => {
-				if (position.path.startsWith('DocType: ')) {
-					return `
        
+			position_dom = positions
+				.map((position) => {
+					if (position.path.startsWith("DocType: ")) {
+						return `
        
 						${position.path}
 					
        `;
-				} else {
-					return `
        
+					} else {
+						return ``;
-				}
-			}).join('');
+					}
+				})
+				.join("");
 		}
-		this.form.get_field('positions').$wrapper.html(position_dom);
+		this.form.get_field("positions").$wrapper.html(position_dom);
 	}
 
 	setup_contributions(contributions) {
 		const contributions_exists = contributions && contributions.length;
 		if (contributions_exists) {
-			let contributions_html = contributions.map(c => {
+			let contributions_html = contributions.map((c) => {
 				return `
 					
        
 						
        ${c.translated}
        
@@ -293,62 +291,70 @@ class TranslationTool {
 					
        
 				`;
 			});
-			this.form.get_field('contributed_translations').html(contributions_html);
+			this.form.get_field("contributed_translations").html(contributions_html);
 		}
-		this.form.set_df_property('contributed_translations_section', 'hidden', !contributions_exists);
+		this.form.set_df_property(
+			"contributed_translations_section",
+			"hidden",
+			!contributions_exists
+		);
 	}
 	show_confirmation_dialog() {
 		this.confirmation_dialog = new frappe.ui.Dialog({
 			fields: [
 				{
-					label: __('Language'),
-					fieldname: 'language',
-					fieldtype: 'Data',
+					label: __("Language"),
+					fieldname: "language",
+					fieldtype: "Data",
 					read_only: 1,
 					bold: 1,
-					default: this.language
+					default: this.language,
 				},
 				{
-					fieldtype: 'HTML',
-					fieldname: 'edited_translations'
-				}
+					fieldtype: "HTML",
+					fieldname: "edited_translations",
+				},
 			],
-			title: __('Confirm Translations'),
+			title: __("Confirm Translations"),
 			no_submit_on_enter: true,
-			primary_action_label: __('Submit'),
-			primary_action: values => {
+			primary_action_label: __("Submit"),
+			primary_action: (values) => {
 				this.create_translations(values).then(this.confirmation_dialog.hide());
-			}
+			},
 		});
-		this.confirmation_dialog.get_field('edited_translations').html(`
+		this.confirmation_dialog.get_field("edited_translations").html(`
 			
        
 				
        Progress / Wait Event
 				
        
 				
        
-					
-					
+					
+					
-				${Object.values(this.edited_translations).map(t => `
+				${Object.values(this.edited_translations)
+					.map(
+						(t) => `
 					
-				`).join('')}
+				`
+					)
+					.join("")}
 			
        
 				
        ${__('Source Text')}${__('Translated Text')}${__("Source Text")}${__("Translated Text")}
 				
        
 						${t.source_text}
 						${t.translated_text}
 					
        
 		`);
 		this.confirmation_dialog.show();
 	}
 	create_translations() {
-		frappe.dom.freeze(__('Submitting...'));
+		frappe.dom.freeze(__("Submitting..."));
 		return frappe
-			.xcall(
-				'frappe.core.doctype.translation.translation.create_translations',
-				{
-					translation_map: this.edited_translations,
-					language: this.language
-				}
-			)
+			.xcall("frappe.core.doctype.translation.translation.create_translations", {
+				translation_map: this.edited_translations,
+				language: this.language,
+			})
 			.then(() => {
 				frappe.dom.unfreeze();
-				frappe.show_alert({ message: __('Successfully Submitted!'), indicator: 'success'});
+				frappe.show_alert({
+					message: __("Successfully Submitted!"),
+					indicator: "success",
+				});
 				this.edited_translations = {};
 				this.update_header();
 				this.fetch_messages_then_render();
@@ -359,11 +365,11 @@ class TranslationTool {
 	setup_local_contributions() {
 		// TODO: Refactor
 		frappe
-			.xcall('frappe.translate.get_contributions', {
-				language: this.language
+			.xcall("frappe.translate.get_contributions", {
+				language: this.language,
 			})
-			.then(messages => {
-				let template = message => `
+			.then((messages) => {
+				let template = (message) => `
 					
        
 				`;
 
-				let html = messages.map(template).join('');
-				this.wrapper.find('.translation-item-tr').html(html);
+				let html = messages.map(template).join("");
+				this.wrapper.find(".translation-item-tr").html(html);
 			});
 	}
 
 	show_translation_status_modal(e, $el) {
-		let message_id = decodeURIComponent($el.data('message-id'));
+		let message_id = decodeURIComponent($el.data("message-id"));
 
-		frappe.xcall('frappe.translate.get_contribution_status', { message_id })
-			.then(doc => {
-				let d = new frappe.ui.Dialog({
-					title: __('Contribution Status'),
-					fields: [
-						{
-							fieldname: 'source_message',
-							label: __('Source Message'),
-							fieldtype: 'Data',
-							read_only: 1
+		frappe.xcall("frappe.translate.get_contribution_status", { message_id }).then((doc) => {
+			let d = new frappe.ui.Dialog({
+				title: __("Contribution Status"),
+				fields: [
+					{
+						fieldname: "source_message",
+						label: __("Source Message"),
+						fieldtype: "Data",
+						read_only: 1,
+					},
+					{
+						fieldname: "translated",
+						label: __("Translated Message"),
+						fieldtype: "Data",
+						read_only: 1,
+					},
+					{
+						fieldname: "contribution_status",
+						label: __("Contribution Status"),
+						fieldtype: "Data",
+						read_only: 1,
+					},
+					{
+						fieldname: "modified_by",
+						label: __("Verified By"),
+						fieldtype: "Data",
+						read_only: 1,
+						depends_on: (doc) => {
+							return doc.contribution_status == "Verified";
 						},
-						{
-							fieldname: 'translated',
-							label: __('Translated Message'),
-							fieldtype: 'Data',
-							read_only: 1
-						},
-						{
-							fieldname: 'contribution_status',
-							label: __('Contribution Status'),
-							fieldtype: 'Data',
-							read_only: 1
-						},
-						{
-							fieldname: 'modified_by',
-							label: __('Verified By'),
-							fieldtype: 'Data',
-							read_only: 1,
-							depends_on: doc => {
-								return doc.contribution_status == 'Verified';
-							}
-						},
-					]
-				});
-				d.set_values(doc);
-				d.show();
+					},
+				],
 			});
+			d.set_values(doc);
+			d.show();
+		});
 	}
 
 	update_header() {
-		let edited_translations_count = Object.keys(this.edited_translations)
-			.length;
+		let edited_translations_count = Object.keys(this.edited_translations).length;
 		if (edited_translations_count) {
-			let message = '';
+			let message = "";
 			if (edited_translations_count == 1) {
-				message = __('{0} translation pending', [edited_translations_count]);
+				message = __("{0} translation pending", [edited_translations_count]);
 			} else {
-				message = __('{0} translations pending', [edited_translations_count]);
+				message = __("{0} translations pending", [edited_translations_count]);
 			}
-			this.page.set_indicator(message, 'orange');
+			this.page.set_indicator(message, "orange");
 		} else {
-			this.page.set_indicator('');
+			this.page.set_indicator("");
 		}
-		this.page.btn_primary.prop('disabled', !edited_translations_count);
+		this.page.btn_primary.prop("disabled", !edited_translations_count);
 	}
 
 	get_indicator_color(message_obj) {
-		return !message_obj.translated ? 'red' : message_obj.translated_by_google ? 'orange' : 'blue';
+		return !message_obj.translated
+			? "red"
+			: message_obj.translated_by_google
+			? "orange"
+			: "blue";
 	}
 
 	get_indicator_status_text(message_obj) {
 		if (!message_obj.translated) {
-			return __('Untranslated');
+			return __("Untranslated");
 		} else if (message_obj.translated_by_google) {
-			return __('Google Translation');
+			return __("Google Translation");
 		} else {
-			return __('Community Contribution');
+			return __("Community Contribution");
 		}
 	}
 
 	get_contribution_indicator_color(message_obj) {
-		return message_obj.contribution_status == 'Pending' ? 'orange' : 'green';
+		return message_obj.contribution_status == "Pending" ? "orange" : "green";
 	}
 
 	get_code_url(path, line_no, app) {
diff --git a/frappe/desk/page/user_profile/user_profile.js b/frappe/desk/page/user_profile/user_profile.js
index 5890975e69..e4cef59cc7 100644
--- a/frappe/desk/page/user_profile/user_profile.js
+++ b/frappe/desk/page/user_profile/user_profile.js
@@ -1,5 +1,5 @@
-frappe.pages['user-profile'].on_page_load = function (wrapper) {
-	frappe.require('user_profile_controller.bundle.js', () => {
+frappe.pages["user-profile"].on_page_load = function (wrapper) {
+	frappe.require("user_profile_controller.bundle.js", () => {
 		let user_profile = new frappe.ui.UserProfile(wrapper);
 		user_profile.show();
 	});
diff --git a/frappe/desk/page/user_profile/user_profile_controller.js b/frappe/desk/page/user_profile/user_profile_controller.js
index 40b542d5c3..5103bd8a19 100644
--- a/frappe/desk/page/user_profile/user_profile_controller.js
+++ b/frappe/desk/page/user_profile/user_profile_controller.js
@@ -1,5 +1,5 @@
 import BaseTimeline from "../../../public/js/frappe/form/footer/base_timeline";
-frappe.provide('frappe.energy_points');
+frappe.provide("frappe.energy_points");
 
 class UserProfile {
 	constructor(wrapper) {
@@ -7,9 +7,9 @@ class UserProfile {
 		this.page = frappe.ui.make_app_page({
 			parent: wrapper,
 		});
-		this.sidebar = this.wrapper.find('.layout-side-section');
-		this.main_section = this.wrapper.find('.layout-main-section');
-		this.wrapper.bind('show', () => {
+		this.sidebar = this.wrapper.find(".layout-side-section");
+		this.main_section = this.wrapper.find(".layout-main-section");
+		this.wrapper.bind("show", () => {
 			this.show();
 		});
 	}
@@ -17,13 +17,13 @@ class UserProfile {
 	show() {
 		let route = frappe.get_route();
 		this.user_id = route[1] || frappe.session.user;
-		frappe.dom.freeze(__('Loading user profile') + '...');
-		frappe.db.exists('User', this.user_id).then(exists => {
+		frappe.dom.freeze(__("Loading user profile") + "...");
+		frappe.db.exists("User", this.user_id).then((exists) => {
 			frappe.dom.unfreeze();
 			if (exists) {
 				this.make_user_profile();
 			} else {
-				frappe.msgprint(__('User does not exist'));
+				frappe.msgprint(__("User does not exist"));
 			}
 		});
 	}
@@ -32,7 +32,7 @@ class UserProfile {
 		this.user = frappe.user_info(this.user_id);
 		this.page.set_title(this.user.fullname);
 		this.setup_user_search();
-		this.main_section.empty().append(frappe.render_template('user_profile'));
+		this.main_section.empty().append(frappe.render_template("user_profile"));
 		this.energy_points = 0;
 		this.review_points = 0;
 		this.rank = 0;
@@ -41,91 +41,92 @@ class UserProfile {
 		this.render_points_and_rank();
 		this.render_heatmap();
 		this.render_line_chart();
-		this.render_percentage_chart('type', 'Type Distribution');
+		this.render_percentage_chart("type", "Type Distribution");
 		this.create_percentage_chart_filters();
 		this.setup_user_activity_timeline();
 	}
 
 	setup_user_search() {
 		this.$user_search_button = this.page.set_secondary_action(
-			__('Change User'),
+			__("Change User"),
 			() => this.show_user_search_dialog(),
-			{ icon: 'change', size: 'sm' }
+			{ icon: "change", size: "sm" }
 		);
 	}
 
 	show_user_search_dialog() {
 		let dialog = new frappe.ui.Dialog({
-			title: __('Change User'),
+			title: __("Change User"),
 			fields: [
 				{
-					fieldtype: 'Link',
-					fieldname: 'user',
-					options: 'User',
-					label: __('User'),
-				}
+					fieldtype: "Link",
+					fieldname: "user",
+					options: "User",
+					label: __("User"),
+				},
 			],
-			primary_action_label: __('Go'),
+			primary_action_label: __("Go"),
 			primary_action: ({ user }) => {
 				dialog.hide();
-				frappe.set_route('user-profile', user);
-			}
+				frappe.set_route("user-profile", user);
+			},
 		});
 		dialog.show();
 	}
 
 	render_heatmap() {
-		this.heatmap = new frappe.Chart('.performance-heatmap', {
-			type: 'heatmap',
-			countLabel: 'Energy Points',
+		this.heatmap = new frappe.Chart(".performance-heatmap", {
+			type: "heatmap",
+			countLabel: "Energy Points",
 			data: {},
 			discreteDomains: 1,
 			radius: 3,
-			height: 150
+			height: 150,
 		});
 		this.update_heatmap_data();
 		this.create_heatmap_chart_filters();
 	}
 
 	update_heatmap_data(date_from) {
-		frappe.xcall('frappe.desk.page.user_profile.user_profile.get_energy_points_heatmap_data', {
-			user: this.user_id,
-			date: date_from || frappe.datetime.year_start(),
-		}).then((r) => {
-			this.heatmap.update({ dataPoints: r });
-		});
+		frappe
+			.xcall("frappe.desk.page.user_profile.user_profile.get_energy_points_heatmap_data", {
+				user: this.user_id,
+				date: date_from || frappe.datetime.year_start(),
+			})
+			.then((r) => {
+				this.heatmap.update({ dataPoints: r });
+			});
 	}
 
-
 	render_line_chart() {
 		this.line_chart_filters = [
-			['Energy Point Log', 'user', '=', this.user_id, false],
-			['Energy Point Log', 'type', '!=', 'Review', false]
+			["Energy Point Log", "user", "=", this.user_id, false],
+			["Energy Point Log", "type", "!=", "Review", false],
 		];
 
 		this.line_chart_config = {
-			timespan: 'Last Month',
-			time_interval: 'Daily',
-			type: 'Line',
-			value_based_on: 'points',
-			chart_type: 'Sum',
-			document_type: 'Energy Point Log',
-			name: 'Energy Points',
-			width: 'half',
-			based_on: 'creation'
+			timespan: "Last Month",
+			time_interval: "Daily",
+			type: "Line",
+			value_based_on: "points",
+			chart_type: "Sum",
+			document_type: "Energy Point Log",
+			name: "Energy Points",
+			width: "half",
+			based_on: "creation",
 		};
 
-		this.line_chart = new frappe.Chart('.performance-line-chart', {
-			type: 'line',
+		this.line_chart = new frappe.Chart(".performance-line-chart", {
+			type: "line",
 			height: 200,
 			data: {
 				labels: [],
-				datasets: [{}]
+				datasets: [{}],
 			},
-			colors: ['purple'],
+			colors: ["purple"],
 			axisOptions: {
-				xIsSeries: 1
-			}
+				xIsSeries: 1,
+			},
 		});
 		this.update_line_chart_data();
 		this.create_line_chart_filters();
@@ -134,217 +135,258 @@ class UserProfile {
 	update_line_chart_data() {
 		this.line_chart_config.filters_json = JSON.stringify(this.line_chart_filters);
 
-		frappe.xcall('frappe.desk.doctype.dashboard_chart.dashboard_chart.get', {
-			chart: this.line_chart_config,
-			no_cache: 1,
-		}).then(chart => {
-			this.line_chart.update(chart);
-		});
+		frappe
+			.xcall("frappe.desk.doctype.dashboard_chart.dashboard_chart.get", {
+				chart: this.line_chart_config,
+				no_cache: 1,
+			})
+			.then((chart) => {
+				this.line_chart.update(chart);
+			});
 	}
 
 	// eslint-disable-next-line no-unused-vars
 	render_percentage_chart(field, title) {
-		frappe.xcall('frappe.desk.page.user_profile.user_profile.get_energy_points_percentage_chart_data', {
-			user: this.user_id,
-			field: field
-		}).then(chart => {
-			if (chart.labels.length) {
-				this.percentage_chart = new frappe.Chart('.performance-percentage-chart', {
-					type: 'percentage',
-					data: {
-						labels: chart.labels,
-						datasets: chart.datasets
-					},
-					truncateLegends: 1,
-					barOptions: {
-						height: 11,
-						depth: 1
-					},
-					height: 200,
-					maxSlices: 8,
-					colors: ['purple', 'blue', 'cyan', 'teal', 'pink', 'red', 'orange', 'yellow'],
-				});
-			} else {
-				this.wrapper.find('.percentage-chart-container').hide();
-			}
-		});
+		frappe
+			.xcall(
+				"frappe.desk.page.user_profile.user_profile.get_energy_points_percentage_chart_data",
+				{
+					user: this.user_id,
+					field: field,
+				}
+			)
+			.then((chart) => {
+				if (chart.labels.length) {
+					this.percentage_chart = new frappe.Chart(".performance-percentage-chart", {
+						type: "percentage",
+						data: {
+							labels: chart.labels,
+							datasets: chart.datasets,
+						},
+						truncateLegends: 1,
+						barOptions: {
+							height: 11,
+							depth: 1,
+						},
+						height: 200,
+						maxSlices: 8,
+						colors: [
+							"purple",
+							"blue",
+							"cyan",
+							"teal",
+							"pink",
+							"red",
+							"orange",
+							"yellow",
+						],
+					});
+				} else {
+					this.wrapper.find(".percentage-chart-container").hide();
+				}
+			});
 	}
 
 	create_line_chart_filters() {
 		let filters = [
 			{
-				label: 'All',
-				options: ['All', 'Auto', 'Criticism', 'Appreciation', 'Revert'],
+				label: "All",
+				options: ["All", "Auto", "Criticism", "Appreciation", "Revert"],
 				action: (selected_item) => {
-					if (selected_item === 'All') {
+					if (selected_item === "All") {
 						this.line_chart_filters = [
-							['Energy Point Log', 'user', '=', this.user_id, false],
-							['Energy Point Log', 'type', '!=', 'Review', false]
+							["Energy Point Log", "user", "=", this.user_id, false],
+							["Energy Point Log", "type", "!=", "Review", false],
 						];
 					} else {
-						this.line_chart_filters[1] = ['Energy Point Log', 'type', '=', selected_item, false];
+						this.line_chart_filters[1] = [
+							"Energy Point Log",
+							"type",
+							"=",
+							selected_item,
+							false,
+						];
 					}
 					this.update_line_chart_data();
-				}
+				},
 			},
 			{
-				label: 'Last Month',
-				options: ['Last Week', 'Last Month', 'Last Quarter', 'Last Year'],
+				label: "Last Month",
+				options: ["Last Week", "Last Month", "Last Quarter", "Last Year"],
 				action: (selected_item) => {
 					this.line_chart_config.timespan = selected_item;
 					this.update_line_chart_data();
-				}
+				},
 			},
 			{
-				label: 'Daily',
-				options: ['Daily', 'Weekly', 'Monthly'],
+				label: "Daily",
+				options: ["Daily", "Weekly", "Monthly"],
 				action: (selected_item) => {
 					this.line_chart_config.time_interval = selected_item;
 					this.update_line_chart_data();
-				}
+				},
 			},
 		];
-		frappe.dashboard_utils.render_chart_filters(filters, 'chart-filter', '.line-chart-options', 1);
+		frappe.dashboard_utils.render_chart_filters(
+			filters,
+			"chart-filter",
+			".line-chart-options",
+			1
+		);
 	}
 
 	create_percentage_chart_filters() {
 		let filters = [
 			{
-				label: 'Type',
-				options: ['Type', 'Reference Doctype', 'Rule'],
-				fieldnames: ['type', 'reference_doctype', 'rule'],
+				label: "Type",
+				options: ["Type", "Reference Doctype", "Rule"],
+				fieldnames: ["type", "reference_doctype", "rule"],
 				action: (selected_item, fieldname) => {
-					let title = selected_item + ' Distribution';
+					let title = selected_item + " Distribution";
 					this.render_percentage_chart(fieldname, title);
-				}
+				},
 			},
 		];
-		frappe.dashboard_utils.render_chart_filters(filters, 'chart-filter', '.percentage-chart-options');
+		frappe.dashboard_utils.render_chart_filters(
+			filters,
+			"chart-filter",
+			".percentage-chart-options"
+		);
 	}
 
 	create_heatmap_chart_filters() {
 		let filters = [
 			{
 				label: frappe.dashboard_utils.get_year(frappe.datetime.now_date()),
-				options: frappe.dashboard_utils.get_years_since_creation(frappe.boot.user.creation),
+				options: frappe.dashboard_utils.get_years_since_creation(
+					frappe.boot.user.creation
+				),
 				action: (selected_item) => {
 					this.update_heatmap_data(frappe.datetime.obj_to_str(selected_item));
-				}
+				},
 			},
 		];
-		frappe.dashboard_utils.render_chart_filters(filters, 'chart-filter', '.heatmap-options');
+		frappe.dashboard_utils.render_chart_filters(filters, "chart-filter", ".heatmap-options");
 	}
 
-
 	edit_profile() {
 		let edit_profile_dialog = new frappe.ui.Dialog({
-			title: __('Edit Profile'),
+			title: __("Edit Profile"),
 			fields: [
 				{
-					fieldtype: 'Attach Image',
-					fieldname: 'user_image',
-					label: 'Profile Image',
+					fieldtype: "Attach Image",
+					fieldname: "user_image",
+					label: "Profile Image",
 				},
 				{
-					fieldtype: 'Data',
-					fieldname: 'interest',
-					label: 'Interests',
+					fieldtype: "Data",
+					fieldname: "interest",
+					label: "Interests",
 				},
 				{
-					fieldtype: 'Column Break'
+					fieldtype: "Column Break",
 				},
 				{
-					fieldtype: 'Data',
-					fieldname: 'location',
-					label: 'Location',
+					fieldtype: "Data",
+					fieldname: "location",
+					label: "Location",
 				},
 				{
-					fieldtype: 'Section Break',
-					fieldname: 'Interest',
+					fieldtype: "Section Break",
+					fieldname: "Interest",
 				},
 				{
-					fieldtype: 'Small Text',
-					fieldname: 'bio',
-					label: 'Bio',
-				}
+					fieldtype: "Small Text",
+					fieldname: "bio",
+					label: "Bio",
+				},
 			],
-			primary_action: values => {
+			primary_action: (values) => {
 				edit_profile_dialog.disable_primary_action();
-				frappe.xcall('frappe.desk.page.user_profile.user_profile.update_profile_info', {
-					profile_info: values
-				}).then(user => {
-					user.image = user.user_image;
-					this.user = Object.assign(values, user);
-					edit_profile_dialog.hide();
-					this.render_user_details();
-				}).finally(() => {
-					edit_profile_dialog.enable_primary_action();
-				});
+				frappe
+					.xcall("frappe.desk.page.user_profile.user_profile.update_profile_info", {
+						profile_info: values,
+					})
+					.then((user) => {
+						user.image = user.user_image;
+						this.user = Object.assign(values, user);
+						edit_profile_dialog.hide();
+						this.render_user_details();
+					})
+					.finally(() => {
+						edit_profile_dialog.enable_primary_action();
+					});
 			},
-			primary_action_label: __('Save')
+			primary_action_label: __("Save"),
 		});
 
 		edit_profile_dialog.set_values({
 			user_image: this.user.image,
 			location: this.user.location,
 			interest: this.user.interest,
-			bio: this.user.bio
+			bio: this.user.bio,
 		});
 		edit_profile_dialog.show();
 	}
 
 	render_user_details() {
-		this.sidebar.empty().append(frappe.render_template('user_profile_sidebar', {
-			user_image: this.user.image,
-			user_abbr: this.user.abbr,
-			user_location: this.user.location,
-			user_interest: this.user.interest,
-			user_bio: this.user.bio,
-		}));
+		this.sidebar.empty().append(
+			frappe.render_template("user_profile_sidebar", {
+				user_image: this.user.image,
+				user_abbr: this.user.abbr,
+				user_location: this.user.location,
+				user_interest: this.user.interest,
+				user_bio: this.user.bio,
+			})
+		);
 
 		this.setup_user_profile_links();
 	}
 
 	setup_user_profile_links() {
 		if (this.user_id !== frappe.session.user) {
-			this.wrapper.find('.profile-links').hide();
+			this.wrapper.find(".profile-links").hide();
 		} else {
-			this.wrapper.find('.edit-profile-link').on('click', () => {
+			this.wrapper.find(".edit-profile-link").on("click", () => {
 				this.edit_profile();
 			});
 
-			this.wrapper.find('.user-settings-link').on('click', () => {
+			this.wrapper.find(".user-settings-link").on("click", () => {
 				this.go_to_user_settings();
 			});
 		}
 	}
 
 	get_user_rank() {
-		return frappe.xcall('frappe.desk.page.user_profile.user_profile.get_user_rank', {
-			user: this.user_id,
-		}).then(r => {
-			if (r.monthly_rank.length) this.month_rank = r.monthly_rank[0];
-			if (r.all_time_rank.length) this.rank = r.all_time_rank[0];
-		});
+		return frappe
+			.xcall("frappe.desk.page.user_profile.user_profile.get_user_rank", {
+				user: this.user_id,
+			})
+			.then((r) => {
+				if (r.monthly_rank.length) this.month_rank = r.monthly_rank[0];
+				if (r.all_time_rank.length) this.rank = r.all_time_rank[0];
+			});
 	}
 
 	get_user_points() {
-		return frappe.xcall(
-			'frappe.social.doctype.energy_point_log.energy_point_log.get_user_energy_and_review_points',
-			{
-				user: this.user_id,
-			}
-		).then(r => {
-			if (r[this.user_id]) {
-				this.energy_points = r[this.user_id].energy_points;
-				this.review_points = r[this.user_id].review_points;
-			}
-		});
+		return frappe
+			.xcall(
+				"frappe.social.doctype.energy_point_log.energy_point_log.get_user_energy_and_review_points",
+				{
+					user: this.user_id,
+				}
+			)
+			.then((r) => {
+				if (r[this.user_id]) {
+					this.energy_points = r[this.user_id].energy_points;
+					this.review_points = r[this.user_id].review_points;
+				}
+			});
 	}
 
 	render_points_and_rank() {
-		let $profile_details = this.wrapper.find('.user-stats');
-		let $profile_details_wrapper = this.wrapper.find('.user-stats-detail');
+		let $profile_details = this.wrapper.find(".user-stats");
+		let $profile_details_wrapper = this.wrapper.find(".user-stats-detail");
 
 		const _get_stat_dom = (value, label, icon) => {
 			return `
        
@@ -359,10 +401,10 @@ class UserProfile {
 		this.get_user_rank().then(() => {
 			this.get_user_points().then(() => {
 				let html = $(`
-					${_get_stat_dom(this.energy_points, __('Energy Points'), "color-energy-points")}
-					${_get_stat_dom(this.review_points, __('Review Points'), "color-review-points")}
-					${_get_stat_dom(this.rank, __('Rank'), "color-rank")}
-					${_get_stat_dom(this.month_rank, __('Monthly Rank'), "color-monthly-rank")}
+					${_get_stat_dom(this.energy_points, __("Energy Points"), "color-energy-points")}
+					${_get_stat_dom(this.review_points, __("Review Points"), "color-review-points")}
+					${_get_stat_dom(this.rank, __("Rank"), "color-rank")}
+					${_get_stat_dom(this.month_rank, __("Monthly Rank"), "color-monthly-rank")}
 				`);
 
 				$profile_details.append(html);
@@ -372,14 +414,14 @@ class UserProfile {
 	}
 
 	go_to_user_settings() {
-		frappe.set_route('Form', 'User', this.user_id);
+		frappe.set_route("Form", "User", this.user_id);
 	}
 
 	setup_user_activity_timeline() {
 		this.user_activity_timeline = new UserProfileTimeline({
-			parent: this.wrapper.find('.recent-activity-list'),
-			footer: this.wrapper.find('.recent-activity-footer'),
-			user: this.user_id
+			parent: this.wrapper.find(".recent-activity-list"),
+			footer: this.wrapper.find(".recent-activity-footer"),
+			user: this.user_id,
 		});
 
 		this.user_activity_timeline.refresh();
@@ -397,24 +439,27 @@ class UserProfileTimeline extends BaseTimeline {
 		return this.get_user_activity_data().then((activities) => {
 			if (!activities.length) {
 				this.show_more_button.hide();
-				this.timeline_wrapper.html(`
        ${__('No activities to show')}
        `);
+				this.timeline_wrapper.html(`
        ${__("No activities to show")}
        `);
 				return;
 			}
 			this.show_more_button.toggle(activities.length === this.activity_limit);
-			this.timeline_items = activities.map((activity) => this.get_activity_timeline_item(activity));
+			this.timeline_items = activities.map((activity) =>
+				this.get_activity_timeline_item(activity)
+			);
 		});
 	}
 
 	get_user_activity_data() {
-		return frappe.xcall('frappe.desk.page.user_profile.user_profile.get_energy_points_list', {
+		return frappe.xcall("frappe.desk.page.user_profile.user_profile.get_energy_points_list", {
 			start: this.activity_start,
 			limit: this.activity_limit,
-			user: this.user
+			user: this.user,
 		});
 	}
 
 	get_activity_timeline_item(data) {
-		let icon = data.type == 'Appreciation' ? 'clap': data.type == 'Criticism' ? 'criticize': null;
+		let icon =
+			data.type == "Appreciation" ? "clap" : data.type == "Criticism" ? "criticize" : null;
 		return {
 			icon: icon,
 			creation: data.creation,
@@ -424,23 +469,27 @@ class UserProfileTimeline extends BaseTimeline {
 	}
 
 	setup_show_more_activity() {
-		this.show_more_button = $(`        ${__('Show More Activity')}`);
+		this.show_more_button = $(
+			`${__("Show More Activity")}`
+		);
 		this.show_more_button.hide();
 		this.footer.append(this.show_more_button);
-		this.show_more_button.on('click', () => this.show_more_activity());
+		this.show_more_button.on("click", () => this.show_more_activity());
 	}
 
 	show_more_activity() {
 		this.activity_start += this.activity_limit;
-		this.get_user_activity_data().then(activities => {
+		this.get_user_activity_data().then((activities) => {
 			if (!activities.length || activities.length < this.activity_limit) {
 				this.show_more_button.hide();
 			}
-			let timeline_items = activities.map((activity) => this.get_activity_timeline_item(activity));
+			let timeline_items = activities.map((activity) =>
+				this.get_activity_timeline_item(activity)
+			);
 			timeline_items.map((item) => this.add_timeline_item(item, true));
 		});
 	}
 }
 
-frappe.provide('frappe.ui');
+frappe.provide("frappe.ui");
 frappe.ui.UserProfile = UserProfile;
diff --git a/frappe/desk/query_report.py b/frappe/desk/query_report.py
index 7de8ccabbf..e88a453e64 100644
--- a/frappe/desk/query_report.py
+++ b/frappe/desk/query_report.py
@@ -1,6 +1,7 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
 
+import datetime
 import json
 import os
 from datetime import timedelta
@@ -58,6 +59,8 @@ def get_report_doc(report_name):
 
 
 def get_report_result(report, filters):
+	res = None
+
 	if report.report_type == "Query Report":
 		res = report.execute_query_report(filters)
 
@@ -84,7 +87,7 @@ def generate_report_result(
 	res = get_report_result(report, filters) or []
 
 	columns, result, message, chart, report_summary, skip_total_row = ljust_list(res, 6)
-	columns = [get_column_as_dict(col) for col in columns]
+	columns = [get_column_as_dict(col) for col in (columns or [])]
 	report_column_names = [col["fieldname"] for col in columns]
 
 	# convert to list of dicts
@@ -185,7 +188,7 @@ def get_script(report_name):
 
 	script = None
 	if os.path.exists(script_path):
-		with open(script_path, "r") as f:
+		with open(script_path) as f:
 			script = f.read()
 			script += f"\n\n//# sourceURL={scrub(report.name)}.js"
 
@@ -314,7 +317,7 @@ def get_prepared_report_result(report, filters, dn="", user=None):
 
 				latest_report_data = {"columns": columns, "result": data}
 		except Exception:
-			frappe.log_error(frappe.get_traceback())
+			doc.log_error("Prepared report failed")
 			frappe.delete_doc("Prepared Report", doc.name)
 			frappe.db.commit()
 			doc = None
@@ -382,6 +385,18 @@ def format_duration_fields(data: frappe._dict) -> None:
 
 
 def build_xlsx_data(data, visible_idx, include_indentation, ignore_visible_idx=False):
+	EXCEL_TYPES = (
+		str,
+		bool,
+		type(None),
+		int,
+		float,
+		datetime.datetime,
+		datetime.date,
+		datetime.time,
+		datetime.timedelta,
+	)
+
 	result = [[]]
 	column_widths = []
 
@@ -406,6 +421,9 @@ def build_xlsx_data(data, visible_idx, include_indentation, ignore_visible_idx=F
 					label = column.get("label")
 					fieldname = column.get("fieldname")
 					cell_value = row.get(fieldname, row.get(label, ""))
+					if not isinstance(cell_value, EXCEL_TYPES):
+						cell_value = cstr(cell_value)
+
 					if cint(include_indentation) and "indent" in row and col_idx == 0:
 						cell_value = ("    " * cint(row["indent"])) + cstr(cell_value)
 					row_data.append(cell_value)
diff --git a/frappe/desk/report/todo/todo.js b/frappe/desk/report/todo/todo.js
index bb2e0f7846..52fee62afd 100644
--- a/frappe/desk/report/todo/todo.js
+++ b/frappe/desk/report/todo/todo.js
@@ -3,7 +3,5 @@
 /* eslint-disable */
 
 frappe.query_reports["ToDo"] = {
-	"filters": [
-
-	]
-}
+	filters: [],
+};
diff --git a/frappe/desk/report/todo/todo.py b/frappe/desk/report/todo/todo.py
index 7c566c74ea..e75b8d1900 100644
--- a/frappe/desk/report/todo/todo.py
+++ b/frappe/desk/report/todo/todo.py
@@ -46,7 +46,7 @@ def execute(filters=None):
 	for todo in todo_list:
 		if todo.owner == frappe.session.user or todo.assigned_by == frappe.session.user:
 			if todo.reference_type:
-				todo.reference = """%s: %s""" % (
+				todo.reference = """{}: {}""".format(
 					todo.reference_type,
 					todo.reference_name,
 					todo.reference_type,
diff --git a/frappe/desk/report_dump.py b/frappe/desk/report_dump.py
index ac01cf892f..6650d24757 100644
--- a/frappe/desk/report_dump.py
+++ b/frappe/desk/report_dump.py
@@ -53,7 +53,7 @@ def get_data(doctypes, last_modified):
 		out[dt]["data"] = [
 			list(t)
 			for t in frappe.db.sql(
-				"""select %s from %s %s %s""" % (",".join(args["columns"]), table, conditions, order_by)
+				"""select {} from {} {} {}""".format(",".join(args["columns"]), table, conditions, order_by)
 			)
 		]
 
diff --git a/frappe/desk/reportview.py b/frappe/desk/reportview.py
index b45f80f6ff..679b052baf 100644
--- a/frappe/desk/reportview.py
+++ b/frappe/desk/reportview.py
@@ -13,6 +13,7 @@ from frappe.core.doctype.access_log.access_log import make_access_log
 from frappe.model import child_table_fields, default_fields, optional_fields
 from frappe.model.base_document import get_controller
 from frappe.model.db_query import DatabaseQuery
+from frappe.model.utils import is_virtual_doctype
 from frappe.utils import add_user_info, cstr, format_duration
 
 
@@ -23,7 +24,7 @@ def get():
 	# If virtual doctype get data from controller het_list method
 	if is_virtual_doctype(args.doctype):
 		controller = get_controller(args.doctype)
-		data = compress(controller(args.doctype).get_list(args))
+		data = compress(controller.get_list(args))
 	else:
 		data = compress(execute(**args), args=args)
 	return data
@@ -36,7 +37,7 @@ def get_list():
 
 	if is_virtual_doctype(args.doctype):
 		controller = get_controller(args.doctype)
-		data = controller(args.doctype).get_list(args)
+		data = controller.get_list(args)
 	else:
 		# uncompressed (refactored from frappe.model.db_query.get_list)
 		data = execute(**args)
@@ -51,7 +52,7 @@ def get_count():
 
 	if is_virtual_doctype(args.doctype):
 		controller = get_controller(args.doctype)
-		data = controller(args.doctype).get_count(args)
+		data = controller.get_count(args)
 	else:
 		distinct = "distinct " if args.distinct == "true" else ""
 		args.fields = [f"count({distinct}`tab{args.doctype}`.name) as total_count"]
@@ -166,12 +167,12 @@ def setup_group_by(data):
 
 
 def raise_invalid_field(fieldname):
-	frappe.throw(_("Field not permitted in query") + ": {0}".format(fieldname), frappe.DataError)
+	frappe.throw(_("Field not permitted in query") + f": {fieldname}", frappe.DataError)
 
 
 def is_standard(fieldname):
 	if "." in fieldname:
-		parenttype, fieldname = get_parenttype_and_fieldname(fieldname, None)
+		fieldname = fieldname.split(".")[1].strip("`")
 	return (
 		fieldname in default_fields or fieldname in optional_fields or fieldname in child_table_fields
 	)
@@ -224,7 +225,7 @@ def parse_json(data):
 	if isinstance(data.get("or_filters"), str):
 		data["or_filters"] = json.loads(data["or_filters"])
 	if isinstance(data.get("fields"), str):
-		data["fields"] = json.loads(data["fields"])
+		data["fields"] = ["*"] if data["fields"] == "*" else json.loads(data["fields"])
 	if isinstance(data.get("docstatus"), str):
 		data["docstatus"] = json.loads(data["docstatus"])
 	if isinstance(data.get("save_user_settings"), str):
@@ -235,7 +236,16 @@ def parse_json(data):
 
 def get_parenttype_and_fieldname(field, data):
 	if "." in field:
-		parenttype, fieldname = field.split(".")[0][4:-1], field.split(".")[1].strip("`")
+		parts = field.split(".")
+		parenttype = parts[0]
+		fieldname = parts[1]
+		if parenttype.startswith("`tab"):
+			# `tabChild DocType`.`fieldname`
+			parenttype = parenttype[4:-1]
+			fieldname = fieldname.strip("`")
+		else:
+			# tablefield.fieldname
+			parenttype = frappe.get_meta(data.doctype).get_field(parenttype).options
 	else:
 		parenttype = data.doctype
 		fieldname = field.strip("`")
@@ -262,7 +272,7 @@ def compress(data, args=None):
 		values.append(new_row)
 
 		# add user info for assignments (avatar)
-		if row._assign:
+		if row.get("_assign", ""):
 			for user in json.loads(row._assign):
 				add_user_info(user, user_info)
 
@@ -518,7 +528,7 @@ def get_sidebar_stats(stats, doctype, filters=None):
 	if is_virtual_doctype(doctype):
 		controller = get_controller(doctype)
 		args = {"stats": stats, "filters": filters}
-		data = controller(doctype).get_stats(args)
+		data = controller.get_stats(args)
 	else:
 		data = get_stats(stats, doctype, filters)
 
@@ -674,8 +684,7 @@ def build_match_conditions(doctype, user=None, as_condition=True):
 	)
 	if as_condition:
 		return match_conditions.replace("%", "%%")
-	else:
-		return match_conditions
+	return match_conditions
 
 
 def get_filters_cond(
@@ -722,7 +731,3 @@ def get_filters_cond(
 	else:
 		cond = ""
 	return cond
-
-
-def is_virtual_doctype(doctype):
-	return frappe.db.get_value("DocType", doctype, "is_virtual")
diff --git a/frappe/desk/search.py b/frappe/desk/search.py
index ba4c5fb4fb..8ae635093c 100644
--- a/frappe/desk/search.py
+++ b/frappe/desk/search.py
@@ -1,12 +1,10 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
 
+import functools
 import json
 import re
 
-import wrapt
-
-# Search
 import frappe
 from frappe import _, is_whitelisted
 from frappe.permissions import has_permission
@@ -36,7 +34,7 @@ def sanitize_searchfield(searchfield):
 			_raise_exception(searchfield)
 
 		# to avoid and, or and like
-		elif any(" {0} ".format(keyword) in searchfield.split() for keyword in blacklisted_keywords):
+		elif any(f" {keyword} " in searchfield.split() for keyword in blacklisted_keywords):
 			_raise_exception(searchfield)
 
 		# to avoid select, delete, drop, update and case
@@ -168,7 +166,7 @@ def search_widget(
 							in ["Data", "Text", "Small Text", "Long Text", "Link", "Select", "Read Only", "Text Editor"]
 						)
 					):
-						or_filters.append([doctype, f.strip(), "like", "%{0}%".format(txt)])
+						or_filters.append([doctype, f.strip(), "like", f"%{txt}%"])
 
 			if meta.get("fields", {"fieldname": "enabled", "fieldtype": "Check"}):
 				filters.append([doctype, "enabled", "=", 1])
@@ -179,7 +177,7 @@ def search_widget(
 			fields = get_std_fields_list(meta, searchfield or "name")
 			if filter_fields:
 				fields = list(set(fields + json.loads(filter_fields)))
-			formatted_fields = ["`tab%s`.`%s`" % (meta.name, f.strip()) for f in fields]
+			formatted_fields = [f"`tab{meta.name}`.`{f.strip()}`" for f in fields]
 
 			title_field_query = get_title_field_query(meta)
 
@@ -199,7 +197,7 @@ def search_widget(
 
 			order_by_based_on_meta = get_order_by(doctype, meta)
 			# 2 is the index of _relevance column
-			order_by = "_relevance, {0}, `tab{1}`.idx desc".format(order_by_based_on_meta, doctype)
+			order_by = f"_relevance, {order_by_based_on_meta}, `tab{doctype}`.idx desc"
 
 			ptype = "select" if frappe.only_has_select_perm(doctype) else "read"
 			ignore_permissions = (
@@ -272,7 +270,7 @@ def get_title_field_query(meta):
 	field = None
 
 	if title_field and show_title_field_in_link:
-		field = "`tab{0}`.{1} as `label`".format(meta.name, title_field)
+		field = f"`tab{meta.name}`.{title_field} as `label`"
 
 	return field
 
@@ -314,17 +312,20 @@ def relevance_sorter(key, query, as_dict):
 	return (cstr(value).lower().startswith(query.lower()) is not True, value)
 
 
-@wrapt.decorator
-def validate_and_sanitize_search_inputs(fn, instance, args, kwargs):
-	kwargs.update(dict(zip(fn.__code__.co_varnames, args)))
-	sanitize_searchfield(kwargs["searchfield"])
-	kwargs["start"] = cint(kwargs["start"])
-	kwargs["page_len"] = cint(kwargs["page_len"])
+def validate_and_sanitize_search_inputs(fn):
+	@functools.wraps(fn)
+	def wrapper(*args, **kwargs):
+		kwargs.update(dict(zip(fn.__code__.co_varnames, args)))
+		sanitize_searchfield(kwargs["searchfield"])
+		kwargs["start"] = cint(kwargs["start"])
+		kwargs["page_len"] = cint(kwargs["page_len"])
 
-	if kwargs["doctype"] and not frappe.db.exists("DocType", kwargs["doctype"]):
-		return []
+		if kwargs["doctype"] and not frappe.db.exists("DocType", kwargs["doctype"]):
+			return []
 
-	return fn(**kwargs)
+		return fn(**kwargs)
+
+	return wrapper
 
 
 @frappe.whitelist()
diff --git a/frappe/desk/treeview.py b/frappe/desk/treeview.py
index d7a8db8792..f8b2a67c82 100644
--- a/frappe/desk/treeview.py
+++ b/frappe/desk/treeview.py
@@ -43,7 +43,7 @@ def get_children(doctype, parent="", **filters):
 
 def _get_children(doctype, parent="", ignore_permissions=False):
 	parent_field = "parent_" + doctype.lower().replace(" ", "_")
-	filters = [["ifnull(`{0}`,'')".format(parent_field), "=", parent], ["docstatus", "<", 2]]
+	filters = [[f"ifnull(`{parent_field}`,'')", "=", parent], ["docstatus", "<", 2]]
 
 	meta = frappe.get_meta(doctype)
 
@@ -51,7 +51,7 @@ def _get_children(doctype, parent="", ignore_permissions=False):
 		doctype,
 		fields=[
 			"name as value",
-			"{0} as title".format(meta.get("title_field") or "name"),
+			"{} as title".format(meta.get("title_field") or "name"),
 			"is_group as expandable",
 		],
 		filters=filters,
diff --git a/frappe/email/__init__.py b/frappe/email/__init__.py
index fae60baebf..1f6af4a3e7 100644
--- a/frappe/email/__init__.py
+++ b/frappe/email/__init__.py
@@ -10,31 +10,26 @@ def sendmail_to_system_managers(subject, content):
 
 
 @frappe.whitelist()
-def get_contact_list(txt, page_length=20):
+def get_contact_list(txt, page_length=20) -> list[dict]:
 	"""Returns contacts (from autosuggest)"""
 
-	cached_contacts = get_cached_contacts(txt)
-	if cached_contacts:
+	if cached_contacts := get_cached_contacts(txt):
 		return cached_contacts[:page_length]
 
-	try:
-		match_conditions = build_match_conditions("Contact")
-		match_conditions = "and {0}".format(match_conditions) if match_conditions else ""
+	reportview_conditions = build_match_conditions("Contact")
+	match_conditions = f"and {reportview_conditions}" if reportview_conditions else ""
 
-		out = frappe.db.sql(
-			"""select email_id as value,
-			concat(first_name, ifnull(concat(' ',last_name), '' )) as description
-			from tabContact
-			where name like %(txt)s or email_id like %(txt)s
-			%(condition)s
-			limit %(page_length)s""",
-			{"txt": "%" + txt + "%", "condition": match_conditions, "page_length": page_length},
-			as_dict=True,
-		)
-		out = filter(None, out)
-
-	except:
-		raise
+	out = frappe.db.sql(
+		f"""select email_id as value,
+		concat(first_name, ifnull(concat(' ',last_name), '' )) as description
+		from tabContact
+		where name like %(txt)s or email_id like %(txt)s
+		{match_conditions}
+		limit %(page_length)s""",
+		{"txt": f"%{txt}%", "page_length": page_length},
+		as_dict=True,
+	)
+	out = list(filter(None, out))
 
 	update_contact_cache(out)
 
diff --git a/frappe/email/doctype/auto_email_report/auto_email_report.js b/frappe/email/doctype/auto_email_report/auto_email_report.js
index 3423c3ccba..6930078512 100644
--- a/frappe/email/doctype/auto_email_report/auto_email_report.js
+++ b/frappe/email/doctype/auto_email_report/auto_email_report.js
@@ -1,139 +1,156 @@
 // Copyright (c) 2016, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Auto Email Report', {
-	refresh: function(frm) {
-		frm.trigger('fetch_report_filters');
-		if(!frm.is_new()) {
-			frm.add_custom_button(__('Download'), function() {
+frappe.ui.form.on("Auto Email Report", {
+	refresh: function (frm) {
+		frm.trigger("fetch_report_filters");
+		if (!frm.is_new()) {
+			frm.add_custom_button(__("Download"), function () {
 				var w = window.open(
 					frappe.urllib.get_full_url(
-						"/api/method/frappe.email.doctype.auto_email_report.auto_email_report.download?"
-						+"name="+encodeURIComponent(frm.doc.name)));
-				if(!w) {
-					frappe.msgprint(__("Please enable pop-ups")); return;
+						"/api/method/frappe.email.doctype.auto_email_report.auto_email_report.download?" +
+							"name=" +
+							encodeURIComponent(frm.doc.name)
+					)
+				);
+				if (!w) {
+					frappe.msgprint(__("Please enable pop-ups"));
+					return;
 				}
 			});
-			frm.add_custom_button(__('Send Now'), function() {
+			frm.add_custom_button(__("Send Now"), function () {
 				frappe.call({
-					method: 'frappe.email.doctype.auto_email_report.auto_email_report.send_now',
-					args: {name: frm.doc.name},
-					callback: function() {
-						frappe.msgprint(__('Scheduled to send'));
-					}
+					method: "frappe.email.doctype.auto_email_report.auto_email_report.send_now",
+					args: { name: frm.doc.name },
+					callback: function () {
+						frappe.msgprint(__("Scheduled to send"));
+					},
 				});
 			});
 		} else {
-			if(!frm.doc.user) {
-				frm.set_value('user', frappe.session.user);
+			if (!frm.doc.user) {
+				frm.set_value("user", frappe.session.user);
 			}
-			if(!frm.doc.email_to) {
-				frm.set_value('email_to', frappe.session.user);
+			if (!frm.doc.email_to) {
+				frm.set_value("email_to", frappe.session.user);
 			}
 		}
 	},
-	report: function(frm) {
-		frm.set_value('filters', '');
-		frm.trigger('fetch_report_filters');
+	report: function (frm) {
+		frm.set_value("filters", "");
+		frm.trigger("fetch_report_filters");
 	},
 	fetch_report_filters(frm) {
-		if (frm.doc.report
-			&& frm.doc.report_type !== 'Report Builder'
-			&& frm.script_setup_for !== frm.doc.report
+		if (
+			frm.doc.report &&
+			frm.doc.report_type !== "Report Builder" &&
+			frm.script_setup_for !== frm.doc.report
 		) {
 			frappe.call({
 				method: "frappe.desk.query_report.get_script",
 				args: {
-					report_name: frm.doc.report
+					report_name: frm.doc.report,
 				},
-				callback: function(r) {
+				callback: function (r) {
 					frappe.dom.eval(r.message.script || "");
 					frm.script_setup_for = frm.doc.report;
-					frm.trigger('show_filters');
-				}
+					frm.trigger("show_filters");
+				},
 			});
 		} else {
-			frm.trigger('show_filters');
+			frm.trigger("show_filters");
 		}
 	},
-	show_filters: function(frm) {
-		var wrapper = $(frm.get_field('filters_display').wrapper);
+	show_filters: function (frm) {
+		var wrapper = $(frm.get_field("filters_display").wrapper);
 		wrapper.empty();
-		if(frm.doc.report_type === 'Custom Report' || (frm.doc.report_type !== 'Report Builder'
-			&& frappe.query_reports[frm.doc.report]
-			&& frappe.query_reports[frm.doc.report].filters)) {
-
+		if (
+			frm.doc.report_type === "Custom Report" ||
+			(frm.doc.report_type !== "Report Builder" &&
+				frappe.query_reports[frm.doc.report] &&
+				frappe.query_reports[frm.doc.report].filters)
+		) {
 			// make a table to show filters
-			var table = $('\
-				\
-				
        '+__('Filter')+''+__('Value')+'
        ').appendTo(wrapper);
-			$('
        ' + __("Click table to edit") + '
        ').appendTo(wrapper);
+			var table = $(
+				'\
+				\
+				
        ' +
+					__("Filter") +
+					"" +
+					__("Value") +
+					"
        "
+			).appendTo(wrapper);
+			$('
        ' + __("Click table to edit") + "
        ").appendTo(
+				wrapper
+			);
 
-			var filters = JSON.parse(frm.doc.filters || '{}');
+			var filters = JSON.parse(frm.doc.filters || "{}");
 
 			let report_filters;
 
-			if (frm.doc.report_type === 'Custom Report'
-				&& frappe.query_reports[frm.doc.reference_report]
-				&& frappe.query_reports[frm.doc.reference_report].filters) {
+			if (
+				frm.doc.report_type === "Custom Report" &&
+				frappe.query_reports[frm.doc.reference_report] &&
+				frappe.query_reports[frm.doc.reference_report].filters
+			) {
 				report_filters = frappe.query_reports[frm.doc.reference_report].filters;
 			} else {
 				report_filters = frappe.query_reports[frm.doc.report].filters;
 			}
 
-			if(report_filters && report_filters.length > 0) {
-				frm.set_value('filter_meta', JSON.stringify(report_filters));
+			if (report_filters && report_filters.length > 0) {
+				frm.set_value("filter_meta", JSON.stringify(report_filters));
 				if (frm.is_dirty()) {
 					frm.save();
 				}
 			}
 
-			var report_filters_list = []
-			$.each(report_filters, function(key, val){
+			var report_filters_list = [];
+			$.each(report_filters, function (key, val) {
 				// Remove break fieldtype from the filters
-				if(val.fieldtype != 'Break') {
-					report_filters_list.push(val)
+				if (val.fieldtype != "Break") {
+					report_filters_list.push(val);
 				}
-			})
+			});
 			report_filters = report_filters_list;
 
 			const mandatory_css = {
 				"background-color": "var(--error-bg)",
-				"font-weight": "bold"
+				"font-weight": "bold",
 			};
 
-			report_filters.forEach(f => {
+			report_filters.forEach((f) => {
 				const css = f.reqd ? mandatory_css : {};
 				const row = $("").appendTo(table.find("tbody"));
 				$("" + f.label + "").appendTo(row);
-				$("" + frappe.format(filters[f.fieldname], f) +"")
+				$("" + frappe.format(filters[f.fieldname], f) + "")
 					.css(css)
 					.appendTo(row);
 			});
 
-			table.on('click', function() {
+			table.on("click", function () {
 				var dialog = new frappe.ui.Dialog({
 					fields: report_filters,
-					primary_action: function() {
+					primary_action: function () {
 						var values = this.get_values();
-						if(values) {
+						if (values) {
 							this.hide();
-							frm.set_value('filters', JSON.stringify(values));
-							frm.trigger('show_filters');
+							frm.set_value("filters", JSON.stringify(values));
+							frm.trigger("show_filters");
 						}
-					}
+					},
 				});
 				dialog.show();
 				dialog.set_values(filters);
-			})
+			});
 
 			// populate dynamic date field selection
 			let date_fields = report_filters
-				.filter(df => df.fieldtype === 'Date')
-				.map(df => ({ label: df.label, value: df.fieldname }));
-			frm.set_df_property('from_date_field', 'options', date_fields);
-			frm.set_df_property('to_date_field', 'options', date_fields);
-			frm.toggle_display('dynamic_report_filters_section', date_fields.length > 0);
+				.filter((df) => df.fieldtype === "Date")
+				.map((df) => ({ label: df.label, value: df.fieldname }));
+			frm.set_df_property("from_date_field", "options", date_fields);
+			frm.set_df_property("to_date_field", "options", date_fields);
+			frm.toggle_display("dynamic_report_filters_section", date_fields.length > 0);
 		}
-	}
+	},
 });
diff --git a/frappe/email/doctype/auto_email_report/auto_email_report.py b/frappe/email/doctype/auto_email_report/auto_email_report.py
index f4fdcf4275..b9b5e4e8d7 100644
--- a/frappe/email/doctype/auto_email_report/auto_email_report.py
+++ b/frappe/email/doctype/auto_email_report/auto_email_report.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2015, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
@@ -12,6 +11,7 @@ from frappe.model.document import Document
 from frappe.model.naming import append_number_if_name_exists
 from frappe.utils import (
 	add_to_date,
+	cint,
 	format_time,
 	get_link_to_form,
 	get_url_to_report,
@@ -51,14 +51,18 @@ class AutoEmailReport(Document):
 		self.email_to = "\n".join(valid)
 
 	def validate_report_count(self):
-		"""check that there are only 3 enabled reports per user"""
-		count = frappe.db.sql(
-			"select count(*) from `tabAuto Email Report` where user=%s and enabled=1", self.user
-		)[0][0]
-		max_reports_per_user = frappe.local.conf.max_reports_per_user or 3
+		count = frappe.db.count("Auto Email Report", {"user": self.user, "enabled": 1})
+
+		max_reports_per_user = (
+			cint(frappe.local.conf.max_reports_per_user)  # kept for backward compatibilty
+			or cint(frappe.db.get_single_value("System Settings", "max_auto_email_report_per_user"))
+			or 20
+		)
 
 		if count > max_reports_per_user + (-1 if self.flags.in_insert else 0):
-			frappe.throw(_("Only {0} emailed reports are allowed per user").format(max_reports_per_user))
+			msg = _("Only {0} emailed reports are allowed per user.").format(max_reports_per_user)
+			msg += " " + _("To allow more reports update limit in System Settings.")
+			frappe.throw(msg, title=_("Report limit reached"))
 
 	def validate_report_format(self):
 		"""check if user has select correct report format"""
@@ -160,7 +164,7 @@ class AutoEmailReport(Document):
 		)
 
 	def get_file_name(self):
-		return "{0}.{1}".format(self.report.replace(" ", "-").replace("/", "-"), self.format.lower())
+		return "{}.{}".format(self.report.replace(" ", "-").replace("/", "-"), self.format.lower())
 
 	def prepare_dynamic_filters(self):
 		self.filters = frappe.parse_json(self.filters)
@@ -255,7 +259,7 @@ def send_daily():
 		try:
 			auto_email_report.send()
 		except Exception as e:
-			frappe.log_error(e, _("Failed to send {0} Auto Email Report").format(auto_email_report.name))
+			auto_email_report.log_error(f"Failed to send {auto_email_report.name} Auto Email Report")
 
 
 def send_monthly():
diff --git a/frappe/email/doctype/auto_email_report/test_auto_email_report.py b/frappe/email/doctype/auto_email_report/test_auto_email_report.py
index aa8dffb9e0..ee0a363bd9 100644
--- a/frappe/email/doctype/auto_email_report/test_auto_email_report.py
+++ b/frappe/email/doctype/auto_email_report/test_auto_email_report.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2015, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
 import json
diff --git a/frappe/email/doctype/document_follow/document_follow.js b/frappe/email/doctype/document_follow/document_follow.js
index 3fdf055d36..59efb37341 100644
--- a/frappe/email/doctype/document_follow/document_follow.js
+++ b/frappe/email/doctype/document_follow/document_follow.js
@@ -1,6 +1,4 @@
 // Copyright (c) 2019, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Document Follow', {
-
-});
+frappe.ui.form.on("Document Follow", {});
diff --git a/frappe/email/doctype/document_follow/document_follow.py b/frappe/email/doctype/document_follow/document_follow.py
index 3b7d411fb5..6f9d63f3be 100644
--- a/frappe/email/doctype/document_follow/document_follow.py
+++ b/frappe/email/doctype/document_follow/document_follow.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2019, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
diff --git a/frappe/email/doctype/document_follow/test_document_follow.py b/frappe/email/doctype/document_follow/test_document_follow.py
index 1f31338351..159a399ee8 100644
--- a/frappe/email/doctype/document_follow/test_document_follow.py
+++ b/frappe/email/doctype/document_follow/test_document_follow.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2019, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
 import unittest
diff --git a/frappe/email/doctype/email_account/email_account.js b/frappe/email/doctype/email_account/email_account.js
index 54f0d2372d..3e42af7051 100644
--- a/frappe/email/doctype/email_account/email_account.js
+++ b/frappe/email/doctype/email_account/email_account.js
@@ -1,176 +1,246 @@
 frappe.email_defaults = {
-	"GMail": {
-		"email_server": "imap.gmail.com",
-		"use_ssl": 1,
-		"enable_outgoing": 1,
-		"smtp_server": "smtp.gmail.com",
-		"smtp_port": 587,
-		"use_tls": 1,
-		"use_imap": 1
+	GMail: {
+		email_server: "imap.gmail.com",
+		incoming_port: 993,
+		use_ssl: 1,
+		enable_outgoing: 1,
+		smtp_server: "smtp.gmail.com",
+		smtp_port: 587,
+		use_tls: 1,
+		use_imap: 1,
 	},
 	"Outlook.com": {
-		"email_server": "imap-mail.outlook.com",
-		"use_ssl": 1,
-		"enable_outgoing": 1,
-		"smtp_server": "smtp-mail.outlook.com",
-		"smtp_port": 587,
-		"use_tls": 1,
-		"use_imap": 1
+		email_server: "imap-mail.outlook.com",
+		use_ssl: 1,
+		enable_outgoing: 1,
+		smtp_server: "smtp-mail.outlook.com",
+		smtp_port: 587,
+		use_tls: 1,
+		use_imap: 1,
 	},
-	"Sendgrid": {
-		"enable_outgoing": 1,
-		"smtp_server": "smtp.sendgrid.net",
-		"smtp_port": 587,
-		"use_tls": 1,
+	Sendgrid: {
+		enable_outgoing: 1,
+		smtp_server: "smtp.sendgrid.net",
+		smtp_port: 587,
+		use_tls: 1,
 	},
-	"SparkPost": {
-		"enable_incoming": 0,
-		"enable_outgoing": 1,
-		"smtp_server": "smtp.sparkpostmail.com",
-		"smtp_port": 587,
-		"use_tls": 1
+	SparkPost: {
+		enable_incoming: 0,
+		enable_outgoing: 1,
+		smtp_server: "smtp.sparkpostmail.com",
+		smtp_port: 587,
+		use_tls: 1,
 	},
 	"Yahoo Mail": {
-		"email_server": "imap.mail.yahoo.com",
-		"use_ssl": 1,
-		"enable_outgoing": 1,
-		"smtp_server": "smtp.mail.yahoo.com",
-		"smtp_port": 587,
-		"use_tls": 1,
-		"use_imap": 1
+		email_server: "imap.mail.yahoo.com",
+		use_ssl: 1,
+		enable_outgoing: 1,
+		smtp_server: "smtp.mail.yahoo.com",
+		smtp_port: 587,
+		use_tls: 1,
+		use_imap: 1,
 	},
 	"Yandex.Mail": {
-		"email_server": "imap.yandex.com",
-		"use_ssl": 1,
-		"enable_outgoing": 1,
-		"smtp_server": "smtp.yandex.com",
-		"smtp_port": 587,
-		"use_tls": 1,
-		"use_imap": 1
+		email_server: "imap.yandex.com",
+		use_ssl: 1,
+		enable_outgoing: 1,
+		smtp_server: "smtp.yandex.com",
+		smtp_port: 587,
+		use_tls: 1,
+		use_imap: 1,
 	},
 };
 
 frappe.email_defaults_pop = {
-	"GMail": {
-		"email_server": "pop.gmail.com"
+	GMail: {
+		email_server: "pop.gmail.com",
 	},
 	"Outlook.com": {
-		"email_server": "pop3-mail.outlook.com"
+		email_server: "pop3-mail.outlook.com",
 	},
 	"Yahoo Mail": {
-		"email_server": "pop.mail.yahoo.com"
+		email_server: "pop.mail.yahoo.com",
 	},
 	"Yandex.Mail": {
-		"email_server": "pop.yandex.com"
+		email_server: "pop.yandex.com",
 	},
-
 };
 
+function oauth_access(frm) {
+	return frappe.call({
+		method: "frappe.email.oauth.oauth_access",
+		args: {
+			email_account: frm.doc.name,
+			service: frm.doc.service || "",
+		},
+		callback: function (r) {
+			if (!r.exc) {
+				window.open(r.message.url, "_self");
+			}
+		},
+	});
+}
+
+function set_default_max_attachment_size(frm, field) {
+	if (frm.doc.__islocal && !frm.doc[field]) {
+		frappe.call({
+			method: "frappe.core.api.file.get_max_file_size",
+			callback: function (r) {
+				if (!r.exc) {
+					frm.set_value(field, Number(r.message) / (1024 * 1024));
+				}
+			},
+		});
+	}
+}
+
 frappe.ui.form.on("Email Account", {
-	service: function(frm) {
-		$.each(frappe.email_defaults[frm.doc.service], function(key, value) {
+	service: function (frm) {
+		$.each(frappe.email_defaults[frm.doc.service], function (key, value) {
 			frm.set_value(key, value);
 		});
 		if (!frm.doc.use_imap) {
-			$.each(frappe.email_defaults_pop[frm.doc.service], function(key, value) {
+			$.each(frappe.email_defaults_pop[frm.doc.service], function (key, value) {
 				frm.set_value(key, value);
 			});
 		}
 		frm.events.show_gmail_message_for_less_secure_apps(frm);
+		frm.events.toggle_auth_method(frm);
 	},
 
-	use_imap: function(frm) {
+	use_imap: function (frm) {
 		if (!frm.doc.use_imap) {
-			$.each(frappe.email_defaults_pop[frm.doc.service], function(key, value) {
+			$.each(frappe.email_defaults_pop[frm.doc.service], function (key, value) {
 				frm.set_value(key, value);
 			});
-		}
-		else{
-			$.each(frappe.email_defaults[frm.doc.service], function(key, value) {
+		} else {
+			$.each(frappe.email_defaults[frm.doc.service], function (key, value) {
 				frm.set_value(key, value);
 			});
 		}
 	},
 
-	enable_incoming: function(frm) {
-		frm.doc.no_remaining = null; //perform full sync
-		//frm.set_df_property("append_to", "reqd", frm.doc.enable_incoming);
+	enable_incoming: function (frm) {
 		frm.trigger("warn_autoreply_on_incoming");
 	},
 
-	enable_auto_reply: function(frm) {
+	enable_auto_reply: function (frm) {
 		frm.trigger("warn_autoreply_on_incoming");
 	},
 
-	notify_if_unreplied: function(frm) {
+	notify_if_unreplied: function (frm) {
 		frm.set_df_property("send_notification_to", "reqd", frm.doc.notify_if_unreplied);
 	},
 
-	onload: function(frm) {
+	onload: function (frm) {
+		if (frappe.utils.get_query_params().successful_authorization === "1") {
+			frappe.show_alert(__("Successfully Authorized"));
+			// FIXME: find better alternative
+			window.history.replaceState(null, "", window.location.pathname);
+		}
+
 		frm.set_df_property("append_to", "only_select", true);
-		frm.set_query("append_to", "frappe.email.doctype.email_account.email_account.get_append_to");
-		frm.set_query("append_to", "imap_folder", function() {
+		frm.set_query(
+			"append_to",
+			"frappe.email.doctype.email_account.email_account.get_append_to"
+		);
+		frm.set_query("append_to", "imap_folder", function () {
 			return {
-				query: "frappe.email.doctype.email_account.email_account.get_append_to"
+				query: "frappe.email.doctype.email_account.email_account.get_append_to",
 			};
 		});
 		if (frm.doc.__islocal) {
-			frm.add_child("imap_folder", {"folder_name": "INBOX"});
+			frm.add_child("imap_folder", { folder_name: "INBOX" });
 			frm.refresh_field("imap_folder");
 		}
+		frm.toggle_display(["auth_method"], frm.doc.service === "GMail");
+		set_default_max_attachment_size(frm, "attachment_limit");
 	},
 
-	refresh: function(frm) {
+	refresh: function (frm) {
 		frm.events.set_domain_fields(frm);
 		frm.events.enable_incoming(frm);
 		frm.events.notify_if_unreplied(frm);
 		frm.events.show_gmail_message_for_less_secure_apps(frm);
+		frm.events.show_oauth_authorization_message(frm);
 
 		if (frappe.route_flags.delete_user_from_locals && frappe.route_flags.linked_user) {
 			delete frappe.route_flags.delete_user_from_locals;
-			delete locals['User'][frappe.route_flags.linked_user];
+			delete locals["User"][frappe.route_flags.linked_user];
 		}
 	},
 
-	show_gmail_message_for_less_secure_apps: function(frm) {
+	after_save(frm) {
+		if (frm.doc.auth_method === "OAuth" && !frm.doc.refresh_token) {
+			oauth_access(frm);
+		}
+	},
+
+	toggle_auth_method: function (frm) {
+		if (frm.doc.service !== "GMail") {
+			frm.toggle_display(["auth_method"], false);
+			frm.doc.auth_method = "Basic";
+		} else {
+			frm.toggle_display(["auth_method"], true);
+		}
+	},
+
+	show_gmail_message_for_less_secure_apps: function (frm) {
 		frm.dashboard.clear_headline();
-		if (frm.doc.service==="GMail") {
-			frm.dashboard.set_headline_alert('Gmail will only work if you allow access for less secure \
-				apps in Gmail settings. Read this for details');
+		let msg = __(
+			"GMail will only work if you enable 2-step authentication and use app-specific password OR use OAuth."
+		);
+		let cta = __("Read the step by step guide here.");
+		msg += ` ${cta}`;
+		if (frm.doc.service === "GMail") {
+			frm.dashboard.set_headline_alert(msg);
 		}
 	},
 
-	email_id:function(frm) {
+	show_oauth_authorization_message(frm) {
+		if (frm.doc.auth_method === "OAuth" && !frm.doc.refresh_token) {
+			let msg = __(
+				'OAuth has been enabled but not authorised. Please use "Authorise API Access" button to do the same.'
+			);
+			frm.dashboard.clear_headline();
+			frm.dashboard.set_headline_alert(msg, "yellow");
+		}
+	},
+
+	authorize_api_access: function (frm) {
+		oauth_access(frm);
+	},
+
+	email_id: function (frm) {
 		//pull domain and if no matching domain go create one
 		frm.events.update_domain(frm);
 	},
 
-	update_domain: function(frm) {
+	update_domain: function (frm) {
 		if (!frm.doc.email_id && !frm.doc.service) {
 			return;
 		}
 
 		frappe.call({
-			method: 'get_domain',
+			method: "get_domain",
 			doc: frm.doc,
 			args: {
-				"email_id": frm.doc.email_id
+				email_id: frm.doc.email_id,
 			},
-			callback: function(r) {
+			callback: function (r) {
 				if (r.message) {
 					frm.events.set_domain_fields(frm, r.message);
 				}
-			}
+			},
 		});
 	},
 
-	set_domain_fields: function(frm, args) {
+	set_domain_fields: function (frm, args) {
 		if (!args) {
-			args = frappe.route_flags.set_domain_values? frappe.route_options: {};
+			args = frappe.route_flags.set_domain_values ? frappe.route_options : {};
 		}
 
-		for(var field in args) {
+		for (var field in args) {
 			frm.set_value(field, args[field]);
 		}
 
@@ -178,24 +248,28 @@ frappe.ui.form.on("Email Account", {
 		frappe.route_options = {};
 	},
 
-	email_sync_option: function(frm) {
+	email_sync_option: function (frm) {
 		// confirm if the ALL sync option is selected
 
 		if (frm.doc.email_sync_option == "ALL") {
-			var msg = __("You are selecting Sync Option as ALL, It will resync all read as well as unread message from server. This may also cause the duplication of Communication (emails).");
-			frappe.confirm(msg, null, function() {
+			var msg = __(
+				"You are selecting Sync Option as ALL, It will resync all read as well as unread message from server. This may also cause the duplication of Communication (emails)."
+			);
+			frappe.confirm(msg, null, function () {
 				frm.set_value("email_sync_option", "UNSEEN");
 			});
 		}
 	},
 
-	warn_autoreply_on_incoming: function(frm) {
+	warn_autoreply_on_incoming: function (frm) {
 		if (frm.doc.enable_incoming && frm.doc.enable_auto_reply && frm.doc.__islocal) {
-			var msg = __("Enabling auto reply on an incoming email account will send automated replies to all the synchronized emails. Do you wish to continue?");
-			frappe.confirm(msg, null, function() {
+			var msg = __(
+				"Enabling auto reply on an incoming email account will send automated replies to all the synchronized emails. Do you wish to continue?"
+			);
+			frappe.confirm(msg, null, function () {
 				frm.set_value("enable_auto_reply", 0);
-				frappe.show_alert({message: __("Disabled Auto Reply"), indicator: "blue"});
+				frappe.show_alert({ message: __("Disabled Auto Reply"), indicator: "blue" });
 			});
 		}
-	}
+	},
 });
diff --git a/frappe/email/doctype/email_account/email_account.json b/frappe/email/doctype/email_account/email_account.json
index 65053bab3d..740f6039ed 100644
--- a/frappe/email/doctype/email_account/email_account.json
+++ b/frappe/email/doctype/email_account/email_account.json
@@ -14,10 +14,14 @@
   "domain",
   "service",
   "authentication_column",
+  "auth_method",
+  "authorize_api_access",
   "password",
   "awaiting_password",
   "ascii_encode_password",
   "column_break_10",
+  "refresh_token",
+  "access_token",
   "login_id_is_different",
   "login_id",
   "mailbox_settings",
@@ -44,9 +48,9 @@
   "send_notification_to",
   "outgoing_mail_settings",
   "enable_outgoing",
-  "smtp_server",
   "use_tls",
   "use_ssl_for_outgoing",
+  "smtp_server",
   "smtp_port",
   "column_break_38",
   "default_outgoing",
@@ -66,7 +70,6 @@
   "brand_logo",
   "uidvalidity",
   "uidnext",
-  "no_remaining",
   "no_failed"
  ],
  "fields": [
@@ -79,7 +82,8 @@
    "in_list_view": 1,
    "label": "Email Address",
    "options": "Email",
-   "reqd": 1
+   "reqd": 1,
+   "unique": 1
   },
   {
    "default": "0",
@@ -87,7 +91,7 @@
    "fieldtype": "Check",
    "hide_days": 1,
    "hide_seconds": 1,
-   "label": "Use different login"
+   "label": "Use different Email ID"
   },
   {
    "depends_on": "login_id_is_different",
@@ -95,9 +99,10 @@
    "fieldtype": "Data",
    "hide_days": 1,
    "hide_seconds": 1,
-   "label": "Email Login ID"
+   "label": "Alternative Email ID"
   },
   {
+   "depends_on": "eval: doc.auth_method === \"Basic\"",
    "fieldname": "password",
    "fieldtype": "Password",
    "hide_days": 1,
@@ -106,6 +111,7 @@
   },
   {
    "default": "0",
+   "depends_on": "eval: doc.auth_method === \"Basic\"",
    "fieldname": "awaiting_password",
    "fieldtype": "Check",
    "hide_days": 1,
@@ -114,6 +120,7 @@
   },
   {
    "default": "0",
+   "depends_on": "eval: doc.auth_method === \"Basic\"",
    "fieldname": "ascii_encode_password",
    "fieldtype": "Check",
    "hide_days": 1,
@@ -182,7 +189,7 @@
    "fieldtype": "Data",
    "hide_days": 1,
    "hide_seconds": 1,
-   "label": "Email Server"
+   "label": "Incoming Server"
   },
   {
    "default": "0",
@@ -207,7 +214,7 @@
   },
   {
    "depends_on": "eval: doc.enable_incoming && !doc.use_imap",
-   "description": "Append as communication against this DocType (must have fields, \"Status\", \"Subject\")",
+   "description": "Append as communication against this DocType (must have field \"Status\" and both \"Sender\" and \"Subject\" defined in the related doctype Email Settings)",
    "fieldname": "append_to",
    "fieldtype": "Link",
    "hide_days": 1,
@@ -304,7 +311,7 @@
    "fieldtype": "Data",
    "hide_days": 1,
    "hide_seconds": 1,
-   "label": "SMTP Server"
+   "label": "Outgoing Server"
   },
   {
    "default": "0",
@@ -464,15 +471,6 @@
    "label": "UIDNEXT",
    "no_copy": 1
   },
-  {
-   "fieldname": "no_remaining",
-   "fieldtype": "Data",
-   "hidden": 1,
-   "hide_days": 1,
-   "hide_seconds": 1,
-   "label": "No of emails remaining to be synced",
-   "no_copy": 1
-  },
   {
    "fieldname": "no_failed",
    "fieldtype": "Int",
@@ -524,7 +522,7 @@
    "fieldtype": "Check",
    "hide_days": 1,
    "hide_seconds": 1,
-   "label": "Use SSL for Outgoing"
+   "label": "Use SSL"
   },
   {
    "default": "1",
@@ -576,12 +574,39 @@
    "fieldname": "section_break_25",
    "fieldtype": "Section Break",
    "label": "IMAP Details"
+  },
+  {
+   "depends_on": "eval: doc.service === \"GMail\" && doc.auth_method === \"OAuth\" && !doc.__islocal && !doc.__unsaved",
+   "fieldname": "authorize_api_access",
+   "fieldtype": "Button",
+   "label": "Authorize API Access"
+  },
+  {
+   "fieldname": "refresh_token",
+   "fieldtype": "Small Text",
+   "hidden": 1,
+   "label": "Refresh Token",
+   "read_only": 1
+  },
+  {
+   "fieldname": "access_token",
+   "fieldtype": "Small Text",
+   "hidden": 1,
+   "label": "Access Token",
+   "read_only": 1
+  },
+  {
+   "default": "Basic",
+   "fieldname": "auth_method",
+   "fieldtype": "Select",
+   "label": "Method",
+   "options": "Basic\nOAuth"
   }
  ],
  "icon": "fa fa-inbox",
  "index_web_pages_for_search": 1,
  "links": [],
- "modified": "2021-11-30 09:03:25.728637",
+ "modified": "2022-07-13 13:05:45.445572",
  "modified_by": "Administrator",
  "module": "Email",
  "name": "Email Account",
@@ -603,5 +628,6 @@
  ],
  "sort_field": "modified",
  "sort_order": "DESC",
+ "states": [],
  "track_changes": 1
 }
diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py
index e60be0d965..c1ed170b6a 100755
--- a/frappe/email/doctype/email_account/email_account.py
+++ b/frappe/email/doctype/email_account/email_account.py
@@ -8,7 +8,6 @@ import socket
 import time
 from datetime import datetime, timedelta
 from poplib import error_proto
-from typing import List
 
 import frappe
 from frappe import _, are_emails_muted, safe_encode
@@ -21,12 +20,9 @@ from frappe.utils import cint, comma_or, cstr, parse_addr, validate_email_addres
 from frappe.utils.background_jobs import enqueue, get_jobs
 from frappe.utils.error import raise_error_on_no_output
 from frappe.utils.jinja import render_template
+from frappe.utils.password import decrypt, encrypt
 from frappe.utils.user import get_system_managers
 
-OUTGOING_EMAIL_ACCOUNT_MISSING = _(
-	"Please setup default Email Account from Setup > Email > Email Account"
-)
-
 
 class SentEmailInInbox(Exception):
 	pass
@@ -68,6 +64,7 @@ class EmailAccount(Document):
 
 	def validate(self):
 		"""Validate Email Address and check POP3/IMAP and SMTP connections is enabled."""
+
 		if self.email_id:
 			validate_email_address(self.email_id, True)
 
@@ -81,25 +78,26 @@ class EmailAccount(Document):
 		if self.enable_incoming and self.use_imap and len(self.imap_folder) <= 0:
 			frappe.throw(_("You need to set one IMAP folder for {0}").format(frappe.bold(self.email_id)))
 
-		duplicate_email_account = frappe.get_all(
-			"Email Account", filters={"email_id": self.email_id, "name": ("!=", self.name)}
-		)
-		if duplicate_email_account:
-			frappe.throw(
-				_("Email ID must be unique, Email Account already exists for {0}").format(
-					frappe.bold(self.email_id)
-				)
-			)
-
 		if frappe.local.flags.in_patch or frappe.local.flags.in_test:
 			return
 
-		if (
-			not self.awaiting_password
-			and not frappe.local.flags.in_install
-			and not frappe.local.flags.in_patch
-		):
-			if self.password or self.smtp_server in ("127.0.0.1", "localhost"):
+		use_oauth = self.auth_method == "OAuth"
+
+		if getattr(self, "service", "") != "GMail" and use_oauth:
+			self.auth_method = "Basic"
+			use_oauth = False
+
+		if use_oauth:
+			# no need for awaiting password for oauth
+			self.awaiting_password = 0
+			self.password = None
+
+		elif self.refresh_token:
+			# clear access & refresh token
+			self.refresh_token = self.access_token = None
+
+		if not frappe.local.flags.in_install and not self.awaiting_password:
+			if self.refresh_token or self.password or self.smtp_server in ("127.0.0.1", "localhost"):
 				if self.enable_incoming:
 					self.get_incoming_server()
 					self.no_failed = 0
@@ -108,7 +106,8 @@ class EmailAccount(Document):
 					self.validate_smtp_conn()
 			else:
 				if self.enable_incoming or (self.enable_outgoing and not self.no_smtp_authentication):
-					frappe.throw(_("Password is required or select Awaiting Password"))
+					if not use_oauth:
+						frappe.throw(_("Password is required or select Awaiting Password"))
 
 		if self.notify_if_unreplied:
 			if not self.send_notification_to:
@@ -116,11 +115,12 @@ class EmailAccount(Document):
 			for e in self.get_unreplied_notification_emails():
 				validate_email_address(e, True)
 
-		for folder in self.imap_folder:
-			if self.enable_incoming and folder.append_to:
-				valid_doctypes = [d[0] for d in get_append_to()]
-				if folder.append_to not in valid_doctypes:
-					frappe.throw(_("Append To can be one of {0}").format(comma_or(valid_doctypes)))
+		if self.enable_incoming:
+			for folder in self.imap_folder:
+				if folder.append_to:
+					valid_doctypes = [d[0] for d in get_append_to()]
+					if folder.append_to not in valid_doctypes:
+						frappe.throw(_("Append To can be one of {0}").format(comma_or(valid_doctypes)))
 
 	def validate_smtp_conn(self):
 		if not self.smtp_server:
@@ -160,6 +160,7 @@ class EmailAccount(Document):
 			awaiting_password=self.awaiting_password,
 			email_id=self.email_id,
 			enable_outgoing=self.enable_outgoing,
+			used_oauth=self.auth_method == "OAuth",
 		)
 
 	def there_must_be_only_one_default(self):
@@ -209,10 +210,14 @@ class EmailAccount(Document):
 				"host": self.email_server,
 				"use_ssl": self.use_ssl,
 				"username": getattr(self, "login_id", None) or self.email_id,
+				"service": getattr(self, "service", ""),
 				"use_imap": self.use_imap,
 				"email_sync_rule": email_sync_rule,
 				"incoming_port": get_port(self),
 				"initial_sync_count": self.initial_sync_count or 100,
+				"use_oauth": self.auth_method == "OAuth",
+				"refresh_token": decrypt(self.refresh_token) if self.refresh_token else None,
+				"access_token": decrypt(self.access_token) if self.access_token else None,
 			}
 		)
 
@@ -263,7 +268,7 @@ class EmailAccount(Document):
 			else:
 				frappe.throw(cstr(e))
 
-		except socket.error:
+		except OSError:
 			if in_receive:
 				# timeout while connecting, see receive.py connect method
 				description = frappe.message_log.pop() if frappe.message_log else "Socket Error"
@@ -279,7 +284,9 @@ class EmailAccount(Document):
 
 	@property
 	def _password(self):
-		raise_exception = not (self.no_smtp_authentication or frappe.flags.in_test)
+		raise_exception = not (
+			self.auth_method == "OAuth" or self.no_smtp_authentication or frappe.flags.in_test
+		)
 		return self.get_password(raise_exception=raise_exception)
 
 	@property
@@ -319,7 +326,7 @@ class EmailAccount(Document):
 	@classmethod
 	@raise_error_on_no_output(
 		keep_quiet=lambda: not cint(frappe.get_system_settings("setup_complete")),
-		error_message=OUTGOING_EMAIL_ACCOUNT_MISSING,
+		error_message=_("Please setup default Email Account from Setup > Email > Email Account"),
 		error_type=frappe.OutgoingEmailError,
 	)  # noqa
 	@cache_email_account("outgoing_email_account")
@@ -398,6 +405,9 @@ class EmailAccount(Document):
 				"default": 0,
 			},
 			"name": {"conf_names": ("email_sender_name",), "default": "Frappe"},
+			"auth_method": {"conf_names": ("auth_method"), "default": "Basic"},
+			"access_token": {"conf_names": ("mail_access_token")},
+			"refresh_token": {"conf_names": ("mail_refresh_token")},
 			"from_site_config": {"default": True},
 		}
 
@@ -405,17 +415,27 @@ class EmailAccount(Document):
 		for doc_field_name, d in field_to_conf_name_map.items():
 			conf_names, default = d.get("conf_names") or [], d.get("default")
 			value = [frappe.conf.get(k) for k in conf_names if frappe.conf.get(k)]
-			account_details[doc_field_name] = (value and value[0]) or default
+
+			if doc_field_name in ("refresh_token", "access_token"):
+				account_details[doc_field_name] = value and encrypt(value[0])
+			else:
+				account_details[doc_field_name] = (value and value[0]) or default
+
 		return account_details
 
 	def sendmail_config(self):
 		return {
+			"email_account": self.name,
 			"server": self.smtp_server,
 			"port": cint(self.smtp_port),
 			"login": getattr(self, "login_id", None) or self.email_id,
 			"password": self._password,
 			"use_ssl": cint(self.use_ssl_for_outgoing),
 			"use_tls": cint(self.use_tls),
+			"service": getattr(self, "service", ""),
+			"use_oauth": self.auth_method == "OAuth",
+			"refresh_token": decrypt(self.refresh_token) if self.refresh_token else None,
+			"access_token": decrypt(self.access_token) if self.access_token else None,
 		}
 
 	def get_smtp_server(self):
@@ -448,10 +468,10 @@ class EmailAccount(Document):
 			frappe.cache().set_value("workers:no-internet", True)
 
 	def set_failed_attempts_count(self, value):
-		frappe.cache().set("{0}:email-account-failed-attempts".format(self.name), value)
+		frappe.cache().set(f"{self.name}:email-account-failed-attempts", value)
 
 	def get_failed_attempts_count(self):
-		return cint(frappe.cache().get("{0}:email-account-failed-attempts".format(self.name)))
+		return cint(frappe.cache().get(f"{self.name}:email-account-failed-attempts"))
 
 	def receive(self):
 		"""Called by scheduler to receive emails from this EMail account using POP3/IMAP."""
@@ -473,7 +493,7 @@ class EmailAccount(Document):
 				frappe.db.rollback()
 			except Exception:
 				frappe.db.rollback()
-				frappe.log_error(title="EmailAccount.receive")
+				self.log_error(title="EmailAccount.receive")
 				if self.use_imap:
 					self.handle_bad_emails(mail.uid, mail.raw_message, frappe.get_traceback())
 				exceptions.append(frappe.get_traceback())
@@ -489,7 +509,7 @@ class EmailAccount(Document):
 		if exceptions:
 			raise Exception(frappe.as_json(exceptions))
 
-	def get_inbound_mails(self) -> List[InboundMail]:
+	def get_inbound_mails(self) -> list[InboundMail]:
 		"""retrive and return inbound mails."""
 		mails = []
 
@@ -499,7 +519,7 @@ class EmailAccount(Document):
 				seen_status = messages.get("seen_status", {}).get(uid)
 				if self.email_sync_option != "UNSEEN" or seen_status != "SEEN":
 					# only append the emails with status != 'SEEN' if sync option is set to 'UNSEEN'
-					mails.append(InboundMail(message, self, uid, seen_status, append_to))
+					mails.append(InboundMail(message, self, frappe.safe_decode(uid), seen_status, append_to))
 
 		if not self.enable_incoming:
 			return []
@@ -521,7 +541,7 @@ class EmailAccount(Document):
 			# close connection to mailserver
 			email_server.logout()
 		except Exception:
-			frappe.log_error(title=_("Error while connecting to email account {0}").format(self.name))
+			self.log_error(title=_("Error while connecting to email account {0}").format(self.name))
 			return []
 		return mails
 
@@ -599,7 +619,7 @@ class EmailAccount(Document):
 		if self.email_sync_option == "ALL":
 			max_uid = get_max_email_uid(self.name)
 			last_uid = max_uid + int(self.initial_sync_count or 100) if max_uid == 1 else "*"
-			return "UID {}:{}".format(max_uid, last_uid)
+			return f"UID {max_uid}:{last_uid}"
 		else:
 			return self.email_sync_option or "UNSEEN"
 
@@ -667,7 +687,7 @@ class EmailAccount(Document):
 		try:
 			email_server = self.get_incoming_server(in_receive=True)
 		except Exception:
-			frappe.log_error(title=_("Error while connecting to email account {0}").format(self.name))
+			self.log_error("Email Connection Error")
 
 		if not email_server:
 			return
@@ -679,7 +699,7 @@ class EmailAccount(Document):
 				message = safe_encode(message)
 				email_server.imap.append("Sent", "\\Seen", imaplib.Time2Internaldate(time.time()), message)
 			except Exception:
-				frappe.log_error(title="EmailAccount.append_email_to_sent_folder")
+				self.log_error("Unable to add to Sent folder")
 
 
 @frappe.whitelist()
@@ -776,22 +796,33 @@ def notify_unreplied():
 
 def pull(now=False):
 	"""Will be called via scheduler, pull emails from all enabled Email accounts."""
+
 	if frappe.cache().get_value("workers:no-internet") == True:
 		if test_internet():
 			frappe.cache().set_value("workers:no-internet", False)
 		else:
 			return
-	queued_jobs = get_jobs(site=frappe.local.site, key="job_name")[frappe.local.site]
-	for email_account in frappe.get_list(
-		"Email Account", filters={"enable_incoming": 1, "awaiting_password": 0}
-	):
+
+	doctype = frappe.qb.DocType("Email Account")
+	email_accounts = (
+		frappe.qb.from_(doctype)
+		.select(doctype.name)
+		.where(doctype.enable_incoming == 1)
+		.where(
+			(doctype.awaiting_password == 0)
+			| ((doctype.auth_method == "OAuth") & (doctype.refresh_token.isnotnull()))
+		)
+		.run(as_dict=1)
+	)
+	for email_account in email_accounts:
 		if now:
 			pull_from_email_account(email_account.name)
 
 		else:
 			# job_name is used to prevent duplicates in queue
-			job_name = "pull_from_email_account|{0}".format(email_account.name)
+			job_name = f"pull_from_email_account|{email_account.name}"
 
+			queued_jobs = get_jobs(site=frappe.local.site, key="job_name")[frappe.local.site]
 			if job_name not in queued_jobs:
 				enqueue(
 					pull_from_email_account,
@@ -829,7 +860,9 @@ def get_max_email_uid(email_account):
 		return max_uid
 
 
-def setup_user_email_inbox(email_account, awaiting_password, email_id, enable_outgoing):
+def setup_user_email_inbox(
+	email_account, awaiting_password, email_id, enable_outgoing, used_oauth
+):
 	"""setup email inbox for user"""
 	from frappe.core.doctype.user.user import ask_pass_update
 
@@ -840,6 +873,7 @@ def setup_user_email_inbox(email_account, awaiting_password, email_id, enable_ou
 		row.email_id = email_id
 		row.email_account = email_account
 		row.awaiting_password = awaiting_password or 0
+		row.used_oauth = used_oauth or 0
 		row.enable_outgoing = enable_outgoing or 0
 
 		user.save(ignore_permissions=True)
@@ -872,8 +906,10 @@ def setup_user_email_inbox(email_account, awaiting_password, email_id, enable_ou
 	if update_user_email_settings:
 		UserEmail = frappe.qb.DocType("User Email")
 		frappe.qb.update(UserEmail).set(UserEmail.awaiting_password, (awaiting_password or 0)).set(
-			UserEmail.enable_outgoing, enable_outgoing
-		).where(UserEmail.email_account == email_account).run()
+			UserEmail.enable_outgoing, (enable_outgoing or 0)
+		).set(UserEmail.used_oauth, (used_oauth or 0)).where(
+			UserEmail.email_account == email_account
+		).run()
 
 	else:
 		users = " and ".join([frappe.bold(user.get("name")) for user in user_names])
@@ -898,10 +934,10 @@ def remove_user_email_inbox(email_account):
 		doc.save(ignore_permissions=True)
 
 
-@frappe.whitelist(allow_guest=False)
-def set_email_password(email_account, user, password):
+@frappe.whitelist()
+def set_email_password(email_account, password):
 	account = frappe.get_doc("Email Account", email_account)
-	if account.awaiting_password:
+	if account.awaiting_password and not account.auth_method == "OAuth":
 		account.awaiting_password = 0
 		account.password = password
 		try:
diff --git a/frappe/email/doctype/email_account/email_account_list.js b/frappe/email/doctype/email_account/email_account_list.js
index 5ec56fb3db..5913706cbf 100644
--- a/frappe/email/doctype/email_account/email_account_list.js
+++ b/frappe/email/doctype/email_account/email_account_list.js
@@ -1,23 +1,24 @@
 frappe.listview_settings["Email Account"] = {
 	add_fields: ["default_incoming", "default_outgoing", "enable_incoming", "enable_outgoing"],
-	get_indicator: function(doc) {
-		if(doc.default_incoming && doc.default_outgoing) {
-			var color = (doc.enable_incoming && doc.enable_outgoing) ? "blue" : "gray";
-			return [__("Default Sending and Inbox"), color, "default_incoming,=,Yes|default_outgoing,=,Yes"]
-		}
-		else if(doc.default_incoming) {
+	get_indicator: function (doc) {
+		if (doc.default_incoming && doc.default_outgoing) {
+			var color = doc.enable_incoming && doc.enable_outgoing ? "blue" : "gray";
+			return [
+				__("Default Sending and Inbox"),
+				color,
+				"default_incoming,=,Yes|default_outgoing,=,Yes",
+			];
+		} else if (doc.default_incoming) {
 			color = doc.enable_incoming ? "blue" : "gray";
 			return [__("Default Inbox"), color, "default_incoming,=,Yes"];
-		}
-		else if(doc.default_outgoing) {
+		} else if (doc.default_outgoing) {
 			color = doc.enable_outgoing ? "blue" : "gray";
 			return [__("Default Sending"), color, "default_outgoing,=,Yes"];
-		}
-		else {
+		} else {
 			color = doc.enable_incoming ? "blue" : "gray";
 			return [__("Inbox"), color, "is_global,=,No|is_default=No"];
 		}
-	}
-}
+	},
+};
 
 frappe.help.youtube_id["Email Account"] = "YFYe0DrB95o";
diff --git a/frappe/email/doctype/email_account/test_email_account.py b/frappe/email/doctype/email_account/test_email_account.py
index a027a81bd7..9ab004cdd0 100644
--- a/frappe/email/doctype/email_account/test_email_account.py
+++ b/frappe/email/doctype/email_account/test_email_account.py
@@ -39,7 +39,7 @@ class TestEmailAccount(unittest.TestCase):
 		frappe.db.delete("Unhandled Email")
 
 	def get_test_mail(self, fname):
-		with open(os.path.join(os.path.dirname(__file__), "test_mails", fname), "r") as f:
+		with open(os.path.join(os.path.dirname(__file__), "test_mails", fname)) as f:
 			return f.read()
 
 	def test_incoming(self):
@@ -211,7 +211,7 @@ class TestEmailAccount(unittest.TestCase):
 
 		sent_mail = email.message_from_string(frappe.get_last_doc("Email Queue").message)
 
-		with open(os.path.join(os.path.dirname(__file__), "test_mails", "reply-1.raw"), "r") as f:
+		with open(os.path.join(os.path.dirname(__file__), "test_mails", "reply-1.raw")) as f:
 			raw = f.read()
 			raw = raw.replace("<-- in-reply-to -->", sent_mail.get("Message-Id"))
 
@@ -233,10 +233,10 @@ class TestEmailAccount(unittest.TestCase):
 	def test_threading_by_subject(self):
 		cleanup(["in", ["test_sender@example.com", "test@example.com"]])
 
-		with open(os.path.join(os.path.dirname(__file__), "test_mails", "reply-2.raw"), "r") as f:
+		with open(os.path.join(os.path.dirname(__file__), "test_mails", "reply-2.raw")) as f:
 			test_mails = [f.read()]
 
-		with open(os.path.join(os.path.dirname(__file__), "test_mails", "reply-3.raw"), "r") as f:
+		with open(os.path.join(os.path.dirname(__file__), "test_mails", "reply-3.raw")) as f:
 			test_mails.append(f.read())
 
 		# parse reply
@@ -280,11 +280,11 @@ class TestEmailAccount(unittest.TestCase):
 		last_mail = frappe.get_doc("Email Queue", dict(reference_name=event.name))
 
 		# get test mail with message-id as in-reply-to
-		with open(os.path.join(os.path.dirname(__file__), "test_mails", "reply-4.raw"), "r") as f:
+		with open(os.path.join(os.path.dirname(__file__), "test_mails", "reply-4.raw")) as f:
 			messages = {
 				# append_to = ToDo
 				'"INBOX"': {
-					"latest_messages": [f.read().replace("{{ message_id }}", last_mail.message_id)],
+					"latest_messages": [f.read().replace("{{ message_id }}", "<" + last_mail.message_id + ">")],
 					"seen_status": {2: "UNSEEN"},
 					"uid_list": [2],
 				}
@@ -451,7 +451,7 @@ class TestInboundMail(unittest.TestCase):
 		frappe.db.delete("ToDo")
 
 	def get_test_mail(self, fname):
-		with open(os.path.join(os.path.dirname(__file__), "test_mails", fname), "r") as f:
+		with open(os.path.join(os.path.dirname(__file__), "test_mails", fname)) as f:
 			return f.read()
 
 	def new_doc(self, doctype, **data):
diff --git a/frappe/email/doctype/email_account/test_records.json b/frappe/email/doctype/email_account/test_records.json
index 66eb5a9b2e..2e204e5277 100644
--- a/frappe/email/doctype/email_account/test_records.json
+++ b/frappe/email/doctype/email_account/test_records.json
@@ -18,7 +18,6 @@
 		"unreplied_for_mins": 20,
 		"send_notification_to": "test_unreplied@example.com",
 		"pop3_server": "pop.test.example.com",
-		"no_remaining":"0",
 		"append_to": "ToDo",
 		"imap_folder": [{"folder_name": "INBOX", "append_to": "ToDo"}, {"folder_name": "Test Folder", "append_to": "Communication"}],
 		"track_email_status": 1
diff --git a/frappe/email/doctype/email_domain/email_domain.js b/frappe/email/doctype/email_domain/email_domain.js
index 1716bf9900..ea93758905 100644
--- a/frappe/email/doctype/email_domain/email_domain.js
+++ b/frappe/email/doctype/email_domain/email_domain.js
@@ -1,29 +1,28 @@
-
 frappe.ui.form.on("Email Domain", {
-	email_id:function(frm){
-		frm.set_value("domain_name",frm.doc.email_id.split("@")[1])
+	email_id: function (frm) {
+		frm.set_value("domain_name", frm.doc.email_id.split("@")[1]);
 	},
 
-	refresh:function(frm){
-		if (frm.doc.email_id){
-			frm.set_value("domain_name",frm.doc.email_id.split("@")[1])
+	refresh: function (frm) {
+		if (frm.doc.email_id) {
+			frm.set_value("domain_name", frm.doc.email_id.split("@")[1]);
 		}
 
 		if (frm.doc.__islocal != 1 && frappe.route_flags.return_to_email_account) {
 			var route = frappe.get_prev_route();
 			delete frappe.route_flags.return_to_email_account;
-			frappe.route_flags.set_domain_values = true
+			frappe.route_flags.set_domain_values = true;
 
-			frappe.route_options = {
+			(frappe.route_options = {
 				domain: frm.doc.name,
 				use_imap: frm.doc.use_imap,
 				email_server: frm.doc.email_server,
 				use_ssl: frm.doc.use_ssl,
 				smtp_server: frm.doc.smtp_server,
 				use_tls: frm.doc.use_tls,
-				smtp_port: frm.doc.smtp_port
-			},
-			frappe.set_route(route);
+				smtp_port: frm.doc.smtp_port,
+			}),
+				frappe.set_route(route);
 		}
-	}
-})
\ No newline at end of file
+	},
+});
diff --git a/frappe/email/doctype/email_domain/email_domain.json b/frappe/email/doctype/email_domain/email_domain.json
index a4ca19a0bd..aba5a8569f 100644
--- a/frappe/email/doctype/email_domain/email_domain.json
+++ b/frappe/email/doctype/email_domain/email_domain.json
@@ -73,7 +73,7 @@
    "label": "Attachment Limit (MB)"
   },
   {
-   "description": "Append as communication against this DocType (must have fields, \"Status\", \"Subject\")",
+   "description": "Append as communication against this DocType (must have field \"Status\" and both \"Sender\" and \"Subject\" defined in the related doctype Email Settings)",
    "fieldname": "append_to",
    "fieldtype": "Link",
    "hidden": 1,
@@ -143,4 +143,4 @@
  ],
  "sort_field": "modified",
  "sort_order": "DESC"
-}
\ No newline at end of file
+}
diff --git a/frappe/email/doctype/email_domain/email_domain.py b/frappe/email/doctype/email_domain/email_domain.py
index ff59725fd7..ab6546e4e6 100644
--- a/frappe/email/doctype/email_domain/email_domain.py
+++ b/frappe/email/doctype/email_domain/email_domain.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors
 # License: MIT. See LICENSE
 
@@ -53,12 +52,10 @@ class EmailDomain(Document):
 						test = poplib.POP3(self.email_server, port=get_port(self))
 
 			except Exception as e:
-				logger.warn(
-					'Incoming email account "{host}" not correct'.format(host=self.email_server), exc_info=e
-				)
+				logger.warn(f'Incoming email account "{self.email_server}" not correct', exc_info=e)
 				frappe.throw(
 					title=_("Incoming email account not correct"),
-					msg='Error connecting IMAP/POP3 "{host}": {e}'.format(host=self.email_server, e=e),
+					msg=f'Error connecting IMAP/POP3 "{self.email_server}": {e}',
 				)
 
 			finally:
@@ -94,12 +91,10 @@ class EmailDomain(Document):
 					sess = smtplib.SMTP(cstr(self.smtp_server or ""), cint(self.smtp_port) or None)
 				sess.quit()
 			except Exception as e:
-				logger.warn(
-					'Outgoing email account "{host}" not correct'.format(host=self.smtp_server), exc_info=e
-				)
+				logger.warn(f'Outgoing email account "{self.smtp_server}" not correct', exc_info=e)
 				frappe.throw(
 					title=_("Outgoing email account not correct"),
-					msg='Error connecting SMTP "{host}": {e}'.format(host=self.smtp_server, e=e),
+					msg=f'Error connecting SMTP "{self.smtp_server}": {e}',
 				)
 
 	def on_update(self):
diff --git a/frappe/email/doctype/email_domain/test_email_domain.py b/frappe/email/doctype/email_domain/test_email_domain.py
index 0162e4efe0..55a8d620a8 100644
--- a/frappe/email/doctype/email_domain/test_email_domain.py
+++ b/frappe/email/doctype/email_domain/test_email_domain.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
 import unittest
diff --git a/frappe/email/doctype/email_flag_queue/email_flag_queue.js b/frappe/email/doctype/email_flag_queue/email_flag_queue.js
index 19c4d4b0c1..ee062815f5 100644
--- a/frappe/email/doctype/email_flag_queue/email_flag_queue.js
+++ b/frappe/email/doctype/email_flag_queue/email_flag_queue.js
@@ -1,8 +1,6 @@
 // Copyright (c) 2016, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Email Flag Queue', {
-	refresh: function(frm) {
-
-	}
+frappe.ui.form.on("Email Flag Queue", {
+	refresh: function (frm) {},
 });
diff --git a/frappe/email/doctype/email_flag_queue/email_flag_queue.py b/frappe/email/doctype/email_flag_queue/email_flag_queue.py
index 1b29d2f9d8..cf28ce0628 100644
--- a/frappe/email/doctype/email_flag_queue/email_flag_queue.py
+++ b/frappe/email/doctype/email_flag_queue/email_flag_queue.py
@@ -1,8 +1,6 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2015, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
-import frappe
 from frappe.model.document import Document
 
 
diff --git a/frappe/email/doctype/email_flag_queue/test_email_flag_queue.py b/frappe/email/doctype/email_flag_queue/test_email_flag_queue.py
index f52aeb61fa..8bfc9230a4 100644
--- a/frappe/email/doctype/email_flag_queue/test_email_flag_queue.py
+++ b/frappe/email/doctype/email_flag_queue/test_email_flag_queue.py
@@ -1,10 +1,7 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2015, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
 import unittest
 
-import frappe
-
 # test_records = frappe.get_test_records('Email Flag Queue')
 
 
diff --git a/frappe/email/doctype/email_group/email_group.js b/frappe/email/doctype/email_group/email_group.js
index 404600c97d..5ad4a39dd9 100644
--- a/frappe/email/doctype/email_group/email_group.js
+++ b/frappe/email/doctype/email_group/email_group.js
@@ -1,46 +1,74 @@
 // Copyright (c) 2016, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on("Email Group", "refresh", function(frm) {
-	if(!frm.is_new()) {
-		frm.add_custom_button(__("Import Subscribers"), function() {
-			frappe.prompt({fieldtype:"Select", options: frm.doc.__onload.import_types,
-				label:__("Import Email From"), fieldname:"doctype", reqd:1},
-				function(data) {
-					frappe.call({
-						method: "frappe.email.doctype.email_group.email_group.import_from",
-						args: {
-							"name": frm.doc.name,
-							"doctype": data.doctype
-						},
-						callback: function(r) {
-							frm.set_value("total_subscribers", r.message);
-						}
-					})
-				}, __("Import Subscribers"), __("Import"));
-		}, __("Action"));
+frappe.ui.form.on("Email Group", "refresh", function (frm) {
+	if (!frm.is_new()) {
+		frm.add_custom_button(
+			__("Import Subscribers"),
+			function () {
+				frappe.prompt(
+					{
+						fieldtype: "Select",
+						options: frm.doc.__onload.import_types,
+						label: __("Import Email From"),
+						fieldname: "doctype",
+						reqd: 1,
+					},
+					function (data) {
+						frappe.call({
+							method: "frappe.email.doctype.email_group.email_group.import_from",
+							args: {
+								name: frm.doc.name,
+								doctype: data.doctype,
+							},
+							callback: function (r) {
+								frm.set_value("total_subscribers", r.message);
+							},
+						});
+					},
+					__("Import Subscribers"),
+					__("Import")
+				);
+			},
+			__("Action")
+		);
 
-		frm.add_custom_button(__("Add Subscribers"), function() {
-			frappe.prompt({fieldtype:"Text",
-				label:__("Email Addresses"), fieldname:"email_list", reqd:1},
-				function(data) {
-					frappe.call({
-						method: "frappe.email.doctype.email_group.email_group.add_subscribers",
-						args: {
-							"name": frm.doc.name,
-							"email_list": data.email_list
-						},
-						callback: function(r) {
-							frm.set_value("total_subscribers", r.message);
-						}
-					})
-				}, __("Add Subscribers"), __("Add"));
-		}, __("Action"));
-
-		frm.add_custom_button(__("New Newsletter"), function() {
-			frappe.route_options = {"email_group": frm.doc.name};
-			frappe.new_doc("Newsletter");
-		}, __("Action"));
+		frm.add_custom_button(
+			__("Add Subscribers"),
+			function () {
+				frappe.prompt(
+					{
+						fieldtype: "Text",
+						label: __("Email Addresses"),
+						fieldname: "email_list",
+						reqd: 1,
+					},
+					function (data) {
+						frappe.call({
+							method: "frappe.email.doctype.email_group.email_group.add_subscribers",
+							args: {
+								name: frm.doc.name,
+								email_list: data.email_list,
+							},
+							callback: function (r) {
+								frm.set_value("total_subscribers", r.message);
+							},
+						});
+					},
+					__("Add Subscribers"),
+					__("Add")
+				);
+			},
+			__("Action")
+		);
 
+		frm.add_custom_button(
+			__("New Newsletter"),
+			function () {
+				frappe.route_options = { email_group: frm.doc.name };
+				frappe.new_doc("Newsletter");
+			},
+			__("Action")
+		);
 	}
 });
diff --git a/frappe/email/doctype/email_group/email_group.py b/frappe/email/doctype/email_group/email_group.py
index 6489e82341..ea62a6e9ec 100755
--- a/frappe/email/doctype/email_group/email_group.py
+++ b/frappe/email/doctype/email_group/email_group.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2015, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
@@ -12,7 +11,7 @@ class EmailGroup(Document):
 	def onload(self):
 		singles = [d.name for d in frappe.db.get_all("DocType", "name", {"issingle": 1})]
 		self.get("__onload").import_types = [
-			{"value": d.parent, "label": "{0} ({1})".format(d.parent, d.label)}
+			{"value": d.parent, "label": f"{d.parent} ({d.label})"}
 			for d in frappe.db.get_all("DocField", ("parent", "label"), {"options": "Email"})
 			if d.parent not in singles
 		]
diff --git a/frappe/email/doctype/email_group/test_email_group.py b/frappe/email/doctype/email_group/test_email_group.py
index 4013c17d93..aa41285f90 100644
--- a/frappe/email/doctype/email_group/test_email_group.py
+++ b/frappe/email/doctype/email_group/test_email_group.py
@@ -1,10 +1,7 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2015, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
 import unittest
 
-import frappe
-
 # test_records = frappe.get_test_records('Email Group')
 
 
diff --git a/frappe/email/doctype/email_group_member/email_group_member.js b/frappe/email/doctype/email_group_member/email_group_member.js
index 417eb70119..ca2c17ad81 100644
--- a/frappe/email/doctype/email_group_member/email_group_member.js
+++ b/frappe/email/doctype/email_group_member/email_group_member.js
@@ -1,8 +1,6 @@
 // Copyright (c) 2016, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Email Group Member', {
-	refresh: function(frm) {
-
-	}
+frappe.ui.form.on("Email Group Member", {
+	refresh: function (frm) {},
 });
diff --git a/frappe/email/doctype/email_group_member/email_group_member.json b/frappe/email/doctype/email_group_member/email_group_member.json
index b3b87c6211..0e32135b72 100644
--- a/frappe/email/doctype/email_group_member/email_group_member.json
+++ b/frappe/email/doctype/email_group_member/email_group_member.json
@@ -1,151 +1,70 @@
 {
- "allow_copy": 0, 
- "allow_import": 1, 
- "allow_rename": 0, 
- "autoname": "hash", 
- "beta": 0, 
- "creation": "2015-03-18 06:15:59.321619", 
- "custom": 0, 
- "docstatus": 0, 
- "doctype": "DocType", 
- "document_type": "Document", 
- "editable_grid": 0, 
- "engine": "InnoDB", 
+ "actions": [],
+ "allow_import": 1,
+ "autoname": "hash",
+ "creation": "2015-03-18 06:15:59.321619",
+ "doctype": "DocType",
+ "document_type": "Document",
+ "engine": "InnoDB",
+ "field_order": [
+  "email_group",
+  "email",
+  "unsubscribed"
+ ],
  "fields": [
   {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "email_group", 
-   "fieldtype": "Link", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 1, 
-   "label": "Email Group", 
-   "length": 0, 
-   "no_copy": 0, 
-   "options": "Email Group", 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "fieldname": "email_group",
+   "fieldtype": "Link",
+   "in_list_view": 1,
+   "in_standard_filter": 1,
+   "label": "Email Group",
+   "options": "Email Group",
+   "reqd": 1,
+   "search_index": 1
+  },
   {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "email", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 1, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Email", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "fieldname": "email",
+   "fieldtype": "Data",
+   "in_global_search": 1,
+   "in_list_view": 1,
+   "label": "Email",
+   "reqd": 1
+  },
   {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "unsubscribed", 
-   "fieldtype": "Check", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Unsubscribed", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
+   "default": "0",
+   "fieldname": "unsubscribed",
+   "fieldtype": "Check",
+   "in_list_view": 1,
+   "label": "Unsubscribed",
+   "search_index": 1
   }
- ], 
- "hide_heading": 0, 
- "hide_toolbar": 0, 
- "idx": 0, 
- "image_view": 0, 
- "in_create": 0, 
-
- "is_submittable": 0, 
- "issingle": 0, 
- "istable": 0, 
- "max_attachments": 0, 
- "modified": "2017-02-17 17:00:42.551806", 
- "modified_by": "Administrator", 
- "module": "Email", 
- "name": "Email Group Member", 
- "name_case": "", 
- "owner": "Administrator", 
+ ],
+ "links": [],
+ "modified": "2022-07-11 16:38:34.165271",
+ "modified_by": "Administrator",
+ "module": "Email",
+ "name": "Email Group Member",
+ "naming_rule": "Random",
+ "owner": "Administrator",
  "permissions": [
   {
-   "amend": 0, 
-   "apply_user_permissions": 0, 
-   "cancel": 0, 
-   "create": 1, 
-   "delete": 1, 
-   "email": 1, 
-   "export": 1, 
-   "if_owner": 0, 
-   "import": 1, 
-   "permlevel": 0, 
-   "print": 1, 
-   "read": 1, 
-   "report": 1, 
-   "role": "Newsletter Manager", 
-   "set_user_permissions": 0, 
-   "share": 1, 
-   "submit": 0, 
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "export": 1,
+   "import": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "Newsletter Manager",
+   "share": 1,
    "write": 1
   }
- ], 
- "quick_entry": 1, 
- "read_only": 0, 
- "read_only_onload": 0, 
- "show_name_in_global_search": 0, 
- "sort_field": "modified", 
- "sort_order": "DESC", 
- "title_field": "email", 
- "track_changes": 1, 
- "track_seen": 0
+ ],
+ "quick_entry": 1,
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "states": [],
+ "title_field": "email",
+ "track_changes": 1
 }
\ No newline at end of file
diff --git a/frappe/email/doctype/email_group_member/email_group_member.py b/frappe/email/doctype/email_group_member/email_group_member.py
index 76990b18fb..c0995d393b 100644
--- a/frappe/email/doctype/email_group_member/email_group_member.py
+++ b/frappe/email/doctype/email_group_member/email_group_member.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2015, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
diff --git a/frappe/email/doctype/email_group_member/test_email_group_member.py b/frappe/email/doctype/email_group_member/test_email_group_member.py
index 562b0327f2..749792fe52 100644
--- a/frappe/email/doctype/email_group_member/test_email_group_member.py
+++ b/frappe/email/doctype/email_group_member/test_email_group_member.py
@@ -1,10 +1,7 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2015, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
 import unittest
 
-import frappe
-
 # test_records = frappe.get_test_records('Email Group Member')
 
 
diff --git a/frappe/email/doctype/email_queue/email_queue.js b/frappe/email/doctype/email_queue/email_queue.js
index b6ef0ec082..2ac4b6f7fe 100644
--- a/frappe/email/doctype/email_queue/email_queue.js
+++ b/frappe/email/doctype/email_queue/email_queue.js
@@ -2,37 +2,37 @@
 // For license information, please see license.txt
 
 frappe.ui.form.on("Email Queue", {
-	refresh: function(frm) {
-		if (["Not Sent","Partially Sent"].indexOf(frm.doc.status)!=-1) {
-			let button = frm.add_custom_button("Send Now", function() {
+	refresh: function (frm) {
+		if (["Not Sent", "Partially Sent"].indexOf(frm.doc.status) != -1) {
+			let button = frm.add_custom_button("Send Now", function () {
 				frappe.call({
-					method: 'frappe.email.doctype.email_queue.email_queue.send_now',
+					method: "frappe.email.doctype.email_queue.email_queue.send_now",
 					args: {
-						name: frm.doc.name
+						name: frm.doc.name,
 					},
 					btn: button,
-					callback: function() {
+					callback: function () {
 						frm.reload_doc();
-					}
+					},
 				});
 			});
 		}
 
-		if (["Error","Partially Errored"].indexOf(frm.doc.status)!=-1) {
-			let button = frm.add_custom_button("Retry Sending", function() {
+		if (["Error", "Partially Errored"].indexOf(frm.doc.status) != -1) {
+			let button = frm.add_custom_button("Retry Sending", function () {
 				frm.call({
 					method: "retry_sending",
 					args: {
-						name: frm.doc.name
+						name: frm.doc.name,
 					},
 					btn: button,
-					callback: function(r) {
+					callback: function (r) {
 						if (!r.exc) {
 							frm.set_value("status", "Not Sent");
 						}
-					}
-				})
+					},
+				});
 			});
 		}
-	}
+	},
 });
diff --git a/frappe/email/doctype/email_queue/email_queue.json b/frappe/email/doctype/email_queue/email_queue.json
index f251786c90..c9ec374687 100644
--- a/frappe/email/doctype/email_queue/email_queue.json
+++ b/frappe/email/doctype/email_queue/email_queue.json
@@ -83,7 +83,8 @@
    "fieldname": "reference_name",
    "fieldtype": "Data",
    "label": "Reference DocName",
-   "read_only": 1
+   "read_only": 1,
+   "search_index": 1
   },
   {
    "fieldname": "communication",
@@ -152,10 +153,11 @@
  "idx": 1,
  "in_create": 1,
  "links": [],
- "modified": "2021-04-29 06:33:25.191729",
+ "modified": "2022-07-12 15:17:37.934316",
  "modified_by": "Administrator",
  "module": "Email",
  "name": "Email Queue",
+ "naming_rule": "Random",
  "owner": "Administrator",
  "permissions": [
   {
@@ -169,5 +171,6 @@
  ],
  "sort_field": "modified",
  "sort_order": "DESC",
+ "states": [],
  "track_changes": 1
 }
\ No newline at end of file
diff --git a/frappe/email/doctype/email_queue/email_queue.py b/frappe/email/doctype/email_queue/email_queue.py
index 38577eeb97..3c020eea39 100644
--- a/frappe/email/doctype/email_queue/email_queue.py
+++ b/frappe/email/doctype/email_queue/email_queue.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2015, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
@@ -9,19 +8,28 @@ import traceback
 from email.parser import Parser
 from email.policy import SMTPUTF8
 
-from html2text import html2text
 from rq.timeouts import JobTimeoutException
 
 import frappe
 from frappe import _, safe_encode, task
+from frappe.core.utils import html2text
 from frappe.email.doctype.email_account.email_account import EmailAccount
 from frappe.email.email_body import add_attachment, get_email, get_formatted_html
 from frappe.email.queue import get_unsubcribed_url, get_unsubscribe_message
+from frappe.email.smtp import SMTPServer
 from frappe.model.document import Document
-from frappe.query_builder.utils import DocType
-from frappe.utils import add_days, cint, cstr, get_hook_method, nowdate, split_emails
-
-MAX_RETRY_COUNT = 3
+from frappe.query_builder import DocType, Interval
+from frappe.query_builder.functions import Now
+from frappe.utils import (
+	add_days,
+	cint,
+	cstr,
+	get_hook_method,
+	get_string_between,
+	nowdate,
+	sbool,
+	split_emails,
+)
 
 
 class EmailQueue(Document):
@@ -93,7 +101,7 @@ class EmailQueue(Document):
 
 	def get_email_account(self):
 		if self.email_account:
-			return frappe.get_doc("Email Account", self.email_account)
+			return frappe.get_cached_doc("Email Account", self.email_account)
 
 		return EmailAccount.find_outgoing(
 			match_by_email=self.sender, match_by_doctype=self.reference_doctype
@@ -103,18 +111,21 @@ class EmailQueue(Document):
 		return self.status in ["Not Sent", "Partially Sent"]
 
 	def can_send_now(self):
-		hold_queue = cint(frappe.defaults.get_defaults().get("hold_queue")) == 1
-		if frappe.are_emails_muted() or not self.is_to_be_sent() or hold_queue:
+		if (
+			frappe.are_emails_muted()
+			or not self.is_to_be_sent()
+			or cint(frappe.db.get_default("suspend_email_queue")) == 1
+		):
 			return False
 
 		return True
 
-	def send(self, is_background_task=False):
+	def send(self, is_background_task: bool = False, smtp_server_instance: SMTPServer = None):
 		"""Send emails to recipients."""
 		if not self.can_send_now():
 			return
 
-		with SendMailContext(self, is_background_task) as ctx:
+		with SendMailContext(self, is_background_task, smtp_server_instance) as ctx:
 			message = None
 			for recipient in self.recipients:
 				if not recipient.is_mail_to_be_sent():
@@ -136,23 +147,59 @@ class EmailQueue(Document):
 			if ctx.email_account_doc.append_emails_to_sent_folder and ctx.sent_to:
 				ctx.email_account_doc.append_email_to_sent_folder(message)
 
+	@staticmethod
+	def clear_old_logs(days=30):
+		"""Remove low priority older than 31 days in Outbox or configured in Log Settings.
+		Note: Used separate query to avoid deadlock
+		"""
+		days = days or 31
+		email_queue = frappe.qb.DocType("Email Queue")
+		email_recipient = frappe.qb.DocType("Email Queue Recipient")
+
+		# Delete queue table
+		(
+			frappe.qb.from_(email_queue)
+			.delete()
+			.where(email_queue.modified < (Now() - Interval(days=days)))
+		).run()
+
+		# delete child tables, note that this has potential to leave some orphan
+		# child table behind if modified time was later than parent doc (rare).
+		# But it's safe since child table doesn't contain links.
+		(
+			frappe.qb.from_(email_recipient)
+			.delete()
+			.where(email_recipient.modified < (Now() - Interval(days=days)))
+		).run()
+
 
 @task(queue="short")
-def send_mail(email_queue_name, is_background_task=False):
-	"""This is equalent to EmqilQueue.send.
+def send_mail(email_queue_name, is_background_task=False, smtp_server_instance: SMTPServer = None):
+	"""This is equivalent to EmailQueue.send.
 
 	This provides a way to make sending mail as a background job.
 	"""
 	record = EmailQueue.find(email_queue_name)
-	record.send(is_background_task=is_background_task)
+	record.send(is_background_task=is_background_task, smtp_server_instance=smtp_server_instance)
 
 
 class SendMailContext:
-	def __init__(self, queue_doc: Document, is_background_task: bool = False):
-		self.queue_doc = queue_doc
+	def __init__(
+		self,
+		queue_doc: Document,
+		is_background_task: bool = False,
+		smtp_server_instance: SMTPServer = None,
+	):
+		self.queue_doc: EmailQueue = queue_doc
 		self.is_background_task = is_background_task
 		self.email_account_doc = queue_doc.get_email_account()
-		self.smtp_server = self.email_account_doc.get_smtp_server()
+
+		self.smtp_server = smtp_server_instance or self.email_account_doc.get_smtp_server()
+
+		# if smtp_server_instance is passed, then retain smtp session
+		# Note: smtp session will have to be manually closed
+		self.retain_smtp_session = bool(smtp_server_instance)
+
 		self.sent_to = [rec.recipient for rec in self.queue_doc.recipients if rec.is_main_sent()]
 
 	def __enter__(self):
@@ -169,14 +216,16 @@ class SendMailContext:
 			JobTimeoutException,
 		]
 
-		self.smtp_server.quit()
+		if not self.retain_smtp_session:
+			self.smtp_server.quit()
+
 		self.log_exception(exc_type, exc_val, exc_tb)
 
 		if exc_type in exceptions:
-			email_status = (self.sent_to and "Partially Sent") or "Not Sent"
+			email_status = "Partially Sent" if self.sent_to else "Not Sent"
 			self.queue_doc.update_status(status=email_status, commit=True)
 		elif exc_type:
-			if self.queue_doc.retry < MAX_RETRY_COUNT:
+			if self.queue_doc.retry < get_email_retry_limit():
 				update_fields = {"status": "Not Sent", "retry": self.queue_doc.retry + 1}
 			else:
 				update_fields = {"status": (self.sent_to and "Partially Errored") or "Error"}
@@ -185,12 +234,12 @@ class SendMailContext:
 			email_status = self.is_mail_sent_to_all() and "Sent"
 			email_status = email_status or (self.sent_to and "Partially Sent") or "Not Sent"
 
-			update_fields = {"status": email_status}
-			if self.email_account_doc.is_exists_in_db():
-				update_fields["email_account"] = self.email_account_doc.name
-			else:
-				update_fields["email_account"] = None
-
+			update_fields = {
+				"status": email_status,
+				"email_account": self.email_account_doc.name
+				if self.email_account_doc.is_exists_in_db()
+				else None,
+			}
 			self.queue_doc.update_status(**update_fields, commit=True)
 
 	def log_exception(self, exc_type, exc_val, exc_tb):
@@ -198,10 +247,7 @@ class SendMailContext:
 			traceback_string = "".join(traceback.format_tb(exc_tb))
 			traceback_string += f"\n Queue Name: {self.queue_doc.name}"
 
-			if self.is_background_task:
-				frappe.log_error(title="frappe.email.queue.flush", message=traceback_string)
-			else:
-				frappe.log_error(message=traceback_string)
+			self.queue_doc.log_error("Email sending failed", traceback_string)
 
 	@property
 	def smtp_session(self):
@@ -215,12 +261,13 @@ class SendMailContext:
 		self.sent_to.append(recipient.recipient)
 
 	def is_mail_sent_to_all(self):
-		return sorted(self.sent_to) == sorted([rec.recipient for rec in self.queue_doc.recipients])
+		return sorted(self.sent_to) == sorted(rec.recipient for rec in self.queue_doc.recipients)
 
 	def get_message_object(self, message):
 		return Parser(policy=SMTPUTF8).parsestr(message)
 
 	def message_placeholder(self, placeholder_key):
+		# sourcery skip: avoid-builtin-shadow
 		map = {
 			"tracker": "",
 			"unsubscribe_url": "",
@@ -241,7 +288,7 @@ class SendMailContext:
 		)
 		message = message.replace(self.message_placeholder("cc"), self.get_receivers_str())
 		message = message.replace(
-			self.message_placeholder("recipient"), self.get_receipient_str(recipient_email)
+			self.message_placeholder("recipient"), self.get_recipient_str(recipient_email)
 		)
 		message = self.include_attachments(message)
 		return message
@@ -256,16 +303,16 @@ class SendMailContext:
 			).decode()
 		return message
 
-	def get_unsubscribe_str(self, recipient_email):
+	def get_unsubscribe_str(self, recipient_email: str) -> str:
 		unsubscribe_url = ""
+
 		if self.queue_doc.add_unsubscribe_link and self.queue_doc.reference_doctype:
-			doctype, doc_name = self.queue_doc.reference_doctype, self.queue_doc.reference_name
 			unsubscribe_url = get_unsubcribed_url(
-				doctype,
-				doc_name,
-				recipient_email,
-				self.queue_doc.unsubscribe_method,
-				self.queue_doc.unsubscribe_param,
+				reference_doctype=self.queue_doc.reference_doctype,
+				reference_name=self.queue_doc.reference_name,
+				email=recipient_email,
+				unsubscribe_method=self.queue_doc.unsubscribe_method,
+				unsubscribe_params=self.queue_doc.unsubscribe_param,
 			)
 
 		return quopri.encodestring(unsubscribe_url.encode()).decode()
@@ -276,14 +323,11 @@ class SendMailContext:
 			to_str = ", ".join(self.queue_doc.to)
 			cc_str = ", ".join(self.queue_doc.cc)
 			message = f"This email was sent to {to_str}"
-			message = message + f" and copied to {cc_str}" if cc_str else message
+			message = f"{message} and copied to {cc_str}" if cc_str else message
 		return message
 
-	def get_receipient_str(self, recipient_email):
-		message = ""
-		if self.queue_doc.expose_recipients != "header":
-			message = recipient_email
-		return message
+	def get_recipient_str(self, recipient_email):
+		return recipient_email if self.queue_doc.expose_recipients != "header" else ""
 
 	def include_attachments(self, message):
 		message_obj = self.get_message_object(message)
@@ -319,6 +363,8 @@ class SendMailContext:
 @frappe.whitelist()
 def retry_sending(name):
 	doc = frappe.get_doc("Email Queue", name)
+	doc.check_permission()
+
 	if doc and (doc.status == "Error" or doc.status == "Partially Errored"):
 		doc.status = "Not Sent"
 		for d in doc.recipients:
@@ -331,9 +377,16 @@ def retry_sending(name):
 def send_now(name):
 	record = EmailQueue.find(name)
 	if record:
+		record.check_permission()
 		record.send()
 
 
+@frappe.whitelist()
+def toggle_sending(enable):
+	frappe.only_for("System Manager")
+	frappe.db.set_default("suspend_email_queue", 0 if sbool(enable) else 1)
+
+
 def on_doctype_update():
 	"""Add index in `tabCommunication` for `(reference_doctype, reference_name)`"""
 	frappe.db.add_index(
@@ -341,6 +394,10 @@ def on_doctype_update():
 	)
 
 
+def get_email_retry_limit():
+	return cint(frappe.db.get_system_setting("email_retry_limit")) or 3
+
+
 class QueueBuilder:
 	"""Builds Email Queue from the given data"""
 
@@ -596,7 +653,6 @@ class QueueBuilder:
 		if not (final_recipients + self.final_cc()):
 			return []
 
-		email_queues = []
 		queue_data = self.as_dict(include_recipients=False)
 		if not queue_data:
 			return []
@@ -604,17 +660,35 @@ class QueueBuilder:
 		if not queue_separately:
 			recipients = list(set(final_recipients + self.final_cc() + self.bcc))
 			q = EmailQueue.new({**queue_data, **{"recipients": recipients}}, ignore_permissions=True)
-			email_queues.append(q)
+			send_now and q.send()
 		else:
-			for r in final_recipients:
-				recipients = [r] if email_queues else list(set([r] + self.final_cc() + self.bcc))
-				q = EmailQueue.new({**queue_data, **{"recipients": recipients}}, ignore_permissions=True)
-				email_queues.append(q)
+			if send_now and len(final_recipients) >= 1000:
+				# force queueing if there are too many recipients to avoid timeouts
+				send_now = False
+			for recipients in frappe.utils.create_batch(final_recipients, 1000):
+				frappe.enqueue(
+					self.send_emails,
+					queue_data=queue_data,
+					final_recipients=recipients,
+					job_name=frappe.utils.get_job_name(
+						"send_bulk_emails_for", self.reference_doctype, self.reference_name
+					),
+					now=frappe.flags.in_test or send_now,
+					queue="long",
+				)
 
-		if send_now:
-			for doc in email_queues:
-				doc.send()
-		return email_queues
+	def send_emails(self, queue_data, final_recipients):
+		# This is used to bulk send emails from same sender to multiple recipients separately
+		# This re-uses smtp server instance to minimize the cost of new session creation
+		smtp_server_instance = None
+		for r in final_recipients:
+			recipients = list(set([r] + self.final_cc() + self.bcc))
+			q = EmailQueue.new({**queue_data, **{"recipients": recipients}}, ignore_permissions=True)
+			if not smtp_server_instance:
+				email_account = q.get_email_account()
+				smtp_server_instance = email_account.get_smtp_server()
+			q.send(smtp_server_instance=smtp_server_instance)
+		smtp_server_instance.quit()
 
 	def as_dict(self, include_recipients=True):
 		email_account = self.get_outgoing_email_account()
@@ -626,17 +700,19 @@ class QueueBuilder:
 		except frappe.InvalidEmailAddressError:
 			# bad Email Address - don't add to queue
 			frappe.log_error(
-				"Invalid Email ID Sender: {0}, Recipients: {1}, \nTraceback: {2} ".format(
+				title="Invalid email address",
+				message="Invalid email address Sender: {}, Recipients: {}, \nTraceback: {} ".format(
 					self.sender, ", ".join(self.final_recipients()), traceback.format_exc()
 				),
-				"Email Not Sent",
+				reference_doctype=self.reference_doctype,
+				reference_name=self.reference_name,
 			)
 			return
 
 		d = {
 			"priority": self.send_priority,
 			"attachments": json.dumps(self.get_attachments()),
-			"message_id": mail.msg_root["Message-Id"].strip(" <>"),
+			"message_id": get_string_between("<", mail.msg_root["Message-Id"], ">"),
 			"message": mail_to_string,
 			"sender": self.sender,
 			"reference_doctype": self.reference_doctype,
diff --git a/frappe/email/doctype/email_queue/email_queue_list.js b/frappe/email/doctype/email_queue/email_queue_list.js
index 0445a3ca19..b00503b6f8 100644
--- a/frappe/email/doctype/email_queue/email_queue_list.js
+++ b/frappe/email/doctype/email_queue/email_queue_list.js
@@ -1,23 +1,41 @@
-frappe.listview_settings['Email Queue'] = {
-	get_indicator: function(doc) {
-		var colour = {'Sent': 'green', 'Sending': 'blue', 'Not Sent': 'grey', 'Error': 'red', 'Expired': 'orange'};
+frappe.listview_settings["Email Queue"] = {
+	get_indicator: function (doc) {
+		var colour = {
+			Sent: "green",
+			Sending: "blue",
+			"Not Sent": "grey",
+			Error: "red",
+			Expired: "orange",
+		};
 		return [__(doc.status), colour[doc.status], "status,=," + doc.status];
 	},
-	refresh: function(doclist){
-		if (has_common(frappe.user_roles, ["Administrator", "System Manager"])){
-			if (cint(frappe.defaults.get_default("hold_queue"))){
-				doclist.page.clear_inner_toolbar()
-				doclist.page.add_inner_button(__("Resume Sending"), function() {
-					frappe.defaults.set_default("hold_queue", 0);
-					cur_list.refresh();
-				})
-			} else {
-				doclist.page.clear_inner_toolbar()
-				doclist.page.add_inner_button(__("Suspend Sending"), function() {
-					frappe.defaults.set_default("hold_queue", 1)
-					cur_list.refresh();
-				})
-			}
-		}
-	}
+	refresh: show_toggle_sending_button,
+	onload: function (list_view) {
+		frappe.require("logtypes.bundle.js", () => {
+			frappe.utils.logtypes.show_log_retention_message(list_view.doctype);
+		});
+	},
+};
+
+function show_toggle_sending_button(list_view) {
+	if (!has_common(frappe.user_roles, ["Administrator", "System Manager"])) return;
+
+	const sending_disabled = cint(frappe.sys_defaults.suspend_email_queue);
+	const label = sending_disabled ? __("Resume Sending") : __("Suspend Sending");
+
+	list_view.page.add_inner_button(label, async () => {
+		await frappe.xcall(
+			"frappe.email.doctype.email_queue.email_queue.toggle_sending",
+
+			// enable if disabled
+			{ enable: sending_disabled }
+		);
+
+		// set new value for suspend_email_queue in sys_defaults
+		frappe.sys_defaults.suspend_email_queue = sending_disabled ? 0 : 1;
+
+		// clear the button and show one with the opposite label
+		list_view.page.remove_inner_button(label);
+		show_toggle_sending_button(list_view);
+	});
 }
diff --git a/frappe/email/doctype/email_queue/test_email_queue.py b/frappe/email/doctype/email_queue/test_email_queue.py
index b90390916a..5a608b1b23 100644
--- a/frappe/email/doctype/email_queue/test_email_queue.py
+++ b/frappe/email/doctype/email_queue/test_email_queue.py
@@ -1,12 +1,41 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2015, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
-import unittest
 
 import frappe
-
-# test_records = frappe.get_test_records('Email Queue')
+from frappe.tests.utils import FrappeTestCase
 
 
-class TestEmailQueue(unittest.TestCase):
-	pass
+class TestEmailQueue(FrappeTestCase):
+	def test_email_queue_deletion_based_on_modified_date(self):
+		from frappe.email.doctype.email_queue.email_queue import EmailQueue
+
+		old_record = frappe.get_doc(
+			{
+				"doctype": "Email Queue",
+				"sender": "Test ",
+				"show_as_cc": "",
+				"message": "Test message",
+				"status": "Sent",
+				"priority": 1,
+				"recipients": [
+					{
+						"recipient": "test_auth@test.com",
+					}
+				],
+			}
+		).insert()
+
+		old_record.modified = "2010-01-01 00:00:01"
+		old_record.recipients[0].modified = old_record.modified
+		old_record.db_update_all()
+
+		new_record = frappe.copy_doc(old_record)
+		new_record.insert()
+
+		EmailQueue.clear_old_logs()
+
+		self.assertFalse(frappe.db.exists("Email Queue", old_record.name))
+		self.assertFalse(frappe.db.exists("Email Queue Recipient", {"parent": old_record.name}))
+
+		self.assertTrue(frappe.db.exists("Email Queue", new_record.name))
+		self.assertTrue(frappe.db.exists("Email Queue Recipient", {"parent": new_record.name}))
diff --git a/frappe/email/doctype/email_queue_recipient/email_queue_recipient.json b/frappe/email/doctype/email_queue_recipient/email_queue_recipient.json
index 8d1f77e818..c217886ce6 100644
--- a/frappe/email/doctype/email_queue_recipient/email_queue_recipient.json
+++ b/frappe/email/doctype/email_queue_recipient/email_queue_recipient.json
@@ -1,122 +1,46 @@
 {
- "allow_copy": 0, 
- "allow_import": 0, 
- "allow_rename": 0, 
- "beta": 0, 
- "creation": "2016-12-08 12:01:07.993900", 
- "custom": 0, 
- "docstatus": 0, 
- "doctype": "DocType", 
- "document_type": "", 
- "editable_grid": 0, 
- "engine": "InnoDB", 
+ "actions": [],
+ "creation": "2016-12-08 12:01:07.993900",
+ "doctype": "DocType",
+ "engine": "InnoDB",
+ "field_order": [
+  "recipient",
+  "status",
+  "error"
+ ],
  "fields": [
   {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "recipient", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0,
-   "in_list_view": 1, 
-   "label": "Recipient", 
-   "length": 0, 
-   "no_copy": 0, 
-   "options": "Email", 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "fieldname": "recipient",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Recipient",
+   "options": "Email"
+  },
   {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "default": "Not Sent", 
-   "fieldname": "status", 
-   "fieldtype": "Select", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0,
-   "in_list_view": 1, 
-   "label": "Status", 
-   "length": 0, 
-   "no_copy": 0, 
-   "options": "\nNot Sent\nSending\nSent\nError\nExpired", 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "default": "Not Sent",
+   "fieldname": "status",
+   "fieldtype": "Select",
+   "in_list_view": 1,
+   "label": "Status",
+   "options": "\nNot Sent\nSending\nSent\nError\nExpired",
+   "search_index": 1
+  },
   {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "error", 
-   "fieldtype": "Code", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0,
-   "in_list_view": 0, 
-   "label": "Error", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
+   "fieldname": "error",
+   "fieldtype": "Code",
+   "label": "Error"
   }
- ], 
- "hide_heading": 0, 
- "hide_toolbar": 0, 
- "idx": 0, 
- "image_view": 0, 
- "in_create": 0, 
-
- "is_submittable": 0, 
- "issingle": 0, 
- "istable": 1, 
- "max_attachments": 0, 
- "modified": "2016-12-08 14:05:33.578240", 
- "modified_by": "Administrator", 
- "module": "Email", 
- "name": "Email Queue Recipient", 
- "name_case": "", 
- "owner": "Administrator", 
- "permissions": [], 
- "quick_entry": 1, 
- "read_only": 0, 
- "read_only_onload": 0, 
- "sort_field": "modified", 
- "sort_order": "DESC", 
- "track_seen": 0
+ ],
+ "istable": 1,
+ "links": [],
+ "modified": "2022-07-11 16:38:10.644417",
+ "modified_by": "Administrator",
+ "module": "Email",
+ "name": "Email Queue Recipient",
+ "owner": "Administrator",
+ "permissions": [],
+ "quick_entry": 1,
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "states": []
 }
\ No newline at end of file
diff --git a/frappe/email/doctype/email_queue_recipient/email_queue_recipient.py b/frappe/email/doctype/email_queue_recipient/email_queue_recipient.py
index d2e4753ddf..bcb8d9b05d 100644
--- a/frappe/email/doctype/email_queue_recipient/email_queue_recipient.py
+++ b/frappe/email/doctype/email_queue_recipient/email_queue_recipient.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2015, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
diff --git a/frappe/email/doctype/email_rule/email_rule.js b/frappe/email/doctype/email_rule/email_rule.js
index 974bcd4e51..2670ef3e1c 100644
--- a/frappe/email/doctype/email_rule/email_rule.js
+++ b/frappe/email/doctype/email_rule/email_rule.js
@@ -1,8 +1,6 @@
 // Copyright (c) 2017, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Email Rule', {
-	refresh: function(frm) {
-
-	}
+frappe.ui.form.on("Email Rule", {
+	refresh: function (frm) {},
 });
diff --git a/frappe/email/doctype/email_rule/email_rule.json b/frappe/email/doctype/email_rule/email_rule.json
index b4e505b8c6..20e296290d 100644
--- a/frappe/email/doctype/email_rule/email_rule.json
+++ b/frappe/email/doctype/email_rule/email_rule.json
@@ -1,128 +1,49 @@
 {
+ "actions": [],
  "allow_copy": 1,
- "allow_guest_to_view": 0,
- "allow_import": 0,
- "allow_rename": 0,
  "autoname": "field:email_id",
- "beta": 0,
  "creation": "2017-03-13 09:20:56.387135",
- "custom": 0,
- "docstatus": 0,
  "doctype": "DocType",
- "document_type": "",
  "editable_grid": 1,
  "engine": "InnoDB",
+ "field_order": [
+  "email_id",
+  "is_spam"
+ ],
  "fields": [
   {
-   "allow_bulk_edit": 0,
-   "allow_in_quick_entry": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
    "fieldname": "email_id",
    "fieldtype": "Data",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
    "label": "Email ID",
-   "length": 0,
-   "no_copy": 0,
    "options": "Email",
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "translatable": 0,
    "unique": 1
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_in_quick_entry": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
+   "default": "0",
    "fieldname": "is_spam",
    "fieldtype": "Check",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
-   "label": "Is Spam",
-   "length": 0,
-   "no_copy": 0,
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "translatable": 0,
-   "unique": 0
+   "label": "Is Spam"
   }
  ],
- "has_web_view": 0,
- "hide_heading": 0,
- "hide_toolbar": 0,
- "idx": 0,
- "image_view": 0,
- "in_create": 0,
- "is_submittable": 0,
- "issingle": 0,
- "istable": 0,
- "max_attachments": 0,
- "modified": "2018-08-31 06:08:12.645682",
+ "links": [],
+ "modified": "2022-08-03 12:20:51.443237",
  "modified_by": "Administrator",
  "module": "Email",
  "name": "Email Rule",
- "name_case": "",
  "owner": "Administrator",
  "permissions": [
   {
-   "amend": 0,
-   "cancel": 0,
-   "create": 0,
-   "delete": 0,
    "email": 1,
    "export": 1,
-   "if_owner": 0,
-   "import": 0,
-   "permlevel": 0,
    "print": 1,
    "read": 1,
    "report": 1,
    "role": "System Manager",
-   "set_user_permissions": 0,
-   "share": 1,
-   "submit": 0,
-   "write": 0
+   "share": 1
   }
  ],
  "quick_entry": 1,
- "read_only": 0,
- "read_only_onload": 0,
- "show_name_in_global_search": 0,
  "sort_field": "modified",
  "sort_order": "DESC",
- "track_changes": 0,
- "track_seen": 0,
- "track_views": 0
+ "states": []
 }
\ No newline at end of file
diff --git a/frappe/email/doctype/email_rule/email_rule.py b/frappe/email/doctype/email_rule/email_rule.py
index f603aed77e..ab04632280 100644
--- a/frappe/email/doctype/email_rule/email_rule.py
+++ b/frappe/email/doctype/email_rule/email_rule.py
@@ -1,8 +1,6 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2017, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
-import frappe
 from frappe.model.document import Document
 
 
diff --git a/frappe/email/doctype/email_rule/test_email_rule.py b/frappe/email/doctype/email_rule/test_email_rule.py
index 2cea421bd6..1da5d34d6b 100644
--- a/frappe/email/doctype/email_rule/test_email_rule.py
+++ b/frappe/email/doctype/email_rule/test_email_rule.py
@@ -1,10 +1,7 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2017, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
 import unittest
 
-import frappe
-
 
 class TestEmailRule(unittest.TestCase):
 	pass
diff --git a/frappe/email/doctype/email_template/email_template.js b/frappe/email/doctype/email_template/email_template.js
index 62ce4d94ad..33327005a5 100644
--- a/frappe/email/doctype/email_template/email_template.js
+++ b/frappe/email/doctype/email_template/email_template.js
@@ -1,8 +1,6 @@
 // Copyright (c) 2018, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Email Template', {
-	refresh: function() {
-
-	}
+frappe.ui.form.on("Email Template", {
+	refresh: function () {},
 });
diff --git a/frappe/email/doctype/email_template/test_email_template.py b/frappe/email/doctype/email_template/test_email_template.py
index d37b5497fb..291f7e1df0 100644
--- a/frappe/email/doctype/email_template/test_email_template.py
+++ b/frappe/email/doctype/email_template/test_email_template.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2018, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
 import unittest
diff --git a/frappe/email/doctype/email_unsubscribe/email_unsubscribe.js b/frappe/email/doctype/email_unsubscribe/email_unsubscribe.js
index 9a022cf4ca..5f1e28e9b9 100644
--- a/frappe/email/doctype/email_unsubscribe/email_unsubscribe.js
+++ b/frappe/email/doctype/email_unsubscribe/email_unsubscribe.js
@@ -1,8 +1,6 @@
 // Copyright (c) 2016, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Email Unsubscribe', {
-	refresh: function(frm) {
-
-	}
+frappe.ui.form.on("Email Unsubscribe", {
+	refresh: function (frm) {},
 });
diff --git a/frappe/email/doctype/email_unsubscribe/email_unsubscribe.json b/frappe/email/doctype/email_unsubscribe/email_unsubscribe.json
index bf633ead4b..38df531c35 100644
--- a/frappe/email/doctype/email_unsubscribe/email_unsubscribe.json
+++ b/frappe/email/doctype/email_unsubscribe/email_unsubscribe.json
@@ -1,175 +1,70 @@
 {
- "allow_copy": 0,
- "allow_import": 0,
- "allow_rename": 0,
- "beta": 0,
+ "actions": [],
  "creation": "2015-03-18 09:41:20.216320",
- "custom": 0,
- "docstatus": 0,
  "doctype": "DocType",
  "document_type": "System",
- "editable_grid": 0,
  "engine": "InnoDB",
+ "field_order": [
+  "email",
+  "reference_doctype",
+  "reference_name",
+  "global_unsubscribe"
+ ],
  "fields": [
   {
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
    "fieldname": "email",
    "fieldtype": "Data",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
    "in_list_view": 1,
-   "in_standard_filter": 0,
    "label": "Email",
-   "length": 0,
-   "no_copy": 0,
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
    "reqd": 1,
-   "search_index": 1,
-   "set_only_once": 0,
-   "unique": 0
+   "search_index": 1
   },
   {
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
    "fieldname": "reference_doctype",
    "fieldtype": "Link",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
    "in_list_view": 1,
    "in_standard_filter": 1,
    "label": "Reference Document Type",
-   "length": 0,
-   "no_copy": 0,
-   "options": "DocType",
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "unique": 0
+   "options": "DocType"
   },
   {
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
    "fieldname": "reference_name",
    "fieldtype": "Dynamic Link",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
    "in_list_view": 1,
-   "in_standard_filter": 0,
    "label": "Reference Name",
-   "length": 0,
-   "no_copy": 0,
-   "options": "reference_doctype",
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "unique": 0
+   "options": "reference_doctype"
   },
   {
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
+   "default": "0",
    "fieldname": "global_unsubscribe",
    "fieldtype": "Check",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
    "in_list_view": 1,
-   "in_standard_filter": 0,
-   "label": "Global Unsubscribe",
-   "length": 0,
-   "no_copy": 0,
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "unique": 0
+   "label": "Global Unsubscribe"
   }
  ],
- "hide_heading": 0,
- "hide_toolbar": 0,
- "idx": 0,
- "image_view": 0,
- "in_create": 0,
-
- "is_submittable": 0,
- "issingle": 0,
- "istable": 0,
- "max_attachments": 0,
- "modified": "2019-09-05 14:22:27.664645",
+ "links": [],
+ "modified": "2022-08-03 12:20:51.694626",
  "modified_by": "Administrator",
  "module": "Email",
  "name": "Email Unsubscribe",
- "name_case": "",
  "owner": "Administrator",
  "permissions": [
   {
-   "amend": 0,
-   "apply_user_permissions": 0,
-   "cancel": 0,
    "create": 1,
    "delete": 1,
    "email": 1,
    "export": 1,
-   "if_owner": 0,
-   "import": 0,
-   "is_custom": 0,
-   "permlevel": 0,
    "print": 1,
    "read": 1,
    "report": 1,
    "role": "System Manager",
-   "set_user_permissions": 0,
    "share": 1,
-   "submit": 0,
    "write": 1
   }
  ],
  "quick_entry": 1,
- "read_only": 0,
- "read_only_onload": 0,
  "sort_field": "modified",
  "sort_order": "DESC",
- "track_changes": 1,
- "track_seen": 0
-}
+ "states": [],
+ "track_changes": 1
+}
\ No newline at end of file
diff --git a/frappe/email/doctype/email_unsubscribe/email_unsubscribe.py b/frappe/email/doctype/email_unsubscribe/email_unsubscribe.py
index 131546a3c2..c9ab17d61d 100644
--- a/frappe/email/doctype/email_unsubscribe/email_unsubscribe.py
+++ b/frappe/email/doctype/email_unsubscribe/email_unsubscribe.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors
 # License: MIT. See LICENSE
 
diff --git a/frappe/email/doctype/email_unsubscribe/test_email_unsubscribe.py b/frappe/email/doctype/email_unsubscribe/test_email_unsubscribe.py
index 7f9173d0b0..9ba99a6690 100644
--- a/frappe/email/doctype/email_unsubscribe/test_email_unsubscribe.py
+++ b/frappe/email/doctype/email_unsubscribe/test_email_unsubscribe.py
@@ -1,10 +1,7 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
 import unittest
 
-import frappe
-
 # test_records = frappe.get_test_records('Email Unsubscribe')
 
 
diff --git a/frappe/email/doctype/newsletter/newsletter.js b/frappe/email/doctype/newsletter/newsletter.js
index 55805ad485..8a81bbaab3 100644
--- a/frappe/email/doctype/newsletter/newsletter.js
+++ b/frappe/email/doctype/newsletter/newsletter.js
@@ -1,105 +1,146 @@
 // Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 // License: GNU General Public License v3. See license.txt
 
-frappe.ui.form.on('Newsletter', {
+frappe.ui.form.on("Newsletter", {
 	refresh(frm) {
 		let doc = frm.doc;
 		let can_write = in_list(frappe.boot.user.can_write, doc.doctype);
 		if (!frm.is_new() && !frm.is_dirty() && !doc.email_sent && can_write) {
-			frm.add_custom_button(__('Send a test email'), () => {
-				frm.events.send_test_email(frm);
-			}, __('Preview'));
+			frm.add_custom_button(
+				__("Send a test email"),
+				() => {
+					frm.events.send_test_email(frm);
+				},
+				__("Preview")
+			);
 
-			frm.add_custom_button(__('Check broken links'), () => {
-				frm.dashboard.set_headline(__('Checking broken links...'));
-				frm.call('find_broken_links').then(r => {
-					frm.dashboard.set_headline('');
-					let links = r.message;
-					if (links && links.length) {
-						let html = '
          ' + links.map(link => `
        • ${link}
          • `).join('') + '
        ';
-						frm.dashboard.set_headline(__("Following links are broken in the email content: {0}", [html]));
-					} else {
-						frm.dashboard.set_headline(__("No broken links found in the email content"));
-						setTimeout(() => {
-							frm.dashboard.set_headline('');
-						}, 3000);
-					}
-				});
-			}, __('Preview'));
-
-			frm.add_custom_button(__('Send now'), () => {
-				if (frm.doc.schedule_send) {
-					frappe.confirm(__("This newsletter was scheduled to send on a later date. Are you sure you want to send it now?"), function () {
-						frm.call('send_emails').then(() => frm.refresh());
+			frm.add_custom_button(
+				__("Check broken links"),
+				() => {
+					frm.dashboard.set_headline(__("Checking broken links..."));
+					frm.call("find_broken_links").then((r) => {
+						frm.dashboard.set_headline("");
+						let links = r.message;
+						if (links && links.length) {
+							let html =
+								"
          " +
+								links.map((link) => `
        • ${link}
          • `).join("") +
+								"
        ";
+							frm.dashboard.set_headline(
+								__("Following links are broken in the email content: {0}", [html])
+							);
+						} else {
+							frm.dashboard.set_headline(
+								__("No broken links found in the email content")
+							);
+							setTimeout(() => {
+								frm.dashboard.set_headline("");
+							}, 3000);
+						}
 					});
-					return;
-				}
-				frappe.confirm(__("Are you sure you want to send this newsletter now?"), function () {
-					frm.call('send_emails').then(() => frm.refresh());
-				});
-			}, __('Send'));
+				},
+				__("Preview")
+			);
 
-			frm.add_custom_button(__('Schedule sending'), () => {
-				frm.events.schedule_send_dialog(frm);
-			}, __('Send'));
+			frm.add_custom_button(
+				__("Send now"),
+				() => {
+					if (frm.doc.schedule_send) {
+						frappe.confirm(
+							__(
+								"This newsletter was scheduled to send on a later date. Are you sure you want to send it now?"
+							),
+							function () {
+								frm.events.send_emails(frm);
+							}
+						);
+						return;
+					}
+					frappe.confirm(
+						__("Are you sure you want to send this newsletter now?"),
+						() => {
+							frm.events.send_emails(frm);
+						}
+					);
+				},
+				__("Send")
+			);
+
+			frm.add_custom_button(
+				__("Schedule sending"),
+				() => {
+					frm.events.schedule_send_dialog(frm);
+				},
+				__("Send")
+			);
 		}
 
-		frm.events.setup_dashboard(frm);
-		frm.events.setup_sending_status(frm);
+		frm.events.update_sending_status(frm);
 
 		if (frm.is_new() && !doc.sender_email) {
 			let { fullname, email } = frappe.user_info(doc.owner);
-			frm.set_value('sender_email', email);
-			frm.set_value('sender_name', fullname);
+			frm.set_value("sender_email", email);
+			frm.set_value("sender_name", fullname);
 		}
 
-		frm.trigger('update_schedule_message');
+		frm.trigger("update_schedule_message");
+	},
+
+	send_emails(frm) {
+		frappe.dom.freeze(__("Queuing emails..."));
+		frm.call("send_emails").then(() => {
+			frm.refresh();
+			frappe.dom.unfreeze();
+			frappe.show_alert(
+				__("Queued {0} emails", [frappe.utils.shorten_number(frm.doc.total_recipients)])
+			);
+		});
 	},
 
 	schedule_send_dialog(frm) {
 		let hours = frappe.utils.range(24);
-		let time_slots = hours.map(hour => {
-			return `${(hour + '').padStart(2, '0')}:00`;
+		let time_slots = hours.map((hour) => {
+			return `${(hour + "").padStart(2, "0")}:00`;
 		});
 		let d = new frappe.ui.Dialog({
-			title: __('Schedule Newsletter'),
+			title: __("Schedule Newsletter"),
 			fields: [
 				{
-					label: __('Date'),
-					fieldname: 'date',
-					fieldtype: 'Date',
+					label: __("Date"),
+					fieldname: "date",
+					fieldtype: "Date",
 					options: {
-						minDate: new Date()
-					}
+						minDate: new Date(),
+					},
 				},
 				{
-					label: __('Time'),
-					fieldname: 'time',
-					fieldtype: 'Select',
+					label: __("Time"),
+					fieldname: "time",
+					fieldtype: "Select",
 					options: time_slots,
 				},
 			],
-			primary_action_label: __('Schedule'),
+			primary_action_label: __("Schedule"),
 			primary_action({ date, time }) {
-				frm.set_value('schedule_sending', 1);
-				frm.set_value('schedule_send', `${date} ${time}:00`);
+				frm.set_value("schedule_sending", 1);
+				frm.set_value("schedule_send", `${date} ${time}:00`);
 				d.hide();
 				frm.save();
 			},
-			secondary_action_label: __('Cancel Scheduling'),
+			secondary_action_label: __("Cancel Scheduling"),
 			secondary_action() {
-				frm.set_value('schedule_sending', 0);
-				frm.set_value('schedule_send', '');
+				frm.set_value("schedule_sending", 0);
+				frm.set_value("schedule_send", "");
 				d.hide();
 				frm.save();
-			}
+			},
 		});
 		if (frm.doc.schedule_sending) {
-			let parts = frm.doc.schedule_send.split(' ');
+			let parts = frm.doc.schedule_send.split(" ");
 			if (parts.length === 2) {
 				let [date, time] = parts;
-				d.set_value('date', date);
-				d.set_value('time', time.slice(0, 5));
+				d.set_value("date", date);
+				d.set_value("time", time.slice(0, 5));
 			}
 		}
 		d.show();
@@ -107,98 +148,63 @@ frappe.ui.form.on('Newsletter', {
 
 	send_test_email(frm) {
 		let d = new frappe.ui.Dialog({
-			title: __('Send Test Email'),
+			title: __("Send Test Email"),
 			fields: [
 				{
-					label: __('Email'),
-					fieldname: 'email',
-					fieldtype: 'Data',
-					options: 'Email',
-				}
+					label: __("Email"),
+					fieldname: "email",
+					fieldtype: "Data",
+					options: "Email",
+				},
 			],
-			primary_action_label: __('Send'),
+			primary_action_label: __("Send"),
 			primary_action({ email }) {
-				d.get_primary_btn().text(__('Sending...')).prop('disabled', true);
-				frm.call('send_test_email', { email })
-					.then(() => {
-						d.get_primary_btn().text(__('Send again')).prop('disabled', false);
-					});
-			}
+				d.get_primary_btn().text(__("Sending...")).prop("disabled", true);
+				frm.call("send_test_email", { email }).then(() => {
+					d.get_primary_btn().text(__("Send again")).prop("disabled", false);
+				});
+			},
 		});
 		d.show();
 	},
 
-	setup_dashboard(frm) {
-		if (!frm.doc.__islocal && cint(frm.doc.email_sent)
-			&& frm.doc.__onload && frm.doc.__onload.status_count) {
-			var stat = frm.doc.__onload.status_count;
-			var total = frm.doc.scheduled_to_send;
-			if (total) {
-				$.each(stat, function (k, v) {
-					stat[k] = flt(v * 100 / total, 2) + '%';
-				});
-
-				frm.dashboard.add_progress("Status", [
-					{
-						title: stat["Not Sent"] + " Queued",
-						width: stat["Not Sent"],
-						progress_class: "progress-bar-info"
-					},
-					{
-						title: stat["Sent"] + " Sent",
-						width: stat["Sent"],
-						progress_class: "progress-bar-success"
-					},
-					{
-						title: stat["Sending"] + " Sending",
-						width: stat["Sending"],
-						progress_class: "progress-bar-warning"
-					},
-					{
-						title: stat["Error"] + "% Error",
-						width: stat["Error"],
-						progress_class: "progress-bar-danger"
-					}
-				]);
-			}
-		}
-	},
-
-	setup_sending_status(frm) {
-		frm.call('get_sending_status').then(r => {
-			if (r.message) {
-				frm.events.update_sending_progress(frm, r.message.sent, r.message.total);
-			}
-			if (r.message.sent >= r.message.total) {
+	async update_sending_status(frm) {
+		if (frm.doc.email_sent && frm.$wrapper.is(":visible") && !frm.waiting_for_request) {
+			frm.waiting_for_request = true;
+			let res = await frm.call("get_sending_status");
+			frm.waiting_for_request = false;
+			let stats = res.message;
+			stats && frm.events.update_sending_progress(frm, stats);
+			if (
+				stats.sent + stats.error >= frm.doc.total_recipients ||
+				(!stats.total && !stats.emails_queued)
+			) {
+				frm.sending_status && clearInterval(frm.sending_status);
+				frm.sending_status = null;
 				return;
 			}
-			if (frm.sending_status) return;
+		}
 
-			frm.sending_status = setInterval(() => {
-				if (frm.doc.email_sent && frm.$wrapper.is(':visible')) {
-					frm.call('get_sending_status').then(r => {
-						if (r.message) {
-							let { sent, total } = r.message;
-							frm.events.update_sending_progress(frm, sent, total);
-
-							if (sent >= total) {
-								clearInterval(frm.sending_status);
-								frm.sending_status = null;
-								return;
-							}
-						}
-					});
-				}
-			}, 5000);
-		});
+		if (frm.sending_status) return;
+		frm.sending_status = setInterval(() => frm.events.update_sending_status(frm), 5000);
 	},
 
-	update_sending_progress(frm, sent, total) {
-		if (sent >= total) {
+	update_sending_progress(frm, stats) {
+		if (stats.sent + stats.error >= frm.doc.total_recipients || !frm.doc.email_sent) {
+			frm.doc.email_sent && frm.page.set_indicator(__("Sent"), "green");
 			frm.dashboard.hide_progress();
 			return;
 		}
-		frm.dashboard.show_progress(__('Sending emails'), sent * 100 / total, __("{0} of {1} sent", [sent, total]));
+		if (stats.total) {
+			frm.page.set_indicator(__("Sending"), "blue");
+			frm.dashboard.show_progress(
+				__("Sending emails"),
+				(stats.sent * 100) / frm.doc.total_recipients,
+				__("{0} of {1} sent", [stats.sent, frm.doc.total_recipients])
+			);
+		} else if (stats.emails_queued) {
+			frm.page.set_indicator(__("Queued"), "blue");
+		}
 	},
 
 	on_hide(frm) {
@@ -211,9 +217,11 @@ frappe.ui.form.on('Newsletter', {
 	update_schedule_message(frm) {
 		if (!frm.doc.email_sent && frm.doc.schedule_send) {
 			let datetime = frappe.datetime.global_date_format(frm.doc.schedule_send);
-			frm.dashboard.set_headline_alert(__('This newsletter is scheduled to be sent on {0}', [datetime.bold()]));
+			frm.dashboard.set_headline_alert(
+				__("This newsletter is scheduled to be sent on {0}", [datetime.bold()])
+			);
 		} else {
 			frm.dashboard.clear_headline();
 		}
-	}
+	},
 });
diff --git a/frappe/email/doctype/newsletter/newsletter.py b/frappe/email/doctype/newsletter/newsletter.py
index 45a4539866..b0cbb1993d 100644
--- a/frappe/email/doctype/newsletter/newsletter.py
+++ b/frappe/email/doctype/newsletter/newsletter.py
@@ -1,12 +1,12 @@
 # Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors
 # MIT License. See LICENSE
 
-from typing import Dict, List
 
 import frappe
 import frappe.utils
 from frappe import _
 from frappe.email.doctype.email_group.email_group import add_subscribers
+from frappe.utils.safe_exec import is_job_queued
 from frappe.utils.verified_command import get_signed_params, verify_request
 from frappe.website.website_generator import WebsiteGenerator
 
@@ -21,7 +21,7 @@ class Newsletter(WebsiteGenerator):
 		self.validate_publishing()
 
 	@property
-	def newsletter_recipients(self) -> List[str]:
+	def newsletter_recipients(self) -> list[str]:
 		if getattr(self, "_recipients", None) is None:
 			self._recipients = self.get_recipients()
 		return self._recipients
@@ -36,13 +36,19 @@ class Newsletter(WebsiteGenerator):
 			order_by="status",
 		)
 		sent = 0
+		error = 0
 		total = 0
 		for row in count_by_status:
 			if row.status == "Sent":
 				sent = row.count
+			elif row.status == "Error":
+				error = row.count
 			total += row.count
-
-		return {"sent": sent, "total": total}
+		emails_queued = is_job_queued(
+			job_name=frappe.utils.get_job_name("send_bulk_emails_for", self.doctype, self.name),
+			queue="long",
+		)
+		return {"sent": sent, "error": error, "total": total, "emails_queued": emails_queued}
 
 	@frappe.whitelist()
 	def send_test_email(self, email):
@@ -66,7 +72,7 @@ class Newsletter(WebsiteGenerator):
 				response = requests.head(url, verify=False, timeout=5)
 				if response.status_code >= 400:
 					broken_links.append(url)
-			except:
+			except Exception:
 				broken_links.append(url)
 		return broken_links
 
@@ -76,7 +82,6 @@ class Newsletter(WebsiteGenerator):
 		self.schedule_sending = False
 		self.schedule_send = None
 		self.queue_all()
-		frappe.msgprint(_("Email queued to {0} recipients").format(self.total_recipients))
 
 	def validate_send(self):
 		"""Validate if Newsletter can be sent."""
@@ -112,7 +117,7 @@ class Newsletter(WebsiteGenerator):
 		if self.send_webview_link and not self.published:
 			frappe.throw(_("Newsletter must be published to send webview link in email"))
 
-	def get_linked_email_queue(self) -> List[str]:
+	def get_linked_email_queue(self) -> list[str]:
 		"""Get list of email queue linked to this newsletter."""
 		return frappe.get_all(
 			"Email Queue",
@@ -123,8 +128,8 @@ class Newsletter(WebsiteGenerator):
 			pluck="name",
 		)
 
-	def get_success_recipients(self) -> List[str]:
-		"""Recipients who have already recieved the newsletter.
+	def get_success_recipients(self) -> list[str]:
+		"""Recipients who have already received the newsletter.
 
 		Couldn't think of a better name ;)
 		"""
@@ -132,16 +137,17 @@ class Newsletter(WebsiteGenerator):
 			"Email Queue Recipient",
 			filters={
 				"status": ("in", ["Not Sent", "Sending", "Sent"]),
-				"parentfield": ("in", self.get_linked_email_queue()),
+				"parent": ("in", self.get_linked_email_queue()),
 			},
 			pluck="recipient",
 		)
 
-	def get_pending_recipients(self) -> List[str]:
+	def get_pending_recipients(self) -> list[str]:
 		"""Get list of pending recipients of the newsletter. These
 		recipients may not have receive the newsletter in the previous iteration.
 		"""
-		return [x for x in self.newsletter_recipients if x not in self.get_success_recipients()]
+		success_recipients = set(self.get_success_recipients())
+		return [x for x in self.newsletter_recipients if x not in success_recipients]
 
 	def queue_all(self):
 		"""Queue Newsletter to all the recipients generated from the `Email Group` table"""
@@ -156,11 +162,11 @@ class Newsletter(WebsiteGenerator):
 		self.total_recipients = len(recipients)
 		self.save()
 
-	def get_newsletter_attachments(self) -> List[Dict[str, str]]:
+	def get_newsletter_attachments(self) -> list[dict[str, str]]:
 		"""Get list of attachments on current Newsletter"""
 		return [{"file_url": row.attachment} for row in self.attachments]
 
-	def send_newsletter(self, emails: List[str]):
+	def send_newsletter(self, emails: list[str]):
 		"""Trigger email generation for `emails` and add it in Email Queue."""
 		attachments = self.get_newsletter_attachments()
 		sender = self.send_from or frappe.utils.get_formatted_email(self.owner)
@@ -197,7 +203,7 @@ class Newsletter(WebsiteGenerator):
 
 		return frappe.render_template(message, {"doc": self.as_dict()})
 
-	def get_recipients(self) -> List[str]:
+	def get_recipients(self) -> list[str]:
 		"""Get recipients from Email Group"""
 		emails = frappe.get_all(
 			"Email Group Member",
@@ -206,7 +212,7 @@ class Newsletter(WebsiteGenerator):
 		)
 		return list(set(emails))
 
-	def get_email_groups(self) -> List[str]:
+	def get_email_groups(self) -> list[str]:
 		# wondering why the 'or'? i can't figure out why both aren't equivalent - @gavin
 		return [x.email_group for x in self.email_group] or frappe.get_all(
 			"Newsletter Email Group",
@@ -214,7 +220,7 @@ class Newsletter(WebsiteGenerator):
 			pluck="email_group",
 		)
 
-	def get_attachments(self) -> List[Dict[str, str]]:
+	def get_attachments(self) -> list[dict[str, str]]:
 		return frappe.get_all(
 			"File",
 			fields=["name", "file_name", "file_url", "is_private"],
@@ -237,7 +243,7 @@ def confirmed_unsubscribe(email, group):
 
 
 @frappe.whitelist(allow_guest=True)
-def subscribe(email, email_group=_("Website")):
+def subscribe(email, email_group=_("Website")):  # noqa
 	"""API endpoint to subscribe an email to a particular email group. Triggers a confirmation email."""
 
 	# build subscription confirmation URL
@@ -267,8 +273,8 @@ def subscribe(email, email_group=_("Website")):
 			_("Click here to verify"),
 		)
 		content = """
-			
        {0}. {1}.
        
-			
        {3}
        
+			
        {}. {}.
        
+			
        {}
        
 		""".format(
 			*translatable_content
 		)
@@ -282,7 +288,7 @@ def subscribe(email, email_group=_("Website")):
 
 
 @frappe.whitelist(allow_guest=True)
-def confirm_subscription(email, email_group=_("Website")):
+def confirm_subscription(email, email_group=_("Website")):  # noqa
 	"""API endpoint to confirm email subscription.
 	This endpoint is called when user clicks on the link sent to their mail.
 	"""
@@ -329,19 +335,17 @@ def send_scheduled_email():
 		pluck="name",
 	)
 
-	for newsletter in scheduled_newsletter:
+	for newsletter_name in scheduled_newsletter:
 		try:
-			frappe.get_doc("Newsletter", newsletter).queue_all()
+			newsletter = frappe.get_doc("Newsletter", newsletter_name)
+			newsletter.queue_all()
 
 		except Exception:
 			frappe.db.rollback()
 
 			# wasn't able to send emails :(
-			frappe.db.set_value("Newsletter", newsletter, "email_sent", 0)
-			message = (
-				f"Newsletter {newsletter} failed to send" "\n\n" f"Traceback: {frappe.get_traceback()}"
-			)
-			frappe.log_error(title="Send Newsletter", message=message)
+			frappe.db.set_value("Newsletter", newsletter_name, "email_sent", 0)
+			newsletter.log_error("Failed to send newsletter")
 
 		if not frappe.flags.in_test:
 			frappe.db.commit()
diff --git a/frappe/email/doctype/newsletter/newsletter_list.js b/frappe/email/doctype/newsletter/newsletter_list.js
index 9ded6148e0..0921de02b4 100644
--- a/frappe/email/doctype/newsletter/newsletter_list.js
+++ b/frappe/email/doctype/newsletter/newsletter_list.js
@@ -1,12 +1,12 @@
-frappe.listview_settings['Newsletter'] = {
+frappe.listview_settings["Newsletter"] = {
 	add_fields: ["subject", "email_sent", "schedule_sending"],
-	get_indicator: function(doc) {
+	get_indicator: function (doc) {
 		if (doc.email_sent) {
 			return [__("Sent"), "green", "email_sent,=,Yes"];
 		} else if (doc.schedule_sending) {
-			return [__("Scheduled"), "orange", "email_sent,=,No|schedule_sending,=,Yes"];
+			return [__("Scheduled"), "purple", "email_sent,=,No|schedule_sending,=,Yes"];
 		} else {
-			return [__("Not Sent"), "orange", "email_sent,=,No"];
+			return [__("Not Sent"), "gray", "email_sent,=,No"];
 		}
-	}
+	},
 };
diff --git a/frappe/email/doctype/newsletter/test_newsletter.py b/frappe/email/doctype/newsletter/test_newsletter.py
index c62b7e84aa..524289db7f 100644
--- a/frappe/email/doctype/newsletter/test_newsletter.py
+++ b/frappe/email/doctype/newsletter/test_newsletter.py
@@ -1,13 +1,10 @@
 # Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors
 # MIT License. See LICENSE
 
-import unittest
 from random import choice
-from typing import Union
 from unittest.mock import MagicMock, PropertyMock, patch
 
 import frappe
-from frappe.desk.form.load import run_onload
 from frappe.email.doctype.newsletter.exceptions import (
 	NewsletterAlreadySentError,
 	NoRecipientFoundError,
@@ -18,9 +15,9 @@ from frappe.email.doctype.newsletter.newsletter import (
 	send_scheduled_email,
 )
 from frappe.email.queue import flush
+from frappe.tests.utils import FrappeTestCase
 from frappe.utils import add_days, getdate
 
-test_dependencies = ["Email Group"]
 emails = [
 	"test_subscriber1@example.com",
 	"test_subscriber2@example.com",
@@ -64,17 +61,24 @@ class TestNewsletterMixin:
 		for email in emails:
 			doctype = "Email Group Member"
 			email_filters = {"email": email, "email_group": "_Test Email Group"}
+
+			savepoint = "setup_email_group"
+			frappe.db.savepoint(savepoint)
+
 			try:
 				frappe.get_doc(
 					{
 						"doctype": doctype,
 						**email_filters,
 					}
-				).insert()
+				).insert(ignore_if_duplicate=True)
 			except Exception:
+				frappe.db.rollback(save_point=savepoint)
 				frappe.db.update(doctype, email_filters, "unsubscribed", 0)
 
-	def send_newsletter(self, published=0, schedule_send=None) -> Union[str, None]:
+			frappe.db.release_savepoint(savepoint)
+
+	def send_newsletter(self, published=0, schedule_send=None) -> str | None:
 		frappe.db.delete("Email Queue")
 		frappe.db.delete("Email Queue Recipient")
 		frappe.db.delete("Newsletter")
@@ -128,7 +132,7 @@ class TestNewsletterMixin:
 		return newsletter
 
 
-class TestNewsletter(TestNewsletterMixin, unittest.TestCase):
+class TestNewsletter(TestNewsletterMixin, FrappeTestCase):
 	def test_send(self):
 		self.send_newsletter()
 
@@ -221,3 +225,24 @@ class TestNewsletter(TestNewsletterMixin, unittest.TestCase):
 
 		newsletter.reload()
 		self.assertEqual(newsletter.email_sent, 0)
+
+	def test_retry_partially_sent_newsletter(self):
+		frappe.db.delete("Email Queue")
+		frappe.db.delete("Email Queue Recipient")
+		frappe.db.delete("Newsletter")
+
+		newsletter = self.get_newsletter()
+		newsletter.send_emails()
+		email_queue_list = [frappe.get_doc("Email Queue", e.name) for e in frappe.get_all("Email Queue")]
+		self.assertEqual(len(email_queue_list), 4)
+
+		# emulate partial send
+		email_queue_list[0].status = "Error"
+		email_queue_list[0].recipients[0].status = "Error"
+		email_queue_list[0].save()
+		newsletter.email_sent = False
+
+		# retry
+		newsletter.send_emails()
+		email_queue_list = [frappe.get_doc("Email Queue", e.name) for e in frappe.get_all("Email Queue")]
+		self.assertEqual(len(email_queue_list), 5)
diff --git a/frappe/email/doctype/newsletter_email_group/newsletter_email_group.py b/frappe/email/doctype/newsletter_email_group/newsletter_email_group.py
index b7a00ac7d2..41ada8a491 100644
--- a/frappe/email/doctype/newsletter_email_group/newsletter_email_group.py
+++ b/frappe/email/doctype/newsletter_email_group/newsletter_email_group.py
@@ -1,8 +1,6 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2015, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
-import frappe
 from frappe.model.document import Document
 
 
diff --git a/frappe/email/doctype/notification/notification.js b/frappe/email/doctype/notification/notification.js
index f14447707f..4e3b1eae53 100644
--- a/frappe/email/doctype/notification/notification.js
+++ b/frappe/email/doctype/notification/notification.js
@@ -1,100 +1,98 @@
 // Copyright (c) 2018, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-this.frm.add_fetch('sender', 'email_id', 'sender_email');
+this.frm.add_fetch("sender", "email_id", "sender_email");
 
-this.frm.fields_dict.sender.get_query = function() {
+this.frm.fields_dict.sender.get_query = function () {
 	return {
 		filters: {
-			enable_outgoing: 1
-		}
+			enable_outgoing: 1,
+		},
 	};
 };
 
 frappe.notification = {
-	setup_fieldname_select: function(frm) {
+	setup_fieldname_select: function (frm) {
 		// get the doctype to update fields
 		if (!frm.doc.document_type) {
 			return;
 		}
 
-		frappe.model.with_doctype(frm.doc.document_type, function() {
-			let get_select_options = function(df, parent_field) {
+		frappe.model.with_doctype(frm.doc.document_type, function () {
+			let get_select_options = function (df, parent_field) {
 				// Append parent_field name along with fieldname for child table fields
-				let select_value = parent_field ? df.fieldname + ',' + parent_field : df.fieldname;
+				let select_value = parent_field ? df.fieldname + "," + parent_field : df.fieldname;
 
 				return {
 					value: select_value,
-					label: df.fieldname + ' (' + __(df.label) + ')'
+					label: df.fieldname + " (" + __(df.label) + ")",
 				};
 			};
 
-			let get_date_change_options = function() {
-				let date_options = $.map(fields, function(d) {
-					return d.fieldtype == 'Date' || d.fieldtype == 'Datetime'
+			let get_date_change_options = function () {
+				let date_options = $.map(fields, function (d) {
+					return d.fieldtype == "Date" || d.fieldtype == "Datetime"
 						? get_select_options(d)
 						: null;
 				});
 				// append creation and modified date to Date Change field
 				return date_options.concat([
-					{ value: 'creation', label: `creation (${__('Created On')})` },
-					{ value: 'modified', label: `modified (${__('Last Modified Date')})` }
+					{ value: "creation", label: `creation (${__("Created On")})` },
+					{ value: "modified", label: `modified (${__("Last Modified Date")})` },
 				]);
 			};
 
-			let fields = frappe.get_doc('DocType', frm.doc.document_type).fields;
-			let options = $.map(fields, function(d) {
+			let fields = frappe.get_doc("DocType", frm.doc.document_type).fields;
+			let options = $.map(fields, function (d) {
 				return in_list(frappe.model.no_value_type, d.fieldtype)
-					? null : get_select_options(d);
+					? null
+					: get_select_options(d);
 			});
 
 			// set value changed options
-			frm.set_df_property('value_changed', 'options', [''].concat(options));
-			frm.set_df_property(
-				'set_property_after_alert',
-				'options',
-				[''].concat(options)
-			);
+			frm.set_df_property("value_changed", "options", [""].concat(options));
+			frm.set_df_property("set_property_after_alert", "options", [""].concat(options));
 
 			// set date changed options
-			frm.set_df_property('date_changed', 'options', get_date_change_options());
+			frm.set_df_property("date_changed", "options", get_date_change_options());
 
 			let receiver_fields = [];
-			if (frm.doc.channel === 'Email') {
-				receiver_fields = $.map(fields, function(d) {
-
+			if (frm.doc.channel === "Email") {
+				receiver_fields = $.map(fields, function (d) {
 					// Add User and Email fields from child into select dropdown
-					if (d.fieldtype == 'Table') {
-						let child_fields = frappe.get_doc('DocType', d.options).fields;
-						return $.map(child_fields, function(df) {
-							return df.options == 'Email' ||
-								(df.options == 'User' && df.fieldtype == 'Link')
-								? get_select_options(df, d.fieldname) : null;
+					if (d.fieldtype == "Table") {
+						let child_fields = frappe.get_doc("DocType", d.options).fields;
+						return $.map(child_fields, function (df) {
+							return df.options == "Email" ||
+								(df.options == "User" && df.fieldtype == "Link")
+								? get_select_options(df, d.fieldname)
+								: null;
 						});
-					// Add User and Email fields from parent into select dropdown
+						// Add User and Email fields from parent into select dropdown
 					} else {
-						return d.options == 'Email' ||
-							(d.options == 'User' && d.fieldtype == 'Link')
-							? get_select_options(d) : null;
+						return d.options == "Email" ||
+							(d.options == "User" && d.fieldtype == "Link")
+							? get_select_options(d)
+							: null;
 					}
 				});
-			} else if (in_list(['WhatsApp', 'SMS'], frm.doc.channel)) {
-				receiver_fields = $.map(fields, function(d) {
-					return d.options == 'Phone' ? get_select_options(d) : null;
+			} else if (in_list(["WhatsApp", "SMS"], frm.doc.channel)) {
+				receiver_fields = $.map(fields, function (d) {
+					return d.options == "Phone" ? get_select_options(d) : null;
 				});
 			}
 
 			// set email recipient options
 			frm.fields_dict.recipients.grid.update_docfield_property(
-				'receiver_by_document_field',
-				'options',
-				[''].concat(["owner"]).concat(receiver_fields)
+				"receiver_by_document_field",
+				"options",
+				[""].concat(["owner"]).concat(receiver_fields)
 			);
 		});
 	},
-	setup_example_message: function(frm) {
-		let template = '';
-		if (frm.doc.channel === 'Email') {
+	setup_example_message: function (frm) {
+		let template = "";
+		if (frm.doc.channel === "Email") {
 			template = `
        Message Example
        
 
 
        <h3>Order Overdue</h3>
@@ -114,7 +112,7 @@ Last comment: {{ comments[-1].comment }} by {{ comments[-1].by }}
 </ul>
 
        
 			`;
-		} else if (in_list(['Slack', 'System Notification', 'SMS'], frm.doc.channel)) {
+		} else if (in_list(["Slack", "System Notification", "SMS"], frm.doc.channel)) {
 			template = `
        Message Example
        
 
 
        *Order Overdue*
@@ -133,71 +131,72 @@ Last comment: {{ comments[-1].comment }} by {{ comments[-1].by }}
 
        `;
 		}
 		if (template) {
-			frm.set_df_property('message_examples', 'options', template);
+			frm.set_df_property("message_examples", "options", template);
 		}
-
-	}
+	},
 };
 
-frappe.ui.form.on('Notification', {
-	onload: function(frm) {
-		frm.set_query('document_type', function() {
+frappe.ui.form.on("Notification", {
+	onload: function (frm) {
+		frm.set_query("document_type", function () {
 			return {
 				filters: {
-					istable: 0
-				}
+					istable: 0,
+				},
 			};
 		});
-		frm.set_query('print_format', function() {
+		frm.set_query("print_format", function () {
 			return {
 				filters: {
-					doc_type: frm.doc.document_type
-				}
+					doc_type: frm.doc.document_type,
+				},
 			};
 		});
 	},
-	refresh: function(frm) {
+	refresh: function (frm) {
 		frappe.notification.setup_fieldname_select(frm);
 		frappe.notification.setup_example_message(frm);
-		frm.get_field('is_standard').toggle(frappe.boot.developer_mode);
-		frm.trigger('event');
+		frm.get_field("is_standard").toggle(frappe.boot.developer_mode);
+		frm.trigger("event");
 	},
-	document_type: function(frm) {
+	document_type: function (frm) {
 		frappe.notification.setup_fieldname_select(frm);
 	},
-	view_properties: function(frm) {
+	view_properties: function (frm) {
 		frappe.route_options = { doc_type: frm.doc.document_type };
-		frappe.set_route('Form', 'Customize Form');
+		frappe.set_route("Form", "Customize Form");
 	},
-	event: function(frm) {
-		if (in_list(['Days Before', 'Days After'], frm.doc.event)) {
-			frm.add_custom_button(__('Get Alerts for Today'), function() {
+	event: function (frm) {
+		if (in_list(["Days Before", "Days After"], frm.doc.event)) {
+			frm.add_custom_button(__("Get Alerts for Today"), function () {
 				frappe.call({
-					method:
-						'frappe.email.doctype.notification.notification.get_documents_for_today',
+					method: "frappe.email.doctype.notification.notification.get_documents_for_today",
 					args: {
-						notification: frm.doc.name
+						notification: frm.doc.name,
 					},
-					callback: function(r) {
+					callback: function (r) {
 						if (r.message) {
 							frappe.msgprint(r.message);
 						} else {
-							frappe.msgprint(__('No alerts for today'));
+							frappe.msgprint(__("No alerts for today"));
 						}
-					}
+					},
 				});
 			});
 		}
 	},
-	channel: function(frm) {
-		frm.toggle_reqd('recipients', frm.doc.channel == 'Email');
+	channel: function (frm) {
+		frm.toggle_reqd("recipients", frm.doc.channel == "Email");
 		frappe.notification.setup_fieldname_select(frm);
 		frappe.notification.setup_example_message(frm);
-		if (frm.doc.channel === 'SMS' && frm.doc.__islocal) {
-			frm.set_df_property('channel',
-				'description', `To use SMS Channel, initialize SMS Settings.`);
+		if (frm.doc.channel === "SMS" && frm.doc.__islocal) {
+			frm.set_df_property(
+				"channel",
+				"description",
+				`To use SMS Channel, initialize SMS Settings.`
+			);
 		} else {
-			frm.set_df_property('channel', 'description', ` `);
+			frm.set_df_property("channel", "description", ` `);
 		}
-	}
+	},
 });
diff --git a/frappe/email/doctype/notification/notification.py b/frappe/email/doctype/notification/notification.py
index 5c27eb95eb..8d0857ac60 100644
--- a/frappe/email/doctype/notification/notification.py
+++ b/frappe/email/doctype/notification/notification.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2018, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
@@ -13,7 +12,7 @@ from frappe.desk.doctype.notification_log.notification_log import enqueue_create
 from frappe.integrations.doctype.slack_webhook_url.slack_webhook_url import send_slack_message
 from frappe.model.document import Document
 from frappe.modules.utils import export_module_json, get_doc_module
-from frappe.utils import add_to_date, is_html, nowdate, parse_val, validate_email_address
+from frappe.utils import add_to_date, cast, is_html, nowdate, validate_email_address
 from frappe.utils.jinja import validate_template
 from frappe.utils.safe_exec import get_safe_globals
 
@@ -140,8 +139,8 @@ def get_context(context):
 			if self.channel == "System Notification" or self.send_system_notification:
 				self.create_system_notification(doc, context)
 
-		except:
-			frappe.log_error(title="Failed to send notification", message=frappe.get_traceback())
+		except Exception:
+			self.log_error("Failed to send Notification")
 
 		if self.set_property_after_alert:
 			allow_update = True
@@ -168,7 +167,7 @@ def get_context(context):
 					doc.save(ignore_permissions=True)
 					doc.flags.in_notification_update = False
 			except Exception:
-				frappe.log_error(title="Document update failed", message=frappe.get_traceback())
+				self.log_error("Document update failed")
 
 	def create_system_notification(self, doc, context):
 		subject = self.subject
@@ -368,7 +367,7 @@ def get_context(context):
 			template = ""
 			template_path = os.path.join(os.path.dirname(module.__file__), frappe.scrub(self.name) + extn)
 			if os.path.exists(template_path):
-				with open(template_path, "r") as f:
+				with open(template_path) as f:
 					template = f.read()
 			return template
 
@@ -417,7 +416,7 @@ def trigger_notifications(doc, method=None):
 				frappe.db.commit()
 
 
-def evaluate_alert(doc, alert, event):
+def evaluate_alert(doc: Document, alert, event):
 	from jinja2 import TemplateError
 
 	try:
@@ -433,14 +432,14 @@ def evaluate_alert(doc, alert, event):
 		if event == "Value Change" and not doc.is_new():
 			if not frappe.db.has_column(doc.doctype, alert.value_changed):
 				alert.db_set("enabled", 0)
-				frappe.log_error("Notification {0} has been disabled due to missing field".format(alert.name))
+				alert.log_error(f"Notification {alert.name} has been disabled due to missing field")
 				return
 
 			doc_before_save = doc.get_doc_before_save()
 			field_value_before_save = doc_before_save.get(alert.value_changed) if doc_before_save else None
 
-			field_value_before_save = parse_val(field_value_before_save)
-			if doc.get(alert.value_changed) == field_value_before_save:
+			fieldtype = doc.meta.get_field(alert.value_changed).fieldtype
+			if cast(fieldtype, doc.get(alert.value_changed)) == cast(fieldtype, field_value_before_save):
 				# value not changed
 				return
 
diff --git a/frappe/email/doctype/notification/test_notification.py b/frappe/email/doctype/notification/test_notification.py
index 4d8b26c559..0f570b1fd3 100644
--- a/frappe/email/doctype/notification/test_notification.py
+++ b/frappe/email/doctype/notification/test_notification.py
@@ -1,7 +1,7 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2018, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
 import unittest
+from contextlib import contextmanager
 
 import frappe
 import frappe.utils
@@ -11,6 +11,15 @@ from frappe.desk.form import assign_to
 test_dependencies = ["User", "Notification"]
 
 
+@contextmanager
+def get_test_notification(config):
+	try:
+		notification = frappe.get_doc(doctype="Notification", **config).insert()
+		yield notification
+	finally:
+		notification.delete()
+
+
 class TestNotification(unittest.TestCase):
 	def setUp(self):
 		frappe.db.delete("Email Queue")
@@ -84,7 +93,7 @@ class TestNotification(unittest.TestCase):
 	def test_condition(self):
 		"""Check notification is triggered based on a condition."""
 		event = frappe.new_doc("Event")
-		event.subject = ("test",)
+		event.subject = "test"
 		event.event_type = "Private"
 		event.starts_on = "2014-06-06 12:00:00"
 		event.insert()
@@ -137,7 +146,7 @@ class TestNotification(unittest.TestCase):
 
 	def test_value_changed(self):
 		event = frappe.new_doc("Event")
-		event.subject = ("test",)
+		event.subject = "test"
 		event.event_type = "Private"
 		event.starts_on = "2014-06-06 12:00:00"
 		event.insert()
@@ -186,7 +195,7 @@ class TestNotification(unittest.TestCase):
 		frappe.db.commit()
 
 		event = frappe.new_doc("Event")
-		event.subject = ("test-2",)
+		event.subject = "test-2"
 		event.event_type = "Private"
 		event.starts_on = "2014-06-06 12:00:00"
 		event.insert()
@@ -200,9 +209,8 @@ class TestNotification(unittest.TestCase):
 		event.delete()
 
 	def test_date_changed(self):
-
 		event = frappe.new_doc("Event")
-		event.subject = ("test",)
+		event.subject = "test"
 		event.event_type = "Private"
 		event.starts_on = "2014-01-01 12:00:00"
 		event.insert()
@@ -345,6 +353,31 @@ class TestNotification(unittest.TestCase):
 		self.assertTrue("test2@example.com" in recipients)
 		self.assertTrue("test1@example.com" in recipients)
 
+	def test_notification_value_change_casted_types(self):
+		"""Make sure value change event dont fire because of incorrect type comparisons."""
+		frappe.set_user("Administrator")
+
+		notification = {
+			"document_type": "User",
+			"subject": "User changed birthdate",
+			"event": "Value Change",
+			"channel": "System Notification",
+			"value_changed": "birth_date",
+			"recipients": [{"receiver_by_document_field": "email"}],
+		}
+
+		with get_test_notification(notification) as n:
+			frappe.db.delete("Notification Log", {"subject": n.subject})
+
+			user = frappe.get_doc("User", "test@example.com")
+			user.birth_date = frappe.utils.add_days(user.birth_date, 1)
+			user.save()
+
+			user.reload()
+			user.birth_date = frappe.utils.getdate(user.birth_date)
+			user.save()
+			self.assertEqual(1, frappe.db.count("Notification Log", {"subject": n.subject}))
+
 	@classmethod
 	def tearDownClass(cls):
 		frappe.delete_doc_if_exists("Notification", "ToDo Status Update")
diff --git a/frappe/email/doctype/notification_recipient/notification_recipient.py b/frappe/email/doctype/notification_recipient/notification_recipient.py
index 9de15f46c0..75bb274599 100644
--- a/frappe/email/doctype/notification_recipient/notification_recipient.py
+++ b/frappe/email/doctype/notification_recipient/notification_recipient.py
@@ -1,8 +1,6 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2018, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
-import frappe
 from frappe.model.document import Document
 
 
diff --git a/frappe/email/doctype/unhandled_email/test_unhandled_email.py b/frappe/email/doctype/unhandled_email/test_unhandled_email.py
index 694b3e03a6..debc52d685 100644
--- a/frappe/email/doctype/unhandled_email/test_unhandled_email.py
+++ b/frappe/email/doctype/unhandled_email/test_unhandled_email.py
@@ -1,10 +1,7 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
 import unittest
 
-import frappe
-
 # test_records = frappe.get_test_records('Unhandled Emails')
 
 
diff --git a/frappe/email/doctype/unhandled_email/unhandled_email.json b/frappe/email/doctype/unhandled_email/unhandled_email.json
index de4407f38f..d904536936 100644
--- a/frappe/email/doctype/unhandled_email/unhandled_email.json
+++ b/frappe/email/doctype/unhandled_email/unhandled_email.json
@@ -1,212 +1,60 @@
 {
- "allow_copy": 0, 
- "allow_guest_to_view": 0, 
- "allow_import": 0, 
- "allow_rename": 0, 
- "beta": 0, 
- "creation": "2016-04-14 09:41:45.892975", 
- "custom": 0, 
- "docstatus": 0, 
- "doctype": "DocType", 
- "document_type": "Setup", 
- "editable_grid": 0, 
+ "actions": [],
+ "creation": "2016-04-14 09:41:45.892975",
+ "doctype": "DocType",
+ "document_type": "Setup",
+ "engine": "InnoDB",
+ "field_order": [
+  "email_account",
+  "uid",
+  "reason",
+  "message_id",
+  "raw"
+ ],
  "fields": [
   {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "email_account", 
-   "fieldtype": "Link", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Email Account", 
-   "length": 0, 
-   "no_copy": 0, 
-   "options": "Email Account", 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "fieldname": "email_account",
+   "fieldtype": "Link",
+   "in_list_view": 1,
+   "label": "Email Account",
+   "options": "Email Account"
+  },
   {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "uid", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "UID", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "fieldname": "uid",
+   "fieldtype": "Data",
+   "label": "UID"
+  },
   {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "reason", 
-   "fieldtype": "Long Text", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Reason", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "fieldname": "reason",
+   "fieldtype": "Long Text",
+   "in_list_view": 1,
+   "label": "Reason"
+  },
   {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "message_id", 
-   "fieldtype": "Code", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Message-id", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "fieldname": "message_id",
+   "fieldtype": "Code",
+   "label": "Message-id"
+  },
   {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "raw", 
-   "fieldtype": "Code", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Raw Email", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
+   "fieldname": "raw",
+   "fieldtype": "Code",
+   "label": "Raw Email"
   }
- ], 
- "has_web_view": 0, 
- "hide_heading": 0, 
- "hide_toolbar": 0, 
- "idx": 0, 
- "image_view": 0, 
- "in_create": 1, 
- "is_submittable": 0, 
- "issingle": 0, 
- "istable": 0, 
- "max_attachments": 0, 
- "modified": "2017-09-19 16:28:00.042256", 
- "modified_by": "Administrator", 
- "module": "Email", 
- "name": "Unhandled Email", 
- "name_case": "", 
- "owner": "Administrator", 
+ ],
+ "in_create": 1,
+ "links": [],
+ "modified": "2022-08-03 12:20:51.822287",
+ "modified_by": "Administrator",
+ "module": "Email",
+ "name": "Unhandled Email",
+ "owner": "Administrator",
  "permissions": [
   {
-   "amend": 0, 
-   "apply_user_permissions": 0, 
-   "cancel": 0, 
-   "create": 0, 
-   "delete": 0, 
-   "email": 0, 
-   "export": 0, 
-   "if_owner": 0, 
-   "import": 0, 
-   "permlevel": 0, 
-   "print": 0, 
-   "read": 1, 
-   "report": 0, 
-   "role": "System Manager", 
-   "set_user_permissions": 0, 
-   "share": 0, 
-   "submit": 0, 
-   "write": 0
+   "read": 1,
+   "role": "System Manager"
   }
- ], 
- "quick_entry": 0, 
- "read_only": 0, 
- "read_only_onload": 0, 
- "show_name_in_global_search": 0, 
- "sort_field": "modified", 
- "sort_order": "DESC", 
- "track_changes": 0, 
- "track_seen": 0
+ ],
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "states": []
 }
\ No newline at end of file
diff --git a/frappe/email/doctype/unhandled_email/unhandled_email.py b/frappe/email/doctype/unhandled_email/unhandled_email.py
index e703f1ec97..1c315e2423 100644
--- a/frappe/email/doctype/unhandled_email/unhandled_email.py
+++ b/frappe/email/doctype/unhandled_email/unhandled_email.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors
 # License: MIT. See LICENSE
 
diff --git a/frappe/email/email_body.py b/frappe/email/email_body.py
index 07f698f740..20f81cb89b 100755
--- a/frappe/email/email_body.py
+++ b/frappe/email/email_body.py
@@ -24,6 +24,8 @@ from frappe.utils import (
 )
 from frappe.utils.pdf import get_pdf
 
+EMBED_PATTERN = re.compile("""embed=["'](.*?)["']""")
+
 
 def get_email(
 	recipients,
@@ -190,7 +192,7 @@ class EMail:
 	def set_part_html(self, message, inline_images):
 		from email.mime.text import MIMEText
 
-		has_inline_images = re.search("""embed=['"].*?['"]""", message)
+		has_inline_images = EMBED_PATTERN.search(message)
 
 		if has_inline_images:
 			# process inline images
@@ -265,28 +267,27 @@ class EMail:
 		validate_email_address(strip(self.sender), True)
 		self.reply_to = validate_email_address(strip(self.reply_to) or self.sender, True)
 
+		self.set_header("X-Original-From", self.sender)
 		self.replace_sender()
 		self.replace_sender_name()
 
-		self.recipients = [strip(r) for r in self.recipients]
-		self.cc = [strip(r) for r in self.cc]
-		self.bcc = [strip(r) for r in self.bcc]
+		self.recipients = [strip(r) for r in self.recipients if r not in frappe.STANDARD_USERS]
+		self.cc = [strip(r) for r in self.cc if r not in frappe.STANDARD_USERS]
+		self.bcc = [strip(r) for r in self.bcc if r not in frappe.STANDARD_USERS]
 
 		for e in self.recipients + (self.cc or []) + (self.bcc or []):
 			validate_email_address(e, True)
 
 	def replace_sender(self):
 		if cint(self.email_account.always_use_account_email_id_as_sender):
-			self.set_header("X-Original-From", self.sender)
-			sender_name, sender_email = parse_addr(self.sender)
+			sender_name, _ = parse_addr(self.sender)
 			self.sender = email.utils.formataddr(
 				(str(Header(sender_name or self.email_account.name, "utf-8")), self.email_account.email_id)
 			)
 
 	def replace_sender_name(self):
 		if cint(self.email_account.always_use_account_name_as_sender_name):
-			self.set_header("X-Original-From", self.sender)
-			sender_name, sender_email = parse_addr(self.sender)
+			_, sender_email = parse_addr(self.sender)
 			self.sender = email.utils.formataddr(
 				(str(Header(self.email_account.name, "utf-8")), sender_email)
 			)
@@ -330,12 +331,12 @@ class EMail:
 
 	def set_header(self, key, value):
 		if key in self.msg_root:
+			# delete key if found
+			# this is done because adding the same key doesn't override
+			# the existing key, rather appends another header with same key.
 			del self.msg_root[key]
 
-		try:
-			self.msg_root[key] = value
-		except ValueError:
-			self.msg_root[key] = sanitize_email_header(value)
+		self.msg_root[key] = sanitize_email_header(value)
 
 	def as_string(self):
 		"""validate, build message and convert to string"""
@@ -351,7 +352,7 @@ def get_formatted_html(
 	print_html=None,
 	email_account=None,
 	header=None,
-	unsubscribe_link=None,
+	unsubscribe_link: frappe._dict | None = None,
 	sender=None,
 	with_container=False,
 ):
@@ -451,7 +452,7 @@ def add_attachment(fname, fcontent, content_type=None, parent=None, content_id=N
 		attachment_type = "inline" if inline else "attachment"
 		part.add_header("Content-Disposition", attachment_type, filename=str(fname))
 	if content_id:
-		part.add_header("Content-ID", "<{0}>".format(content_id))
+		part.add_header("Content-ID", f"<{content_id}>")
 
 	parent.attach(part)
 
@@ -499,7 +500,7 @@ def replace_filename_with_cid(message):
 	inline_images = []
 
 	while True:
-		matches = re.search("""embed=["'](.*?)["']""", message)
+		matches = EMBED_PATTERN.search(message)
 		if not matches:
 			break
 		groups = matches.groups()
@@ -510,7 +511,7 @@ def replace_filename_with_cid(message):
 
 		filecontent = get_filecontent_from_path(img_path)
 		if not filecontent:
-			message = re.sub("""embed=['"]{0}['"]""".format(img_path), "", message)
+			message = re.sub(f"""embed=['"]{img_path}['"]""", "", message)
 			continue
 
 		content_id = random_string(10)
@@ -519,9 +520,7 @@ def replace_filename_with_cid(message):
 			{"filename": filename, "filecontent": filecontent, "content_id": content_id}
 		)
 
-		message = re.sub(
-			"""embed=['"]{0}['"]""".format(img_path), 'src="cid:{0}"'.format(content_id), message
-		)
+		message = re.sub(f"""embed=['"]{img_path}['"]""", f'src="cid:{content_id}"', message)
 
 	return (message, inline_images)
 
@@ -580,8 +579,17 @@ def get_header(header=None):
 	return email_header
 
 
-def sanitize_email_header(str):
-	return str.replace("\r", "").replace("\n", "")
+def sanitize_email_header(header: str):
+	"""
+	Removes all line boundaries in the headers.
+
+	Email Policy (python's std) has some bugs in it which uses splitlines
+	and raises ValueError (ref: https://github.com/python/cpython/blob/main/Lib/email/policy.py#L143).
+	Hence removing all line boundaries while sanitization of headers to prevent such faliures.
+	The line boundaries which are removed can be found here: https://docs.python.org/3/library/stdtypes.html#str.splitlines
+	"""
+
+	return "".join(header.splitlines())
 
 
 def get_brand_logo(email_account):
diff --git a/frappe/email/oauth.py b/frappe/email/oauth.py
new file mode 100644
index 0000000000..89b6df15d8
--- /dev/null
+++ b/frappe/email/oauth.py
@@ -0,0 +1,168 @@
+import base64
+from imaplib import IMAP4
+from poplib import POP3
+from smtplib import SMTP
+from urllib.parse import quote
+
+import frappe
+from frappe.integrations.google_oauth import GoogleOAuth
+from frappe.utils.password import encrypt
+
+
+class OAuthenticationError(Exception):
+	pass
+
+
+class Oauth:
+	def __init__(
+		self,
+		conn: IMAP4 | POP3 | SMTP,
+		email_account: str,
+		email: str,
+		access_token: str,
+		refresh_token: str,
+		service: str,
+		mechanism: str = "XOAUTH2",
+	) -> None:
+
+		self.email_account = email_account
+		self.email = email
+		self.service = service
+		self._mechanism = mechanism
+		self._conn = conn
+		self._access_token = access_token
+		self._refresh_token = refresh_token
+
+		self._validate()
+
+	def _validate(self) -> None:
+		if self.service != "GMail":
+			raise NotImplementedError(
+				f"Service {self.service} currently doesn't have oauth implementation."
+			)
+
+		if not self._refresh_token:
+			frappe.throw(
+				frappe._("Please Authorize OAuth."),
+				OAuthenticationError,
+				frappe._("OAuth Error"),
+			)
+
+	@property
+	def _auth_string(self) -> str:
+		return f"user={self.email}\1auth=Bearer {self._access_token}\1\1"
+
+	def connect(self, _retry: int = 0) -> None:
+		"""Connection method with retry on exception for Oauth"""
+		try:
+			if isinstance(self._conn, POP3):
+				res = self._connect_pop()
+
+				if not res.startswith(b"+OK"):
+					raise
+
+			elif isinstance(self._conn, IMAP4):
+				self._connect_imap()
+
+			else:
+				# SMTP
+				self._connect_smtp()
+
+		except Exception as e:
+			# maybe the access token expired - refreshing
+			access_token = self._refresh_access_token()
+
+			if not access_token or _retry > 0:
+				frappe.log_error(
+					"OAuth Error - Authentication Failed", str(e), "Email Account", self.email_account
+				)
+				# raising a bare exception here as we have a lot of exception handling present
+				# where the connect method is called from - hence just logging and raising.
+				raise
+
+			self._access_token = access_token
+			self.connect(_retry + 1)
+
+	def _connect_pop(self) -> bytes:
+		# poplib doesn't have AUTH command implementation
+		res = self._conn._shortcmd(
+			"AUTH {} {}".format(
+				self._mechanism, base64.b64encode(bytes(self._auth_string, "utf-8")).decode("utf-8")
+			)
+		)
+
+		return res
+
+	def _connect_imap(self) -> None:
+		self._conn.authenticate(self._mechanism, lambda x: self._auth_string)
+
+	def _connect_smtp(self) -> None:
+		self._conn.auth(self._mechanism, lambda x: self._auth_string, initial_response_ok=False)
+
+	def _refresh_access_token(self) -> str:
+		"""Refreshes access token via calling `refresh_access_token` method of oauth service object"""
+		service_obj = self._get_service_object()
+		access_token = service_obj.refresh_access_token(self._refresh_token).get("access_token")
+
+		if access_token:
+			# set the new access token in db
+			frappe.db.set_value(
+				"Email Account",
+				self.email_account,
+				"access_token",
+				encrypt(access_token),
+				update_modified=False,
+			)
+
+		return access_token
+
+	def _get_service_object(self):
+		"""Get Oauth service object"""
+
+		return {
+			"GMail": GoogleOAuth("mail", validate=False),
+		}[self.service]
+
+
+@frappe.whitelist(methods=["POST"])
+def oauth_access(email_account: str, service: str):
+	"""Used as a default endpoint/caller for all oauth services.
+	Returns authorization url for redirection"""
+
+	if not service:
+		frappe.throw(frappe._("No Service is selected. Please select one and try again!"))
+
+	doctype = "Email Account"
+
+	if service == "GMail":
+		return authorize_google_access(email_account, doctype)
+
+	raise NotImplementedError(f"Service {service} currently doesn't have oauth implementation.")
+
+
+def authorize_google_access(email_account, doctype: str = "Email Account", code: str = None):
+	"""Facilitates google oauth for email.
+	This is invoked 2 times - first time when user clicks `Authorze API Access` for getting the authorization url
+	and second time for setting the refresh and access token in db when google redirects back with oauth code."""
+
+	oauth_obj = GoogleOAuth("mail")
+
+	if not code:
+		return oauth_obj.get_authentication_url(
+			{
+				"redirect": f"/app/Form/{quote(doctype)}/{quote(email_account)}",
+				"success_query_param": "successful_authorization=1",
+				"email_account": email_account,
+			},
+		)
+
+	res = oauth_obj.authorize(code)
+	frappe.db.set_value(
+		doctype,
+		email_account,
+		{
+			"refresh_token": encrypt(res.get("refresh_token")),
+			"access_token": encrypt(res.get("access_token")),
+		},
+		update_modified=False,
+	)
diff --git a/frappe/email/queue.py b/frappe/email/queue.py
index b0a3b0583b..bc02c6be32 100755
--- a/frappe/email/queue.py
+++ b/frappe/email/queue.py
@@ -67,37 +67,24 @@ def get_emails_sent_today(email_account=None):
 	return frappe.db.sql(q, q_args)[0][0]
 
 
-def get_unsubscribe_message(unsubscribe_message, expose_recipients):
-	if unsubscribe_message:
-		unsubscribe_html = """{0}""".format(
-			unsubscribe_message
-		)
-	else:
-		unsubscribe_link = """{0}""".format(
-			_("Unsubscribe")
-		)
-		unsubscribe_html = _("{0} to stop receiving emails of this type").format(unsubscribe_link)
-
-	html = """"""
 
+	text = f"\n\n{unsubscribe_message}: \n"
 	if expose_recipients == "footer":
-		text = "\n"
-	else:
-		text = ""
-	text += "\n\n{unsubscribe_message}: \n".format(
-		unsubscribe_message=unsubscribe_message
-	)
+		text = f"\n{text}"
 
-	return frappe._dict({"html": html, "text": text})
+	return frappe._dict(html=html, text=text)
 
 
 def get_unsubcribed_url(
@@ -161,7 +148,7 @@ def flush(from_test=False):
 		msgprint(_("Emails are muted"))
 		from_test = True
 
-	if cint(frappe.defaults.get_defaults().get("hold_queue")) == 1:
+	if cint(frappe.db.get_default("suspend_email_queue")) == 1:
 		return
 
 	for row in get_queue():
@@ -170,7 +157,7 @@ def flush(from_test=False):
 			is_background_task = not from_test
 			func(email_queue_name=row.name, is_background_task=is_background_task)
 		except Exception:
-			frappe.log_error()
+			frappe.get_doc("Email Queue", row.name).log_error()
 
 
 def get_queue():
@@ -190,25 +177,6 @@ def get_queue():
 	)
 
 
-def clear_outbox(days: int = None) -> None:
-	"""Remove low priority older than 31 days in Outbox or configured in Log Settings.
-	Note: Used separate query to avoid deadlock
-	"""
-	days = days or 31
-	email_queue = DocType("Email Queue")
-
-	email_queues = (
-		frappe.qb.from_(email_queue)
-		.select(email_queue.name)
-		.where(email_queue.modified < (Now() - Interval(days=days)))
-		.run(pluck=True)
-	)
-
-	if email_queues:
-		frappe.db.delete("Email Queue", {"name": ("in", email_queues)})
-		frappe.db.delete("Email Queue Recipient", {"parent": ("in", email_queues)})
-
-
 def set_expiry_for_email_queue():
 	"""Mark emails as expire that has not sent for 7 days.
 	Called daily via scheduler.
diff --git a/frappe/email/receive.py b/frappe/email/receive.py
index 4a6db65a84..e26748dd07 100644
--- a/frappe/email/receive.py
+++ b/frappe/email/receive.py
@@ -17,7 +17,8 @@ from email_reply_parser import EmailReplyParser
 
 import frappe
 from frappe import _, safe_decode, safe_encode
-from frappe.core.doctype.file.file import MaxFileSizeReachedError, get_random_filename
+from frappe.core.doctype.file import MaxFileSizeReachedError, get_random_filename
+from frappe.email.oauth import Oauth
 from frappe.utils import (
 	add_days,
 	cint,
@@ -25,6 +26,7 @@ from frappe.utils import (
 	cstr,
 	extract_email_id,
 	get_datetime,
+	get_string_between,
 	markdown,
 	now,
 	parse_addr,
@@ -37,6 +39,9 @@ from frappe.utils.user import is_system_user
 # fix due to a python bug in poplib that limits it to 2048
 poplib._MAXLINE = 20480
 
+THREAD_ID_PATTERN = re.compile(r"(?<=\[)[\w/-]+")
+WORDS_PATTERN = re.compile(r"\w+")
+
 
 class EmailSizeExceededError(frappe.ValidationError):
 	pass
@@ -94,7 +99,20 @@ class EmailServer:
 				self.imap = Timed_IMAP4(
 					self.settings.host, self.settings.incoming_port, timeout=frappe.conf.get("pop_timeout")
 				)
-			self.imap.login(self.settings.username, self.settings.password)
+
+			if self.settings.use_oauth:
+				Oauth(
+					self.imap,
+					self.settings.email_account,
+					self.settings.username,
+					self.settings.access_token,
+					self.settings.refresh_token,
+					self.settings.service,
+				).connect()
+
+			else:
+				self.imap.login(self.settings.username, self.settings.password)
+
 			# connection established!
 			return True
 
@@ -115,15 +133,26 @@ class EmailServer:
 					self.settings.host, self.settings.incoming_port, timeout=frappe.conf.get("pop_timeout")
 				)
 
-			self.pop.user(self.settings.username)
-			self.pop.pass_(self.settings.password)
+			if self.settings.use_oauth:
+				Oauth(
+					self.pop,
+					self.settings.email_account,
+					self.settings.username,
+					self.settings.access_token,
+					self.settings.refresh_token,
+					self.settings.service,
+				).connect()
+
+			else:
+				self.pop.user(self.settings.username)
+				self.pop.pass_(self.settings.password)
 
 			# connection established!
 			return True
 
 		except _socket.error:
 			# log performs rollback and logs error in Error Log
-			frappe.log_error("receive.connect_pop")
+			self.log_error("POP: Unable to connect")
 
 			# Invalid mail server -- due to refusing connection
 			frappe.msgprint(_("Invalid Mail Server. Please rectify and try again."))
@@ -265,14 +294,14 @@ class EmailServer:
 				1 if uidnext < (sync_count + 1) or (uidnext - sync_count) < 1 else uidnext - sync_count
 			)
 			# sync last 100 email
-			self.settings.email_sync_rule = "UID {}:{}".format(from_uid, uidnext)
+			self.settings.email_sync_rule = f"UID {from_uid}:{uidnext}"
 			self.uid_reindexed = True
 
 		elif uid_validity == current_uid_validity:
 			return
 
 	def parse_imap_response(self, cmd, response):
-		pattern = r"(?<={cmd} )[0-9]*".format(cmd=cmd)
+		pattern = rf"(?<={cmd} )[0-9]*"
 		match = re.search(pattern, response.decode("utf-8"), re.U | re.I)
 
 		if match:
@@ -306,7 +335,7 @@ class EmailServer:
 
 			else:
 				# log performs rollback and logs error in Error Log
-				frappe.log_error("receive.get_messages", self.make_error_msg(msg_num, incoming_mail))
+				self.log_error("Unable to fetch email", self.make_error_msg(msg_num, incoming_mail))
 				self.errors = True
 				frappe.db.rollback()
 
@@ -331,8 +360,7 @@ class EmailServer:
 
 		flags = []
 		for flag in imaplib.ParseFlags(flag_string) or []:
-			pattern = re.compile(r"\w+")
-			match = re.search(pattern, frappe.as_unicode(flag))
+			match = WORDS_PATTERN.search(frappe.as_unicode(flag))
 			flags.append(match.group(0))
 
 		if "Seen" in flags:
@@ -374,7 +402,7 @@ class EmailServer:
 			try:
 				# retrieve headers
 				incoming_mail = Email(b"\n".join(self.pop.top(msg_num, 5)[1]))
-			except:
+			except Exception:
 				pass
 
 		if incoming_mail:
@@ -425,14 +453,16 @@ class Email:
 		self.set_content_and_type()
 		self.set_subject()
 		self.set_from()
-		self.message_id = (self.mail.get("Message-ID") or "").strip(" <>")
+
+		message_id = self.mail.get("Message-ID") or ""
+		self.message_id = get_string_between("<", message_id, ">")
 
 		if self.mail["Date"]:
 			try:
 				utc = email.utils.mktime_tz(email.utils.parsedate_tz(self.mail["Date"]))
 				utc_dt = datetime.datetime.utcfromtimestamp(utc)
 				self.date = convert_utc_to_user_timezone(utc_dt).strftime("%Y-%m-%d %H:%M:%S")
-			except:
+			except Exception:
 				self.date = now()
 		else:
 			self.date = now()
@@ -441,7 +471,8 @@ class Email:
 
 	@property
 	def in_reply_to(self):
-		return (self.mail.get("In-Reply-To") or "").strip(" <>")
+		in_reply_to = self.mail.get("In-Reply-To") or ""
+		return get_string_between("<", in_reply_to, ">")
 
 	def parse(self):
 		"""Walk and process multi-part email."""
@@ -528,10 +559,10 @@ class Email:
 		for key in ("From", "To", "Subject", "Date"):
 			value = cstr(message.get(key))
 			if value:
-				headers.append("{label}: {value}".format(label=_(key), value=escape(value)))
+				headers.append(f"{_(key)}: {escape(value)}")
 
 		self.text_content += "\n".join(headers)
-		self.html_content += "
        " + "\n".join("
        {0}
        ".format(h) for h in headers)
+		self.html_content += "
        " + "\n".join(f"
        {h}
        " for h in headers)
 
 		if not message.is_multipart() and message.get_content_type() == "text/plain":
 			# email.parser didn't parse it!
@@ -566,7 +597,7 @@ class Email:
 				try:
 					fname = fname.replace("\n", " ").replace("\r", "")
 					fname = cstr(decode_header(fname)[0][0])
-				except:
+				except Exception:
 					fname = get_random_filename(content_type=content_type)
 			else:
 				fname = get_random_filename(content_type=content_type)
@@ -618,7 +649,7 @@ class Email:
 
 	def get_thread_id(self):
 		"""Extract thread ID from `[]`"""
-		l = re.findall(r"(?<=\[)[\w/-]+", self.subject)
+		l = THREAD_ID_PATTERN.findall(self.subject)
 		return l and l[0] or None
 
 	def is_reply(self):
@@ -704,7 +735,7 @@ class InboundMail(Email):
 		content = self.content
 		for file in attachments:
 			if file.name in self.cid_map and self.cid_map[file.name]:
-				content = content.replace("cid:{0}".format(self.cid_map[file.name]), file.file_url)
+				content = content.replace(f"cid:{self.cid_map[file.name]}", file.file_url)
 		return content
 
 	def is_notification(self):
@@ -889,7 +920,7 @@ class InboundMail(Email):
 		users = frappe.get_all(
 			"User Email", filters={"email_account": email_account.name}, fields=["parent"]
 		)
-		return list(set([user.get("parent") for user in users]))
+		return list({user.get("parent") for user in users})
 
 	@staticmethod
 	def clean_subject(subject):
@@ -940,7 +971,7 @@ class InboundMail(Email):
 		}
 
 
-class TimerMixin(object):
+class TimerMixin:
 	def __init__(self, *args, **kwargs):
 		self.timeout = kwargs.pop("timeout", 0.0)
 		self.elapsed_time = 0.0
diff --git a/frappe/email/smtp.py b/frappe/email/smtp.py
index 1c91356506..10eb2f7681 100644
--- a/frappe/email/smtp.py
+++ b/frappe/email/smtp.py
@@ -1,25 +1,12 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
+# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
 
-import email.utils
 import smtplib
-import sys
-
-import _socket
 
 import frappe
 from frappe import _
-from frappe.utils import cint, cstr, parse_addr
-
-CONNECTION_FAILED = _("Could not connect to outgoing email server")
-AUTH_ERROR_TITLE = _("Invalid Credentials")
-AUTH_ERROR = _("Incorrect email or password. Please check your login credentials.")
-SOCKET_ERROR_TITLE = _("Incorrect Configuration")
-SOCKET_ERROR = _("Invalid Outgoing Mail Server or Port")
-SEND_MAIL_FAILED = _("Unable to send emails at this time")
-EMAIL_ACCOUNT_MISSING = _(
-	"Email Account not setup. Please create a new Email Account from Setup > Email > Email Account"
-)
+from frappe.email.oauth import Oauth
+from frappe.utils import cint, cstr
 
 
 class InvalidEmailCredentials(frappe.ValidationError):
@@ -57,17 +44,40 @@ def send(email, append_to=None, retry=1):
 
 
 class SMTPServer:
-	def __init__(self, server, login=None, password=None, port=None, use_tls=None, use_ssl=None):
+	def __init__(
+		self,
+		server,
+		login=None,
+		email_account=None,
+		password=None,
+		port=None,
+		use_tls=None,
+		use_ssl=None,
+		use_oauth=0,
+		refresh_token=None,
+		access_token=None,
+		service=None,
+	):
 		self.login = login
+		self.email_account = email_account
 		self.password = password
 		self._server = server
 		self._port = port
 		self.use_tls = use_tls
 		self.use_ssl = use_ssl
+		self.use_oauth = use_oauth
+		self.refresh_token = refresh_token
+		self.access_token = access_token
+		self.service = service
 		self._session = None
 
 		if not self.server:
-			frappe.msgprint(EMAIL_ACCOUNT_MISSING, raise_exception=frappe.OutgoingEmailError)
+			frappe.msgprint(
+				_(
+					"Email Account not setup. Please create a new Email Account from Setup > Email > Email Account"
+				),
+				raise_exception=frappe.OutgoingEmailError,
+			)
 
 	@property
 	def port(self):
@@ -95,10 +105,18 @@ class SMTPServer:
 		try:
 			_session = SMTP(self.server, self.port)
 			if not _session:
-				frappe.msgprint(CONNECTION_FAILED, raise_exception=frappe.OutgoingEmailError)
+				frappe.msgprint(
+					_("Could not connect to outgoing email server"), raise_exception=frappe.OutgoingEmailError
+				)
 
 			self.secure_session(_session)
-			if self.login and self.password:
+
+			if self.use_oauth:
+				Oauth(
+					_session, self.email_account, self.login, self.access_token, self.refresh_token, self.service
+				).connect()
+
+			elif self.password:
 				res = _session.login(str(self.login or ""), str(self.password or ""))
 
 				# check if logged correctly
@@ -108,16 +126,12 @@ class SMTPServer:
 			self._session = _session
 			return self._session
 
-		except smtplib.SMTPAuthenticationError as e:
+		except smtplib.SMTPAuthenticationError:
 			self.throw_invalid_credentials_exception()
 
-		except _socket.error as e:
+		except OSError:
 			# Invalid mail server -- due to refusing connection
-			frappe.throw(SOCKET_ERROR, title=SOCKET_ERROR_TITLE)
-
-		except smtplib.SMTPException:
-			frappe.msgprint(SEND_MAIL_FAILED)
-			raise
+			frappe.throw(_("Invalid Outgoing Mail Server or Port"), title=_("Incorrect Configuration"))
 
 	def is_session_active(self):
 		if self._session:
@@ -132,4 +146,8 @@ class SMTPServer:
 
 	@classmethod
 	def throw_invalid_credentials_exception(cls):
-		frappe.throw(AUTH_ERROR, title=AUTH_ERROR_TITLE, exc=InvalidEmailCredentials)
+		frappe.throw(
+			_("Please check your email login credentials."),
+			title=_("Invalid Credentials"),
+			exc=InvalidEmailCredentials,
+		)
diff --git a/frappe/email/test_email_body.py b/frappe/email/test_email_body.py
index 3de21f64ce..d5b1013a73 100644
--- a/frappe/email/test_email_body.py
+++ b/frappe/email/test_email_body.py
@@ -5,6 +5,7 @@ import base64
 import os
 import unittest
 
+import frappe
 from frappe import safe_decode
 from frappe.email.doctype.email_queue.email_queue import QueueBuilder, SendMailContext
 from frappe.email.email_body import (
@@ -54,26 +55,27 @@ This is the text version of this email
 		uni_chr1 = chr(40960)
 		uni_chr2 = chr(1972)
 
-		queue_doc = QueueBuilder(
+		QueueBuilder(
 			recipients=["test@example.com"],
 			sender="me@example.com",
 			subject="Test Subject",
-			message="
        " + uni_chr1 + "abcd" + uni_chr2 + "
        ",
+			message=f"
        {uni_chr1}abcd{uni_chr2}
        ",
 			text_content="whatever",
-		).process()[0]
+		).process()
+		queue_doc = frappe.get_last_doc("Email Queue")
 		mail_ctx = SendMailContext(queue_doc=queue_doc)
 		result = mail_ctx.build_message(recipient_email="test@test.com")
 		self.assertTrue(b"
        =EA=80=80abcd=DE=B4
        " in result)
 
 	def test_prepare_message_returns_cr_lf(self):
-		queue_doc = QueueBuilder(
+		QueueBuilder(
 			recipients=["test@example.com"],
 			sender="me@example.com",
 			subject="Test Subject",
 			message="
        \n this is a test of newlines\n" + "
        ",
 			text_content="whatever",
-		).process()[0]
-
+		).process()
+		queue_doc = frappe.get_last_doc("Email Queue")
 		mail_ctx = SendMailContext(queue_doc=queue_doc)
 		result = safe_decode(mail_ctx.build_message(recipient_email="test@test.com"))
 
@@ -128,7 +130,7 @@ w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
 
 		processed_message = """
 			
        
-				test
+				test
 				
 			
        
 		""".format(
@@ -152,20 +154,19 @@ w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
 
        Hey John Doe!
        
 
        This is embedded image you asked for
        
 """
-		email_string = (
-			get_email(
-				recipients=["test@example.com"],
-				sender="me@example.com",
-				subject="Test Subject",
-				content=email_html,
-				header=["Email Title", "green"],
-			)
-			.as_string()
-			.replace("\r\n", "\n")
-		)
+		email_string = get_email(
+			recipients=["test@example.com"],
+			sender="me@example.com",
+			subject="Test Subject\u2028, with line break, \nand Line feed \rand carriage return.",
+			content=email_html,
+			header=["Email Title", "green"],
+		).as_string()
 		# REDESIGN-TODO: Add style for indicators in email
 		self.assertTrue("""""" in email_string)
 		self.assertTrue("Email Title" in email_string)
+		self.assertIn(
+			"Subject: Test Subject, with line break, and Line feed and carriage return.", email_string
+		)
 
 	def test_get_email_header(self):
 		html = get_header(["This is test", "orange"])
diff --git a/frappe/email/utils.py b/frappe/email/utils.py
index 147284a625..7fc2e0ff89 100644
--- a/frappe/email/utils.py
+++ b/frappe/email/utils.py
@@ -1,5 +1,6 @@
 # Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and contributors
 # License: MIT. See LICENSE
+
 import imaplib
 import poplib
 
diff --git a/frappe/event_streaming/doctype/document_type_field_mapping/document_type_field_mapping.py b/frappe/event_streaming/doctype/document_type_field_mapping/document_type_field_mapping.py
index 3019d70035..96d9e0fcb3 100644
--- a/frappe/event_streaming/doctype/document_type_field_mapping/document_type_field_mapping.py
+++ b/frappe/event_streaming/doctype/document_type_field_mapping/document_type_field_mapping.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2019, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
diff --git a/frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.js b/frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.js
index 4653bf4d03..ad9ab0f51d 100644
--- a/frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.js
+++ b/frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.js
@@ -1,28 +1,37 @@
 // Copyright (c) 2019, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Document Type Mapping', {
-	local_doctype: function(frm) {
+frappe.ui.form.on("Document Type Mapping", {
+	local_doctype: function (frm) {
 		if (frm.doc.local_doctype) {
-			frappe.model.clear_table(frm.doc, 'field_mapping');
+			frappe.model.clear_table(frm.doc, "field_mapping");
 			let fields = frm.events.get_fields(frm);
-			$.each(fields, function(i, data) {
-				let row = frappe.model.add_child(frm.doc, 'Document Type Mapping', 'field_mapping');
+			$.each(fields, function (i, data) {
+				let row = frappe.model.add_child(
+					frm.doc,
+					"Document Type Field Mapping",
+					"field_mapping"
+				);
 				row.local_fieldname = data;
 			});
-			refresh_field('field_mapping');
+			refresh_field("field_mapping");
 		}
 	},
 
-	get_fields: function(frm) {
+	get_fields: function (frm) {
 		let filtered_fields = [];
-		frappe.model.with_doctype(frm.doc.local_doctype, ()=> {
-			frappe.get_meta(frm.doc.local_doctype).fields.map( field => {
-				if (field.fieldname !== 'remote_docname' && field.fieldname !== 'remote_site_name' && frappe.model.is_value_type(field) && !field.hidden) {
+		frappe.model.with_doctype(frm.doc.local_doctype, () => {
+			frappe.get_meta(frm.doc.local_doctype).fields.map((field) => {
+				if (
+					field.fieldname !== "remote_docname" &&
+					field.fieldname !== "remote_site_name" &&
+					frappe.model.is_value_type(field) &&
+					!field.hidden
+				) {
 					filtered_fields.push(field.fieldname);
 				}
 			});
 		});
 		return filtered_fields;
-	}
+	},
 });
diff --git a/frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.py b/frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.py
index bcd2b275d1..04b5015296 100644
--- a/frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.py
+++ b/frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2019, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 import json
diff --git a/frappe/event_streaming/doctype/document_type_mapping/test_document_type_mapping.py b/frappe/event_streaming/doctype/document_type_mapping/test_document_type_mapping.py
index 1d5c4862de..676d5040ff 100644
--- a/frappe/event_streaming/doctype/document_type_mapping/test_document_type_mapping.py
+++ b/frappe/event_streaming/doctype/document_type_mapping/test_document_type_mapping.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2019, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
 # import frappe
diff --git a/frappe/event_streaming/doctype/event_consumer/event_consumer.js b/frappe/event_streaming/doctype/event_consumer/event_consumer.js
index 66d92699fa..2bcf96f9f3 100644
--- a/frappe/event_streaming/doctype/event_consumer/event_consumer.js
+++ b/frappe/event_streaming/doctype/event_consumer/event_consumer.js
@@ -1,19 +1,17 @@
 // Copyright (c) 2019, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Event Consumer', {
-	refresh: function(frm) {
+frappe.ui.form.on("Event Consumer", {
+	refresh: function (frm) {
 		// formatter for subscribed doctype approval status
-		frm.set_indicator_formatter('status',
-			function(doc) {
-				let indicator = 'orange';
-				if (doc.status == 'Approved') {
-					indicator = 'green';
-				} else if (doc.status == 'Rejected') {
-					indicator = 'red';
-				}
-				return indicator;
+		frm.set_indicator_formatter("status", function (doc) {
+			let indicator = "orange";
+			if (doc.status == "Approved") {
+				indicator = "green";
+			} else if (doc.status == "Rejected") {
+				indicator = "red";
 			}
-		);
-	}
+			return indicator;
+		});
+	},
 });
diff --git a/frappe/event_streaming/doctype/event_consumer/event_consumer.py b/frappe/event_streaming/doctype/event_consumer/event_consumer.py
index 287a1fca03..a2ae6f6651 100644
--- a/frappe/event_streaming/doctype/event_consumer/event_consumer.py
+++ b/frappe/event_streaming/doctype/event_consumer/event_consumer.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2019, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
@@ -213,5 +212,5 @@ def has_consumer_access(consumer, update_log):
 			else:
 				return frappe.safe_eval(condition, frappe._dict(doc=doc))
 	except Exception as e:
-		frappe.log_error(title="has_consumer_access error", message=e)
+		consumer.log_error("has_consumer_access error")
 	return False
diff --git a/frappe/event_streaming/doctype/event_consumer/test_event_consumer.py b/frappe/event_streaming/doctype/event_consumer/test_event_consumer.py
index 605fc7982a..6f04af643e 100644
--- a/frappe/event_streaming/doctype/event_consumer/test_event_consumer.py
+++ b/frappe/event_streaming/doctype/event_consumer/test_event_consumer.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2019, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
 # import frappe
diff --git a/frappe/event_streaming/doctype/event_consumer_document_type/event_consumer_document_type.py b/frappe/event_streaming/doctype/event_consumer_document_type/event_consumer_document_type.py
index b33313087f..1ed15c5a75 100644
--- a/frappe/event_streaming/doctype/event_consumer_document_type/event_consumer_document_type.py
+++ b/frappe/event_streaming/doctype/event_consumer_document_type/event_consumer_document_type.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2019, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
diff --git a/frappe/event_streaming/doctype/event_producer/event_producer.js b/frappe/event_streaming/doctype/event_producer/event_producer.js
index c2c3389e92..23ca482433 100644
--- a/frappe/event_streaming/doctype/event_producer/event_producer.js
+++ b/frappe/event_streaming/doctype/event_producer/event_producer.js
@@ -1,27 +1,25 @@
 // Copyright (c) 2019, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Event Producer', {
-	refresh: function(frm) {
-		frm.set_query('ref_doctype', 'producer_doctypes', function() {
+frappe.ui.form.on("Event Producer", {
+	refresh: function (frm) {
+		frm.set_query("ref_doctype", "producer_doctypes", function () {
 			return {
 				filters: {
 					issingle: 0,
-					istable: 0
-				}
+					istable: 0,
+				},
 			};
 		});
 
-		frm.set_indicator_formatter('status',
-			function(doc) {
-				let indicator = 'orange';
-				if (doc.status == 'Approved') {
-					indicator = 'green';
-				} else if (doc.status == 'Rejected') {
-					indicator = 'red';
-				}
-				return indicator;
+		frm.set_indicator_formatter("status", function (doc) {
+			let indicator = "orange";
+			if (doc.status == "Approved") {
+				indicator = "green";
+			} else if (doc.status == "Rejected") {
+				indicator = "red";
 			}
-		);
-	}
+			return indicator;
+		});
+	},
 });
diff --git a/frappe/event_streaming/doctype/event_producer/event_producer.py b/frappe/event_streaming/doctype/event_producer/event_producer.py
index f639e48b50..f91c8a4fd4 100644
--- a/frappe/event_streaming/doctype/event_producer/event_producer.py
+++ b/frappe/event_streaming/doctype/event_producer/event_producer.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2019, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
@@ -315,8 +314,9 @@ def set_insert(update, producer_site, event_producer):
 	else:
 		# if event consumer is not saving documents with the same name as the producer
 		# store the remote docname in a custom field for future updates
-		local_doc = doc.insert(set_child_names=False)
-		set_custom_fields(local_doc, update.docname, event_producer)
+		doc.remote_docname = update.docname
+		doc.remote_site_name = event_producer
+		doc.insert(set_child_names=False)
 
 
 def set_update(update, producer_site):
@@ -567,9 +567,3 @@ def resync(update):
 		update = get_mapped_update(update, producer_site)
 		update.data = json.loads(update.data)
 	return sync(update, producer_site, event_producer, in_retry=True)
-
-
-def set_custom_fields(local_doc, remote_docname, remote_site_name):
-	"""sets custom field in doc for storing remote docname"""
-	frappe.db.set_value(local_doc.doctype, local_doc.name, "remote_docname", remote_docname)
-	frappe.db.set_value(local_doc.doctype, local_doc.name, "remote_site_name", remote_site_name)
diff --git a/frappe/event_streaming/doctype/event_producer/test_event_producer.py b/frappe/event_streaming/doctype/event_producer/test_event_producer.py
index 4464b0a434..168c9a61cf 100644
--- a/frappe/event_streaming/doctype/event_producer/test_event_producer.py
+++ b/frappe/event_streaming/doctype/event_producer/test_event_producer.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2019, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
 import json
@@ -8,6 +7,8 @@ import frappe
 from frappe.core.doctype.user.user import generate_keys
 from frappe.event_streaming.doctype.event_producer.event_producer import pull_from_node
 from frappe.frappeclient import FrappeClient
+from frappe.query_builder.utils import db_type_is
+from frappe.tests.test_query_builder import run_only_if
 
 producer_url = "http://test_site_producer:8000"
 
@@ -51,7 +52,9 @@ class TestEventProducer(unittest.TestCase):
 		self.pull_producer_data()
 		self.assertFalse(frappe.db.exists("ToDo", producer_doc.name))
 
+	@run_only_if(db_type_is.MARIADB)
 	def test_multiple_doctypes_sync(self):
+		# TODO: This test is extremely flaky with Postgres. Rewrite this!
 		producer = get_remote_site()
 
 		# insert todo and note in producer
diff --git a/frappe/event_streaming/doctype/event_producer_document_type/event_producer_document_type.py b/frappe/event_streaming/doctype/event_producer_document_type/event_producer_document_type.py
index 3e9623f56f..8f4c936792 100644
--- a/frappe/event_streaming/doctype/event_producer_document_type/event_producer_document_type.py
+++ b/frappe/event_streaming/doctype/event_producer_document_type/event_producer_document_type.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2019, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
diff --git a/frappe/event_streaming/doctype/event_producer_last_update/event_producer_last_update.js b/frappe/event_streaming/doctype/event_producer_last_update/event_producer_last_update.js
index 15730e4c5f..6d18be43e3 100644
--- a/frappe/event_streaming/doctype/event_producer_last_update/event_producer_last_update.js
+++ b/frappe/event_streaming/doctype/event_producer_last_update/event_producer_last_update.js
@@ -1,8 +1,7 @@
 // Copyright (c) 2020, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Event Producer Last Update', {
+frappe.ui.form.on("Event Producer Last Update", {
 	// refresh: function(frm) {
-
 	// }
 });
diff --git a/frappe/event_streaming/doctype/event_producer_last_update/event_producer_last_update.py b/frappe/event_streaming/doctype/event_producer_last_update/event_producer_last_update.py
index 8e32e6fe6f..ec5cee7e78 100644
--- a/frappe/event_streaming/doctype/event_producer_last_update/event_producer_last_update.py
+++ b/frappe/event_streaming/doctype/event_producer_last_update/event_producer_last_update.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2020, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
diff --git a/frappe/event_streaming/doctype/event_producer_last_update/test_event_producer_last_update.py b/frappe/event_streaming/doctype/event_producer_last_update/test_event_producer_last_update.py
index 6054ec873f..ccdea6c694 100644
--- a/frappe/event_streaming/doctype/event_producer_last_update/test_event_producer_last_update.py
+++ b/frappe/event_streaming/doctype/event_producer_last_update/test_event_producer_last_update.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2020, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
 # import frappe
diff --git a/frappe/event_streaming/doctype/event_sync_log/event_sync_log.js b/frappe/event_streaming/doctype/event_sync_log/event_sync_log.js
index 5199e3f02d..7cc3198bae 100644
--- a/frappe/event_streaming/doctype/event_sync_log/event_sync_log.js
+++ b/frappe/event_streaming/doctype/event_sync_log/event_sync_log.js
@@ -1,24 +1,24 @@
 // Copyright (c) 2019, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Event Sync Log', {
-	refresh: function(frm) {
-		if (frm.doc.status == 'Failed') {
-			frm.add_custom_button(__('Resync'), function() {
+frappe.ui.form.on("Event Sync Log", {
+	refresh: function (frm) {
+		if (frm.doc.status == "Failed") {
+			frm.add_custom_button(__("Resync"), function () {
 				frappe.call({
 					method: "frappe.event_streaming.doctype.event_producer.event_producer.resync",
 					args: {
 						update: frm.doc,
 					},
-					callback: function(r) {
+					callback: function (r) {
 						if (r.message) {
 							frappe.msgprint(r.message);
-							frm.set_value('status', r.message);
+							frm.set_value("status", r.message);
 							frm.save();
 						}
-					}
+					},
 				});
 			});
 		}
-	}
+	},
 });
diff --git a/frappe/event_streaming/doctype/event_sync_log/event_sync_log.py b/frappe/event_streaming/doctype/event_sync_log/event_sync_log.py
index c26ca46e05..a1d82ad08f 100644
--- a/frappe/event_streaming/doctype/event_sync_log/event_sync_log.py
+++ b/frappe/event_streaming/doctype/event_sync_log/event_sync_log.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2019, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
diff --git a/frappe/event_streaming/doctype/event_sync_log/event_sync_log_list.js b/frappe/event_streaming/doctype/event_sync_log/event_sync_log_list.js
index 75d67003c4..97d2ee0a1d 100644
--- a/frappe/event_streaming/doctype/event_sync_log/event_sync_log_list.js
+++ b/frappe/event_streaming/doctype/event_sync_log/event_sync_log_list.js
@@ -1,9 +1,9 @@
-frappe.listview_settings['Event Sync Log'] = {
-	get_indicator: function(doc) {
+frappe.listview_settings["Event Sync Log"] = {
+	get_indicator: function (doc) {
 		var colors = {
-			"Failed": "red",
-			"Synced": "green"
+			Failed: "red",
+			Synced: "green",
 		};
 		return [__(doc.status), colors[doc.status], "status,=," + doc.status];
-	}
+	},
 };
diff --git a/frappe/event_streaming/doctype/event_sync_log/test_event_sync_log.py b/frappe/event_streaming/doctype/event_sync_log/test_event_sync_log.py
index da90c8e634..13028cbac7 100644
--- a/frappe/event_streaming/doctype/event_sync_log/test_event_sync_log.py
+++ b/frappe/event_streaming/doctype/event_sync_log/test_event_sync_log.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2019, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
 # import frappe
diff --git a/frappe/event_streaming/doctype/event_update_log/event_update_log.js b/frappe/event_streaming/doctype/event_update_log/event_update_log.js
index c5e8ed5915..d901799780 100644
--- a/frappe/event_streaming/doctype/event_update_log/event_update_log.js
+++ b/frappe/event_streaming/doctype/event_update_log/event_update_log.js
@@ -1,8 +1,7 @@
 // Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Event Update Log', {
+frappe.ui.form.on("Event Update Log", {
 	// refresh: function(frm) {
-
 	// }
 });
diff --git a/frappe/event_streaming/doctype/event_update_log/event_update_log.py b/frappe/event_streaming/doctype/event_update_log/event_update_log.py
index 658a3b47cc..e40f600484 100644
--- a/frappe/event_streaming/doctype/event_update_log/event_update_log.py
+++ b/frappe/event_streaming/doctype/event_update_log/event_update_log.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and contributors
 # License: MIT. See LICENSE
 
diff --git a/frappe/event_streaming/doctype/event_update_log/test_event_update_log.py b/frappe/event_streaming/doctype/event_update_log/test_event_update_log.py
index 673164b8d7..0cbff47912 100644
--- a/frappe/event_streaming/doctype/event_update_log/test_event_update_log.py
+++ b/frappe/event_streaming/doctype/event_update_log/test_event_update_log.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
 # import frappe
diff --git a/frappe/event_streaming/doctype/event_update_log_consumer/event_update_log_consumer.py b/frappe/event_streaming/doctype/event_update_log_consumer/event_update_log_consumer.py
index 4f00504538..69da7db92e 100644
--- a/frappe/event_streaming/doctype/event_update_log_consumer/event_update_log_consumer.py
+++ b/frappe/event_streaming/doctype/event_update_log_consumer/event_update_log_consumer.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2020, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
diff --git a/frappe/exceptions.py b/frappe/exceptions.py
index a8569481d3..c3bb45caea 100644
--- a/frappe/exceptions.py
+++ b/frappe/exceptions.py
@@ -76,7 +76,7 @@ class ImproperDBConfigurationError(Exception):
 	def __init__(self, reason, msg=None):
 		if not msg:
 			msg = "MariaDb is not properly configured"
-		super(ImproperDBConfigurationError, self).__init__(msg)
+		super().__init__(msg)
 		self.reason = reason
 
 
@@ -263,3 +263,15 @@ class ExecutableNotFound(FileNotFoundError):
 
 class InvalidRemoteException(Exception):
 	pass
+
+
+class LinkExpired(ValidationError):
+	http_status_code = 410
+	title = "Link Expired"
+	message = "The link has expired"
+
+
+class InvalidKeyError(ValidationError):
+	http_status_code = 401
+	title = "Invalid Key"
+	message = "The document key is invalid"
diff --git a/frappe/frappeclient.py b/frappe/frappeclient.py
index 04087463bc..474a5b06c4 100644
--- a/frappe/frappeclient.py
+++ b/frappe/frappeclient.py
@@ -26,7 +26,7 @@ class FrappeException(Exception):
 	pass
 
 
-class FrappeClient(object):
+class FrappeClient:
 	def __init__(
 		self,
 		url,
@@ -85,11 +85,9 @@ class FrappeClient(object):
 
 	def setup_key_authentication_headers(self):
 		if self.api_key and self.api_secret:
-			token = base64.b64encode(
-				("{}:{}".format(self.api_key, self.api_secret)).encode("utf-8")
-			).decode("utf-8")
+			token = base64.b64encode((f"{self.api_key}:{self.api_secret}").encode()).decode("utf-8")
 			auth_header = {
-				"Authorization": "Basic {}".format(token),
+				"Authorization": f"Basic {token}",
 			}
 			self.headers.update(auth_header)
 
@@ -266,7 +264,7 @@ class FrappeClient(object):
 		# build - attach children to parents
 		if tables:
 			docs = [frappe._dict(doc) for doc in docs]
-			docs_map = dict((doc.name, doc) for doc in docs)
+			docs_map = {doc.name: doc for doc in docs}
 
 			for fieldname in tables:
 				for child in tables[fieldname]:
diff --git a/frappe/geo/country_info.json b/frappe/geo/country_info.json
index 7ffdf0a8bf..ed44b1c7f8 100644
--- a/frappe/geo/country_info.json
+++ b/frappe/geo/country_info.json
@@ -8,7 +8,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Asia/Kabul"
-  ]
+  ],
+  "isd": "+93"
  },
  "Albania": {
   "code": "al",
@@ -20,7 +21,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Europe/Tirane"
-  ]
+  ],
+  "isd": "+355"
  },
  "Algeria": {
   "code": "dz",
@@ -32,11 +34,13 @@
   "number_format": "#,###.##",
   "timezones": [
    "Africa/Algiers"
-  ]
+  ],
+  "isd": "+213"
  },
  "American Samoa": {
   "code": "as",
-  "number_format": "#,###.##"
+  "number_format": "#,###.##",
+  "isd": "+1684"
  },
  "Andorra": {
   "code": "ad",
@@ -48,7 +52,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Europe/Andorra"
-  ]
+  ],
+  "isd": "+376"
  },
  "Angola": {
   "code": "ao",
@@ -60,7 +65,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Africa/Luanda"
-  ]
+  ],
+  "isd": "+244"
  },
  "Anguilla": {
   "code": "ai",
@@ -72,7 +78,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "America/Anguilla"
-  ]
+  ],
+  "isd": "+1264"
  },
  "Antarctica": {
   "code": "aq",
@@ -88,7 +95,8 @@
    "Antarctica/Rothera",
    "Antarctica/Syowa",
    "Antarctica/Vostok"
-  ]
+  ],
+  "isd": "+672"
  },
  "Antigua and Barbuda": {
   "code": "ag",
@@ -100,7 +108,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "America/Antigua"
-  ]
+  ],
+  "isd": "+1268"
  },
  "Argentina": {
   "code": "ar",
@@ -123,7 +132,8 @@
    "America/Argentina/San_Luis",
    "America/Argentina/Tucuman",
    "America/Argentina/Ushuaia"
-  ]
+  ],
+  "isd": "+54"
  },
  "Armenia": {
   "code": "am",
@@ -135,7 +145,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Asia/Yerevan"
-  ]
+  ],
+  "isd": "+374"
  },
  "Aruba": {
   "code": "aw",
@@ -147,7 +158,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "America/Aruba"
-  ]
+  ],
+  "isd": "+297"
  },
  "Australia": {
   "code": "au",
@@ -170,7 +182,8 @@
    "Australia/Melbourne",
    "Australia/Perth",
    "Australia/Sydney"
-  ]
+  ],
+  "isd": "+61"
  },
  "Austria": {
   "code": "at",
@@ -182,7 +195,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Europe/Vienna"
-  ]
+  ],
+  "isd": "+43"
  },
  "Azerbaijan": {
   "code": "az",
@@ -192,7 +206,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Asia/Baku"
-  ]
+  ],
+  "isd": "+994"
  },
  "Bahamas": {
   "code": "bs",
@@ -201,7 +216,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "America/Nassau"
-  ]
+  ],
+  "isd": "+1242"
  },
  "Bahrain": {
   "code": "bh",
@@ -213,7 +229,8 @@
   "number_format": "#,###.###",
   "timezones": [
    "Asia/Bahrain"
-  ]
+  ],
+  "isd": "+973"
  },
  "Bangladesh": {
   "code": "bd",
@@ -225,7 +242,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Asia/Dhaka"
-  ]
+  ],
+  "isd": "+880"
  },
  "Barbados": {
   "code": "bb",
@@ -237,7 +255,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "America/Barbados"
-  ]
+  ],
+  "isd": "+1246"
  },
  "Belarus": {
   "code": "by",
@@ -247,7 +266,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Europe/Minsk"
-  ]
+  ],
+  "isd": "+375"
  },
  "Belgium": {
   "code": "be",
@@ -259,7 +279,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Europe/Brussels"
-  ]
+  ],
+  "isd": "+32"
  },
  "Belize": {
   "code": "bz",
@@ -272,7 +293,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "America/Belize"
-  ]
+  ],
+  "isd": "+501"
  },
  "Benin": {
   "code": "bj",
@@ -284,7 +306,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Africa/Porto-Novo"
-  ]
+  ],
+  "isd": "+229"
  },
  "Bermuda": {
   "code": "bm",
@@ -296,7 +319,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Atlantic/Bermuda"
-  ]
+  ],
+  "isd": "+1441"
  },
  "Bhutan": {
   "code": "bt",
@@ -308,13 +332,15 @@
   "number_format": "#,###.##",
   "timezones": [
    "Asia/Thimphu"
-  ]
+  ],
+  "isd": "+975"
  },
  "Bolivia, Plurinational State of": {
   "code": "bo",
   "currency": "BOB",
   "currency_name": "Boliviano",
-  "number_format": "#,###.##"
+  "number_format": "#,###.##",
+  "isd": "+591"
  },
  "Bonaire, Sint Eustatius and Saba": {
   "code": "bq",
@@ -329,7 +355,8 @@
   "number_format": "#.###,##",
   "timezones": [
    "Europe/Sarajevo"
-  ]
+  ],
+  "isd": "+387"
  },
  "Botswana": {
   "code": "bw",
@@ -341,11 +368,13 @@
   "number_format": "#,###.##",
   "timezones": [
    "Africa/Gaborone"
-  ]
+  ],
+  "isd": "+267"
  },
  "Bouvet Island": {
   "code": "bv",
-  "number_format": "#,###.##"
+  "number_format": "#,###.##",
+  "isd": "+47"
  },
  "Brazil": {
   "code": "br",
@@ -372,7 +401,8 @@
    "America/Rio_Branco",
    "America/Santarem",
    "America/Sao_Paulo"
-  ]
+  ],
+  "isd": "+55"
  },
  "British Indian Ocean Territory": {
   "code": "io",
@@ -382,7 +412,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Indian/Chagos"
-  ]
+  ],
+  "isd": "+246"
  },
  "Brunei Darussalam": {
   "code": "bn",
@@ -391,7 +422,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Asia/Brunei"
-  ]
+  ],
+  "isd": "+673"
  },
  "Bulgaria": {
   "code": "bg",
@@ -403,7 +435,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Europe/Sofia"
-  ]
+  ],
+  "isd": "+359"
  },
  "Burkina Faso": {
   "code": "bf",
@@ -415,7 +448,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Africa/Ouagadougou"
-  ]
+  ],
+  "isd": "+226"
  },
  "Burundi": {
   "code": "bi",
@@ -427,7 +461,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Africa/Bujumbura"
-  ]
+  ],
+  "isd": "+257"
  },
  "Cambodia": {
   "code": "kh",
@@ -439,7 +474,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Asia/Phnom_Penh"
-  ]
+  ],
+  "isd": "+855"
  },
  "Cameroon": {
   "code": "cm",
@@ -451,7 +487,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Africa/Douala"
-  ]
+  ],
+  "isd": "+237"
  },
  "Canada": {
   "code": "ca",
@@ -491,7 +528,8 @@
    "America/Whitehorse",
    "America/Winnipeg",
    "America/Yellowknife"
-  ]
+  ],
+  "isd": "+1"
  },
  "Cape Verde": {
   "code": "cv",
@@ -503,7 +541,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Atlantic/Cape_Verde"
-  ]
+  ],
+  "isd": "+238"
  },
  "Cayman Islands": {
   "code": "ky",
@@ -515,7 +554,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "America/Cayman"
-  ]
+  ],
+  "isd": "+ 345"
  },
  "Central African Republic": {
   "code": "cf",
@@ -527,7 +567,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Africa/Bangui"
-  ]
+  ],
+  "isd": "+236"
  },
  "Chad": {
   "code": "td",
@@ -539,7 +580,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Africa/Ndjamena"
-  ]
+  ],
+  "isd": "+235"
  },
  "Chile": {
   "code": "cl",
@@ -552,7 +594,8 @@
   "timezones": [
    "America/Santiago",
    "Pacific/Easter"
-  ]
+  ],
+  "isd": "+56"
  },
  "China": {
   "code": "cn",
@@ -568,14 +611,16 @@
    "Asia/Kashgar",
    "Asia/Shanghai",
    "Asia/Urumqi"
-  ]
+  ],
+  "isd": "+86"
  },
  "Christmas Island": {
   "code": "cx",
   "number_format": "#,###.##",
   "timezones": [
    "Indian/Christmas"
-  ]
+  ],
+  "isd": "+61"
  },
  "Cocos (Keeling) Islands": {
   "code": "cc",
@@ -585,7 +630,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Indian/Cocos"
-  ]
+  ],
+  "isd": "+61"
  },
  "Colombia": {
   "code": "co",
@@ -597,7 +643,8 @@
   "number_format": "#.###,##",
   "timezones": [
    "America/Bogota"
-  ]
+  ],
+  "isd": "+57"
  },
  "Comoros": {
   "code": "km",
@@ -609,7 +656,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Indian/Comoro"
-  ]
+  ],
+  "isd": "+269"
  },
  "Congo": {
   "code": "cg",
@@ -618,7 +666,8 @@
   "currency_name": "Central African CFA Franc",
   "currency_symbol": "FCFA",
   "currency_fraction": "Centime",
-  "currency_fraction_units": 100
+  "currency_fraction_units": 100,
+  "isd": "+242"
  },
  "Congo, The Democratic Republic of the": {
   "code": "cd",
@@ -627,7 +676,8 @@
   "currency_name": "Congolese franc",
   "currency_symbol": "FC",
   "currency_fraction": "Centime",
-  "currency_fraction_units": 100
+  "currency_fraction_units": 100,
+  "isd": "+243"
  },
  "Cook Islands": {
   "code": "ck",
@@ -637,7 +687,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Pacific/Rarotonga"
-  ]
+  ],
+  "isd": "+682"
  },
  "Costa Rica": {
   "code": "cr",
@@ -649,7 +700,8 @@
   "number_format": "#.###,##",
   "timezones": [
    "America/Costa_Rica"
-  ]
+  ],
+  "isd": "+506"
  },
  "Croatia": {
   "code": "hr",
@@ -661,7 +713,8 @@
   "number_format": "#.###,##",
   "timezones": [
    "Europe/Zagreb"
-  ]
+  ],
+  "isd": "+385"
  },
  "Cuba": {
   "code": "cu",
@@ -673,7 +726,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "America/Havana"
-  ]
+  ],
+  "isd": "+53"
  },
  "Cura\u00e7ao": {
   "code": "cw",
@@ -692,7 +746,8 @@
   "number_format": "#.###,##",
   "timezones": [
    "Asia/Nicosia"
-  ]
+  ],
+  "isd": "+357"
  },
  "Czech Republic": {
   "code": "cz",
@@ -704,7 +759,8 @@
   "number_format": "#.###,##",
   "timezones": [
    "Europe/Prague"
-  ]
+  ],
+  "isd": "+420"
  },
  "Denmark": {
   "code": "dk",
@@ -716,7 +772,8 @@
   "number_format": "#.###,##",
   "timezones": [
    "Europe/Copenhagen"
-  ]
+  ],
+  "isd": "+45"
  },
  "Djibouti": {
   "code": "dj",
@@ -728,7 +785,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Africa/Djibouti"
-  ]
+  ],
+  "isd": "+253"
  },
  "Dominica": {
   "code": "dm",
@@ -740,7 +798,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "America/Dominica"
-  ]
+  ],
+  "isd": "+1767"
  },
  "Dominican Republic": {
   "code": "do",
@@ -752,7 +811,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "America/Santo_Domingo"
-  ]
+  ],
+  "isd": "+1849"
  },
  "Ecuador": {
   "code": "ec",
@@ -763,7 +823,8 @@
   "timezones": [
    "America/Guayaquil",
    "Pacific/Galapagos"
-  ]
+  ],
+  "isd": "+593"
  },
  "Egypt": {
   "code": "eg",
@@ -775,12 +836,13 @@
   "number_format": "#,###.##",
   "timezones": [
    "Africa/Cairo"
-  ]
+  ],
+  "isd": "+20"
  },
  "El Salvador": {
   "code": "sv",
   "currency": "USD",
-  "currency_fraction": "Centavo",
+  "currency_fraction": "Cent",
   "currency_fraction_units": 100,
   "smallest_currency_fraction_value": 0.01,
   "currency_name": "Dolar estadounidense",
@@ -789,7 +851,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "America/El_Salvador"
-  ]
+  ],
+  "isd": "+503"
  },
  "Equatorial Guinea": {
   "code": "gq",
@@ -801,7 +864,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Africa/Malabo"
-  ]
+  ],
+  "isd": "+240"
  },
  "Eritrea": {
   "code": "er",
@@ -813,7 +877,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Africa/Asmara"
-  ]
+  ],
+  "isd": "+291"
  },
  "Estonia": {
   "code": "ee",
@@ -825,7 +890,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Europe/Tallinn"
-  ]
+  ],
+  "isd": "+372"
  },
  "Ethiopia": {
   "code": "et",
@@ -837,13 +903,15 @@
   "number_format": "#,###.##",
   "timezones": [
    "Africa/Addis_Ababa"
-  ]
+  ],
+  "isd": "+251"
  },
  "Falkland Islands (Malvinas)": {
   "code": "fk",
   "currency": "FKP",
   "currency_name": "Falkland Islands Pound",
-  "number_format": "#,###.##"
+  "number_format": "#,###.##",
+  "isd": "+500"
  },
  "Faroe Islands": {
   "code": "fo",
@@ -853,7 +921,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Atlantic/Faroe"
-  ]
+  ],
+  "isd": "+298"
  },
  "Fiji": {
   "code": "fj",
@@ -865,7 +934,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Pacific/Fiji"
-  ]
+  ],
+  "isd": "+679"
  },
  "Finland": {
   "code": "fi",
@@ -877,7 +947,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Europe/Helsinki"
-  ]
+  ],
+  "isd": "+358"
  },
  "France": {
   "code": "fr",
@@ -890,14 +961,16 @@
   "date_format": "dd/mm/yyyy",
   "timezones": [
    "Europe/Paris"
-  ]
+  ],
+  "isd": "+33"
  },
  "French Guiana": {
   "code": "gf",
   "number_format": "#,###.##",
   "timezones": [
    "America/Cayenne"
-  ]
+  ],
+  "isd": "+594"
  },
  "French Polynesia": {
   "code": "pf",
@@ -909,11 +982,13 @@
    "Pacific/Gambier",
    "Pacific/Marquesas",
    "Pacific/Tahiti"
-  ]
+  ],
+  "isd": "+689"
  },
  "French Southern Territories": {
   "code": "tf",
-  "number_format": "#,###.##"
+  "number_format": "#,###.##",
+  "isd": "+262"
  },
  "Gabon": {
   "code": "ga",
@@ -925,7 +1000,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Africa/Libreville"
-  ]
+  ],
+  "isd": "+241"
  },
  "Gambia": {
   "code": "gm",
@@ -934,7 +1010,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Africa/Banjul"
-  ]
+  ],
+  "isd": "+220"
  },
  "Georgia": {
   "code": "ge",
@@ -944,7 +1021,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Asia/Tbilisi"
-  ]
+  ],
+  "isd": "+995"
  },
  "Germany": {
   "code": "de",
@@ -954,9 +1032,12 @@
   "smallest_currency_fraction_value": 0.01,
   "currency_symbol": "\u20ac",
   "number_format": "#.###,##",
+  "date_format": "dd.mm.yyyy",
+  "time_format": "HH:mm",
   "timezones": [
    "Europe/Berlin"
-  ]
+  ],
+  "isd": "+49"
  },
  "Ghana": {
   "code": "gh",
@@ -967,7 +1048,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Africa/Accra"
-  ]
+  ],
+  "isd": "+233"
  },
  "Gibraltar": {
   "code": "gi",
@@ -979,7 +1061,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Europe/Gibraltar"
-  ]
+  ],
+  "isd": "+350"
  },
  "Greece": {
   "code": "gr",
@@ -991,7 +1074,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Europe/Athens"
-  ]
+  ],
+  "isd": "+30"
  },
  "Greenland": {
   "code": "gl",
@@ -1001,7 +1085,8 @@
    "America/Godthab",
    "America/Scoresbysund",
    "America/Thule"
-  ]
+  ],
+  "isd": "+299"
  },
  "Grenada": {
   "code": "gd",
@@ -1013,21 +1098,24 @@
   "number_format": "#,###.##",
   "timezones": [
    "America/Grenada"
-  ]
+  ],
+  "isd": "+1473"
  },
  "Guadeloupe": {
   "code": "gp",
   "number_format": "#,###.##",
   "timezones": [
    "America/Guadeloupe"
-  ]
+  ],
+  "isd": "+590"
  },
  "Guam": {
   "code": "gu",
   "number_format": "#,###.##",
   "timezones": [
    "Pacific/Guam"
-  ]
+  ],
+  "isd": "+1671"
  },
  "Guatemala": {
   "code": "gt",
@@ -1039,7 +1127,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "America/Guatemala"
-  ]
+  ],
+  "isd": "+502"
  },
  "Guernsey": {
   "code": "gg",
@@ -1049,7 +1138,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Europe/London"
-  ]
+  ],
+  "isd": "+44"
  },
  "Guinea": {
   "code": "gn",
@@ -1061,7 +1151,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Africa/Conakry"
-  ]
+  ],
+  "isd": "+224"
  },
  "Guinea-Bissau": {
   "code": "gw",
@@ -1073,7 +1164,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Africa/Bissau"
-  ]
+  ],
+  "isd": "+245"
  },
  "Guyana": {
   "code": "gy",
@@ -1085,7 +1177,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "America/Guyana"
-  ]
+  ],
+  "isd": "+592"
  },
  "Haiti": {
   "code": "ht",
@@ -1098,15 +1191,18 @@
   "timezones": [
    "America/Guatemala",
    "America/Port-au-Prince"
-  ]
+  ],
+  "isd": "+509"
  },
  "Heard Island and McDonald Islands": {
   "code": "hm",
-  "number_format": "#,###.##"
+  "number_format": "#,###.##",
+  "isd": "+0"
  },
  "Holy See (Vatican City State)": {
   "code": "va",
-  "number_format": "#,###.##"
+  "number_format": "#,###.##",
+  "isd": "+379"
  },
  "Honduras": {
   "code": "hn",
@@ -1118,7 +1214,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "America/Tegucigalpa"
-  ]
+  ],
+  "isd": "+504"
  },
  "Hong Kong": {
   "code": "hk",
@@ -1130,7 +1227,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Asia/Hong_Kong"
-  ]
+  ],
+  "isd": "+852"
  },
  "Hungary": {
   "code": "hu",
@@ -1143,7 +1241,8 @@
   "number_format": "#.###",
   "timezones": [
    "Europe/Budapest"
-  ]
+  ],
+  "isd": "+36"
  },
  "Iceland": {
   "code": "is",
@@ -1155,7 +1254,8 @@
   "number_format": "#.###",
   "timezones": [
    "Atlantic/Reykjavik"
-  ]
+  ],
+  "isd": "+354"
  },
  "India": {
   "code": "in",
@@ -1167,7 +1267,8 @@
   "number_format": "#,##,###.##",
   "timezones": [
    "Asia/Kolkata"
-  ]
+  ],
+  "isd": "+91"
  },
  "Indonesia": {
   "code": "id",
@@ -1182,7 +1283,8 @@
    "Asia/Jayapura",
    "Asia/Makassar",
    "Asia/Pontianak"
-  ]
+  ],
+  "isd": "+62"
  },
  "Iran": {
   "code": "ir",
@@ -1192,7 +1294,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Asia/Tehran"
-  ]
+  ],
+  "isd": "+98"
  },
  "Iraq": {
   "code": "iq",
@@ -1204,7 +1307,8 @@
   "number_format": "#,###.###",
   "timezones": [
    "Asia/Baghdad"
-  ]
+  ],
+  "isd": "+964"
  },
  "Ireland": {
   "code": "ie",
@@ -1216,7 +1320,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Europe/Dublin"
-  ]
+  ],
+  "isd": "+353"
  },
  "Isle of Man": {
   "code": "im",
@@ -1226,7 +1331,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Europe/London"
-  ]
+  ],
+  "isd": "+44"
  },
  "Israel": {
   "code": "il",
@@ -1238,7 +1344,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Asia/Jerusalem"
-  ]
+  ],
+  "isd": "+972"
  },
  "Italy": {
   "code": "it",
@@ -1251,7 +1358,8 @@
   "date_format": "dd/mm/yyyy",
   "timezones": [
    "Europe/Rome"
-  ]
+  ],
+  "isd": "+39"
  },
  "Ivory Coast": {
   "code": "ci",
@@ -1263,7 +1371,8 @@
   "number_format": "#,###.##",
   "timeszones": [
    "Africa/Abidjan"
-  ]
+  ],
+  "isd": "+225"
  },
  "Jamaica": {
   "code": "jm",
@@ -1275,7 +1384,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "America/Jamaica"
-  ]
+  ],
+  "isd": "+1876"
  },
  "Japan": {
   "code": "jp",
@@ -1287,7 +1397,8 @@
   "number_format": "#,###",
   "timezones": [
    "Asia/Tokyo"
-  ]
+  ],
+  "isd": "+81"
  },
  "Jersey": {
   "code": "je",
@@ -1297,7 +1408,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Europe/London"
-  ]
+  ],
+  "isd": "+44"
  },
  "Jordan": {
   "code": "jo",
@@ -1309,7 +1421,8 @@
   "number_format": "#,###.###",
   "timezones": [
    "Asia/Amman"
-  ]
+  ],
+  "isd": "+962"
  },
  "Kazakhstan": {
   "code": "kz",
@@ -1325,7 +1438,8 @@
    "Asia/Aqtobe",
    "Asia/Oral",
    "Asia/Qyzylorda"
-  ]
+  ],
+  "isd": "+7"
  },
  "Kenya": {
   "code": "ke",
@@ -1337,7 +1451,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Africa/Nairobi"
-  ]
+  ],
+  "isd": "+254"
  },
  "Kiribati": {
   "code": "ki",
@@ -1349,19 +1464,22 @@
    "Pacific/Enderbury",
    "Pacific/Kiritimati",
    "Pacific/Tarawa"
-  ]
+  ],
+  "isd": "+686"
  },
  "Korea, Democratic Peoples Republic of": {
   "code": "kp",
   "currency": "KPW",
   "currency_name": "North Korean Won",
-  "number_format": "#,###.##"
+  "number_format": "#,###.##",
+  "isd": "+850"
  },
  "Korea, Republic of": {
   "code": "kr",
   "currency": "KRW",
   "currency_name": "Won",
-  "number_format": "#,###"
+  "number_format": "#,###",
+  "isd": "+82"
  },
  "Kuwait": {
   "code": "kw",
@@ -1373,7 +1491,8 @@
   "number_format": "#,###.###",
   "timezones": [
    "Asia/Kuwait"
-  ]
+  ],
+  "isd": "+965"
  },
  "Kyrgyzstan": {
   "code": "kg",
@@ -1385,16 +1504,18 @@
   "number_format": "#,###.##",
   "timezones": [
    "Asia/Bishkek"
-  ]
+  ],
+  "isd": "+996"
  },
  "Lao Peoples Democratic Republic": {
   "code": "la",
   "currency": "LAK",
   "currency_name": "Kip",
   "number_format": "#,###.##",
-  "timezones":[
-      "Asia/Vientiane"
-  ]
+  "timezones": [
+   "Asia/Vientiane"
+  ],
+  "isd": "+856"
  },
  "Latvia": {
   "code": "lv",
@@ -1406,7 +1527,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Europe/Riga"
-  ]
+  ],
+  "isd": "+371"
  },
  "Lebanon": {
   "code": "lb",
@@ -1418,7 +1540,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Asia/Beirut"
-  ]
+  ],
+  "isd": "+961"
  },
  "Lesotho": {
   "code": "ls",
@@ -1430,7 +1553,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Africa/Maseru"
-  ]
+  ],
+  "isd": "+266"
  },
  "Liberia": {
   "code": "lr",
@@ -1442,7 +1566,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Africa/Monrovia"
-  ]
+  ],
+  "isd": "+231"
  },
  "Libya": {
   "code": "ly",
@@ -1454,7 +1579,8 @@
   "number_format": "#,###.###",
   "timezones": [
    "Africa/Tripoli"
-  ]
+  ],
+  "isd": "+218"
  },
  "Liechtenstein": {
   "code": "li",
@@ -1464,7 +1590,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Europe/Vaduz"
-  ]
+  ],
+  "isd": "+423"
  },
  "Lithuania": {
   "code": "lt",
@@ -1477,7 +1604,8 @@
   "number_format": "# ###,##",
   "timezones": [
    "Europe/Vilnius"
-  ]
+  ],
+  "isd": "+370"
  },
  "Luxembourg": {
   "code": "lu",
@@ -1489,13 +1617,15 @@
   "number_format": "#,###.##",
   "timezones": [
    "Europe/Luxembourg"
-  ]
+  ],
+  "isd": "+352"
  },
  "Macao": {
   "code": "mo",
   "currency": "MOP",
   "currency_name": "Pataca",
-  "number_format": "#,###.##"
+  "number_format": "#,###.##",
+  "isd": "+853"
  },
  "Macedonia": {
   "code": "mk",
@@ -1504,7 +1634,8 @@
   "currency_fraction_units": 100,
   "currency_name": "Denar",
   "currency_symbol": "\u0434\u0435\u043d",
-  "number_format": "#,###.##"
+  "number_format": "#,###.##",
+  "isd": "+389"
  },
  "Madagascar": {
   "code": "mg",
@@ -1514,7 +1645,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Indian/Antananarivo"
-  ]
+  ],
+  "isd": "+261"
  },
  "Malawi": {
   "code": "mw",
@@ -1526,7 +1658,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Africa/Blantyre"
-  ]
+  ],
+  "isd": "+265"
  },
  "Malaysia": {
   "code": "my",
@@ -1539,7 +1672,8 @@
   "timezones": [
    "Asia/Kuala_Lumpur",
    "Asia/Kuching"
-  ]
+  ],
+  "isd": "+60"
  },
  "Maldives": {
   "code": "mv",
@@ -1551,7 +1685,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Indian/Maldives"
-  ]
+  ],
+  "isd": "+960"
  },
  "Mali": {
   "code": "ml",
@@ -1563,19 +1698,22 @@
   "number_format": "#,###.##",
   "timezones": [
    "Africa/Bamako"
-  ]
+  ],
+  "isd": "+223"
  },
  "Malta": {
   "code": "mt",
-  "currency": "MTL",
+  "currency": "EUR",
   "currency_fraction": "Cent",
   "currency_fraction_units": 100,
-  "currency_name": "Maltese Lira",
+  "smallest_currency_fraction_value": 0.01,
   "currency_symbol": "\u20ac",
   "number_format": "#,###.##",
+  "date_format": "dd/mm/yyyy",
   "timezones": [
    "Europe/Malta"
-  ]
+  ],
+  "isd": "+356"
  },
  "Marshall Islands": {
   "code": "mh",
@@ -1586,14 +1724,16 @@
   "timezones": [
    "Pacific/Kwajalein",
    "Pacific/Majuro"
-  ]
+  ],
+  "isd": "+692"
  },
  "Martinique": {
   "code": "mq",
   "number_format": "#,###.##",
   "timezones": [
    "America/Martinique"
-  ]
+  ],
+  "isd": "+596"
  },
  "Mauritania": {
   "code": "mr",
@@ -1605,7 +1745,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Africa/Nouakchott"
-  ]
+  ],
+  "isd": "+222"
  },
  "Mauritius": {
   "code": "mu",
@@ -1617,14 +1758,16 @@
   "number_format": "#,###",
   "timezones": [
    "Indian/Mauritius"
-  ]
+  ],
+  "isd": "+230"
  },
  "Mayotte": {
   "code": "yt",
   "number_format": "#,###.##",
   "timezones": [
    "Indian/Mayotte"
-  ]
+  ],
+  "isd": "+262"
  },
  "Mexico": {
   "code": "mx",
@@ -1647,17 +1790,20 @@
    "America/Ojinaga",
    "America/Santa_Isabel",
    "America/Tijuana"
-  ]
+  ],
+  "isd": "+52"
  },
  "Micronesia, Federated States of": {
   "code": "fm",
-  "number_format": "#,###.##"
+  "number_format": "#,###.##",
+  "isd": "+691"
  },
  "Moldova, Republic of": {
   "code": "md",
   "currency": "MDL",
   "currency_name": "Moldovan Leu",
-  "number_format": "#,###.##"
+  "number_format": "#,###.##",
+  "isd": "+373"
  },
  "Monaco": {
   "code": "mc",
@@ -1669,7 +1815,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Europe/Monaco"
-  ]
+  ],
+  "isd": "+377"
  },
  "Mongolia": {
   "code": "mn",
@@ -1684,7 +1831,8 @@
    "Asia/Choibalsan",
    "Asia/Hovd",
    "Asia/Ulaanbaatar"
-  ]
+  ],
+  "isd": "+976"
  },
  "Montenegro": {
   "code": "me",
@@ -1696,7 +1844,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Europe/Belgrade"
-  ]
+  ],
+  "isd": "+382"
  },
  "Montserrat": {
   "code": "ms",
@@ -1708,7 +1857,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "America/Montserrat"
-  ]
+  ],
+  "isd": "+1664"
  },
  "Morocco": {
   "code": "ma",
@@ -1720,7 +1870,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Africa/Casablanca"
-  ]
+  ],
+  "isd": "+212"
  },
  "Mozambique": {
   "code": "mz",
@@ -1731,7 +1882,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Africa/Maputo"
-  ]
+  ],
+  "isd": "+258"
  },
  "Myanmar": {
   "code": "mm",
@@ -1740,7 +1892,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Asia/Rangoon"
-  ]
+  ],
+  "isd": "+95"
  },
  "Namibia": {
   "code": "na",
@@ -1752,7 +1905,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Africa/Windhoek"
-  ]
+  ],
+  "isd": "+264"
  },
  "Nauru": {
   "code": "nr",
@@ -1762,7 +1916,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Pacific/Nauru"
-  ]
+  ],
+  "isd": "+674"
  },
  "Nepal": {
   "code": "np",
@@ -1774,7 +1929,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Asia/Kathmandu"
-  ]
+  ],
+  "isd": "+977"
  },
  "Netherlands": {
   "code": "nl",
@@ -1786,7 +1942,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Europe/Amsterdam"
-  ]
+  ],
+  "isd": "+31"
  },
  "New Caledonia": {
   "code": "nc",
@@ -1796,7 +1953,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Pacific/Noumea"
-  ]
+  ],
+  "isd": "+687"
  },
  "New Zealand": {
   "code": "nz",
@@ -1809,7 +1967,8 @@
   "timezones": [
    "Pacific/Auckland",
    "Pacific/Chatham"
-  ]
+  ],
+  "isd": "+64"
  },
  "Nicaragua": {
   "code": "ni",
@@ -1821,7 +1980,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "America/Managua"
-  ]
+  ],
+  "isd": "+505"
  },
  "Niger": {
   "code": "ne",
@@ -1833,7 +1993,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Africa/Niamey"
-  ]
+  ],
+  "isd": "+227"
  },
  "Nigeria": {
   "code": "ng",
@@ -1845,7 +2006,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Africa/Lagos"
-  ]
+  ],
+  "isd": "+234"
  },
  "Niue": {
   "code": "nu",
@@ -1855,21 +2017,24 @@
   "number_format": "#,###.##",
   "timezones": [
    "Pacific/Niue"
-  ]
+  ],
+  "isd": "+683"
  },
  "Norfolk Island": {
   "code": "nf",
   "number_format": "#,###.##",
   "timezones": [
    "Pacific/Norfolk"
-  ]
+  ],
+  "isd": "+672"
  },
  "Northern Mariana Islands": {
   "code": "mp",
   "number_format": "#,###.##",
   "timezones": [
    "Pacific/Saipan"
-  ]
+  ],
+  "isd": "+1670"
  },
  "Norway": {
   "code": "no",
@@ -1881,7 +2046,8 @@
   "number_format": "#.###,##",
   "timezones": [
    "Europe/Oslo"
-  ]
+  ],
+  "isd": "+47"
  },
  "Oman": {
   "code": "om",
@@ -1893,7 +2059,8 @@
   "number_format": "#,###.###",
   "timezones": [
    "Asia/Muscat"
-  ]
+  ],
+  "isd": "+968"
  },
  "Pakistan": {
   "code": "pk",
@@ -1905,7 +2072,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Asia/Karachi"
-  ]
+  ],
+  "isd": "+92"
  },
  "Palau": {
   "code": "pw",
@@ -1916,11 +2084,13 @@
   "number_format": "#,###.##",
   "timezones": [
    "Pacific/Palau"
-  ]
+  ],
+  "isd": "+680"
  },
  "Palestinian Territory, Occupied": {
   "code": "ps",
-  "number_format": "#,###.##"
+  "number_format": "#,###.##",
+  "isd": "+970"
  },
  "Panama": {
   "code": "pa",
@@ -1930,7 +2100,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "America/Panama"
-  ]
+  ],
+  "isd": "+507"
  },
  "Papua New Guinea": {
   "code": "pg",
@@ -1942,7 +2113,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Pacific/Port_Moresby"
-  ]
+  ],
+  "isd": "+675"
  },
  "Paraguay": {
   "code": "py",
@@ -1954,7 +2126,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "America/Asuncion"
-  ]
+  ],
+  "isd": "+595"
  },
  "Peru": {
   "code": "pe",
@@ -1966,7 +2139,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "America/Lima"
-  ]
+  ],
+  "isd": "+51"
  },
  "Philippines": {
   "code": "ph",
@@ -1979,14 +2153,16 @@
   "number_format": "#,###.##",
   "timezones": [
    "Asia/Manila"
-  ]
+  ],
+  "isd": "+63"
  },
  "Pitcairn": {
   "code": "pn",
   "number_format": "#,###.##",
   "timezones": [
    "Pacific/Pitcairn"
-  ]
+  ],
+  "isd": "+64"
  },
  "Poland": {
   "code": "pl",
@@ -1997,7 +2173,8 @@
   "number_format": "#.###,##",
   "timezones": [
    "Europe/Warsaw"
-  ]
+  ],
+  "isd": "+48"
  },
  "Portugal": {
   "code": "pt",
@@ -2011,14 +2188,16 @@
    "Atlantic/Azores",
    "Atlantic/Madeira",
    "Europe/Lisbon"
-  ]
+  ],
+  "isd": "+351"
  },
  "Puerto Rico": {
   "code": "pr",
   "number_format": "#,###.##",
   "timezones": [
    "America/Puerto_Rico"
-  ]
+  ],
+  "isd": "+1939"
  },
  "Qatar": {
   "code": "qa",
@@ -2030,7 +2209,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Asia/Qatar"
-  ]
+  ],
+  "isd": "+974"
  },
  "Romania": {
   "code": "ro",
@@ -2042,13 +2222,15 @@
   "number_format": "#,###.##",
   "timezones": [
    "Europe/Bucharest"
-  ]
+  ],
+  "isd": "+40"
  },
  "Russian Federation": {
   "code": "ru",
   "currency": "RUB",
   "currency_name": "Russian Ruble",
-  "number_format": "#.###,##"
+  "number_format": "#.###,##",
+  "isd": "+7"
  },
  "Rwanda": {
   "code": "rw",
@@ -2060,21 +2242,25 @@
   "number_format": "#,###.##",
   "timezones": [
    "Africa/Kigali"
-  ]
+  ],
+  "isd": "+250"
  },
  "R\u00e9union": {
   "code": "re",
-  "number_format": "#,###.##"
+  "number_format": "#,###.##",
+  "isd": "+262"
  },
  "Saint Barth\u00e9lemy": {
   "code": "bl",
-  "number_format": "#,###.##"
+  "number_format": "#,###.##",
+  "isd": "+590"
  },
  "Saint Helena, Ascension and Tristan da Cunha": {
   "code": "sh",
   "currency": "SHP",
   "currency_name": "Saint Helena Pound",
-  "number_format": "#,###.##"
+  "number_format": "#,###.##",
+  "isd": "+290"
  },
  "Saint Kitts and Nevis": {
   "code": "kn",
@@ -2086,7 +2272,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "America/St_Kitts"
-  ]
+  ],
+  "isd": "+1869"
  },
  "Saint Lucia": {
   "code": "lc",
@@ -2098,15 +2285,18 @@
   "number_format": "#,###.##",
   "timezones": [
    "America/St_Lucia"
-  ]
+  ],
+  "isd": "+1758"
  },
  "Saint Martin (French part)": {
   "code": "mf",
-  "number_format": "#,###.##"
+  "number_format": "#,###.##",
+  "isd": "+590"
  },
  "Saint Pierre and Miquelon": {
   "code": "pm",
-  "number_format": "#,###.##"
+  "number_format": "#,###.##",
+  "isd": "+508"
  },
  "Saint Vincent and the Grenadines": {
   "code": "vc",
@@ -2118,7 +2308,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "America/St_Vincent"
-  ]
+  ],
+  "isd": "+1784"
  },
  "Samoa": {
   "code": "ws",
@@ -2130,7 +2321,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Pacific/Apia"
-  ]
+  ],
+  "isd": "+685"
  },
  "San Marino": {
   "code": "sm",
@@ -2142,13 +2334,15 @@
   "number_format": "#,###.##",
   "timezones": [
    "Europe/Rome"
-  ]
+  ],
+  "isd": "+378"
  },
  "Sao Tome and Principe": {
   "code": "st",
   "currency": "STD",
   "currency_name": "Dobra",
-  "number_format": "#,###.##"
+  "number_format": "#,###.##",
+  "isd": "+239"
  },
  "Saudi Arabia": {
   "code": "sa",
@@ -2160,7 +2354,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Asia/Riyadh"
-  ]
+  ],
+  "isd": "+966"
  },
  "Senegal": {
   "code": "sn",
@@ -2172,7 +2367,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Africa/Dakar"
-  ]
+  ],
+  "isd": "+221"
  },
  "Serbia": {
   "code": "rs",
@@ -2184,7 +2380,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Europe/Belgrade"
-  ]
+  ],
+  "isd": "+381"
  },
  "Seychelles": {
   "code": "sc",
@@ -2196,7 +2393,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Indian/Mahe"
-  ]
+  ],
+  "isd": "+248"
  },
  "Sierra Leone": {
   "code": "sl",
@@ -2208,19 +2406,21 @@
   "number_format": "#,###.##",
   "timezones": [
    "Africa/Freetown"
-  ]
+  ],
+  "isd": "+232"
  },
  "Singapore": {
   "code": "sg",
   "currency": "SGD",
-  "currency_fraction": "Sen",
+  "currency_fraction": "Cent",
   "currency_fraction_units": 100,
   "currency_name": "Singapore Dollar",
   "currency_symbol": "$",
   "number_format": "#,###.##",
   "timezones": [
    "Asia/Singapore"
-  ]
+  ],
+  "isd": "+65"
  },
  "Sint Maarten (Dutch part)": {
   "code": "sx",
@@ -2236,7 +2436,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Europe/Bratislava"
-  ]
+  ],
+  "isd": "+421"
  },
  "Slovenia": {
   "code": "si",
@@ -2248,7 +2449,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Europe/Belgrade"
-  ]
+  ],
+  "isd": "+386"
  },
  "Solomon Islands": {
   "code": "sb",
@@ -2260,7 +2462,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Pacific/Guadalcanal"
-  ]
+  ],
+  "isd": "+677"
  },
  "Somalia": {
   "code": "so",
@@ -2272,7 +2475,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Africa/Mogadishu"
-  ]
+  ],
+  "isd": "+252"
  },
  "South Africa": {
   "code": "za",
@@ -2285,14 +2489,16 @@
   "number_format": "# ###.##",
   "timezones": [
    "Africa/Johannesburg"
-  ]
+  ],
+  "isd": "+27"
  },
  "South Georgia and the South Sandwich Islands": {
   "code": "gs",
   "currency_fraction": "Penny",
   "currency_fraction_units": 100,
   "currency_symbol": "\u00a3",
-  "number_format": "#,###.##"
+  "number_format": "#,###.##",
+  "isd": "+500"
  },
  "South Sudan": {
   "code": "ss",
@@ -2302,7 +2508,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Africa/Juba"
-  ]
+  ],
+  "isd": "+211"
  },
  "Spain": {
   "code": "es",
@@ -2316,7 +2523,8 @@
    "Africa/Ceuta",
    "Atlantic/Canary",
    "Europe/Madrid"
-  ]
+  ],
+  "isd": "+34"
  },
  "Sri Lanka": {
   "code": "lk",
@@ -2328,7 +2536,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Asia/Colombo"
-  ]
+  ],
+  "isd": "+94"
  },
  "Sudan": {
   "code": "sd",
@@ -2338,7 +2547,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Africa/Khartoum"
-  ]
+  ],
+  "isd": "+249"
  },
  "Suriname": {
   "code": "sr",
@@ -2349,11 +2559,13 @@
   "number_format": "#,###.##",
   "timezones": [
    "America/Paramaribo"
-  ]
+  ],
+  "isd": "+597"
  },
  "Svalbard and Jan Mayen": {
   "code": "sj",
-  "number_format": "#,###.##"
+  "number_format": "#,###.##",
+  "isd": "+47"
  },
  "Swaziland": {
   "code": "sz",
@@ -2365,7 +2577,8 @@
   "number_format": "#, ###.##",
   "timezones": [
    "Africa/Mbabane"
-  ]
+  ],
+  "isd": "+268"
  },
  "Sweden": {
   "code": "se",
@@ -2377,7 +2590,8 @@
   "number_format": "#.###,##",
   "timezones": [
    "Europe/Stockholm"
-  ]
+  ],
+  "isd": "+46"
  },
  "Switzerland": {
   "code": "ch",
@@ -2390,19 +2604,22 @@
   "number_format": "#'###.##",
   "timezones": [
    "Europe/Zurich"
-  ]
+  ],
+  "isd": "+41"
  },
  "Syria": {
   "code": "sy",
   "currency": "SYP",
   "currency_name": "Syrian Pound",
-  "number_format": "#,###.##"
+  "number_format": "#,###.##",
+  "isd": "+963"
  },
  "Taiwan": {
   "code": "tw",
   "currency": "TWD",
   "date_format": "yyyy-mm-dd",
-  "number_format": "#,###.##"
+  "number_format": "#,###.##",
+  "isd": "+886"
  },
  "Tajikistan": {
   "code": "tj",
@@ -2412,13 +2629,15 @@
   "number_format": "#,###.##",
   "timezones": [
    "Asia/Dushanbe"
-  ]
+  ],
+  "isd": "+992"
  },
  "Tanzania": {
   "code": "tz",
   "currency": "TZS",
   "currency_name": "Tanzanian Shilling",
-  "number_format": "#,###.##"
+  "number_format": "#,###.##",
+  "isd": "+255"
  },
  "Thailand": {
   "code": "th",
@@ -2430,11 +2649,13 @@
   "number_format": "#,###.##",
   "timezones": [
    "Asia/Bangkok"
-  ]
+  ],
+  "isd": "+66"
  },
  "Timor-Leste": {
   "code": "tl",
-  "number_format": "#,###.##"
+  "number_format": "#,###.##",
+  "isd": "+670"
  },
  "Togo": {
   "code": "tg",
@@ -2446,14 +2667,16 @@
   "number_format": "#,###.##",
   "timezones": [
    "Africa/Lome"
-  ]
+  ],
+  "isd": "+228"
  },
  "Tokelau": {
   "code": "tk",
   "number_format": "#,###.##",
   "timezones": [
    "Pacific/Fakaofo"
-  ]
+  ],
+  "isd": "+690"
  },
  "Tonga": {
   "code": "to",
@@ -2465,7 +2688,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Pacific/Tongatapu"
-  ]
+  ],
+  "isd": "+676"
  },
  "Trinidad and Tobago": {
   "code": "tt",
@@ -2477,7 +2701,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "America/Port_of_Spain"
-  ]
+  ],
+  "isd": "+1868"
  },
  "Tunisia": {
   "code": "tn",
@@ -2489,7 +2714,8 @@
   "number_format": "#,###.###",
   "timezones": [
    "Africa/Tunis"
-  ]
+  ],
+  "isd": "+216"
  },
  "Turkey": {
   "code": "tr",
@@ -2500,7 +2726,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Europe/Istanbul"
-  ]
+  ],
+  "isd": "+90"
  },
  "Turkmenistan": {
   "code": "tm",
@@ -2512,14 +2739,16 @@
   "number_format": "#,###.##",
   "timezones": [
    "Asia/Ashgabat"
-  ]
+  ],
+  "isd": "+993"
  },
  "Turks and Caicos Islands": {
   "code": "tc",
   "currency_fraction": "Cent",
   "currency_fraction_units": 100,
   "currency_symbol": "$",
-  "number_format": "#,###.##"
+  "number_format": "#,###.##",
+  "isd": "+1649"
  },
  "Tuvalu": {
   "code": "tv",
@@ -2529,7 +2758,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Pacific/Funafuti"
-  ]
+  ],
+  "isd": "+688"
  },
  "Uganda": {
   "code": "ug",
@@ -2541,7 +2771,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Africa/Kampala"
-  ]
+  ],
+  "isd": "+256"
  },
  "Ukraine": {
   "code": "ua",
@@ -2556,7 +2787,8 @@
    "Europe/Simferopol",
    "Europe/Uzhgorod",
    "Europe/Zaporozhye"
-  ]
+  ],
+  "isd": "+380"
  },
  "United Arab Emirates": {
   "code": "ae",
@@ -2568,7 +2800,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Asia/Dubai"
-  ]
+  ],
+  "isd": "+971"
  },
  "United Kingdom": {
   "code": "gb",
@@ -2580,7 +2813,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Europe/London"
-  ]
+  ],
+  "isd": "+44"
  },
  "United States": {
   "code": "us",
@@ -2623,7 +2857,8 @@
    "America/Sitka",
    "America/Yakutat",
    "Pacific/Honolulu"
-  ]
+  ],
+  "isd": "+1"
  },
  "United States Minor Outlying Islands": {
   "code": "um",
@@ -2639,7 +2874,8 @@
   "number_format": "#.###,##",
   "timezones": [
    "America/Montevideo"
-  ]
+  ],
+  "isd": "+598"
  },
  "Uzbekistan": {
   "code": "uz",
@@ -2652,7 +2888,8 @@
   "timezones": [
    "Asia/Samarkand",
    "Asia/Tashkent"
-  ]
+  ],
+  "isd": "+998"
  },
  "Vanuatu": {
   "code": "vu",
@@ -2664,7 +2901,8 @@
   "number_format": "#,###",
   "timezones": [
    "Pacific/Efate"
-  ]
+  ],
+  "isd": "+678"
  },
  "Venezuela, Bolivarian Republic of": {
   "code": "ve",
@@ -2672,28 +2910,33 @@
   "currency": "VEF",
   "currency_symbol": "Bs.",
   "currency_fraction": "Centimos",
-  "currency_fraction_units": 100
+  "currency_fraction_units": 100,
+  "isd": "+58"
  },
  "Vietnam": {
   "code": "vn",
   "currency": "VND",
   "currency_name": "Dong",
-  "number_format": "#.###"
+  "number_format": "#.###",
+  "isd": "+84"
  },
  "Virgin Islands, British": {
   "code": "vg",
-  "number_format": "#,###.##"
+  "number_format": "#,###.##",
+  "isd": "+1284"
  },
  "Virgin Islands, U.S.": {
   "code": "vi",
-  "number_format": "#,###.##"
+  "number_format": "#,###.##",
+  "isd": "+1340"
  },
  "Wallis and Futuna": {
   "code": "wf",
   "currency_fraction": "Centime",
   "currency_fraction_units": 100,
   "currency_symbol": "Fr",
-  "number_format": "#,###.##"
+  "number_format": "#,###.##",
+  "isd": "+681"
  },
  "Western Sahara": {
   "code": "eh",
@@ -2713,7 +2956,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Asia/Aden"
-  ]
+  ],
+  "isd": "+967"
  },
  "Zambia": {
   "code": "zm",
@@ -2725,7 +2969,8 @@
   "number_format": "#,###.##",
   "timezones": [
    "Africa/Lusaka"
-  ]
+  ],
+  "isd": "+260"
  },
  "Zimbabwe": {
   "code": "zw",
@@ -2737,10 +2982,12 @@
   "number_format": "# ###.##",
   "timezones": [
    "Africa/Harare"
-  ]
+  ],
+  "isd": "+263"
  },
  "\u00c5land Islands": {
   "code": "ax",
-  "number_format": "#,###.##"
+  "number_format": "#,###.##",
+  "isd": "+358"
  }
 }
diff --git a/frappe/geo/country_info.py b/frappe/geo/country_info.py
index 1de8467a6a..2aefa27170 100644
--- a/frappe/geo/country_info.py
+++ b/frappe/geo/country_info.py
@@ -22,7 +22,7 @@ def get_country_info(country=None):
 
 
 def get_all():
-	with open(os.path.join(os.path.dirname(__file__), "country_info.json"), "r") as local_info:
+	with open(os.path.join(os.path.dirname(__file__), "country_info.json")) as local_info:
 		all_data = json.loads(local_info.read())
 	return all_data
 
@@ -59,7 +59,7 @@ def get_translated_dict():
 
 
 def update():
-	with open(os.path.join(os.path.dirname(__file__), "currency_info.json"), "r") as nformats:
+	with open(os.path.join(os.path.dirname(__file__), "currency_info.json")) as nformats:
 		nformats = json.loads(nformats.read())
 
 	all_data = get_all()
diff --git a/frappe/geo/doctype/country/__init__.py b/frappe/geo/doctype/country/__init__.py
index 8b13789179..e69de29bb2 100644
--- a/frappe/geo/doctype/country/__init__.py
+++ b/frappe/geo/doctype/country/__init__.py
@@ -1 +0,0 @@
-
diff --git a/frappe/geo/doctype/country/country.js b/frappe/geo/doctype/country/country.js
index 62159a1fe7..75bb3f46d5 100644
--- a/frappe/geo/doctype/country/country.js
+++ b/frappe/geo/doctype/country/country.js
@@ -1,8 +1,6 @@
 // Copyright (c) 2016, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Country', {
-	refresh: function(frm) {
-
-	}
+frappe.ui.form.on("Country", {
+	refresh: function (frm) {},
 });
diff --git a/frappe/geo/doctype/country/country.py b/frappe/geo/doctype/country/country.py
index b3ba1b7127..f6be7a078d 100644
--- a/frappe/geo/doctype/country/country.py
+++ b/frappe/geo/doctype/country/country.py
@@ -1,7 +1,6 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
+# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
 
-import frappe
 from frappe.model.document import Document
 
 
diff --git a/frappe/geo/doctype/currency/__init__.py b/frappe/geo/doctype/currency/__init__.py
index 8b13789179..e69de29bb2 100644
--- a/frappe/geo/doctype/currency/__init__.py
+++ b/frappe/geo/doctype/currency/__init__.py
@@ -1 +0,0 @@
-
diff --git a/frappe/geo/doctype/currency/currency.js b/frappe/geo/doctype/currency/currency.js
index af2d6ebc4e..08915893a5 100644
--- a/frappe/geo/doctype/currency/currency.js
+++ b/frappe/geo/doctype/currency/currency.js
@@ -1,11 +1,11 @@
 // Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 // License: See license.txt
 
-frappe.ui.form.on('Currency', {
+frappe.ui.form.on("Currency", {
 	refresh(frm) {
 		frm.set_intro("");
-		if(!frm.doc.enabled) {
+		if (!frm.doc.enabled) {
 			frm.set_intro(__("This Currency is disabled. Enable to use in transactions"));
 		}
-	}
+	},
 });
diff --git a/frappe/geo/doctype/currency/currency.json b/frappe/geo/doctype/currency/currency.json
index db3fa5a19f..c51ab7f063 100644
--- a/frappe/geo/doctype/currency/currency.json
+++ b/frappe/geo/doctype/currency/currency.json
@@ -15,6 +15,7 @@
   "fraction_units",
   "smallest_currency_fraction_value",
   "symbol",
+  "symbol_on_right",
   "number_format"
  ],
  "fields": [
@@ -69,16 +70,23 @@
    "in_list_view": 1,
    "label": "Number Format",
    "options": "\n#,###.##\n#.###,##\n# ###.##\n# ###,##\n#'###.##\n#, ###.##\n#,##,###.##\n#,###.###\n#.###\n#,###"
+  },
+  {
+   "default": "0",
+   "fieldname": "symbol_on_right",
+   "fieldtype": "Check",
+   "label": "Show Currency Symbol on Right Side"
   }
  ],
  "icon": "fa fa-bitcoin",
  "idx": 1,
  "index_web_pages_for_search": 1,
  "links": [],
- "modified": "2020-10-29 06:33:12.879978",
+ "modified": "2022-07-04 09:42:52.425440",
  "modified_by": "Administrator",
  "module": "Geo",
  "name": "Currency",
+ "naming_rule": "By fieldname",
  "owner": "Administrator",
  "permissions": [
   {
@@ -109,5 +117,6 @@
  ],
  "sort_field": "modified",
  "sort_order": "DESC",
+ "states": [],
  "track_changes": 1
 }
\ No newline at end of file
diff --git a/frappe/geo/doctype/currency/currency.py b/frappe/geo/doctype/currency/currency.py
index dd5df57bab..93bcc063f8 100644
--- a/frappe/geo/doctype/currency/currency.py
+++ b/frappe/geo/doctype/currency/currency.py
@@ -2,7 +2,6 @@
 # License: MIT. See LICENSE
 
 import frappe
-from frappe import _, throw
 from frappe.model.document import Document
 
 
diff --git a/frappe/geo/doctype/currency/test_currency.py b/frappe/geo/doctype/currency/test_currency.py
index f93a452462..b02dd4258c 100644
--- a/frappe/geo/doctype/currency/test_currency.py
+++ b/frappe/geo/doctype/currency/test_currency.py
@@ -4,5 +4,10 @@
 # pre loaded
 
 import frappe
+from frappe.tests.utils import FrappeTestCase
 
-test_records = frappe.get_test_records("Currency")
+
+class TestUser(FrappeTestCase):
+	def test_default_currency_on_setup(self):
+		usd = frappe.get_doc("Currency", "USD")
+		self.assertDocumentEqual({"enabled": 1, "fraction": "Cent"}, usd)
diff --git a/frappe/geo/utils.py b/frappe/geo/utils.py
index 577c5de2ff..53caed452d 100644
--- a/frappe/geo/utils.py
+++ b/frappe/geo/utils.py
@@ -1,9 +1,6 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2020, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
-from pymysql import InternalError
-
 import frappe
 
 
@@ -65,9 +62,9 @@ def return_location(doctype, filters_sql):
 	if filters_sql:
 		try:
 			coords = frappe.db.sql(
-				"""SELECT name, location FROM `tab{}`  WHERE {}""".format(doctype, filters_sql), as_dict=True
+				f"""SELECT name, location FROM `tab{doctype}`  WHERE {filters_sql}""", as_dict=True
 			)
-		except InternalError:
+		except frappe.db.InternalError:
 			frappe.msgprint(frappe._("This Doctype does not contain location fields"), raise_exception=True)
 			return
 	else:
@@ -80,10 +77,10 @@ def return_coordinates(doctype, filters_sql):
 	if filters_sql:
 		try:
 			coords = frappe.db.sql(
-				"""SELECT name, latitude, longitude FROM `tab{}`  WHERE {}""".format(doctype, filters_sql),
+				f"""SELECT name, latitude, longitude FROM `tab{doctype}`  WHERE {filters_sql}""",
 				as_dict=True,
 			)
-		except InternalError:
+		except frappe.db.InternalError:
 			frappe.msgprint(
 				frappe._("This Doctype does not contain latitude and longitude fields"), raise_exception=True
 			)
diff --git a/frappe/handler.py b/frappe/handler.py
index 7b010eb716..cee6d3fbde 100644
--- a/frappe/handler.py
+++ b/frappe/handler.py
@@ -1,7 +1,9 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
+# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
 
+import os
 from mimetypes import guess_type
+from typing import TYPE_CHECKING
 
 from werkzeug.wrappers import Response
 
@@ -15,6 +17,10 @@ from frappe.utils.csvutils import build_csv_response
 from frappe.utils.image import optimize_image
 from frappe.utils.response import build_response
 
+if TYPE_CHECKING:
+	from frappe.core.doctype.file.file import File
+	from frappe.core.doctype.user.user import User
+
 ALLOWED_MIMETYPES = (
 	"image/png",
 	"image/jpeg",
@@ -25,6 +31,7 @@ ALLOWED_MIMETYPES = (
 	"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
 	"application/vnd.oasis.opendocument.text",
 	"application/vnd.oasis.opendocument.spreadsheet",
+	"text/plain",
 )
 
 
@@ -165,9 +172,9 @@ def upload_file():
 		if frappe.get_system_settings("allow_guests_to_upload_files"):
 			ignore_permissions = True
 		else:
-			return
+			raise frappe.PermissionError
 	else:
-		user = frappe.get_doc("User", frappe.session.user)
+		user: "User" = frappe.get_doc("User", frappe.session.user)
 		ignore_permissions = False
 
 	files = frappe.request.files
@@ -199,17 +206,19 @@ def upload_file():
 	frappe.local.uploaded_file = content
 	frappe.local.uploaded_filename = filename
 
-	if not file_url and (frappe.session.user == "Guest" or (user and not user.has_desk_access())):
+	if content is not None and (
+		frappe.session.user == "Guest" or (user and not user.has_desk_access())
+	):
 		filetype = guess_type(filename)[0]
 		if filetype not in ALLOWED_MIMETYPES:
-			frappe.throw(_("You can only upload JPG, PNG, PDF, or Microsoft documents."))
+			frappe.throw(_("You can only upload JPG, PNG, PDF, TXT or Microsoft documents."))
 
 	if method:
 		method = frappe.get_attr(method)
 		is_whitelisted(method)
 		return method()
 	else:
-		ret = frappe.get_doc(
+		return frappe.get_doc(
 			{
 				"doctype": "File",
 				"attached_to_doctype": doctype,
@@ -221,9 +230,26 @@ def upload_file():
 				"is_private": cint(is_private),
 				"content": content,
 			}
-		)
-		ret.save(ignore_permissions=ignore_permissions)
-		return ret
+		).save(ignore_permissions=ignore_permissions)
+
+
+@frappe.whitelist(allow_guest=True)
+def download_file(file_url: str):
+	"""
+	Download file using token and REST API. Valid session or
+	token is required to download private files.
+
+	Method : GET
+	Endpoints : download_file, frappe.core.doctype.file.file.download_file
+	URL Params : file_name = /path/to/file relative to site path
+	"""
+	file: "File" = frappe.get_doc("File", {"file_url": file_url})
+	if not file.is_downloadable():
+		raise frappe.PermissionError
+
+	frappe.local.response.filename = os.path.basename(file_url)
+	frappe.local.response.filecontent = file.get_content()
+	frappe.local.response.type = "download"
 
 
 def get_attr(cmd):
diff --git a/frappe/hooks.py b/frappe/hooks.py
index d3de3877ba..14e76adc22 100644
--- a/frappe/hooks.py
+++ b/frappe/hooks.py
@@ -4,8 +4,6 @@ app_name = "frappe"
 app_title = "Frappe Framework"
 app_publisher = "Frappe Technologies"
 app_description = "Full stack web framework with Python, Javascript, MariaDB, Redis, Node"
-app_icon = "octicon octicon-circuit-board"
-app_color = "orange"
 source_link = "https://github.com/frappe/frappe"
 app_license = "MIT"
 app_logo_url = "/assets/frappe/images/frappe-framework-logo.svg"
@@ -148,7 +146,7 @@ doc_events = {
 			"frappe.core.doctype.activity_log.feed.update_feed",
 			"frappe.workflow.doctype.workflow_action.workflow_action.process_workflow_actions",
 			"frappe.automation.doctype.assignment_rule.assignment_rule.apply",
-			"frappe.core.doctype.file.file.attach_files_to_document",
+			"frappe.core.doctype.file.utils.attach_files_to_document",
 			"frappe.event_streaming.doctype.event_update_log.event_update_log.notify_consumers",
 			"frappe.automation.doctype.assignment_rule.assignment_rule.update_due_date",
 			"frappe.core.doctype.user_type.user_type.apply_permissions_for_non_standard_user_type",
@@ -182,12 +180,10 @@ doc_events = {
 		"on_update": "frappe.integrations.doctype.google_contacts.google_contacts.update_contacts_to_google_contacts",
 	},
 	"DocType": {
-		"after_insert": "frappe.cache_manager.build_domain_restriced_doctype_cache",
-		"after_save": "frappe.cache_manager.build_domain_restriced_doctype_cache",
+		"on_update": "frappe.cache_manager.build_domain_restriced_doctype_cache",
 	},
 	"Page": {
-		"after_insert": "frappe.cache_manager.build_domain_restriced_page_cache",
-		"after_save": "frappe.cache_manager.build_domain_restriced_page_cache",
+		"on_update": "frappe.cache_manager.build_domain_restriced_page_cache",
 	},
 }
 
@@ -203,7 +199,6 @@ scheduler_events = {
 		"frappe.email.queue.flush",
 		"frappe.email.doctype.email_account.email_account.pull",
 		"frappe.email.doctype.email_account.email_account.notify_unreplied",
-		"frappe.integrations.doctype.razorpay_settings.razorpay_settings.capture_payment",
 		"frappe.utils.global_search.sync_global_search",
 		"frappe.monitor.flush",
 	],
@@ -221,12 +216,9 @@ scheduler_events = {
 	"daily": [
 		"frappe.email.queue.set_expiry_for_email_queue",
 		"frappe.desk.notifications.clear_notifications",
-		"frappe.core.doctype.error_log.error_log.set_old_logs_as_seen",
 		"frappe.desk.doctype.event.event.send_event_digest",
 		"frappe.sessions.clear_expired_sessions",
 		"frappe.email.doctype.notification.notification.trigger_daily_alerts",
-		"frappe.utils.scheduler.restrict_scheduler_events_if_dormant",
-		"frappe.email.doctype.auto_email_report.auto_email_report.send_daily",
 		"frappe.website.doctype.personal_data_deletion_request.personal_data_deletion_request.remove_unverified_record",
 		"frappe.desk.form.document_follow.send_daily_updates",
 		"frappe.social.doctype.energy_point_settings.energy_point_settings.allocate_review_points",
@@ -241,12 +233,12 @@ scheduler_events = {
 		"frappe.integrations.doctype.dropbox_settings.dropbox_settings.take_backups_daily",
 		"frappe.utils.change_log.check_for_update",
 		"frappe.integrations.doctype.s3_backup_settings.s3_backup_settings.take_backups_daily",
+		"frappe.email.doctype.auto_email_report.auto_email_report.send_daily",
 		"frappe.integrations.doctype.google_drive.google_drive.daily_backup",
 	],
 	"weekly_long": [
 		"frappe.integrations.doctype.dropbox_settings.dropbox_settings.take_backups_weekly",
 		"frappe.integrations.doctype.s3_backup_settings.s3_backup_settings.take_backups_weekly",
-		"frappe.desk.doctype.route_history.route_history.flush_old_route_records",
 		"frappe.desk.form.document_follow.send_weekly_updates",
 		"frappe.social.doctype.energy_point_log.energy_point_log.send_weekly_summary",
 		"frappe.integrations.doctype.google_drive.google_drive.weekly_backup",
@@ -281,7 +273,7 @@ setup_wizard_exception = [
 	"frappe.desk.page.setup_wizard.setup_wizard.log_setup_wizard_exception",
 ]
 
-before_migrate = ["frappe.patches.v11_0.sync_user_permission_doctype_before_migrate.execute"]
+before_migrate = []
 after_migrate = ["frappe.website.doctype.website_theme.website_theme.after_migrate"]
 
 otp_methods = ["OTP App", "Email", "SMS"]
@@ -370,4 +362,16 @@ global_search_doctypes = {
 	]
 }
 
+override_whitelisted_methods = {
+	"frappe.core.doctype.file.file.download_file": "download_file",
+	"frappe.core.doctype.file.file.unzip_file": "frappe.core.api.file.unzip_file",
+	"frappe.core.doctype.file.file.get_attached_images": "frappe.core.api.file.get_attached_images",
+	"frappe.core.doctype.file.file.get_files_in_folder": "frappe.core.api.file.get_files_in_folder",
+	"frappe.core.doctype.file.file.get_files_by_search_text": "frappe.core.api.file.get_files_by_search_text",
+	"frappe.core.doctype.file.file.get_max_file_size": "frappe.core.api.file.get_max_file_size",
+	"frappe.core.doctype.file.file.create_new_folder": "frappe.core.api.file.create_new_folder",
+	"frappe.core.doctype.file.file.move_file": "frappe.core.api.file.move_file",
+	"frappe.core.doctype.file.file.zip_files": "frappe.core.api.file.zip_files",
+}
+
 translated_search_doctypes = ["DocType", "Role", "Country", "Gender", "Salutation"]
diff --git a/frappe/installer.py b/frappe/installer.py
index 634d6287f8..32ab45e383 100644
--- a/frappe/installer.py
+++ b/frappe/installer.py
@@ -1,15 +1,29 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
+# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
 
 import json
 import os
 import sys
 from collections import OrderedDict
-from typing import Dict, List, Tuple
+
+import click
 
 import frappe
 from frappe.defaults import _clear_cache
-from frappe.utils import is_git_url
+from frappe.utils import cint, is_git_url
+
+
+def _is_scheduler_enabled() -> bool:
+	enable_scheduler = False
+	try:
+		frappe.connect()
+		enable_scheduler = cint(frappe.db.get_single_value("System Settings", "enable_scheduler"))
+	except Exception:
+		pass
+	finally:
+		frappe.db.close()
+
+	return bool(enable_scheduler)
 
 
 def _new_site(
@@ -28,15 +42,13 @@ def _new_site(
 	db_type=None,
 	db_host=None,
 	db_port=None,
-	new_site=False,
 ):
 	"""Install a new Frappe site"""
 
-	from frappe.commands.scheduler import _is_scheduler_enabled
 	from frappe.utils import get_site_path, scheduler, touch_file
 
 	if not force and os.path.exists(site):
-		print("Site {0} already exists".format(site))
+		print(f"Site {site} already exists")
 		sys.exit(1)
 
 	if no_mariadb_socket and not db_type == "mariadb":
@@ -80,7 +92,13 @@ def _new_site(
 	)
 
 	for app in apps_to_install:
-		install_app(app, verbose=verbose, set_as_patched=not source_sql)
+		# NOTE: not using force here for 2 reasons:
+		# 	1. It's not really needed here as we've freshly installed a new db
+		# 	2. If someone uses a sql file to do restore and that file already had
+		# 		installed_apps then it might cause problems as that sql file can be of any previous version(s)
+		# 		which might be incompatible with the current version and using force might cause problems.
+		# 		Example: the DocType DocType might not have `migration_hash` column which will cause failure in the restore.
+		install_app(app, verbose=verbose, set_as_patched=not source_sql, force=False)
 
 	os.remove(installing)
 
@@ -143,7 +161,7 @@ def install_db(
 	frappe.flags.in_install_db = False
 
 
-def find_org(org_repo: str) -> Tuple[str, str]:
+def find_org(org_repo: str) -> tuple[str, str]:
 	"""find the org a repo is in
 
 	find_org()
@@ -171,7 +189,7 @@ def find_org(org_repo: str) -> Tuple[str, str]:
 	raise InvalidRemoteException
 
 
-def fetch_details_from_tag(_tag: str) -> Tuple[str, str, str]:
+def fetch_details_from_tag(_tag: str) -> tuple[str, str, str]:
 	"""parse org, repo, tag from string
 
 	fetch_details_from_tag()
@@ -226,7 +244,7 @@ def parse_app_name(name: str) -> str:
 	return repo
 
 
-def install_app(name, verbose=False, set_as_patched=True):
+def install_app(name, verbose=False, set_as_patched=True, force=False):
 	from frappe.core.doctype.scheduled_job_type.scheduled_job_type import sync_jobs
 	from frappe.model.sync import sync_for
 	from frappe.modules.utils import sync_customizations
@@ -243,7 +261,7 @@ def install_app(name, verbose=False, set_as_patched=True):
 	if app_hooks.required_apps:
 		for app in app_hooks.required_apps:
 			required_app = parse_app_name(app)
-			install_app(required_app, verbose=verbose)
+			install_app(required_app, verbose=verbose, force=force)
 
 	frappe.flags.in_install = name
 	frappe.clear_cache()
@@ -251,11 +269,11 @@ def install_app(name, verbose=False, set_as_patched=True):
 	if name not in frappe.get_all_apps():
 		raise Exception("App not in apps.txt")
 
-	if name in installed_apps:
-		frappe.msgprint(frappe._("App {0} already installed").format(name))
+	if not force and name in installed_apps:
+		click.secho(f"App {name} already installed", fg="yellow")
 		return
 
-	print("\nInstalling {0}...".format(name))
+	print(f"\nInstalling {name}...")
 
 	if name != "frappe":
 		frappe.only_for("System Manager")
@@ -266,9 +284,9 @@ def install_app(name, verbose=False, set_as_patched=True):
 			return
 
 	if name != "frappe":
-		add_module_defs(name)
+		add_module_defs(name, ignore_if_duplicate=force)
 
-	sync_for(name, force=True, reset_permissions=True)
+	sync_for(name, force=force, reset_permissions=True)
 
 	add_to_installed_apps(name)
 
@@ -315,7 +333,6 @@ def remove_from_installed_apps(app_name):
 
 def remove_app(app_name, dry_run=False, yes=False, no_backup=False, force=False):
 	"""Remove app and all linked to the app's module with the app from a site."""
-	import click
 
 	site = frappe.local.site
 	app_hooks = frappe.get_hooks(app_name=app_name)
@@ -364,7 +381,7 @@ def remove_app(app_name, dry_run=False, yes=False, no_backup=False, force=False)
 	frappe.flags.in_uninstall = False
 
 
-def _delete_modules(modules: List[str], dry_run: bool) -> List[str]:
+def _delete_modules(modules: list[str], dry_run: bool) -> list[str]:
 	"""Delete modules belonging to the app and all related doctypes.
 
 	Note: All record linked linked to Module Def are also deleted.
@@ -397,7 +414,7 @@ def _delete_modules(modules: List[str], dry_run: bool) -> List[str]:
 
 
 def _delete_linked_documents(
-	module_name: str, doctype_linkfield_map: Dict[str, str], dry_run: bool
+	module_name: str, doctype_linkfield_map: dict[str, str], dry_run: bool
 ) -> None:
 
 	"""Deleted all records linked with module def"""
@@ -408,7 +425,7 @@ def _delete_linked_documents(
 				frappe.delete_doc(doctype, record, ignore_on_trash=True, force=True)
 
 
-def _get_module_linked_doctype_field_map() -> Dict[str, str]:
+def _get_module_linked_doctype_field_map() -> dict[str, str]:
 	"""Get all the doctypes which have module linked with them.
 
 	returns ordered dictionary with doctype->link field mapping."""
@@ -437,7 +454,7 @@ def _get_module_linked_doctype_field_map() -> Dict[str, str]:
 	return doctype_to_field_map
 
 
-def _delete_doctypes(doctypes: List[str], dry_run: bool) -> None:
+def _delete_doctypes(doctypes: list[str], dry_run: bool) -> None:
 	for doctype in set(doctypes):
 		print(f"* dropping Table for '{doctype}'...")
 		if not dry_run:
@@ -466,13 +483,20 @@ def set_all_patches_as_completed(app):
 
 
 def init_singles():
-	singles = [single["name"] for single in frappe.get_all("DocType", filters={"issingle": True})]
+	singles = frappe.get_all("DocType", filters={"issingle": True}, pluck="name")
 	for single in singles:
-		if not frappe.db.get_singles_dict(single):
+		if frappe.db.get_singles_dict(single):
+			continue
+
+		try:
 			doc = frappe.new_doc(single)
 			doc.flags.ignore_mandatory = True
 			doc.flags.ignore_validate = True
 			doc.save()
+		except ImportError:
+			# The doctype exists, but controller is deleted,
+			# no need to attempt to init such single, ref: #16917
+			continue
 
 
 def make_conf(
@@ -515,7 +539,7 @@ def update_site_config(key, value, validate=True, site_config_path=None):
 	if not site_config_path:
 		site_config_path = get_site_config_path()
 
-	with open(site_config_path, "r") as f:
+	with open(site_config_path) as f:
 		site_config = json.loads(f.read())
 
 	# In case of non-int value
@@ -573,13 +597,13 @@ def make_site_dirs():
 		os.makedirs(path, exist_ok=True)
 
 
-def add_module_defs(app):
+def add_module_defs(app, ignore_if_duplicate=False):
 	modules = frappe.get_module_list(app)
 	for module in modules:
 		d = frappe.new_doc("Module Def")
 		d.app_name = app
 		d.module_name = module
-		d.insert(ignore_permissions=True, ignore_if_duplicate=True)
+		d.insert(ignore_permissions=True, ignore_if_duplicate=ignore_if_duplicate)
 
 
 def remove_missing_apps():
@@ -650,7 +674,7 @@ def extract_sql_gzip(sql_gz_path):
 	try:
 		original_file = sql_gz_path
 		decompressed_file = original_file.rstrip(".gz")
-		cmd = "gzip --decompress --force < {0} > {1}".format(original_file, decompressed_file)
+		cmd = f"gzip --decompress --force < {original_file} > {decompressed_file}"
 		subprocess.check_call(cmd, shell=True)
 	except Exception:
 		raise
@@ -682,7 +706,7 @@ def extract_files(site_name, file_path):
 			subprocess.check_output(["tar", "xvf", tar_path, "--strip", "2"], cwd=abs_site_path)
 		elif file_path.endswith(".tgz"):
 			subprocess.check_output(["tar", "zxvf", tar_path, "--strip", "2"], cwd=abs_site_path)
-	except:
+	except Exception:
 		raise
 	finally:
 		frappe.destroy()
@@ -752,11 +776,9 @@ def partial_restore(sql_file_path, verbose=False):
 	elif frappe.conf.db_type == "postgres":
 		import warnings
 
-		from click import style
-
 		from frappe.database.postgres.setup_db import import_db_from_sql
 
-		warn = style(
+		warn = click.style(
 			"Delete the tables you want to restore manually before attempting"
 			" partial restore operation for PostreSQL databases",
 			fg="yellow",
@@ -788,7 +810,7 @@ def validate_database_sql(path, _raise=True):
 
 	# dont bother checking if empty file
 	if not empty_file:
-		with open(path, "r") as f:
+		with open(path) as f:
 			for line in f:
 				if "tabDefaultValue" in line:
 					missing_table = False
@@ -798,8 +820,6 @@ def validate_database_sql(path, _raise=True):
 			error_message = "Table `tabDefaultValue` not found in file."
 
 	if error_message:
-		import click
-
 		click.secho(error_message, fg="red")
 
 	if _raise and (missing_table or empty_file):
diff --git a/frappe/integrations/doctype/braintree_settings/__init__.py b/frappe/integrations/doctype/braintree_settings/__init__.py
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/frappe/integrations/doctype/braintree_settings/braintree_settings.js b/frappe/integrations/doctype/braintree_settings/braintree_settings.js
deleted file mode 100644
index c844022cec..0000000000
--- a/frappe/integrations/doctype/braintree_settings/braintree_settings.js
+++ /dev/null
@@ -1,6 +0,0 @@
-// Copyright (c) 2018, Frappe Technologies and contributors
-// For license information, please see license.txt
-
-frappe.ui.form.on('Braintree Settings', {
-
-});
diff --git a/frappe/integrations/doctype/braintree_settings/braintree_settings.json b/frappe/integrations/doctype/braintree_settings/braintree_settings.json
deleted file mode 100644
index eebf64dfd1..0000000000
--- a/frappe/integrations/doctype/braintree_settings/braintree_settings.json
+++ /dev/null
@@ -1,273 +0,0 @@
-{
- "allow_copy": 0, 
- "allow_guest_to_view": 0, 
- "allow_import": 0, 
- "allow_rename": 0, 
- "autoname": "field:gateway_name", 
- "beta": 0, 
- "creation": "2018-02-05 13:46:12.101852", 
- "custom": 0, 
- "docstatus": 0, 
- "doctype": "DocType", 
- "document_type": "", 
- "editable_grid": 1, 
- "engine": "InnoDB", 
- "fields": [
-  {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "gateway_name", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Payment Gateway Name", 
-   "length": 0, 
-   "no_copy": 0, 
-   "options": "Company", 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "section_break_2", 
-   "fieldtype": "Section Break", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "merchant_id", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Merchant ID", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "public_key", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Public Key", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "private_key", 
-   "fieldtype": "Password", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Private Key", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "use_sandbox", 
-   "fieldtype": "Check", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Use Sandbox", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "header_img", 
-   "fieldtype": "Attach Image", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Header Image", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }
- ], 
- "has_web_view": 0, 
- "hide_heading": 0, 
- "hide_toolbar": 0, 
- "idx": 0, 
- "image_view": 0, 
- "in_create": 0, 
- "is_submittable": 0, 
- "issingle": 0, 
- "istable": 0, 
- "max_attachments": 0, 
- "modified": "2018-02-05 14:33:06.050377", 
- "modified_by": "Administrator", 
- "module": "Integrations", 
- "name": "Braintree Settings", 
- "name_case": "", 
- "owner": "Administrator", 
- "permissions": [
-  {
-   "amend": 0, 
-   "apply_user_permissions": 0, 
-   "cancel": 0, 
-   "create": 1, 
-   "delete": 1, 
-   "email": 1, 
-   "export": 0, 
-   "if_owner": 0, 
-   "import": 0, 
-   "permlevel": 0, 
-   "print": 1, 
-   "read": 1, 
-   "report": 0, 
-   "role": "System Manager", 
-   "set_user_permissions": 0, 
-   "share": 1, 
-   "submit": 0, 
-   "write": 1
-  }
- ], 
- "quick_entry": 1, 
- "read_only": 0, 
- "read_only_onload": 0, 
- "show_name_in_global_search": 0, 
- "sort_field": "modified", 
- "sort_order": "DESC", 
- "track_changes": 0, 
- "track_seen": 0
-}
\ No newline at end of file
diff --git a/frappe/integrations/doctype/braintree_settings/braintree_settings.py b/frappe/integrations/doctype/braintree_settings/braintree_settings.py
deleted file mode 100644
index ca7ab0cfdf..0000000000
--- a/frappe/integrations/doctype/braintree_settings/braintree_settings.py
+++ /dev/null
@@ -1,288 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright (c) 2018, Frappe Technologies and contributors
-# License: MIT. See LICENSE
-
-from urllib.parse import urlencode
-
-import braintree
-
-import frappe
-from frappe import _
-from frappe.integrations.utils import create_payment_gateway, create_request_log
-from frappe.model.document import Document
-from frappe.utils import call_hook_method, get_url
-
-
-class BraintreeSettings(Document):
-	supported_currencies = [
-		"AED",
-		"AMD",
-		"AOA",
-		"ARS",
-		"AUD",
-		"AWG",
-		"AZN",
-		"BAM",
-		"BBD",
-		"BDT",
-		"BGN",
-		"BIF",
-		"BMD",
-		"BND",
-		"BOB",
-		"BRL",
-		"BSD",
-		"BWP",
-		"BYN",
-		"BZD",
-		"CAD",
-		"CHF",
-		"CLP",
-		"CNY",
-		"COP",
-		"CRC",
-		"CVE",
-		"CZK",
-		"DJF",
-		"DKK",
-		"DOP",
-		"DZD",
-		"EGP",
-		"ETB",
-		"EUR",
-		"FJD",
-		"FKP",
-		"GBP",
-		"GEL",
-		"GHS",
-		"GIP",
-		"GMD",
-		"GNF",
-		"GTQ",
-		"GYD",
-		"HKD",
-		"HNL",
-		"HRK",
-		"HTG",
-		"HUF",
-		"IDR",
-		"ILS",
-		"INR",
-		"ISK",
-		"JMD",
-		"JPY",
-		"KES",
-		"KGS",
-		"KHR",
-		"KMF",
-		"KRW",
-		"KYD",
-		"KZT",
-		"LAK",
-		"LBP",
-		"LKR",
-		"LRD",
-		"LSL",
-		"LTL",
-		"MAD",
-		"MDL",
-		"MKD",
-		"MNT",
-		"MOP",
-		"MUR",
-		"MVR",
-		"MWK",
-		"MXN",
-		"MYR",
-		"MZN",
-		"NAD",
-		"NGN",
-		"NIO",
-		"NOK",
-		"NPR",
-		"NZD",
-		"PAB",
-		"PEN",
-		"PGK",
-		"PHP",
-		"PKR",
-		"PLN",
-		"PYG",
-		"QAR",
-		"RON",
-		"RSD",
-		"RUB",
-		"RWF",
-		"SAR",
-		"SBD",
-		"SCR",
-		"SEK",
-		"SGD",
-		"SHP",
-		"SLL",
-		"SOS",
-		"SRD",
-		"STD",
-		"SVC",
-		"SYP",
-		"SZL",
-		"THB",
-		"TJS",
-		"TOP",
-		"TRY",
-		"TTD",
-		"TWD",
-		"TZS",
-		"UAH",
-		"UGX",
-		"USD",
-		"UYU",
-		"UZS",
-		"VEF",
-		"VND",
-		"VUV",
-		"WST",
-		"XAF",
-		"XCD",
-		"XOF",
-		"XPF",
-		"YER",
-		"ZAR",
-		"ZMK",
-		"ZWD",
-	]
-
-	def validate(self):
-		if not self.flags.ignore_mandatory:
-			self.configure_braintree()
-
-	def on_update(self):
-		create_payment_gateway(
-			"Braintree-" + self.gateway_name, settings="Braintree Settings", controller=self.gateway_name
-		)
-		call_hook_method("payment_gateway_enabled", gateway="Braintree-" + self.gateway_name)
-
-	def configure_braintree(self):
-		if self.use_sandbox:
-			environment = "sandbox"
-		else:
-			environment = "production"
-
-		braintree.Configuration.configure(
-			environment=environment,
-			merchant_id=self.merchant_id,
-			public_key=self.public_key,
-			private_key=self.get_password(fieldname="private_key", raise_exception=False),
-		)
-
-	def validate_transaction_currency(self, currency):
-		if currency not in self.supported_currencies:
-			frappe.throw(
-				_(
-					"Please select another payment method. Stripe does not support transactions in currency '{0}'"
-				).format(currency)
-			)
-
-	def get_payment_url(self, **kwargs):
-		return get_url("./integrations/braintree_checkout?{0}".format(urlencode(kwargs)))
-
-	def create_payment_request(self, data):
-		self.data = frappe._dict(data)
-
-		try:
-			self.integration_request = create_request_log(self.data, "Host", "Braintree")
-			return self.create_charge_on_braintree()
-
-		except Exception:
-			frappe.log_error(frappe.get_traceback())
-			return {
-				"redirect_to": frappe.redirect_to_message(
-					_("Server Error"),
-					_(
-						"There seems to be an issue with the server's braintree configuration. Don't worry, in case of failure, the amount will get refunded to your account."
-					),
-				),
-				"status": 401,
-			}
-
-	def create_charge_on_braintree(self):
-		self.configure_braintree()
-
-		redirect_to = self.data.get("redirect_to") or None
-		redirect_message = self.data.get("redirect_message") or None
-
-		result = braintree.Transaction.sale(
-			{
-				"amount": self.data.amount,
-				"payment_method_nonce": self.data.payload_nonce,
-				"options": {"submit_for_settlement": True},
-			}
-		)
-
-		if result.is_success:
-			self.integration_request.db_set("status", "Completed", update_modified=False)
-			self.flags.status_changed_to = "Completed"
-			self.integration_request.db_set("output", result.transaction.status, update_modified=False)
-
-		elif result.transaction:
-			self.integration_request.db_set("status", "Failed", update_modified=False)
-			error_log = frappe.log_error(
-				"code: "
-				+ str(result.transaction.processor_response_code)
-				+ " | text: "
-				+ str(result.transaction.processor_response_text),
-				"Braintree Payment Error",
-			)
-			self.integration_request.db_set("error", error_log.error, update_modified=False)
-		else:
-			self.integration_request.db_set("status", "Failed", update_modified=False)
-			for error in result.errors.deep_errors:
-				error_log = frappe.log_error(
-					"code: " + str(error.code) + " | message: " + str(error.message), "Braintree Payment Error"
-				)
-				self.integration_request.db_set("error", error_log.error, update_modified=False)
-
-		if self.flags.status_changed_to == "Completed":
-			status = "Completed"
-			if self.data.reference_doctype and self.data.reference_docname:
-				custom_redirect_to = None
-				try:
-					custom_redirect_to = frappe.get_doc(
-						self.data.reference_doctype, self.data.reference_docname
-					).run_method("on_payment_authorized", self.flags.status_changed_to)
-					braintree_success_page = frappe.get_hooks("braintree_success_page")
-					if braintree_success_page:
-						custom_redirect_to = frappe.get_attr(braintree_success_page[-1])(self.data)
-				except Exception:
-					frappe.log_error(frappe.get_traceback())
-
-				if custom_redirect_to:
-					redirect_to = custom_redirect_to
-
-			redirect_url = "payment-success"
-		else:
-			status = "Error"
-			redirect_url = "payment-failed"
-
-		if redirect_to:
-			redirect_url += "?" + urlencode({"redirect_to": redirect_to})
-		if redirect_message:
-			redirect_url += "&" + urlencode({"redirect_message": redirect_message})
-
-		return {"redirect_to": redirect_url, "status": status}
-
-
-def get_gateway_controller(doc):
-	payment_request = frappe.get_doc("Payment Request", doc)
-	gateway_controller = frappe.db.get_value(
-		"Payment Gateway", payment_request.payment_gateway, "gateway_controller"
-	)
-	return gateway_controller
-
-
-def get_client_token(doc):
-	gateway_controller = get_gateway_controller(doc)
-	settings = frappe.get_doc("Braintree Settings", gateway_controller)
-	settings.configure_braintree()
-
-	return braintree.ClientToken.generate()
diff --git a/frappe/integrations/doctype/braintree_settings/test_braintree_settings.py b/frappe/integrations/doctype/braintree_settings/test_braintree_settings.py
deleted file mode 100644
index 475a62be79..0000000000
--- a/frappe/integrations/doctype/braintree_settings/test_braintree_settings.py
+++ /dev/null
@@ -1,8 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright (c) 2018, Frappe Technologies and Contributors
-# License: MIT. See LICENSE
-import unittest
-
-
-class TestBraintreeSettings(unittest.TestCase):
-	pass
diff --git a/frappe/integrations/doctype/connected_app/connected_app.js b/frappe/integrations/doctype/connected_app/connected_app.js
index 4d20f65559..11dcda235e 100644
--- a/frappe/integrations/doctype/connected_app/connected_app.js
+++ b/frappe/integrations/doctype/connected_app/connected_app.js
@@ -1,38 +1,38 @@
 // Copyright (c) 2019, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Connected App', {
-	refresh: frm => {
-		frm.add_custom_button(__('Get OpenID Configuration'), async () => {
+frappe.ui.form.on("Connected App", {
+	refresh: (frm) => {
+		frm.add_custom_button(__("Get OpenID Configuration"), async () => {
 			if (!frm.doc.openid_configuration) {
-				frappe.msgprint(__('Please enter OpenID Configuration URL'));
+				frappe.msgprint(__("Please enter OpenID Configuration URL"));
 			} else {
 				try {
 					const response = await fetch(frm.doc.openid_configuration);
 					const oidc = await response.json();
-					frm.set_value('authorization_uri', oidc.authorization_endpoint);
-					frm.set_value('token_uri', oidc.token_endpoint);
-					frm.set_value('userinfo_uri', oidc.userinfo_endpoint);
-					frm.set_value('introspection_uri', oidc.introspection_endpoint);
-					frm.set_value('revocation_uri', oidc.revocation_endpoint);
+					frm.set_value("authorization_uri", oidc.authorization_endpoint);
+					frm.set_value("token_uri", oidc.token_endpoint);
+					frm.set_value("userinfo_uri", oidc.userinfo_endpoint);
+					frm.set_value("introspection_uri", oidc.introspection_endpoint);
+					frm.set_value("revocation_uri", oidc.revocation_endpoint);
 				} catch (error) {
-					frappe.msgprint(__('Please check OpenID Configuration URL'));
+					frappe.msgprint(__("Please check OpenID Configuration URL"));
 				}
 			}
 		});
 
 		if (!frm.is_new()) {
-			frm.add_custom_button(__('Connect to {}', [frm.doc.provider_name]), async () => {
+			frm.add_custom_button(__("Connect to {}", [frm.doc.provider_name]), async () => {
 				frappe.call({
-					method: 'initiate_web_application_flow',
+					method: "initiate_web_application_flow",
 					doc: frm.doc,
-					callback: function(r) {
-						window.open(r.message, '_blank');
-					}
+					callback: function (r) {
+						window.open(r.message, "_blank");
+					},
 				});
 			});
 		}
 
-		frm.toggle_display('sb_client_credentials_section', !frm.is_new());
-	}
+		frm.toggle_display("sb_client_credentials_section", !frm.is_new());
+	},
 });
diff --git a/frappe/integrations/doctype/connected_app/connected_app.py b/frappe/integrations/doctype/connected_app/connected_app.py
index e472193da8..308d1ca84a 100644
--- a/frappe/integrations/doctype/connected_app/connected_app.py
+++ b/frappe/integrations/doctype/connected_app/connected_app.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2019, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
diff --git a/frappe/integrations/doctype/connected_app/test_connected_app.py b/frappe/integrations/doctype/connected_app/test_connected_app.py
index 1597ec75bf..1acedff160 100644
--- a/frappe/integrations/doctype/connected_app/test_connected_app.py
+++ b/frappe/integrations/doctype/connected_app/test_connected_app.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2019, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 import unittest
diff --git a/frappe/integrations/doctype/dropbox_settings/dropbox_settings.js b/frappe/integrations/doctype/dropbox_settings/dropbox_settings.js
index ea731fafc2..9a5e9a4dc7 100644
--- a/frappe/integrations/doctype/dropbox_settings/dropbox_settings.js
+++ b/frappe/integrations/doctype/dropbox_settings/dropbox_settings.js
@@ -1,51 +1,54 @@
 // Copyright (c) 2016, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Dropbox Settings', {
-	refresh: function(frm) {
-		frm.toggle_display(["app_access_key", "app_secret_key"], !(frm.doc.__onload && frm.doc.__onload.dropbox_setup_via_site_config));
+frappe.ui.form.on("Dropbox Settings", {
+	refresh: function (frm) {
+		frm.toggle_display(
+			["app_access_key", "app_secret_key"],
+			!(frm.doc.__onload && frm.doc.__onload.dropbox_setup_via_site_config)
+		);
 		frm.clear_custom_buttons();
 		frm.events.take_backup(frm);
 	},
 
-	allow_dropbox_access: function(frm) {
+	allow_dropbox_access: function (frm) {
 		if (frm.doc.app_access_key && frm.doc.app_secret_key) {
 			frappe.call({
 				method: "frappe.integrations.doctype.dropbox_settings.dropbox_settings.get_dropbox_authorize_url",
 				freeze: true,
-				callback: function(r) {
-					if(!r.exc) {
+				callback: function (r) {
+					if (!r.exc) {
 						window.open(r.message.auth_url);
 					}
-				}
-			})
-		}
-		else if (frm.doc.__onload && frm.doc.__onload.dropbox_setup_via_site_config) {
+				},
+			});
+		} else if (frm.doc.__onload && frm.doc.__onload.dropbox_setup_via_site_config) {
 			frappe.call({
 				method: "frappe.integrations.doctype.dropbox_settings.dropbox_settings.get_redirect_url",
 				freeze: true,
-				callback: function(r) {
-					if(!r.exc) {
+				callback: function (r) {
+					if (!r.exc) {
 						window.open(r.message.auth_url);
 					}
-				}
-			})
-		}
-		else {
-			frappe.msgprint(__("Please enter values for App Access Key and App Secret Key"))
+				},
+			});
+		} else {
+			frappe.msgprint(__("Please enter values for App Access Key and App Secret Key"));
 		}
 	},
 
-	take_backup: function(frm) {
-		if (frm.doc.enabled && ((frm.doc.app_access_key && frm.doc.app_secret_key)
-			|| (frm.doc.__onload && frm.doc.__onload.dropbox_setup_via_site_config))) {
-			frm.add_custom_button(__("Take Backup Now"), function(frm){
+	take_backup: function (frm) {
+		if (
+			frm.doc.enabled &&
+			((frm.doc.app_access_key && frm.doc.app_secret_key) ||
+				(frm.doc.__onload && frm.doc.__onload.dropbox_setup_via_site_config))
+		) {
+			frm.add_custom_button(__("Take Backup Now"), function (frm) {
 				frappe.call({
 					method: "frappe.integrations.doctype.dropbox_settings.dropbox_settings.take_backup",
-					freeze: true
-				})
-			}).addClass("btn-primary")
+					freeze: true,
+				});
+			}).addClass("btn-primary");
 		}
-	}
+	},
 });
-
diff --git a/frappe/integrations/doctype/dropbox_settings/dropbox_settings.py b/frappe/integrations/doctype/dropbox_settings/dropbox_settings.py
index 8a6a7a4bfb..dc9db2ccda 100644
--- a/frappe/integrations/doctype/dropbox_settings/dropbox_settings.py
+++ b/frappe/integrations/doctype/dropbox_settings/dropbox_settings.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2015, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
@@ -63,21 +62,21 @@ def take_backups_weekly():
 
 
 def take_backups_if(freq):
-	if frappe.db.get_value("Dropbox Settings", None, "backup_frequency") == freq:
+	if frappe.db.get_single_value("Dropbox Settings", "backup_frequency") == freq:
 		take_backup_to_dropbox()
 
 
 def take_backup_to_dropbox(retry_count=0, upload_db_backup=True):
 	did_not_upload, error_log = [], []
 	try:
-		if cint(frappe.db.get_value("Dropbox Settings", None, "enabled")):
+		if cint(frappe.db.get_single_value("Dropbox Settings", "enabled")):
 			validate_file_size()
 
 			did_not_upload, error_log = backup_to_dropbox(upload_db_backup)
 			if did_not_upload:
 				raise Exception
 
-			if cint(frappe.db.get_value("Dropbox Settings", None, "send_email_for_successful_backup")):
+			if cint(frappe.db.get_single_value("Dropbox Settings", "send_email_for_successful_backup")):
 				send_email(True, "Dropbox", "Dropbox Settings", "send_notifications_to")
 	except JobTimeoutException:
 		if retry_count < 2:
@@ -89,7 +88,7 @@ def take_backup_to_dropbox(retry_count=0, upload_db_backup=True):
 				"frappe.integrations.doctype.dropbox_settings.dropbox_settings.take_backup_to_dropbox",
 				queue="long",
 				timeout=1500,
-				**args
+				**args,
 			)
 	except Exception:
 		if isinstance(error_log, str):
@@ -212,7 +211,7 @@ def upload_file_to_dropbox(filename, folder, dropbox_client):
 	mode = dropbox.files.WriteMode.overwrite
 
 	f = open(encode(filename), "rb")
-	path = "{0}/{1}".format(folder, os.path.basename(filename))
+	path = f"{folder}/{os.path.basename(filename)}"
 
 	try:
 		if file_size <= chunk_size:
@@ -234,7 +233,7 @@ def upload_file_to_dropbox(filename, folder, dropbox_client):
 					cursor.offset = f.tell()
 	except dropbox.exceptions.ApiError as e:
 		if isinstance(e.error, dropbox.files.UploadError):
-			error = "File Path: {path}\n".format(path=path)
+			error = f"File Path: {path}\n"
 			error += frappe.get_traceback()
 			frappe.log_error(error)
 		else:
@@ -326,7 +325,7 @@ def delete_older_backups(dropbox_client, folder_path, to_keep):
 def get_redirect_url():
 	if not frappe.conf.dropbox_broker_site:
 		frappe.conf.dropbox_broker_site = "https://dropbox.erpnext.com"
-	url = "{0}/api/method/dropbox_erpnext_broker.www.setup_dropbox.get_authotize_url".format(
+	url = "{}/api/method/dropbox_erpnext_broker.www.setup_dropbox.get_authotize_url".format(
 		frappe.conf.dropbox_broker_site
 	)
 
diff --git a/frappe/integrations/doctype/dropbox_settings/test_dropbox_settings.py b/frappe/integrations/doctype/dropbox_settings/test_dropbox_settings.py
index e73cf03268..b165e03780 100644
--- a/frappe/integrations/doctype/dropbox_settings/test_dropbox_settings.py
+++ b/frappe/integrations/doctype/dropbox_settings/test_dropbox_settings.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2019, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
 # import frappe
diff --git a/frappe/integrations/doctype/google_calendar/google_calendar.js b/frappe/integrations/doctype/google_calendar/google_calendar.js
index f30c52b2f2..977dee8dfe 100644
--- a/frappe/integrations/doctype/google_calendar/google_calendar.js
+++ b/frappe/integrations/doctype/google_calendar/google_calendar.js
@@ -2,15 +2,22 @@
 // For license information, please see license.txt
 
 frappe.ui.form.on("Google Calendar", {
-	refresh: function(frm) {
+	refresh: function (frm) {
 		if (frm.is_new()) {
-			frm.dashboard.set_headline(__("To use Google Calendar, enable {0}.", [`${__('Google Settings')}`]));
+			frm.dashboard.set_headline(
+				__("To use Google Calendar, enable {0}.", [
+					`${__("Google Settings")}`,
+				])
+			);
 		}
 
 		frappe.realtime.on("import_google_calendar", (data) => {
 			if (data.progress) {
-				frm.dashboard.show_progress("Syncing Google Calendar", data.progress / data.total * 100,
-					__("Syncing {0} of {1}", [data.progress, data.total]));
+				frm.dashboard.show_progress(
+					"Syncing Google Calendar",
+					(data.progress / data.total) * 100,
+					__("Syncing {0} of {1}", [data.progress, data.total])
+				);
 				if (data.progress === data.total) {
 					frm.dashboard.hide_progress("Syncing Google Calendar");
 				}
@@ -21,38 +28,40 @@ frappe.ui.form.on("Google Calendar", {
 			frm.add_custom_button(__("Sync Calendar"), function () {
 				frappe.show_alert({
 					indicator: "green",
-					message: __("Syncing")
-				});
-				frappe.call({
-					method: "frappe.integrations.doctype.google_calendar.google_calendar.sync",
-					args: {
-						"g_calendar": frm.doc.name
-					},
-				}).then((r) => {
-					frappe.hide_progress();
-					frappe.msgprint(r.message);
+					message: __("Syncing"),
 				});
+				frappe
+					.call({
+						method: "frappe.integrations.doctype.google_calendar.google_calendar.sync",
+						args: {
+							g_calendar: frm.doc.name,
+						},
+					})
+					.then((r) => {
+						frappe.hide_progress();
+						frappe.msgprint(r.message);
+					});
 			});
 		}
 	},
-	authorize_google_calendar_access: function(frm) {
+	authorize_google_calendar_access: function (frm) {
 		let reauthorize = 0;
-		if(frm.doc.authorization_code) {
+		if (frm.doc.authorization_code) {
 			reauthorize = 1;
 		}
 
 		frappe.call({
 			method: "frappe.integrations.doctype.google_calendar.google_calendar.authorize_access",
 			args: {
-				"g_calendar": frm.doc.name,
-				"reauthorize": reauthorize
+				g_calendar: frm.doc.name,
+				reauthorize: reauthorize,
 			},
-			callback: function(r) {
-				if(!r.exc) {
+			callback: function (r) {
+				if (!r.exc) {
 					frm.save();
 					window.open(r.message.url);
 				}
-			}
+			},
 		});
-	}
+	},
 });
diff --git a/frappe/integrations/doctype/google_calendar/google_calendar.py b/frappe/integrations/doctype/google_calendar/google_calendar.py
index 71f0e83f80..09ed012454 100644
--- a/frappe/integrations/doctype/google_calendar/google_calendar.py
+++ b/frappe/integrations/doctype/google_calendar/google_calendar.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2019, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
@@ -14,7 +13,7 @@ from googleapiclient.errors import HttpError
 
 import frappe
 from frappe import _
-from frappe.integrations.doctype.google_settings.google_settings import get_auth_url
+from frappe.integrations.google_oauth import GoogleOAuth
 from frappe.model.document import Document
 from frappe.utils import (
 	add_days,
@@ -91,7 +90,7 @@ class GoogleCalendar(Document):
 		}
 
 		try:
-			r = requests.post(get_auth_url(), data=data).json()
+			r = requests.post(GoogleOAuth.OAUTH_URL, data=data).json()
 		except requests.exceptions.HTTPError:
 			button_label = frappe.bold(_("Allow Google Calendar Access"))
 			frappe.throw(
@@ -131,7 +130,7 @@ def authorize_access(g_calendar, reauthorize=None):
 				"redirect_uri": redirect_uri,
 				"grant_type": "authorization_code",
 			}
-			r = requests.post(get_auth_url(), data=data).json()
+			r = requests.post(GoogleOAuth.OAUTH_URL, data=data).json()
 
 			if "refresh_token" in r:
 				frappe.db.set_value(
@@ -140,7 +139,7 @@ def authorize_access(g_calendar, reauthorize=None):
 				frappe.db.commit()
 
 			frappe.local.response["type"] = "redirect"
-			frappe.local.response["location"] = "/app/Form/{0}/{1}".format(
+			frappe.local.response["location"] = "/app/Form/{}/{}".format(
 				quote("Google Calendar"), quote(google_calendar.name)
 			)
 
@@ -192,7 +191,7 @@ def get_google_calendar_object(g_calendar):
 	credentials_dict = {
 		"token": account.get_access_token(),
 		"refresh_token": account.get_password(fieldname="refresh_token", raise_exception=False),
-		"token_uri": get_auth_url(),
+		"token_uri": GoogleOAuth.OAUTH_URL,
 		"client_id": google_settings.client_id,
 		"client_secret": google_settings.get_password(fieldname="client_secret", raise_exception=False),
 		"scopes": "https://www.googleapis.com/auth/calendar/v3",
diff --git a/frappe/integrations/doctype/google_contacts/google_contacts.js b/frappe/integrations/doctype/google_contacts/google_contacts.js
index 7cbef46699..06289b0ca5 100644
--- a/frappe/integrations/doctype/google_contacts/google_contacts.js
+++ b/frappe/integrations/doctype/google_contacts/google_contacts.js
@@ -1,59 +1,63 @@
 // Copyright (c) 2019, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Google Contacts', {
-	refresh: function(frm) {
+frappe.ui.form.on("Google Contacts", {
+	refresh: function (frm) {
 		if (!frm.doc.enable) {
-			frm.dashboard.set_headline(__("To use Google Contacts, enable {0}.", [`${__('Google Settings')}`]));
+			frm.dashboard.set_headline(
+				__("To use Google Contacts, enable {0}.", [
+					`${__("Google Settings")}`,
+				])
+			);
 		}
 
-		frappe.realtime.on('import_google_contacts', (data) => {
+		frappe.realtime.on("import_google_contacts", (data) => {
 			if (data.progress) {
-				frm.dashboard.show_progress('Import Google Contacts', data.progress / data.total * 100,
-					__('Importing {0} of {1}', [data.progress, data.total]));
+				frm.dashboard.show_progress(
+					"Import Google Contacts",
+					(data.progress / data.total) * 100,
+					__("Importing {0} of {1}", [data.progress, data.total])
+				);
 				if (data.progress === data.total) {
-					frm.dashboard.hide_progress('Import Google Contacts');
+					frm.dashboard.hide_progress("Import Google Contacts");
 				}
 			}
 		});
 
 		if (frm.doc.refresh_token) {
-			let sync_button = frm.add_custom_button(__('Sync Contacts'), function () {
+			let sync_button = frm.add_custom_button(__("Sync Contacts"), function () {
 				frappe.show_alert({
-					indicator: 'green',
-					message: __('Syncing')
-				});
-				frappe.call({
-					method: "frappe.integrations.doctype.google_contacts.google_contacts.sync",
-					args: {
-						"g_contact": frm.doc.name
-					},
-					btn: sync_button
-				}).then((r) => {
-					frappe.hide_progress();
-					frappe.msgprint(r.message);
+					indicator: "green",
+					message: __("Syncing"),
 				});
+				frappe
+					.call({
+						method: "frappe.integrations.doctype.google_contacts.google_contacts.sync",
+						args: {
+							g_contact: frm.doc.name,
+						},
+						btn: sync_button,
+					})
+					.then((r) => {
+						frappe.hide_progress();
+						frappe.msgprint(r.message);
+					});
 			});
 		}
 	},
-	authorize_google_contacts_access: function(frm) {
-		let reauthorize = 0;
-		if(frm.doc.authorization_code) {
-			reauthorize = 1;
-		}
-
+	authorize_google_contacts_access: function (frm) {
 		frappe.call({
 			method: "frappe.integrations.doctype.google_contacts.google_contacts.authorize_access",
 			args: {
-				"g_contact": frm.doc.name,
-				"reauthorize": reauthorize
+				g_contact: frm.doc.name,
+				reauthorize: frm.doc.authorization_code ? 1 : 0,
 			},
-			callback: function(r) {
-				if(!r.exc) {
+			callback: function (r) {
+				if (!r.exc) {
 					frm.save();
 					window.open(r.message.url);
 				}
-			}
+			},
 		});
-	}
+	},
 });
diff --git a/frappe/integrations/doctype/google_contacts/google_contacts.py b/frappe/integrations/doctype/google_contacts/google_contacts.py
index c26366f71a..9a20d5e905 100644
--- a/frappe/integrations/doctype/google_contacts/google_contacts.py
+++ b/frappe/integrations/doctype/google_contacts/google_contacts.py
@@ -1,20 +1,15 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2019, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
 
-import google.oauth2.credentials
-import requests
-from googleapiclient.discovery import build
+from urllib.parse import quote
+
 from googleapiclient.errors import HttpError
 
 import frappe
 from frappe import _
-from frappe.integrations.doctype.google_settings.google_settings import get_auth_url
+from frappe.integrations.google_oauth import GoogleOAuth
 from frappe.model.document import Document
-from frappe.utils import get_request_site_address
-
-SCOPES = "https://www.googleapis.com/auth/contacts"
 
 
 class GoogleContacts(Document):
@@ -23,120 +18,56 @@ class GoogleContacts(Document):
 			frappe.throw(_("Enable Google API in Google Settings."))
 
 	def get_access_token(self):
-		google_settings = frappe.get_doc("Google Settings")
-
-		if not google_settings.enable:
-			frappe.throw(_("Google Contacts Integration is disabled."))
-
 		if not self.refresh_token:
 			button_label = frappe.bold(_("Allow Google Contacts Access"))
 			raise frappe.ValidationError(_("Click on {0} to generate Refresh Token.").format(button_label))
 
-		data = {
-			"client_id": google_settings.client_id,
-			"client_secret": google_settings.get_password(fieldname="client_secret", raise_exception=False),
-			"refresh_token": self.get_password(fieldname="refresh_token", raise_exception=False),
-			"grant_type": "refresh_token",
-			"scope": SCOPES,
-		}
-
-		try:
-			r = requests.post(get_auth_url(), data=data).json()
-		except requests.exceptions.HTTPError:
-			button_label = frappe.bold(_("Allow Google Contacts Access"))
-			frappe.throw(
-				_(
-					"Something went wrong during the token generation. Click on {0} to generate a new one."
-				).format(button_label)
-			)
+		oauth_obj = GoogleOAuth("contacts")
+		r = oauth_obj.refresh_access_token(
+			self.get_password(fieldname="refresh_token", raise_exception=False)
+		)
 
 		return r.get("access_token")
 
 
-@frappe.whitelist()
-def authorize_access(g_contact, reauthorize=None):
+@frappe.whitelist(methods=["POST"])
+def authorize_access(g_contact, reauthorize=False, code=None):
 	"""
 	If no Authorization code get it from Google and then request for Refresh Token.
 	Google Contact Name is set to flags to set_value after Authorization Code is obtained.
 	"""
 
-	google_settings = frappe.get_doc("Google Settings")
-	google_contact = frappe.get_doc("Google Contacts", g_contact)
-
-	redirect_uri = (
-		get_request_site_address(True)
-		+ "?cmd=frappe.integrations.doctype.google_contacts.google_contacts.google_callback"
+	oauth_code = (
+		frappe.db.get_value("Google Contacts", g_contact, "authorization_code") if not code else code
 	)
+	oauth_obj = GoogleOAuth("contacts")
 
-	if not google_contact.authorization_code or reauthorize:
-		frappe.cache().hset("google_contacts", "google_contact", google_contact.name)
-		return get_authentication_url(client_id=google_settings.client_id, redirect_uri=redirect_uri)
-	else:
-		try:
-			data = {
-				"code": google_contact.authorization_code,
-				"client_id": google_settings.client_id,
-				"client_secret": google_settings.get_password(
-					fieldname="client_secret", raise_exception=False
-				),
-				"redirect_uri": redirect_uri,
-				"grant_type": "authorization_code",
-			}
-			r = requests.post(get_auth_url(), data=data).json()
-
-			if "refresh_token" in r:
-				frappe.db.set_value(
-					"Google Contacts", google_contact.name, "refresh_token", r.get("refresh_token")
-				)
-				frappe.db.commit()
-
-			frappe.local.response["type"] = "redirect"
-			frappe.local.response["location"] = "/app/Form/Google%20Contacts/{}".format(google_contact.name)
-
-			frappe.msgprint(_("Google Contacts has been configured."))
-		except Exception as e:
-			frappe.throw(e)
-
-
-def get_authentication_url(client_id=None, redirect_uri=None):
-	return {
-		"url": "https://accounts.google.com/o/oauth2/v2/auth?access_type=offline&response_type=code&prompt=consent&client_id={}&include_granted_scopes=true&scope={}&redirect_uri={}".format(
-			client_id, SCOPES, redirect_uri
+	if not oauth_code or reauthorize:
+		return oauth_obj.get_authentication_url(
+			{
+				"g_contact": g_contact,
+				"redirect": f"/app/Form/{quote('Google Contacts')}/{quote(g_contact)}",
+			},
 		)
-	}
 
-
-@frappe.whitelist()
-def google_callback(code=None):
-	"""
-	Authorization code is sent to callback as per the API configuration
-	"""
-	google_contact = frappe.cache().hget("google_contacts", "google_contact")
-	frappe.db.set_value("Google Contacts", google_contact, "authorization_code", code)
-	frappe.db.commit()
-
-	authorize_access(google_contact)
+	r = oauth_obj.authorize(oauth_code)
+	frappe.db.set_value(
+		"Google Contacts",
+		g_contact,
+		{"authorization_code": oauth_code, "refresh_token": r.get("refresh_token")},
+	)
 
 
 def get_google_contacts_object(g_contact):
 	"""
 	Returns an object of Google Calendar along with Google Calendar doc.
 	"""
-	google_settings = frappe.get_doc("Google Settings")
 	account = frappe.get_doc("Google Contacts", g_contact)
+	oauth_obj = GoogleOAuth("contacts")
 
-	credentials_dict = {
-		"token": account.get_access_token(),
-		"refresh_token": account.get_password(fieldname="refresh_token", raise_exception=False),
-		"token_uri": get_auth_url(),
-		"client_id": google_settings.client_id,
-		"client_secret": google_settings.get_password(fieldname="client_secret", raise_exception=False),
-		"scopes": "https://www.googleapis.com/auth/contacts",
-	}
-
-	credentials = google.oauth2.credentials.Credentials(**credentials_dict)
-	google_contacts = build(
-		serviceName="people", version="v1", credentials=credentials, static_discovery=False
+	google_contacts = oauth_obj.get_google_service_object(
+		account.get_access_token(),
+		account.get_password(fieldname="indexing_refresh_token", raise_exception=False),
 	)
 
 	return google_contacts, account
diff --git a/frappe/integrations/doctype/google_drive/google_drive.js b/frappe/integrations/doctype/google_drive/google_drive.js
index c314d02e7e..208c1e5e1a 100644
--- a/frappe/integrations/doctype/google_drive/google_drive.js
+++ b/frappe/integrations/doctype/google_drive/google_drive.js
@@ -1,16 +1,23 @@
 // Copyright (c) 2019, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Google Drive', {
-	refresh: function(frm) {
+frappe.ui.form.on("Google Drive", {
+	refresh: function (frm) {
 		if (!frm.doc.enable) {
-			frm.dashboard.set_headline(__("To use Google Drive, enable {0}.", [`${__('Google Settings')}`]));
+			frm.dashboard.set_headline(
+				__("To use Google Drive, enable {0}.", [
+					`${__("Google Settings")}`,
+				])
+			);
 		}
 
 		frappe.realtime.on("upload_to_google_drive", (data) => {
 			if (data.progress) {
-				frm.dashboard.show_progress("Uploading to Google Drive", data.progress / data.total * 100,
-					__("{0}", [data.message]));
+				frm.dashboard.show_progress(
+					"Uploading to Google Drive",
+					(data.progress / data.total) * 100,
+					__("{0}", [data.message])
+				);
 				if (data.progress === data.total) {
 					frm.dashboard.hide_progress("Uploading to Google Drive");
 				}
@@ -21,42 +28,43 @@ frappe.ui.form.on('Google Drive', {
 			let sync_button = frm.add_custom_button(__("Take Backup"), function () {
 				frappe.show_alert({
 					indicator: "green",
-					message: __("Backing up to Google Drive.")
-				});
-				frappe.call({
-					method: "frappe.integrations.doctype.google_drive.google_drive.take_backup",
-					btn: sync_button
-				}).then((r) => {
-					frappe.msgprint(r.message);
+					message: __("Backing up to Google Drive."),
 				});
+				frappe
+					.call({
+						method: "frappe.integrations.doctype.google_drive.google_drive.take_backup",
+						btn: sync_button,
+					})
+					.then((r) => {
+						frappe.msgprint(r.message);
+					});
 			});
 		}
 
 		if (frm.doc.enable && frm.doc.backup_folder_name && !frm.doc.refresh_token) {
-			frm.dashboard.set_headline(__("Click on Authorize Google Drive Access to authorize Google Drive Access."));
+			frm.dashboard.set_headline(
+				__(
+					"Click on Authorize Google Drive Access to authorize Google Drive Access."
+				)
+			);
 		}
 
 		if (frm.doc.enable && frm.doc.refresh_token && frm.doc.authorization_code) {
 			frm.page.set_indicator("Authorized", "green");
 		}
 	},
-	authorize_google_drive_access: function(frm) {
-		let reauthorize = 0;
-		if (frm.doc.authorization_code) {
-			reauthorize = 1;
-		}
-
+	authorize_google_drive_access: function (frm) {
 		frappe.call({
 			method: "frappe.integrations.doctype.google_drive.google_drive.authorize_access",
 			args: {
-				"reauthorize": reauthorize
+				reauthorize: frm.doc.authorization_code ? 1 : 0,
 			},
-			callback: function(r) {
+			callback: function (r) {
 				if (!r.exc) {
 					frm.save();
 					window.open(r.message.url);
 				}
-			}
+			},
 		});
-	}
+	},
 });
diff --git a/frappe/integrations/doctype/google_drive/google_drive.py b/frappe/integrations/doctype/google_drive/google_drive.py
index bbb1e8485e..6ea1294cb0 100644
--- a/frappe/integrations/doctype/google_drive/google_drive.py
+++ b/frappe/integrations/doctype/google_drive/google_drive.py
@@ -1,31 +1,25 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2019, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
 import os
 from urllib.parse import quote
 
-import google.oauth2.credentials
-import requests
 from apiclient.http import MediaFileUpload
-from googleapiclient.discovery import build
 from googleapiclient.errors import HttpError
 
 import frappe
 from frappe import _
-from frappe.integrations.doctype.google_settings.google_settings import get_auth_url
+from frappe.integrations.google_oauth import GoogleOAuth
 from frappe.integrations.offsite_backup_utils import (
 	get_latest_backup_file,
 	send_email,
 	validate_file_size,
 )
 from frappe.model.document import Document
-from frappe.utils import get_backups_path, get_bench_path, get_request_site_address
+from frappe.utils import get_backups_path, get_bench_path
 from frappe.utils.background_jobs import enqueue
 from frappe.utils.backups import new_backup
 
-SCOPES = "https://www.googleapis.com/auth/drive"
-
 
 class GoogleDrive(Document):
 	def validate(self):
@@ -34,118 +28,57 @@ class GoogleDrive(Document):
 			self.backup_folder_id = ""
 
 	def get_access_token(self):
-		google_settings = frappe.get_doc("Google Settings")
-
-		if not google_settings.enable:
-			frappe.throw(_("Google Integration is disabled."))
-
 		if not self.refresh_token:
 			button_label = frappe.bold(_("Allow Google Drive Access"))
 			raise frappe.ValidationError(_("Click on {0} to generate Refresh Token.").format(button_label))
 
-		data = {
-			"client_id": google_settings.client_id,
-			"client_secret": google_settings.get_password(fieldname="client_secret", raise_exception=False),
-			"refresh_token": self.get_password(fieldname="refresh_token", raise_exception=False),
-			"grant_type": "refresh_token",
-			"scope": SCOPES,
-		}
-
-		try:
-			r = requests.post(get_auth_url(), data=data).json()
-		except requests.exceptions.HTTPError:
-			button_label = frappe.bold(_("Allow Google Drive Access"))
-			frappe.throw(
-				_(
-					"Something went wrong during the token generation. Click on {0} to generate a new one."
-				).format(button_label)
-			)
+		oauth_obj = GoogleOAuth("drive")
+		r = oauth_obj.refresh_access_token(
+			self.get_password(fieldname="refresh_token", raise_exception=False)
+		)
 
 		return r.get("access_token")
 
 
-@frappe.whitelist()
-def authorize_access(reauthorize=None):
+@frappe.whitelist(methods=["POST"])
+def authorize_access(reauthorize=False, code=None):
 	"""
 	If no Authorization code get it from Google and then request for Refresh Token.
 	Google Contact Name is set to flags to set_value after Authorization Code is obtained.
 	"""
 
-	google_settings = frappe.get_doc("Google Settings")
-	google_drive = frappe.get_doc("Google Drive")
-
-	redirect_uri = (
-		get_request_site_address(True)
-		+ "?cmd=frappe.integrations.doctype.google_drive.google_drive.google_callback"
+	oauth_code = (
+		frappe.db.get_single_value("Google Drive", "authorization_code") if not code else code
 	)
+	oauth_obj = GoogleOAuth("drive")
 
-	if not google_drive.authorization_code or reauthorize:
+	if not oauth_code or reauthorize:
 		if reauthorize:
 			frappe.db.set_value("Google Drive", None, "backup_folder_id", "")
-		return get_authentication_url(client_id=google_settings.client_id, redirect_uri=redirect_uri)
-	else:
-		try:
-			data = {
-				"code": google_drive.authorization_code,
-				"client_id": google_settings.client_id,
-				"client_secret": google_settings.get_password(
-					fieldname="client_secret", raise_exception=False
-				),
-				"redirect_uri": redirect_uri,
-				"grant_type": "authorization_code",
-			}
-			r = requests.post(get_auth_url(), data=data).json()
-
-			if "refresh_token" in r:
-				frappe.db.set_value("Google Drive", google_drive.name, "refresh_token", r.get("refresh_token"))
-				frappe.db.commit()
-
-			frappe.local.response["type"] = "redirect"
-			frappe.local.response["location"] = "/app/Form/{0}".format(quote("Google Drive"))
-
-			frappe.msgprint(_("Google Drive has been configured."))
-		except Exception as e:
-			frappe.throw(e)
-
-
-def get_authentication_url(client_id, redirect_uri):
-	return {
-		"url": "https://accounts.google.com/o/oauth2/v2/auth?access_type=offline&response_type=code&prompt=consent&client_id={}&include_granted_scopes=true&scope={}&redirect_uri={}".format(
-			client_id, SCOPES, redirect_uri
+		return oauth_obj.get_authentication_url(
+			{
+				"redirect": f"/app/Form/{quote('Google Drive')}",
+			},
 		)
-	}
 
-
-@frappe.whitelist()
-def google_callback(code=None):
-	"""
-	Authorization code is sent to callback as per the API configuration
-	"""
-	frappe.db.set_value("Google Drive", None, "authorization_code", code)
-	frappe.db.commit()
-
-	authorize_access()
+	r = oauth_obj.authorize(oauth_code)
+	frappe.db.set_value(
+		"Google Drive",
+		"Google Drive",
+		{"authorization_code": oauth_code, "refresh_token": r.get("refresh_token")},
+	)
 
 
 def get_google_drive_object():
 	"""
 	Returns an object of Google Drive.
 	"""
-	google_settings = frappe.get_doc("Google Settings")
 	account = frappe.get_doc("Google Drive")
+	oauth_obj = GoogleOAuth("drive")
 
-	credentials_dict = {
-		"token": account.get_access_token(),
-		"refresh_token": account.get_password(fieldname="refresh_token", raise_exception=False),
-		"token_uri": get_auth_url(),
-		"client_id": google_settings.client_id,
-		"client_secret": google_settings.get_password(fieldname="client_secret", raise_exception=False),
-		"scopes": "https://www.googleapis.com/auth/drive/v3",
-	}
-
-	credentials = google.oauth2.credentials.Credentials(**credentials_dict)
-	google_drive = build(
-		serviceName="drive", version="v3", credentials=credentials, static_discovery=False
+	google_drive = oauth_obj.get_google_service_object(
+		account.get_access_token(),
+		account.get_password(fieldname="indexing_refresh_token", raise_exception=False),
 	)
 
 	return google_drive, account
@@ -243,7 +176,7 @@ def upload_system_backup_to_google_drive():
 			media = MediaFileUpload(
 				get_absolute_path(filename=fileurl), mimetype="application/gzip", resumable=True
 			)
-		except IOError as e:
+		except OSError as e:
 			frappe.throw(_("Google Drive - Could not locate - {0}").format(e))
 
 		try:
@@ -259,20 +192,20 @@ def upload_system_backup_to_google_drive():
 
 
 def daily_backup():
-	drive_settings = frappe.db.get_singles_dict("Google Drive")
+	drive_settings = frappe.db.get_singles_dict("Google Drive", cast=True)
 	if drive_settings.enable and drive_settings.frequency == "Daily":
 		upload_system_backup_to_google_drive()
 
 
 def weekly_backup():
-	drive_settings = frappe.db.get_singles_dict("Google Drive")
+	drive_settings = frappe.db.get_singles_dict("Google Drive", cast=True)
 	if drive_settings.enable and drive_settings.frequency == "Weekly":
 		upload_system_backup_to_google_drive()
 
 
 def get_absolute_path(filename):
 	file_path = os.path.join(get_backups_path()[2:], os.path.basename(filename))
-	return "{0}/sites/{1}".format(get_bench_path(), file_path)
+	return f"{get_bench_path()}/sites/{file_path}"
 
 
 def set_progress(progress, message):
diff --git a/frappe/integrations/doctype/google_drive/test_google_drive.py b/frappe/integrations/doctype/google_drive/test_google_drive.py
index 17f5b152ca..4dcc79afd6 100644
--- a/frappe/integrations/doctype/google_drive/test_google_drive.py
+++ b/frappe/integrations/doctype/google_drive/test_google_drive.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2019, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
 # import frappe
diff --git a/frappe/integrations/doctype/google_settings/google_settings.js b/frappe/integrations/doctype/google_settings/google_settings.js
index 01a127db7f..58093034b5 100644
--- a/frappe/integrations/doctype/google_settings/google_settings.js
+++ b/frappe/integrations/doctype/google_settings/google_settings.js
@@ -1,8 +1,14 @@
 // Copyright (c) 2019, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Google Settings', {
-	refresh: function(frm) {
-		frm.dashboard.set_headline(__("For more information, {0}.", [`${__('Click here')}`]));
-	}
+frappe.ui.form.on("Google Settings", {
+	refresh: function (frm) {
+		frm.dashboard.set_headline(
+			__("For more information, {0}.", [
+				`${__(
+					"Click here"
+				)}`,
+			])
+		);
+	},
 });
diff --git a/frappe/integrations/doctype/google_settings/google_settings.py b/frappe/integrations/doctype/google_settings/google_settings.py
index 0d5f9cb00d..e464e0d090 100644
--- a/frappe/integrations/doctype/google_settings/google_settings.py
+++ b/frappe/integrations/doctype/google_settings/google_settings.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2019, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
@@ -10,10 +9,6 @@ class GoogleSettings(Document):
 	pass
 
 
-def get_auth_url():
-	return "https://www.googleapis.com/oauth2/v4/token"
-
-
 @frappe.whitelist()
 def get_file_picker_settings():
 	"""Return all the data FileUploader needs to start the Google Drive Picker."""
diff --git a/frappe/integrations/doctype/google_settings/test_google_settings.py b/frappe/integrations/doctype/google_settings/test_google_settings.py
index 53d59b1be0..8d07ffa54f 100644
--- a/frappe/integrations/doctype/google_settings/test_google_settings.py
+++ b/frappe/integrations/doctype/google_settings/test_google_settings.py
@@ -1,7 +1,5 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2021, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
-from __future__ import unicode_literals
 
 import unittest
 
diff --git a/frappe/integrations/doctype/integration_request/integration_request.js b/frappe/integrations/doctype/integration_request/integration_request.js
index 4b3b9a2de7..ac810f4d73 100644
--- a/frappe/integrations/doctype/integration_request/integration_request.js
+++ b/frappe/integrations/doctype/integration_request/integration_request.js
@@ -1,8 +1,6 @@
 // Copyright (c) 2016, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Integration Request', {
-	refresh: function(frm) {
-
-	}
+frappe.ui.form.on("Integration Request", {
+	refresh: function (frm) {},
 });
diff --git a/frappe/integrations/doctype/integration_request/integration_request.json b/frappe/integrations/doctype/integration_request/integration_request.json
index 8a3fbc41ba..98db8ea748 100644
--- a/frappe/integrations/doctype/integration_request/integration_request.json
+++ b/frappe/integrations/doctype/integration_request/integration_request.json
@@ -1,334 +1,154 @@
 {
- "allow_copy": 0,
- "allow_events_in_timeline": 0,
- "allow_guest_to_view": 0,
- "allow_import": 0,
- "allow_rename": 0,
- "beta": 0,
- "creation": "2016-08-04 04:58:40.457416",
- "custom": 0,
- "docstatus": 0,
+ "actions": [],
+ "creation": "2022-03-28 12:25:29.929952",
  "doctype": "DocType",
- "document_type": "",
  "editable_grid": 1,
  "engine": "InnoDB",
+ "field_order": [
+  "request_id",
+  "integration_request_service",
+  "is_remote_request",
+  "column_break_5",
+  "request_description",
+  "status",
+  "section_break_8",
+  "url",
+  "request_headers",
+  "data",
+  "response_section",
+  "output",
+  "error",
+  "reference_section",
+  "reference_doctype",
+  "column_break_16",
+  "reference_docname"
+ ],
  "fields": [
   {
-   "allow_bulk_edit": 0,
-   "allow_in_quick_entry": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
-   "fetch_if_empty": 0,
-   "fieldname": "integration_type",
-   "fieldtype": "Select",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
-   "label": "Integration Type",
-   "length": 0,
-   "no_copy": 0,
-   "options": "\nHost\nRemote\nSubscription Notification",
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "translatable": 0,
-   "unique": 0
-  },
-  {
-   "allow_bulk_edit": 0,
-   "allow_in_quick_entry": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
-   "fetch_if_empty": 0,
    "fieldname": "integration_request_service",
    "fieldtype": "Data",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
-   "label": "Integration Request Service",
-   "length": 0,
-   "no_copy": 0,
-   "options": "",
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 1,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "translatable": 0,
-   "unique": 0
+   "label": "Service",
+   "read_only": 1
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_in_quick_entry": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
    "default": "Queued",
-   "fetch_if_empty": 0,
    "fieldname": "status",
    "fieldtype": "Select",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
    "in_list_view": 1,
    "in_standard_filter": 1,
    "label": "Status",
-   "length": 0,
-   "no_copy": 0,
    "options": "\nQueued\nAuthorized\nCompleted\nCancelled\nFailed",
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "translatable": 0,
-   "unique": 0
+   "read_only": 1
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_in_quick_entry": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
-   "fetch_if_empty": 0,
    "fieldname": "data",
    "fieldtype": "Code",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
-   "label": "Data",
-   "length": 0,
-   "no_copy": 0,
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 1,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "translatable": 0,
-   "unique": 0
+   "label": "Request Data",
+   "read_only": 1
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_in_quick_entry": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
-   "fetch_if_empty": 0,
    "fieldname": "output",
    "fieldtype": "Code",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
    "label": "Output",
-   "length": 0,
-   "no_copy": 0,
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 1,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "translatable": 0,
-   "unique": 0
+   "read_only": 1
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_in_quick_entry": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
-   "fetch_if_empty": 0,
    "fieldname": "error",
    "fieldtype": "Code",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
    "label": "Error",
-   "length": 0,
-   "no_copy": 0,
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 1,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "translatable": 0,
-   "unique": 0
+   "read_only": 1
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_in_quick_entry": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
-   "fetch_if_empty": 0,
    "fieldname": "reference_doctype",
    "fieldtype": "Link",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
    "label": "Reference Document Type",
-   "length": 0,
-   "no_copy": 0,
    "options": "DocType",
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 1,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "translatable": 0,
-   "unique": 0
+   "read_only": 1
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_in_quick_entry": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
-   "fetch_if_empty": 0,
    "fieldname": "reference_docname",
    "fieldtype": "Dynamic Link",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
-   "label": "Reference Docname",
-   "length": 0,
-   "no_copy": 0,
+   "label": "Reference Document Name",
    "options": "reference_doctype",
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 1,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "translatable": 0,
-   "unique": 0
+   "read_only": 1
+  },
+  {
+   "default": "0",
+   "fieldname": "is_remote_request",
+   "fieldtype": "Check",
+   "label": "Is Remote Request?",
+   "read_only": 1
+  },
+  {
+   "fieldname": "request_description",
+   "fieldtype": "Data",
+   "label": "Request Description",
+   "read_only": 1
+  },
+  {
+   "fieldname": "request_id",
+   "fieldtype": "Data",
+   "label": "Request ID",
+   "read_only": 1
+  },
+  {
+   "fieldname": "column_break_5",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "section_break_8",
+   "fieldtype": "Section Break"
+  },
+  {
+   "fieldname": "url",
+   "fieldtype": "Data",
+   "label": "URL",
+   "read_only": 1
+  },
+  {
+   "fieldname": "response_section",
+   "fieldtype": "Section Break",
+   "label": "Response"
+  },
+  {
+   "depends_on": "eval:doc.reference_doctype",
+   "fieldname": "reference_section",
+   "fieldtype": "Section Break",
+   "label": "Reference"
+  },
+  {
+   "fieldname": "column_break_16",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "request_headers",
+   "fieldtype": "Code",
+   "label": "Request Headers",
+   "read_only": 1
   }
  ],
- "has_web_view": 0,
- "hide_heading": 0,
- "hide_toolbar": 0,
- "idx": 0,
- "image_view": 0,
  "in_create": 1,
- "is_submittable": 0,
- "issingle": 0,
- "istable": 0,
- "max_attachments": 0,
- "modified": "2019-09-05 14:22:27.664645",
+ "links": [],
+ "modified": "2022-04-07 11:32:27.557548",
  "modified_by": "Administrator",
  "module": "Integrations",
  "name": "Integration Request",
- "name_case": "",
  "owner": "Administrator",
  "permissions": [
   {
-   "amend": 0,
-   "cancel": 0,
-   "create": 1,
    "delete": 1,
    "email": 1,
    "export": 1,
-   "if_owner": 0,
-   "import": 0,
-   "permlevel": 0,
    "print": 1,
    "read": 1,
    "report": 1,
    "role": "System Manager",
-   "set_user_permissions": 0,
-   "share": 1,
-   "submit": 0,
-   "write": 1
+   "share": 1
   }
  ],
- "quick_entry": 0,
- "read_only": 0,
- "read_only_onload": 0,
- "show_name_in_global_search": 0,
  "sort_field": "modified",
  "sort_order": "DESC",
+ "states": [],
  "title_field": "integration_request_service",
- "track_changes": 1,
- "track_seen": 0,
- "track_views": 0
+ "track_changes": 1
 }
\ No newline at end of file
diff --git a/frappe/integrations/doctype/integration_request/integration_request.py b/frappe/integrations/doctype/integration_request/integration_request.py
index 4c99613161..334736bc9b 100644
--- a/frappe/integrations/doctype/integration_request/integration_request.py
+++ b/frappe/integrations/doctype/integration_request/integration_request.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2015, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
diff --git a/frappe/integrations/doctype/integration_request/test_integration_request.py b/frappe/integrations/doctype/integration_request/test_integration_request.py
index d14af481e8..45963d5096 100644
--- a/frappe/integrations/doctype/integration_request/test_integration_request.py
+++ b/frappe/integrations/doctype/integration_request/test_integration_request.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2015, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
 import unittest
diff --git a/frappe/integrations/doctype/ldap_group_mapping/ldap_group_mapping.json b/frappe/integrations/doctype/ldap_group_mapping/ldap_group_mapping.json
index 92db68e962..9bfe1eac56 100644
--- a/frappe/integrations/doctype/ldap_group_mapping/ldap_group_mapping.json
+++ b/frappe/integrations/doctype/ldap_group_mapping/ldap_group_mapping.json
@@ -1,4 +1,5 @@
 {
+ "actions": [],
  "creation": "2019-05-29 01:24:29.585060",
  "doctype": "DocType",
  "editable_grid": 1,
@@ -19,13 +20,14 @@
    "fieldname": "erpnext_role",
    "fieldtype": "Link",
    "in_list_view": 1,
-   "label": "ERPNext Role",
+   "label": "User Role",
    "options": "Role",
    "reqd": 1
   }
  ],
  "istable": 1,
- "modified": "2019-07-15 06:46:38.050408",
+ "links": [],
+ "modified": "2022-07-07 16:28:44.828514",
  "modified_by": "Administrator",
  "module": "Integrations",
  "name": "LDAP Group Mapping",
@@ -34,5 +36,6 @@
  "quick_entry": 1,
  "sort_field": "modified",
  "sort_order": "ASC",
+ "states": [],
  "track_changes": 1
 }
\ No newline at end of file
diff --git a/frappe/integrations/doctype/ldap_group_mapping/ldap_group_mapping.py b/frappe/integrations/doctype/ldap_group_mapping/ldap_group_mapping.py
index f1b242e4bb..853cfc96a1 100644
--- a/frappe/integrations/doctype/ldap_group_mapping/ldap_group_mapping.py
+++ b/frappe/integrations/doctype/ldap_group_mapping/ldap_group_mapping.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2019, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
diff --git a/frappe/integrations/doctype/ldap_settings/ldap_settings.js b/frappe/integrations/doctype/ldap_settings/ldap_settings.js
index 9ac95883b7..2ca7370ecf 100644
--- a/frappe/integrations/doctype/ldap_settings/ldap_settings.js
+++ b/frappe/integrations/doctype/ldap_settings/ldap_settings.js
@@ -1,8 +1,6 @@
 // Copyright (c) 2016, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('LDAP Settings', {
-	refresh: function(frm) {
-
-	}
+frappe.ui.form.on("LDAP Settings", {
+	refresh: function (frm) {},
 });
diff --git a/frappe/integrations/doctype/ldap_settings/ldap_settings.json b/frappe/integrations/doctype/ldap_settings/ldap_settings.json
index fd45a71538..f5472a5097 100644
--- a/frappe/integrations/doctype/ldap_settings/ldap_settings.json
+++ b/frappe/integrations/doctype/ldap_settings/ldap_settings.json
@@ -42,7 +42,10 @@
   "column_break_33",
   "ldap_group_member_attribute",
   "ldap_group_mappings_section",
+  "default_user_type",
+  "column_break_38",
   "default_role",
+  "section_break_40",
   "ldap_groups",
   "ldap_group_field"
  ],
@@ -79,9 +82,11 @@
    "reqd": 1
   },
   {
+   "depends_on": "eval: doc.default_user_type == \"System User\"",
    "fieldname": "default_role",
    "fieldtype": "Link",
-   "label": "Default Role on Creation",
+   "label": "Default User Role",
+   "mandatory_depends_on": "eval: doc.default_user_type == \"System User\"",
    "options": "Role",
    "reqd": 1
   },
@@ -249,10 +254,10 @@
    "label": "Group Object Class"
   },
   {
-    "description": "string value, i.e. {0} or uid={0},ou=users,dc=example,dc=com",
-    "fieldname": "ldap_custom_group_search",
-    "fieldtype": "Data",
-    "label": "Custom Group Search"
+   "description": "string value, i.e. {0} or uid={0},ou=users,dc=example,dc=com",
+   "fieldname": "ldap_custom_group_search",
+   "fieldtype": "Data",
+   "label": "Custom Group Search"
   },
   {
    "description": "Requires any valid fdn path. i.e. ou=users,dc=example,dc=com",
@@ -268,12 +273,28 @@
    "fieldtype": "Data",
    "label": "LDAP search path for Groups",
    "reqd": 1
+  },
+  {
+   "fieldname": "default_user_type",
+   "fieldtype": "Link",
+   "label": "Default User Type",
+   "options": "User Type",
+   "reqd": 1
+  },
+  {
+   "fieldname": "column_break_38",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "section_break_40",
+   "fieldtype": "Section Break",
+   "hide_border": 1
   }
  ],
  "in_create": 1,
  "issingle": 1,
  "links": [],
- "modified": "2021-07-27 11:51:43.328271",
+ "modified": "2022-07-07 16:51:46.230793",
  "modified_by": "Administrator",
  "module": "Integrations",
  "name": "LDAP Settings",
@@ -294,5 +315,6 @@
  "read_only": 1,
  "sort_field": "modified",
  "sort_order": "DESC",
+ "states": [],
  "track_changes": 1
 }
\ No newline at end of file
diff --git a/frappe/integrations/doctype/ldap_settings/ldap_settings.py b/frappe/integrations/doctype/ldap_settings/ldap_settings.py
index a14124234f..735b96968c 100644
--- a/frappe/integrations/doctype/ldap_settings/ldap_settings.py
+++ b/frappe/integrations/doctype/ldap_settings/ldap_settings.py
@@ -1,20 +1,37 @@
-# -*- coding: utf-8 -*-
-# Copyright (c) 2015, Frappe Technologies and contributors
+# Copyright (c) 2022, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
+import ssl
+from typing import TYPE_CHECKING
+
+import ldap3
+from ldap3 import AUTO_BIND_TLS_BEFORE_BIND, HASHED_SALTED_SHA, MODIFY_REPLACE
+from ldap3.abstract.entry import Entry
+from ldap3.core.exceptions import (
+	LDAPAttributeError,
+	LDAPInvalidCredentialsResult,
+	LDAPInvalidFilterError,
+	LDAPNoSuchObjectResult,
+)
+from ldap3.utils.hashed import hashed
+
 import frappe
 from frappe import _, safe_encode
 from frappe.model.document import Document
 from frappe.twofactor import authenticate_for_2factor, confirm_otp_token, should_run_2fa
 
+if TYPE_CHECKING:
+	from frappe.core.doctype.user.user import User
+
 
 class LDAPSettings(Document):
 	def validate(self):
+		self.default_user_type = self.default_user_type or "System User"
+
 		if not self.enabled:
 			return
 
 		if not self.flags.ignore_mandatory:
-
 			if (
 				self.ldap_search_string.count("(") == self.ldap_search_string.count(")")
 				and self.ldap_search_string.startswith("(")
@@ -29,8 +46,6 @@ class LDAPSettings(Document):
 
 				try:
 					if conn.result["type"] == "bindResponse" and self.base_dn:
-						import ldap3
-
 						conn.search(
 							search_base=self.ldap_search_path_user,
 							search_filter="(objectClass=*)",
@@ -41,13 +56,13 @@ class LDAPSettings(Document):
 							search_base=self.ldap_search_path_group, search_filter="(objectClass=*)", attributes=["cn"]
 						)
 
-				except ldap3.core.exceptions.LDAPAttributeError as ex:
+				except LDAPAttributeError as ex:
 					frappe.throw(
 						_("LDAP settings incorrect. validation response was: {0}").format(ex),
 						title=_("Misconfigured"),
 					)
 
-				except ldap3.core.exceptions.LDAPNoSuchObjectResult:
+				except LDAPNoSuchObjectResult:
 					frappe.throw(
 						_("Ensure the user and group search paths are correct."), title=_("Misconfigured")
 					)
@@ -76,12 +91,8 @@ class LDAPSettings(Document):
 					)
 				)
 
-	def connect_to_ldap(self, base_dn, password, read_only=True):
+	def connect_to_ldap(self, base_dn, password, read_only=True) -> ldap3.Connection:
 		try:
-			import ssl
-
-			import ldap3
-
 			if self.require_trusted_certificate == "Yes":
 				tls_configuration = ldap3.Tls(validate=ssl.CERT_REQUIRED, version=ssl.PROTOCOL_TLS_CLIENT)
 			else:
@@ -95,9 +106,9 @@ class LDAPSettings(Document):
 				tls_configuration.ca_certs_file = self.local_ca_certs_file
 
 			server = ldap3.Server(host=self.ldap_server_url, tls=tls_configuration)
-			bind_type = ldap3.AUTO_BIND_TLS_BEFORE_BIND if self.ssl_tls_mode == "StartTLS" else True
+			bind_type = AUTO_BIND_TLS_BEFORE_BIND if self.ssl_tls_mode == "StartTLS" else True
 
-			conn = ldap3.Connection(
+			return ldap3.Connection(
 				server=server,
 				user=base_dn,
 				password=password,
@@ -106,42 +117,38 @@ class LDAPSettings(Document):
 				raise_exceptions=True,
 			)
 
-			return conn
-
 		except ImportError:
 			msg = _("Please Install the ldap3 library via pip to use ldap functionality.")
 			frappe.throw(msg, title=_("LDAP Not Installed"))
-		except ldap3.core.exceptions.LDAPInvalidCredentialsResult:
+		except LDAPInvalidCredentialsResult:
 			frappe.throw(_("Invalid username or password"))
 		except Exception as ex:
 			frappe.throw(_(str(ex)))
 
 	@staticmethod
-	def get_ldap_client_settings():
+	def get_ldap_client_settings() -> dict:
 		# return the settings to be used on the client side.
 		result = {"enabled": False}
-		ldap = frappe.get_doc("LDAP Settings")
+		ldap = frappe.get_cached_doc("LDAP Settings")
 		if ldap.enabled:
 			result["enabled"] = True
 			result["method"] = "frappe.integrations.doctype.ldap_settings.ldap_settings.login"
 		return result
 
 	@classmethod
-	def update_user_fields(cls, user, user_data):
-
+	def update_user_fields(cls, user: "User", user_data: dict):
 		updatable_data = {key: value for key, value in user_data.items() if key != "email"}
 
 		for key, value in updatable_data.items():
 			setattr(user, key, value)
 		user.save(ignore_permissions=True)
 
-	def sync_roles(self, user, additional_groups=None):
-
-		current_roles = set(d.role for d in user.get("roles"))
-
-		needed_roles = set()
-		needed_roles.add(self.default_role)
-
+	def sync_roles(self, user: "User", additional_groups: list = None):
+		current_roles = {d.role for d in user.get("roles")}
+		if self.default_user_type == "System User":
+			needed_roles = {self.default_role}
+		else:
+			needed_roles = set()
 		lower_groups = [g.lower() for g in additional_groups or []]
 
 		all_mapped_roles = {r.erpnext_role for r in self.ldap_groups}
@@ -158,28 +165,31 @@ class LDAPSettings(Document):
 
 		user.remove_roles(*roles_to_remove)
 
-	def create_or_update_user(self, user_data, groups=None):
-		user = None
+	def create_or_update_user(self, user_data: dict, groups: list = None):
+		user: "User" = None
+		role: str = None
+
 		if frappe.db.exists("User", user_data["email"]):
 			user = frappe.get_doc("User", user_data["email"])
 			LDAPSettings.update_user_fields(user=user, user_data=user_data)
 		else:
-			doc = user_data
-			doc.update(
-				{
-					"doctype": "User",
-					"send_welcome_email": 0,
-					"language": "",
-					"user_type": "System User",
-					# "roles": [{
-					# 	"role": self.default_role
-					# }]
-				}
-			)
+			doc = user_data | {
+				"doctype": "User",
+				"send_welcome_email": 0,
+				"language": "",
+				"user_type": self.default_user_type,
+			}
 			user = frappe.get_doc(doc)
 			user.insert(ignore_permissions=True)
-		# always add default role.
-		user.add_roles(self.default_role)
+
+		if self.default_user_type == "System User":
+			role = self.default_role
+		else:
+			role = frappe.db.get_value("User Type", user.user_type, "role")
+
+		if role:
+			user.add_roles(role)
+
 		self.sync_roles(user, groups)
 
 		return user
@@ -204,40 +214,28 @@ class LDAPSettings(Document):
 
 		return ldap_attributes
 
-	def fetch_ldap_groups(self, user, conn):
-		import ldap3
+	def fetch_ldap_groups(self, user: Entry, conn: ldap3.Connection) -> list:
+		if not isinstance(user, Entry):
+			raise TypeError("Invalid type, attribute 'user' must be of type 'ldap3.abstract.entry.Entry'")
 
-		if type(user) is not ldap3.abstract.entry.Entry:
-			raise TypeError(
-				"Invalid type, attribute {0} must be of type '{1}'".format(
-					"user", "ldap3.abstract.entry.Entry"
-				)
-			)
-
-		if type(conn) is not ldap3.core.connection.Connection:
-			raise TypeError(
-				"Invalid type, attribute {0} must be of type '{1}'".format("conn", "ldap3.Connection")
-			)
+		if not isinstance(conn, ldap3.Connection):
+			raise TypeError("Invalid type, attribute 'conn' must be of type 'ldap3.Connection'")
 
 		fetch_ldap_groups = None
-
 		ldap_object_class = None
 		ldap_group_members_attribute = None
 
 		if self.ldap_directory_server.lower() == "active directory":
-
 			ldap_object_class = "Group"
 			ldap_group_members_attribute = "member"
 			user_search_str = user.entry_dn
 
 		elif self.ldap_directory_server.lower() == "openldap":
-
 			ldap_object_class = "posixgroup"
 			ldap_group_members_attribute = "memberuid"
 			user_search_str = getattr(user, self.ldap_username_field).value
 
 		elif self.ldap_directory_server.lower() == "custom":
-
 			ldap_object_class = self.ldap_group_objectclass
 			ldap_group_members_attribute = self.ldap_group_member_attribute
 			ldap_custom_group_search = self.ldap_custom_group_search or "{0}"
@@ -248,68 +246,55 @@ class LDAPSettings(Document):
 			# this path will be hit for everyone with preconfigured ldap settings. this must be taken into account so as not to break ldap for those users.
 
 			if self.ldap_group_field:
-
 				fetch_ldap_groups = getattr(user, self.ldap_group_field).values
 
 		if ldap_object_class is not None:
 			conn.search(
 				search_base=self.ldap_search_path_group,
-				search_filter="(&(objectClass={0})({1}={2}))".format(
-					ldap_object_class, ldap_group_members_attribute, user_search_str
-				),
+				search_filter=f"(&(objectClass={ldap_object_class})({ldap_group_members_attribute}={user_search_str}))",
 				attributes=["cn"],
 			)  # Build search query
 
 		if len(conn.entries) >= 1:
-
 			fetch_ldap_groups = []
 			for group in conn.entries:
 				fetch_ldap_groups.append(group["cn"].value)
 
 		return fetch_ldap_groups
 
-	def authenticate(self, username, password):
-
+	def authenticate(self, username: str, password: str):
 		if not self.enabled:
 			frappe.throw(_("LDAP is not enabled."))
 
 		user_filter = self.ldap_search_string.format(username)
 		ldap_attributes = self.get_ldap_attributes()
-
 		conn = self.connect_to_ldap(self.base_dn, self.get_password(raise_exception=False))
 
 		try:
-			import ldap3
-
 			conn.search(
 				search_base=self.ldap_search_path_user,
-				search_filter="{0}".format(user_filter),
+				search_filter=f"{user_filter}",
 				attributes=ldap_attributes,
 			)
 
 			if len(conn.entries) == 1 and conn.entries[0]:
 				user = conn.entries[0]
-
 				groups = self.fetch_ldap_groups(user, conn)
 
 				# only try and connect as the user, once we have their fqdn entry.
 				if user.entry_dn and password and conn.rebind(user=user.entry_dn, password=password):
-
 					return self.create_or_update_user(self.convert_ldap_entry_to_dict(user), groups=groups)
 
-			raise ldap3.core.exceptions.LDAPInvalidCredentialsResult  # even though nothing foundor failed authentication raise invalid credentials
+			raise LDAPInvalidCredentialsResult  # even though nothing foundor failed authentication raise invalid credentials
 
-		except ldap3.core.exceptions.LDAPInvalidFilterError:
+		except LDAPInvalidFilterError:
 			frappe.throw(_("Please use a valid LDAP search filter"), title=_("Misconfigured"))
 
-		except ldap3.core.exceptions.LDAPInvalidCredentialsResult:
+		except LDAPInvalidCredentialsResult:
 			frappe.throw(_("Invalid username or password"))
 
 	def reset_password(self, user, password, logout_sessions=False):
-		from ldap3 import HASHED_SALTED_SHA, MODIFY_REPLACE
-		from ldap3.utils.hashed import hashed
-
-		search_filter = "({0}={1})".format(self.ldap_email_field, user)
+		search_filter = f"({self.ldap_email_field}={user})"
 
 		conn = self.connect_to_ldap(
 			self.base_dn, self.get_password(raise_exception=False), read_only=False
@@ -337,8 +322,7 @@ class LDAPSettings(Document):
 		else:
 			frappe.throw(_("No LDAP User found for email: {0}").format(user))
 
-	def convert_ldap_entry_to_dict(self, user_entry):
-
+	def convert_ldap_entry_to_dict(self, user_entry: Entry):
 		# support multiple email values
 		email = user_entry[self.ldap_email_field]
 
@@ -349,7 +333,6 @@ class LDAPSettings(Document):
 		}
 
 		# optional fields
-
 		if self.ldap_middle_name_field:
 			data["middle_name"] = user_entry[self.ldap_middle_name_field].value
 
@@ -369,7 +352,7 @@ class LDAPSettings(Document):
 def login():
 	# LDAP LOGIN LOGIC
 	args = frappe.form_dict
-	ldap = frappe.get_doc("LDAP Settings")
+	ldap: LDAPSettings = frappe.get_doc("LDAP Settings")
 
 	user = ldap.authenticate(frappe.as_unicode(args.usr), frappe.as_unicode(args.pwd))
 
@@ -386,7 +369,7 @@ def login():
 
 @frappe.whitelist()
 def reset_password(user, password, logout):
-	ldap = frappe.get_doc("LDAP Settings")
+	ldap: LDAPSettings = frappe.get_doc("LDAP Settings")
 	if not ldap.enabled:
 		frappe.throw(_("LDAP is not enabled."))
 	ldap.reset_password(user, password, logout_sessions=int(logout))
diff --git a/frappe/integrations/doctype/ldap_settings/test_ldap_settings.py b/frappe/integrations/doctype/ldap_settings/test_ldap_settings.py
index 0651932843..9080e0c82a 100644
--- a/frappe/integrations/doctype/ldap_settings/test_ldap_settings.py
+++ b/frappe/integrations/doctype/ldap_settings/test_ldap_settings.py
@@ -1,16 +1,16 @@
-# -*- coding: utf-8 -*-
-# Copyright (c) 2019, Frappe Technologies and Contributors
+# Copyright (c) 2022, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
+import contextlib
 import functools
 import os
 import ssl
-import unittest
-from unittest import mock
+from unittest import TestCase, mock
 
 import ldap3
 from ldap3 import MOCK_SYNC, OFFLINE_AD_2012_R2, OFFLINE_SLAPD_2_4, Connection, Server
 
 import frappe
+from frappe.exceptions import MandatoryError, ValidationError
 from frappe.integrations.doctype.ldap_settings.ldap_settings import LDAPSettings
 
 
@@ -23,15 +23,19 @@ class LDAP_TestCase:
 	LDAP_LDIF_JSON = None
 	TEST_VALUES_LDAP_COMPLEX_SEARCH_STRING = None
 
+	# for adding type hints during development ^_^
+	assertTrue = TestCase.assertTrue
+	assertEqual = TestCase.assertEqual
+	assertIn = TestCase.assertIn
+
 	def mock_ldap_connection(f):
 		@functools.wraps(f)
 		def wrapped(self, *args, **kwargs):
 
 			with mock.patch(
-				"frappe.integrations.doctype.ldap_settings.ldap_settings.LDAPSettings.connect_to_ldap"
-			) as mock_connection:
-				mock_connection.return_value = self.connection
-
+				"frappe.integrations.doctype.ldap_settings.ldap_settings.LDAPSettings.connect_to_ldap",
+				return_value=self.connection,
+			):
 				self.test_class = LDAPSettings(self.doc)
 
 				# Create a clean doc
@@ -48,80 +52,66 @@ class LDAP_TestCase:
 		return wrapped
 
 	def clean_test_users():
-		try:  # clean up test user 1
+		with contextlib.suppress(Exception):
 			frappe.get_doc("User", "posix.user1@unit.testing").delete()
-		except Exception:
-			pass
-
-		try:  # clean up test user 2
+		with contextlib.suppress(Exception):
 			frappe.get_doc("User", "posix.user2@unit.testing").delete()
-		except Exception:
-			pass
+		with contextlib.suppress(Exception):
+			frappe.get_doc("User", "website_ldap_user@test.com").delete()
 
 	@classmethod
-	def setUpClass(self, ldapServer="OpenLDAP"):
-
-		self.clean_test_users()
+	def setUpClass(cls):
+		cls.clean_test_users()
 		# Save user data for restoration in tearDownClass()
-		self.user_ldap_settings = frappe.get_doc("LDAP Settings")
+		cls.user_ldap_settings = frappe.get_doc("LDAP Settings")
 
 		# Create test user1
-		self.user1doc = {
+		cls.user1doc = {
 			"username": "posix.user",
 			"email": "posix.user1@unit.testing",
 			"first_name": "posix",
+			"doctype": "User",
+			"send_welcome_email": 0,
+			"language": "",
+			"user_type": "System User",
 		}
-		self.user1doc.update(
-			{
-				"doctype": "User",
-				"send_welcome_email": 0,
-				"language": "",
-				"user_type": "System User",
-			}
-		)
 
-		user = frappe.get_doc(self.user1doc)
+		user = frappe.get_doc(cls.user1doc)
 		user.insert(ignore_permissions=True)
 
-		# Create test user1
-		self.user2doc = {
+		cls.user2doc = {
 			"username": "posix.user2",
 			"email": "posix.user2@unit.testing",
 			"first_name": "posix",
+			"doctype": "User",
+			"send_welcome_email": 0,
+			"language": "",
+			"user_type": "System User",
 		}
-		self.user2doc.update(
-			{
-				"doctype": "User",
-				"send_welcome_email": 0,
-				"language": "",
-				"user_type": "System User",
-			}
-		)
-
-		user = frappe.get_doc(self.user2doc)
+		user = frappe.get_doc(cls.user2doc)
 		user.insert(ignore_permissions=True)
 
 		# Setup Mock OpenLDAP Directory
-		self.ldap_dc_path = "dc=unit,dc=testing"
-		self.ldap_user_path = "ou=users," + self.ldap_dc_path
-		self.ldap_group_path = "ou=groups," + self.ldap_dc_path
-		self.base_dn = "cn=base_dn_user," + self.ldap_dc_path
-		self.base_password = "my_password"
-		self.ldap_server = "ldap://my_fake_server:389"
+		cls.ldap_dc_path = "dc=unit,dc=testing"
+		cls.ldap_user_path = f"ou=users,{cls.ldap_dc_path}"
+		cls.ldap_group_path = f"ou=groups,{cls.ldap_dc_path}"
+		cls.base_dn = f"cn=base_dn_user,{cls.ldap_dc_path}"
+		cls.base_password = "my_password"
+		cls.ldap_server = "ldap://my_fake_server:389"
 
-		self.doc = {
+		cls.doc = {
 			"doctype": "LDAP Settings",
 			"enabled": True,
-			"ldap_directory_server": self.TEST_LDAP_SERVER,
-			"ldap_server_url": self.ldap_server,
-			"base_dn": self.base_dn,
-			"password": self.base_password,
-			"ldap_search_path_user": self.ldap_user_path,
-			"ldap_search_string": self.TEST_LDAP_SEARCH_STRING,
-			"ldap_search_path_group": self.ldap_group_path,
+			"ldap_directory_server": cls.TEST_LDAP_SERVER,
+			"ldap_server_url": cls.ldap_server,
+			"base_dn": cls.base_dn,
+			"password": cls.base_password,
+			"ldap_search_path_user": cls.ldap_user_path,
+			"ldap_search_string": cls.TEST_LDAP_SEARCH_STRING,
+			"ldap_search_path_group": cls.ldap_group_path,
 			"ldap_user_creation_and_mapping_section": "",
 			"ldap_email_field": "mail",
-			"ldap_username_field": self.LDAP_USERNAME_FIELD,
+			"ldap_username_field": cls.LDAP_USERNAME_FIELD,
 			"ldap_first_name_field": "givenname",
 			"ldap_middle_name_field": "",
 			"ldap_last_name_field": "sn",
@@ -136,50 +126,41 @@ class LDAP_TestCase:
 			"ldap_group_objectclass": "",
 			"ldap_group_member_attribute": "",
 			"default_role": "Newsletter Manager",
-			"ldap_groups": self.DOCUMENT_GROUP_MAPPINGS,
+			"ldap_groups": cls.DOCUMENT_GROUP_MAPPINGS,
 			"ldap_group_field": "",
+			"default_user_type": "System User",
 		}
 
-		self.server = Server(host=self.ldap_server, port=389, get_info=self.LDAP_SCHEMA)
-
-		self.connection = Connection(
-			self.server,
-			user=self.base_dn,
-			password=self.base_password,
+		cls.server = Server(host=cls.ldap_server, port=389, get_info=cls.LDAP_SCHEMA)
+		cls.connection = Connection(
+			cls.server,
+			user=cls.base_dn,
+			password=cls.base_password,
 			read_only=True,
 			client_strategy=MOCK_SYNC,
 		)
-
-		self.connection.strategy.entries_from_json(
-			os.path.abspath(os.path.dirname(__file__)) + "/" + self.LDAP_LDIF_JSON
+		cls.connection.strategy.entries_from_json(
+			f"{os.path.abspath(os.path.dirname(__file__))}/{cls.LDAP_LDIF_JSON}"
 		)
-
-		self.connection.bind()
+		cls.connection.bind()
 
 	@classmethod
-	def tearDownClass(self):
-		try:
+	def tearDownClass(cls):
+		with contextlib.suppress(Exception):
 			frappe.get_doc("LDAP Settings").delete()
 
-		except Exception:
-			pass
-
-		try:
-			# return doc back to user data
-			self.user_ldap_settings.save()
-
-		except Exception:
-			pass
+		# return doc back to user data
+		with contextlib.suppress(Exception):
+			cls.user_ldap_settings.save()
 
 		# Clean-up test users
-		self.clean_test_users()
+		cls.clean_test_users()
 
 		# Clear OpenLDAP connection
-		self.connection = None
+		cls.connection = None
 
 	@mock_ldap_connection
 	def test_mandatory_fields(self):
-
 		mandatory_fields = [
 			"ldap_server_url",
 			"ldap_directory_server",
@@ -196,26 +177,14 @@ class LDAP_TestCase:
 		]  # fields that are required to have ldap functioning need to be mandatory
 
 		for mandatory_field in mandatory_fields:
-
 			localdoc = self.doc.copy()
 			localdoc[mandatory_field] = ""
 
-			try:
-
+			with contextlib.suppress(MandatoryError, ValidationError):
 				frappe.get_doc(localdoc).save()
-
-				self.fail("Document LDAP Settings field [{0}] is not mandatory".format(mandatory_field))
-
-			except frappe.exceptions.MandatoryError:
-				pass
-
-			except frappe.exceptions.ValidationError:
-				if mandatory_field == "ldap_search_string":
-					# additional validation is done on this field, pass in this instance
-					pass
+				self.fail(f"Document LDAP Settings field [{mandatory_field}] is not mandatory")
 
 		for non_mandatory_field in self.doc:  # Ensure remaining fields have not been made mandatory
-
 			if non_mandatory_field == "doctype" or non_mandatory_field in mandatory_fields:
 				continue
 
@@ -223,17 +192,12 @@ class LDAP_TestCase:
 			localdoc[non_mandatory_field] = ""
 
 			try:
-
 				frappe.get_doc(localdoc).save()
-
-			except frappe.exceptions.MandatoryError:
-				self.fail(
-					"Document LDAP Settings field [{0}] should not be mandatory".format(non_mandatory_field)
-				)
+			except MandatoryError:
+				self.fail(f"Document LDAP Settings field [{non_mandatory_field}] should not be mandatory")
 
 	@mock_ldap_connection
 	def test_validation_ldap_search_string(self):
-
 		invalid_ldap_search_strings = [
 			"",
 			"uid={0}",
@@ -245,19 +209,26 @@ class LDAP_TestCase:
 		]  # ldap search string must be enclosed in '()' for ldap search to work for finding user and have the same number of opening and closing brackets.
 
 		for invalid_search_string in invalid_ldap_search_strings:
-
 			localdoc = self.doc.copy()
 			localdoc["ldap_search_string"] = invalid_search_string
 
-			try:
+			with contextlib.suppress(ValidationError):
 				frappe.get_doc(localdoc).save()
-
-				self.fail("LDAP search string [{0}] should not validate".format(invalid_search_string))
-
-			except frappe.exceptions.ValidationError:
-				pass
+				self.fail(f"LDAP search string [{invalid_search_string}] should not validate")
 
 	def test_connect_to_ldap(self):
+		# prevent these parameters for security or lack of the und user from being able to configure
+		prevent_connection_parameters = {
+			"mode": {
+				"IP_V4_ONLY": "Locks the user to IPv4 without frappe providing a way to configure",
+				"IP_V6_ONLY": "Locks the user to IPv6 without frappe providing a way to configure",
+			},
+			"auto_bind": {
+				"NONE": "ldap3.Connection must autobind with base_dn",
+				"NO_TLS": "ldap3.Connection must have TLS",
+				"TLS_AFTER_BIND": "[Security] ldap3.Connection TLS bind must occur before bind",
+			},
+		}
 
 		# setup a clean doc with ldap disabled so no validation occurs (this is tested seperatly)
 		local_doc = self.doc.copy()
@@ -265,48 +236,25 @@ class LDAP_TestCase:
 		self.test_class = LDAPSettings(self.doc)
 
 		with mock.patch("ldap3.Server") as ldap3_server_method:
-
-			with mock.patch("ldap3.Connection") as ldap3_connection_method:
-				ldap3_connection_method.return_value = self.connection
-
+			with mock.patch("ldap3.Connection", return_value=self.connection) as ldap3_connection_method:
 				with mock.patch("ldap3.Tls") as ldap3_Tls_method:
-
 					function_return = self.test_class.connect_to_ldap(
 						base_dn=self.base_dn, password=self.base_password
 					)
-
 					args, kwargs = ldap3_connection_method.call_args
 
-					prevent_connection_parameters = {
-						# prevent these parameters for security or lack of the und user from being able to configure
-						"mode": {
-							"IP_V4_ONLY": "Locks the user to IPv4 without frappe providing a way to configure",
-							"IP_V6_ONLY": "Locks the user to IPv6 without frappe providing a way to configure",
-						},
-						"auto_bind": {
-							"NONE": "ldap3.Connection must autobind with base_dn",
-							"NO_TLS": "ldap3.Connection must have TLS",
-							"TLS_AFTER_BIND": "[Security] ldap3.Connection TLS bind must occur before bind",
-						},
-					}
-
 					for connection_arg in kwargs:
-
 						if (
 							connection_arg in prevent_connection_parameters
 							and kwargs[connection_arg] in prevent_connection_parameters[connection_arg]
 						):
-
 							self.fail(
-								"ldap3.Connection was called with {0}, failed reason: [{1}]".format(
-									kwargs[connection_arg],
-									prevent_connection_parameters[connection_arg][kwargs[connection_arg]],
-								)
+								f"ldap3.Connection was called with {kwargs[connection_arg]}, failed reason: [{prevent_connection_parameters[connection_arg][kwargs[connection_arg]]}]"
 							)
 
+					tls_version = ssl.PROTOCOL_TLS_CLIENT
 					if local_doc["require_trusted_certificate"] == "Yes":
 						tls_validate = ssl.CERT_REQUIRED
-						tls_version = ssl.PROTOCOL_TLS_CLIENT
 						tls_configuration = ldap3.Tls(validate=tls_validate, version=tls_version)
 
 						self.assertTrue(
@@ -316,7 +264,6 @@ class LDAP_TestCase:
 
 					else:
 						tls_validate = ssl.CERT_NONE
-						tls_version = ssl.PROTOCOL_TLS_CLIENT
 						tls_configuration = ldap3.Tls(validate=tls_validate, version=tls_version)
 
 						self.assertTrue(kwargs["auto_bind"], "ldap3.Connection must autobind")
@@ -350,7 +297,7 @@ class LDAP_TestCase:
 					)
 
 					self.assertTrue(
-						type(function_return) is ldap3.core.connection.Connection,
+						type(function_return) is Connection,
 						"The return type must be of ldap3.Connection",
 					)
 
@@ -367,24 +314,20 @@ class LDAP_TestCase:
 
 	@mock_ldap_connection
 	def test_get_ldap_client_settings(self):
-
 		result = self.test_class.get_ldap_client_settings()
 
 		self.assertIsInstance(result, dict)
-
 		self.assertTrue(result["enabled"] == self.doc["enabled"])  # settings should match doc
 
 		localdoc = self.doc.copy()
 		localdoc["enabled"] = False
 		frappe.get_doc(localdoc).save()
-
 		result = self.test_class.get_ldap_client_settings()
 
 		self.assertFalse(result["enabled"])  # must match the edited doc
 
 	@mock_ldap_connection
 	def test_update_user_fields(self):
-
 		test_user_data = {
 			"username": "posix.user",
 			"email": "posix.user1@unit.testing",
@@ -394,11 +337,8 @@ class LDAP_TestCase:
 			"phone": "08 1234 5678",
 			"mobile_no": "0421 123 456",
 		}
-
 		test_user = frappe.get_doc("User", test_user_data["email"])
-
 		self.test_class.update_user_fields(test_user, test_user_data)
-
 		updated_user = frappe.get_doc("User", test_user_data["email"])
 
 		self.assertTrue(updated_user.middle_name == test_user_data["middle_name"])
@@ -406,9 +346,23 @@ class LDAP_TestCase:
 		self.assertTrue(updated_user.phone == test_user_data["phone"])
 		self.assertTrue(updated_user.mobile_no == test_user_data["mobile_no"])
 
+		self.assertEqual(updated_user.user_type, self.test_class.default_user_type)
+		self.assertIn(self.test_class.default_role, frappe.get_roles(updated_user.name))
+
+	@mock_ldap_connection
+	def test_create_website_user(self):
+		new_test_user_data = {
+			"username": "website_ldap_user.test",
+			"email": "website_ldap_user@test.com",
+			"first_name": "Website User - LDAP Test",
+		}
+		self.test_class.default_user_type = "Website User"
+		self.test_class.create_or_update_user(user_data=new_test_user_data, groups=[])
+		new_user = frappe.get_doc("User", new_test_user_data["email"])
+		self.assertEqual(new_user.user_type, "Website User")
+
 	@mock_ldap_connection
 	def test_sync_roles(self):
-
 		if self.TEST_LDAP_SERVER.lower() == "openldap":
 			test_user_data = {
 				"posix.user1": [
@@ -460,9 +414,8 @@ class LDAP_TestCase:
 		user.insert(ignore_permissions=True)
 
 		for test_user in test_user_data:
-
-			test_user_doc = frappe.get_doc("User", test_user + "@unit.testing")
-			test_user_roles = frappe.get_roles(test_user + "@unit.testing")
+			test_user_doc = frappe.get_doc("User", f"{test_user}@unit.testing")
+			test_user_roles = frappe.get_roles(f"{test_user}@unit.testing")
 
 			self.assertTrue(
 				len(test_user_roles) == 2, "User should only be a part of the All and Guest roles"
@@ -470,28 +423,22 @@ class LDAP_TestCase:
 
 			self.test_class.sync_roles(test_user_doc, test_user_data[test_user])  # update user roles
 
-			frappe.get_doc("User", test_user + "@unit.testing")
-			updated_user_roles = frappe.get_roles(test_user + "@unit.testing")
+			frappe.get_doc("User", f"{test_user}@unit.testing")
+			updated_user_roles = frappe.get_roles(f"{test_user}@unit.testing")
 
 			self.assertTrue(
 				len(updated_user_roles) == len(test_user_data[test_user]),
-				"syncing of the user roles failed. {0} != {1} for user {2}".format(
-					len(updated_user_roles), len(test_user_data[test_user]), test_user
-				),
+				f"syncing of the user roles failed. {len(updated_user_roles)} != {len(test_user_data[test_user])} for user {test_user}",
 			)
 
 			for user_role in updated_user_roles:  # match each users role mapped to ldap groups
-
 				self.assertTrue(
 					role_to_group_map[user_role] in test_user_data[test_user],
-					"during sync_roles(), the user was given role {0} which should not have occured".format(
-						user_role
-					),
+					f"during sync_roles(), the user was given role {user_role} which should not have occured",
 				)
 
 	@mock_ldap_connection
 	def test_create_or_update_user(self):
-
 		test_user_data = {
 			"posix.user1": [
 				"Users",
@@ -501,28 +448,21 @@ class LDAP_TestCase:
 				"frappe_default_guest",
 			],
 		}
-
 		test_user = "posix.user1"
 
-		frappe.get_doc("User", test_user + "@unit.testing").delete()  # remove user 1
+		frappe.get_doc("User", f"{test_user}@unit.testing").delete()
 
 		with self.assertRaises(
 			frappe.exceptions.DoesNotExistError
 		):  # ensure user deleted so function can be tested
-			frappe.get_doc("User", test_user + "@unit.testing")
+			frappe.get_doc("User", f"{test_user}@unit.testing")
 
 		with mock.patch(
 			"frappe.integrations.doctype.ldap_settings.ldap_settings.LDAPSettings.update_user_fields"
 		) as update_user_fields_method:
-
-			update_user_fields_method.return_value = None
-
 			with mock.patch(
 				"frappe.integrations.doctype.ldap_settings.ldap_settings.LDAPSettings.sync_roles"
 			) as sync_roles_method:
-
-				sync_roles_method.return_value = None
-
 				# New user
 				self.test_class.create_or_update_user(self.user1doc, test_user_data[test_user])
 
@@ -542,14 +482,11 @@ class LDAP_TestCase:
 
 	@mock_ldap_connection
 	def test_get_ldap_attributes(self):
-
 		method_return = self.test_class.get_ldap_attributes()
-
 		self.assertTrue(type(method_return) is list)
 
 	@mock_ldap_connection
 	def test_fetch_ldap_groups(self):
-
 		if self.TEST_LDAP_SERVER.lower() == "openldap":
 			test_users = {"posix.user": ["Users", "Administrators"], "posix.user2": ["Users", "Group3"]}
 		elif self.TEST_LDAP_SERVER.lower() == "active directory":
@@ -559,7 +496,6 @@ class LDAP_TestCase:
 			}
 
 		for test_user in test_users:
-
 			self.connection.search(
 				search_base=self.ldap_user_path,
 				search_filter=self.TEST_LDAP_SEARCH_STRING.format(test_user),
@@ -572,18 +508,13 @@ class LDAP_TestCase:
 			self.assertTrue(len(method_return) == len(test_users[test_user]))
 
 			for returned_group in method_return:
-
 				self.assertTrue(returned_group in test_users[test_user])
 
 	@mock_ldap_connection
 	def test_authenticate(self):
-
 		with mock.patch(
 			"frappe.integrations.doctype.ldap_settings.ldap_settings.LDAPSettings.fetch_ldap_groups"
 		) as fetch_ldap_groups_function:
-
-			fetch_ldap_groups_function.return_value = None
-
 			self.assertTrue(self.test_class.authenticate("posix.user", "posix_user_password"))
 
 		self.assertTrue(
@@ -602,25 +533,19 @@ class LDAP_TestCase:
 		]  # All invalid users should return 'invalid username or password'
 
 		for username, password in enumerate(invalid_users):
-
 			with self.assertRaises(frappe.exceptions.ValidationError) as display_massage:
-
 				self.test_class.authenticate(username, password)
 
 			self.assertTrue(
 				str(display_massage.exception).lower() == "invalid username or password",
-				"invalid credentials passed authentication [user: {0}, password: {1}]".format(
-					username, password
-				),
+				f"invalid credentials passed authentication [user: {username}, password: {password}]",
 			)
 
 	@mock_ldap_connection
 	def test_complex_ldap_search_filter(self):
-
 		ldap_search_filters = self.TEST_VALUES_LDAP_COMPLEX_SEARCH_STRING
 
 		for search_filter in ldap_search_filters:
-
 			self.test_class.ldap_search_string = search_filter
 
 			if (
@@ -637,55 +562,44 @@ class LDAP_TestCase:
 				self.assertTrue(self.test_class.authenticate("posix.user", "posix_user_password"))
 
 	def test_reset_password(self):
-
 		self.test_class = LDAPSettings(self.doc)
 
 		# Create a clean doc
 		localdoc = self.doc.copy()
-
 		localdoc["enabled"] = False
 		frappe.get_doc(localdoc).save()
 
 		with mock.patch(
-			"frappe.integrations.doctype.ldap_settings.ldap_settings.LDAPSettings.connect_to_ldap"
+			"frappe.integrations.doctype.ldap_settings.ldap_settings.LDAPSettings.connect_to_ldap",
+			return_value=self.connection,
 		) as connect_to_ldap:
-			connect_to_ldap.return_value = self.connection
-
 			with self.assertRaises(
 				frappe.exceptions.ValidationError
 			) as validation:  # Fail if username string used
 				self.test_class.reset_password("posix.user", "posix_user_password")
-
 			self.assertTrue(str(validation.exception) == "No LDAP User found for email: posix.user")
 
-			try:
+			with contextlib.suppress(Exception):
 				self.test_class.reset_password(
 					"posix.user1@unit.testing", "posix_user_password"
 				)  # Change Password
-
-			except Exception:  # An exception from the tested class is ok, as long as the connection to LDAP was made writeable
-				pass
-
 			connect_to_ldap.assert_called_with(self.base_dn, self.base_password, read_only=False)
 
 	@mock_ldap_connection
 	def test_convert_ldap_entry_to_dict(self):
-
 		self.connection.search(
 			search_base=self.ldap_user_path,
 			search_filter=self.TEST_LDAP_SEARCH_STRING.format("posix.user"),
 			attributes=self.test_class.get_ldap_attributes(),
 		)
-
 		test_ldap_entry = self.connection.entries[0]
-
 		method_return = self.test_class.convert_ldap_entry_to_dict(test_ldap_entry)
 
 		self.assertTrue(type(method_return) is dict)  # must be dict
 		self.assertTrue(len(method_return) == 6)  # there are 6 fields in mock_ldap for use
 
 
-class Test_OpenLDAP(LDAP_TestCase, unittest.TestCase):
+class Test_OpenLDAP(LDAP_TestCase, TestCase):
 	TEST_LDAP_SERVER = "OpenLDAP"
 	TEST_LDAP_SEARCH_STRING = "(uid={0})"
 	DOCUMENT_GROUP_MAPPINGS = [
@@ -709,7 +623,7 @@ class Test_OpenLDAP(LDAP_TestCase, unittest.TestCase):
 	]
 
 
-class Test_ActiveDirectory(LDAP_TestCase, unittest.TestCase):
+class Test_ActiveDirectory(LDAP_TestCase, TestCase):
 	TEST_LDAP_SERVER = "Active Directory"
 	TEST_LDAP_SEARCH_STRING = "(samaccountname={0})"
 	DOCUMENT_GROUP_MAPPINGS = [
diff --git a/frappe/integrations/doctype/oauth_authorization_code/oauth_authorization_code.js b/frappe/integrations/doctype/oauth_authorization_code/oauth_authorization_code.js
index 32746e6752..83ad1b3ee5 100644
--- a/frappe/integrations/doctype/oauth_authorization_code/oauth_authorization_code.js
+++ b/frappe/integrations/doctype/oauth_authorization_code/oauth_authorization_code.js
@@ -1,8 +1,6 @@
 // Copyright (c) 2016, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('OAuth Authorization Code', {
-	refresh: function(frm) {
-
-	}
+frappe.ui.form.on("OAuth Authorization Code", {
+	refresh: function (frm) {},
 });
diff --git a/frappe/integrations/doctype/oauth_authorization_code/oauth_authorization_code.py b/frappe/integrations/doctype/oauth_authorization_code/oauth_authorization_code.py
index 4ef6f65dc7..431d27bc04 100644
--- a/frappe/integrations/doctype/oauth_authorization_code/oauth_authorization_code.py
+++ b/frappe/integrations/doctype/oauth_authorization_code/oauth_authorization_code.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2015, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
diff --git a/frappe/integrations/doctype/oauth_authorization_code/test_oauth_authorization_code.py b/frappe/integrations/doctype/oauth_authorization_code/test_oauth_authorization_code.py
index 72cb789ebb..2036a42f15 100644
--- a/frappe/integrations/doctype/oauth_authorization_code/test_oauth_authorization_code.py
+++ b/frappe/integrations/doctype/oauth_authorization_code/test_oauth_authorization_code.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2015, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
 import unittest
diff --git a/frappe/integrations/doctype/oauth_bearer_token/oauth_bearer_token.js b/frappe/integrations/doctype/oauth_bearer_token/oauth_bearer_token.js
index da69753903..7794f2fb70 100644
--- a/frappe/integrations/doctype/oauth_bearer_token/oauth_bearer_token.js
+++ b/frappe/integrations/doctype/oauth_bearer_token/oauth_bearer_token.js
@@ -1,8 +1,6 @@
 // Copyright (c) 2016, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('OAuth Bearer Token', {
-	refresh: function(frm) {
-
-	}
+frappe.ui.form.on("OAuth Bearer Token", {
+	refresh: function (frm) {},
 });
diff --git a/frappe/integrations/doctype/oauth_bearer_token/oauth_bearer_token.py b/frappe/integrations/doctype/oauth_bearer_token/oauth_bearer_token.py
index 515d3d2ba3..2a17035571 100644
--- a/frappe/integrations/doctype/oauth_bearer_token/oauth_bearer_token.py
+++ b/frappe/integrations/doctype/oauth_bearer_token/oauth_bearer_token.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2015, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
diff --git a/frappe/integrations/doctype/oauth_bearer_token/test_oauth_bearer_token.py b/frappe/integrations/doctype/oauth_bearer_token/test_oauth_bearer_token.py
index 9dea8f482a..3439096809 100644
--- a/frappe/integrations/doctype/oauth_bearer_token/test_oauth_bearer_token.py
+++ b/frappe/integrations/doctype/oauth_bearer_token/test_oauth_bearer_token.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2015, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
 import unittest
diff --git a/frappe/integrations/doctype/oauth_client/oauth_client.js b/frappe/integrations/doctype/oauth_client/oauth_client.js
index b0caa562b1..3ddd1a046b 100644
--- a/frappe/integrations/doctype/oauth_client/oauth_client.js
+++ b/frappe/integrations/doctype/oauth_client/oauth_client.js
@@ -1,8 +1,6 @@
 // Copyright (c) 2016, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('OAuth Client', {
-	refresh: function(frm) {
-
-	}
+frappe.ui.form.on("OAuth Client", {
+	refresh: function (frm) {},
 });
diff --git a/frappe/integrations/doctype/oauth_client/oauth_client.json b/frappe/integrations/doctype/oauth_client/oauth_client.json
index d0d45c36ab..f4ccde8174 100644
--- a/frappe/integrations/doctype/oauth_client/oauth_client.json
+++ b/frappe/integrations/doctype/oauth_client/oauth_client.json
@@ -1,517 +1,144 @@
 {
- "allow_copy": 0,
- "allow_guest_to_view": 0,
- "allow_import": 0,
- "allow_rename": 0,
- "autoname": "",
- "beta": 0,
+ "actions": [],
  "creation": "2016-08-24 14:07:21.955052",
- "custom": 0,
- "docstatus": 0,
  "doctype": "DocType",
  "document_type": "Document",
  "editable_grid": 1,
  "engine": "InnoDB",
+ "field_order": [
+  "client_id",
+  "app_name",
+  "user",
+  "cb_1",
+  "client_secret",
+  "skip_authorization",
+  "sb_1",
+  "scopes",
+  "cb_3",
+  "redirect_uris",
+  "default_redirect_uri",
+  "sb_advanced",
+  "grant_type",
+  "cb_2",
+  "response_type"
+ ],
  "fields": [
   {
-   "allow_bulk_edit": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
-   "default": "",
    "fieldname": "client_id",
    "fieldtype": "Data",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
    "label": "App Client ID",
-   "length": 0,
-   "no_copy": 0,
-   "permlevel": 0,
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 1,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "unique": 0
+   "read_only": 1
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
    "fieldname": "app_name",
    "fieldtype": "Data",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
    "label": "App Name",
-   "length": 0,
-   "no_copy": 0,
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 1,
-   "search_index": 0,
-   "set_only_once": 0,
-   "unique": 0
+   "reqd": 1
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
    "fieldname": "user",
    "fieldtype": "Link",
    "hidden": 1,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
    "label": "User",
-   "length": 0,
-   "no_copy": 0,
-   "options": "User",
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "unique": 0
+   "options": "User"
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
    "fieldname": "cb_1",
-   "fieldtype": "Column Break",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
-   "length": 0,
-   "no_copy": 0,
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "unique": 0
+   "fieldtype": "Column Break"
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
    "fieldname": "client_secret",
    "fieldtype": "Data",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
    "label": "App Client Secret",
-   "length": 0,
-   "no_copy": 0,
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 1,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "unique": 0
+   "read_only": 1
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
+   "default": "0",
    "description": "If checked, users will not see the Confirm Access dialog.",
    "fieldname": "skip_authorization",
    "fieldtype": "Check",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
-   "label": "Skip Authorization",
-   "length": 0,
-   "no_copy": 0,
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "unique": 0
+   "label": "Skip Authorization"
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
-   "description": "",
    "fieldname": "sb_1",
-   "fieldtype": "Section Break",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
-   "label": "",
-   "length": 0,
-   "no_copy": 0,
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "unique": 0
+   "fieldtype": "Section Break"
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
    "default": "all openid",
    "description": "A list of resources which the Client App will have access to after the user allows it.
         e.g. project",
    "fieldname": "scopes",
    "fieldtype": "Text",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
    "label": "Scopes",
-   "length": 0,
-   "no_copy": 0,
-   "permlevel": 0,
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 1,
-   "search_index": 0,
-   "set_only_once": 0,
-   "unique": 0
+   "reqd": 1
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
    "fieldname": "cb_3",
-   "fieldtype": "Column Break",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
-   "length": 0,
-   "no_copy": 0,
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "unique": 0
+   "fieldtype": "Column Break"
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
    "description": "URIs for receiving authorization code once the user allows access, as well as failure responses. Typically a REST endpoint exposed by the Client App.\n
        e.g. http://hostname//api/method/frappe.www.login.login_via_facebook",
    "fieldname": "redirect_uris",
    "fieldtype": "Text",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
-   "label": "Redirect URIs",
-   "length": 0,
-   "no_copy": 0,
-   "permlevel": 0,
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "unique": 0
+   "label": "Redirect URIs"
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
    "fieldname": "default_redirect_uri",
    "fieldtype": "Data",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
    "label": "Default Redirect URI",
-   "length": 0,
-   "no_copy": 0,
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 1,
-   "search_index": 0,
-   "set_only_once": 0,
-   "unique": 0
+   "reqd": 1
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
    "collapsible": 1,
    "collapsible_depends_on": "1",
-   "columns": 0,
    "fieldname": "sb_advanced",
    "fieldtype": "Section Break",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
-   "label": "Advanced Settings",
-   "length": 0,
-   "no_copy": 0,
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "unique": 0
+   "label": "Advanced Settings"
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
    "fieldname": "grant_type",
    "fieldtype": "Select",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
    "in_list_view": 1,
    "in_standard_filter": 1,
    "label": "Grant Type",
-   "length": 0,
-   "no_copy": 0,
-   "options": "Authorization Code\nImplicit",
-   "permlevel": 0,
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "unique": 0
+   "options": "Authorization Code\nImplicit"
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
    "fieldname": "cb_2",
-   "fieldtype": "Column Break",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
-   "length": 0,
-   "no_copy": 0,
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "unique": 0
+   "fieldtype": "Column Break"
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
    "default": "Code",
    "fieldname": "response_type",
    "fieldtype": "Select",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
    "in_list_view": 1,
    "in_standard_filter": 1,
    "label": "Response Type",
-   "length": 0,
-   "no_copy": 0,
-   "options": "Code\nToken",
-   "permlevel": 0,
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "unique": 0
+   "options": "Code\nToken"
   }
  ],
- "has_web_view": 0,
- "hide_heading": 0,
- "hide_toolbar": 0,
- "idx": 0,
- "image_view": 0,
- "in_create": 0,
- "is_submittable": 0,
- "issingle": 0,
- "istable": 0,
- "max_attachments": 0,
- "modified": "2020-04-07 21:07:39.476360",
+ "links": [],
+ "modified": "2022-08-03 12:21:52.062755",
  "modified_by": "Administrator",
  "module": "Integrations",
  "name": "OAuth Client",
- "name_case": "",
  "owner": "Administrator",
  "permissions": [
   {
-   "amend": 0,
-   "apply_user_permissions": 0,
-   "cancel": 0,
    "create": 1,
    "delete": 1,
    "email": 1,
    "export": 1,
-   "if_owner": 0,
-   "import": 0,
-   "permlevel": 0,
    "print": 1,
    "read": 1,
    "report": 1,
    "role": "System Manager",
-   "set_user_permissions": 0,
    "share": 1,
-   "submit": 0,
    "write": 1
   }
  ],
- "quick_entry": 0,
- "read_only": 0,
- "read_only_onload": 0,
- "show_name_in_global_search": 0,
  "sort_field": "modified",
  "sort_order": "DESC",
+ "states": [],
  "title_field": "app_name",
- "track_changes": 1,
- "track_seen": 0
+ "track_changes": 1
 }
\ No newline at end of file
diff --git a/frappe/integrations/doctype/oauth_client/oauth_client.py b/frappe/integrations/doctype/oauth_client/oauth_client.py
index 09f6e3aced..ab40467751 100644
--- a/frappe/integrations/doctype/oauth_client/oauth_client.py
+++ b/frappe/integrations/doctype/oauth_client/oauth_client.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2015, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
diff --git a/frappe/integrations/doctype/oauth_client/test_oauth_client.py b/frappe/integrations/doctype/oauth_client/test_oauth_client.py
index dd1b25239a..8fd732673e 100644
--- a/frappe/integrations/doctype/oauth_client/test_oauth_client.py
+++ b/frappe/integrations/doctype/oauth_client/test_oauth_client.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2015, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
 import unittest
diff --git a/frappe/integrations/doctype/oauth_client/test_records.json b/frappe/integrations/doctype/oauth_client/test_records.json
deleted file mode 100644
index 11e6338a87..0000000000
--- a/frappe/integrations/doctype/oauth_client/test_records.json
+++ /dev/null
@@ -1,15 +0,0 @@
-[
- {
-  "app_name": "_Test OAuth Client",
-  "client_secret": "test_client_secret", 
-  "default_redirect_uri": "http://localhost", 
-  "docstatus": 0, 
-  "doctype": "OAuth Client", 
-  "grant_type": "Authorization Code", 
-  "name": "test_client_id", 
-  "redirect_uris": "http://localhost", 
-  "response_type": "Code", 
-  "scopes": "all openid", 
-  "skip_authorization": 1
- }
-]
diff --git a/frappe/integrations/doctype/oauth_provider_settings/oauth_provider_settings.js b/frappe/integrations/doctype/oauth_provider_settings/oauth_provider_settings.js
index 6d7d071934..0071b4e977 100644
--- a/frappe/integrations/doctype/oauth_provider_settings/oauth_provider_settings.js
+++ b/frappe/integrations/doctype/oauth_provider_settings/oauth_provider_settings.js
@@ -1,8 +1,6 @@
 // Copyright (c) 2016, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('OAuth Provider Settings', {
-	refresh: function(frm) {
-
-	}
+frappe.ui.form.on("OAuth Provider Settings", {
+	refresh: function (frm) {},
 });
diff --git a/frappe/integrations/doctype/oauth_provider_settings/oauth_provider_settings.json b/frappe/integrations/doctype/oauth_provider_settings/oauth_provider_settings.json
index bf19eee6b1..219a87f2f4 100644
--- a/frappe/integrations/doctype/oauth_provider_settings/oauth_provider_settings.json
+++ b/frappe/integrations/doctype/oauth_provider_settings/oauth_provider_settings.json
@@ -1,90 +1,43 @@
 {
- "allow_copy": 0, 
- "allow_import": 0, 
- "allow_rename": 0, 
- "beta": 0, 
- "creation": "2016-09-03 11:42:42.575525", 
- "custom": 0, 
- "docstatus": 0, 
- "doctype": "DocType", 
- "document_type": "Setup", 
- "editable_grid": 1, 
- "engine": "InnoDB", 
+ "actions": [],
+ "creation": "2016-09-03 11:42:42.575525",
+ "doctype": "DocType",
+ "document_type": "Setup",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "skip_authorization"
+ ],
  "fields": [
   {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "skip_authorization", 
-   "fieldtype": "Select", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Skip Authorization", 
-   "length": 0, 
-   "no_copy": 0, 
-   "options": "Force\nAuto", 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
+   "fieldname": "skip_authorization",
+   "fieldtype": "Select",
+   "label": "Skip Authorization",
+   "options": "Force\nAuto"
   }
- ], 
- "hide_heading": 0, 
- "hide_toolbar": 0, 
- "idx": 0, 
- "image_view": 0, 
- "in_create": 0, 
-
- "is_submittable": 0, 
- "issingle": 1, 
- "istable": 0, 
- "max_attachments": 0, 
- "modified": "2016-12-29 14:40:30.718685", 
- "modified_by": "Administrator", 
- "module": "Integrations", 
- "name": "OAuth Provider Settings", 
- "name_case": "", 
- "owner": "Administrator", 
+ ],
+ "issingle": 1,
+ "links": [],
+ "modified": "2022-08-03 12:20:52.328415",
+ "modified_by": "Administrator",
+ "module": "Integrations",
+ "name": "OAuth Provider Settings",
+ "owner": "Administrator",
  "permissions": [
   {
-   "amend": 0, 
-   "apply_user_permissions": 0, 
-   "cancel": 0, 
-   "create": 1, 
-   "delete": 1, 
-   "email": 1, 
-   "export": 0, 
-   "if_owner": 0, 
-   "import": 0, 
-   "is_custom": 0, 
-   "permlevel": 0, 
-   "print": 1, 
-   "read": 1, 
-   "report": 0, 
-   "role": "System Manager", 
-   "set_user_permissions": 0, 
-   "share": 1, 
-   "submit": 0, 
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "print": 1,
+   "read": 1,
+   "role": "System Manager",
+   "share": 1,
    "write": 1
   }
- ], 
- "quick_entry": 1, 
- "read_only": 0, 
- "read_only_onload": 0, 
- "sort_field": "modified", 
- "sort_order": "DESC", 
- "track_changes": 1, 
- "track_seen": 0
+ ],
+ "quick_entry": 1,
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "states": [],
+ "track_changes": 1
 }
\ No newline at end of file
diff --git a/frappe/integrations/doctype/oauth_provider_settings/oauth_provider_settings.py b/frappe/integrations/doctype/oauth_provider_settings/oauth_provider_settings.py
index 2aefd591a1..5a918db587 100644
--- a/frappe/integrations/doctype/oauth_provider_settings/oauth_provider_settings.py
+++ b/frappe/integrations/doctype/oauth_provider_settings/oauth_provider_settings.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2015, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
@@ -15,7 +14,9 @@ def get_oauth_settings():
 	"""Returns oauth settings"""
 	out = frappe._dict(
 		{
-			"skip_authorization": frappe.db.get_value("OAuth Provider Settings", None, "skip_authorization")
+			"skip_authorization": frappe.db.get_single_value(
+				"OAuth Provider Settings", "skip_authorization"
+			)
 		}
 	)
 
diff --git a/frappe/integrations/doctype/oauth_scope/oauth_scope.py b/frappe/integrations/doctype/oauth_scope/oauth_scope.py
index a30d087cc0..1db49a3818 100644
--- a/frappe/integrations/doctype/oauth_scope/oauth_scope.py
+++ b/frappe/integrations/doctype/oauth_scope/oauth_scope.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2020, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
diff --git a/frappe/integrations/doctype/paypal_settings/__init__.py b/frappe/integrations/doctype/paypal_settings/__init__.py
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/frappe/integrations/doctype/paypal_settings/paypal_settings.js b/frappe/integrations/doctype/paypal_settings/paypal_settings.js
deleted file mode 100644
index 63480bc927..0000000000
--- a/frappe/integrations/doctype/paypal_settings/paypal_settings.js
+++ /dev/null
@@ -1,8 +0,0 @@
-// Copyright (c) 2016, Frappe Technologies and contributors
-// For license information, please see license.txt
-
-frappe.ui.form.on('PayPal Settings', {
-	refresh: function(frm) {
-
-	}
-});
diff --git a/frappe/integrations/doctype/paypal_settings/paypal_settings.json b/frappe/integrations/doctype/paypal_settings/paypal_settings.json
deleted file mode 100644
index 8d48496a4c..0000000000
--- a/frappe/integrations/doctype/paypal_settings/paypal_settings.json
+++ /dev/null
@@ -1,202 +0,0 @@
-{
- "allow_copy": 0, 
- "allow_import": 0, 
- "allow_rename": 0, 
- "beta": 0, 
- "creation": "2016-09-21 08:03:01.009852", 
- "custom": 0, 
- "docstatus": 0, 
- "doctype": "DocType", 
- "document_type": "System", 
- "editable_grid": 1, 
- "fields": [
-  {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "api_username", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "API Username", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "api_password", 
-   "fieldtype": "Password", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "API Password", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "signature", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Signature", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "description": "Check this if you are testing your payment using the Sandbox API", 
-   "fieldname": "paypal_sandbox", 
-   "fieldtype": "Check", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Use Sandbox", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "description": "Mention transaction completion page URL", 
-   "fieldname": "redirect_to", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Redirect To", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }
- ], 
- "hide_heading": 0, 
- "hide_toolbar": 0, 
- "idx": 0, 
- "image_view": 0, 
- "in_create": 1, 
-
- "is_submittable": 0, 
- "issingle": 1, 
- "istable": 0, 
- "max_attachments": 0, 
- "modified": "2016-12-29 14:40:31.574789", 
- "modified_by": "Administrator", 
- "module": "Integrations", 
- "name": "PayPal Settings", 
- "name_case": "", 
- "owner": "Administrator", 
- "permissions": [
-  {
-   "amend": 0, 
-   "apply_user_permissions": 0, 
-   "cancel": 0, 
-   "create": 1, 
-   "delete": 1, 
-   "email": 1, 
-   "export": 1, 
-   "if_owner": 0, 
-   "import": 0, 
-   "is_custom": 0, 
-   "permlevel": 0, 
-   "print": 1, 
-   "read": 1, 
-   "report": 0, 
-   "role": "System Manager", 
-   "set_user_permissions": 0, 
-   "share": 1, 
-   "submit": 0, 
-   "write": 1
-  }
- ], 
- "quick_entry": 0, 
- "read_only": 1, 
- "read_only_onload": 0, 
- "sort_field": "modified", 
- "sort_order": "DESC", 
- "track_changes": 1, 
- "track_seen": 0
-}
\ No newline at end of file
diff --git a/frappe/integrations/doctype/paypal_settings/paypal_settings.py b/frappe/integrations/doctype/paypal_settings/paypal_settings.py
deleted file mode 100644
index ab7512f403..0000000000
--- a/frappe/integrations/doctype/paypal_settings/paypal_settings.py
+++ /dev/null
@@ -1,508 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright (c) 2015, Frappe Technologies and contributors
-# License: MIT. See LICENSE
-
-"""
-# Integrating PayPal
-
-### 1. Validate Currency Support
-
-Example:
-
-	from frappe.integrations.utils import get_payment_gateway_controller
-
-	controller = get_payment_gateway_controller("PayPal")
-	controller().validate_transaction_currency(currency)
-
-### 2. Redirect for payment
-
-Example:
-
-	payment_details = {
-		"amount": 600,
-		"title": "Payment for bill : 111",
-		"description": "payment via cart",
-		"reference_doctype": "Payment Request",
-		"reference_docname": "PR0001",
-		"payer_email": "NuranVerkleij@example.com",
-		"payer_name": "Nuran Verkleij",
-		"order_id": "111",
-		"currency": "USD",
-		"payment_gateway": "Razorpay",
-		"subscription_details": {
-			"plan_id": "plan_12313", # if Required
-			"start_date": "2018-08-30",
-			"billing_period": "Month" #(Day, Week, SemiMonth, Month, Year),
-			"billing_frequency": 1,
-			"customer_notify": 1,
-			"upfront_amount": 1000
-		}
-	}
-
-	# redirect the user to this url
-	url = controller().get_payment_url(**payment_details)
-
-
-### 3. On Completion of Payment
-
-Write a method for `on_payment_authorized` in the reference doctype
-
-Example:
-
-	def on_payment_authorized(payment_status):
-		# your code to handle callback
-
-##### Note:
-
-payment_status - payment gateway will put payment status on callback.
-For paypal payment status parameter is one from: [Completed, Cancelled, Failed]
-
-
-More Details:
-
        For details on how to get your API credentials, follow this link: https://developer.paypal.com/docs/classic/api/apiCredentials/
        
-
-"""
-
-import json
-from urllib.parse import urlencode
-
-import pytz
-
-import frappe
-from frappe import _
-from frappe.integrations.utils import create_payment_gateway, create_request_log, make_post_request
-from frappe.model.document import Document
-from frappe.utils import call_hook_method, cint, get_datetime, get_url
-
-api_path = "/api/method/frappe.integrations.doctype.paypal_settings.paypal_settings"
-
-
-class PayPalSettings(Document):
-	supported_currencies = [
-		"AUD",
-		"BRL",
-		"CAD",
-		"CZK",
-		"DKK",
-		"EUR",
-		"HKD",
-		"HUF",
-		"ILS",
-		"JPY",
-		"MYR",
-		"MXN",
-		"TWD",
-		"NZD",
-		"NOK",
-		"PHP",
-		"PLN",
-		"GBP",
-		"RUB",
-		"SGD",
-		"SEK",
-		"CHF",
-		"THB",
-		"TRY",
-		"USD",
-	]
-
-	def __setup__(self):
-		setattr(self, "use_sandbox", 0)
-
-	def setup_sandbox_env(self, token):
-		data = json.loads(frappe.db.get_value("Integration Request", token, "data"))
-		setattr(self, "use_sandbox", cint(frappe._dict(data).use_sandbox) or 0)
-
-	def validate(self):
-		create_payment_gateway("PayPal")
-		call_hook_method("payment_gateway_enabled", gateway="PayPal")
-		if not self.flags.ignore_mandatory:
-			self.validate_paypal_credentails()
-
-	def on_update(self):
-		pass
-
-	def validate_transaction_currency(self, currency):
-		if currency not in self.supported_currencies:
-			frappe.throw(
-				_(
-					"Please select another payment method. PayPal does not support transactions in currency '{0}'"
-				).format(currency)
-			)
-
-	def get_paypal_params_and_url(self):
-		params = {
-			"USER": self.api_username,
-			"PWD": self.get_password(fieldname="api_password", raise_exception=False),
-			"SIGNATURE": self.signature,
-			"VERSION": "98",
-			"METHOD": "GetPalDetails",
-		}
-
-		if hasattr(self, "use_sandbox") and self.use_sandbox:
-			params.update(
-				{
-					"USER": frappe.conf.sandbox_api_username,
-					"PWD": frappe.conf.sandbox_api_password,
-					"SIGNATURE": frappe.conf.sandbox_signature,
-				}
-			)
-
-		api_url = (
-			"https://api-3t.sandbox.paypal.com/nvp"
-			if (self.paypal_sandbox or self.use_sandbox)
-			else "https://api-3t.paypal.com/nvp"
-		)
-
-		return params, api_url
-
-	def validate_paypal_credentails(self):
-		params, url = self.get_paypal_params_and_url()
-		params = urlencode(params)
-
-		try:
-			res = make_post_request(url=url, data=params.encode("utf-8"))
-
-			if res["ACK"][0] == "Failure":
-				raise Exception
-
-		except Exception:
-			frappe.throw(_("Invalid payment gateway credentials"))
-
-	def get_payment_url(self, **kwargs):
-		setattr(self, "use_sandbox", cint(kwargs.get("use_sandbox", 0)))
-
-		response = self.execute_set_express_checkout(**kwargs)
-
-		if self.paypal_sandbox or self.use_sandbox:
-			return_url = "https://www.sandbox.paypal.com/cgi-bin/webscr?cmd=_express-checkout&token={0}"
-		else:
-			return_url = "https://www.paypal.com/cgi-bin/webscr?cmd=_express-checkout&token={0}"
-
-		kwargs.update(
-			{"token": response.get("TOKEN")[0], "correlation_id": response.get("CORRELATIONID")[0]}
-		)
-		self.integration_request = create_request_log(
-			kwargs, "Remote", "PayPal", response.get("TOKEN")[0]
-		)
-
-		return return_url.format(kwargs["token"])
-
-	def execute_set_express_checkout(self, **kwargs):
-		params, url = self.get_paypal_params_and_url()
-
-		params.update(
-			{
-				"METHOD": "SetExpressCheckout",
-				"returnUrl": get_url("{0}.get_express_checkout_details".format(api_path)),
-				"cancelUrl": get_url("/payment-cancel"),
-				"PAYMENTREQUEST_0_PAYMENTACTION": "SALE",
-				"PAYMENTREQUEST_0_AMT": kwargs["amount"],
-				"PAYMENTREQUEST_0_CURRENCYCODE": kwargs["currency"].upper(),
-			}
-		)
-
-		if kwargs.get("subscription_details"):
-			self.configure_recurring_payments(params, kwargs)
-
-		params = urlencode(params)
-		response = make_post_request(url, data=params.encode("utf-8"))
-
-		if response.get("ACK")[0] != "Success":
-			frappe.throw(_("Looks like something is wrong with this site's Paypal configuration."))
-
-		return response
-
-	def configure_recurring_payments(self, params, kwargs):
-		# removing the params as we have to setup rucurring payments
-		for param in (
-			"PAYMENTREQUEST_0_PAYMENTACTION",
-			"PAYMENTREQUEST_0_AMT",
-			"PAYMENTREQUEST_0_CURRENCYCODE",
-		):
-			del params[param]
-
-		params.update(
-			{
-				"L_BILLINGTYPE0": "RecurringPayments",  # The type of billing agreement
-				"L_BILLINGAGREEMENTDESCRIPTION0": kwargs["description"],
-			}
-		)
-
-
-def get_paypal_and_transaction_details(token):
-	doc = frappe.get_doc("PayPal Settings")
-	doc.setup_sandbox_env(token)
-	params, url = doc.get_paypal_params_and_url()
-
-	integration_request = frappe.get_doc("Integration Request", token)
-	data = json.loads(integration_request.data)
-
-	return data, params, url
-
-
-def setup_redirect(data, redirect_url, custom_redirect_to=None, redirect=True):
-	redirect_to = data.get("redirect_to") or None
-	redirect_message = data.get("redirect_message") or None
-
-	if custom_redirect_to:
-		redirect_to = custom_redirect_to
-
-	if redirect_to:
-		redirect_url += "&" + urlencode({"redirect_to": redirect_to})
-	if redirect_message:
-		redirect_url += "&" + urlencode({"redirect_message": redirect_message})
-
-	# this is done so that functions called via hooks can update flags.redirect_to
-	if redirect:
-		frappe.local.response["type"] = "redirect"
-		frappe.local.response["location"] = get_url(redirect_url)
-
-
-@frappe.whitelist(allow_guest=True, xss_safe=True)
-def get_express_checkout_details(token):
-	try:
-		doc = frappe.get_doc("PayPal Settings")
-		doc.setup_sandbox_env(token)
-
-		params, url = doc.get_paypal_params_and_url()
-		params.update({"METHOD": "GetExpressCheckoutDetails", "TOKEN": token})
-
-		response = make_post_request(url, data=params)
-
-		if response.get("ACK")[0] != "Success":
-			frappe.respond_as_web_page(
-				_("Something went wrong"),
-				_(
-					"Looks like something went wrong during the transaction. Since we haven't confirmed the payment, Paypal will automatically refund you this amount. If it doesn't, please send us an email and mention the Correlation ID: {0}."
-				).format(response.get("CORRELATIONID", [None])[0]),
-				indicator_color="red",
-				http_status_code=frappe.ValidationError.http_status_code,
-			)
-
-			return
-
-		doc = frappe.get_doc("Integration Request", token)
-		update_integration_request_status(
-			token,
-			{"payerid": response.get("PAYERID")[0], "payer_email": response.get("EMAIL")[0]},
-			"Authorized",
-			doc=doc,
-		)
-
-		frappe.local.response["type"] = "redirect"
-		frappe.local.response["location"] = get_redirect_uri(doc, token, response.get("PAYERID")[0])
-
-	except Exception:
-		frappe.log_error(frappe.get_traceback())
-
-
-@frappe.whitelist(allow_guest=True, xss_safe=True)
-def confirm_payment(token):
-	try:
-		custom_redirect_to = None
-		data, params, url = get_paypal_and_transaction_details(token)
-
-		params.update(
-			{
-				"METHOD": "DoExpressCheckoutPayment",
-				"PAYERID": data.get("payerid"),
-				"TOKEN": token,
-				"PAYMENTREQUEST_0_PAYMENTACTION": "SALE",
-				"PAYMENTREQUEST_0_AMT": data.get("amount"),
-				"PAYMENTREQUEST_0_CURRENCYCODE": data.get("currency").upper(),
-			}
-		)
-
-		response = make_post_request(url, data=params)
-
-		if response.get("ACK")[0] == "Success":
-			update_integration_request_status(
-				token,
-				{
-					"transaction_id": response.get("PAYMENTINFO_0_TRANSACTIONID")[0],
-					"correlation_id": response.get("CORRELATIONID")[0],
-				},
-				"Completed",
-			)
-
-			if data.get("reference_doctype") and data.get("reference_docname"):
-				custom_redirect_to = frappe.get_doc(
-					data.get("reference_doctype"), data.get("reference_docname")
-				).run_method("on_payment_authorized", "Completed")
-				frappe.db.commit()
-
-			redirect_url = "/integrations/payment-success?doctype={0}&docname={1}".format(
-				data.get("reference_doctype"), data.get("reference_docname")
-			)
-		else:
-			redirect_url = "/integrations/payment-failed"
-
-		setup_redirect(data, redirect_url, custom_redirect_to)
-
-	except Exception:
-		frappe.log_error(frappe.get_traceback())
-
-
-@frappe.whitelist(allow_guest=True, xss_safe=True)
-def create_recurring_profile(token, payerid):
-	try:
-		custom_redirect_to = None
-		updating = False
-		data, params, url = get_paypal_and_transaction_details(token)
-
-		addons = data.get("addons")
-		subscription_details = data.get("subscription_details")
-
-		if data.get("subscription_id"):
-			if addons:
-				updating = True
-			manage_recurring_payment_profile_status(data["subscription_id"], "Cancel", params, url)
-
-		params.update(
-			{
-				"METHOD": "CreateRecurringPaymentsProfile",
-				"PAYERID": payerid,
-				"TOKEN": token,
-				"DESC": data.get("description"),
-				"BILLINGPERIOD": subscription_details.get("billing_period"),
-				"BILLINGFREQUENCY": subscription_details.get("billing_frequency"),
-				"AMT": data.get("amount")
-				if data.get("subscription_amount") == data.get("amount")
-				else data.get("subscription_amount"),
-				"CURRENCYCODE": data.get("currency").upper(),
-				"INITAMT": data.get("upfront_amount"),
-			}
-		)
-
-		status_changed_to = "Completed" if data.get("starting_immediately") or updating else "Verified"
-
-		starts_at = get_datetime(subscription_details.get("start_date")) or frappe.utils.now_datetime()
-		starts_at = starts_at.replace(tzinfo=pytz.timezone(frappe.utils.get_time_zone())).astimezone(
-			pytz.utc
-		)
-
-		# "PROFILESTARTDATE": datetime.utcfromtimestamp(get_timestamp(starts_at)).isoformat()
-		params.update({"PROFILESTARTDATE": starts_at.isoformat()})
-
-		response = make_post_request(url, data=params)
-
-		if response.get("ACK")[0] == "Success":
-			update_integration_request_status(
-				token,
-				{
-					"profile_id": response.get("PROFILEID")[0],
-				},
-				"Completed",
-			)
-
-			if data.get("reference_doctype") and data.get("reference_docname"):
-				data["subscription_id"] = response.get("PROFILEID")[0]
-
-				frappe.flags.data = data
-				custom_redirect_to = frappe.get_doc(
-					data.get("reference_doctype"), data.get("reference_docname")
-				).run_method("on_payment_authorized", status_changed_to)
-				frappe.db.commit()
-
-			redirect_url = "/integrations/payment-success?doctype={0}&docname={1}".format(
-				data.get("reference_doctype"), data.get("reference_docname")
-			)
-		else:
-			redirect_url = "/integrations/payment-failed"
-
-		setup_redirect(data, redirect_url, custom_redirect_to)
-
-	except Exception:
-		frappe.log_error(frappe.get_traceback())
-
-
-def update_integration_request_status(token, data, status, error=False, doc=None):
-	if not doc:
-		doc = frappe.get_doc("Integration Request", token)
-
-	doc.update_status(data, status)
-
-
-def get_redirect_uri(doc, token, payerid):
-	data = json.loads(doc.data)
-
-	if data.get("subscription_details") or data.get("subscription_id"):
-		return get_url(
-			"{0}.create_recurring_profile?token={1}&payerid={2}".format(api_path, token, payerid)
-		)
-	else:
-		return get_url("{0}.confirm_payment?token={1}".format(api_path, token))
-
-
-def manage_recurring_payment_profile_status(profile_id, action, args, url):
-	args.update(
-		{"METHOD": "ManageRecurringPaymentsProfileStatus", "PROFILEID": profile_id, "ACTION": action}
-	)
-
-	response = make_post_request(url, data=args)
-
-	# error code 11556 indicates profile is not in active state(or already cancelled)
-	# thus could not cancel the subscription.
-	# thus raise an exception only if the error code is not equal to 11556
-
-	if response.get("ACK")[0] != "Success" and response.get("L_ERRORCODE0", [])[0] != "11556":
-		frappe.throw(_("Failed while amending subscription"))
-
-
-@frappe.whitelist(allow_guest=True)
-def ipn_handler():
-	try:
-		data = frappe.local.form_dict
-
-		validate_ipn_request(data)
-
-		data.update({"payment_gateway": "PayPal"})
-
-		doc = frappe.get_doc(
-			{
-				"data": json.dumps(frappe.local.form_dict),
-				"doctype": "Integration Request",
-				"integration_type": "Subscription Notification",
-				"status": "Queued",
-			}
-		).insert(ignore_permissions=True)
-		frappe.db.commit()
-
-		frappe.enqueue(
-			method="frappe.integrations.doctype.paypal_settings.paypal_settings.handle_subscription_notification",
-			queue="long",
-			timeout=600,
-			is_async=True,
-			**{"doctype": "Integration Request", "docname": doc.name}
-		)
-
-	except frappe.InvalidStatusError:
-		pass
-	except Exception as e:
-		frappe.log(frappe.log_error(title=e))
-
-
-def validate_ipn_request(data):
-	def _throw():
-		frappe.throw(_("In Valid Request"), exc=frappe.InvalidStatusError)
-
-	if not data.get("recurring_payment_id"):
-		_throw()
-
-	doc = frappe.get_doc("PayPal Settings")
-	params, url = doc.get_paypal_params_and_url()
-
-	params.update(
-		{"METHOD": "GetRecurringPaymentsProfileDetails", "PROFILEID": data.get("recurring_payment_id")}
-	)
-
-	params = urlencode(params)
-	res = make_post_request(url=url, data=params.encode("utf-8"))
-
-	if res["ACK"][0] != "Success":
-		_throw()
-
-
-def handle_subscription_notification(doctype, docname):
-	call_hook_method("handle_subscription_notification", doctype=doctype, docname=docname)
diff --git a/frappe/integrations/doctype/paytm_settings/__init__.py b/frappe/integrations/doctype/paytm_settings/__init__.py
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/frappe/integrations/doctype/paytm_settings/paytm_settings.js b/frappe/integrations/doctype/paytm_settings/paytm_settings.js
deleted file mode 100644
index fe2ee7c952..0000000000
--- a/frappe/integrations/doctype/paytm_settings/paytm_settings.js
+++ /dev/null
@@ -1,8 +0,0 @@
-// Copyright (c) 2020, Frappe Technologies and contributors
-// For license information, please see license.txt
-
-frappe.ui.form.on('Paytm Settings', {
-	refresh: function(frm) {
-		frm.dashboard.set_headline(__("For more information, {0}.", [`${__('Click here')}`]));
-	}
-});
diff --git a/frappe/integrations/doctype/paytm_settings/paytm_settings.json b/frappe/integrations/doctype/paytm_settings/paytm_settings.json
deleted file mode 100644
index 93fbd0df09..0000000000
--- a/frappe/integrations/doctype/paytm_settings/paytm_settings.json
+++ /dev/null
@@ -1,89 +0,0 @@
-{
- "actions": [],
- "creation": "2020-04-02 00:11:22.846697",
- "doctype": "DocType",
- "editable_grid": 1,
- "engine": "InnoDB",
- "field_order": [
-  "merchant_id",
-  "merchant_key",
-  "staging",
-  "column_break_4",
-  "industry_type_id",
-  "website"
- ],
- "fields": [
-  {
-   "fieldname": "merchant_id",
-   "fieldtype": "Data",
-   "in_list_view": 1,
-   "label": "Merchant ID",
-   "reqd": 1,
-   "show_days": 1,
-   "show_seconds": 1
-  },
-  {
-   "fieldname": "merchant_key",
-   "fieldtype": "Password",
-   "in_list_view": 1,
-   "label": "Merchant Key",
-   "reqd": 1,
-   "show_days": 1,
-   "show_seconds": 1
-  },
-  {
-   "default": "0",
-   "fieldname": "staging",
-   "fieldtype": "Check",
-   "label": "Staging",
-   "show_days": 1,
-   "show_seconds": 1
-  },
-  {
-   "depends_on": "eval: !doc.staging",
-   "fieldname": "website",
-   "fieldtype": "Data",
-   "label": "Website",
-   "mandatory_depends_on": "eval: !doc.staging",
-   "show_days": 1,
-   "show_seconds": 1
-  },
-  {
-   "fieldname": "column_break_4",
-   "fieldtype": "Column Break",
-   "show_days": 1,
-   "show_seconds": 1
-  },
-  {
-   "depends_on": "eval: !doc.staging",
-   "fieldname": "industry_type_id",
-   "fieldtype": "Data",
-   "label": "Industry Type ID",
-   "mandatory_depends_on": "eval: !doc.staging",
-   "show_days": 1,
-   "show_seconds": 1
-  }
- ],
- "issingle": 1,
- "links": [],
- "modified": "2020-06-08 13:36:09.703143",
- "modified_by": "Administrator",
- "module": "Integrations",
- "name": "Paytm Settings",
- "owner": "Administrator",
- "permissions": [
-  {
-   "create": 1,
-   "delete": 1,
-   "email": 1,
-   "print": 1,
-   "read": 1,
-   "role": "System Manager",
-   "share": 1,
-   "write": 1
-  }
- ],
- "sort_field": "modified",
- "sort_order": "DESC",
- "track_changes": 1
-}
\ No newline at end of file
diff --git a/frappe/integrations/doctype/paytm_settings/paytm_settings.py b/frappe/integrations/doctype/paytm_settings/paytm_settings.py
deleted file mode 100644
index 0888fd35b7..0000000000
--- a/frappe/integrations/doctype/paytm_settings/paytm_settings.py
+++ /dev/null
@@ -1,182 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright (c) 2020, Frappe Technologies and contributors
-# License: MIT. See LICENSE
-
-import json
-from urllib.parse import urlencode
-
-import requests
-from paytmchecksum import generateSignature, verifySignature
-
-import frappe
-from frappe import _
-from frappe.integrations.utils import create_payment_gateway, create_request_log
-from frappe.model.document import Document
-from frappe.utils import call_hook_method, cint, cstr, flt, get_request_site_address, get_url
-from frappe.utils.password import get_decrypted_password
-
-
-class PaytmSettings(Document):
-	supported_currencies = ["INR"]
-
-	def validate(self):
-		create_payment_gateway("Paytm")
-		call_hook_method("payment_gateway_enabled", gateway="Paytm")
-
-	def validate_transaction_currency(self, currency):
-		if currency not in self.supported_currencies:
-			frappe.throw(
-				_(
-					"Please select another payment method. Paytm does not support transactions in currency '{0}'"
-				).format(currency)
-			)
-
-	def get_payment_url(self, **kwargs):
-		"""Return payment url with several params"""
-		# create unique order id by making it equal to the integration request
-		integration_request = create_request_log(kwargs, "Host", "Paytm")
-		kwargs.update(dict(order_id=integration_request.name))
-
-		return get_url("./integrations/paytm_checkout?{0}".format(urlencode(kwargs)))
-
-
-def get_paytm_config():
-	"""Returns paytm config"""
-
-	paytm_config = frappe.db.get_singles_dict("Paytm Settings")
-	paytm_config.update(
-		dict(merchant_key=get_decrypted_password("Paytm Settings", "Paytm Settings", "merchant_key"))
-	)
-
-	if cint(paytm_config.staging):
-		paytm_config.update(
-			dict(
-				website="WEBSTAGING",
-				url="https://securegw-stage.paytm.in/order/process",
-				transaction_status_url="https://securegw-stage.paytm.in/order/status",
-				industry_type_id="RETAIL",
-			)
-		)
-	else:
-		paytm_config.update(
-			dict(
-				url="https://securegw.paytm.in/order/process",
-				transaction_status_url="https://securegw.paytm.in/order/status",
-			)
-		)
-	return paytm_config
-
-
-def get_paytm_params(payment_details, order_id, paytm_config):
-
-	# initialize a dictionary
-	paytm_params = dict()
-
-	redirect_uri = (
-		get_request_site_address(True)
-		+ "/api/method/frappe.integrations.doctype.paytm_settings.paytm_settings.verify_transaction"
-	)
-
-	paytm_params.update(
-		{
-			"MID": paytm_config.merchant_id,
-			"WEBSITE": paytm_config.website,
-			"INDUSTRY_TYPE_ID": paytm_config.industry_type_id,
-			"CHANNEL_ID": "WEB",
-			"ORDER_ID": order_id,
-			"CUST_ID": payment_details["payer_email"],
-			"EMAIL": payment_details["payer_email"],
-			"TXN_AMOUNT": cstr(flt(payment_details["amount"], 2)),
-			"CALLBACK_URL": redirect_uri,
-		}
-	)
-
-	checksum = generateSignature(paytm_params, paytm_config.merchant_key)
-
-	paytm_params.update({"CHECKSUMHASH": checksum})
-
-	return paytm_params
-
-
-@frappe.whitelist(allow_guest=True)
-def verify_transaction(**paytm_params):
-	"""Verify checksum for received data in the callback and then verify the transaction"""
-	paytm_config = get_paytm_config()
-	is_valid_checksum = False
-
-	paytm_params.pop("cmd", None)
-	paytm_checksum = paytm_params.pop("CHECKSUMHASH", None)
-
-	if paytm_params and paytm_config and paytm_checksum:
-		# Verify checksum
-		is_valid_checksum = verifySignature(paytm_params, paytm_config.merchant_key, paytm_checksum)
-
-	if is_valid_checksum and paytm_params.get("RESPCODE") == "01":
-		verify_transaction_status(paytm_config, paytm_params["ORDERID"])
-	else:
-		frappe.respond_as_web_page(
-			"Payment Failed",
-			"Transaction failed to complete. In case of any deductions, deducted amount will get refunded to your account.",
-			http_status_code=401,
-			indicator_color="red",
-		)
-		frappe.log_error(
-			"Order unsuccessful. Failed Response:" + cstr(paytm_params), "Paytm Payment Failed"
-		)
-
-
-def verify_transaction_status(paytm_config, order_id):
-	"""Verify transaction completion after checksum has been verified"""
-	paytm_params = dict(MID=paytm_config.merchant_id, ORDERID=order_id)
-
-	checksum = generateSignature(paytm_params, paytm_config.merchant_key)
-	paytm_params["CHECKSUMHASH"] = checksum
-
-	post_data = json.dumps(paytm_params)
-	url = paytm_config.transaction_status_url
-
-	response = requests.post(url, data=post_data, headers={"Content-type": "application/json"}).json()
-	finalize_request(order_id, response)
-
-
-def finalize_request(order_id, transaction_response):
-	request = frappe.get_doc("Integration Request", order_id)
-	transaction_data = frappe._dict(json.loads(request.data))
-	redirect_to = transaction_data.get("redirect_to") or None
-	redirect_message = transaction_data.get("redirect_message") or None
-
-	if transaction_response["STATUS"] == "TXN_SUCCESS":
-		if transaction_data.reference_doctype and transaction_data.reference_docname:
-			custom_redirect_to = None
-			try:
-				custom_redirect_to = frappe.get_doc(
-					transaction_data.reference_doctype, transaction_data.reference_docname
-				).run_method("on_payment_authorized", "Completed")
-				request.db_set("status", "Completed")
-			except Exception:
-				request.db_set("status", "Failed")
-				frappe.log_error(frappe.get_traceback())
-
-			if custom_redirect_to:
-				redirect_to = custom_redirect_to
-
-			redirect_url = "/integrations/payment-success"
-	else:
-		request.db_set("status", "Failed")
-		redirect_url = "/integrations/payment-failed"
-
-	if redirect_to:
-		redirect_url += "?" + urlencode({"redirect_to": redirect_to})
-	if redirect_message:
-		redirect_url += "&" + urlencode({"redirect_message": redirect_message})
-
-	frappe.local.response["type"] = "redirect"
-	frappe.local.response["location"] = redirect_url
-
-
-def get_gateway_controller(doctype, docname):
-	reference_doc = frappe.get_doc(doctype, docname)
-	gateway_controller = frappe.db.get_value(
-		"Payment Gateway", reference_doc.payment_gateway, "gateway_controller"
-	)
-	return gateway_controller
diff --git a/frappe/integrations/doctype/paytm_settings/test_paytm_settings.py b/frappe/integrations/doctype/paytm_settings/test_paytm_settings.py
deleted file mode 100644
index d9e72e344c..0000000000
--- a/frappe/integrations/doctype/paytm_settings/test_paytm_settings.py
+++ /dev/null
@@ -1,9 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright (c) 2020, Frappe Technologies and Contributors
-# License: MIT. See LICENSE
-# import frappe
-import unittest
-
-
-class TestPaytmSettings(unittest.TestCase):
-	pass
diff --git a/frappe/integrations/doctype/query_parameters/query_parameters.py b/frappe/integrations/doctype/query_parameters/query_parameters.py
index 09f039a764..b501c0c3e8 100644
--- a/frappe/integrations/doctype/query_parameters/query_parameters.py
+++ b/frappe/integrations/doctype/query_parameters/query_parameters.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2020, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
diff --git a/frappe/integrations/doctype/razorpay_settings/__init__.py b/frappe/integrations/doctype/razorpay_settings/__init__.py
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/frappe/integrations/doctype/razorpay_settings/razorpay_settings.js b/frappe/integrations/doctype/razorpay_settings/razorpay_settings.js
deleted file mode 100644
index 6915c5c582..0000000000
--- a/frappe/integrations/doctype/razorpay_settings/razorpay_settings.js
+++ /dev/null
@@ -1,8 +0,0 @@
-// Copyright (c) 2016, Frappe Technologies and contributors
-// For license information, please see license.txt
-
-frappe.ui.form.on('Razorpay Settings', {
-	refresh: function(frm) {
-
-	}
-});
\ No newline at end of file
diff --git a/frappe/integrations/doctype/razorpay_settings/razorpay_settings.json b/frappe/integrations/doctype/razorpay_settings/razorpay_settings.json
deleted file mode 100644
index 3fdea79e2b..0000000000
--- a/frappe/integrations/doctype/razorpay_settings/razorpay_settings.json
+++ /dev/null
@@ -1,145 +0,0 @@
-{
- "allow_copy": 0, 
- "allow_import": 0, 
- "allow_rename": 0, 
- "beta": 0, 
- "creation": "2016-09-20 03:44:03.799402", 
- "custom": 0, 
- "docstatus": 0, 
- "doctype": "DocType", 
- "document_type": "System", 
- "editable_grid": 1, 
- "fields": [
-  {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "api_key", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "API Key", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "api_secret", 
-   "fieldtype": "Password", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "API Secret", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "description": "Mention transaction completion page URL", 
-   "fieldname": "redirect_to", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Redirect To", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }
- ], 
- "hide_heading": 0, 
- "hide_toolbar": 0, 
- "idx": 0, 
- "image_view": 0, 
- "in_create": 1, 
-
- "is_submittable": 0, 
- "issingle": 1, 
- "istable": 0, 
- "max_attachments": 0, 
- "modified": "2016-12-29 14:40:31.658270", 
- "modified_by": "Administrator", 
- "module": "Integrations", 
- "name": "Razorpay Settings", 
- "name_case": "", 
- "owner": "Administrator", 
- "permissions": [
-  {
-   "amend": 0, 
-   "apply_user_permissions": 0, 
-   "cancel": 0, 
-   "create": 1, 
-   "delete": 1, 
-   "email": 1, 
-   "export": 1, 
-   "if_owner": 0, 
-   "import": 0, 
-   "is_custom": 0, 
-   "permlevel": 0, 
-   "print": 1, 
-   "read": 1, 
-   "report": 0, 
-   "role": "System Manager", 
-   "set_user_permissions": 0, 
-   "share": 1, 
-   "submit": 0, 
-   "write": 1
-  }
- ], 
- "quick_entry": 0, 
- "read_only": 1, 
- "read_only_onload": 0, 
- "sort_field": "modified", 
- "sort_order": "DESC", 
- "track_changes": 1, 
- "track_seen": 0
-}
\ No newline at end of file
diff --git a/frappe/integrations/doctype/razorpay_settings/razorpay_settings.py b/frappe/integrations/doctype/razorpay_settings/razorpay_settings.py
deleted file mode 100644
index c4ffb74325..0000000000
--- a/frappe/integrations/doctype/razorpay_settings/razorpay_settings.py
+++ /dev/null
@@ -1,536 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright (c) 2015, Frappe Technologies and contributors
-# License: MIT. See LICENSE
-
-"""
-# Integrating RazorPay
-
-### Validate Currency
-
-Example:
-
-	from frappe.integrations.utils import get_payment_gateway_controller
-
-	controller = get_payment_gateway_controller("Razorpay")
-	controller().validate_transaction_currency(currency)
-
-### 2. Redirect for payment
-
-Example:
-
-	payment_details = {
-		"amount": 600,
-		"title": "Payment for bill : 111",
-		"description": "payment via cart",
-		"reference_doctype": "Payment Request",
-		"reference_docname": "PR0001",
-		"payer_email": "NuranVerkleij@example.com",
-		"payer_name": "Nuran Verkleij",
-		"order_id": "111",
-		"currency": "INR",
-		"payment_gateway": "Razorpay",
-		"subscription_details": {
-			"plan_id": "plan_12313", # if Required
-			"start_date": "2018-08-30",
-			"billing_period": "Month" #(Day, Week, Month, Year),
-			"billing_frequency": 1,
-			"customer_notify": 1,
-			"upfront_amount": 1000
-		}
-	}
-
-	# Redirect the user to this url
-	url = controller().get_payment_url(**payment_details)
-
-
-### 3. On Completion of Payment
-
-Write a method for `on_payment_authorized` in the reference doctype
-
-Example:
-
-	def on_payment_authorized(payment_status):
-		# this method will be called when payment is complete
-
-
-##### Notes:
-
-payment_status - payment gateway will put payment status on callback.
-For razorpay payment status is Authorized
-
-"""
-
-import hashlib
-import hmac
-import json
-from urllib.parse import urlencode
-
-import razorpay
-
-import frappe
-from frappe import _
-from frappe.integrations.utils import (
-	create_payment_gateway,
-	create_request_log,
-	make_get_request,
-	make_post_request,
-)
-from frappe.model.document import Document
-from frappe.utils import call_hook_method, cint, get_timestamp, get_url
-
-
-class RazorpaySettings(Document):
-	supported_currencies = ["INR"]
-
-	def init_client(self):
-		if self.api_key:
-			secret = self.get_password(fieldname="api_secret", raise_exception=False)
-			self.client = razorpay.Client(auth=(self.api_key, secret))
-
-	def validate(self):
-		create_payment_gateway("Razorpay")
-		call_hook_method("payment_gateway_enabled", gateway="Razorpay")
-		if not self.flags.ignore_mandatory:
-			self.validate_razorpay_credentails()
-
-	def validate_razorpay_credentails(self):
-		if self.api_key and self.api_secret:
-			try:
-				make_get_request(
-					url="https://api.razorpay.com/v1/payments",
-					auth=(self.api_key, self.get_password(fieldname="api_secret", raise_exception=False)),
-				)
-			except Exception:
-				frappe.throw(_("Seems API Key or API Secret is wrong !!!"))
-
-	def validate_transaction_currency(self, currency):
-		if currency not in self.supported_currencies:
-			frappe.throw(
-				_(
-					"Please select another payment method. Razorpay does not support transactions in currency '{0}'"
-				).format(currency)
-			)
-
-	def setup_addon(self, settings, **kwargs):
-		"""
-		Addon template:
-		{
-		        "item": {
-		                "name": row.upgrade_type,
-		                "amount": row.amount,
-		                "currency": currency,
-		                "description": "add-on description"
-		        },
-		        "quantity": 1 (The total amount is calculated as item.amount * quantity)
-		}
-		"""
-		url = "https://api.razorpay.com/v1/subscriptions/{0}/addons".format(
-			kwargs.get("subscription_id")
-		)
-
-		try:
-			if not frappe.conf.converted_rupee_to_paisa:
-				convert_rupee_to_paisa(**kwargs)
-
-			for addon in kwargs.get("addons"):
-				resp = make_post_request(
-					url,
-					auth=(settings.api_key, settings.api_secret),
-					data=json.dumps(addon),
-					headers={"content-type": "application/json"},
-				)
-				if not resp.get("id"):
-					frappe.log_error(str(resp), "Razorpay Failed while creating subscription")
-		except:
-			frappe.log_error(frappe.get_traceback())
-			# failed
-			pass
-
-	def setup_subscription(self, settings, **kwargs):
-		start_date = (
-			get_timestamp(kwargs.get("subscription_details").get("start_date"))
-			if kwargs.get("subscription_details").get("start_date")
-			else None
-		)
-
-		subscription_details = {
-			"plan_id": kwargs.get("subscription_details").get("plan_id"),
-			"total_count": kwargs.get("subscription_details").get("billing_frequency"),
-			"customer_notify": kwargs.get("subscription_details").get("customer_notify"),
-		}
-
-		if start_date:
-			subscription_details["start_at"] = cint(start_date)
-
-		if kwargs.get("addons"):
-			convert_rupee_to_paisa(**kwargs)
-			subscription_details.update({"addons": kwargs.get("addons")})
-
-		try:
-			resp = make_post_request(
-				"https://api.razorpay.com/v1/subscriptions",
-				auth=(settings.api_key, settings.api_secret),
-				data=json.dumps(subscription_details),
-				headers={"content-type": "application/json"},
-			)
-
-			if resp.get("status") == "created":
-				kwargs["subscription_id"] = resp.get("id")
-				frappe.flags.status = "created"
-				return kwargs
-			else:
-				frappe.log_error(str(resp), "Razorpay Failed while creating subscription")
-
-		except:
-			frappe.log_error(frappe.get_traceback())
-			# failed
-			pass
-
-	def prepare_subscription_details(self, settings, **kwargs):
-		if not kwargs.get("subscription_id"):
-			kwargs = self.setup_subscription(settings, **kwargs)
-
-		if frappe.flags.status != "created":
-			kwargs["subscription_id"] = None
-
-		return kwargs
-
-	def get_payment_url(self, **kwargs):
-		integration_request = create_request_log(kwargs, "Host", "Razorpay")
-		return get_url("./integrations/razorpay_checkout?token={0}".format(integration_request.name))
-
-	def create_order(self, **kwargs):
-		# Creating Orders https://razorpay.com/docs/api/orders/
-
-		# convert rupees to paisa
-		kwargs["amount"] *= 100
-
-		# Create integration log
-		integration_request = create_request_log(kwargs, "Host", "Razorpay")
-
-		# Setup payment options
-		payment_options = {
-			"amount": kwargs.get("amount"),
-			"currency": kwargs.get("currency", "INR"),
-			"receipt": kwargs.get("receipt"),
-			"payment_capture": kwargs.get("payment_capture"),
-		}
-		if self.api_key and self.api_secret:
-			try:
-				order = make_post_request(
-					"https://api.razorpay.com/v1/orders",
-					auth=(self.api_key, self.get_password(fieldname="api_secret", raise_exception=False)),
-					data=payment_options,
-				)
-				order["integration_request"] = integration_request.name
-				return order  # Order returned to be consumed by razorpay.js
-			except Exception:
-				frappe.log(frappe.get_traceback())
-				frappe.throw(_("Could not create razorpay order"))
-
-	def create_request(self, data):
-		self.data = frappe._dict(data)
-
-		try:
-			self.integration_request = frappe.get_doc("Integration Request", self.data.token)
-			self.integration_request.update_status(self.data, "Queued")
-			return self.authorize_payment()
-
-		except Exception:
-			frappe.log_error(frappe.get_traceback())
-			return {
-				"redirect_to": frappe.redirect_to_message(
-					_("Server Error"),
-					_(
-						"Seems issue with server's razorpay config. Don't worry, in case of failure amount will get refunded to your account."
-					),
-				),
-				"status": 401,
-			}
-
-	def authorize_payment(self):
-		"""
-		An authorization is performed when user’s payment details are successfully authenticated by the bank.
-		The money is deducted from the customer’s account, but will not be transferred to the merchant’s account
-		until it is explicitly captured by merchant.
-		"""
-		data = json.loads(self.integration_request.data)
-		settings = self.get_settings(data)
-
-		try:
-			resp = make_get_request(
-				"https://api.razorpay.com/v1/payments/{0}".format(self.data.razorpay_payment_id),
-				auth=(settings.api_key, settings.api_secret),
-			)
-
-			if resp.get("status") == "authorized":
-				self.integration_request.update_status(data, "Authorized")
-				self.flags.status_changed_to = "Authorized"
-
-			if resp.get("status") == "captured":
-				self.integration_request.update_status(data, "Completed")
-				self.flags.status_changed_to = "Completed"
-
-			elif data.get("subscription_id"):
-				if resp.get("status") == "refunded":
-					# if subscription start date is in future then
-					# razorpay refunds the amount after authorizing the card details
-					# thus changing status to Verified
-
-					self.integration_request.update_status(data, "Completed")
-					self.flags.status_changed_to = "Verified"
-
-			else:
-				frappe.log_error(str(resp), "Razorpay Payment not authorized")
-
-		except:
-			frappe.log_error(frappe.get_traceback())
-			# failed
-			pass
-
-		status = frappe.flags.integration_request.status_code
-
-		redirect_to = data.get("redirect_to") or None
-		redirect_message = data.get("redirect_message") or None
-		if self.flags.status_changed_to in ("Authorized", "Verified", "Completed"):
-			if self.data.reference_doctype and self.data.reference_docname:
-				custom_redirect_to = None
-				try:
-					frappe.flags.data = data
-					custom_redirect_to = frappe.get_doc(
-						self.data.reference_doctype, self.data.reference_docname
-					).run_method("on_payment_authorized", self.flags.status_changed_to)
-
-				except Exception:
-					frappe.log_error(frappe.get_traceback())
-
-				if custom_redirect_to:
-					redirect_to = custom_redirect_to
-
-			redirect_url = "payment-success?doctype={0}&docname={1}".format(
-				self.data.reference_doctype, self.data.reference_docname
-			)
-		else:
-			redirect_url = "payment-failed"
-
-		if redirect_to:
-			redirect_url += "&" + urlencode({"redirect_to": redirect_to})
-		if redirect_message:
-			redirect_url += "&" + urlencode({"redirect_message": redirect_message})
-
-		return {"redirect_to": redirect_url, "status": status}
-
-	def get_settings(self, data):
-		settings = frappe._dict(
-			{
-				"api_key": self.api_key,
-				"api_secret": self.get_password(fieldname="api_secret", raise_exception=False),
-			}
-		)
-
-		if cint(data.get("notes", {}).get("use_sandbox")) or data.get("use_sandbox"):
-			settings.update(
-				{
-					"api_key": frappe.conf.sandbox_api_key,
-					"api_secret": frappe.conf.sandbox_api_secret,
-				}
-			)
-
-		return settings
-
-	def cancel_subscription(self, subscription_id):
-		settings = self.get_settings({})
-
-		try:
-			resp = make_post_request(
-				"https://api.razorpay.com/v1/subscriptions/{0}/cancel".format(subscription_id),
-				auth=(settings.api_key, settings.api_secret),
-			)
-		except Exception:
-			frappe.log_error(frappe.get_traceback())
-
-	def verify_signature(self, body, signature, key):
-		key = bytes(key, "utf-8")
-		body = bytes(body, "utf-8")
-
-		dig = hmac.new(key=key, msg=body, digestmod=hashlib.sha256)
-
-		generated_signature = dig.hexdigest()
-		result = hmac.compare_digest(generated_signature, signature)
-
-		if not result:
-			frappe.throw(_("Razorpay Signature Verification Failed"), exc=frappe.PermissionError)
-
-		return result
-
-
-def capture_payment(is_sandbox=False, sanbox_response=None):
-	"""
-	Verifies the purchase as complete by the merchant.
-	After capture, the amount is transferred to the merchant within T+3 days
-	where T is the day on which payment is captured.
-
-	Note: Attempting to capture a payment whose status is not authorized will produce an error.
-	"""
-	controller = frappe.get_doc("Razorpay Settings")
-
-	for doc in frappe.get_all(
-		"Integration Request",
-		filters={"status": "Authorized", "integration_request_service": "Razorpay"},
-		fields=["name", "data"],
-	):
-		try:
-			if is_sandbox:
-				resp = sanbox_response
-			else:
-				data = json.loads(doc.data)
-				settings = controller.get_settings(data)
-
-				resp = make_get_request(
-					"https://api.razorpay.com/v1/payments/{0}".format(data.get("razorpay_payment_id")),
-					auth=(settings.api_key, settings.api_secret),
-					data={"amount": data.get("amount")},
-				)
-
-				if resp.get("status") == "authorized":
-					resp = make_post_request(
-						"https://api.razorpay.com/v1/payments/{0}/capture".format(data.get("razorpay_payment_id")),
-						auth=(settings.api_key, settings.api_secret),
-						data={"amount": data.get("amount")},
-					)
-
-			if resp.get("status") == "captured":
-				frappe.db.set_value("Integration Request", doc.name, "status", "Completed")
-
-		except Exception:
-			doc = frappe.get_doc("Integration Request", doc.name)
-			doc.status = "Failed"
-			doc.error = frappe.get_traceback()
-			doc.save()
-			frappe.log_error(doc.error, "{0} Failed".format(doc.name))
-
-
-@frappe.whitelist(allow_guest=True)
-def get_api_key():
-	controller = frappe.get_doc("Razorpay Settings")
-	return controller.api_key
-
-
-@frappe.whitelist(allow_guest=True)
-def get_order(doctype, docname):
-	# Order returned to be consumed by razorpay.js
-	doc = frappe.get_doc(doctype, docname)
-	try:
-		# Do not use run_method here as it fails silently
-		return doc.get_razorpay_order()
-	except AttributeError:
-		frappe.log_error(frappe.get_traceback(), _("Controller method get_razorpay_order missing"))
-		frappe.throw(_("Could not create Razorpay order. Please contact Administrator"))
-
-
-@frappe.whitelist(allow_guest=True)
-def order_payment_success(integration_request, params):
-	"""Called by razorpay.js on order payment success, the params
-	contains razorpay_payment_id, razorpay_order_id, razorpay_signature
-	that is updated in the data field of integration request
-
-	Args:
-	        integration_request (string): Name for integration request doc
-	        params (string): Params to be updated for integration request.
-	"""
-	params = json.loads(params)
-	integration = frappe.get_doc("Integration Request", integration_request)
-
-	# Update integration request
-	integration.update_status(params, integration.status)
-	integration.reload()
-
-	data = json.loads(integration.data)
-	controller = frappe.get_doc("Razorpay Settings")
-
-	# Update payment and integration data for payment controller object
-	controller.integration_request = integration
-	controller.data = frappe._dict(data)
-
-	# Authorize payment
-	controller.authorize_payment()
-
-
-@frappe.whitelist(allow_guest=True)
-def order_payment_failure(integration_request, params):
-	"""Called by razorpay.js on failure
-
-	Args:
-	        integration_request (TYPE): Description
-	        params (TYPE): error data to be updated
-	"""
-	frappe.log_error(params, "Razorpay Payment Failure")
-	params = json.loads(params)
-	integration = frappe.get_doc("Integration Request", integration_request)
-	integration.update_status(params, integration.status)
-
-
-def convert_rupee_to_paisa(**kwargs):
-	for addon in kwargs.get("addons"):
-		addon["item"]["amount"] *= 100
-
-	frappe.conf.converted_rupee_to_paisa = True
-
-
-@frappe.whitelist(allow_guest=True)
-def razorpay_subscription_callback():
-	try:
-		data = frappe.local.form_dict
-
-		validate_payment_callback(data)
-
-		data.update({"payment_gateway": "Razorpay"})
-
-		doc = frappe.get_doc(
-			{
-				"data": json.dumps(frappe.local.form_dict),
-				"doctype": "Integration Request",
-				"integration_type": "Subscription Notification",
-				"status": "Queued",
-			}
-		).insert(ignore_permissions=True)
-		frappe.db.commit()
-
-		frappe.enqueue(
-			method="frappe.integrations.doctype.razorpay_settings.razorpay_settings.handle_subscription_notification",
-			queue="long",
-			timeout=600,
-			is_async=True,
-			**{"doctype": "Integration Request", "docname": doc.name}
-		)
-
-	except frappe.InvalidStatusError:
-		pass
-	except Exception as e:
-		frappe.log(frappe.log_error(title=e))
-
-
-def validate_payment_callback(data):
-	def _throw():
-		frappe.throw(_("Invalid Subscription"), exc=frappe.InvalidStatusError)
-
-	subscription_id = data.get("payload").get("subscription").get("entity").get("id")
-
-	if not (subscription_id):
-		_throw()
-
-	controller = frappe.get_doc("Razorpay Settings")
-
-	settings = controller.get_settings(data)
-
-	resp = make_get_request(
-		"https://api.razorpay.com/v1/subscriptions/{0}".format(subscription_id),
-		auth=(settings.api_key, settings.api_secret),
-	)
-
-	if resp.get("status") != "active":
-		_throw()
-
-
-def handle_subscription_notification(doctype, docname):
-	call_hook_method("handle_subscription_notification", doctype=doctype, docname=docname)
diff --git a/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.js b/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.js
index 1a1b8a7c67..6db4087cf3 100755
--- a/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.js
+++ b/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.js
@@ -1,26 +1,26 @@
 // Copyright (c) 2017, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('S3 Backup Settings', {
-	refresh: function(frm) {
+frappe.ui.form.on("S3 Backup Settings", {
+	refresh: function (frm) {
 		frm.clear_custom_buttons();
 		frm.events.take_backup(frm);
 	},
 
-	take_backup: function(frm) {
+	take_backup: function (frm) {
 		if (frm.doc.access_key_id && frm.doc.secret_access_key) {
-			frm.add_custom_button(__("Take Backup Now"), function(){
+			frm.add_custom_button(__("Take Backup Now"), function () {
 				frm.dashboard.set_headline_alert("S3 Backup Started!");
 				frappe.call({
 					method: "frappe.integrations.doctype.s3_backup_settings.s3_backup_settings.take_backups_s3",
-					callback: function(r) {
-						if(!r.exc) {
+					callback: function (r) {
+						if (!r.exc) {
 							frappe.msgprint(__("S3 Backup complete!"));
 							frm.dashboard.clear_headline();
 						}
-					}
+					},
 				});
 			}).addClass("btn-primary");
 		}
-	}
+	},
 });
diff --git a/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.py b/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.py
index 015d4f0467..568ff71b00 100755
--- a/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.py
+++ b/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2017, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 import os
@@ -77,8 +76,8 @@ def take_backups_monthly():
 
 
 def take_backups_if(freq):
-	if cint(frappe.db.get_value("S3 Backup Settings", None, "enabled")):
-		if frappe.db.get_value("S3 Backup Settings", None, "frequency") == freq:
+	if cint(frappe.db.get_single_value("S3 Backup Settings", "enabled")):
+		if frappe.db.get_single_value("S3 Backup Settings", "frequency") == freq:
 			take_backups_s3()
 
 
diff --git a/frappe/integrations/doctype/s3_backup_settings/test_s3_backup_settings.py b/frappe/integrations/doctype/s3_backup_settings/test_s3_backup_settings.py
index c6dbee2c20..48b1ccd113 100755
--- a/frappe/integrations/doctype/s3_backup_settings/test_s3_backup_settings.py
+++ b/frappe/integrations/doctype/s3_backup_settings/test_s3_backup_settings.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2017, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
 import unittest
diff --git a/frappe/integrations/doctype/slack_webhook_url/slack_webhook_url.js b/frappe/integrations/doctype/slack_webhook_url/slack_webhook_url.js
index b7a972bdc1..49991fcffe 100644
--- a/frappe/integrations/doctype/slack_webhook_url/slack_webhook_url.js
+++ b/frappe/integrations/doctype/slack_webhook_url/slack_webhook_url.js
@@ -1,6 +1,4 @@
 // Copyright (c) 2018, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Slack Webhook URL', {
-
-});
+frappe.ui.form.on("Slack Webhook URL", {});
diff --git a/frappe/integrations/doctype/slack_webhook_url/slack_webhook_url.py b/frappe/integrations/doctype/slack_webhook_url/slack_webhook_url.py
index cf6a23fb34..d71d7075a6 100644
--- a/frappe/integrations/doctype/slack_webhook_url/slack_webhook_url.py
+++ b/frappe/integrations/doctype/slack_webhook_url/slack_webhook_url.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2018, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
diff --git a/frappe/integrations/doctype/slack_webhook_url/test_slack_webhook_url.py b/frappe/integrations/doctype/slack_webhook_url/test_slack_webhook_url.py
index 4bd71033bb..16b1bcd3c2 100644
--- a/frappe/integrations/doctype/slack_webhook_url/test_slack_webhook_url.py
+++ b/frappe/integrations/doctype/slack_webhook_url/test_slack_webhook_url.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2018, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
 import unittest
diff --git a/frappe/integrations/doctype/social_login_key/social_login_key.js b/frappe/integrations/doctype/social_login_key/social_login_key.js
index e2cbb3459f..033beffff0 100644
--- a/frappe/integrations/doctype/social_login_key/social_login_key.js
+++ b/frappe/integrations/doctype/social_login_key/social_login_key.js
@@ -1,12 +1,19 @@
 // Copyright (c) 2017, Frappe Technologies and contributors
 // For license information, please see license.txt
 const fields = [
-	"provider_name", "base_url", "custom_base_url",
-	"icon", "authorize_url", "access_token_url", "redirect_url",
-	"api_endpoint", "api_endpoint_args", "auth_url_data"
+	"provider_name",
+	"base_url",
+	"custom_base_url",
+	"icon",
+	"authorize_url",
+	"access_token_url",
+	"redirect_url",
+	"api_endpoint",
+	"api_endpoint_args",
+	"auth_url_data",
 ];
 
-frappe.ui.form.on('Social Login Key', {
+frappe.ui.form.on("Social Login Key", {
 	refresh(frm) {
 		frm.trigger("setup_fields");
 	},
@@ -16,23 +23,25 @@ frappe.ui.form.on('Social Login Key', {
 	},
 
 	social_login_provider(frm) {
-		if(frm.doc.social_login_provider != "Custom") {
-			frappe.call({
-				"doc": frm.doc,
-				"method": "get_social_login_provider",
-				"args": {
-					"provider": frm.doc.social_login_provider
-				}
-			}).done((r) => {
-				const provider = r.message;
-				for(var field of fields) {
-					frm.set_value(field, provider[field]);
-					frm.set_df_property(field, "read_only", 1);
-					if (frm.doc.custom_base_url) {
-						frm.toggle_enable("base_url", 1);
+		if (frm.doc.social_login_provider != "Custom") {
+			frappe
+				.call({
+					doc: frm.doc,
+					method: "get_social_login_provider",
+					args: {
+						provider: frm.doc.social_login_provider,
+					},
+				})
+				.done((r) => {
+					const provider = r.message;
+					for (var field of fields) {
+						frm.set_value(field, provider[field]);
+						frm.set_df_property(field, "read_only", 1);
+						if (frm.doc.custom_base_url) {
+							frm.toggle_enable("base_url", 1);
+						}
 					}
-				}
-			});
+				});
 		} else {
 			frm.trigger("clear_fields");
 			frm.trigger("setup_fields");
@@ -41,38 +50,35 @@ frappe.ui.form.on('Social Login Key', {
 
 	setup_fields(frm) {
 		// set custom_base_url to read only for "Custom" provider
-		if(frm.doc.social_login_provider == "Custom") {
+		if (frm.doc.social_login_provider == "Custom") {
 			frm.set_value("custom_base_url", 1);
 			frm.set_df_property("custom_base_url", "read_only", 1);
 		}
 
 		// set fields to read only for providers from template
-		for(var f of fields) {
-			if(frm.doc.social_login_provider != "Custom"){
+		for (var f of fields) {
+			if (frm.doc.social_login_provider != "Custom") {
 				frm.set_df_property(f, "read_only", 1);
 			}
 		}
 
 		// enable base_url for providers with custom_base_url
-		if(frm.doc.custom_base_url) {
+		if (frm.doc.custom_base_url) {
 			frm.set_df_property("base_url", "read_only", 0);
 			frm.fields_dict["sb_identity_details"].collapse(false);
 		}
 
 		// hide social_login_provider and provider_name for non local
-		if(!frm.doc.__islocal &&
-			(frm.doc.social_login_provider ||
-				frm.doc.provider_name)) {
+		if (!frm.doc.__islocal && (frm.doc.social_login_provider || frm.doc.provider_name)) {
 			frm.set_df_property("social_login_provider", "hidden", 1);
 			frm.set_df_property("provider_name", "hidden", 1);
 		}
 	},
 
 	clear_fields(frm) {
-		for(var field of fields){
+		for (var field of fields) {
 			frm.set_value(field, "");
 			frm.set_df_property(field, "read_only", 0);
 		}
-	}
-
+	},
 });
diff --git a/frappe/integrations/doctype/social_login_key/social_login_key.py b/frappe/integrations/doctype/social_login_key/social_login_key.py
index 8c19767107..e38f19bb2b 100644
--- a/frappe/integrations/doctype/social_login_key/social_login_key.py
+++ b/frappe/integrations/doctype/social_login_key/social_login_key.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2017, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
@@ -69,7 +68,7 @@ class SocialLoginKey(Document):
 
 		if self.provider_name in icon_map:
 			icon_file = icon_map[self.provider_name]
-			self.icon = "/assets/frappe/icons/social/{0}".format(icon_file)
+			self.icon = f"/assets/frappe/icons/social/{icon_file}"
 
 	@frappe.whitelist()
 	def get_social_login_provider(self, provider, initialize=False):
diff --git a/frappe/integrations/doctype/social_login_key/test_social_login_key.py b/frappe/integrations/doctype/social_login_key/test_social_login_key.py
index be4c1f7c49..c51ccb2c0f 100644
--- a/frappe/integrations/doctype/social_login_key/test_social_login_key.py
+++ b/frappe/integrations/doctype/social_login_key/test_social_login_key.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2017, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
 import unittest
diff --git a/frappe/integrations/doctype/stripe_settings/__init__.py b/frappe/integrations/doctype/stripe_settings/__init__.py
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/frappe/integrations/doctype/stripe_settings/stripe_settings.js b/frappe/integrations/doctype/stripe_settings/stripe_settings.js
deleted file mode 100644
index 578ae94906..0000000000
--- a/frappe/integrations/doctype/stripe_settings/stripe_settings.js
+++ /dev/null
@@ -1,8 +0,0 @@
-// Copyright (c) 2017, Frappe Technologies and contributors
-// For license information, please see license.txt
-
-frappe.ui.form.on('Stripe Settings', {
-	refresh: function(frm) {
-
-	}
-});
diff --git a/frappe/integrations/doctype/stripe_settings/stripe_settings.json b/frappe/integrations/doctype/stripe_settings/stripe_settings.json
deleted file mode 100644
index 306355319b..0000000000
--- a/frappe/integrations/doctype/stripe_settings/stripe_settings.json
+++ /dev/null
@@ -1,315 +0,0 @@
-{
- "allow_copy": 0, 
- "allow_guest_to_view": 0, 
- "allow_import": 0, 
- "allow_rename": 0, 
- "autoname": "field:gateway_name", 
- "beta": 0, 
- "creation": "2017-03-09 17:18:29.458397", 
- "custom": 0, 
- "docstatus": 0, 
- "doctype": "DocType", 
- "document_type": "", 
- "editable_grid": 1, 
- "engine": "InnoDB", 
- "fields": [
-  {
-   "allow_bulk_edit": 0, 
-   "allow_in_quick_entry": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "gateway_name", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Payment Gateway Name", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "translatable": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_in_quick_entry": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "publishable_key", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Publishable Key", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "translatable": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_in_quick_entry": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "column_break_3", 
-   "fieldtype": "Column Break", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "translatable": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_in_quick_entry": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "secret_key", 
-   "fieldtype": "Password", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Secret Key", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "translatable": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_in_quick_entry": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "section_break_5", 
-   "fieldtype": "Section Break", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "translatable": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_in_quick_entry": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "header_img", 
-   "fieldtype": "Attach Image", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Header Image", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "translatable": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_in_quick_entry": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "column_break_7", 
-   "fieldtype": "Column Break", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "translatable": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_in_quick_entry": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "redirect_url", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Redirect URL", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "translatable": 0, 
-   "unique": 0
-  }
- ], 
- "has_web_view": 0, 
- "hide_heading": 0, 
- "hide_toolbar": 0, 
- "idx": 0, 
- "image_view": 0, 
- "in_create": 0, 
- "is_submittable": 0, 
- "issingle": 0, 
- "istable": 0, 
- "max_attachments": 0, 
- "modified": "2018-05-23 13:32:14.429916", 
- "modified_by": "Administrator", 
- "module": "Integrations", 
- "name": "Stripe Settings", 
- "name_case": "", 
- "owner": "Administrator", 
- "permissions": [
-  {
-   "amend": 0, 
-   "cancel": 0, 
-   "create": 1, 
-   "delete": 1, 
-   "email": 1, 
-   "export": 0, 
-   "if_owner": 0, 
-   "import": 0, 
-   "permlevel": 0, 
-   "print": 1, 
-   "read": 1, 
-   "report": 0, 
-   "role": "System Manager", 
-   "set_user_permissions": 0, 
-   "share": 1, 
-   "submit": 0, 
-   "write": 1
-  }
- ], 
- "quick_entry": 1, 
- "read_only": 0, 
- "read_only_onload": 0, 
- "show_name_in_global_search": 0, 
- "sort_field": "modified", 
- "sort_order": "DESC", 
- "track_changes": 0, 
- "track_seen": 0
-}
\ No newline at end of file
diff --git a/frappe/integrations/doctype/stripe_settings/stripe_settings.py b/frappe/integrations/doctype/stripe_settings/stripe_settings.py
deleted file mode 100644
index 4ebf902e84..0000000000
--- a/frappe/integrations/doctype/stripe_settings/stripe_settings.py
+++ /dev/null
@@ -1,281 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright (c) 2017, Frappe Technologies and contributors
-# License: MIT. See LICENSE
-
-from urllib.parse import urlencode
-
-import frappe
-from frappe import _
-from frappe.integrations.utils import (
-	create_payment_gateway,
-	create_request_log,
-	make_get_request,
-	make_post_request,
-)
-from frappe.model.document import Document
-from frappe.utils import call_hook_method, cint, flt, get_url
-
-
-class StripeSettings(Document):
-	supported_currencies = [
-		"AED",
-		"ALL",
-		"ANG",
-		"ARS",
-		"AUD",
-		"AWG",
-		"BBD",
-		"BDT",
-		"BIF",
-		"BMD",
-		"BND",
-		"BOB",
-		"BRL",
-		"BSD",
-		"BWP",
-		"BZD",
-		"CAD",
-		"CHF",
-		"CLP",
-		"CNY",
-		"COP",
-		"CRC",
-		"CVE",
-		"CZK",
-		"DJF",
-		"DKK",
-		"DOP",
-		"DZD",
-		"EGP",
-		"ETB",
-		"EUR",
-		"FJD",
-		"FKP",
-		"GBP",
-		"GIP",
-		"GMD",
-		"GNF",
-		"GTQ",
-		"GYD",
-		"HKD",
-		"HNL",
-		"HRK",
-		"HTG",
-		"HUF",
-		"IDR",
-		"ILS",
-		"INR",
-		"ISK",
-		"JMD",
-		"JPY",
-		"KES",
-		"KHR",
-		"KMF",
-		"KRW",
-		"KYD",
-		"KZT",
-		"LAK",
-		"LBP",
-		"LKR",
-		"LRD",
-		"MAD",
-		"MDL",
-		"MNT",
-		"MOP",
-		"MRO",
-		"MUR",
-		"MVR",
-		"MWK",
-		"MXN",
-		"MYR",
-		"NAD",
-		"NGN",
-		"NIO",
-		"NOK",
-		"NPR",
-		"NZD",
-		"PAB",
-		"PEN",
-		"PGK",
-		"PHP",
-		"PKR",
-		"PLN",
-		"PYG",
-		"QAR",
-		"RUB",
-		"SAR",
-		"SBD",
-		"SCR",
-		"SEK",
-		"SGD",
-		"SHP",
-		"SLL",
-		"SOS",
-		"STD",
-		"SVC",
-		"SZL",
-		"THB",
-		"TOP",
-		"TTD",
-		"TWD",
-		"TZS",
-		"UAH",
-		"UGX",
-		"USD",
-		"UYU",
-		"UZS",
-		"VND",
-		"VUV",
-		"WST",
-		"XAF",
-		"XOF",
-		"XPF",
-		"YER",
-		"ZAR",
-	]
-
-	currency_wise_minimum_charge_amount = {
-		"JPY": 50,
-		"MXN": 10,
-		"DKK": 2.50,
-		"HKD": 4.00,
-		"NOK": 3.00,
-		"SEK": 3.00,
-		"USD": 0.50,
-		"AUD": 0.50,
-		"BRL": 0.50,
-		"CAD": 0.50,
-		"CHF": 0.50,
-		"EUR": 0.50,
-		"GBP": 0.30,
-		"NZD": 0.50,
-		"SGD": 0.50,
-	}
-
-	def on_update(self):
-		create_payment_gateway(
-			"Stripe-" + self.gateway_name, settings="Stripe Settings", controller=self.gateway_name
-		)
-		call_hook_method("payment_gateway_enabled", gateway="Stripe-" + self.gateway_name)
-		if not self.flags.ignore_mandatory:
-			self.validate_stripe_credentails()
-
-	def validate_stripe_credentails(self):
-		if self.publishable_key and self.secret_key:
-			header = {
-				"Authorization": "Bearer {0}".format(
-					self.get_password(fieldname="secret_key", raise_exception=False)
-				)
-			}
-			try:
-				make_get_request(url="https://api.stripe.com/v1/charges", headers=header)
-			except Exception:
-				frappe.throw(_("Seems Publishable Key or Secret Key is wrong !!!"))
-
-	def validate_transaction_currency(self, currency):
-		if currency not in self.supported_currencies:
-			frappe.throw(
-				_(
-					"Please select another payment method. Stripe does not support transactions in currency '{0}'"
-				).format(currency)
-			)
-
-	def validate_minimum_transaction_amount(self, currency, amount):
-		if currency in self.currency_wise_minimum_charge_amount:
-			if flt(amount) < self.currency_wise_minimum_charge_amount.get(currency, 0.0):
-				frappe.throw(
-					_("For currency {0}, the minimum transaction amount should be {1}").format(
-						currency, self.currency_wise_minimum_charge_amount.get(currency, 0.0)
-					)
-				)
-
-	def get_payment_url(self, **kwargs):
-		return get_url("./integrations/stripe_checkout?{0}".format(urlencode(kwargs)))
-
-	def create_request(self, data):
-		import stripe
-
-		self.data = frappe._dict(data)
-		stripe.api_key = self.get_password(fieldname="secret_key", raise_exception=False)
-		stripe.default_http_client = stripe.http_client.RequestsClient()
-
-		try:
-			self.integration_request = create_request_log(self.data, "Host", "Stripe")
-			return self.create_charge_on_stripe()
-
-		except Exception:
-			frappe.log_error(frappe.get_traceback())
-			return {
-				"redirect_to": frappe.redirect_to_message(
-					_("Server Error"),
-					_(
-						"It seems that there is an issue with the server's stripe configuration. In case of failure, the amount will get refunded to your account."
-					),
-				),
-				"status": 401,
-			}
-
-	def create_charge_on_stripe(self):
-		import stripe
-
-		try:
-			charge = stripe.Charge.create(
-				amount=cint(flt(self.data.amount) * 100),
-				currency=self.data.currency,
-				source=self.data.stripe_token_id,
-				description=self.data.description,
-				receipt_email=self.data.payer_email,
-			)
-
-			if charge.captured == True:
-				self.integration_request.db_set("status", "Completed", update_modified=False)
-				self.flags.status_changed_to = "Completed"
-
-			else:
-				frappe.log_error(charge.failure_message, "Stripe Payment not completed")
-
-		except Exception:
-			frappe.log_error(frappe.get_traceback())
-
-		return self.finalize_request()
-
-	def finalize_request(self):
-		redirect_to = self.data.get("redirect_to") or None
-		redirect_message = self.data.get("redirect_message") or None
-		status = self.integration_request.status
-
-		if self.flags.status_changed_to == "Completed":
-			if self.data.reference_doctype and self.data.reference_docname:
-				custom_redirect_to = None
-				try:
-					custom_redirect_to = frappe.get_doc(
-						self.data.reference_doctype, self.data.reference_docname
-					).run_method("on_payment_authorized", self.flags.status_changed_to)
-				except Exception:
-					frappe.log_error(frappe.get_traceback())
-
-				if custom_redirect_to:
-					redirect_to = custom_redirect_to
-
-				redirect_url = "payment-success"
-
-			if self.redirect_url:
-				redirect_url = self.redirect_url
-				redirect_to = None
-		else:
-			redirect_url = "payment-failed"
-
-		if redirect_to:
-			redirect_url += "?" + urlencode({"redirect_to": redirect_to})
-		if redirect_message:
-			redirect_url += "&" + urlencode({"redirect_message": redirect_message})
-
-		return {"redirect_to": redirect_url, "status": status}
-
-
-def get_gateway_controller(doctype, docname):
-	reference_doc = frappe.get_doc(doctype, docname)
-	gateway_controller = frappe.db.get_value(
-		"Payment Gateway", reference_doc.payment_gateway, "gateway_controller"
-	)
-	return gateway_controller
diff --git a/frappe/integrations/doctype/stripe_settings/test_stripe_settings.py b/frappe/integrations/doctype/stripe_settings/test_stripe_settings.py
deleted file mode 100644
index e13359fa6d..0000000000
--- a/frappe/integrations/doctype/stripe_settings/test_stripe_settings.py
+++ /dev/null
@@ -1,8 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright (c) 2018, Frappe Technologies and Contributors
-# License: MIT. See LICENSE
-import unittest
-
-
-class TestStripeSettings(unittest.TestCase):
-	pass
diff --git a/frappe/integrations/doctype/token_cache/test_token_cache.py b/frappe/integrations/doctype/token_cache/test_token_cache.py
index 6a3b16e72c..a9366d84d3 100644
--- a/frappe/integrations/doctype/token_cache/test_token_cache.py
+++ b/frappe/integrations/doctype/token_cache/test_token_cache.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2019, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 import unittest
diff --git a/frappe/integrations/doctype/token_cache/token_cache.js b/frappe/integrations/doctype/token_cache/token_cache.js
index b7cac9b804..c8074c876b 100644
--- a/frappe/integrations/doctype/token_cache/token_cache.js
+++ b/frappe/integrations/doctype/token_cache/token_cache.js
@@ -1,8 +1,7 @@
 // Copyright (c) 2019, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Token Cache', {
+frappe.ui.form.on("Token Cache", {
 	// refresh: function(frm) {
-
 	// }
 });
diff --git a/frappe/integrations/doctype/token_cache/token_cache.py b/frappe/integrations/doctype/token_cache/token_cache.py
index 7d961fe1cc..25f07a16ba 100644
--- a/frappe/integrations/doctype/token_cache/token_cache.py
+++ b/frappe/integrations/doctype/token_cache/token_cache.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2019, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
diff --git a/frappe/integrations/doctype/webhook/__init__.py b/frappe/integrations/doctype/webhook/__init__.py
index 915d2819ee..192cd2fa12 100644
--- a/frappe/integrations/doctype/webhook/__init__.py
+++ b/frappe/integrations/doctype/webhook/__init__.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2017, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
@@ -18,6 +17,7 @@ def run_webhooks(doc, method):
 	if frappe.flags.webhooks_executed is None:
 		frappe.flags.webhooks_executed = {}
 
+	# TODO: remove this hazardous unnecessary cache in flags
 	if frappe.flags.webhooks is None:
 		# load webhooks from cache
 		webhooks = frappe.cache().get_value("webhooks")
diff --git a/frappe/integrations/doctype/webhook/test_webhook.py b/frappe/integrations/doctype/webhook/test_webhook.py
index 5386a75573..7d9d05cd9e 100644
--- a/frappe/integrations/doctype/webhook/test_webhook.py
+++ b/frappe/integrations/doctype/webhook/test_webhook.py
@@ -1,7 +1,8 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2017, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
+import json
 import unittest
+from contextlib import contextmanager
 
 import frappe
 from frappe.integrations.doctype.webhook.webhook import (
@@ -11,6 +12,16 @@ from frappe.integrations.doctype.webhook.webhook import (
 )
 
 
+@contextmanager
+def get_test_webhook(config):
+	wh = frappe.get_doc(config).insert()
+	wh.reload()
+	try:
+		yield wh
+	finally:
+		wh.delete()
+
+
 class TestWebhook(unittest.TestCase):
 	@classmethod
 	def setUpClass(cls):
@@ -166,3 +177,31 @@ class TestWebhook(unittest.TestCase):
 		enqueue_webhook(user, webhook)
 
 		self.assertTrue(frappe.db.get_all("Webhook Request Log", pluck="name"))
+
+	def test_webhook_with_array_body(self):
+		"""Check if array request body are supported."""
+		wh_config = {
+			"doctype": "Webhook",
+			"webhook_doctype": "Note",
+			"webhook_docevent": "after_insert",
+			"enabled": 1,
+			"request_url": "https://httpbin.org/post",
+			"request_method": "POST",
+			"request_structure": "JSON",
+			"webhook_json": '[\r\n{% for n in range(3) %}\r\n    {\r\n        "title": "{{ doc.title }}",\r\n        "n": {{ n }}\r\n    }\r\n    {%- if not loop.last -%}\r\n        , \r\n    {%endif%}\r\n{%endfor%}\r\n]',
+			"meets_condition": "Yes",
+			"webhook_headers": [
+				{
+					"key": "Content-Type",
+					"value": "application/json",
+				}
+			],
+		}
+
+		with get_test_webhook(wh_config) as wh:
+			doc = frappe.new_doc("Note")
+			doc.title = "Test Webhook Note"
+
+			enqueue_webhook(doc, wh)
+			log = frappe.get_last_doc("Webhook Request Log")
+			self.assertEqual(len(json.loads(log.response)["json"]), 3)
diff --git a/frappe/integrations/doctype/webhook/webhook.js b/frappe/integrations/doctype/webhook/webhook.js
index 0953e60625..302cacc194 100644
--- a/frappe/integrations/doctype/webhook/webhook.js
+++ b/frappe/integrations/doctype/webhook/webhook.js
@@ -6,27 +6,41 @@ frappe.webhook = {
 		if (frm.doc.webhook_doctype) {
 			frappe.model.with_doctype(frm.doc.webhook_doctype, () => {
 				// get doctype fields
-				let fields = $.map(frappe.get_doc("DocType", frm.doc.webhook_doctype).fields, (d) => {
-					if (frappe.model.no_value_type.includes(d.fieldtype) && !(frappe.model.table_fields.includes(d.fieldtype))) {
-						return null;
-					} else if (d.fieldtype === 'Currency' || d.fieldtype === 'Float') {
-						return { label: d.label, value: d.fieldname };
-					} else {
-						return {  label: `${__(d.label)} (${d.fieldtype})`, value: d.fieldname };
+				let fields = $.map(
+					frappe.get_doc("DocType", frm.doc.webhook_doctype).fields,
+					(d) => {
+						if (
+							frappe.model.no_value_type.includes(d.fieldtype) &&
+							!frappe.model.table_fields.includes(d.fieldtype)
+						) {
+							return null;
+						} else if (d.fieldtype === "Currency" || d.fieldtype === "Float") {
+							return { label: d.label, value: d.fieldname };
+						} else {
+							return {
+								label: `${__(d.label)} (${d.fieldtype})`,
+								value: d.fieldname,
+							};
+						}
 					}
-				});
+				);
 
 				// add meta fields
 				for (let field of frappe.model.std_fields) {
 					if (field.fieldname == "name") {
 						fields.unshift({ label: "Name (Doc Name)", value: "name" });
 					} else {
-						fields.push({ label: `${__(field.label)} (${field.fieldtype})`, value: field.fieldname });
+						fields.push({
+							label: `${__(field.label)} (${field.fieldtype})`,
+							value: field.fieldname,
+						});
 					}
 				}
 
 				frm.fields_dict.webhook_data.grid.update_docfield_property(
-					'fieldname', 'options', [""].concat(fields)
+					"fieldname",
+					"options",
+					[""].concat(fields)
 				);
 			});
 		}
@@ -42,22 +56,29 @@ frappe.webhook = {
 			}
 
 			if (header_value) {
-				let header_row = (frm.doc.webhook_headers || []).find(row => row.key === 'Content-Type');
+				let header_row = (frm.doc.webhook_headers || []).find(
+					(row) => row.key === "Content-Type"
+				);
 				if (header_row) {
-					frappe.model.set_value(header_row.doctype, header_row.name, "value", header_value);
+					frappe.model.set_value(
+						header_row.doctype,
+						header_row.name,
+						"value",
+						header_value
+					);
 				} else {
 					frm.add_child("webhook_headers", {
-						"key": "Content-Type",
-						"value": header_value
+						key: "Content-Type",
+						value: header_value,
 					});
 				}
 				frm.refresh();
 			}
 		}
-	}
+	},
 };
 
-frappe.ui.form.on('Webhook', {
+frappe.ui.form.on("Webhook", {
 	refresh: (frm) => {
 		frappe.webhook.set_fieldname_select(frm);
 	},
@@ -71,14 +92,27 @@ frappe.ui.form.on('Webhook', {
 	},
 
 	enable_security: (frm) => {
-		frm.toggle_reqd('webhook_secret', frm.doc.enable_security);
-	}
+		frm.toggle_reqd("webhook_secret", frm.doc.enable_security);
+	},
+
+	preview_document: (frm) => {
+		frappe.call({
+			method: "generate_preview",
+			doc: frm.doc,
+			callback: (r) => {
+				frm.refresh_field("meets_condition");
+				frm.refresh_field("preview_request_body");
+			},
+		});
+	},
 });
 
 frappe.ui.form.on("Webhook Data", {
 	fieldname: (frm, cdt, cdn) => {
 		let row = locals[cdt][cdn];
-		let df = frappe.get_meta(frm.doc.webhook_doctype).fields.filter((field) => field.fieldname == row.fieldname);
+		let df = frappe
+			.get_meta(frm.doc.webhook_doctype)
+			.fields.filter((field) => field.fieldname == row.fieldname);
 
 		if (!df.length) {
 			// check if field is a meta field
@@ -87,5 +121,5 @@ frappe.ui.form.on("Webhook Data", {
 
 		row.key = df.length ? df[0].fieldname : "name";
 		frm.refresh_field("webhook_data");
-	}
+	},
 });
diff --git a/frappe/integrations/doctype/webhook/webhook.json b/frappe/integrations/doctype/webhook/webhook.json
index 880874cb25..a21e460659 100644
--- a/frappe/integrations/doctype/webhook/webhook.json
+++ b/frappe/integrations/doctype/webhook/webhook.json
@@ -28,7 +28,13 @@
   "webhook_headers",
   "sb_webhook_data",
   "webhook_data",
-  "webhook_json"
+  "webhook_json",
+  "preview_tab",
+  "preview_document",
+  "column_break_26",
+  "meets_condition",
+  "section_break_28",
+  "preview_request_body"
  ],
  "fields": [
   {
@@ -163,13 +169,45 @@
    "label": "Request Method",
    "options": "POST\nPUT\nDELETE",
    "reqd": 1
+  },
+  {
+   "fieldname": "preview_tab",
+   "fieldtype": "Tab Break",
+   "label": "Preview"
+  },
+  {
+   "fieldname": "preview_document",
+   "fieldtype": "Dynamic Link",
+   "label": "Select Document",
+   "options": "webhook_doctype"
+  },
+  {
+   "fieldname": "preview_request_body",
+   "fieldtype": "Code",
+   "is_virtual": 1,
+   "label": "Request Body"
+  },
+  {
+   "fieldname": "meets_condition",
+   "fieldtype": "Data",
+   "is_virtual": 1,
+   "label": "Meets Condition?"
+  },
+  {
+   "fieldname": "column_break_26",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "section_break_28",
+   "fieldtype": "Section Break"
   }
  ],
  "links": [],
- "modified": "2021-05-25 11:11:28.555291",
+ "modified": "2022-07-11 08:54:10.740512",
  "modified_by": "Administrator",
  "module": "Integrations",
  "name": "Webhook",
+ "naming_rule": "By \"Naming Series\" field",
  "owner": "Administrator",
  "permissions": [
   {
@@ -187,6 +225,7 @@
  ],
  "sort_field": "modified",
  "sort_order": "DESC",
+ "states": [],
  "title_field": "webhook_doctype",
  "track_changes": 1
 }
\ No newline at end of file
diff --git a/frappe/integrations/doctype/webhook/webhook.py b/frappe/integrations/doctype/webhook/webhook.py
index 04a1c6c21e..64a98a61e1 100644
--- a/frappe/integrations/doctype/webhook/webhook.py
+++ b/frappe/integrations/doctype/webhook/webhook.py
@@ -1,9 +1,7 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2017, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
 import base64
-import datetime
 import hashlib
 import hmac
 import json
@@ -28,6 +26,7 @@ class Webhook(Document):
 		self.validate_request_url()
 		self.validate_request_body()
 		self.validate_repeating_fields()
+		self.preview_document = None
 
 	def on_update(self):
 		frappe.cache().delete_value("webhooks")
@@ -48,7 +47,7 @@ class Webhook(Document):
 			try:
 				frappe.safe_eval(self.condition, eval_locals=get_context(temp_doc))
 			except Exception as e:
-				frappe.throw(_(e))
+				frappe.throw(_("Invalid Condition: {}").format(e))
 
 	def validate_request_url(self):
 		try:
@@ -75,13 +74,45 @@ class Webhook(Document):
 		if len(webhook_data) != len(set(webhook_data)):
 			frappe.throw(_("Same Field is entered more than once"))
 
+	@frappe.whitelist()
+	def generate_preview(self):
+		# This function doesn't need to do anything specific as virtual fields
+		# get evaluated automatically.
+		pass
+
+	@property
+	def meets_condition(self):
+		if not self.condition:
+			return _("Yes")
+
+		if not (self.preview_document and self.webhook_doctype):
+			return _("Select a document to check if it meets conditions.")
+
+		try:
+			doc = frappe.get_cached_doc(self.webhook_doctype, self.preview_document)
+			met_condition = frappe.safe_eval(self.condition, eval_locals=get_context(doc))
+		except Exception as e:
+			return _("Failed to evaluate conditions: {}").format(e)
+		return _("Yes") if met_condition else _("No")
+
+	@property
+	def preview_request_body(self):
+		if not (self.preview_document and self.webhook_doctype):
+			return _("Select a document to preview request data")
+
+		try:
+			doc = frappe.get_cached_doc(self.webhook_doctype, self.preview_document)
+			return frappe.as_json(get_webhook_data(doc, self))
+		except Exception as e:
+			return _("Failed to compute request body: {}").format(e)
+
 
 def get_context(doc):
 	return {"doc": doc, "utils": get_safe_globals().get("frappe").get("utils")}
 
 
-def enqueue_webhook(doc, webhook):
-	webhook = frappe.get_doc("Webhook", webhook.get("name"))
+def enqueue_webhook(doc, webhook) -> None:
+	webhook: Webhook = frappe.get_doc("Webhook", webhook.get("name"))
 	headers = get_webhook_headers(doc, webhook)
 	data = get_webhook_data(doc, webhook)
 
@@ -98,6 +129,11 @@ def enqueue_webhook(doc, webhook):
 			frappe.logger().debug({"webhook_success": r.text})
 			log_request(webhook.request_url, headers, data, r)
 			break
+
+		except requests.exceptions.ReadTimeout as e:
+			frappe.logger().debug({"webhook_error": e, "try": i + 1})
+			log_request(webhook.request_url, headers, data)
+
 		except Exception as e:
 			frappe.logger().debug({"webhook_error": e, "try": i + 1})
 			log_request(webhook.request_url, headers, data, r)
@@ -105,18 +141,18 @@ def enqueue_webhook(doc, webhook):
 			if i != 2:
 				continue
 			else:
-				raise e
+				webhook.log_error("Webhook failed")
 
 
-def log_request(url, headers, data, res):
+def log_request(url: str, headers: dict, data: dict, res: requests.Response | None = None):
 	request_log = frappe.get_doc(
 		{
 			"doctype": "Webhook Request Log",
 			"user": frappe.session.user if frappe.session.user else None,
 			"url": url,
-			"headers": json.dumps(headers, indent=4) if headers else None,
-			"data": json.dumps(data, indent=4) if isinstance(data, dict) else data,
-			"response": json.dumps(res.json(), indent=4) if res else None,
+			"headers": frappe.as_json(headers) if headers else None,
+			"data": frappe.as_json(data) if data else None,
+			"response": frappe.as_json(res.json()) if res else None,
 		}
 	)
 
diff --git a/frappe/integrations/doctype/webhook_data/webhook_data.json b/frappe/integrations/doctype/webhook_data/webhook_data.json
index 96ae7f786a..2ace6a9237 100644
--- a/frappe/integrations/doctype/webhook_data/webhook_data.json
+++ b/frappe/integrations/doctype/webhook_data/webhook_data.json
@@ -1,130 +1,43 @@
 {
- "allow_copy": 0, 
- "allow_guest_to_view": 0, 
- "allow_import": 0, 
- "allow_rename": 0, 
- "beta": 0, 
- "creation": "2017-09-14 12:08:50.302810", 
- "custom": 0, 
- "docstatus": 0, 
- "doctype": "DocType", 
- "document_type": "", 
- "editable_grid": 1, 
- "engine": "InnoDB", 
+ "actions": [],
+ "creation": "2017-09-14 12:08:50.302810",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "fieldname",
+  "cb_doc_data",
+  "key"
+ ],
  "fields": [
   {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "fieldname", 
-   "fieldtype": "Select", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Fieldname", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "fieldname": "fieldname",
+   "fieldtype": "Select",
+   "in_list_view": 1,
+   "label": "Fieldname",
+   "reqd": 1
+  },
   {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "cb_doc_data", 
-   "fieldtype": "Column Break", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "fieldname": "cb_doc_data",
+   "fieldtype": "Column Break"
+  },
   {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "key", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Key", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
+   "fieldname": "key",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Key",
+   "reqd": 1
   }
- ], 
- "has_web_view": 0, 
- "hide_heading": 0, 
- "hide_toolbar": 0, 
- "idx": 0, 
- "image_view": 0, 
- "in_create": 0, 
- "is_submittable": 0, 
- "issingle": 0, 
- "istable": 1, 
- "max_attachments": 0, 
- "modified": "2017-09-14 13:16:58.252176", 
- "modified_by": "Administrator", 
- "module": "Integrations", 
- "name": "Webhook Data", 
- "name_case": "", 
- "owner": "Administrator", 
- "permissions": [], 
- "quick_entry": 0, 
- "read_only": 0, 
- "read_only_onload": 0, 
- "show_name_in_global_search": 0, 
- "sort_field": "modified", 
- "sort_order": "DESC", 
- "track_changes": 0, 
- "track_seen": 0
+ ],
+ "istable": 1,
+ "links": [],
+ "modified": "2022-08-03 12:20:52.208987",
+ "modified_by": "Administrator",
+ "module": "Integrations",
+ "name": "Webhook Data",
+ "owner": "Administrator",
+ "permissions": [],
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "states": []
 }
\ No newline at end of file
diff --git a/frappe/integrations/doctype/webhook_data/webhook_data.py b/frappe/integrations/doctype/webhook_data/webhook_data.py
index 1bb0d901df..39016cb864 100644
--- a/frappe/integrations/doctype/webhook_data/webhook_data.py
+++ b/frappe/integrations/doctype/webhook_data/webhook_data.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2017, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
diff --git a/frappe/integrations/doctype/webhook_header/webhook_header.json b/frappe/integrations/doctype/webhook_header/webhook_header.json
index 315d28335f..4aea5d02ed 100644
--- a/frappe/integrations/doctype/webhook_header/webhook_header.json
+++ b/frappe/integrations/doctype/webhook_header/webhook_header.json
@@ -1,101 +1,38 @@
 {
- "allow_copy": 0, 
- "allow_guest_to_view": 0, 
- "allow_import": 0, 
- "allow_rename": 0, 
- "beta": 0, 
- "creation": "2017-09-08 16:27:39.195379", 
- "custom": 0, 
- "docstatus": 0, 
- "doctype": "DocType", 
- "document_type": "", 
- "editable_grid": 1, 
- "engine": "InnoDB", 
+ "actions": [],
+ "creation": "2017-09-08 16:27:39.195379",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "key",
+  "value"
+ ],
  "fields": [
   {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "key", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Key", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "fieldname": "key",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Key"
+  },
   {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "value", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Value", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
+   "fieldname": "value",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Value"
   }
- ], 
- "has_web_view": 0, 
- "hide_heading": 0, 
- "hide_toolbar": 0, 
- "idx": 0, 
- "image_view": 0, 
- "in_create": 0, 
- "is_submittable": 0, 
- "issingle": 0, 
- "istable": 1, 
- "max_attachments": 0, 
- "modified": "2017-09-08 16:28:20.025612", 
- "modified_by": "Administrator", 
- "module": "Integrations", 
- "name": "Webhook Header", 
- "name_case": "", 
- "owner": "Administrator", 
- "permissions": [], 
- "quick_entry": 1, 
- "read_only": 0, 
- "read_only_onload": 0, 
- "show_name_in_global_search": 0, 
- "sort_field": "modified", 
- "sort_order": "DESC", 
- "track_changes": 1, 
- "track_seen": 0
+ ],
+ "istable": 1,
+ "links": [],
+ "modified": "2022-08-03 12:20:51.949422",
+ "modified_by": "Administrator",
+ "module": "Integrations",
+ "name": "Webhook Header",
+ "owner": "Administrator",
+ "permissions": [],
+ "quick_entry": 1,
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "states": [],
+ "track_changes": 1
 }
\ No newline at end of file
diff --git a/frappe/integrations/doctype/webhook_header/webhook_header.py b/frappe/integrations/doctype/webhook_header/webhook_header.py
index 9478d227e4..f5f85d1afe 100644
--- a/frappe/integrations/doctype/webhook_header/webhook_header.py
+++ b/frappe/integrations/doctype/webhook_header/webhook_header.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2017, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
diff --git a/frappe/integrations/doctype/webhook_request_log/webhook_request_log.js b/frappe/integrations/doctype/webhook_request_log/webhook_request_log.js
index 9ec4f11536..1cb8c5ec76 100644
--- a/frappe/integrations/doctype/webhook_request_log/webhook_request_log.js
+++ b/frappe/integrations/doctype/webhook_request_log/webhook_request_log.js
@@ -1,8 +1,7 @@
 // Copyright (c) 2021, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Webhook Request Log', {
+frappe.ui.form.on("Webhook Request Log", {
 	// refresh: function(frm) {
-
 	// }
 });
diff --git a/frappe/integrations/doctype/webhook_request_log/webhook_request_log.json b/frappe/integrations/doctype/webhook_request_log/webhook_request_log.json
index 96690f6e8c..d9410a2f82 100644
--- a/frappe/integrations/doctype/webhook_request_log/webhook_request_log.json
+++ b/frappe/integrations/doctype/webhook_request_log/webhook_request_log.json
@@ -1,6 +1,6 @@
 {
  "actions": [],
- "autoname": "WEBHOOK-REQ-.#####",
+ "autoname": "hash",
  "creation": "2021-05-24 21:35:59.104776",
  "doctype": "DocType",
  "editable_grid": 1,
@@ -56,10 +56,11 @@
  "in_create": 1,
  "index_web_pages_for_search": 1,
  "links": [],
- "modified": "2021-05-26 23:57:58.495261",
+ "modified": "2022-05-03 09:33:49.240777",
  "modified_by": "Administrator",
  "module": "Integrations",
  "name": "Webhook Request Log",
+ "naming_rule": "Random",
  "owner": "Administrator",
  "permissions": [
   {
@@ -77,5 +78,6 @@
  ],
  "sort_field": "modified",
  "sort_order": "DESC",
+ "states": [],
  "track_changes": 1
 }
\ No newline at end of file
diff --git a/frappe/integrations/frappe_providers/__init__.py b/frappe/integrations/frappe_providers/__init__.py
index 161937a936..630c2c08b4 100644
--- a/frappe/integrations/frappe_providers/__init__.py
+++ b/frappe/integrations/frappe_providers/__init__.py
@@ -9,5 +9,5 @@ def migrate_to(local_site, frappe_provider):
 	if frappe_provider in ("frappe.cloud", "frappecloud.com"):
 		return frappecloud_migrator(local_site)
 	else:
-		print("{} is not supported yet".format(frappe_provider))
+		print(f"{frappe_provider} is not supported yet")
 		sys.exit(1)
diff --git a/frappe/integrations/frappe_providers/frappecloud.py b/frappe/integrations/frappe_providers/frappecloud.py
index 0a01989d5e..bae811d41d 100644
--- a/frappe/integrations/frappe_providers/frappecloud.py
+++ b/frappe/integrations/frappe_providers/frappecloud.py
@@ -1,14 +1,14 @@
 import click
 import requests
-from html2text import html2text
 
 import frappe
+from frappe.core.utils import html2text
 
 
 def frappecloud_migrator(local_site):
 	print("Retrieving Site Migrator...")
 	remote_site = frappe.conf.frappecloud_url or "frappecloud.com"
-	request_url = "https://{}/api/method/press.api.script".format(remote_site)
+	request_url = f"https://{remote_site}/api/method/press.api.script"
 	request = requests.get(request_url)
 
 	if request.status_code / 100 != 2:
@@ -32,5 +32,5 @@ def frappecloud_migrator(local_site):
 	py = sys.executable
 	script = tempfile.NamedTemporaryFile(mode="w")
 	script.write(script_contents)
-	print("Site Migrator stored at {}".format(script.name))
+	print(f"Site Migrator stored at {script.name}")
 	os.execv(py, [py, script.name, local_site])
diff --git a/frappe/integrations/google_oauth.py b/frappe/integrations/google_oauth.py
new file mode 100644
index 0000000000..8bc54e0b1d
--- /dev/null
+++ b/frappe/integrations/google_oauth.py
@@ -0,0 +1,201 @@
+import json
+
+from google.oauth2.credentials import Credentials
+from googleapiclient.discovery import build
+from requests import get, post
+
+import frappe
+from frappe.utils import get_request_site_address
+
+CALLBACK_METHOD = "/api/method/frappe.integrations.google_oauth.callback"
+_SCOPES = {
+	"mail": ("https://mail.google.com/"),
+	"contacts": ("https://www.googleapis.com/auth/contacts"),
+	"drive": ("https://www.googleapis.com/auth/drive"),
+	"indexing": ("https://www.googleapis.com/auth/indexing"),
+}
+_SERVICES = {
+	"contacts": ("people", "v1"),
+	"drive": ("drive", "v3"),
+	"indexing": ("indexing", "v3"),
+}
+_DOMAIN_CALLBACK_METHODS = {
+	"mail": "frappe.email.oauth.authorize_google_access",
+	"contacts": "frappe.integrations.doctype.google_contacts.google_contacts.authorize_access",
+	"drive": "frappe.integrations.doctype.google_drive.google_drive.authorize_access",
+	"indexing": "frappe.website.doctype.website_settings.google_indexing.authorize_access",
+}
+
+
+class GoogleAuthenticationError(Exception):
+	pass
+
+
+class GoogleOAuth:
+	OAUTH_URL = "https://oauth2.googleapis.com/token"
+
+	def __init__(self, domain: str, validate: bool = True):
+		self.google_settings = frappe.get_single("Google Settings")
+		self.domain = domain.lower()
+		self.scopes = (
+			" ".join(_SCOPES[self.domain])
+			if isinstance(_SCOPES[self.domain], (list, tuple))
+			else _SCOPES[self.domain]
+		)
+
+		if validate:
+			self.validate_google_settings()
+
+	def validate_google_settings(self):
+		google_settings = "Google Settings"
+
+		if not self.google_settings.enable:
+			frappe.throw(frappe._("Please enable {} before continuing.").format(google_settings))
+
+		if not (self.google_settings.client_id and self.google_settings.client_secret):
+			frappe.throw(frappe._("Please update {} before continuing.").format(google_settings))
+
+	def authorize(self, oauth_code: str) -> dict[str, str | int]:
+		"""Returns a dict with access and refresh token.
+
+		:param oauth_code: code got back from google upon successful auhtorization
+		"""
+
+		data = {
+			"code": oauth_code,
+			"client_id": self.google_settings.client_id,
+			"client_secret": self.google_settings.get_password(
+				fieldname="client_secret", raise_exception=False
+			),
+			"grant_type": "authorization_code",
+			"scope": self.scopes,
+			"redirect_uri": get_request_site_address(True) + CALLBACK_METHOD,
+		}
+
+		return handle_response(
+			post(self.OAUTH_URL, data=data).json(),
+			"Google Oauth Authorization Error",
+			"Something went wrong during the authorization.",
+		)
+
+	def refresh_access_token(self, refresh_token: str) -> dict[str, str | int]:
+		"""Refreshes google access token using refresh token"""
+
+		data = {
+			"client_id": self.google_settings.client_id,
+			"client_secret": self.google_settings.get_password(
+				fieldname="client_secret", raise_exception=False
+			),
+			"refresh_token": refresh_token,
+			"grant_type": "refresh_token",
+			"scope": self.scopes,
+		}
+
+		return handle_response(
+			post(self.OAUTH_URL, data=data).json(),
+			"Google Oauth Access Token Refresh Error",
+			"Something went wrong during the access token generation.",
+			raise_err=True,
+		)
+
+	def get_authentication_url(self, state: dict[str, str]) -> dict[str, str]:
+		"""Returns google authentication url.
+
+		:param state: dict of values which you need on callback (for calling methods, redirection back to the form, doc name, etc)
+		"""
+
+		state.update({"domain": self.domain})
+		state = json.dumps(state)
+		callback_url = get_request_site_address(True) + CALLBACK_METHOD
+
+		return {
+			"url": "https://accounts.google.com/o/oauth2/v2/auth?"
+			+ "access_type=offline&response_type=code&prompt=consent&include_granted_scopes=true&"
+			+ "client_id={}&scope={}&redirect_uri={}&state={}".format(
+				self.google_settings.client_id, self.scopes, callback_url, state
+			)
+		}
+
+	def get_google_service_object(self, access_token: str, refresh_token: str):
+		"""Returns google service object"""
+
+		credentials_dict = {
+			"token": access_token,
+			"refresh_token": refresh_token,
+			"token_uri": self.OAUTH_URL,
+			"client_id": self.google_settings.client_id,
+			"client_secret": self.google_settings.get_password(
+				fieldname="client_secret", raise_exception=False
+			),
+			"scopes": self.scopes,
+		}
+
+		return build(
+			serviceName=_SERVICES[self.domain][0],
+			version=_SERVICES[self.domain][1],
+			credentials=Credentials(**credentials_dict),
+			static_discovery=False,
+		)
+
+
+def handle_response(
+	response: dict[str, str | int],
+	error_title: str,
+	error_message: str,
+	raise_err: bool = False,
+):
+	if "error" in response:
+		frappe.log_error(
+			frappe._(error_title), frappe._(response.get("error_description", error_message))
+		)
+
+		if raise_err:
+			frappe.throw(frappe._(error_title), GoogleAuthenticationError, frappe._(error_message))
+
+		return {}
+
+	return response
+
+
+def is_valid_access_token(access_token: str) -> bool:
+	response = get(
+		"https://oauth2.googleapis.com/tokeninfo", params={"access_token": access_token}
+	).json()
+
+	if "error" in response:
+		return False
+
+	return True
+
+
+@frappe.whitelist(methods=["GET"])
+def callback(state: str, code: str = None, error: str = None) -> None:
+	"""Common callback for google integrations.
+	Invokes functions using `frappe.get_attr` and also adds required (keyworded) arguments
+	along with committing and redirecting us back to frappe site."""
+
+	state = json.loads(state)
+	redirect = state.pop("redirect", "/app")
+	success_query_param = state.pop("success_query_param", "")
+	failure_query_param = state.pop("failure_query_param", "")
+
+	if not error:
+		if (domain := state.pop("domain")) in _DOMAIN_CALLBACK_METHODS:
+			state.update({"code": code})
+			frappe.get_attr(_DOMAIN_CALLBACK_METHODS[domain])(**state)
+
+			# GET request, hence using commit to persist changes
+			frappe.db.commit()  # nosemgrep
+		else:
+			return frappe.respond_as_web_page(
+				"Invalid Google Callback",
+				"The callback domain provided is not valid for Google Authentication",
+				http_status_code=400,
+				indicator_color="red",
+				width=640,
+			)
+
+	frappe.local.response["type"] = "redirect"
+	frappe.local.response[
+		"location"
+	] = f"{redirect}?{failure_query_param if error else success_query_param}"
diff --git a/frappe/integrations/offsite_backup_utils.py b/frappe/integrations/offsite_backup_utils.py
index 307f1525fe..620f692ad0 100644
--- a/frappe/integrations/offsite_backup_utils.py
+++ b/frappe/integrations/offsite_backup_utils.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2019, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
@@ -13,8 +12,8 @@ def send_email(success, service_name, doctype, email_field, error_status=None):
 	recipients = get_recipients(doctype, email_field)
 	if not recipients:
 		frappe.log_error(
-			"No Email Recipient found for {0}".format(service_name),
-			"{0}: Failed to send backup status email".format(service_name),
+			f"No Email Recipient found for {service_name}",
+			f"{service_name}: Failed to send backup status email",
 		)
 		return
 
@@ -25,15 +24,15 @@ def send_email(success, service_name, doctype, email_field, error_status=None):
 		subject = "Backup Upload Successful"
 		message = """
 
        Backup Uploaded Successfully!
        
-
        Hi there, this is just to inform you that your backup was successfully uploaded to your {0} bucket. So relax!
        """.format(
+
        Hi there, this is just to inform you that your backup was successfully uploaded to your {} bucket. So relax!
        """.format(
 			service_name
 		)
 	else:
 		subject = "[Warning] Backup Upload Failed"
 		message = """
 
        Backup Upload Failed!
        
-
        Oops, your automated backup to {0} failed.
        
-
        Error message: {1}
        
+
        Oops, your automated backup to {} failed.
        
+
        Error message: {}
        
 
        Please contact your system manager for more information.
        """.format(
 			service_name, error_status
 		)
diff --git a/frappe/integrations/utils.py b/frappe/integrations/utils.py
index 191cd1f23b..5ae8965c83 100644
--- a/frappe/integrations/utils.py
+++ b/frappe/integrations/utils.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2019, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
@@ -42,22 +41,45 @@ def make_put_request(url, **kwargs):
 	return make_request("PUT", url, **kwargs)
 
 
-def create_request_log(data, integration_type, service_name, name=None, error=None):
-	if isinstance(data, str):
-		data = json.loads(data)
+def create_request_log(
+	data,
+	integration_type=None,
+	service_name=None,
+	name=None,
+	error=None,
+	request_headers=None,
+	output=None,
+	**kwargs,
+):
+	"""
+	DEPRECATED: The parameter integration_type will be removed in the next major release.
+	Use is_remote_request instead.
+	"""
+	if integration_type == "Remote":
+		kwargs["is_remote_request"] = 1
 
-	if isinstance(error, str):
-		error = json.loads(error)
+	elif integration_type == "Subscription Notification":
+		kwargs["request_description"] = integration_type
+
+	reference_doctype = reference_docname = None
+	if "reference_doctype" not in kwargs:
+		if isinstance(data, str):
+			data = json.loads(data)
+
+		reference_doctype = data.get("reference_doctype")
+		reference_docname = data.get("reference_docname")
 
 	integration_request = frappe.get_doc(
 		{
 			"doctype": "Integration Request",
-			"integration_type": integration_type,
 			"integration_request_service": service_name,
-			"reference_doctype": data.get("reference_doctype"),
-			"reference_docname": data.get("reference_docname"),
-			"error": json.dumps(error, default=json_handler),
-			"data": json.dumps(data, default=json_handler),
+			"request_headers": get_json(request_headers),
+			"data": get_json(data),
+			"output": get_json(output),
+			"error": get_json(error),
+			"reference_doctype": reference_doctype,
+			"reference_docname": reference_docname,
+			**kwargs,
 		}
 	)
 
@@ -70,52 +92,8 @@ def create_request_log(data, integration_type, service_name, name=None, error=No
 	return integration_request
 
 
-def get_payment_gateway_controller(payment_gateway):
-	"""Return payment gateway controller"""
-	gateway = frappe.get_doc("Payment Gateway", payment_gateway)
-	if gateway.gateway_controller is None:
-		try:
-			return frappe.get_doc("{0} Settings".format(payment_gateway))
-		except Exception:
-			frappe.throw(_("{0} Settings not found").format(payment_gateway))
-	else:
-		try:
-			return frappe.get_doc(gateway.gateway_settings, gateway.gateway_controller)
-		except Exception:
-			frappe.throw(_("{0} Settings not found").format(payment_gateway))
-
-
-@frappe.whitelist(allow_guest=True, xss_safe=True)
-def get_checkout_url(**kwargs):
-	try:
-		if kwargs.get("payment_gateway"):
-			doc = frappe.get_doc("{0} Settings".format(kwargs.get("payment_gateway")))
-			return doc.get_payment_url(**kwargs)
-		else:
-			raise Exception
-	except Exception:
-		frappe.respond_as_web_page(
-			_("Something went wrong"),
-			_(
-				"Looks like something is wrong with this site's payment gateway configuration. No payment has been made."
-			),
-			indicator_color="red",
-			http_status_code=frappe.ValidationError.http_status_code,
-		)
-
-
-def create_payment_gateway(gateway, settings=None, controller=None):
-	# NOTE: we don't translate Payment Gateway name because it is an internal doctype
-	if not frappe.db.exists("Payment Gateway", gateway):
-		payment_gateway = frappe.get_doc(
-			{
-				"doctype": "Payment Gateway",
-				"gateway": gateway,
-				"gateway_settings": settings,
-				"gateway_controller": controller,
-			}
-		)
-		payment_gateway.insert(ignore_permissions=True)
+def get_json(obj):
+	return obj if isinstance(obj, str) else frappe.as_json(obj, indent=1)
 
 
 def json_handler(obj):
diff --git a/frappe/integrations/workspace/integrations/integrations.json b/frappe/integrations/workspace/integrations/integrations.json
index bbd2e1199f..8d1dfd64af 100644
--- a/frappe/integrations/workspace/integrations/integrations.json
+++ b/frappe/integrations/workspace/integrations/integrations.json
@@ -1,6 +1,6 @@
 {
  "charts": [],
- "content": "[{\"type\":\"header\",\"data\":{\"text\":\"Reports & Masters\",\"col\":12}},{\"type\":\"card\",\"data\":{\"card_name\":\"Backup\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Google Services\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Authentication\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Payments\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Settings\",\"col\":4}}]",
+ "content": "[{\"type\":\"header\",\"data\":{\"text\":\"Reports & Masters\",\"col\":12}},{\"type\":\"card\",\"data\":{\"card_name\":\"Backup\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Google Services\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Authentication\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Settings\",\"col\":4}}]",
  "creation": "2020-03-02 15:16:18.714190",
  "docstatus": 0,
  "doctype": "Workspace",
@@ -106,11 +106,52 @@
   {
    "hidden": 0,
    "is_query_report": 0,
-   "label": "Authentication",
+   "label": "Settings",
    "link_count": 0,
    "onboard": 0,
    "type": "Card Break"
   },
+  {
+   "dependencies": "",
+   "hidden": 0,
+   "is_query_report": 0,
+   "label": "Webhook",
+   "link_count": 0,
+   "link_to": "Webhook",
+   "link_type": "DocType",
+   "onboard": 0,
+   "type": "Link"
+  },
+  {
+   "dependencies": "",
+   "hidden": 0,
+   "is_query_report": 0,
+   "label": "Slack Webhook URL",
+   "link_count": 0,
+   "link_to": "Slack Webhook URL",
+   "link_type": "DocType",
+   "onboard": 0,
+   "type": "Link"
+  },
+  {
+   "dependencies": "",
+   "hidden": 0,
+   "is_query_report": 0,
+   "label": "SMS Settings",
+   "link_count": 0,
+   "link_to": "SMS Settings",
+   "link_type": "DocType",
+   "onboard": 0,
+   "type": "Link"
+  },
+  {
+   "hidden": 0,
+   "is_query_report": 0,
+   "label": "Authentication",
+   "link_count": 4,
+   "onboard": 0,
+   "type": "Card Break"
+  },
   {
    "dependencies": "",
    "hidden": 0,
@@ -154,119 +195,16 @@
    "link_type": "DocType",
    "onboard": 0,
    "type": "Link"
-  },
-  {
-   "hidden": 0,
-   "is_query_report": 0,
-   "label": "Payments",
-   "link_count": 0,
-   "onboard": 0,
-   "type": "Card Break"
-  },
-  {
-   "dependencies": "",
-   "hidden": 0,
-   "is_query_report": 0,
-   "label": "Braintree Settings",
-   "link_count": 0,
-   "link_to": "Braintree Settings",
-   "link_type": "DocType",
-   "onboard": 0,
-   "type": "Link"
-  },
-  {
-   "dependencies": "",
-   "hidden": 0,
-   "is_query_report": 0,
-   "label": "PayPal Settings",
-   "link_count": 0,
-   "link_to": "PayPal Settings",
-   "link_type": "DocType",
-   "onboard": 0,
-   "type": "Link"
-  },
-  {
-   "dependencies": "",
-   "hidden": 0,
-   "is_query_report": 0,
-   "label": "Razorpay Settings",
-   "link_count": 0,
-   "link_to": "Razorpay Settings",
-   "link_type": "DocType",
-   "onboard": 0,
-   "type": "Link"
-  },
-  {
-   "dependencies": "",
-   "hidden": 0,
-   "is_query_report": 0,
-   "label": "Stripe Settings",
-   "link_count": 0,
-   "link_to": "Stripe Settings",
-   "link_type": "DocType",
-   "onboard": 0,
-   "type": "Link"
-  },
-  {
-   "dependencies": "",
-   "hidden": 0,
-   "is_query_report": 0,
-   "label": "Paytm Settings",
-   "link_count": 0,
-   "link_to": "Paytm Settings",
-   "link_type": "DocType",
-   "onboard": 0,
-   "type": "Link"
-  },
-  {
-   "hidden": 0,
-   "is_query_report": 0,
-   "label": "Settings",
-   "link_count": 0,
-   "onboard": 0,
-   "type": "Card Break"
-  },
-  {
-   "dependencies": "",
-   "hidden": 0,
-   "is_query_report": 0,
-   "label": "Webhook",
-   "link_count": 0,
-   "link_to": "Webhook",
-   "link_type": "DocType",
-   "onboard": 0,
-   "type": "Link"
-  },
-  {
-   "dependencies": "",
-   "hidden": 0,
-   "is_query_report": 0,
-   "label": "Slack Webhook URL",
-   "link_count": 0,
-   "link_to": "Slack Webhook URL",
-   "link_type": "DocType",
-   "onboard": 0,
-   "type": "Link"
-  },
-  {
-   "dependencies": "",
-   "hidden": 0,
-   "is_query_report": 0,
-   "label": "SMS Settings",
-   "link_count": 0,
-   "link_to": "SMS Settings",
-   "link_type": "DocType",
-   "onboard": 0,
-   "type": "Link"
   }
  ],
- "modified": "2022-01-13 17:39:01.292154",
+ "modified": "2022-07-23 18:00:28.805405",
  "modified_by": "Administrator",
  "module": "Integrations",
  "name": "Integrations",
  "owner": "Administrator",
  "parent_page": "",
  "public": 1,
+ "quick_lists": [],
  "restrict_to_domain": "",
  "roles": [],
  "sequence_id": 15.0,
diff --git a/frappe/middlewares.py b/frappe/middlewares.py
index cd47b7210f..168d129ebe 100644
--- a/frappe/middlewares.py
+++ b/frappe/middlewares.py
@@ -13,7 +13,7 @@ from frappe.utils import cstr, get_site_name
 class StaticDataMiddleware(SharedDataMiddleware):
 	def __call__(self, environ, start_response):
 		self.environ = environ
-		return super(StaticDataMiddleware, self).__call__(environ, start_response)
+		return super().__call__(environ, start_response)
 
 	def get_directory_loader(self, directory):
 		def loader(path):
diff --git a/frappe/migrate.py b/frappe/migrate.py
index bb83fa5b6d..1c249dfdb1 100644
--- a/frappe/migrate.py
+++ b/frappe/migrate.py
@@ -159,13 +159,13 @@ class SiteMigration:
 		"""Run Migrate operation on site specified. This method initializes
 		and destroys connections to the site database.
 		"""
-		if not self.required_services_running():
-			raise SystemExit(1)
-
 		if site:
 			frappe.init(site=site)
 			frappe.connect()
 
+		if not self.required_services_running():
+			raise SystemExit(1)
+
 		self.setUp()
 		try:
 			self.pre_schema_updates()
diff --git a/frappe/model/__init__.py b/frappe/model/__init__.py
index bd607e7119..29991fa403 100644
--- a/frappe/model/__init__.py
+++ b/frappe/model/__init__.py
@@ -36,10 +36,14 @@ data_fieldtypes = (
 	"Geolocation",
 	"Duration",
 	"Icon",
+	"Phone",
 	"Autocomplete",
 	"JSON",
 )
 
+float_like_fields = {"Float", "Currency", "Percent"}
+datetime_fields = {"Datetime", "Date", "Time"}
+
 attachment_fieldtypes = (
 	"Attach",
 	"Attach Image",
diff --git a/frappe/model/base_document.py b/frappe/model/base_document.py
index 4f2ddd3bb6..1162ceacd3 100644
--- a/frappe/model/base_document.py
+++ b/frappe/model/base_document.py
@@ -4,8 +4,15 @@ import datetime
 import json
 
 import frappe
-from frappe import _
-from frappe.model import child_table_fields, default_fields, display_fieldtypes, table_fields
+from frappe import _, _dict
+from frappe.model import (
+	child_table_fields,
+	datetime_fields,
+	default_fields,
+	display_fieldtypes,
+	float_like_fields,
+	table_fields,
+)
 from frappe.model.docstatus import DocStatus
 from frappe.model.naming import set_new_name
 from frappe.model.utils.link_count import notify_link_count
@@ -13,9 +20,18 @@ from frappe.modules import load_doctype_module
 from frappe.utils import cast_fieldtype, cint, cstr, flt, now, sanitize_html, strip_html
 from frappe.utils.html_utils import unescape_html
 
-max_positive_value = {"smallint": 2**15, "int": 2**31, "bigint": 2**63}
+max_positive_value = {"smallint": 2**15 - 1, "int": 2**31 - 1, "bigint": 2**63 - 1}
 
-DOCTYPES_FOR_DOCTYPE = ("DocType", "DocField", "DocPerm", "DocType Action", "DocType Link")
+DOCTYPE_TABLE_FIELDS = [
+	_dict(fieldname="fields", options="DocField"),
+	_dict(fieldname="permissions", options="DocPerm"),
+	_dict(fieldname="actions", options="DocType Action"),
+	_dict(fieldname="links", options="DocType Link"),
+	_dict(fieldname="states", options="DocType State"),
+]
+
+TABLE_DOCTYPES_FOR_DOCTYPE = {df["fieldname"]: df["options"] for df in DOCTYPE_TABLE_FIELDS}
+DOCTYPES_FOR_DOCTYPE = {"DocType", *TABLE_DOCTYPES_FOR_DOCTYPE.values()}
 
 
 def get_controller(doctype):
@@ -42,9 +58,7 @@ def get_controller(doctype):
 				module_path, classname = import_path.rsplit(".", 1)
 				module = frappe.get_module(module_path)
 				if not hasattr(module, classname):
-					raise ImportError(
-						"{0}: {1} does not exist in module {2}".format(doctype, classname, module_path)
-					)
+					raise ImportError(f"{doctype}: {classname} does not exist in module {module_path}")
 			else:
 				module = load_doctype_module(doctype, module_name)
 				classname = doctype.replace(" ", "").replace("-", "")
@@ -69,13 +83,29 @@ def get_controller(doctype):
 	return site_controllers[doctype]
 
 
-class BaseDocument(object):
-	ignore_in_setter = ("doctype", "_meta", "meta", "_table_fields", "_valid_columns")
+class BaseDocument:
+	_reserved_keywords = {
+		"doctype",
+		"meta",
+		"_meta",
+		"flags",
+		"_table_fields",
+		"_valid_columns",
+		"_table_fieldnames",
+		"_reserved_keywords",
+		"dont_update_if_missing",
+	}
 
 	def __init__(self, d):
 		if d.get("doctype"):
 			self.doctype = d["doctype"]
 
+		self._table_fieldnames = (
+			d["_table_fieldnames"]  # from cache
+			if "_table_fieldnames" in d
+			else {df.fieldname for df in self._get_table_fields()}
+		)
+
 		self.update(d)
 		self.dont_update_if_missing = []
 
@@ -84,14 +114,29 @@ class BaseDocument(object):
 
 	@property
 	def meta(self):
-		if not getattr(self, "_meta", None):
-			self._meta = frappe.get_meta(self.doctype)
+		if not (meta := getattr(self, "_meta", None)):
+			self._meta = meta = frappe.get_meta(self.doctype)
 
-		return self._meta
+		return meta
 
 	def __getstate__(self):
-		self._meta = None
-		return self.__dict__
+		"""
+		Called when pickling.
+		Returns a copy of `__dict__` excluding unpicklable values like `_meta`.
+
+		More info: https://docs.python.org/3/library/pickle.html#handling-stateful-objects
+		"""
+
+		# Always use the dict.copy() method to avoid modifying the original state
+		state = self.__dict__.copy()
+		self.remove_unpicklable_values(state)
+
+		return state
+
+	def remove_unpicklable_values(self, state):
+		"""Remove unpicklable values before pickling"""
+
+		state.pop("_meta", None)
 
 	def update(self, d):
 		"""Update multiple fields of a doctype using a dictionary of key-value pairs.
@@ -136,17 +181,12 @@ class BaseDocument(object):
 
 		if filters:
 			if isinstance(filters, dict):
-				value = _filter(self.__dict__.get(key, []), filters, limit=limit)
-			else:
-				default = filters
-				filters = None
-				value = self.__dict__.get(key, default)
-		else:
-			value = self.__dict__.get(key, default)
+				return _filter(self.__dict__.get(key, []), filters, limit=limit)
 
-		if value is None and key in (d.fieldname for d in self.meta.get_table_fields()):
-			value = []
-			self.set(key, value)
+			# perhaps you wanted to set a default instead
+			default = filters
+
+		value = self.__dict__.get(key, default)
 
 		if limit and isinstance(value, (list, tuple)) and len(value) > limit:
 			value = value[:limit]
@@ -157,14 +197,19 @@ class BaseDocument(object):
 		return self.get(key, filters=filters, limit=1)[0]
 
 	def set(self, key, value, as_value=False):
-		if key in self.ignore_in_setter:
+		if key in self._reserved_keywords:
 			return
 
-		if isinstance(value, list) and not as_value:
+		if not as_value and key in self._table_fieldnames:
 			self.__dict__[key] = []
-			self.extend(key, value)
-		else:
-			self.__dict__[key] = value
+
+			# if value is falsy, just init to an empty list
+			if value:
+				self.extend(key, value)
+
+			return
+
+		self.__dict__[key] = value
 
 	def delete_key(self, key):
 		if key in self.__dict__:
@@ -182,41 +227,27 @@ class BaseDocument(object):
 		"""
 		if value is None:
 			value = {}
-		if isinstance(value, (dict, BaseDocument)):
-			if not self.__dict__.get(key):
-				self.__dict__[key] = []
 
-			value = self._init_child(value, key)
-			self.__dict__[key].append(value)
+		if (table := self.__dict__.get(key)) is None:
+			self.__dict__[key] = table = []
 
-			# reference parent document
-			value.parent_doc = self
+		value = self._init_child(value, key)
+		table.append(value)
 
-			return value
-		else:
+		# reference parent document
+		value.parent_doc = self
 
-			# metaclasses may have arbitrary lists
-			# which we can ignore
-			if getattr(self, "_metaclass", None) or self.__class__.__name__ in (
-				"Meta",
-				"FormMeta",
-				"DocField",
-			):
-				return value
-
-			raise ValueError(
-				'Document for field "{0}" attached to child table of "{1}" must be a dict or BaseDocument, not {2} ({3})'.format(
-					key, self.name, str(type(value))[1:-1], value
-				)
-			)
+		return value
 
 	def extend(self, key, value):
-		if isinstance(value, list):
-			for v in value:
-				self.append(key, v)
-		else:
+		try:
+			value = iter(value)
+		except TypeError:
 			raise ValueError
 
+		for v in value:
+			self.append(key, v)
+
 	def remove(self, doc):
 		# Usage: from the parent doc, pass the child table doc
 		# to remove that child doc from the child table, thus removing it from the parent doc
@@ -224,16 +255,12 @@ class BaseDocument(object):
 			self.get(doc.parentfield).remove(doc)
 
 	def _init_child(self, value, key):
-		if not self.doctype:
-			return value
-
 		if not isinstance(value, BaseDocument):
-			value["doctype"] = self.get_table_field_doctype(key)
-			if not value["doctype"]:
+			if not (doctype := self.get_table_field_doctype(key)):
 				raise AttributeError(key)
 
-			value = get_controller(value["doctype"])(value)
-			value.init_valid_columns()
+			value["doctype"] = doctype
+			value = get_controller(doctype)(value)
 
 		value.parent = self.name
 		value.parenttype = self.doctype
@@ -243,19 +270,38 @@ class BaseDocument(object):
 			value.docstatus = DocStatus.draft()
 
 		if not getattr(value, "idx", None):
-			value.idx = len(self.get(key) or []) + 1
+			if table := getattr(self, key, None):
+				value.idx = len(table) + 1
+			else:
+				value.idx = 1
 
 		if not getattr(value, "name", None):
 			value.__dict__["__islocal"] = 1
 
 		return value
 
+	def _get_table_fields(self):
+		"""
+		To get table fields during Document init
+		Meta.get_table_fields goes into recursion for special doctypes
+		"""
+
+		if self.doctype == "DocType":
+			return DOCTYPE_TABLE_FIELDS
+
+		# child tables don't have child tables
+		if self.doctype in DOCTYPES_FOR_DOCTYPE or getattr(self, "parentfield", None):
+			return ()
+
+		return self.meta.get_table_fields()
+
 	def get_valid_dict(
 		self, sanitize=True, convert_dates_to_str=False, ignore_nulls=False, ignore_virtual=False
-	):
-		d = frappe._dict()
+	) -> dict:
+		d = _dict()
 		for fieldname in self.meta.get_valid_columns():
-			d[fieldname] = self.get(fieldname)
+			# column is valid, we can use getattr
+			d[fieldname] = getattr(self, fieldname, None)
 
 			# if no need for sanitization and value is None, continue
 			if not sanitize and d[fieldname] is None:
@@ -263,25 +309,24 @@ class BaseDocument(object):
 
 			df = self.meta.get_field(fieldname)
 
-			if df and df.get("is_virtual"):
-				if ignore_virtual:
-					del d[fieldname]
-					continue
+			if df:
+				if getattr(df, "is_virtual", False):
+					if ignore_virtual:
+						del d[fieldname]
+						continue
 
-				from frappe.utils.safe_exec import get_safe_globals
+					if d[fieldname] is None and (options := getattr(df, "options", None)):
+						from frappe.utils.safe_exec import get_safe_globals
 
-				if d[fieldname] is None:
-					if df.get("options"):
 						d[fieldname] = frappe.safe_eval(
-							code=df.get("options"),
+							code=options,
 							eval_globals=get_safe_globals(),
 							eval_locals={"doc": self},
 						)
-					else:
-						_val = getattr(self, fieldname, None)
-						if _val and not callable(_val):
-							d[fieldname] = _val
-			elif df:
+
+				if isinstance(d[fieldname], list) and df.fieldtype not in table_fields:
+					frappe.throw(_("Value for {0} cannot be a list").format(_(df.label)))
+
 				if df.fieldtype == "Check":
 					d[fieldname] = 1 if cint(d[fieldname]) else 0
 
@@ -291,29 +336,34 @@ class BaseDocument(object):
 				elif df.fieldtype == "JSON" and isinstance(d[fieldname], dict):
 					d[fieldname] = json.dumps(d[fieldname], sort_keys=True, indent=4, separators=(",", ": "))
 
-				elif df.fieldtype in ("Currency", "Float", "Percent") and not isinstance(d[fieldname], float):
+				elif df.fieldtype in float_like_fields and not isinstance(d[fieldname], float):
 					d[fieldname] = flt(d[fieldname])
 
-				elif df.fieldtype in ("Datetime", "Date", "Time") and d[fieldname] == "":
+				elif (df.fieldtype in datetime_fields and d[fieldname] == "") or (
+					getattr(df, "unique", False) and cstr(d[fieldname]).strip() == ""
+				):
 					d[fieldname] = None
 
-				elif df.get("unique") and cstr(d[fieldname]).strip() == "":
-					# unique empty field should be set to None
-					d[fieldname] = None
-
-				if isinstance(d[fieldname], list) and df.fieldtype not in table_fields:
-					frappe.throw(_("Value for {0} cannot be a list").format(_(df.label)))
-
 			if convert_dates_to_str and isinstance(
 				d[fieldname], (datetime.datetime, datetime.date, datetime.time, datetime.timedelta)
 			):
 				d[fieldname] = str(d[fieldname])
 
-			if d[fieldname] is None and ignore_nulls:
+			if ignore_nulls and d[fieldname] is None:
 				del d[fieldname]
 
 		return d
 
+	def init_child_tables(self):
+		"""
+		This is needed so that one can loop over child table properties
+		without worrying about whether or not they have values
+		"""
+
+		for fieldname in self._table_fieldnames:
+			if self.__dict__.get(fieldname) is None:
+				self.__dict__[fieldname] = []
+
 	def init_valid_columns(self):
 		for key in default_fields:
 			if key not in self.__dict__:
@@ -329,7 +379,7 @@ class BaseDocument(object):
 			if key not in self.__dict__:
 				self.__dict__[key] = None
 
-	def get_valid_columns(self):
+	def get_valid_columns(self) -> list[str]:
 		if self.doctype not in frappe.local.valid_columns:
 			if self.doctype in DOCTYPES_FOR_DOCTYPE:
 				from frappe.model.meta import get_table_columns
@@ -342,12 +392,12 @@ class BaseDocument(object):
 
 		return frappe.local.valid_columns[self.doctype]
 
-	def is_new(self):
+	def is_new(self) -> bool:
 		return self.get("__islocal")
 
 	@property
 	def docstatus(self):
-		return DocStatus(self.get("docstatus"))
+		return DocStatus(cint(self.get("docstatus")))
 
 	@docstatus.setter
 	def docstatus(self, value):
@@ -359,13 +409,13 @@ class BaseDocument(object):
 		no_default_fields=False,
 		convert_dates_to_str=False,
 		no_child_table_fields=False,
-	):
-		doc = self.get_valid_dict(convert_dates_to_str=convert_dates_to_str)
+	) -> dict:
+		doc = self.get_valid_dict(convert_dates_to_str=convert_dates_to_str, ignore_nulls=no_nulls)
 		doc["doctype"] = self.doctype
 
-		for df in self.meta.get_table_fields():
-			children = self.get(df.fieldname) or []
-			doc[df.fieldname] = [
+		for fieldname in self._table_fieldnames:
+			children = self.get(fieldname) or []
+			doc[fieldname] = [
 				d.as_dict(
 					convert_dates_to_str=convert_dates_to_str,
 					no_nulls=no_nulls,
@@ -375,20 +425,15 @@ class BaseDocument(object):
 				for d in children
 			]
 
-		if no_nulls:
-			for k in list(doc):
-				if doc[k] is None:
-					del doc[k]
-
 		if no_default_fields:
-			for k in list(doc):
-				if k in default_fields:
-					del doc[k]
+			for key in default_fields:
+				if key in doc:
+					del doc[key]
 
 		if no_child_table_fields:
-			for k in list(doc):
-				if k in child_table_fields:
-					del doc[k]
+			for key in child_table_fields:
+				if key in doc:
+					del doc[key]
 
 		for key in (
 			"_user_tags",
@@ -398,8 +443,8 @@ class BaseDocument(object):
 			"__run_link_triggers",
 			"__unsaved",
 		):
-			if self.get(key):
-				doc[key] = self.get(key)
+			if value := getattr(self, key, None):
+				doc[key] = value
 
 		return doc
 
@@ -410,10 +455,9 @@ class BaseDocument(object):
 		try:
 			return self.meta.get_field(fieldname).options
 		except AttributeError:
-			if self.doctype == "DocType":
-				return dict(links="DocType Link", actions="DocType Action", states="DocType State").get(
-					fieldname
-				)
+			if self.doctype == "DocType" and (table_doctype := TABLE_DOCTYPES_FOR_DOCTYPE.get(fieldname)):
+				return table_doctype
+
 			raise
 
 	def get_parentfield_of_doctype(self, doctype):
@@ -495,7 +539,9 @@ class BaseDocument(object):
 			return
 
 		d = self.get_valid_dict(
-			convert_dates_to_str=True, ignore_nulls=self.doctype in DOCTYPES_FOR_DOCTYPE
+			convert_dates_to_str=True,
+			ignore_nulls=self.doctype in DOCTYPES_FOR_DOCTYPE,
+			ignore_virtual=True,
 		)
 
 		# don't update name, as case might've been changed
@@ -522,8 +568,8 @@ class BaseDocument(object):
 		"""Raw update parent + children
 		DOES NOT VALIDATE AND CALL TRIGGERS"""
 		self.db_update()
-		for df in self.meta.get_table_fields():
-			for doc in self.get(df.fieldname):
+		for fieldname in self._table_fieldnames:
+			for doc in self.get(fieldname):
 				doc.db_update()
 
 	def show_unique_validation_message(self, e):
@@ -635,7 +681,7 @@ class BaseDocument(object):
 		if self.meta.istable:
 			for fieldname in ("parent", "parenttype"):
 				if not self.get(fieldname):
-					missing.append((fieldname, get_msg(frappe._dict(label=fieldname))))
+					missing.append((fieldname, get_msg(_dict(label=fieldname))))
 
 		return missing
 
@@ -647,7 +693,7 @@ class BaseDocument(object):
 			if self.get("parentfield"):
 				return "{} #{}: {}: {}".format(_("Row"), self.idx, _(df.label), docname)
 
-			return "{}: {}".format(_(df.label), docname)
+			return f"{_(df.label)}: {docname}"
 
 		invalid_links = []
 		cancelled_links = []
@@ -682,7 +728,7 @@ class BaseDocument(object):
 				if not frappe.get_meta(doctype).get("is_virtual"):
 					if not fields_to_fetch:
 						# cache a single value type
-						values = frappe._dict(name=frappe.db.get_value(doctype, docname, "name", cache=True))
+						values = _dict(name=frappe.db.get_value(doctype, docname, "name", cache=True))
 					else:
 						values_to_fetch = ["name"] + [_df.fetch_from.split(".")[-1] for _df in fields_to_fetch]
 
@@ -771,6 +817,10 @@ class BaseDocument(object):
 
 	def _validate_data_fields(self):
 		# data_field options defined in frappe.model.data_field_options
+		for phone_field in self.meta.get_phone_fields():
+			phone = self.get(phone_field.fieldname)
+			frappe.utils.validate_phone_number_with_country_code(phone, phone_field.fieldname)
+
 		for data_field in self.meta.get_data_fields():
 			data = self.get(data_field.fieldname)
 			data_field_options = data_field.get("options")
@@ -869,7 +919,7 @@ class BaseDocument(object):
 		autoname = self.meta.autoname or ""
 		_empty, _field_specifier, fieldname = autoname.partition("field:")
 
-		if fieldname and self.name and self.name != self.get("fieldname"):
+		if fieldname and self.name and self.name != self.get(fieldname):
 			self.set(fieldname, self.name)
 
 	def throw_length_exceeded_error(self, df, max_length, value):
@@ -877,7 +927,7 @@ class BaseDocument(object):
 		if self.get("parentfield"):
 			reference = _("{0}, Row {1}").format(_(self.doctype), self.idx)
 		else:
-			reference = "{0} {1}".format(_(self.doctype), self.name)
+			reference = f"{_(self.doctype)} {self.name}"
 
 		frappe.throw(
 			_("{0}: '{1}' ({3}) will get truncated, as max characters allowed is {2}").format(
@@ -1008,10 +1058,10 @@ class BaseDocument(object):
 		cache_key = parentfield or "main"
 
 		if not hasattr(self, "_precision"):
-			self._precision = frappe._dict()
+			self._precision = _dict()
 
 		if cache_key not in self._precision:
-			self._precision[cache_key] = frappe._dict()
+			self._precision[cache_key] = _dict()
 
 		if fieldname not in self._precision[cache_key]:
 			self._precision[cache_key][fieldname] = None
@@ -1132,7 +1182,7 @@ class BaseDocument(object):
 		return cast_fieldtype(df.fieldtype, value, show_warning=False)
 
 	def _extract_images_from_text_editor(self):
-		from frappe.core.doctype.file.file import extract_images_from_doc
+		from frappe.core.doctype.file.utils import extract_images_from_doc
 
 		if self.doctype != "DocType":
 			for df in self.meta.get("fields", {"fieldtype": ("=", "Text Editor")}):
diff --git a/frappe/model/create_new.py b/frappe/model/create_new.py
index 8671008f82..51810c3e18 100644
--- a/frappe/model/create_new.py
+++ b/frappe/model/create_new.py
@@ -23,8 +23,6 @@ def get_new_doc(doctype, parent_doc=None, parentfield=None, as_dict=False):
 
 	doc = copy.deepcopy(frappe.local.new_doc_templates[doctype])
 
-	# doc = make_new_doc(doctype)
-
 	set_dynamic_default_values(doc, parent_doc, parentfield)
 
 	if as_dict:
diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py
index a7d9536ebc..a29ede37bf 100644
--- a/frappe/model/db_query.py
+++ b/frappe/model/db_query.py
@@ -6,7 +6,6 @@ import copy
 import json
 import re
 from datetime import datetime
-from typing import List
 
 import frappe
 import frappe.defaults
@@ -14,6 +13,7 @@ import frappe.permissions
 import frappe.share
 from frappe import _
 from frappe.core.doctype.server_script.server_script_utils import get_server_script_map
+from frappe.database.utils import FallBackDateTimeStr
 from frappe.model import optional_fields
 from frappe.model.meta import get_table_columns
 from frappe.model.utils.user_settings import get_user_settings, update_user_settings
@@ -29,11 +29,34 @@ from frappe.utils import (
 	make_filter_tuple,
 )
 
+LOCATE_PATTERN = re.compile(r"locate\([^,]+,\s*[`\"]?name[`\"]?\s*\)", flags=re.IGNORECASE)
+LOCATE_CAST_PATTERN = re.compile(
+	r"locate\(([^,]+),\s*([`\"]?name[`\"]?)\s*\)", flags=re.IGNORECASE
+)
+FUNC_IFNULL_PATTERN = re.compile(
+	r"(strpos|ifnull|coalesce)\(\s*[`\"]?name[`\"]?\s*,", flags=re.IGNORECASE
+)
+CAST_VARCHAR_PATTERN = re.compile(
+	r"([`\"]?tab[\w`\" -]+\.[`\"]?name[`\"]?)(?!\w)", flags=re.IGNORECASE
+)
+ORDER_BY_PATTERN = re.compile(r"\ order\ by\ |\ asc|\ ASC|\ desc|\ DESC", flags=re.IGNORECASE)
+SUB_QUERY_PATTERN = re.compile("^.*[,();@].*")
+IS_QUERY_PATTERN = re.compile(r"^(select|delete|update|drop|create)\s")
+IS_QUERY_PREDICATE_PATTERN = re.compile(
+	r"\s*[0-9a-zA-z]*\s*( from | group by | order by | where | join )"
+)
+FIELD_QUOTE_PATTERN = re.compile(r"[0-9a-zA-Z]+\s*'")
+FIELD_COMMA_PATTERN = re.compile(r"[0-9a-zA-Z]+\s*,")
+STRICT_FIELD_PATTERN = re.compile(r".*/\*.*")
+STRICT_UNION_PATTERN = re.compile(r".*\s(union).*\s")
+ORDER_GROUP_PATTERN = re.compile(r".*[^a-z0-9-_ ,`'\"\.\(\)].*")
 
-class DatabaseQuery(object):
+
+class DatabaseQuery:
 	def __init__(self, doctype, user=None):
 		self.doctype = doctype
 		self.tables = []
+		self.link_tables = []
 		self.conditions = []
 		self.or_conditions = []
 		self.fields = None
@@ -75,7 +98,7 @@ class DatabaseQuery(object):
 		pluck=None,
 		ignore_ddl=False,
 		parent_doctype=None,
-	) -> List:
+	) -> list:
 
 		if (
 			not ignore_permissions
@@ -213,9 +236,13 @@ class DatabaseQuery(object):
 
 		# left join parent, child tables
 		for child in self.tables[1:]:
-			parent_name = self.cast_name(f"{self.tables[0]}.name")
+			parent_name = cast_name(f"{self.tables[0]}.name")
 			args.tables += f" {self.join} {child} on ({child}.parent = {parent_name})"
 
+		# left join link tables
+		for link in self.link_tables:
+			args.tables += f" {self.join} `tab{link.doctype}` on (`tab{link.doctype}`.`name` = {self.tables[0]}.`{link.fieldname}`)"
+
 		if self.grouped_or_conditions:
 			self.conditions.append(f"({' or '.join(self.grouped_or_conditions)})")
 
@@ -225,6 +252,7 @@ class DatabaseQuery(object):
 			args.conditions += (" or " if args.conditions else "") + " or ".join(self.or_conditions)
 
 		self.set_field_tables()
+		self.cast_name_fields()
 
 		fields = []
 
@@ -260,7 +288,7 @@ class DatabaseQuery(object):
 		return args
 
 	def prepare_select_args(self, args):
-		order_field = re.sub(r"\ order\ by\ |\ asc|\ ASC|\ desc|\ DESC", "", args.order_by)
+		order_field = ORDER_BY_PATTERN.sub("", args.order_by)
 
 		if order_field not in args.fields:
 			extracted_column = order_column = order_field.replace("`", "")
@@ -286,6 +314,23 @@ class DatabaseQuery(object):
 		# remove empty strings / nulls in fields
 		self.fields = [f for f in self.fields if f]
 
+		# convert child_table.fieldname to `tabChild DocType`.`fieldname`
+		for field in self.fields:
+			if "." in field and "tab" not in field:
+				original_field = field
+				alias = None
+				if " as " in field:
+					field, alias = field.split(" as ")
+				linked_fieldname, fieldname = field.split(".")
+				linked_field = frappe.get_meta(self.doctype).get_field(linked_fieldname)
+				linked_doctype = linked_field.options
+				if linked_field.fieldtype == "Link":
+					self.append_link_table(linked_doctype, linked_fieldname)
+				field = f"`tab{linked_doctype}`.`{fieldname}`"
+				if alias:
+					field = f"{field} as {alias}"
+				self.fields[self.fields.index(original_field)] = field
+
 		for filter_name in ["filters", "or_filters"]:
 			filters = getattr(self, filter_name)
 			if isinstance(filters, str):
@@ -308,8 +353,6 @@ class DatabaseQuery(object):
 		As field contains `,` and mysql function `version()`, with the help of regex
 		the system will filter out this field.
 		"""
-
-		sub_query_regex = re.compile("^.*[,();@].*")
 		blacklisted_keywords = ["select", "create", "insert", "delete", "drop", "update", "case", "show"]
 		blacklisted_functions = [
 			"concat",
@@ -333,19 +376,14 @@ class DatabaseQuery(object):
 			frappe.throw(_("Use of sub-query or function is restricted"), frappe.DataError)
 
 		def _is_query(field):
-			if re.compile(r"^(select|delete|update|drop|create)\s").match(field):
+			if IS_QUERY_PATTERN.match(field):
 				_raise_exception()
 
-			elif re.compile(r"\s*[0-9a-zA-z]*\s*( from | group by | order by | where | join )").match(
-				field
-			):
+			elif IS_QUERY_PREDICATE_PATTERN.match(field):
 				_raise_exception()
 
 		for field in self.fields:
-			if sub_query_regex.match(field):
-				if any(keyword in field.lower().split() for keyword in blacklisted_keywords):
-					_raise_exception()
-
+			if SUB_QUERY_PATTERN.match(field):
 				if any(f"({keyword}" in field.lower() for keyword in blacklisted_keywords):
 					_raise_exception()
 
@@ -356,19 +394,19 @@ class DatabaseQuery(object):
 					# prevent access to global variables
 					_raise_exception()
 
-			if re.compile(r"[0-9a-zA-Z]+\s*'").match(field):
+			if FIELD_QUOTE_PATTERN.match(field):
 				_raise_exception()
 
-			if re.compile(r"[0-9a-zA-Z]+\s*,").match(field):
+			if FIELD_COMMA_PATTERN.match(field):
 				_raise_exception()
 
 			_is_query(field)
 
 			if self.strict:
-				if re.compile(r".*/\*.*").match(field):
+				if STRICT_FIELD_PATTERN.match(field):
 					frappe.throw(_("Illegal SQL Query"))
 
-				if re.compile(r".*\s(union).*\s").match(field.lower()):
+				if STRICT_UNION_PATTERN.match(field.lower()):
 					frappe.throw(_("Illegal SQL Query"))
 
 	def extract_tables(self):
@@ -385,16 +423,8 @@ class DatabaseQuery(object):
 		]
 		# add tables from fields
 		if self.fields:
-			for i, field in enumerate(self.fields):
-				# add cast in locate/strpos
-				func_found = False
-				for func in sql_functions:
-					if func in field.lower():
-						self.fields[i] = self.cast_name(field, func)
-						func_found = True
-						break
-
-				if func_found or not ("tab" in field and "." in field):
+			for field in self.fields:
+				if not ("tab" in field and "." in field) or any(x for x in sql_functions if x in field):
 					continue
 
 				table_name = field.split(".")[0]
@@ -403,44 +433,27 @@ class DatabaseQuery(object):
 					table_name = table_name[13:]
 				if not table_name[0] == "`":
 					table_name = f"`{table_name}`"
-				if table_name not in self.tables:
+				if table_name not in self.tables and table_name not in (
+					d.table_name for d in self.link_tables
+				):
 					self.append_table(table_name)
 
-	def cast_name(
-		self,
-		column: str,
-		sql_function: str = "",
-	) -> str:
-		if frappe.db.db_type == "postgres":
-			if "name" in column.lower():
-				if "cast(" not in column.lower() or "::" not in column:
-					if not sql_function:
-						return f"cast({column} as varchar)"
-
-					elif sql_function == "locate(":
-						return re.sub(
-							r"locate\(([^,]+),([^)]+)\)",
-							r"locate(\1, cast(\2 as varchar))",
-							column,
-							flags=re.IGNORECASE,
-						)
-
-					elif sql_function == "strpos(":
-						return re.sub(
-							r"strpos\(([^,]+),([^)]+)\)",
-							r"strpos(cast(\1 as varchar), \2)",
-							column,
-							flags=re.IGNORECASE,
-						)
-
-					elif sql_function == "ifnull(":
-						return re.sub(r"ifnull\(([^,]+)", r"ifnull(cast(\1 as varchar)", column, flags=re.IGNORECASE)
-
-		return column
-
 	def append_table(self, table_name):
 		self.tables.append(table_name)
 		doctype = table_name[4:-1]
+		self.check_read_permission(doctype)
+
+	def append_link_table(self, doctype, fieldname):
+		for d in self.link_tables:
+			if d.doctype == doctype and d.fieldname == fieldname:
+				return
+
+		self.check_read_permission(doctype)
+		self.link_tables.append(
+			frappe._dict(doctype=doctype, fieldname=fieldname, table_name=f"`tab{doctype}`")
+		)
+
+	def check_read_permission(self, doctype):
 		ptype = "select" if frappe.only_has_select_perm(doctype) else "read"
 
 		if not self.flags.ignore_permissions and not frappe.has_permission(
@@ -457,11 +470,15 @@ class DatabaseQuery(object):
 			methods = ("count(", "avg(", "sum(", "extract(", "dayofyear(")
 			return field.lower().startswith(methods)
 
-		if len(self.tables) > 1:
+		if len(self.tables) > 1 or len(self.link_tables) > 0:
 			for idx, field in enumerate(self.fields):
 				if "." not in field and not _in_standard_sql_methods(field):
 					self.fields[idx] = f"{self.tables[0]}.{field}"
 
+	def cast_name_fields(self):
+		for i, field in enumerate(self.fields):
+			self.fields[i] = cast_name(field)
+
 	def get_table_columns(self):
 		try:
 			return get_table_columns(self.doctype)
@@ -541,10 +558,7 @@ class DatabaseQuery(object):
 		if tname not in self.tables:
 			self.append_table(tname)
 
-		if "ifnull(" in f.fieldname:
-			column_name = self.cast_name(f.fieldname, "ifnull(")
-		else:
-			column_name = self.cast_name(f"{tname}.`{f.fieldname}`")
+		column_name = cast_name(f.fieldname if "ifnull(" in f.fieldname else f"{tname}.`{f.fieldname}`")
 
 		if f.operator.lower() in additional_filters_config:
 			f.update(get_additional_filter_field(additional_filters_config, f, f.value))
@@ -619,11 +633,11 @@ class DatabaseQuery(object):
 				date_range = get_date_range(f.operator.lower(), f.value)
 				f.operator = "Between"
 				f.value = date_range
-				fallback = "'0001-01-01 00:00:00'"
+				fallback = f"'{FallBackDateTimeStr}'"
 
 			if f.operator in (">", "<") and (f.fieldname in ("creation", "modified")):
 				value = cstr(f.value)
-				fallback = "'0001-01-01 00:00:00'"
+				fallback = f"'{FallBackDateTimeStr}'"
 
 			elif f.operator.lower() in ("between") and (
 				f.fieldname in ("creation", "modified")
@@ -631,7 +645,7 @@ class DatabaseQuery(object):
 			):
 
 				value = get_between_date_filter(f.value, df)
-				fallback = "'0001-01-01 00:00:00'"
+				fallback = f"'{FallBackDateTimeStr}'"
 
 			elif f.operator.lower() == "is":
 				if f.value == "set":
@@ -652,7 +666,7 @@ class DatabaseQuery(object):
 
 			elif (df and df.fieldtype == "Datetime") or isinstance(f.value, datetime):
 				value = frappe.db.format_datetime(f.value)
-				fallback = "'0001-01-01 00:00:00'"
+				fallback = f"'{FallBackDateTimeStr}'"
 
 			elif df and df.fieldtype == "Time":
 				value = get_time(f.value).strftime("%H:%M:%S.%f")
@@ -706,7 +720,7 @@ class DatabaseQuery(object):
 
 		return condition
 
-	def build_match_conditions(self, as_condition=True):
+	def build_match_conditions(self, as_condition=True) -> str | list:
 		"""add match conditions if applicable"""
 		self.match_filters = []
 		self.match_conditions = []
@@ -729,7 +743,7 @@ class DatabaseQuery(object):
 		):
 			only_if_shared = True
 			if not self.shared:
-				frappe.throw(_("No permission to read {0}").format(self.doctype), frappe.PermissionError)
+				frappe.throw(_("No permission to read {0}").format(_(self.doctype)), frappe.PermissionError)
 			else:
 				self.conditions.append(self.get_share_condition())
 
@@ -766,7 +780,10 @@ class DatabaseQuery(object):
 			return self.match_filters
 
 	def get_share_condition(self):
-		return f"`tab{self.doctype}`.name in ({', '.join(frappe.db.escape(s, percent=False) for s in self.shared)})"
+		return (
+			cast_name(f"`tab{self.doctype}`.name")
+			+ f" in ({', '.join(frappe.db.escape(s, percent=False) for s in self.shared)})"
+		)
 
 	def add_user_permissions(self, user_permissions):
 		meta = frappe.get_meta(self.doctype)
@@ -794,7 +811,9 @@ class DatabaseQuery(object):
 				if frappe.get_system_settings("apply_strict_user_permissions"):
 					condition = ""
 				else:
-					empty_value_condition = f"ifnull(`tab{self.doctype}`.`{df.get('fieldname')}`, '')=''"
+					empty_value_condition = cast_name(
+						f"ifnull(`tab{self.doctype}`.`{df.get('fieldname')}`, '')=''"
+					)
 					condition = empty_value_condition + " or "
 
 				for permission in user_permission_values:
@@ -815,7 +834,7 @@ class DatabaseQuery(object):
 
 				if docs:
 					values = ", ".join(frappe.db.escape(doc, percent=False) for doc in docs)
-					condition += f"`tab{self.doctype}`.`{df.get('fieldname')}` in ({values})"
+					condition += cast_name(f"`tab{self.doctype}`.`{df.get('fieldname')}`") + f" in ({values})"
 					match_conditions.append(f"({condition})")
 					match_filters[df.get("options")] = docs
 
@@ -894,7 +913,7 @@ class DatabaseQuery(object):
 		if "select" in _lower and "from" in _lower:
 			frappe.throw(_("Cannot use sub-query in order by"))
 
-		if re.compile(r".*[^a-z0-9-_ ,`'\"\.\(\)].*").match(_lower):
+		if ORDER_GROUP_PATTERN.match(_lower):
 			frappe.throw(_("Illegal SQL Query"))
 
 		for field in parameters.split(","):
@@ -907,7 +926,7 @@ class DatabaseQuery(object):
 
 	def add_limit(self):
 		if self.limit_page_length:
-			return "limit %s offset %s" % (self.limit_page_length, self.limit_start)
+			return f"limit {self.limit_page_length} offset {self.limit_start}"
 		else:
 			return ""
 
@@ -933,6 +952,38 @@ class DatabaseQuery(object):
 		update_user_settings(self.doctype, user_settings)
 
 
+def cast_name(column: str) -> str:
+	"""Casts name field to varchar for postgres
+
+	Handles majorly 4 cases:
+	1. locate
+	2. strpos
+	3. ifnull
+	4. coalesce
+
+	Uses regex substitution.
+
+	Example:
+	input - "ifnull(`tabBlog Post`.`name`, '')=''"
+	output - "ifnull(cast(`tabBlog Post`.`name` as varchar), '')=''" """
+
+	if frappe.db.db_type == "mariadb":
+		return column
+
+	kwargs = {"string": column}
+	if "cast(" not in column.lower() and "::" not in column:
+		if LOCATE_PATTERN.search(**kwargs):
+			return LOCATE_CAST_PATTERN.sub(r"locate(\1, cast(\2 as varchar))", **kwargs)
+
+		elif match := FUNC_IFNULL_PATTERN.search(**kwargs):
+			func = match.groups()[0]
+			return re.sub(rf"{func}\(\s*([`\"]?name[`\"]?)\s*,", rf"{func}(cast(\1 as varchar),", **kwargs)
+
+		return CAST_VARCHAR_PATTERN.sub(r"cast(\1 as varchar)", **kwargs)
+
+	return column
+
+
 def check_parent_permission(parent, child_doctype):
 	if parent:
 		# User may pass fake parent and get the information from the child table
@@ -981,11 +1032,11 @@ def is_parent_only_filter(doctype, filters):
 	only_parent_doctype = True
 
 	if isinstance(filters, list):
-		for flt in filters:
-			if doctype not in flt:
+		for filter in filters:
+			if doctype not in filter:
 				only_parent_doctype = False
-			if "Between" in flt:
-				flt[3] = get_between_date_filter(flt[3])
+			if "Between" in filter:
+				filter[3] = get_between_date_filter(flt[3])
 
 	return only_parent_doctype
 
@@ -1019,12 +1070,12 @@ def get_between_date_filter(value, df=None):
 		to_date = add_to_date(to_date, days=1)
 
 	if df and df.fieldtype == "Datetime":
-		data = "'%s' AND '%s'" % (
+		data = "'{}' AND '{}'".format(
 			frappe.db.format_datetime(from_date),
 			frappe.db.format_datetime(to_date),
 		)
 	else:
-		data = "'%s' AND '%s'" % (frappe.db.format_date(from_date), frappe.db.format_date(to_date))
+		data = f"'{frappe.db.format_date(from_date)}' AND '{frappe.db.format_date(to_date)}'"
 
 	return data
 
@@ -1040,7 +1091,7 @@ def get_additional_filter_field(additional_filters_config, f, value):
 	return f
 
 
-def get_date_range(operator, value):
+def get_date_range(operator: str, value: str):
 	timespan_map = {
 		"1 week": "week",
 		"1 month": "month",
@@ -1053,7 +1104,10 @@ def get_date_range(operator, value):
 		"next": "next",
 	}
 
-	timespan = period_map[operator] + " " + timespan_map[value] if operator != "timespan" else value
+	if operator != "timespan":
+		timespan = f"{period_map[operator]} {timespan_map[value]}"
+	else:
+		timespan = value
 
 	return get_timespan_date_range(timespan)
 
diff --git a/frappe/model/delete_doc.py b/frappe/model/delete_doc.py
index 733e8ca367..332a4337e2 100644
--- a/frappe/model/delete_doc.py
+++ b/frappe/model/delete_doc.py
@@ -11,6 +11,7 @@ from frappe import _, get_module_path
 from frappe.desk.doctype.tag.tag import delete_tags_for_document
 from frappe.model.dynamic_links import get_dynamic_link_map
 from frappe.model.naming import revert_series_if_last
+from frappe.model.utils import is_virtual_doctype
 from frappe.utils.file_manager import remove_all
 from frappe.utils.global_search import delete_for_document
 from frappe.utils.password import delete_all_passwords_for
@@ -29,6 +30,8 @@ doctypes_to_skip = (
 	"Tag Link",
 	"Notification Log",
 	"Email Queue",
+	"Document Share Key",
+	"Integration Request",
 )
 
 
@@ -55,11 +58,16 @@ def delete_doc(
 		doctype = frappe.form_dict.get("dt")
 		name = frappe.form_dict.get("dn")
 
+	is_virtual = is_virtual_doctype(doctype)
+
 	names = name
 	if isinstance(name, str) or isinstance(name, int):
 		names = [name]
 
 	for name in names or []:
+		if is_virtual:
+			frappe.get_doc(doctype, name).delete()
+			continue
 
 		# already deleted..?
 		if not frappe.db.exists(doctype, name):
@@ -87,12 +95,6 @@ def delete_doc(
 
 				update_flags(doc, flags, ignore_permissions)
 				check_permission_and_not_submitted(doc)
-
-				frappe.db.delete("Custom Field", {"dt": name})
-				frappe.db.delete("Client Script", {"dt": name})
-				frappe.db.delete("Property Setter", {"doc_type": name})
-				frappe.db.delete("Report", {"ref_doctype": name})
-				frappe.db.delete("Custom DocPerm", {"parent": name})
 				frappe.db.delete("__global_search", {"doctype": name})
 
 			delete_from_table(doctype, name, ignore_doctypes, None)
@@ -106,7 +108,7 @@ def delete_doc(
 			):
 				try:
 					delete_controllers(name, doc.module)
-				except (FileNotFoundError, OSError, KeyError):
+				except (OSError, KeyError):
 					# in case a doctype doesnt have any controller code  nor any app and module
 					pass
 
@@ -192,39 +194,27 @@ def update_naming_series(doc):
 			revert_series_if_last(doc.meta.autoname, doc.name, doc)
 
 
-def delete_from_table(doctype, name, ignore_doctypes, doc):
+def delete_from_table(doctype: str, name: str, ignore_doctypes: list[str], doc):
 	if doctype != "DocType" and doctype == name:
 		frappe.db.delete("Singles", {"doctype": name})
 	else:
 		frappe.db.delete(doctype, {"name": name})
-	# get child tables
 	if doc:
-		tables = [d.options for d in doc.meta.get_table_fields()]
+		child_doctypes = [
+			d.options for d in doc.meta.get_table_fields() if frappe.get_meta(d.options).is_virtual == 0
+		]
 
 	else:
+		child_doctypes = frappe.get_all(
+			"DocField",
+			fields="options",
+			filters={"fieldtype": ["in", frappe.model.table_fields], "parent": doctype},
+			pluck="options",
+		)
 
-		def get_table_fields(field_doctype):
-			if field_doctype == "Custom Field":
-				return []
-
-			return [
-				r[0]
-				for r in frappe.get_all(
-					field_doctype,
-					fields="options",
-					filters={"fieldtype": ["in", frappe.model.table_fields], "parent": doctype},
-					as_list=1,
-				)
-			]
-
-		tables = get_table_fields("DocField")
-		if not frappe.flags.in_install == "frappe":
-			tables += get_table_fields("Custom Field")
-
-	# delete from child tables
-	for t in list(set(tables)):
-		if t not in ignore_doctypes:
-			frappe.db.delete(t, {"parenttype": doctype, "parent": name})
+	child_doctypes_to_delete = set(child_doctypes) - set(ignore_doctypes)
+	for child_doctype in child_doctypes_to_delete:
+		frappe.db.delete(child_doctype, {"parenttype": doctype, "parent": name})
 
 
 def update_flags(doc, flags=None, ignore_permissions=False):
@@ -354,7 +344,7 @@ def check_if_doc_is_dynamically_linked(doc, method="Delete"):
 
 					reference_doctype = refdoc.parenttype if meta.istable else df.parent
 					reference_docname = refdoc.parent if meta.istable else refdoc.name
-					at_position = "at Row: {0}".format(refdoc.idx) if meta.istable else ""
+					at_position = f"at Row: {refdoc.idx}" if meta.istable else ""
 
 					raise_link_exists_exception(doc, reference_doctype, reference_docname, at_position)
 
@@ -447,7 +437,7 @@ def insert_feed(doc):
 			"doctype": "Comment",
 			"comment_type": "Deleted",
 			"reference_doctype": doc.doctype,
-			"subject": "{0} {1}".format(_(doc.doctype), doc.name),
+			"subject": f"{_(doc.doctype)} {doc.name}",
 			"full_name": get_fullname(doc.owner),
 		}
 	).insert(ignore_permissions=True)
diff --git a/frappe/model/docfield.py b/frappe/model/docfield.py
index 195385a2e1..c54a3855cb 100644
--- a/frappe/model/docfield.py
+++ b/frappe/model/docfield.py
@@ -45,7 +45,7 @@ def update_parent_field(f, new):
 	if f["fieldtype"] in frappe.model.table_fields:
 		frappe.db.begin()
 		frappe.db.sql(
-			"""update `tab%s` set parentfield=%s where parentfield=%s""" % (f["options"], "%s", "%s"),
+			"""update `tab{}` set parentfield={} where parentfield={}""".format(f["options"], "%s", "%s"),
 			(new, f["fieldname"]),
 		)
 		frappe.db.commit()
@@ -56,7 +56,7 @@ def get_change_column_query(f, new):
 	desc = frappe.db.sql("desc `tab%s`" % f["parent"])
 	for d in desc:
 		if d[0] == f["fieldname"]:
-			return "alter table `tab%s` change `%s` `%s` %s" % (f["parent"], f["fieldname"], new, d[1])
+			return "alter table `tab{}` change `{}` `{}` {}".format(f["parent"], f["fieldname"], new, d[1])
 
 
 def supports_translation(fieldtype):
diff --git a/frappe/model/document.py b/frappe/model/document.py
index 07ea58d8e9..c5b6607da6 100644
--- a/frappe/model/document.py
+++ b/frappe/model/document.py
@@ -3,7 +3,6 @@
 import hashlib
 import json
 import time
-from typing import List
 
 from werkzeug.exceptions import NotFound
 
@@ -112,7 +111,8 @@ class Document(BaseDocument):
 
 		if kwargs:
 			# init base document
-			super(Document, self).__init__(kwargs)
+			super().__init__(kwargs)
+			self.init_child_tables()
 			self.init_valid_columns()
 
 		else:
@@ -135,7 +135,7 @@ class Document(BaseDocument):
 				single_doc["name"] = self.doctype
 				del single_doc["__islocal"]
 
-			super(Document, self).__init__(single_doc)
+			super().__init__(single_doc)
 			self.init_valid_columns()
 			self._fix_numeric_types()
 
@@ -148,33 +148,40 @@ class Document(BaseDocument):
 					_("{0} {1} not found").format(_(self.doctype), self.name), frappe.DoesNotExistError
 				)
 
-			super(Document, self).__init__(d)
+			super().__init__(d)
 
-		if self.name == "DocType" and self.doctype == "DocType":
-			from frappe.model.meta import DOCTYPE_TABLE_FIELDS
-
-			table_fields = DOCTYPE_TABLE_FIELDS
-		else:
-			table_fields = self.meta.get_table_fields()
-
-		for df in table_fields:
-			children = frappe.db.get_values(
-				df.options,
-				{"parent": self.name, "parenttype": self.doctype, "parentfield": df.fieldname},
-				"*",
-				as_dict=True,
-				order_by="idx asc",
-			)
-			if children:
-				self.set(df.fieldname, children)
-			else:
+		for df in self._get_table_fields():
+			# Make sure not to query the DB for a child table, if it is a virtual one.
+			# During frappe is installed, the property "is_virtual" is not available in tabDocType, so
+			# we need to filter those cases for the access to frappe.db.get_value() as it would crash otherwise.
+			if (
+				hasattr(self, "doctype")
+				and not hasattr(self, "module")
+				and frappe.db.get_value("DocType", df.options, "is_virtual", cache=True)
+			):
 				self.set(df.fieldname, [])
+				continue
+
+			children = (
+				frappe.db.get_values(
+					df.options,
+					{"parent": self.name, "parenttype": self.doctype, "parentfield": df.fieldname},
+					"*",
+					as_dict=True,
+					order_by="idx asc",
+				)
+				or []
+			)
+
+			self.set(df.fieldname, children)
 
 		# sometimes __setup__ can depend on child values, hence calling again at the end
 		if hasattr(self, "__setup__"):
 			self.__setup__()
 
-	reload = load_from_db
+	def reload(self):
+		"""Reload document from database"""
+		self.load_from_db()
 
 	def get_latest(self):
 		if not getattr(self, "latest", None):
@@ -384,7 +391,10 @@ class Document(BaseDocument):
 			d.db_update()
 			rows.append(d.name)
 
-		if df.options in (self.flags.ignore_children_type or []):
+		if (
+			df.options in (self.flags.ignore_children_type or [])
+			or frappe.get_meta(df.options).is_virtual == 1
+		):
 			# do not delete rows for this because of flags
 			# hack for docperm :(
 			return
@@ -392,9 +402,9 @@ class Document(BaseDocument):
 		if rows:
 			# select rows that do not match the ones in the document
 			deleted_rows = frappe.db.sql(
-				"""select name from `tab{0}` where parent=%s
+				"""select name from `tab{}` where parent=%s
 				and parenttype=%s and parentfield=%s
-				and name not in ({1})""".format(
+				and name not in ({})""".format(
 					df.options, ",".join(["%s"] * len(rows))
 				),
 				[self.name, self.doctype, fieldname] + rows,
@@ -409,7 +419,7 @@ class Document(BaseDocument):
 				df.options, {"parent": self.name, "parenttype": self.doctype, "parentfield": fieldname}
 			)
 
-	def get_doc_before_save(self):
+	def get_doc_before_save(self) -> "Document":
 		return getattr(self, "_doc_before_save", None)
 
 	def has_value_changed(self, fieldname):
@@ -443,7 +453,7 @@ class Document(BaseDocument):
 
 	def get_title(self):
 		"""Get the document title based on title_field or `title` or `name`"""
-		return self.get(self.meta.get_title_field())
+		return self.get(self.meta.get_title_field()) or ""
 
 	def set_title_field(self):
 		"""Set title field based on template"""
@@ -528,6 +538,7 @@ class Document(BaseDocument):
 			d._validate_non_negative()
 			d._validate_length()
 			d._validate_code_fields()
+			d._sync_autoname_field()
 			d._extract_images_from_text_editor()
 			d._sanitize_content()
 			d._save_passwords()
@@ -746,7 +757,7 @@ class Document(BaseDocument):
 					conflict = True
 			else:
 				tmp = frappe.db.sql(
-					"""select modified, docstatus from `tab{0}`
+					"""select modified, docstatus from `tab{}`
 					where name = %s for update""".format(
 						self.doctype
 					),
@@ -769,7 +780,7 @@ class Document(BaseDocument):
 			if conflict:
 				frappe.msgprint(
 					_("Error: Document has been modified after you have opened it")
-					+ (" (%s, %s). " % (modified, self.modified))
+					+ (f" ({modified}, {self.modified}). ")
 					+ _("Please refresh to get the latest document."),
 					raise_exception=frappe.TimestampMismatchError,
 				)
@@ -864,7 +875,7 @@ class Document(BaseDocument):
 
 		raise frappe.MandatoryError(
 			"[{doctype}, {name}]: {fields}".format(
-				fields=", ".join((each[0] for each in missing)), doctype=self.doctype, name=self.name
+				fields=", ".join(each[0] for each in missing), doctype=self.doctype, name=self.name
 			)
 		)
 
@@ -880,14 +891,14 @@ class Document(BaseDocument):
 			cancelled_links.extend(result[1])
 
 		if invalid_links:
-			msg = ", ".join((each[2] for each in invalid_links))
+			msg = ", ".join(each[2] for each in invalid_links)
 			frappe.throw(_("Could not find {0}").format(msg), frappe.LinkValidationError)
 
 		if cancelled_links:
-			msg = ", ".join((each[2] for each in cancelled_links))
+			msg = ", ".join(each[2] for each in cancelled_links)
 			frappe.throw(_("Cannot link cancelled document: {0}").format(msg), frappe.CancelledLinkError)
 
-	def get_all_children(self, parenttype=None) -> List["Document"]:
+	def get_all_children(self, parenttype=None) -> list["Document"]:
 		"""Returns all children documents from **Table** type fields in a list."""
 
 		children = []
@@ -896,8 +907,7 @@ class Document(BaseDocument):
 			if parenttype and df.options != parenttype:
 				continue
 
-			value = self.get(df.fieldname)
-			if isinstance(value, list):
+			if value := self.get(df.fieldname):
 				children.extend(value)
 
 		return children
@@ -988,6 +998,16 @@ class Document(BaseDocument):
 		self.docstatus = DocStatus.cancelled()
 		return self.save()
 
+	@whitelist.__func__
+	def _rename(
+		self, name: str, merge: bool = False, force: bool = False, validate_rename: bool = True
+	):
+		"""Rename the document. Triggers frappe.rename_doc, then reloads."""
+		from frappe.model.rename_doc import rename_doc
+
+		self.name = rename_doc(doc=self, new=name, merge=merge, force=force, validate=validate_rename)
+		self.reload()
+
 	@whitelist.__func__
 	def submit(self):
 		"""Submit the document. Sets `docstatus` = 1, then saves."""
@@ -998,10 +1018,21 @@ class Document(BaseDocument):
 		"""Cancel the document. Sets `docstatus` = 2, then saves."""
 		return self._cancel()
 
-	def delete(self, ignore_permissions=False):
+	@whitelist.__func__
+	def rename(
+		self, name: str, merge: bool = False, force: bool = False, validate_rename: bool = True
+	):
+		"""Rename the document to `name`. This transforms the current object."""
+		return self._rename(name=name, merge=merge, force=force, validate_rename=validate_rename)
+
+	def delete(self, ignore_permissions=False, force=False):
 		"""Delete document."""
-		frappe.delete_doc(
-			self.doctype, self.name, ignore_permissions=ignore_permissions, flags=self.flags
+		return frappe.delete_doc(
+			self.doctype,
+			self.name,
+			ignore_permissions=ignore_permissions,
+			flags=self.flags,
+			force=force,
 		)
 
 	def run_before_save_methods(self):
@@ -1067,7 +1098,9 @@ class Document(BaseDocument):
 			self.run_method("on_update_after_submit")
 
 		self.clear_cache()
-		self.notify_update()
+
+		if self.flags.get("notify_update", True):
+			self.notify_update()
 
 		update_global_search(self)
 
@@ -1120,7 +1153,7 @@ class Document(BaseDocument):
 		:param fieldname: fieldname of the property to be updated, or a {"field":"value"} dictionary
 		:param value: value of the property to be updated
 		:param update_modified: default True. updates the `modified` and `modified_by` properties
-		:param notify: default False. run doc.notify_updated() to send updates via socketio
+		:param notify: default False. run doc.notify_update() to send updates via socketio
 		:param commit: default False. run frappe.db.commit()
 		"""
 		if isinstance(fieldname, dict):
@@ -1186,11 +1219,10 @@ class Document(BaseDocument):
 			return
 
 		version = frappe.new_doc("Version")
-		if not self._doc_before_save:
-			version.for_insert(self)
-			version.insert(ignore_permissions=True)
-		elif version.set_diff(self._doc_before_save, self):
+
+		if is_useful_diff := version.update_version_info(self._doc_before_save, self):
 			version.insert(ignore_permissions=True)
+
 			if not frappe.flags.in_migrate:
 				# follow since you made a change?
 				if frappe.get_cached_value("User", frappe.session.user, "follow_created_documents"):
@@ -1243,7 +1275,7 @@ class Document(BaseDocument):
 	def is_whitelisted(self, method_name):
 		method = getattr(self, method_name, None)
 		if not method:
-			raise NotFound("Method {0} not found".format(method_name))
+			raise NotFound(f"Method {method_name} not found")
 
 		is_whitelisted(getattr(method, "__func__", method))
 
@@ -1361,10 +1393,40 @@ class Document(BaseDocument):
 			).insert(ignore_permissions=True)
 			frappe.local.flags.commit = True
 
+	def log_error(self, title=None, message=None):
+		"""Helper function to create an Error Log"""
+		return frappe.log_error(
+			message=message, title=title, reference_doctype=self.doctype, reference_name=self.name
+		)
+
 	def get_signature(self):
 		"""Returns signature (hash) for private URL."""
 		return hashlib.sha224(get_datetime_str(self.creation).encode()).hexdigest()
 
+	def get_document_share_key(self, expires_on=None, no_expiry=False):
+		if no_expiry:
+			expires_on = None
+
+		existing_key = frappe.db.exists(
+			"Document Share Key",
+			{
+				"reference_doctype": self.doctype,
+				"reference_docname": self.name,
+				"expires_on": expires_on,
+			},
+		)
+		if existing_key:
+			doc = frappe.get_doc("Document Share Key", existing_key)
+		else:
+			doc = frappe.new_doc("Document Share Key")
+			doc.reference_doctype = self.doctype
+			doc.reference_docname = self.name
+			doc.expires_on = expires_on
+			doc.flags.no_expiry = no_expiry
+			doc.insert(ignore_permissions=True)
+
+		return doc.key
+
 	def get_liked_by(self):
 		liked_by = getattr(self, "_liked_by", None)
 		if liked_by:
@@ -1391,21 +1453,22 @@ class Document(BaseDocument):
 		# See: Stock Reconciliation
 		from frappe.utils.background_jobs import enqueue
 
-		if hasattr(self, "_" + action):
-			action = "_" + action
+		if hasattr(self, f"_{action}"):
+			action = f"_{action}"
 
-		if file_lock.lock_exists(self.get_signature()):
+		try:
+			self.lock()
+		except frappe.DocumentLockedError:
 			frappe.throw(
 				_("This document is currently queued for execution. Please try again"),
 				title=_("Document Queued"),
 			)
 
-		self.lock()
-		enqueue(
+		return enqueue(
 			"frappe.model.document.execute_action",
-			doctype=self.doctype,
-			name=self.name,
-			action=action,
+			__doctype=self.doctype,
+			__name=self.name,
+			__action=action,
 			**kwargs,
 		)
 
@@ -1426,10 +1489,13 @@ class Document(BaseDocument):
 			if lock_exists:
 				raise frappe.DocumentLockedError
 		file_lock.create_lock(signature)
+		frappe.local.locked_documents.append(self)
 
 	def unlock(self):
 		"""Delete the lock file for this document"""
 		file_lock.delete_lock(self.get_signature())
+		if self in frappe.local.locked_documents:
+			frappe.local.locked_documents.remove(self)
 
 	# validation helpers
 	def validate_from_to_dates(self, from_date_field, to_date_field):
@@ -1488,12 +1554,12 @@ class Document(BaseDocument):
 		return f"{doctype}({name})"
 
 
-def execute_action(doctype, name, action, **kwargs):
+def execute_action(__doctype, __name, __action, **kwargs):
 	"""Execute an action on a document (called by background worker)"""
-	doc = frappe.get_doc(doctype, name)
+	doc = frappe.get_doc(__doctype, __name)
 	doc.unlock()
 	try:
-		getattr(doc, action)(**kwargs)
+		getattr(doc, __action)(**kwargs)
 	except Exception:
 		frappe.db.rollback()
 
@@ -1504,4 +1570,4 @@ def execute_action(doctype, name, action, **kwargs):
 			msg = "
        " + frappe.get_traceback() + "
        "
 
 		doc.add_comment("Comment", _("Action Failed") + "

        " + msg)
-		doc.notify_update()
+	doc.notify_update()
diff --git a/frappe/model/mapper.py b/frappe/model/mapper.py
index 6a6522ad07..9df79ef276 100644
--- a/frappe/model/mapper.py
+++ b/frappe/model/mapper.py
@@ -231,7 +231,7 @@ def map_fetch_fields(target_doc, df, no_copy_fields):
 	linked_doc = None
 
 	# options should be like "link_fieldname.fieldname_in_liked_doc"
-	for fetch_df in target_doc.meta.get("fields", {"fetch_from": "^{0}.".format(df.fieldname)}):
+	for fetch_df in target_doc.meta.get("fields", {"fetch_from": f"^{df.fieldname}."}):
 		if not (fetch_df.fieldtype == "Read Only" or fetch_df.read_only):
 			continue
 
@@ -243,7 +243,7 @@ def map_fetch_fields(target_doc, df, no_copy_fields):
 			if not linked_doc:
 				try:
 					linked_doc = frappe.get_doc(df.options, target_doc.get(df.fieldname))
-				except:
+				except Exception:
 					return
 
 			val = linked_doc.get(source_fieldname)
diff --git a/frappe/model/meta.py b/frappe/model/meta.py
index 407f7d0811..014dd5faf1 100644
--- a/frappe/model/meta.py
+++ b/frappe/model/meta.py
@@ -30,14 +30,18 @@ from frappe.model import (
 	optional_fields,
 	table_fields,
 )
-from frappe.model.base_document import BaseDocument
+from frappe.model.base_document import (
+	DOCTYPE_TABLE_FIELDS,
+	TABLE_DOCTYPES_FOR_DOCTYPE,
+	BaseDocument,
+)
 from frappe.model.document import Document
 from frappe.model.workflow import get_workflow_name
 from frappe.modules import load_doctype_module
 from frappe.utils import cast, cint, cstr
 
 
-def get_meta(doctype, cached=True):
+def get_meta(doctype, cached=True) -> "Meta":
 	if cached:
 		if not frappe.local.meta_cache.get(doctype):
 			meta = frappe.cache().hget("meta", doctype)
@@ -63,7 +67,7 @@ def get_table_columns(doctype):
 
 def load_doctype_from_file(doctype):
 	fname = frappe.scrub(doctype)
-	with open(frappe.get_app_path("frappe", "core", "doctype", fname, fname + ".json"), "r") as f:
+	with open(frappe.get_app_path("frappe", "core", "doctype", fname, fname + ".json")) as f:
 		txt = json.loads(f.read())
 
 	for d in txt.get("fields", []):
@@ -99,19 +103,19 @@ class Meta(Document):
 	def __init__(self, doctype):
 		self._fields = {}
 		if isinstance(doctype, dict):
-			super(Meta, self).__init__(doctype)
+			super().__init__(doctype)
 
 		elif isinstance(doctype, Document):
-			super(Meta, self).__init__(doctype.as_dict())
+			super().__init__(doctype.as_dict())
 			self.process()
 
 		else:
-			super(Meta, self).__init__("DocType", doctype)
+			super().__init__("DocType", doctype)
 			self.process()
 
 	def load_from_db(self):
 		try:
-			super(Meta, self).load_from_db()
+			super().load_from_db()
 		except frappe.DoesNotExistError:
 			if self.doctype == "DocType" and self.name in self.special_doctypes:
 				self.__dict__.update(load_doctype_from_file(self.name))
@@ -148,9 +152,9 @@ class Meta(Document):
 					out[key] = value
 
 			# set empty lists for unset table fields
-			for table_field in DOCTYPE_TABLE_FIELDS:
-				if out.get(table_field.fieldname) is None:
-					out[table_field.fieldname] = []
+			for fieldname in TABLE_DOCTYPES_FOR_DOCTYPE.keys():
+				if out.get(fieldname) is None:
+					out[fieldname] = []
 
 			return out
 
@@ -162,6 +166,9 @@ class Meta(Document):
 	def get_data_fields(self):
 		return self.get("fields", {"fieldtype": "Data"})
 
+	def get_phone_fields(self):
+		return self.get("fields", {"fieldtype": "Phone"})
+
 	def get_dynamic_link_fields(self):
 		if not hasattr(self, "_dynamic_link_fields"):
 			self._dynamic_link_fields = self.get("fields", {"fieldtype": "Dynamic Link"})
@@ -222,13 +229,7 @@ class Meta(Document):
 		return self._valid_columns
 
 	def get_table_field_doctype(self, fieldname):
-		return {
-			"fields": "DocField",
-			"permissions": "DocPerm",
-			"actions": "DocType Action",
-			"links": "DocType Link",
-			"states": "DocType State",
-		}.get(fieldname)
+		return TABLE_DOCTYPES_FOR_DOCTYPE.get(fieldname)
 
 	def get_field(self, fieldname):
 		"""Return docfield from meta"""
@@ -250,10 +251,15 @@ class Meta(Document):
 		else:
 			label = {
 				"name": _("ID"),
-				"owner": _("Created By"),
-				"modified_by": _("Modified By"),
 				"creation": _("Created On"),
-				"modified": _("Last Modified On"),
+				"docstatus": _("Document Status"),
+				"idx": _("Index"),
+				"modified": _("Last Updated On"),
+				"modified_by": _("Last Updated By"),
+				"owner": _("Created By"),
+				"_user_tags": _("Tags"),
+				"_liked_by": _("Liked By"),
+				"_comments": _("Comments"),
 				"_assign": _("Assigned To"),
 			}.get(fieldname) or _("No Label")
 		return label
@@ -340,6 +346,16 @@ class Meta(Document):
 	def get_workflow(self):
 		return get_workflow_name(self.name)
 
+	def get_naming_series_options(self) -> list[str]:
+		"""Get list naming series options."""
+
+		field = self.get_field("naming_series")
+		if field:
+			options = field.options or ""
+
+			return options.split("\n")
+		return []
+
 	def add_custom_fields(self):
 		if not frappe.db.table_exists("Custom Field"):
 			return
@@ -417,7 +433,7 @@ class Meta(Document):
 
 			# set the fields in order if specified
 			# order is saved as `links_order`
-			order = json.loads(self.get("{}_order".format(fieldname)) or "[]")
+			order = json.loads(self.get(f"{fieldname}_order") or "[]")
 			if order:
 				name_map = {d.name: d for d in self.get(fieldname)}
 				new_list = []
@@ -639,14 +655,6 @@ class Meta(Document):
 		return self.has_field("lft") and self.has_field("rgt")
 
 
-DOCTYPE_TABLE_FIELDS = [
-	frappe._dict({"fieldname": "fields", "options": "DocField"}),
-	frappe._dict({"fieldname": "permissions", "options": "DocPerm"}),
-	frappe._dict({"fieldname": "actions", "options": "DocType Action"}),
-	frappe._dict({"fieldname": "links", "options": "DocType Link"}),
-	frappe._dict({"fieldname": "states", "options": "DocType State"}),
-]
-
 #######
 
 
@@ -781,7 +789,10 @@ def trim_table(doctype, dry_run=True):
 	ignore_fields = default_fields + optional_fields + child_table_fields
 	columns = frappe.db.get_table_columns(doctype)
 	fields = frappe.get_meta(doctype, cached=False).get_fieldnames_with_value()
-	is_internal = lambda f: f not in ignore_fields and not f.startswith("_")
+
+	def is_internal(field):
+		return field not in ignore_fields and not field.startswith("_")
+
 	columns_to_remove = [f for f in list(set(columns) - set(fields)) if is_internal(f)]
 	DROPPED_COLUMNS = columns_to_remove[:]
 
diff --git a/frappe/model/naming.py b/frappe/model/naming.py
index 9d1079d995..49a58da314 100644
--- a/frappe/model/naming.py
+++ b/frappe/model/naming.py
@@ -1,20 +1,130 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
 
+import datetime
 import re
-from typing import TYPE_CHECKING, Optional, Union
+from typing import TYPE_CHECKING, Callable, Optional
 
 import frappe
 from frappe import _
-from frappe.database.sequence import get_next_val, set_next_val
 from frappe.model import log_types
 from frappe.query_builder import DocType
 from frappe.utils import cint, cstr, now_datetime
 
 if TYPE_CHECKING:
+	from frappe.model.document import Document
 	from frappe.model.meta import Meta
 
 
+# NOTE: This is used to keep track of status of sites
+# whether `log_types` have autoincremented naming set for the site or not.
+autoincremented_site_status_map = {}
+
+NAMING_SERIES_PATTERN = re.compile(r"^[\w\- \/.#{}]+$", re.UNICODE)
+BRACED_PARAMS_PATTERN = re.compile(r"(\{[\w | #]+\})")
+
+
+# Types that can be using in naming series fields
+NAMING_SERIES_PART_TYPES = (
+	int,
+	str,
+	datetime.datetime,
+	datetime.date,
+	datetime.time,
+	datetime.timedelta,
+)
+
+
+class InvalidNamingSeriesError(frappe.ValidationError):
+	pass
+
+
+class NamingSeries:
+	__slots__ = ("series",)
+
+	def __init__(self, series: str):
+		self.series = series
+
+		# Add default number part if missing
+		if "#" not in self.series:
+			self.series += ".#####"
+
+	def validate(self):
+		if "." not in self.series:
+			frappe.throw(
+				_("Invalid naming series {}: dot (.) missing").format(frappe.bold(self.series)),
+				exc=InvalidNamingSeriesError,
+			)
+
+		if not NAMING_SERIES_PATTERN.match(self.series):
+			frappe.throw(
+				_(
+					'Special Characters except "-", "#", ".", "/", "{" and "}" not allowed in naming series',
+				),
+				exc=InvalidNamingSeriesError,
+			)
+
+	def generate_next_name(self, doc: "Document") -> str:
+		self.validate()
+		parts = self.series.split(".")
+		return parse_naming_series(parts, doc=doc)
+
+	def get_prefix(self) -> str:
+		"""Naming series stores prefix to maintain a counter in DB. This prefix can be used to update counter or validations.
+
+		e.g. `SINV-.YY.-.####` has prefix of `SINV-22-` in database for year 2022.
+		"""
+
+		prefix = None
+
+		def fake_counter_backend(partial_series, digits):
+			nonlocal prefix
+			prefix = partial_series
+			return "#" * digits
+
+		# This function evaluates all parts till we hit numerical parts and then
+		# sends prefix + digits to DB to find next number.
+		# Instead of reimplementing the whole parsing logic in multiple places we
+		# can just ask this function to give us the prefix.
+		parse_naming_series(self.series, number_generator=fake_counter_backend)
+
+		if prefix is None:
+			frappe.throw(_("Invalid Naming Series: {}").format(self.series))
+
+		return prefix
+
+	def get_preview(self, doc=None) -> list[str]:
+		"""Generate preview of naming series without using DB counters"""
+		generated_names = []
+		for count in range(1, 4):
+
+			def fake_counter(_prefix, digits):
+				# ignore B023: binding `count` is not necessary because
+				# function is evaluated immediately and it can not be done
+				# because of function signature requirement
+				return str(count).zfill(digits)  # noqa: B023
+
+			generated_names.append(parse_naming_series(self.series, doc=doc, number_generator=fake_counter))
+		return generated_names
+
+	def update_counter(self, new_count: int) -> None:
+		"""Warning: Incorrectly updating series can result in unusable transactions"""
+		Series = frappe.qb.DocType("Series")
+		prefix = self.get_prefix()
+
+		# Initialize if not present in DB
+		if frappe.db.get_value("Series", prefix, "name", order_by="name") is None:
+			frappe.qb.into(Series).insert(prefix, 0).columns("name", "current").run()
+
+		(
+			frappe.qb.update(Series).set(Series.current, cint(new_count)).where(Series.name == prefix)
+		).run()
+
+	def get_current_value(self) -> int:
+		prefix = self.get_prefix()
+		return cint(frappe.db.get_value("Series", prefix, "current", order_by="name"))
+
+
 def set_new_name(doc):
 	"""
 	Sets the `name` property for the document based on various rules.
@@ -36,7 +146,7 @@ def set_new_name(doc):
 		doc.name = None
 
 	if is_autoincremented(doc.doctype, meta):
-		doc.name = get_next_val(doc.doctype)
+		doc.name = frappe.db.get_next_sequence_val(doc.doctype)
 		return
 
 	if getattr(doc, "amended_from", None):
@@ -71,12 +181,11 @@ def set_new_name(doc):
 	doc.name = validate_name(doc.doctype, doc.name, meta.get_field("name_case"))
 
 
-def is_autoincremented(doctype: str, meta: "Meta" = None):
+def is_autoincremented(doctype: str, meta: Optional["Meta"] = None) -> bool:
+	"""Checks if the doctype has autoincrement autoname set"""
+
 	if doctype in log_types:
-		if (
-			frappe.local.autoincremented_status_map.get(frappe.local.site) is None
-			or frappe.local.autoincremented_status_map[frappe.local.site] == -1
-		):
+		if autoincremented_site_status_map.get(frappe.local.site) is None:
 			if (
 				frappe.db.sql(
 					f"""select data_type FROM information_schema.columns
@@ -84,22 +193,19 @@ def is_autoincremented(doctype: str, meta: "Meta" = None):
 				)[0][0]
 				== "bigint"
 			):
-				frappe.local.autoincremented_status_map[frappe.local.site] = 1
+				autoincremented_site_status_map[frappe.local.site] = 1
 				return True
 			else:
-				frappe.local.autoincremented_status_map[frappe.local.site] = 0
+				autoincremented_site_status_map[frappe.local.site] = 0
 
-		elif frappe.local.autoincremented_status_map[frappe.local.site]:
+		elif autoincremented_site_status_map[frappe.local.site]:
 			return True
 
 	else:
 		if not meta:
 			meta = frappe.get_meta(doctype)
 
-		if getattr(meta, "issingle", False):
-			return False
-
-		if meta.autoname == "autoincrement":
+		if not getattr(meta, "issingle", False) and meta.autoname == "autoincrement":
 			return True
 
 	return False
@@ -175,32 +281,43 @@ def make_autoname(key="", doctype="", doc=""):
 	if key == "hash":
 		return frappe.generate_hash(doctype, 10)
 
-	if "#" not in key:
-		key = key + ".#####"
-	elif "." not in key:
-		error_message = _("Invalid naming series (. missing)")
-		if doctype:
-			error_message = _("Invalid naming series (. missing) for {0}").format(doctype)
-
-		frappe.throw(error_message)
-
-	parts = key.split(".")
-	n = parse_naming_series(parts, doctype, doc)
-	return n
+	series = NamingSeries(key)
+	return series.generate_next_name(doc)
 
 
-def parse_naming_series(parts, doctype="", doc=""):
-	n = ""
+def parse_naming_series(
+	parts: list[str] | str,
+	doctype=None,
+	doc: Optional["Document"] = None,
+	number_generator: Callable[[str, int], str] | None = None,
+) -> str:
+
+	"""Parse the naming series and get next name.
+
+	args:
+	        parts: naming series parts (split by `.`)
+	        doc: document to use for series that have parts using fieldnames
+	        number_generator: Use different counter backend other than `tabSeries`. Primarily used for testing.
+	"""
+
+	name = ""
 	if isinstance(parts, str):
 		parts = parts.split(".")
+
+	if not number_generator:
+		number_generator = getseries
+
 	series_set = False
 	today = now_datetime()
 	for e in parts:
+		if not e:
+			continue
+
 		part = ""
 		if e.startswith("#"):
 			if not series_set:
 				digits = len(e)
-				part = getseries(n, digits)
+				part = number_generator(name, digits)
 				series_set = True
 		elif e == "YY":
 			part = today.strftime("%y")
@@ -225,9 +342,11 @@ def parse_naming_series(parts, doctype="", doc=""):
 			part = e
 
 		if isinstance(part, str):
-			n += part
+			name += part
+		elif isinstance(part, NAMING_SERIES_PART_TYPES):
+			name += cstr(part).strip()
 
-	return n
+	return name
 
 
 def determine_consecutive_week_number(datetime):
@@ -311,25 +430,27 @@ def revert_series_if_last(key, name, doc=None):
 		frappe.db.sql("UPDATE `tabSeries` SET `current` = `current` - 1 WHERE `name`=%s", prefix)
 
 
-def get_default_naming_series(doctype):
+def get_default_naming_series(doctype: str) -> str | None:
 	"""get default value for `naming_series` property"""
-	naming_series = frappe.get_meta(doctype).get_field("naming_series").options or ""
-	if naming_series:
-		naming_series = naming_series.split("\n")
-		return naming_series[0] or naming_series[1]
-	else:
-		return None
+	naming_series_options = frappe.get_meta(doctype).get_naming_series_options()
+
+	# Return first truthy options
+	# Empty strings are used to avoid populating forms by default
+	for option in naming_series_options:
+		if option:
+			return option
 
 
-def validate_name(doctype: str, name: Union[int, str], case: Optional[str] = None):
+def validate_name(doctype: str, name: int | str, case: str | None = None):
+
 	if not name:
 		frappe.throw(_("No Name Specified for {0}").format(doctype))
 
 	if isinstance(name, int):
 		if is_autoincremented(doctype):
-			# this will set the sequence val to be the provided name and set it to be used
-			# so that the sequence will start from the next val of the setted val(name)
-			set_next_val(doctype, name, is_val_used=True)
+			# this will set the sequence value to be the provided name/value and set it to be used
+			# so that the sequence will start from the next value
+			frappe.db.set_next_sequence_val(doctype, name, is_val_used=True)
 			return name
 
 		frappe.throw(_("Invalid name type (integer) for varchar name column"), frappe.NameError)
@@ -348,8 +469,8 @@ def validate_name(doctype: str, name: Union[int, str], case: Optional[str] = Non
 		frappe.throw(_("Name of {0} cannot be {1}").format(doctype, name), frappe.NameError)
 
 	special_characters = "<>"
-	if re.findall("[{0}]+".format(special_characters), name):
-		message = ", ".join("'{0}'".format(c) for c in special_characters)
+	if re.findall(f"[{special_characters}]+", name):
+		message = ", ".join(f"'{c}'" for c in special_characters)
 		frappe.throw(
 			_("Name cannot contain special characters like {0}").format(message), frappe.NameError
 		)
@@ -363,7 +484,7 @@ def append_number_if_name_exists(doctype, value, fieldname="name", separator="-"
 	filters.update({fieldname: value})
 	exists = frappe.db.exists(doctype, filters)
 
-	regex = "^{value}{separator}\\d+$".format(value=re.escape(value), separator=separator)
+	regex = f"^{re.escape(value)}{separator}\\d+$"
 
 	if exists:
 		last = frappe.db.sql(
@@ -381,7 +502,7 @@ def append_number_if_name_exists(doctype, value, fieldname="name", separator="-"
 		else:
 			count = "1"
 
-		value = "{0}{1}{2}".format(value, separator, count)
+		value = f"{value}{separator}{count}"
 
 	return value
 
@@ -435,6 +556,6 @@ def _format_autoname(autoname, doc):
 		return parse_naming_series([trimmed_param], doc=doc)
 
 	# Replace braced params with their parsed value
-	name = re.sub(r"(\{[\w | #]+\})", get_param_value_for_match, autoname_value)
+	name = BRACED_PARAMS_PATTERN.sub(get_param_value_for_match, autoname_value)
 
 	return name
diff --git a/frappe/model/rename_doc.py b/frappe/model/rename_doc.py
index dee364ae8d..2a04ee7e11 100644
--- a/frappe/model/rename_doc.py
+++ b/frappe/model/rename_doc.py
@@ -1,15 +1,18 @@
 # Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
-from typing import TYPE_CHECKING, Dict, List, Optional
+from types import NoneType
+from typing import TYPE_CHECKING
 
 import frappe
 from frappe import _, bold
+from frappe.model.document import Document
 from frappe.model.dynamic_links import get_dynamic_link_map
 from frappe.model.naming import validate_name
 from frappe.model.utils.user_settings import sync_user_settings, update_user_settings_data
 from frappe.query_builder import Field
-from frappe.utils import cint
+from frappe.utils.data import sbool
 from frappe.utils.password import rename_password
+from frappe.utils.scheduler import is_scheduler_inactive
 
 if TYPE_CHECKING:
 	from frappe.model.meta import Meta
@@ -20,13 +23,22 @@ def update_document_title(
 	*,
 	doctype: str,
 	docname: str,
-	title: Optional[str] = None,
-	name: Optional[str] = None,
+	title: str | None = None,
+	name: str | None = None,
 	merge: bool = False,
+	enqueue: bool = False,
 	**kwargs,
 ) -> str:
 	"""
-	Update title from header in form view
+	Update the name or title of a document. Returns `name` if document was renamed,
+	`docname` if renaming operation was queued.
+
+	:param doctype: DocType of the document
+	:param docname: Name of the document
+	:param title: New Title of the document
+	:param name: New Name of the document
+	:param merge: Merge the current Document with the existing one if exists
+	:param enqueue: Enqueue the rename operation, title is updated in current process
 	"""
 
 	# to maintain backwards API compatibility
@@ -35,9 +47,13 @@ def update_document_title(
 
 	# TODO: omit this after runtime type checking (ref: https://github.com/frappe/frappe/pull/14927)
 	for obj in [docname, updated_title, updated_name]:
-		if not isinstance(obj, (str, type(None))):
+		if not isinstance(obj, (str, NoneType)):
 			frappe.throw(f"{obj=} must be of type str or None")
 
+	# handle bad API usages
+	merge = sbool(merge)
+	enqueue = sbool(enqueue)
+
 	doc = frappe.get_doc(doctype, docname)
 	doc.check_permission(permtype="write")
 
@@ -49,11 +65,34 @@ def update_document_title(
 	name_updated = updated_name and (updated_name != doc.name)
 
 	if name_updated:
-		docname = rename_doc(doctype=doctype, old=docname, new=updated_name, merge=merge)
+		if enqueue and not is_scheduler_inactive():
+			current_name = doc.name
+
+			# before_name hook may have DocType specific validations or transformations
+			transformed_name = doc.run_method("before_rename", current_name, updated_name, merge)
+			if isinstance(transformed_name, dict):
+				transformed_name = transformed_name.get("new")
+			transformed_name = transformed_name or updated_name
+
+			# run rename validations before queueing
+			# use savepoints to avoid partial renames / commits
+			validate_rename(
+				doctype=doctype,
+				old=current_name,
+				new=transformed_name,
+				meta=doc.meta,
+				merge=merge,
+				save_point=True,
+			)
+
+			doc.queue_action("rename", name=transformed_name, merge=merge)
+		else:
+			doc.rename(updated_name, merge=merge)
 
 	if title_updated:
 		try:
-			frappe.db.set_value(doctype, docname, title_field, updated_title)
+			setattr(doc, title_field, updated_title)
+			doc.save()
 			frappe.msgprint(_("Saved"), alert=True, indicator="green")
 		except Exception as e:
 			if frappe.db.is_duplicate_entry(e):
@@ -64,44 +103,64 @@ def update_document_title(
 				)
 			raise
 
-	return docname
+	return doc.name
 
 
 def rename_doc(
-	doctype: str,
-	old: str,
-	new: str,
+	doctype: str | None = None,
+	old: str | None = None,
+	new: str = None,
 	force: bool = False,
 	merge: bool = False,
 	ignore_permissions: bool = False,
 	ignore_if_exists: bool = False,
 	show_alert: bool = True,
 	rebuild_search: bool = True,
+	doc: Document | None = None,
+	validate: bool = True,
 ) -> str:
-	"""Rename a doc(dt, old) to doc(dt, new) and update all linked fields of type "Link"."""
-	if not frappe.db.exists(doctype, old):
-		frappe.errprint(_("Failed: {0} to {1} because {0} doesn't exist.").format(old, new))
-		return
+	"""Rename a doc(dt, old) to doc(dt, new) and update all linked fields of type "Link".
 
-	if ignore_if_exists and frappe.db.exists(doctype, new):
-		frappe.errprint(_("Failed: {0} to {1} because {1} already exists.").format(old, new))
-		return
+	doc: Document object to be renamed.
+	new: New name for the record. If None, and doctype is specified, new name may be automatically generated via before_rename hooks.
+	doctype: DocType of the document. Not required if doc is passed.
+	old: Current name of the document. Not required if doc is passed.
+	force: Allow even if document is not allowed to be renamed.
+	merge: Merge with existing document of new name.
+	ignore_permissions: Ignore user permissions while renaming.
+	ignore_if_exists: Don't raise exception if document with new name already exists. This will quietely overwrite the existing document.
+	show_alert: Display alert if document is renamed successfully.
+	rebuild_search: Rebuild linked doctype search after renaming.
+	validate: Validate before renaming. If False, it is assumed that the caller has already validated.
+	"""
+	old_usage_style = doctype and old and new
+	new_usage_style = doc and new
 
-	if old == new:
-		frappe.errprint(
-			_("Ignored: {0} to {1} no changes made because old and new name are the same.").format(old, new)
+	if not (new_usage_style or old_usage_style):
+		raise TypeError(
+			"{doctype, old, new} or {doc, new} are required arguments for frappe.model.rename_doc"
 		)
-		return
 
-	force = cint(force)
-	merge = cint(merge)
+	old = old or doc.name
+	doctype = doctype or doc.doctype
+	force = sbool(force)
+	merge = sbool(merge)
 	meta = frappe.get_meta(doctype)
 
-	# call before_rename
-	old_doc = frappe.get_doc(doctype, old)
-	out = old_doc.run_method("before_rename", old, new, merge) or {}
-	new = (out.get("new") or new) if isinstance(out, dict) else (out or new)
-	new = validate_rename(doctype, new, meta, merge, force, ignore_permissions)
+	if validate:
+		old_doc = doc or frappe.get_doc(doctype, old)
+		out = old_doc.run_method("before_rename", old, new, merge) or {}
+		new = (out.get("new") or new) if isinstance(out, dict) else (out or new)
+		new = validate_rename(
+			doctype=doctype,
+			old=old,
+			new=new,
+			meta=meta,
+			merge=merge,
+			force=force,
+			ignore_permissions=ignore_permissions,
+			ignore_if_exists=ignore_if_exists,
+		)
 
 	if not merge:
 		rename_parent_and_child(doctype, old, new, meta)
@@ -139,11 +198,12 @@ def rename_doc(
 		rename_password(doctype, old, new)
 
 	# update user_permissions
-	frappe.db.sql(
-		"""UPDATE `tabDefaultValue` SET `defvalue`=%s WHERE `parenttype`='User Permission'
-		AND `defkey`=%s AND `defvalue`=%s""",
-		(new, doctype, old),
-	)
+	DefaultValue = frappe.qb.DocType("DefaultValue")
+	frappe.qb.update(DefaultValue).set(DefaultValue.defvalue, new).where(
+		(DefaultValue.parenttype == "User Permission")
+		& (DefaultValue.defkey == doctype)
+		& (DefaultValue.defvalue == old)
+	).run()
 
 	if merge:
 		new_doc.add_comment("Edit", _("merged {0} into {1}").format(frappe.bold(old), frappe.bold(new)))
@@ -194,7 +254,7 @@ def update_assignments(old: str, new: str, doctype: str) -> None:
 	frappe.db.set_value(doctype, new, "_assign", frappe.as_json(unique_assignments, indent=0))
 
 
-def update_user_settings(old: str, new: str, link_fields: List[Dict]) -> None:
+def update_user_settings(old: str, new: str, link_fields: list[dict]) -> None:
 	"""
 	Update the user settings of all the linked doctypes while renaming.
 	"""
@@ -207,15 +267,13 @@ def update_user_settings(old: str, new: str, link_fields: List[Dict]) -> None:
 
 	# find the user settings for the linked doctypes
 	linked_doctypes = {d.parent for d in link_fields if not d.issingle}
-	user_settings_details = frappe.db.sql(
-		"""SELECT `user`, `doctype`, `data`
-			FROM `__UserSettings`
-			WHERE `data` like %s
-			AND `doctype` IN ('{doctypes}')""".format(
-			doctypes="', '".join(linked_doctypes)
-		),
-		(old),
-		as_dict=1,
+	UserSettings = frappe.qb.Table("__UserSettings")
+
+	user_settings_details = (
+		frappe.qb.from_(UserSettings)
+		.select("user", "doctype", "data")
+		.where(UserSettings.data.like(old) & UserSettings.doctype.isin(linked_doctypes))
+		.run(as_dict=True)
 	)
 
 	# create the dict using the doctype name as key and values as list of the user settings
@@ -240,37 +298,33 @@ def update_customizations(old: str, new: str) -> None:
 
 
 def update_attachments(doctype: str, old: str, new: str) -> None:
-	try:
-		if old != "File Data" and doctype != "DocType":
-			frappe.db.sql(
-				"""update `tabFile` set attached_to_name=%s
-				where attached_to_name=%s and attached_to_doctype=%s""",
-				(new, old, doctype),
-			)
-	except frappe.db.ProgrammingError as e:
-		if not frappe.db.is_column_missing(e):
-			raise
+	if doctype != "DocType":
+		File = frappe.qb.DocType("File")
+
+		frappe.qb.update(File).set(File.attached_to_name, new).where(
+			(File.attached_to_name == old) & (File.attached_to_doctype == doctype)
+		).run()
 
 
 def rename_versions(doctype: str, old: str, new: str) -> None:
-	frappe.db.sql(
-		"""UPDATE `tabVersion` SET `docname`=%s WHERE `ref_doctype`=%s AND `docname`=%s""",
-		(new, doctype, old),
-	)
+	Version = frappe.qb.DocType("Version")
+
+	frappe.qb.update(Version).set(Version.docname, new).where(
+		(Version.docname == old) & (Version.ref_doctype == doctype)
+	).run()
 
 
 def rename_eps_records(doctype: str, old: str, new: str) -> None:
-	epl = frappe.qb.DocType("Energy Point Log")
-	(
-		frappe.qb.update(epl)
-		.set(epl.reference_name, new)
-		.where((epl.reference_doctype == doctype) & (epl.reference_name == old))
+	EPL = frappe.qb.DocType("Energy Point Log")
+
+	frappe.qb.update(EPL).set(EPL.reference_name, new).where(
+		(EPL.reference_doctype == doctype) & (EPL.reference_name == old)
 	).run()
 
 
 def rename_parent_and_child(doctype: str, old: str, new: str, meta: "Meta") -> None:
-	# rename the doc
-	frappe.db.sql("UPDATE `tab{0}` SET `name`={1} WHERE `name`={1}".format(doctype, "%s"), (new, old))
+	frappe.qb.update(doctype).set("name", new).where(Field("name") == old).run()
+
 	update_autoname_field(doctype, new, meta)
 	update_child_docs(old, new, meta)
 
@@ -280,20 +334,36 @@ def update_autoname_field(doctype: str, new: str, meta: "Meta") -> None:
 	if meta.get("autoname"):
 		field = meta.get("autoname").split(":")
 		if field and field[0] == "field":
-			frappe.db.sql(
-				"UPDATE `tab{0}` SET `{1}`={2} WHERE `name`={2}".format(doctype, field[1], "%s"), (new, new)
-			)
+			frappe.qb.update(doctype).set(field[1], new).where(Field("name") == new).run()
 
 
 def validate_rename(
-	doctype: str, new: str, meta: "Meta", merge: bool, force: bool, ignore_permissions: bool
+	doctype: str,
+	old: str,
+	new: str,
+	meta: "Meta",
+	merge: bool,
+	force: bool = False,
+	ignore_permissions: bool = False,
+	ignore_if_exists: bool = False,
+	save_point=False,
 ) -> str:
 	# using for update so that it gets locked and someone else cannot edit it while this rename is going on!
+	if save_point:
+		_SAVE_POINT = f"validate_rename_{frappe.generate_hash(length=8)}"
+		frappe.db.savepoint(_SAVE_POINT)
+
 	exists = (
 		frappe.qb.from_(doctype).where(Field("name") == new).for_update().select("name").run(pluck=True)
 	)
 	exists = exists[0] if exists else None
 
+	if not frappe.db.exists(doctype, old):
+		frappe.throw(_("Can't rename {0} to {1} because {0} doesn't exist.").format(old, new))
+
+	if old == new:
+		frappe.throw(_("No changes made because old and new name are the same.").format(old, new))
+
 	if merge and not exists:
 		frappe.throw(_("{0} {1} does not exist, select a new target to merge").format(doctype, new))
 
@@ -301,7 +371,7 @@ def validate_rename(
 		# for fixing case, accents
 		exists = None
 
-	if (not merge) and exists:
+	if not merge and exists and not ignore_if_exists:
 		frappe.throw(_("Another {0} with name {1} exists, select another name").format(doctype, new))
 
 	if not (
@@ -315,6 +385,9 @@ def validate_rename(
 	# validate naming like it's done in doc.py
 	new = validate_name(doctype, new)
 
+	if save_point:
+		frappe.db.rollback(save_point=_SAVE_POINT)
+
 	return new
 
 
@@ -337,12 +410,10 @@ def rename_doctype(doctype: str, old: str, new: str) -> None:
 def update_child_docs(old: str, new: str, meta: "Meta") -> None:
 	# update "parent"
 	for df in meta.get_table_fields():
-		frappe.db.sql(
-			"update `tab%s` set parent=%s where parent=%s" % (df.options, "%s", "%s"), (new, old)
-		)
+		frappe.qb.update(df.options).set("parent", new).where(Field("parent") == old).run()
 
 
-def update_link_field_values(link_fields: List[Dict], old: str, new: str, doctype: str) -> None:
+def update_link_field_values(link_fields: list[dict], old: str, new: str, doctype: str) -> None:
 	for field in link_fields:
 		if field["issingle"]:
 			try:
@@ -378,63 +449,52 @@ def update_link_field_values(link_fields: List[Dict], old: str, new: str, doctyp
 			field["parent"] = new
 
 
-def get_link_fields(doctype: str) -> List[Dict]:
+def get_link_fields(doctype: str) -> list[dict]:
 	# get link fields from tabDocField
 	if not frappe.flags.link_fields:
 		frappe.flags.link_fields = {}
 
 	if doctype not in frappe.flags.link_fields:
-		link_fields = frappe.db.sql(
-			"""\
-			select parent, fieldname,
-				(select issingle from tabDocType dt
-				where dt.name = df.parent) as issingle
-			from tabDocField df
-			where
-				df.options=%s and df.fieldtype='Link'""",
-			(doctype,),
-			as_dict=1,
+		dt = frappe.qb.DocType("DocType")
+		df = frappe.qb.DocType("DocField")
+		cf = frappe.qb.DocType("Custom Field")
+		ps = frappe.qb.DocType("Property Setter")
+
+		st_issingle = frappe.qb.from_(dt).select(dt.issingle).where(dt.name == df.parent).as_("issingle")
+		standard_fields = (
+			frappe.qb.from_(df)
+			.select(df.parent, df.fieldname, st_issingle)
+			.where((df.options == doctype) & (df.fieldtype == "Link"))
+			.run(as_dict=True)
 		)
 
-		# get link fields from tabCustom Field
-		custom_link_fields = frappe.db.sql(
-			"""\
-			select dt as parent, fieldname,
-				(select issingle from tabDocType dt
-				where dt.name = df.dt) as issingle
-			from `tabCustom Field` df
-			where
-				df.options=%s and df.fieldtype='Link'""",
-			(doctype,),
-			as_dict=1,
+		cf_issingle = frappe.qb.from_(dt).select(dt.issingle).where(dt.name == cf.dt).as_("issingle")
+		custom_fields = (
+			frappe.qb.from_(cf)
+			.select(cf.dt.as_("parent"), cf.fieldname, cf_issingle)
+			.where((cf.options == doctype) & (cf.fieldtype == "Link"))
+			.run(as_dict=True)
 		)
 
-		# add custom link fields list to link fields list
-		link_fields += custom_link_fields
-
-		# remove fields whose options have been changed using property setter
-		property_setter_link_fields = frappe.db.sql(
-			"""\
-			select ps.doc_type as parent, ps.field_name as fieldname,
-				(select issingle from tabDocType dt
-				where dt.name = ps.doc_type) as issingle
-			from `tabProperty Setter` ps
-			where
-				ps.property_type='options' and
-				ps.field_name is not null and
-				ps.value=%s""",
-			(doctype,),
-			as_dict=1,
+		ps_issingle = (
+			frappe.qb.from_(dt).select(dt.issingle).where(dt.name == ps.doc_type).as_("issingle")
+		)
+		property_setter_fields = (
+			frappe.qb.from_(ps)
+			.select(ps.doc_type.as_("parent"), ps.field_name.as_("fieldname"), ps_issingle)
+			.where((ps.property == "options") & (ps.value == doctype) & (ps.field_name.notnull()))
+			.run(as_dict=True)
 		)
 
-		link_fields += property_setter_link_fields
-
-		frappe.flags.link_fields[doctype] = link_fields
+		frappe.flags.link_fields[doctype] = standard_fields + custom_fields + property_setter_fields
 
 	return frappe.flags.link_fields[doctype]
 
 
 def update_options_for_fieldtype(fieldtype: str, old: str, new: str) -> None:
+	CustomField = frappe.qb.DocType("Custom Field")
+	PropertySetter = frappe.qb.DocType("Property Setter")
+
 	if frappe.conf.developer_mode:
 		for name in frappe.get_all("DocField", filters={"options": old}, pluck="parent"):
 			doctype = frappe.get_doc("DocType", name)
@@ -446,132 +506,106 @@ def update_options_for_fieldtype(fieldtype: str, old: str, new: str) -> None:
 			if save:
 				doctype.save()
 	else:
-		frappe.db.sql(
-			"""update `tabDocField` set options=%s
-			where fieldtype=%s and options=%s""",
-			(new, fieldtype, old),
-		)
+		DocField = frappe.qb.DocType("DocField")
+		frappe.qb.update(DocField).set(DocField.options, new).where(
+			(DocField.fieldtype == fieldtype) & (DocField.options == old)
+		).run()
 
-	frappe.db.sql(
-		"""update `tabCustom Field` set options=%s
-		where fieldtype=%s and options=%s""",
-		(new, fieldtype, old),
-	)
+	frappe.qb.update(CustomField).set(CustomField.options, new).where(
+		(CustomField.fieldtype == fieldtype) & (CustomField.options == old)
+	).run()
 
-	frappe.db.sql(
-		"""update `tabProperty Setter` set value=%s
-		where property='options' and value=%s""",
-		(new, old),
-	)
+	frappe.qb.update(PropertySetter).set(PropertySetter.value, new).where(
+		(PropertySetter.property == "options") & (PropertySetter.value == old)
+	).run()
 
 
-def get_select_fields(old: str, new: str) -> List[Dict]:
+def get_select_fields(old: str, new: str) -> list[dict]:
 	"""
 	get select type fields where doctype's name is hardcoded as
 	new line separated list
 	"""
+	df = frappe.qb.DocType("DocField")
+	dt = frappe.qb.DocType("DocType")
+	cf = frappe.qb.DocType("Custom Field")
+	ps = frappe.qb.DocType("Property Setter")
+
 	# get link fields from tabDocField
-	select_fields = frappe.db.sql(
-		"""
-		select parent, fieldname,
-			(select issingle from tabDocType dt
-			where dt.name = df.parent) as issingle
-		from tabDocField df
-		where
-			df.parent != %s and df.fieldtype = 'Select' and
-			df.options like {0} """.format(
-			frappe.db.escape("%" + old + "%")
-		),
-		(new,),
-		as_dict=1,
+	st_issingle = frappe.qb.from_(dt).select(dt.issingle).where(dt.name == df.parent).as_("issingle")
+	standard_fields = (
+		frappe.qb.from_(df)
+		.select(df.parent, df.fieldname, st_issingle)
+		.where((df.parent != new) & (df.fieldtype == "Select") & (df.options.like(f"%{old}%")))
+		.run(as_dict=True)
 	)
 
 	# get link fields from tabCustom Field
-	custom_select_fields = frappe.db.sql(
-		"""
-		select dt as parent, fieldname,
-			(select issingle from tabDocType dt
-			where dt.name = df.dt) as issingle
-		from `tabCustom Field` df
-		where
-			df.dt != %s and df.fieldtype = 'Select' and
-			df.options like {0} """.format(
-			frappe.db.escape("%" + old + "%")
-		),
-		(new,),
-		as_dict=1,
+	cf_issingle = frappe.qb.from_(dt).select(dt.issingle).where(dt.name == cf.dt).as_("issingle")
+	custom_select_fields = (
+		frappe.qb.from_(cf)
+		.select(cf.dt.as_("parent"), cf.fieldname, cf_issingle)
+		.where((cf.dt != new) & (cf.fieldtype == "Select") & (cf.options.like(f"%{old}%")))
+		.run(as_dict=True)
 	)
 
-	# add custom link fields list to link fields list
-	select_fields += custom_select_fields
-
 	# remove fields whose options have been changed using property setter
-	property_setter_select_fields = frappe.db.sql(
-		"""
-		select ps.doc_type as parent, ps.field_name as fieldname,
-			(select issingle from tabDocType dt
-			where dt.name = ps.doc_type) as issingle
-		from `tabProperty Setter` ps
-		where
-			ps.doc_type != %s and
-			ps.property_type='options' and
-			ps.field_name is not null and
-			ps.value like {0} """.format(
-			frappe.db.escape("%" + old + "%")
-		),
-		(new,),
-		as_dict=1,
+	ps_issingle = (
+		frappe.qb.from_(dt).select(dt.issingle).where(dt.name == ps.doc_type).as_("issingle")
+	)
+	property_setter_select_fields = (
+		frappe.qb.from_(ps)
+		.select(ps.doc_type.as_("parent"), ps.field_name.as_("fieldname"), ps_issingle)
+		.where(
+			(ps.doc_type != new)
+			& (ps.property == "options")
+			& (ps.field_name.notnull())
+			& (ps.value.like(f"%{old}%"))
+		)
+		.run(as_dict=True)
 	)
 
-	select_fields += property_setter_select_fields
-
-	return select_fields
+	return standard_fields + custom_select_fields + property_setter_select_fields
 
 
 def update_select_field_values(old: str, new: str):
-	frappe.db.sql(
-		"""
-		update `tabDocField` set options=replace(options, %s, %s)
-		where
-			parent != %s and fieldtype = 'Select' and
-			(options like {0} or options like {1})""".format(
-			frappe.db.escape("%" + "\n" + old + "%"), frappe.db.escape("%" + old + "\n" + "%")
-		),
-		(old, new, new),
-	)
+	from frappe.query_builder.functions import Replace
 
-	frappe.db.sql(
-		"""
-		update `tabCustom Field` set options=replace(options, %s, %s)
-		where
-			dt != %s and fieldtype = 'Select' and
-			(options like {0} or options like {1})""".format(
-			frappe.db.escape("%" + "\n" + old + "%"), frappe.db.escape("%" + old + "\n" + "%")
-		),
-		(old, new, new),
-	)
+	DocField = frappe.qb.DocType("DocField")
+	CustomField = frappe.qb.DocType("Custom Field")
+	PropertySetter = frappe.qb.DocType("Property Setter")
 
-	frappe.db.sql(
-		"""
-		update `tabProperty Setter` set value=replace(value, %s, %s)
-		where
-			doc_type != %s and field_name is not null and
-			property='options' and
-			(value like {0} or value like {1})""".format(
-			frappe.db.escape("%" + "\n" + old + "%"), frappe.db.escape("%" + old + "\n" + "%")
-		),
-		(old, new, new),
-	)
+	frappe.qb.update(DocField).set(DocField.options, Replace(DocField.options, old, new)).where(
+		(DocField.fieldtype == "Select")
+		& (DocField.parent != new)
+		& (DocField.options.like(f"%\n{old}%") | DocField.options.like(f"%{old}\n%"))
+	).run()
+
+	frappe.qb.update(CustomField).set(
+		CustomField.options, Replace(CustomField.options, old, new)
+	).where(
+		(CustomField.fieldtype == "Select")
+		& (CustomField.dt != new)
+		& (CustomField.options.like(f"%\n{old}%") | CustomField.options.like(f"%{old}\n%"))
+	).run()
+
+	frappe.qb.update(PropertySetter).set(
+		PropertySetter.value, Replace(PropertySetter.value, old, new)
+	).where(
+		(PropertySetter.property == "options")
+		& (PropertySetter.field_name.notnull())
+		& (PropertySetter.doc_type != new)
+		& (PropertySetter.value.like(f"%\n{old}%") | PropertySetter.value.like(f"%{old}\n%"))
+	).run()
 
 
 def update_parenttype_values(old: str, new: str):
-	child_doctypes = frappe.db.get_all(
+	child_doctypes = frappe.get_all(
 		"DocField",
 		fields=["options", "fieldname"],
 		filters={"parent": new, "fieldtype": ["in", frappe.model.table_fields]},
 	)
 
-	custom_child_doctypes = frappe.db.get_all(
+	custom_child_doctypes = frappe.get_all(
 		"Custom Field",
 		fields=["options", "fieldname"],
 		filters={"dt": new, "fieldtype": ["in", frappe.model.table_fields]},
@@ -586,40 +620,35 @@ def update_parenttype_values(old: str, new: str):
 		pluck="value",
 	)
 
-	child_doctypes = list(d["options"] for d in child_doctypes)
-	child_doctypes += property_setter_child_doctypes
+	child_doctypes = set(list(d["options"] for d in child_doctypes) + property_setter_child_doctypes)
 
 	for doctype in child_doctypes:
-		frappe.db.sql(f"update `tab{doctype}` set parenttype=%s where parenttype=%s", (new, old))
+		table = frappe.qb.DocType(doctype)
+		frappe.qb.update(table).set(table.parenttype, new).where(table.parenttype == old).run()
 
 
 def rename_dynamic_links(doctype: str, old: str, new: str):
+	Singles = frappe.qb.DocType("Singles")
 	for df in get_dynamic_link_map().get(doctype, []):
 		# dynamic link in single, just one value to check
 		if frappe.get_meta(df.parent).issingle:
 			refdoc = frappe.db.get_singles_dict(df.parent)
 			if refdoc.get(df.options) == doctype and refdoc.get(df.fieldname) == old:
-
-				frappe.db.sql(
-					"""update tabSingles set value=%s where
-					field=%s and value=%s and doctype=%s""",
-					(new, df.fieldname, old, df.parent),
-				)
+				frappe.qb.update(Singles).set(Singles.value, new).where(
+					(Singles.field == df.fieldname) & (Singles.doctype == df.parent) & (Singles.value == old)
+				).run()
 		else:
 			# because the table hasn't been renamed yet!
 			parent = df.parent if df.parent != new else old
-			frappe.db.sql(
-				"""update `tab{parent}` set {fieldname}=%s
-				where {options}=%s and {fieldname}=%s""".format(
-					parent=parent, fieldname=df.fieldname, options=df.options
-				),
-				(new, doctype, old),
-			)
+
+			frappe.qb.update(parent).set(df.fieldname, new).where(
+				(Field(df.options) == doctype) & (Field(df.fieldname) == old)
+			).run()
 
 
 def bulk_rename(
-	doctype: str, rows: Optional[List[List]] = None, via_console: bool = False
-) -> Optional[List[str]]:
+	doctype: str, rows: list[list] | None = None, via_console: bool = False
+) -> list[str] | None:
 	"""Bulk rename documents
 
 	:param doctype: DocType to be renamed
@@ -660,7 +689,7 @@ def bulk_rename(
 
 
 def update_linked_doctypes(
-	doctype: str, docname: str, linked_to: str, value: str, ignore_doctypes: Optional[List] = None
+	doctype: str, docname: str, linked_to: str, value: str, ignore_doctypes: list | None = None
 ) -> None:
 	from frappe.model.utils.rename_doc import update_linked_doctypes
 
@@ -676,8 +705,8 @@ def update_linked_doctypes(
 
 
 def get_fetch_fields(
-	doctype: str, linked_to: str, ignore_doctypes: Optional[List] = None
-) -> List[Dict]:
+	doctype: str, linked_to: str, ignore_doctypes: list | None = None
+) -> list[dict]:
 	from frappe.model.utils.rename_doc import get_fetch_fields
 
 	show_deprecation_warning("get_fetch_fields")
diff --git a/frappe/model/sync.py b/frappe/model/sync.py
index a56d1f267f..df3999054a 100644
--- a/frappe/model/sync.py
+++ b/frappe/model/sync.py
@@ -53,22 +53,6 @@ def sync_for(app_name, force=0, reset_permissions=False):
 				os.path.join(FRAPPE_PATH, "website", "doctype", website_module, f"{website_module}.json")
 			)
 
-		for data_migration_module in [
-			"data_migration_mapping_detail",
-			"data_migration_mapping",
-			"data_migration_plan_mapping",
-			"data_migration_plan",
-		]:
-			files.append(
-				os.path.join(
-					FRAPPE_PATH,
-					"data_migration",
-					"doctype",
-					data_migration_module,
-					f"{data_migration_module}.json",
-				)
-			)
-
 		for desk_module in [
 			"number_card",
 			"dashboard_chart",
@@ -80,6 +64,7 @@ def sync_for(app_name, force=0, reset_permissions=False):
 			"workspace_link",
 			"workspace_chart",
 			"workspace_shortcut",
+			"workspace_quick_list",
 			"workspace",
 		]:
 			files.append(os.path.join(FRAPPE_PATH, "desk", "doctype", desk_module, f"{desk_module}.json"))
@@ -99,7 +84,7 @@ def sync_for(app_name, force=0, reset_permissions=False):
 			frappe.db.commit()
 
 			# show progress bar
-			update_progress_bar("Updating DocTypes for {0}".format(app_name), i, l)
+			update_progress_bar(f"Updating DocTypes for {app_name}", i, l)
 
 		# print each progress bar on new line
 		print()
@@ -123,8 +108,6 @@ def get_doc_files(files, start_path):
 		"web_template",
 		"notification",
 		"print_style",
-		"data_migration_mapping",
-		"data_migration_plan",
 		"workspace",
 		"onboarding_step",
 		"module_onboarding",
diff --git a/frappe/model/utils/__init__.py b/frappe/model/utils/__init__.py
index a0dd0d89e8..2220b3904f 100644
--- a/frappe/model/utils/__init__.py
+++ b/frappe/model/utils/__init__.py
@@ -1,12 +1,12 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
-import io
 import re
 
 import frappe
 from frappe import _
 from frappe.build import html_to_js_template
 from frappe.utils import cstr
+from frappe.utils.caching import site_cache
 
 STANDARD_FIELD_CONVERSION_MAP = {
 	"name": "Link",
@@ -21,10 +21,7 @@ STANDARD_FIELD_CONVERSION_MAP = {
 	"_assign": "Text",
 	"docstatus": "Int",
 }
-
-"""
-Model utilities, unclassified functions
-"""
+INCLUDE_DIRECTIVE_PATTERN = re.compile(r"""{% include\s['"](.*)['"]\s%}""")
 
 
 def set_default(doc, key):
@@ -50,7 +47,7 @@ def set_field_property(filters, key, value):
 	for d in docs:
 		d.get("fields", filters)[0].set(key, value)
 		d.save()
-		print("Updated {0}".format(d.name))
+		print(f"Updated {d.name}")
 
 	frappe.db.commit()
 
@@ -67,18 +64,18 @@ def render_include(content):
 	# try 5 levels of includes
 	for i in range(5):
 		if "{% include" in content:
-			paths = re.findall(r"""{% include\s['"](.*)['"]\s%}""", content)
+			paths = INCLUDE_DIRECTIVE_PATTERN.findall(content)
 			if not paths:
 				frappe.throw(_("Invalid include path"), InvalidIncludePath)
 
 			for path in paths:
 				app, app_path = path.split("/", 1)
-				with io.open(frappe.get_app_path(app, app_path), "r", encoding="utf-8") as f:
+				with open(frappe.get_app_path(app, app_path), encoding="utf-8") as f:
 					include = f.read()
 					if path.endswith(".html"):
 						include = html_to_js_template(path, include)
 
-					content = re.sub(r"""{{% include\s['"]{0}['"]\s%}}""".format(path), include, content)
+					content = re.sub(rf"""{{% include\s['"]{path}['"]\s%}}""", include, content)
 
 		else:
 			break
@@ -93,12 +90,44 @@ def get_fetch_values(doctype, fieldname, value):
 	:param fieldname: Link fieldname selected
 	:param value: Value selected
 	"""
-	out = {}
-	meta = frappe.get_meta(doctype)
-	link_df = meta.get_field(fieldname)
-	for df in meta.get_fields_to_fetch(fieldname):
-		# example shipping_address.gistin
-		link_field, source_fieldname = df.fetch_from.split(".", 1)
-		out[df.fieldname] = frappe.db.get_value(link_df.options, value, source_fieldname)
 
-	return out
+	result = frappe._dict()
+	meta = frappe.get_meta(doctype)
+
+	# fieldname in target doctype: fieldname in source doctype
+	fields_to_fetch = {
+		df.fieldname: df.fetch_from.split(".", 1)[1] for df in meta.get_fields_to_fetch(fieldname)
+	}
+
+	# nothing to fetch
+	if not fields_to_fetch:
+		return result
+
+	# initialise empty values for target fields
+	for target_fieldname in fields_to_fetch:
+		result[target_fieldname] = None
+
+	# fetch only if Link field has a truthy value
+	if not value:
+		return result
+
+	db_values = frappe.db.get_value(
+		meta.get_options(fieldname),  # source doctype
+		value,
+		tuple(set(fields_to_fetch.values())),  # unique source fieldnames
+		as_dict=True,
+	)
+
+	# if value doesn't exist in source doctype, get_value returns None
+	if not db_values:
+		return result
+
+	for target_fieldname, source_fieldname in fields_to_fetch.items():
+		result[target_fieldname] = db_values.get(source_fieldname)
+
+	return result
+
+
+@site_cache(maxsize=128)
+def is_virtual_doctype(doctype):
+	return frappe.db.get_value("DocType", doctype, "is_virtual")
diff --git a/frappe/model/utils/link_count.py b/frappe/model/utils/link_count.py
index 25dfe58139..9a7694b9f8 100644
--- a/frappe/model/utils/link_count.py
+++ b/frappe/model/utils/link_count.py
@@ -42,7 +42,7 @@ def update_link_count():
 			if key[0] not in ignore_doctypes:
 				try:
 					frappe.db.sql(
-						"update `tab{0}` set idx = idx + {1} where name=%s".format(key[0], count),
+						f"update `tab{key[0]}` set idx = idx + {count} where name=%s",
 						key[1],
 						auto_commit=1,
 					)
diff --git a/frappe/model/utils/rename_doc.py b/frappe/model/utils/rename_doc.py
index 00e2d78d5f..ae6649f057 100644
--- a/frappe/model/utils/rename_doc.py
+++ b/frappe/model/utils/rename_doc.py
@@ -2,14 +2,13 @@
 # License: MIT. See LICENSE
 
 from itertools import product
-from typing import Dict, List, Optional
 
 import frappe
 from frappe.model.rename_doc import get_link_fields
 
 
 def update_linked_doctypes(
-	doctype: str, docname: str, linked_to: str, value: str, ignore_doctypes: Optional[List] = None
+	doctype: str, docname: str, linked_to: str, value: str, ignore_doctypes: list | None = None
 ):
 	"""
 	linked_doctype_info_list = list formed by get_fetch_fields() function
@@ -31,8 +30,8 @@ def update_linked_doctypes(
 
 
 def get_fetch_fields(
-	doctype: str, linked_to: str, ignore_doctypes: Optional[List] = None
-) -> List[Dict]:
+	doctype: str, linked_to: str, ignore_doctypes: list | None = None
+) -> list[dict]:
 	"""
 	doctype = Master DocType in which the changes are being made
 	linked_to = DocType name of the field thats being updated in Master
diff --git a/frappe/model/utils/rename_field.py b/frappe/model/utils/rename_field.py
index 56e69455ef..9e4fc5d84a 100644
--- a/frappe/model/utils/rename_field.py
+++ b/frappe/model/utils/rename_field.py
@@ -40,7 +40,7 @@ def rename_field(doctype, old_fieldname, new_fieldname):
 			)
 		else:
 			# copy field value
-			frappe.db.sql("""update `tab%s` set `%s`=`%s`""" % (doctype, new_fieldname, old_fieldname))
+			frappe.db.sql(f"""update `tab{doctype}` set `{new_fieldname}`=`{old_fieldname}`""")
 
 		update_reports(doctype, old_fieldname, new_fieldname)
 		update_users_report_view_settings(doctype, old_fieldname, new_fieldname)
diff --git a/frappe/model/utils/user_settings.py b/frappe/model/utils/user_settings.py
index a6ae1a818e..c12c7e27ba 100644
--- a/frappe/model/utils/user_settings.py
+++ b/frappe/model/utils/user_settings.py
@@ -11,9 +11,7 @@ filter_dict = {"doctype": 0, "docfield": 1, "operator": 2, "value": 3}
 
 
 def get_user_settings(doctype, for_update=False):
-	user_settings = frappe.cache().hget(
-		"_user_settings", "{0}::{1}".format(doctype, frappe.session.user)
-	)
+	user_settings = frappe.cache().hget("_user_settings", f"{doctype}::{frappe.session.user}")
 
 	if user_settings is None:
 		user_settings = frappe.db.sql(
@@ -43,9 +41,7 @@ def update_user_settings(doctype, user_settings, for_update=False):
 
 		current.update(user_settings)
 
-	frappe.cache().hset(
-		"_user_settings", "{0}::{1}".format(doctype, frappe.session.user), json.dumps(current)
-	)
+	frappe.cache().hset("_user_settings", f"{doctype}::{frappe.session.user}", json.dumps(current))
 
 
 def sync_user_settings():
@@ -103,6 +99,4 @@ def update_user_settings_data(
 			)
 
 			# clear that user settings from the redis cache
-			frappe.cache().hset(
-				"_user_settings", "{0}::{1}".format(user_setting.doctype, user_setting.user), None
-			)
+			frappe.cache().hset("_user_settings", f"{user_setting.doctype}::{user_setting.user}", None)
diff --git a/frappe/model/virtual_doctype.py b/frappe/model/virtual_doctype.py
new file mode 100644
index 0000000000..dc228f9577
--- /dev/null
+++ b/frappe/model/virtual_doctype.py
@@ -0,0 +1,52 @@
+from typing import Protocol
+
+import frappe
+
+
+class VirtualDoctype(Protocol):
+	"""This class documents requirements that must be met by a doctype controller to function as virtual doctype
+
+
+	Additional requirements:
+	- DocType controller has to inherit from `frappe.model.document.Document` class
+
+	Note:
+	- "Backend" here means any storage service, it can be a database, flat file or network call to API.
+	"""
+
+	# ============ class/static methods ============
+
+	@staticmethod
+	def get_list(args) -> list[frappe._dict]:
+		"""Similar to reportview.get_list"""
+		...
+
+	@staticmethod
+	def get_count(args) -> int:
+		"""Similar to reportview.get_count, return total count of documents on listview."""
+		...
+
+	@staticmethod
+	def get_stats(args):
+		"""Similar to reportview.get_stats, return sidebar stats."""
+		...
+
+	# ============ instance methods ============
+
+	def db_insert(self, *args, **kwargs) -> None:
+		"""Serialize the `Document` object and insert it in backend."""
+		...
+
+	def load_from_db(self) -> None:
+		"""Using self.name initialize current document from backend data.
+
+		This is responsible for updatinng __dict__ of class with all the fields on doctype."""
+		...
+
+	def db_update(self, *args, **kwargs) -> None:
+		"""Serialize the `Document` object and update existing document in backend."""
+		...
+
+	def delete(self, *args, **kwargs) -> None:
+		"""Delete the current document from backend"""
+		...
diff --git a/frappe/model/workflow.py b/frappe/model/workflow.py
index 0edffaf2fb..923fbc1b3b 100644
--- a/frappe/model/workflow.py
+++ b/frappe/model/workflow.py
@@ -246,16 +246,17 @@ def bulk_workflow_approval(docnames, doctype, action):
 		except Exception as e:
 			if not frappe.message_log:
 				# Exception is	raised manually and not from msgprint or throw
-				message = "{0}".format(e.__class__.__name__)
+				message = f"{e.__class__.__name__}"
 				if e.args:
-					message += " : {0}".format(e.args[0])
+					message += f" : {e.args[0]}"
 				message_dict = {"docname": docname, "message": message}
 				failed_transactions[docname].append(message_dict)
 
 			frappe.db.rollback()
 			frappe.log_error(
-				frappe.get_traceback(),
-				"Workflow {0} threw an error for {1} {2}".format(action, doctype, docname),
+				title=f"Workflow {action} threw an error for {doctype} {docname}",
+				reference_doctype="Workflow",
+				reference_name=action,
 			)
 		finally:
 			if not message_dict:
@@ -285,19 +286,19 @@ def bulk_workflow_approval(docnames, doctype, action):
 
 def print_workflow_log(messages, title, doctype, indicator):
 	if messages.keys():
-		msg = "
        {0}
        ".format(title)
+		msg = f"
        {title}
        "
 
 		for doc in messages.keys():
 			if len(messages[doc]):
-				html = "
        {0}".format(frappe.utils.get_link_to_form(doctype, doc))
+				html = f"
        {frappe.utils.get_link_to_form(doctype, doc)}"
 				for log in messages[doc]:
 					if log.get("message"):
-						html += "
        {0}
        ".format(
+						html += "
        {}
        ".format(
 							log.get("message")
 						)
 				html += "
        "
 			else:
-				html = "
        {0}
        ".format(doc)
+				html = f"
        {doc}
        "
 			msg += html
 
 		frappe.msgprint(msg, title=_("Workflow Status"), indicator=indicator, is_minimizable=True)
diff --git a/frappe/modules.txt b/frappe/modules.txt
index a707ca853e..fb7817f6ba 100644
--- a/frappe/modules.txt
+++ b/frappe/modules.txt
@@ -8,7 +8,6 @@ Desk
 Integrations
 Printing
 Contacts
-Data Migration
 Social
 Automation
 Event Streaming
\ No newline at end of file
diff --git a/frappe/modules/export_file.py b/frappe/modules/export_file.py
index a1d527c91d..b448c04f2f 100644
--- a/frappe/modules/export_file.py
+++ b/frappe/modules/export_file.py
@@ -115,7 +115,7 @@ def create_folder(module, dt, dn, create_init):
 def get_custom_module_path(module):
 	package = frappe.db.get_value("Module Def", module, "package")
 	if not package:
-		frappe.throw("Package must be set for custom Module {module}".format(module=module))
+		frappe.throw(f"Package must be set for custom Module {module}")
 
 	path = os.path.join(get_package_path(package), scrub(module))
 	if not os.path.exists(path):
diff --git a/frappe/modules/import_file.py b/frappe/modules/import_file.py
index 00483bf6a5..3690da0657 100644
--- a/frappe/modules/import_file.py
+++ b/frappe/modules/import_file.py
@@ -109,7 +109,7 @@ def import_file_by_path(
 	"""
 	try:
 		docs = read_doc_from_file(path)
-	except IOError:
+	except OSError:
 		print(f"{path} missing")
 		return
 
@@ -144,7 +144,6 @@ def import_file_by_path(
 
 			import_doc(
 				docdict=doc,
-				force=force,
 				data_import=data_import,
 				pre_process=pre_process,
 				ignore_version=ignore_version,
@@ -173,14 +172,14 @@ def import_file_by_path(
 def read_doc_from_file(path):
 	doc = None
 	if os.path.exists(path):
-		with open(path, "r") as f:
+		with open(path) as f:
 			try:
 				doc = json.loads(f.read())
 			except ValueError:
-				print("bad json: {0}".format(path))
+				print(f"bad json: {path}")
 				raise
 	else:
-		raise IOError("%s missing" % path)
+		raise OSError("%s missing" % path)
 
 	return doc
 
@@ -203,7 +202,6 @@ def update_modified(original_modified, doc):
 
 def import_doc(
 	docdict,
-	force=False,
 	data_import=False,
 	pre_process=None,
 	ignore_version=None,
@@ -256,7 +254,7 @@ def load_code_properties(doc, path):
 			for key, extn in doc.get_code_fields().items():
 				codefile = os.path.join(dirname, filename.split(".")[0] + "." + extn)
 				if os.path.exists(codefile):
-					with open(codefile, "r") as txtfile:
+					with open(codefile) as txtfile:
 						doc.set(key, txtfile.read())
 
 
@@ -287,6 +285,6 @@ def reset_tree_properties(doc):
 	# "rgt". They are automatically set and kept up-to-date. Importing them
 	# would destroy any existing tree structure.
 	if getattr(doc.meta, "is_tree", None) and any([doc.lft, doc.rgt]):
-		print('Ignoring values of `lft` and `rgt` for {} "{}"'.format(doc.doctype, doc.name))
+		print(f'Ignoring values of `lft` and `rgt` for {doc.doctype} "{doc.name}"')
 		doc.lft = None
 		doc.rgt = None
diff --git a/frappe/modules/patch_handler.py b/frappe/modules/patch_handler.py
index ae6d9a6de2..f389312a4f 100644
--- a/frappe/modules/patch_handler.py
+++ b/frappe/modules/patch_handler.py
@@ -38,7 +38,6 @@ import configparser
 import time
 from enum import Enum
 from textwrap import dedent, indent
-from typing import List, Optional
 
 import frappe
 
@@ -52,7 +51,7 @@ class PatchType(Enum):
 	post_model_sync = "post_model_sync"
 
 
-def run_all(skip_failing: bool = False, patch_type: Optional[PatchType] = None) -> None:
+def run_all(skip_failing: bool = False, patch_type: PatchType | None = None) -> None:
 	"""run all pending patches"""
 	executed = set(frappe.get_all("Patch Log", fields="patch", pluck="patch"))
 
@@ -81,7 +80,7 @@ def run_all(skip_failing: bool = False, patch_type: Optional[PatchType] = None)
 		run_patch(patch)
 
 
-def get_all_patches(patch_type: Optional[PatchType] = None) -> List[str]:
+def get_all_patches(patch_type: PatchType | None = None) -> list[str]:
 
 	if patch_type and not isinstance(patch_type, PatchType):
 		frappe.throw(f"Unsupported patch type specified: {patch_type}")
@@ -93,7 +92,7 @@ def get_all_patches(patch_type: Optional[PatchType] = None) -> List[str]:
 	return patches
 
 
-def get_patches_from_app(app: str, patch_type: Optional[PatchType] = None) -> List[str]:
+def get_patches_from_app(app: str, patch_type: PatchType | None = None) -> list[str]:
 	"""Get patches from an app's patches.txt
 
 	patches.txt can be:
diff --git a/frappe/modules/utils.py b/frappe/modules/utils.py
index 1f89dae716..4fb1cfe2a1 100644
--- a/frappe/modules/utils.py
+++ b/frappe/modules/utils.py
@@ -7,7 +7,7 @@ import json
 import os
 from glob import glob
 from textwrap import dedent
-from typing import TYPE_CHECKING, Dict, Optional, Tuple, Union
+from typing import TYPE_CHECKING, Union
 
 import frappe
 from frappe import _, get_module_path, scrub
@@ -100,7 +100,7 @@ def export_customizations(
 		return file_path
 
 
-def sync_customizations(app: Optional[str] = None):
+def sync_customizations(app: str | None = None):
 	"""Sync custom fields and property setters from custom folder in each app module"""
 	apps = frappe.get_installed_apps() if not app else [app]
 
@@ -111,14 +111,14 @@ def sync_customizations(app: Optional[str] = None):
 				continue
 
 			for json_file in glob(os.path.join(module_custom_folder, "*.json")):
-				with open(os.path.join(module_custom_folder, json_file), "r") as f:
+				with open(os.path.join(module_custom_folder, json_file)) as f:
 					data = json.loads(f.read())
 
 				if data.get("sync_on_migrate"):
 					sync_customizations_for_doctype(data, module_custom_folder)
 
 
-def sync_customizations_for_doctype(data: Dict, folder: str):
+def sync_customizations_for_doctype(data: dict, folder: str):
 	"""Sync doctype customzations for a particular data set"""
 	from frappe.core.doctype.doctype.doctype import validate_fields_for_doctype
 
@@ -176,7 +176,7 @@ def sync_customizations_for_doctype(data: Dict, folder: str):
 		frappe.db.updatedb(doctype)
 
 
-def scrub_dt_dn(dt: str, dn: str) -> Tuple[str, str]:
+def scrub_dt_dn(dt: str, dn: str) -> tuple[str, str]:
 	"""Returns in lowercase and code friendly names of doctype and name for certain types"""
 	return scrub(dt), scrub(dn)
 
@@ -210,10 +210,15 @@ def export_doc(doctype, name, module=None):
 
 def get_doctype_module(doctype: str) -> str:
 	"""Returns **Module Def** name of given doctype."""
-	return frappe.cache().get_value(
+	doctype_module_map = frappe.cache().get_value(
 		"doctype_modules",
 		generator=lambda: dict(frappe.qb.from_("DocType").select("name", "module").run()),
-	)[doctype]
+	)
+
+	if module_name := doctype_module_map.get(doctype):
+		return module_name
+	else:
+		frappe.throw(_("DocType {} not found").format(doctype), exc=frappe.DoesNotExistError)
 
 
 def load_doctype_module(doctype, module=None, prefix="", suffix=""):
@@ -231,13 +236,15 @@ def load_doctype_module(doctype, module=None, prefix="", suffix=""):
 		try:
 			doctype_python_modules[key] = frappe.get_module(module_name)
 		except ImportError as e:
-			raise ImportError(f"Module import failed for {doctype} ({module_name} Error: {e})")
+			msg = f"Module import failed for {doctype}, the DocType you're trying to open might be deleted."
+			msg += f"
         Error: {e}"
+			raise ImportError(msg) from e
 
 	return doctype_python_modules[key]
 
 
 def get_module_name(
-	doctype: str, module: str, prefix: str = "", suffix: str = "", app: Optional[str] = None
+	doctype: str, module: str, prefix: str = "", suffix: str = "", app: str | None = None
 ):
 	app = scrub(app or get_module_app(module))
 	module = scrub(module)
@@ -246,18 +253,21 @@ def get_module_name(
 
 
 def get_module_app(module: str) -> str:
-	return frappe.local.module_app[scrub(module)]
+	app = frappe.local.module_app.get(scrub(module))
+	if app is None:
+		frappe.throw(_("Module {} not found").format(module), exc=frappe.DoesNotExistError)
+	return app
 
 
 def get_app_publisher(module: str) -> str:
-	app = frappe.local.module_app[scrub(module)]
+	app = get_module_app(module)
 	if not app:
 		frappe.throw(_("App not found for module: {0}").format(module))
 	return frappe.get_hooks(hook="app_publisher", app_name=app)[0]
 
 
 def make_boilerplate(
-	template: str, doc: Union["Document", "frappe._dict"], opts: Union[Dict, "frappe._dict"] = None
+	template: str, doc: Union["Document", "frappe._dict"], opts: Union[dict, "frappe._dict"] = None
 ):
 	target_path = get_doc_path(doc.module, doc.doctype, doc.name)
 	template_name = template.replace("controller", scrub(doc.name))
@@ -306,7 +316,7 @@ def make_boilerplate(
 			"""
 		)
 
-	with open(target_file_path, "w") as target, open(template_file_path, "r") as source:
+	with open(target_file_path, "w") as target, open(template_file_path) as source:
 		template = source.read()
 		controller_file_content = cstr(template).format(
 			app_publisher=app_publisher,
diff --git a/frappe/monitor.py b/frappe/monitor.py
index 74f9e06ef3..8d5391cb77 100644
--- a/frappe/monitor.py
+++ b/frappe/monitor.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
 
@@ -31,6 +30,8 @@ def log_file():
 
 
 class Monitor:
+	__slots__ = ("data",)
+
 	def __init__(self, transaction_type, method, kwargs):
 		try:
 			self.data = frappe._dict(
diff --git a/frappe/oauth.py b/frappe/oauth.py
index e7fa101bfd..68e21ac88b 100644
--- a/frappe/oauth.py
+++ b/frappe/oauth.py
@@ -323,10 +323,7 @@ class OAuthWebRequestValidator(RequestValidator):
 		# Check whether frappe server URL is set
 		id_token_header = {"typ": "jwt", "alg": "HS256"}
 
-		user = frappe.get_doc(
-			"User",
-			frappe.session.user,
-		)
+		user = frappe.get_doc("User", request.user)
 
 		if request.nonce:
 			id_token["nonce"] = request.nonce
diff --git a/frappe/parallel_test_runner.py b/frappe/parallel_test_runner.py
index f5367e9dc6..39a00235cb 100644
--- a/frappe/parallel_test_runner.py
+++ b/frappe/parallel_test_runner.py
@@ -46,7 +46,7 @@ class ParallelTestRunner:
 
 		if hasattr(test_module, "global_test_dependencies"):
 			for doctype in test_module.global_test_dependencies:
-				make_test_records(doctype)
+				make_test_records(doctype, commit=True)
 
 		elapsed = time.time() - start_time
 		elapsed = click.style(f" ({elapsed:.03}s)", fg="red")
@@ -76,17 +76,17 @@ class ParallelTestRunner:
 	def create_test_dependency_records(self, module, path, filename):
 		if hasattr(module, "test_dependencies"):
 			for doctype in module.test_dependencies:
-				make_test_records(doctype)
+				make_test_records(doctype, commit=True)
 
 		if os.path.basename(os.path.dirname(path)) == "doctype":
 			# test_data_migration_connector.py > data_migration_connector.json
 			test_record_filename = re.sub("^test_", "", filename).replace(".py", ".json")
 			test_record_file_path = os.path.join(path, test_record_filename)
 			if os.path.exists(test_record_file_path):
-				with open(test_record_file_path, "r") as f:
+				with open(test_record_file_path) as f:
 					doc = json.loads(f.read())
 					doctype = doc["name"]
-					make_test_records(doctype)
+					make_test_records(doctype, commit=True)
 
 	def get_module(self, path, filename):
 		app_path = frappe.get_pymodule_path(self.app)
@@ -117,6 +117,7 @@ class ParallelTestRunner:
 
 class ParallelTestResult(unittest.TextTestResult):
 	def startTest(self, test):
+		self.tb_locals = True
 		self._started_at = time.time()
 		super(unittest.TextTestResult, self).startTest(test)
 		test_class = unittest.util.strclass(test.__class__)
diff --git a/frappe/patches.txt b/frappe/patches.txt
index bc2bc22637..2f6ebd334a 100644
--- a/frappe/patches.txt
+++ b/frappe/patches.txt
@@ -66,7 +66,6 @@ execute:frappe.delete_doc_if_exists('Page', 'user-permissions')
 frappe.patches.v10_0.set_no_copy_to_workflow_state
 frappe.patches.v10_0.increase_single_table_column_length
 frappe.patches.v11_0.create_contact_for_user
-frappe.patches.v11_0.sync_stripe_settings_before_migrate
 frappe.patches.v11_0.update_list_user_settings
 frappe.patches.v11_0.rename_workflow_action_to_workflow_action_master #13-06-2018
 frappe.patches.v11_0.rename_email_alert_to_notification #13-06-2018
@@ -106,13 +105,12 @@ frappe.patches.v12_0.set_default_incoming_email_port
 frappe.patches.v12_0.update_global_search
 frappe.patches.v12_0.setup_tags
 frappe.patches.v12_0.update_auto_repeat_status_and_not_submittable
-frappe.patches.v12_0.copy_to_parent_for_tags
 frappe.patches.v12_0.create_notification_settings_for_user
 frappe.patches.v11_0.make_all_prepared_report_attachments_private #2019-11-26
 frappe.patches.v12_0.setup_email_linking
 frappe.patches.v12_0.change_existing_dashboard_chart_filters
 frappe.patches.v12_0.set_correct_assign_value_in_docs #2020-07-13
-execute:frappe.delete_doc("Test Runner")
+execute:frappe.delete_doc('DocType', 'Test Runner') # 2022-05-19
 execute:frappe.delete_doc_if_exists('DocType', 'Google Maps Settings')
 execute:frappe.db.set_default('desktop:home_page', 'workspace')
 execute:frappe.delete_doc_if_exists('DocType', 'GSuite Settings')
@@ -122,7 +120,7 @@ execute:frappe.delete_doc_if_exists('DocType', 'GCalendar Settings')
 frappe.patches.v12_0.remove_example_email_thread_notify
 execute:from frappe.desk.page.setup_wizard.install_fixtures import update_genders;update_genders()
 frappe.patches.v12_0.set_correct_url_in_files
-execute:frappe.reload_doc('core', 'doctype', 'doctype')
+execute:frappe.reload_doc('core', 'doctype', 'doctype') #2022-06-21
 execute:frappe.reload_doc('custom', 'doctype', 'property_setter')
 frappe.patches.v13_0.remove_invalid_options_for_data_fields
 frappe.patches.v13_0.website_theme_custom_scss
@@ -184,12 +182,20 @@ frappe.patches.v13_0.queryreport_columns
 frappe.patches.v13_0.jinja_hook
 frappe.patches.v13_0.update_notification_channel_if_empty
 frappe.patches.v13_0.set_first_day_of_the_week
+frappe.patches.v13_0.encrypt_2fa_secrets
+frappe.patches.v13_0.reset_corrupt_defaults
 execute:frappe.reload_doc('custom', 'doctype', 'custom_field')
 frappe.patches.v14_0.update_workspace2 # 20.09.2021
 frappe.patches.v14_0.save_ratings_in_fraction #23-12-2021
 frappe.patches.v14_0.transform_todo_schema
 frappe.patches.v14_0.remove_post_and_post_comment
 frappe.patches.v14_0.reset_creation_datetime
+frappe.patches.v14_0.remove_is_first_startup
+frappe.patches.v14_0.clear_long_pending_stale_logs
+frappe.patches.v14_0.log_settings_migration
+frappe.patches.v14_0.setup_likes_from_feedback
+frappe.patches.v14_0.update_webforms
+frappe.patches.v14_0.delete_payment_gateways
 
 [post_model_sync]
 frappe.patches.v14_0.drop_data_import_legacy
@@ -199,3 +205,8 @@ frappe.patches.v14_0.remove_db_aggregation
 frappe.patches.v14_0.update_color_names_in_kanban_board_column
 frappe.patches.v14_0.update_is_system_generated_flag
 frappe.patches.v14_0.update_auto_account_deletion_duration
+frappe.patches.v14_0.update_integration_request
+frappe.patches.v14_0.set_document_expiry_default
+frappe.patches.v14_0.delete_data_migration_tool
+frappe.patches.v14_0.set_suspend_email_queue_default
+frappe.patches.v14_0.different_encryption_key
diff --git a/frappe/patches/v10_0/increase_single_table_column_length.py b/frappe/patches/v10_0/increase_single_table_column_length.py
index 21b5a790ab..0b8e94a3f7 100644
--- a/frappe/patches/v10_0/increase_single_table_column_length.py
+++ b/frappe/patches/v10_0/increase_single_table_column_length.py
@@ -6,4 +6,4 @@ import frappe
 
 def execute():
 	for col in ("field", "doctype"):
-		frappe.db.sql_ddl("alter table `tabSingles` modify column `{0}` varchar(255)".format(col))
+		frappe.db.sql_ddl(f"alter table `tabSingles` modify column `{col}` varchar(255)")
diff --git a/frappe/patches/v11_0/delete_duplicate_user_permissions.py b/frappe/patches/v11_0/delete_duplicate_user_permissions.py
index b986c6f825..f2ca6d51fe 100644
--- a/frappe/patches/v11_0/delete_duplicate_user_permissions.py
+++ b/frappe/patches/v11_0/delete_duplicate_user_permissions.py
@@ -13,7 +13,7 @@ def execute():
 	for record in duplicateRecords:
 		frappe.db.sql(
 			"""delete from `tabUser Permission`
-			where allow=%s and user=%s and for_value=%s limit {0}""".format(
+			where allow=%s and user=%s and for_value=%s limit {}""".format(
 				record.count - 1
 			),
 			(record.allow, record.user, record.for_value),
diff --git a/frappe/patches/v11_0/drop_column_apply_user_permissions.py b/frappe/patches/v11_0/drop_column_apply_user_permissions.py
index bfc4aee72c..0a0091624e 100644
--- a/frappe/patches/v11_0/drop_column_apply_user_permissions.py
+++ b/frappe/patches/v11_0/drop_column_apply_user_permissions.py
@@ -8,7 +8,7 @@ def execute():
 	for doctype in to_remove:
 		if frappe.db.table_exists(doctype):
 			if column in frappe.db.get_table_columns(doctype):
-				frappe.db.sql("alter table `tab{0}` drop column {1}".format(doctype, column))
+				frappe.db.sql(f"alter table `tab{doctype}` drop column {column}")
 
 	frappe.reload_doc("core", "doctype", "docperm", force=True)
 	frappe.reload_doc("core", "doctype", "custom_docperm", force=True)
diff --git a/frappe/patches/v11_0/fix_order_by_in_reports_json.py b/frappe/patches/v11_0/fix_order_by_in_reports_json.py
index 3dfec0954f..4e955a338b 100644
--- a/frappe/patches/v11_0/fix_order_by_in_reports_json.py
+++ b/frappe/patches/v11_0/fix_order_by_in_reports_json.py
@@ -28,8 +28,8 @@ def execute():
 
 			sort_by = parts[1].split(" ")
 
-			json_data["order_by"] = "`tab{0}`.`{1}`".format(doc.ref_doctype, sort_by[0])
-			json_data["order_by"] += " {0}".format(sort_by[1]) if len(sort_by) > 1 else ""
+			json_data["order_by"] = f"`tab{doc.ref_doctype}`.`{sort_by[0]}`"
+			json_data["order_by"] += f" {sort_by[1]}" if len(sort_by) > 1 else ""
 
 			doc.json = json.dumps(json_data)
 			doc.save()
diff --git a/frappe/patches/v11_0/set_dropbox_file_backup.py b/frappe/patches/v11_0/set_dropbox_file_backup.py
index c9dec31414..396491e8b3 100644
--- a/frappe/patches/v11_0/set_dropbox_file_backup.py
+++ b/frappe/patches/v11_0/set_dropbox_file_backup.py
@@ -4,6 +4,6 @@ from frappe.utils import cint
 
 def execute():
 	frappe.reload_doctype("Dropbox Settings")
-	check_dropbox_enabled = cint(frappe.db.get_value("Dropbox Settings", None, "enabled"))
+	check_dropbox_enabled = cint(frappe.db.get_single_value("Dropbox Settings", "enabled"))
 	if check_dropbox_enabled == 1:
 		frappe.db.set_value("Dropbox Settings", None, "file_backup", 1)
diff --git a/frappe/patches/v11_0/sync_stripe_settings_before_migrate.py b/frappe/patches/v11_0/sync_stripe_settings_before_migrate.py
deleted file mode 100644
index 019ecef67c..0000000000
--- a/frappe/patches/v11_0/sync_stripe_settings_before_migrate.py
+++ /dev/null
@@ -1,25 +0,0 @@
-import frappe
-from frappe.utils.password import get_decrypted_password
-
-
-def execute():
-	publishable_key = frappe.db.sql(
-		"select value from tabSingles where doctype='Stripe Settings' and field='publishable_key'"
-	)
-	if publishable_key:
-		secret_key = get_decrypted_password(
-			"Stripe Settings", "Stripe Settings", fieldname="secret_key", raise_exception=False
-		)
-		if secret_key:
-			frappe.reload_doc("integrations", "doctype", "stripe_settings")
-			frappe.db.commit()
-
-			settings = frappe.new_doc("Stripe Settings")
-			settings.gateway_name = (
-				frappe.db.get_value("Global Defaults", None, "default_company") or "Stripe Settings"
-			)
-			settings.publishable_key = publishable_key
-			settings.secret_key = secret_key
-			settings.save(ignore_permissions=True)
-
-	frappe.db.delete("Singles", {"doctype": "Stripe Settings"})
diff --git a/frappe/patches/v11_0/sync_user_permission_doctype_before_migrate.py b/frappe/patches/v11_0/sync_user_permission_doctype_before_migrate.py
deleted file mode 100644
index 6b7a7695f6..0000000000
--- a/frappe/patches/v11_0/sync_user_permission_doctype_before_migrate.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import frappe
-
-
-def execute():
-	frappe.flags.in_patch = True
-	frappe.reload_doc("core", "doctype", "user_permission")
-	frappe.db.commit()
diff --git a/frappe/patches/v11_0/update_list_user_settings.py b/frappe/patches/v11_0/update_list_user_settings.py
index 146b29346c..5209b9e384 100644
--- a/frappe/patches/v11_0/update_list_user_settings.py
+++ b/frappe/patches/v11_0/update_list_user_settings.py
@@ -13,7 +13,7 @@ def execute():
 		# get user_settings for each user
 		settings = frappe.db.sql(
 			"select * from `__UserSettings` \
-			where user={0}".format(
+			where user={}".format(
 				frappe.db.escape(user.user)
 			),
 			as_dict=True,
diff --git a/frappe/patches/v12_0/copy_to_parent_for_tags.py b/frappe/patches/v12_0/copy_to_parent_for_tags.py
deleted file mode 100644
index ae3702a0d5..0000000000
--- a/frappe/patches/v12_0/copy_to_parent_for_tags.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import frappe
-
-
-def execute():
-
-	frappe.db.sql("UPDATE `tabTag Link` SET parenttype=document_type")
-	frappe.db.sql("UPDATE `tabTag Link` SET parent=document_name")
diff --git a/frappe/patches/v12_0/delete_duplicate_indexes.py b/frappe/patches/v12_0/delete_duplicate_indexes.py
index 6a6b0b3204..1cb94ca50c 100644
--- a/frappe/patches/v12_0/delete_duplicate_indexes.py
+++ b/frappe/patches/v12_0/delete_duplicate_indexes.py
@@ -1,5 +1,3 @@
-from pymysql import InternalError
-
 import frappe
 
 # This patch deletes all the duplicate indexes created for same column
@@ -43,13 +41,13 @@ def execute():
 	# build drop index query
 	for (table_name, index_list) in final_deletion_map.items():
 		query_list = []
-		alter_query = "ALTER TABLE `{}`".format(table_name)
+		alter_query = f"ALTER TABLE `{table_name}`"
 
 		for index in index_list:
-			query_list.append("{} DROP INDEX `{}`".format(alter_query, index))
+			query_list.append(f"{alter_query} DROP INDEX `{index}`")
 
 		for query in query_list:
 			try:
 				frappe.db.sql(query)
-			except InternalError:
+			except frappe.db.InternalError:
 				pass
diff --git a/frappe/patches/v12_0/delete_gsuite_if_exists.py b/frappe/patches/v12_0/delete_gsuite_if_exists.py
deleted file mode 100644
index 1fb3a8c2d0..0000000000
--- a/frappe/patches/v12_0/delete_gsuite_if_exists.py
+++ /dev/null
@@ -1,9 +0,0 @@
-import frappe
-
-
-def execute():
-	"""
-	Remove GSuite Template and GSuite Settings
-	"""
-	frappe.delete_doc_if_exists("DocType", "GSuite Settings")
-	frappe.delete_doc_if_exists("DocType", "GSuite Templates")
diff --git a/frappe/patches/v12_0/fix_public_private_files.py b/frappe/patches/v12_0/fix_public_private_files.py
index e1ad2f1862..382e3c2db1 100644
--- a/frappe/patches/v12_0/fix_public_private_files.py
+++ b/frappe/patches/v12_0/fix_public_private_files.py
@@ -30,7 +30,7 @@ def generate_file(file_name):
 
 		file_doc.file_url = new_doc.file_url
 		file_doc.save()
-	except IOError:
+	except OSError:
 		pass
 	except Exception as e:
 		print(e)
diff --git a/frappe/patches/v12_0/init_desk_settings.py b/frappe/patches/v12_0/init_desk_settings.py
deleted file mode 100644
index 5ec9764e8f..0000000000
--- a/frappe/patches/v12_0/init_desk_settings.py
+++ /dev/null
@@ -1,11 +0,0 @@
-import json
-
-import frappe
-from frappe.config import get_modules_from_all_apps_for_user
-from frappe.desk.moduleview import get_onboard_items
-
-
-def execute():
-	"""Reset the initial customizations for desk, with modules, indices and links."""
-	frappe.reload_doc("core", "doctype", "user")
-	frappe.db.sql("""update tabUser set home_settings = ''""")
diff --git a/frappe/patches/v12_0/move_timeline_links_to_dynamic_links.py b/frappe/patches/v12_0/move_timeline_links_to_dynamic_links.py
index 2207edd958..4d2061c5ac 100644
--- a/frappe/patches/v12_0/move_timeline_links_to_dynamic_links.py
+++ b/frappe/patches/v12_0/move_timeline_links_to_dynamic_links.py
@@ -22,7 +22,7 @@ def execute():
 		if communication.timeline_doctype and communication.timeline_name:
 			name += 1
 			values.append(
-				"""({0}, "{1}", "timeline_links", "Communication", "{2}", "{3}", "{4}", "{5}", "{6}", "{7}")""".format(
+				"""({}, "{}", "timeline_links", "Communication", "{}", "{}", "{}", "{}", "{}", "{}")""".format(
 					counter,
 					str(name),
 					frappe.db.escape(communication.name),
@@ -37,7 +37,7 @@ def execute():
 		if communication.link_doctype and communication.link_name:
 			name += 1
 			values.append(
-				"""({0}, "{1}", "timeline_links", "Communication", "{2}", "{3}", "{4}", "{5}", "{6}", "{7}")""".format(
+				"""({}, "{}", "timeline_links", "Communication", "{}", "{}", "{}", "{}", "{}", "{}")""".format(
 					counter,
 					str(name),
 					frappe.db.escape(communication.name),
@@ -55,7 +55,7 @@ def execute():
 				INSERT INTO `tabCommunication Link`
 					(`idx`, `name`, `parentfield`, `parenttype`, `parent`, `link_doctype`, `link_name`, `creation`,
 					`modified`, `modified_by`)
-				VALUES {0}
+				VALUES {}
 			""".format(
 					", ".join([d for d in values])
 				)
diff --git a/frappe/patches/v12_0/remove_gcalendar_gmaps.py b/frappe/patches/v12_0/remove_gcalendar_gmaps.py
deleted file mode 100644
index 1177441130..0000000000
--- a/frappe/patches/v12_0/remove_gcalendar_gmaps.py
+++ /dev/null
@@ -1,11 +0,0 @@
-import frappe
-
-
-def execute():
-	"""
-	Remove GCalendar and GCalendar Settings
-	Remove Google Maps Settings as its been merged with Delivery Trips
-	"""
-	frappe.delete_doc_if_exists("DocType", "GCalendar Account")
-	frappe.delete_doc_if_exists("DocType", "GCalendar Settings")
-	frappe.delete_doc_if_exists("DocType", "Google Maps Settings")
diff --git a/frappe/patches/v12_0/set_correct_url_in_files.py b/frappe/patches/v12_0/set_correct_url_in_files.py
index dca42a3c04..fee0b5d6fc 100644
--- a/frappe/patches/v12_0/set_correct_url_in_files.py
+++ b/frappe/patches/v12_0/set_correct_url_in_files.py
@@ -30,12 +30,10 @@ def execute():
 			if file_is_private:
 				public_file_url = os.path.join(public_file_path, file_name)
 				if os.path.exists(public_file_url):
-					frappe.db.set_value(
-						"File", file.name, {"file_url": "/files/{0}".format(file_name), "is_private": 0}
-					)
+					frappe.db.set_value("File", file.name, {"file_url": f"/files/{file_name}", "is_private": 0})
 			else:
 				private_file_url = os.path.join(private_file_path, file_name)
 				if os.path.exists(private_file_url):
 					frappe.db.set_value(
-						"File", file.name, {"file_url": "/private/files/{0}".format(file_name), "is_private": 1}
+						"File", file.name, {"file_url": f"/private/files/{file_name}", "is_private": 1}
 					)
diff --git a/frappe/patches/v12_0/setup_tags.py b/frappe/patches/v12_0/setup_tags.py
index 46482a102b..6bff8d3dac 100644
--- a/frappe/patches/v12_0/setup_tags.py
+++ b/frappe/patches/v12_0/setup_tags.py
@@ -17,7 +17,7 @@ def execute():
 			continue
 
 		for _user_tags in frappe.db.sql(
-			"select `name`, `_user_tags` from `tab{0}`".format(doctype.name), as_dict=True
+			f"select `name`, `_user_tags` from `tab{doctype.name}`", as_dict=True
 		):
 			if not _user_tags.get("_user_tags"):
 				continue
diff --git a/frappe/patches/v12_0/webpage_migrate_description_to_meta_tag.py b/frappe/patches/v12_0/webpage_migrate_description_to_meta_tag.py
deleted file mode 100644
index 32473481b8..0000000000
--- a/frappe/patches/v12_0/webpage_migrate_description_to_meta_tag.py
+++ /dev/null
@@ -1,12 +0,0 @@
-import frappe
-
-
-def execute():
-	web_pages = frappe.get_all("Web Page", ["name", "description"])
-
-	for web_page in web_pages:
-		if web_page.description and web_page.route:
-			doc = frappe.new_doc("Website Route Meta")
-			doc.name = web_page.route
-			doc.append("meta_tags", {"key": "description", "value": web_page.description})
-			doc.save()
diff --git a/frappe/patches/v12_0/website_meta_tag_parent.py b/frappe/patches/v12_0/website_meta_tag_parent.py
deleted file mode 100644
index 8920189826..0000000000
--- a/frappe/patches/v12_0/website_meta_tag_parent.py
+++ /dev/null
@@ -1,12 +0,0 @@
-import frappe
-
-
-def execute():
-	# convert all /path to path
-	frappe.db.sql(
-		"""
-		UPDATE `tabWebsite Meta Tag`
-		SET parent = SUBSTR(parent, 2)
-		WHERE parent like '/%'
-	"""
-	)
diff --git a/frappe/patches/v13_0/cleanup_desk_cards.py b/frappe/patches/v13_0/cleanup_desk_cards.py
deleted file mode 100644
index 988e98a647..0000000000
--- a/frappe/patches/v13_0/cleanup_desk_cards.py
+++ /dev/null
@@ -1,75 +0,0 @@
-from json import loads
-
-import frappe
-from frappe.desk.doctype.workspace.workspace import get_link_type, get_report_type
-
-
-def execute():
-	frappe.reload_doc("desk", "doctype", "workspace")
-
-	pages = frappe.db.sql("Select `name` from `tabDesk Page`")
-	# pages = frappe.get_all("Workspace", filters={"is_standard": 0}, pluck="name")
-
-	for page in pages:
-		rebuild_links(page[0])
-
-	frappe.delete_doc("DocType", "Desk Card")
-
-
-def rebuild_links(page):
-	# Empty links table
-
-	try:
-		doc = frappe.get_doc("Workspace", page)
-	except frappe.DoesNotExistError:
-		db_doc = get_doc_from_db(page)
-
-		doc = frappe.get_doc(db_doc)
-		doc.insert(ignore_permissions=True)
-
-	doc.links = []
-
-	for card in get_all_cards(page):
-		if isinstance(card.links, str):
-			links = loads(card.links)
-		else:
-			links = card.links
-
-		doc.append(
-			"links",
-			{"label": card.label, "type": "Card Break", "icon": card.icon, "hidden": card.hidden or False},
-		)
-
-		for link in links:
-			if not frappe.db.exists(get_link_type(link.get("type")), link.get("name")):
-				continue
-
-			doc.append(
-				"links",
-				{
-					"label": link.get("label") or link.get("name"),
-					"type": "Link",
-					"link_type": get_link_type(link.get("type")),
-					"link_to": link.get("name"),
-					"onboard": link.get("onboard"),
-					"dependencies": ", ".join(link.get("dependencies", [])),
-					"is_query_report": get_report_type(link.get("name"))
-					if link.get("type").lower() == "report"
-					else 0,
-				},
-			)
-
-		try:
-			doc.save(ignore_permissions=True)
-		except frappe.LinkValidationError:
-			print(doc.as_dict())
-
-
-def get_doc_from_db(page):
-	result = frappe.db.sql("SELECT * FROM `tabDesk Page` WHERE name=%s", [page], as_dict=True)
-	if result:
-		return result[0].update({"doctype": "Workspace"})
-
-
-def get_all_cards(page):
-	return frappe.db.get_all("Desk Card", filters={"parent": page}, fields=["*"], order_by="idx")
diff --git a/frappe/patches/v13_0/encrypt_2fa_secrets.py b/frappe/patches/v13_0/encrypt_2fa_secrets.py
new file mode 100644
index 0000000000..1814ff50c5
--- /dev/null
+++ b/frappe/patches/v13_0/encrypt_2fa_secrets.py
@@ -0,0 +1,45 @@
+import frappe
+import frappe.defaults
+from frappe.cache_manager import clear_defaults_cache
+from frappe.twofactor import PARENT_FOR_DEFAULTS
+from frappe.utils.password import encrypt
+
+DOCTYPE = "DefaultValue"
+OLD_PARENT = "__default"
+
+
+def execute():
+	table = frappe.qb.DocType(DOCTYPE)
+
+	# set parent for `*_otplogin`
+	(
+		frappe.qb.update(table)
+		.set(table.parent, PARENT_FOR_DEFAULTS)
+		.where(table.parent == OLD_PARENT)
+		.where(table.defkey.like("%_otplogin"))
+	).run()
+
+	# update records for `*_otpsecret`
+	secrets = {
+		key: value
+		for key, value in frappe.defaults.get_defaults_for(parent=OLD_PARENT).items()
+		if key.endswith("_otpsecret")
+	}
+
+	if not secrets:
+		return
+
+	defvalue_cases = frappe.qb.terms.Case()
+
+	for key, value in secrets.items():
+		defvalue_cases.when(table.defkey == key, encrypt(value))
+
+	(
+		frappe.qb.update(table)
+		.set(table.parent, PARENT_FOR_DEFAULTS)
+		.set(table.defvalue, defvalue_cases)
+		.where(table.parent == OLD_PARENT)
+		.where(table.defkey.like("%_otpsecret"))
+	).run()
+
+	clear_defaults_cache()
diff --git a/frappe/patches/v13_0/generate_theme_files_in_public_folder.py b/frappe/patches/v13_0/generate_theme_files_in_public_folder.py
index 9b905a9bbb..29ddca1108 100644
--- a/frappe/patches/v13_0/generate_theme_files_in_public_folder.py
+++ b/frappe/patches/v13_0/generate_theme_files_in_public_folder.py
@@ -14,6 +14,6 @@ def execute():
 		try:
 			doc.generate_bootstrap_theme()
 			doc.save()
-		except:  # noqa: E722
+		except Exception:
 			print("Ignoring....")
 			print(frappe.get_traceback())
diff --git a/frappe/patches/v13_0/reset_corrupt_defaults.py b/frappe/patches/v13_0/reset_corrupt_defaults.py
new file mode 100644
index 0000000000..10e81c7ff1
--- /dev/null
+++ b/frappe/patches/v13_0/reset_corrupt_defaults.py
@@ -0,0 +1,33 @@
+import frappe
+from frappe.patches.v13_0.encrypt_2fa_secrets import DOCTYPE
+from frappe.patches.v13_0.encrypt_2fa_secrets import PARENT_FOR_DEFAULTS as TWOFACTOR_PARENT
+from frappe.utils import cint
+
+
+def execute():
+	"""
+	This patch is needed to fix parent incorrectly set as `__2fa` because of
+	https://github.com/frappe/frappe/commit/a822092211533ff17ff9b92dd86f6f868ed63e2e
+	"""
+
+	if not frappe.db.get_value(
+		DOCTYPE, {"parent": TWOFACTOR_PARENT, "defkey": ("not like", "%_otp%")}, "defkey"
+	):
+		return
+
+	# system settings
+	system_settings = frappe.get_single("System Settings")
+	system_settings.set_defaults()
+
+	# home page
+	frappe.db.set_default(
+		"desktop:home_page", "workspace" if cint(system_settings.setup_complete) else "setup-wizard"
+	)
+
+	# letter head
+	try:
+		letter_head = frappe.get_doc("Letter Head", {"is_default": 1})
+		letter_head.set_as_default()
+
+	except frappe.DoesNotExistError:
+		pass
diff --git a/frappe/patches/v13_0/set_unique_for_page_view.py b/frappe/patches/v13_0/set_unique_for_page_view.py
index 1a674f6697..45351afdde 100644
--- a/frappe/patches/v13_0/set_unique_for_page_view.py
+++ b/frappe/patches/v13_0/set_unique_for_page_view.py
@@ -4,6 +4,4 @@ import frappe
 def execute():
 	frappe.reload_doc("website", "doctype", "web_page_view", force=True)
 	site_url = frappe.utils.get_site_url(frappe.local.site)
-	frappe.db.sql(
-		"""UPDATE `tabWeb Page View` set is_unique=1 where referrer LIKE '%{0}%'""".format(site_url)
-	)
+	frappe.db.sql(f"""UPDATE `tabWeb Page View` set is_unique=1 where referrer LIKE '%{site_url}%'""")
diff --git a/frappe/patches/v14_0/clear_long_pending_stale_logs.py b/frappe/patches/v14_0/clear_long_pending_stale_logs.py
new file mode 100644
index 0000000000..53127cb197
--- /dev/null
+++ b/frappe/patches/v14_0/clear_long_pending_stale_logs.py
@@ -0,0 +1,41 @@
+import frappe
+from frappe.core.doctype.log_settings.log_settings import clear_log_table
+from frappe.utils import add_to_date, today
+
+
+def execute():
+	"""Due to large size of log tables on old sites some table cleanups never finished during daily log clean up. This patch discards such data by using "big delete" code.
+
+	ref: https://github.com/frappe/frappe/issues/16971
+	"""
+
+	DOCTYPE_RETENTION_MAP = {
+		"Error Log": get_current_setting("clear_error_log_after") or 90,
+		"Activity Log": get_current_setting("clear_activity_log_after") or 90,
+		"Email Queue": get_current_setting("clear_email_queue_after") or 30,
+		# child table on email queue
+		"Email Queue Recipient": get_current_setting("clear_email_queue_after") or 30,
+		"Error Snapshot": get_current_setting("clear_error_log_after") or 90,
+		# newly added
+		"Scheduled Job Log": 90,
+	}
+
+	for doctype, retention in DOCTYPE_RETENTION_MAP.items():
+		if is_log_cleanup_stuck(doctype, retention):
+			print(f"Clearing old {doctype} records")
+			clear_log_table(doctype, retention)
+
+
+def is_log_cleanup_stuck(doctype: str, retention: int) -> bool:
+	"""Check if doctype has data significantly older than configured cleanup period"""
+	threshold = add_to_date(today(), days=retention * -2)
+
+	return bool(frappe.db.exists(doctype, {"modified": ("<", threshold)}))
+
+
+def get_current_setting(fieldname):
+	try:
+		return frappe.db.get_single_value("Log Settings", fieldname)
+	except Exception:
+		# Field might be gone if patch is reattempted
+		pass
diff --git a/frappe/patches/v14_0/copy_mail_data.py b/frappe/patches/v14_0/copy_mail_data.py
index 6b976ba6fb..c44b7c9e92 100644
--- a/frappe/patches/v14_0/copy_mail_data.py
+++ b/frappe/patches/v14_0/copy_mail_data.py
@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 import frappe
 
 
diff --git a/frappe/patches/v14_0/delete_data_migration_tool.py b/frappe/patches/v14_0/delete_data_migration_tool.py
new file mode 100644
index 0000000000..d0416cb1e7
--- /dev/null
+++ b/frappe/patches/v14_0/delete_data_migration_tool.py
@@ -0,0 +1,12 @@
+# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors
+# MIT License. See license.txt
+
+import frappe
+
+
+def execute():
+	doctypes = frappe.db.get_all("DocType", {"module": "Data Migration", "custom": 0}, pluck="name")
+	for doctype in doctypes:
+		frappe.delete_doc("DocType", doctype, ignore_missing=True)
+
+	frappe.delete_doc("Module Def", "Data Migration", ignore_missing=True, force=True)
diff --git a/frappe/patches/v14_0/delete_payment_gateways.py b/frappe/patches/v14_0/delete_payment_gateways.py
new file mode 100644
index 0000000000..c06f63a2d3
--- /dev/null
+++ b/frappe/patches/v14_0/delete_payment_gateways.py
@@ -0,0 +1,16 @@
+import frappe
+
+
+def execute():
+	if "payments" in frappe.get_installed_apps():
+		return
+
+	for doctype in (
+		"Payment Gateway",
+		"Razorpay Settings",
+		"Braintree Settings",
+		"PayPal Settings",
+		"Paytm Settings",
+		"Stripe Settings",
+	):
+		frappe.delete_doc_if_exists("DocType", doctype, force=True)
diff --git a/frappe/patches/v14_0/different_encryption_key.py b/frappe/patches/v14_0/different_encryption_key.py
new file mode 100644
index 0000000000..3b80e15a73
--- /dev/null
+++ b/frappe/patches/v14_0/different_encryption_key.py
@@ -0,0 +1,16 @@
+import pathlib
+
+import frappe
+from frappe.installer import update_site_config
+from frappe.utils.backups import BACKUP_ENCRYPTION_CONFIG_KEY, get_backup_path
+
+
+def execute():
+	if frappe.conf.get(BACKUP_ENCRYPTION_CONFIG_KEY):
+		return
+
+	backup_path = pathlib.Path(get_backup_path())
+	encrypted_backups_present = bool(list(backup_path.glob("*-enc*")))
+
+	if encrypted_backups_present:
+		update_site_config(BACKUP_ENCRYPTION_CONFIG_KEY, frappe.local.conf.encryption_key)
diff --git a/frappe/patches/v14_0/log_settings_migration.py b/frappe/patches/v14_0/log_settings_migration.py
new file mode 100644
index 0000000000..203405e69b
--- /dev/null
+++ b/frappe/patches/v14_0/log_settings_migration.py
@@ -0,0 +1,29 @@
+import frappe
+
+
+def execute():
+	old_settings = {
+		"Error Log": get_current_setting("clear_error_log_after"),
+		"Activity Log": get_current_setting("clear_activity_log_after"),
+		"Email Queue": get_current_setting("clear_email_queue_after"),
+	}
+
+	frappe.reload_doc("core", "doctype", "Logs To Clear")
+	frappe.reload_doc("core", "doctype", "Log Settings")
+
+	log_settings = frappe.get_doc("Log Settings")
+	log_settings.add_default_logtypes()
+
+	for doctype, retention in old_settings.items():
+		if retention:
+			log_settings.register_doctype(doctype, retention)
+
+	log_settings.save()
+
+
+def get_current_setting(fieldname):
+	try:
+		return frappe.db.get_single_value("Log Settings", fieldname)
+	except Exception:
+		# Field might be gone if patch is reattempted
+		pass
diff --git a/frappe/patches/v14_0/remove_db_aggregation.py b/frappe/patches/v14_0/remove_db_aggregation.py
index 6dc34a784b..4b0a58c2d6 100644
--- a/frappe/patches/v14_0/remove_db_aggregation.py
+++ b/frappe/patches/v14_0/remove_db_aggregation.py
@@ -30,6 +30,6 @@ def execute():
 		name, script = server_script["name"], server_script["script"]
 
 		for agg in ["avg", "max", "min", "sum"]:
-			script = re.sub(f"frappe.db.{agg}\(", f"frappe.qb.{agg}(", script)
+			script = re.sub(f"frappe.db.{agg}\\(", f"frappe.qb.{agg}(", script)
 
 		frappe.db.update("Server Script", name, "script", script)
diff --git a/frappe/patches/v14_0/remove_is_first_startup.py b/frappe/patches/v14_0/remove_is_first_startup.py
new file mode 100644
index 0000000000..cae38ce2ab
--- /dev/null
+++ b/frappe/patches/v14_0/remove_is_first_startup.py
@@ -0,0 +1,8 @@
+import frappe
+
+
+def execute():
+	singles = frappe.qb.Table("tabSingles")
+	frappe.qb.from_(singles).delete().where(
+		(singles.doctype == "System Settings") & (singles.field == "is_first_startup")
+	).run()
diff --git a/frappe/patches/v14_0/set_document_expiry_default.py b/frappe/patches/v14_0/set_document_expiry_default.py
new file mode 100644
index 0000000000..59a9db6c4d
--- /dev/null
+++ b/frappe/patches/v14_0/set_document_expiry_default.py
@@ -0,0 +1,9 @@
+import frappe
+
+
+def execute():
+	frappe.db.set_value(
+		"System Settings",
+		"System Settings",
+		{"document_share_key_expiry": 30, "allow_older_web_view_links": 1},
+	)
diff --git a/frappe/patches/v14_0/set_suspend_email_queue_default.py b/frappe/patches/v14_0/set_suspend_email_queue_default.py
new file mode 100644
index 0000000000..8cdb05a177
--- /dev/null
+++ b/frappe/patches/v14_0/set_suspend_email_queue_default.py
@@ -0,0 +1,13 @@
+import frappe
+from frappe.cache_manager import clear_defaults_cache
+
+
+def execute():
+	frappe.db.set_default(
+		"suspend_email_queue",
+		frappe.db.get_default("hold_queue", "Administrator") or 0,
+		parent="__default",
+	)
+
+	frappe.db.delete("DefaultValue", {"defkey": "hold_queue"})
+	clear_defaults_cache()
diff --git a/frappe/patches/v14_0/setup_likes_from_feedback.py b/frappe/patches/v14_0/setup_likes_from_feedback.py
new file mode 100644
index 0000000000..d88f69ce4b
--- /dev/null
+++ b/frappe/patches/v14_0/setup_likes_from_feedback.py
@@ -0,0 +1,30 @@
+import frappe
+
+
+def execute():
+	frappe.reload_doctype("Comment")
+
+	if frappe.db.count("Feedback") > 20000:
+		frappe.db.auto_commit_on_many_writes = True
+
+	for feedback in frappe.get_all("Feedback", fields=["*"]):
+		if feedback.like:
+			new_comment = frappe.new_doc("Comment")
+			new_comment.comment_type = "Like"
+			new_comment.comment_email = feedback.owner
+			new_comment.content = "Liked by: " + feedback.owner
+			new_comment.reference_doctype = feedback.reference_doctype
+			new_comment.reference_name = feedback.reference_name
+			new_comment.creation = feedback.creation
+			new_comment.modified = feedback.modified
+			new_comment.owner = feedback.owner
+			new_comment.modified_by = feedback.modified_by
+			new_comment.ip_address = feedback.ip_address
+			new_comment.db_insert()
+
+	if frappe.db.auto_commit_on_many_writes:
+		frappe.db.auto_commit_on_many_writes = False
+
+	# clean up
+	frappe.db.delete("Feedback")
+	frappe.db.commit()
diff --git a/frappe/patches/v14_0/update_color_names_in_kanban_board_column.py b/frappe/patches/v14_0/update_color_names_in_kanban_board_column.py
index f8d6f236cd..b568151273 100644
--- a/frappe/patches/v14_0/update_color_names_in_kanban_board_column.py
+++ b/frappe/patches/v14_0/update_color_names_in_kanban_board_column.py
@@ -1,7 +1,6 @@
 # Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors
 # MIT License. See license.txt
 
-from __future__ import unicode_literals
 
 import frappe
 
diff --git a/frappe/patches/v14_0/update_github_endpoints.py b/frappe/patches/v14_0/update_github_endpoints.py
index 8f9a06a043..5ea638f0a6 100644
--- a/frappe/patches/v14_0/update_github_endpoints.py
+++ b/frappe/patches/v14_0/update_github_endpoints.py
@@ -1,10 +1,9 @@
 import frappe
 import json
 
+
 def execute():
 	if frappe.db.exists("Social Login Key", "github"):
-		frappe.db.set_value("Social Login Key", "github", "auth_url_data",
-			json.dumps({
-				"scope": "user:email"
-			})
+		frappe.db.set_value(
+			"Social Login Key", "github", "auth_url_data", json.dumps({"scope": "user:email"})
 		)
diff --git a/frappe/patches/v14_0/update_integration_request.py b/frappe/patches/v14_0/update_integration_request.py
new file mode 100644
index 0000000000..d067411166
--- /dev/null
+++ b/frappe/patches/v14_0/update_integration_request.py
@@ -0,0 +1,21 @@
+import frappe
+
+
+def execute():
+	doctype = "Integration Request"
+
+	if not frappe.db.has_column(doctype, "integration_type"):
+		return
+
+	frappe.db.set_value(
+		doctype,
+		{"integration_type": "Remote", "integration_request_service": ("!=", "PayPal")},
+		"is_remote_request",
+		1,
+	)
+	frappe.db.set_value(
+		doctype,
+		{"integration_type": "Subscription Notification"},
+		"request_description",
+		"Subscription Notification",
+	)
diff --git a/frappe/patches/v14_0/update_webforms.py b/frappe/patches/v14_0/update_webforms.py
new file mode 100644
index 0000000000..46918f216e
--- /dev/null
+++ b/frappe/patches/v14_0/update_webforms.py
@@ -0,0 +1,14 @@
+# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors
+# MIT License. See license.txt
+
+
+import frappe
+
+
+def execute():
+	frappe.reload_doc("website", "doctype", "web_form_list_column")
+	frappe.reload_doctype("Web Form")
+
+	for web_form in frappe.db.get_all("Web Form", fields=["*"]):
+		if web_form.allow_multiple and not web_form.show_list:
+			frappe.db.set_value("Web Form", web_form.name, "show_list", True)
diff --git a/frappe/patches/v14_0/update_workspace2.py b/frappe/patches/v14_0/update_workspace2.py
index c6586f46a1..a6c9db503f 100644
--- a/frappe/patches/v14_0/update_workspace2.py
+++ b/frappe/patches/v14_0/update_workspace2.py
@@ -7,6 +7,16 @@ from frappe import _
 def execute():
 	frappe.reload_doc("desk", "doctype", "workspace", force=True)
 
+	child_tables = frappe.get_all(
+		"DocField",
+		pluck="options",
+		filters={"fieldtype": ["in", frappe.model.table_fields], "parent": "Workspace"},
+	)
+
+	for child_table in child_tables:
+		if child_table != "Has Role":
+			frappe.reload_doc("desk", "doctype", child_table, force=True)
+
 	for seq, workspace in enumerate(frappe.get_all("Workspace", order_by="name asc")):
 		doc = frappe.get_doc("Workspace", workspace.name)
 		content = create_content(doc)
diff --git a/frappe/permissions.py b/frappe/permissions.py
index 8980d2e63e..9b781015b4 100644
--- a/frappe/permissions.py
+++ b/frappe/permissions.py
@@ -6,7 +6,7 @@ import frappe
 import frappe.share
 from frappe import _, msgprint
 from frappe.query_builder import DocType
-from frappe.utils import cint
+from frappe.utils import cint, cstr
 
 rights = (
 	"select",
@@ -28,6 +28,14 @@ rights = (
 
 
 def check_admin_or_system_manager(user=None):
+	from frappe.utils.commands import warn
+
+	warn(
+		"The function check_admin_or_system_manager will be deprecated in version 15."
+		'Please use frappe.only_for("System Manager") instead.',
+		category=PendingDeprecationWarning,
+	)
+
 	if not user:
 		user = frappe.session.user
 
@@ -360,9 +368,7 @@ def has_controller_permissions(doc, ptype, user=None):
 
 
 def get_doctypes_with_read():
-	return list(
-		{p.parent if type(p.parent) == str else p.parent.encode("UTF8") for p in get_valid_perms()}
-	)
+	return list({cstr(p.parent) for p in get_valid_perms() if p.parent})
 
 
 def get_valid_perms(doctype=None, user=None):
@@ -420,7 +426,7 @@ def get_roles(user=None, with_standard=True):
 
 	# filter standard if required
 	if not with_standard:
-		roles = filter(lambda x: x not in ["All", "Guest", "Administrator"], roles)
+		roles = [r for r in roles if r not in ["All", "Guest", "Administrator"]]
 
 	return roles
 
@@ -491,6 +497,7 @@ def add_user_permission(
 				for_value=name,
 				is_default=is_default,
 				applicable_for=applicable_for,
+				apply_to_all_doctypes=0 if applicable_for else 1,
 				hide_descendants=hide_descendants,
 			)
 		).insert(ignore_permissions=ignore_permissions)
@@ -515,7 +522,7 @@ def clear_user_permissions_for_doctype(doctype, user=None):
 def can_import(doctype, raise_exception=False):
 	if not ("System Manager" in frappe.get_roles() or has_permission(doctype, "import")):
 		if raise_exception:
-			raise frappe.PermissionError("You are not allowed to import: {doctype}".format(doctype=doctype))
+			raise frappe.PermissionError(f"You are not allowed to import: {doctype}")
 		else:
 			return False
 	return True
@@ -605,19 +612,17 @@ def reset_perms(doctype):
 	frappe.db.delete("Custom DocPerm", {"parent": doctype})
 
 
-def get_linked_doctypes(dt):
-	return list(
-		set(
-			[dt]
-			+ [
-				d.options
-				for d in frappe.get_meta(dt).get(
-					"fields",
-					{"fieldtype": "Link", "ignore_user_permissions": ("!=", 1), "options": ("!=", "[Select]")},
-				)
-			]
+def get_linked_doctypes(dt: str) -> list:
+	meta = frappe.get_meta(dt)
+	linked_doctypes = [dt] + [
+		d.options
+		for d in meta.get(
+			"fields",
+			{"fieldtype": "Link", "ignore_user_permissions": ("!=", 1), "options": ("!=", "[Select]")},
 		)
-	)
+	]
+
+	return list(set(linked_doctypes))
 
 
 def get_doc_name(doc):
diff --git a/frappe/printing/doctype/letter_head/letter_head.js b/frappe/printing/doctype/letter_head/letter_head.js
index ca4dad2d07..55d97cf37f 100644
--- a/frappe/printing/doctype/letter_head/letter_head.js
+++ b/frappe/printing/doctype/letter_head/letter_head.js
@@ -1,8 +1,8 @@
 // Copyright (c) 2017, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Letter Head', {
-	refresh: function(frm) {
+frappe.ui.form.on("Letter Head", {
+	refresh: function (frm) {
 		frm.flag_public_attachments = true;
-	}
+	},
 });
diff --git a/frappe/printing/doctype/letter_head/letter_head.json b/frappe/printing/doctype/letter_head/letter_head.json
index f723a6b489..d49b65ab36 100644
--- a/frappe/printing/doctype/letter_head/letter_head.json
+++ b/frappe/printing/doctype/letter_head/letter_head.json
@@ -9,6 +9,7 @@
  "field_order": [
   "letter_head_name",
   "source",
+  "footer_source",
   "column_break_3",
   "disabled",
   "is_default",
@@ -20,7 +21,12 @@
   "header_section",
   "content",
   "footer_section",
-  "footer"
+  "footer",
+  "footer_image_section",
+  "footer_image",
+  "footer_image_height",
+  "footer_image_width",
+  "footer_align"
  ],
  "fields": [
   {
@@ -93,7 +99,7 @@
    "oldfieldtype": "Text Editor"
   },
   {
-   "collapsible": 1,
+   "depends_on": "eval:doc.footer_source==='HTML' && doc.letter_head_name",
    "fieldname": "footer_section",
    "fieldtype": "Section Break",
    "label": "Footer"
@@ -121,13 +127,48 @@
    "fieldname": "image_width",
    "fieldtype": "Float",
    "label": "Image Width"
+  },
+  {
+   "depends_on": "eval:doc.footer_source==='Image' && doc.letter_head_name",
+   "fieldname": "footer_image_section",
+   "fieldtype": "Section Break",
+   "label": "Footer Image"
+  },
+  {
+   "fieldname": "footer_image",
+   "fieldtype": "Attach Image",
+   "label": "Image"
+  },
+  {
+   "fieldname": "footer_image_height",
+   "fieldtype": "Float",
+   "label": "Image Height"
+  },
+  {
+   "fieldname": "footer_image_width",
+   "fieldtype": "Float",
+   "label": "Image Width"
+  },
+  {
+   "fieldname": "footer_align",
+   "fieldtype": "Select",
+   "label": "Align",
+   "options": "Left\nRight\nCenter"
+  },
+  {
+   "default": "HTML",
+   "depends_on": "letter_head_name",
+   "fieldname": "footer_source",
+   "fieldtype": "Select",
+   "label": "Footer Based On",
+   "options": "Image\nHTML"
   }
  ],
  "icon": "fa fa-font",
  "idx": 1,
  "links": [],
  "max_attachments": 3,
- "modified": "2021-10-03 14:37:58.314696",
+ "modified": "2022-06-16 23:10:46.852116",
  "modified_by": "Administrator",
  "module": "Printing",
  "name": "Letter Head",
@@ -152,5 +193,6 @@
  ],
  "sort_field": "modified",
  "sort_order": "ASC",
+ "states": [],
  "track_changes": 1
 }
\ No newline at end of file
diff --git a/frappe/printing/doctype/letter_head/letter_head.py b/frappe/printing/doctype/letter_head/letter_head.py
index 98c2fc7c2b..c48fd1fe25 100644
--- a/frappe/printing/doctype/letter_head/letter_head.py
+++ b/frappe/printing/doctype/letter_head/letter_head.py
@@ -26,21 +26,56 @@ class LetterHead(Document):
 
 	def set_image(self):
 		if self.source == "Image":
-			if self.image and is_image(self.image):
-				self.image_width = flt(self.image_width)
-				self.image_height = flt(self.image_height)
-				dimension = "width" if self.image_width > self.image_height else "height"
-				dimension_value = self.get("image_" + dimension)
-				self.content = f"""
-				
        
-					{self.name}
-				
        
-				"""
-				frappe.msgprint(frappe._("Header HTML set from attachment {0}").format(self.image), alert=True)
-			else:
-				frappe.msgprint(
-					frappe._("Please attach an image file to set HTML"), alert=True, indicator="orange"
-				)
+			self.set_image_as_html(
+				field="image",
+				width="image_width",
+				height="image_height",
+				align="align",
+				html_field="content",
+				dimension_prefix="image_",
+				success_msg=_("Header HTML set from attachment {0}").format(self.image),
+				failure_msg=_("Please attach an image file to set HTML for Letter Head."),
+			)
+
+		if self.footer_source == "Image":
+			self.set_image_as_html(
+				field="footer_image",
+				width="footer_image_width",
+				height="footer_image_height",
+				align="footer_align",
+				html_field="footer",
+				dimension_prefix="footer_image_",
+				success_msg=_("Footer HTML set from attachment {0}").format(self.footer_image),
+				failure_msg=_("Please attach an image file to set HTML for Footer."),
+			)
+
+	def set_image_as_html(
+		self, field, width, height, dimension_prefix, align, html_field, success_msg, failure_msg
+	):
+		if not self.get(field) or not is_image(self.get(field)):
+			frappe.msgprint(failure_msg, alert=True, indicator="orange")
+			return
+
+		self.set(width, flt(self.get(width)))
+		self.set(height, flt(self.get(height)))
+
+		# To preserve the aspect ratio of the image, apply constraints only on
+		# the greater dimension and allow the other to scale accordingly
+		dimension = "width" if self.get(width) > self.get(height) else "height"
+		dimension_value = self.get(f"{dimension_prefix}{dimension}")
+
+		if not dimension_value:
+			dimension_value = ""
+
+		self.set(
+			html_field,
+			f"""
        
+{self.get(
+
        """,
+		)
+
+		frappe.msgprint(success_msg, alert=True)
 
 	def on_update(self):
 		self.set_as_default()
diff --git a/frappe/printing/doctype/letter_head/test_letter_head.py b/frappe/printing/doctype/letter_head/test_letter_head.py
index 9357d15315..75019ce275 100644
--- a/frappe/printing/doctype/letter_head/test_letter_head.py
+++ b/frappe/printing/doctype/letter_head/test_letter_head.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2017, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
 import unittest
diff --git a/frappe/printing/doctype/network_printer_settings/network_printer_settings.js b/frappe/printing/doctype/network_printer_settings/network_printer_settings.js
index 043afd388f..2d094d8038 100644
--- a/frappe/printing/doctype/network_printer_settings/network_printer_settings.js
+++ b/frappe/printing/doctype/network_printer_settings/network_printer_settings.js
@@ -1,29 +1,29 @@
 // Copyright (c) 2021, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Network Printer Settings', {
-	onload (frm) {
+frappe.ui.form.on("Network Printer Settings", {
+	onload(frm) {
 		frm.trigger("connect_print_server");
 	},
-	server_ip (frm) {
+	server_ip(frm) {
 		frm.trigger("connect_print_server");
 	},
-	port (frm) {
+	port(frm) {
 		frm.trigger("connect_print_server");
 	},
-	connect_print_server (frm) {
+	connect_print_server(frm) {
 		if (frm.doc.server_ip && frm.doc.port) {
 			frappe.call({
-				"doc": frm.doc,
-				"method": "get_printers_list",
-				"args": {
+				doc: frm.doc,
+				method: "get_printers_list",
+				args: {
 					ip: frm.doc.server_ip,
-					port: frm.doc.port
+					port: frm.doc.port,
+				},
+				callback: function (data) {
+					frm.set_df_property("printer_name", "options", [""].concat(data.message));
 				},
-				callback: function(data) {
-					frm.set_df_property('printer_name', 'options', [""].concat(data.message));
-				}
 			});
 		}
-	}
+	},
 });
diff --git a/frappe/printing/doctype/print_format/print_format.js b/frappe/printing/doctype/print_format/print_format.js
index 3fd1d9d148..94f0ae5b1c 100644
--- a/frappe/printing/doctype/print_format/print_format.js
+++ b/frappe/printing/doctype/print_format/print_format.js
@@ -17,9 +17,9 @@ frappe.ui.form.on("Print Format", {
 		if (frm.doc.standard === "Yes" && frappe.session.user !== "Administrator") {
 			frm.set_intro(__("Please duplicate this to make changes"));
 		}
-		frm.trigger('render_buttons');
-		frm.toggle_display('standard', frappe.boot.developer_mode);
-		frm.trigger('hide_absolute_value_field');
+		frm.trigger("render_buttons");
+		frm.toggle_display("standard", frappe.boot.developer_mode);
+		frm.trigger("hide_absolute_value_field");
 	},
 	render_buttons: function (frm) {
 		frm.page.clear_inner_toolbar();
@@ -36,22 +36,21 @@ frappe.ui.form.on("Print Format", {
 						frappe.set_route("print-format-builder", frm.doc.name);
 					}
 				});
-			}
-			else if (frm.doc.custom_format && !frm.doc.raw_printing) {
+			} else if (frm.doc.custom_format && !frm.doc.raw_printing) {
 				frm.set_df_property("html", "reqd", 1);
 			}
 			if (frappe.model.can_read(frm.doc.doc_type)) {
-				frappe.db.get_value('DocType', frm.doc.doc_type, 'default_print_format', (r) => {
+				frappe.db.get_value("DocType", frm.doc.doc_type, "default_print_format", (r) => {
 					if (r.default_print_format != frm.doc.name) {
 						frm.add_custom_button(__("Set as Default"), function () {
 							frappe.call({
 								method: "frappe.printing.doctype.print_format.print_format.make_default",
 								args: {
-									name: frm.doc.name
+									name: frm.doc.name,
 								},
-								callback: function() {
+								callback: function () {
 									frm.refresh();
-								}
+								},
 							});
 						});
 					}
@@ -61,13 +60,13 @@ frappe.ui.form.on("Print Format", {
 	},
 	custom_format: function (frm) {
 		var value = frm.doc.custom_format ? 0 : 1;
-		frm.set_value('align_labels_right', value);
-		frm.set_value('show_section_headings', value);
-		frm.set_value('line_breaks', value);
-		frm.trigger('render_buttons');
+		frm.set_value("align_labels_right", value);
+		frm.set_value("show_section_headings", value);
+		frm.set_value("line_breaks", value);
+		frm.trigger("render_buttons");
 	},
 	doc_type: function (frm) {
-		frm.trigger('hide_absolute_value_field');
+		frm.trigger("hide_absolute_value_field");
 	},
 	hide_absolute_value_field: function (frm) {
 		// TODO: make it work with frm.doc.doc_type
@@ -76,9 +75,11 @@ frappe.ui.form.on("Print Format", {
 		if (doctype) {
 			frappe.model.with_doctype(doctype, () => {
 				const meta = frappe.get_meta(doctype);
-				const has_int_float_currency_field = meta.fields.filter(df => in_list(['Int', 'Float', 'Currency'], df.fieldtype));
-				frm.toggle_display('absolute_value', has_int_float_currency_field.length);
+				const has_int_float_currency_field = meta.fields.filter((df) =>
+					in_list(["Int", "Float", "Currency"], df.fieldtype)
+				);
+				frm.toggle_display("absolute_value", has_int_float_currency_field.length);
 			});
 		}
-	}
+	},
 });
diff --git a/frappe/printing/doctype/print_format/print_format.py b/frappe/printing/doctype/print_format/print_format.py
index e1e043beae..5bf4b83ec0 100644
--- a/frappe/printing/doctype/print_format/print_format.py
+++ b/frappe/printing/doctype/print_format/print_format.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2017, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
@@ -56,7 +55,7 @@ class PrintFormat(Document):
 			frappe.throw(_("{0} is required").format(frappe.bold(_("HTML"))), frappe.MandatoryError)
 
 	def extract_images(self):
-		from frappe.core.doctype.file.file import extract_images_from_html
+		from frappe.core.doctype.file.utils import extract_images_from_html
 
 		if self.print_format_builder_beta:
 			return
diff --git a/frappe/printing/doctype/print_format/test_print_format.py b/frappe/printing/doctype/print_format/test_print_format.py
index e97b07ed6e..0b53e1dd13 100644
--- a/frappe/printing/doctype/print_format/test_print_format.py
+++ b/frappe/printing/doctype/print_format/test_print_format.py
@@ -46,7 +46,7 @@ class TestPrintFormat(unittest.TestCase):
 
 		self.assertTrue(os.path.exists(exported_doc_path))
 
-		with open(exported_doc_path, "r") as f:
+		with open(exported_doc_path) as f:
 			exported_doc = frappe.parse_json(f.read())
 
 		for key, value in exported_doc.items():
diff --git a/frappe/printing/doctype/print_format_field_template/print_format_field_template.js b/frappe/printing/doctype/print_format_field_template/print_format_field_template.js
index 7fbb0d7359..4aa00ae5e7 100644
--- a/frappe/printing/doctype/print_format_field_template/print_format_field_template.js
+++ b/frappe/printing/doctype/print_format_field_template/print_format_field_template.js
@@ -1,8 +1,7 @@
 // Copyright (c) 2021, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Print Format Field Template', {
+frappe.ui.form.on("Print Format Field Template", {
 	// refresh: function(frm) {
-
 	// }
 });
diff --git a/frappe/printing/doctype/print_heading/print_heading.js b/frappe/printing/doctype/print_heading/print_heading.js
index 39f26a2e0f..3a2c615363 100644
--- a/frappe/printing/doctype/print_heading/print_heading.js
+++ b/frappe/printing/doctype/print_heading/print_heading.js
@@ -1,8 +1,6 @@
 // Copyright (c) 2017, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Print Heading', {
-	refresh: function(frm) {
-
-	}
+frappe.ui.form.on("Print Heading", {
+	refresh: function (frm) {},
 });
diff --git a/frappe/printing/doctype/print_heading/print_heading.py b/frappe/printing/doctype/print_heading/print_heading.py
index c905e68d47..9daee06f03 100644
--- a/frappe/printing/doctype/print_heading/print_heading.py
+++ b/frappe/printing/doctype/print_heading/print_heading.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2017, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
diff --git a/frappe/printing/doctype/print_heading/test_print_heading.py b/frappe/printing/doctype/print_heading/test_print_heading.py
index 02eddb072f..74ff7ce74f 100644
--- a/frappe/printing/doctype/print_heading/test_print_heading.py
+++ b/frappe/printing/doctype/print_heading/test_print_heading.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2017, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
 import unittest
diff --git a/frappe/printing/doctype/print_settings/print_settings.js b/frappe/printing/doctype/print_settings/print_settings.js
index b1311166ee..dc939c298d 100644
--- a/frappe/printing/doctype/print_settings/print_settings.js
+++ b/frappe/printing/doctype/print_settings/print_settings.js
@@ -1,19 +1,23 @@
 // Copyright (c) 2018, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Print Settings', {
-	print_style: function(frm) {
-		frappe.db.get_value('Print Style', frm.doc.print_style, 'preview').then((r) => {
-			if(r.message.preview) {
+frappe.ui.form.on("Print Settings", {
+	print_style: function (frm) {
+		frappe.db.get_value("Print Style", frm.doc.print_style, "preview").then((r) => {
+			if (r.message.preview) {
 				frm.get_field("print_style_preview").$wrapper.html(
-					``);
+					``
+				);
 			} else {
 				frm.get_field("print_style_preview").$wrapper.html(
-					`
        ${__("No Preview")}
        `);
+					`
        ${__(
+						"No Preview"
+					)}
        `
+				);
 			}
 		});
 	},
-	onload: function(frm) {
+	onload: function (frm) {
 		frm.script_manager.trigger("print_style");
-	}
+	},
 });
diff --git a/frappe/printing/doctype/print_settings/print_settings.py b/frappe/printing/doctype/print_settings/print_settings.py
index f52d08e6ec..36eaac2e68 100644
--- a/frappe/printing/doctype/print_settings/print_settings.py
+++ b/frappe/printing/doctype/print_settings/print_settings.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2018, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
diff --git a/frappe/printing/doctype/print_settings/test_print_settings.py b/frappe/printing/doctype/print_settings/test_print_settings.py
index ba22df4438..6a6437bf97 100644
--- a/frappe/printing/doctype/print_settings/test_print_settings.py
+++ b/frappe/printing/doctype/print_settings/test_print_settings.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2018, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
 import unittest
diff --git a/frappe/printing/doctype/print_style/print_style.js b/frappe/printing/doctype/print_style/print_style.js
index 44c4a528f4..3177e1aa09 100644
--- a/frappe/printing/doctype/print_style/print_style.js
+++ b/frappe/printing/doctype/print_style/print_style.js
@@ -1,10 +1,10 @@
 // Copyright (c) 2017, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Print Style', {
-	refresh: function(frm) {
-		frm.add_custom_button(__('Print Settings'), () => {
-			frappe.set_route('Form', 'Print Settings');
-		})
-	}
+frappe.ui.form.on("Print Style", {
+	refresh: function (frm) {
+		frm.add_custom_button(__("Print Settings"), () => {
+			frappe.set_route("Form", "Print Settings");
+		});
+	},
 });
diff --git a/frappe/printing/doctype/print_style/print_style.json b/frappe/printing/doctype/print_style/print_style.json
index 29e88a460a..1d3c9a6189 100644
--- a/frappe/printing/doctype/print_style/print_style.json
+++ b/frappe/printing/doctype/print_style/print_style.json
@@ -1,214 +1,75 @@
 {
- "allow_copy": 0, 
- "allow_guest_to_view": 0, 
- "allow_import": 0, 
- "allow_rename": 1, 
- "autoname": "field:print_style_name", 
- "beta": 0, 
- "creation": "2017-08-17 01:25:56.910716", 
- "custom": 0, 
- "docstatus": 0, 
- "doctype": "DocType", 
- "document_type": "", 
- "editable_grid": 1, 
- "engine": "InnoDB", 
+ "actions": [],
+ "allow_rename": 1,
+ "autoname": "field:print_style_name",
+ "creation": "2017-08-17 01:25:56.910716",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "print_style_name",
+  "disabled",
+  "standard",
+  "css",
+  "preview"
+ ],
  "fields": [
   {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "print_style_name", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Print Style Name", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "fieldname": "print_style_name",
+   "fieldtype": "Data",
+   "label": "Print Style Name",
+   "reqd": 1,
+   "unique": 1
+  },
   {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "disabled", 
-   "fieldtype": "Check", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Disabled", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "default": "0",
+   "fieldname": "disabled",
+   "fieldtype": "Check",
+   "in_list_view": 1,
+   "label": "Disabled"
+  },
   {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "standard", 
-   "fieldtype": "Check", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Standard", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "default": "0",
+   "fieldname": "standard",
+   "fieldtype": "Check",
+   "label": "Standard"
+  },
   {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "css", 
-   "fieldtype": "Code", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "CSS", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "fieldname": "css",
+   "fieldtype": "Code",
+   "label": "CSS",
+   "reqd": 1
+  },
   {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "preview", 
-   "fieldtype": "Attach Image", 
-   "hidden": 1, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Preview", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
+   "fieldname": "preview",
+   "fieldtype": "Attach Image",
+   "hidden": 1,
+   "label": "Preview"
   }
- ], 
- "has_web_view": 0, 
- "hide_heading": 0, 
- "hide_toolbar": 0, 
- "idx": 0, 
- "image_field": "preview", 
- "image_view": 0, 
- "in_create": 0, 
- "is_submittable": 0, 
- "issingle": 0, 
- "istable": 0, 
- "max_attachments": 0, 
- "modified": "2017-08-17 02:18:08.132853", 
- "modified_by": "Administrator", 
- "module": "Printing", 
- "name": "Print Style", 
- "name_case": "", 
- "owner": "Administrator", 
+ ],
+ "image_field": "preview",
+ "links": [],
+ "modified": "2022-08-03 12:20:51.295775",
+ "modified_by": "Administrator",
+ "module": "Printing",
+ "name": "Print Style",
+ "owner": "Administrator",
  "permissions": [
   {
-   "amend": 0, 
-   "apply_user_permissions": 0, 
-   "cancel": 0, 
-   "create": 1, 
-   "delete": 1, 
-   "email": 1, 
-   "export": 1, 
-   "if_owner": 0, 
-   "import": 0, 
-   "permlevel": 0, 
-   "print": 1, 
-   "read": 1, 
-   "report": 1, 
-   "role": "System Manager", 
-   "set_user_permissions": 0, 
-   "share": 1, 
-   "submit": 0, 
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "export": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "System Manager",
+   "share": 1,
    "write": 1
   }
- ], 
- "quick_entry": 0, 
- "read_only": 0, 
- "read_only_onload": 0, 
- "show_name_in_global_search": 0, 
- "sort_field": "modified", 
- "sort_order": "DESC", 
- "track_changes": 1, 
- "track_seen": 0
+ ],
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "states": [],
+ "track_changes": 1
 }
\ No newline at end of file
diff --git a/frappe/printing/doctype/print_style/print_style.py b/frappe/printing/doctype/print_style/print_style.py
index 00de829deb..2b0fbfe929 100644
--- a/frappe/printing/doctype/print_style/print_style.py
+++ b/frappe/printing/doctype/print_style/print_style.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2017, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
diff --git a/frappe/printing/doctype/print_style/test_print_style.py b/frappe/printing/doctype/print_style/test_print_style.py
index ad2b61cc87..f8ce54b9bb 100644
--- a/frappe/printing/doctype/print_style/test_print_style.py
+++ b/frappe/printing/doctype/print_style/test_print_style.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2017, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
 import unittest
diff --git a/frappe/printing/page/print/print.js b/frappe/printing/page/print/print.js
index 122aea9fa1..90e7328a30 100644
--- a/frappe/printing/page/print/print.js
+++ b/frappe/printing/page/print/print.js
@@ -1,14 +1,14 @@
-frappe.pages['print'].on_page_load = function(wrapper) {
+frappe.pages["print"].on_page_load = function (wrapper) {
 	frappe.ui.make_app_page({
 		parent: wrapper,
 	});
 
 	let print_view = new frappe.ui.form.PrintView(wrapper);
 
-	$(wrapper).bind('show', () => {
+	$(wrapper).bind("show", () => {
 		const route = frappe.get_route();
 		const doctype = route[1];
-		const docname = route[2];
+		const docname = route.slice(2).join("/");
 		if (!frappe.route_options || !frappe.route_options.frm) {
 			frappe.model.with_doc(doctype, docname, () => {
 				let frm = { doctype: doctype, docname: docname };
@@ -19,7 +19,9 @@ frappe.pages['print'].on_page_load = function(wrapper) {
 				});
 			});
 		} else {
-			print_view.frm = frappe.route_options.frm;
+			print_view.frm = frappe.route_options.frm.doctype
+				? frappe.route_options.frm
+				: frappe.route_options.frm.frm;
 			frappe.route_options.frm = null;
 			print_view.show(print_view.frm);
 		}
@@ -36,7 +38,7 @@ frappe.ui.form.PrintView = class {
 	make() {
 		this.print_wrapper = this.page.main.empty().html(
 			`
        
-				${frappe.render_template('print_skeleton_loading')}
+				${frappe.render_template("print_skeleton_loading")}
 				
 			
        
@@ -48,10 +50,7 @@ frappe.ui.form.PrintView = class {
 		`
 		);
 
-		this.print_settings = frappe.model.get_doc(
-			':Print Settings',
-			'Print Settings'
-		);
+		this.print_settings = frappe.model.get_doc(":Print Settings", "Print Settings");
 		this.setup_menu();
 		this.setup_toolbar();
 		this.setup_sidebar();
@@ -63,83 +62,75 @@ frappe.ui.form.PrintView = class {
 	}
 
 	setup_toolbar() {
-		this.page.set_primary_action(
-			__('Print'),
-			() => this.printit(), 'printer'
-		);
+		this.page.set_primary_action(__("Print"), () => this.printit(), "printer");
 
-		this.page.add_button(
-			__('Full Page'),
-			() => this.render_page('/printview?'),
-			{ icon: 'full-page' }
-		);
+		this.page.add_button(__("Full Page"), () => this.render_page("/printview?"), {
+			icon: "full-page",
+		});
 
-		this.page.add_button(
-			__('PDF'),
-			() => this.render_pdf(),
-			{ icon: 'small-file' }
-		);
+		this.page.add_button(__("PDF"), () => this.render_pdf(), { icon: "small-file" });
 
-		this.page.add_button(
-			__('Refresh'),
-			() => this.refresh_print_format(),
-			{ icon: 'refresh' }
+		this.page.add_button(__("Refresh"), () => this.refresh_print_format(), {
+			icon: "refresh",
+		});
+
+		this.page.add_action_icon(
+			"file",
+			() => {
+				this.go_to_form_view();
+			},
+			"",
+			__("Form")
 		);
 	}
 
 	setup_sidebar() {
-		this.sidebar = this.page.sidebar.addClass('print-preview-sidebar');
+		this.sidebar = this.page.sidebar.addClass("print-preview-sidebar");
 
-		this.print_sel = this.add_sidebar_item(
-			{
-				fieldtype: 'Select',
-				fieldname: 'print_format',
-				label: 'Print Format',
-				options: [this.get_default_option_for_select(__('Select Print Format'))],
-				change: () => this.refresh_print_format(),
-				default: __('Select Print Format')
-			},
-		).$input;
+		this.print_sel = this.add_sidebar_item({
+			fieldtype: "Select",
+			fieldname: "print_format",
+			label: "Print Format",
+			options: [this.get_default_option_for_select(__("Select Print Format"))],
+			change: () => this.refresh_print_format(),
+			default: __("Select Print Format"),
+		}).$input;
 
-		this.language_sel = this.add_sidebar_item(
-			{
-				fieldtype: 'Select',
-				fieldname: 'language',
-				placeholder: 'Language',
-				options: [
-					this.get_default_option_for_select(__('Select Language')),
-					...this.get_language_options()
-				],
-				default: __('Select Language'),
-				change: () => {
-					this.set_user_lang();
-					this.preview();
-				},
+		this.language_sel = this.add_sidebar_item({
+			fieldtype: "Select",
+			fieldname: "language",
+			placeholder: "Language",
+			options: [
+				this.get_default_option_for_select(__("Select Language")),
+				...this.get_language_options(),
+			],
+			default: __("Select Language"),
+			change: () => {
+				this.set_user_lang();
+				this.preview();
 			},
-		).$input;
+		}).$input;
 
-		this.letterhead_selector_df = this.add_sidebar_item(
-			{
-				fieldtype: 'Autocomplete',
-				fieldname: 'letterhead',
-				label: __('Select Letterhead'),
-				placeholder: __('Select Letterhead'),
-				options: [__('No Letterhead')],
-				change: () => this.preview(),
-				default: this.print_settings.with_letterhead
-					? __('No Letterhead')
-					: __('Select Letterhead')
-			},
-		);
+		this.letterhead_selector_df = this.add_sidebar_item({
+			fieldtype: "Autocomplete",
+			fieldname: "letterhead",
+			label: __("Select Letterhead"),
+			placeholder: __("Select Letterhead"),
+			options: [__("No Letterhead")],
+			change: () => this.preview(),
+			default: this.print_settings.with_letterhead
+				? __("No Letterhead")
+				: __("Select Letterhead"),
+		});
 		this.letterhead_selector = this.letterhead_selector_df.$input;
-		this.sidebar_dynamic_section = $(
-			`
        `
-		).appendTo(this.sidebar);
+		this.sidebar_dynamic_section = $(`
        `).appendTo(
+			this.sidebar
+		);
 	}
 
 	add_sidebar_item(df, is_dynamic) {
-		if (df.fieldtype == 'Select') {
-			df.input_class = 'btn btn-default btn-sm text-left';
+		if (df.fieldtype == "Select") {
+			df.input_class = "btn btn-default btn-sm text-left";
 		}
 
 		let field = frappe.ui.form.make_control({
@@ -159,31 +150,29 @@ frappe.ui.form.PrintView = class {
 		return {
 			label: value,
 			value: value,
-			disabled: true
+			disabled: true,
 		};
 	}
 
 	setup_menu() {
 		this.page.clear_menu();
 
-		this.page.add_menu_item(__('Print Settings'), () => {
-			frappe.set_route('Form', 'Print Settings');
+		this.page.add_menu_item(__("Print Settings"), () => {
+			frappe.set_route("Form", "Print Settings");
 		});
 
-		if (this.print_settings.enable_raw_printing == '1') {
-			this.page.add_menu_item(__('Raw Printing Setting'), () => {
+		if (this.print_settings.enable_raw_printing == "1") {
+			this.page.add_menu_item(__("Raw Printing Setting"), () => {
 				this.printer_setting_dialog();
 			});
 		}
 
-		if (frappe.model.can_create('Print Format')) {
-			this.page.add_menu_item(__('Customize'), () =>
-				this.edit_print_format()
-			);
+		if (frappe.model.can_create("Print Format")) {
+			this.page.add_menu_item(__("Customize"), () => this.edit_print_format());
 		}
 
 		if (cint(this.print_settings.enable_print_server)) {
-			this.page.add_menu_item(__('Select Network Printer'), () =>
+			this.page.add_menu_item(__("Select Network Printer"), () =>
 				this.network_printer_setting_dialog()
 			);
 		}
@@ -198,7 +187,7 @@ frappe.ui.form.PrintView = class {
 		// print format builder beta
 		this.page.add_inner_message(`
 			
-				${__('Try the new Print Format Builder')}
+				${__("Try the new Print Format Builder")}
 			
 		`);
 
@@ -221,23 +210,26 @@ frappe.ui.form.PrintView = class {
 		this.additional_settings = {};
 		this.sidebar_dynamic_section.empty();
 		frappe
-			.xcall('frappe.printing.page.print.print.get_print_settings_to_show', {
+			.xcall("frappe.printing.page.print.print.get_print_settings_to_show", {
 				doctype: this.frm.doc.doctype,
-				docname: this.frm.doc.name
+				docname: this.frm.doc.name,
 			})
 			.then((settings) => this.add_settings_to_sidebar(settings));
 	}
 
 	add_settings_to_sidebar(settings) {
 		for (let df of settings) {
-			let field = this.add_sidebar_item({
-				...df,
-				change: () => {
-					const val = field.get_value();
-					this.additional_settings[field.df.fieldname] = val;
-					this.preview();
+			let field = this.add_sidebar_item(
+				{
+					...df,
+					change: () => {
+						const val = field.get_value();
+						this.additional_settings[field.df.fieldname] = val;
+						this.preview();
+					},
 				},
-			}, true);
+				true
+			);
 		}
 	}
 
@@ -246,19 +238,18 @@ frappe.ui.form.PrintView = class {
 		let is_custom_format =
 			print_format.name &&
 			(print_format.print_format_builder || print_format.print_format_builder_beta) &&
-			print_format.standard === 'No';
-		let is_standard_but_editable =
-			print_format.name && print_format.custom_format;
+			print_format.standard === "No";
+		let is_standard_but_editable = print_format.name && print_format.custom_format;
 
 		if (is_standard_but_editable) {
-			frappe.set_route('Form', 'Print Format', print_format.name);
+			frappe.set_route("Form", "Print Format", print_format.name);
 			return;
 		}
 		if (is_custom_format) {
 			if (print_format.print_format_builder_beta) {
-				frappe.set_route('print-format-builder-beta', print_format.name);
+				frappe.set_route("print-format-builder-beta", print_format.name);
 			} else {
-				frappe.set_route('print-format-builder', print_format.name);
+				frappe.set_route("print-format-builder", print_format.name);
 			}
 			return;
 		}
@@ -266,21 +257,21 @@ frappe.ui.form.PrintView = class {
 		frappe.prompt(
 			[
 				{
-					label: __('New Print Format Name'),
-					fieldname: 'print_format_name',
-					fieldtype: 'Data',
+					label: __("New Print Format Name"),
+					fieldname: "print_format_name",
+					fieldtype: "Data",
 					reqd: 1,
 				},
 				{
-					label: __('Based On'),
-					fieldname: 'based_on',
-					fieldtype: 'Read Only',
-					default: print_format.name || 'Standard',
+					label: __("Based On"),
+					fieldname: "based_on",
+					fieldtype: "Read Only",
+					default: print_format.name || "Standard",
 				},
 				{
-					label: __('Use the new Print Format Builder'),
-					fieldname: 'beta',
-					fieldtype: 'Check'
+					label: __("Use the new Print Format Builder"),
+					fieldname: "beta",
+					fieldtype: "Check",
 				},
 			],
 			(data) => {
@@ -289,13 +280,13 @@ frappe.ui.form.PrintView = class {
 					doctype: this.frm.doctype,
 					name: data.print_format_name,
 					based_on: data.based_on,
-					beta: data.beta
+					beta: data.beta,
 				};
-				frappe.set_route('print-format-builder');
+				frappe.set_route("print-format-builder");
 				this.print_sel.val(data.print_format_name);
 			},
-			__('New Custom Print Format'),
-			__('Start')
+			__("New Custom Print Format"),
+			__("Start")
 		);
 	}
 
@@ -316,7 +307,7 @@ frappe.ui.form.PrintView = class {
 
 	setup_customize_dialog() {
 		let print_format = this.get_print_format();
-		$(document).on('new-print-format', (e) => {
+		$(document).on("new-print-format", (e) => {
 			this.refresh_print_options();
 			if (e.print_format) {
 				this.print_sel.val(e.print_format);
@@ -325,16 +316,16 @@ frappe.ui.form.PrintView = class {
 			frappe.prompt(
 				[
 					{
-						label: __('New Print Format Name'),
-						fieldname: 'print_format_name',
-						fieldtype: 'Data',
+						label: __("New Print Format Name"),
+						fieldname: "print_format_name",
+						fieldtype: "Data",
 						reqd: 1,
 					},
 					{
-						label: __('Based On'),
-						fieldname: 'based_on',
-						fieldtype: 'Read Only',
-						default: print_format.name || 'Standard',
+						label: __("Based On"),
+						fieldname: "based_on",
+						fieldtype: "Read Only",
+						default: print_format.name || "Standard",
 					},
 				],
 				(data) => {
@@ -344,30 +335,30 @@ frappe.ui.form.PrintView = class {
 						name: data.print_format_name,
 						based_on: data.based_on,
 					};
-					frappe.set_route('print-format-builder');
+					frappe.set_route("print-format-builder");
 				},
-				__('New Custom Print Format'),
-				__('Start')
+				__("New Custom Print Format"),
+				__("Start")
 			);
 		});
 	}
 
 	setup_keyboard_shortcuts() {
-		this.wrapper.find('.print-toolbar a.btn-default').each((i, el) => {
+		this.wrapper.find(".print-toolbar a.btn-default").each((i, el) => {
 			frappe.ui.keys.get_shortcut_group(this.frm.page).add($(el));
 		});
 	}
 
 	set_letterhead_options() {
-		let letterhead_options = [__('No Letterhead')];
+		let letterhead_options = [__("No Letterhead")];
 		let default_letterhead;
 		let doc_letterhead = this.frm.doc.letter_head;
 
 		return frappe.db
-			.get_list('Letter Head', {
-				filters: {'disabled': 0},
-				fields: ['name', 'is_default'],
-				limit: 0
+			.get_list("Letter Head", {
+				filters: { disabled: 0 },
+				fields: ["name", "is_default"],
+				limit: 0,
 			})
 			.then((letterheads) => {
 				letterheads.map((letterhead) => {
@@ -377,8 +368,7 @@ frappe.ui.form.PrintView = class {
 
 				this.letterhead_selector_df.set_data(letterhead_options);
 				let selected_letterhead = doc_letterhead || default_letterhead;
-				if (selected_letterhead)
-					this.letterhead_selector.val(selected_letterhead);
+				if (selected_letterhead) this.letterhead_selector.val(selected_letterhead);
 			});
 	}
 
@@ -393,31 +383,29 @@ frappe.ui.form.PrintView = class {
 	set_default_print_language() {
 		let print_format = this.get_print_format();
 		this.lang_code =
-			print_format.default_print_language ||
-			this.frm.doc.language ||
-			frappe.boot.lang;
+			print_format.default_print_language || this.frm.doc.language || frappe.boot.lang;
 		this.language_sel.val(this.lang_code);
 	}
 
 	toggle_raw_printing() {
 		const is_raw_printing = this.is_raw_printing();
-		this.wrapper.find('.btn-print-preview').toggle(!is_raw_printing);
-		this.wrapper.find('.btn-download-pdf').toggle(!is_raw_printing);
+		this.wrapper.find(".btn-print-preview").toggle(!is_raw_printing);
+		this.wrapper.find(".btn-download-pdf").toggle(!is_raw_printing);
 	}
 
 	preview() {
 		let print_format = this.get_print_format();
 		if (print_format.print_format_builder_beta) {
-			this.print_wrapper.find('.print-preview-wrapper').hide();
-			this.print_wrapper.find('.preview-beta-wrapper').show();
+			this.print_wrapper.find(".print-preview-wrapper").hide();
+			this.print_wrapper.find(".preview-beta-wrapper").show();
 			this.preview_beta();
 			return;
 		}
 
-		this.print_wrapper.find('.preview-beta-wrapper').hide();
-		this.print_wrapper.find('.print-preview-wrapper').show();
+		this.print_wrapper.find(".preview-beta-wrapper").hide();
+		this.print_wrapper.find(".print-preview-wrapper").show();
 
-		const $print_format = this.print_wrapper.find('iframe');
+		const $print_format = this.print_wrapper.find("iframe");
 		this.$print_format_body = $print_format.contents();
 		this.get_print_html((out) => {
 			if (!out.html) {
@@ -427,83 +415,95 @@ frappe.ui.form.PrintView = class {
 			this.setup_print_format_dom(out, $print_format);
 
 			const print_height = $print_format.get(0).offsetHeight;
-			const $message = this.wrapper.find('.page-break-message');
+			const $message = this.wrapper.find(".page-break-message");
 
 			const print_height_inches = frappe.dom.pixel_to_inches(print_height);
 			// if contents are large enough, indicate that it will get printed on multiple pages
 			// Maximum height for an A4 document is 11.69 inches
 			if (print_height_inches > 11.69) {
-				$message.text(__('This may get printed on multiple pages'));
+				$message.text(__("This may get printed on multiple pages"));
 			} else {
-				$message.text('');
+				$message.text("");
 			}
 		});
 	}
 
 	preview_beta() {
 		let print_format = this.get_print_format();
-		const iframe = this.print_wrapper.find('.preview-beta-wrapper iframe');
+		const iframe = this.print_wrapper.find(".preview-beta-wrapper iframe");
 		let params = new URLSearchParams({
 			doctype: this.frm.doc.doctype,
 			name: this.frm.doc.name,
-			print_format: print_format.name
+			print_format: print_format.name,
 		});
 		let letterhead = this.get_letterhead();
 		if (letterhead) {
 			params.append("letterhead", letterhead);
 		}
-		iframe.prop('src', `/printpreview?${params.toString()}`);
+		iframe.prop("src", `/printpreview?${params.toString()}`);
 	}
 
 	setup_print_format_dom(out, $print_format) {
-		this.print_wrapper.find('.print-format-skeleton').remove();
+		this.print_wrapper.find(".print-format-skeleton").remove();
 		let base_url = frappe.urllib.get_base_url();
-		let print_css = frappe.assets.bundled_asset('print.bundle.css', frappe.utils.is_rtl(this.lang_code));
-		this.$print_format_body.find('html').attr('dir', frappe.utils.is_rtl(this.lang_code) ? 'rtl': 'ltr');
-		this.$print_format_body.find('html').attr('lang', this.lang_code);
-		this.$print_format_body.find('head').html(
+		let print_css = frappe.assets.bundled_asset(
+			"print.bundle.css",
+			frappe.utils.is_rtl(this.lang_code)
+		);
+		this.$print_format_body
+			.find("html")
+			.attr("dir", frappe.utils.is_rtl(this.lang_code) ? "rtl" : "ltr");
+		this.$print_format_body.find("html").attr("lang", this.lang_code);
+		this.$print_format_body.find("head").html(
 			`
 			`
 		);
 
-		this.$print_format_body.find('body').html(
-			`
        ${out.html}
        `
-		);
+		this.$print_format_body
+			.find("body")
+			.html(`
        ${out.html}
        `);
 
 		this.show_footer();
 
-		this.$print_format_body.find('.print-format').css({
-			display: 'flex',
-			flexDirection: 'column',
+		this.$print_format_body.find(".print-format").css({
+			display: "flex",
+			flexDirection: "column",
 		});
 
-		this.$print_format_body.find('.page-break').css({
-			display: 'flex',
-			'flex-direction': 'column',
-			flex: '1',
+		this.$print_format_body.find(".page-break").css({
+			display: "flex",
+			"flex-direction": "column",
+			flex: "1",
 		});
 
 		setTimeout(() => {
-			$print_format.height(this.$print_format_body.find('.print-format').outerHeight());
+			$print_format.height(this.$print_format_body.find(".print-format").outerHeight());
 		}, 500);
 	}
 
 	hide() {
-		if (this.frm.setup_done && this.frm.page.current_view_name === 'print') {
+		if (this.frm.setup_done && this.frm.page.current_view_name === "print") {
 			this.frm.page.set_view(
-				this.frm.page.previous_view_name === 'print'
-					? 'main'
-					: this.frm.page.previous_view_name || 'main'
+				this.frm.page.previous_view_name === "print"
+					? "main"
+					: this.frm.page.previous_view_name || "main"
 			);
 		}
 	}
 
+	go_to_form_view() {
+		frappe.route_options = {
+			frm: this,
+		};
+		frappe.set_route("Form", this.frm.doctype, this.frm.docname);
+	}
+
 	show_footer() {
 		// footer is hidden by default as reqd by pdf generation
 		// simple hack to show it in print preview
 
-		this.$print_format_body.find('#footer-html').attr(
-			'style',
+		this.$print_format_body.find("#footer-html").attr(
+			"style",
 			`
 			display: block !important;
 			order: 1;
@@ -517,7 +517,7 @@ frappe.ui.form.PrintView = class {
 		let me = this;
 
 		if (cint(me.print_settings.enable_print_server)) {
-			if (localStorage.getItem('network_printer')) {
+			if (localStorage.getItem("network_printer")) {
 				me.print_by_server();
 			} else {
 				me.network_printer_setting_dialog(() => me.print_by_server());
@@ -525,10 +525,10 @@ frappe.ui.form.PrintView = class {
 		} else if (me.get_mapped_printer().length === 1) {
 			// printer is already mapped in localstorage (applies for both raw and pdf )
 			if (me.is_raw_printing()) {
-				me.get_raw_commands(function(out) {
+				me.get_raw_commands(function (out) {
 					frappe.ui.form
 						.qz_connect()
-						.then(function() {
+						.then(function () {
 							let printer_map = me.get_mapped_printer()[0];
 							let data = [out.raw_commands];
 							let config = qz.configs.create(printer_map.printer);
@@ -544,9 +544,9 @@ frappe.ui.form.PrintView = class {
 					{
 						message: __('PDF printing via "Raw Print" is not supported.'),
 						subtitle: __(
-							'Please remove the printer mapping in Printer Settings and try again.'
+							"Please remove the printer mapping in Printer Settings and try again."
 						),
-						indicator: 'info',
+						indicator: "info",
 					},
 					14
 				);
@@ -556,61 +556,61 @@ frappe.ui.form.PrintView = class {
 			// printer not mapped in localstorage and the current print format is raw printing
 			frappe.show_alert(
 				{
-					message: __('Printer mapping not set.'),
+					message: __("Printer mapping not set."),
 					subtitle: __(
-						'Please set a printer mapping for this print format in the Printer Settings'
+						"Please set a printer mapping for this print format in the Printer Settings"
 					),
-					indicator: 'warning',
+					indicator: "warning",
 				},
 				14
 			);
 			me.printer_setting_dialog();
 		} else {
-			me.render_page('/printview?', true);
+			me.render_page("/printview?", true);
 		}
 	}
 
 	print_by_server() {
 		let me = this;
-		if (localStorage.getItem('network_printer')) {
+		if (localStorage.getItem("network_printer")) {
 			frappe.call({
-				method: 'frappe.utils.print_format.print_by_server',
+				method: "frappe.utils.print_format.print_by_server",
 				args: {
 					doctype: me.frm.doc.doctype,
 					name: me.frm.doc.name,
-					printer_setting: localStorage.getItem('network_printer'),
+					printer_setting: localStorage.getItem("network_printer"),
 					print_format: me.selected_format(),
 					no_letterhead: me.with_letterhead(),
 					letterhead: me.get_letterhead(),
 				},
-				callback: function() {},
+				callback: function () {},
 			});
 		}
 	}
 	network_printer_setting_dialog(callback) {
 		frappe.call({
-			method: 'frappe.printing.doctype.network_printer_settings.network_printer_settings.get_network_printer_settings',
-			callback: function(r) {
+			method: "frappe.printing.doctype.network_printer_settings.network_printer_settings.get_network_printer_settings",
+			callback: function (r) {
 				if (r.message) {
 					let d = new frappe.ui.Dialog({
-						title: __('Select Network Printer'),
+						title: __("Select Network Printer"),
 						fields: [
 							{
-								"label": "Printer",
-								"fieldname": "printer",
-								"fieldtype": "Select",
-								"reqd": 1,
-								"options": r.message
-							}
+								label: "Printer",
+								fieldname: "printer",
+								fieldtype: "Select",
+								reqd: 1,
+								options: r.message,
+							},
 						],
-						primary_action: function() {
-							localStorage.setItem('network_printer', d.get_values().printer);
+						primary_action: function () {
+							localStorage.setItem("network_printer", d.get_values().printer);
 							if (typeof callback == "function") {
 								callback();
 							}
 							d.hide();
 						},
-						primary_action_label: __('Select')
+						primary_action_label: __("Select"),
 					});
 					d.show();
 				}
@@ -625,15 +625,15 @@ frappe.ui.form.PrintView = class {
 				doctype: this.frm.doc.doctype,
 				name: this.frm.doc.name,
 				print_format: print_format.name,
-				letterhead: this.get_letterhead()
+				letterhead: this.get_letterhead(),
 			});
 			let w = window.open(`/api/method/frappe.utils.weasyprint.download_pdf?${params}`);
 			if (!w) {
-				frappe.msgprint(__('Please enable pop-ups'));
+				frappe.msgprint(__("Please enable pop-ups"));
 				return;
 			}
 		} else {
-			this.render_page('/api/method/frappe.utils.print_format.download_pdf?');
+			this.render_page("/api/method/frappe.utils.print_format.download_pdf?");
 		}
 	}
 
@@ -641,24 +641,24 @@ frappe.ui.form.PrintView = class {
 		let w = window.open(
 			frappe.urllib.get_full_url(
 				method +
-					'doctype=' +
+					"doctype=" +
 					encodeURIComponent(this.frm.doc.doctype) +
-					'&name=' +
+					"&name=" +
 					encodeURIComponent(this.frm.doc.name) +
-					(printit ? '&trigger_print=1' : '') +
-					'&format=' +
+					(printit ? "&trigger_print=1" : "") +
+					"&format=" +
 					encodeURIComponent(this.selected_format()) +
-					'&no_letterhead=' +
-					(this.with_letterhead() ? '0' : '1') +
-					'&letterhead=' +
+					"&no_letterhead=" +
+					(this.with_letterhead() ? "0" : "1") +
+					"&letterhead=" +
 					encodeURIComponent(this.get_letterhead()) +
-					'&settings=' +
+					"&settings=" +
 					encodeURIComponent(JSON.stringify(this.additional_settings)) +
-					(this.lang_code ? '&_lang=' + this.lang_code : '')
+					(this.lang_code ? "&_lang=" + this.lang_code : "")
 			)
 		);
 		if (!w) {
-			frappe.msgprint(__('Please enable pop-ups'));
+			frappe.msgprint(__("Please enable pop-ups"));
 			return;
 		}
 	}
@@ -675,7 +675,7 @@ frappe.ui.form.PrintView = class {
 			this._req.abort();
 		}
 		this._req = frappe.call({
-			method: 'frappe.www.printview.get_html_and_style',
+			method: "frappe.www.printview.get_html_and_style",
 			args: {
 				doc: this.frm.doc,
 				print_format: this.selected_format(),
@@ -684,7 +684,7 @@ frappe.ui.form.PrintView = class {
 				settings: this.additional_settings,
 				_lang: this.lang_code,
 			},
-			callback: function(r) {
+			callback: function (r) {
 				if (!r.exc) {
 					callback(r.message);
 				}
@@ -698,20 +698,20 @@ frappe.ui.form.PrintView = class {
 
 	get_no_preview_html() {
 		return `
        
-			${__('No Preview Available')}
+			${__("No Preview Available")}
 		
        `;
 	}
 
 	get_raw_commands(callback) {
 		// fetches rendered raw commands from the server for the current print format.
 		frappe.call({
-			method: 'frappe.www.printview.get_rendered_raw_commands',
+			method: "frappe.www.printview.get_rendered_raw_commands",
 			args: {
 				doc: this.frm.doc,
 				print_format: this.selected_format(),
 				_lang: this.lang_code,
 			},
-			callback: function(r) {
+			callback: function (r) {
 				if (!r.exc) {
 					callback(r.message);
 				}
@@ -734,9 +734,7 @@ frappe.ui.form.PrintView = class {
 	get_print_format_printer_map() {
 		// returns the whole object "print_format_printer_map" stored in the localStorage.
 		try {
-			let print_format_printer_map = JSON.parse(
-				localStorage.print_format_printer_map
-			);
+			let print_format_printer_map = JSON.parse(localStorage.print_format_printer_map);
 			return print_format_printer_map;
 		} catch (e) {
 			return {};
@@ -746,18 +744,20 @@ frappe.ui.form.PrintView = class {
 	refresh_print_options() {
 		this.print_formats = frappe.meta.get_print_formats(this.frm.doctype);
 		const print_format_select_val = this.print_sel.val();
-		this.print_sel.empty().add_options([
-			this.get_default_option_for_select(__('Select Print Format')),
-			...this.print_formats
-		]);
-		return this.print_formats.includes(print_format_select_val)
-			&& this.print_sel.val(print_format_select_val);
+		this.print_sel
+			.empty()
+			.add_options([
+				this.get_default_option_for_select(__("Select Print Format")),
+				...this.print_formats,
+			]);
+		return (
+			this.print_formats.includes(print_format_select_val) &&
+			this.print_sel.val(print_format_select_val)
+		);
 	}
 
 	selected_format() {
-		return (
-			this.print_sel.val() || this.frm.meta.default_print_format || 'Standard'
-		);
+		return this.print_sel.val() || this.frm.meta.default_print_format || "Standard";
 	}
 
 	is_raw_printing(format) {
@@ -770,19 +770,19 @@ frappe.ui.form.PrintView = class {
 			format = this.selected_format();
 		}
 
-		if (locals['Print Format'] && locals['Print Format'][format]) {
-			print_format = locals['Print Format'][format];
+		if (locals["Print Format"] && locals["Print Format"][format]) {
+			print_format = locals["Print Format"][format];
 		}
 
 		return print_format;
 	}
 
 	with_letterhead() {
-		return cint(this.get_letterhead() !== __('No Letterhead'));
+		return cint(this.get_letterhead() !== __("No Letterhead"));
 	}
 
 	set_style(style) {
-		frappe.dom.set_style(style || frappe.boot.print_css, 'print-style');
+		frappe.dom.set_style(style || frappe.boot.print_css, "print-style");
 	}
 
 	printer_setting_dialog() {
@@ -793,15 +793,15 @@ frappe.ui.form.PrintView = class {
 		frappe.ui.form.qz_get_printer_list().then((data) => {
 			this.printer_list = data;
 			const dialog = new frappe.ui.Dialog({
-				title: __('Printer Settings'),
+				title: __("Printer Settings"),
 				fields: [
 					{
-						fieldtype: 'Section Break',
+						fieldtype: "Section Break",
 					},
 					{
-						fieldname: 'printer_mapping',
-						fieldtype: 'Table',
-						label: __('Printer Mapping'),
+						fieldname: "printer_mapping",
+						fieldtype: "Table",
+						label: __("Printer Mapping"),
 						in_place_edit: true,
 						data: this.data,
 						get_data: () => {
@@ -809,28 +809,28 @@ frappe.ui.form.PrintView = class {
 						},
 						fields: [
 							{
-								fieldtype: 'Select',
-								fieldname: 'print_format',
+								fieldtype: "Select",
+								fieldname: "print_format",
 								default: 0,
 								options: this.print_formats,
 								read_only: 0,
 								in_list_view: 1,
-								label: __('Print Format'),
+								label: __("Print Format"),
 							},
 							{
-								fieldtype: 'Select',
-								fieldname: 'printer',
+								fieldtype: "Select",
+								fieldname: "printer",
 								default: 0,
 								options: this.printer_list,
 								read_only: 0,
 								in_list_view: 1,
-								label: __('Printer'),
+								label: __("Printer"),
 							},
 						],
 					},
 				],
 				primary_action: () => {
-					let printer_mapping = dialog.get_values()['printer_mapping'];
+					let printer_mapping = dialog.get_values()["printer_mapping"];
 					if (printer_mapping && printer_mapping.length) {
 						let print_format_list = printer_mapping.map((a) => a.print_format);
 						let has_duplicate = print_format_list.some(
@@ -839,7 +839,7 @@ frappe.ui.form.PrintView = class {
 						if (has_duplicate)
 							frappe.throw(
 								__(
-									'Cannot have multiple printers mapped to a single print format.'
+									"Cannot have multiple printers mapped to a single print format."
 								)
 							);
 					} else {
@@ -852,11 +852,11 @@ frappe.ui.form.PrintView = class {
 					);
 					dialog.hide();
 				},
-				primary_action_label: __('Save'),
+				primary_action_label: __("Save"),
 			});
 			dialog.show();
 			if (!(this.printer_list && this.printer_list.length)) {
-				frappe.throw(__('No Printer is Available.'));
+				frappe.throw(__("No Printer is Available."));
 			}
 		});
 	}
diff --git a/frappe/printing/page/print/print.py b/frappe/printing/page/print/print.py
index 82ad22656d..baf98438cd 100644
--- a/frappe/printing/page/print/print.py
+++ b/frappe/printing/page/print/print.py
@@ -14,6 +14,8 @@ def get_print_settings_to_show(doctype, docname):
 	print_settings_fields = []
 	for fieldname in fields:
 		df = print_settings.meta.get_field(fieldname)
+		if not df:
+			continue
 		df.default = print_settings.get(fieldname)
 		print_settings_fields.append(df)
 
diff --git a/frappe/printing/page/print_format_builder/print_format_builder.js b/frappe/printing/page/print_format_builder/print_format_builder.js
index 313e8da539..4154861f84 100644
--- a/frappe/printing/page/print_format_builder/print_format_builder.js
+++ b/frappe/printing/page/print_format_builder/print_format_builder.js
@@ -1,17 +1,17 @@
-frappe.pages['print-format-builder'].on_page_load = function(wrapper) {
+frappe.pages["print-format-builder"].on_page_load = function (wrapper) {
 	frappe.print_format_builder = new frappe.PrintFormatBuilder(wrapper);
 	frappe.breadcrumbs.add("Setup", "Print Format");
-}
+};
 
-frappe.pages['print-format-builder'].on_page_show = function(wrapper) {
+frappe.pages["print-format-builder"].on_page_show = function (wrapper) {
 	var route = frappe.get_route();
-	if(route.length>1) {
-		frappe.model.with_doc('Print Format', route[1], function() {
-			frappe.print_format_builder.print_format = frappe.get_doc('Print Format', route[1]);
+	if (route.length > 1) {
+		frappe.model.with_doc("Print Format", route[1], function () {
+			frappe.print_format_builder.print_format = frappe.get_doc("Print Format", route[1]);
 			frappe.print_format_builder.refresh();
 		});
-	} else if(frappe.route_options) {
-		if(frappe.route_options.make_new) {
+	} else if (frappe.route_options) {
+		if (frappe.route_options.make_new) {
 			let { doctype, name, based_on, beta } = frappe.route_options;
 			frappe.route_options = null;
 			frappe.print_format_builder.setup_new_print_format(doctype, name, based_on, beta);
@@ -21,7 +21,7 @@ frappe.pages['print-format-builder'].on_page_show = function(wrapper) {
 			frappe.print_format_builder.refresh();
 		}
 	}
-}
+};
 
 frappe.PrintFormatBuilder = class PrintFormatBuilder {
 	constructor(parent) {
@@ -31,7 +31,7 @@ frappe.PrintFormatBuilder = class PrintFormatBuilder {
 	}
 	refresh() {
 		this.custom_html_count = 0;
-		if(!this.print_format) {
+		if (!this.print_format) {
 			this.show_start();
 		} else {
 			this.page.set_title(this.print_format.name);
@@ -44,10 +44,14 @@ frappe.PrintFormatBuilder = class PrintFormatBuilder {
 			title: __("Print Format Builder"),
 		});
 
-		this.page.main.css({"border-color": "transparent"});
+		this.page.main.css({ "border-color": "transparent" });
 
-		this.page.sidebar = $('
        ').appendTo(this.page.sidebar);
-		this.page.main = $('
        ').appendTo(this.page.main);
+		this.page.sidebar = $('
        ').appendTo(
+			this.page.sidebar
+		);
+		this.page.main = $(
+			'
        '
+		).appendTo(this.page.main);
 
 		// future-bindings for buttons on sections / fields
 		// bind only once
@@ -73,20 +77,20 @@ frappe.PrintFormatBuilder = class PrintFormatBuilder {
 				fieldtype: "Link",
 				options: "Print Format",
 				filters: {
-					print_format_builder: 1
+					print_format_builder: 1,
 				},
 				label: __("Select Print Format to Edit"),
-				only_select: true
+				only_select: true,
 			},
-			render_input: true
+			render_input: true,
 		});
 
 		// create a new print format.
-		this.page.main.find(".btn-edit-print-format").on("click", function() {
+		this.page.main.find(".btn-edit-print-format").on("click", function () {
 			var name = me.print_format_input.get_value();
-			if(!name) return;
-			frappe.model.with_doc("Print Format", name, function(doc) {
-				frappe.set_route('print-format-builder', name);
+			if (!name) return;
+			frappe.model.with_doc("Print Format", name, function (doc) {
+				frappe.set_route("print-format-builder", name);
 			});
 		});
 	}
@@ -98,12 +102,12 @@ frappe.PrintFormatBuilder = class PrintFormatBuilder {
 				fieldtype: "Link",
 				options: "DocType",
 				filters: {
-					"istable": 0,
-					"issingle": 0
+					istable: 0,
+					issingle: 0,
 				},
-				label: __("Select a DocType to make a new format")
+				label: __("Select a DocType to make a new format"),
 			},
-			render_input: true
+			render_input: true,
 		});
 
 		this.name_input = frappe.ui.form.make_control({
@@ -112,34 +116,33 @@ frappe.PrintFormatBuilder = class PrintFormatBuilder {
 				fieldtype: "Data",
 				label: __("Name of the new Print Format"),
 			},
-			render_input: true
+			render_input: true,
 		});
 
-		this.page.main.find(".btn-new-print-format").on("click", function() {
+		this.page.main.find(".btn-new-print-format").on("click", function () {
 			var doctype = me.doctype_input.get_value(),
 				name = me.name_input.get_value();
-			if(!(doctype && name)) {
+			if (!(doctype && name)) {
 				frappe.msgprint(__("Both DocType and Name required"));
 				return;
 			}
 			me.setup_new_print_format(doctype, name);
-
 		});
 	}
 	setup_new_print_format(doctype, name, based_on, beta) {
 		frappe.call({
-			method: 'frappe.printing.page.print_format_builder.print_format_builder.create_custom_format',
+			method: "frappe.printing.page.print_format_builder.print_format_builder.create_custom_format",
 			args: {
 				doctype: doctype,
 				name: name,
 				based_on: based_on,
-				beta: Boolean(beta)
+				beta: Boolean(beta),
 			},
 			callback: (r) => {
 				if (r.message) {
 					let print_format = r.message;
 					if (print_format.print_format_builder_beta) {
-						frappe.set_route('print-format-builder-beta', print_format.name);
+						frappe.set_route("print-format-builder-beta", print_format.name);
 					} else {
 						this.print_format = print_format;
 						this.refresh();
@@ -150,20 +153,24 @@ frappe.PrintFormatBuilder = class PrintFormatBuilder {
 	}
 	setup_print_format() {
 		var me = this;
-		frappe.model.with_doctype(this.print_format.doc_type, function(doctype) {
+		frappe.model.with_doctype(this.print_format.doc_type, function (doctype) {
 			me.meta = frappe.get_meta(me.print_format.doc_type);
 			me.setup_sidebar();
 			me.render_layout();
-			me.page.set_primary_action(__("Save"), function() {
+			me.page.set_primary_action(__("Save"), function () {
 				me.save_print_format();
 			});
 			me.page.clear_menu();
-			me.page.add_menu_item(__("Start new Format"), function() {
-				me.print_format = null;
-				me.refresh();
-			}, true);
+			me.page.add_menu_item(
+				__("Start new Format"),
+				function () {
+					me.print_format = null;
+					me.refresh();
+				},
+				true
+			);
 			me.page.clear_inner_toolbar();
-			me.page.add_inner_button(__("Edit Properties"), function() {
+			me.page.add_inner_button(__("Edit Properties"), function () {
 				frappe.set_route("Form", "Print Format", me.print_format.name);
 			});
 		});
@@ -172,7 +179,7 @@ frappe.PrintFormatBuilder = class PrintFormatBuilder {
 		// prepend custom HTML field
 		var fields = [this.get_custom_html_field()].concat(this.meta.fields);
 		this.page.sidebar.html(
-			$(frappe.render_template("print_format_builder_sidebar", {fields: fields}))
+			$(frappe.render_template("print_format_builder_sidebar", { fields: fields }))
 		);
 		this.setup_field_filter();
 	}
@@ -180,15 +187,18 @@ frappe.PrintFormatBuilder = class PrintFormatBuilder {
 		return {
 			fieldtype: "Custom HTML",
 			fieldname: "_custom_html",
-			label: __("Custom HTML")
+			label: __("Custom HTML"),
 		};
 	}
 	render_layout() {
 		this.page.main.empty();
 		this.prepare_data();
-		$(frappe.render_template("print_format_builder_layout", {
-			data: this.layout_data, me: this}))
-			.appendTo(this.page.main);
+		$(
+			frappe.render_template("print_format_builder_layout", {
+				data: this.layout_data,
+				me: this,
+			})
+		).appendTo(this.page.main);
 		this.setup_sortable();
 		this.setup_add_section();
 		this.setup_edit_heading();
@@ -198,82 +208,89 @@ frappe.PrintFormatBuilder = class PrintFormatBuilder {
 	prepare_data() {
 		this.print_heading_template = null;
 		this.data = JSON.parse(this.print_format.format_data || "[]");
-		if(!this.data.length) {
+		if (!this.data.length) {
 			// new layout
 			this.data = this.meta.fields;
 		} else {
 			// extract print_heading_template if found
-			if(this.data[0].fieldname==="print_heading_template") {
+			if (this.data[0].fieldname === "print_heading_template") {
 				this.print_heading_template = this.data[0].options;
 				this.data = this.data.splice(1);
 			}
 		}
 
-		if(!this.print_heading_template) {
+		if (!this.print_heading_template) {
 			// default print heading template
-			this.print_heading_template = '
        \
-				
        '+__(this.print_format.doc_type)
-					+'

        {{ doc.name }}\
+			this.print_heading_template =
+				'
        \
+				
        ' +
+				__(this.print_format.doc_type) +
+				'

        {{ doc.name }}\
 				
        ';
 		}
 
 		this.layout_data = [];
 		this.fields_dict = {};
 		this.custom_html_dict = {};
-		var section = null, column = null, me = this, custom_html_count = 0;
+		var section = null,
+			column = null,
+			me = this,
+			custom_html_count = 0;
 
 		// create a new placeholder for column and set
 		// it as "column"
-		var set_column = function() {
-			if(!section) set_section();
+		var set_column = function () {
+			if (!section) set_section();
 			column = me.get_new_column();
 			section.columns.push(column);
 			section.no_of_columns += 1;
-		}
+		};
 
-		var set_section = function(label) {
+		var set_section = function (label) {
 			section = me.get_new_section();
-			if(label) section.label = label;
+			if (label) section.label = label;
 			column = null;
 			me.layout_data.push(section);
-		}
+		};
 
 		// break the layout into sections and columns
 		// so that it is easier to render in a template
-		$.each(this.data, function(i, f) {
+		$.each(this.data, function (i, f) {
 			me.fields_dict[f.fieldname] = f;
-			if(!f.name && f.fieldname) {
+			if (!f.name && f.fieldname) {
 				// from format_data (designed format)
 				// print_hide should always be false
-				if(f.fieldname==="_custom_html") {
+				if (f.fieldname === "_custom_html") {
 					f.label = "Custom HTML";
 					f.fieldtype = "Custom HTML";
 
 					// set custom html id to map data properties later
 					custom_html_count++;
 					f.custom_html_id = custom_html_count;
-					me.custom_html_dict[f.custom_html_id] = f
+					me.custom_html_dict[f.custom_html_id] = f;
 				} else {
-					f = $.extend(frappe.meta.get_docfield(me.print_format.doc_type,
-						f.fieldname) || {}, f);
+					f = $.extend(
+						frappe.meta.get_docfield(me.print_format.doc_type, f.fieldname) || {},
+						f
+					);
 				}
 			}
 
-			if(f.fieldtype==="Section Break") {
+			if (f.fieldtype === "Section Break") {
 				set_section(f.label);
-
-			} else if(f.fieldtype==="Column Break") {
+			} else if (f.fieldtype === "Column Break") {
 				set_column();
+			} else if (
+				!in_list(["Section Break", "Column Break", "Tab Break", "Fold"], f.fieldtype) &&
+				f.label
+			) {
+				if (!column) set_column();
 
-			} else if (!in_list(["Section Break", "Column Break", "Tab Break", "Fold"], f.fieldtype)
-				&& f.label) {
-				if(!column) set_column();
-
-				if(f.fieldtype==="Table") {
+				if (f.fieldtype === "Table") {
 					me.add_table_properties(f);
 				}
 
-				if(!f.print_hide) {
+				if (!f.print_hide) {
 					column.fields.push(f);
 					section.has_fields = true;
 				}
@@ -281,33 +298,38 @@ frappe.PrintFormatBuilder = class PrintFormatBuilder {
 		});
 
 		// strip out empty sections
-		this.layout_data = $.map(this.layout_data, function(s) {
-			return s.has_fields ? s : null
+		this.layout_data = $.map(this.layout_data, function (s) {
+			return s.has_fields ? s : null;
 		});
 	}
 	get_new_section() {
-		return {columns: [], no_of_columns: 0, label:''};
+		return { columns: [], no_of_columns: 0, label: "" };
 	}
 	get_new_column() {
-		return {fields: []}
+		return { fields: [] };
 	}
 	add_table_properties(f) {
 		// build table columns and widths in a dict
 		// visible_columns
 		var me = this;
-		if(!f.visible_columns) {
+		if (!f.visible_columns) {
 			me.init_visible_columns(f);
 		}
 	}
 	init_visible_columns(f) {
-		f.visible_columns = []
-		$.each(frappe.get_meta(f.options).fields, function(i, _f) {
-			if (!in_list(["Section Break", "Column Break", "Tab Break"], _f.fieldtype) &&
-				!_f.print_hide && f.label) {
-
+		f.visible_columns = [];
+		$.each(frappe.get_meta(f.options).fields, function (i, _f) {
+			if (
+				!in_list(["Section Break", "Column Break", "Tab Break"], _f.fieldtype) &&
+				!_f.print_hide &&
+				f.label
+			) {
 				// column names set as fieldname|width
-				f.visible_columns.push({fieldname: _f.fieldname,
-					print_width: (_f.width || ""), print_hide:0});
+				f.visible_columns.push({
+					fieldname: _f.fieldname,
+					print_width: _f.width || "",
+					print_hide: 0,
+				});
 			}
 		});
 	}
@@ -315,118 +337,125 @@ frappe.PrintFormatBuilder = class PrintFormatBuilder {
 		var me = this;
 
 		// drag from fields library
-		Sortable.create(this.page.sidebar.find(".print-format-builder-sidebar-fields").get(0),
-			{
-				group: {
-					name:'field', put: true, pull:"clone"
-				},
-				sort: false,
-				onAdd: function(evt) {
-					// on drop, trash!
-					$(evt.item).fadeOut();
-				}
-			});
+		Sortable.create(this.page.sidebar.find(".print-format-builder-sidebar-fields").get(0), {
+			group: {
+				name: "field",
+				put: true,
+				pull: "clone",
+			},
+			sort: false,
+			onAdd: function (evt) {
+				// on drop, trash!
+				$(evt.item).fadeOut();
+			},
+		});
 
 		// sort, drag and drop between columns
-		this.page.main.find(".print-format-builder-column").each(function() {
+		this.page.main.find(".print-format-builder-column").each(function () {
 			me.setup_sortable_for_column(this);
 		});
 
 		// section sorting
-		Sortable.create(this.page.main.find(".print-format-builder-layout").get(0),
-			{ handle: ".print-format-builder-section-head" }
-		);
+		Sortable.create(this.page.main.find(".print-format-builder-layout").get(0), {
+			handle: ".print-format-builder-section-head",
+		});
 	}
 	setup_sortable_for_column(col) {
 		var me = this;
 		Sortable.create(col, {
 			group: {
-				name: 'field',
+				name: "field",
 				put: true,
-				pull: true
+				pull: true,
 			},
-			onAdd: function(evt) {
+			onAdd: function (evt) {
 				// on drop, change the HTML
 
 				var $item = $(evt.item);
-				if(!$item.hasClass("print-format-builder-field")) {
+				if (!$item.hasClass("print-format-builder-field")) {
 					var fieldname = $item.attr("data-fieldname");
 
-					if(fieldname==="_custom_html") {
+					if (fieldname === "_custom_html") {
 						var field = me.get_custom_html_field();
 					} else {
-						var field = frappe.meta.get_docfield(me.print_format.doc_type,
-							fieldname);
+						var field = frappe.meta.get_docfield(me.print_format.doc_type, fieldname);
 					}
 
-					var html = frappe.render_template("print_format_builder_field",
-						{field: field, me:me});
+					var html = frappe.render_template("print_format_builder_field", {
+						field: field,
+						me: me,
+					});
 
 					$item.replaceWith(html);
 				}
-			}
+			},
 		});
-
 	}
 	setup_field_filter() {
 		var me = this;
-		this.page.sidebar.find(".filter-fields").on("keyup", function() {
+		this.page.sidebar.find(".filter-fields").on("keyup", function () {
 			var text = $(this).val();
-			me.page.sidebar.find(".field-label").each(function() {
-				var show = !text || $(this).text().toLowerCase().indexOf(text.toLowerCase())!==-1;
+			me.page.sidebar.find(".field-label").each(function () {
+				var show =
+					!text || $(this).text().toLowerCase().indexOf(text.toLowerCase()) !== -1;
 				$(this).parent().toggle(show);
-			})
+			});
 		});
 	}
 	setup_section_settings() {
 		var me = this;
-		this.page.main.on("click", ".section-settings", function() {
+		this.page.main.on("click", ".section-settings", function () {
 			var section = $(this).parent().parent();
 			var no_of_columns = section.find(".section-column").length;
-			var label = section.attr('data-label');
+			var label = section.attr("data-label");
 
 			// new dialog
 			var d = new frappe.ui.Dialog({
 				title: "Edit Section",
 				fields: [
 					{
-						label:__("No of Columns"),
-						fieldname:"no_of_columns",
-						fieldtype:"Select",
+						label: __("No of Columns"),
+						fieldname: "no_of_columns",
+						fieldtype: "Select",
 						options: ["1", "2", "3", "4"],
 					},
 					{
-						label:__("Section Heading"),
-						fieldname:"label",
-						fieldtype:"Data",
-						description: __('Will only be shown if section headings are enabled')
+						label: __("Section Heading"),
+						fieldname: "label",
+						fieldtype: "Data",
+						description: __("Will only be shown if section headings are enabled"),
 					},
 					{
 						label: __("Remove Section"),
 						fieldname: "remove_section",
 						fieldtype: "Button",
-						click: function() {
+						click: function () {
 							d.hide();
-							section.fadeOut(function() {section.remove()});
+							section.fadeOut(function () {
+								section.remove();
+							});
 						},
 						input_class: "btn-danger",
 						input_css: {
-							"margin-top": "20px"
-						}
-					}
+							"margin-top": "20px",
+						},
+					},
 				],
 			});
 
 			d.set_input("no_of_columns", no_of_columns + "");
 			d.set_input("label", label || "");
 
-			d.set_primary_action(__("Update"), function() {
+			d.set_primary_action(__("Update"), function () {
 				// resize number of columns
-				me.update_columns_in_section(section, no_of_columns,
-					cint(d.get_value("no_of_columns")));
+				me.update_columns_in_section(
+					section,
+					no_of_columns,
+					cint(d.get_value("no_of_columns"))
+				);
 
-				section.attr('data-label', d.get_value('label') || '');
-				section.find('.section-label').html(d.get_value('label') || '');
+				section.attr("data-label", d.get_value("label") || "");
+				section.find(".section-label").html(d.get_value("label") || "");
 
 				d.hide();
 			});
@@ -437,49 +466,52 @@ frappe.PrintFormatBuilder = class PrintFormatBuilder {
 		});
 	}
 	setup_field_settings() {
-		this.page.main.find(".field-settings").on("click", e => {
+		this.page.main.find(".field-settings").on("click", (e) => {
 			const field = $(e.currentTarget).parent();
 			// new dialog
 			var d = new frappe.ui.Dialog({
-				title: "Set Properties",
+				title: __("Set Properties"),
 				fields: [
 					{
 						label: __("Label"),
 						fieldname: "label",
-						fieldtype: "Data"
+						fieldtype: "Data",
 					},
 					{
 						label: __("Align Value"),
 						fieldname: "align",
 						fieldtype: "Select",
-						options: [{'label': __('Left'), 'value': 'left'}, {'label': __('Right'), 'value': 'right'}]
+						options: [
+							{ label: __("Left", null, "alignment"), value: "left" },
+							{ label: __("Right", null, "alignment"), value: "right" },
+						],
 					},
 					{
 						label: __("Remove Field"),
 						fieldtype: "Button",
-						click: function() {
+						click: function () {
 							d.hide();
 							field.remove();
 						},
 						input_class: "btn-danger",
-					}
+					},
 				],
 			});
 
-			d.set_value('label', field.attr("data-label"));
+			d.set_value("label", field.attr("data-label"));
 
-			d.set_primary_action(__("Update"), function() {
-				field.attr('data-align', d.get_value('align'));
-				field.attr('data-label', d.get_value('label'));
-				field.find('.field-label').html(d.get_value('label'));
+			d.set_primary_action(__("Update"), function () {
+				field.attr("data-align", d.get_value("align"));
+				field.attr("data-label", d.get_value("label"));
+				field.find(".field-label").html(d.get_value("label"));
 				d.hide();
 			});
 
 			// set current value
-			if (field.attr('data-align')) {
-				d.set_value('align', field.attr('data-align'));
+			if (field.attr("data-align")) {
+				d.set_value("align", field.attr("data-align"));
 			} else {
-				d.set_value('align', 'left');
+				d.set_value("align", "left");
 			}
 
 			d.show();
@@ -494,32 +526,33 @@ frappe.PrintFormatBuilder = class PrintFormatBuilder {
 		// this is based on a dummy attribute custom_html_id, since all custom html
 		// fields have the same fieldname `_custom_html`
 		var me = this;
-		this.page.main.find('[data-fieldtype="Custom HTML"]').each(function() {
-			var fieldname = $(this).attr('data-fieldname');
-			var content = $($(this).find('.html-content')[0]);
-			var html = me.custom_html_dict[parseInt(content.attr('data-custom-html-id'))].options;
-			content.data('content', html);
-		})
+		this.page.main.find('[data-fieldtype="Custom HTML"]').each(function () {
+			var fieldname = $(this).attr("data-fieldname");
+			var content = $($(this).find(".html-content")[0]);
+			var html = me.custom_html_dict[parseInt(content.attr("data-custom-html-id"))].options;
+			content.data("content", html);
+		});
 	}
 	update_columns_in_section(section, no_of_columns, new_no_of_columns) {
 		var col_size = 12 / new_no_of_columns,
 			me = this,
-			resize = function() {
-				section.find(".section-column")
+			resize = function () {
+				section
+					.find(".section-column")
 					.removeClass()
 					.addClass("section-column")
-					.addClass("col-md-" + col_size)
+					.addClass("col-md-" + col_size);
 			};
 
-		if(new_no_of_columns < no_of_columns) {
+		if (new_no_of_columns < no_of_columns) {
 			// move contents of last n columns to previous column
-			for(var i=no_of_columns; i > new_no_of_columns; i--) {
-				var $col = $(section.find(".print-format-builder-column").get(i-1));
-				var prev = section.find(".print-format-builder-column").get(i-2);
+			for (var i = no_of_columns; i > new_no_of_columns; i--) {
+				var $col = $(section.find(".print-format-builder-column").get(i - 1));
+				var prev = section.find(".print-format-builder-column").get(i - 2);
 
 				// append each field to prev
 				$col.parent().addClass("to-drop");
-				$col.find(".print-format-builder-field").each(function() {
+				$col.find(".print-format-builder-field").each(function () {
 					$(this).appendTo(prev);
 				});
 			}
@@ -529,32 +562,33 @@ frappe.PrintFormatBuilder = class PrintFormatBuilder {
 
 			// resize
 			resize();
-
-		} else if(new_no_of_columns > no_of_columns) {
+		} else if (new_no_of_columns > no_of_columns) {
 			// add empty column and resize old columns
-			for(var i=no_of_columns; i < new_no_of_columns; i++) {
-				var col = $('
        \
-					
        ')
-				.appendTo(section);
-				me.setup_sortable_for_column(col
-					.find(".print-format-builder-column").get(0));
+			for (var i = no_of_columns; i < new_no_of_columns; i++) {
+				var col = $(
+					'
        \
+					
        '
+				).appendTo(section);
+				me.setup_sortable_for_column(col.find(".print-format-builder-column").get(0));
 			}
 			// resize
 			resize();
 		}
-
 	}
 	setup_add_section() {
 		var me = this;
-		this.page.main.find(".print-format-builder-add-section").on("click", function() {
+		this.page.main.find(".print-format-builder-add-section").on("click", function () {
 			// boostrap new section info
 			var section = me.get_new_section();
 			section.columns.push(me.get_new_column());
 			section.no_of_columns = 1;
 
-			var $section = $(frappe.render_template("print_format_builder_section",
-				{section: section, me: me}))
-				.appendTo(me.page.main.find(".print-format-builder-layout"))
+			var $section = $(
+				frappe.render_template("print_format_builder_section", {
+					section: section,
+					me: me,
+				})
+			).appendTo(me.page.main.find(".print-format-builder-layout"));
 
 			me.setup_sortable_for_column($section.find(".print-format-builder-column").get(0));
 		});
@@ -564,23 +598,25 @@ frappe.PrintFormatBuilder = class PrintFormatBuilder {
 		var $heading = this.page.main.find(".print-format-builder-print-heading");
 
 		// set content property
-		$heading.data('content', this.print_heading_template);
+		$heading.data("content", this.print_heading_template);
 
-		this.page.main.find(".edit-heading").on("click", function() {
+		this.page.main.find(".edit-heading").on("click", function () {
 			var d = me.get_edit_html_dialog(__("Edit Heading"), __("Heading"), $heading);
-		})
+		});
 	}
 	setup_column_selector() {
 		var me = this;
-		this.page.main.on("click", ".select-columns", function() {
+		this.page.main.on("click", ".select-columns", function () {
 			var parent = $(this).parents(".print-format-builder-field:first"),
 				doctype = parent.attr("data-doctype"),
 				label = parent.attr("data-label"),
 				columns = parent.attr("data-columns").split(","),
-				column_names = $.map(columns, function(v) { return v.split("|")[0]; }),
+				column_names = $.map(columns, function (v) {
+					return v.split("|")[0];
+				}),
 				widths = {};
 
-			$.each(columns, function(i, v) {
+			$.each(columns, function (i, v) {
 				var parts = v.split("|");
 				widths[parts[0]] = parts[1] || "";
 			});
@@ -591,46 +627,51 @@ frappe.PrintFormatBuilder = class PrintFormatBuilder {
 
 			var $body = $(d.body);
 
-
 			var doc_fields = frappe.get_meta(doctype).fields;
 			var docfields_by_name = {};
 
 			// docfields by fieldname
-			$.each(doc_fields, function(j, f) {
-				if(f) docfields_by_name[f.fieldname] = f;
-			})
+			$.each(doc_fields, function (j, f) {
+				if (f) docfields_by_name[f.fieldname] = f;
+			});
 
 			// add field which are in column_names first to preserve order
 			var fields = [];
-			$.each(column_names, function(i, v) {
-				if(in_list(Object.keys(docfields_by_name), v)) {
+			$.each(column_names, function (i, v) {
+				if (in_list(Object.keys(docfields_by_name), v)) {
 					fields.push(docfields_by_name[v]);
 				}
-			})
+			});
 			// add remaining fields
-			$.each(doc_fields, function(j, f) {
-				if (f && !in_list(column_names, f.fieldname)
-					&& !in_list(["Section Break", "Column Break", "Tab Break"], f.fieldtype) && f.label) {
+			$.each(doc_fields, function (j, f) {
+				if (
+					f &&
+					!in_list(column_names, f.fieldname) &&
+					!in_list(["Section Break", "Column Break", "Tab Break"], f.fieldtype) &&
+					f.label
+				) {
 					fields.push(f);
 				}
-			})
+			});
 			// render checkboxes
-			$(frappe.render_template("print_format_builder_column_selector", {
-				fields: fields,
-				column_names: column_names,
-				widths: widths
-			})).appendTo(d.body);
+			$(
+				frappe.render_template("print_format_builder_column_selector", {
+					fields: fields,
+					column_names: column_names,
+					widths: widths,
+				})
+			).appendTo(d.body);
 
 			Sortable.create($body.find(".column-selector-list").get(0));
 
-			var get_width_input = function(fieldname) {
-				return $body.find(".column-width[data-fieldname='"+ fieldname +"']")
-			}
+			var get_width_input = function (fieldname) {
+				return $body.find(".column-width[data-fieldname='" + fieldname + "']");
+			};
 
 			// update data-columns property on update
-			d.set_primary_action(__("Update"), function() {
+			d.set_primary_action(__("Update"), function () {
 				var visible_columns = [];
-				$body.find("input:checked").each(function() {
+				$body.find("input:checked").each(function () {
 					var fieldname = $(this).attr("data-fieldname"),
 						width = get_width_input(fieldname).val() || "";
 					visible_columns.push(fieldname + "|" + width);
@@ -642,17 +683,17 @@ frappe.PrintFormatBuilder = class PrintFormatBuilder {
 			let update_column_count_message = () => {
 				// show a warning if user selects more than 10 columns for a table
 				let columns_count = $body.find("input:checked").length;
-				$body.find('.help-message').toggle(columns_count > 10);
-			}
+				$body.find(".help-message").toggle(columns_count > 10);
+			};
 			update_column_count_message();
 
 			// enable / disable input based on selection
-			$body.on("click", "input[type='checkbox']", function() {
+			$body.on("click", "input[type='checkbox']", function () {
 				var disabled = !$(this).prop("checked"),
 					input = get_width_input($(this).attr("data-fieldname"));
 
 				input.prop("disabled", disabled);
-				if(disabled) input.val("");
+				if (disabled) input.val("");
 
 				update_column_count_message();
 			});
@@ -663,19 +704,24 @@ frappe.PrintFormatBuilder = class PrintFormatBuilder {
 		});
 	}
 	get_visible_columns_string(f) {
-		if(!f.visible_columns) {
+		if (!f.visible_columns) {
 			this.init_visible_columns(f);
 		}
-		return $.map(f.visible_columns, function(v) { return v.fieldname + "|" + (v.print_width || "") }).join(",");
+		return $.map(f.visible_columns, function (v) {
+			return v.fieldname + "|" + (v.print_width || "");
+		}).join(",");
 	}
 	get_no_content() {
-		return __("Edit to add content")
+		return __("Edit to add content");
 	}
 	setup_edit_custom_html() {
 		var me = this;
-		this.page.main.on("click", ".edit-html", function() {
-			me.get_edit_html_dialog(__("Edit Custom HTML"), __("Custom HTML"),
-				$(this).parents(".print-format-builder-field:first").find(".html-content"));
+		this.page.main.on("click", ".edit-html", function () {
+			me.get_edit_html_dialog(
+				__("Edit Custom HTML"),
+				__("Custom HTML"),
+				$(this).parents(".print-format-builder-field:first").find(".html-content")
+			);
 		});
 	}
 	get_edit_html_dialog(title, label, $content) {
@@ -687,26 +733,31 @@ frappe.PrintFormatBuilder = class PrintFormatBuilder {
 					fieldname: "content",
 					fieldtype: "Code",
 					label: label,
-					options: "HTML"
+					options: "HTML",
 				},
 				{
 					fieldname: "help",
 					fieldtype: "HTML",
-					options: '
        '
-						+ __("You can add dynamic properties from the document by using Jinja templating.")
-						+ __("For example: If you want to include the document ID, use {0}", ["{{ doc.name }}"])
-					+ '
        '
-				}
-			]
+					options:
+						"
        " +
+						__(
+							"You can add dynamic properties from the document by using Jinja templating."
+						) +
+						__("For example: If you want to include the document ID, use {0}", [
+							"{{ doc.name }}",
+						]) +
+						"
        ",
+				},
+			],
 		});
 
 		// set existing content in input
-		var content = $content.data('content') || "";
-		if(content.indexOf(me.get_no_content())!==-1) content = "";
+		var content = $content.data("content") || "";
+		if (content.indexOf(me.get_no_content()) !== -1) content = "";
 		d.set_input("content", content);
 
-		d.set_primary_action(__("Update"), function() {
-			$($content[0]).data('content', d.get_value("content"));
+		d.set_primary_action(__("Update"), function () {
+			$($content[0]).data("content", d.get_value("content"));
 			$content.html(d.get_value("content"));
 			d.hide();
 		});
@@ -724,55 +775,59 @@ frappe.PrintFormatBuilder = class PrintFormatBuilder {
 		// before rendering
 		data.push({
 			fieldname: "print_heading_template",
-			fieldtype:"Custom HTML",
-			options: this.page.main.find(".print-format-builder-print-heading").data('content')
+			fieldtype: "Custom HTML",
+			options: this.page.main.find(".print-format-builder-print-heading").data("content"),
 		});
 
 		// add pages
-		this.page.main.find(".print-format-builder-section").each(function() {
-			var section = {"fieldtype": "Section Break", 'label': $(this).attr('data-label') || '' };
+		this.page.main.find(".print-format-builder-section").each(function () {
+			var section = { fieldtype: "Section Break", label: $(this).attr("data-label") || "" };
 			data.push(section);
-			$(this).find(".print-format-builder-column").each(function() {
-				data.push({"fieldtype": "Column Break"});
-				$(this).find(".print-format-builder-field").each(function() {
-					var $this = $(this),
-						fieldtype = $this.attr("data-fieldtype"),
-						align = $this.attr('data-align'),
-						label = $this.attr('data-label'),
-						df = {
-							fieldname: $this.attr("data-fieldname"),
-							print_hide: 0
-						};
+			$(this)
+				.find(".print-format-builder-column")
+				.each(function () {
+					data.push({ fieldtype: "Column Break" });
+					$(this)
+						.find(".print-format-builder-field")
+						.each(function () {
+							var $this = $(this),
+								fieldtype = $this.attr("data-fieldtype"),
+								align = $this.attr("data-align"),
+								label = $this.attr("data-label"),
+								df = {
+									fieldname: $this.attr("data-fieldname"),
+									print_hide: 0,
+								};
 
-					if(align) {
-						df.align = align;
-					}
+							if (align) {
+								df.align = align;
+							}
 
-					if(label) {
-						df.label = label;
-					}
+							if (label) {
+								df.label = label;
+							}
 
-					if(fieldtype==="Table") {
-						// append the user selected columns to visible_columns
-						var columns = $this.attr("data-columns").split(",");
-						df.visible_columns = [];
-						$.each(columns, function(i, c) {
-							var parts = c.split("|");
-							df.visible_columns.push({
-								fieldname:parts[0],
-								print_width:parts[1],
-								print_hide:0
-							});
+							if (fieldtype === "Table") {
+								// append the user selected columns to visible_columns
+								var columns = $this.attr("data-columns").split(",");
+								df.visible_columns = [];
+								$.each(columns, function (i, c) {
+									var parts = c.split("|");
+									df.visible_columns.push({
+										fieldname: parts[0],
+										print_width: parts[1],
+										print_hide: 0,
+									});
+								});
+							}
+							if (fieldtype === "Custom HTML") {
+								// custom html as HTML field
+								df.fieldtype = "HTML";
+								df.options = $($this.find(".html-content")[0]).data("content");
+							}
+							data.push(df);
 						});
-					}
-					if(fieldtype==="Custom HTML") {
-						// custom html as HTML field
-						df.fieldtype = "HTML";
-						df.options = $($this.find(".html-content")[0]).data('content');
-					}
-					data.push(df);
 				});
-			});
 		});
 
 		// save format_data
@@ -786,11 +841,11 @@ frappe.PrintFormatBuilder = class PrintFormatBuilder {
 			},
 			freeze: true,
 			btn: this.page.btn_primary,
-			callback: function(r) {
+			callback: function (r) {
 				me.print_format = r.message;
-				locals['Print Format'][me.print_format.name] = r.message;
-				frappe.show_alert({message: __("Saved"), indicator: 'green'});
-			}
+				locals["Print Format"][me.print_format.name] = r.message;
+				frappe.show_alert({ message: __("Saved"), indicator: "green" });
+			},
 		});
 	}
 };
diff --git a/frappe/printing/page/print_format_builder/print_format_builder_column_selector.html b/frappe/printing/page/print_format_builder/print_format_builder_column_selector.html
index 15d029704b..495b837f48 100644
--- a/frappe/printing/page/print_format_builder/print_format_builder_column_selector.html
+++ b/frappe/printing/page/print_format_builder/print_format_builder_column_selector.html
@@ -13,14 +13,14 @@
 		
        
 			
        
 				
        
-					
+					
 				
        
 				
        
 					
 				
        
 			
        
diff --git a/frappe/printing/page/print_format_builder/print_format_builder_field.html b/frappe/printing/page/print_format_builder/print_format_builder_field.html
index 0e207222a5..beb9de2f5a 100644
--- a/frappe/printing/page/print_format_builder/print_format_builder_field.html
+++ b/frappe/printing/page/print_format_builder/print_format_builder_field.html
@@ -13,11 +13,11 @@
 	{% if !in_list(["Table", "HTML", "Custom HTML"], field.fieldtype) %}
 	
 		
-			
+			
 		
 	
 	
-		
+		
 	
 	{% endif %}
 	{% if(field.fieldtype==="Custom HTML") { %}
@@ -32,7 +32,7 @@
 	{% } else { %}
 		{% if(field.fieldtype==="Table") { %}
 			
-				
+				
 			
 			{{ __(field.label) }}
 				 ({%= __("Table") %})
diff --git a/frappe/printing/page/print_format_builder/print_format_builder_layout.html b/frappe/printing/page/print_format_builder/print_format_builder_layout.html
index f45a760cbc..cc78ddceb8 100644
--- a/frappe/printing/page/print_format_builder/print_format_builder_layout.html
+++ b/frappe/printing/page/print_format_builder/print_format_builder_layout.html
@@ -7,7 +7,7 @@
 			
 				{%= __("Edit Heading") %}
 				
-					
+					
 				
 			
 		
        
@@ -23,7 +23,7 @@
 	
        
 	
        
 		
-			
+			
 		
 		{%= __("Add a new section") %}
 	
        
diff --git a/frappe/printing/page/print_format_builder/print_format_builder_section.html b/frappe/printing/page/print_format_builder/print_format_builder_section.html
index 70a97a15f0..108e5ff01c 100644
--- a/frappe/printing/page/print_format_builder/print_format_builder_section.html
+++ b/frappe/printing/page/print_format_builder/print_format_builder_section.html
@@ -1,10 +1,10 @@
 
        
 	
        
 		
        
-			
+			
 		
        
 		
-			
+			
 		
 		{{ section.label || "" }}
 	
        
diff --git a/frappe/printing/page/print_format_builder/print_format_builder_sidebar.html b/frappe/printing/page/print_format_builder/print_format_builder_sidebar.html
index c608eecbbd..2b5b040bca 100644
--- a/frappe/printing/page/print_format_builder/print_format_builder_sidebar.html
+++ b/frappe/printing/page/print_format_builder/print_format_builder_sidebar.html
@@ -11,7 +11,7 @@
 				{%= (f.fieldtype==="Custom HTML") ? "sidebar-custom-field" : "" %}"
 				style="display: inline-block">
 				
-					
+					
 				
 				{%= __(f.label) || __(f.fieldname) %}
 			
        
diff --git a/frappe/printing/page/print_format_builder_beta/print_format_builder_beta.js b/frappe/printing/page/print_format_builder_beta/print_format_builder_beta.js
index e923bbcb00..d9445d1c3a 100644
--- a/frappe/printing/page/print_format_builder_beta/print_format_builder_beta.js
+++ b/frappe/printing/page/print_format_builder_beta/print_format_builder_beta.js
@@ -1,8 +1,8 @@
-frappe.pages["print-format-builder-beta"].on_page_load = function(wrapper) {
+frappe.pages["print-format-builder-beta"].on_page_load = function (wrapper) {
 	frappe.ui.make_app_page({
 		parent: wrapper,
 		title: __("Print Format Builder"),
-		single_column: true
+		single_column: true,
 	});
 
 	// hot reload in development
@@ -12,7 +12,7 @@ frappe.pages["print-format-builder-beta"].on_page_load = function(wrapper) {
 	}
 };
 
-frappe.pages["print-format-builder-beta"].on_page_show = function(wrapper) {
+frappe.pages["print-format-builder-beta"].on_page_show = function (wrapper) {
 	load_print_format_builder_beta(wrapper);
 };
 
@@ -26,7 +26,7 @@ function load_print_format_builder_beta(wrapper) {
 			frappe.print_format_builder = new frappe.ui.PrintFormatBuilder({
 				wrapper: $parent,
 				page: wrapper.page,
-				print_format: route[1]
+				print_format: route[1],
 			});
 		});
 	} else {
@@ -39,14 +39,12 @@ function load_print_format_builder_beta(wrapper) {
 					fieldtype: "Select",
 					options: [
 						{ label: __("Create New"), value: "Create" },
-						{ label: __("Edit Existing"), value: "Edit" }
+						{ label: __("Edit Existing"), value: "Edit" },
 					],
 					change() {
 						let action = d.get_value("action");
-						d.get_primary_btn().text(
-							action === "Create" ? __("Create") : __("Edit")
-						);
-					}
+						d.get_primary_btn().text(action === "Create" ? __("Create") : __("Edit"));
+					},
 				},
 				{
 					label: __("Select Document Type"),
@@ -54,19 +52,17 @@ function load_print_format_builder_beta(wrapper) {
 					fieldtype: "Link",
 					options: "DocType",
 					filters: {
-						istable: 0
+						istable: 0,
 					},
 					reqd: 1,
-					default: frappe.route_options
-						? frappe.route_options.doctype
-						: null
+					default: frappe.route_options ? frappe.route_options.doctype : null,
 				},
 				{
 					label: __("Print Format Name"),
 					fieldname: "print_format_name",
 					fieldtype: "Data",
-					depends_on: doc => doc.action === "Create",
-					mandatory_depends_on: doc => doc.action === "Create"
+					depends_on: (doc) => doc.action === "Create",
+					mandatory_depends_on: (doc) => doc.action === "Create",
 				},
 				{
 					label: __("Select Print Format"),
@@ -74,25 +70,20 @@ function load_print_format_builder_beta(wrapper) {
 					fieldtype: "Link",
 					options: "Print Format",
 					only_select: 1,
-					depends_on: doc => doc.action === "Edit",
+					depends_on: (doc) => doc.action === "Edit",
 					get_query() {
 						return {
 							filters: {
 								doc_type: d.get_value("doctype"),
-								print_format_builder_beta: 1
-							}
+								print_format_builder_beta: 1,
+							},
 						};
 					},
-					mandatory_depends_on: doc => doc.action === "Edit"
-				}
+					mandatory_depends_on: (doc) => doc.action === "Edit",
+				},
 			],
 			primary_action_label: __("Edit"),
-			primary_action({
-				action,
-				doctype,
-				print_format,
-				print_format_name
-			}) {
+			primary_action({ action, doctype, print_format, print_format_name }) {
 				if (action === "Edit") {
 					frappe.set_route("print-format-builder-beta", print_format);
 				} else if (action === "Create") {
@@ -102,19 +93,16 @@ function load_print_format_builder_beta(wrapper) {
 							doctype: "Print Format",
 							name: print_format_name,
 							doc_type: doctype,
-							print_format_builder_beta: 1
+							print_format_builder_beta: 1,
 						})
-						.then(doc => {
-							frappe.set_route(
-								"print-format-builder-beta",
-								doc.name
-							);
+						.then((doc) => {
+							frappe.set_route("print-format-builder-beta", doc.name);
 						})
 						.finally(() => {
 							d.get_primary_btn().prop("disabled", false);
 						});
 				}
-			}
+			},
 		});
 		d.set_value("action", "Create");
 		d.show();
diff --git a/frappe/printing/page/print_format_builder_beta/print_format_builder_beta.py b/frappe/printing/page/print_format_builder_beta/print_format_builder_beta.py
index 668be8d05a..d54624aaef 100644
--- a/frappe/printing/page/print_format_builder_beta/print_format_builder_beta.py
+++ b/frappe/printing/page/print_format_builder_beta/print_format_builder_beta.py
@@ -1,7 +1,6 @@
 # Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors
 # MIT License. See license.txt
 
-from __future__ import unicode_literals
 
 import functools
 
@@ -13,7 +12,7 @@ def get_google_fonts():
 	return _get_google_fonts()
 
 
-@functools.lru_cache()
+@functools.lru_cache
 def _get_google_fonts():
 	file_path = frappe.get_app_path("frappe", "data", "google_fonts.json")
 	return frappe.parse_json(frappe.read_file(file_path))
diff --git a/frappe/public/css/bootstrap.css b/frappe/public/css/bootstrap.css
index 16afe3cd54..d4ce177bae 100644
--- a/frappe/public/css/bootstrap.css
+++ b/frappe/public/css/bootstrap.css
@@ -259,799 +259,6 @@ th {
     border: 1px solid #ddd;
   }
 }
-@font-face {
-  font-family: 'Glyphicons Halflings';
-
-  src: url('/assets/frappe/css/fonts/glyphicons/glyphicons-halflings-regular.eot');
-  src: url('/assets/frappe/css/fonts/glyphicons/glyphicons-halflings-regular.eot?#iefix') format('embedded-opentype'), url('/assets/frappe/css/fonts/glyphicons/glyphicons-halflings-regular.woff2') format('woff2'), url('/assets/frappe/css/fonts/glyphicons/glyphicons-halflings-regular.woff') format('woff'), url('/assets/frappe/css/fonts/glyphicons/glyphicons-halflings-regular.ttf') format('truetype'), url('/assets/frappe/css/fonts/glyphicons/glyphicons-halflings-regular.svg#glyphicons_halflingsregular') format('svg');
-}
-.glyphicon {
-  position: relative;
-  top: 1px;
-  display: inline-block;
-  font-family: 'Glyphicons Halflings';
-  font-style: normal;
-  font-weight: normal;
-  line-height: 1;
-
-  -webkit-font-smoothing: antialiased;
-  -moz-osx-font-smoothing: grayscale;
-}
-.glyphicon-asterisk:before {
-  content: "\2a";
-}
-.glyphicon-plus:before {
-  content: "\2b";
-}
-.glyphicon-euro:before,
-.glyphicon-eur:before {
-  content: "\20ac";
-}
-.glyphicon-minus:before {
-  content: "\2212";
-}
-.glyphicon-cloud:before {
-  content: "\2601";
-}
-.glyphicon-envelope:before {
-  content: "\2709";
-}
-.glyphicon-pencil:before {
-  content: "\270f";
-}
-.glyphicon-glass:before {
-  content: "\e001";
-}
-.glyphicon-music:before {
-  content: "\e002";
-}
-.glyphicon-search:before {
-  content: "\e003";
-}
-.glyphicon-heart:before {
-  content: "\e005";
-}
-.glyphicon-star:before {
-  content: "\e006";
-}
-.glyphicon-star-empty:before {
-  content: "\e007";
-}
-.glyphicon-user:before {
-  content: "\e008";
-}
-.glyphicon-film:before {
-  content: "\e009";
-}
-.glyphicon-th-large:before {
-  content: "\e010";
-}
-.glyphicon-th:before {
-  content: "\e011";
-}
-.glyphicon-th-list:before {
-  content: "\e012";
-}
-.glyphicon-ok:before {
-  content: "\e013";
-}
-.glyphicon-remove:before {
-  content: "\e014";
-}
-.glyphicon-zoom-in:before {
-  content: "\e015";
-}
-.glyphicon-zoom-out:before {
-  content: "\e016";
-}
-.glyphicon-off:before {
-  content: "\e017";
-}
-.glyphicon-signal:before {
-  content: "\e018";
-}
-.glyphicon-cog:before {
-  content: "\e019";
-}
-.glyphicon-trash:before {
-  content: "\e020";
-}
-.glyphicon-home:before {
-  content: "\e021";
-}
-.glyphicon-file:before {
-  content: "\e022";
-}
-.glyphicon-time:before {
-  content: "\e023";
-}
-.glyphicon-road:before {
-  content: "\e024";
-}
-.glyphicon-download-alt:before {
-  content: "\e025";
-}
-.glyphicon-download:before {
-  content: "\e026";
-}
-.glyphicon-upload:before {
-  content: "\e027";
-}
-.glyphicon-inbox:before {
-  content: "\e028";
-}
-.glyphicon-play-circle:before {
-  content: "\e029";
-}
-.glyphicon-repeat:before {
-  content: "\e030";
-}
-.glyphicon-refresh:before {
-  content: "\e031";
-}
-.glyphicon-list-alt:before {
-  content: "\e032";
-}
-.glyphicon-lock:before {
-  content: "\e033";
-}
-.glyphicon-flag:before {
-  content: "\e034";
-}
-.glyphicon-headphones:before {
-  content: "\e035";
-}
-.glyphicon-volume-off:before {
-  content: "\e036";
-}
-.glyphicon-volume-down:before {
-  content: "\e037";
-}
-.glyphicon-volume-up:before {
-  content: "\e038";
-}
-.glyphicon-qrcode:before {
-  content: "\e039";
-}
-.glyphicon-barcode:before {
-  content: "\e040";
-}
-.glyphicon-tag:before {
-  content: "\e041";
-}
-.glyphicon-tags:before {
-  content: "\e042";
-}
-.glyphicon-book:before {
-  content: "\e043";
-}
-.glyphicon-bookmark:before {
-  content: "\e044";
-}
-.glyphicon-print:before {
-  content: "\e045";
-}
-.glyphicon-camera:before {
-  content: "\e046";
-}
-.glyphicon-font:before {
-  content: "\e047";
-}
-.glyphicon-bold:before {
-  content: "\e048";
-}
-.glyphicon-italic:before {
-  content: "\e049";
-}
-.glyphicon-text-height:before {
-  content: "\e050";
-}
-.glyphicon-text-width:before {
-  content: "\e051";
-}
-.glyphicon-align-left:before {
-  content: "\e052";
-}
-.glyphicon-align-center:before {
-  content: "\e053";
-}
-.glyphicon-align-right:before {
-  content: "\e054";
-}
-.glyphicon-align-justify:before {
-  content: "\e055";
-}
-.glyphicon-list:before {
-  content: "\e056";
-}
-.glyphicon-indent-left:before {
-  content: "\e057";
-}
-.glyphicon-indent-right:before {
-  content: "\e058";
-}
-.glyphicon-facetime-video:before {
-  content: "\e059";
-}
-.glyphicon-picture:before {
-  content: "\e060";
-}
-.glyphicon-map-marker:before {
-  content: "\e062";
-}
-.glyphicon-adjust:before {
-  content: "\e063";
-}
-.glyphicon-tint:before {
-  content: "\e064";
-}
-.glyphicon-edit:before {
-  content: "\e065";
-}
-.glyphicon-share:before {
-  content: "\e066";
-}
-.glyphicon-check:before {
-  content: "\e067";
-}
-.glyphicon-move:before {
-  content: "\e068";
-}
-.glyphicon-step-backward:before {
-  content: "\e069";
-}
-.glyphicon-fast-backward:before {
-  content: "\e070";
-}
-.glyphicon-backward:before {
-  content: "\e071";
-}
-.glyphicon-play:before {
-  content: "\e072";
-}
-.glyphicon-pause:before {
-  content: "\e073";
-}
-.glyphicon-stop:before {
-  content: "\e074";
-}
-.glyphicon-forward:before {
-  content: "\e075";
-}
-.glyphicon-fast-forward:before {
-  content: "\e076";
-}
-.glyphicon-step-forward:before {
-  content: "\e077";
-}
-.glyphicon-eject:before {
-  content: "\e078";
-}
-.glyphicon-chevron-left:before {
-  content: "\e079";
-}
-.glyphicon-chevron-right:before {
-  content: "\e080";
-}
-.glyphicon-plus-sign:before {
-  content: "\e081";
-}
-.glyphicon-minus-sign:before {
-  content: "\e082";
-}
-.glyphicon-remove-sign:before {
-  content: "\e083";
-}
-.glyphicon-ok-sign:before {
-  content: "\e084";
-}
-.glyphicon-question-sign:before {
-  content: "\e085";
-}
-.glyphicon-info-sign:before {
-  content: "\e086";
-}
-.glyphicon-screenshot:before {
-  content: "\e087";
-}
-.glyphicon-remove-circle:before {
-  content: "\e088";
-}
-.glyphicon-ok-circle:before {
-  content: "\e089";
-}
-.glyphicon-ban-circle:before {
-  content: "\e090";
-}
-.glyphicon-arrow-left:before {
-  content: "\e091";
-}
-.glyphicon-arrow-right:before {
-  content: "\e092";
-}
-.glyphicon-arrow-up:before {
-  content: "\e093";
-}
-.glyphicon-arrow-down:before {
-  content: "\e094";
-}
-.glyphicon-share-alt:before {
-  content: "\e095";
-}
-.glyphicon-resize-full:before {
-  content: "\e096";
-}
-.glyphicon-resize-small:before {
-  content: "\e097";
-}
-.glyphicon-exclamation-sign:before {
-  content: "\e101";
-}
-.glyphicon-gift:before {
-  content: "\e102";
-}
-.glyphicon-leaf:before {
-  content: "\e103";
-}
-.glyphicon-fire:before {
-  content: "\e104";
-}
-.glyphicon-eye-open:before {
-  content: "\e105";
-}
-.glyphicon-eye-close:before {
-  content: "\e106";
-}
-.glyphicon-warning-sign:before {
-  content: "\e107";
-}
-.glyphicon-plane:before {
-  content: "\e108";
-}
-.glyphicon-calendar:before {
-  content: "\e109";
-}
-.glyphicon-random:before {
-  content: "\e110";
-}
-.glyphicon-comment:before {
-  content: "\e111";
-}
-.glyphicon-magnet:before {
-  content: "\e112";
-}
-.glyphicon-chevron-up:before {
-  content: "\e113";
-}
-.glyphicon-chevron-down:before {
-  content: "\e114";
-}
-.glyphicon-retweet:before {
-  content: "\e115";
-}
-.glyphicon-shopping-cart:before {
-  content: "\e116";
-}
-.glyphicon-folder-close:before {
-  content: "\e117";
-}
-.glyphicon-folder-open:before {
-  content: "\e118";
-}
-.glyphicon-resize-vertical:before {
-  content: "\e119";
-}
-.glyphicon-resize-horizontal:before {
-  content: "\e120";
-}
-.glyphicon-hdd:before {
-  content: "\e121";
-}
-.glyphicon-bullhorn:before {
-  content: "\e122";
-}
-.glyphicon-bell:before {
-  content: "\e123";
-}
-.glyphicon-certificate:before {
-  content: "\e124";
-}
-.glyphicon-thumbs-up:before {
-  content: "\e125";
-}
-.glyphicon-thumbs-down:before {
-  content: "\e126";
-}
-.glyphicon-hand-right:before {
-  content: "\e127";
-}
-.glyphicon-hand-left:before {
-  content: "\e128";
-}
-.glyphicon-hand-up:before {
-  content: "\e129";
-}
-.glyphicon-hand-down:before {
-  content: "\e130";
-}
-.glyphicon-circle-arrow-right:before {
-  content: "\e131";
-}
-.glyphicon-circle-arrow-left:before {
-  content: "\e132";
-}
-.glyphicon-circle-arrow-up:before {
-  content: "\e133";
-}
-.glyphicon-circle-arrow-down:before {
-  content: "\e134";
-}
-.glyphicon-globe:before {
-  content: "\e135";
-}
-.glyphicon-wrench:before {
-  content: "\e136";
-}
-.glyphicon-tasks:before {
-  content: "\e137";
-}
-.glyphicon-filter:before {
-  content: "\e138";
-}
-.glyphicon-briefcase:before {
-  content: "\e139";
-}
-.glyphicon-fullscreen:before {
-  content: "\e140";
-}
-.glyphicon-dashboard:before {
-  content: "\e141";
-}
-.glyphicon-paperclip:before {
-  content: "\e142";
-}
-.glyphicon-heart-empty:before {
-  content: "\e143";
-}
-.glyphicon-link:before {
-  content: "\e144";
-}
-.glyphicon-phone:before {
-  content: "\e145";
-}
-.glyphicon-pushpin:before {
-  content: "\e146";
-}
-.glyphicon-usd:before {
-  content: "\e148";
-}
-.glyphicon-gbp:before {
-  content: "\e149";
-}
-.glyphicon-sort:before {
-  content: "\e150";
-}
-.glyphicon-sort-by-alphabet:before {
-  content: "\e151";
-}
-.glyphicon-sort-by-alphabet-alt:before {
-  content: "\e152";
-}
-.glyphicon-sort-by-order:before {
-  content: "\e153";
-}
-.glyphicon-sort-by-order-alt:before {
-  content: "\e154";
-}
-.glyphicon-sort-by-attributes:before {
-  content: "\e155";
-}
-.glyphicon-sort-by-attributes-alt:before {
-  content: "\e156";
-}
-.glyphicon-unchecked:before {
-  content: "\e157";
-}
-.glyphicon-expand:before {
-  content: "\e158";
-}
-.glyphicon-collapse-down:before {
-  content: "\e159";
-}
-.glyphicon-collapse-up:before {
-  content: "\e160";
-}
-.glyphicon-log-in:before {
-  content: "\e161";
-}
-.glyphicon-flash:before {
-  content: "\e162";
-}
-.glyphicon-log-out:before {
-  content: "\e163";
-}
-.glyphicon-new-window:before {
-  content: "\e164";
-}
-.glyphicon-record:before {
-  content: "\e165";
-}
-.glyphicon-save:before {
-  content: "\e166";
-}
-.glyphicon-open:before {
-  content: "\e167";
-}
-.glyphicon-saved:before {
-  content: "\e168";
-}
-.glyphicon-import:before {
-  content: "\e169";
-}
-.glyphicon-export:before {
-  content: "\e170";
-}
-.glyphicon-send:before {
-  content: "\e171";
-}
-.glyphicon-floppy-disk:before {
-  content: "\e172";
-}
-.glyphicon-floppy-saved:before {
-  content: "\e173";
-}
-.glyphicon-floppy-remove:before {
-  content: "\e174";
-}
-.glyphicon-floppy-save:before {
-  content: "\e175";
-}
-.glyphicon-floppy-open:before {
-  content: "\e176";
-}
-.glyphicon-credit-card:before {
-  content: "\e177";
-}
-.glyphicon-transfer:before {
-  content: "\e178";
-}
-.glyphicon-cutlery:before {
-  content: "\e179";
-}
-.glyphicon-header:before {
-  content: "\e180";
-}
-.glyphicon-compressed:before {
-  content: "\e181";
-}
-.glyphicon-earphone:before {
-  content: "\e182";
-}
-.glyphicon-phone-alt:before {
-  content: "\e183";
-}
-.glyphicon-tower:before {
-  content: "\e184";
-}
-.glyphicon-stats:before {
-  content: "\e185";
-}
-.glyphicon-sd-video:before {
-  content: "\e186";
-}
-.glyphicon-hd-video:before {
-  content: "\e187";
-}
-.glyphicon-subtitles:before {
-  content: "\e188";
-}
-.glyphicon-sound-stereo:before {
-  content: "\e189";
-}
-.glyphicon-sound-dolby:before {
-  content: "\e190";
-}
-.glyphicon-sound-5-1:before {
-  content: "\e191";
-}
-.glyphicon-sound-6-1:before {
-  content: "\e192";
-}
-.glyphicon-sound-7-1:before {
-  content: "\e193";
-}
-.glyphicon-copyright-mark:before {
-  content: "\e194";
-}
-.glyphicon-registration-mark:before {
-  content: "\e195";
-}
-.glyphicon-cloud-download:before {
-  content: "\e197";
-}
-.glyphicon-cloud-upload:before {
-  content: "\e198";
-}
-.glyphicon-tree-conifer:before {
-  content: "\e199";
-}
-.glyphicon-tree-deciduous:before {
-  content: "\e200";
-}
-.glyphicon-cd:before {
-  content: "\e201";
-}
-.glyphicon-save-file:before {
-  content: "\e202";
-}
-.glyphicon-open-file:before {
-  content: "\e203";
-}
-.glyphicon-level-up:before {
-  content: "\e204";
-}
-.glyphicon-copy:before {
-  content: "\e205";
-}
-.glyphicon-paste:before {
-  content: "\e206";
-}
-.glyphicon-alert:before {
-  content: "\e209";
-}
-.glyphicon-equalizer:before {
-  content: "\e210";
-}
-.glyphicon-king:before {
-  content: "\e211";
-}
-.glyphicon-queen:before {
-  content: "\e212";
-}
-.glyphicon-pawn:before {
-  content: "\e213";
-}
-.glyphicon-bishop:before {
-  content: "\e214";
-}
-.glyphicon-knight:before {
-  content: "\e215";
-}
-.glyphicon-baby-formula:before {
-  content: "\e216";
-}
-.glyphicon-tent:before {
-  content: "\26fa";
-}
-.glyphicon-blackboard:before {
-  content: "\e218";
-}
-.glyphicon-bed:before {
-  content: "\e219";
-}
-.glyphicon-apple:before {
-  content: "\f8ff";
-}
-.glyphicon-erase:before {
-  content: "\e221";
-}
-.glyphicon-hourglass:before {
-  content: "\231b";
-}
-.glyphicon-lamp:before {
-  content: "\e223";
-}
-.glyphicon-duplicate:before {
-  content: "\e224";
-}
-.glyphicon-piggy-bank:before {
-  content: "\e225";
-}
-.glyphicon-scissors:before {
-  content: "\e226";
-}
-.glyphicon-bitcoin:before {
-  content: "\e227";
-}
-.glyphicon-yen:before {
-  content: "\00a5";
-}
-.glyphicon-ruble:before {
-  content: "\20bd";
-}
-.glyphicon-scale:before {
-  content: "\e230";
-}
-.glyphicon-ice-lolly:before {
-  content: "\e231";
-}
-.glyphicon-ice-lolly-tasted:before {
-  content: "\e232";
-}
-.glyphicon-education:before {
-  content: "\e233";
-}
-.glyphicon-option-horizontal:before {
-  content: "\e234";
-}
-.glyphicon-option-vertical:before {
-  content: "\e235";
-}
-.glyphicon-menu-hamburger:before {
-  content: "\e236";
-}
-.glyphicon-modal-window:before {
-  content: "\e237";
-}
-.glyphicon-oil:before {
-  content: "\e238";
-}
-.glyphicon-grain:before {
-  content: "\e239";
-}
-.glyphicon-sunglasses:before {
-  content: "\e240";
-}
-.glyphicon-text-size:before {
-  content: "\e241";
-}
-.glyphicon-text-color:before {
-  content: "\e242";
-}
-.glyphicon-text-background:before {
-  content: "\e243";
-}
-.glyphicon-object-align-top:before {
-  content: "\e244";
-}
-.glyphicon-object-align-bottom:before {
-  content: "\e245";
-}
-.glyphicon-object-align-horizontal:before {
-  content: "\e246";
-}
-.glyphicon-object-align-left:before {
-  content: "\e247";
-}
-.glyphicon-object-align-vertical:before {
-  content: "\e248";
-}
-.glyphicon-object-align-right:before {
-  content: "\e249";
-}
-.glyphicon-triangle-right:before {
-  content: "\e250";
-}
-.glyphicon-triangle-left:before {
-  content: "\e251";
-}
-.glyphicon-triangle-bottom:before {
-  content: "\e252";
-}
-.glyphicon-triangle-top:before {
-  content: "\e253";
-}
-.glyphicon-console:before {
-  content: "\e254";
-}
-.glyphicon-superscript:before {
-  content: "\e255";
-}
-.glyphicon-subscript:before {
-  content: "\e256";
-}
-.glyphicon-menu-left:before {
-  content: "\e257";
-}
-.glyphicon-menu-right:before {
-  content: "\e258";
-}
-.glyphicon-menu-down:before {
-  content: "\e259";
-}
-.glyphicon-menu-up:before {
-  content: "\e260";
-}
 * {
   -webkit-box-sizing: border-box;
      -moz-box-sizing: border-box;
diff --git a/frappe/public/css/font-awesome.css b/frappe/public/css/font-awesome.css
deleted file mode 100644
index afe8a7f458..0000000000
--- a/frappe/public/css/font-awesome.css
+++ /dev/null
@@ -1,2337 +0,0 @@
-/*!
- *  Font Awesome 4.7.0 by @davegandy - http://fontawesome.io - @fontawesome
- *  License - http://fontawesome.io/license (Font: SIL OFL 1.1, CSS: MIT License)
- */
-/* FONT PATH
- * -------------------------- */
-@font-face {
-  font-family: 'FontAwesome';
-  src: url('/assets/frappe/css/fonts/fontawesome/fontawesome-webfont.eot?v=4.7.0');
-  src: url('/assets/frappe/css/fonts/fontawesome/fontawesome-webfont.eot?#iefix&v=4.7.0') format('embedded-opentype'), url('/assets/frappe/css/fonts/fontawesome/fontawesome-webfont.woff2?v=4.7.0') format('woff2'), url('/assets/frappe/css/fonts/fontawesome/fontawesome-webfont.woff?v=4.7.0') format('woff'), url('/assets/frappe/css/fonts/fontawesome/fontawesome-webfont.ttf?v=4.7.0') format('truetype'), url('/assets/frappe/css/fonts/fontawesome/fontawesome-webfont.svg?v=4.7.0#fontawesomeregular') format('svg');
-  font-weight: normal;
-  font-style: normal;
-}
-.fa {
-  display: inline-block;
-  font: normal normal normal 14px/1 FontAwesome;
-  font-size: inherit;
-  text-rendering: auto;
-  -webkit-font-smoothing: antialiased;
-  -moz-osx-font-smoothing: grayscale;
-}
-/* makes the font 33% larger relative to the icon container */
-.fa-lg {
-  font-size: 1.33333333em;
-  line-height: 0.75em;
-  vertical-align: -15%;
-}
-.fa-2x {
-  font-size: 2em;
-}
-.fa-3x {
-  font-size: 3em;
-}
-.fa-4x {
-  font-size: 4em;
-}
-.fa-5x {
-  font-size: 5em;
-}
-.fa-fw {
-  width: 1.28571429em;
-  text-align: center;
-}
-.fa-ul {
-  padding-left: 0;
-  margin-left: 2.14285714em;
-  list-style-type: none;
-}
-.fa-ul > li {
-  position: relative;
-}
-.fa-li {
-  position: absolute;
-  left: -2.14285714em;
-  width: 2.14285714em;
-  top: 0.14285714em;
-  text-align: center;
-}
-.fa-li.fa-lg {
-  left: -1.85714286em;
-}
-.fa-border {
-  padding: .2em .25em .15em;
-  border: solid 0.08em #eeeeee;
-  border-radius: .1em;
-}
-.fa-pull-left {
-  float: left;
-}
-.fa-pull-right {
-  float: right;
-}
-.fa.fa-pull-left {
-  margin-right: .3em;
-}
-.fa.fa-pull-right {
-  margin-left: .3em;
-}
-/* Deprecated as of 4.4.0 */
-.pull-right {
-  float: right;
-}
-.pull-left {
-  float: left;
-}
-.fa.pull-left {
-  margin-right: .3em;
-}
-.fa.pull-right {
-  margin-left: .3em;
-}
-.fa-spin {
-  -webkit-animation: fa-spin 2s infinite linear;
-  animation: fa-spin 2s infinite linear;
-}
-.fa-pulse {
-  -webkit-animation: fa-spin 1s infinite steps(8);
-  animation: fa-spin 1s infinite steps(8);
-}
-@-webkit-keyframes fa-spin {
-  0% {
-    -webkit-transform: rotate(0deg);
-    transform: rotate(0deg);
-  }
-  100% {
-    -webkit-transform: rotate(359deg);
-    transform: rotate(359deg);
-  }
-}
-@keyframes fa-spin {
-  0% {
-    -webkit-transform: rotate(0deg);
-    transform: rotate(0deg);
-  }
-  100% {
-    -webkit-transform: rotate(359deg);
-    transform: rotate(359deg);
-  }
-}
-.fa-rotate-90 {
-  -ms-filter: "progid:DXImageTransform.Microsoft.BasicImage(rotation=1)";
-  -webkit-transform: rotate(90deg);
-  -ms-transform: rotate(90deg);
-  transform: rotate(90deg);
-}
-.fa-rotate-180 {
-  -ms-filter: "progid:DXImageTransform.Microsoft.BasicImage(rotation=2)";
-  -webkit-transform: rotate(180deg);
-  -ms-transform: rotate(180deg);
-  transform: rotate(180deg);
-}
-.fa-rotate-270 {
-  -ms-filter: "progid:DXImageTransform.Microsoft.BasicImage(rotation=3)";
-  -webkit-transform: rotate(270deg);
-  -ms-transform: rotate(270deg);
-  transform: rotate(270deg);
-}
-.fa-flip-horizontal {
-  -ms-filter: "progid:DXImageTransform.Microsoft.BasicImage(rotation=0, mirror=1)";
-  -webkit-transform: scale(-1, 1);
-  -ms-transform: scale(-1, 1);
-  transform: scale(-1, 1);
-}
-.fa-flip-vertical {
-  -ms-filter: "progid:DXImageTransform.Microsoft.BasicImage(rotation=2, mirror=1)";
-  -webkit-transform: scale(1, -1);
-  -ms-transform: scale(1, -1);
-  transform: scale(1, -1);
-}
-:root .fa-rotate-90,
-:root .fa-rotate-180,
-:root .fa-rotate-270,
-:root .fa-flip-horizontal,
-:root .fa-flip-vertical {
-  filter: none;
-}
-.fa-stack {
-  position: relative;
-  display: inline-block;
-  width: 2em;
-  height: 2em;
-  line-height: 2em;
-  vertical-align: middle;
-}
-.fa-stack-1x,
-.fa-stack-2x {
-  position: absolute;
-  left: 0;
-  width: 100%;
-  text-align: center;
-}
-.fa-stack-1x {
-  line-height: inherit;
-}
-.fa-stack-2x {
-  font-size: 2em;
-}
-.fa-inverse {
-  color: #ffffff;
-}
-/* Font Awesome uses the Unicode Private Use Area (PUA) to ensure screen
-   readers do not read off random characters that represent icons */
-.fa-glass:before {
-  content: "\f000";
-}
-.fa-music:before {
-  content: "\f001";
-}
-.fa-search:before {
-  content: "\f002";
-}
-.fa-envelope-o:before {
-  content: "\f003";
-}
-.fa-heart:before {
-  content: "\f004";
-}
-.fa-star:before {
-  content: "\f005";
-}
-.fa-star-o:before {
-  content: "\f006";
-}
-.fa-user:before {
-  content: "\f007";
-}
-.fa-film:before {
-  content: "\f008";
-}
-.fa-th-large:before {
-  content: "\f009";
-}
-.fa-th:before {
-  content: "\f00a";
-}
-.fa-th-list:before {
-  content: "\f00b";
-}
-.fa-check:before {
-  content: "\f00c";
-}
-.fa-remove:before,
-.fa-close:before,
-.fa-times:before {
-  content: "\f00d";
-}
-.fa-search-plus:before {
-  content: "\f00e";
-}
-.fa-search-minus:before {
-  content: "\f010";
-}
-.fa-power-off:before {
-  content: "\f011";
-}
-.fa-signal:before {
-  content: "\f012";
-}
-.fa-gear:before,
-.fa-cog:before {
-  content: "\f013";
-}
-.fa-trash-o:before {
-  content: "\f014";
-}
-.fa-home:before {
-  content: "\f015";
-}
-.fa-file-o:before {
-  content: "\f016";
-}
-.fa-clock-o:before {
-  content: "\f017";
-}
-.fa-road:before {
-  content: "\f018";
-}
-.fa-download:before {
-  content: "\f019";
-}
-.fa-arrow-circle-o-down:before {
-  content: "\f01a";
-}
-.fa-arrow-circle-o-up:before {
-  content: "\f01b";
-}
-.fa-inbox:before {
-  content: "\f01c";
-}
-.fa-play-circle-o:before {
-  content: "\f01d";
-}
-.fa-rotate-right:before,
-.fa-repeat:before {
-  content: "\f01e";
-}
-.fa-refresh:before {
-  content: "\f021";
-}
-.fa-list-alt:before {
-  content: "\f022";
-}
-.fa-lock:before {
-  content: "\f023";
-}
-.fa-flag:before {
-  content: "\f024";
-}
-.fa-headphones:before {
-  content: "\f025";
-}
-.fa-volume-off:before {
-  content: "\f026";
-}
-.fa-volume-down:before {
-  content: "\f027";
-}
-.fa-volume-up:before {
-  content: "\f028";
-}
-.fa-qrcode:before {
-  content: "\f029";
-}
-.fa-barcode:before {
-  content: "\f02a";
-}
-.fa-tag:before {
-  content: "\f02b";
-}
-.fa-tags:before {
-  content: "\f02c";
-}
-.fa-book:before {
-  content: "\f02d";
-}
-.fa-bookmark:before {
-  content: "\f02e";
-}
-.fa-print:before {
-  content: "\f02f";
-}
-.fa-camera:before {
-  content: "\f030";
-}
-.fa-font:before {
-  content: "\f031";
-}
-.fa-bold:before {
-  content: "\f032";
-}
-.fa-italic:before {
-  content: "\f033";
-}
-.fa-text-height:before {
-  content: "\f034";
-}
-.fa-text-width:before {
-  content: "\f035";
-}
-.fa-align-left:before {
-  content: "\f036";
-}
-.fa-align-center:before {
-  content: "\f037";
-}
-.fa-align-right:before {
-  content: "\f038";
-}
-.fa-align-justify:before {
-  content: "\f039";
-}
-.fa-list:before {
-  content: "\f03a";
-}
-.fa-dedent:before,
-.fa-outdent:before {
-  content: "\f03b";
-}
-.fa-indent:before {
-  content: "\f03c";
-}
-.fa-video-camera:before {
-  content: "\f03d";
-}
-.fa-photo:before,
-.fa-image:before,
-.fa-picture-o:before {
-  content: "\f03e";
-}
-.fa-pencil:before {
-  content: "\f040";
-}
-.fa-map-marker:before {
-  content: "\f041";
-}
-.fa-adjust:before {
-  content: "\f042";
-}
-.fa-tint:before {
-  content: "\f043";
-}
-.fa-edit:before,
-.fa-pencil-square-o:before {
-  content: "\f044";
-}
-.fa-share-square-o:before {
-  content: "\f045";
-}
-.fa-check-square-o:before {
-  content: "\f046";
-}
-.fa-arrows:before {
-  content: "\f047";
-}
-.fa-step-backward:before {
-  content: "\f048";
-}
-.fa-fast-backward:before {
-  content: "\f049";
-}
-.fa-backward:before {
-  content: "\f04a";
-}
-.fa-play:before {
-  content: "\f04b";
-}
-.fa-pause:before {
-  content: "\f04c";
-}
-.fa-stop:before {
-  content: "\f04d";
-}
-.fa-forward:before {
-  content: "\f04e";
-}
-.fa-fast-forward:before {
-  content: "\f050";
-}
-.fa-step-forward:before {
-  content: "\f051";
-}
-.fa-eject:before {
-  content: "\f052";
-}
-.fa-chevron-left:before {
-  content: "\f053";
-}
-.fa-chevron-right:before {
-  content: "\f054";
-}
-.fa-plus-circle:before {
-  content: "\f055";
-}
-.fa-minus-circle:before {
-  content: "\f056";
-}
-.fa-times-circle:before {
-  content: "\f057";
-}
-.fa-check-circle:before {
-  content: "\f058";
-}
-.fa-question-circle:before {
-  content: "\f059";
-}
-.fa-info-circle:before {
-  content: "\f05a";
-}
-.fa-crosshairs:before {
-  content: "\f05b";
-}
-.fa-times-circle-o:before {
-  content: "\f05c";
-}
-.fa-check-circle-o:before {
-  content: "\f05d";
-}
-.fa-ban:before {
-  content: "\f05e";
-}
-.fa-arrow-left:before {
-  content: "\f060";
-}
-.fa-arrow-right:before {
-  content: "\f061";
-}
-.fa-arrow-up:before {
-  content: "\f062";
-}
-.fa-arrow-down:before {
-  content: "\f063";
-}
-.fa-mail-forward:before,
-.fa-share:before {
-  content: "\f064";
-}
-.fa-expand:before {
-  content: "\f065";
-}
-.fa-compress:before {
-  content: "\f066";
-}
-.fa-plus:before {
-  content: "\f067";
-}
-.fa-minus:before {
-  content: "\f068";
-}
-.fa-asterisk:before {
-  content: "\f069";
-}
-.fa-exclamation-circle:before {
-  content: "\f06a";
-}
-.fa-gift:before {
-  content: "\f06b";
-}
-.fa-leaf:before {
-  content: "\f06c";
-}
-.fa-fire:before {
-  content: "\f06d";
-}
-.fa-eye:before {
-  content: "\f06e";
-}
-.fa-eye-slash:before {
-  content: "\f070";
-}
-.fa-warning:before,
-.fa-exclamation-triangle:before {
-  content: "\f071";
-}
-.fa-plane:before {
-  content: "\f072";
-}
-.fa-calendar:before {
-  content: "\f073";
-}
-.fa-random:before {
-  content: "\f074";
-}
-.fa-comment:before {
-  content: "\f075";
-}
-.fa-magnet:before {
-  content: "\f076";
-}
-.fa-chevron-up:before {
-  content: "\f077";
-}
-.fa-chevron-down:before {
-  content: "\f078";
-}
-.fa-retweet:before {
-  content: "\f079";
-}
-.fa-shopping-cart:before {
-  content: "\f07a";
-}
-.fa-folder:before {
-  content: "\f07b";
-}
-.fa-folder-open:before {
-  content: "\f07c";
-}
-.fa-arrows-v:before {
-  content: "\f07d";
-}
-.fa-arrows-h:before {
-  content: "\f07e";
-}
-.fa-bar-chart-o:before,
-.fa-bar-chart:before {
-  content: "\f080";
-}
-.fa-twitter-square:before {
-  content: "\f081";
-}
-.fa-facebook-square:before {
-  content: "\f082";
-}
-.fa-camera-retro:before {
-  content: "\f083";
-}
-.fa-key:before {
-  content: "\f084";
-}
-.fa-gears:before,
-.fa-cogs:before {
-  content: "\f085";
-}
-.fa-comments:before {
-  content: "\f086";
-}
-.fa-thumbs-o-up:before {
-  content: "\f087";
-}
-.fa-thumbs-o-down:before {
-  content: "\f088";
-}
-.fa-star-half:before {
-  content: "\f089";
-}
-.fa-heart-o:before {
-  content: "\f08a";
-}
-.fa-sign-out:before {
-  content: "\f08b";
-}
-.fa-linkedin-square:before {
-  content: "\f08c";
-}
-.fa-thumb-tack:before {
-  content: "\f08d";
-}
-.fa-external-link:before {
-  content: "\f08e";
-}
-.fa-sign-in:before {
-  content: "\f090";
-}
-.fa-trophy:before {
-  content: "\f091";
-}
-.fa-github-square:before {
-  content: "\f092";
-}
-.fa-upload:before {
-  content: "\f093";
-}
-.fa-lemon-o:before {
-  content: "\f094";
-}
-.fa-phone:before {
-  content: "\f095";
-}
-.fa-square-o:before {
-  content: "\f096";
-}
-.fa-bookmark-o:before {
-  content: "\f097";
-}
-.fa-phone-square:before {
-  content: "\f098";
-}
-.fa-twitter:before {
-  content: "\f099";
-}
-.fa-facebook-f:before,
-.fa-facebook:before {
-  content: "\f09a";
-}
-.fa-github:before {
-  content: "\f09b";
-}
-.fa-unlock:before {
-  content: "\f09c";
-}
-.fa-credit-card:before {
-  content: "\f09d";
-}
-.fa-feed:before,
-.fa-rss:before {
-  content: "\f09e";
-}
-.fa-hdd-o:before {
-  content: "\f0a0";
-}
-.fa-bullhorn:before {
-  content: "\f0a1";
-}
-.fa-bell:before {
-  content: "\f0f3";
-}
-.fa-certificate:before {
-  content: "\f0a3";
-}
-.fa-hand-o-right:before {
-  content: "\f0a4";
-}
-.fa-hand-o-left:before {
-  content: "\f0a5";
-}
-.fa-hand-o-up:before {
-  content: "\f0a6";
-}
-.fa-hand-o-down:before {
-  content: "\f0a7";
-}
-.fa-arrow-circle-left:before {
-  content: "\f0a8";
-}
-.fa-arrow-circle-right:before {
-  content: "\f0a9";
-}
-.fa-arrow-circle-up:before {
-  content: "\f0aa";
-}
-.fa-arrow-circle-down:before {
-  content: "\f0ab";
-}
-.fa-globe:before {
-  content: "\f0ac";
-}
-.fa-wrench:before {
-  content: "\f0ad";
-}
-.fa-tasks:before {
-  content: "\f0ae";
-}
-.fa-filter:before {
-  content: "\f0b0";
-}
-.fa-briefcase:before {
-  content: "\f0b1";
-}
-.fa-arrows-alt:before {
-  content: "\f0b2";
-}
-.fa-group:before,
-.fa-users:before {
-  content: "\f0c0";
-}
-.fa-chain:before,
-.fa-link:before {
-  content: "\f0c1";
-}
-.fa-cloud:before {
-  content: "\f0c2";
-}
-.fa-flask:before {
-  content: "\f0c3";
-}
-.fa-cut:before,
-.fa-scissors:before {
-  content: "\f0c4";
-}
-.fa-copy:before,
-.fa-files-o:before {
-  content: "\f0c5";
-}
-.fa-paperclip:before {
-  content: "\f0c6";
-}
-.fa-save:before,
-.fa-floppy-o:before {
-  content: "\f0c7";
-}
-.fa-square:before {
-  content: "\f0c8";
-}
-.fa-navicon:before,
-.fa-reorder:before,
-.fa-bars:before {
-  content: "\f0c9";
-}
-.fa-list-ul:before {
-  content: "\f0ca";
-}
-.fa-list-ol:before {
-  content: "\f0cb";
-}
-.fa-strikethrough:before {
-  content: "\f0cc";
-}
-.fa-underline:before {
-  content: "\f0cd";
-}
-.fa-table:before {
-  content: "\f0ce";
-}
-.fa-magic:before {
-  content: "\f0d0";
-}
-.fa-truck:before {
-  content: "\f0d1";
-}
-.fa-pinterest:before {
-  content: "\f0d2";
-}
-.fa-pinterest-square:before {
-  content: "\f0d3";
-}
-.fa-google-plus-square:before {
-  content: "\f0d4";
-}
-.fa-google-plus:before {
-  content: "\f0d5";
-}
-.fa-money:before {
-  content: "\f0d6";
-}
-.fa-caret-down:before {
-  content: "\f0d7";
-}
-.fa-caret-up:before {
-  content: "\f0d8";
-}
-.fa-caret-left:before {
-  content: "\f0d9";
-}
-.fa-caret-right:before {
-  content: "\f0da";
-}
-.fa-columns:before {
-  content: "\f0db";
-}
-.fa-unsorted:before,
-.fa-sort:before {
-  content: "\f0dc";
-}
-.fa-sort-down:before,
-.fa-sort-desc:before {
-  content: "\f0dd";
-}
-.fa-sort-up:before,
-.fa-sort-asc:before {
-  content: "\f0de";
-}
-.fa-envelope:before {
-  content: "\f0e0";
-}
-.fa-linkedin:before {
-  content: "\f0e1";
-}
-.fa-rotate-left:before,
-.fa-undo:before {
-  content: "\f0e2";
-}
-.fa-legal:before,
-.fa-gavel:before {
-  content: "\f0e3";
-}
-.fa-dashboard:before,
-.fa-tachometer:before {
-  content: "\f0e4";
-}
-.fa-comment-o:before {
-  content: "\f0e5";
-}
-.fa-comments-o:before {
-  content: "\f0e6";
-}
-.fa-flash:before,
-.fa-bolt:before {
-  content: "\f0e7";
-}
-.fa-sitemap:before {
-  content: "\f0e8";
-}
-.fa-umbrella:before {
-  content: "\f0e9";
-}
-.fa-paste:before,
-.fa-clipboard:before {
-  content: "\f0ea";
-}
-.fa-lightbulb-o:before {
-  content: "\f0eb";
-}
-.fa-exchange:before {
-  content: "\f0ec";
-}
-.fa-cloud-download:before {
-  content: "\f0ed";
-}
-.fa-cloud-upload:before {
-  content: "\f0ee";
-}
-.fa-user-md:before {
-  content: "\f0f0";
-}
-.fa-stethoscope:before {
-  content: "\f0f1";
-}
-.fa-suitcase:before {
-  content: "\f0f2";
-}
-.fa-bell-o:before {
-  content: "\f0a2";
-}
-.fa-coffee:before {
-  content: "\f0f4";
-}
-.fa-cutlery:before {
-  content: "\f0f5";
-}
-.fa-file-text-o:before {
-  content: "\f0f6";
-}
-.fa-building-o:before {
-  content: "\f0f7";
-}
-.fa-hospital-o:before {
-  content: "\f0f8";
-}
-.fa-ambulance:before {
-  content: "\f0f9";
-}
-.fa-medkit:before {
-  content: "\f0fa";
-}
-.fa-fighter-jet:before {
-  content: "\f0fb";
-}
-.fa-beer:before {
-  content: "\f0fc";
-}
-.fa-h-square:before {
-  content: "\f0fd";
-}
-.fa-plus-square:before {
-  content: "\f0fe";
-}
-.fa-angle-double-left:before {
-  content: "\f100";
-}
-.fa-angle-double-right:before {
-  content: "\f101";
-}
-.fa-angle-double-up:before {
-  content: "\f102";
-}
-.fa-angle-double-down:before {
-  content: "\f103";
-}
-.fa-angle-left:before {
-  content: "\f104";
-}
-.fa-angle-right:before {
-  content: "\f105";
-}
-.fa-angle-up:before {
-  content: "\f106";
-}
-.fa-angle-down:before {
-  content: "\f107";
-}
-.fa-desktop:before {
-  content: "\f108";
-}
-.fa-laptop:before {
-  content: "\f109";
-}
-.fa-tablet:before {
-  content: "\f10a";
-}
-.fa-mobile-phone:before,
-.fa-mobile:before {
-  content: "\f10b";
-}
-.fa-circle-o:before {
-  content: "\f10c";
-}
-.fa-quote-left:before {
-  content: "\f10d";
-}
-.fa-quote-right:before {
-  content: "\f10e";
-}
-.fa-spinner:before {
-  content: "\f110";
-}
-.fa-circle:before {
-  content: "\f111";
-}
-.fa-mail-reply:before,
-.fa-reply:before {
-  content: "\f112";
-}
-.fa-github-alt:before {
-  content: "\f113";
-}
-.fa-folder-o:before {
-  content: "\f114";
-}
-.fa-folder-open-o:before {
-  content: "\f115";
-}
-.fa-smile-o:before {
-  content: "\f118";
-}
-.fa-frown-o:before {
-  content: "\f119";
-}
-.fa-meh-o:before {
-  content: "\f11a";
-}
-.fa-gamepad:before {
-  content: "\f11b";
-}
-.fa-keyboard-o:before {
-  content: "\f11c";
-}
-.fa-flag-o:before {
-  content: "\f11d";
-}
-.fa-flag-checkered:before {
-  content: "\f11e";
-}
-.fa-terminal:before {
-  content: "\f120";
-}
-.fa-code:before {
-  content: "\f121";
-}
-.fa-mail-reply-all:before,
-.fa-reply-all:before {
-  content: "\f122";
-}
-.fa-star-half-empty:before,
-.fa-star-half-full:before,
-.fa-star-half-o:before {
-  content: "\f123";
-}
-.fa-location-arrow:before {
-  content: "\f124";
-}
-.fa-crop:before {
-  content: "\f125";
-}
-.fa-code-fork:before {
-  content: "\f126";
-}
-.fa-unlink:before,
-.fa-chain-broken:before {
-  content: "\f127";
-}
-.fa-question:before {
-  content: "\f128";
-}
-.fa-info:before {
-  content: "\f129";
-}
-.fa-exclamation:before {
-  content: "\f12a";
-}
-.fa-superscript:before {
-  content: "\f12b";
-}
-.fa-subscript:before {
-  content: "\f12c";
-}
-.fa-eraser:before {
-  content: "\f12d";
-}
-.fa-puzzle-piece:before {
-  content: "\f12e";
-}
-.fa-microphone:before {
-  content: "\f130";
-}
-.fa-microphone-slash:before {
-  content: "\f131";
-}
-.fa-shield:before {
-  content: "\f132";
-}
-.fa-calendar-o:before {
-  content: "\f133";
-}
-.fa-fire-extinguisher:before {
-  content: "\f134";
-}
-.fa-rocket:before {
-  content: "\f135";
-}
-.fa-maxcdn:before {
-  content: "\f136";
-}
-.fa-chevron-circle-left:before {
-  content: "\f137";
-}
-.fa-chevron-circle-right:before {
-  content: "\f138";
-}
-.fa-chevron-circle-up:before {
-  content: "\f139";
-}
-.fa-chevron-circle-down:before {
-  content: "\f13a";
-}
-.fa-html5:before {
-  content: "\f13b";
-}
-.fa-css3:before {
-  content: "\f13c";
-}
-.fa-anchor:before {
-  content: "\f13d";
-}
-.fa-unlock-alt:before {
-  content: "\f13e";
-}
-.fa-bullseye:before {
-  content: "\f140";
-}
-.fa-ellipsis-h:before {
-  content: "\f141";
-}
-.fa-ellipsis-v:before {
-  content: "\f142";
-}
-.fa-rss-square:before {
-  content: "\f143";
-}
-.fa-play-circle:before {
-  content: "\f144";
-}
-.fa-ticket:before {
-  content: "\f145";
-}
-.fa-minus-square:before {
-  content: "\f146";
-}
-.fa-minus-square-o:before {
-  content: "\f147";
-}
-.fa-level-up:before {
-  content: "\f148";
-}
-.fa-level-down:before {
-  content: "\f149";
-}
-.fa-check-square:before {
-  content: "\f14a";
-}
-.fa-pencil-square:before {
-  content: "\f14b";
-}
-.fa-external-link-square:before {
-  content: "\f14c";
-}
-.fa-share-square:before {
-  content: "\f14d";
-}
-.fa-compass:before {
-  content: "\f14e";
-}
-.fa-toggle-down:before,
-.fa-caret-square-o-down:before {
-  content: "\f150";
-}
-.fa-toggle-up:before,
-.fa-caret-square-o-up:before {
-  content: "\f151";
-}
-.fa-toggle-right:before,
-.fa-caret-square-o-right:before {
-  content: "\f152";
-}
-.fa-euro:before,
-.fa-eur:before {
-  content: "\f153";
-}
-.fa-gbp:before {
-  content: "\f154";
-}
-.fa-dollar:before,
-.fa-usd:before {
-  content: "\f155";
-}
-.fa-rupee:before,
-.fa-inr:before {
-  content: "\f156";
-}
-.fa-cny:before,
-.fa-rmb:before,
-.fa-yen:before,
-.fa-jpy:before {
-  content: "\f157";
-}
-.fa-ruble:before,
-.fa-rouble:before,
-.fa-rub:before {
-  content: "\f158";
-}
-.fa-won:before,
-.fa-krw:before {
-  content: "\f159";
-}
-.fa-bitcoin:before,
-.fa-btc:before {
-  content: "\f15a";
-}
-.fa-file:before {
-  content: "\f15b";
-}
-.fa-file-text:before {
-  content: "\f15c";
-}
-.fa-sort-alpha-asc:before {
-  content: "\f15d";
-}
-.fa-sort-alpha-desc:before {
-  content: "\f15e";
-}
-.fa-sort-amount-asc:before {
-  content: "\f160";
-}
-.fa-sort-amount-desc:before {
-  content: "\f161";
-}
-.fa-sort-numeric-asc:before {
-  content: "\f162";
-}
-.fa-sort-numeric-desc:before {
-  content: "\f163";
-}
-.fa-thumbs-up:before {
-  content: "\f164";
-}
-.fa-thumbs-down:before {
-  content: "\f165";
-}
-.fa-youtube-square:before {
-  content: "\f166";
-}
-.fa-youtube:before {
-  content: "\f167";
-}
-.fa-xing:before {
-  content: "\f168";
-}
-.fa-xing-square:before {
-  content: "\f169";
-}
-.fa-youtube-play:before {
-  content: "\f16a";
-}
-.fa-dropbox:before {
-  content: "\f16b";
-}
-.fa-stack-overflow:before {
-  content: "\f16c";
-}
-.fa-instagram:before {
-  content: "\f16d";
-}
-.fa-flickr:before {
-  content: "\f16e";
-}
-.fa-adn:before {
-  content: "\f170";
-}
-.fa-bitbucket:before {
-  content: "\f171";
-}
-.fa-bitbucket-square:before {
-  content: "\f172";
-}
-.fa-tumblr:before {
-  content: "\f173";
-}
-.fa-tumblr-square:before {
-  content: "\f174";
-}
-.fa-long-arrow-down:before {
-  content: "\f175";
-}
-.fa-long-arrow-up:before {
-  content: "\f176";
-}
-.fa-long-arrow-left:before {
-  content: "\f177";
-}
-.fa-long-arrow-right:before {
-  content: "\f178";
-}
-.fa-apple:before {
-  content: "\f179";
-}
-.fa-windows:before {
-  content: "\f17a";
-}
-.fa-android:before {
-  content: "\f17b";
-}
-.fa-linux:before {
-  content: "\f17c";
-}
-.fa-dribbble:before {
-  content: "\f17d";
-}
-.fa-skype:before {
-  content: "\f17e";
-}
-.fa-foursquare:before {
-  content: "\f180";
-}
-.fa-trello:before {
-  content: "\f181";
-}
-.fa-female:before {
-  content: "\f182";
-}
-.fa-male:before {
-  content: "\f183";
-}
-.fa-gittip:before,
-.fa-gratipay:before {
-  content: "\f184";
-}
-.fa-sun-o:before {
-  content: "\f185";
-}
-.fa-moon-o:before {
-  content: "\f186";
-}
-.fa-archive:before {
-  content: "\f187";
-}
-.fa-bug:before {
-  content: "\f188";
-}
-.fa-vk:before {
-  content: "\f189";
-}
-.fa-weibo:before {
-  content: "\f18a";
-}
-.fa-renren:before {
-  content: "\f18b";
-}
-.fa-pagelines:before {
-  content: "\f18c";
-}
-.fa-stack-exchange:before {
-  content: "\f18d";
-}
-.fa-arrow-circle-o-right:before {
-  content: "\f18e";
-}
-.fa-arrow-circle-o-left:before {
-  content: "\f190";
-}
-.fa-toggle-left:before,
-.fa-caret-square-o-left:before {
-  content: "\f191";
-}
-.fa-dot-circle-o:before {
-  content: "\f192";
-}
-.fa-wheelchair:before {
-  content: "\f193";
-}
-.fa-vimeo-square:before {
-  content: "\f194";
-}
-.fa-turkish-lira:before,
-.fa-try:before {
-  content: "\f195";
-}
-.fa-plus-square-o:before {
-  content: "\f196";
-}
-.fa-space-shuttle:before {
-  content: "\f197";
-}
-.fa-slack:before {
-  content: "\f198";
-}
-.fa-envelope-square:before {
-  content: "\f199";
-}
-.fa-wordpress:before {
-  content: "\f19a";
-}
-.fa-openid:before {
-  content: "\f19b";
-}
-.fa-institution:before,
-.fa-bank:before,
-.fa-university:before {
-  content: "\f19c";
-}
-.fa-mortar-board:before,
-.fa-graduation-cap:before {
-  content: "\f19d";
-}
-.fa-yahoo:before {
-  content: "\f19e";
-}
-.fa-google:before {
-  content: "\f1a0";
-}
-.fa-reddit:before {
-  content: "\f1a1";
-}
-.fa-reddit-square:before {
-  content: "\f1a2";
-}
-.fa-stumbleupon-circle:before {
-  content: "\f1a3";
-}
-.fa-stumbleupon:before {
-  content: "\f1a4";
-}
-.fa-delicious:before {
-  content: "\f1a5";
-}
-.fa-digg:before {
-  content: "\f1a6";
-}
-.fa-pied-piper-pp:before {
-  content: "\f1a7";
-}
-.fa-pied-piper-alt:before {
-  content: "\f1a8";
-}
-.fa-drupal:before {
-  content: "\f1a9";
-}
-.fa-joomla:before {
-  content: "\f1aa";
-}
-.fa-language:before {
-  content: "\f1ab";
-}
-.fa-fax:before {
-  content: "\f1ac";
-}
-.fa-building:before {
-  content: "\f1ad";
-}
-.fa-child:before {
-  content: "\f1ae";
-}
-.fa-paw:before {
-  content: "\f1b0";
-}
-.fa-spoon:before {
-  content: "\f1b1";
-}
-.fa-cube:before {
-  content: "\f1b2";
-}
-.fa-cubes:before {
-  content: "\f1b3";
-}
-.fa-behance:before {
-  content: "\f1b4";
-}
-.fa-behance-square:before {
-  content: "\f1b5";
-}
-.fa-steam:before {
-  content: "\f1b6";
-}
-.fa-steam-square:before {
-  content: "\f1b7";
-}
-.fa-recycle:before {
-  content: "\f1b8";
-}
-.fa-automobile:before,
-.fa-car:before {
-  content: "\f1b9";
-}
-.fa-cab:before,
-.fa-taxi:before {
-  content: "\f1ba";
-}
-.fa-tree:before {
-  content: "\f1bb";
-}
-.fa-spotify:before {
-  content: "\f1bc";
-}
-.fa-deviantart:before {
-  content: "\f1bd";
-}
-.fa-soundcloud:before {
-  content: "\f1be";
-}
-.fa-database:before {
-  content: "\f1c0";
-}
-.fa-file-pdf-o:before {
-  content: "\f1c1";
-}
-.fa-file-word-o:before {
-  content: "\f1c2";
-}
-.fa-file-excel-o:before {
-  content: "\f1c3";
-}
-.fa-file-powerpoint-o:before {
-  content: "\f1c4";
-}
-.fa-file-photo-o:before,
-.fa-file-picture-o:before,
-.fa-file-image-o:before {
-  content: "\f1c5";
-}
-.fa-file-zip-o:before,
-.fa-file-archive-o:before {
-  content: "\f1c6";
-}
-.fa-file-sound-o:before,
-.fa-file-audio-o:before {
-  content: "\f1c7";
-}
-.fa-file-movie-o:before,
-.fa-file-video-o:before {
-  content: "\f1c8";
-}
-.fa-file-code-o:before {
-  content: "\f1c9";
-}
-.fa-vine:before {
-  content: "\f1ca";
-}
-.fa-codepen:before {
-  content: "\f1cb";
-}
-.fa-jsfiddle:before {
-  content: "\f1cc";
-}
-.fa-life-bouy:before,
-.fa-life-buoy:before,
-.fa-life-saver:before,
-.fa-support:before,
-.fa-life-ring:before {
-  content: "\f1cd";
-}
-.fa-circle-o-notch:before {
-  content: "\f1ce";
-}
-.fa-ra:before,
-.fa-resistance:before,
-.fa-rebel:before {
-  content: "\f1d0";
-}
-.fa-ge:before,
-.fa-empire:before {
-  content: "\f1d1";
-}
-.fa-git-square:before {
-  content: "\f1d2";
-}
-.fa-git:before {
-  content: "\f1d3";
-}
-.fa-y-combinator-square:before,
-.fa-yc-square:before,
-.fa-hacker-news:before {
-  content: "\f1d4";
-}
-.fa-tencent-weibo:before {
-  content: "\f1d5";
-}
-.fa-qq:before {
-  content: "\f1d6";
-}
-.fa-wechat:before,
-.fa-weixin:before {
-  content: "\f1d7";
-}
-.fa-send:before,
-.fa-paper-plane:before {
-  content: "\f1d8";
-}
-.fa-send-o:before,
-.fa-paper-plane-o:before {
-  content: "\f1d9";
-}
-.fa-history:before {
-  content: "\f1da";
-}
-.fa-circle-thin:before {
-  content: "\f1db";
-}
-.fa-header:before {
-  content: "\f1dc";
-}
-.fa-paragraph:before {
-  content: "\f1dd";
-}
-.fa-sliders:before {
-  content: "\f1de";
-}
-.fa-share-alt:before {
-  content: "\f1e0";
-}
-.fa-share-alt-square:before {
-  content: "\f1e1";
-}
-.fa-bomb:before {
-  content: "\f1e2";
-}
-.fa-soccer-ball-o:before,
-.fa-futbol-o:before {
-  content: "\f1e3";
-}
-.fa-tty:before {
-  content: "\f1e4";
-}
-.fa-binoculars:before {
-  content: "\f1e5";
-}
-.fa-plug:before {
-  content: "\f1e6";
-}
-.fa-slideshare:before {
-  content: "\f1e7";
-}
-.fa-twitch:before {
-  content: "\f1e8";
-}
-.fa-yelp:before {
-  content: "\f1e9";
-}
-.fa-newspaper-o:before {
-  content: "\f1ea";
-}
-.fa-wifi:before {
-  content: "\f1eb";
-}
-.fa-calculator:before {
-  content: "\f1ec";
-}
-.fa-paypal:before {
-  content: "\f1ed";
-}
-.fa-google-wallet:before {
-  content: "\f1ee";
-}
-.fa-cc-visa:before {
-  content: "\f1f0";
-}
-.fa-cc-mastercard:before {
-  content: "\f1f1";
-}
-.fa-cc-discover:before {
-  content: "\f1f2";
-}
-.fa-cc-amex:before {
-  content: "\f1f3";
-}
-.fa-cc-paypal:before {
-  content: "\f1f4";
-}
-.fa-cc-stripe:before {
-  content: "\f1f5";
-}
-.fa-bell-slash:before {
-  content: "\f1f6";
-}
-.fa-bell-slash-o:before {
-  content: "\f1f7";
-}
-.fa-trash:before {
-  content: "\f1f8";
-}
-.fa-copyright:before {
-  content: "\f1f9";
-}
-.fa-at:before {
-  content: "\f1fa";
-}
-.fa-eyedropper:before {
-  content: "\f1fb";
-}
-.fa-paint-brush:before {
-  content: "\f1fc";
-}
-.fa-birthday-cake:before {
-  content: "\f1fd";
-}
-.fa-area-chart:before {
-  content: "\f1fe";
-}
-.fa-pie-chart:before {
-  content: "\f200";
-}
-.fa-line-chart:before {
-  content: "\f201";
-}
-.fa-lastfm:before {
-  content: "\f202";
-}
-.fa-lastfm-square:before {
-  content: "\f203";
-}
-.fa-toggle-off:before {
-  content: "\f204";
-}
-.fa-toggle-on:before {
-  content: "\f205";
-}
-.fa-bicycle:before {
-  content: "\f206";
-}
-.fa-bus:before {
-  content: "\f207";
-}
-.fa-ioxhost:before {
-  content: "\f208";
-}
-.fa-angellist:before {
-  content: "\f209";
-}
-.fa-cc:before {
-  content: "\f20a";
-}
-.fa-shekel:before,
-.fa-sheqel:before,
-.fa-ils:before {
-  content: "\f20b";
-}
-.fa-meanpath:before {
-  content: "\f20c";
-}
-.fa-buysellads:before {
-  content: "\f20d";
-}
-.fa-connectdevelop:before {
-  content: "\f20e";
-}
-.fa-dashcube:before {
-  content: "\f210";
-}
-.fa-forumbee:before {
-  content: "\f211";
-}
-.fa-leanpub:before {
-  content: "\f212";
-}
-.fa-sellsy:before {
-  content: "\f213";
-}
-.fa-shirtsinbulk:before {
-  content: "\f214";
-}
-.fa-simplybuilt:before {
-  content: "\f215";
-}
-.fa-skyatlas:before {
-  content: "\f216";
-}
-.fa-cart-plus:before {
-  content: "\f217";
-}
-.fa-cart-arrow-down:before {
-  content: "\f218";
-}
-.fa-diamond:before {
-  content: "\f219";
-}
-.fa-ship:before {
-  content: "\f21a";
-}
-.fa-user-secret:before {
-  content: "\f21b";
-}
-.fa-motorcycle:before {
-  content: "\f21c";
-}
-.fa-street-view:before {
-  content: "\f21d";
-}
-.fa-heartbeat:before {
-  content: "\f21e";
-}
-.fa-venus:before {
-  content: "\f221";
-}
-.fa-mars:before {
-  content: "\f222";
-}
-.fa-mercury:before {
-  content: "\f223";
-}
-.fa-intersex:before,
-.fa-transgender:before {
-  content: "\f224";
-}
-.fa-transgender-alt:before {
-  content: "\f225";
-}
-.fa-venus-double:before {
-  content: "\f226";
-}
-.fa-mars-double:before {
-  content: "\f227";
-}
-.fa-venus-mars:before {
-  content: "\f228";
-}
-.fa-mars-stroke:before {
-  content: "\f229";
-}
-.fa-mars-stroke-v:before {
-  content: "\f22a";
-}
-.fa-mars-stroke-h:before {
-  content: "\f22b";
-}
-.fa-neuter:before {
-  content: "\f22c";
-}
-.fa-genderless:before {
-  content: "\f22d";
-}
-.fa-facebook-official:before {
-  content: "\f230";
-}
-.fa-pinterest-p:before {
-  content: "\f231";
-}
-.fa-whatsapp:before {
-  content: "\f232";
-}
-.fa-server:before {
-  content: "\f233";
-}
-.fa-user-plus:before {
-  content: "\f234";
-}
-.fa-user-times:before {
-  content: "\f235";
-}
-.fa-hotel:before,
-.fa-bed:before {
-  content: "\f236";
-}
-.fa-viacoin:before {
-  content: "\f237";
-}
-.fa-train:before {
-  content: "\f238";
-}
-.fa-subway:before {
-  content: "\f239";
-}
-.fa-medium:before {
-  content: "\f23a";
-}
-.fa-yc:before,
-.fa-y-combinator:before {
-  content: "\f23b";
-}
-.fa-optin-monster:before {
-  content: "\f23c";
-}
-.fa-opencart:before {
-  content: "\f23d";
-}
-.fa-expeditedssl:before {
-  content: "\f23e";
-}
-.fa-battery-4:before,
-.fa-battery:before,
-.fa-battery-full:before {
-  content: "\f240";
-}
-.fa-battery-3:before,
-.fa-battery-three-quarters:before {
-  content: "\f241";
-}
-.fa-battery-2:before,
-.fa-battery-half:before {
-  content: "\f242";
-}
-.fa-battery-1:before,
-.fa-battery-quarter:before {
-  content: "\f243";
-}
-.fa-battery-0:before,
-.fa-battery-empty:before {
-  content: "\f244";
-}
-.fa-mouse-pointer:before {
-  content: "\f245";
-}
-.fa-i-cursor:before {
-  content: "\f246";
-}
-.fa-object-group:before {
-  content: "\f247";
-}
-.fa-object-ungroup:before {
-  content: "\f248";
-}
-.fa-sticky-note:before {
-  content: "\f249";
-}
-.fa-sticky-note-o:before {
-  content: "\f24a";
-}
-.fa-cc-jcb:before {
-  content: "\f24b";
-}
-.fa-cc-diners-club:before {
-  content: "\f24c";
-}
-.fa-clone:before {
-  content: "\f24d";
-}
-.fa-balance-scale:before {
-  content: "\f24e";
-}
-.fa-hourglass-o:before {
-  content: "\f250";
-}
-.fa-hourglass-1:before,
-.fa-hourglass-start:before {
-  content: "\f251";
-}
-.fa-hourglass-2:before,
-.fa-hourglass-half:before {
-  content: "\f252";
-}
-.fa-hourglass-3:before,
-.fa-hourglass-end:before {
-  content: "\f253";
-}
-.fa-hourglass:before {
-  content: "\f254";
-}
-.fa-hand-grab-o:before,
-.fa-hand-rock-o:before {
-  content: "\f255";
-}
-.fa-hand-stop-o:before,
-.fa-hand-paper-o:before {
-  content: "\f256";
-}
-.fa-hand-scissors-o:before {
-  content: "\f257";
-}
-.fa-hand-lizard-o:before {
-  content: "\f258";
-}
-.fa-hand-spock-o:before {
-  content: "\f259";
-}
-.fa-hand-pointer-o:before {
-  content: "\f25a";
-}
-.fa-hand-peace-o:before {
-  content: "\f25b";
-}
-.fa-trademark:before {
-  content: "\f25c";
-}
-.fa-registered:before {
-  content: "\f25d";
-}
-.fa-creative-commons:before {
-  content: "\f25e";
-}
-.fa-gg:before {
-  content: "\f260";
-}
-.fa-gg-circle:before {
-  content: "\f261";
-}
-.fa-tripadvisor:before {
-  content: "\f262";
-}
-.fa-odnoklassniki:before {
-  content: "\f263";
-}
-.fa-odnoklassniki-square:before {
-  content: "\f264";
-}
-.fa-get-pocket:before {
-  content: "\f265";
-}
-.fa-wikipedia-w:before {
-  content: "\f266";
-}
-.fa-safari:before {
-  content: "\f267";
-}
-.fa-chrome:before {
-  content: "\f268";
-}
-.fa-firefox:before {
-  content: "\f269";
-}
-.fa-opera:before {
-  content: "\f26a";
-}
-.fa-internet-explorer:before {
-  content: "\f26b";
-}
-.fa-tv:before,
-.fa-television:before {
-  content: "\f26c";
-}
-.fa-contao:before {
-  content: "\f26d";
-}
-.fa-500px:before {
-  content: "\f26e";
-}
-.fa-amazon:before {
-  content: "\f270";
-}
-.fa-calendar-plus-o:before {
-  content: "\f271";
-}
-.fa-calendar-minus-o:before {
-  content: "\f272";
-}
-.fa-calendar-times-o:before {
-  content: "\f273";
-}
-.fa-calendar-check-o:before {
-  content: "\f274";
-}
-.fa-industry:before {
-  content: "\f275";
-}
-.fa-map-pin:before {
-  content: "\f276";
-}
-.fa-map-signs:before {
-  content: "\f277";
-}
-.fa-map-o:before {
-  content: "\f278";
-}
-.fa-map:before {
-  content: "\f279";
-}
-.fa-commenting:before {
-  content: "\f27a";
-}
-.fa-commenting-o:before {
-  content: "\f27b";
-}
-.fa-houzz:before {
-  content: "\f27c";
-}
-.fa-vimeo:before {
-  content: "\f27d";
-}
-.fa-black-tie:before {
-  content: "\f27e";
-}
-.fa-fonticons:before {
-  content: "\f280";
-}
-.fa-reddit-alien:before {
-  content: "\f281";
-}
-.fa-edge:before {
-  content: "\f282";
-}
-.fa-credit-card-alt:before {
-  content: "\f283";
-}
-.fa-codiepie:before {
-  content: "\f284";
-}
-.fa-modx:before {
-  content: "\f285";
-}
-.fa-fort-awesome:before {
-  content: "\f286";
-}
-.fa-usb:before {
-  content: "\f287";
-}
-.fa-product-hunt:before {
-  content: "\f288";
-}
-.fa-mixcloud:before {
-  content: "\f289";
-}
-.fa-scribd:before {
-  content: "\f28a";
-}
-.fa-pause-circle:before {
-  content: "\f28b";
-}
-.fa-pause-circle-o:before {
-  content: "\f28c";
-}
-.fa-stop-circle:before {
-  content: "\f28d";
-}
-.fa-stop-circle-o:before {
-  content: "\f28e";
-}
-.fa-shopping-bag:before {
-  content: "\f290";
-}
-.fa-shopping-basket:before {
-  content: "\f291";
-}
-.fa-hashtag:before {
-  content: "\f292";
-}
-.fa-bluetooth:before {
-  content: "\f293";
-}
-.fa-bluetooth-b:before {
-  content: "\f294";
-}
-.fa-percent:before {
-  content: "\f295";
-}
-.fa-gitlab:before {
-  content: "\f296";
-}
-.fa-wpbeginner:before {
-  content: "\f297";
-}
-.fa-wpforms:before {
-  content: "\f298";
-}
-.fa-envira:before {
-  content: "\f299";
-}
-.fa-universal-access:before {
-  content: "\f29a";
-}
-.fa-wheelchair-alt:before {
-  content: "\f29b";
-}
-.fa-question-circle-o:before {
-  content: "\f29c";
-}
-.fa-blind:before {
-  content: "\f29d";
-}
-.fa-audio-description:before {
-  content: "\f29e";
-}
-.fa-volume-control-phone:before {
-  content: "\f2a0";
-}
-.fa-braille:before {
-  content: "\f2a1";
-}
-.fa-assistive-listening-systems:before {
-  content: "\f2a2";
-}
-.fa-asl-interpreting:before,
-.fa-american-sign-language-interpreting:before {
-  content: "\f2a3";
-}
-.fa-deafness:before,
-.fa-hard-of-hearing:before,
-.fa-deaf:before {
-  content: "\f2a4";
-}
-.fa-glide:before {
-  content: "\f2a5";
-}
-.fa-glide-g:before {
-  content: "\f2a6";
-}
-.fa-signing:before,
-.fa-sign-language:before {
-  content: "\f2a7";
-}
-.fa-low-vision:before {
-  content: "\f2a8";
-}
-.fa-viadeo:before {
-  content: "\f2a9";
-}
-.fa-viadeo-square:before {
-  content: "\f2aa";
-}
-.fa-snapchat:before {
-  content: "\f2ab";
-}
-.fa-snapchat-ghost:before {
-  content: "\f2ac";
-}
-.fa-snapchat-square:before {
-  content: "\f2ad";
-}
-.fa-pied-piper:before {
-  content: "\f2ae";
-}
-.fa-first-order:before {
-  content: "\f2b0";
-}
-.fa-yoast:before {
-  content: "\f2b1";
-}
-.fa-themeisle:before {
-  content: "\f2b2";
-}
-.fa-google-plus-circle:before,
-.fa-google-plus-official:before {
-  content: "\f2b3";
-}
-.fa-fa:before,
-.fa-font-awesome:before {
-  content: "\f2b4";
-}
-.fa-handshake-o:before {
-  content: "\f2b5";
-}
-.fa-envelope-open:before {
-  content: "\f2b6";
-}
-.fa-envelope-open-o:before {
-  content: "\f2b7";
-}
-.fa-linode:before {
-  content: "\f2b8";
-}
-.fa-address-book:before {
-  content: "\f2b9";
-}
-.fa-address-book-o:before {
-  content: "\f2ba";
-}
-.fa-vcard:before,
-.fa-address-card:before {
-  content: "\f2bb";
-}
-.fa-vcard-o:before,
-.fa-address-card-o:before {
-  content: "\f2bc";
-}
-.fa-user-circle:before {
-  content: "\f2bd";
-}
-.fa-user-circle-o:before {
-  content: "\f2be";
-}
-.fa-user-o:before {
-  content: "\f2c0";
-}
-.fa-id-badge:before {
-  content: "\f2c1";
-}
-.fa-drivers-license:before,
-.fa-id-card:before {
-  content: "\f2c2";
-}
-.fa-drivers-license-o:before,
-.fa-id-card-o:before {
-  content: "\f2c3";
-}
-.fa-quora:before {
-  content: "\f2c4";
-}
-.fa-free-code-camp:before {
-  content: "\f2c5";
-}
-.fa-telegram:before {
-  content: "\f2c6";
-}
-.fa-thermometer-4:before,
-.fa-thermometer:before,
-.fa-thermometer-full:before {
-  content: "\f2c7";
-}
-.fa-thermometer-3:before,
-.fa-thermometer-three-quarters:before {
-  content: "\f2c8";
-}
-.fa-thermometer-2:before,
-.fa-thermometer-half:before {
-  content: "\f2c9";
-}
-.fa-thermometer-1:before,
-.fa-thermometer-quarter:before {
-  content: "\f2ca";
-}
-.fa-thermometer-0:before,
-.fa-thermometer-empty:before {
-  content: "\f2cb";
-}
-.fa-shower:before {
-  content: "\f2cc";
-}
-.fa-bathtub:before,
-.fa-s15:before,
-.fa-bath:before {
-  content: "\f2cd";
-}
-.fa-podcast:before {
-  content: "\f2ce";
-}
-.fa-window-maximize:before {
-  content: "\f2d0";
-}
-.fa-window-minimize:before {
-  content: "\f2d1";
-}
-.fa-window-restore:before {
-  content: "\f2d2";
-}
-.fa-times-rectangle:before,
-.fa-window-close:before {
-  content: "\f2d3";
-}
-.fa-times-rectangle-o:before,
-.fa-window-close-o:before {
-  content: "\f2d4";
-}
-.fa-bandcamp:before {
-  content: "\f2d5";
-}
-.fa-grav:before {
-  content: "\f2d6";
-}
-.fa-etsy:before {
-  content: "\f2d7";
-}
-.fa-imdb:before {
-  content: "\f2d8";
-}
-.fa-ravelry:before {
-  content: "\f2d9";
-}
-.fa-eercast:before {
-  content: "\f2da";
-}
-.fa-microchip:before {
-  content: "\f2db";
-}
-.fa-snowflake-o:before {
-  content: "\f2dc";
-}
-.fa-superpowers:before {
-  content: "\f2dd";
-}
-.fa-wpexplorer:before {
-  content: "\f2de";
-}
-.fa-meetup:before {
-  content: "\f2e0";
-}
-.sr-only {
-  position: absolute;
-  width: 1px;
-  height: 1px;
-  padding: 0;
-  margin: -1px;
-  overflow: hidden;
-  clip: rect(0, 0, 0, 0);
-  border: 0;
-}
-.sr-only-focusable:active,
-.sr-only-focusable:focus {
-  position: static;
-  width: auto;
-  height: auto;
-  margin: 0;
-  overflow: visible;
-  clip: auto;
-}
diff --git a/frappe/public/css/fonts/fontawesome/LICENSE b/frappe/public/css/fonts/fontawesome/LICENSE
new file mode 100644
index 0000000000..304228182c
--- /dev/null
+++ b/frappe/public/css/fonts/fontawesome/LICENSE
@@ -0,0 +1,4 @@
+The Font Awesome font is licensed under the SIL OFL 1.1:
+http://scripts.sil.org/OFL
+Font Awesome CSS, LESS, and Sass files are licensed under the MIT License:
+https://opensource.org/licenses/mit-license.html
diff --git a/frappe/public/css/font-awesome.min.css b/frappe/public/css/fonts/fontawesome/font-awesome.min.css
similarity index 100%
rename from frappe/public/css/font-awesome.min.css
rename to frappe/public/css/fonts/fontawesome/font-awesome.min.css
diff --git a/frappe/public/css/fonts/glyphicons/glyphicons-halflings-regular.eot b/frappe/public/css/fonts/glyphicons/glyphicons-halflings-regular.eot
deleted file mode 100644
index b93a4953ff..0000000000
Binary files a/frappe/public/css/fonts/glyphicons/glyphicons-halflings-regular.eot and /dev/null differ
diff --git a/frappe/public/css/fonts/glyphicons/glyphicons-halflings-regular.svg b/frappe/public/css/fonts/glyphicons/glyphicons-halflings-regular.svg
deleted file mode 100644
index 94fb5490a2..0000000000
--- a/frappe/public/css/fonts/glyphicons/glyphicons-halflings-regular.svg
+++ /dev/null
@@ -1,288 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- 
\ No newline at end of file
diff --git a/frappe/public/css/fonts/glyphicons/glyphicons-halflings-regular.ttf b/frappe/public/css/fonts/glyphicons/glyphicons-halflings-regular.ttf
deleted file mode 100644
index 1413fc609a..0000000000
Binary files a/frappe/public/css/fonts/glyphicons/glyphicons-halflings-regular.ttf and /dev/null differ
diff --git a/frappe/public/css/fonts/glyphicons/glyphicons-halflings-regular.woff b/frappe/public/css/fonts/glyphicons/glyphicons-halflings-regular.woff
deleted file mode 100644
index 9e612858f8..0000000000
Binary files a/frappe/public/css/fonts/glyphicons/glyphicons-halflings-regular.woff and /dev/null differ
diff --git a/frappe/public/css/fonts/glyphicons/glyphicons-halflings-regular.woff2 b/frappe/public/css/fonts/glyphicons/glyphicons-halflings-regular.woff2
deleted file mode 100644
index 64539b54c3..0000000000
Binary files a/frappe/public/css/fonts/glyphicons/glyphicons-halflings-regular.woff2 and /dev/null differ
diff --git a/frappe/public/css/fonts/inter/LICENSE.txt b/frappe/public/css/fonts/inter/LICENSE.txt
new file mode 100644
index 0000000000..65ec0f9103
--- /dev/null
+++ b/frappe/public/css/fonts/inter/LICENSE.txt
@@ -0,0 +1,94 @@
+Copyright (c) 2016-2020 The Inter Project Authors.
+"Inter" is trademark of Rasmus Andersson.
+https://github.com/rsms/inter
+
+This Font Software is licensed under the SIL Open Font License, Version 1.1.
+This license is copied below, and is also available with a FAQ at:
+http://scripts.sil.org/OFL
+
+-----------------------------------------------------------
+SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007
+-----------------------------------------------------------
+
+PREAMBLE
+The goals of the Open Font License (OFL) are to stimulate worldwide
+development of collaborative font projects, to support the font creation
+efforts of academic and linguistic communities, and to provide a free and
+open framework in which fonts may be shared and improved in partnership
+with others.
+
+The OFL allows the licensed fonts to be used, studied, modified and
+redistributed freely as long as they are not sold by themselves. The
+fonts, including any derivative works, can be bundled, embedded,
+redistributed and/or sold with any software provided that any reserved
+names are not used by derivative works. The fonts and derivatives,
+however, cannot be released under any other type of license. The
+requirement for fonts to remain under this license does not apply
+to any document created using the fonts or their derivatives.
+
+DEFINITIONS
+"Font Software" refers to the set of files released by the Copyright
+Holder(s) under this license and clearly marked as such. This may
+include source files, build scripts and documentation.
+
+"Reserved Font Name" refers to any names specified as such after the
+copyright statement(s).
+
+"Original Version" refers to the collection of Font Software components as
+distributed by the Copyright Holder(s).
+
+"Modified Version" refers to any derivative made by adding to, deleting,
+or substituting -- in part or in whole -- any of the components of the
+Original Version, by changing formats or by porting the Font Software to a
+new environment.
+
+"Author" refers to any designer, engineer, programmer, technical
+writer or other person who contributed to the Font Software.
+
+PERMISSION AND CONDITIONS
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of the Font Software, to use, study, copy, merge, embed, modify,
+redistribute, and sell modified and unmodified copies of the Font
+Software, subject to the following conditions:
+
+1) Neither the Font Software nor any of its individual components,
+in Original or Modified Versions, may be sold by itself.
+
+2) Original or Modified Versions of the Font Software may be bundled,
+redistributed and/or sold with any software, provided that each copy
+contains the above copyright notice and this license. These can be
+included either as stand-alone text files, human-readable headers or
+in the appropriate machine-readable metadata fields within text or
+binary files as long as those fields can be easily viewed by the user.
+
+3) No Modified Version of the Font Software may use the Reserved Font
+Name(s) unless explicit written permission is granted by the corresponding
+Copyright Holder. This restriction only applies to the primary font name as
+presented to the users.
+
+4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font
+Software shall not be used to promote, endorse or advertise any
+Modified Version, except to acknowledge the contribution(s) of the
+Copyright Holder(s) and the Author(s) or with their explicit written
+permission.
+
+5) The Font Software, modified or unmodified, in part or in whole,
+must be distributed entirely under this license, and must not be
+distributed under any other license. The requirement for fonts to
+remain under this license does not apply to any document created
+using the Font Software.
+
+TERMINATION
+This license becomes null and void if any of the above conditions are
+not met.
+
+DISCLAIMER
+THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
+OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE
+COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL
+DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM
+OTHER DEALINGS IN THE FONT SOFTWARE.
\ No newline at end of file
diff --git a/frappe/public/css/tree_grid.css b/frappe/public/css/tree_grid.css
index 8e23edfc0a..dc26510728 100644
--- a/frappe/public/css/tree_grid.css
+++ b/frappe/public/css/tree_grid.css
@@ -13,10 +13,3 @@
 	visibility: visible;
 }
 
-.toggle.expand {
-	background: url("/assets/frappe/js/lib/slickgrid/images/expand.gif") no-repeat center center;
-}
-
-.toggle.collapse {
-	background: url("/assets/frappe/js/lib/slickgrid/images/collapse.gif") no-repeat center center;
-}
diff --git a/frappe/public/icons/social/fair.svg b/frappe/public/icons/social/fair.svg
index 4cfc77a71f..60c00dc7af 100644
--- a/frappe/public/icons/social/fair.svg
+++ b/frappe/public/icons/social/fair.svg
@@ -4,11 +4,11 @@
 
 
 
-
+
 
 
 
 
-
+
 
 
diff --git a/frappe/public/icons/social/office_365.svg b/frappe/public/icons/social/office_365.svg
index d93167741d..77c3559408 100644
--- a/frappe/public/icons/social/office_365.svg
+++ b/frappe/public/icons/social/office_365.svg
@@ -35,11 +35,11 @@
     
         
             
-                
+                
                 
             
             
-                
+                
                 
             
             
diff --git a/frappe/public/icons/timeless/symbol-defs.svg b/frappe/public/icons/timeless/icons.svg
similarity index 97%
rename from frappe/public/icons/timeless/symbol-defs.svg
rename to frappe/public/icons/timeless/icons.svg
index bf4e02a7af..af63d0c8ff 100644
--- a/frappe/public/icons/timeless/symbol-defs.svg
+++ b/frappe/public/icons/timeless/icons.svg
@@ -2,11 +2,11 @@
 	
 		
 	
-	
+
 	
 		
 	
-	
+
 	
 		
 	
@@ -14,15 +14,15 @@
 	
 		
 	
-	
+
 	
 		
 	
-	
+
 	
 		
 	
-	
+
 	
 		
 		
@@ -52,22 +52,22 @@
 		
 		
 	
-	
+
 	
 		
 		
 		
 		
 	
-	
+
 	
 		
 	
-	
+
 	
 		
 	
-	
+
 	
 		
 		
@@ -77,7 +77,7 @@
 		
 		
 	
-	
+
 	
 		
 	
@@ -95,19 +95,19 @@
 		
 		
 	
-	
+
 	
 		
 		
 		
 		
 	
-	
+
 	
 		
 		
 	
-	
+
 	
 		
 	
@@ -303,7 +303,7 @@
 		
 		
 	
-	
+
 	
 		
 		
@@ -613,8 +613,8 @@
 	
 		
-		
+		
 	
 
 	
@@ -947,5 +947,8 @@
 			
 		
 	
+	
+		
+	
 
 
diff --git a/frappe/public/images/chat/wallpaper-default.jpg b/frappe/public/images/chat/wallpaper-default.jpg
deleted file mode 100644
index b7818e9bf0..0000000000
Binary files a/frappe/public/images/chat/wallpaper-default.jpg and /dev/null differ
diff --git a/frappe/public/images/chat/wallpaper-primary.jpg b/frappe/public/images/chat/wallpaper-primary.jpg
deleted file mode 100644
index 125fff145d..0000000000
Binary files a/frappe/public/images/chat/wallpaper-primary.jpg and /dev/null differ
diff --git a/frappe/public/images/help/style-settings-help.png b/frappe/public/images/help/style-settings-help.png
deleted file mode 100644
index cc49373b59..0000000000
Binary files a/frappe/public/images/help/style-settings-help.png and /dev/null differ
diff --git a/frappe/public/images/leaflet/spritesheet.svg b/frappe/public/images/leaflet/spritesheet.svg
index 3c00f30314..05feda62df 100644
--- a/frappe/public/images/leaflet/spritesheet.svg
+++ b/frappe/public/images/leaflet/spritesheet.svg
@@ -134,14 +134,14 @@
      transform="translate(120,0)"
      style="fill:#bbbbbb">
     
     
 						
        ${title}
        
 						
 					
        
 					
        
@@ -50,15 +50,14 @@ frappe.ui.Dialog = class Dialog extends frappe.ui.Dialog {
 	}
 
 	set_primary_action(label, click) {
-		this.$wrapper.find('.modal-footer').removeClass('hidden');
-		return super.set_primary_action(label, click)
-			.removeClass('hidden');
+		this.$wrapper.find(".modal-footer").removeClass("hidden");
+		return super.set_primary_action(label, click).removeClass("hidden");
 	}
 
 	make() {
 		super.make();
 		if (this.fields) {
-			this.$wrapper.find('.section-body').addClass('w-100');
+			this.$wrapper.find(".section-body").addClass("w-100");
 		}
 	}
 };
diff --git a/frappe/public/js/checkout.bundle.js b/frappe/public/js/checkout.bundle.js
deleted file mode 100644
index 954e838fa8..0000000000
--- a/frappe/public/js/checkout.bundle.js
+++ /dev/null
@@ -1 +0,0 @@
-import "./integrations/razorpay";
diff --git a/frappe/public/js/desk.bundle.js b/frappe/public/js/desk.bundle.js
index e056a34be2..6697c034bc 100644
--- a/frappe/public/js/desk.bundle.js
+++ b/frappe/public/js/desk.bundle.js
@@ -107,6 +107,4 @@ import "./frappe/utils/dashboard_utils.js";
 import "./frappe/ui/chart.js";
 import "./frappe/ui/datatable.js";
 import "./frappe/ui/driver.js";
-import "./frappe/ui/plyr.js";
-import "./frappe/barcode_scanner/index.js";
 import "./frappe/scanner";
diff --git a/frappe/public/js/form.bundle.js b/frappe/public/js/form.bundle.js
index 2719535599..f2d19f5f3b 100644
--- a/frappe/public/js/form.bundle.js
+++ b/frappe/public/js/form.bundle.js
@@ -10,8 +10,7 @@ import "./frappe/form/templates/set_sharing.html";
 import "./frappe/form/templates/timeline_message_box.html";
 import "./frappe/form/templates/users_in_sidebar.html";
 
-import "./frappe/form/controls/control.js";
 import "./frappe/views/formview.js";
 import "./frappe/form/form.js";
 import "./frappe/meta_tag.js";
-import "./frappe/doctype/"
+import "./frappe/doctype/";
diff --git a/frappe/public/js/frappe-web.bundle.js b/frappe/public/js/frappe-web.bundle.js
index b8d4006090..e2098dd56c 100644
--- a/frappe/public/js/frappe-web.bundle.js
+++ b/frappe/public/js/frappe-web.bundle.js
@@ -1,15 +1,15 @@
 import "./jquery-bootstrap";
 import "./frappe/class.js";
 import "./frappe/polyfill.js";
-import "./lib/md5.min.js";
 import "./lib/moment.js";
 import "./frappe/provide.js";
+import "./frappe/translate.js";
+import "./frappe/form/formatters.js";
 import "./frappe/format.js";
 import "./frappe/utils/number_format.js";
 import "./frappe/utils/utils.js";
 import "./frappe/utils/common.js";
 import "./frappe/ui/messages.js";
-import "./frappe/translate.js";
 import "./frappe/utils/pretty_date.js";
 import "./frappe/utils/datetime.js";
 import "./frappe/microtemplate.js";
@@ -23,6 +23,5 @@ import "./frappe/model/perm.js";
 
 import "./bootstrap-4-web.bundle";
 
-
 import "../../website/js/website.js";
 import "./frappe/socketio_client.js";
diff --git a/frappe/public/js/frappe/assets.js b/frappe/public/js/frappe/assets.js
index 0a3fb33cb8..b8af761ee8 100644
--- a/frappe/public/js/frappe/assets.js
+++ b/frappe/public/js/frappe/assets.js
@@ -5,13 +5,13 @@
 // will try and get from localStorage if latest are available
 // depends on frappe.versions to manage versioning
 
-frappe.require = function(items, callback) {
-	if(typeof items === "string") {
+frappe.require = function (items, callback) {
+	if (typeof items === "string") {
 		items = [items];
 	}
-	items = items.map(item => frappe.assets.bundled_asset(item));
+	items = items.map((item) => frappe.assets.bundled_asset(item));
 
-	return new Promise(resolve => {
+	return new Promise((resolve) => {
 		frappe.assets.execute(items, () => {
 			resolve();
 			callback && callback();
@@ -20,16 +20,16 @@ frappe.require = function(items, callback) {
 };
 
 frappe.assets = {
-	check: function() {
+	check: function () {
 		// if version is different then clear localstorage
-		if(window._version_number != localStorage.getItem("_version_number")) {
+		if (window._version_number != localStorage.getItem("_version_number")) {
 			frappe.assets.clear_local_storage();
 			console.log("Cleared App Cache.");
 		}
 
-		if(localStorage._last_load) {
+		if (localStorage._last_load) {
 			var not_updated_since = new Date() - new Date(localStorage._last_load);
-			if(not_updated_since < 10000 || not_updated_since > 86400000) {
+			if (not_updated_since < 10000 || not_updated_since > 86400000) {
 				frappe.assets.clear_local_storage();
 			}
 		} else {
@@ -39,42 +39,47 @@ frappe.assets = {
 		frappe.assets.init_local_storage();
 	},
 
-	init_local_storage: function() {
+	init_local_storage: function () {
 		localStorage._last_load = new Date();
 		localStorage._version_number = window._version_number;
-		if(frappe.boot) localStorage.metadata_version = frappe.boot.metadata_version;
+		if (frappe.boot) localStorage.metadata_version = frappe.boot.metadata_version;
 	},
 
-	clear_local_storage: function() {
-		$.each(["_last_load", "_version_number", "metadata_version", "page_info",
-			"last_visited"], function(i, key) {
-			localStorage.removeItem(key);
-		});
+	clear_local_storage: function () {
+		$.each(
+			["_last_load", "_version_number", "metadata_version", "page_info", "last_visited"],
+			function (i, key) {
+				localStorage.removeItem(key);
+			}
+		);
 
 		// clear assets
-		for(var key in localStorage) {
-			if(key.indexOf("desk_assets:")===0 || key.indexOf("_page:")===0
-				|| key.indexOf("_doctype:")===0 || key.indexOf("preferred_breadcrumbs:")===0) {
+		for (var key in localStorage) {
+			if (
+				key.indexOf("desk_assets:") === 0 ||
+				key.indexOf("_page:") === 0 ||
+				key.indexOf("_doctype:") === 0 ||
+				key.indexOf("preferred_breadcrumbs:") === 0
+			) {
 				localStorage.removeItem(key);
 			}
 		}
 		console.log("localStorage cleared");
 	},
 
-
 	// keep track of executed assets
-	executed_ : [],
+	executed_: [],
 
 	// pass on to the handler to set
-	execute: function(items, callback) {
-		var to_fetch = []
-		for(var i=0, l=items.length; i {
-		if (
-			window.cordova &&
-			window.cordova.plugins &&
-			window.cordova.plugins.barcodeScanner
-		) {
-			window.cordova.plugins.barcodeScanner.scan(result => {
-				if (!result.cancelled) {
-					resolve(result.text);
-				}
-			}, reject);
-		} else {
-			frappe.require('barcode_scanner.bundle.js', () => {
-				frappe.barcode.get_barcode().then(barcode => {
-					resolve(barcode);
-				});
-			});
-		}
-	});
-};
diff --git a/frappe/public/js/frappe/barcode_scanner/quagga.js b/frappe/public/js/frappe/barcode_scanner/quagga.js
deleted file mode 100644
index fcab3b4dbe..0000000000
--- a/frappe/public/js/frappe/barcode_scanner/quagga.js
+++ /dev/null
@@ -1,94 +0,0 @@
-import Quagga from 'quagga/dist/quagga';
-frappe.provide('frappe.barcode');
-
-Quagga.onProcessed(function(result) {
-	let drawingCtx = Quagga.canvas.ctx.overlay,
-		drawingCanvas = Quagga.canvas.dom.overlay;
-
-	if (result) {
-		if (result.boxes) {
-			drawingCtx.clearRect(
-				0,
-				0,
-				parseInt(drawingCanvas.getAttribute('width')),
-				parseInt(drawingCanvas.getAttribute('height'))
-			);
-			result.boxes
-				.filter(function(box) {
-					return box !== result.box;
-				})
-				.forEach(function(box) {
-					Quagga.ImageDebug.drawPath(box, { x: 0, y: 1 }, drawingCtx, {
-						color: 'green',
-						lineWidth: 2
-					});
-				});
-		}
-
-		if (result.box) {
-			Quagga.ImageDebug.drawPath(result.box, { x: 0, y: 1 }, drawingCtx, {
-				color: '#00F',
-				lineWidth: 2
-			});
-		}
-
-		if (result.codeResult && result.codeResult.code) {
-			Quagga.ImageDebug.drawPath(result.line, { x: 'x', y: 'y' }, drawingCtx, {
-				color: 'red',
-				lineWidth: 3
-			});
-		}
-	}
-});
-
-frappe.barcode.get_barcode = function() {
-	return new Promise(resolve => {
-		let d = new frappe.ui.Dialog({
-			title: __('Scan Barcode'),
-			fields: [
-				{
-					fieldtype: 'HTML',
-					fieldname: 'scan_area'
-				}
-			],
-			on_page_show() {
-				let $scan_area = d.get_field('scan_area').$wrapper;
-				$scan_area.addClass('barcode-scanner');
-
-				Quagga.init(
-					{
-						inputStream: {
-							name: 'Live',
-							type: 'LiveStream',
-							target: $scan_area.get(0)
-						},
-						decoder: {
-							readers: ['code_128_reader']
-						}
-					},
-					function(err) {
-						if (err) {
-							// eslint-disable-next-line
-							console.log(err);
-							return;
-						}
-						// eslint-disable-next-line
-						console.log('Initialization finished. Ready to start');
-						Quagga.start();
-					}
-				);
-
-				Quagga.onDetected(function(result) {
-					let code = result.codeResult.code;
-					if (code) {
-						Quagga.stop();
-						d.hide();
-						resolve(code);
-					}
-				});
-			}
-		});
-
-		d.show();
-	});
-};
diff --git a/frappe/public/js/frappe/build_events/build_events.bundle.js b/frappe/public/js/frappe/build_events/build_events.bundle.js
index 21960f0b00..afb68ba75a 100644
--- a/frappe/public/js/frappe/build_events/build_events.bundle.js
+++ b/frappe/public/js/frappe/build_events/build_events.bundle.js
@@ -5,7 +5,7 @@ let $container = $("#build-events-overlay");
 let success = null;
 let error = null;
 
-frappe.realtime.on("build_event", data => {
+frappe.realtime.on("build_event", (data) => {
 	if (data.success) {
 		// remove executed cache for rebuilt files
 		let changed_files = data.changed_files;
@@ -17,19 +17,19 @@ frappe.realtime.on("build_event", data => {
 						let filename = parts[0].split("/").slice(-1)[0];
 
 						frappe.assets.executed_ = frappe.assets.executed_.filter(
-							asset => !asset.includes(`${filename}.bundle`)
+							(asset) => !asset.includes(`${filename}.bundle`)
 						);
 					}
 				}
 			}
 		}
 		// update assets json
-		frappe.call("frappe.sessions.get_boot_assets_json").then(r => {
+		frappe.call("frappe.sessions.get_boot_assets_json").then((r) => {
 			if (r.message) {
 				frappe.boot.assets_json = r.message;
 
 				if (frappe.hot_update) {
-					frappe.hot_update.forEach(callback => {
+					frappe.hot_update.forEach((callback) => {
 						callback();
 					});
 				}
@@ -47,12 +47,10 @@ function show_build_success(data) {
 	}
 
 	if (!success) {
-		let target = $('
        ')
-			.appendTo($container)
-			.get(0);
+		let target = $('
        ').appendTo($container).get(0);
 		let vm = new Vue({
 			el: target,
-			render: h => h(BuildSuccess)
+			render: (h) => h(BuildSuccess),
 		});
 		success = vm.$children[0];
 	}
@@ -64,12 +62,10 @@ function show_build_error(data) {
 		success.hide();
 	}
 	if (!error) {
-		let target = $('
        ')
-			.appendTo($container)
-			.get(0);
+		let target = $('
        ').appendTo($container).get(0);
 		let vm = new Vue({
 			el: target,
-			render: h => h(BuildError)
+			render: (h) => h(BuildError),
 		});
 		error = vm.$children[0];
 	}
diff --git a/frappe/public/js/frappe/class.js b/frappe/public/js/frappe/class.js
index 79ef2792ae..f1370d6e55 100644
--- a/frappe/public/js/frappe/class.js
+++ b/frappe/public/js/frappe/class.js
@@ -20,64 +20,73 @@ To subclass, use:
  *  (inspired by base2 and Prototype)
  * MIT Licensed.
  */
-(function(global) {
+(function (global) {
 	"use strict";
-	var fnTest = /xyz/.test(function(){xyz;}) ? /\b_super\b/ : /.*/;
+	var fnTest = /xyz/.test(function () {
+		xyz;
+	})
+		? /\b_super\b/
+		: /.*/;
 
 	// The base Class implementation (does nothing)
-	function Class(){}
+	function Class() {}
 
 	// Create a new Class that inherits from this class
-	Class.extend = function(props) {
-	  var _super = this.prototype;
+	Class.extend = function (props) {
+		var _super = this.prototype;
 
-	  // Set up the prototype to inherit from the base class
-	  // (but without running the init constructor)
-	  var proto = Object.create(_super);
+		// Set up the prototype to inherit from the base class
+		// (but without running the init constructor)
+		var proto = Object.create(_super);
 
-	  // Copy the properties over onto the new prototype
-	  for (var name in props) {
-		// Check if we're overwriting an existing function
-		proto[name] = typeof props[name] === "function" &&
-		  typeof _super[name] == "function" && fnTest.test(props[name])
-		  ? (function(name, fn){
-			  return function() {
-				var tmp = this._super;
+		// Copy the properties over onto the new prototype
+		for (var name in props) {
+			// Check if we're overwriting an existing function
+			proto[name] =
+				typeof props[name] === "function" &&
+				typeof _super[name] == "function" &&
+				fnTest.test(props[name])
+					? (function (name, fn) {
+							return function () {
+								var tmp = this._super;
 
-				// Add a new ._super() method that is the same method
-				// but on the super-class
-				this._super = _super[name];
+								// Add a new ._super() method that is the same method
+								// but on the super-class
+								this._super = _super[name];
 
-				// The method only need to be bound temporarily, so we
-				// remove it when we're done executing
-				var ret = fn.apply(this, arguments);
-				this._super = tmp;
+								// The method only need to be bound temporarily, so we
+								// remove it when we're done executing
+								var ret = fn.apply(this, arguments);
+								this._super = tmp;
 
-				return ret;
-			  };
-			})(name, props[name])
-		  : props[name];
-	  }
+								return ret;
+							};
+					  })(name, props[name])
+					: props[name];
+		}
 
-	  // The new constructor
-	  var newClass = typeof proto.init === "function"
-		? proto.hasOwnProperty("init")
-		  ? proto.init // All construction is actually done in the init method
-		  : function SubClass(){ _super.init.apply(this, arguments); }
-		: function EmptyClass(){};
+		// The new constructor
+		var newClass =
+			typeof proto.init === "function"
+				? proto.hasOwnProperty("init")
+					? proto.init // All construction is actually done in the init method
+					: function SubClass() {
+							_super.init.apply(this, arguments);
+					  }
+				: function EmptyClass() {};
 
-	  // Populate our constructed prototype object
-	  newClass.prototype = proto;
+		// Populate our constructed prototype object
+		newClass.prototype = proto;
 
-	  // Enforce the constructor to be what we expect
-	  proto.constructor = newClass;
+		// Enforce the constructor to be what we expect
+		proto.constructor = newClass;
 
-	  // And make this class extendable
-	  newClass.extend = Class.extend;
+		// And make this class extendable
+		newClass.extend = Class.extend;
 
-	  return newClass;
+		return newClass;
 	};
 
 	// export
 	global.Class = Class;
-  })(window);
+})(window);
diff --git a/frappe/public/js/frappe/color_picker/color_picker.js b/frappe/public/js/frappe/color_picker/color_picker.js
index aa26bb1c90..5f6c940425 100644
--- a/frappe/public/js/frappe/color_picker/color_picker.js
+++ b/frappe/public/js/frappe/color_picker/color_picker.js
@@ -1,4 +1,4 @@
-import utils from './utils';
+import utils from "./utils";
 
 class Picker {
 	constructor(opts) {
@@ -16,14 +16,14 @@ class Picker {
 	}
 
 	setup_picker() {
-		let color_picker_template = document.createElement('template');
+		let color_picker_template = document.createElement("template");
 		color_picker_template.innerHTML = `
 			
        
 				
        
-					${__('SWATCHES')}
        
+					${__("SWATCHES")}
        
 					
        
 				
        
-				${__('COLOR PICKER')}
        
+				${__("COLOR PICKER")}
        
 				
        
 					
        
 				
        
@@ -32,13 +32,14 @@ class Picker {
 				
        
 			
        
 		`.trim();
-		this.color_picker_wrapper = color_picker_template.content.firstElementChild.cloneNode(true);
+		this.color_picker_wrapper =
+			color_picker_template.content.firstElementChild.cloneNode(true);
 		this.parent.appendChild(this.color_picker_wrapper);
-		this.color_map = this.color_picker_wrapper.getElementsByClassName('color-map')[0];
-		this.color_selector_circle = this.color_map.getElementsByClassName('color-selector')[0];
-		this.hue_map = this.color_picker_wrapper.getElementsByClassName('hue-map')[0];
-		this.swatches_wrapper = this.color_picker_wrapper.getElementsByClassName('swatches')[0];
-		this.hue_selector_circle = this.hue_map.getElementsByClassName('hue-selector')[0];
+		this.color_map = this.color_picker_wrapper.getElementsByClassName("color-map")[0];
+		this.color_selector_circle = this.color_map.getElementsByClassName("color-selector")[0];
+		this.hue_map = this.color_picker_wrapper.getElementsByClassName("hue-map")[0];
+		this.swatches_wrapper = this.color_picker_wrapper.getElementsByClassName("swatches")[0];
+		this.hue_selector_circle = this.hue_map.getElementsByClassName("hue-selector")[0];
 		this.refresh();
 		this.setup_events();
 		this.setup_swatches();
@@ -50,9 +51,9 @@ class Picker {
 	}
 
 	setup_swatches() {
-		let swatch_template = document.createElement('template');
+		let swatch_template = document.createElement("template");
 		swatch_template.innerHTML = '
        ';
-		this.swatches.forEach(color => {
+		this.swatches.forEach((color) => {
 			let swatch = swatch_template.content.firstElementChild.cloneNode(true);
 			this.swatches_wrapper.appendChild(swatch);
 			const set_values = () => {
@@ -60,7 +61,7 @@ class Picker {
 				this.set_selector_position();
 				this.update_color_map();
 			};
-			swatch.addEventListener('click', () => {
+			swatch.addEventListener("click", () => {
 				set_values();
 			});
 			swatch.onkeydown = (e) => {
@@ -78,8 +79,8 @@ class Picker {
 		this.hue = utils.get_hue(this.get_color());
 		this.color_selector_position = this.get_pointer_coords();
 		this.hue_selector_position = {
-			x: this.hue * this.hue_map.offsetWidth / 360,
-			y: this.hue_map.offsetHeight / 2
+			x: (this.hue * this.hue_map.offsetWidth) / 360,
+			y: this.hue_map.offsetHeight / 2,
 		};
 		this.update_color_selector(silent);
 		this.update_hue_selector(silent);
@@ -102,8 +103,8 @@ class Picker {
 		let h = this.color_map.offsetHeight;
 		let color = utils.hsv_to_hex(
 			this.hue,
-			Math.round(x / w * 100),
-			Math.round((1 - (y / h)) * 100),
+			Math.round((x / w) * 100),
+			Math.round((1 - y / h) * 100)
 		);
 		this.set_color(color);
 	}
@@ -112,8 +113,10 @@ class Picker {
 		let x = this.color_selector_position.x;
 		let y = this.color_selector_position.y;
 		// set color selector position and background
-		this.color_selector_circle.style.top = (y - this.color_selector_circle.offsetHeight / 2) + 'px';
-		this.color_selector_circle.style.left = (x - this.color_selector_circle.offsetWidth / 2) + 'px';
+		this.color_selector_circle.style.top =
+			y - this.color_selector_circle.offsetHeight / 2 + "px";
+		this.color_selector_circle.style.left =
+			x - this.color_selector_circle.offsetWidth / 2 + "px";
 		this.color_map.style.color = this.get_color();
 		!silent && this.on_change && this.on_change(this.get_color());
 	}
@@ -122,7 +125,7 @@ class Picker {
 		// eslint-disable-next-line no-unused-vars
 		let on_drag = (x, y) => {
 			this.hue_selector_position.x = x;
-			this.hue = Math.round(x * 360 / this.hue_map.offsetWidth);
+			this.hue = Math.round((x * 360) / this.hue_map.offsetWidth);
 			this.update_color_map();
 			this.update_hue_selector();
 			this.update_color();
@@ -141,10 +144,10 @@ class Picker {
 
 	update_hue_selector() {
 		let x = this.hue_selector_position.x - 1;
-		let y = (this.hue_map.offsetHeight / 2) - 1;
+		let y = this.hue_map.offsetHeight / 2 - 1;
 		// set color selector position and background
-		this.hue_selector_circle.style.top = (y - this.hue_selector_circle.offsetHeight / 2) + 'px';
-		this.hue_selector_circle.style.left = (x - this.hue_selector_circle.offsetWidth / 2) + 'px';
+		this.hue_selector_circle.style.top = y - this.hue_selector_circle.offsetHeight / 2 + "px";
+		this.hue_selector_circle.style.left = x - this.hue_selector_circle.offsetWidth / 2 + "px";
 		this.hue_map.style.color = `hsl(${this.hue}, 100%, 50%)`;
 	}
 
@@ -155,9 +158,9 @@ class Picker {
 		[h, s, v] = utils.get_hsv(this.get_color());
 		let width = this.color_map.offsetWidth;
 		let height = this.color_map.offsetHeight;
-		let x = utils.clamp(0, s * width / 100, width);
-		let y = utils.clamp(0, (1 - (v / 100)) * height, height);
-		return {x, y};
+		let x = utils.clamp(0, (s * width) / 100, width);
+		let y = utils.clamp(0, (1 - v / 100) * height, height);
+		return { x, y };
 	}
 
 	setup_drag_event(element, callback) {
@@ -174,14 +177,14 @@ class Picker {
 			}
 		};
 
-		element.addEventListener("mousedown", () => element.drag_enabled = true);
-		document.addEventListener("mouseup", () => element.drag_enabled = false);
+		element.addEventListener("mousedown", () => (element.drag_enabled = true));
+		document.addEventListener("mouseup", () => (element.drag_enabled = false));
 		document.addEventListener("mousemove", on_drag);
 		element.addEventListener("click", (event) => on_drag(event, true));
 
-		element.addEventListener("touchstart", () => element.drag_enabled = true);
-		element.addEventListener("touchend", () => element.drag_enabled = false);
-		element.addEventListener("touchcancel", () => element.drag_enabled = false);
+		element.addEventListener("touchstart", () => (element.drag_enabled = true));
+		element.addEventListener("touchend", () => (element.drag_enabled = false));
+		element.addEventListener("touchcancel", () => (element.drag_enabled = false));
 		element.addEventListener("touchmove", (event) => {
 			if (event.touches.length == 1) {
 				on_drag(event);
@@ -192,12 +195,12 @@ class Picker {
 	}
 
 	set_color(color) {
-		this.color = color || '#ffffff';
+		this.color = color || "#ffffff";
 	}
 
 	get_color() {
-		return this.color || '#ffffff';
+		return this.color || "#ffffff";
 	}
 }
 
-export default Picker;
\ No newline at end of file
+export default Picker;
diff --git a/frappe/public/js/frappe/color_picker/utils.js b/frappe/public/js/frappe/color_picker/utils.js
index fd11f210fe..70dfee220c 100644
--- a/frappe/public/js/frappe/color_picker/utils.js
+++ b/frappe/public/js/frappe/color_picker/utils.js
@@ -5,7 +5,7 @@ export default {
 
 	get_hsv(rgb) {
 		if (typeof rgb === "string") {
-			if (rgb.startsWith('#')) {
+			if (rgb.startsWith("#")) {
 				rgb = this.hex_to_rgb_values(rgb);
 			} else {
 				rgb = this.get_rgb_values(rgb);
@@ -15,7 +15,10 @@ export default {
 	},
 
 	get_rgb_values(rgb_string) {
-		return rgb_string.replace(/[()rgb\s]/g, '').split(',').map(s => parseInt(s));
+		return rgb_string
+			.replace(/[()rgb\s]/g, "")
+			.split(",")
+			.map((s) => parseInt(s));
 	},
 
 	rgb_to_hsv(r, g, b) {
@@ -23,8 +26,11 @@ export default {
 		g /= 255;
 		b /= 255;
 
-		let max = Math.max(r, g, b), min = Math.min(r, g, b);
-		let h, s, v = max;
+		let max = Math.max(r, g, b),
+			min = Math.min(r, g, b);
+		let h,
+			s,
+			v = max;
 
 		let d = max - min;
 		s = max == 0 ? 0 : d / max;
@@ -33,9 +39,15 @@ export default {
 			h = 0; // achromatic
 		} else {
 			switch (max) {
-				case r: h = (g - b) / d + (g < b ? 6 : 0); break;
-				case g: h = (b - r) / d + 2; break;
-				case b: h = (r - g) / d + 4; break;
+				case r:
+					h = (g - b) / d + (g < b ? 6 : 0);
+					break;
+				case g:
+					h = (b - r) / d + 2;
+					break;
+				case b:
+					h = (r - g) / d + 4;
+					break;
 			}
 
 			h /= 6;
@@ -49,7 +61,9 @@ export default {
 	},
 
 	rgb_to_hex(r, g, b) {
-		return "#" + this.component_to_hex(r) + this.component_to_hex(g) + this.component_to_hex(b);
+		return (
+			"#" + this.component_to_hex(r) + this.component_to_hex(g) + this.component_to_hex(b)
+		);
 	},
 
 	hex_to_rgb_values(hex) {
@@ -70,12 +84,24 @@ export default {
 		let t = v * (1 - (1 - f) * s);
 
 		switch (i % 6) {
-			case 0: r = v, g = t, b = p; break;
-			case 1: r = q, g = v, b = p; break;
-			case 2: r = p, g = v, b = t; break;
-			case 3: r = p, g = q, b = v; break;
-			case 4: r = t, g = p, b = v; break;
-			case 5: r = v, g = p, b = q; break;
+			case 0:
+				(r = v), (g = t), (b = p);
+				break;
+			case 1:
+				(r = q), (g = v), (b = p);
+				break;
+			case 2:
+				(r = p), (g = v), (b = t);
+				break;
+			case 3:
+				(r = p), (g = q), (b = v);
+				break;
+			case 4:
+				(r = t), (g = p), (b = v);
+				break;
+			case 5:
+				(r = v), (g = p), (b = q);
+				break;
 		}
 		r = Math.round(r * 255);
 		g = Math.round(g * 255);
@@ -85,5 +111,5 @@ export default {
 
 	clamp(min, val, max) {
 		return val > max ? max : val < min ? min : val;
-	}
-};
\ No newline at end of file
+	},
+};
diff --git a/frappe/public/js/frappe/data_import/data_exporter.js b/frappe/public/js/frappe/data_import/data_exporter.js
index 8fa5a08945..ccf449b443 100644
--- a/frappe/public/js/frappe/data_import/data_exporter.js
+++ b/frappe/public/js/frappe/data_import/data_exporter.js
@@ -1,4 +1,4 @@
-frappe.provide('frappe.data_import');
+frappe.provide("frappe.data_import");
 
 frappe.data_import.DataExporter = class DataExporter {
 	constructor(doctype, exporting_for) {
@@ -11,82 +11,83 @@ frappe.data_import.DataExporter = class DataExporter {
 
 	make_dialog() {
 		this.dialog = new frappe.ui.Dialog({
-			title: __('Export Data'),
+			title: __("Export Data"),
 			fields: [
 				{
-					fieldtype: 'Select',
-					fieldname: 'file_type',
-					label: __('File Type'),
-					options: ['Excel', 'CSV'],
-					default: 'CSV'
+					fieldtype: "Select",
+					fieldname: "file_type",
+					label: __("File Type"),
+					options: ["Excel", "CSV"],
+					default: "CSV",
 				},
 				{
-					fieldtype: 'Select',
-					fieldname: 'export_records',
-					label: __('Export Type'),
+					fieldtype: "Select",
+					fieldname: "export_records",
+					label: __("Export Type"),
 					options: [
 						{
-							label: __('All Records'),
-							value: 'all'
+							label: __("All Records"),
+							value: "all",
 						},
 						{
-							label: __('Filtered Records'),
-							value: 'by_filter'
+							label: __("Filtered Records"),
+							value: "by_filter",
 						},
 						{
-							label: __('5 Records'),
-							value: '5_records'
+							label: __("5 Records"),
+							value: "5_records",
 						},
 						{
-							label: __('Blank Template'),
-							value: 'blank_template'
-						}
+							label: __("Blank Template"),
+							value: "blank_template",
+						},
 					],
-					default: this.exporting_for === 'Insert New Records' ? 'blank_template' : 'all',
+					default:
+						this.exporting_for === "Insert New Records" ? "blank_template" : "all",
 					change: () => {
 						this.update_record_count_message();
-					}
+					},
 				},
 				{
-					fieldtype: 'HTML',
-					fieldname: 'filter_area',
-					depends_on: doc => doc.export_records === 'by_filter'
+					fieldtype: "HTML",
+					fieldname: "filter_area",
+					depends_on: (doc) => doc.export_records === "by_filter",
 				},
 				{
-					fieldtype: 'Section Break'
+					fieldtype: "Section Break",
 				},
 				{
-					fieldtype: 'HTML',
-					fieldname: 'select_all_buttons'
+					fieldtype: "HTML",
+					fieldname: "select_all_buttons",
 				},
 				{
 					label: __(this.doctype),
 					fieldname: this.doctype,
-					fieldtype: 'MultiCheck',
+					fieldtype: "MultiCheck",
 					columns: 2,
 					on_change: () => this.update_primary_action(),
-					options: this.get_multicheck_options(this.doctype)
+					options: this.get_multicheck_options(this.doctype),
 				},
-				...frappe.meta.get_table_fields(this.doctype).map(df => {
+				...frappe.meta.get_table_fields(this.doctype).map((df) => {
 					let doctype = df.options;
 					let child_fieldname = df.fieldname;
 					let label = df.reqd
 						? // prettier-ignore
 						  __('{0} ({1}) (1 row mandatory)', [__(df.label || df.fieldname), __(doctype)])
-						: __('{0} ({1})', [__(df.label || df.fieldname), __(doctype)]);
+						: __("{0} ({1})", [__(df.label || df.fieldname), __(doctype)]);
 					return {
 						label,
 						fieldname: child_fieldname,
-						fieldtype: 'MultiCheck',
+						fieldtype: "MultiCheck",
 						columns: 2,
 						on_change: () => this.update_primary_action(),
-						options: this.get_multicheck_options(doctype, child_fieldname)
+						options: this.get_multicheck_options(doctype, child_fieldname),
 					};
-				})
+				}),
 			],
-			primary_action_label: __('Export'),
-			primary_action: values => this.export_records(values),
-			on_page_show: () => this.select_mandatory()
+			primary_action_label: __("Export"),
+			primary_action: (values) => this.export_records(values),
+			on_page_show: () => this.select_mandatory(),
 		});
 
 		this.make_filter_area();
@@ -97,12 +98,11 @@ frappe.data_import.DataExporter = class DataExporter {
 	}
 
 	export_records() {
-		let method =
-			'/api/method/frappe.core.doctype.data_import.data_import.download_template';
+		let method = "/api/method/frappe.core.doctype.data_import.data_import.download_template";
 
 		let multicheck_fields = this.dialog.fields
-			.filter(df => df.fieldtype === 'MultiCheck')
-			.map(df => df.fieldname);
+			.filter((df) => df.fieldtype === "MultiCheck")
+			.map((df) => df.fieldname);
 
 		let values = this.dialog.get_values();
 
@@ -114,7 +114,7 @@ frappe.data_import.DataExporter = class DataExporter {
 		}
 
 		let filters = null;
-		if (values.export_records === 'by_filter') {
+		if (values.export_records === "by_filter") {
 			filters = this.get_filters();
 		}
 
@@ -123,111 +123,108 @@ frappe.data_import.DataExporter = class DataExporter {
 			file_type: values.file_type,
 			export_records: values.export_records,
 			export_fields: doctype_field_map,
-			export_filters: filters
+			export_filters: filters,
 		});
 	}
 
 	make_filter_area() {
 		this.filter_group = new frappe.ui.FilterGroup({
-			parent: this.dialog.get_field('filter_area').$wrapper,
+			parent: this.dialog.get_field("filter_area").$wrapper,
 			doctype: this.doctype,
 			on_change: () => {
 				this.update_record_count_message();
-			}
+			},
 		});
 	}
 
 	make_select_all_buttons() {
-		let for_insert = this.exporting_for === 'Insert New Records';
-		let section_title = for_insert ? __('Select Fields To Insert') : __('Select Fields To Update');
+		let for_insert = this.exporting_for === "Insert New Records";
+		let section_title = for_insert
+			? __("Select Fields To Insert")
+			: __("Select Fields To Update");
 		let $select_all_buttons = $(`
 			
        
 				
        ${section_title}
        
 				
-				${for_insert ? ``: ''}
+				${
+					for_insert
+						? ``
+						: ""
+				}
 				
 			
        
 		`);
 		frappe.utils.bind_actions_with_object($select_all_buttons, this);
-		this.dialog
-			.get_field('select_all_buttons')
-			.$wrapper.html($select_all_buttons);
+		this.dialog.get_field("select_all_buttons").$wrapper.html($select_all_buttons);
 	}
 
 	select_all() {
-		this.dialog.$wrapper
-			.find(':checkbox')
-			.prop('checked', true)
-			.trigger('change');
+		this.dialog.$wrapper.find(":checkbox").prop("checked", true).trigger("change");
 	}
 
 	select_mandatory() {
 		let mandatory_table_fields = frappe.meta
 			.get_table_fields(this.doctype)
-			.filter(df => df.reqd)
-			.map(df => df.fieldname);
+			.filter((df) => df.reqd)
+			.map((df) => df.fieldname);
 		mandatory_table_fields.push(this.doctype);
 
 		let multicheck_fields = this.dialog.fields
-			.filter(df => df.fieldtype === 'MultiCheck')
-			.map(df => df.fieldname)
-			.filter(doctype => mandatory_table_fields.includes(doctype));
+			.filter((df) => df.fieldtype === "MultiCheck")
+			.map((df) => df.fieldname)
+			.filter((doctype) => mandatory_table_fields.includes(doctype));
 
 		let checkboxes = [].concat(
-			...multicheck_fields.map(fieldname => {
+			...multicheck_fields.map((fieldname) => {
 				let field = this.dialog.get_field(fieldname);
 				return field.options
-					.filter(option => option.danger)
-					.map(option => option.$checkbox.find('input').get(0));
+					.filter((option) => option.danger)
+					.map((option) => option.$checkbox.find("input").get(0));
 			})
 		);
 
 		this.unselect_all();
-		$(checkboxes)
-			.prop('checked', true)
-			.trigger('change');
+		$(checkboxes).prop("checked", true).trigger("change");
 	}
 
 	unselect_all() {
 		let update_existing_records =
-			this.dialog.get_value('exporting_for') == 'Update Existing Records';
+			this.dialog.get_value("exporting_for") == "Update Existing Records";
 		this.dialog.$wrapper
-			.find(
-				`:checkbox${update_existing_records ? ':not([data-unit=name])' : ''}`
-			)
-			.prop('checked', false)
-			.trigger('change');
+			.find(`:checkbox${update_existing_records ? ":not([data-unit=name])" : ""}`)
+			.prop("checked", false)
+			.trigger("change");
 	}
 
 	update_record_count_message() {
-		let export_records = this.dialog.get_value('export_records');
+		let export_records = this.dialog.get_value("export_records");
 		let count_method = {
 			all: () => frappe.db.count(this.doctype),
 			by_filter: () =>
 				frappe.db.count(this.doctype, {
-					filters: this.get_filters()
+					filters: this.get_filters(),
 				}),
 			blank_template: () => Promise.resolve(0),
-			'5_records': () => Promise.resolve(5)
+			"5_records": () => Promise.resolve(5),
 		};
 
-		count_method[export_records]().then(value => {
-			let message = '';
+		count_method[export_records]().then((value) => {
+			let message = "";
 			value = parseInt(value, 10);
 			if (value === 0) {
-				message = __('No records will be exported');
+				message = __("No records will be exported");
 			} else if (value === 1) {
-				message = __('1 record will be exported');
+				message = __("1 record will be exported");
 			} else {
-				message = __('{0} records will be exported', [value]);
+				message = __("{0} records will be exported", [value]);
 			}
-			this.dialog.set_df_property('export_records', 'description', message);
+			this.dialog.set_df_property("export_records", "description", message);
 
 			this.update_primary_action(value);
 		});
@@ -237,23 +234,23 @@ frappe.data_import.DataExporter = class DataExporter {
 		let $primary_action = this.dialog.get_primary_btn();
 
 		if (no_of_records != null) {
-			let label = '';
+			let label = "";
 			if (no_of_records === 0) {
-				label = __('Export');
+				label = __("Export");
 			} else if (no_of_records === 1) {
-				label = __('Export 1 record');
+				label = __("Export 1 record");
 			} else {
-				label = __('Export {0} records', [no_of_records]);
+				label = __("Export {0} records", [no_of_records]);
 			}
 			$primary_action.html(label);
 		} else {
 			let parent_fields = this.dialog.get_value(this.doctype);
-			$primary_action.prop('disabled', parent_fields.length === 0);
+			$primary_action.prop("disabled", parent_fields.length === 0);
 		}
 	}
 
 	get_filters() {
-		return this.filter_group.get_filters().map(filter => {
+		return this.filter_group.get_filters().map((filter) => {
 			return filter.slice(0, 4);
 		});
 	}
@@ -265,42 +262,40 @@ frappe.data_import.DataExporter = class DataExporter {
 
 		let autoname_field = null;
 		let meta = frappe.get_meta(doctype);
-		if (meta.autoname && meta.autoname.startsWith('field:')) {
-			let fieldname = meta.autoname.slice('field:'.length);
+		if (meta.autoname && meta.autoname.startsWith("field:")) {
+			let fieldname = meta.autoname.slice("field:".length);
 			autoname_field = frappe.meta.get_field(doctype, fieldname);
 		}
 
-		let fields = child_fieldname
-			? this.column_map[child_fieldname]
-			: this.column_map[doctype];
+		let fields = child_fieldname ? this.column_map[child_fieldname] : this.column_map[doctype];
 
-		let is_field_mandatory = df => {
-			if (df.reqd && this.exporting_for == 'Insert New Records') {
+		let is_field_mandatory = (df) => {
+			if (df.reqd && this.exporting_for == "Insert New Records") {
 				return true;
 			}
 			if (autoname_field && df.fieldname == autoname_field.fieldname) {
 				return true;
 			}
-			if (df.fieldname === 'name') {
+			if (df.fieldname === "name") {
 				return true;
 			}
 			return false;
 		};
 
 		return fields
-			.filter(df => {
-				if (autoname_field && df.fieldname === 'name') {
+			.filter((df) => {
+				if (autoname_field && df.fieldname === "name") {
 					return false;
 				}
 				return true;
 			})
-			.map(df => {
+			.map((df) => {
 				return {
 					label: __(df.label),
 					value: df.fieldname,
 					danger: is_field_mandatory(df),
 					checked: false,
-					description: `${df.fieldname} ${df.reqd ? __('(Mandatory)') : ''}`
+					description: `${df.fieldname} ${df.reqd ? __("(Mandatory)") : ""}`,
 				};
 			});
 	}
@@ -309,46 +304,42 @@ frappe.data_import.DataExporter = class DataExporter {
 export function get_columns_for_picker(doctype) {
 	let out = {};
 
-	const exportable_fields = df => {
+	const exportable_fields = (df) => {
 		let keep = true;
 		if (frappe.model.no_value_type.includes(df.fieldtype)) {
 			keep = false;
 		}
-		if (['lft', 'rgt'].includes(df.fieldname)) {
+		if (["lft", "rgt"].includes(df.fieldname)) {
 			keep = false;
 		}
 		return keep;
 	};
 
 	// parent
-	let doctype_fields = frappe.meta
-		.get_docfields(doctype)
-		.filter(exportable_fields);
+	let doctype_fields = frappe.meta.get_docfields(doctype).filter(exportable_fields);
 
 	out[doctype] = [
 		{
-			label: __('ID'),
-			fieldname: 'name',
-			fieldtype: 'Data',
-			reqd: 1
-		}
+			label: __("ID"),
+			fieldname: "name",
+			fieldtype: "Data",
+			reqd: 1,
+		},
 	].concat(doctype_fields);
 
 	// children
 	const table_fields = frappe.meta.get_table_fields(doctype);
-	table_fields.forEach(df => {
+	table_fields.forEach((df) => {
 		const cdt = df.options;
-		const child_table_fields = frappe.meta
-			.get_docfields(cdt)
-			.filter(exportable_fields);
+		const child_table_fields = frappe.meta.get_docfields(cdt).filter(exportable_fields);
 
 		out[df.fieldname] = [
 			{
-				label: __('ID'),
-				fieldname: 'name',
-				fieldtype: 'Data',
-				reqd: 1
-			}
+				label: __("ID"),
+				fieldname: "name",
+				fieldtype: "Data",
+				reqd: 1,
+			},
 		].concat(child_table_fields);
 	});
 
diff --git a/frappe/public/js/frappe/data_import/import_preview.js b/frappe/public/js/frappe/data_import/import_preview.js
index b153718c70..4b0f18e3e1 100644
--- a/frappe/public/js/frappe/data_import/import_preview.js
+++ b/frappe/public/js/frappe/data_import/import_preview.js
@@ -1,17 +1,10 @@
-import DataTable from 'frappe-datatable';
-import { get_columns_for_picker } from './data_exporter';
+import DataTable from "frappe-datatable";
+import { get_columns_for_picker } from "./data_exporter";
 
-frappe.provide('frappe.data_import');
+frappe.provide("frappe.data_import");
 
 frappe.data_import.ImportPreview = class ImportPreview {
-	constructor({
-		wrapper,
-		doctype,
-		preview_data,
-		frm,
-		import_log,
-		events = {}
-	}) {
+	constructor({ wrapper, doctype, preview_data, frm, import_log, events = {} }) {
 		this.wrapper = wrapper;
 		this.doctype = doctype;
 		this.preview_data = preview_data;
@@ -49,22 +42,22 @@ frappe.data_import.ImportPreview = class ImportPreview {
 		`);
 		frappe.utils.bind_actions_with_object(this.wrapper, this);
 
-		this.$table_preview = this.wrapper.find('.table-preview');
+		this.$table_preview = this.wrapper.find(".table-preview");
 	}
 
 	prepare_columns() {
 		this.columns = this.preview_data.columns.map((col, i) => {
 			let df = col.df;
 			let column_width = 120;
-			if (col.header_title === 'Sr. No') {
+			if (col.header_title === "Sr. No") {
 				return {
-					id: 'srno',
-					name: 'Sr. No',
-					content: 'Sr. No',
+					id: "srno",
+					name: "Sr. No",
+					content: "Sr. No",
 					editable: false,
 					focusable: false,
-					align: 'left',
-					width: 60
+					align: "left",
+					width: 60,
 				};
 			}
 
@@ -76,37 +69,37 @@ frappe.data_import.ImportPreview = class ImportPreview {
 					column_width += 50;
 				}
 				let column_title = `
-					${col.header_title || `${__('Untitled Column')}`}
-					${!col.df ? show_warnings_button : ''}
+					${col.header_title || `${__("Untitled Column")}`}
+					${!col.df ? show_warnings_button : ""}
 				`;
 				return {
 					id: frappe.utils.get_random(6),
-					name: col.header_title || (df ? df.label : 'Untitled Column'),
+					name: col.header_title || (df ? df.label : "Untitled Column"),
 					content: column_title,
 					skip_import: true,
 					editable: false,
 					focusable: false,
-					align: 'left',
+					align: "left",
 					width: column_width,
-					format: value => `
        ${value}
        `
+					format: (value) => `
        ${value}
        `,
 				};
 			}
 
 			let date_format = col.date_format
 				? col.date_format
-					.replace('%Y', 'yyyy')
-					.replace('%y', 'yy')
-					.replace('%m', 'mm')
-					.replace('%d', 'dd')
-					.replace('%H', 'HH')
-					.replace('%M', 'mm')
-					.replace('%S', 'ss')
-					.replace('%b', 'Mon')
+						.replace("%Y", "yyyy")
+						.replace("%y", "yy")
+						.replace("%m", "mm")
+						.replace("%d", "dd")
+						.replace("%H", "HH")
+						.replace("%M", "mm")
+						.replace("%S", "ss")
+						.replace("%b", "Mon")
 				: null;
 
 			let column_title = `
 				${col.header_title || df.label}
-				${date_format ? `(${date_format})` : ''}
+				${date_format ? `(${date_format})` : ""}
 			`;
 
 			return {
@@ -115,17 +108,17 @@ frappe.data_import.ImportPreview = class ImportPreview {
 				content: column_title,
 				df: df,
 				editable: false,
-				align: 'left',
-				width: column_width
+				align: "left",
+				width: column_width,
 			};
 		});
 	}
 
 	prepare_data() {
-		this.data = this.data.map(row => {
-			return row.map(cell => {
+		this.data = this.data.map((row) => {
+			return row.map((cell) => {
 				if (cell == null) {
-					return '';
+					return "";
 				}
 				return cell;
 			});
@@ -140,104 +133,100 @@ frappe.data_import.ImportPreview = class ImportPreview {
 		this.datatable = new DataTable(this.$table_preview.get(0), {
 			data: this.data,
 			columns: this.columns,
-			layout: this.columns.length < 10 ? 'fluid' : 'fixed',
+			layout: this.columns.length < 10 ? "fluid" : "fixed",
 			cellHeight: 35,
 			language: frappe.boot.lang,
 			translations: frappe.utils.datatable.get_translations(),
 			serialNoColumn: false,
 			checkboxColumn: false,
-			noDataMessage: __('No Data'),
-			disableReorderColumn: true
+			noDataMessage: __("No Data"),
+			disableReorderColumn: true,
 		});
 
-		let {
-			max_rows_exceeded,
-			max_rows_in_preview,
-			total_number_of_rows
-		} = this.preview_data;
+		let { max_rows_exceeded, max_rows_in_preview, total_number_of_rows } = this.preview_data;
 		if (max_rows_exceeded) {
 			let parts = [max_rows_in_preview, total_number_of_rows];
-			this.wrapper.find('.table-message').html(`
+			this.wrapper.find(".table-message").html(`
 				
        
-				${__('Showing only first {0} rows out of {1}', parts)}
+				${__("Showing only first {0} rows out of {1}", parts)}
 				
        
 			`);
 		}
 
 		if (this.data.length === 0) {
-			this.datatable.style.setStyle('.dt-scrollable', {
-				height: 'auto'
+			this.datatable.style.setStyle(".dt-scrollable", {
+				height: "auto",
 			});
 		}
 
-		this.datatable.style.setStyle('.dt-dropdown', {
-			display: 'none'
+		this.datatable.style.setStyle(".dt-dropdown", {
+			display: "none",
 		});
 	}
 
 	setup_styles() {
 		// import success checkbox
 		this.datatable.style.setStyle(`svg.import-success`, {
-			width: '16px',
-			fill: frappe.ui.color.get_color_shade('green', 'dark')
+			width: "16px",
+			fill: frappe.ui.color.get_color_shade("green", "dark"),
 		});
 		// make successfully imported rows readonly
 		let row_classes = this.datatable
 			.getRows()
-			.filter(row => this.is_row_imported(row))
-			.map(row => row.meta.rowIndex)
-			.map(i => `.dt-row-${i} .dt-cell`)
-			.join(',');
+			.filter((row) => this.is_row_imported(row))
+			.map((row) => row.meta.rowIndex)
+			.map((i) => `.dt-row-${i} .dt-cell`)
+			.join(",");
 		this.datatable.style.setStyle(row_classes, {
-			pointerEvents: 'none',
-			backgroundColor: frappe.ui.color.get_color_shade('gray', 'extra-light'),
-			color: frappe.ui.color.get_color_shade('gray', 'dark')
+			pointerEvents: "none",
+			backgroundColor: frappe.ui.color.get_color_shade("gray", "extra-light"),
+			color: frappe.ui.color.get_color_shade("gray", "dark"),
 		});
 	}
 
 	add_actions() {
 		let actions = [
 			{
-				label: __('Map Columns'),
-				handler: 'show_column_mapper',
-				condition: this.frm.doc.status !== 'Success'
+				label: __("Map Columns"),
+				handler: "show_column_mapper",
+				condition: this.frm.doc.status !== "Success",
 			},
 			{
-				label: __('Export Errored Rows'),
-				handler: 'export_errored_rows',
-				condition: this.import_log.filter(log => !log.success).length > 0
+				label: __("Export Errored Rows"),
+				handler: "export_errored_rows",
+				condition: this.import_log.filter((log) => !log.success).length > 0,
 			},
 			{
-				label: __('Show Warnings'),
-				handler: 'show_warnings',
-				condition: this.preview_data.warnings.length > 0
-			}
+				label: __("Show Warnings"),
+				handler: "show_warnings",
+				condition: this.preview_data.warnings.length > 0,
+			},
 		];
 
 		let html = actions
-			.filter(action => action.condition)
-			.map(action => {
+			.filter((action) => action.condition)
+			.map((action) => {
 				return `
 			`;
 			});
 
-		this.wrapper.find('.table-actions').html(html);
+		this.wrapper.find(".table-actions").html(html);
 	}
 
 	export_errored_rows() {
-		this.frm.trigger('export_errored_rows');
+		this.frm.trigger("export_errored_rows");
 	}
 
 	show_warnings() {
-		this.frm.scroll_to_field('import_warnings');
+		this.frm.scroll_to_field("import_warnings");
 	}
 
 	show_column_warning(_, $target) {
 		let $warning = this.frm
-			.get_field('import_warnings')
-			.$wrapper.find(`[data-col=${$target.data('col')}]`);
+			.get_field("import_warnings")
+			.$wrapper.find(`[data-col=${$target.data("col")}]`);
 		frappe.utils.scroll_to($warning, true, 30);
 	}
 
@@ -246,7 +235,7 @@ frappe.data_import.ImportPreview = class ImportPreview {
 		let changed = [];
 		let fields = this.preview_data.columns.map((col, i) => {
 			let df = col.df;
-			if (col.header_title === 'Sr. No') return [];
+			if (col.header_title === "Sr. No") return [];
 
 			let fieldname;
 			if (!df) {
@@ -260,61 +249,61 @@ frappe.data_import.ImportPreview = class ImportPreview {
 			}
 			return [
 				{
-					label: '',
-					fieldtype: 'Data',
+					label: "",
+					fieldtype: "Data",
 					default: col.header_title,
 					fieldname: `Column ${i}`,
-					read_only: 1
+					read_only: 1,
 				},
 				{
-					fieldtype: 'Column Break'
+					fieldtype: "Column Break",
 				},
 				{
-					fieldtype: 'Autocomplete',
+					fieldtype: "Autocomplete",
 					fieldname: i,
-					label: '',
+					label: "",
 					max_items: Infinity,
 					options: [
 						{
 							label: __("Don't Import"),
-							value: "Don't Import"
-						}
+							value: "Don't Import",
+						},
 					].concat(get_fields_as_options(this.doctype, column_picker_fields)),
 					default: fieldname || "Don't Import",
 					change() {
 						changed.push(i);
-					}
+					},
 				},
 				{
-					fieldtype: 'Section Break'
-				}
+					fieldtype: "Section Break",
+				},
 			];
 		});
 		// flatten the array
 		fields = fields.reduce((acc, curr) => [...acc, ...curr]);
-		let file_name = (this.frm.doc.import_file || '').split('/').pop();
+		let file_name = (this.frm.doc.import_file || "").split("/").pop();
 		let parts = [file_name.bold(), this.doctype.bold()];
 		fields = [
 			{
-				fieldtype: 'HTML',
-				fieldname: 'heading',
+				fieldtype: "HTML",
+				fieldname: "heading",
 				options: `
 					
        
-					${__('Map columns from {0} to fields in {1}', parts)}
+					${__("Map columns from {0} to fields in {1}", parts)}
 					
        
-				`
+				`,
 			},
 			{
-				fieldtype: 'Section Break'
-			}
+				fieldtype: "Section Break",
+			},
 		].concat(fields);
 
 		let dialog = new frappe.ui.Dialog({
-			title: __('Map Columns'),
+			title: __("Map Columns"),
 			fields,
-			primary_action: values => {
+			primary_action: (values) => {
 				let changed_map = {};
-				changed.map(i => {
+				changed.map((i) => {
 					let header_row_index = i - 1;
 					changed_map[header_row_index] = values[i];
 				});
@@ -322,29 +311,29 @@ frappe.data_import.ImportPreview = class ImportPreview {
 					this.events.remap_column(changed_map);
 				}
 				dialog.hide();
-			}
+			},
 		});
-		dialog.$body.addClass('map-columns');
+		dialog.$body.addClass("map-columns");
 		dialog.show();
 	}
 
 	is_row_imported(row) {
 		let serial_no = row[0].content;
-		return this.import_log.find(log => {
-			return log.success && JSON.parse(log.row_indexes || '[]').includes(serial_no);
+		return this.import_log.find((log) => {
+			return log.success && JSON.parse(log.row_indexes || "[]").includes(serial_no);
 		});
 	}
 };
 
 function get_fields_as_options(doctype, column_map) {
 	let keys = [doctype];
-	frappe.meta.get_table_fields(doctype).forEach(df => {
+	frappe.meta.get_table_fields(doctype).forEach((df) => {
 		keys.push(df.fieldname);
 	});
 	// flatten array
 	return [].concat(
-		...keys.map(key => {
-			return column_map[key].map(df => {
+		...keys.map((key) => {
+			return column_map[key].map((df) => {
 				let label = __(df.label);
 				let value = df.fieldname;
 				if (doctype !== key) {
@@ -355,7 +344,7 @@ function get_fields_as_options(doctype, column_map) {
 				return {
 					label,
 					value,
-					description: value
+					description: value,
 				};
 			});
 		})
diff --git a/frappe/public/js/frappe/data_import/index.js b/frappe/public/js/frappe/data_import/index.js
index 626d17c85d..8f49e4b87b 100644
--- a/frappe/public/js/frappe/data_import/index.js
+++ b/frappe/public/js/frappe/data_import/index.js
@@ -1,2 +1,2 @@
-import './import_preview';
-import './data_exporter';
+import "./import_preview";
+import "./data_exporter";
diff --git a/frappe/public/js/frappe/db.js b/frappe/public/js/frappe/db.js
index 2467302c76..7e9dd1e875 100644
--- a/frappe/public/js/frappe/db.js
+++ b/frappe/public/js/frappe/db.js
@@ -2,125 +2,131 @@
 // MIT License. See license.txt
 
 frappe.db = {
-	get_list: function(doctype, args) {
+	get_list: function (doctype, args) {
 		if (!args) {
 			args = {};
 		}
 		args.doctype = doctype;
 		if (!args.fields) {
-			args.fields = ['name'];
+			args.fields = ["name"];
 		}
-		if (!('limit' in args)) {
+		if (!("limit" in args)) {
 			args.limit = 20;
 		}
-		return new Promise ((resolve) => {
+		return new Promise((resolve) => {
 			frappe.call({
-				method: 'frappe.desk.reportview.get_list',
+				method: "frappe.desk.reportview.get_list",
 				args: args,
-				type: 'GET',
-				callback: function(r) {
+				type: "GET",
+				callback: function (r) {
 					resolve(r.message);
-				}
+				},
 			});
 		});
 	},
-	exists: function(doctype, name) {
-		return new Promise ((resolve) => {
-			frappe.db.get_value(doctype, {name: name}, 'name').then((r) => {
-				(r.message && r.message.name) ? resolve(true) : resolve(false);
+	exists: function (doctype, name) {
+		return new Promise((resolve) => {
+			frappe.db.get_value(doctype, { name: name }, "name").then((r) => {
+				r.message && r.message.name ? resolve(true) : resolve(false);
 			});
 		});
 	},
-	get_value: function(doctype, filters, fieldname, callback, parent_doc) {
+	get_value: function (doctype, filters, fieldname, callback, parent_doc) {
 		return frappe.call({
 			method: "frappe.client.get_value",
-			type: 'GET',
+			type: "GET",
 			args: {
 				doctype: doctype,
 				fieldname: fieldname,
 				filters: filters,
-				parent: parent_doc
+				parent: parent_doc,
 			},
-			callback: function(r) {
+			callback: function (r) {
 				callback && callback(r.message);
-			}
+			},
 		});
 	},
 	get_single_value: (doctype, field) => {
-		return new Promise(resolve => {
-			frappe.call({
-				method: 'frappe.client.get_single_value',
-				args: { doctype, field },
-				type: 'GET',
-			}).then(r => resolve(r ? r.message : null));
+		return new Promise((resolve) => {
+			frappe
+				.call({
+					method: "frappe.client.get_single_value",
+					args: { doctype, field },
+					type: "GET",
+				})
+				.then((r) => resolve(r ? r.message : null));
 		});
 	},
-	set_value: function(doctype, docname, fieldname, value, callback) {
+	set_value: function (doctype, docname, fieldname, value, callback) {
 		return frappe.call({
 			method: "frappe.client.set_value",
 			args: {
 				doctype: doctype,
 				name: docname,
 				fieldname: fieldname,
-				value: value
+				value: value,
 			},
-			callback: function(r) {
+			callback: function (r) {
 				callback && callback(r.message);
-			}
+			},
 		});
 	},
-	get_doc: function(doctype, name, filters = null) {
+	get_doc: function (doctype, name, filters = null) {
 		return new Promise((resolve, reject) => {
-			frappe.call({
-				method: "frappe.client.get",
-				type: 'GET',
-				args: { doctype, name, filters },
-				callback: r => {
-					frappe.model.sync(r.message);
-					resolve(r.message);
-				}
-			}).fail(reject);
+			frappe
+				.call({
+					method: "frappe.client.get",
+					type: "GET",
+					args: { doctype, name, filters },
+					callback: (r) => {
+						frappe.model.sync(r.message);
+						resolve(r.message);
+					},
+				})
+				.fail(reject);
 		});
 	},
-	insert: function(doc) {
-		return frappe.xcall('frappe.client.insert', { doc });
+	insert: function (doc) {
+		return frappe.xcall("frappe.client.insert", { doc });
 	},
-	delete_doc: function(doctype, name) {
-		return new Promise(resolve => {
-			frappe.call('frappe.client.delete', { doctype, name }, r => resolve(r.message));
+	delete_doc: function (doctype, name) {
+		return new Promise((resolve) => {
+			frappe.call("frappe.client.delete", { doctype, name }, (r) => resolve(r.message));
 		});
 	},
-	count: function(doctype, args={}) {
+	count: function (doctype, args = {}) {
 		let filters = args.filters || {};
 
 		// has a filter with childtable?
-		const distinct = Array.isArray(filters) && filters.some(filter => {
-			return filter[0] !== doctype;
-		});
+		const distinct =
+			Array.isArray(filters) &&
+			filters.some((filter) => {
+				return filter[0] !== doctype;
+			});
 
 		const fields = [];
 
-		return frappe.xcall('frappe.desk.reportview.get_count', {
+		return frappe.xcall("frappe.desk.reportview.get_count", {
 			doctype,
 			filters,
 			fields,
 			distinct,
 		});
 	},
-	get_link_options(doctype, txt = '', filters={}) {
-		return new Promise(resolve => {
+	get_link_options(doctype, txt = "", filters = {}) {
+		return new Promise((resolve) => {
 			frappe.call({
-				type: 'GET',
-				method: 'frappe.desk.search.search_link',
+				type: "GET",
+				method: "frappe.desk.search.search_link",
 				args: {
 					doctype,
 					txt,
-					filters
+					filters,
 				},
 				callback(r) {
 					resolve(r.results);
-				}
+				},
 			});
 		});
-	}
+	},
 };
diff --git a/frappe/public/js/frappe/defaults.js b/frappe/public/js/frappe/defaults.js
index 6115afb784..03258f4691 100644
--- a/frappe/public/js/frappe/defaults.js
+++ b/frappe/public/js/frappe/defaults.js
@@ -2,124 +2,107 @@
 // MIT License. See license.txt
 
 frappe.defaults = {
-	get_user_default: function(key) {
+	get_user_default: function (key) {
 		var defaults = frappe.boot.user.defaults;
 		var d = defaults[key];
-		if(!d && frappe.defaults.is_a_user_permission_key(key))
+		if (!d && frappe.defaults.is_a_user_permission_key(key))
 			d = defaults[frappe.model.scrub(key)];
-		if($.isArray(d)) d = d[0];
+		if ($.isArray(d)) d = d[0];
 
-		if(!frappe.defaults.in_user_permission(key, d)) {
+		if (!frappe.defaults.in_user_permission(key, d)) {
 			return;
 		}
 
 		return d;
 	},
-	get_user_defaults: function(key) {
+	get_user_defaults: function (key) {
 		var defaults = frappe.boot.user.defaults;
 		var d = defaults[key];
 
 		if (frappe.defaults.is_a_user_permission_key(key)) {
-			if (d && $.isArray(d) && d.length===1) {
+			if (d && $.isArray(d) && d.length === 1) {
 				// Use User Permission value when only when it has a single value
 				d = d[0];
 			} else {
 				d = defaults[key] || defaults[frappe.model.scrub(key)];
 			}
 		}
-		if(!$.isArray(d)) d = [d];
+		if (!$.isArray(d)) d = [d];
 
 		// filter out values which are not permitted to the user
-		d.filter(item => {
-			if(frappe.defaults.in_user_permission(key, item)) {
+		d.filter((item) => {
+			if (frappe.defaults.in_user_permission(key, item)) {
 				return item;
 			}
 		});
 		return d;
 	},
-	get_global_default: function(key) {
+	get_global_default: function (key) {
 		var d = frappe.sys_defaults[key];
-		if($.isArray(d)) d = d[0];
+		if ($.isArray(d)) d = d[0];
 		return d;
 	},
-	get_global_defaults: function(key) {
+	get_global_defaults: function (key) {
 		var d = frappe.sys_defaults[key];
-		if(!$.isArray(d)) d = [d];
+		if (!$.isArray(d)) d = [d];
 		return d;
 	},
-	set_default: function(key, value, callback) {
-		if(typeof value!=="string")
-			value = JSON.stringify(value);
-
-		frappe.boot.user.defaults[key] = value;
-		return frappe.call({
-			method: "frappe.client.set_default",
-			args: {
-				key: key,
-				value: value
-			},
-			callback: callback || function(r) {}
-		});
-	},
-	set_user_default_local: function(key, value) {
+	set_user_default_local: function (key, value) {
 		frappe.boot.user.defaults[key] = value;
 	},
-	get_default: function(key) {
+	get_default: function (key) {
 		var defaults = frappe.boot.user.defaults;
 		var value = defaults[key];
 		if (frappe.defaults.is_a_user_permission_key(key)) {
-			if (value && $.isArray(value) && value.length===1) {
+			if (value && $.isArray(value) && value.length === 1) {
 				value = value[0];
 			} else {
 				value = defaults[frappe.model.scrub(key)];
 			}
 		}
 
-		if(!frappe.defaults.in_user_permission(key, value)) {
+		if (!frappe.defaults.in_user_permission(key, value)) {
 			return;
 		}
 
-		if(value) {
+		if (value) {
 			try {
-				return JSON.parse(value)
-			} catch(e) {
+				return JSON.parse(value);
+			} catch (e) {
 				return value;
 			}
 		}
 	},
 
-	is_a_user_permission_key: function(key) {
-		return key.indexOf(":")===-1 && key !== frappe.model.scrub(key);
+	is_a_user_permission_key: function (key) {
+		return key.indexOf(":") === -1 && key !== frappe.model.scrub(key);
 	},
 
-	in_user_permission: function(key, value) {
+	in_user_permission: function (key, value) {
 		let user_permission = this.get_user_permissions()[frappe.model.unscrub(key)];
 
 		if (user_permission && user_permission.length) {
-
-			let doc_found = user_permission.some(perm => {
+			let doc_found = user_permission.some((perm) => {
 				return perm.doc === value;
 			});
 			return doc_found;
-
 		} else {
 			// there is no user permission for this doctype
 			// so we can allow this doc i.e., value
 			return true;
 		}
-
 	},
 
-	get_user_permissions: function() {
+	get_user_permissions: function () {
 		return this._user_permissions || {};
 	},
 
-	update_user_permissions: function() {
-		const method = 'frappe.core.doctype.user_permission.user_permission.get_user_permissions';
-		frappe.call(method).then(r => {
+	update_user_permissions: function () {
+		const method = "frappe.core.doctype.user_permission.user_permission.get_user_permissions";
+		frappe.call(method).then((r) => {
 			if (r.message) {
 				this._user_permissions = Object.assign({}, r.message);
 			}
 		});
-	}
+	},
 };
diff --git a/frappe/public/js/frappe/desk.js b/frappe/public/js/frappe/desk.js
index 72e8010605..d942c3b849 100644
--- a/frappe/public/js/frappe/desk.js
+++ b/frappe/public/js/frappe/desk.js
@@ -4,21 +4,22 @@
 
 // __('Modules') __('Domains') __('Places') __('Administration') # for translation, don't remove
 
-frappe.start_app = function() {
-	if (!frappe.Application)
-		return;
+frappe.start_app = function () {
+	if (!frappe.Application) return;
 	frappe.assets.check();
-	frappe.provide('frappe.app');
-	frappe.provide('frappe.desk');
+	frappe.provide("frappe.app");
+	frappe.provide("frappe.desk");
 	frappe.app = new frappe.Application();
 };
 
-$(document).ready(function() {
+$(document).ready(function () {
 	if (!frappe.utils.supportsES6) {
 		frappe.msgprint({
-			indicator: 'red',
-			title: __('Browser not supported'),
-			message: __('Some of the features might not work in your browser. Please update your browser to the latest version.')
+			indicator: "red",
+			title: __("Browser not supported"),
+			message: __(
+				"Some of the features might not work in your browser. Please update your browser to the latest version."
+			),
 		});
 	}
 	frappe.start_app();
@@ -33,13 +34,13 @@ frappe.Application = class Application {
 		frappe.socketio.init();
 		frappe.model.init();
 
-		if(frappe.boot.status==='failed') {
+		if (frappe.boot.status === "failed") {
 			frappe.msgprint({
 				message: frappe.boot.error,
-				title: __('Session Start Failed'),
-				indicator: 'red',
+				title: __("Session Start Failed"),
+				indicator: "red",
 			});
-			throw 'boot failed';
+			throw "boot failed";
 		}
 
 		this.setup_frappe_vue();
@@ -56,8 +57,8 @@ frappe.Application = class Application {
 		frappe.ui.keys.setup();
 
 		frappe.ui.keys.add_shortcut({
-			shortcut: 'shift+ctrl+g',
-			description: __('Switch Theme'),
+			shortcut: "shift+ctrl+g",
+			description: __("Switch Theme"),
 			action: () => {
 				if (frappe.theme_switcher && frappe.theme_switcher.dialog.is_visible) {
 					frappe.theme_switcher.hide();
@@ -65,7 +66,7 @@ frappe.Application = class Application {
 					frappe.theme_switcher = new frappe.ui.ThemeSwitcher();
 					frappe.theme_switcher.show();
 				}
-			}
+			},
 		});
 
 		frappe.ui.add_system_theme_switch_listener();
@@ -76,7 +77,7 @@ frappe.Application = class Application {
 		});
 		observer.observe(root, {
 			attributes: true,
-			attributeFilter: ['data-theme-mode']
+			attributeFilter: ["data-theme-mode"],
 		});
 
 		frappe.ui.set_theme();
@@ -86,15 +87,15 @@ frappe.Application = class Application {
 		this.set_route();
 
 		// trigger app startup
-		$(document).trigger('startup');
+		$(document).trigger("startup");
 
-		$(document).trigger('app_ready');
+		$(document).trigger("app_ready");
 
 		if (frappe.boot.messages) {
 			frappe.msgprint(frappe.boot.messages);
 		}
 
-		if (frappe.user_roles.includes('System Manager')) {
+		if (frappe.user_roles.includes("System Manager")) {
 			// delayed following requests to make boot faster
 			setTimeout(() => {
 				this.show_change_log();
@@ -103,11 +104,10 @@ frappe.Application = class Application {
 		}
 
 		if (!frappe.boot.developer_mode) {
-			let console_security_message = __("Using this console may allow attackers to impersonate you and steal your information. Do not enter or paste code that you do not understand.");
-			console.log(
-				`%c${console_security_message}`,
-				"font-size: large"
+			let console_security_message = __(
+				"Using this console may allow attackers to impersonate you and steal your information. Do not enter or paste code that you do not understand."
 			);
+			console.log(`%c${console_security_message}`, "font-size: large");
 		}
 
 		this.show_notes();
@@ -117,13 +117,15 @@ frappe.Application = class Application {
 			frappe.ui.startup_setup_dialog.show();
 		}
 
-		frappe.realtime.on("version-update", function() {
+		frappe.realtime.on("version-update", function () {
 			var dialog = frappe.msgprint({
-				message:__("The application has been updated to a new version, please refresh this page"),
-				indicator: 'green',
-				title: __('Version Updated')
+				message: __(
+					"The application has been updated to a new version, please refresh this page"
+				),
+				indicator: "green",
+				title: __("Version Updated"),
 			});
-			dialog.set_primary_action(__("Refresh"), function() {
+			dialog.set_primary_action(__("Refresh"), function () {
 				location.reload(true);
 			});
 			dialog.get_close_btn().toggle(false);
@@ -133,9 +135,9 @@ frappe.Application = class Application {
 		this.setup_build_events();
 
 		if (frappe.sys_defaults.email_user_password) {
-			var email_list =  frappe.sys_defaults.email_user_password.split(',');
+			var email_list = frappe.sys_defaults.email_user_password.split(",");
 			for (var u in email_list) {
-				if (email_list[u]===frappe.user.name) {
+				if (email_list[u] === frappe.user.name) {
 					this.set_password(email_list[u]);
 				}
 			}
@@ -145,32 +147,32 @@ frappe.Application = class Application {
 		this.link_preview = new frappe.ui.LinkPreview();
 
 		if (!frappe.boot.developer_mode) {
-			setInterval(function() {
+			setInterval(function () {
 				frappe.call({
-					method: 'frappe.core.page.background_jobs.background_jobs.get_scheduler_status',
-					callback: function(r) {
+					method: "frappe.core.page.background_jobs.background_jobs.get_scheduler_status",
+					callback: function (r) {
 						if (r.message[0] == __("Inactive")) {
-							frappe.call('frappe.utils.scheduler.activate_scheduler');
+							frappe.call("frappe.utils.scheduler.activate_scheduler");
 						}
-					}
+					},
 				});
 			}, 300000); // check every 5 minutes
 
 			if (frappe.user.has_role("System Manager")) {
-				setInterval(function() {
+				setInterval(function () {
 					frappe.call({
-						method: 'frappe.core.doctype.log_settings.log_settings.has_unseen_error_log',
+						method: "frappe.core.doctype.log_settings.log_settings.has_unseen_error_log",
 						args: {
-							user: frappe.session.user
+							user: frappe.session.user,
 						},
-						callback: function(r) {
+						callback: function (r) {
 							if (r.message.show_alert) {
 								frappe.show_alert({
-									indicator: 'red',
-									message: r.message.message
+									indicator: "red",
+									message: r.message.message,
 								});
 							}
-						}
+						},
 					});
 				}, 600000); // check every 10 minutes
 			}
@@ -186,8 +188,8 @@ frappe.Application = class Application {
 			// route to home page
 			frappe.router.route();
 		}
-		frappe.after_ajax(() => frappe.flags.setting_original_route = false);
-		frappe.router.on('change', () => {
+		frappe.after_ajax(() => (frappe.flags.setting_original_route = false));
+		frappe.router.on("change", () => {
 			$(".tooltip").hide();
 		});
 	}
@@ -198,67 +200,75 @@ frappe.Application = class Application {
 	}
 
 	set_password(user) {
-		var me=this;
+		var me = this;
 		frappe.call({
-			method: 'frappe.core.doctype.user.user.get_email_awaiting',
+			method: "frappe.core.doctype.user.user.get_email_awaiting",
 			args: {
-				"user": user
+				user: user,
 			},
-			callback: function(email_account) {
+			callback: function (email_account) {
 				email_account = email_account["message"];
 				if (email_account) {
 					var i = 0;
 					if (i < email_account.length) {
-						me.email_password_prompt( email_account, user, i);
+						me.email_password_prompt(email_account, user, i);
 					}
 				}
-			}
+			},
 		});
 	}
 
-	email_password_prompt(email_account,user,i) {
+	email_password_prompt(email_account, user, i) {
 		var me = this;
 		const email_id = email_account[i]["email_id"];
 		let d = new frappe.ui.Dialog({
-			title: __('Password missing in Email Account'),
+			title: __("Password missing in Email Account"),
 			fields: [
 				{
-					'fieldname': 'password',
-					'fieldtype': 'Password',
-					'label': __('Please enter the password for: {0}', [email_id], "Email Account"),
-					'reqd': 1
+					fieldname: "password",
+					fieldtype: "Password",
+					label: __(
+						"Please enter the password for: {0}",
+						[email_id],
+						"Email Account"
+					),
+					reqd: 1,
 				},
 				{
-					"fieldname": "submit",
-					"fieldtype": "Button",
-					"label": __("Submit", null, "Submit password for Email Account")
-				}
-			]
+					fieldname: "submit",
+					fieldtype: "Button",
+					label: __("Submit", null, "Submit password for Email Account"),
+				},
+			],
 		});
-		d.get_input("submit").on("click", function() {
+		d.get_input("submit").on("click", function () {
 			//setup spinner
 			d.hide();
 			var s = new frappe.ui.Dialog({
 				title: __("Checking one moment"),
-				fields: [{
-					"fieldtype": "HTML",
-					"fieldname": "checking"
-				}]
+				fields: [
+					{
+						fieldtype: "HTML",
+						fieldname: "checking",
+					},
+				],
 			});
 			s.fields_dict.checking.$wrapper.html('');
 			s.show();
 			frappe.call({
-				method: 'frappe.email.doctype.email_account.email_account.set_email_password',
+				method: "frappe.email.doctype.email_account.email_account.set_email_password",
 				args: {
-					"email_account": email_account[i]["email_account"],
-					"user": user,
-					"password": d.get_value("password")
+					email_account: email_account[i]["email_account"],
+					password: d.get_value("password"),
 				},
-				callback: function(passed) {
+				callback: function (passed) {
 					s.hide();
-					d.hide();//hide waiting indication
+					d.hide(); //hide waiting indication
 					if (!passed["message"]) {
-						frappe.show_alert({message: __("Login Failed please try again"), indicator: 'error'}, 5);
+						frappe.show_alert(
+							{ message: __("Login Failed please try again"), indicator: "error" },
+							5
+						);
 						me.email_password_prompt(email_account, user, i);
 					} else {
 						if (i + 1 < email_account.length) {
@@ -266,14 +276,13 @@ frappe.Application = class Application {
 							me.email_password_prompt(email_account, user, i);
 						}
 					}
-
-				}
+				},
 			});
 		});
 		d.show();
 	}
 	load_bootinfo() {
-		if(frappe.boot) {
+		if (frappe.boot) {
 			this.setup_workspaces();
 			frappe.model.sync(frappe.boot.docs);
 			this.check_metadata_cache_status();
@@ -281,7 +290,7 @@ frappe.Application = class Application {
 			this.sync_pages();
 			frappe.router.setup();
 			this.setup_moment();
-			if(frappe.boot.print_css) {
+			if (frappe.boot.print_css) {
 				frappe.dom.set_style(frappe.boot.print_css, "print-style");
 			}
 			frappe.user.name = frappe.boot.user.name;
@@ -295,7 +304,7 @@ frappe.Application = class Application {
 		frappe.modules = {};
 		frappe.workspaces = {};
 		for (let page of frappe.boot.allowed_workspaces || []) {
-			frappe.modules[page.module]=page;
+			frappe.modules[page.module] = page;
 			frappe.workspaces[frappe.router.slug(page.name)] = page;
 		}
 	}
@@ -303,13 +312,16 @@ frappe.Application = class Application {
 	load_user_permissions() {
 		frappe.defaults.update_user_permissions();
 
-		frappe.realtime.on('update_user_permissions', frappe.utils.debounce(() => {
-			frappe.defaults.update_user_permissions();
-		}, 500));
+		frappe.realtime.on(
+			"update_user_permissions",
+			frappe.utils.debounce(() => {
+				frappe.defaults.update_user_permissions();
+			}, 500)
+		);
 	}
 
 	check_metadata_cache_status() {
-		if(frappe.boot.metadata_version != localStorage.metadata_version) {
+		if (frappe.boot.metadata_version != localStorage.metadata_version) {
 			frappe.assets.clear_local_storage();
 			frappe.assets.init_local_storage();
 		}
@@ -325,56 +337,71 @@ frappe.Application = class Application {
 		frappe.user_roles = frappe.boot.user.roles;
 		frappe.sys_defaults = frappe.boot.sysdefaults;
 
-		frappe.ui.py_date_format = frappe.boot.sysdefaults.date_format.replace('dd', '%d').replace('mm', '%m').replace('yyyy', '%Y');
+		frappe.ui.py_date_format = frappe.boot.sysdefaults.date_format
+			.replace("dd", "%d")
+			.replace("mm", "%m")
+			.replace("yyyy", "%Y");
 		frappe.boot.user.last_selected_values = {};
 
 		// Proxy for user globals
 		Object.defineProperties(window, {
-			'user': {
-				get: function() {
-					console.warn('Please use `frappe.session.user` instead of `user`. It will be deprecated soon.');
+			user: {
+				get: function () {
+					console.warn(
+						"Please use `frappe.session.user` instead of `user`. It will be deprecated soon."
+					);
 					return frappe.session.user;
-				}
+				},
 			},
-			'user_fullname': {
-				get: function() {
-					console.warn('Please use `frappe.session.user_fullname` instead of `user_fullname`. It will be deprecated soon.');
+			user_fullname: {
+				get: function () {
+					console.warn(
+						"Please use `frappe.session.user_fullname` instead of `user_fullname`. It will be deprecated soon."
+					);
 					return frappe.session.user;
-				}
+				},
 			},
-			'user_email': {
-				get: function() {
-					console.warn('Please use `frappe.session.user_email` instead of `user_email`. It will be deprecated soon.');
+			user_email: {
+				get: function () {
+					console.warn(
+						"Please use `frappe.session.user_email` instead of `user_email`. It will be deprecated soon."
+					);
 					return frappe.session.user_email;
-				}
+				},
 			},
-			'user_defaults': {
-				get: function() {
-					console.warn('Please use `frappe.user_defaults` instead of `user_defaults`. It will be deprecated soon.');
+			user_defaults: {
+				get: function () {
+					console.warn(
+						"Please use `frappe.user_defaults` instead of `user_defaults`. It will be deprecated soon."
+					);
 					return frappe.user_defaults;
-				}
+				},
 			},
-			'roles': {
-				get: function() {
-					console.warn('Please use `frappe.user_roles` instead of `roles`. It will be deprecated soon.');
+			roles: {
+				get: function () {
+					console.warn(
+						"Please use `frappe.user_roles` instead of `roles`. It will be deprecated soon."
+					);
 					return frappe.user_roles;
-				}
+				},
 			},
-			'sys_defaults': {
-				get: function() {
-					console.warn('Please use `frappe.sys_defaults` instead of `sys_defaults`. It will be deprecated soon.');
+			sys_defaults: {
+				get: function () {
+					console.warn(
+						"Please use `frappe.sys_defaults` instead of `sys_defaults`. It will be deprecated soon."
+					);
 					return frappe.user_roles;
-				}
-			}
+				},
+			},
 		});
 	}
 	sync_pages() {
 		// clear cached pages if timestamp is not found
-		if(localStorage["page_info"]) {
+		if (localStorage["page_info"]) {
 			frappe.boot.allowed_pages = [];
 			var page_info = JSON.parse(localStorage["page_info"]);
-			$.each(frappe.boot.page_info, function(name, p) {
-				if(!page_info[name] || (page_info[name].modified != p.modified)) {
+			$.each(frappe.boot.page_info, function (name, p) {
+				if (!page_info[name] || page_info[name].modified != p.modified) {
 					delete localStorage["_page:" + name];
 				}
 				frappe.boot.allowed_pages.push(name);
@@ -385,95 +412,98 @@ frappe.Application = class Application {
 		localStorage["page_info"] = JSON.stringify(frappe.boot.page_info);
 	}
 	set_as_guest() {
-		frappe.session.user = 'Guest';
-		frappe.session.user_email = '';
-		frappe.session.user_fullname = 'Guest';
+		frappe.session.user = "Guest";
+		frappe.session.user_email = "";
+		frappe.session.user_fullname = "Guest";
 
 		frappe.user_defaults = {};
-		frappe.user_roles = ['Guest'];
+		frappe.user_roles = ["Guest"];
 		frappe.sys_defaults = {};
 	}
 	make_page_container() {
 		if ($("#body").length) {
 			$(".splash").remove();
-			frappe.temp_container = $("
 		
@@ -171,7 +171,8 @@ export default {
 			default: () => ({
 				max_file_size: null, // 2048 -> 2KB
 				max_number_of_files: null,
-				allowed_file_types: [] // ['image/*', 'video/*', '.jpg', '.gif', '.pdf']
+				allowed_file_types: [], // ['image/*', 'video/*', '.jpg', '.gif', '.pdf'],
+				crop_image_aspect_ratio: null // 1, 16 / 9, 4 / 3, NaN (free)
 			})
 		},
 		attach_doc_image: {
@@ -203,7 +204,8 @@ export default {
 			allow_web_link: true,
 			google_drive_settings: {
 				enabled: false
-			}
+			},
+			wrapper_ready: false
 		}
 	},
 	created() {
@@ -220,11 +222,14 @@ export default {
 			});
 		}
 		if (this.restrictions.max_file_size == null) {
-			frappe.call('frappe.core.doctype.file.file.get_max_file_size')
+			frappe.call('frappe.core.api.file.get_max_file_size')
 				.then(res => {
 					this.restrictions.max_file_size = Number(res.message);
 				});
 		}
+		if (this.restrictions.max_number_of_files == null && this.doctype) {
+			this.restrictions.max_number_of_files = frappe.get_meta(self.doctype).max_attachments;
+		}
 	},
 	watch: {
 		files(newvalue, oldvalue) {
@@ -281,16 +286,32 @@ export default {
 				return file;
 			});
 		},
+		show_max_files_number_warning(file) {
+			console.warn(
+				`File skipped because it exceeds the allowed specified limit of ${max_number_of_files} uploads`,
+				file,
+			);
+			if (this.doctype) {
+				MSG = __('File "{0}" was skipped because only {1} uploads are allowed for DocType "{2}"', [file.name, max_number_of_files, this.doctype])
+			} else {
+				MSG = __('File "{0}" was skipped because only {1} uploads are allowed', [file.name, max_number_of_files])
+			}
+			frappe.show_alert({
+				message: MSG,
+				indicator: "orange",
+			});
+		},
 		add_files(file_array) {
 			let files = Array.from(file_array)
 				.filter(this.check_restrictions)
 				.map(file => {
 					let is_image = file.type.startsWith('image');
+					let size_kb = file.size / 1024;
 					return {
 						file_obj: file,
 						cropper_file: file,
 						crop_box_data: null,
-						optimize: this.attach_doc_image ? true : false,
+						optimize: size_kb > 200 && is_image && !file.type.includes('svg'),
 						name: file.name,
 						doc: null,
 						progress: 0,
@@ -299,24 +320,35 @@ export default {
 						request_succeeded: false,
 						error_message: null,
 						uploading: false,
-						private: !is_image
-					}
+						private: true
+					};
 				});
+
+			// pop extra files as per FileUploader.restrictions.max_number_of_files
+			max_number_of_files = this.restrictions.max_number_of_files;
+			if (max_number_of_files && files.length > max_number_of_files) {
+				files.slice(max_number_of_files).forEach(file => {
+					this.show_max_files_number_warning(file, this.doctype);
+				});
+
+				files = files.slice(0, max_number_of_files);
+			}
+
 			this.files = this.files.concat(files);
-			if(this.files.length != 0 && this.attach_doc_image) {
-				this.toggle_image_cropper(0);
+			// if only one file is allowed and crop_image_aspect_ratio is set, open cropper immediately
+			if (this.files.length === 1 && !this.allow_multiple && this.restrictions.crop_image_aspect_ratio != null) {
+				if (!this.files[0].file_obj.type.includes('svg')) {
+					this.toggle_image_cropper(0);
+				}
 			}
 		},
 		check_restrictions(file) {
-			let { max_file_size, allowed_file_types } = this.restrictions;
-
-			let mime_type = file.type;
-			let extension = '.' + file.name.split('.').pop();
+			let { max_file_size, allowed_file_types = [] } = this.restrictions;
 
 			let is_correct_type = true;
 			let valid_file_size = true;
 
-			if (allowed_file_types.length) {
+			if (allowed_file_types && allowed_file_types.length) {
 				is_correct_type = allowed_file_types.some((type) => {
 					// is this is a mime-type
 					if (type.includes('/')) {
diff --git a/frappe/public/js/frappe/file_uploader/ImageCropper.vue b/frappe/public/js/frappe/file_uploader/ImageCropper.vue
index 09b50390fe..0711cc7cc9 100644
--- a/frappe/public/js/frappe/file_uploader/ImageCropper.vue
+++ b/frappe/public/js/frappe/file_uploader/ImageCropper.vue
@@ -1,12 +1,39 @@
 
@@ -15,22 +42,31 @@
 import Cropper from "cropperjs";
 export default {
 	name: "ImageCropper",
-	props: ["file", "attach_doc_image"],
+	props: ["file", "fixed_aspect_ratio"],
 	data() {
+		let aspect_ratio =
+			this.fixed_aspect_ratio != null ? this.fixed_aspect_ratio : NaN;
 		return {
 			src: null,
 			cropper: null,
-			image: null
+			image: null,
+			aspect_ratio
 		};
 	},
+	watch: {
+		aspect_ratio(value) {
+			if (this.cropper) {
+				this.cropper.setAspectRatio(value);
+			}
+		}
+	},
 	mounted() {
 		if (window.FileReader) {
 			let fr = new FileReader();
 			fr.onload = () => (this.src = fr.result);
 			fr.readAsDataURL(this.file.cropper_file);
 		}
-		aspect_ratio = this.attach_doc_image ? 1 : NaN;
-		crop_box = this.file.crop_box_data;
+		let crop_box = this.file.crop_box_data;
 		this.image = this.$refs.image;
 		this.image.onload = () => {
 			this.cropper = new Cropper(this.image, {
@@ -38,13 +74,31 @@ export default {
 				scalable: false,
 				viewMode: 1,
 				data: crop_box,
-				aspectRatio: aspect_ratio
+				aspectRatio: this.aspect_ratio
 			});
+			window.cropper = this.cropper;
 		};
 	},
 	computed: {
-		crop_button_text() {
-			return this.attach_doc_image ? "Upload" : "Crop";
+		aspect_ratio_buttons() {
+			return [
+				{
+					label: __("1:1"),
+					value: 1
+				},
+				{
+					label: __("4:3"),
+					value: 4 / 3
+				},
+				{
+					label: __("16:9"),
+					value: 16 / 9
+				},
+				{
+					label: __("Free"),
+					value: NaN
+				}
+			];
 		}
 	},
 	methods: {
@@ -58,9 +112,6 @@ export default {
 				});
 				this.file.file_obj = cropped_file_obj;
 				this.$emit("toggle_image_cropper");
-				if(this.attach_doc_image) {
-					this.$emit("upload_after_crop");
-				}
 			}, file_type);
 		}
 	}
@@ -75,6 +126,8 @@ img {
 
 .image-cropper-actions {
 	display: flex;
-	justify-content: flex-end;
+	align-items: center;
+	justify-content: space-between;
+	margin-top: var(--margin-md);
 }
 
diff --git a/frappe/public/js/frappe/file_uploader/index.js b/frappe/public/js/frappe/file_uploader/index.js
index ec90b19a1a..3508a44bc0 100644
--- a/frappe/public/js/frappe/file_uploader/index.js
+++ b/frappe/public/js/frappe/file_uploader/index.js
@@ -1,4 +1,4 @@
-import FileUploaderComponent from './FileUploader.vue';
+import FileUploaderComponent from "./FileUploader.vue";
 
 export default class FileUploader {
 	constructor({
@@ -10,76 +10,83 @@ export default class FileUploader {
 		fieldname,
 		files,
 		folder,
-		restrictions,
+		restrictions = {},
 		upload_notes,
 		allow_multiple,
 		as_dataurl,
 		disable_file_browser,
+		dialog_title,
 		attach_doc_image,
-		frm
+		frm,
 	} = {}) {
-
 		frm && frm.attachments.max_reached(true);
 
 		if (!wrapper) {
-			this.make_dialog();
+			this.make_dialog(dialog_title);
 		} else {
 			this.wrapper = wrapper.get ? wrapper.get(0) : wrapper;
 		}
 
-		if (attach_doc_image) {
-			restrictions.allowed_file_types = ['image/jpeg', 'image/png'];
-		}
-
 		this.$fileuploader = new Vue({
 			el: this.wrapper,
-			render: h => h(FileUploaderComponent, {
-				props: {
-					show_upload_button: !Boolean(this.dialog),
-					doctype,
-					docname,
-					fieldname,
-					method,
-					folder,
-					on_success,
-					restrictions,
-					upload_notes,
-					allow_multiple,
-					as_dataurl,
-					disable_file_browser,
-					attach_doc_image,
-				}
-			})
+			render: (h) =>
+				h(FileUploaderComponent, {
+					props: {
+						show_upload_button: !Boolean(this.dialog),
+						doctype,
+						docname,
+						fieldname,
+						method,
+						folder,
+						on_success,
+						restrictions,
+						upload_notes,
+						allow_multiple,
+						as_dataurl,
+						disable_file_browser,
+						attach_doc_image,
+					},
+				}),
 		});
 
 		this.uploader = this.$fileuploader.$children[0];
 
-		this.uploader.$watch('files', (files) => {
-			let all_private = files.every(file => file.private);
-			if (this.dialog) {
-				this.dialog.set_secondary_action_label(all_private ? __('Set all public') : __('Set all private'));
-			}
-		}, { deep: true });
+		if (!this.dialog) {
+			this.uploader.wrapper_ready = true;
+		}
 
-		this.uploader.$watch('trigger_upload', (trigger_upload) => {
+		this.uploader.$watch(
+			"files",
+			(files) => {
+				let all_private = files.every((file) => file.private);
+				if (this.dialog) {
+					this.dialog.set_secondary_action_label(
+						all_private ? __("Set all public") : __("Set all private")
+					);
+				}
+			},
+			{ deep: true }
+		);
+
+		this.uploader.$watch("trigger_upload", (trigger_upload) => {
 			if (trigger_upload) {
 				this.upload_files();
 			}
 		});
 
-		this.uploader.$watch('close_dialog', (close_dialog) => {
+		this.uploader.$watch("close_dialog", (close_dialog) => {
 			if (close_dialog) {
 				this.dialog && this.dialog.hide();
 			}
 		});
 
-		this.uploader.$watch('hide_dialog_footer', (hide_dialog_footer) => {
+		this.uploader.$watch("hide_dialog_footer", (hide_dialog_footer) => {
 			if (hide_dialog_footer) {
-				this.dialog && this.dialog.footer.addClass('hide');
-				this.dialog.$wrapper.data('bs.modal')._config.backdrop = 'static';
+				this.dialog && this.dialog.footer.addClass("hide");
+				this.dialog.$wrapper.data("bs.modal")._config.backdrop = "static";
 			} else {
-				this.dialog && this.dialog.footer.removeClass('hide');
-				this.dialog.$wrapper.data('bs.modal')._config.backdrop = true;
+				this.dialog && this.dialog.footer.removeClass("hide");
+				this.dialog.$wrapper.data("bs.modal")._config.backdrop = true;
 			}
 		});
 
@@ -89,26 +96,29 @@ export default class FileUploader {
 	}
 
 	upload_files() {
-		this.dialog && this.dialog.get_primary_btn().prop('disabled', true);
-		this.dialog && this.dialog.get_secondary_btn().prop('disabled', true);
+		this.dialog && this.dialog.get_primary_btn().prop("disabled", true);
+		this.dialog && this.dialog.get_secondary_btn().prop("disabled", true);
 		return this.uploader.upload_files();
 	}
 
-	make_dialog() {
+	make_dialog(title) {
 		this.dialog = new frappe.ui.Dialog({
-			title: __('Upload'),
-			primary_action_label: __('Upload'),
+			title: title || __("Upload"),
+			primary_action_label: __("Upload"),
 			primary_action: () => this.upload_files(),
-			secondary_action_label: __('Set all private'),
+			secondary_action_label: __("Set all private"),
 			secondary_action: () => {
 				this.uploader.toggle_all_private();
-			}
+			},
+			on_page_show: () => {
+				this.uploader.wrapper_ready = true;
+			},
 		});
 
 		this.wrapper = this.dialog.body;
 		this.dialog.show();
-		this.dialog.$wrapper.on('hidden.bs.modal', function() {
-			$(this).data('bs.modal', null);
+		this.dialog.$wrapper.on("hidden.bs.modal", function () {
+			$(this).data("bs.modal", null);
 			$(this).remove();
 		});
 	}
diff --git a/frappe/public/js/frappe/form/column.js b/frappe/public/js/frappe/form/column.js
index 27231da7e5..939ce6e2d7 100644
--- a/frappe/public/js/frappe/form/column.js
+++ b/frappe/public/js/frappe/form/column.js
@@ -26,8 +26,7 @@ export default class Column {
 				
-			`)
-				.appendTo(this.wrapper);
+			`).appendTo(this.wrapper);
 		}
 	}
 
@@ -40,10 +39,9 @@ export default class Column {
 			.removeClass()
 			.addClass("form-column")
 			.addClass("col-sm-" + colspan);
-
 	}
 
 	refresh() {
 		this.section.refresh();
 	}
-}
\ No newline at end of file
+}
diff --git a/frappe/public/js/frappe/form/controls/attach.js b/frappe/public/js/frappe/form/controls/attach.js
index a91058a208..8b0e19ad64 100644
--- a/frappe/public/js/frappe/form/controls/attach.js
+++ b/frappe/public/js/frappe/form/controls/attach.js
@@ -5,12 +5,12 @@ frappe.ui.form.ControlAttach = class ControlAttach extends frappe.ui.form.Contro
 			.html(__("Attach"))
 			.prependTo(me.input_area)
 			.on({
-				click: function() {
+				click: function () {
 					me.on_attach_click();
 				},
-				attach_doc_image: function() {
+				attach_doc_image: function () {
 					me.on_attach_doc_image();
-				}
+				},
 			});
 		this.$value = $(
 			`
        
@@ -19,10 +19,11 @@ frappe.ui.form.ControlAttach = class ControlAttach extends frappe.ui.form.Contro
 					
 				
        
 				
-			
        `)
+			
        `
+		)
 			.prependTo(me.input_area)
 			.toggle(false);
 		this.input = this.$input.get(0);
@@ -34,13 +35,13 @@ frappe.ui.form.ControlAttach = class ControlAttach extends frappe.ui.form.Contro
 	}
 	clear_attachment() {
 		let me = this;
-		if(this.frm) {
+		if (this.frm) {
 			me.parse_validate_and_set_in_model(null);
 			me.refresh();
 			me.frm.attachments.remove_attachment_by_filename(me.value, async () => {
 				await me.parse_validate_and_set_in_model(null);
 				me.refresh();
-				me.frm.doc.docstatus == 1 ? me.frm.save('Update') : me.frm.save();
+				me.frm.doc.docstatus == 1 ? me.frm.save("Update") : me.frm.save();
 			});
 		} else {
 			this.dataurl = null;
@@ -61,16 +62,18 @@ frappe.ui.form.ControlAttach = class ControlAttach extends frappe.ui.form.Contro
 	}
 	on_attach_doc_image() {
 		this.set_upload_options();
-		this.upload_options["attach_doc_image"] = true;
+		this.upload_options.restrictions.allowed_file_types = ["image/*"];
+		this.upload_options.restrictions.crop_image_aspect_ratio = 1;
 		this.file_uploader = new frappe.ui.FileUploader(this.upload_options);
 	}
 	set_upload_options() {
 		let options = {
 			allow_multiple: false,
-			on_success: file => {
+			on_success: (file) => {
 				this.on_upload_complete(file);
 				this.toggle_reload_button();
-			}
+			},
+			restrictions: {},
 		};
 
 		if (this.frm) {
@@ -87,7 +90,7 @@ frappe.ui.form.ControlAttach = class ControlAttach extends frappe.ui.form.Contro
 
 	set_input(value, dataurl) {
 		this.value = value;
-		if(this.value) {
+		if (this.value) {
 			this.$input.toggle(false);
 			// value can also be using this format: FILENAME,DATA_URL
 			// Important: We have to be careful because normal filenames may also contain ","
@@ -95,9 +98,11 @@ frappe.ui.form.ControlAttach = class ControlAttach extends frappe.ui.form.Contro
 			let filename;
 			if (file_url_parts) {
 				filename = file_url_parts[1];
-				dataurl = file_url_parts[2] + ':' + file_url_parts[3];
+				dataurl = file_url_parts[2] + ":" + file_url_parts[3];
 			}
-			this.$value.toggle(true).find(".attached-file-link")
+			this.$value
+				.toggle(true)
+				.find(".attached-file-link")
 				.html(filename || this.value)
 				.attr("href", dataurl || this.value);
 		} else {
@@ -111,16 +116,17 @@ frappe.ui.form.ControlAttach = class ControlAttach extends frappe.ui.form.Contro
 	}
 
 	async on_upload_complete(attachment) {
-		if(this.frm) {
+		if (this.frm) {
 			await this.parse_validate_and_set_in_model(attachment.file_url);
 			this.frm.attachments.update_attachment(attachment);
-			this.frm.doc.docstatus == 1 ? this.frm.save('Update') : this.frm.save();
+			this.frm.doc.docstatus == 1 ? this.frm.save("Update") : this.frm.save();
 		}
 		this.set_value(attachment.file_url);
 	}
 
 	toggle_reload_button() {
-		this.$value.find('[data-action="reload_attachment"]')
+		this.$value
+			.find('[data-action="reload_attachment"]')
 			.toggle(this.file_uploader && this.file_uploader.uploader.files.length > 0);
 	}
 };
diff --git a/frappe/public/js/frappe/form/controls/attach_image.js b/frappe/public/js/frappe/form/controls/attach_image.js
index 7c24ec9551..21c17f9754 100644
--- a/frappe/public/js/frappe/form/controls/attach_image.js
+++ b/frappe/public/js/frappe/form/controls/attach_image.js
@@ -2,10 +2,10 @@ frappe.ui.form.ControlAttachImage = class ControlAttachImage extends frappe.ui.f
 	make_input() {
 		super.make_input();
 
-		let $file_link = this.$value.find('.attached-file-link');
+		let $file_link = this.$value.find(".attached-file-link");
 		$file_link.popover({
-			trigger: 'hover',
-			placement: 'top',
+			trigger: "hover",
+			placement: "top",
 			content: () => {
 				return `
        
 					
 				
        `;
 			},
-			html: true
+			html: true,
 		});
 	}
 	set_upload_options() {
 		super.set_upload_options();
-		this.upload_options.restrictions = {};
-		this.upload_options.restrictions.allowed_file_types = ['image/*'];
+		this.upload_options.restrictions.allowed_file_types = ["image/*"];
 	}
 };
diff --git a/frappe/public/js/frappe/form/controls/autocomplete.js b/frappe/public/js/frappe/form/controls/autocomplete.js
index a509af4121..27bf75e807 100644
--- a/frappe/public/js/frappe/form/controls/autocomplete.js
+++ b/frappe/public/js/frappe/form/controls/autocomplete.js
@@ -1,7 +1,7 @@
-import Awesomplete from 'awesomplete';
+import Awesomplete from "awesomplete";
 
 frappe.ui.form.ControlAutocomplete = class ControlAutoComplete extends frappe.ui.form.ControlData {
-	static trigger_change_on_input_event = false
+	static trigger_change_on_input_event = false;
 	make_input() {
 		super.make_input();
 		this.setup_awesomplete();
@@ -19,7 +19,7 @@ frappe.ui.form.ControlAutocomplete = class ControlAutoComplete extends frappe.ui
 		if (value == null) {
 			return "";
 		} else if (this._data && this._data.length) {
-			const item = this._data.find(i => i.value == value);
+			const item = this._data.find((i) => i.value == value);
 			return item ? item.label : value;
 		} else {
 			return value;
@@ -29,7 +29,7 @@ frappe.ui.form.ControlAutocomplete = class ControlAutoComplete extends frappe.ui
 	get_input_value() {
 		if (this.$input) {
 			const label = this.$input.val();
-			const item = this._data?.find(i => i.label == label);
+			const item = this._data?.find((i) => i.label == label);
 			return item ? item.value : label;
 		}
 	}
@@ -41,22 +41,22 @@ frappe.ui.form.ControlAutocomplete = class ControlAutoComplete extends frappe.ui
 			maxItems: this.df.max_items || 99,
 			autoFirst: true,
 			list: this.get_data(),
-			data: function(item) {
-				if (typeof item !== 'object') {
+			data: function (item) {
+				if (typeof item !== "object") {
 					var d = { value: item };
 					item = d;
 				}
 
 				return {
 					label: item.label || item.value,
-					value: item.value
+					value: item.value,
 				};
 			},
-			filter: function(item, input) {
+			filter: function (item, input) {
 				let hay = item.label + item.value;
 				return Awesomplete.FILTER_CONTAINS(hay, input);
 			},
-			item: function(item) {
+			item: function (item) {
 				var d = this.get_item(item.value);
 				if (!d) {
 					d = item;
@@ -67,20 +67,20 @@ frappe.ui.form.ControlAutocomplete = class ControlAutoComplete extends frappe.ui
 				}
 
 				var _label = me.translate_values ? __(d.label) : d.label;
-				var html = '' + _label + '';
+				var html = "" + _label + "";
 				if (d.description) {
-					html += '
        ' + __(d.description) + '';
+					html += '
        ' + __(d.description) + "";
 				}
 
-				return $('
      • ')
-					.data('item.autocomplete', d)
-					.prop('aria-selected', 'false')
-					.html('
        ' + html + '
        ')
+				return $("
      • ")
+					.data("item.autocomplete", d)
+					.prop("aria-selected", "false")
+					.html("
        " + html + "
        ")
 					.get(0);
 			},
 			sort: () => {
 				return 0;
-			}
+			},
 		};
 	}
 
@@ -97,42 +97,41 @@ frappe.ui.form.ControlAutocomplete = class ControlAutoComplete extends frappe.ui
 	}
 
 	setup_awesomplete() {
-		this.awesomplete = new Awesomplete(
-			this.input,
-			this.get_awesomplete_settings()
-		);
+		this.awesomplete = new Awesomplete(this.input, this.get_awesomplete_settings());
 
-		$(this.input_area)
-			.find('.awesomplete ul')
-			.css('min-width', '100%');
+		$(this.input_area).find(".awesomplete ul").css("min-width", "100%");
 
 		this.init_option_cache();
 
-		this.$input.on('input', frappe.utils.debounce((e) => {
-			const cached_options = this.$input.cache[this.doctype][this.df.fieldname][e.target.value];
-			if (cached_options && cached_options.length) {
-				this.set_data(cached_options);
-			} else if (this.get_query || this.df.get_query) {
-				this.execute_query_if_exists(e.target.value);
-			} else {
-				this.awesomplete.list = this.get_data();
-			}
-		}, 500));
+		this.$input.on(
+			"input",
+			frappe.utils.debounce((e) => {
+				const cached_options =
+					this.$input.cache[this.doctype][this.df.fieldname][e.target.value];
+				if (cached_options && cached_options.length) {
+					this.set_data(cached_options);
+				} else if (this.get_query || this.df.get_query) {
+					this.execute_query_if_exists(e.target.value);
+				} else {
+					this.awesomplete.list = this.get_data();
+				}
+			}, 500)
+		);
 
-		this.$input.on('focus', () => {
+		this.$input.on("focus", () => {
 			if (!this.$input.val()) {
-				this.$input.val('');
-				this.$input.trigger('input');
+				this.$input.val("");
+				this.$input.trigger("input");
 			}
 		});
 
 		this.$input.on("blur", () => {
-			if(this.selected) {
+			if (this.selected) {
 				this.selected = false;
 				return;
 			}
 			var value = this.get_input_value();
-			if(value!==this.last_value) {
+			if (value !== this.last_value) {
 				this.parse_validate_and_set_in_model(value);
 			}
 		});
@@ -145,35 +144,35 @@ frappe.ui.form.ControlAutocomplete = class ControlAutoComplete extends frappe.ui
 			this.autocomplete_open = false;
 		});
 
-		this.$input.on('awesomplete-selectcomplete', () => {
-			this.$input.trigger('change');
+		this.$input.on("awesomplete-selectcomplete", () => {
+			this.$input.trigger("change");
 		});
 	}
 
 	validate(value) {
 		if (this.df.ignore_validation) {
-			return value || '';
+			return value || "";
 		}
-		let valid_values = this.awesomplete._list.map(d => d.value);
+		let valid_values = this.awesomplete._list.map((d) => d.value);
 		if (!valid_values.length) {
 			return value;
 		}
 		if (valid_values.includes(value)) {
 			return value;
 		} else {
-			return '';
+			return "";
 		}
 	}
 
 	parse_options(options) {
-		if (typeof options === 'string' && options[0] === '[') {
+		if (typeof options === "string" && options[0] === "[") {
 			options = frappe.utils.parse_json(options);
 		}
-		if (typeof options === 'string') {
-			options = options.split('\n');
+		if (typeof options === "string") {
+			options = options.split("\n");
 		}
-		if (typeof options[0] === 'string') {
-			options = options.map(o => ({ label: o, value: o }));
+		if (typeof options[0] === "string") {
+			options = options.map((o) => ({ label: o, value: o }));
 		}
 		return options;
 	}
@@ -186,8 +185,8 @@ frappe.ui.form.ControlAutocomplete = class ControlAutoComplete extends frappe.ui
 			return;
 		}
 
-		let set_nulls = function(obj) {
-			$.each(obj, function(key, value) {
+		let set_nulls = function (obj) {
+			$.each(obj, function (key, value) {
 				if (value !== undefined) {
 					obj[key] = value;
 				}
@@ -195,7 +194,7 @@ frappe.ui.form.ControlAutocomplete = class ControlAutoComplete extends frappe.ui
 			return obj;
 		};
 
-		let process_query_object = function(obj) {
+		let process_query_object = function (obj) {
 			if (obj.query) {
 				args.query = obj.query;
 			}
@@ -217,11 +216,7 @@ frappe.ui.form.ControlAutocomplete = class ControlAutoComplete extends frappe.ui
 			args.query = get_query;
 		} else {
 			// get_query by function
-			var q = get_query(
-				(this.frm && this.frm.doc) || this.doc,
-				this.doctype,
-				this.docname
-			);
+			var q = get_query((this.frm && this.frm.doc) || this.doc, this.doctype, this.docname);
 
 			if (typeof q === "string") {
 				// returns a string
@@ -237,13 +232,13 @@ frappe.ui.form.ControlAutocomplete = class ControlAutoComplete extends frappe.ui
 				method: args.query,
 				args: args,
 				callback: ({ message }) => {
-					if(!this.$input.is(":focus")) {
+					if (!this.$input.is(":focus")) {
 						return;
 					}
 					this.$input.cache[this.doctype][this.df.fieldname][term] = message;
 					this.set_data(message);
-				}
-			})
+				},
+			});
 		}
 	}
 
diff --git a/frappe/public/js/frappe/form/controls/barcode.js b/frappe/public/js/frappe/form/controls/barcode.js
index 3a678b6a97..c130ecc039 100644
--- a/frappe/public/js/frappe/form/controls/barcode.js
+++ b/frappe/public/js/frappe/form/controls/barcode.js
@@ -1,36 +1,34 @@
-import JsBarcode from 'jsbarcode';
+import JsBarcode from "jsbarcode";
 
 frappe.ui.form.ControlBarcode = class ControlBarcode extends frappe.ui.form.ControlData {
 	make_wrapper() {
 		// Create the elements for barcode area
 		super.make_wrapper();
 
-		this.default_svg = '';
-		let $input_wrapper = this.$wrapper.find('.control-input-wrapper');
-		this.barcode_area = $(
-			`
        ${this.default_svg}
        `
-		);
+		this.default_svg = "";
+		let $input_wrapper = this.$wrapper.find(".control-input-wrapper");
+		this.barcode_area = $(`
        ${this.default_svg}
        `);
 		this.barcode_area.appendTo($input_wrapper);
 	}
 
 	parse(value) {
 		// Parse raw value
 		if (value) {
-			if (value.startsWith('