diff --git a/.editorconfig b/.editorconfig index f4c7f1528c..b901702d08 100644 --- a/.editorconfig +++ b/.editorconfig @@ -12,3 +12,10 @@ charset = utf-8 [{*.py,*.js,*.vue,*.css,*.scss,*.html}] indent_style = tab indent_size = 4 +max_line_length = 99 + +# JSON files - mostly doctype schema files +[{*.json}] +insert_final_newline = false +indent_style = space +indent_size = 2 diff --git a/.flake8 b/.flake8 index 4b852abd7c..e783fbbeb3 100644 --- a/.flake8 +++ b/.flake8 @@ -1,37 +1,75 @@ [flake8] ignore = + B001, + B007, + B009, + B010, + B950, + E101, + E111, + E114, + E116, + E117, E121, + E122, + E123, + E124, + E125, E126, E127, E128, + E131, + E201, + E202, E203, + E211, + E221, + E222, + E223, + E224, E225, E226, + E228, E231, E241, + E242, E251, E261, + E262, E265, + E266, + E271, + E272, + E273, + E274, + E301, E302, E303, E305, + E306, E402, E501, + E502, + E701, + E702, + E703, E741, + F401, + F403, + F405, + W191, W291, W292, W293, W391, W503, W504, - F403, - B007, - B950, - W191, - E124, # closing bracket, irritating while writing QB code - E131, # continuation line unaligned for hanging indent - E123, # closing bracket does not match indentation of opening bracket's line - E101, # ensured by use of black + E711, + E129, + F841, + E713, + E712, + B028, max-line-length = 200 -exclude=.github/helper/semgrep_rules +exclude=,test_*.py diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index c3ad43c5be..5a96c3fea8 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -25,3 +25,12 @@ c0c5b2ebdddbe8898ce2d5e5365f4931ff73b6bf # update python code to use 3.10 supported features 81b37cb7d2160866afa2496873656afe53f0c145 + +# mass minified JSON schema +85e3ee940353d7b0b517b33815148672e9a8b15b + +# format JS files with pretter +40f27f908a3890c9a90d2d96794fc31fcea63c59 + +# db.get_all -> get_all +2eec621e95564c359ad22da79501a855c1f32b03 diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 26bb7ab280..6fe6ab1dcd 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,5 +1,5 @@ blank_issues_enabled: false contact_links: - name: Community Forum - url: https://discuss.erpnext.com/ + url: https://discuss.frappe.io/c/framework/5 about: For general QnA, discussions and community help. diff --git a/.github/helper/ci.py b/.github/helper/ci.py new file mode 100644 index 0000000000..1f35d0b18d --- /dev/null +++ b/.github/helper/ci.py @@ -0,0 +1,109 @@ +# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors +# MIT License. See LICENSE +import os +from pathlib import Path + +STANDARD_INCLUSIONS = ["*.py"] + +STANDARD_EXCLUSIONS = [ + "*.js", + "*.xml", + "*.pyc", + "*.css", + "*.less", + "*.scss", + "*.vue", + "*.html", + "*/test_*", + "*/node_modules/*", + "*/doctype/*/*_dashboard.py", + "*/patches/*", + ".github/*", +] + +# tested via commands' test suite +TESTED_VIA_CLI = [ + "*/frappe/installer.py", + "*/frappe/utils/install.py", + "*/frappe/utils/scheduler.py", + "*/frappe/utils/doctor.py", + "*/frappe/build.py", + "*/frappe/database/__init__.py", + "*/frappe/database/db_manager.py", + "*/frappe/database/**/setup_db.py", +] + +FRAPPE_EXCLUSIONS = [ + "*/tests/*", + "*/commands/*", + "*/frappe/change_log/*", + "*/frappe/exceptions*", + "*/frappe/desk/page/setup_wizard/setup_wizard.py", + "*/frappe/coverage.py", + "*frappe/setup.py", + "*/frappe/hooks.py", + "*/doctype/*/*_dashboard.py", + "*/patches/*", + "*/.github/helper/ci.py", +] + TESTED_VIA_CLI + + +def get_bench_path(): + return Path(__file__).resolve().parents[4] + + +class CodeCoverage: + def __init__(self, with_coverage, app): + self.with_coverage = with_coverage + self.app = app or "frappe" + + def __enter__(self): + if self.with_coverage: + import os + + from coverage import Coverage + + # Generate coverage report only for app that is being tested + source_path = os.path.join(get_bench_path(), "apps", self.app) + print(f"Source path: {source_path}") + omit = STANDARD_EXCLUSIONS[:] + + if self.app == "frappe": + omit.extend(FRAPPE_EXCLUSIONS) + + self.coverage = Coverage(source=[source_path], omit=omit, include=STANDARD_INCLUSIONS) + self.coverage.start() + + def __exit__(self, exc_type, exc_value, traceback): + if self.with_coverage: + self.coverage.stop() + self.coverage.save() + self.coverage.xml_report() + + +if __name__ == "__main__": + app = "frappe" + site = os.environ.get("SITE") or "test_site" + use_orchestrator = bool(os.environ.get("ORCHESTRATOR_URL")) + build_number = 1 + total_builds = 1 + + try: + build_number = int(os.environ.get("BUILD_NUMBER")) + except Exception: + pass + + try: + total_builds = int(os.environ.get("TOTAL_BUILDS")) + except Exception: + pass + + with CodeCoverage(with_coverage=True, app=app): + if use_orchestrator: + from frappe.parallel_test_runner import ParallelTestWithOrchestrator + + ParallelTestWithOrchestrator(app, site=site) + else: + from frappe.parallel_test_runner import ParallelTestRunner + + ParallelTestRunner(app, site=site, build_number=build_number, total_builds=total_builds) diff --git a/.github/helper/consumer_db/mariadb.json b/.github/helper/db/mariadb.json similarity index 90% rename from .github/helper/consumer_db/mariadb.json rename to .github/helper/db/mariadb.json index 2e32157e1a..e86e701dc3 100644 --- a/.github/helper/consumer_db/mariadb.json +++ b/.github/helper/db/mariadb.json @@ -1,7 +1,7 @@ { "db_host": "127.0.0.1", "db_port": 3306, - "db_name": "test_frappe_consumer", + "db_name": "test_frappe", "db_password": "test_frappe", "allow_tests": true, "db_type": "mariadb", @@ -13,5 +13,6 @@ "root_login": "root", "root_password": "travis", "host_name": "http://test_site:8000", + "monitor": 1, "server_script_enabled": true } diff --git a/.github/helper/consumer_db/postgres.json b/.github/helper/db/postgres.json similarity index 92% rename from .github/helper/consumer_db/postgres.json rename to .github/helper/db/postgres.json index 9532670029..6ca83b9e96 100644 --- a/.github/helper/consumer_db/postgres.json +++ b/.github/helper/db/postgres.json @@ -1,7 +1,7 @@ { "db_host": "127.0.0.1", "db_port": 5432, - "db_name": "test_frappe_consumer", + "db_name": "test_frappe", "db_password": "test_frappe", "db_type": "postgres", "allow_tests": true, diff --git a/.github/helper/documentation.py b/.github/helper/documentation.py index aece5f543b..b541583fd6 100644 --- a/.github/helper/documentation.py +++ b/.github/helper/documentation.py @@ -3,48 +3,71 @@ import requests from urllib.parse import urlparse -docs_repos = [ - "frappe_docs", - "erpnext_documentation", +WEBSITE_REPOS = [ "erpnext_com", "frappe_io", ] +DOCUMENTATION_DOMAINS = [ + "docs.erpnext.com", + "frappeframework.com", +] -def uri_validator(x): - result = urlparse(x) - return all([result.scheme, result.netloc, result.path]) -def docs_link_exists(body): - for line in body.splitlines(): - for word in line.split(): - if word.startswith('http') and uri_validator(word): - parsed_url = urlparse(word) - if parsed_url.netloc == "github.com": - parts = parsed_url.path.split('/') - if len(parts) == 5 and parts[1] == "frappe" and parts[2] in docs_repos: - return True - if parsed_url.netloc in ["docs.erpnext.com", "frappeframework.com"]: - return True +def is_valid_url(url: str) -> bool: + parts = urlparse(url) + return all((parts.scheme, parts.netloc, parts.path)) + + +def is_documentation_link(word: str) -> bool: + if not word.startswith("http") or not is_valid_url(word): + return False + + parsed_url = urlparse(word) + if parsed_url.netloc in DOCUMENTATION_DOMAINS: + return True + + if parsed_url.netloc == "github.com": + parts = parsed_url.path.split("/") + if len(parts) == 5 and parts[1] == "frappe" and parts[2] in WEBSITE_REPOS: + return True + + return False + + +def contains_documentation_link(body: str) -> bool: + return any( + is_documentation_link(word) + for line in body.splitlines() + for word in line.split() + ) + + +def check_pull_request(number: str) -> "tuple[int, str]": + response = requests.get(f"https://api.github.com/repos/frappe/frappe/pulls/{number}") + if not response.ok: + return 1, "Pull Request Not Found! ⚠️" + + payload = response.json() + title = (payload.get("title") or "").lower().strip() + head_sha = (payload.get("head") or {}).get("sha") + body = (payload.get("body") or "").lower() + + if ( + not title.startswith("feat") + or not head_sha + or "no-docs" in body + or "backport" in body + ): + return 0, "Skipping documentation checks... 🏃" + + if contains_documentation_link(body): + return 0, "Documentation Link Found. You're Awesome! 🎉" + + return 1, "Documentation Link Not Found! ⚠️" if __name__ == "__main__": - pr = sys.argv[1] - response = requests.get("https://api.github.com/repos/frappe/frappe/pulls/{}".format(pr)) - - if response.ok: - payload = response.json() - title = (payload.get("title") or "").lower() - head_sha = (payload.get("head") or {}).get("sha") - body = (payload.get("body") or "").lower() - - if title.startswith("feat") and head_sha and "no-docs" not in body: - if docs_link_exists(body): - print("Documentation Link Found. You're Awesome! 🎉") - - else: - print("Documentation Link Not Found! ⚠️") - sys.exit(1) - - else: - print("Skipping documentation checks... 🏃") + exit_code, message = check_pull_request(sys.argv[1]) + print(message) + sys.exit(exit_code) diff --git a/.github/helper/flake8.conf b/.github/helper/flake8.conf deleted file mode 100644 index 20d4b912ca..0000000000 --- a/.github/helper/flake8.conf +++ /dev/null @@ -1,75 +0,0 @@ -[flake8] -ignore = - B001, - B007, - B009, - B010, - B950, - E101, - E111, - E114, - E116, - E117, - E121, - E122, - E123, - E124, - E125, - E126, - E127, - E128, - E131, - E201, - E202, - E203, - E211, - E221, - E222, - E223, - E224, - E225, - E226, - E228, - E231, - E241, - E242, - E251, - E261, - E262, - E265, - E266, - E271, - E272, - E273, - E274, - E301, - E302, - E303, - E305, - E306, - E402, - E501, - E502, - E701, - E702, - E703, - E741, - F401, - F403, - F405, - W191, - W291, - W292, - W293, - W391, - W503, - W504, - E711, - E129, - F841, - E713, - E712, - - -max-line-length = 200 -exclude=.github/helper/semgrep_rules,test_*.py diff --git a/.github/helper/install.sh b/.github/helper/install.sh index b36c1e4b12..39880e35e7 100644 --- a/.github/helper/install.sh +++ b/.github/helper/install.sh @@ -1,67 +1,71 @@ #!/bin/bash - set -e - cd ~ || exit -pip install frappe-bench +echo "Setting Up Bench..." +pip install frappe-bench bench -v init frappe-bench --skip-assets --python "$(which python)" --frappe-path "${GITHUB_WORKSPACE}" +cd ./frappe-bench || exit + +bench -v setup requirements --dev +if [ "$TYPE" == "ui" ] +then + bench -v setup requirements --node; +fi + +echo "Setting Up Sites & Database..." mkdir ~/frappe-bench/sites/test_site -cp "${GITHUB_WORKSPACE}/.github/helper/consumer_db/$DB.json" ~/frappe-bench/sites/test_site/site_config.json +cp "${GITHUB_WORKSPACE}/.github/helper/db/$DB.json" ~/frappe-bench/sites/test_site/site_config.json -if [ "$TYPE" == "server" ]; then - mkdir ~/frappe-bench/sites/test_site_producer; - cp "${GITHUB_WORKSPACE}/.github/helper/producer_db/$DB.json" ~/frappe-bench/sites/test_site_producer/site_config.json; +if [ "$DB" == "mariadb" ] +then + mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "SET GLOBAL character_set_server = 'utf8mb4'"; + mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "SET GLOBAL collation_server = 'utf8mb4_unicode_ci'"; + + mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "CREATE DATABASE test_frappe"; + mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "CREATE USER 'test_frappe'@'localhost' IDENTIFIED BY 'test_frappe'"; + mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "GRANT ALL PRIVILEGES ON \`test_frappe\`.* TO 'test_frappe'@'localhost'"; + + mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "FLUSH PRIVILEGES"; +fi +if [ "$DB" == "postgres" ] +then + echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE DATABASE test_frappe" -U postgres; + echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE USER test_frappe WITH PASSWORD 'test_frappe'" -U postgres; fi -if [ "$DB" == "mariadb" ];then - curl -LsS -O https://downloads.mariadb.com/MariaDB/mariadb_repo_setup - sudo bash mariadb_repo_setup --mariadb-server-version=10.6 - sudo apt install mariadb-client - - mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "SET GLOBAL character_set_server = 'utf8mb4'"; - mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "SET GLOBAL collation_server = 'utf8mb4_unicode_ci'"; - - mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "CREATE DATABASE test_frappe_consumer"; - mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "CREATE USER 'test_frappe_consumer'@'localhost' IDENTIFIED BY 'test_frappe_consumer'"; - mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "GRANT ALL PRIVILEGES ON \`test_frappe_consumer\`.* TO 'test_frappe_consumer'@'localhost'"; - - mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "CREATE DATABASE test_frappe_producer"; - mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "CREATE USER 'test_frappe_producer'@'localhost' IDENTIFIED BY 'test_frappe_producer'"; - mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "GRANT ALL PRIVILEGES ON \`test_frappe_producer\`.* TO 'test_frappe_producer'@'localhost'"; - - mariadb --host 127.0.0.1 --port 3306 -u root -ptravis -e "FLUSH PRIVILEGES"; - fi - -if [ "$DB" == "postgres" ];then - echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE DATABASE test_frappe_consumer" -U postgres; - echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE USER test_frappe_consumer WITH PASSWORD 'test_frappe'" -U postgres; - - echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE DATABASE test_frappe_producer" -U postgres; - echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE USER test_frappe_producer WITH PASSWORD 'test_frappe'" -U postgres; -fi - -cd ./frappe-bench || exit +echo "Setting Up Procfile..." sed -i 's/^watch:/# watch:/g' Procfile sed -i 's/^schedule:/# schedule:/g' Procfile -if [ "$TYPE" == "server" ]; then sed -i 's/^socketio:/# socketio:/g' Procfile; fi -if [ "$TYPE" == "server" ]; then sed -i 's/^redis_socketio:/# redis_socketio:/g' Procfile; fi +if [ "$TYPE" == "server" ] +then + sed -i 's/^socketio:/# socketio:/g' Procfile + sed -i 's/^redis_socketio:/# redis_socketio:/g' Procfile +fi -if [ "$TYPE" == "ui" ]; then bench -v setup requirements --node; fi -bench -v setup requirements --dev +if [ "$TYPE" == "ui" ] +then + sed -i 's/^web: bench serve/web: bench serve --with-coverage/g' Procfile +fi -if [ "$TYPE" == "ui" ]; then sed -i 's/^web: bench serve/web: bench serve --with-coverage/g' Procfile; fi +echo "Starting Bench..." -# install node-sass which is required for website theme test -cd ./apps/frappe || exit -yarn add node-sass@4.13.1 -cd ../.. +bench start &> bench_start.log & + +if [ "$TYPE" == "server" ] +then + CI=Yes bench build --app frappe & + build_pid=$! +fi -bench start & bench --site test_site reinstall --yes -if [ "$TYPE" == "server" ]; then bench --site test_site_producer reinstall --yes; fi -if [ "$TYPE" == "server" ]; then CI=Yes bench build --app frappe; fi + +if [ "$TYPE" == "server" ] +then + # wait till assets are built succesfully + wait $build_pid +fi diff --git a/.github/helper/install_dependencies.sh b/.github/helper/install_dependencies.sh index 18bc4e6f9a..b8da022665 100644 --- a/.github/helper/install_dependencies.sh +++ b/.github/helper/install_dependencies.sh @@ -1,19 +1,13 @@ #!/bin/bash - set -e -# Check for merge conflicts before proceeding -python -m compileall -f "${GITHUB_WORKSPACE}" -if grep -lr --exclude-dir=node_modules "^<<<<<<< " "${GITHUB_WORKSPACE}" - then echo "Found merge conflicts" - exit 1 -fi +echo "Setting Up System Dependencies..." - # install wkhtmltopdf -wget -O /tmp/wkhtmltox.tar.xz https://github.com/frappe/wkhtmltopdf/raw/master/wkhtmltox-0.12.3_linux-generic-amd64.tar.xz -tar -xf /tmp/wkhtmltox.tar.xz -C /tmp -sudo mv /tmp/wkhtmltox/bin/wkhtmltopdf /usr/local/bin/wkhtmltopdf -sudo chmod o+x /usr/local/bin/wkhtmltopdf +sudo apt update +sudo apt install libcups2-dev redis-server mariadb-client-10.6 -# install cups -sudo apt update && sudo apt install libcups2-dev redis-server +install_wkhtmltopdf() { + wget -q https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.focal_amd64.deb + sudo apt install ./wkhtmltox_0.12.6-1.focal_amd64.deb +} +install_wkhtmltopdf & diff --git a/.github/helper/producer_db/mariadb.json b/.github/helper/producer_db/mariadb.json deleted file mode 100644 index c1db0d765f..0000000000 --- a/.github/helper/producer_db/mariadb.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_host": "127.0.0.1", - "db_port": 3306, - "db_name": "test_frappe_producer", - "db_password": "test_frappe", - "allow_tests": true, - "db_type": "mariadb", - "auto_email_id": "test@example.com", - "mail_server": "smtp.example.com", - "mail_login": "test@example.com", - "mail_password": "test", - "admin_password": "admin", - "root_login": "root", - "root_password": "travis", - "host_name": "http://test_site_producer:8000" -} diff --git a/.github/helper/producer_db/postgres.json b/.github/helper/producer_db/postgres.json deleted file mode 100644 index 8b9d2a20fd..0000000000 --- a/.github/helper/producer_db/postgres.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_host": "127.0.0.1", - "db_port": 5432, - "db_name": "test_frappe_producer", - "db_password": "test_frappe", - "db_type": "postgres", - "allow_tests": true, - "auto_email_id": "test@example.com", - "mail_server": "smtp.example.com", - "mail_login": "test@example.com", - "mail_password": "test", - "admin_password": "admin", - "root_login": "postgres", - "root_password": "travis", - "host_name": "http://test_site_producer:8000" -} diff --git a/.github/helper/roulette.py b/.github/helper/roulette.py index 554f4ae5f5..e3b212fa89 100644 --- a/.github/helper/roulette.py +++ b/.github/helper/roulette.py @@ -4,8 +4,10 @@ import re import shlex import subprocess import sys +import time import urllib.request from functools import lru_cache +from urllib.error import HTTPError @lru_cache(maxsize=None) @@ -15,41 +17,78 @@ def fetch_pr_data(pr_number, repo, endpoint=""): if endpoint: api_url += f"/{endpoint}" - req = urllib.request.Request(api_url) - res = urllib.request.urlopen(req) - return json.loads(res.read().decode('utf8')) + res = req(api_url) + return json.loads(res.read().decode("utf8")) + + +def req(url): + "Simple resilient request call to handle rate limits." + headers = None + token = os.environ.get("GITHUB_TOKEN") + if token: + headers = {"authorization": f"Bearer {token}"} + + retries = 0 + while True: + try: + req = urllib.request.Request(url, headers=headers) + return urllib.request.urlopen(req) + except HTTPError as exc: + if exc.code == 403 and retries < 5: + retries += 1 + time.sleep(retries) + continue + raise + def get_files_list(pr_number, repo="frappe/frappe"): return [change["filename"] for change in fetch_pr_data(pr_number, repo, "files")] + def get_output(command, shell=True): print(command) command = shlex.split(command) return subprocess.check_output(command, shell=shell, encoding="utf8").strip() + def has_skip_ci_label(pr_number, repo="frappe/frappe"): return has_label(pr_number, "Skip CI", repo) + def has_run_server_tests_label(pr_number, repo="frappe/frappe"): return has_label(pr_number, "Run Server Tests", repo) + def has_run_ui_tests_label(pr_number, repo="frappe/frappe"): return has_label(pr_number, "Run UI Tests", repo) + def has_label(pr_number, label, repo="frappe/frappe"): - return any([fetched_label["name"] for fetched_label in fetch_pr_data(pr_number, repo)["labels"] if fetched_label["name"] == label]) + return any( + [ + fetched_label["name"] + for fetched_label in fetch_pr_data(pr_number, repo)["labels"] + if fetched_label["name"] == label + ] + ) + def is_py(file): return file.endswith("py") + def is_ci(file): return ".github" in file + def is_frontend_code(file): - return file.lower().endswith((".css", ".scss", ".less", ".sass", ".styl", ".js", ".ts", ".vue", ".html")) + return file.lower().endswith( + (".css", ".scss", ".less", ".sass", ".styl", ".js", ".ts", ".vue", ".html") + ) + def is_docs(file): - regex = re.compile(r'\.(md|png|jpg|jpeg|csv|svg)$|^.github|LICENSE') + regex = re.compile(r"\.(md|png|jpg|jpeg|csv|svg)$|^.github|LICENSE") return bool(regex.search(file)) @@ -61,8 +100,7 @@ if __name__ == "__main__": # this is a push build, run all builds if not pr_number: - os.system('echo "::set-output name=build::strawberry"') - os.system('echo "::set-output name=build-server::strawberry"') + os.system('echo "build=strawberry" >> $GITHUB_OUTPUT') sys.exit(0) files_list = files_list or get_files_list(pr_number=pr_number, repo=repo) @@ -78,8 +116,13 @@ if __name__ == "__main__": only_py_changed = updated_py_file_count == len(files_list) if has_skip_ci_label(pr_number, repo): - print("Found `Skip CI` label on pr, stopping build process.") - sys.exit(0) + if build_type == "ui" and has_run_ui_tests_label(pr_number, repo): + print("Running UI tests only.") + elif build_type == "server" and has_run_server_tests_label(pr_number, repo): + print("Running server tests only.") + else: + print("Found `Skip CI` label on pr, stopping build process.") + sys.exit(0) elif ci_files_changed: print("CI related files were updated, running all build processes.") @@ -88,7 +131,11 @@ if __name__ == "__main__": print("Only docs were updated, stopping build process.") sys.exit(0) - elif only_frontend_code_changed and build_type == "server" and not has_run_server_tests_label(pr_number, repo): + elif ( + only_frontend_code_changed + and build_type == "server" + and not has_run_server_tests_label(pr_number, repo) + ): print("Only Frontend code was updated; Stopping Python build process.") sys.exit(0) @@ -96,4 +143,4 @@ if __name__ == "__main__": print("Only Python code was updated, stopping Cypress build process.") sys.exit(0) - os.system('echo "::set-output name=build::strawberry"') + os.system('echo "build=strawberry" >> $GITHUB_OUTPUT') diff --git a/.github/helper/translation.py b/.github/helper/translation.py index 9146b3b32b..72f661d3e1 100644 --- a/.github/helper/translation.py +++ b/.github/helper/translation.py @@ -20,19 +20,12 @@ for _file in files_to_scan: if 'frappe-lint: disable-translate' in line: continue - start_matches = start_pattern.search(line) - if start_matches: - starts_with_f = starts_with_f_pattern.search(line) - - if starts_with_f: - has_f_string = f_string_pattern.search(line) - if has_f_string: + if start_matches := start_pattern.search(line): + if starts_with_f := starts_with_f_pattern.search(line): + if has_f_string := f_string_pattern.search(line): errors_encounter += 1 print(f'\nF-strings are not supported for translations at line number {line_number}\n{line.strip()[:100]}') - continue - else: - continue - + continue match = pattern.search(line) error_found = False diff --git a/.github/labeler.yml b/.github/labeler.yml new file mode 100644 index 0000000000..42e52e553f --- /dev/null +++ b/.github/labeler.yml @@ -0,0 +1,4 @@ +# Any python files modifed but no test files modified +add-test-cases: +- any: ['frappe/**/*.py'] + all: ['!frappe/**/test*.py'] diff --git a/.github/semantic.yml b/.github/semantic.yml deleted file mode 100644 index fa15046b4a..0000000000 --- a/.github/semantic.yml +++ /dev/null @@ -1,30 +0,0 @@ -# Always validate the PR title AND all the commits -titleAndCommits: true - -# Allow use of Merge commits (eg on github: "Merge branch 'master' into feature/ride-unicorns") -# this is only relevant when using commitsOnly: true (or titleAndCommits: true) -allowMergeCommits: true - -# Allow use of Revert commits (eg on github: "Revert "feat: ride unicorns"") -# this is only relevant when using commitsOnly: true (or titleAndCommits: true) -allowRevertCommits: true - -# For allowed PR types: https://github.com/commitizen/conventional-commit-types/blob/v3.0.0/index.json -# Tool Reference: https://github.com/zeke/semantic-pull-requests - -# By default types specified in commitizen/conventional-commit-types is used. -# See: https://github.com/commitizen/conventional-commit-types/blob/v3.0.0/index.json -# You can override the valid types -types: - - BREAKING CHANGE - - feat - - fix - - docs - - style - - refactor - - perf - - test - - build - - ci - - chore - - revert diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml new file mode 100644 index 0000000000..8422378ef4 --- /dev/null +++ b/.github/workflows/backport.yml @@ -0,0 +1,26 @@ +name: Backport +on: + pull_request_target: + types: + - closed + - labeled + +jobs: + main: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout Actions + uses: actions/checkout@v3 + with: + repository: "frappe/backport" + path: ./actions + ref: develop + - name: Install Actions + run: npm install --production --prefix ./actions + - name: Run backport + uses: ./actions/backport + with: + token: ${{secrets.RELEASE_TOKEN}} + labelsToAdd: "backport" + title: "{{originalTitle}}" diff --git a/.github/workflows/release.yml b/.github/workflows/create-release.yml similarity index 90% rename from .github/workflows/release.yml rename to .github/workflows/create-release.yml index f73bed09c7..6dcbccd85e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/create-release.yml @@ -16,10 +16,10 @@ jobs: with: fetch-depth: 0 persist-credentials: false - - name: Setup Node.js v14 + - name: Setup Node.js uses: actions/setup-node@v3 with: - node-version: 14 + node-version: 18 - name: Setup dependencies run: | npm install @semantic-release/git @semantic-release/exec --no-save @@ -31,4 +31,4 @@ jobs: GIT_AUTHOR_EMAIL: "developers@frappe.io" GIT_COMMITTER_NAME: "Frappe PR Bot" GIT_COMMITTER_EMAIL: "developers@frappe.io" - run: npx semantic-release \ No newline at end of file + run: npx semantic-release diff --git a/.github/workflows/deps-checker.yml b/.github/workflows/deps-checker.yml deleted file mode 100644 index d3fa8c80fb..0000000000 --- a/.github/workflows/deps-checker.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: 'Python Dependency Check' -on: - pull_request: - workflow_dispatch: - push: - branches: [ develop ] - -permissions: - contents: read - -jobs: - deps-vulnerable-check: - name: 'Vulnerable Dependency' - runs-on: ubuntu-latest - - steps: - - uses: actions/setup-python@v4 - with: - python-version: '3.10' - - uses: actions/checkout@v3 - - run: pip install pip-audit - - run: pip-audit ${GITHUB_WORKSPACE} diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml deleted file mode 100644 index 988c2dcc6c..0000000000 --- a/.github/workflows/docker-release.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: 'Trigger Docker build on release' -on: - release: - types: [released] -permissions: - contents: read - -jobs: - curl: - permissions: - contents: none - name: 'Trigger Docker build on release' - runs-on: ubuntu-latest - container: - image: alpine:latest - steps: - - name: curl - run: | - apk add curl bash - curl -X POST -H "Accept: application/vnd.github.v3+json" -H "Authorization: Bearer ${{ secrets.CI_PAT }}" https://api.github.com/repos/frappe/frappe_docker/actions/workflows/build_stable.yml/dispatches -d '{"ref":"main"}' diff --git a/.github/workflows/docs-checker.yml b/.github/workflows/docs-checker.yml deleted file mode 100644 index e61ee6355a..0000000000 --- a/.github/workflows/docs-checker.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: 'Documentation Check' -on: - pull_request: - types: [ opened, synchronize, reopened, edited ] - -permissions: - contents: read - -jobs: - docs-required: - name: 'Documentation Required' - runs-on: ubuntu-latest - - steps: - - name: 'Setup Environment' - uses: actions/setup-python@v4 - with: - python-version: '3.10' - - - name: 'Clone repo' - uses: actions/checkout@v3 - - - name: Validate Docs - env: - PR_NUMBER: ${{ github.event.number }} - run: | - pip install requests --quiet - python $GITHUB_WORKSPACE/.github/helper/documentation.py $PR_NUMBER diff --git a/.github/workflows/initiate_release.yml b/.github/workflows/initiate_release.yml new file mode 100644 index 0000000000..178ceeeb8b --- /dev/null +++ b/.github/workflows/initiate_release.yml @@ -0,0 +1,32 @@ +# This workflow is agnostic to branches. Only maintain on develop branch. +# To add/remove versions just modify the matrix. + +name: Create weekly release pull requests +on: + schedule: + # 9:30 UTC => 3 PM IST Tuesday + - cron: "30 9 * * 2" + workflow_dispatch: + +jobs: + release: + name: Release + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + version: ["13", "14"] + + steps: + - uses: octokit/request-action@v2.x + with: + route: POST /repos/{owner}/{repo}/pulls + owner: frappe + repo: frappe + title: |- + "chore: release v${{ matrix.version }}" + body: "Automated weekly release." + base: version-${{ matrix.version }} + head: version-${{ matrix.version }}-hotfix + env: + GITHUB_TOKEN: ${{ secrets.RELEASE_TOKEN }} diff --git a/.github/workflows/labeller.yml b/.github/workflows/labeller.yml new file mode 100644 index 0000000000..97fa4a1a2c --- /dev/null +++ b/.github/workflows/labeller.yml @@ -0,0 +1,12 @@ +name: "Pull Request Labeler" +on: + pull_request_target: + types: [opened, reopened] + +jobs: + triage: + runs-on: ubuntu-latest + steps: + - uses: actions/labeler@v4 + with: + repo-token: "${{ secrets.GITHUB_TOKEN }}" diff --git a/.github/workflows/linters.yml b/.github/workflows/linters.yml index 6d1029d51d..be343c1254 100644 --- a/.github/workflows/linters.yml +++ b/.github/workflows/linters.yml @@ -1,29 +1,100 @@ name: Linters on: - pull_request: { } + pull_request: + workflow_dispatch: + push: + branches: [ develop ] + +permissions: + contents: read + +concurrency: + group: commitcheck-frappe-${{ github.event_name }}-${{ github.event.number }} + cancel-in-progress: true jobs: - - linters: - name: Frappe Linter + commit-lint: + name: 'Semantic Commits' runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + steps: - uses: actions/checkout@v3 + with: + fetch-depth: 200 + - uses: actions/setup-node@v3 + with: + node-version: 16 + check-latest: true - - name: Set up Python + - name: Check commit titles + run: | + npm install @commitlint/cli @commitlint/config-conventional + npx commitlint --verbose --from ${{ github.event.pull_request.base.sha }} --to ${{ github.event.pull_request.head.sha }} + + docs-required: + name: 'Documentation Required' + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + + steps: + - name: 'Setup Environment' uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: '3.11' + - uses: actions/checkout@v3 - - name: Install and Run Pre-commit - uses: pre-commit/action@v3.0.0 + - name: Validate Docs + env: + PR_NUMBER: ${{ github.event.number }} + run: | + pip install requests --quiet + python $GITHUB_WORKSPACE/.github/helper/documentation.py $PR_NUMBER + + linter: + name: 'Frappe Linter' + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + - uses: pre-commit/action@v3.0.0 - name: Download Semgrep rules run: git clone --depth 1 https://github.com/frappe/semgrep-rules.git frappe-semgrep-rules - - name: Download semgrep - run: pip install semgrep==0.97.0 - - name: Run Semgrep rules - run: semgrep ci --config ./frappe-semgrep-rules/rules --config r/python.lang.correctness + run: | + pip install semgrep==0.97.0 + semgrep ci --config ./frappe-semgrep-rules/rules --config r/python.lang.correctness + + deps-vulnerable-check: + name: 'Vulnerable Dependency Check' + runs-on: ubuntu-latest + + steps: + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - uses: actions/checkout@v3 + + - name: Cache pip + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/*requirements.txt', '**/pyproject.toml', '**/setup.py') }} + restore-keys: | + ${{ runner.os }}-pip- + ${{ runner.os }}- + + - name: Install and run pip-audit + run: | + pip install pip-audit + cd ${GITHUB_WORKSPACE} + sed -i '/dropbox/d' pyproject.toml # Remove dropbox temporarily https://github.com/dropbox/dropbox-sdk-python/pull/456 + pip-audit --desc on . diff --git a/.github/workflows/lock.yml b/.github/workflows/lock.yml new file mode 100644 index 0000000000..bb90188c4c --- /dev/null +++ b/.github/workflows/lock.yml @@ -0,0 +1,21 @@ +name: 'Lock threads' + +on: + schedule: + - cron: '0 0 * * *' + workflow_dispatch: + +permissions: + issues: write + pull-requests: write + + +jobs: + lock: + runs-on: ubuntu-latest + steps: + - uses: dessant/lock-threads@v4 + with: + github-token: ${{ github.token }} + issue-inactive-days: 14 + pr-inactive-days: 14 \ No newline at end of file diff --git a/.github/workflows/publish-assets-releases.yml b/.github/workflows/on_release.yml similarity index 67% rename from .github/workflows/publish-assets-releases.yml rename to .github/workflows/on_release.yml index 2612c45bea..851b5b1d6a 100644 --- a/.github/workflows/publish-assets-releases.yml +++ b/.github/workflows/on_release.yml @@ -1,8 +1,11 @@ -name: 'Frappe Assets' +name: 'Release' on: release: - types: [ created ] + types: [released] + +permissions: + contents: read env: GITHUB_TOKEN: ${{ github.token }} @@ -16,9 +19,11 @@ jobs: - uses: actions/checkout@v3 with: path: 'frappe' + - uses: actions/setup-node@v3 with: - python-version: '12.x' + node-version: 16 + - uses: actions/setup-python@v4 with: python-version: '3.10' @@ -36,7 +41,7 @@ jobs: - name: Get release id: get_release - uses: bruceadams/get-release@v1.2.3 + uses: bruceadams/get-release@v1.3.1 - name: Upload built Assets to Release uses: actions/upload-release-asset@v1.0.2 @@ -45,3 +50,16 @@ jobs: asset_path: build/assets.tar.gz asset_name: assets.tar.gz asset_content_type: application/octet-stream + + docker-release: + name: 'Trigger Docker build on release' + runs-on: ubuntu-latest + permissions: + contents: none + container: + image: alpine:latest + steps: + - name: curl + run: | + apk add curl bash + curl -X POST -H "Accept: application/vnd.github.v3+json" -H "Authorization: Bearer ${{ secrets.CI_PAT }}" https://api.github.com/repos/frappe/frappe_docker/actions/workflows/build_stable.yml/dispatches -d '{"ref":"main"}' diff --git a/.github/workflows/patch-mariadb-tests.yml b/.github/workflows/patch-mariadb-tests.yml index 73e0dda5de..4b487d2aea 100644 --- a/.github/workflows/patch-mariadb-tests.yml +++ b/.github/workflows/patch-mariadb-tests.yml @@ -1,21 +1,45 @@ -name: Patch - -on: [pull_request, workflow_dispatch] +name: Patch (MariaDB) +on: + pull_request: + workflow_dispatch: concurrency: - group: patch-mariadb-develop-${{ github.event.number }} + group: patch-mariadb-develop-${{ github.event_name }}-${{ github.event.number }} cancel-in-progress: true permissions: + # Do not change this as GITHUB_TOKEN is being used by roulette contents: read jobs: - test: + checkrun: + name: Build Check runs-on: ubuntu-latest - timeout-minutes: 60 - name: Patch Test + outputs: + build: ${{ steps.check-build.outputs.build }} + + steps: + - name: Clone + uses: actions/checkout@v3 + + - name: Check if build should be run + id: check-build + run: | + python "${GITHUB_WORKSPACE}/.github/helper/roulette.py" + env: + TYPE: "server" + PR_NUMBER: ${{ github.event.number }} + REPO_NAME: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + test: + name: Patch + runs-on: ubuntu-latest + needs: checkrun + if: ${{ needs.checkrun.outputs.build == 'strawberry' }} + timeout-minutes: 60 services: mariadb: @@ -30,6 +54,13 @@ jobs: - name: Clone uses: actions/checkout@v3 + - name: Check for Merge Conflicts + run: | + if grep -lr --exclude-dir=node_modules "^<<<<<<< " "${GITHUB_WORKSPACE}" + then echo "Found merge conflicts" + exit 1 + fi + - name: Setup Python uses: "gabrielfalcao/pyenv-action@v10" with: @@ -38,24 +69,13 @@ jobs: - name: Setup Node uses: actions/setup-node@v3 with: - node-version: 14 + node-version: 16 check-latest: true - - name: Check if build should be run - id: check-build - run: | - python "${GITHUB_WORKSPACE}/.github/helper/roulette.py" - env: - TYPE: "server" - PR_NUMBER: ${{ github.event.number }} - REPO_NAME: ${{ github.repository }} - - name: Add to Hosts - if: ${{ steps.check-build.outputs.build == 'strawberry' }} run: echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts - name: Cache pip - if: ${{ steps.check-build.outputs.build == 'strawberry' }} uses: actions/cache@v3 with: path: ~/.cache/pip @@ -64,26 +84,11 @@ jobs: ${{ runner.os }}-pip- ${{ runner.os }}- - - name: Cache node modules - if: ${{ steps.check-build.outputs.build == 'strawberry' }} - uses: actions/cache@v3 - env: - cache-name: cache-node-modules - with: - path: ~/.npm - key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }} - restore-keys: | - ${{ runner.os }}-build-${{ env.cache-name }}- - ${{ runner.os }}-build- - ${{ runner.os }}- - - name: Get yarn cache directory path - if: ${{ steps.check-build.outputs.build == 'strawberry' }} id: yarn-cache-dir-path - run: echo "::set-output name=dir::$(yarn cache dir)" + run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT - uses: actions/cache@v3 - if: ${{ steps.check-build.outputs.build == 'strawberry' }} id: yarn-cache with: path: ${{ steps.yarn-cache-dir-path.outputs.dir }} @@ -92,25 +97,18 @@ jobs: ${{ runner.os }}-yarn- - name: Install Dependencies - if: ${{ steps.check-build.outputs.build == 'strawberry' }} - run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh - env: - BEFORE: ${{ env.GITHUB_EVENT_PATH.before }} - AFTER: ${{ env.GITHUB_EVENT_PATH.after }} - TYPE: server - - - name: Install - if: ${{ steps.check-build.outputs.build == 'strawberry' }} run: | + bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh pip install frappe-bench pyenv global $(pyenv versions | grep '3.10') bash ${GITHUB_WORKSPACE}/.github/helper/install.sh env: - DB: mariadb + BEFORE: ${{ env.GITHUB_EVENT_PATH.before }} + AFTER: ${{ env.GITHUB_EVENT_PATH.after }} TYPE: server + DB: mariadb - name: Run Patch Tests - if: ${{ steps.check-build.outputs.build == 'strawberry' }} run: | cd ~/frappe-bench/ wget https://frappeframework.com/files/v10-frappe.sql.gz @@ -120,23 +118,28 @@ jobs: cd apps/frappe/ git remote set-url upstream https://github.com/frappe/frappe.git - pyenv global $(pyenv versions | grep '3.7') - for version in $(seq 12 13) - do - echo "Updating to v$version" - branch_name="version-$version-hotfix" - git fetch --depth 1 upstream $branch_name:$branch_name - git checkout -q -f $branch_name - pip install -U frappe-bench + function update_to_version() { + version=$1 + branch_name="version-$version-hotfix" + echo "Updating to v$version" + git fetch --depth 1 upstream $branch_name:$branch_name + git checkout -q -f $branch_name + pip install -U frappe-bench - rm -rf ~/frappe-bench/env - bench -v setup env - bench --site test_site migrate - done + rm -rf ~/frappe-bench/env + bench -v setup env + bench --site test_site migrate + } + + pyenv global $(pyenv versions | grep '3.7') + update_to_version 12 + update_to_version 13 + + pyenv global $(pyenv versions | grep '3.10') + update_to_version 14 echo "Updating to last commit" git checkout -q -f "$GITHUB_SHA" - pyenv global $(pyenv versions | grep '3.10') rm -rf ~/frappe-bench/env bench -v setup env bench --site test_site migrate diff --git a/.github/workflows/publish-assets-develop.yml b/.github/workflows/publish-assets-develop.yml index b216718b99..4feaebe15d 100644 --- a/.github/workflows/publish-assets-develop.yml +++ b/.github/workflows/publish-assets-develop.yml @@ -1,6 +1,7 @@ name: 'Frappe Assets' on: + workflow_dispatch: push: branches: [ develop ] @@ -15,10 +16,10 @@ jobs: path: 'frappe' - uses: actions/setup-node@v3 with: - node-version: 14 + node-version: 16 - uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: '3.11' - name: Set up bench and build assets run: | npm install -g yarn diff --git a/.github/workflows/release_notes.yml b/.github/workflows/release_notes.yml new file mode 100644 index 0000000000..bf761f23ba --- /dev/null +++ b/.github/workflows/release_notes.yml @@ -0,0 +1,38 @@ +# This action: +# +# 1. Generates release notes using github API. +# 2. Strips unnecessary info like chore/style etc from notes. +# 3. Updates release info. + +# This action needs to be maintained on all branches that do releases. + +name: 'Release Notes' + +on: + workflow_dispatch: + inputs: + tag_name: + description: 'Tag of release like v13.0.0' + required: true + type: string + release: + types: [released] + +permissions: + contents: read + +jobs: + regen-notes: + name: 'Regenerate release notes' + runs-on: ubuntu-latest + + steps: + - name: Update notes + run: | + NEW_NOTES=$(gh api --method POST -H "Accept: application/vnd.github+json" /repos/frappe/frappe/releases/generate-notes -f tag_name=$RELEASE_TAG | jq -r '.body' | sed -E '/^\* (chore|ci|test|docs|style)/d' ) + RELEASE_ID=$(gh api -H "Accept: application/vnd.github+json" /repos/frappe/frappe/releases/tags/$RELEASE_TAG | jq -r '.id') + gh api --method PATCH -H "Accept: application/vnd.github+json" /repos/frappe/frappe/releases/$RELEASE_ID -f body=$NEW_NOTES + + env: + GITHUB_TOKEN: ${{ secrets.RELEASE_TOKEN }} + RELEASE_TAG: ${{ github.event.inputs.tag_name || github.event.release.tag_name }} diff --git a/.github/workflows/server-mariadb-tests.yml b/.github/workflows/server-mariadb-tests.yml deleted file mode 100644 index 29d88fd9a5..0000000000 --- a/.github/workflows/server-mariadb-tests.yml +++ /dev/null @@ -1,135 +0,0 @@ -name: Server - -on: - pull_request: - workflow_dispatch: - push: - branches: [ develop ] - -concurrency: - group: server-mariadb-develop-${{ github.event.number }} - cancel-in-progress: true - - -permissions: - contents: read - -jobs: - test: - runs-on: ubuntu-latest - timeout-minutes: 60 - - strategy: - fail-fast: false - matrix: - container: [1, 2] - - name: Python Unit Tests (MariaDB) - - services: - mariadb: - image: mariadb:10.6 - env: - MARIADB_ROOT_PASSWORD: travis - ports: - - 3306:3306 - options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=3 - - steps: - - name: Clone - uses: actions/checkout@v3 - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.10' - - - name: Check if build should be run - id: check-build - run: | - python "${GITHUB_WORKSPACE}/.github/helper/roulette.py" - env: - TYPE: "server" - PR_NUMBER: ${{ github.event.number }} - REPO_NAME: ${{ github.repository }} - - - uses: actions/setup-node@v3 - if: ${{ steps.check-build.outputs.build == 'strawberry' }} - with: - node-version: 14 - check-latest: true - - - name: Add to Hosts - if: ${{ steps.check-build.outputs.build == 'strawberry' }} - run: | - echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts - echo "127.0.0.1 test_site_producer" | sudo tee -a /etc/hosts - - - name: Cache pip - if: ${{ steps.check-build.outputs.build == 'strawberry' }} - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/*requirements.txt', '**/pyproject.toml', '**/setup.py') }} - restore-keys: | - ${{ runner.os }}-pip- - ${{ runner.os }}- - - - name: Cache node modules - if: ${{ steps.check-build.outputs.build == 'strawberry' }} - uses: actions/cache@v3 - env: - cache-name: cache-node-modules - with: - path: ~/.npm - key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }} - restore-keys: | - ${{ runner.os }}-build-${{ env.cache-name }}- - ${{ runner.os }}-build- - ${{ runner.os }}- - - - name: Get yarn cache directory path - if: ${{ steps.check-build.outputs.build == 'strawberry' }} - id: yarn-cache-dir-path - run: echo "::set-output name=dir::$(yarn cache dir)" - - - uses: actions/cache@v3 - if: ${{ steps.check-build.outputs.build == 'strawberry' }} - id: yarn-cache - with: - path: ${{ steps.yarn-cache-dir-path.outputs.dir }} - key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} - restore-keys: | - ${{ runner.os }}-yarn- - - - name: Install Dependencies - if: ${{ steps.check-build.outputs.build == 'strawberry' }} - run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh - env: - BEFORE: ${{ env.GITHUB_EVENT_PATH.before }} - AFTER: ${{ env.GITHUB_EVENT_PATH.after }} - TYPE: server - - - name: Install - if: ${{ steps.check-build.outputs.build == 'strawberry' }} - run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh - env: - DB: mariadb - TYPE: server - - - name: Run Tests - if: ${{ steps.check-build.outputs.build == 'strawberry' }} - run: cd ~/frappe-bench/ && bench --site test_site run-parallel-tests --use-orchestrator --with-coverage - env: - CI_BUILD_ID: ${{ github.run_id }} - ORCHESTRATOR_URL: http://test-orchestrator.frappe.io - - - name: Upload coverage data - if: ${{ steps.check-build.outputs.build == 'strawberry' }} - uses: codecov/codecov-action@v3 - with: - name: MariaDB - fail_ci_if_error: true - files: /home/runner/frappe-bench/sites/coverage.xml - verbose: true - flags: server diff --git a/.github/workflows/server-postgres-tests.yml b/.github/workflows/server-tests.yml similarity index 52% rename from .github/workflows/server-postgres-tests.yml rename to .github/workflows/server-tests.yml index 8f015f43e6..3b76da1973 100644 --- a/.github/workflows/server-postgres-tests.yml +++ b/.github/workflows/server-tests.yml @@ -7,25 +7,58 @@ on: branches: [ develop ] concurrency: - group: server-postgres-develop-${{ github.event.number }} + group: server-develop-${{ github.event_name }}-${{ github.event.number }} cancel-in-progress: true + permissions: + # Do not change this as GITHUB_TOKEN is being used by roulette contents: read jobs: - test: + checkrun: + name: Build Check runs-on: ubuntu-latest + + outputs: + build: ${{ steps.check-build.outputs.build }} + + steps: + - name: Clone + uses: actions/checkout@v3 + + - name: Check if build should be run + id: check-build + run: | + python "${GITHUB_WORKSPACE}/.github/helper/roulette.py" + env: + TYPE: "server" + PR_NUMBER: ${{ github.event.number }} + REPO_NAME: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + test: + name: Unit Tests + runs-on: ubuntu-latest + needs: checkrun + if: ${{ needs.checkrun.outputs.build == 'strawberry' }} timeout-minutes: 60 strategy: fail-fast: false matrix: + db: ["mariadb", "postgres"] container: [1, 2] - name: Python Unit Tests (Postgres) - services: + mariadb: + image: mariadb:10.6 + env: + MARIADB_ROOT_PASSWORD: travis + ports: + - 3306:3306 + options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=3 + postgres: image: postgres:12.4 env: @@ -45,31 +78,26 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: '3.11' - - name: Check if build should be run - id: check-build + - name: Check for valid Python & Merge Conflicts run: | - python "${GITHUB_WORKSPACE}/.github/helper/roulette.py" - env: - TYPE: "server" - PR_NUMBER: ${{ github.event.number }} - REPO_NAME: ${{ github.repository }} + python -m compileall -q -f "${GITHUB_WORKSPACE}" + if grep -lr --exclude-dir=node_modules "^<<<<<<< " "${GITHUB_WORKSPACE}" + then echo "Found merge conflicts" + exit 1 + fi - uses: actions/setup-node@v3 - if: ${{ steps.check-build.outputs.build == 'strawberry' }} with: - node-version: '14' + node-version: 16 check-latest: true - name: Add to Hosts - if: ${{ steps.check-build.outputs.build == 'strawberry' }} run: | echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts - echo "127.0.0.1 test_site_producer" | sudo tee -a /etc/hosts - name: Cache pip - if: ${{ steps.check-build.outputs.build == 'strawberry' }} uses: actions/cache@v3 with: path: ~/.cache/pip @@ -78,26 +106,11 @@ jobs: ${{ runner.os }}-pip- ${{ runner.os }}- - - name: Cache node modules - if: ${{ steps.check-build.outputs.build == 'strawberry' }} - uses: actions/cache@v3 - env: - cache-name: cache-node-modules - with: - path: ~/.npm - key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }} - restore-keys: | - ${{ runner.os }}-build-${{ env.cache-name }}- - ${{ runner.os }}-build- - ${{ runner.os }}- - - name: Get yarn cache directory path - if: ${{ steps.check-build.outputs.build == 'strawberry' }} id: yarn-cache-dir-path - run: echo "::set-output name=dir::$(yarn cache dir)" + run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT - uses: actions/cache@v3 - if: ${{ steps.check-build.outputs.build == 'strawberry' }} id: yarn-cache with: path: ${{ steps.yarn-cache-dir-path.outputs.dir }} @@ -106,33 +119,45 @@ jobs: ${{ runner.os }}-yarn- - name: Install Dependencies - if: ${{ steps.check-build.outputs.build == 'strawberry' }} - run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh + run: | + bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh + bash ${GITHUB_WORKSPACE}/.github/helper/install.sh env: BEFORE: ${{ env.GITHUB_EVENT_PATH.before }} AFTER: ${{ env.GITHUB_EVENT_PATH.after }} TYPE: server - - - name: Install - if: ${{ steps.check-build.outputs.build == 'strawberry' }} - run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh - env: - DB: postgres - TYPE: server + DB: ${{ matrix.db }} - name: Run Tests - if: ${{ steps.check-build.outputs.build == 'strawberry' }} - run: cd ~/frappe-bench/ && bench --site test_site run-parallel-tests --use-orchestrator --with-coverage + run: cd ~/frappe-bench/sites && ../env/bin/python3 ../apps/frappe/.github/helper/ci.py env: + SITE: test_site CI_BUILD_ID: ${{ github.run_id }} - ORCHESTRATOR_URL: http://test-orchestrator.frappe.io + BUILD_NUMBER: ${{ matrix.container }} + TOTAL_BUILDS: 2 + + - name: Upload coverage data + uses: actions/upload-artifact@v3 + with: + name: coverage-${{ matrix.db }}-${{ matrix.container }} + path: /home/runner/frappe-bench/sites/coverage.xml + + coverage: + name: Coverage Wrap Up + needs: [test, checkrun] + runs-on: ubuntu-latest + if: ${{ needs.checkrun.outputs.build == 'strawberry' }} + steps: + - name: Clone + uses: actions/checkout@v3 + + - name: Download artifacts + uses: actions/download-artifact@v3 - name: Upload coverage data - if: ${{ steps.check-build.outputs.build == 'strawberry' }} uses: codecov/codecov-action@v3 with: - name: Postgres + name: Server fail_ci_if_error: true - files: /home/runner/frappe-bench/sites/coverage.xml verbose: true flags: server diff --git a/.github/workflows/ui-tests.yml b/.github/workflows/ui-tests.yml index da6b095451..1b88bc73ce 100644 --- a/.github/workflows/ui-tests.yml +++ b/.github/workflows/ui-tests.yml @@ -7,21 +7,46 @@ on: branches: [ develop ] concurrency: - group: ui-develop-${{ github.event.number }} + group: ui-develop-${{ github.event_name }}-${{ github.event.number }} cancel-in-progress: true permissions: + # Do not change this as GITHUB_TOKEN is being used by roulette contents: read jobs: + checkrun: + name: Build Check + runs-on: ubuntu-latest + + outputs: + build: ${{ steps.check-build.outputs.build }} + + steps: + - name: Clone + uses: actions/checkout@v3 + + - name: Check if build should be run + id: check-build + run: | + python "${GITHUB_WORKSPACE}/.github/helper/roulette.py" + env: + TYPE: "ui" + PR_NUMBER: ${{ github.event.number }} + REPO_NAME: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + test: runs-on: ubuntu-latest + needs: checkrun + if: ${{ needs.checkrun.outputs.build == 'strawberry' && github.repository_owner == 'frappe' }} timeout-minutes: 60 strategy: fail-fast: false matrix: - containers: [1, 2, 3] + # Make sure you modify coverage submission file list if changing this + container: [1, 2, 3] name: UI Tests (Cypress) @@ -41,31 +66,26 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: '3.11' - - name: Check if build should be run - id: check-build + - name: Check for valid Python & Merge Conflicts run: | - python "${GITHUB_WORKSPACE}/.github/helper/roulette.py" - env: - TYPE: "ui" - PR_NUMBER: ${{ github.event.number }} - REPO_NAME: ${{ github.repository }} + python -m compileall -q -f "${GITHUB_WORKSPACE}" + if grep -lr --exclude-dir=node_modules "^<<<<<<< " "${GITHUB_WORKSPACE}" + then echo "Found merge conflicts" + exit 1 + fi - uses: actions/setup-node@v3 - if: ${{ steps.check-build.outputs.build == 'strawberry' }} with: - node-version: 14 + node-version: 16 check-latest: true - name: Add to Hosts - if: ${{ steps.check-build.outputs.build == 'strawberry' }} run: | echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts - echo "127.0.0.1 test_site_producer" | sudo tee -a /etc/hosts - name: Cache pip - if: ${{ steps.check-build.outputs.build == 'strawberry' }} uses: actions/cache@v3 with: path: ~/.cache/pip @@ -74,104 +94,105 @@ jobs: ${{ runner.os }}-pip- ${{ runner.os }}- - - name: Cache node modules - if: ${{ steps.check-build.outputs.build == 'strawberry' }} - uses: actions/cache@v3 - env: - cache-name: cache-node-modules - with: - path: ~/.npm - key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }} - restore-keys: | - ${{ runner.os }}-build-${{ env.cache-name }}- - ${{ runner.os }}-build- - ${{ runner.os }}- - - name: Get yarn cache directory path - if: ${{ steps.check-build.outputs.build == 'strawberry' }} id: yarn-cache-dir-path - run: echo "::set-output name=dir::$(yarn cache dir)" + run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT - uses: actions/cache@v3 - if: ${{ steps.check-build.outputs.build == 'strawberry' }} id: yarn-cache with: path: ${{ steps.yarn-cache-dir-path.outputs.dir }} - key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} + key: ${{ runner.os }}-yarn-ui-${{ hashFiles('**/yarn.lock') }} restore-keys: | - ${{ runner.os }}-yarn- + ${{ runner.os }}-yarn-ui- - name: Cache cypress binary - if: ${{ steps.check-build.outputs.build == 'strawberry' }} uses: actions/cache@v3 with: - path: ~/.cache - key: ${{ runner.os }}-cypress- - restore-keys: | - ${{ runner.os }}-cypress- - ${{ runner.os }}- + path: ~/.cache/Cypress + key: ${{ runner.os }}-cypress - name: Install Dependencies - if: ${{ steps.check-build.outputs.build == 'strawberry' }} - run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh + run: | + bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh + bash ${GITHUB_WORKSPACE}/.github/helper/install.sh env: BEFORE: ${{ env.GITHUB_EVENT_PATH.before }} AFTER: ${{ env.GITHUB_EVENT_PATH.after }} TYPE: ui - - - name: Install - if: ${{ steps.check-build.outputs.build == 'strawberry' }} - run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh - env: DB: mariadb - TYPE: ui + + - name: Verify yarn.lock + run: | + cd ~/frappe-bench/apps/frappe + git diff --exit-code yarn.lock - name: Instrument Source Code - if: ${{ steps.check-build.outputs.build == 'strawberry' }} run: cd ~/frappe-bench/apps/frappe/ && npx nyc instrument -x 'frappe/public/dist/**' -x 'frappe/public/js/lib/**' -x '**/*.bundle.js' --compact=false --in-place frappe - name: Build - if: ${{ steps.check-build.outputs.build == 'strawberry' }} run: cd ~/frappe-bench/ && bench build --apps frappe - name: Site Setup - if: ${{ steps.check-build.outputs.build == 'strawberry' }} - run: cd ~/frappe-bench/ && bench --site test_site execute frappe.utils.install.complete_setup_wizard + run: | + cd ~/frappe-bench/ + bench --site test_site execute frappe.utils.install.complete_setup_wizard + bench --site test_site execute frappe.tests.ui_test_helpers.create_test_user - name: UI Tests - if: ${{ steps.check-build.outputs.build == 'strawberry' }} run: cd ~/frappe-bench/ && bench --site test_site run-ui-tests frappe --with-coverage --headless --parallel --ci-build-id $GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT env: CYPRESS_RECORD_KEY: 4a48f41c-11b3-425b-aa88-c58048fa69eb - - name: Stop server - if: ${{ steps.check-build.outputs.build-server == 'strawberry' }} + - name: Stop server and wait for coverage file run: | - ps -ef | grep "frappe serve" | awk '{print $2}' | xargs kill -s SIGINT 2> /dev/null || true + ps -ef | grep "[f]rappe serve" | awk '{print $2}' | xargs kill -s SIGINT sleep 5 + ( tail -f /home/runner/frappe-bench/sites/coverage.xml & ) | grep -q "\/coverage" - - name: Check If Coverage Report Exists - id: check_coverage - uses: andstor/file-existence-action@v1 + - name: Upload JS coverage data + uses: actions/upload-artifact@v3 with: - files: "/home/runner/frappe-bench/apps/frappe/.cypress-coverage/clover.xml" + name: coverage-js-${{ matrix.container }} + path: /home/runner/frappe-bench/apps/frappe/.cypress-coverage/clover.xml - - name: Upload Coverage Data - if: ${{ steps.check-build.outputs.build == 'strawberry' && steps.check_coverage.outputs.files_exists == 'true' }} + - name: Upload python coverage data + uses: actions/upload-artifact@v3 + with: + name: coverage-py-${{ matrix.container }} + path: /home/runner/frappe-bench/sites/coverage.xml + + - name: Show bench output + if: ${{ always() }} + run: cat ~/frappe-bench/bench_start.log || true + + + coverage: + name: Coverage Wrap Up + needs: [test, checkrun] + if: ${{ needs.checkrun.outputs.build == 'strawberry' }} + runs-on: ubuntu-latest + steps: + - name: Clone + uses: actions/checkout@v3 + + - name: Download artifacts + uses: actions/download-artifact@v3 + + - name: Upload python coverage data + uses: codecov/codecov-action@v3 + with: + name: UIBackend + fail_ci_if_error: true + verbose: true + files: ./coverage-py-1/coverage.xml,./coverage-py-2/coverage.xml,./coverage-py-3/coverage.xml + flags: server-ui + + - name: Upload JS coverage data uses: codecov/codecov-action@v3 with: name: Cypress fail_ci_if_error: true - directory: /home/runner/frappe-bench/apps/frappe/.cypress-coverage/ + files: ./coverage-js-1/clover.xml,./coverage-js-2/clover.xml,./coverage-js-3/clover.xml verbose: true flags: ui-tests - - - name: Upload Server Coverage Data - if: ${{ steps.check-build.outputs.build-server == 'strawberry' }} - uses: codecov/codecov-action@v3 - with: - name: MariaDB - fail_ci_if_error: true - files: /home/runner/frappe-bench/sites/coverage.xml - verbose: true - flags: server diff --git a/.mergify.yml b/.mergify.yml index d9896df921..0881dd591b 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -4,11 +4,13 @@ pull_request_rules: - and: - and: - author!=surajshetty3416 - - author!=gavindsouza - author!=deepeshgarg007 - author!=ankush + - author!=frappe-pr-bot - author!=mergify[bot] - or: + - base=version-15 + - base=version-14 - base=version-13 - base=version-12 actions: @@ -20,15 +22,6 @@ pull_request_rules: - name: Automatic merge on CI success and review conditions: - - status-success=Sider - - status-success=Python Unit Tests (MariaDB) (1) - - status-success=Python Unit Tests (MariaDB) (2) - - status-success=Python Unit Tests (Postgres) (1) - - status-success=Python Unit Tests (Postgres) (2) - - status-success=UI Tests (Cypress) (1) - - status-success=UI Tests (Cypress) (2) - - status-success=UI Tests (Cypress) (3) - - status-success=security/snyk (frappe) - label!=dont-merge - label!=squash - "#approved-reviews-by>=1" @@ -37,15 +30,6 @@ pull_request_rules: method: merge - name: Automatic squash on CI success and review conditions: - - status-success=Sider - - status-success=Python Unit Tests (MariaDB) (1) - - status-success=Python Unit Tests (MariaDB) (2) - - status-success=Python Unit Tests (Postgres) (1) - - status-success=Python Unit Tests (Postgres) (2) - - status-success=UI Tests (Cypress) (1) - - status-success=UI Tests (Cypress) (2) - - status-success=UI Tests (Cypress) (3) - - status-success=security/snyk (frappe) - label!=dont-merge - label=squash - "#approved-reviews-by>=1" @@ -77,22 +61,13 @@ pull_request_rules: assignees: - "{{ author }}" - - name: backport to version-13-pre-release + - name: backport to version-14-hotfix conditions: - - label="backport version-13-pre-release" + - label="backport version-14-hotfix" actions: backport: branches: - - version-13-pre-release + - version-14-hotfix assignees: - "{{ author }}" - - name: backport to version-12-hotfix - conditions: - - label="backport version-12-hotfix" - actions: - backport: - branches: - - version-12-hotfix - assignees: - - "{{ author }}" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b231221517..0c6bbe8ec9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,23 +26,38 @@ repos: - id: pyupgrade args: ['--py310-plus'] - - repo: https://github.com/adityahase/black - rev: 9cb0a69f4d0030cdf687eddf314468b39ed54119 + - repo: https://github.com/frappe/black + rev: 951ccf4d5bb0d692b457a5ebc4215d755618eb68 hooks: - id: black - additional_dependencies: ['click==8.0.4'] - - repo: https://github.com/timothycrosley/isort - rev: 5.9.1 + - repo: https://github.com/pre-commit/mirrors-prettier + rev: v2.7.1 + hooks: + - id: prettier + types_or: [javascript] + # Ignore any files that might contain jinja / bundles + exclude: | + (?x)^( + frappe/public/dist/.*| + .*node_modules.*| + .*boilerplate.*| + frappe/www/website_script.js| + frappe/templates/includes/.*| + frappe/public/js/lib/.* + )$ + + + - repo: https://github.com/PyCQA/isort + rev: 5.12.0 hooks: - id: isort - - repo: https://gitlab.com/pycqa/flake8 - rev: 3.9.2 + - repo: https://github.com/PyCQA/flake8 + rev: 5.0.4 hooks: - id: flake8 additional_dependencies: ['flake8-bugbear',] - args: ['--config', '.github/helper/flake8.conf'] ci: autoupdate_schedule: weekly diff --git a/.releaserc b/.releaserc index c9ca71bbf5..86f4f3cda0 100644 --- a/.releaserc +++ b/.releaserc @@ -13,9 +13,9 @@ [ "@semantic-release/git", { "assets": ["frappe/__init__.py"], - "message": "chore(release): Bumped to Version ${nextRelease.version}\n\n${nextRelease.notes}" + "message": "chore(release): Bumped to Version ${nextRelease.version}" } ], "@semantic-release/github" ] -} \ No newline at end of file +} diff --git a/.snyk b/.snyk deleted file mode 100644 index 6c6555a819..0000000000 --- a/.snyk +++ /dev/null @@ -1,101 +0,0 @@ -# Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities. -version: v1.19.0 -# ignores vulnerabilities until expiry date; change duration by modifying expiry date -ignore: - SNYK-JS-AWESOMPLETE-174474: - - awesomplete: - reason: No patch available - expires: '2019-06-11T14:12:04.995Z' - 'npm:mem:20180117': - - showdown > yargs > os-locale > mem: - reason: No patch available - expires: '2019-06-11T14:12:04.995Z' - SNYK-PYTHON-PYYAML-550022: - - '*': - reason: Project is not directly dependant on the package - expires: 2021-04-01T18:02:21.256Z -# patches apply the minimum changes required to fix a vulnerability -patch: - 'npm:extend:20180424': - - superagent > extend: - patched: '2019-05-09T10:14:19.246Z' - SNYK-JS-LODASH-450202: - - frappe-datatable > lodash: - patched: '2020-01-31T01:33:09.889Z' - SNYK-JS-LODASH-567746: - - frappe-datatable > lodash: - patched: '2020-04-30T23:02:32.330Z' - - quagga > lodash: - patched: '2020-04-30T23:02:32.330Z' - - snyk > lodash: - patched: '2020-04-30T23:02:32.330Z' - - tailwindcss > lodash: - patched: '2020-04-30T23:02:32.330Z' - - '@tailwindcss/ui > @tailwindcss/custom-forms > lodash': - patched: '2020-04-30T23:02:32.330Z' - - snyk > @snyk/dep-graph > lodash: - patched: '2020-04-30T23:02:32.330Z' - - snyk > inquirer > lodash: - patched: '2020-04-30T23:02:32.330Z' - - snyk > snyk-config > lodash: - patched: '2020-04-30T23:02:32.330Z' - - snyk > snyk-mvn-plugin > lodash: - patched: '2020-04-30T23:02:32.330Z' - - snyk > snyk-nodejs-lockfile-parser > lodash: - patched: '2020-04-30T23:02:32.330Z' - - snyk > snyk-nuget-plugin > lodash: - patched: '2020-04-30T23:02:32.330Z' - - snyk > @snyk/dep-graph > graphlib > lodash: - patched: '2020-04-30T23:02:32.330Z' - - snyk > snyk-go-plugin > graphlib > lodash: - patched: '2020-04-30T23:02:32.330Z' - - snyk > snyk-nodejs-lockfile-parser > graphlib > lodash: - patched: '2020-04-30T23:02:32.330Z' - - snyk > @snyk/snyk-cocoapods-plugin > @snyk/dep-graph > lodash: - patched: '2020-04-30T23:02:32.330Z' - - snyk > snyk-nuget-plugin > dotnet-deps-parser > lodash: - patched: '2020-04-30T23:02:32.330Z' - - snyk > snyk-php-plugin > @snyk/composer-lockfile-parser > lodash: - patched: '2020-04-30T23:02:32.330Z' - - snyk > @snyk/snyk-cocoapods-plugin > @snyk/dep-graph > graphlib > lodash: - patched: '2020-04-30T23:02:32.330Z' - - snyk > @snyk/snyk-cocoapods-plugin > @snyk/cocoapods-lockfile-parser > @snyk/ruby-semver > lodash: - patched: '2020-04-30T23:02:32.330Z' - - snyk > @snyk/snyk-cocoapods-plugin > @snyk/cocoapods-lockfile-parser > @snyk/dep-graph > graphlib > lodash: - patched: '2020-04-30T23:02:32.330Z' - - quill-image-resize > lodash: - patched: '2020-08-24T23:06:37.710Z' - - node-sass > lodash: - patched: '2020-09-15T23:06:41.931Z' - - node-sass > sass-graph > lodash: - patched: '2020-09-15T23:06:41.931Z' - - node-sass > gaze > globule > lodash: - patched: '2020-09-15T23:06:41.931Z' - - snyk > graphlib > lodash: - patched: '2020-09-16T23:06:38.881Z' - - snyk > @snyk/snyk-cocoapods-plugin > @snyk/dep-graph > graphlib > lodash: - patched: '2020-09-16T23:06:38.881Z' - - snyk > snyk-cpp-plugin > @snyk/dep-graph > graphlib > lodash: - patched: '2020-09-16T23:06:38.881Z' - - snyk > snyk-go-plugin > @snyk/dep-graph > graphlib > lodash: - patched: '2020-09-16T23:06:38.881Z' - - snyk > snyk-gradle-plugin > @snyk/dep-graph > graphlib > lodash: - patched: '2020-09-16T23:06:38.881Z' - - snyk > snyk-docker-plugin > snyk-nodejs-lockfile-parser > graphlib > lodash: - patched: '2020-09-16T23:06:38.881Z' - - snyk > snyk-mvn-plugin > @snyk/java-call-graph-builder > graphlib > lodash: - patched: '2020-09-16T23:06:38.881Z' - - snyk > @snyk/snyk-cocoapods-plugin > @snyk/cocoapods-lockfile-parser > @snyk/dep-graph > graphlib > lodash: - patched: '2020-09-16T23:06:38.881Z' - - snyk > snyk-php-plugin > @snyk/cli-interface > @snyk/dep-graph > graphlib > lodash: - patched: '2020-09-16T23:06:38.881Z' - - snyk > snyk-gradle-plugin > @snyk/cli-interface > @snyk/dep-graph > graphlib > lodash: - patched: '2020-09-16T23:06:38.881Z' - - snyk > snyk-mvn-plugin > @snyk/cli-interface > @snyk/dep-graph > graphlib > lodash: - patched: '2020-09-16T23:06:38.881Z' - - snyk > @snyk/dep-graph > graphlib > lodash: - patched: '2020-09-16T23:06:38.881Z' - - snyk > snyk-nodejs-lockfile-parser > graphlib > lodash: - patched: '2020-09-16T23:06:38.881Z' - - snyk > snyk-go-plugin > graphlib > lodash: - patched: '2020-09-16T23:06:38.881Z' diff --git a/.stylelintrc b/.stylelintrc deleted file mode 100644 index 1e05d1fb41..0000000000 --- a/.stylelintrc +++ /dev/null @@ -1,9 +0,0 @@ -{ - "extends": ["stylelint-config-recommended"], - "plugins": ["stylelint-scss"], - "rules": { - "at-rule-no-unknown": null, - "scss/at-rule-no-unknown": true, - "no-descending-specificity": null - } -} \ No newline at end of file diff --git a/CODEOWNERS b/CODEOWNERS index 59832e8636..861016710a 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -6,13 +6,7 @@ * @frappe/frappe-review-team templates/ @surajshetty3416 www/ @surajshetty3416 -patches/ @surajshetty3416 @gavindsouza -event_streaming/ @ruchamahabal +patches/ @surajshetty3416 data_import* @netchampfaris core/ @surajshetty3416 -database @gavindsouza -model @gavindsouza -pyproject.toml @gavindsouza -query_builder/ @gavindsouza -commands/ @gavindsouza workspace @shariquerik diff --git a/README.md b/README.md index 4942d87e18..562437d5d1 100644 --- a/README.md +++ b/README.md @@ -14,25 +14,28 @@
- - + + + + + + + + + + + - - - - - -
-Full-stack web application framework that uses Python and MariaDB on the server side and a tightly integrated client side library. Built for [ERPNext](https://erpnext.com) +Full-stack web application framework that uses Python and MariaDB on the server side and a tightly integrated client side library. Built for [ERPNext](https://erpnext.com).
@@ -72,3 +75,5 @@ Full-stack web application framework that uses Python and MariaDB on the server ## License This repository has been released under the [MIT License](LICENSE). + +By contributing to Frappe, you agree that your contributions will be licensed under its MIT License. diff --git a/SECURITY.md b/SECURITY.md index 1126c8b4da..cbe6016713 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,7 +1,7 @@ # Security Policy -The Frappe team and community take security issues in the Frappe Framework seriously. To report a security issue, fill out the form at [https://erpnext.com/security/report](https://erpnext.com/security/report). +The Frappe team and community take security issues in the Frappe Framework seriously. To report a security issue, please go through the information mentioned [here](https://frappe.io/security). You can help us make Frappe and consequently all Frappe dependent apps like [ERPNext](https://erpnext.com) more secure by following the [Reporting guidelines](https://erpnext.com/security). -We appreciate your efforts to responsibly disclose your findings. We'll endeavor to respond quickly, and will keep you updated throughout the process. \ No newline at end of file +We appreciate your efforts to responsibly disclose your findings. We'll endeavor to respond quickly, and will keep you updated throughout the process. diff --git a/attributions.md b/attributions.md index 5afc9f9d46..611b9b1d9a 100644 --- a/attributions.md +++ b/attributions.md @@ -11,6 +11,7 @@ The following 3rd-party software packages may be used by or distributed with ### Icon Fonts diff --git a/bandit.yml b/bandit.yml deleted file mode 100644 index b8560e97c8..0000000000 --- a/bandit.yml +++ /dev/null @@ -1 +0,0 @@ -skips: ['E0203', 'B605', 'B404', 'B603', 'B607'] \ No newline at end of file diff --git a/codecov.yml b/codecov.yml index 1326403cfe..3fca9d9384 100644 --- a/codecov.yml +++ b/codecov.yml @@ -2,17 +2,16 @@ codecov: require_ci_to_pass: yes coverage: + range: 60..90 status: project: - default: false - server: + default: target: auto threshold: 0.5% flags: - server patch: - default: false - server: + default: target: 85% threshold: 0% only_pulls: true @@ -23,13 +22,39 @@ coverage: comment: layout: "diff, flags" require_changes: true + show_critical_paths: true flags: server: paths: - - ".*\\.py" + - "**/*.py" carryforward: true ui-tests: paths: - - ".*\\.js" + - "**/*.js" carryforward: true + server-ui: + paths: + - "**/*.py" + carryforward: true + +profiling: + critical_files_paths: + - /frappe/api.py + - /frappe/app.py + - /frappe/auth.py + - /frappe/boot.py + - /frappe/client.py + - /frappe/handler.py + - /frappe/migrate.py + - /frappe/sessions.py + - /frappe/utils/* + - /frappe/desk/reportview.py + - /frappe/desk/form/* + - /frappe/model/* + - /frappe/core/doctype/doctype/* + - /frappe/core/doctype/data_import/* + - /frappe/core/doctype/user/* + - /frappe/core/doctype/user/* + - /frappe/query_builder/* + - /frappe/database/* diff --git a/commitlint.config.js b/commitlint.config.js new file mode 100644 index 0000000000..09de8b8272 --- /dev/null +++ b/commitlint.config.js @@ -0,0 +1,25 @@ +module.exports = { + parserPreset: "conventional-changelog-conventionalcommits", + rules: { + "subject-empty": [2, "never"], + "type-case": [2, "always", "lower-case"], + "type-empty": [2, "never"], + "type-enum": [ + 2, + "always", + [ + "build", + "chore", + "ci", + "docs", + "feat", + "fix", + "perf", + "refactor", + "revert", + "style", + "test", + ], + ], + }, +}; diff --git a/cypress.config.js b/cypress.config.js new file mode 100644 index 0000000000..bfd0bc0025 --- /dev/null +++ b/cypress.config.js @@ -0,0 +1,24 @@ +const { defineConfig } = require("cypress"); + +module.exports = defineConfig({ + projectId: "92odwv", + adminPassword: "admin", + testUser: "frappe@example.com", + defaultCommandTimeout: 20000, + pageLoadTimeout: 15000, + video: true, + videoUploadOnPasses: false, + retries: { + runMode: 2, + openMode: 2, + }, + e2e: { + // We've imported your old cypress plugins here. + // You may want to clean this up later by importing these. + setupNodeEvents(on, config) { + return require("./cypress/plugins/index.js")(on, config); + }, + baseUrl: "http://test_site_ui:8000", + specPattern: ["./cypress/integration/*.js", "**/ui_test_*.js"], + }, +}); diff --git a/cypress.json b/cypress.json deleted file mode 100644 index 15f8f230fa..0000000000 --- a/cypress.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "baseUrl": "http://test_site_ui:8000", - "projectId": "92odwv", - "adminPassword": "admin", - "defaultCommandTimeout": 20000, - "pageLoadTimeout": 15000, - "video": true, - "videoUploadOnPasses": false, - "retries": { - "runMode": 2, - "openMode": 2 - }, - "integrationFolder": ".", - "testFiles": ["cypress/integration/*.js", "**/ui_test_*.js"] -} diff --git a/cypress/fixtures/child_table_doctype.js b/cypress/fixtures/child_table_doctype.js index f65e5d1765..88a925aca3 100644 --- a/cypress/fixtures/child_table_doctype.js +++ b/cypress/fixtures/child_table_doctype.js @@ -13,8 +13,8 @@ export default { fieldtype: "Data", in_list_view: 1, label: "Title", - unique: 1 - } + unique: 1, + }, ], links: [], istable: 1, @@ -24,7 +24,7 @@ export default { naming_rule: "By fieldname", owner: "Administrator", permissions: [], - sort_field: 'modified', - sort_order: 'ASC', - track_changes: 1 -}; \ No newline at end of file + sort_field: "modified", + sort_order: "ASC", + track_changes: 1, +}; diff --git a/cypress/fixtures/child_table_doctype_1.js b/cypress/fixtures/child_table_doctype_1.js index 4657d63e2e..abf8873bff 100644 --- a/cypress/fixtures/child_table_doctype_1.js +++ b/cypress/fixtures/child_table_doctype_1.js @@ -12,38 +12,38 @@ export default { fieldname: "data", fieldtype: "Data", in_list_view: 1, - label: "Data" + label: "Data", }, { fieldname: "barcode", fieldtype: "Barcode", in_list_view: 1, - label: "Barcode" + label: "Barcode", }, { fieldname: "check", fieldtype: "Check", in_list_view: 1, - label: "Check" + label: "Check", }, { fieldname: "rating", fieldtype: "Rating", in_list_view: 1, - label: "Rating" + label: "Rating", }, { fieldname: "duration", fieldtype: "Duration", in_list_view: 1, - label: "Duration" + label: "Duration", }, { fieldname: "date", fieldtype: "Date", in_list_view: 1, - label: "Date" - } + label: "Date", + }, ], links: [], istable: 1, @@ -53,7 +53,7 @@ export default { naming_rule: "By fieldname", owner: "Administrator", permissions: [], - sort_field: 'modified', - sort_order: 'ASC', - track_changes: 1 -}; \ No newline at end of file + sort_field: "modified", + sort_order: "ASC", + track_changes: 1, +}; diff --git a/cypress/fixtures/custom_submittable_doctype.js b/cypress/fixtures/custom_submittable_doctype.js index c88d37b373..30aa698db4 100644 --- a/cypress/fixtures/custom_submittable_doctype.js +++ b/cypress/fixtures/custom_submittable_doctype.js @@ -1,37 +1,37 @@ export default { - name: 'Custom Submittable DocType', + name: "Custom Submittable DocType", custom: 1, actions: [], is_submittable: 1, - creation: '2019-12-10 06:29:07.215072', - doctype: 'DocType', + creation: "2019-12-10 06:29:07.215072", + doctype: "DocType", editable_grid: 1, - engine: 'InnoDB', + engine: "InnoDB", fields: [ { - fieldname: 'enabled', - fieldtype: 'Check', - label: 'Enabled', + fieldname: "enabled", + fieldtype: "Check", + label: "Enabled", allow_on_submit: 1, - reqd: 1 + reqd: 1, }, { - fieldname: 'title', - fieldtype: 'Data', - label: 'title', - reqd: 1 + fieldname: "title", + fieldtype: "Data", + label: "title", + reqd: 1, }, { - fieldname: 'description', - fieldtype: 'Text Editor', - label: 'Description' - } + fieldname: "description", + fieldtype: "Text Editor", + label: "Description", + }, ], links: [], - modified: '2019-12-10 14:40:53.127615', - modified_by: 'Administrator', - module: 'Custom', - owner: 'Administrator', + modified: "2019-12-10 14:40:53.127615", + modified_by: "Administrator", + module: "Custom", + owner: "Administrator", permissions: [ { create: 1, @@ -39,15 +39,15 @@ export default { email: 1, print: 1, read: 1, - role: 'System Manager', + role: "System Manager", share: 1, write: 1, submit: 1, - cancel: 1 - } + cancel: 1, + }, ], quick_entry: 1, - sort_field: 'modified', - sort_order: 'ASC', - track_changes: 1 -}; \ No newline at end of file + sort_field: "modified", + sort_order: "ASC", + track_changes: 1, +}; diff --git a/cypress/fixtures/data_field_validation_doctype.js b/cypress/fixtures/data_field_validation_doctype.js index da091af7e5..2901630d3f 100644 --- a/cypress/fixtures/data_field_validation_doctype.js +++ b/cypress/fixtures/data_field_validation_doctype.js @@ -1,51 +1,51 @@ export default { - name: 'Validation Test', + name: "Validation Test", custom: 1, actions: [], - creation: '2019-03-15 06:29:07.215072', - doctype: 'DocType', + creation: "2019-03-15 06:29:07.215072", + doctype: "DocType", editable_grid: 1, - engine: 'InnoDB', + engine: "InnoDB", fields: [ { - fieldname: 'email', - fieldtype: 'Data', - label: 'Email', - options: 'Email' + fieldname: "email", + fieldtype: "Data", + label: "Email", + options: "Email", }, { - fieldname: 'URL', - fieldtype: 'Data', - label: 'URL', - options: 'URL' + fieldname: "URL", + fieldtype: "Data", + label: "URL", + options: "URL", }, { - fieldname: 'Phone', - fieldtype: 'Data', - label: 'Phone', - options: 'Phone' + fieldname: "Phone", + fieldtype: "Data", + label: "Phone", + options: "Phone", }, { - fieldname: 'person_name', - fieldtype: 'Data', - label: 'Person Name', - options: 'Name' + fieldname: "person_name", + fieldtype: "Data", + label: "Person Name", + options: "Name", }, { - fieldname: 'read_only_url', - fieldtype: 'Data', - label: 'Read Only URL', - options: 'URL', - read_only: '1', - default: 'https://frappe.io' - } + fieldname: "read_only_url", + fieldtype: "Data", + label: "Read Only URL", + options: "URL", + read_only: "1", + default: "https://frappe.io", + }, ], issingle: 1, links: [], - modified: '2021-04-19 14:40:53.127615', - modified_by: 'Administrator', - module: 'Custom', - owner: 'Administrator', + modified: "2021-04-19 14:40:53.127615", + modified_by: "Administrator", + module: "Custom", + owner: "Administrator", permissions: [ { create: 1, @@ -53,13 +53,13 @@ export default { email: 1, print: 1, read: 1, - role: 'System Manager', + role: "System Manager", share: 1, - write: 1 - } + write: 1, + }, ], quick_entry: 1, - sort_field: 'modified', - sort_order: 'ASC', - track_changes: 1 + sort_field: "modified", + sort_order: "ASC", + track_changes: 1, }; diff --git a/cypress/fixtures/datetime_doctype.js b/cypress/fixtures/datetime_doctype.js index b8c89ced5c..f1a77ba6bb 100644 --- a/cypress/fixtures/datetime_doctype.js +++ b/cypress/fixtures/datetime_doctype.js @@ -1,34 +1,34 @@ export default { - name: 'DateTime Test', + name: "DateTime Test", custom: 1, actions: [], - creation: '2019-03-15 06:29:07.215072', - doctype: 'DocType', + creation: "2019-03-15 06:29:07.215072", + doctype: "DocType", editable_grid: 1, - engine: 'InnoDB', + engine: "InnoDB", fields: [ { - fieldname: 'date', - fieldtype: 'Date', - label: 'Date' + fieldname: "date", + fieldtype: "Date", + label: "Date", }, { - fieldname: 'time', - fieldtype: 'Time', - label: 'Time' + fieldname: "time", + fieldtype: "Time", + label: "Time", }, { - fieldname: 'datetime', - fieldtype: 'Datetime', - label: 'Datetime' - } + fieldname: "datetime", + fieldtype: "Datetime", + label: "Datetime", + }, ], issingle: 1, links: [], - modified: '2019-12-09 14:40:53.127615', - modified_by: 'Administrator', - module: 'Custom', - owner: 'Administrator', + modified: "2019-12-09 14:40:53.127615", + modified_by: "Administrator", + module: "Custom", + owner: "Administrator", permissions: [ { create: 1, @@ -36,13 +36,13 @@ export default { email: 1, print: 1, read: 1, - role: 'System Manager', + role: "System Manager", share: 1, - write: 1 - } + write: 1, + }, ], quick_entry: 1, - sort_field: 'modified', - sort_order: 'ASC', - track_changes: 1 + sort_field: "modified", + sort_order: "ASC", + track_changes: 1, }; diff --git a/cypress/fixtures/doctype_to_link.js b/cypress/fixtures/doctype_to_link.js index f5335b1755..ff5d1b5c68 100644 --- a/cypress/fixtures/doctype_to_link.js +++ b/cypress/fixtures/doctype_to_link.js @@ -10,18 +10,18 @@ export default { engine: "InnoDB", fields: [ { - "fieldname": "title", - "fieldtype": "Data", - "label": "Title", - "unique": 1 - } + fieldname: "title", + fieldtype: "Data", + label: "Title", + unique: 1, + }, ], links: [ { - "group": "Child Doctype", - "link_doctype": "Doctype With Child Table", - "link_fieldname": "title" - } + group: "Child Doctype", + link_doctype: "Doctype With Child Table", + link_fieldname: "title", + }, ], modified: "2022-02-10 12:03:12.603763", modified_by: "Administrator", @@ -34,12 +34,12 @@ export default { email: 1, print: 1, read: 1, - role: 'System Manager', + role: "System Manager", share: 1, - write: 1 - } + write: 1, + }, ], - sort_field: 'modified', - sort_order: 'ASC', - track_changes: 1 -}; \ No newline at end of file + sort_field: "modified", + sort_order: "ASC", + track_changes: 1, +}; diff --git a/cypress/fixtures/doctype_with_child_table.js b/cypress/fixtures/doctype_with_child_table.js index 014074b0b5..7caba516cf 100644 --- a/cypress/fixtures/doctype_with_child_table.js +++ b/cypress/fixtures/doctype_with_child_table.js @@ -12,21 +12,21 @@ export default { fieldname: "title", fieldtype: "Data", label: "Title", - unique: 1 + unique: 1, }, { fieldname: "child_table", fieldtype: "Table", label: "Child Table", options: "Child Table Doctype", - reqd: 1 + reqd: 1, }, { fieldname: "child_table_1", fieldtype: "Table", label: "Child Table 1", - options: "Child Table Doctype 1" - } + options: "Child Table Doctype 1", + }, ], links: [], modified: "2022-02-10 12:03:12.603763", @@ -41,12 +41,12 @@ export default { email: 1, print: 1, read: 1, - role: 'System Manager', + role: "System Manager", share: 1, - write: 1 - } + write: 1, + }, ], - sort_field: 'modified', - sort_order: 'ASC', - track_changes: 1 + sort_field: "modified", + sort_order: "ASC", + track_changes: 1, }; diff --git a/cypress/fixtures/doctype_with_phone.js b/cypress/fixtures/doctype_with_phone.js index c62922ade2..06a24a5be5 100644 --- a/cypress/fixtures/doctype_with_phone.js +++ b/cypress/fixtures/doctype_with_phone.js @@ -4,29 +4,28 @@ export default { custom: 1, is_submittable: 1, autoname: "field:title", - creation: '2022-03-30 06:29:07.215072', - doctype: 'DocType', - engine: 'InnoDB', + creation: "2022-03-30 06:29:07.215072", + doctype: "DocType", + engine: "InnoDB", fields: [ - { - fieldname: 'title', - fieldtype: 'Data', - label: 'title', + fieldname: "title", + fieldtype: "Data", + label: "title", unique: 1, }, { - fieldname: 'phone', - fieldtype: 'Phone', - label: 'Phone' - } + fieldname: "phone", + fieldtype: "Phone", + label: "Phone", + }, ], links: [], - modified: '2019-03-30 14:40:53.127615', - modified_by: 'Administrator', + modified: "2019-03-30 14:40:53.127615", + modified_by: "Administrator", naming_rule: "By fieldname", - module: 'Custom', - owner: 'Administrator', + module: "Custom", + owner: "Administrator", permissions: [ { create: 1, @@ -34,14 +33,14 @@ export default { email: 1, print: 1, read: 1, - role: 'System Manager', + role: "System Manager", share: 1, write: 1, submit: 1, - cancel: 1 - } + cancel: 1, + }, ], - sort_field: 'modified', - sort_order: 'ASC', - track_changes: 1 + sort_field: "modified", + sort_order: "ASC", + track_changes: 1, }; diff --git a/cypress/fixtures/doctype_with_tab_break.js b/cypress/fixtures/doctype_with_tab_break.js index 74e5e6abba..44d6c16682 100644 --- a/cypress/fixtures/doctype_with_tab_break.js +++ b/cypress/fixtures/doctype_with_tab_break.js @@ -1,39 +1,39 @@ export default { - name: 'Form With Tab Break', + name: "Form With Tab Break", custom: 1, actions: [], - doctype: 'DocType', - engine: 'InnoDB', + doctype: "DocType", + engine: "InnoDB", fields: [ { - fieldname: 'username', - fieldtype: 'Data', - label: 'Name', - options: 'Name' + fieldname: "username", + fieldtype: "Data", + label: "Name", + options: "Name", }, { - fieldname: 'tab', - fieldtype: 'Tab Break', - label: 'Tab 2', + fieldname: "tab", + fieldtype: "Tab Break", + label: "Tab 2", }, { - fieldname: 'Phone', - fieldtype: 'Data', - label: 'Phone', - options: 'Phone', - reqd: 1 + fieldname: "Phone", + fieldtype: "Data", + label: "Phone", + options: "Phone", + reqd: 1, }, ], links: [ { - "group": "Profile", - "link_doctype": "Contact", - "link_fieldname": "user" + group: "Profile", + link_doctype: "Contact", + link_fieldname: "user", }, ], - modified_by: 'Administrator', - module: 'Custom', - owner: 'Administrator', + modified_by: "Administrator", + module: "Custom", + owner: "Administrator", permissions: [ { create: 1, @@ -41,14 +41,14 @@ export default { email: 1, print: 1, read: 1, - role: 'System Manager', + role: "System Manager", share: 1, - write: 1 - } + write: 1, + }, ], quick_entry: 1, autoname: "format: Test-{####}", - sort_field: 'modified', - sort_order: 'ASC', - track_changes: 1 + sort_field: "modified", + sort_order: "ASC", + track_changes: 1, }; diff --git a/cypress/fixtures/form_builder_doctype.js b/cypress/fixtures/form_builder_doctype.js new file mode 100644 index 0000000000..08b598f82a --- /dev/null +++ b/cypress/fixtures/form_builder_doctype.js @@ -0,0 +1,65 @@ +export default { + name: "Form Builder Doctype", + custom: 1, + actions: [], + doctype: "DocType", + engine: "InnoDB", + fields: [ + { + fieldname: "data3", + fieldtype: "Data", + label: "Data 3", + }, + { + fieldname: "tab", + fieldtype: "Tab Break", + label: "Tab 2", + }, + { + fieldname: "data", + fieldtype: "Data", + label: "Data", + }, + { + fieldname: "check", + fieldtype: "Check", + label: "Check", + }, + { + fieldname: "column_1", + fieldtype: "Column Break", + }, + { + fieldname: "data1", + fieldtype: "Data", + label: "Data 1", + }, + { + fieldname: "section_1", + fieldtype: "Section Break", + }, + { + fieldname: "data2", + fieldtype: "Data", + label: "Data 2", + }, + ], + modified_by: "Administrator", + module: "Custom", + owner: "Administrator", + permissions: [ + { + create: 1, + delete: 1, + email: 1, + print: 1, + read: 1, + role: "System Manager", + share: 1, + write: 1, + }, + ], + sort_field: "modified", + sort_order: "ASC", + track_changes: 1, +}; diff --git a/cypress/integration/api.js b/cypress/integration/api.js index e8c39e6e25..420cea25fd 100644 --- a/cypress/integration/api.js +++ b/cypress/integration/api.js @@ -1,42 +1,43 @@ -context('API Resources', () => { +context("API Resources", () => { before(() => { - cy.visit('/login'); + cy.visit("/login"); cy.login(); - cy.visit('/app/website'); + cy.visit("/app/website"); }); - it('Creates two Comments', () => { - cy.insert_doc('Comment', { comment_type: 'Comment', content: "hello" }); - cy.insert_doc('Comment', { comment_type: 'Comment', content: "world" }); + it("Creates two Comments", () => { + cy.insert_doc("Comment", { comment_type: "Comment", content: "hello" }); + cy.insert_doc("Comment", { comment_type: "Comment", content: "world" }); }); - it('Lists the Comments', () => { - cy.get_list('Comment') - .its('data') - .then(data => expect(data.length).to.be.at.least(2)); + it("Lists the Comments", () => { + cy.get_list("Comment") + .its("data") + .then((data) => expect(data.length).to.be.at.least(2)); - cy.get_list('Comment', ['name', 'content'], [['content', '=', 'hello']]) - .then(body => { - expect(body).to.have.property('data'); - expect(body.data).to.have.lengthOf(1); - expect(body.data[0]).to.have.property('content'); - expect(body.data[0]).to.have.property('name'); - }); + cy.get_list("Comment", ["name", "content"], [["content", "=", "hello"]]).then((body) => { + expect(body).to.have.property("data"); + expect(body.data).to.have.lengthOf(1); + expect(body.data[0]).to.have.property("content"); + expect(body.data[0]).to.have.property("name"); + }); }); - it('Gets each Comment', () => { - cy.get_list('Comment').then(body => body.data.forEach(comment => { - cy.get_doc('Comment', comment.name); - })); + it("Gets each Comment", () => { + cy.get_list("Comment").then((body) => + body.data.forEach((comment) => { + cy.get_doc("Comment", comment.name); + }) + ); }); - it('Removes the Comments', () => { - cy.get_list('Comment').then(body => { + it("Removes the Comments", () => { + cy.get_list("Comment").then((body) => { let comment_names = []; - body.data.map(comment => comment_names.push(comment.name)); + body.data.map((comment) => comment_names.push(comment.name)); comment_names = [...new Set(comment_names)]; // remove duplicates comment_names.forEach((comment_name) => { - cy.remove_doc('Comment', comment_name); + cy.remove_doc("Comment", comment_name); }); }); }); diff --git a/cypress/integration/assignment_rule.js b/cypress/integration/assignment_rule.js new file mode 100644 index 0000000000..5431561272 --- /dev/null +++ b/cypress/integration/assignment_rule.js @@ -0,0 +1,16 @@ +context("Assignment Rule", () => { + before(() => { + cy.login(); + }); + + it("Custom grid buttons work", () => { + cy.new_form("Assignment Rule"); + cy.findByRole("button", { name: "All Days" }).should("be.visible").click(); + cy.wait(2000); + cy.window() + .its("cur_frm") + .then((frm) => { + expect(frm.doc.assignment_days.length).to.equal(7); + }); + }); +}); diff --git a/cypress/integration/awesome_bar.js b/cypress/integration/awesome_bar.js index 938034a34a..71e5e498cf 100644 --- a/cypress/integration/awesome_bar.js +++ b/cypress/integration/awesome_bar.js @@ -1,48 +1,57 @@ -context('Awesome Bar', () => { +context("Awesome Bar", () => { before(() => { - cy.visit('/login'); + cy.visit("/login"); cy.login(); - cy.visit('/app/website'); + cy.visit("/app/website"); }); beforeEach(() => { - cy.get('.navbar .navbar-home').click(); - cy.findByPlaceholderText('Search or type a command (Ctrl + G)').clear(); + cy.get(".navbar .navbar-home").click(); + cy.findByPlaceholderText("Search or type a command (Ctrl + G)").clear(); }); - it('navigates to doctype list', () => { - cy.findByPlaceholderText('Search or type a command (Ctrl + G)').type('todo', { delay: 700 }); - cy.get('.awesomplete').findByRole('listbox').should('be.visible'); - cy.findByPlaceholderText('Search or type a command (Ctrl + G)').type('{enter}', { delay: 700 }); + it("navigates to doctype list", () => { + cy.findByPlaceholderText("Search or type a command (Ctrl + G)").type("todo", { + delay: 700, + }); + cy.get(".awesomplete").findByRole("listbox").should("be.visible"); + cy.findByPlaceholderText("Search or type a command (Ctrl + G)").type("{enter}", { + delay: 700, + }); - cy.get('.title-text').should('contain', 'To Do'); + cy.get(".title-text").should("contain", "To Do"); - cy.location('pathname').should('eq', '/app/todo'); + cy.location("pathname").should("eq", "/app/todo"); }); - it('find text in doctype list', () => { - cy.findByPlaceholderText('Search or type a command (Ctrl + G)') - .type('test in todo{enter}', { delay: 700 }); + it("find text in doctype list", () => { + cy.findByPlaceholderText("Search or type a command (Ctrl + G)").type( + "test in todo{enter}", + { delay: 700 } + ); - cy.get('.title-text').should('contain', 'To Do'); + cy.get(".title-text").should("contain", "To Do"); - cy.findByPlaceholderText('ID') - .should('have.value', '%test%'); + cy.findByPlaceholderText("ID").should("have.value", "%test%"); cy.clear_filters(); }); - it('navigates to new form', () => { - cy.findByPlaceholderText('Search or type a command (Ctrl + G)') - .type('new blog post{enter}', { delay: 700 }); + it("navigates to new form", () => { + cy.findByPlaceholderText("Search or type a command (Ctrl + G)").type( + "new blog post{enter}", + { delay: 700 } + ); - cy.get('.title-text:visible').should('have.text', 'New Blog Post'); + cy.get(".title-text:visible").should("have.text", "New Blog Post"); }); - it('calculates math expressions', () => { - cy.findByPlaceholderText('Search or type a command (Ctrl + G)') - .type('55 + 32{downarrow}{enter}', { delay: 700 }); + it("calculates math expressions", () => { + cy.findByPlaceholderText("Search or type a command (Ctrl + G)").type( + "55 + 32{downarrow}{enter}", + { delay: 700 } + ); - cy.get('.modal-title').should('contain', 'Result'); - cy.get('.msgprint').should('contain', '55 + 32 = 87'); + cy.get(".modal-title").should("contain", "Result"); + cy.get(".msgprint").should("contain", "55 + 32 = 87"); }); }); diff --git a/cypress/integration/control_attach.js b/cypress/integration/control_attach.js index 0552780737..96b8c73b6e 100644 --- a/cypress/integration/control_attach.js +++ b/cypress/integration/control_attach.js @@ -1,90 +1,95 @@ -context('Attach Control', () => { +context("Attach Control", () => { before(() => { cy.login(); - cy.visit('/app/doctype'); - return cy.window().its('frappe').then(frappe => { - return frappe.xcall('frappe.tests.ui_test_helpers.create_doctype', { - name: 'Test Attach Control', - fields: [ - { - "label": "Attach File or Image", - "fieldname": "attach", - "fieldtype": "Attach", - "in_list_view": 1, - }, - ] + cy.visit("/app/doctype"); + return cy + .window() + .its("frappe") + .then((frappe) => { + return frappe.xcall("frappe.tests.ui_test_helpers.create_doctype", { + name: "Test Attach Control", + fields: [ + { + label: "Attach File or Image", + fieldname: "attach", + fieldtype: "Attach", + in_list_view: 1, + }, + ], + }); }); - }); }); it('Checking functionality for "Link" button in the "Attach" fieldtype', () => { //Navigating to the new form for the newly created doctype - cy.new_form('Test Attach Control'); + cy.new_form("Test Attach Control"); //Clicking on the attach button which is displayed as part of creating a doctype with "Attach" fieldtype - cy.findByRole('button', {name: 'Attach'}).click(); + cy.findByRole("button", { name: "Attach" }).click(); //Clicking on "Link" button to attach a file using the "Link" button - cy.findByRole('button', {name: 'Link'}).click(); - cy.findByPlaceholderText('Attach a web link').type('https://wallpaperplay.com/walls/full/8/2/b/72402.jpg'); - + cy.findByRole("button", { name: "Link" }).click(); + cy.findByPlaceholderText("Attach a web link").type( + "https://wallpaperplay.com/walls/full/8/2/b/72402.jpg" + ); + //Clicking on the Upload button to upload the file cy.intercept("POST", "/api/method/upload_file").as("upload_image"); - cy.get('.modal-footer').findByRole("button", {name: "Upload"}).click({delay: 500}); + cy.get(".modal-footer").findByRole("button", { name: "Upload" }).click({ delay: 500 }); cy.wait("@upload_image"); - cy.findByRole('button', {name: 'Save'}).click(); + cy.findByRole("button", { name: "Save" }).click(); //Checking if the URL of the attached image is getting displayed in the field of the newly created doctype - cy.get('.attached-file > .ellipsis > .attached-file-link') - .should('have.attr', 'href') - .and('equal', 'https://wallpaperplay.com/walls/full/8/2/b/72402.jpg'); + cy.get(".attached-file > .ellipsis > .attached-file-link") + .should("have.attr", "href") + .and("equal", "https://wallpaperplay.com/walls/full/8/2/b/72402.jpg"); //Clicking on the "Clear" button cy.get('[data-action="clear_attachment"]').click(); //Checking if clicking on the clear button clears the field of the doctype form and again displays the attach button - cy.get('.control-input > .btn-sm').should('contain', 'Attach'); + cy.get(".control-input > .btn-sm").should("contain", "Attach"); //Deleting the doc - cy.go_to_list('Test Attach Control'); - cy.get('.list-row-checkbox').eq(0).click(); - cy.get('.actions-btn-group > .btn').contains('Actions').click(); + cy.go_to_list("Test Attach Control"); + cy.get(".list-row-checkbox").eq(0).click(); + cy.get(".actions-btn-group > .btn").contains("Actions").click(); cy.get('.actions-btn-group > .dropdown-menu [data-label="Delete"]').click(); - cy.click_modal_primary_button('Yes'); + cy.click_modal_primary_button("Yes"); }); it('Checking functionality for "Library" button in the "Attach" fieldtype', () => { //Navigating to the new form for the newly created doctype - cy.new_form('Test Attach Control'); + cy.new_form("Test Attach Control"); //Clicking on the attach button which is displayed as part of creating a doctype with "Attach" fieldtype - cy.findByRole('button', {name: 'Attach'}).click(); + cy.findByRole("button", { name: "Attach" }).click(); //Clicking on "Library" button to attach a file using the "Library" button - cy.findByRole('button', {name: 'Library'}).click(); - cy.contains('72402.jpg').click(); + cy.findByRole("button", { name: "Library" }).click(); + cy.contains("72402.jpg").click(); //Clicking on the Upload button to upload the file cy.intercept("POST", "/api/method/upload_file").as("upload_image"); - cy.get('.modal-footer').findByRole("button", {name: "Upload"}).click({delay: 500}); + cy.get(".modal-footer").findByRole("button", { name: "Upload" }).click({ delay: 500 }); cy.wait("@upload_image"); - cy.findByRole('button', {name: 'Save'}).click(); + cy.findByRole("button", { name: "Save" }).click(); //Checking if the URL of the attached image is getting displayed in the field of the newly created doctype - cy.get('.attached-file > .ellipsis > .attached-file-link') - .should('have.attr', 'href') - .and('equal', 'https://wallpaperplay.com/walls/full/8/2/b/72402.jpg'); + cy.get(".attached-file > .ellipsis > .attached-file-link") + .should("have.attr", "href") + .and("equal", "https://wallpaperplay.com/walls/full/8/2/b/72402.jpg"); //Clicking on the "Clear" button cy.get('[data-action="clear_attachment"]').click(); //Checking if clicking on the clear button clears the field of the doctype form and again displays the attach button - cy.get('.control-input > .btn-sm').should('contain', 'Attach'); + cy.get(".control-input > .btn-sm").should("contain", "Attach"); //Deleting the doc - cy.go_to_list('Test Attach Control'); - cy.get('.list-row-checkbox').eq(0).click(); - cy.get('.actions-btn-group > .btn').contains('Actions').click(); + cy.go_to_list("Test Attach Control"); + cy.get(".list-row-checkbox").eq(0).click(); + cy.get(".actions-btn-group > .btn").contains("Actions").click(); cy.get('.actions-btn-group > .dropdown-menu [data-label="Delete"]').click(); - cy.click_modal_primary_button('Yes'); + cy.click_modal_primary_button("Yes"); }); -}); \ No newline at end of file +}); diff --git a/cypress/integration/control_autocomplete.js b/cypress/integration/control_autocomplete.js index 3bf3e829f9..6dc57fcf43 100644 --- a/cypress/integration/control_autocomplete.js +++ b/cypress/integration/control_autocomplete.js @@ -1,57 +1,64 @@ -context('Control Autocomplete', () => { +context("Control Autocomplete", () => { before(() => { cy.login(); - cy.visit('/app/website'); + cy.visit("/app/website"); }); function get_dialog_with_autocomplete(options) { - cy.visit('/app/website'); + cy.visit("/app/website"); return cy.dialog({ - title: 'Autocomplete', + title: "Autocomplete", fields: [ { - 'label': 'Select an option', - 'fieldname': 'autocomplete', - 'fieldtype': 'Autocomplete', - 'options': options || ['Option 1', 'Option 2', 'Option 3'], - } - ] + label: "Select an option", + fieldname: "autocomplete", + fieldtype: "Autocomplete", + options: options || ["Option 1", "Option 2", "Option 3"], + }, + ], }); } - it('should set the valid value', () => { - get_dialog_with_autocomplete().as('dialog'); + it("should set the valid value", () => { + get_dialog_with_autocomplete().as("dialog"); - cy.get('.frappe-control[data-fieldname=autocomplete] input').focus().as('input'); + cy.get(".frappe-control[data-fieldname=autocomplete] input").focus().as("input"); cy.wait(1000); - cy.get('@input').type('2', { delay: 300 }); - cy.get('.frappe-control[data-fieldname=autocomplete]').findByRole('listbox').should('be.visible'); - cy.get('.frappe-control[data-fieldname=autocomplete] input').type('{enter}', { delay: 300 }); - cy.get('.frappe-control[data-fieldname=autocomplete] input').blur(); - cy.get('@dialog').then(dialog => { - let value = dialog.get_value('autocomplete'); - expect(value).to.eq('Option 2'); + cy.get("@input").type("2", { delay: 300 }); + cy.get(".frappe-control[data-fieldname=autocomplete]") + .findByRole("listbox") + .should("be.visible"); + cy.get(".frappe-control[data-fieldname=autocomplete] input").type("{enter}", { + delay: 300, + }); + cy.get(".frappe-control[data-fieldname=autocomplete] input").blur(); + cy.get("@dialog").then((dialog) => { + let value = dialog.get_value("autocomplete"); + expect(value).to.eq("Option 2"); dialog.clear(); }); }); - it('should set the valid value with different label', () => { + it("should set the valid value with different label", () => { const options_with_label = [ { label: "Option 1", value: "option_1" }, - { label: "Option 2", value: "option_2" } + { label: "Option 2", value: "option_2" }, ]; - get_dialog_with_autocomplete(options_with_label).as('dialog'); + get_dialog_with_autocomplete(options_with_label).as("dialog"); - cy.get('.frappe-control[data-fieldname=autocomplete] input').focus().as('input'); - cy.get('.frappe-control[data-fieldname=autocomplete]').findByRole('listbox').should('be.visible'); - cy.get('@input').type('2', { delay: 300 }); - cy.get('.frappe-control[data-fieldname=autocomplete] input').type('{enter}', { delay: 300 }); - cy.get('.frappe-control[data-fieldname=autocomplete] input').blur(); - cy.get('@dialog').then(dialog => { - let value = dialog.get_value('autocomplete'); - expect(value).to.eq('option_2'); + cy.get(".frappe-control[data-fieldname=autocomplete] input").focus().as("input"); + cy.get(".frappe-control[data-fieldname=autocomplete]") + .findByRole("listbox") + .should("be.visible"); + cy.get("@input").type("2", { delay: 300 }); + cy.get(".frappe-control[data-fieldname=autocomplete] input").type("{enter}", { + delay: 300, + }); + cy.get(".frappe-control[data-fieldname=autocomplete] input").blur(); + cy.get("@dialog").then((dialog) => { + let value = dialog.get_value("autocomplete"); + expect(value).to.eq("option_2"); dialog.clear(); }); }); - }); diff --git a/cypress/integration/control_barcode.js b/cypress/integration/control_barcode.js index 85a3182397..96a1bb43d4 100644 --- a/cypress/integration/control_barcode.js +++ b/cypress/integration/control_barcode.js @@ -1,55 +1,57 @@ -context('Control Barcode', () => { +context("Control Barcode", () => { beforeEach(() => { cy.login(); - cy.visit('/app/website'); + cy.visit("/app/website"); }); function get_dialog_with_barcode() { return cy.dialog({ - title: 'Barcode', + title: "Barcode", fields: [ { - label: 'Barcode', - fieldname: 'barcode', - fieldtype: 'Barcode' - } - ] + label: "Barcode", + fieldname: "barcode", + fieldtype: "Barcode", + }, + ], }); } - it('should generate barcode on setting a value', () => { - get_dialog_with_barcode().as('dialog'); + it("should generate barcode on setting a value", () => { + get_dialog_with_barcode().as("dialog"); cy.focused().blur(); - cy.get('.frappe-control[data-fieldname=barcode]').findByRole('textbox') - .type('123456789') + cy.get(".frappe-control[data-fieldname=barcode]") + .findByRole("textbox") + .type("123456789") .blur(); - cy.get('.frappe-control[data-fieldname=barcode] svg[data-barcode-value="123456789"]') - .should('exist'); + cy.get( + '.frappe-control[data-fieldname=barcode] svg[data-barcode-value="123456789"]' + ).should("exist"); - cy.get('@dialog').then(dialog => { - let value = dialog.get_value('barcode'); - expect(value).to.contain(' { + let value = dialog.get_value("barcode"); + expect(value).to.contain(" { - get_dialog_with_barcode().as('dialog'); + it("should reset when input is cleared", () => { + get_dialog_with_barcode().as("dialog"); cy.focused().blur(); - cy.get('.frappe-control[data-fieldname=barcode]').findByRole('textbox') - .type('123456789') + cy.get(".frappe-control[data-fieldname=barcode]") + .findByRole("textbox") + .type("123456789") .blur(); - cy.get('.frappe-control[data-fieldname=barcode]').findByRole('textbox') - .clear() - .blur(); - cy.get('.frappe-control[data-fieldname=barcode] svg[data-barcode-value="123456789"]') - .should('not.exist'); + cy.get(".frappe-control[data-fieldname=barcode]").findByRole("textbox").clear().blur(); + cy.get( + '.frappe-control[data-fieldname=barcode] svg[data-barcode-value="123456789"]' + ).should("not.exist"); - cy.get('@dialog').then(dialog => { - let value = dialog.get_value('barcode'); - expect(value).to.equal(''); + cy.get("@dialog").then((dialog) => { + let value = dialog.get_value("barcode"); + expect(value).to.equal(""); }); }); }); diff --git a/cypress/integration/control_color.js b/cypress/integration/control_color.js index 8d55003618..e97dbe0f06 100644 --- a/cypress/integration/control_color.js +++ b/cypress/integration/control_color.js @@ -1,77 +1,80 @@ -context('Control Color', () => { +context("Control Color", () => { before(() => { cy.login(); - cy.visit('/app/website'); + cy.visit("/app/website"); }); function get_dialog_with_color() { return cy.dialog({ - title: 'Color', - fields: [{ - label: 'Color', - fieldname: 'color', - fieldtype: 'Color' - }] + title: "Color", + fields: [ + { + label: "Color", + fieldname: "color", + fieldtype: "Color", + }, + ], }); } - it('Verifying if the color control is selecting correct', () => { - get_dialog_with_color().as('dialog'); - cy.findByPlaceholderText('Choose a color').click(); + it("Verifying if the color control is selecting correct", () => { + get_dialog_with_color().as("dialog"); + cy.findByPlaceholderText("Choose a color").click(); ///Selecting a color from the color palette cy.get('[style="background-color: rgb(79, 157, 217);"]').click(); //Checking if the css attribute is correct - cy.get('.color-map').should('have.css', 'color', 'rgb(79, 157, 217)'); - cy.get('.hue-map').should('have.css', 'color', 'rgb(0, 145, 255)'); + cy.get(".color-map").should("have.css", "color", "rgb(79, 157, 217)"); + cy.get(".hue-map").should("have.css", "color", "rgb(0, 144, 255)"); //Checking if the correct color is being selected - cy.get('@dialog').then(dialog => { - let value = dialog.get_value('color'); - expect(value).to.equal('#4F9DD9'); + cy.get("@dialog").then((dialog) => { + let value = dialog.get_value("color"); + expect(value).to.equal("#4F9DD9"); }); //Selecting a color cy.get('[style="background-color: rgb(203, 41, 41);"]').click(); //Checking if the correct css is being selected - cy.get('.color-map').should('have.css', 'color', 'rgb(203, 41, 41)'); - cy.get('.hue-map').should('have.css', 'color', 'rgb(255, 0, 0)'); + cy.get(".color-map").should("have.css", "color", "rgb(203, 41, 41)"); + cy.get(".hue-map").should("have.css", "color", "rgb(255, 0, 0)"); //Checking if the correct color is being selected - cy.get('@dialog').then(dialog => { - let value = dialog.get_value('color'); - expect(value).to.equal('#CB2929'); + cy.get("@dialog").then((dialog) => { + let value = dialog.get_value("color"); + expect(value).to.equal("#CB2929"); }); //Selecting color from the palette - cy.get('.color-map > .color-selector').click(65, 87, {force: true}); - cy.get('.color-map').should('have.css', 'color', 'rgb(56, 0, 0)'); + cy.get(".color-map > .color-selector").click(65, 87, { force: true }); + cy.get(".color-map").should("have.css", "color", "rgb(56, 0, 0)"); //Checking if the expected color is selected and getting displayed - cy.get('@dialog').then(dialog => { - let value = dialog.get_value('color'); - expect(value).to.equal('#380000'); + cy.get("@dialog").then((dialog) => { + let value = dialog.get_value("color"); + expect(value).to.equal("#380000"); }); //Selecting the color from the hue map - cy.get('.hue-map > .hue-selector').click(35, -1, {force: true}); - cy.get('.color-map').should('have.css', 'color', 'rgb(56, 45, 0)'); - cy.get('.hue-map').should('have.css', 'color', 'rgb(255, 204, 0)'); - cy.get('.color-map > .color-selector').click(55, 12, {force: true}); - cy.get('.color-map').should('have.css', 'color', 'rgb(46, 37, 0)'); + cy.get(".hue-map > .hue-selector").click(35, -1, { force: true }); + cy.get(".color-map").should("have.css", "color", "rgb(56, 45, 0)"); + cy.get(".hue-map").should("have.css", "color", "rgb(255, 204, 0)"); + cy.get(".color-map > .color-selector").click(55, 12, { force: true }); + cy.get(".color-map").should("have.css", "color", "rgb(46, 37, 0)"); //Checking if the correct color is being displayed - cy.get('@dialog').then(dialog => { - let value = dialog.get_value('color'); - expect(value).to.equal('#2e2500'); + cy.get("@dialog").then((dialog) => { + let value = dialog.get_value("color"); + expect(value).to.equal("#2e2500"); }); //Clearing the field and checking if the field contains the placeholder "Choose a color" - cy.get('.input-with-feedback').click({force: true}); - cy.get_field('color', 'Color').type('{selectall}').clear(); - cy.get_field('color', 'Color').invoke('attr', 'placeholder').should('contain', 'Choose a color'); - + cy.get(".input-with-feedback").click({ force: true }); + cy.get_field("color", "Color").type("{selectall}").clear(); + cy.get_field("color", "Color") + .invoke("attr", "placeholder") + .should("contain", "Choose a color"); }); -}); \ No newline at end of file +}); diff --git a/cypress/integration/control_data.js b/cypress/integration/control_data.js index 78cece627b..4c7ee589ab 100644 --- a/cypress/integration/control_data.js +++ b/cypress/integration/control_data.js @@ -1,134 +1,145 @@ -context('Data Control', () => { +context("Data Control", () => { before(() => { cy.login(); - cy.visit('/app/doctype'); - return cy.window().its('frappe').then(frappe => { - return frappe.xcall('frappe.tests.ui_test_helpers.create_doctype', { - name: 'Test Data Control', - fields: [ - { - "label": "Name", - "fieldname": "name1", - "fieldtype": "Data", - "options": "Name", - "in_list_view": 1, - "reqd": 1, - }, - { - "label": "Email-ID", - "fieldname": "email", - "fieldtype": "Data", - "options": "Email", - "in_list_view": 1, - "reqd": 1, - }, - { - "label": "Phone No.", - "fieldname": "phone", - "fieldtype": "Data", - "options": "Phone", - "in_list_view": 1, - "reqd": 1, - }, - ] + cy.visit("/app/doctype"); + return cy + .window() + .its("frappe") + .then((frappe) => { + return frappe.xcall("frappe.tests.ui_test_helpers.create_doctype", { + name: "Test Data Control", + fields: [ + { + label: "Name", + fieldname: "name1", + fieldtype: "Data", + options: "Name", + in_list_view: 1, + reqd: 1, + }, + { + label: "Email-ID", + fieldname: "email", + fieldtype: "Data", + options: "Email", + in_list_view: 1, + reqd: 1, + }, + { + label: "Phone No.", + fieldname: "phone", + fieldtype: "Data", + options: "Phone", + in_list_view: 1, + reqd: 1, + }, + ], + }); }); - }); }); - it('check custom formatters', () => { + it("check custom formatters", () => { cy.visit(`/app/doctype/User`); - cy.get('[data-fieldname="fields"] .grid-row[data-idx="2"] [data-fieldname="fieldtype"] .static-area').should('have.text', '🔵 Section Break'); + cy.get( + '[data-fieldname="fields"] .grid-row[data-idx="3"] [data-fieldname="fieldtype"] .static-area' + ).should("have.text", "Section Break"); }); it('Verifying data control by inputting different patterns for "Name" field', () => { - cy.new_form('Test Data Control'); + cy.new_form("Test Data Control"); //Checking the URL for the new form of the doctype - cy.location("pathname").should('eq', '/app/test-data-control/new-test-data-control-1'); - cy.get('.title-text').should('have.text', 'New Test Data Control'); - cy.get('.frappe-control[data-fieldname="name1"]').find('label').should('have.class', 'reqd'); - cy.get('.frappe-control[data-fieldname="email"]').find('label').should('have.class', 'reqd'); - cy.get('.frappe-control[data-fieldname="phone"]').find('label').should('have.class', 'reqd'); + cy.location("pathname").should("eq", "/app/test-data-control/new-test-data-control-1"); + cy.get(".title-text").should("have.text", "New Test Data Control"); + cy.get('.frappe-control[data-fieldname="name1"]') + .find("label") + .should("have.class", "reqd"); + cy.get('.frappe-control[data-fieldname="email"]') + .find("label") + .should("have.class", "reqd"); + cy.get('.frappe-control[data-fieldname="phone"]') + .find("label") + .should("have.class", "reqd"); //Checking if the status is "Not Saved" initially - cy.get('.indicator-pill').should('have.text', 'Not Saved'); + cy.get(".indicator-pill").should("have.text", "Not Saved"); //Inputting data in the field - cy.fill_field('name1', '@@###', 'Data'); - cy.fill_field('email', 'test@example.com', 'Data'); - cy.fill_field('phone', '9834280031', 'Data'); + cy.fill_field("name1", "@@###", "Data"); + cy.fill_field("email", "test@example.com", "Data"); + cy.fill_field("phone", "9834280031", "Data"); //Checking if the border color of the field changes to red - cy.get('.frappe-control[data-fieldname="name1"]').should('have.class', 'has-error'); + cy.get('.frappe-control[data-fieldname="name1"]').should("have.class", "has-error"); cy.save(); //Checking for the error message - cy.get('.modal-title').should('have.text', 'Message'); - cy.get('.msgprint').should('have.text', '@@### is not a valid Name'); + cy.get(".modal-title").should("have.text", "Message"); + cy.get(".msgprint").should("have.text", "@@### is not a valid Name"); cy.hide_dialog(); - cy.get_field('name1', 'Data').clear({force: true}); - cy.fill_field('name1', 'Komal{}/!', 'Data'); - cy.get('.frappe-control[data-fieldname="name1"]').should('have.class', 'has-error'); + cy.get_field("name1", "Data").clear({ force: true }); + cy.fill_field("name1", "Komal{}/!", "Data"); + cy.get('.frappe-control[data-fieldname="name1"]').should("have.class", "has-error"); cy.save(); - cy.get('.modal-title').should('have.text', 'Message'); - cy.get('.msgprint').should('have.text', 'Komal{}/! is not a valid Name'); + cy.get(".modal-title").should("have.text", "Message"); + cy.get(".msgprint").should("have.text", "Komal{}/! is not a valid Name"); cy.hide_dialog(); }); it('Verifying data control by inputting different patterns for "Email" field', () => { - cy.get_field('name1', 'Data').clear({force: true}); - cy.fill_field('name1', 'Komal', 'Data'); - cy.get_field('email', 'Data').clear({force: true}); - cy.fill_field('email', 'komal', 'Data'); - cy.get('.frappe-control[data-fieldname="email"]').should('have.class', 'has-error'); + cy.get_field("name1", "Data").clear({ force: true }); + cy.fill_field("name1", "Komal", "Data"); + cy.get_field("email", "Data").clear({ force: true }); + cy.fill_field("email", "komal", "Data"); + cy.get('.frappe-control[data-fieldname="email"]').should("have.class", "has-error"); cy.save(); - cy.get('.modal-title').should('have.text', 'Message'); - cy.get('.msgprint').should('have.text', 'komal is not a valid Email Address'); + cy.get(".modal-title").should("have.text", "Message"); + cy.get(".msgprint").should("have.text", "komal is not a valid Email Address"); cy.hide_dialog(); - cy.get_field('email', 'Data').clear({force: true}); - cy.fill_field('email', 'komal@test', 'Data'); - cy.get('.frappe-control[data-fieldname="email"]').should('have.class', 'has-error'); + cy.get_field("email", "Data").clear({ force: true }); + cy.fill_field("email", "komal@test", "Data"); + cy.get('.frappe-control[data-fieldname="email"]').should("have.class", "has-error"); cy.save(); - cy.get('.modal-title').should('have.text', 'Message'); - cy.get('.msgprint').should('have.text', 'komal@test is not a valid Email Address'); + cy.get(".modal-title").should("have.text", "Message"); + cy.get(".msgprint").should("have.text", "komal@test is not a valid Email Address"); cy.hide_dialog(); }); it('Verifying data control by inputting different patterns for "Phone" field', () => { - cy.get_field('email', 'Data').clear({force: true}); - cy.fill_field('email', 'komal@test.com', 'Data'); - cy.get_field('phone', 'Data').clear({force: true}); - cy.fill_field('phone', 'komal', 'Data'); - cy.get('.frappe-control[data-fieldname="phone"]').should('have.class', 'has-error'); - cy.findByRole('button', {name: 'Save'}).click({force: true}); - cy.get('.modal-title').should('have.text', 'Message'); - cy.get('.msgprint').should('have.text', 'komal is not a valid Phone Number'); + cy.get_field("email", "Data").clear({ force: true }); + cy.fill_field("email", "komal@test.com", "Data"); + cy.get_field("phone", "Data").clear({ force: true }); + cy.fill_field("phone", "komal", "Data"); + cy.get('.frappe-control[data-fieldname="phone"]').should("have.class", "has-error"); + cy.findByRole("button", { name: "Save" }).click({ force: true }); + cy.get(".modal-title").should("have.text", "Message"); + cy.get(".msgprint").should("have.text", "komal is not a valid Phone Number"); cy.hide_dialog(); }); - it('Inputting correct data and saving the doc', () => { + it("Inputting correct data and saving the doc", () => { //Inputting the data as expected and saving the document - cy.get_field('name1', 'Data').clear({force: true}); - cy.get_field('email', 'Data').clear({force: true}); - cy.get_field('phone', 'Data').clear({force: true}); - cy.fill_field('name1', 'Komal', 'Data'); - cy.fill_field('email', 'komal@test.com', 'Data'); - cy.fill_field('phone', '9432380001', 'Data'); - cy.findByRole('button', {name: 'Save'}).click({force: true}); + cy.get_field("name1", "Data").clear({ force: true }); + cy.get_field("email", "Data").clear({ force: true }); + cy.get_field("phone", "Data").clear({ force: true }); + cy.fill_field("name1", "Komal", "Data"); + cy.fill_field("email", "komal@test.com", "Data"); + cy.fill_field("phone", "9432380001", "Data"); + cy.findByRole("button", { name: "Save" }).click({ force: true }); //Checking if the fields contains the data which has been filled in - cy.location("pathname").should('not.be', '/app/test-data-control/new-test-data-control-1'); - cy.get_field('name1').should('have.value', 'Komal'); - cy.get_field('email').should('have.value', 'komal@test.com'); - cy.get_field('phone').should('have.value', '9432380001'); + cy.location("pathname").should("not.be", "/app/test-data-control/new-test-data-control-1"); + cy.get_field("name1").should("have.value", "Komal"); + cy.get_field("email").should("have.value", "komal@test.com"); + cy.get_field("phone").should("have.value", "9432380001"); }); - it('Deleting the doc', () => { + it("Deleting the doc", () => { //Deleting the inserted document - cy.go_to_list('Test Data Control'); - cy.get('.list-row-checkbox').eq(0).click({force: true}); - cy.get('.actions-btn-group > .btn').contains('Actions').click(); + cy.go_to_list("Test Data Control"); + cy.get(".list-row-checkbox").eq(0).click({ force: true }); + cy.get(".actions-btn-group > .btn").contains("Actions").click(); cy.get('.actions-btn-group > .dropdown-menu [data-label="Delete"]').click(); - cy.click_modal_primary_button('Yes'); + cy.click_modal_primary_button("Yes"); }); }); diff --git a/cypress/integration/control_date.js b/cypress/integration/control_date.js index 6d9f0b9bcc..442538661e 100644 --- a/cypress/integration/control_date.js +++ b/cypress/integration/control_date.js @@ -1,82 +1,83 @@ -context('Date Control', () => { +context("Date Control", () => { before(() => { cy.login(); - cy.visit('/app'); + cy.visit("/app"); }); function get_dialog(date_field_options) { return cy.dialog({ - title: 'Date', - fields: [{ - "label": "Date", - "fieldname": "date", - "fieldtype": "Date", - "in_list_view": 1, - ...date_field_options - }] + title: "Date", + fields: [ + { + label: "Date", + fieldname: "date", + fieldtype: "Date", + in_list_view: 1, + ...date_field_options, + }, + ], }); } - it('Selecting a date from the datepicker', () => { + it("Selecting a date from the datepicker & check prev & next button", () => { cy.clear_dialogs(); cy.clear_datepickers(); - get_dialog().as('dialog'); - cy.get_field('date', 'Date').click(); - cy.get('.datepicker--nav-title').click(); - cy.get('.datepicker--nav-title').click({force: true}); - + get_dialog().as("dialog"); + cy.get_field("date", "Date").click(); + cy.get(".datepicker--nav-title").click(); + cy.get(".datepicker--nav-title").click({ force: true }); //Inputing values in the date field - cy.get('.datepicker--years > .datepicker--cells > .datepicker--cell[data-year=2020]').click(); - cy.get('.datepicker--months > .datepicker--cells > .datepicker--cell[data-month=0]').click(); - cy.get('.datepicker--days > .datepicker--cells > .datepicker--cell[data-date=15]').click(); + cy.get( + ".datepicker--years > .datepicker--cells > .datepicker--cell[data-year=2020]" + ).click(); + cy.get( + ".datepicker--months > .datepicker--cells > .datepicker--cell[data-month=0]" + ).click(); + cy.get(".datepicker--days > .datepicker--cells > .datepicker--cell[data-date=15]").click(); // Verify if the selected date is set the date field - cy.window().its('cur_dialog.fields_dict.date.value').should('be.equal', '2020-01-15'); - }); + cy.window().its("cur_dialog.fields_dict.date.value").should("be.equal", "2020-01-15"); - it('Checking next and previous button', () => { - cy.clear_dialogs(); - cy.clear_datepickers(); - - get_dialog({ default: '2020-01-15' }).as('dialog'); - cy.get_field('date', 'Date').click(); + cy.get_field("date", "Date").click(); //Clicking on the next button in the datepicker - cy.get('.datepicker--nav-action[data-action=next]').click(); + cy.get(".datepicker--nav-action[data-action=next]").click(); //Selecting a date from the datepicker - cy.get('.datepicker--cell[data-date=15]').click({force: true}); + cy.get(".datepicker--cell[data-date=15]").click({ force: true }); //Verifying if the selected date has been displayed in the date field - cy.window().its('cur_dialog.fields_dict.date.value').should('be.equal', '2020-02-15'); + cy.window().its("cur_dialog.fields_dict.date.value").should("be.equal", "2020-02-15"); cy.wait(500); - cy.get_field('date', 'Date').click(); + cy.get_field("date", "Date").click(); //Clicking on the previous button in the datepicker - cy.get('.datepicker--nav-action[data-action=prev]').click(); + cy.get(".datepicker--nav-action[data-action=prev]").click(); //Selecting a date from the datepicker - cy.get('.datepicker--cell[data-date=15]').click({force: true}); + cy.get(".datepicker--cell[data-date=15]").click({ force: true }); //Verifying if the selected date has been displayed in the date field - cy.window().its('cur_dialog.fields_dict.date.value').should('be.equal', '2020-01-15'); + cy.window().its("cur_dialog.fields_dict.date.value").should("be.equal", "2020-01-15"); }); it('Clicking on "Today" button gives todays date', () => { cy.clear_dialogs(); cy.clear_datepickers(); - get_dialog().as('dialog'); - cy.get_field('date', 'Date').click(); + get_dialog().as("dialog"); + cy.get_field("date", "Date").click(); //Clicking on "Today" button - cy.get('.datepicker--button').click(); + cy.get(".datepicker--button").click(); //Verifying if clicking on "Today" button matches today's date - cy.window().then(win => { - expect(win.cur_dialog.fields_dict.date.value).to.be.equal(win.frappe.datetime.get_today()); + cy.window().then((win) => { + expect(win.cur_dialog.fields_dict.date.value).to.be.equal( + win.frappe.datetime.get_today() + ); }); }); -}); \ No newline at end of file +}); diff --git a/cypress/integration/control_date_range.js b/cypress/integration/control_date_range.js index 6f26b35f84..f95a3825cc 100644 --- a/cypress/integration/control_date_range.js +++ b/cypress/integration/control_date_range.js @@ -1,42 +1,48 @@ -context('Date Range Control', () => { +context("Date Range Control", () => { before(() => { cy.login(); - cy.visit('/app'); + cy.visit("/app"); }); function get_dialog() { return cy.dialog({ - title: 'Date Range', - fields: [{ - "label": "Date Range", - "fieldname": "date_range", - "fieldtype": "Date Range", - }] + title: "Date Range", + fields: [ + { + label: "Date Range", + fieldname: "date_range", + fieldtype: "Date Range", + }, + ], }); } - it('Selecting a date range from the datepicker', () => { + it("Selecting a date range from the datepicker", () => { cy.clear_dialogs(); cy.clear_datepickers(); - get_dialog().as('dialog'); - cy.get_field('date_range', 'Date Range').click(); - cy.get('.datepicker--nav-title').click(); - cy.get('.datepicker--nav-title').click({force: true}); + get_dialog().as("dialog"); + cy.get_field("date_range", "Date Range").click(); + cy.get(".datepicker--nav-title").click(); + cy.get(".datepicker--nav-title").click({ force: true }); //Inputing date range values in the date range field - cy.get('.datepicker--years > .datepicker--cells > .datepicker--cell[data-year=2020]').click(); - cy.get('.datepicker--months > .datepicker--cells > .datepicker--cell[data-month=0]').click(); - cy.get('.datepicker--cell[data-date=1]:first').click({force: true}); - cy.get('.datepicker--cell[data-date=15]:first').click({force: true}); + cy.get( + ".datepicker--years > .datepicker--cells > .datepicker--cell[data-year=2020]" + ).click(); + cy.get( + ".datepicker--months > .datepicker--cells > .datepicker--cell[data-month=0]" + ).click(); + cy.get(".datepicker--cell[data-date=1]:first").click({ force: true }); + cy.get(".datepicker--cell[data-date=15]:first").click({ force: true }); // Verify if the selected date range values is set in the date range field cy.window() - .its('cur_dialog') - .then(dialog => { + .its("cur_dialog") + .then((dialog) => { let date_range = dialog.get_value("date_range"); - expect(date_range[0]).to.equal('2020-01-01'); - expect(date_range[1]).to.equal('2020-01-15'); + expect(date_range[0]).to.equal("2020-01-01"); + expect(date_range[1]).to.equal("2020-01-15"); }); }); -}); \ No newline at end of file +}); diff --git a/cypress/integration/control_duration.js b/cypress/integration/control_duration.js index 09629a344f..a391eec7c1 100644 --- a/cypress/integration/control_duration.js +++ b/cypress/integration/control_duration.js @@ -1,46 +1,46 @@ -context('Control Duration', () => { +context("Control Duration", () => { before(() => { cy.login(); - cy.visit('/app/website'); + cy.visit("/app/website"); }); function get_dialog_with_duration(hide_days = 0, hide_seconds = 0) { return cy.dialog({ - title: 'Duration', - fields: [{ - 'fieldname': 'duration', - 'fieldtype': 'Duration', - 'hide_days': hide_days, - 'hide_seconds': hide_seconds - }] + title: "Duration", + fields: [ + { + fieldname: "duration", + fieldtype: "Duration", + hide_days: hide_days, + hide_seconds: hide_seconds, + }, + ], }); } - it('should set duration', () => { - get_dialog_with_duration().as('dialog'); - cy.get('.frappe-control[data-fieldname=duration] input') - .first() - .click(); - cy.get('.duration-input[data-duration=days]') + it("should set duration", () => { + get_dialog_with_duration().as("dialog"); + cy.get(".frappe-control[data-fieldname=duration] input").first().click(); + cy.get(".duration-input[data-duration=days]") .type(45, { force: true }) .blur({ force: true }); - cy.get('.duration-input[data-duration=minutes]') - .type(30) - .blur({ force: true }); - cy.get('.frappe-control[data-fieldname=duration] input').first().should('have.value', '45d 30m'); - cy.get('.frappe-control[data-fieldname=duration] input').first().blur(); - cy.get('.duration-picker').should('not.be.visible'); - cy.get('@dialog').then(dialog => { - let value = dialog.get_value('duration'); + cy.get(".duration-input[data-duration=minutes]").type(30).blur({ force: true }); + cy.get(".frappe-control[data-fieldname=duration] input") + .first() + .should("have.value", "45d 30m"); + cy.get(".frappe-control[data-fieldname=duration] input").first().blur(); + cy.get(".duration-picker").should("not.be.visible"); + cy.get("@dialog").then((dialog) => { + let value = dialog.get_value("duration"); expect(value).to.equal(3889800); cy.hide_dialog(); }); }); - it('should hide days or seconds according to duration options', () => { - get_dialog_with_duration(1, 1).as('dialog'); - cy.get('.frappe-control[data-fieldname=duration] input').first(); - cy.get('.duration-input[data-duration=days]').should('not.be.visible'); - cy.get('.duration-input[data-duration=seconds]').should('not.be.visible'); + it("should hide days or seconds according to duration options", () => { + get_dialog_with_duration(1, 1).as("dialog"); + cy.get(".frappe-control[data-fieldname=duration] input").first(); + cy.get(".duration-input[data-duration=days]").should("not.be.visible"); + cy.get(".duration-input[data-duration=seconds]").should("not.be.visible"); }); -}); \ No newline at end of file +}); diff --git a/cypress/integration/control_dynamic_link.js b/cypress/integration/control_dynamic_link.js index 32b2c274a8..7f34f7ad42 100644 --- a/cypress/integration/control_dynamic_link.js +++ b/cypress/integration/control_dynamic_link.js @@ -1,133 +1,159 @@ -context('Dynamic Link', () => { +context("Dynamic Link", () => { before(() => { cy.login(); - cy.visit('/app/doctype'); - return cy.window().its('frappe').then(frappe => { - return frappe.xcall('frappe.tests.ui_test_helpers.create_doctype', { - name: 'Test Dynamic Link', - fields: [ - { - "label": "Document Type", - "fieldname": "doc_type", - "fieldtype": "Link", - "options": "DocType", - "in_list_view": 1, - "in_standard_filter": 1, - }, - { - "label": "Document ID", - "fieldname": "doc_id", - "fieldtype": "Dynamic Link", - "options": "doc_type", - "in_list_view": 1, - "in_standard_filter": 1, - }, - ] + cy.visit("/app/doctype"); + return cy + .window() + .its("frappe") + .then((frappe) => { + return frappe.xcall("frappe.tests.ui_test_helpers.create_doctype", { + name: "Test Dynamic Link", + fields: [ + { + label: "Document Type", + fieldname: "doc_type", + fieldtype: "Link", + options: "DocType", + in_list_view: 1, + in_standard_filter: 1, + }, + { + label: "Document ID", + fieldname: "doc_id", + fieldtype: "Dynamic Link", + options: "doc_type", + in_list_view: 1, + in_standard_filter: 1, + }, + ], + }); }); - }); }); - function get_dialog_with_dynamic_link() { return cy.dialog({ - title: 'Dynamic Link', - fields: [{ - "label": "Document Type", - "fieldname": "doc_type", - "fieldtype": "Link", - "options": "DocType", - "in_list_view": 1, - }, - { - "label": "Document ID", - "fieldname": "doc_id", - "fieldtype": "Dynamic Link", - "options": "doc_type", - "in_list_view": 1, - }] + title: "Dynamic Link", + fields: [ + { + label: "Document Type", + fieldname: "doc_type", + fieldtype: "Link", + options: "DocType", + in_list_view: 1, + }, + { + label: "Document ID", + fieldname: "doc_id", + fieldtype: "Dynamic Link", + options: "doc_type", + in_list_view: 1, + }, + ], }); } function get_dialog_with_dynamic_link_option() { return cy.dialog({ - title: 'Dynamic Link', - fields: [{ - "label": "Document Type", - "fieldname": "doc_type", - "fieldtype": "Link", - "options": "DocType", - "in_list_view": 1, - }, - { - "label": "Document ID", - "fieldname": "doc_id", - "fieldtype": "Dynamic Link", - "get_options": () => { - return "User"; + title: "Dynamic Link", + fields: [ + { + label: "Document Type", + fieldname: "doc_type", + fieldtype: "Link", + options: "DocType", + in_list_view: 1, }, - "in_list_view": 1, - }] + { + label: "Document ID", + fieldname: "doc_id", + fieldtype: "Dynamic Link", + get_options: () => { + return "User"; + }, + in_list_view: 1, + }, + ], }); } - it('Creating a dynamic link by passing option as function and verifying it in a dialog', () => { - get_dialog_with_dynamic_link_option().as('dialog'); - cy.get_field('doc_type').clear(); - cy.fill_field('doc_type', 'User', 'Link'); - cy.get_field('doc_id').click(); + it("Creating a dynamic link by passing option as function and verifying it in a dialog", () => { + get_dialog_with_dynamic_link_option().as("dialog"); + cy.get_field("doc_type").clear(); + cy.fill_field("doc_type", "User", "Link"); + cy.get_field("doc_id").click(); //Checking if the listbox have length greater than 0 - cy.get('[data-fieldname="doc_id"]').find('.awesomplete').find("li").its('length').should('be.gte', 0); - cy.get('.btn-modal-close').click({force: true}); + cy.get('[data-fieldname="doc_id"]') + .find(".awesomplete") + .find("li") + .its("length") + .should("be.gte", 0); + cy.get(".btn-modal-close").click({ force: true }); }); - it('Creating a dynamic link and verifying it in a dialog', () => { - get_dialog_with_dynamic_link().as('dialog'); - cy.get_field('doc_type').clear(); - cy.fill_field('doc_type', 'User', 'Link'); - cy.get_field('doc_id').click(); + it("Creating a dynamic link and verifying it in a dialog", () => { + get_dialog_with_dynamic_link().as("dialog"); + cy.get_field("doc_type").clear(); + cy.fill_field("doc_type", "User", "Link"); + cy.get_field("doc_id").click(); //Checking if the listbox have length greater than 0 - cy.get('[data-fieldname="doc_id"]').find('.awesomplete').find("li").its('length').should('be.gte', 0); - cy.get('.btn-modal-close').click({force: true, multiple: true}); + cy.get('[data-fieldname="doc_id"]') + .find(".awesomplete") + .find("li") + .its("length") + .should("be.gte", 0); + cy.get(".btn-modal-close").click({ force: true, multiple: true }); }); - it('Creating a dynamic link and verifying it', () => { - cy.visit('/app/test-dynamic-link'); + it("Creating a dynamic link and verifying it", () => { + cy.visit("/app/test-dynamic-link"); //Clicking on the Document ID field - cy.get_field('doc_type').clear(); + cy.get_field("doc_type").clear(); //Entering User in the Doctype field - cy.fill_field('doc_type', 'User', 'Link', {delay: 500}); - cy.get_field('doc_id').click(); + cy.fill_field("doc_type", "User", "Link", { delay: 500 }); + cy.get_field("doc_id").click(); //Checking if the listbox have length greater than 0 - cy.get('[data-fieldname="doc_id"]').find('.awesomplete').find("li").its('length').should('be.gte', 0); + cy.get('[data-fieldname="doc_id"]') + .find(".awesomplete") + .find("li") + .its("length") + .should("be.gte", 0); //Opening a new form for dynamic link doctype - cy.new_form('Test Dynamic Link'); - cy.get_field('doc_type').clear(); + cy.new_form("Test Dynamic Link"); + cy.get_field("doc_type").clear(); //Entering User in the Doctype field - cy.fill_field('doc_type', 'User', 'Link', {delay: 500}); - cy.get_field('doc_id').click(); + cy.fill_field("doc_type", "User", "Link", { delay: 500 }); + cy.get_field("doc_id").click(); //Checking if the listbox have length greater than 0 - cy.get('[data-fieldname="doc_id"]').find('.awesomplete').find("li").its('length').should('be.gte', 0); - cy.get_field('doc_type').clear(); + cy.get('[data-fieldname="doc_id"]') + .find(".awesomplete") + .find("li") + .its("length") + .should("be.gte", 0); + cy.get_field("doc_type").clear(); //Entering System Settings in the Doctype field - cy.intercept('/api/method/frappe.desk.search.search_link').as('search_query'); - cy.fill_field('doc_type', 'System Settings', 'Link', {delay: 500}); - cy.wait('@search_query'); - cy.get(`[data-fieldname="doc_type"] ul:visible li:first-child`) - .click({scrollBehavior: false}); + cy.intercept("/api/method/frappe.desk.search.search_link").as("search_query"); + cy.fill_field("doc_type", "System Settings", "Link", { delay: 500 }); + cy.wait("@search_query"); + cy.get(`[data-fieldname="doc_type"] ul:visible li:first-child`).click({ + scrollBehavior: false, + }); - cy.get_field('doc_id').click(); + cy.get_field("doc_id").click(); //Checking if the system throws error - cy.get('.modal-title').should('have.text', 'Error'); - cy.get('.msgprint').should('have.text', 'System Settings is not a valid DocType for Dynamic Link'); + cy.get(".modal-title").should("have.text", "Error"); + cy.get(".msgprint").should( + "have.text", + "System Settings is not a valid DocType for Dynamic Link" + ); }); }); diff --git a/cypress/integration/control_float.js b/cypress/integration/control_float.js index 670d1fe73e..65aa21ed69 100644 --- a/cypress/integration/control_float.js +++ b/cypress/integration/control_float.js @@ -11,9 +11,9 @@ context("Control Float", () => { { fieldname: "float_number", fieldtype: "Float", - Label: "Float" - } - ] + Label: "Float", + }, + ], }); } @@ -21,27 +21,22 @@ context("Control Float", () => { get_dialog_with_float().as("dialog"); let data = get_data(); - data.forEach(x => { + data.forEach((x) => { cy.window() .its("frappe") - .then(frappe => { + .then((frappe) => { frappe.boot.sysdefaults.number_format = x.number_format; }); - x.values.forEach(d => { + x.values.forEach((d) => { cy.get_field("float_number", "Float").clear(); + cy.wait(200); cy.fill_field("float_number", d.input, "Float").blur(); - cy.get_field("float_number", "Float").should( - "have.value", - d.blur_expected - ); + cy.get_field("float_number", "Float").should("have.value", d.blur_expected); cy.get_field("float_number", "Float").focus(); cy.get_field("float_number", "Float").blur(); cy.get_field("float_number", "Float").focus(); - cy.get_field("float_number", "Float").should( - "have.value", - d.focus_expected - ); + cy.get_field("float_number", "Float").should("have.value", d.focus_expected); }); }); }); @@ -54,19 +49,19 @@ context("Control Float", () => { { input: "364.87,334", blur_expected: "36.487,334", - focus_expected: "36487.334" + focus_expected: "36487.334", }, { input: "36487,334", blur_expected: "36.487,334", - focus_expected: "36487.334" + focus_expected: "36487.334", }, { input: "100", blur_expected: "100,000", - focus_expected: "100" - } - ] + focus_expected: "100", + }, + ], }, { number_format: "#,###.##", @@ -74,20 +69,20 @@ context("Control Float", () => { { input: "364,87.334", blur_expected: "36,487.334", - focus_expected: "36487.334" + focus_expected: "36487.334", }, { input: "36487.334", blur_expected: "36,487.334", - focus_expected: "36487.334" + focus_expected: "36487.334", }, { input: "100", blur_expected: "100.000", - focus_expected: "100" - } - ] - } + focus_expected: "100", + }, + ], + }, ]; } }); diff --git a/cypress/integration/control_icon.js b/cypress/integration/control_icon.js index d89eba8840..a965ed0f9e 100644 --- a/cypress/integration/control_icon.js +++ b/cypress/integration/control_icon.js @@ -1,50 +1,55 @@ -context('Control Icon', () => { +context("Control Icon", () => { before(() => { cy.login(); - cy.visit('/app/website'); + cy.visit("/app/website"); }); function get_dialog_with_icon() { return cy.dialog({ - title: 'Icon', - fields: [{ - label: 'Icon', - fieldname: 'icon', - fieldtype: 'Icon' - }] + title: "Icon", + fields: [ + { + label: "Icon", + fieldname: "icon", + fieldtype: "Icon", + }, + ], }); } - it('should set icon', () => { - get_dialog_with_icon().as('dialog'); - cy.get('.frappe-control[data-fieldname=icon]').findByRole('textbox').click(); + it("should set icon", () => { + get_dialog_with_icon().as("dialog"); + cy.get(".frappe-control[data-fieldname=icon]").findByRole("textbox").click(); - cy.get('.icon-picker .icon-wrapper[id=heart-active]').first().click(); - cy.get('.frappe-control[data-fieldname=icon]').findByRole('textbox').should('have.value', 'heart-active'); - cy.get('@dialog').then(dialog => { - let value = dialog.get_value('icon'); - expect(value).to.equal('heart-active'); + cy.get(".icon-picker .icon-wrapper[id=heart-active]").first().click(); + cy.get(".frappe-control[data-fieldname=icon]") + .findByRole("textbox") + .should("have.value", "heart-active"); + cy.get("@dialog").then((dialog) => { + let value = dialog.get_value("icon"); + expect(value).to.equal("heart-active"); }); - cy.get('.icon-picker .icon-wrapper[id=heart]').first().click(); - cy.get('.frappe-control[data-fieldname=icon]').findByRole('textbox').should('have.value', 'heart'); - cy.get('@dialog').then(dialog => { - let value = dialog.get_value('icon'); - expect(value).to.equal('heart'); + cy.get(".icon-picker .icon-wrapper[id=heart]").first().click(); + cy.get(".frappe-control[data-fieldname=icon]") + .findByRole("textbox") + .should("have.value", "heart"); + cy.get("@dialog").then((dialog) => { + let value = dialog.get_value("icon"); + expect(value).to.equal("heart"); }); }); - it('search for icon and clear search input', () => { - let search_text = 'ed'; - cy.get('.icon-picker').findByRole('searchbox').click().type(search_text); - cy.get('.icon-section .icon-wrapper:not(.hidden)').then(i => { - cy.get(`.icon-section .icon-wrapper[id*='${search_text}']`).then(icons => { + it("search for icon and clear search input", () => { + let search_text = "ed"; + cy.get(".icon-picker").findByRole("searchbox").click().type(search_text); + cy.get(".icon-section .icon-wrapper:not(.hidden)").then((i) => { + cy.get(`.icon-section .icon-wrapper[id*='${search_text}']`).then((icons) => { expect(i.length).to.equal(icons.length); }); }); - cy.get('.icon-picker').findByRole('searchbox').clear().blur(); - cy.get('.icon-section .icon-wrapper').should('not.have.class', 'hidden'); + cy.get(".icon-picker").findByRole("searchbox").clear().blur(); + cy.get(".icon-section .icon-wrapper").should("not.have.class", "hidden"); }); - -}); \ No newline at end of file +}); diff --git a/cypress/integration/control_link.js b/cypress/integration/control_link.js index 44153f7e4a..d3462492f6 100644 --- a/cypress/integration/control_link.js +++ b/cypress/integration/control_link.js @@ -1,93 +1,88 @@ -context('Control Link', () => { +context("Control Link", () => { before(() => { cy.login(); - cy.visit('/app/website'); + cy.visit("/app/website"); }); beforeEach(() => { - cy.visit('/app/website'); + cy.visit("/app/website"); cy.create_records({ - doctype: 'ToDo', - description: 'this is a test todo for link' - }).as('todos'); + doctype: "ToDo", + description: "this is a test todo for link", + }).as("todos"); }); function get_dialog_with_link() { return cy.dialog({ - title: 'Link', + title: "Link", fields: [ { - 'label': 'Select ToDo', - 'fieldname': 'link', - 'fieldtype': 'Link', - 'options': 'ToDo', - } - ] + label: "Select ToDo", + fieldname: "link", + fieldtype: "Link", + options: "ToDo", + }, + ], }); } - function get_dialog_with_user_link() { + function get_dialog_with_gender_link() { return cy.dialog({ - title: 'Link', + title: "Link", fields: [ { - 'label': 'Select User', - 'fieldname': 'link', - 'fieldtype': 'Link', - 'options': 'User', - } - ] + label: "Select Gender", + fieldname: "link", + fieldtype: "Link", + options: "Gender", + }, + ], }); } - it('should set the valid value', () => { - get_dialog_with_link().as('dialog'); + it("should set the valid value", () => { + get_dialog_with_link().as("dialog"); - cy.insert_doc("Property Setter", { - "doctype": "Property Setter", - "doc_type": "User", - "property": "translate_link_fields", - "property_type": "Check", - "doctype_or_field": "DocType", - "value": "0" - }, true); + cy.insert_doc( + "Property Setter", + { + doctype: "Property Setter", + doc_type: "ToDo", + property: "show_title_field_in_link", + property_type: "Check", + doctype_or_field: "DocType", + value: "0", + }, + true + ); - cy.insert_doc("Property Setter", { - "doctype": "Property Setter", - "doc_type": "ToDo", - "property": "show_title_field_in_link", - "property_type": "Check", - "doctype_or_field": "DocType", - "value": "0" - }, true); + cy.intercept("POST", "/api/method/frappe.desk.search.search_link").as("search_link"); - cy.intercept('POST', '/api/method/frappe.desk.search.search_link').as('search_link'); - - cy.get('.frappe-control[data-fieldname=link] input').focus().as('input'); - cy.wait('@search_link'); - cy.get('@input').type('todo for link', { delay: 200 }); - cy.wait('@search_link'); - cy.get('.frappe-control[data-fieldname=link]').findByRole('listbox').should('be.visible'); - cy.get('.frappe-control[data-fieldname=link] input').type('{enter}', { delay: 100 }); - cy.get('.frappe-control[data-fieldname=link] input').blur(); - cy.get('@dialog').then(dialog => { - cy.get('@todos').then(todos => { - let value = dialog.get_value('link'); + cy.get(".frappe-control[data-fieldname=link] input").focus().as("input"); + cy.wait("@search_link"); + cy.get("@input").type("todo for link", { delay: 200 }); + cy.wait("@search_link"); + cy.get(".frappe-control[data-fieldname=link]").findByRole("listbox").should("be.visible"); + cy.get(".frappe-control[data-fieldname=link] input").type("{enter}", { delay: 100 }); + cy.get(".frappe-control[data-fieldname=link] input").blur(); + cy.get("@dialog").then((dialog) => { + cy.get("@todos").then((todos) => { + let value = dialog.get_value("link"); expect(value).to.eq(todos[0]); }); }); }); - it('should unset invalid value', () => { - get_dialog_with_link().as('dialog'); + it("should unset invalid value", () => { + get_dialog_with_link().as("dialog"); - cy.intercept('POST', '/api/method/frappe.client.validate_link').as('validate_link'); + cy.intercept("POST", "/api/method/frappe.client.validate_link").as("validate_link"); - cy.get('.frappe-control[data-fieldname=link] input') - .type('invalid value', { delay: 100 }) + cy.get(".frappe-control[data-fieldname=link] input") + .type("invalid value", { delay: 100 }) .blur(); - cy.wait('@validate_link'); - cy.get('.frappe-control[data-fieldname=link] input').should('have.value', ''); + cy.wait("@validate_link"); + cy.get(".frappe-control[data-fieldname=link] input").should("have.value", ""); }); it("should be possible set empty value explicitly", () => { @@ -95,295 +90,246 @@ context('Control Link', () => { cy.intercept("POST", "/api/method/frappe.client.validate_link").as("validate_link"); - cy.get(".frappe-control[data-fieldname=link] input") - .type(" ", { delay: 100 }) - .blur(); + cy.get(".frappe-control[data-fieldname=link] input").type(" ", { delay: 100 }).blur(); cy.wait("@validate_link"); cy.get(".frappe-control[data-fieldname=link] input").should("have.value", ""); cy.window() .its("cur_dialog") .then((dialog) => { - expect(dialog.get_value("link")).to.equal(''); + expect(dialog.get_value("link")).to.equal(""); }); }); - it('should route to form on arrow click', () => { - get_dialog_with_link().as('dialog'); + it("should route to form on arrow click", () => { + get_dialog_with_link().as("dialog"); - cy.intercept('POST', '/api/method/frappe.client.validate_link').as('validate_link'); - cy.intercept('POST', '/api/method/frappe.desk.search.search_link').as('search_link'); + cy.intercept("POST", "/api/method/frappe.client.validate_link").as("validate_link"); + cy.intercept("POST", "/api/method/frappe.desk.search.search_link").as("search_link"); - cy.get('@todos').then(todos => { - cy.get('.frappe-control[data-fieldname=link] input').as('input'); - cy.get('@input').focus(); - cy.wait('@search_link'); - cy.get('@input').type(todos[0]).blur(); - cy.wait('@validate_link'); - cy.get('@input').focus(); + cy.get("@todos").then((todos) => { + cy.get(".frappe-control[data-fieldname=link] input").as("input"); + cy.get("@input").focus(); + cy.wait("@search_link"); + cy.get("@input").type(todos[0]).blur(); + cy.wait("@validate_link"); + cy.get("@input").focus(); cy.wait(500); // wait for arrow to show - cy.get('.frappe-control[data-fieldname=link] .btn-open') - .should('be.visible') - .click(); - cy.location('pathname').should('eq', `/app/todo/${todos[0]}`); + cy.get(".frappe-control[data-fieldname=link] .btn-open").should("be.visible").click(); + cy.location("pathname").should("eq", `/app/todo/${todos[0]}`); }); }); - it('show title field in link', () => { - - cy.insert_doc("Property Setter", { - "doctype": "Property Setter", - "doc_type": "User", - "property": "translate_link_fields", - "property_type": "Check", - "doctype_or_field": "DocType", - "value": "0" - }, true); - - cy.insert_doc("Property Setter", { - "doctype": "Property Setter", - "doc_type": "ToDo", - "property": "show_title_field_in_link", - "property_type": "Check", - "doctype_or_field": "DocType", - "value": "1" - }, true); + it("show title field in link", () => { + cy.insert_doc( + "Property Setter", + { + doctype: "Property Setter", + doc_type: "ToDo", + property: "show_title_field_in_link", + property_type: "Check", + doctype_or_field: "DocType", + value: "1", + }, + true + ); cy.clear_cache(); cy.wait(500); - get_dialog_with_link().as('dialog'); - cy.window().its('frappe').then(frappe => { - if (!frappe.boot) { - frappe.boot = { - link_title_doctypes: ['ToDo'] - }; - } else { - frappe.boot.link_title_doctypes = ['ToDo']; - } - }); + get_dialog_with_link().as("dialog"); + cy.window() + .its("frappe") + .then((frappe) => { + if (!frappe.boot) { + frappe.boot = { + link_title_doctypes: ["ToDo"], + }; + } else { + frappe.boot.link_title_doctypes = ["ToDo"]; + } + }); - cy.intercept('POST', '/api/method/frappe.desk.search.search_link').as('search_link'); + cy.intercept("POST", "/api/method/frappe.desk.search.search_link").as("search_link"); - cy.get('.frappe-control[data-fieldname=link] input').focus().as('input'); - cy.wait('@search_link'); - cy.get('@input').type('todo for link'); - cy.wait('@search_link'); - cy.get('.frappe-control[data-fieldname=link] ul').should('be.visible'); - cy.get('.frappe-control[data-fieldname=link] input').type('{enter}', { delay: 100 }); - cy.get('.frappe-control[data-fieldname=link] input').blur(); - cy.get('@dialog').then(dialog => { - cy.get('@todos').then(todos => { - let field = dialog.get_field('link'); + cy.get(".frappe-control[data-fieldname=link] input").focus().as("input"); + cy.wait("@search_link"); + cy.get("@input").type("todo for link"); + cy.wait("@search_link"); + cy.get(".frappe-control[data-fieldname=link] ul").should("be.visible"); + cy.get(".frappe-control[data-fieldname=link] input").type("{enter}", { delay: 100 }); + cy.get(".frappe-control[data-fieldname=link] input").blur(); + cy.get("@dialog").then((dialog) => { + cy.get("@todos").then((todos) => { + let field = dialog.get_field("link"); let value = field.get_value(); let label = field.get_label_value(); expect(value).to.eq(todos[0]); - expect(label).to.eq('this is a test todo for link'); + expect(label).to.eq("this is a test todo for link"); }); }); }); - it('should update dependant fields (via fetch_from)', () => { - cy.get('@todos').then(todos => { + it("should update dependant fields (via fetch_from)", () => { + cy.get("@todos").then((todos) => { cy.visit(`/app/todo/${todos[0]}`); - cy.intercept('POST', '/api/method/frappe.desk.search.search_link').as('search_link'); - cy.intercept('POST', '/api/method/frappe.client.validate_link').as('validate_link'); + cy.intercept("POST", "/api/method/frappe.desk.search.search_link").as("search_link"); + cy.intercept("POST", "/api/method/frappe.client.validate_link").as("validate_link"); - cy.get('.frappe-control[data-fieldname=assigned_by] input').focus().as('input'); - cy.get('@input').type('Administrator', {delay: 100}).blur(); - cy.wait('@validate_link'); - cy.get('.frappe-control[data-fieldname=assigned_by_full_name] .control-value').should( - 'contain', 'Administrator' + cy.get(".frappe-control[data-fieldname=assigned_by] input").focus().as("input"); + cy.get("@input").type(cy.config("testUser"), { delay: 100 }).blur(); + cy.wait("@validate_link"); + cy.get(".frappe-control[data-fieldname=assigned_by_full_name] .control-value").should( + "contain", + "Frappe" ); - cy.window() - .its("cur_frm.doc.assigned_by") - .should("eq", "Administrator"); + cy.window().its("cur_frm.doc.assigned_by").should("eq", cy.config("testUser")); // invalid input - cy.get('@input').clear().type('invalid input', {delay: 100}).blur(); - cy.get('.frappe-control[data-fieldname=assigned_by_full_name] .control-value').should( - 'contain', '' + cy.get("@input").clear().type("invalid input", { delay: 100 }).blur(); + cy.get(".frappe-control[data-fieldname=assigned_by_full_name] .control-value").should( + "contain", + "" ); - cy.window() - .its("cur_frm.doc.assigned_by") - .should("eq", null); + cy.window().its("cur_frm.doc.assigned_by").should("eq", null); // set valid value again - cy.get('@input').clear().focus(); - cy.wait('@search_link'); - cy.get('@input').type('Administrator', {delay: 100}).blur(); - cy.wait('@validate_link'); + cy.get("@input").clear().focus(); + cy.wait("@search_link"); + cy.get("@input").type(cy.config("testUser"), { delay: 100 }).blur(); + cy.wait("@validate_link"); - cy.window() - .its("cur_frm.doc.assigned_by") - .should("eq", "Administrator"); + cy.window().its("cur_frm.doc.assigned_by").should("eq", cy.config("testUser")); // clear input - cy.get('@input').clear().blur(); - cy.get('.frappe-control[data-fieldname=assigned_by_full_name] .control-value').should( - 'contain', '' + cy.get("@input").clear().blur(); + cy.get(".frappe-control[data-fieldname=assigned_by_full_name] .control-value").should( + "contain", + "" ); - cy.window() - .its("cur_frm.doc.assigned_by") - .should("eq", ""); + cy.window().its("cur_frm.doc.assigned_by").should("eq", ""); }); }); it("should set default values", () => { - cy.insert_doc("Property Setter", { - "doctype_or_field": "DocField", - "doc_type": "ToDo", - "field_name": "assigned_by", - "property": "default", - "property_type": "Text", - "value": "Administrator" - }, true); + cy.insert_doc( + "Property Setter", + { + doctype_or_field: "DocField", + doc_type: "ToDo", + field_name: "assigned_by", + property: "default", + property_type: "Text", + value: "Administrator", + }, + true + ); cy.reload(); cy.new_form("ToDo"); - cy.fill_field("description", "new", "Text Editor"); - cy.intercept("POST", "/api/method/frappe.desk.form.save.savedocs").as("save_form"); - cy.findByRole("button", {name: "Save"}).click(); - cy.wait("@save_form"); + cy.fill_field("description", "new", "Text Editor").wait(200); + cy.save(); cy.get(".frappe-control[data-fieldname=assigned_by_full_name] .control-value").should( - "contain", "Administrator" + "contain", + "Administrator" ); // if user clears default value explicitly, system should not reset default again cy.get_field("assigned_by").clear().blur(); - cy.intercept("POST", "/api/method/frappe.desk.form.save.savedocs").as("save_form"); - cy.findByRole("button", {name: "Save"}).click(); - cy.wait("@save_form"); + cy.save(); cy.get_field("assigned_by").should("have.value", ""); cy.get(".frappe-control[data-fieldname=assigned_by_full_name] .control-value").should( - "contain", "" + "contain", + "" ); }); - it('show translated text for link with show_title_field_in_link enabled', () => { - cy.insert_doc("Property Setter", { - "doctype": "Property Setter", - "doc_type": "ToDo", - "property": "translate_link_fields", - "property_type": "Check", - "doctype_or_field": "DocType", - "value": "1" - }, true); + it("show translated text for Gender link field with language de with input in de", () => { + cy.call("frappe.tests.ui_test_helpers.insert_translations").then(() => { + cy.window() + .its("frappe") + .then((frappe) => { + cy.set_value("User", frappe.user.name, { language: "de" }); + }); - cy.insert_doc("Property Setter", { - "doctype": "Property Setter", - "doc_type": "ToDo", - "property": "show_title_field_in_link", - "property_type": "Check", - "doctype_or_field": "DocType", - "value": "1" - }, true); + cy.clear_cache(); + cy.wait(500); - cy.window().its('frappe').then(frappe => { - cy.insert_doc("Translation", { - doctype: "Translation", - language: frappe.boot.lang, - source_text: "this is a test todo for link", - translated_text: "this is a translated test todo for link", - }); - }); + get_dialog_with_gender_link().as("dialog"); + cy.intercept("POST", "/api/method/frappe.desk.search.search_link").as("search_link"); - cy.clear_cache(); - cy.wait(500); - - cy.window().its('frappe').then(frappe => { - if (!frappe.boot) { - frappe.boot = { - link_title_doctypes: ['ToDo'], - translatable_doctypes: ['ToDo'] - }; - } else { - frappe.boot.link_title_doctypes = ['ToDo']; - frappe.boot.translatable_doctypes = ['ToDo']; - } - }); - - get_dialog_with_link().as('dialog'); - cy.intercept('POST', '/api/method/frappe.desk.search.search_link').as('search_link'); - - cy.get('.frappe-control[data-fieldname=link] input').focus().as('input'); - cy.wait('@search_link'); - cy.get('@input').type('todo for link', { delay: 100 }); - cy.wait('@search_link'); - cy.get('.frappe-control[data-fieldname=link] ul').should('be.visible'); - cy.get('.frappe-control[data-fieldname=link] input').type('{enter}', { delay: 100 }); - cy.get('.frappe-control[data-fieldname=link] input').blur(); - cy.get('@dialog').then(dialog => { - cy.get('@todos').then(todos => { - let field = dialog.get_field('link'); + cy.get(".frappe-control[data-fieldname=link] input").focus().as("input"); + cy.wait("@search_link"); + cy.get("@input").type("Sonstiges", { delay: 100 }); + cy.wait("@search_link"); + cy.get(".frappe-control[data-fieldname=link] ul").should("be.visible"); + cy.get(".frappe-control[data-fieldname=link] input").type("{enter}", { delay: 100 }); + cy.get(".frappe-control[data-fieldname=link] input").blur(); + cy.get("@dialog").then((dialog) => { + let field = dialog.get_field("link"); let value = field.get_value(); let label = field.get_label_value(); - expect(value).to.eq(todos[0]); - expect(label).to.eq('this is a translated test todo for link'); + expect(value).to.eq("Other"); + expect(label).to.eq("Sonstiges"); }); }); }); - it('show translated text for link with show_title_field_in_link disabled', () => { - cy.insert_doc("Property Setter", { - "doctype": "Property Setter", - "doc_type": "User", - "property": "translate_link_fields", - "property_type": "Check", - "doctype_or_field": "DocType", - "value": "1" - }, true); - - cy.insert_doc("Property Setter", { - "doctype": "Property Setter", - "doc_type": "ToDo", - "property": "show_title_field_in_link", - "property_type": "Check", - "doctype_or_field": "DocType", - "value": "0" - }, true); - - cy.window().its('frappe').then(frappe => { - cy.insert_doc("Translation", { - doctype: "Translation", - language: frappe.boot.lang, - source_text: "test@erpnext.com", - translated_text: "translatedtest@erpnext.com", + it("show text for Gender link field with language en", () => { + cy.window() + .its("frappe") + .then((frappe) => { + cy.set_value("User", frappe.user.name, { language: "en" }); }); - }); cy.clear_cache(); cy.wait(500); - cy.window().its('frappe').then(frappe => { - if (!frappe.boot) { - frappe.boot = { - translatable_doctypes: ['User'] - }; - } else { - frappe.boot.translatable_doctypes = ['User']; - } - }); + get_dialog_with_gender_link().as("dialog"); + cy.intercept("POST", "/api/method/frappe.desk.search.search_link").as("search_link"); - get_dialog_with_user_link().as('dialog'); - cy.intercept('POST', '/api/method/frappe.desk.search.search_link').as('search_link'); - - cy.get('.frappe-control[data-fieldname=link] input').focus().as('input'); - cy.wait('@search_link'); - cy.get('@input').type('test@erpnext.com', { delay: 100 }); - cy.wait('@search_link'); - cy.get('.frappe-control[data-fieldname=link] ul').should('be.visible'); - cy.get('.frappe-control[data-fieldname=link] input').type('{enter}', { delay: 100 }); - cy.get('.frappe-control[data-fieldname=link] input').blur(); - cy.get('@dialog').then(dialog => { - let field = dialog.get_field('link'); + cy.get(".frappe-control[data-fieldname=link] input").focus().as("input"); + cy.wait("@search_link"); + cy.get("@input").type("Non-Conforming", { delay: 100 }); + cy.wait("@search_link"); + cy.get(".frappe-control[data-fieldname=link] ul").should("be.visible"); + cy.get(".frappe-control[data-fieldname=link] input").type("{enter}", { delay: 100 }); + cy.get(".frappe-control[data-fieldname=link] input").blur(); + cy.get("@dialog").then((dialog) => { + let field = dialog.get_field("link"); let value = field.get_value(); let label = field.get_label_value(); - expect(value).to.eq('test@erpnext.com'); - expect(label).to.eq('translatedtest@erpnext.com'); + expect(value).to.eq("Non-Conforming"); + expect(label).to.eq("Non-Conforming"); }); }); + + it("show custom link option", () => { + cy.window() + .its("frappe") + .then((frappe) => { + frappe.ui.form.ControlLink.link_options = (link) => { + return [ + { + html: + "" + + " " + + "Custom Link Option" + + "", + label: "Custom Link Option", + value: "custom__link_option", + action: () => {}, + }, + ]; + }; + + get_dialog_with_link().as("dialog"); + cy.get(".frappe-control[data-fieldname=link] input").focus().as("input"); + cy.get("@input").type("custom", { delay: 100 }); + cy.get(".custom-link-option").should("be.visible"); + }); + }); }); diff --git a/cypress/integration/control_markdown_editor.js b/cypress/integration/control_markdown_editor.js index 34f94f13bf..24ab32f48a 100644 --- a/cypress/integration/control_markdown_editor.js +++ b/cypress/integration/control_markdown_editor.js @@ -7,10 +7,10 @@ context("Control Markdown Editor", () => { it("should allow inserting images by drag and drop", () => { cy.visit("/app/web-page/new"); cy.fill_field("content_type", "Markdown", "Select"); - cy.get_field("main_section_md", "Markdown Editor").attachFile( - "sample_image.jpg", + cy.get_field("main_section_md", "Markdown Editor").selectFile( + "cypress/fixtures/sample_image.jpg", { - subjectType: "drag-n-drop" + action: "drag-drop", } ); cy.click_modal_primary_button("Upload"); diff --git a/cypress/integration/control_phone.js b/cypress/integration/control_phone.js index 5a26decdee..b56343c2d8 100644 --- a/cypress/integration/control_phone.js +++ b/cypress/integration/control_phone.js @@ -1,4 +1,4 @@ -import doctype_with_phone from '../fixtures/doctype_with_phone'; +import doctype_with_phone from "../fixtures/doctype_with_phone"; context("Control Phone", () => { before(() => { @@ -9,10 +9,12 @@ context("Control Phone", () => { function get_dialog_with_phone() { return cy.dialog({ title: "Phone", - fields: [{ - "fieldname": "phone", - "fieldtype": "Phone", - }] + fields: [ + { + fieldname: "phone", + fieldtype: "Phone", + }, + ], }); } @@ -27,18 +29,16 @@ context("Control Phone", () => { let phone_number = "9312672712"; cy.get(".selected-phone > img").click().first(); - cy.get_field("phone") - .first() - .click({multiple: true}); + cy.get_field("phone").first().click({ multiple: true }); cy.get(".frappe-control[data-fieldname=phone]") .findByRole("textbox") .first() - .type(phone_number, {force: true}); + .type(phone_number, { force: true }); cy.get_field("phone").first().should("have.value", phone_number); - cy.get_field("phone").first().blur({force: true}); + cy.get_field("phone").first().blur({ force: true }); cy.wait(100); - cy.get("@dialog").then(dialog => { + cy.get("@dialog").then((dialog) => { let value = dialog.get_value("phone"); expect(value).to.equal("+91-" + phone_number); }); @@ -48,10 +48,12 @@ context("Control Phone", () => { let search_text = "india"; cy.get(".selected-phone").click().first(); cy.get(".phone-picker").findByRole("searchbox").click().type(search_text); - cy.get(".phone-section .phone-wrapper:not(.hidden)").then(i => { - cy.get(`.phone-section .phone-wrapper[id*="${search_text.toLowerCase()}"]`).then(countries => { - expect(i.length).to.equal(countries.length); - }); + cy.get(".phone-section .phone-wrapper:not(.hidden)").then((i) => { + cy.get(`.phone-section .phone-wrapper[id*="${search_text.toLowerCase()}"]`).then( + (countries) => { + expect(i.length).to.equal(countries.length); + } + ); }); cy.get(".phone-picker").findByRole("searchbox").clear().blur(); diff --git a/cypress/integration/control_rating.js b/cypress/integration/control_rating.js index 15c11b352b..613a6e9f92 100644 --- a/cypress/integration/control_rating.js +++ b/cypress/integration/control_rating.js @@ -1,56 +1,54 @@ -context('Control Rating', () => { +context("Control Rating", () => { before(() => { cy.login(); - cy.visit('/app/website'); + cy.visit("/app/website"); }); function get_dialog_with_rating() { return cy.dialog({ - title: 'Rating', - fields: [{ - 'fieldname': 'rate', - 'fieldtype': 'Rating', - 'options': 7 - }] + title: "Rating", + fields: [ + { + fieldname: "rate", + fieldtype: "Rating", + options: 7, + }, + ], }); } - it('click on the star rating to record value', () => { - get_dialog_with_rating().as('dialog'); + it("click on the star rating to record value", () => { + get_dialog_with_rating().as("dialog"); - cy.get('div.rating') - .children('svg') - .find('.right-half') + cy.get("div.rating") + .children("svg") + .find(".right-half") .first() .click() - .should('have.class', 'star-click'); - cy.get('@dialog').then(dialog => { - var value = dialog.get_value('rate'); - expect(value).to.equal(1/7); + .should("have.class", "star-click"); + cy.get("@dialog").then((dialog) => { + var value = dialog.get_value("rate"); + expect(value).to.equal(1 / 7); dialog.hide(); }); }); - it('hover on the star', () => { + it("hover on the star", () => { get_dialog_with_rating(); - cy.get('div.rating') - .children('svg') - .find('.right-half') + cy.get("div.rating") + .children("svg") + .find(".right-half") .first() - .invoke('trigger', 'mouseenter') - .should('have.class', 'star-hover') - .invoke('trigger', 'mouseleave') - .should('not.have.class', 'star-hover'); + .invoke("trigger", "mouseenter") + .should("have.class", "star-hover") + .invoke("trigger", "mouseleave") + .should("not.have.class", "star-hover"); }); - it('check number of stars in rating', () => { + it("check number of stars in rating", () => { get_dialog_with_rating(); - cy.get('div.rating') - .first() - .children('svg') - .should('have.length', 7); + cy.get("div.rating").first().children("svg").should("have.length", 7); }); - }); diff --git a/cypress/integration/control_select.js b/cypress/integration/control_select.js index 8e18d21260..5f7a07e0c4 100644 --- a/cypress/integration/control_select.js +++ b/cypress/integration/control_select.js @@ -1,37 +1,40 @@ -context('Control Select', () => { +context("Control Select", () => { before(() => { cy.login(); - cy.visit('/app/website'); + cy.visit("/app/website"); }); function get_dialog_with_select() { return cy.dialog({ - title: 'Select', - fields: [{ - 'fieldname': 'select_control', - 'fieldtype': 'Select', - 'placeholder': 'Select an Option', - 'options': ['', 'Option 1', 'Option 2', 'Option 2'], - }] + title: "Select", + fields: [ + { + fieldname: "select_control", + fieldtype: "Select", + placeholder: "Select an Option", + options: ["", "Option 1", "Option 2", "Option 2"], + }, + ], }); } - it('toggles placholder on clicking an option', () => { - get_dialog_with_select().as('dialog'); + it("toggles placholder on clicking an option", () => { + get_dialog_with_select().as("dialog"); - cy.get('.frappe-control[data-fieldname=select_control] .control-input').as('control'); - cy.get('.frappe-control[data-fieldname=select_control] .control-input select').as('select'); - cy.get('@control').get('.select-icon').should('exist'); - cy.get('@control').get('.placeholder').should('have.css', 'display', 'block'); - cy.get('@select').select('Option 1'); - cy.findByDisplayValue('Option 1').should('exist'); - cy.get('@control').get('.placeholder').should('have.css', 'display', 'none'); - cy.get('@select').invoke('val', ''); - cy.findByDisplayValue('Option 1').should('not.exist'); - cy.get('@control').get('.placeholder').should('have.css', 'display', 'block'); + cy.get(".frappe-control[data-fieldname=select_control] .control-input").as("control"); + cy.get(".frappe-control[data-fieldname=select_control] .control-input select").as( + "select" + ); + cy.get("@control").get(".select-icon").should("exist"); + cy.get("@control").get(".placeholder").should("have.css", "display", "block"); + cy.get("@select").select("Option 1"); + cy.findByDisplayValue("Option 1").should("exist"); + cy.get("@control").get(".placeholder").should("have.css", "display", "none"); + cy.get("@select").invoke("val", ""); + cy.findByDisplayValue("Option 1").should("not.exist"); + cy.get("@control").get(".placeholder").should("have.css", "display", "block"); - - cy.get('@dialog').then(dialog => { + cy.get("@dialog").then((dialog) => { dialog.hide(); }); }); diff --git a/cypress/integration/custom_buttons.js b/cypress/integration/custom_buttons.js index 6045d009c2..ddbd19731a 100644 --- a/cypress/integration/custom_buttons.js +++ b/cypress/integration/custom_buttons.js @@ -31,10 +31,7 @@ const check_button_count = (label, group = "TestGroup") => { .should("be.visible"); //reset viewport - cy.viewport( - Cypress.config("viewportWidth"), - Cypress.config("viewportHeight") - ); + cy.viewport(Cypress.config("viewportWidth"), Cypress.config("viewportHeight")); }; describe( diff --git a/cypress/integration/customize_form.js b/cypress/integration/customize_form.js index 3857d7ccd8..cd03f7b54c 100644 --- a/cypress/integration/customize_form.js +++ b/cypress/integration/customize_form.js @@ -1,19 +1,19 @@ -context('Customize Form', () => { +context("Customize Form", () => { before(() => { cy.login(); - cy.visit('/app/customize-form'); + cy.visit("/app/customize-form"); }); - it('Changing to naming rule should update autoname', () => { + it("Changing to naming rule should update autoname", () => { cy.fill_field("doc_type", "ToDo", "Link").blur(); cy.click_form_section("Naming"); const naming_rule_default_autoname_map = { "Set by user": "prompt", "By fieldname": "field:", 'By "Naming Series" field': "naming_series:", - "Expression": "format:", + Expression: "format:", "Expression (old style)": "", - "Random": "hash", - "By script": "" + Random: "hash", + "By script": "", }; Cypress._.forOwn(naming_rule_default_autoname_map, (value, naming_rule) => { cy.fill_field("naming_rule", naming_rule, "Select"); diff --git a/cypress/integration/dashboard.js b/cypress/integration/dashboard.js new file mode 100644 index 0000000000..6eb28567bc --- /dev/null +++ b/cypress/integration/dashboard.js @@ -0,0 +1,50 @@ +describe("Dashboard view", { scrollBehavior: false }, () => { + before(() => { + cy.login(); + cy.visit("/app"); + }); + + it("should load", () => { + const chart = "TODO-YEARLY-TRENDS"; + const dashboard = "TODO-TEST-DASHBOARD"; // check slash in name intentionally. + + cy.insert_doc( + "Dashboard Chart", + { + is_standard: 0, + chart_name: chart, + chart_type: "Count", + document_type: "ToDo", + parent_document_type: "", + based_on: "creation", + group_by_type: "Count", + timespan: "Last Year", + time_interval: "Yearly", + timeseries: 1, + type: "Line", + filters_json: "[]", + }, + true + ); + + cy.insert_doc( + "Dashboard", + { + name: dashboard, + dashboard_name: dashboard, + is_standard: 0, + charts: [ + { + chart: chart, + }, + ], + }, + true + ); + + cy.visit(`/app/dashboard-view/${dashboard}`); + + // expect chart to be loaded + cy.findByText(chart).should("be.visible"); + }); +}); diff --git a/cypress/integration/dashboard_chart.js b/cypress/integration/dashboard_chart.js index ae71fcda3a..6023a50abe 100644 --- a/cypress/integration/dashboard_chart.js +++ b/cypress/integration/dashboard_chart.js @@ -1,22 +1,22 @@ -context('Dashboard Chart', () => { +context("Dashboard Chart", () => { before(() => { cy.login(); - cy.visit('/app/website'); + cy.visit("/app/website"); }); - it('Check filter populate for child table doctype', () => { - cy.visit('/app/dashboard-chart/new-dashboard-chart-1'); - cy.get('[data-fieldname="parent_document_type"]').should('have.css', 'display', 'none'); + it("Check filter populate for child table doctype", () => { + cy.visit("/app/dashboard-chart/new-dashboard-chart-1"); + cy.get('[data-fieldname="parent_document_type"]').should("have.css", "display", "none"); - cy.get_field('document_type', 'Link'); - cy.fill_field('document_type', 'Workspace Link', 'Link').focus().blur(); - cy.get_field('document_type', 'Link').should('have.value', 'Workspace Link'); + cy.get_field("document_type", "Link"); + cy.fill_field("document_type", "Workspace Link", "Link").focus().blur(); + cy.get_field("document_type", "Link").should("have.value", "Workspace Link"); - cy.fill_field('chart_name', 'Test Chart', 'Data'); + cy.fill_field("chart_name", "Test Chart", "Data"); cy.get('[data-fieldname="filters_json"]').click().wait(200); - cy.get('.modal-body .filter-action-buttons .add-filter').click(); - cy.get('.modal-body .fieldname-select-area').click(); - cy.get('.modal-actions .btn-modal-close').click(); + cy.get(".modal-body .filter-action-buttons .add-filter").click(); + cy.get(".modal-body .fieldname-select-area").click(); + cy.get(".modal-actions .btn-modal-close").click(); }); -}); \ No newline at end of file +}); diff --git a/cypress/integration/dashboard_links.js b/cypress/integration/dashboard_links.js index 019de1991d..ebcdfa0048 100644 --- a/cypress/integration/dashboard_links.js +++ b/cypress/integration/dashboard_links.js @@ -1,91 +1,94 @@ -import doctype_with_child_table from '../fixtures/doctype_with_child_table'; -import child_table_doctype from '../fixtures/child_table_doctype'; -import child_table_doctype_1 from '../fixtures/child_table_doctype_1'; -import doctype_to_link from '../fixtures/doctype_to_link'; +import doctype_with_child_table from "../fixtures/doctype_with_child_table"; +import child_table_doctype from "../fixtures/child_table_doctype"; +import child_table_doctype_1 from "../fixtures/child_table_doctype_1"; +import doctype_to_link from "../fixtures/doctype_to_link"; const doctype_to_link_name = doctype_to_link.name; const child_table_doctype_name = child_table_doctype.name; -context('Dashboard links', () => { +context("Dashboard links", () => { before(() => { - cy.visit('/login'); - cy.login(); - cy.insert_doc('DocType', child_table_doctype, true); - cy.insert_doc('DocType', child_table_doctype_1, true); - cy.insert_doc('DocType', doctype_with_child_table, true); - cy.insert_doc('DocType', doctype_to_link, true); - return cy.window().its('frappe').then(frappe => { - return frappe.xcall("frappe.tests.ui_test_helpers.update_child_table", { - name: child_table_doctype_name + cy.visit("/login"); + cy.login("Administrator"); + cy.insert_doc("DocType", child_table_doctype, true); + cy.insert_doc("DocType", child_table_doctype_1, true); + cy.insert_doc("DocType", doctype_with_child_table, true); + cy.insert_doc("DocType", doctype_to_link, true); + return cy + .window() + .its("frappe") + .then((frappe) => { + frappe.call("frappe.tests.ui_test_helpers.update_child_table", { + name: child_table_doctype_name, + }); }); - }); }); - it('Adding a new contact, checking for the counter on the dashboard and deleting the created contact', () => { - cy.visit('/app/contact'); + it("Adding a new contact, checking for the counter on the dashboard and deleting the created contact", () => { + cy.visit("/app/contact"); cy.clear_filters(); - cy.visit('/app/user'); - cy.get('.list-row-col > .level-item > .ellipsis').eq(0).click({ force: true }); + cy.visit(`/app/user/${cy.config("testUser")}`); //To check if initially the dashboard contains only the "Contact" link and there is no counter - cy.get('[data-doctype="Contact"]').should('contain', 'Contact'); + cy.select_form_tab("Connections"); + cy.get('[data-doctype="Contact"]').should("contain", "Contact"); //Adding a new contact cy.get('.document-link-badge[data-doctype="Contact"]').click(); cy.wait(300); - cy.findByRole('button', {name: 'Add Contact'}).should('be.visible'); - cy.findByRole('button', {name: 'Add Contact'}).click(); - cy.get('[data-doctype="Contact"][data-fieldname="first_name"]').type('Admin'); - cy.findByRole('button', {name: 'Save'}).click(); - cy.visit('/app/user'); - cy.get('.list-row-col > .level-item > .ellipsis').eq(0).click({ force: true }); + cy.findByRole("button", { name: "Add Contact" }).should("be.visible"); + cy.findByRole("button", { name: "Add Contact" }).click(); + cy.get('[data-doctype="Contact"][data-fieldname="first_name"]').type("Admin"); + cy.findByRole("button", { name: "Save" }).click(); + cy.visit(`/app/user/${cy.config("testUser")}`); - //To check if the counter for contact doc is "1" after adding the contact - cy.get('[data-doctype="Contact"] > .count').should('contain', '1'); - cy.get('[data-doctype="Contact"]').contains('Contact').click(); + //To check if the counter for contact doc is "2" after adding additional contact + cy.select_form_tab("Connections"); + cy.get('[data-doctype="Contact"] > .count').should("contain", "2"); + cy.get('[data-doctype="Contact"]').contains("Contact").click(); //Deleting the newly created contact - cy.visit('/app/contact'); - cy.get('.list-subject > .select-like > .list-row-checkbox').eq(0).click({ force: true }); - cy.findByRole('button', {name: 'Actions'}).click(); + cy.visit("/app/contact"); + cy.get(".list-subject > .select-like > .list-row-checkbox").eq(0).click({ force: true }); + cy.findByRole("button", { name: "Actions" }).click(); cy.get('.actions-btn-group [data-label="Delete"]').click(); - cy.findByRole('button', {name: 'Yes'}).click({delay: 700}); - + cy.findByRole("button", { name: "Yes" }).click({ delay: 700 }); //To check if the counter from the "Contact" doc link is removed cy.wait(700); - cy.visit('/app/user'); - cy.get('.list-row-col > .level-item > .ellipsis').eq(0).click({ force: true }); - cy.get('[data-doctype="Contact"]').should('contain', 'Contact'); + cy.visit("/app/user"); + cy.get(".list-row-col > .level-item > .ellipsis").eq(0).click({ force: true }); + cy.get('[data-doctype="Contact"]').should("contain", "Contact"); }); - it('Report link in dashboard', () => { - cy.visit('/app/user'); - cy.visit('/app/user/Administrator'); - cy.get('[data-doctype="Contact"]').should('contain', 'Contact'); - cy.findByText('Connections'); + it("Report link in dashboard", () => { + cy.visit(`/app/user/${cy.config("testUser")}`); + cy.select_form_tab("Connections"); + cy.get('.document-link[data-doctype="Contact"]').contains("Contact"); cy.window() - .its('cur_frm') - .then(cur_frm => { + .its("cur_frm") + .then((cur_frm) => { cur_frm.dashboard.data.reports = [ { - 'label': 'Reports', - 'items': ['Website Analytics'] - } + label: "Reports", + items: ["Website Analytics"], + }, ]; cur_frm.dashboard.render_report_links(); - cy.get('[data-report="Website Analytics"]').contains('Website Analytics').click(); - cy.findByText('Website Analytics'); + cy.get('.document-link[data-report="Website Analytics"]') + .contains("Website Analytics") + .click(); }); }); - it('check if child table is populated with linked field on creation from dashboard link', () => { + it("check if child table is populated with linked field on creation from dashboard link", () => { cy.new_form(doctype_to_link_name); cy.fill_field("title", "Test Linking"); - cy.findByRole("button", {name: "Save"}).click(); + cy.findByRole("button", { name: "Save" }).click(); - cy.get('.document-link .btn-new').click(); - cy.get('.frappe-control[data-fieldname="child_table"] .rows .data-row .col[data-fieldname="doctype_to_link"]') - .should('contain.text', 'Test Linking'); + cy.get(".document-link .btn-new").click(); + cy.get( + '.frappe-control[data-fieldname="child_table"] .rows .data-row .col[data-fieldname="doctype_to_link"]' + ).should("contain.text", "Test Linking"); }); }); diff --git a/cypress/integration/data_field_form_validation.js b/cypress/integration/data_field_form_validation.js index c6feea5550..49513e72fb 100644 --- a/cypress/integration/data_field_form_validation.js +++ b/cypress/integration/data_field_form_validation.js @@ -1,43 +1,45 @@ -import data_field_validation_doctype from '../fixtures/data_field_validation_doctype'; +import data_field_validation_doctype from "../fixtures/data_field_validation_doctype"; const doctype_name = data_field_validation_doctype.name; - -context('Data Field Input Validation in New Form', () => { +context("Data Field Input Validation in New Form", () => { before(() => { cy.login(); - cy.visit('/app/website'); - return cy.insert_doc('DocType', data_field_validation_doctype, true); + cy.visit("/app/website"); + return cy.insert_doc("DocType", data_field_validation_doctype, true); }); function validateField(fieldname, invalid_value, valid_value) { // Invalid, should have has-error class cy.get_field(fieldname).clear().type(invalid_value).blur(); - cy.get(`.frappe-control[data-fieldname="${fieldname}"]`).should('have.class', 'has-error'); + cy.get(`.frappe-control[data-fieldname="${fieldname}"]`).should("have.class", "has-error"); // Valid value, should not have has-error class cy.get_field(fieldname).clear().type(valid_value); - cy.get(`.frappe-control[data-fieldname="${fieldname}"]`).should('not.have.class', 'has-error'); + cy.get(`.frappe-control[data-fieldname="${fieldname}"]`).should( + "not.have.class", + "has-error" + ); } - describe('Data Field Options', () => { - it('should validate email address', () => { + describe("Data Field Options", () => { + it("should validate email address", () => { cy.new_form(doctype_name); - validateField('email', 'captian', 'hello@test.com'); + validateField("email", "captian", "hello@test.com"); }); - it('should validate URL', () => { - validateField('url', 'jkl', 'https://frappe.io'); - validateField('url', 'abcd.com', 'http://google.com/home'); - validateField('url', '&&http://google.uae', 'gopher://frappe.io'); - validateField('url', 'ftt2:://google.in?q=news', 'ftps2://frappe.io/__/#home'); - validateField('url', 'ftt2://', 'ntps://localhost'); // For intranet URLs + it("should validate URL", () => { + validateField("url", "jkl", "https://frappe.io"); + validateField("url", "abcd.com", "http://google.com/home"); + validateField("url", "&&http://google.uae", "gopher://frappe.io"); + validateField("url", "ftt2:://google.in?q=news", "ftps2://frappe.io/__/#home"); + validateField("url", "ftt2://", "ntps://localhost"); // For intranet URLs }); - it('should validate phone number', () => { - validateField('phone', 'america', '89787878'); + it("should validate phone number", () => { + validateField("phone", "america", "89787878"); }); - it('should validate name', () => { - validateField('person_name', ' 777Hello', 'James Bond'); + it("should validate name", () => { + validateField("person_name", " 777Hello", "James Bond"); }); }); -}); \ No newline at end of file +}); diff --git a/cypress/integration/datetime.js b/cypress/integration/datetime.js index 4a24faf40b..7a8a68c1d9 100644 --- a/cypress/integration/datetime.js +++ b/cypress/integration/datetime.js @@ -1,53 +1,52 @@ -import datetime_doctype from '../fixtures/datetime_doctype'; +import datetime_doctype from "../fixtures/datetime_doctype"; const doctype_name = datetime_doctype.name; -context('Control Date, Time and DateTime', () => { +context("Control Date, Time and DateTime", () => { before(() => { cy.login(); - cy.visit('/app/website'); - return cy.insert_doc('DocType', datetime_doctype, true); + cy.visit("/app/website"); + return cy.insert_doc("DocType", datetime_doctype, true); }); - describe('Date formats', () => { + describe("Date formats", () => { let date_formats = [ { - date_format: 'dd-mm-yyyy', + date_format: "dd-mm-yyyy", part: 2, length: 4, - separator: '-' + separator: "-", }, { - date_format: 'mm/dd/yyyy', + date_format: "mm/dd/yyyy", part: 0, length: 2, - separator: '/' - } + separator: "/", + }, ]; - date_formats.forEach(d => { - it('test date format ' + d.date_format, () => { - cy.set_value('System Settings', 'System Settings', { - date_format: d.date_format + date_formats.forEach((d) => { + it("test date format " + d.date_format, () => { + cy.set_value("System Settings", "System Settings", { + date_format: d.date_format, }); cy.window() - .its('frappe') - .then(frappe => { + .its("frappe") + .then((frappe) => { // update sys_defaults value to avoid a reload frappe.sys_defaults.date_format = d.date_format; }); cy.new_form(doctype_name); - cy.get('.form-control[data-fieldname=date]').focus(); - cy.get('.datepickers-container .datepicker.active') - .should('be.visible'); + cy.get(".form-control[data-fieldname=date]").focus(); + cy.get(".datepickers-container .datepicker.active").should("be.visible"); cy.get( - '.datepickers-container .datepicker.active .datepicker--cell-day.-current-' + ".datepickers-container .datepicker.active .datepicker--cell-day.-current-" ).click({ force: true }); cy.window() - .its('cur_frm') - .then(cur_frm => { - let formatted_value = cur_frm.get_field('date').input.value; + .its("cur_frm") + .then((cur_frm) => { + let formatted_value = cur_frm.get_field("date").input.value; let parts = formatted_value.split(d.separator); expect(parts[d.part].length).to.equal(d.length); }); @@ -55,74 +54,72 @@ context('Control Date, Time and DateTime', () => { }); }); - describe('Time formats', () => { + describe("Time formats", () => { let time_formats = [ { - time_format: 'HH:mm:ss', - value: ' 11:00:12', - match_value: '11:00:12' + time_format: "HH:mm:ss", + value: " 11:00:12", + match_value: "11:00:12", }, { - time_format: 'HH:mm', - value: ' 11:00:12', - match_value: '11:00' - } + time_format: "HH:mm", + value: " 11:00:12", + match_value: "11:00", + }, ]; - time_formats.forEach(d => { - it('test time format ' + d.time_format, () => { - cy.set_value('System Settings', 'System Settings', { - time_format: d.time_format + time_formats.forEach((d) => { + it("test time format " + d.time_format, () => { + cy.set_value("System Settings", "System Settings", { + time_format: d.time_format, }); cy.window() - .its('frappe') - .then(frappe => { + .its("frappe") + .then((frappe) => { frappe.sys_defaults.time_format = d.time_format; }); cy.new_form(doctype_name); - cy.fill_field('time', d.value, 'Time').blur(); - cy.get_field('time').should('have.value', d.match_value); + cy.fill_field("time", d.value, "Time").blur(); + cy.get_field("time").should("have.value", d.match_value); }); }); }); - describe('DateTime formats', () => { + describe("DateTime formats", () => { let datetime_formats = [ { - date_format: 'dd.mm.yyyy', - time_format: 'HH:mm:ss', - value: ' 02.12.2019 11:00:12', - doc_value: '2019-12-02 00:30:12', // system timezone (America/New_York) - input_value: '02.12.2019 11:00:12' // admin timezone (Asia/Kolkata) + date_format: "dd.mm.yyyy", + time_format: "HH:mm:ss", + value: " 02.12.2019 11:00:12", + doc_value: "2019-12-02 00:30:12", // system timezone (America/New_York) + input_value: "02.12.2019 11:00:12", // admin timezone (Asia/Kolkata) }, { - date_format: 'mm-dd-yyyy', - time_format: 'HH:mm', - value: ' 12-02-2019 11:00:00', - doc_value: '2019-12-02 00:30:00', // system timezone (America/New_York) - input_value: '12-02-2019 11:00' // admin timezone (Asia/Kolkata) - } + date_format: "mm-dd-yyyy", + time_format: "HH:mm", + value: " 12-02-2019 11:00:00", + doc_value: "2019-12-02 00:30:00", // system timezone (America/New_York) + input_value: "12-02-2019 11:00", // admin timezone (Asia/Kolkata) + }, ]; - datetime_formats.forEach(d => { + datetime_formats.forEach((d) => { it(`test datetime format ${d.date_format} ${d.time_format}`, () => { - cy.set_value('System Settings', 'System Settings', { + cy.set_value("System Settings", "System Settings", { date_format: d.date_format, - time_format: d.time_format + time_format: d.time_format, }); cy.window() - .its('frappe') - .then(frappe => { + .its("frappe") + .then((frappe) => { frappe.sys_defaults.date_format = d.date_format; frappe.sys_defaults.time_format = d.time_format; }); cy.new_form(doctype_name); - cy.fill_field('datetime', d.value, 'Datetime').blur(); - cy.get_field('datetime').should('have.value', d.input_value); + cy.fill_field("datetime", d.value, "Datetime").blur(); + cy.get_field("datetime").should("have.value", d.input_value); - cy.window() - .its('cur_frm.doc.datetime') - .should('eq', d.doc_value); + cy.window().its("cur_frm.doc.datetime").should("eq", d.doc_value); }); }); }); diff --git a/cypress/integration/datetime_field_form_validation.js b/cypress/integration/datetime_field_form_validation.js index ef47a0fbf7..1a549d8a1d 100644 --- a/cypress/integration/datetime_field_form_validation.js +++ b/cypress/integration/datetime_field_form_validation.js @@ -16,4 +16,4 @@ // cy.get('.indicator-pill').should('contain', 'Open').should('have.class', 'red'); // }); // }); -// }); \ No newline at end of file +// }); diff --git a/cypress/integration/depends_on.js b/cypress/integration/depends_on.js index 12f54f2b6e..6419809466 100644 --- a/cypress/integration/depends_on.js +++ b/cypress/integration/depends_on.js @@ -1,135 +1,152 @@ -context('Depends On', () => { +context("Depends On", () => { before(() => { cy.login(); - cy.visit('/app/website'); - return cy.window().its('frappe').then(frappe => { - return frappe.xcall('frappe.tests.ui_test_helpers.create_child_doctype', { - name: 'Child Test Depends On', - fields: [ - { - "label": "Child Test Field", - "fieldname": "child_test_field", - "fieldtype": "Data", - "in_list_view": 1, - }, - { - "label": "Child Dependant Field", - "fieldname": "child_dependant_field", - "fieldtype": "Data", - "in_list_view": 1, - }, - { - "label": "Child Display Dependant Field", - "fieldname": "child_display_dependant_field", - "fieldtype": "Data", - "in_list_view": 1, - }, - ] + cy.visit("/app/website"); + return cy + .window() + .its("frappe") + .then((frappe) => { + return frappe.xcall("frappe.tests.ui_test_helpers.create_child_doctype", { + name: "Child Test Depends On", + fields: [ + { + label: "Child Test Field", + fieldname: "child_test_field", + fieldtype: "Data", + in_list_view: 1, + }, + { + label: "Child Dependant Field", + fieldname: "child_dependant_field", + fieldtype: "Data", + in_list_view: 1, + }, + { + label: "Child Display Dependant Field", + fieldname: "child_display_dependant_field", + fieldtype: "Data", + in_list_view: 1, + }, + ], + }); + }) + .then((frappe) => { + return frappe.xcall("frappe.tests.ui_test_helpers.create_doctype", { + name: "Test Depends On", + fields: [ + { + label: "Test Field", + fieldname: "test_field", + fieldtype: "Data", + }, + { + label: "Dependant Field", + fieldname: "dependant_field", + fieldtype: "Data", + mandatory_depends_on: "eval:doc.test_field=='Some Value'", + read_only_depends_on: "eval:doc.test_field=='Some Other Value'", + }, + { + label: "Display Dependant Field", + fieldname: "display_dependant_field", + fieldtype: "Data", + depends_on: "eval:doc.test_field=='Value'", + }, + { + label: "Child Test Depends On Field", + fieldname: "child_test_depends_on_field", + fieldtype: "Table", + read_only_depends_on: "eval:doc.test_field=='Some Other Value'", + options: "Child Test Depends On", + }, + { + label: "Dependent Tab", + fieldname: "dependent_tab", + fieldtype: "Tab Break", + depends_on: "eval:doc.test_field=='Show Tab'", + }, + { + fieldname: "tab_section", + fieldtype: "Section Break", + }, + { + label: "Field in Tab", + fieldname: "field_in_tab", + fieldtype: "Data", + }, + ], + }); }); - }).then(frappe => { - return frappe.xcall('frappe.tests.ui_test_helpers.create_doctype', { - name: 'Test Depends On', - fields: [ - { - "label": "Test Field", - "fieldname": "test_field", - "fieldtype": "Data", - }, - { - "label": "Dependant Field", - "fieldname": "dependant_field", - "fieldtype": "Data", - "mandatory_depends_on": "eval:doc.test_field=='Some Value'", - "read_only_depends_on": "eval:doc.test_field=='Some Other Value'", - }, - { - "label": "Display Dependant Field", - "fieldname": "display_dependant_field", - "fieldtype": "Data", - 'depends_on': "eval:doc.test_field=='Value'" - }, - { - "label": "Child Test Depends On Field", - "fieldname": "child_test_depends_on_field", - "fieldtype": "Table", - 'read_only_depends_on': "eval:doc.test_field=='Some Other Value'", - 'options': "Child Test Depends On" - }, - { - "label": "Dependent Tab", - "fieldname": "dependent_tab", - "fieldtype": "Tab Break", - "depends_on": "eval:doc.test_field=='Show Tab'" - }, - { - "fieldname": "tab_section", - "fieldtype": "Section Break", - }, - { - "label": "Field in Tab", - "fieldname": "field_in_tab", - "fieldtype": "Data", - } - ] - }); - }); }); - it('should show the tab on other setting field value', () => { - cy.new_form('Test Depends On'); - cy.fill_field('test_field', 'Show Tab'); - cy.get('body').click(); - cy.findByRole("tab", {name: "Dependent Tab"}).should('be.visible'); + it("should show the tab on other setting field value", () => { + cy.new_form("Test Depends On"); + cy.fill_field("test_field", "Show Tab"); + cy.get("body").click(); + cy.findByRole("tab", { name: "Dependent Tab" }).should("be.visible"); }); - it('should set the field as mandatory depending on other fields value', () => { - cy.new_form('Test Depends On'); - cy.fill_field('test_field', 'Some Value'); - cy.findByRole('button', {name: 'Save'}).click(); - cy.get('.msgprint-dialog .modal-title').contains('Missing Fields').should('be.visible'); + it("should set the field as mandatory depending on other fields value", () => { + cy.new_form("Test Depends On"); + cy.fill_field("test_field", "Some Value"); + cy.findByRole("button", { name: "Save" }).click(); + cy.get(".msgprint-dialog .modal-title").contains("Missing Fields").should("be.visible"); cy.hide_dialog(); - cy.fill_field('test_field', 'Random value'); - cy.findByRole('button', {name: 'Save'}).click(); - cy.get('.msgprint-dialog .modal-title').contains('Missing Fields').should('not.be.visible'); + cy.fill_field("test_field", "Random value"); + cy.findByRole("button", { name: "Save" }).click(); + cy.get(".msgprint-dialog .modal-title") + .contains("Missing Fields") + .should("not.be.visible"); }); - it('should set the field as read only depending on other fields value', () => { - cy.new_form('Test Depends On'); - cy.fill_field('dependant_field', 'Some Value'); - cy.fill_field('test_field', 'Some Other Value'); - cy.get('body').click(); - cy.get('.control-input [data-fieldname="dependant_field"]').should('be.disabled'); - cy.fill_field('test_field', 'Random Value'); - cy.get('body').click(); - cy.get('.control-input [data-fieldname="dependant_field"]').should('not.be.disabled'); + it("should set the field as read only depending on other fields value", () => { + cy.new_form("Test Depends On"); + cy.fill_field("dependant_field", "Some Value"); + cy.fill_field("test_field", "Some Other Value"); + cy.get("body").click(); + cy.get('.control-input [data-fieldname="dependant_field"]').should("be.disabled"); + cy.fill_field("test_field", "Random Value"); + cy.get("body").click(); + cy.get('.control-input [data-fieldname="dependant_field"]').should("not.be.disabled"); }); - it('should set the table and its fields as read only depending on other fields value', () => { - cy.new_form('Test Depends On'); - cy.fill_field('dependant_field', 'Some Value'); + it("should set the table and its fields as read only depending on other fields value", () => { + cy.new_form("Test Depends On"); + cy.fill_field("dependant_field", "Some Value"); //cy.fill_field('test_field', 'Some Other Value'); - cy.get('.frappe-control[data-fieldname="child_test_depends_on_field"]').as('table'); - cy.get('@table').findByRole('button', {name: 'Add Row'}).click(); - cy.get('@table').find('[data-idx="1"]').as('row1'); - cy.get('@row1').find('.btn-open-row').click(); - cy.get('@row1').find('.form-in-grid').as('row1-form_in_grid'); + cy.get('.frappe-control[data-fieldname="child_test_depends_on_field"]').as("table"); + cy.get("@table").findByRole("button", { name: "Add Row" }).click(); + cy.get("@table").find('[data-idx="1"]').as("row1"); + cy.get("@row1").find(".btn-open-row").click(); + cy.get("@row1").find(".form-in-grid").as("row1-form_in_grid"); //cy.get('@row1-form_in_grid').find('') - cy.fill_table_field('child_test_depends_on_field', '1', 'child_test_field', 'Some Value'); - cy.fill_table_field('child_test_depends_on_field', '1', 'child_dependant_field', 'Some Other Value'); + cy.fill_table_field("child_test_depends_on_field", "1", "child_test_field", "Some Value"); + cy.fill_table_field( + "child_test_depends_on_field", + "1", + "child_dependant_field", + "Some Other Value" + ); - cy.get('@row1-form_in_grid').find('.grid-collapse-row').click(); + cy.get("@row1-form_in_grid").find(".grid-collapse-row").click(); // set the table to read-only - cy.fill_field('test_field', 'Some Other Value'); + cy.fill_field("test_field", "Some Other Value"); // grid row form fields should be read-only - cy.get('@row1').find('.btn-open-row').click(); + cy.get("@row1").find(".btn-open-row").click(); - cy.get('@row1-form_in_grid').find('.control-input [data-fieldname="child_test_field"]').should('be.disabled'); - cy.get('@row1-form_in_grid').find('.control-input [data-fieldname="child_dependant_field"]').should('be.disabled'); + cy.get("@row1-form_in_grid") + .find('.control-input [data-fieldname="child_test_field"]') + .should("be.disabled"); + cy.get("@row1-form_in_grid") + .find('.control-input [data-fieldname="child_dependant_field"]') + .should("be.disabled"); }); - it('should display the field depending on other fields value', () => { - cy.new_form('Test Depends On'); - cy.get('.control-input [data-fieldname="display_dependant_field"]').should('not.be.visible'); + it("should display the field depending on other fields value", () => { + cy.new_form("Test Depends On"); + cy.get('.control-input [data-fieldname="display_dependant_field"]').should( + "not.be.visible" + ); cy.get('.control-input [data-fieldname="test_field"]').clear(); - cy.fill_field('test_field', 'Value'); - cy.get('body').click(); - cy.get('.control-input [data-fieldname="display_dependant_field"]').should('be.visible'); + cy.fill_field("test_field", "Value"); + cy.get("body").click(); + cy.get('.control-input [data-fieldname="display_dependant_field"]').should("be.visible"); }); }); diff --git a/cypress/integration/discussions.js b/cypress/integration/discussions.js index caf7d6c3f9..55bcabce19 100644 --- a/cypress/integration/discussions.js +++ b/cypress/integration/discussions.js @@ -1,79 +1,101 @@ -context('Discussions', () => { +context("Discussions", () => { before(() => { cy.login(); - cy.visit('/app'); - return cy.window().its('frappe').then(frappe => { - return frappe.call('frappe.tests.ui_test_helpers.create_data_for_discussions'); - }); + cy.visit("/app"); + return cy + .window() + .its("frappe") + .then((frappe) => { + return frappe.call("frappe.tests.ui_test_helpers.create_data_for_discussions"); + }); }); const reply_through_modal = () => { - cy.visit('/test-page-discussions'); + cy.visit("/test-page-discussions"); // Open the modal - cy.get('.reply').click(); + cy.get(".reply").click(); cy.wait(500); - cy.get('.discussion-modal').should('be.visible'); + cy.get(".discussion-modal").should("be.visible"); // Enter title - cy.get('.modal .topic-title').type('Discussion from tests') - .should('have.value', 'Discussion from tests'); + cy.get(".modal .topic-title") + .type("Discussion from tests") + .should("have.value", "Discussion from tests"); // Enter comment - cy.get('.modal .comment-field') - .type('This is a discussion from the cypress ui tests.') - .should('have.value', 'This is a discussion from the cypress ui tests.'); + cy.get(".modal .comment-field") + .type("This is a discussion from the cypress ui tests.") + .should("have.value", "This is a discussion from the cypress ui tests."); // Submit - cy.get('.modal .submit-discussion').click(); + cy.get(".modal .submit-discussion").click(); cy.wait(2000); // Check if discussion is added to page and content is visible - cy.get('.sidebar-parent:first .discussion-topic-title').should('have.text', 'Discussion from tests'); - cy.get('.discussion-on-page:visible').should('have.class', 'show'); - cy.get('.discussion-on-page:visible .reply-card .reply-text') - .should('have.text', 'This is a discussion from the cypress ui tests.\n'); - + cy.get(".sidebar-parent:first .discussion-topic-title").should( + "have.text", + "Discussion from tests" + ); + cy.get(".discussion-on-page:visible").should("have.class", "show"); + cy.get(".discussion-on-page:visible .reply-card .reply-text").should( + "have.text", + "This is a discussion from the cypress ui tests.\n" + ); }; const reply_through_comment_box = () => { - cy.get('.discussion-form:visible .comment-field') - .type('This is a discussion from the cypress ui tests. \n\nThis comment was entered through the commentbox on the page.') - .should('have.value', 'This is a discussion from the cypress ui tests. \n\nThis comment was entered through the commentbox on the page.'); + cy.get(".discussion-form:visible .comment-field") + .type( + "This is a discussion from the cypress ui tests. \n\nThis comment was entered through the commentbox on the page." + ) + .should( + "have.value", + "This is a discussion from the cypress ui tests. \n\nThis comment was entered through the commentbox on the page." + ); - cy.get('.discussion-form:visible .submit-discussion').click(); + cy.get(".discussion-form:visible .submit-discussion").click(); cy.wait(3000); - cy.get('.discussion-on-page:visible').should('have.class', 'show'); - cy.get('.discussion-on-page:visible').children(".reply-card").eq(1).find(".reply-text") - .should('have.text', 'This is a discussion from the cypress ui tests. \n\nThis comment was entered through the commentbox on the page.\n'); + cy.get(".discussion-on-page:visible").should("have.class", "show"); + cy.get(".discussion-on-page:visible") + .children(".reply-card") + .eq(1) + .find(".reply-text") + .should( + "have.text", + "This is a discussion from the cypress ui tests. \n\nThis comment was entered through the commentbox on the page.\n" + ); }; const cancel_and_clear_comment_box = () => { - cy.get('.discussion-form:visible .comment-field') - .type('This is a discussion from the cypress ui tests.') - .should('have.value', 'This is a discussion from the cypress ui tests.'); + cy.get(".discussion-form:visible .comment-field") + .type("This is a discussion from the cypress ui tests.") + .should("have.value", "This is a discussion from the cypress ui tests."); - cy.get('.discussion-form:visible .cancel-comment').click(); - cy.get('.discussion-form:visible .comment-field').should('have.value', ''); + cy.get(".discussion-form:visible .cancel-comment").click(); + cy.get(".discussion-form:visible .comment-field").should("have.value", ""); }; const single_thread_discussion = () => { - cy.visit('/test-single-thread'); - cy.get('.discussions-sidebar').should('have.length', 0); - cy.get('.reply').should('have.length', 0); + cy.visit("/test-single-thread"); + cy.get(".discussions-sidebar").should("have.length", 0); + cy.get(".reply").should("have.length", 0); - cy.get('.discussion-form:visible .comment-field') - .type('This comment is being made on a single thread discussion.') - .should('have.value', 'This comment is being made on a single thread discussion.'); + cy.get(".discussion-form:visible .comment-field") + .type("This comment is being made on a single thread discussion.") + .should("have.value", "This comment is being made on a single thread discussion."); - cy.get('.discussion-form:visible .submit-discussion').click(); + cy.get(".discussion-form:visible .submit-discussion").click(); cy.wait(3000); - cy.get('.discussion-on-page').children(".reply-card").eq(-1).find(".reply-text") - .should('have.text', 'This comment is being made on a single thread discussion.\n'); + cy.get(".discussion-on-page") + .children(".reply-card") + .eq(-1) + .find(".reply-text") + .should("have.text", "This comment is being made on a single thread discussion.\n"); }; - it('reply through modal', reply_through_modal); - it('reply through comment box', reply_through_comment_box); - it('cancel and clear comment box', cancel_and_clear_comment_box); - it('single thread discussion', single_thread_discussion); + it("reply through modal", reply_through_modal); + it("reply through comment box", reply_through_comment_box); + it("cancel and clear comment box", cancel_and_clear_comment_box); + it("single thread discussion", single_thread_discussion); }); diff --git a/cypress/integration/file_uploader.js b/cypress/integration/file_uploader.js index 3d4f92df3c..e1cf91d043 100644 --- a/cypress/integration/file_uploader.js +++ b/cypress/integration/file_uploader.js @@ -1,78 +1,86 @@ -context('FileUploader', () => { +context("FileUploader", () => { before(() => { cy.login(); - cy.visit('/app'); + cy.visit("/app"); }); function open_upload_dialog() { - cy.window().its('frappe').then(frappe => { - new frappe.ui.FileUploader(); - }); + cy.window() + .its("frappe") + .then((frappe) => { + new frappe.ui.FileUploader(); + }); } - it('upload dialog api works', () => { + it("upload dialog api works", () => { open_upload_dialog(); - cy.get_open_dialog().should('contain', 'Drag and drop files'); + cy.get_open_dialog().should("contain", "Drag and drop files"); cy.hide_dialog(); }); - it('should accept dropped files', () => { + it("should accept dropped files", () => { open_upload_dialog(); - cy.get_open_dialog().find('.file-upload-area').attachFile('example.json', { - subjectType: 'drag-n-drop', - }); - - cy.get_open_dialog().find('.file-name').should('contain', 'example.json'); - cy.intercept('POST', '/api/method/upload_file').as('upload_file'); - cy.get_open_dialog().findByRole('button', {name: 'Upload'}).click(); - cy.wait('@upload_file').its('response.statusCode').should('eq', 200); - cy.get('.modal:visible').should('not.exist'); - }); - - it('should accept uploaded files', () => { - open_upload_dialog(); - - cy.get_open_dialog().findByRole('button', {name: 'Library'}).click(); - cy.findByPlaceholderText('Search by filename or extension').type('example.json'); - cy.get_open_dialog().findAllByText('example.json').first().click(); - cy.intercept('POST', '/api/method/upload_file').as('upload_file'); - cy.get_open_dialog().findByRole('button', {name: 'Upload'}).click(); - cy.wait('@upload_file').its('response.body.message') - .should('have.property', 'file_name', 'example.json'); - cy.get('.modal:visible').should('not.exist'); - }); - - it('should accept web links', () => { - open_upload_dialog(); - - cy.get_open_dialog().findByRole('button', {name: 'Link'}).click(); cy.get_open_dialog() - .findByPlaceholderText('Attach a web link') - .type('https://github.com', { delay: 100, force: true }); - cy.intercept('POST', '/api/method/upload_file').as('upload_file'); - cy.get_open_dialog().findByRole('button', {name: 'Upload'}).click(); - cy.wait('@upload_file').its('response.body.message') - .should('have.property', 'file_url', 'https://github.com'); - cy.get('.modal:visible').should('not.exist'); + .find(".file-upload-area") + .selectFile("cypress/fixtures/example.json", { + action: "drag-drop", + }); + + cy.get_open_dialog().find(".file-name").should("contain", "example.json"); + cy.intercept("POST", "/api/method/upload_file").as("upload_file"); + cy.get_open_dialog().findByRole("button", { name: "Upload" }).click(); + cy.wait("@upload_file").its("response.statusCode").should("eq", 200); + cy.get(".modal:visible").should("not.exist"); }); - it('should allow cropping and optimization for valid images', () => { + it("should accept uploaded files", () => { open_upload_dialog(); - cy.get_open_dialog().find('.file-upload-area').attachFile('sample_image.jpg', { - subjectType: 'drag-n-drop', - }); + cy.get_open_dialog().findByRole("button", { name: "Library" }).click(); + cy.findByPlaceholderText("Search by filename or extension").type("example.json"); + cy.get_open_dialog().findAllByText("example.json").first().click(); + cy.intercept("POST", "/api/method/upload_file").as("upload_file"); + cy.get_open_dialog().findByRole("button", { name: "Upload" }).click(); + cy.wait("@upload_file") + .its("response.body.message") + .should("have.property", "file_name", "example.json"); + cy.get(".modal:visible").should("not.exist"); + }); - cy.get_open_dialog().findAllByText('sample_image.jpg').should('exist'); - cy.get_open_dialog().find('.btn-crop').first().click(); - cy.get_open_dialog().findByRole('button', {name: 'Crop'}).click(); - cy.get_open_dialog().findAllByRole('checkbox', {name: 'Optimize'}).should('exist'); - cy.get_open_dialog().findAllByLabelText('Optimize').first().click(); + it("should accept web links", () => { + open_upload_dialog(); - cy.intercept('POST', '/api/method/upload_file').as('upload_file'); - cy.get_open_dialog().findByRole('button', {name: 'Upload'}).click(); - cy.wait('@upload_file').its('response.statusCode').should('eq', 200); - cy.get('.modal:visible').should('not.exist'); + cy.get_open_dialog().findByRole("button", { name: "Link" }).click(); + cy.get_open_dialog() + .findByPlaceholderText("Attach a web link") + .type("https://github.com", { delay: 100, force: true }); + cy.intercept("POST", "/api/method/upload_file").as("upload_file"); + cy.get_open_dialog().findByRole("button", { name: "Upload" }).click(); + cy.wait("@upload_file") + .its("response.body.message") + .should("have.property", "file_url", "https://github.com"); + cy.get(".modal:visible").should("not.exist"); + }); + + it("should allow cropping and optimization for valid images", () => { + open_upload_dialog(); + + cy.get_open_dialog() + .find(".file-upload-area") + .selectFile("cypress/fixtures/sample_image.jpg", { + action: "drag-drop", + }); + + cy.get_open_dialog().findAllByText("sample_image.jpg").should("exist"); + cy.get_open_dialog().find(".btn-crop").first().click(); + cy.get_open_dialog().findByRole("button", { name: "Crop" }).click(); + cy.get_open_dialog().findAllByRole("checkbox", { name: "Optimize" }).should("exist"); + cy.get_open_dialog().findAllByLabelText("Optimize").first().click(); + + cy.intercept("POST", "/api/method/upload_file").as("upload_file"); + cy.get_open_dialog().findByRole("button", { name: "Upload" }).click(); + cy.wait("@upload_file").its("response.statusCode").should("eq", 200); + cy.get(".modal:visible").should("not.exist"); }); }); diff --git a/cypress/integration/first_day_of_the_week.js b/cypress/integration/first_day_of_the_week.js index 1e65b78990..784c068f01 100644 --- a/cypress/integration/first_day_of_the_week.js +++ b/cypress/integration/first_day_of_the_week.js @@ -4,42 +4,48 @@ context("First Day of the Week", () => { }); beforeEach(() => { - cy.visit('/app/system-settings'); - cy.findByText('Date and Number Format').click(); + cy.visit("/app/system-settings"); + cy.findByText("Date and Number Format").click(); }); it("Date control starts with same day as selected in System Settings", () => { - cy.intercept('POST', '/api/method/frappe.core.doctype.system_settings.system_settings.load').as("load_settings"); - cy.fill_field('first_day_of_the_week', 'Tuesday', 'Select'); - cy.findByRole('button', {name: 'Save'}).click(); + cy.intercept( + "POST", + "/api/method/frappe.core.doctype.system_settings.system_settings.load" + ).as("load_settings"); + cy.fill_field("first_day_of_the_week", "Tuesday", "Select"); + cy.findByRole("button", { name: "Save" }).click(); cy.wait("@load_settings"); cy.dialog({ - title: 'Date', + title: "Date", fields: [ { - label: 'Date', - fieldname: 'date', - fieldtype: 'Date' - } - ] + label: "Date", + fieldname: "date", + fieldtype: "Date", + }, + ], }); - cy.get_field('date').click(); - cy.get('.datepicker--day-name').eq(0).should('have.text', 'Tu'); + cy.get_field("date").click(); + cy.get(".datepicker--day-name").eq(0).should("have.text", "Tu"); }); it("Calendar view starts with same day as selected in System Settings", () => { - cy.intercept('POST', '/api/method/frappe.core.doctype.system_settings.system_settings.load').as("load_settings"); - cy.fill_field('first_day_of_the_week', 'Monday', 'Select'); - cy.findByRole('button', {name: 'Save'}).click(); + cy.intercept( + "POST", + "/api/method/frappe.core.doctype.system_settings.system_settings.load" + ).as("load_settings"); + cy.fill_field("first_day_of_the_week", "Monday", "Select"); + cy.findByRole("button", { name: "Save" }).click(); cy.wait("@load_settings"); cy.visit("app/todo/view/calendar/default"); - cy.get('.fc-day-header > span').eq(0).should('have.text', 'Mon'); + cy.get(".fc-day-header > span").eq(0).should("have.text", "Mon"); }); after(() => { - cy.visit('/app/system-settings'); - cy.findByText('Date and Number Format').click(); - cy.fill_field('first_day_of_the_week', 'Sunday', 'Select'); - cy.findByRole('button', {name: 'Save'}).click(); + cy.visit("/app/system-settings"); + cy.findByText("Date and Number Format").click(); + cy.fill_field("first_day_of_the_week", "Sunday", "Select"); + cy.findByRole("button", { name: "Save" }).click(); }); -}); \ No newline at end of file +}); diff --git a/cypress/integration/folder_navigation.js b/cypress/integration/folder_navigation.js index 484419b4aa..ba65454ef6 100644 --- a/cypress/integration/folder_navigation.js +++ b/cypress/integration/folder_navigation.js @@ -1,79 +1,96 @@ -context('Folder Navigation', () => { +context("Folder Navigation", () => { before(() => { - cy.visit('/login'); + cy.visit("/login"); cy.login(); - cy.visit('/app/file'); + cy.visit("/app/file"); }); - it('Adding Folders', () => { + it("Adding Folders", () => { //Adding filter to go into the home folder - cy.get('.filter-selector > .btn').findByText('1 filter').click(); - cy.findByRole('button', {name: 'Clear Filters'}).click(); - cy.get('.filter-action-buttons > .text-muted').findByText('+ Add a Filter').click(); - cy.get('.fieldname-select-area > .awesomplete > .form-control').type('Fol{enter}'); - cy.get('.filter-field > .form-group > .link-field > .awesomplete > .input-with-feedback').type('Home{enter}'); - cy.get('.filter-action-buttons > div > .btn-primary').findByText('Apply Filters').click(); + cy.get(".filter-x-button").click(); + cy.click_filter_button(); + cy.get(".filter-action-buttons > .text-muted").findByText("+ Add a Filter").click(); + cy.get(".fieldname-select-area > .awesomplete > .form-control:last").type("Fol{enter}"); + cy.get( + ".filter-field > .form-group > .link-field > .awesomplete > .input-with-feedback" + ).type("Home{enter}"); + cy.get(".filter-action-buttons > div > .btn-primary").findByText("Apply Filters").click(); //Adding folder (Test Folder) cy.click_menu_button("New Folder"); - cy.fill_field('value', 'Test Folder'); - cy.click_modal_primary_button('Create'); + cy.fill_field("value", "Test Folder"); + cy.click_modal_primary_button("Create"); }); - it('Navigating the nested folders, checking if the URL formed is correct, checking if the added content in the child folder is correct', () => { + it("Navigating the nested folders, checking if the URL formed is correct, checking if the added content in the child folder is correct", () => { //Navigating inside the Attachments folder + cy.wait(500); cy.get('[title="Attachments"] > span').click(); //To check if the URL formed after visiting the attachments folder is correct - cy.location('pathname').should('eq', '/app/file/view/home/Attachments'); - cy.visit('/app/file/view/home/Attachments'); + cy.location("pathname").should("eq", "/app/file/view/home/Attachments"); + cy.visit("/app/file/view/home/Attachments"); //Adding folder inside the attachments folder cy.click_menu_button("New Folder"); - cy.fill_field('value', 'Test Folder'); - cy.click_modal_primary_button('Create'); + cy.fill_field("value", "Test Folder"); + cy.click_modal_primary_button("Create"); //Navigating inside the added folder in the Attachments folder + cy.wait(500); cy.get('[title="Test Folder"] > span').click(); //To check if the URL is correct after visiting the Test Folder - cy.location('pathname').should('eq', '/app/file/view/home/Attachments/Test%20Folder'); - cy.visit('/app/file/view/home/Attachments/Test%20Folder'); + cy.location("pathname").should("eq", "/app/file/view/home/Attachments/Test%20Folder"); + cy.visit("/app/file/view/home/Attachments/Test%20Folder"); //Adding a file inside the Test Folder - cy.findByRole('button', {name: 'Add File'}).eq(0).click({force: true}); - cy.get('.file-uploader').findByText('Link').click(); - cy.get('.input-group > .form-control').type('https://wallpaperplay.com/walls/full/8/2/b/72402.jpg'); - cy.click_modal_primary_button('Upload'); + cy.findByRole("button", { name: "Add File" }).eq(0).click({ force: true }); + cy.get(".file-uploader").findByText("Link").click(); + cy.get(".input-group > input.form-control:visible").as("upload_input"); + cy.get("@upload_input").type("https://wallpaperplay.com/walls/full/8/2/b/72402.jpg", { + waitForAnimations: false, + parseSpecialCharSequences: false, + force: true, + delay: 100, + }); + cy.click_modal_primary_button("Upload"); //To check if the added file is present in the Test Folder - cy.get('span.level-item > span').should('contain', 'Test Folder'); - cy.get('.list-row-container').eq(0).should('contain.text', '72402.jpg'); - cy.get('.list-row-checkbox').eq(0).click(); + cy.visit("/app/file/view/home/Attachments"); + cy.wait(500); + cy.get("span.level-item > a > span").should("contain", "Test Folder"); + cy.visit("/app/file/view/home/Attachments/Test%20Folder"); + + cy.wait(500); + cy.get(".list-row-container").eq(0).should("contain.text", "72402.jpg"); + cy.get(".list-row-checkbox").eq(0).click(); cy.intercept({ - method: 'POST', - url: 'api/method/frappe.desk.reportview.delete_items' - }).as('file_deleted'); + method: "POST", + url: "api/method/frappe.desk.reportview.delete_items", + }).as("file_deleted"); //Deleting the added file from the Test folder cy.click_action_button("Delete"); - cy.click_modal_primary_button('Yes'); - cy.wait('@file_deleted'); + cy.click_modal_primary_button("Yes"); + cy.wait("@file_deleted"); //Deleting the Test Folder - cy.visit('/app/file/view/home/Attachments'); - cy.get('.list-row-checkbox').eq(0).click(); + cy.visit("/app/file/view/home/Attachments"); + cy.get(".list-row-checkbox").eq(0).click(); cy.click_action_button("Delete"); - cy.click_modal_primary_button('Yes'); - cy.wait('@file_deleted'); + cy.click_modal_primary_button("Yes"); + cy.wait("@file_deleted"); }); - it('Deleting Test Folder from the home', () => { - //Deleting the Test Folder added in the home directory - cy.visit('/app/file/view/home'); - cy.get('.level-left > .list-subject > .file-select >.list-row-checkbox').eq(0).click({force: true, delay: 500}); + it("Deleting Test Folder from the home", () => { + //Deleting the Test Folder added in the home directory + cy.visit("/app/file/view/home"); + cy.get(".level-left > .list-subject > .file-select >.list-row-checkbox") + .eq(0) + .click({ force: true, delay: 500 }); cy.click_action_button("Delete"); - cy.click_modal_primary_button('Yes'); + cy.click_modal_primary_button("Yes"); }); }); diff --git a/cypress/integration/form.js b/cypress/integration/form.js index b395ff77b2..8186647a14 100644 --- a/cypress/integration/form.js +++ b/cypress/integration/form.js @@ -1,98 +1,233 @@ -context('Form', () => { +const jump_to_field = (field_label) => { + cy.get("body") + .type("{esc}") // lose focus if any + .type("{ctrl+j}") // jump to field + .type(field_label) + .wait(500) + .type("{enter}") + .wait(200) + .type("{enter}") + .wait(500); +}; + +const type_value = (value) => { + cy.focused().clear().type(value).type("{esc}"); +}; + +context("Form", () => { before(() => { cy.login(); - cy.visit('/app/website'); - return cy.window().its('frappe').then(frappe => { - return frappe.call("frappe.tests.ui_test_helpers.create_contact_records"); - }); + cy.visit("/app/website"); + return cy + .window() + .its("frappe") + .then((frappe) => { + return frappe.call("frappe.tests.ui_test_helpers.create_contact_records"); + }); }); - it('create a new form', () => { - cy.visit('/app/todo/new'); - cy.get_field('description', 'Text Editor').type('this is a test todo', {force: true}).wait(200); - cy.get('.page-title').should('contain', 'Not Saved'); + + beforeEach(() => { + cy.login(); + cy.visit("/app/website"); + }); + + it("create a new form", () => { + cy.visit("/app/todo/new"); + cy.get_field("description", "Text Editor") + .type("this is a test todo", { force: true }) + .wait(200); + cy.get(".page-title").should("contain", "Not Saved"); cy.intercept({ - method: 'POST', - url: 'api/method/frappe.desk.form.save.savedocs' - }).as('form_save'); - cy.get('.primary-action').click(); - cy.wait('@form_save').its('response.statusCode').should('eq', 200); + method: "POST", + url: "api/method/frappe.desk.form.save.savedocs", + }).as("form_save"); + cy.get(".primary-action").click(); + cy.wait("@form_save").its("response.statusCode").should("eq", 200); - cy.go_to_list('ToDo'); - cy.clear_filters() - cy.get('.page-head').findByTitle('To Do').should('exist'); - cy.get('.list-row').should('contain', 'this is a test todo'); + cy.go_to_list("ToDo"); + cy.clear_filters(); + cy.get(".page-head").findByTitle("To Do").should("exist"); + cy.get(".list-row").should("contain", "this is a test todo"); }); - it('navigates between documents with child table list filters applied', () => { - cy.visit('/app/contact'); + it("navigates between documents with child table list filters applied", () => { + cy.visit("/app/contact"); cy.clear_filters(); - cy.get('.standard-filter-section [data-fieldname="name"] input').type('Test Form Contact 3').blur(); - cy.click_listview_row_item_with_text('Test Form Contact 3'); + cy.get('.standard-filter-section [data-fieldname="name"] input') + .type("Test Form Contact 3") + .blur(); + cy.click_listview_row_item_with_text("Test Form Contact 3"); - cy.get('#page-Contact .page-head').findByTitle('Test Form Contact 3').should('exist'); - cy.get('.prev-doc').should('be.visible').click(); - cy.get('.msgprint-dialog .modal-body').contains('No further records').should('be.visible'); + cy.get("#page-Contact .page-head").findByTitle("Test Form Contact 3").should("exist"); + cy.get(".prev-doc").should("be.visible").click(); + cy.get(".msgprint-dialog .modal-body").contains("No further records").should("be.visible"); cy.hide_dialog(); - cy.get('#page-Contact .page-head').findByTitle('Test Form Contact 3').should('exist'); - cy.get('.next-doc').should('be.visible').click(); - cy.get('.msgprint-dialog .modal-body').contains('No further records').should('be.visible'); + cy.get("#page-Contact .page-head").findByTitle("Test Form Contact 3").should("exist"); + cy.get(".next-doc").should("be.visible").click(); + cy.get(".msgprint-dialog .modal-body").contains("No further records").should("be.visible"); cy.hide_dialog(); - cy.get('#page-Contact .page-head').findByTitle('Test Form Contact 3').should('exist'); + cy.get("#page-Contact .page-head").findByTitle("Test Form Contact 3").should("exist"); // clear filters - cy.visit('/app/contact'); + cy.visit("/app/contact"); cy.clear_filters(); }); - it('validates behaviour of Data options validations in child table', () => { + it("validates behaviour of Data options validations in child table", () => { // test email validations for set_invalid controller - let website_input = 'website.in'; - let valid_email = 'user@email.com'; - let expectBackgroundColor = 'rgb(255, 245, 245)'; + let website_input = "website.in"; + let valid_email = "user@email.com"; + let expectBackgroundColor = "rgb(255, 245, 245)"; - cy.visit('/app/contact/new'); - cy.get('.frappe-control[data-fieldname="email_ids"]').as('table'); - cy.get('@table').find('button.grid-add-row').click(); - cy.get('@table').find('button.grid-add-row').click(); - cy.get('@table').find('[data-idx="1"]').as('row1'); - cy.get('@table').find('[data-idx="2"]').as('row2'); - cy.get('@row1').click(); - cy.get('@row1').find('input.input-with-feedback.form-control').as('email_input1'); + cy.visit("/app/contact/new"); + cy.get('.frappe-control[data-fieldname="email_ids"]').as("table"); + cy.get("@table").find("button.grid-add-row").click(); + cy.get("@table").find("button.grid-add-row").click(); + cy.get("@table").find('[data-idx="1"]').as("row1"); + cy.get("@table").find('[data-idx="2"]').as("row2"); + cy.get("@row1").click(); + cy.get("@row1").find("input.input-with-feedback.form-control").as("email_input1"); - cy.get('@email_input1').type(website_input, { waitForAnimations: false }); - cy.fill_field('company_name', 'Test Company'); + cy.get("@email_input1").type(website_input, { waitForAnimations: false }); + cy.fill_field("company_name", "Test Company"); - cy.get('@row2').click(); - cy.get('@row2').find('input.input-with-feedback.form-control').as('email_input2'); - cy.get('@email_input2').type(valid_email, { waitForAnimations: false }); + cy.get("@row2").click(); + cy.get("@row2").find("input.input-with-feedback.form-control").as("email_input2"); + cy.get("@email_input2").type(valid_email, { waitForAnimations: false }); - cy.get('@row1').click(); - cy.get('@email_input1').should($div => { + cy.get("@row1").click(); + cy.get("@email_input1").should(($div) => { const style = window.getComputedStyle($div[0]); expect(style.backgroundColor).to.equal(expectBackgroundColor); }); - cy.get('@email_input1').should('have.class', 'invalid'); + cy.get("@email_input1").should("have.class", "invalid"); - cy.get('@row2').click(); - cy.get('@email_input2').should('not.have.class', 'invalid'); + cy.get("@row2").click(); + cy.get("@email_input2").should("not.have.class", "invalid"); }); - it('Shows version conflict warning', { scrollBehavior: false }, () => { - cy.visit('/app/todo'); + it("Shows version conflict warning", { scrollBehavior: false }, () => { + cy.visit("/app/todo"); - cy.insert_doc("ToDo", {"description": "old"}).then(doc => { + cy.insert_doc("ToDo", { description: "old" }).then((doc) => { cy.visit(`/app/todo/${doc.name}`); // make form dirty cy.fill_field("status", "Cancelled", "Select"); // update doc using api - simulating parallel change by another user - cy.update_doc("ToDo", doc.name, {"status": "Closed"}).then(() => { - cy.findByRole("button", {name: "Refresh"}).click(); + cy.update_doc("ToDo", doc.name, { status: "Closed" }).then(() => { + cy.findByRole("button", { name: "Refresh" }).click(); cy.get_field("status", "Select").should("have.value", "Closed"); - }) - }) + }); + }); + }); + + it("Jump to field in collapsed section", { scrollBehavior: false }, () => { + cy.new_form("User"); + + jump_to_field("Location"); // this is in collapsed section + type_value("Bermuda"); + + cy.get_field("location").should("have.value", "Bermuda"); + }); + + it("let user undo/redo field value changes", { scrollBehavior: false }, () => { + const undo = () => cy.get("body").type("{esc}").type("{ctrl+z}").wait(500); + const redo = () => cy.get("body").type("{esc}").type("{ctrl+y}").wait(500); + + cy.new_form("User"); + + jump_to_field("Email"); + type_value("admin@example.com"); + + jump_to_field("Username"); + type_value("admin42"); + + jump_to_field("Send Welcome Email"); + cy.focused().uncheck(); + + // make a mistake + jump_to_field("Username"); + type_value("admin24"); + + // undo behaviour + undo(); + cy.get_field("username").should("have.value", "admin42"); + + // redo behaviour + redo(); + cy.get_field("username").should("have.value", "admin24"); + + // undo everything & redo everything, ensure same values at the end + undo(); + undo(); + undo(); + undo(); + redo(); + redo(); + redo(); + redo(); + + cy.compare_document({ + username: "admin24", + email: "admin@example.com", + send_welcome_email: 0, + }); + }); + + it("update docfield property using set_df_property in child table", () => { + cy.visit("/app/contact/Test Form Contact 1"); + cy.window() + .its("cur_frm") + .then((frm) => { + cy.get('.frappe-control[data-fieldname="phone_nos"]').as("table"); + + // set property before form_render event of child table + cy.get("@table") + .find('[data-idx="1"]') + .invoke("attr", "data-name") + .then((cdn) => { + frm.set_df_property( + "phone_nos", + "hidden", + 1, + "Contact Phone", + "is_primary_phone", + cdn + ); + }); + + cy.get("@table").find('[data-idx="1"] .edit-grid-row').click(); + cy.get(".grid-row-open").as("table-form"); + cy.get("@table-form") + .find('.frappe-control[data-fieldname="is_primary_phone"]') + .should("be.hidden"); + cy.get("@table-form").find(".grid-footer-toolbar").click(); + + // set property on form_render event of child table + cy.get("@table").find('[data-idx="1"] .edit-grid-row').click(); + cy.get("@table") + .find('[data-idx="1"]') + .invoke("attr", "data-name") + .then((cdn) => { + frm.set_df_property( + "phone_nos", + "hidden", + 0, + "Contact Phone", + "is_primary_phone", + cdn + ); + }); + + cy.get(".grid-row-open").as("table-form"); + cy.get("@table-form") + .find('.frappe-control[data-fieldname="is_primary_phone"]') + .should("be.visible"); + cy.get("@table-form").find(".grid-footer-toolbar").click(); + }); }); }); diff --git a/cypress/integration/form_builder.js b/cypress/integration/form_builder.js new file mode 100644 index 0000000000..968f6aaaf0 --- /dev/null +++ b/cypress/integration/form_builder.js @@ -0,0 +1,301 @@ +import form_builder_doctype from "../fixtures/form_builder_doctype"; +const doctype_name = form_builder_doctype.name; +context("Form Builder", () => { + before(() => { + cy.login(); + cy.visit("/app"); + return cy.insert_doc("DocType", form_builder_doctype, true); + }); + + it("Open Form Builder for Web Form Doctype/Customize Form", () => { + // doctype + cy.visit("/app/form-builder/Web Form"); + cy.get(".form-builder-container").should("exist"); + + // customize form + cy.visit("/app/form-builder/Web Form/customize"); + cy.get(".form-builder-container").should("exist"); + }); + + it("Change Doctype using page title dialog", () => { + cy.intercept("POST", "/api/method/frappe.desk.search.search_link").as("search_link"); + + cy.visit(`/app/form-builder/Web Form`); + cy.get(".form-builder-container").should("exist"); + + cy.get(".page-title").click(); + + cy.get(".frappe-control[data-fieldname='doctype'] input").click().as("input"); + cy.get("@input").type("{rightArrow} Field", { delay: 200 }); + cy.wait("@search_link"); + cy.get("@input").type("{enter}").blur(); + + cy.click_modal_primary_button("Change"); + + cy.get(".page-title .title-text").should("have.text", "Web Form Field"); + }); + + it("Save without change, check form dirty and reset changes", () => { + cy.visit(`/app/form-builder/${doctype_name}`); + + // Save without change + cy.click_doc_primary_button("Save"); + cy.get(".desk-alert.orange .alert-message").should("have.text", "No changes to save"); + + // Check form dirty + cy.get(".tab-content.active .section-columns-container:first .column:first .field:first") + .find("div[title='Double click to edit label']") + .dblclick() + .type("Dirty"); + cy.get(".title-area .indicator-pill.orange").should("have.text", "Not Saved"); + + // Reset changes + cy.get(".page-actions .custom-actions .btn").contains("Reset Changes").click(); + cy.get(".title-area .indicator-pill.orange").should("not.exist"); + }); + + it("Add empty section and save", () => { + cy.visit(`/app/form-builder/${doctype_name}`); + + let first_section = ".tab-content.active .form-section-container:first"; + + // add new section + cy.get(first_section).click(15, 10); + cy.get(first_section).find(".section-actions button:first").click(); + + // save + cy.click_doc_primary_button("Save"); + cy.get(".tab-content.active .form-section-container").should("have.length", 1); + }); + + it("Add Table field and check if columns are rendered", () => { + cy.intercept("POST", "/api/method/frappe.desk.search.search_link").as("search_link"); + + cy.visit(`/app/form-builder/${doctype_name}`); + + let first_field = + ".tab-content.active .section-columns-container:first .column:first .field:first"; + + cy.get(".fields-container .field[title='Table']").drag(first_field, { + target: { x: 100, y: 10 }, + }); + + // save + cy.click_doc_primary_button("Save"); + + // Validate if options is not set + cy.get_open_dialog().find(".msgprint").should("contain", "Options is required"); + cy.hide_dialog(); + + cy.get(first_field).click({ force: true }); + + cy.get(".sidebar-container .frappe-control[data-fieldname='options'] input") + .click() + .as("input"); + cy.get("@input").clear({ force: true }).type("Web Form Field", { delay: 200 }); + cy.wait("@search_link"); + cy.get("@input").type("{enter}").blur(); + + cy.get(first_field) + .find(".table-controls .table-column") + .contains("Field") + .should("exist"); + cy.get(first_field) + .find(".table-controls .table-column") + .contains("Fieldtype") + .should("exist"); + + // validate In List View + cy.get(".sidebar-container .field label .label-area").contains("In List View").click(); + + // save + cy.click_doc_primary_button("Save"); + + cy.get_open_dialog().find(".msgprint").should("contain", "In List View"); + cy.hide_dialog(); + + cy.get(first_field).click({ force: true }); + cy.get(".sidebar-container .field label .label-area").contains("In List View").click(); + + // validate In Global Search + cy.get(".sidebar-container .field label .label-area").contains("In Global Search").click(); + // save + cy.click_doc_primary_button("Save"); + + cy.get_open_dialog().find(".msgprint").should("contain", "In Global Search"); + }); + + it("Drag Field/Column/Section & Tab", () => { + cy.visit(`/app/form-builder/${doctype_name}`); + + let first_column = ".tab-content.active .section-columns-container:first .column:first"; + let first_field = first_column + " .field:first"; + let label = "div[title='Double click to edit label'] span:first"; + + // drag first tab to second position + cy.get(".tabs .tab:first").drag(".tabs .tab:nth-child(2)", { + target: { x: 10, y: 10 }, + force: true, + }); + cy.get(".tabs .tab:first").find(label).should("have.text", "Tab 2"); + + cy.get(".tabs .tab:first").click(); + cy.get(".sidebar-container .tab:first").click(); + + // drag check field to first column + cy.get(".fields-container .field[title='Check']").drag(first_field, { + target: { x: 100, y: 10 }, + }); + cy.get(first_column).find(".field").should("have.length", 3); + + cy.get(first_field) + .find("div[title='Double click to edit label']") + .dblclick() + .type("Test Check{enter}"); + cy.get(first_field).find(label).should("have.text", "Test Check"); + + // drag the first field to second position + cy.get(first_field).drag(first_column + " .field:nth-child(2)", { + target: { x: 100, y: 10 }, + }); + cy.get(first_field).find(label).should("have.text", "Data"); + + // drag first column to second position + cy.get(first_column).click().wait(200); + cy.get(first_column) + .find(".column-actions") + .drag(".section-columns-container:first .column:last", { + target: { x: 100, y: 10 }, + force: true, + }); + cy.get(first_field).find(label).should("have.text", "Data 1"); + + let first_section = ".tab-content.active .form-section-container:first"; + + // drag first section to second position + cy.get(first_section).click().wait(200); + cy.get(first_section) + .find(".section-header") + .drag(".form-section-container:nth-child(2)", { + target: { x: 100, y: 10 }, + force: true, + }); + cy.get(first_field).find(label).should("have.text", "Data 2"); + }); + + it("Add New Tab/Section/Column to Form", () => { + cy.visit(`/app/form-builder/${doctype_name}`); + + let first_section = ".tab-content.active .form-section-container:first"; + + // add new tab + cy.get(".tab-header").realHover().find(".tab-actions .new-tab-btn").click(); + cy.get(".tabs .tab").should("have.length", 3); + + // add new section + cy.get(first_section).click(15, 10); + cy.get(first_section).find(".section-actions button:first").click(); + cy.get(".tab-content.active .form-section-container").should("have.length", 2); + + // add new column + cy.get(first_section).find(".column:first").click(15, 10); + cy.get(first_section).find(".column:first .column-actions button:first").click(); + cy.get(first_section).find(".column").should("have.length", 3); + }); + + it("Remove Tab/Section/Column", () => { + let first_section = ".tab-content.active .form-section-container:first"; + + // remove column + cy.get(first_section).find(".column:first").click(15, 10); + cy.get(first_section).find(".column:first .column-actions button:last").click(); + cy.get(first_section).find(".column").should("have.length", 2); + + // remove section + cy.get(first_section).click(15, 10); + cy.get(first_section).find(".section-actions button:last").click(); + cy.get(".tab-content.active .form-section-container").should("have.length", 1); + + // remove tab + cy.get(".tab-header").realHover().find(".tab-actions .remove-tab-btn").click(); + cy.get(".tabs .tab").should("have.length", 2); + }); + + it("Update Title field Label to New Title through Customize Form", () => { + cy.visit(`/app/form-builder/${doctype_name}`); + + let first_field = + ".tab-content.active .section-columns-container:first .column:first .field:first"; + + cy.get(first_field) + .find("div[title='Double click to edit label']") + .dblclick() + .type("{selectall}New Title"); + + cy.findByRole("button", { name: "Save" }).click({ force: true }); + + cy.visit("/app/form-builder-doctype/new"); + cy.get("[data-fieldname='data3'] .clearfix label").should("have.text", "New Title"); + }); + + it("Validate Duplicate Name & reqd + hidden without default logic", () => { + cy.visit(`/app/form-builder/${doctype_name}`); + + let first_field = + ".tab-content.active .section-columns-container:first .column:first .field:first"; + + cy.get(".fields-container .field[title='Data']").drag(first_field, { + target: { x: 100, y: 10 }, + }); + + cy.get(first_field).click(); + + // validate duplicate name + cy.get(".sidebar-container .frappe-control[data-fieldname='fieldname'] input") + .click() + .as("input"); + cy.get("@input").clear({ force: true }).type("data3"); + + cy.click_doc_primary_button("Save"); + cy.get_open_dialog().find(".msgprint").should("contain", "appears multiple times"); + cy.hide_dialog(); + cy.get(first_field).click(); + cy.get("@input").clear({ force: true }); + + // validate reqd + hidden without default + cy.get(".sidebar-container .field label .label-area").contains("Mandatory").click(); + cy.get(".sidebar-container .field label .label-area").contains("Hidden").click(); + + // save + cy.click_doc_primary_button("Save"); + + cy.get_open_dialog() + .find(".msgprint") + .should("contain", "cannot be hidden and mandatory without any default value"); + }); + + it("Undo/Redo", () => { + cy.visit(`/app/form-builder/${doctype_name}`); + + // click on second tab + cy.get(".tabs .tab:last").click(); + + let first_column = ".tab-content.active .section-columns-container:first .column:first"; + let first_field = first_column + " .field:first"; + let label = "div[title='Double click to edit label'] span:first"; + + // drag the first field to second position + cy.get(first_field).drag(first_column + " .field:nth-child(2)", { + target: { x: 100, y: 10 }, + }); + cy.get(first_field).find(label).should("have.text", "Check"); + + // undo + cy.get("body").type("{ctrl}z"); + cy.get(first_field).find(label).should("have.text", "Data"); + + // redo + cy.get("body").type("{ctrl}{shift}z"); + cy.get(first_field).find(label).should("have.text", "Check"); + }); +}); diff --git a/cypress/integration/form_tab_break.js b/cypress/integration/form_tab_break.js index 45c3c92084..91695cb143 100644 --- a/cypress/integration/form_tab_break.js +++ b/cypress/integration/form_tab_break.js @@ -1,31 +1,30 @@ -import doctype_with_tab_break from '../fixtures/doctype_with_tab_break'; +import doctype_with_tab_break from "../fixtures/doctype_with_tab_break"; const doctype_name = doctype_with_tab_break.name; context("Form Tab Break", () => { before(() => { cy.login(); - cy.visit('/app/website'); - return cy.insert_doc('DocType', doctype_with_tab_break, true); + cy.visit("/app/website"); + return cy.insert_doc("DocType", doctype_with_tab_break, true); }); it("Should switch tab and open correct tabs on validation error", () => { cy.new_form(doctype_name); // test tab switch - cy.findByRole("tab", {name: "Tab 2"}).click(); + cy.findByRole("tab", { name: "Tab 2" }).click(); cy.findByText("Phone"); - cy.findByRole("tab", {name: "Details"}).click(); + cy.findByRole("tab", { name: "Details" }).click(); cy.findByText("Name"); // form should switch to the tab with un-filled mandatory field cy.fill_field("username", "Test"); - cy.findByRole("button", {name: "Save"}).click(); + cy.findByRole("button", { name: "Save" }).click(); cy.findByText("Missing Fields"); cy.hide_dialog(); cy.findByText("Phone"); cy.fill_field("phone", "12345678"); - cy.findByRole("button", {name: "Save"}).click(); + cy.findByRole("button", { name: "Save" }).click(); // After save, first tab should have dashboard cy.get(".form-tabs > .nav-item").eq(0).click(); cy.findByText("Connections"); - }); -}); \ No newline at end of file +}); diff --git a/cypress/integration/form_tour.js b/cypress/integration/form_tour.js index 507a07ab1a..f4ae0dbb6d 100644 --- a/cypress/integration/form_tour.js +++ b/cypress/integration/form_tour.js @@ -1,88 +1,94 @@ -context.skip('Form Tour', () => { +context.skip("Form Tour", () => { before(() => { cy.login(); - cy.visit('/app'); - return cy.window().its('frappe').then(frappe => { - return frappe.call("frappe.tests.ui_test_helpers.create_form_tour"); - }); + cy.visit("/app"); + return cy + .window() + .its("frappe") + .then((frappe) => { + return frappe.call("frappe.tests.ui_test_helpers.create_form_tour"); + }); }); const open_test_form_tour = () => { - cy.visit('/app/form-tour/Test Form Tour'); - cy.findByRole('button', {name: 'Show Tour'}).should('be.visible').as('show_tour'); - cy.get('@show_tour').click(); + cy.visit("/app/form-tour/Test Form Tour"); + cy.findByRole("button", { name: "Show Tour" }).should("be.visible").as("show_tour"); + cy.get("@show_tour").click(); cy.wait(500); - cy.url().should('include', '/app/contact'); + cy.url().should("include", "/app/contact"); }; - it('jump to a form tour', open_test_form_tour); + it("jump to a form tour", open_test_form_tour); - it('navigates a form tour', () => { + it("navigates a form tour", () => { open_test_form_tour(); - cy.get('.frappe-driver').should('be.visible'); - cy.get('.frappe-control[data-fieldname="first_name"]').as('first_name'); - cy.get('@first_name').should('have.class', 'driver-highlighted-element'); - cy.get('.frappe-driver').findByRole('button', {name: 'Next'}).as('next_btn'); + cy.get(".frappe-driver").should("be.visible"); + cy.get('.frappe-control[data-fieldname="first_name"]').as("first_name"); + cy.get("@first_name").should("have.class", "driver-highlighted-element"); + cy.get(".frappe-driver").findByRole("button", { name: "Next" }).as("next_btn"); // next btn shouldn't move to next step, if first name is not entered - cy.get('@next_btn').click(); + cy.get("@next_btn").click(); cy.wait(500); - cy.get('@first_name').should('have.class', 'driver-highlighted-element'); + cy.get("@first_name").should("have.class", "driver-highlighted-element"); // after filling the field, next step should be highlighted - cy.fill_field('first_name', 'Test Name', 'Data'); + cy.fill_field("first_name", "Test Name", "Data"); cy.wait(500); - cy.get('@next_btn').click(); + cy.get("@next_btn").click(); cy.wait(500); // assert field is highlighted - cy.get('.frappe-control[data-fieldname="last_name"]').as('last_name'); - cy.get('@last_name').should('have.class', 'driver-highlighted-element'); + cy.get('.frappe-control[data-fieldname="last_name"]').as("last_name"); + cy.get("@last_name").should("have.class", "driver-highlighted-element"); // after filling the field, next step should be highlighted - cy.fill_field('last_name', 'Test Last Name', 'Data'); + cy.fill_field("last_name", "Test Last Name", "Data"); cy.wait(500); - cy.get('@next_btn').click(); + cy.get("@next_btn").click(); cy.wait(500); // assert field is highlighted - cy.get('.frappe-control[data-fieldname="phone_nos"]').as('phone_nos'); - cy.get('@phone_nos').should('have.class', 'driver-highlighted-element'); + cy.get('.frappe-control[data-fieldname="phone_nos"]').as("phone_nos"); + cy.get("@phone_nos").should("have.class", "driver-highlighted-element"); // move to next step cy.wait(500); - cy.get('@next_btn').click(); + cy.get("@next_btn").click(); cy.wait(500); // assert add row btn is highlighted - cy.get('@phone_nos').find('.grid-add-row').as('add_row'); - cy.get('@add_row').should('have.class', 'driver-highlighted-element'); + cy.get("@phone_nos").find(".grid-add-row").as("add_row"); + cy.get("@add_row").should("have.class", "driver-highlighted-element"); // add a row & move to next step cy.wait(500); - cy.get('@add_row').click(); + cy.get("@add_row").click(); cy.wait(500); // assert table field is highlighted - cy.get('.grid-row-open .frappe-control[data-fieldname="phone"]').as('phone'); - cy.get('@phone').should('have.class', 'driver-highlighted-element'); + cy.get('.grid-row-open .frappe-control[data-fieldname="phone"]').as("phone"); + cy.get("@phone").should("have.class", "driver-highlighted-element"); // enter value in a table field - let field = cy.fill_table_field('phone_nos', '1', 'phone', '1234567890'); + let field = cy.fill_table_field("phone_nos", "1", "phone", "1234567890"); field.blur(); // move to collapse row step cy.wait(500); - cy.get('.driver-popover-title').contains('Test Title 4').siblings().get('@next_btn').click(); + cy.get(".driver-popover-title") + .contains("Test Title 4") + .siblings() + .get("@next_btn") + .click(); cy.wait(500); // collapse row - cy.get('.grid-row-open .grid-collapse-row').click(); + cy.get(".grid-row-open .grid-collapse-row").click(); cy.wait(500); // assert save btn is highlighted - cy.get('.primary-action').should('have.class', 'driver-highlighted-element'); + cy.get(".primary-action").should("have.class", "driver-highlighted-element"); cy.wait(500); - cy.get('.frappe-driver').findByRole('button', {name: 'Save'}).should('be.visible'); - + cy.get(".frappe-driver").findByRole("button", { name: "Save" }).should("be.visible"); }); }); diff --git a/cypress/integration/grid.js b/cypress/integration/grid.js index 4fa52712cf..6cf9e8cdc7 100644 --- a/cypress/integration/grid.js +++ b/cypress/integration/grid.js @@ -1,92 +1,114 @@ -context('Grid', () => { +context("Grid", () => { beforeEach(() => { cy.login(); - cy.visit('/app/website'); + cy.visit("/app/website"); }); before(() => { cy.login(); - cy.visit('/app/website'); - return cy.window().its('frappe').then(frappe => { - return frappe.call("frappe.tests.ui_test_helpers.create_contact_phone_nos_records"); - }); + cy.visit("/app/website"); + return cy + .window() + .its("frappe") + .then((frappe) => { + return frappe.call( + "frappe.tests.ui_test_helpers.create_contact_phone_nos_records" + ); + }); }); - it('update docfield property using update_docfield_property', () => { - cy.visit('/app/contact/Test Contact'); - cy.window().its("cur_frm").then(frm => { - cy.get('.frappe-control[data-fieldname="phone_nos"]').as('table'); - let field = frm.get_field("phone_nos"); - field.grid.update_docfield_property("is_primary_phone", "hidden", true); + it("update docfield property using update_docfield_property", () => { + cy.visit("/app/contact/Test Contact"); + cy.window() + .its("cur_frm") + .then((frm) => { + cy.get('.frappe-control[data-fieldname="phone_nos"]').as("table"); + let field = frm.get_field("phone_nos"); + field.grid.update_docfield_property("is_primary_phone", "hidden", true); - cy.get('@table').find('[data-idx="1"] .edit-grid-row').click(); - cy.get('.grid-row-open').as('table-form'); - cy.get('@table-form').find('.frappe-control[data-fieldname="is_primary_phone"]').should("be.hidden"); - cy.get('@table-form').find('.grid-footer-toolbar').click(); + cy.get("@table").find('[data-idx="1"] .edit-grid-row').click(); + cy.get(".grid-row-open").as("table-form"); + cy.get("@table-form") + .find('.frappe-control[data-fieldname="is_primary_phone"]') + .should("be.hidden"); + cy.get("@table-form").find(".grid-footer-toolbar").click(); - cy.get('@table').find('[data-idx="2"] .edit-grid-row').click(); - cy.get('.grid-row-open').as('table-form'); - cy.get('@table-form').find('.frappe-control[data-fieldname="is_primary_phone"]').should("be.hidden"); - cy.get('@table-form').find('.grid-footer-toolbar').click(); - }); + cy.get("@table").find('[data-idx="2"] .edit-grid-row').click(); + cy.get(".grid-row-open").as("table-form"); + cy.get("@table-form") + .find('.frappe-control[data-fieldname="is_primary_phone"]') + .should("be.hidden"); + cy.get("@table-form").find(".grid-footer-toolbar").click(); + }); }); - it('update docfield property using toggle_display', () => { - cy.visit('/app/contact/Test Contact'); - cy.window().its("cur_frm").then(frm => { - cy.get('.frappe-control[data-fieldname="phone_nos"]').as('table'); - let field = frm.get_field("phone_nos"); - field.grid.toggle_display("is_primary_mobile_no", false); + it("update docfield property using toggle_display", () => { + cy.visit("/app/contact/Test Contact"); + cy.window() + .its("cur_frm") + .then((frm) => { + cy.get('.frappe-control[data-fieldname="phone_nos"]').as("table"); + let field = frm.get_field("phone_nos"); + field.grid.toggle_display("is_primary_mobile_no", false); - cy.get('@table').find('[data-idx="1"] .edit-grid-row').click(); - cy.get('.grid-row-open').as('table-form'); - cy.get('@table-form').find('.frappe-control[data-fieldname="is_primary_mobile_no"]').should("be.hidden"); - cy.get('@table-form').find('.grid-footer-toolbar').click(); + cy.get("@table").find('[data-idx="1"] .edit-grid-row').click(); + cy.get(".grid-row-open").as("table-form"); + cy.get("@table-form") + .find('.frappe-control[data-fieldname="is_primary_mobile_no"]') + .should("be.hidden"); + cy.get("@table-form").find(".grid-footer-toolbar").click(); - cy.get('@table').find('[data-idx="2"] .edit-grid-row').click(); - cy.get('.grid-row-open').as('table-form'); - cy.get('@table-form').find('.frappe-control[data-fieldname="is_primary_mobile_no"]').should("be.hidden"); - cy.get('@table-form').find('.grid-footer-toolbar').click(); - }); + cy.get("@table").find('[data-idx="2"] .edit-grid-row').click(); + cy.get(".grid-row-open").as("table-form"); + cy.get("@table-form") + .find('.frappe-control[data-fieldname="is_primary_mobile_no"]') + .should("be.hidden"); + cy.get("@table-form").find(".grid-footer-toolbar").click(); + }); }); - it('update docfield property using toggle_enable', () => { - cy.visit('/app/contact/Test Contact'); - cy.window().its("cur_frm").then(frm => { - cy.get('.frappe-control[data-fieldname="phone_nos"]').as('table'); - let field = frm.get_field("phone_nos"); - field.grid.toggle_enable("phone", false); + it("update docfield property using toggle_enable", () => { + cy.visit("/app/contact/Test Contact"); + cy.window() + .its("cur_frm") + .then((frm) => { + cy.get('.frappe-control[data-fieldname="phone_nos"]').as("table"); + let field = frm.get_field("phone_nos"); + field.grid.toggle_enable("phone", false); + cy.get("@table").find('[data-idx="1"] .edit-grid-row').click(); + cy.get(".grid-row-open").as("table-form"); + cy.get("@table-form") + .find('.frappe-control[data-fieldname="phone"] .control-value') + .should("have.class", "like-disabled-input"); + cy.get("@table-form").find(".grid-footer-toolbar").click(); - cy.get('@table').find('[data-idx="1"] .edit-grid-row').click(); - cy.get('.grid-row-open').as('table-form'); - cy.get('@table-form').find('.frappe-control[data-fieldname="phone"] .control-value').should('have.class', 'like-disabled-input'); - cy.get('@table-form').find('.grid-footer-toolbar').click(); - - cy.get('@table').find('[data-idx="2"] .edit-grid-row').click(); - cy.get('.grid-row-open').as('table-form'); - cy.get('@table-form').find('.frappe-control[data-fieldname="phone"] .control-value').should('have.class', 'like-disabled-input'); - cy.get('@table-form').find('.grid-footer-toolbar').click(); - }); + cy.get("@table").find('[data-idx="2"] .edit-grid-row').click(); + cy.get(".grid-row-open").as("table-form"); + cy.get("@table-form") + .find('.frappe-control[data-fieldname="phone"] .control-value') + .should("have.class", "like-disabled-input"); + cy.get("@table-form").find(".grid-footer-toolbar").click(); + }); }); - it('update docfield property using toggle_reqd', () => { - cy.visit('/app/contact/Test Contact'); - cy.window().its("cur_frm").then(frm => { - cy.get('.frappe-control[data-fieldname="phone_nos"]').as('table'); - let field = frm.get_field("phone_nos"); - field.grid.toggle_reqd("phone", false); + it("update docfield property using toggle_reqd", () => { + cy.visit("/app/contact/Test Contact"); + cy.window() + .its("cur_frm") + .then((frm) => { + cy.get('.frappe-control[data-fieldname="phone_nos"]').as("table"); + let field = frm.get_field("phone_nos"); + field.grid.toggle_reqd("phone", false); - cy.get('@table').find('[data-idx="1"] .edit-grid-row').click(); - cy.get('.grid-row-open').as('table-form'); - cy.get_field("phone").as('phone-field'); - cy.get('@phone-field').focus().clear().wait(500).blur(); - cy.get('@phone-field').should("not.have.class", "has-error"); - cy.get('@table-form').find('.grid-footer-toolbar').click(); + cy.get("@table").find('[data-idx="1"] .edit-grid-row').click(); + cy.get(".grid-row-open").as("table-form"); + cy.get_field("phone").as("phone-field"); + cy.get("@phone-field").focus().clear().wait(500).blur(); + cy.get("@phone-field").should("not.have.class", "has-error"); + cy.get("@table-form").find(".grid-footer-toolbar").click(); - cy.get('@table').find('[data-idx="2"] .edit-grid-row').click(); - cy.get('.grid-row-open').as('table-form'); - cy.get_field("phone").as('phone-field'); - cy.get('@phone-field').focus().clear().wait(500).blur(); - cy.get('@phone-field').should("not.have.class", "has-error"); - cy.get('@table-form').find('.grid-footer-toolbar').click(); - - }); + cy.get("@table").find('[data-idx="2"] .edit-grid-row').click(); + cy.get(".grid-row-open").as("table-form"); + cy.get_field("phone").as("phone-field"); + cy.get("@phone-field").focus().clear().wait(500).blur(); + cy.get("@phone-field").should("not.have.class", "has-error"); + cy.get("@table-form").find(".grid-footer-toolbar").click(); + }); }); }); - diff --git a/cypress/integration/grid_configuration.js b/cypress/integration/grid_configuration.js index 7193d804c2..9112d7023e 100644 --- a/cypress/integration/grid_configuration.js +++ b/cypress/integration/grid_configuration.js @@ -1,23 +1,23 @@ -context('Grid Configuration', () => { +context("Grid Configuration", () => { beforeEach(() => { cy.login(); - cy.visit('/app/doctype/User'); + cy.visit("/app/doctype/User"); }); - it('Set user wise grid settings', () => { + it("Set user wise grid settings", () => { cy.wait(100); - cy.get('.frappe-control[data-fieldname="fields"]').as('table'); - cy.get('@table').find('.icon-sm').click(); + cy.get('.frappe-control[data-fieldname="fields"]').as("table"); + cy.get("@table").find(".icon-sm").click(); cy.wait(100); - cy.get('.frappe-control[data-fieldname="fields_html"]').as('modal'); - cy.get('@modal').find('.add-new-fields').click(); + cy.get('.frappe-control[data-fieldname="fields_html"]').as("modal"); + cy.get("@modal").find(".add-new-fields").click(); cy.wait(100); cy.get('[type="checkbox"][data-unit="read_only"]').check(); - cy.findByRole('button', {name: 'Add'}).click(); + cy.findByRole("button", { name: "Add" }).click(); cy.wait(100); - cy.get('[data-fieldname="options"]').invoke('attr', 'value', '1'); - cy.get('.form-control.column-width[data-fieldname="options"]').trigger('change'); - cy.findByRole('button', {name: 'Update'}).click(); + cy.get('[data-fieldname="options"]').invoke("attr", "value", "1"); + cy.get('.form-control.column-width[data-fieldname="options"]').trigger("change"); + cy.findByRole("button", { name: "Update" }).click(); cy.wait(200); - cy.get('[title="Read Only"').should('be.visible'); + cy.get('[title="Read Only"').should("be.visible"); }); -}); \ No newline at end of file +}); diff --git a/cypress/integration/grid_keyboard_shortcut.js b/cypress/integration/grid_keyboard_shortcut.js index 9cf39165ad..414e822516 100644 --- a/cypress/integration/grid_keyboard_shortcut.js +++ b/cypress/integration/grid_keyboard_shortcut.js @@ -1,40 +1,47 @@ -context('Grid Keyboard Shortcut', () => { +context("Grid Keyboard Shortcut", () => { let total_count = 0; before(() => { cy.login(); }); beforeEach(() => { cy.reload(); - cy.visit('/app/contact/new-contact-1'); + cy.visit("/app/contact/new-contact-1"); cy.get('.frappe-control[data-fieldname="email_ids"]').find(".grid-add-row").click(); }); - it('Insert new row at the end', () => { - cy.add_new_row_in_grid('{ctrl}{shift}{downarrow}', (cy, total_count) => { - cy.get('[data-name="new-contact-email-1"]').should('have.attr', 'data-idx', `${total_count+1}`); - }, total_count); + it("Insert new row at the end", () => { + cy.add_new_row_in_grid( + "{ctrl}{shift}{downarrow}", + (cy, total_count) => { + cy.get('[data-name="new-contact-email-1"]').should( + "have.attr", + "data-idx", + `${total_count + 1}` + ); + }, + total_count + ); }); - it('Insert new row at the top', () => { - cy.add_new_row_in_grid('{ctrl}{shift}{uparrow}', (cy) => { - cy.get('[data-name="new-contact-email-1"]').should('have.attr', 'data-idx', '2'); + it("Insert new row at the top", () => { + cy.add_new_row_in_grid("{ctrl}{shift}{uparrow}", (cy) => { + cy.get('[data-name="new-contact-email-1"]').should("have.attr", "data-idx", "2"); }); }); - it('Insert new row below', () => { - cy.add_new_row_in_grid('{ctrl}{downarrow}', (cy) => { - cy.get('[data-name="new-contact-email-1"]').should('have.attr', 'data-idx', '1'); + it("Insert new row below", () => { + cy.add_new_row_in_grid("{ctrl}{downarrow}", (cy) => { + cy.get('[data-name="new-contact-email-1"]').should("have.attr", "data-idx", "1"); }); }); - it('Insert new row above', () => { - cy.add_new_row_in_grid('{ctrl}{uparrow}', (cy) => { - cy.get('[data-name="new-contact-email-1"]').should('have.attr', 'data-idx', '2'); + it("Insert new row above", () => { + cy.add_new_row_in_grid("{ctrl}{uparrow}", (cy) => { + cy.get('[data-name="new-contact-email-1"]').should("have.attr", "data-idx", "2"); }); }); }); -Cypress.Commands.add('add_new_row_in_grid', (shortcut_keys, callbackFn, total_count) => { - cy.get('.frappe-control[data-fieldname="email_ids"]').as('table'); - cy.get('@table').find('.grid-body [data-fieldname="email_id"]').first().click(); - cy.get('@table').find('.grid-body [data-fieldname="email_id"]') - .first().type(shortcut_keys); +Cypress.Commands.add("add_new_row_in_grid", (shortcut_keys, callbackFn, total_count) => { + cy.get('.frappe-control[data-fieldname="email_ids"]').as("table"); + cy.get("@table").find('.grid-body [data-fieldname="email_id"]').first().click(); + cy.get("@table").find('.grid-body [data-fieldname="email_id"]').first().type(shortcut_keys); callbackFn(cy, total_count); -}); \ No newline at end of file +}); diff --git a/cypress/integration/grid_pagination.js b/cypress/integration/grid_pagination.js index 84b3320282..097f2a5cdc 100644 --- a/cypress/integration/grid_pagination.js +++ b/cypress/integration/grid_pagination.js @@ -1,65 +1,73 @@ -context('Grid Pagination', () => { +context("Grid Pagination", () => { beforeEach(() => { cy.login(); - cy.visit('/app/website'); + cy.visit("/app/website"); }); before(() => { cy.login(); - cy.visit('/app/website'); - return cy.window().its('frappe').then(frappe => { - return frappe.call("frappe.tests.ui_test_helpers.create_contact_phone_nos_records"); - }); + cy.visit("/app/website"); + return cy + .window() + .its("frappe") + .then((frappe) => { + return frappe.call( + "frappe.tests.ui_test_helpers.create_contact_phone_nos_records" + ); + }); }); - it('creates pages for child table', () => { - cy.visit('/app/contact/Test Contact'); - cy.get('.frappe-control[data-fieldname="phone_nos"]').as('table'); - cy.get('@table').find('.current-page-number').should('have.value', '1'); - cy.get('@table').find('.total-page-number').should('contain', '20'); - cy.get('@table').find('.grid-body .grid-row').should('have.length', 50); + it("creates pages for child table", () => { + cy.visit("/app/contact/Test Contact"); + cy.get('.frappe-control[data-fieldname="phone_nos"]').as("table"); + cy.get("@table").find(".current-page-number").should("have.value", "1"); + cy.get("@table").find(".total-page-number").should("contain", "20"); + cy.get("@table").find(".grid-body .grid-row").should("have.length", 50); }); - it('goes to the next and previous page', () => { - cy.visit('/app/contact/Test Contact'); - cy.get('.frappe-control[data-fieldname="phone_nos"]').as('table'); - cy.get('@table').find('.next-page').click(); - cy.get('@table').find('.current-page-number').should('have.value', '2'); - cy.get('@table').find('.grid-body .grid-row').first().should('have.attr', 'data-idx', '51'); - cy.get('@table').find('.prev-page').click(); - cy.get('@table').find('.current-page-number').should('have.value', '1'); - cy.get('@table').find('.grid-body .grid-row').first().should('have.attr', 'data-idx', '1'); + it("goes to the next and previous page", () => { + cy.visit("/app/contact/Test Contact"); + cy.get('.frappe-control[data-fieldname="phone_nos"]').as("table"); + cy.get("@table").find(".next-page").click(); + cy.get("@table").find(".current-page-number").should("have.value", "2"); + cy.get("@table") + .find(".grid-body .grid-row") + .first() + .should("have.attr", "data-idx", "51"); + cy.get("@table").find(".prev-page").click(); + cy.get("@table").find(".current-page-number").should("have.value", "1"); + cy.get("@table").find(".grid-body .grid-row").first().should("have.attr", "data-idx", "1"); }); - it('adds and deletes rows and changes page', () => { - cy.visit('/app/contact/Test Contact'); - cy.get('.frappe-control[data-fieldname="phone_nos"]').as('table'); - cy.get('@table').findByRole('button', {name: 'Add Row'}).click(); - cy.get('@table').find('.grid-body .row-index').should('contain', 1001); - cy.get('@table').find('.current-page-number').should('have.value', '21'); - cy.get('@table').find('.total-page-number').should('contain', '21'); - cy.get('@table').find('.grid-body .grid-row .grid-row-check').click({ force: true }); - cy.get('@table').findByRole('button', {name: 'Delete'}).click(); - cy.get('@table').find('.grid-body .row-index').last().should('contain', 1000); - cy.get('@table').find('.current-page-number').should('have.value', '20'); - cy.get('@table').find('.total-page-number').should('contain', '20'); + it("adds and deletes rows and changes page", () => { + cy.visit("/app/contact/Test Contact"); + cy.get('.frappe-control[data-fieldname="phone_nos"]').as("table"); + cy.get("@table").findByRole("button", { name: "Add Row" }).click(); + cy.get("@table").find(".grid-body .row-index").should("contain", 1001); + cy.get("@table").find(".current-page-number").should("have.value", "21"); + cy.get("@table").find(".total-page-number").should("contain", "21"); + cy.get("@table").find(".grid-body .grid-row .grid-row-check").click({ force: true }); + cy.get("@table").findByRole("button", { name: "Delete" }).click(); + cy.get("@table").find(".grid-body .row-index").last().should("contain", 1000); + cy.get("@table").find(".current-page-number").should("have.value", "20"); + cy.get("@table").find(".total-page-number").should("contain", "20"); }); - it('go to specific page, use up and down arrow, type characters, 0 page and more than existing page', () => { - cy.visit('/app/contact/Test Contact'); - cy.get('.frappe-control[data-fieldname="phone_nos"]').as('table'); - cy.get('@table').find('.current-page-number').focus().clear().type('17').blur(); - cy.get('@table').find('.grid-body .row-index').should('contain', 801); + it("go to specific page, use up and down arrow, type characters, 0 page and more than existing page", () => { + cy.visit("/app/contact/Test Contact"); + cy.get('.frappe-control[data-fieldname="phone_nos"]').as("table"); + cy.get("@table").find(".current-page-number").focus().clear().type("17").blur(); + cy.get("@table").find(".grid-body .row-index").should("contain", 801); - cy.get('@table').find('.current-page-number').focus().type('{uparrow}{uparrow}'); - cy.get('@table').find('.current-page-number').should('have.value', '19'); + cy.get("@table").find(".current-page-number").focus().type("{uparrow}{uparrow}"); + cy.get("@table").find(".current-page-number").should("have.value", "19"); - cy.get('@table').find('.current-page-number').focus().type('{downarrow}{downarrow}'); - cy.get('@table').find('.current-page-number').should('have.value', '17'); + cy.get("@table").find(".current-page-number").focus().type("{downarrow}{downarrow}"); + cy.get("@table").find(".current-page-number").should("have.value", "17"); - cy.get('@table').find('.current-page-number').focus().clear().type('700').blur(); - cy.get('@table').find('.current-page-number').should('have.value', '20'); + cy.get("@table").find(".current-page-number").focus().clear().type("700").blur(); + cy.get("@table").find(".current-page-number").should("have.value", "20"); - cy.get('@table').find('.current-page-number').focus().clear().type('0').blur(); - cy.get('@table').find('.current-page-number').should('have.value', '1'); + cy.get("@table").find(".current-page-number").focus().clear().type("0").blur(); + cy.get("@table").find(".current-page-number").should("have.value", "1"); - cy.get('@table').find('.current-page-number').focus().clear().type('abc').blur(); - cy.get('@table').find('.current-page-number').should('have.value', '1'); + cy.get("@table").find(".current-page-number").focus().clear().type("abc").blur(); + cy.get("@table").find(".current-page-number").should("have.value", "1"); }); // it('deletes all rows', ()=> { // cy.visit('/app/contact/Test Contact'); @@ -69,4 +77,4 @@ context('Grid Pagination', () => { // cy.get('.modal-dialog .btn-primary').contains('Yes').click(); // cy.get('@table').find('.grid-body .grid-row').should('have.length', 0); // }); -}); \ No newline at end of file +}); diff --git a/cypress/integration/grid_search.js b/cypress/integration/grid_search.js index d30545a2e1..3d43412313 100644 --- a/cypress/integration/grid_search.js +++ b/cypress/integration/grid_search.js @@ -1,107 +1,133 @@ -import doctype_with_child_table from '../fixtures/doctype_with_child_table'; -import child_table_doctype from '../fixtures/child_table_doctype'; -import child_table_doctype_1 from '../fixtures/child_table_doctype_1'; +import doctype_with_child_table from "../fixtures/doctype_with_child_table"; +import child_table_doctype from "../fixtures/child_table_doctype"; +import child_table_doctype_1 from "../fixtures/child_table_doctype_1"; const doctype_with_child_table_name = doctype_with_child_table.name; -context('Grid Search', () => { +context("Grid Search", () => { before(() => { - cy.visit('/login'); + cy.visit("/login"); cy.login(); - cy.visit('/app/website'); - cy.insert_doc('DocType', child_table_doctype, true); - cy.insert_doc('DocType', child_table_doctype_1, true); - cy.insert_doc('DocType', doctype_with_child_table, true); - return cy.window().its('frappe').then(frappe => { - return frappe.xcall("frappe.tests.ui_test_helpers.insert_doctype_with_child_table_record", { - name: doctype_with_child_table_name + cy.visit("/app/website"); + cy.insert_doc("DocType", child_table_doctype, true); + cy.insert_doc("DocType", child_table_doctype_1, true); + cy.insert_doc("DocType", doctype_with_child_table, true); + return cy + .window() + .its("frappe") + .then((frappe) => { + return frappe.xcall( + "frappe.tests.ui_test_helpers.insert_doctype_with_child_table_record", + { + name: doctype_with_child_table_name, + } + ); }); - }); }); - it('Test search row visibility', () => { - cy.window().its('frappe').then(frappe => { - frappe.model.user_settings.save('Doctype With Child Table', 'GridView', { - 'Child Table Doctype 1': [ - {'fieldname': 'data', 'columns': 2}, - {'fieldname': 'barcode', 'columns': 1}, - {'fieldname': 'check', 'columns': 1}, - {'fieldname': 'rating', 'columns': 2}, - {'fieldname': 'duration', 'columns': 2}, - {'fieldname': 'date', 'columns': 2} - ] + it("Test search row visibility", () => { + cy.window() + .its("frappe") + .then((frappe) => { + frappe.model.user_settings.save("Doctype With Child Table", "GridView", { + "Child Table Doctype 1": [ + { fieldname: "data", columns: 2 }, + { fieldname: "barcode", columns: 1 }, + { fieldname: "check", columns: 1 }, + { fieldname: "rating", columns: 2 }, + { fieldname: "duration", columns: 2 }, + { fieldname: "date", columns: 2 }, + ], + }); }); - }); cy.visit(`/app/doctype-with-child-table/Test Grid Search`); - cy.get('.frappe-control[data-fieldname="child_table_1"]').as('table'); - cy.get('@table').find('.grid-row-check:last').click(); - cy.get('@table').find('.grid-footer').contains('Delete').click(); - cy.get('.grid-heading-row .grid-row .search').should('not.exist'); + cy.get('.frappe-control[data-fieldname="child_table_1"]').as("table"); + cy.get("@table").find(".grid-row-check:last").click(); + cy.get("@table").find(".grid-footer").contains("Delete").click(); + cy.get(".grid-heading-row .grid-row .search").should("not.exist"); }); - it('test search field for different fieldtypes', () => { + it("test search field for different fieldtypes", () => { cy.visit(`/app/doctype-with-child-table/Test Grid Search`); - cy.get('.frappe-control[data-fieldname="child_table_1"]').as('table'); + cy.get('.frappe-control[data-fieldname="child_table_1"]').as("table"); // Index Column - cy.get('@table').find('.grid-heading-row .row-index.search input').type('3'); - cy.get('@table').find('.grid-body .rows .grid-row').should('have.length', 2); - cy.get('@table').find('.grid-heading-row .row-index.search input').clear(); + cy.get("@table").find(".grid-heading-row .row-index.search input").type("3"); + cy.get("@table").find(".grid-body .rows .grid-row").should("have.length", 2); + cy.get("@table").find(".grid-heading-row .row-index.search input").clear(); // Data Column - cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Data"]').type('Data'); - cy.get('@table').find('.grid-body .rows .grid-row').should('have.length', 1); - cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Data"]').clear(); + cy.get("@table") + .find('.grid-heading-row .search input[data-fieldtype="Data"]') + .type("Data"); + cy.get("@table").find(".grid-body .rows .grid-row").should("have.length", 1); + cy.get("@table").find('.grid-heading-row .search input[data-fieldtype="Data"]').clear(); // Barcode Column - cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Barcode"]').type('092'); - cy.get('@table').find('.grid-body .rows .grid-row').should('have.length', 4); - cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Barcode"]').clear(); + cy.get("@table") + .find('.grid-heading-row .search input[data-fieldtype="Barcode"]') + .type("092"); + cy.get("@table").find(".grid-body .rows .grid-row").should("have.length", 4); + cy.get("@table").find('.grid-heading-row .search input[data-fieldtype="Barcode"]').clear(); // Check Column - cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Check"]').type('1'); - cy.get('@table').find('.grid-body .rows .grid-row').should('have.length', 9); - cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Check"]').clear(); + cy.get("@table").find('.grid-heading-row .search input[data-fieldtype="Check"]').type("1"); + cy.get("@table").find(".grid-body .rows .grid-row").should("have.length", 9); + cy.get("@table").find('.grid-heading-row .search input[data-fieldtype="Check"]').clear(); - cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Check"]').type('0'); - cy.get('@table').find('.grid-body .rows .grid-row').should('have.length', 11); - cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Check"]').clear(); + cy.get("@table").find('.grid-heading-row .search input[data-fieldtype="Check"]').type("0"); + cy.get("@table").find(".grid-body .rows .grid-row").should("have.length", 11); + cy.get("@table").find('.grid-heading-row .search input[data-fieldtype="Check"]').clear(); // Rating Column - cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Rating"]').type('3'); - cy.get('@table').find('.grid-body .rows .grid-row').should('have.length', 3); - cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Rating"]').clear(); + cy.get("@table") + .find('.grid-heading-row .search input[data-fieldtype="Rating"]') + .type("3"); + cy.get("@table").find(".grid-body .rows .grid-row").should("have.length", 3); + cy.get("@table").find('.grid-heading-row .search input[data-fieldtype="Rating"]').clear(); // Duration Column - cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Duration"]').type('3d'); - cy.get('@table').find('.grid-body .rows .grid-row').should('have.length', 3); - cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Duration"]').clear(); + cy.get("@table") + .find('.grid-heading-row .search input[data-fieldtype="Duration"]') + .type("3d"); + cy.get("@table").find(".grid-body .rows .grid-row").should("have.length", 3); + cy.get("@table") + .find('.grid-heading-row .search input[data-fieldtype="Duration"]') + .clear(); // Date Column - cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Date"]').type('2022'); - cy.get('@table').find('.grid-body .rows .grid-row').should('have.length', 4); - cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Date"]').clear(); + cy.get("@table") + .find('.grid-heading-row .search input[data-fieldtype="Date"]') + .type("2022"); + cy.get("@table").find(".grid-body .rows .grid-row").should("have.length", 4); + cy.get("@table").find('.grid-heading-row .search input[data-fieldtype="Date"]').clear(); }); - it('test with multiple filter', () => { - cy.get('.frappe-control[data-fieldname="child_table_1"]').as('table'); + it("test with multiple filter", () => { + cy.get('.frappe-control[data-fieldname="child_table_1"]').as("table"); // Data Column - cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Data"]').type('a'); - cy.get('@table').find('.grid-body .rows .grid-row').should('have.length', 10); + cy.get("@table").find('.grid-heading-row .search input[data-fieldtype="Data"]').type("a"); + cy.get("@table").find(".grid-body .rows .grid-row").should("have.length", 10); // Barcode Column - cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Barcode"]').type('0'); - cy.get('@table').find('.grid-body .rows .grid-row').should('have.length', 8); + cy.get("@table") + .find('.grid-heading-row .search input[data-fieldtype="Barcode"]') + .type("0"); + cy.get("@table").find(".grid-body .rows .grid-row").should("have.length", 8); // Duration Column - cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Duration"]').type('d'); - cy.get('@table').find('.grid-body .rows .grid-row').should('have.length', 5); + cy.get("@table") + .find('.grid-heading-row .search input[data-fieldtype="Duration"]') + .type("d"); + cy.get("@table").find(".grid-body .rows .grid-row").should("have.length", 5); // Date Column - cy.get('@table').find('.grid-heading-row .search input[data-fieldtype="Date"]').type('02-'); - cy.get('@table').find('.grid-body .rows .grid-row').should('have.length', 2); + cy.get("@table") + .find('.grid-heading-row .search input[data-fieldtype="Date"]') + .type("02-"); + cy.get("@table").find(".grid-body .rows .grid-row").should("have.length", 2); }); -}); \ No newline at end of file +}); diff --git a/cypress/integration/kanban.js b/cypress/integration/kanban.js index 6ebab5d008..f14c991c7c 100644 --- a/cypress/integration/kanban.js +++ b/cypress/integration/kanban.js @@ -1,87 +1,134 @@ -context('Kanban Board', () => { +context("Kanban Board", () => { before(() => { - cy.login(); - cy.visit('/app'); + cy.login("frappe@example.com"); + cy.visit("/app"); }); - it('Create ToDo Kanban', () => { - cy.visit('/app/todo'); + it("Create ToDo Kanban", () => { + cy.visit("/app/todo"); - cy.get('.page-actions .custom-btn-group button').click(); - cy.get('.page-actions .custom-btn-group ul.dropdown-menu li').contains('Kanban').click(); + cy.get(".page-actions .custom-btn-group button").click(); + cy.get(".page-actions .custom-btn-group ul.dropdown-menu li").contains("Kanban").click(); cy.focused().blur(); - cy.fill_field('board_name', 'ToDo Kanban', 'Data'); - cy.fill_field('field_name', 'Status', 'Select'); - cy.click_modal_primary_button('Save'); + cy.fill_field("board_name", "ToDo Kanban", "Data"); + cy.fill_field("field_name", "Status", "Select"); + cy.click_modal_primary_button("Save"); - cy.get('.title-text').should('contain', 'ToDo Kanban'); + cy.get(".title-text").should("contain", "ToDo Kanban"); }); - it('Create ToDo from kanban', () => { + it("Create ToDo from kanban", () => { cy.intercept({ - method: 'POST', - url: 'api/method/frappe.client.save' - }).as('save-todo'); + method: "POST", + url: "api/method/frappe.client.save", + }).as("save-todo"); - cy.click_listview_primary_button('Add ToDo'); + cy.click_listview_primary_button("Add ToDo"); - cy.fill_field('description', 'Test Kanban ToDo', 'Text Editor').wait(300); - cy.get('.modal-footer .btn-primary').last().click(); + cy.fill_field("description", "Test Kanban ToDo", "Text Editor").wait(300); + cy.get(".modal-footer .btn-primary").last().click(); - cy.wait('@save-todo'); + cy.wait("@save-todo"); }); - it('Add and Remove fields', () => { - cy.visit('/app/todo/view/kanban/ToDo Kanban'); + it("Add and Remove fields", () => { + cy.visit("/app/todo/view/kanban/ToDo Kanban"); - cy.intercept('POST', '/api/method/frappe.desk.doctype.kanban_board.kanban_board.save_settings').as('save-kanban'); - cy.intercept('POST', '/api/method/frappe.desk.doctype.kanban_board.kanban_board.update_order').as('update-order'); + cy.intercept( + "POST", + "/api/method/frappe.desk.doctype.kanban_board.kanban_board.save_settings" + ).as("save-kanban"); + cy.intercept( + "POST", + "/api/method/frappe.desk.doctype.kanban_board.kanban_board.update_order" + ).as("update-order"); - cy.get('.page-actions .menu-btn-group > .btn').click(); - cy.get('.page-actions .menu-btn-group .dropdown-menu li').contains('Kanban Settings').click(); - cy.get('.add-new-fields').click(); + cy.get(".page-actions .menu-btn-group > .btn").click(); + cy.get(".page-actions .menu-btn-group .dropdown-menu li") + .contains("Kanban Settings") + .click(); + cy.get(".add-new-fields").click(); - cy.get('.checkbox-options .checkbox').contains('ID').click(); - cy.get('.checkbox-options .checkbox').contains('Status').first().click(); - cy.get('.checkbox-options .checkbox').contains('Priority').click(); + cy.get(".checkbox-options .checkbox").contains("ID").click(); + cy.get(".checkbox-options .checkbox").contains("Status").first().click(); + cy.get(".checkbox-options .checkbox").contains("Priority").click(); - cy.get('.modal-footer .btn-primary').last().click(); + cy.get(".modal-footer .btn-primary").last().click(); - cy.get('.frappe-control .label-area').contains('Show Labels').click(); - cy.click_modal_primary_button('Save'); + cy.get(".frappe-control .label-area").contains("Show Labels").click(); + cy.click_modal_primary_button("Save"); - cy.wait('@save-kanban'); + cy.wait("@save-kanban"); - cy.get('.kanban-column[data-column-value="Open"] .kanban-cards').as('open-cards'); - cy.get('@open-cards').find('.kanban-card .kanban-card-doc').first().should('contain', 'ID:'); - cy.get('@open-cards').find('.kanban-card .kanban-card-doc').first().should('contain', 'Status:'); - cy.get('@open-cards').find('.kanban-card .kanban-card-doc').first().should('contain', 'Priority:'); + cy.get('.kanban-column[data-column-value="Open"] .kanban-cards').as("open-cards"); + cy.get("@open-cards") + .find(".kanban-card .kanban-card-doc") + .first() + .should("contain", "ID:"); + cy.get("@open-cards") + .find(".kanban-card .kanban-card-doc") + .first() + .should("contain", "Status:"); + cy.get("@open-cards") + .find(".kanban-card .kanban-card-doc") + .first() + .should("contain", "Priority:"); - cy.get('.page-actions .menu-btn-group > .btn').click(); - cy.get('.page-actions .menu-btn-group .dropdown-menu li').contains('Kanban Settings').click(); - cy.get_open_dialog().find('.frappe-control[data-fieldname="fields_html"] div[data-label="ID"] .remove-field').click(); + cy.get(".page-actions .menu-btn-group > .btn").click(); + cy.get(".page-actions .menu-btn-group .dropdown-menu li") + .contains("Kanban Settings") + .click(); + cy.get_open_dialog() + .find( + '.frappe-control[data-fieldname="fields_html"] div[data-label="ID"] .remove-field' + ) + .click(); - cy.wait('@update-order'); - cy.get_open_dialog().find('.frappe-control .label-area').contains('Show Labels').click(); - cy.get('.modal-footer .btn-primary').last().click(); + cy.wait("@update-order"); + cy.get_open_dialog().find(".frappe-control .label-area").contains("Show Labels").click(); + cy.get(".modal-footer .btn-primary").last().click(); - cy.wait('@save-kanban'); - - cy.get('@open-cards').find('.kanban-card .kanban-card-doc').first().should('not.contain', 'ID:'); + cy.wait("@save-kanban"); + cy.get("@open-cards") + .find(".kanban-card .kanban-card-doc") + .first() + .should("not.contain", "ID:"); }); - // it('Drag todo', () => { - // cy.intercept({ - // method: 'POST', - // url: 'api/method/frappe.desk.doctype.kanban_board.kanban_board.update_order_for_single_card' - // }).as('drag-completed'); + it("Checks if Kanban Board edits are blocked for non-System Manager and non-owner of the Board", () => { + cy.switch_to_user("Administrator"); - // cy.get('.kanban-card-body') - // .contains('Test Kanban ToDo').first() - // .drag('[data-column-value="Closed"] .kanban-cards', { force: true }); + const noSystemManager = "nosysmanager@example.com"; + cy.call("frappe.tests.ui_test_helpers.create_test_user", { + username: noSystemManager, + }); + cy.remove_role(noSystemManager, "System Manager"); + cy.call("frappe.tests.ui_test_helpers.create_todo", { description: "Frappe User ToDo" }); + cy.call("frappe.tests.ui_test_helpers.create_admin_kanban"); - // cy.wait('@drag-completed'); - // }); -}); \ No newline at end of file + cy.switch_to_user(noSystemManager); + + cy.visit("/app/todo/view/kanban/Admin Kanban"); + + // Menu button should be hidden (dropdown for 'Save Filters' and 'Delete Kanban Board') + cy.get(".no-list-sidebar .menu-btn-group .btn-default[data-original-title='Menu']").should( + "have.length", + 0 + ); + // Kanban Columns should be visible (read-only) + cy.get(".kanban .kanban-column").should("have.length", 2); + // User should be able to add card (has access to ToDo) + cy.get(".kanban .add-card").should("have.length", 2); + // Column actions should be hidden (dropdown for 'Archive' and indicators) + cy.get(".kanban .column-options").should("have.length", 0); + + cy.switch_to_user("Administrator"); + cy.call("frappe.client.delete", { doctype: "User", name: noSystemManager }); + }); + + after(() => { + cy.call("logout"); + }); +}); diff --git a/cypress/integration/list_paging.js b/cypress/integration/list_paging.js index 0cf6f2e565..5195d0b3ae 100644 --- a/cypress/integration/list_paging.js +++ b/cypress/integration/list_paging.js @@ -1,39 +1,42 @@ -context('List Paging', () => { +context("List Paging", () => { before(() => { cy.login(); - cy.visit('/app/website'); - return cy.window().its('frappe').then(frappe => { - return frappe.call("frappe.tests.ui_test_helpers.create_multiple_todo_records"); - }); + cy.visit("/app/website"); + return cy + .window() + .its("frappe") + .then((frappe) => { + return frappe.call("frappe.tests.ui_test_helpers.create_multiple_todo_records"); + }); }); - it('test load more with count selection buttons', () => { - cy.visit('/app/todo/view/report'); - cy.clear_filters() + it("test load more with count selection buttons", () => { + cy.visit("/app/todo/view/report"); + cy.get(".filter-x-button").click(); - cy.get('.list-paging-area .list-count').should('contain.text', '20 of'); - cy.get('.list-paging-area .btn-more').click(); - cy.get('.list-paging-area .list-count').should('contain.text', '40 of'); - cy.get('.list-paging-area .btn-more').click(); - cy.get('.list-paging-area .list-count').should('contain.text', '60 of'); + cy.get(".list-paging-area .list-count").should("contain.text", "20 of"); + cy.get(".list-paging-area .btn-more").click(); + cy.get(".list-paging-area .list-count").should("contain.text", "40 of"); + cy.get(".list-paging-area .btn-more").click(); + cy.get(".list-paging-area .list-count").should("contain.text", "60 of"); cy.get('.list-paging-area .btn-group .btn-paging[data-value="100"]').click(); - cy.get('.list-paging-area .list-count').should('contain.text', '100 of'); - cy.get('.list-paging-area .btn-more').click(); - cy.get('.list-paging-area .list-count').should('contain.text', '200 of'); - cy.get('.list-paging-area .btn-more').click(); - cy.get('.list-paging-area .list-count').should('contain.text', '300 of'); + cy.get(".list-paging-area .list-count").should("contain.text", "100 of"); + cy.get(".list-paging-area .btn-more").click(); + cy.get(".list-paging-area .list-count").should("contain.text", "200 of"); + cy.get(".list-paging-area .btn-more").click(); + cy.get(".list-paging-area .list-count").should("contain.text", "300 of"); // check if refresh works after load more cy.get('.page-head .standard-actions [data-original-title="Refresh"]').click(); - cy.get('.list-paging-area .list-count').should('contain.text', '300 of'); + cy.get(".list-paging-area .list-count").should("contain.text", "300 of"); cy.get('.list-paging-area .btn-group .btn-paging[data-value="500"]').click(); - cy.get('.list-paging-area .list-count').should('contain.text', '500 of'); - cy.get('.list-paging-area .btn-more').click(); + cy.get(".list-paging-area .list-count").should("contain.text", "500 of"); + cy.get(".list-paging-area .btn-more").click(); - cy.get('.list-paging-area .list-count').should('contain.text', '1000 of'); + cy.get(".list-paging-area .list-count").should("contain.text", "1000 of"); }); }); diff --git a/cypress/integration/list_view.js b/cypress/integration/list_view.js index ee12b37638..3fa0758f0c 100644 --- a/cypress/integration/list_view.js +++ b/cypress/integration/list_view.js @@ -1,48 +1,66 @@ -context('List View', () => { +context("List View", () => { before(() => { cy.login(); - cy.visit('/app/website'); - return cy.window().its('frappe').then(frappe => { - return frappe.xcall("frappe.tests.ui_test_helpers.setup_workflow"); - }); + cy.visit("/app/website"); + return cy + .window() + .its("frappe") + .then((frappe) => { + return frappe.xcall("frappe.tests.ui_test_helpers.setup_workflow"); + }); }); - it('Keep checkbox checked after Refresh', () => { - cy.go_to_list('ToDo'); - cy.clear_filters() - cy.get('.list-row-container .list-row-checkbox').click({ multiple: true, force: true }); - cy.get('.actions-btn-group button').contains('Actions').should('be.visible'); - cy.intercept('/api/method/frappe.desk.reportview.get').as('list-refresh'); + it("Keep checkbox checked after Refresh", { scrollBehavior: false }, () => { + cy.go_to_list("ToDo"); + cy.clear_filters(); + cy.get(".list-row-container .list-row-checkbox").click({ + multiple: true, + force: true, + }); + cy.get(".actions-btn-group button").contains("Actions").should("be.visible"); + cy.intercept("/api/method/frappe.desk.reportview.get").as("list-refresh"); cy.wait(3000); // wait before you hit another refresh cy.get('button[data-original-title="Refresh"]').click(); - cy.wait('@list-refresh'); - cy.get('.list-row-container .list-row-checkbox:checked').should('be.visible'); + cy.wait("@list-refresh"); + cy.get(".list-row-container .list-row-checkbox:checked").should("be.visible"); }); - it('enables "Actions" button', () => { - const actions = ['Approve', 'Reject', 'Edit', 'Export', 'Assign To', 'Apply Assignment Rule', 'Add Tags', 'Print', 'Delete']; - cy.go_to_list('ToDo'); - cy.clear_filters() - cy.get('.list-row-container:contains("Pending") .list-row-checkbox').click({ multiple: true, force: true }); - cy.get('.actions-btn-group button').contains('Actions').should('be.visible').click(); - cy.get('.dropdown-menu li:visible .dropdown-item').should('have.length', 9).each((el, index) => { - cy.wrap(el).contains(actions[index]); - }).then((elements) => { - cy.intercept({ - method: 'POST', - url: 'api/method/frappe.model.workflow.bulk_workflow_approval' - }).as('bulk-approval'); - cy.intercept({ - method: 'POST', - url: 'api/method/frappe.desk.reportview.get' - }).as('real-time-update'); - cy.wrap(elements).contains('Approve').click(); - cy.wait(['@bulk-approval', '@real-time-update']); - cy.wait(300); - cy.get_open_dialog().find('.btn-modal-close').click(); - cy.reload(); - cy.clear_filters(); - cy.get('.list-row-container:visible').should('contain', 'Approved'); + it('enables "Actions" button', { scrollBehavior: false }, () => { + const actions = [ + "Approve", + "Reject", + "Edit", + "Export", + "Assign To", + "Apply Assignment Rule", + "Add Tags", + "Print", + "Delete", + ]; + cy.go_to_list("ToDo"); + cy.clear_filters(); + cy.get('.list-row-container:contains("Pending") .list-row-checkbox').click({ + multiple: true, + force: true, }); + cy.get(".actions-btn-group button").contains("Actions").should("be.visible").click(); + cy.get(".dropdown-menu li:visible .dropdown-item") + .should("have.length", 9) + .each((el, index) => { + cy.wrap(el).contains(actions[index]); + }) + .then((elements) => { + cy.intercept({ + method: "POST", + url: "api/method/frappe.model.workflow.bulk_workflow_approval", + }).as("bulk-approval"); + cy.wrap(elements).contains("Approve").click(); + cy.wait("@bulk-approval"); + cy.wait(300); + cy.get_open_dialog().find(".btn-modal-close").click(); + cy.reload(); + cy.clear_filters(); + cy.get(".list-row-container:visible").should("contain", "Approved"); + }); }); }); diff --git a/cypress/integration/list_view_drag_select.js b/cypress/integration/list_view_drag_select.js new file mode 100644 index 0000000000..2dcb31372c --- /dev/null +++ b/cypress/integration/list_view_drag_select.js @@ -0,0 +1,49 @@ +context("List View", () => { + before(() => { + cy.login(); + cy.go_to_list("DocType"); + }); + + it("List view check rows on drag", () => { + cy.get(".filter-x-button").click(); + cy.get(".list-row-checkbox").then(($checkbox) => { + cy.wrap($checkbox).first().trigger("mousedown"); + cy.get(".level.list-row").each(($ele) => { + cy.wrap($ele).trigger("mousemove"); + }); + cy.document().trigger("mouseup"); + }); + + cy.get(".level.list-row .list-row-checkbox").each(($checkbox) => { + cy.wrap($checkbox).should("be.checked"); + }); + }); + + it("Check all rows are checked", () => { + cy.get(".level.list-row .list-row-checkbox") + .its("length") + .then((len) => { + cy.get(".level-item.list-header-meta") + .should("be.visible") + .should("contain.text", `${len} items selected`); + }); + }); + + it("List view uncheck rows on drag", () => { + cy.get(".list-row-checkbox").then(($checkbox) => { + cy.wrap($checkbox).first().trigger("mousedown"); + cy.get(".level.list-row").each(($ele) => { + cy.wrap($ele).trigger("mousemove"); + }); + cy.document().trigger("mouseup"); + }); + + cy.get(".level.list-row .list-row-checkbox").each(($checkbox) => { + cy.wrap($checkbox).should("not.be.checked"); + }); + }); + + it("Check all rows are unchecked", () => { + cy.get(".level-item.list-header-meta").should("not.be.visible"); + }); +}); diff --git a/cypress/integration/list_view_settings.js b/cypress/integration/list_view_settings.js index 61d4b8aae5..ff9a30ce5c 100644 --- a/cypress/integration/list_view_settings.js +++ b/cypress/integration/list_view_settings.js @@ -1,36 +1,42 @@ -context('List View Settings', () => { +context("List View Settings", () => { beforeEach(() => { cy.login(); - cy.visit('/app/website'); + cy.visit("/app/website"); }); - it('Default settings', () => { - cy.visit('/app/List/DocType/List'); - cy.get('.list-count').should('contain', "20 of"); - cy.get('.list-stats').should('contain', "Tags"); + it("Default settings", () => { + cy.visit("/app/List/DocType/List"); + cy.clear_filters(); + cy.get(".list-count").should("contain", "20 of"); + cy.get(".list-stats").should("contain", "Tags"); }); - it('disable count and sidebar stats then verify', () => { + it("disable count and sidebar stats then verify", () => { cy.wait(300); - cy.visit('/app/List/DocType/List'); + cy.visit("/app/List/DocType/List"); + cy.clear_filters(); cy.wait(300); - cy.get('.list-count').should('contain', "20 of"); - cy.get('.menu-btn-group button').click(); - cy.get('.dropdown-menu li').filter(':visible').contains('List Settings').click(); - cy.get('.modal-dialog').should('contain', 'DocType Settings'); + cy.get(".list-count").should("contain", "20 of"); + cy.get("[href='#icon-small-message']").should("be.visible"); + cy.get(".menu-btn-group button").click(); + cy.get(".dropdown-menu li").filter(":visible").contains("List Settings").click(); + cy.get(".modal-dialog").should("contain", "DocType Settings"); - cy.findByLabelText('Disable Count').check({ force: true }); - cy.findByLabelText('Disable Sidebar Stats').check({ force: true }); - cy.findByRole('button', {name: 'Save'}).click(); + cy.findByLabelText("Disable Count").check({ force: true }); + cy.findByLabelText("Disable Comment Count").check({ force: true }); + cy.findByLabelText("Disable Sidebar Stats").check({ force: true }); + cy.findByRole("button", { name: "Save" }).click(); cy.reload({ force: true }); - cy.get('.list-count').should('be.empty'); - cy.get('.list-sidebar .list-tags').should('not.exist'); + cy.get(".list-count").should("be.empty"); + cy.get(".list-sidebar .list-tags").should("not.exist"); + cy.get("[href='#icon-small-message']").should("not.be.visible"); - cy.get('.menu-btn-group button').click({ force: true }); - cy.get('.dropdown-menu li').filter(':visible').contains('List Settings').click(); - cy.get('.modal-dialog').should('contain', 'DocType Settings'); - cy.findByLabelText('Disable Count').uncheck({ force: true }); - cy.findByLabelText('Disable Sidebar Stats').uncheck({ force: true }); - cy.findByRole('button', {name: 'Save'}).click(); + cy.get(".menu-btn-group button").click({ force: true }); + cy.get(".dropdown-menu li").filter(":visible").contains("List Settings").click(); + cy.get(".modal-dialog").should("contain", "DocType Settings"); + cy.findByLabelText("Disable Count").uncheck({ force: true }); + cy.findByLabelText("Disable Comment Count").uncheck({ force: true }); + cy.findByLabelText("Disable Sidebar Stats").uncheck({ force: true }); + cy.findByRole("button", { name: "Save" }).click(); }); }); diff --git a/cypress/integration/login.js b/cypress/integration/login.js index 98739bb4c9..912f34c508 100644 --- a/cypress/integration/login.js +++ b/cypress/integration/login.js @@ -1,68 +1,66 @@ -context('Login', () => { +context("Login", () => { beforeEach(() => { - cy.request('/api/method/logout'); - cy.visit('/login'); - cy.location('pathname').should('eq', '/login'); + cy.request("/api/method/logout"); + cy.visit("/login"); + cy.location("pathname").should("eq", "/login"); }); - it('greets with login screen', () => { - cy.get('.page-card-head').contains('Login'); + it("greets with login screen", () => { + cy.get(".page-card-head").contains("Login"); }); - it('validates password', () => { - cy.get('#login_email').type('Administrator'); - cy.findByRole('button', {name: 'Login'}).click(); - cy.location('pathname').should('eq', '/login'); + it("validates password", () => { + cy.get("#login_email").type("Administrator"); + cy.findByRole("button", { name: "Login" }).click(); + cy.location("pathname").should("eq", "/login"); }); - it('validates email', () => { - cy.get('#login_password').type('qwe'); - cy.findByRole('button', {name: 'Login'}).click(); - cy.location('pathname').should('eq', '/login'); + it("validates email", () => { + cy.get("#login_password").type("qwe"); + cy.findByRole("button", { name: "Login" }).click(); + cy.location("pathname").should("eq", "/login"); }); - it('shows invalid login if incorrect credentials', () => { - cy.get('#login_email').type('Administrator'); - cy.get('#login_password').type('qwer'); + it("shows invalid login if incorrect credentials", () => { + cy.get("#login_email").type("Administrator"); + cy.get("#login_password").type("qwer"); - cy.findByRole('button', {name: 'Login'}).click(); - cy.findByRole('button', {name: 'Invalid Login. Try again.'}).should('exist'); - cy.location('pathname').should('eq', '/login'); + cy.findByRole("button", { name: "Login" }).click(); + cy.findByRole("button", { name: "Invalid Login. Try again." }).should("exist"); + cy.location("pathname").should("eq", "/login"); }); - it('logs in using correct credentials', () => { - cy.get('#login_email').type('Administrator'); - cy.get('#login_password').type(Cypress.config('adminPassword')); + it("logs in using correct credentials", () => { + cy.get("#login_email").type("Administrator"); + cy.get("#login_password").type(Cypress.env("adminPassword")); - cy.findByRole('button', {name: 'Login'}).click(); - cy.location('pathname').should('eq', '/app'); - cy.window().its('frappe.session.user').should('eq', 'Administrator'); + cy.findByRole("button", { name: "Login" }).click(); + cy.location("pathname").should("eq", "/app"); + cy.window().its("frappe.session.user").should("eq", "Administrator"); }); - it('check redirect after login', () => { - + it("check redirect after login", () => { // mock for OAuth 2.0 client_id, redirect_uri, scope and state const payload = new URLSearchParams({ - uuid: '6fed1519-cfd8-4a2d-84a6-9a1799c7c741', - encoded_string: 'hello all', - encoded_url: 'http://test.localhost/callback', - base64_string: 'aGVsbG8gYWxs' + uuid: "6fed1519-cfd8-4a2d-84a6-9a1799c7c741", + encoded_string: "hello all", + encoded_url: "http://test.localhost/callback", + base64_string: "aGVsbG8gYWxs", }); - cy.request('/api/method/logout'); + cy.request("/api/method/logout"); // redirect-to /me page with params to mock OAuth 2.0 like request cy.visit( - '/login?redirect-to=/me?' + - encodeURIComponent(payload.toString().replace("+", " ")) + "/login?redirect-to=/me?" + encodeURIComponent(payload.toString().replace("+", " ")) ); - cy.get('#login_email').type('Administrator'); - cy.get('#login_password').type(Cypress.config('adminPassword')); + cy.get("#login_email").type("Administrator"); + cy.get("#login_password").type(Cypress.env("adminPassword")); - cy.findByRole('button', {name: 'Login'}).click(); + cy.findByRole("button", { name: "Login" }).click(); // verify redirected location and url params after login - cy.url().should('include', '/me?' + payload.toString().replace('+', '%20')); + cy.url().should("include", "/me?" + payload.toString().replace("+", "%20")); }); }); diff --git a/cypress/integration/multi_select_dialog.js b/cypress/integration/multi_select_dialog.js index 607db506c7..1be56d3b3d 100644 --- a/cypress/integration/multi_select_dialog.js +++ b/cypress/integration/multi_select_dialog.js @@ -1,99 +1,102 @@ -context('MultiSelectDialog', () => { +context("MultiSelectDialog", () => { before(() => { cy.login(); - cy.visit('/app'); + cy.visit("/app"); const contact_template = { - "doctype": "Contact", - "first_name": "Test", - "status": "Passive", - "email_ids": [ + doctype: "Contact", + first_name: "Test", + status: "Passive", + email_ids: [ { - "doctype": "Contact Email", - "email_id": "test@example.com", - "is_primary": 0 - } - ] + doctype: "Contact Email", + email_id: "test@example.com", + is_primary: 0, + }, + ], }; - const promises = Array.from({length: 25}) - .map(() => cy.insert_doc('Contact', contact_template, true)); + const promises = Array.from({ length: 25 }).map(() => + cy.insert_doc("Contact", contact_template, true) + ); Promise.all(promises); }); function open_multi_select_dialog() { - cy.window().its('frappe').then(frappe => { - new frappe.ui.form.MultiSelectDialog({ - doctype: "Contact", - target: {}, - setters: { - status: null, - gender: null - }, - add_filters_group: 1, - allow_child_item_selection: 1, - child_fieldname: "email_ids", - child_columns: ["email_id", "is_primary"] + cy.window() + .its("frappe") + .then((frappe) => { + new frappe.ui.form.MultiSelectDialog({ + doctype: "Contact", + target: {}, + setters: { + status: null, + gender: null, + }, + add_filters_group: 1, + allow_child_item_selection: 1, + child_fieldname: "email_ids", + child_columns: ["email_id", "is_primary"], + }); }); - }); } - it('checks multi select dialog api works', () => { + it("checks multi select dialog api works", () => { open_multi_select_dialog(); - cy.get_open_dialog().should('contain', 'Select Contacts'); + cy.get_open_dialog().should("contain", "Select Contacts"); }); - it('checks for filters', () => { - ['search_term', 'status', 'gender'].forEach(fieldname => { - cy.get_open_dialog().get(`.frappe-control[data-fieldname="${fieldname}"]`).should('exist'); + it("checks for filters", () => { + ["search_term", "status", "gender"].forEach((fieldname) => { + cy.get_open_dialog() + .get(`.frappe-control[data-fieldname="${fieldname}"]`) + .should("exist"); }); // add_filters_group: 1 should add a filter group - cy.get_open_dialog().get(`.frappe-control[data-fieldname="filter_area"]`).should('exist'); - + cy.get_open_dialog().get(`.frappe-control[data-fieldname="filter_area"]`).should("exist"); }); - it('checks for child item selection', () => { - cy.get_open_dialog() - .get(`.dt-row-header`).should('not.exist'); + it("checks for child item selection", () => { + cy.get_open_dialog().get(`.dt-row-header`).should("not.exist"); cy.get_open_dialog() .get(`.frappe-control[data-fieldname="allow_child_item_selection"]`) .find('input[data-fieldname="allow_child_item_selection"]') - .should('exist') - .click({force: true}); + .should("exist") + .click({ force: true }); cy.get_open_dialog() .get(`.frappe-control[data-fieldname="child_selection_area"]`) - .should('exist'); + .should("exist"); - cy.get_open_dialog() - .get(`.dt-row-header`).should('contain', 'Contact'); + cy.get_open_dialog().get(`.dt-row-header`).should("contain", "Contact"); - cy.get_open_dialog() - .get(`.dt-row-header`).should('contain', 'Email Id'); + cy.get_open_dialog().get(`.dt-row-header`).should("contain", "Email Id"); - cy.get_open_dialog() - .get(`.dt-row-header`).should('contain', 'Is Primary'); + cy.get_open_dialog().get(`.dt-row-header`).should("contain", "Is Primary"); }); - it('tests more button', () => { + it("tests more button", () => { cy.get_open_dialog() .get(`.frappe-control[data-fieldname="more_child_btn"]`) - .should('exist') - .as('more-btn'); - - cy.get_open_dialog().get('.datatable .dt-scrollable .dt-row').should(($rows) => { - expect($rows).to.have.length(20); - }); + .should("exist") + .as("more-btn"); - cy.intercept('POST', 'api/method/frappe.client.get_list').as('get-more-records'); - cy.get('@more-btn').find('button').click({force: true}); - cy.wait('@get-more-records'); + cy.get_open_dialog() + .get(".datatable .dt-scrollable .dt-row") + .should(($rows) => { + expect($rows).to.have.length(20); + }); - cy.get_open_dialog().get('.datatable .dt-scrollable .dt-row').should(($rows) => { - if ($rows.length <= 20) { - throw new Error("More button doesn't work"); - } - }); + cy.intercept("POST", "api/method/frappe.client.get_list").as("get-more-records"); + cy.get("@more-btn").find("button").click({ force: true }); + cy.wait("@get-more-records"); + cy.get_open_dialog() + .get(".datatable .dt-scrollable .dt-row") + .should(($rows) => { + if ($rows.length <= 20) { + throw new Error("More button doesn't work"); + } + }); }); -}); \ No newline at end of file +}); diff --git a/cypress/integration/navigation.js b/cypress/integration/navigation.js index b4e023c53e..cf1b5dc89d 100644 --- a/cypress/integration/navigation.js +++ b/cypress/integration/navigation.js @@ -1,25 +1,30 @@ -context('Navigation', () => { +context("Navigation", () => { before(() => { cy.login(); }); - it('Navigate to route with hash in document name', () => { - cy.insert_doc('ToDo', {'__newname': 'ABC#123', 'description': 'Test this', 'ignore_duplicate': true}); - cy.visit('/app/todo/ABC#123'); - cy.title().should('eq', 'Test this - ABC#123'); - cy.get_field('description', 'Text Editor').contains('Test this'); - cy.go('back'); - cy.title().should('eq', 'Website'); + it("Navigate to route with hash in document name", () => { + cy.insert_doc("ToDo", { + __newname: "ABC#123", + description: "Test this", + ignore_duplicate: true, + }); + cy.visit("/app/todo/ABC#123"); + cy.title().should("eq", "Test this - ABC#123"); + cy.get_field("description", "Text Editor").contains("Test this"); + cy.go("back"); + cy.title().should("eq", "Website"); }); - it.only('Navigate to previous page after login', () => { - cy.visit('/app/todo'); - cy.get('.page-head').findByTitle('To Do').should('be.visible'); - cy.request('/api/method/logout'); - cy.reload().as('reload'); - cy.get('@reload').get('.page-card .btn-primary').contains('Login').click(); - cy.location('pathname').should('eq', '/login'); + it.only("Navigate to previous page after login", () => { + cy.visit("/app/todo"); + cy.get(".page-head").findByTitle("To Do").should("be.visible"); + cy.clear_filters(); + cy.request("/api/method/logout"); + cy.reload().as("reload"); + cy.get("@reload").get(".page-card .btn-primary").contains("Login").click(); + cy.location("pathname").should("eq", "/login"); cy.login(); - cy.visit('/app'); - cy.location('pathname').should('eq', '/app/todo'); + cy.visit("/app"); + cy.location("pathname").should("eq", "/app/todo"); }); }); diff --git a/cypress/integration/number_card.js b/cypress/integration/number_card.js index a01ff1152d..eb0f19be26 100644 --- a/cypress/integration/number_card.js +++ b/cypress/integration/number_card.js @@ -1,22 +1,22 @@ -context('Number Card', () => { +context("Number Card", () => { before(() => { cy.login(); - cy.visit('/app/website'); + cy.visit("/app/website"); }); - it('Check filter populate for child table doctype', () => { - cy.visit('/app/number-card/new-number-card-1'); - cy.get('[data-fieldname="parent_document_type"]').should('have.css', 'display', 'none'); + it("Check filter populate for child table doctype", () => { + cy.visit("/app/number-card/new-number-card-1"); + cy.get('[data-fieldname="parent_document_type"]').should("have.css", "display", "none"); - cy.get_field('document_type', 'Link'); - cy.fill_field('document_type', 'Workspace Link', 'Link').focus().blur(); - cy.get_field('document_type', 'Link').should('have.value', 'Workspace Link'); + cy.get_field("document_type", "Link"); + cy.fill_field("document_type", "Workspace Link", "Link").focus().blur(); + cy.get_field("document_type", "Link").should("have.value", "Workspace Link"); - cy.fill_field('label', 'Test Number Card', 'Data'); + cy.fill_field("label", "Test Number Card", "Data"); cy.get('[data-fieldname="filters_json"]').click().wait(200); - cy.get('.modal-body .filter-action-buttons .add-filter').click(); - cy.get('.modal-body .fieldname-select-area').click(); - cy.get('.modal-actions .btn-modal-close').click(); + cy.get(".modal-body .filter-action-buttons .add-filter").click(); + cy.get(".modal-body .fieldname-select-area").click(); + cy.get(".modal-actions .btn-modal-close").click(); }); -}); \ No newline at end of file +}); diff --git a/cypress/integration/permissions.js b/cypress/integration/permissions.js new file mode 100644 index 0000000000..4e371cc17f --- /dev/null +++ b/cypress/integration/permissions.js @@ -0,0 +1,41 @@ +context("Permissions API", () => { + before(() => { + cy.visit("/login"); + cy.remove_role("frappe@example.com", "System Manager"); + cy.visit("/app"); + }); + + it("Checks permissions via `has_perm` for Kanban Board DocType", () => { + cy.visit("/app/kanban-board/view/list"); + cy.window() + .its("frappe") + .then((frappe) => { + frappe.model.with_doctype("Kanban Board", function () { + // needed to make sure doc meta is loaded + expect(frappe.perm.has_perm("Kanban Board", 0, "read")).to.equal(true); + expect(frappe.perm.has_perm("Kanban Board", 0, "write")).to.equal(true); + expect(frappe.perm.has_perm("Kanban Board", 0, "print")).to.equal(false); + }); + }); + }); + + it("Checks permissions via `get_perm` for Kanban Board DocType", () => { + cy.visit("/app/kanban-board/view/list"); + cy.window() + .its("frappe") + .then((frappe) => { + frappe.model.with_doctype("Kanban Board", function () { + // needed to make sure doc meta is loaded + const perms = frappe.perm.get_perm("Kanban Board"); + expect(perms.read).to.equal(true); + expect(perms.write).to.equal(true); + expect(perms.rights_without_if_owner).to.include("read"); + }); + }); + }); + + after(() => { + cy.add_role("frappe@example.com", "System Manager"); + cy.call("logout"); + }); +}); diff --git a/cypress/integration/query_report.js b/cypress/integration/query_report.js index 43f26f8b50..5dd0ab2d53 100644 --- a/cypress/integration/query_report.js +++ b/cypress/integration/query_report.js @@ -1,63 +1,91 @@ -context('Query Report', () => { +context("Query Report", () => { before(() => { cy.login(); - cy.visit('/app/website'); - cy.insert_doc('Report', { - 'report_name': 'Test ToDo Report', - 'ref_doctype': 'ToDo', - 'report_type': 'Query Report', - 'query': 'select * from tabToDo' - }, true).as('doc'); + cy.visit("/app/website"); + cy.insert_doc( + "Report", + { + report_name: "Test ToDo Report", + ref_doctype: "ToDo", + report_type: "Query Report", + query: "select * from tabToDo", + }, + true + ).as("doc"); cy.create_records({ - doctype: 'ToDo', - description: 'this is a test todo for query report' - }).as('todos'); + doctype: "ToDo", + description: "this is a test todo for query report", + }).as("todos"); }); - it('add custom column in report', () => { - cy.visit('/app/query-report/Permitted Documents For User'); + it("add custom column in report", () => { + cy.visit("/app/query-report/Permitted Documents For User"); - cy.get('.page-form.flex', { timeout: 60000 }).should('have.length', 1).then(() => { - cy.get('#page-query-report input[data-fieldname="user"]').as('input-user'); - cy.get('@input-user').focus().type('test@erpnext.com', { delay: 100 }).blur(); - cy.wait(300); - cy.get('#page-query-report input[data-fieldname="doctype"]').as('input-role'); - cy.get('@input-role').focus().type('Role', { delay: 100 }).blur(); + cy.get(".page-form.flex", { timeout: 60000 }) + .should("have.length", 1) + .then(() => { + cy.get('#page-query-report input[data-fieldname="user"]').as("input-user"); + cy.get("@input-user").focus().type("test@erpnext.com", { delay: 100 }).blur(); + cy.wait(300); + cy.get('#page-query-report input[data-fieldname="doctype"]').as("input-role"); + cy.get("@input-role").focus().type("Role", { delay: 100 }).blur(); - cy.get('.datatable').should('exist'); - cy.get('#page-query-report .page-actions .menu-btn-group button').click({ force: true }); - cy.get('#page-query-report .menu-btn-group .dropdown-menu').contains('Add Column').click({ force: true }); - cy.get_open_dialog().get('.modal-title').should('contain', 'Add Column'); - cy.get('select[data-fieldname="doctype"]').select("Role", { force: true }); - cy.get('select[data-fieldname="field"]').select("Role Name", { force: true }); - cy.get('select[data-fieldname="insert_after"]').select("Name", { force: true }); - cy.get_open_dialog().findByRole('button', {name: 'Submit'}).click({ force: true }); - cy.get('#page-query-report .page-actions .menu-btn-group button').click({ force: true }); - cy.get('#page-query-report .menu-btn-group .dropdown-menu').contains('Save').click({ timeout: 100, force: true }); - cy.get_open_dialog().get('.modal-title').should('contain', 'Save Report'); + cy.get(".datatable").should("exist"); + cy.get("#page-query-report .page-actions .menu-btn-group button").click({ + force: true, + }); + cy.get("#page-query-report .menu-btn-group .dropdown-menu") + .contains("Add Column") + .click({ force: true }); + cy.get_open_dialog().get(".modal-title").should("contain", "Add Column"); + cy.get('select[data-fieldname="doctype"]').select("Role", { force: true }); + cy.get('select[data-fieldname="field"]').select("Role Name", { force: true }); + cy.get('select[data-fieldname="insert_after"]').select("Name", { force: true }); + cy.get_open_dialog() + .findByRole("button", { name: "Submit" }) + .click({ force: true }); + cy.get("#page-query-report .page-actions .menu-btn-group button").click({ + force: true, + }); + cy.get("#page-query-report .menu-btn-group .dropdown-menu") + .contains("Save") + .click({ timeout: 100, force: true }); + cy.get_open_dialog().get(".modal-title").should("contain", "Save Report"); - cy.get('input[data-fieldname="report_name"]').type("Test Report", { delay: 100, force: true }); - cy.get_open_dialog().findByRole('button', {name: 'Submit'}).click({ timeout: 1000, force: true }); - }); + cy.get('input[data-fieldname="report_name"]').type("Test Report", { + delay: 100, + force: true, + }); + cy.get_open_dialog() + .findByRole("button", { name: "Submit" }) + .click({ timeout: 1000, force: true }); + }); }); let save_report_and_open = (report, update_name) => { - cy.get('#page-query-report .page-actions .menu-btn-group button').click({ force: true }); - cy.get('#page-query-report .menu-btn-group .dropdown-menu').contains('Save').click({ timeout: 100, force: true }); - cy.get_open_dialog().get('.modal-title').should('contain', 'Save Report'); + cy.get("#page-query-report .page-actions .menu-btn-group button").click({ force: true }); + cy.get("#page-query-report .menu-btn-group .dropdown-menu") + .contains("Save") + .click({ timeout: 100, force: true }); + cy.get_open_dialog().get(".modal-title").should("contain", "Save Report"); - cy.get('input[data-fieldname="report_name"]').type(update_name, { delay: 100, force: true }); - cy.get_open_dialog().findByRole('button', {name: 'Submit'}).click({ timeout: 1000, force: true }); + cy.get('input[data-fieldname="report_name"]').type(update_name, { + delay: 100, + force: true, + }); + cy.get_open_dialog() + .findByRole("button", { name: "Submit" }) + .click({ timeout: 1000, force: true }); - cy.visit('/app/query-report/'+report); - cy.get('.datatable').should('exist'); + cy.visit("/app/query-report/" + report); + cy.get(".datatable").should("exist"); }; - it('test multi level query report', () => { - cy.visit('/app/query-report/Test ToDo Report'); - cy.get('.datatable').should('exist'); + it("test multi level query report", () => { + cy.visit("/app/query-report/Test ToDo Report"); + cy.get(".datatable").should("exist"); - save_report_and_open('Test ToDo Report 1', ' 1'); - save_report_and_open('Test ToDo Report 11', '1'); + save_report_and_open("Test ToDo Report 1", " 1"); + save_report_and_open("Test ToDo Report 11", "1"); }); -}); \ No newline at end of file +}); diff --git a/cypress/integration/recorder.js b/cypress/integration/recorder.js index 7d4c83abf5..de95a852fc 100644 --- a/cypress/integration/recorder.js +++ b/cypress/integration/recorder.js @@ -1,66 +1,72 @@ -context('Recorder', () => { +context.skip("Recorder", () => { before(() => { cy.login(); }); beforeEach(() => { - cy.visit('/app/recorder'); - return cy.window().its('frappe').then(frappe => { - // reset recorder - return frappe.xcall("frappe.recorder.stop").then(() => { - return frappe.xcall("frappe.recorder.delete"); + cy.visit("/app/recorder"); + return cy + .window() + .its("frappe") + .then((frappe) => { + // reset recorder + return frappe.xcall("frappe.recorder.stop").then(() => { + return frappe.xcall("frappe.recorder.delete"); + }); }); - }); }); - it('Recorder Empty State', () => { - cy.get('.page-head').findByTitle('Recorder').should('exist'); + it("Recorder Empty State", () => { + cy.get(".page-head").findByTitle("Recorder").should("exist"); - cy.get('.indicator-pill').should('contain', 'Inactive').should('have.class', 'red'); + cy.get(".indicator-pill").should("contain", "Inactive").should("have.class", "red"); - cy.get('.page-actions').findByRole('button', {name: 'Start'}).should('exist'); - cy.get('.page-actions').findByRole('button', {name: 'Clear'}).should('exist'); + cy.get(".page-actions").findByRole("button", { name: "Start" }).should("exist"); + cy.get(".page-actions").findByRole("button", { name: "Clear" }).should("exist"); - cy.get('.msg-box').should('contain', 'Recorder is Inactive'); - cy.get('.msg-box').findByRole('button', {name: 'Start Recording'}).should('exist'); + cy.get(".msg-box").should("contain", "Recorder is Inactive"); + cy.get(".msg-box").findByRole("button", { name: "Start Recording" }).should("exist"); }); - it('Recorder Start', () => { - cy.get('.page-actions').findByRole('button', {name: 'Start'}).click(); - cy.get('.indicator-pill').should('contain', 'Active').should('have.class', 'green'); + it("Recorder Start", () => { + cy.get(".page-actions").findByRole("button", { name: "Start" }).click(); + cy.get(".indicator-pill").should("contain", "Active").should("have.class", "green"); - cy.get('.msg-box').should('contain', 'No Requests found'); + cy.get(".msg-box").should("contain", "No Requests found"); - cy.visit('/app/List/DocType/List'); - cy.intercept('POST', '/api/method/frappe.desk.reportview.get').as('list_refresh'); - cy.wait('@list_refresh'); + cy.visit("/app/List/DocType/List"); + cy.intercept("POST", "/api/method/frappe.desk.reportview.get").as("list_refresh"); + cy.wait("@list_refresh"); - cy.get('.page-head').findByTitle('DocType').should('exist'); - cy.get('.list-count').should('contain', '20 of '); + cy.get(".page-head").findByTitle("DocType").should("exist"); + cy.get(".list-count").should("contain", "20 of "); - cy.visit('/app/recorder'); - cy.get('.page-head').findByTitle('Recorder').should('exist'); - cy.get('.frappe-list .result-list').should('contain', '/api/method/frappe.desk.reportview.get'); + cy.visit("/app/recorder"); + cy.get(".page-head").findByTitle("Recorder").should("exist"); + cy.get(".frappe-list .result-list").should( + "contain", + "/api/method/frappe.desk.reportview.get" + ); }); - it('Recorder View Request', () => { - cy.get('.page-actions').findByRole('button', {name: 'Start'}).click(); + it("Recorder View Request", () => { + cy.get(".page-actions").findByRole("button", { name: "Start" }).click(); - cy.visit('/app/List/DocType/List'); - cy.intercept('POST', '/api/method/frappe.desk.reportview.get').as('list_refresh'); - cy.wait('@list_refresh'); + cy.visit("/app/List/DocType/List"); + cy.intercept("POST", "/api/method/frappe.desk.reportview.get").as("list_refresh"); + cy.wait("@list_refresh"); - cy.get('.page-head').findByTitle('DocType').should('exist'); - cy.get('.list-count').should('contain', '20 of '); + cy.get(".page-head").findByTitle("DocType").should("exist"); + cy.get(".list-count").should("contain", "20 of "); - cy.visit('/app/recorder'); + cy.visit("/app/recorder"); - cy.get('.frappe-list .list-row-container span') - .contains('/api/method/frappe') - .should('be.visible') - .click({force: true}); + cy.get(".frappe-list .list-row-container span") + .contains("/api/method/frappe") + .should("be.visible") + .click({ force: true }); - cy.url().should('include', '/recorder/request'); - cy.get('form').should('contain', '/api/method/frappe'); + cy.url().should("include", "/recorder/request"); + cy.get("form").should("contain", "/api/method/frappe"); }); }); diff --git a/cypress/integration/report_view.js b/cypress/integration/report_view.js index bacbf9c172..27fe840450 100644 --- a/cypress/integration/report_view.js +++ b/cypress/integration/report_view.js @@ -1,42 +1,46 @@ -import custom_submittable_doctype from '../fixtures/custom_submittable_doctype'; +import custom_submittable_doctype from "../fixtures/custom_submittable_doctype"; const doctype_name = custom_submittable_doctype.name; -context('Report View', () => { +context("Report View", () => { before(() => { cy.login(); - cy.visit('/app/website'); - cy.insert_doc('DocType', custom_submittable_doctype, true); + cy.visit("/app/website"); + cy.insert_doc("DocType", custom_submittable_doctype, true); cy.clear_cache(); - cy.insert_doc(doctype_name, { - 'title': 'Doc 1', - 'description': 'Random Text', - 'enabled': 0, - 'docstatus': 1 // submit document - }, true); + cy.insert_doc( + doctype_name, + { + title: "Doc 1", + description: "Random Text", + enabled: 0, + docstatus: 1, // submit document + }, + true + ); }); - it('Field with enabled allow_on_submit should be editable.', () => { - cy.intercept('POST', 'api/method/frappe.client.set_value').as('value-update'); + it("Field with enabled allow_on_submit should be editable.", () => { + cy.intercept("POST", "api/method/frappe.client.set_value").as("value-update"); cy.visit(`/app/List/${doctype_name}/Report`); // check status column added from docstatus - cy.get('.dt-row-0 > .dt-cell--col-3').should('contain', 'Submitted'); - let cell = cy.get('.dt-row-0 > .dt-cell--col-4'); + cy.get(".dt-row-0 > .dt-cell--col-3").should("contain", "Submitted"); + let cell = cy.get(".dt-row-0 > .dt-cell--col-4"); // select the cell cell.dblclick(); - cell.get('.dt-cell__edit--col-4').findByRole('checkbox').check({ force: true }); - cy.get('.dt-row-0 > .dt-cell--col-3').click(); // click outside + cell.get(".dt-cell__edit--col-4").findByRole("checkbox").check({ force: true }); + cy.get(".dt-row-0 > .dt-cell--col-3").click(); // click outside - cy.wait('@value-update'); + cy.wait("@value-update"); - cy.call('frappe.client.get_value', { + cy.call("frappe.client.get_value", { doctype: doctype_name, filters: { - title: 'Doc 1', + title: "Doc 1", }, - fieldname: 'enabled' - }).then(r => { + fieldname: "enabled", + }).then((r) => { expect(r.message.enabled).to.equals(1); }); }); diff --git a/cypress/integration/rounding.js b/cypress/integration/rounding.js new file mode 100644 index 0000000000..1a9cfa685b --- /dev/null +++ b/cypress/integration/rounding.js @@ -0,0 +1,104 @@ +context("Rounding behaviour", () => { + before(() => { + cy.login(); + cy.visit("/app/"); + }); + + it("Commercial Rounding", () => { + cy.window() + .its("flt") + .then((flt) => { + let rounding_method = "Commercial Rounding"; + + expect(flt("0.5", 0, null, rounding_method)).eq(1); + expect(flt("0.3", null, null, rounding_method)).eq(0.3); + + expect(flt("1.5", 0, null, rounding_method)).eq(2); + + // positive rounding to integers + expect(flt(0.4, 0, null, rounding_method)).eq(0); + expect(flt(0.5, 0, null, rounding_method)).eq(1); + expect(flt(1.455, 0, null, rounding_method)).eq(1); + expect(flt(1.5, 0, null, rounding_method)).eq(2); + + // negative rounding to integers + expect(flt(-0.5, 0, null, rounding_method)).eq(-1); + expect(flt(-1.5, 0, null, rounding_method)).eq(-2); + + // negative precision i.e. round to nearest 10th + expect(flt(123, -1, null, rounding_method)).eq(120); + expect(flt(125, -1, null, rounding_method)).eq(130); + expect(flt(134.45, -1, null, rounding_method)).eq(130); + expect(flt(135, -1, null, rounding_method)).eq(140); + + // positive multiple digit rounding + expect(flt(1.25, 1, null, rounding_method)).eq(1.3); + expect(flt(0.15, 1, null, rounding_method)).eq(0.2); + expect(flt(2.675, 2, null, rounding_method)).eq(2.68); + + // negative multiple digit rounding + expect(flt(-1.25, 1, null, rounding_method)).eq(-1.3); + expect(flt(-0.15, 1, null, rounding_method)).eq(-0.2); + }); + }); + + it("Banker's Rounding", () => { + cy.window() + .its("flt") + .then((flt) => { + let rounding_method = "Banker's Rounding"; + + expect(flt("0.5", 0, null, rounding_method)).eq(0); + expect(flt("0.3", null, rounding_method)).eq(0.3); + + expect(flt("1.5", 0, null, rounding_method)).eq(2); + + // positive rounding to integers + expect(flt(0.4, 0, null, rounding_method)).eq(0); + expect(flt(0.5, 0, null, rounding_method)).eq(0); + expect(flt(1.455, 0, null, rounding_method)).eq(1); + expect(flt(1.5, 0, null, rounding_method)).eq(2); + + // negative rounding to integers + expect(flt(-0.5, 0, null, rounding_method)).eq(0); + expect(flt(-1.5, 0, null, rounding_method)).eq(-2); + + // negative precision i.e. round to nearest 10th + expect(flt(123, -1, null, rounding_method)).eq(120); + expect(flt(125, -1, null, rounding_method)).eq(120); + expect(flt(134.45, -1, null, rounding_method)).eq(130); + expect(flt(135, -1, null, rounding_method)).eq(140); + + // positive multiple digit rounding + expect(flt(1.25, 1, null, rounding_method)).eq(1.2); + expect(flt(0.15, 1, null, rounding_method)).eq(0.2); + expect(flt(2.675, 2, null, rounding_method)).eq(2.68); + expect(flt(-2.675, 2, null, rounding_method)).eq(-2.68); + + // negative multiple digit rounding + expect(flt(-1.25, 1, null, rounding_method)).eq(-1.2); + expect(flt(-0.15, 1, null, rounding_method)).eq(-0.2); + + // Nearest number and not even (the default behaviour) + expect(flt(0.5, 0, null, rounding_method)).eq(0); + expect(flt(1.5, 0, null, rounding_method)).eq(2); + expect(flt(2.5, 0, null, rounding_method)).eq(2); + expect(flt(3.5, 0, null, rounding_method)).eq(4); + + expect(flt(0.05, 1, null, rounding_method)).eq(0.0); + expect(flt(1.15, 1, null, rounding_method)).eq(1.2); + expect(flt(2.25, 1, null, rounding_method)).eq(2.2); + expect(flt(3.35, 1, null, rounding_method)).eq(3.4); + + expect(flt(-0.5, 0, null, rounding_method)).eq(0); + expect(flt(-1.5, 0, null, rounding_method)).eq(-2); + expect(flt(-2.5, 0, null, rounding_method)).eq(-2); + expect(flt(-3.5, 0, null, rounding_method)).eq(-4); + + expect(flt(-0.05, 1, null, rounding_method)).eq(0.0); + expect(flt(-1.15, 1, null, rounding_method)).eq(-1.2); + expect(flt(-2.25, 1, null, rounding_method)).eq(-2.2); + expect(flt(-3.35, 1, null, rounding_method)).eq(-3.4); + }); + }); +}); diff --git a/cypress/integration/sidebar.js b/cypress/integration/sidebar.js index 2831c9bad5..0f97cdc7fe 100644 --- a/cypress/integration/sidebar.js +++ b/cypress/integration/sidebar.js @@ -1,55 +1,86 @@ -context('Sidebar', () => { +const verify_attachment_visibility = (document, is_private) => { + cy.visit(`/app/${document}`); + + const assertion = is_private ? "be.checked" : "not.be.checked"; + cy.findByRole("button", { name: "Attach File" }).click(); + + cy.get_open_dialog() + .find(".file-upload-area") + .selectFile("cypress/fixtures/sample_image.jpg", { + action: "drag-drop", + }); + + cy.get_open_dialog().findByRole("checkbox", { name: "Private" }).should(assertion); +}; + +context("Sidebar", () => { before(() => { - cy.visit('/login'); + cy.visit("/login"); cy.login(); - cy.visit('/app/doctype'); + + return cy + .window() + .its("frappe") + .then((frappe) => { + return frappe.call("frappe.tests.ui_test_helpers.create_blog_post"); + }); + }); + + it("Verify attachment visibility config", () => { + verify_attachment_visibility("doctype/Blog Post", true); + verify_attachment_visibility("blog-post/test-blog-attachment-post", false); }); it('Test for checking "Assigned To" counter value, adding filter and adding & removing an assignment', () => { + cy.visit("/app/doctype"); cy.click_sidebar_button("Assigned To"); //To check if no filter is available in "Assigned To" dropdown - cy.get('.empty-state').should('contain', 'No filters found'); - - cy.click_sidebar_button("Created By"); - - //To check if "Created By" dropdown contains filter - cy.get('.group-by-item > .dropdown-item').should('contain', 'Me'); + cy.get(".empty-state").should("contain", "No filters found"); //Assigning a doctype to a user - cy.visit('/app/doctype/ToDo'); - cy.get('.form-assignments > .flex > .text-muted').click(); - cy.get_field('assign_to_me', 'Check').click(); - cy.get('.modal-footer > .standard-actions > .btn-primary').click(); - cy.visit('/app/doctype'); + cy.visit("/app/doctype/ToDo"); + cy.get(".form-assignments > .flex > .text-muted").click(); + cy.get_field("assign_to_me", "Check").click(); + cy.get(".modal-footer > .standard-actions > .btn-primary").click(); + cy.visit("/app/doctype"); cy.click_sidebar_button("Assigned To"); //To check if filter is added in "Assigned To" dropdown after assignment - cy.get('.group-by-field.show > .dropdown-menu > .group-by-item > .dropdown-item').should('contain', '1'); + cy.get(".group-by-field.show > .dropdown-menu > .group-by-item > .dropdown-item").should( + "contain", + "1" + ); //To check if there is no filter added to the listview - cy.get('.filter-selector > .btn').should('contain', 'Filter'); + cy.get(".filter-button").should("contain", "Filter"); //To add a filter to display data into the listview - cy.get('.group-by-field.show > .dropdown-menu > .group-by-item > .dropdown-item').click(); + cy.get(".group-by-field.show > .dropdown-menu > .group-by-item > .dropdown-item").click(); //To check if filter is applied - cy.click_filter_button().should('contain', '1 filter'); - cy.get('.fieldname-select-area > .awesomplete > .form-control').should('have.value', 'Assigned To'); - cy.get('.condition').should('have.value', 'like'); - cy.get('.filter-field > .form-group > .input-with-feedback').should('have.value', '%Administrator%'); + cy.click_filter_button().should("contain", "1 filter"); + cy.get(".fieldname-select-area > .awesomplete > .form-control").should( + "have.value", + "Assigned To" + ); + cy.get(".condition").should("have.value", "like"); + cy.get(".filter-field > .form-group > .input-with-feedback").should( + "have.value", + `%${cy.config("testUser")}%` + ); cy.click_filter_button(); //To remove the applied filter cy.clear_filters(); //To remove the assignment - cy.visit('/app/doctype/ToDo'); - cy.get('.assignments > .avatar-group > .avatar > .avatar-frame').click(); - cy.get('.remove-btn').click({force: true}); + cy.visit("/app/doctype/ToDo"); + cy.get(".assignments > .avatar-group > .avatar > .avatar-frame").click(); + cy.get(".remove-btn").click({ force: true }); cy.hide_dialog(); - cy.visit('/app/doctype'); + cy.visit("/app/doctype"); cy.click_sidebar_button("Assigned To"); - cy.get('.empty-state').should('contain', 'No filters found'); + cy.get(".empty-state").should("contain", "No filters found"); }); }); diff --git a/cypress/integration/table_multiselect.js b/cypress/integration/table_multiselect.js index ae93354964..8261b5b384 100644 --- a/cypress/integration/table_multiselect.js +++ b/cypress/integration/table_multiselect.js @@ -1,51 +1,59 @@ -context('Table MultiSelect', () => { +context("Table MultiSelect", () => { before(() => { cy.login(); }); - let name = 'table multiselect' + Math.random().toString().slice(2, 8); + let name = "table multiselect" + Math.random().toString().slice(2, 8); - it('select value from multiselect dropdown', () => { - cy.new_form('Assignment Rule'); - cy.fill_field('__newname', name); - cy.fill_field('document_type', 'Blog Post'); - cy.get('.section-head').contains('Assignment Rules').scrollIntoView(); - cy.fill_field('assign_condition', 'status=="Open"', 'Code'); - cy.get('input[data-fieldname="users"]').focus().as('input'); - cy.get('input[data-fieldname="users"] + ul').should('be.visible'); - cy.get('@input').type('test{enter}', { delay: 100 }); - cy.get('.frappe-control[data-fieldname="users"] .form-control .tb-selected-value .btn-link-to-form') - .as('selected-value'); - cy.get('@selected-value').should('contain', 'test@erpnext.com'); + it("select value from multiselect dropdown", () => { + cy.new_form("Assignment Rule"); + cy.fill_field("__newname", name); + cy.fill_field("document_type", "Blog Post"); + cy.get(".section-head").contains("Assignment Rules").scrollIntoView(); + cy.fill_field("assign_condition", 'status=="Open"', "Code"); + cy.get('input[data-fieldname="users"]').focus().as("input"); + cy.get('input[data-fieldname="users"] + ul').should("be.visible"); + cy.get("@input").type("test@erpnext", { delay: 100 }); + cy.wait(500); + cy.get("@input").type("{enter}"); + cy.get( + '.frappe-control[data-fieldname="users"] .form-control .tb-selected-value .btn-link-to-form' + ).as("selected-value"); + cy.get("@selected-value").should("contain", "test@erpnext.com"); - cy.intercept('POST', '/api/method/frappe.desk.form.save.savedocs').as('save_form'); + cy.intercept("POST", "/api/method/frappe.desk.form.save.savedocs").as("save_form"); // trigger save - cy.get('.primary-action').click(); - cy.wait('@save_form').its('response.statusCode').should('eq', 200); - cy.get('@selected-value').should('contain', 'test@erpnext.com'); + cy.get(".primary-action").click(); + cy.wait("@save_form").its("response.statusCode").should("eq", 200); + cy.get("@selected-value").should("contain", "test@erpnext.com"); }); - it('delete value using backspace', () => { - cy.go_to_list('Assignment Rule'); - cy.get(`.list-subject:contains("table multiselect")`).last().find('a').click(); - cy.get('input[data-fieldname="users"]').focus().type('{backspace}'); - cy.get('.frappe-control[data-fieldname="users"] .form-control .tb-selected-value') - .should('not.exist'); + it("delete value using backspace", () => { + cy.go_to_list("Assignment Rule"); + cy.get(`.list-subject:contains("table multiselect")`).last().find("a").click(); + cy.get('input[data-fieldname="users"]').focus().type("{backspace}"); + cy.get('.frappe-control[data-fieldname="users"] .form-control .tb-selected-value').should( + "not.exist" + ); }); - it('delete value using x', () => { - cy.go_to_list('Assignment Rule'); - cy.get(`.list-subject:contains("table multiselect")`).last().find('a').click(); - cy.get('.frappe-control[data-fieldname="users"] .form-control .tb-selected-value').as('existing_value'); - cy.get('@existing_value').find('.btn-remove').click(); - cy.get('@existing_value').should('not.exist'); + it("delete value using x", () => { + cy.go_to_list("Assignment Rule"); + cy.get(`.list-subject:contains("table multiselect")`).last().find("a").click(); + cy.get('.frappe-control[data-fieldname="users"] .form-control .tb-selected-value').as( + "existing_value" + ); + cy.get("@existing_value").find(".btn-remove").click(); + cy.get("@existing_value").should("not.exist"); }); - it('navigate to selected value', () => { - cy.go_to_list('Assignment Rule'); - cy.get(`.list-subject:contains("table multiselect")`).last().find('a').click(); - cy.get('.frappe-control[data-fieldname="users"] .form-control .tb-selected-value').as('existing_value'); - cy.get('@existing_value').find('.btn-link-to-form').click(); - cy.location('pathname').should('contain', '/user/test@erpnext.com'); + it("navigate to selected value", () => { + cy.go_to_list("Assignment Rule"); + cy.get(`.list-subject:contains("table multiselect")`).last().find("a").click(); + cy.get('.frappe-control[data-fieldname="users"] .form-control .tb-selected-value').as( + "existing_value" + ); + cy.get("@existing_value").find(".btn-link-to-form").click(); + cy.location("pathname").should("contain", "/user/test%40erpnext.com"); }); -}); \ No newline at end of file +}); diff --git a/cypress/integration/theme_switcher_dialog.js b/cypress/integration/theme_switcher_dialog.js index b4297e5674..158ff3e244 100644 --- a/cypress/integration/theme_switcher_dialog.js +++ b/cypress/integration/theme_switcher_dialog.js @@ -1,30 +1,29 @@ -context('Theme Switcher Shortcut', () => { +context("Theme Switcher Shortcut", () => { before(() => { cy.login(); - cy.visit('/app'); + cy.visit("/app"); }); beforeEach(() => { cy.reload(); }); - it('Check Toggle', () => { - cy.open_theme_dialog('{ctrl+shift+g}'); - cy.get('.modal-backdrop').should('exist'); - cy.get('.theme-grid > div').first().click(); - cy.close_theme('{ctrl+shift+g}'); - cy.get('.modal-backdrop').should('not.exist'); + it("Check Toggle", () => { + cy.open_theme_dialog("{ctrl+shift+g}"); + cy.get(".modal-backdrop").should("exist"); + cy.get(".theme-grid > div").first().click(); + cy.close_theme("{ctrl+shift+g}"); + cy.get(".modal-backdrop").should("not.exist"); }); - it('Check Enter', () => { - cy.open_theme_dialog('{ctrl+shift+g}'); - cy.get('.theme-grid > div').first().click(); - cy.close_theme('{enter}'); - cy.get('.modal-backdrop').should('not.exist'); + it("Check Enter", () => { + cy.open_theme_dialog("{ctrl+shift+g}"); + cy.get(".theme-grid > div").first().click(); + cy.close_theme("{enter}"); + cy.get(".modal-backdrop").should("not.exist"); }); - }); -Cypress.Commands.add('open_theme_dialog', (shortcut_keys) => { - cy.get('body').type(shortcut_keys); +Cypress.Commands.add("open_theme_dialog", (shortcut_keys) => { + cy.get("body").type(shortcut_keys); +}); +Cypress.Commands.add("close_theme", (shortcut_keys) => { + cy.get(".modal-header").type(shortcut_keys); }); -Cypress.Commands.add('close_theme', (shortcut_keys) => { - cy.get('.modal-header').type(shortcut_keys); -}); \ No newline at end of file diff --git a/cypress/integration/timeline.js b/cypress/integration/timeline.js index e7308fbaa7..c6076088fb 100644 --- a/cypress/integration/timeline.js +++ b/cypress/integration/timeline.js @@ -1,83 +1,91 @@ -import custom_submittable_doctype from '../fixtures/custom_submittable_doctype'; +import custom_submittable_doctype from "../fixtures/custom_submittable_doctype"; -context('Timeline', () => { +context("Timeline", () => { before(() => { - cy.visit('/login'); + cy.visit("/login"); cy.login(); }); - it('Adding new ToDo, adding new comment, verifying comment addition & deletion and deleting ToDo', () => { + it("Adding new ToDo, adding new comment, verifying comment addition & deletion and deleting ToDo", () => { //Adding new ToDo - cy.visit('/app/todo/new-todo-1'); - cy.get('[data-fieldname="description"] .ql-editor.ql-blank').type('Test ToDo', {force: true}).wait(200); - cy.get('.page-head .page-actions').findByRole('button', {name: 'Save'}).click(); + cy.visit("/app/todo/new-todo-1"); + cy.get('[data-fieldname="description"] .ql-editor.ql-blank') + .type("Test ToDo", { force: true }) + .wait(200); + cy.get(".page-head .page-actions").findByRole("button", { name: "Save" }).click(); - cy.go_to_list('ToDo'); - cy.clear_filters() + cy.go_to_list("ToDo"); + cy.clear_filters(); cy.click_listview_row_item(0); //To check if the comment box is initially empty and tying some text into it - cy.get('[data-fieldname="comment"] .ql-editor').should('contain', '').type('Testing Timeline'); + cy.get('[data-fieldname="comment"] .ql-editor') + .should("contain", "") + .type("Testing Timeline"); //Adding new comment - cy.get('.comment-box').findByRole('button', {name: 'Comment'}).click(); + cy.get(".comment-box").findByRole("button", { name: "Comment" }).click(); //To check if the commented text is visible in the timeline content - cy.get('.timeline-content').should('contain', 'Testing Timeline'); + cy.get(".timeline-content").should("contain", "Testing Timeline"); //Editing comment cy.click_timeline_action_btn("Edit"); - cy.get('.timeline-content [data-fieldname="comment"] .ql-editor').first().type(' 123'); + cy.get('.timeline-content [data-fieldname="comment"] .ql-editor').first().type(" 123"); cy.click_timeline_action_btn("Save"); //To check if the edited comment text is visible in timeline content - cy.get('.timeline-content').should('contain', 'Testing Timeline 123'); + cy.get(".timeline-content").should("contain", "Testing Timeline 123"); //Discarding comment cy.click_timeline_action_btn("Edit"); cy.click_timeline_action_btn("Dismiss"); //To check if after discarding the timeline content is same as previous - cy.get('.timeline-content').should('contain', 'Testing Timeline 123'); + cy.get(".timeline-content").should("contain", "Testing Timeline 123"); //Deleting the added comment - cy.get('.timeline-message-box .more-actions > .action-btn').click(); //Menu button in timeline item - cy.get('.timeline-message-box .more-actions .dropdown-item').contains('Delete').click({ force: true }); - cy.get_open_dialog().findByRole('button', {name: 'Yes'}).click({ force: true }); + cy.get(".timeline-message-box .more-actions > .action-btn").click(); //Menu button in timeline item + cy.get(".timeline-message-box .more-actions .dropdown-item") + .contains("Delete") + .click({ force: true }); + cy.get_open_dialog().findByRole("button", { name: "Yes" }).click({ force: true }); - cy.get('.timeline-content').should('not.contain', 'Testing Timeline 123'); + cy.get(".timeline-content").should("not.contain", "Testing Timeline 123"); }); - it('Timeline should have submit and cancel activity information', () => { - cy.visit('/app/doctype'); + it("Timeline should have submit and cancel activity information", () => { + cy.visit("/app/doctype"); //Creating custom doctype - cy.insert_doc('DocType', custom_submittable_doctype, true); + cy.insert_doc("DocType", custom_submittable_doctype, true); - cy.visit('/app/custom-submittable-doctype'); - cy.click_listview_primary_button('Add Custom Submittable DocType'); + cy.visit("/app/custom-submittable-doctype"); + cy.click_listview_primary_button("Add Custom Submittable DocType"); //Adding a new entry for the created custom doctype - cy.fill_field('title', 'Test'); - cy.click_modal_primary_button('Save'); - cy.click_modal_primary_button('Submit'); + cy.fill_field("title", "Test"); + cy.click_modal_primary_button("Save"); + cy.click_modal_primary_button("Submit"); - cy.visit('/app/custom-submittable-doctype'); + cy.visit("/app/custom-submittable-doctype"); cy.click_listview_row_item(0); //To check if the submission of the documemt is visible in the timeline content - cy.get('.timeline-content').should('contain', 'Administrator submitted this document'); - cy.get('[id="page-Custom Submittable DocType"] .page-actions').findByRole('button', {name: 'Cancel'}).click(); - cy.get_open_dialog().findByRole('button', {name: 'Yes'}).click(); + cy.get(".timeline-content").should("contain", "You submitted this document"); + cy.get('[id="page-Custom Submittable DocType"] .page-actions') + .findByRole("button", { name: "Cancel" }) + .click(); + cy.get_open_dialog().findByRole("button", { name: "Yes" }).click(); //To check if the cancellation of the documemt is visible in the timeline content - cy.get('.timeline-content').should('contain', 'Administrator cancelled this document'); + cy.get(".timeline-content").should("contain", "You cancelled this document"); //Deleting the document - cy.visit('/app/custom-submittable-doctype'); + cy.visit("/app/custom-submittable-doctype"); cy.select_listview_row_checkbox(0); - cy.get('.page-actions').findByRole('button', {name: 'Actions'}).click(); + cy.get(".page-actions").findByRole("button", { name: "Actions" }).click(); cy.get('.page-actions .actions-btn-group [data-label="Delete"]').click(); - cy.click_modal_primary_button('Yes'); + cy.click_modal_primary_button("Yes"); }); }); diff --git a/cypress/integration/timeline_email.js b/cypress/integration/timeline_email.js deleted file mode 100644 index 993847bcb8..0000000000 --- a/cypress/integration/timeline_email.js +++ /dev/null @@ -1,76 +0,0 @@ -context('Timeline Email', () => { - before(() => { - cy.visit('/login'); - cy.login(); - cy.visit('/app/todo'); - }); - - it('Adding new ToDo', () => { - cy.click_listview_primary_button('Add ToDo'); - cy.get('.custom-actions:visible > .btn').contains("Edit Full Form").click({delay: 500}); - cy.fill_field("description", "Test ToDo", "Text Editor"); - cy.wait(500); - cy.get('.primary-action').contains('Save').click({force: true}); - cy.wait(700); - }); - - it('Adding email and verifying timeline content for email attachment', () => { - cy.visit('/app/todo'); - cy.click_listview_row_item_with_text('Test ToDo'); - - //Creating a new email - cy.get('.timeline-actions > .timeline-item > .action-buttons > .action-btn').click(); - cy.fill_field('recipients', 'test@example.com', 'MultiSelect'); - cy.get('.modal.show > .modal-dialog > .modal-content > .modal-body > :nth-child(1) > .form-layout > .form-page > :nth-child(3) > .section-body > .form-column > form > [data-fieldtype="Text Editor"] > .form-group > .control-input-wrapper > .control-input > .ql-container > .ql-editor').type('Test Mail'); - - //Adding attachment to the email - cy.get('.add-more-attachments > .btn').click(); - cy.get('.mt-2 > .btn > .mt-1').eq(2).click(); - cy.get('.input-group > .form-control').type('https://wallpaperplay.com/walls/full/8/2/b/72402.jpg'); - cy.get('.btn-primary').contains('Upload').click(); - - //Sending the email - cy.click_modal_primary_button('Send', {delay: 500}); - - //To check if the sent mail content is shown in the timeline content - cy.get('[data-doctype="Communication"] > .timeline-content').should('contain', 'Test Mail'); - - //To check if the attachment of email is shown in the timeline content - cy.get('.timeline-content').should('contain', 'Added 72402.jpg'); - - //Deleting the sent email - cy.get('[title="Open Communication"] > .icon').first().click({force: true}); - cy.get('#page-Communication > .page-head > .container > .row > .col > .standard-actions > .menu-btn-group > .btn').click(); - cy.get('#page-Communication > .page-head > .container > .row > .col > .standard-actions > .menu-btn-group > .dropdown-menu > li > .grey-link').eq(9).click(); - cy.get('.modal.show > .modal-dialog > .modal-content > .modal-footer > .standard-actions > .btn-primary').click(); - }); - - it('Deleting attachment and ToDo', () => { - cy.visit('/app/todo'); - cy.click_listview_row_item_with_text('Test ToDo'); - - //Removing the added attachment - cy.get('.attachment-row > .data-pill > .remove-btn > .icon').click(); - cy.wait(500); - cy.get('.modal-footer:visible > .standard-actions > .btn-primary').contains('Yes').click(); - - //To check if the removed attachment is shown in the timeline content - cy.get('.timeline-content').should('contain', 'Removed 72402.jpg'); - cy.wait(500); - - //To check if the discard button functionality in email is working correctly - cy.get('.timeline-actions > .timeline-item > .action-buttons > .action-btn').click(); - cy.fill_field('recipients', 'test@example.com', 'MultiSelect'); - cy.get('.modal-footer > .standard-actions > .btn-secondary').contains('Discard').click(); - cy.wait(500); - cy.get('.timeline-actions > .timeline-item > .action-buttons > .action-btn').click(); - cy.wait(500); - cy.get_field('recipients', 'MultiSelect').should('have.text', ''); - cy.get('.modal-header:visible > .modal-actions > .btn-modal-close > .icon').click(); - - //Deleting the added ToDo - cy.get('.menu-btn-group:visible > .btn').click(); - cy.get('.menu-btn-group:visible > .dropdown-menu > li > .dropdown-item').contains('Delete').click(); - cy.get('.modal-footer:visible > .standard-actions > .btn-primary').click(); - }); -}); diff --git a/cypress/integration/url_data_field.js b/cypress/integration/url_data_field.js index cf22c62363..c74bc8b1a2 100644 --- a/cypress/integration/url_data_field.js +++ b/cypress/integration/url_data_field.js @@ -1,43 +1,42 @@ -import data_field_validation_doctype from '../fixtures/data_field_validation_doctype'; +import data_field_validation_doctype from "../fixtures/data_field_validation_doctype"; const doctype_name = data_field_validation_doctype.name; -context('URL Data Field Input', () => { +context("URL Data Field Input", () => { before(() => { cy.login(); - cy.visit('/app/website'); - return cy.insert_doc('DocType', data_field_validation_doctype, true); + cy.visit("/app/website"); + return cy.insert_doc("DocType", data_field_validation_doctype, true); }); - - describe('URL Data Field Input ', () => { - it('should not show URL link button without focus', () => { + describe("URL Data Field Input ", () => { + it("should not show URL link button without focus", () => { cy.new_form(doctype_name); - cy.get_field('url').clear().type('https://frappe.io'); - cy.get_field('url').blur().wait(500); - cy.get('.link-btn').should('not.be.visible'); + cy.get_field("url").clear().type("https://frappe.io"); + cy.get_field("url").blur().wait(500); + cy.get(".link-btn").should("not.be.visible"); }); - it('should show URL link button on focus', () => { - cy.get_field('url').focus().wait(500); - cy.get('.link-btn').should('be.visible'); + it("should show URL link button on focus", () => { + cy.get_field("url").focus().wait(500); + cy.get(".link-btn").should("be.visible"); }); - it('should not show URL link button for invalid URL', () => { - cy.get_field('url').clear().type('fuzzbuzz'); - cy.get('.link-btn').should('not.be.visible'); + it("should not show URL link button for invalid URL", () => { + cy.get_field("url").clear().type("fuzzbuzz"); + cy.get(".link-btn").should("not.be.visible"); }); - it('should have valid URL link with target _blank', () => { - cy.get_field('url').clear().type('https://frappe.io'); - cy.get('.link-btn .btn-open').should('have.attr', 'href', 'https://frappe.io'); - cy.get('.link-btn .btn-open').should('have.attr', 'target', '_blank'); + it("should have valid URL link with target _blank", () => { + cy.get_field("url").clear().type("https://frappe.io"); + cy.get(".link-btn .btn-open").should("have.attr", "href", "https://frappe.io"); + cy.get(".link-btn .btn-open").should("have.attr", "target", "_blank"); }); - it('should inject anchor tag in read-only URL data field', () => { + it("should inject anchor tag in read-only URL data field", () => { cy.get('[data-fieldname="read_only_url"]') - .find('a') - .should('have.attr', 'target', '_blank'); + .find("a") + .should("have.attr", "target", "_blank"); }); }); -}); \ No newline at end of file +}); diff --git a/cypress/integration/view_routing.js b/cypress/integration/view_routing.js new file mode 100644 index 0000000000..72fb6836ec --- /dev/null +++ b/cypress/integration/view_routing.js @@ -0,0 +1,231 @@ +context("View", () => { + before(() => { + cy.login(); + cy.visit("/app/website"); + }); + + it("Route to ToDo List View", () => { + cy.visit("/app/todo/view/list"); + cy.wait(500); + cy.window() + .its("cur_list") + .then((list) => { + expect(list.view_name).to.equal("List"); + }); + }); + + it("Route to ToDo Report View", () => { + cy.visit("/app/todo/view/report"); + cy.wait(500); + cy.window() + .its("cur_list") + .then((list) => { + expect(list.view_name).to.equal("Report"); + }); + }); + + it("Route to ToDo Dashboard View", () => { + cy.visit("/app/todo/view/dashboard"); + cy.wait(500); + cy.window() + .its("cur_list") + .then((list) => { + expect(list.view_name).to.equal("Dashboard"); + }); + }); + + it("Route to ToDo Gantt View", () => { + cy.visit("/app/todo/view/gantt"); + cy.wait(500); + cy.window() + .its("cur_list") + .then((list) => { + expect(list.view_name).to.equal("Gantt"); + }); + }); + + it("Route to ToDo Kanban View", () => { + cy.call("frappe.tests.ui_test_helpers.create_kanban").then(() => { + cy.visit("/app/note/view/kanban/_Note _Kanban"); + cy.wait(500); + cy.window() + .its("cur_list") + .then((list) => { + expect(list.view_name).to.equal("Kanban"); + }); + }); + }); + + it("Route to ToDo Calendar View", () => { + cy.visit("/app/todo/view/calendar"); + cy.wait(500); + cy.window() + .its("cur_list") + .then((list) => { + expect(list.view_name).to.equal("Calendar"); + }); + }); + + it("Route to Custom Tree View", () => { + cy.call("frappe.tests.ui_test_helpers.setup_tree_doctype").then(() => { + cy.visit("/app/custom-tree/view/tree"); + cy.wait(500); + cy.window() + .its("cur_tree") + .then((list) => { + expect(list.view_name).to.equal("Tree"); + }); + }); + }); + + it("Route to Custom Image View", () => { + cy.call("frappe.tests.ui_test_helpers.setup_image_doctype").then(() => { + cy.visit("app/custom-image/view/image"); + cy.wait(500); + cy.window() + .its("cur_list") + .then((list) => { + expect(list.view_name).to.equal("Image"); + }); + }); + }); + + it("Route to Communication Inbox View", () => { + cy.call("frappe.tests.ui_test_helpers.setup_inbox").then(() => { + cy.visit("app/communication/view/inbox"); + cy.wait(500); + cy.window() + .its("cur_list") + .then((list) => { + expect(list.view_name).to.equal("Inbox"); + }); + }); + }); + + it("Route to File View", () => { + cy.intercept("POST", "/api/method/frappe.desk.reportview.get").as("list_loaded"); + cy.visit("app/file"); + cy.wait("@list_loaded"); + cy.window() + .its("cur_list") + .then((list) => { + expect(list.view_name).to.equal("File"); + expect(list.current_folder).to.equal("Home"); + }); + + cy.visit("app/file/view/home/Attachments"); + cy.wait("@list_loaded"); + cy.window() + .its("cur_list") + .then((list) => { + expect(list.view_name).to.equal("File"); + expect(list.current_folder).to.equal("Home/Attachments"); + }); + }); + + it("Re-route to default view", () => { + cy.call("frappe.tests.ui_test_helpers.setup_default_view", { view: "Report" }).then(() => { + cy.visit("app/event"); + cy.wait(500); + cy.window() + .its("cur_list") + .then((list) => { + expect(list.view_name).to.equal("Report"); + }); + }); + }); + + it("Route to default view from app/{doctype}", () => { + cy.call("frappe.tests.ui_test_helpers.setup_default_view", { view: "Report" }).then(() => { + cy.visit("/app/event"); + cy.wait(500); + cy.window() + .its("cur_list") + .then((list) => { + expect(list.view_name).to.equal("Report"); + }); + }); + }); + + it("Route to default view from app/{doctype}/view", () => { + cy.call("frappe.tests.ui_test_helpers.setup_default_view", { view: "Report" }).then(() => { + cy.visit("/app/event/view"); + cy.wait(500); + cy.window() + .its("cur_list") + .then((list) => { + expect(list.view_name).to.equal("Report"); + }); + }); + }); + + it("Force Route to default view from app/{doctype}", () => { + cy.call("frappe.tests.ui_test_helpers.setup_default_view", { + view: "Report", + force_reroute: true, + }).then(() => { + cy.visit("/app/event"); + cy.wait(500); + cy.window() + .its("cur_list") + .then((list) => { + expect(list.view_name).to.equal("Report"); + }); + }); + }); + + it("Force Route to default view from app/{doctype}/view", () => { + cy.call("frappe.tests.ui_test_helpers.setup_default_view", { + view: "Report", + force_reroute: true, + }).then(() => { + cy.visit("/app/event/view"); + cy.wait(500); + cy.window() + .its("cur_list") + .then((list) => { + expect(list.view_name).to.equal("Report"); + }); + }); + }); + + it("Force Route to default view from app/{doctype}/view", () => { + cy.call("frappe.tests.ui_test_helpers.setup_default_view", { + view: "Report", + force_reroute: true, + }).then(() => { + cy.visit("/app/event/view/list"); + cy.wait(500); + cy.window() + .its("cur_list") + .then((list) => { + expect(list.view_name).to.equal("Report"); + }); + }); + }); + + it("Validate Route History for Default View", () => { + cy.call("frappe.tests.ui_test_helpers.setup_default_view", { view: "Report" }).then(() => { + cy.visit("/app/event"); + cy.visit("/app/event/view/list"); + cy.location("pathname").should("eq", "/app/event/view/list"); + cy.go("back"); + cy.location("pathname").should("eq", "/app/event"); + }); + }); + + it("Route to Form", () => { + const test_user = cy.config("testUser"); + cy.visit(`/app/user/${test_user}`); + cy.window() + .its("cur_frm") + .then((frm) => { + expect(frm.doc.name).to.equal(test_user); + }); + }); + + it("Route to Settings Workspace", () => { + cy.visit("/app/settings"); + cy.get(".title-text").should("contain", "Settings"); + }); +}); diff --git a/cypress/integration/web_form.js b/cypress/integration/web_form.js index bd1c7e147e..fcb24219b9 100644 --- a/cypress/integration/web_form.js +++ b/cypress/integration/web_form.js @@ -1,27 +1,276 @@ -context('Web Form', () => { +context("Web Form", () => { before(() => { - cy.login(); + cy.login("Administrator"); + cy.visit("/app/"); + return cy + .window() + .its("frappe") + .then((frappe) => { + return frappe.xcall("frappe.tests.ui_test_helpers.prepare_webform_test"); + }); }); - it('Navigate and Submit a WebForm', () => { - cy.visit('/update-profile'); - cy.get_field('last_name', 'Data').type('_Test User', {force: true}).wait(200); - cy.get('.web-form-actions .btn-primary').click(); - cy.wait(5000); - cy.url().should('include', '/me'); + it("Create Web Form", () => { + cy.visit("/app/web-form/new"); + + cy.intercept("POST", "/api/method/frappe.desk.form.save.savedocs").as("save_form"); + + cy.fill_field("title", "Note"); + cy.fill_field("doc_type", "Note", "Link"); + cy.fill_field("module", "Website", "Link"); + cy.click_custom_action_button("Get Fields"); + cy.click_custom_action_button("Publish"); + + cy.wait("@save_form"); + + cy.get_field("route").should("have.value", "note"); + cy.get(".title-area .indicator-pill").contains("Published"); }); - it('Navigate and Submit a MultiStep WebForm', () => { - cy.call('frappe.tests.ui_test_helpers.update_webform_to_multistep').then(() => { - cy.visit('/update-profile-duplicate'); - cy.get_field('last_name', 'Data').type('_Test User', {force: true}).wait(200); - cy.get('.btn-next').should('be.visible'); - cy.get('.btn-next').click(); - cy.get('.btn-previous').should('be.visible'); - cy.get('.btn-next').should('not.be.visible'); - cy.get('.web-form-actions .btn-primary').click(); - cy.wait(5000); - cy.url().should('include', '/me'); + it("Open Web Form", () => { + cy.visit("/note"); + cy.fill_field("title", "Note 1"); + cy.get(".web-form-actions button").contains("Save").click(); + + cy.url().should("include", "/note/new"); + + cy.request("/api/method/logout"); + cy.visit("/note"); + + cy.url().should("include", "/note/new"); + + cy.fill_field("title", "Guest Note 1"); + cy.get(".web-form-actions button").contains("Save").click(); + + cy.url().should("include", "/note/new"); + + cy.visit("/note"); + cy.url().should("include", "/note/new"); + }); + + it("Login Required", () => { + cy.login("Administrator"); + cy.visit("/app/web-form/note"); + + cy.findByRole("tab", { name: "Settings" }).click(); + cy.get('input[data-fieldname="login_required"]').check({ force: true }); + + cy.save(); + + cy.visit("/note"); + + cy.call("logout"); + + cy.visit("/note"); + cy.get_open_dialog() + .get(".modal-message") + .contains("You are not permitted to access this page without login."); + }); + + it("Show List", () => { + cy.login("Administrator"); + cy.visit("/app/web-form/note"); + + cy.findByRole("tab", { name: "Settings" }).click(); + cy.get(".section-head").contains("List Settings").click(); + cy.get('input[data-fieldname="show_list"]').check(); + + cy.save(); + + cy.visit("/note"); + cy.url().should("include", "/note/list"); + cy.get(".web-list-table").should("be.visible"); + }); + + it("Show Custom List Title", () => { + cy.visit("/app/web-form/note"); + + cy.findByRole("tab", { name: "Settings" }).click(); + cy.fill_field("list_title", "Note List"); + + cy.save(); + + cy.visit("/note"); + cy.url().should("include", "/note/list"); + cy.get(".web-list-header h1").should("contain.text", "Note List"); + }); + + it("Show Custom List Columns", () => { + cy.visit("/note"); + cy.url().should("include", "/note/list"); + + cy.get(".web-list-table thead th").contains("Sr."); + cy.get(".web-list-table thead th").contains("Title"); + + cy.visit("/app/web-form/note"); + + cy.findByRole("tab", { name: "Settings" }).click(); + + cy.get('[data-fieldname="list_columns"] .grid-footer button') + .contains("Add Row") + .as("add-row"); + + cy.get("@add-row").click(); + cy.get('[data-fieldname="list_columns"] .grid-body .rows').as("grid-rows"); + cy.get("@grid-rows").find('.grid-row:first [data-fieldname="fieldname"]').click(); + cy.get("@grid-rows") + .find('.grid-row:first select[data-fieldname="fieldname"]') + .select("Title"); + + cy.get("@add-row").click(); + cy.get("@grid-rows").find('.grid-row[data-idx="2"] [data-fieldname="fieldname"]').click(); + cy.get("@grid-rows") + .find('.grid-row[data-idx="2"] select[data-fieldname="fieldname"]') + .select("Public"); + + cy.get("@add-row").click(); + cy.get("@grid-rows").find('.grid-row:last [data-fieldname="fieldname"]').click(); + cy.get("@grid-rows") + .find('.grid-row:last select[data-fieldname="fieldname"]') + .select("Content"); + + cy.save(); + + cy.visit("/note"); + cy.url().should("include", "/note/list"); + cy.get(".web-list-table thead th").contains("Sr."); + cy.get(".web-list-table thead th").contains("Title"); + cy.get(".web-list-table thead th").contains("Public"); + cy.get(".web-list-table thead th").contains("Content"); + }); + + it("Breadcrumbs", () => { + cy.visit("/note"); + cy.url().should("include", "/note/list"); + cy.get(".web-list-table tbody tr:last").click(); + + cy.get(".breadcrumb-container .breadcrumb .breadcrumb-item:first a") + .should("contain.text", "Note") + .click(); + cy.url().should("include", "/note/list"); + }); + + it("Custom Breadcrumbs", () => { + cy.visit("/app/web-form/note"); + + cy.findByRole("tab", { name: "Customization" }).click(); + cy.fill_field("breadcrumbs", '[{"label": _("Notes"), "route":"note"}]', "Code"); + cy.get(".form-tabs .nav-item .nav-link").contains("Customization").click(); + cy.save(); + + cy.visit("/note"); + cy.url().should("include", "/note/list"); + cy.get(".web-list-table tbody tr:last").click(); + cy.get(".breadcrumb-container .breadcrumb .breadcrumb-item:first a").should( + "contain.text", + "Notes" + ); + }); + + it("Read Only", () => { + cy.login("Administrator"); + cy.visit("/note"); + cy.url().should("include", "/note/list"); + + // Read Only Field + cy.get(".web-list-table tbody tr:last").click(); + cy.get('.frappe-control[data-fieldname="title"] .control-input').should( + "have.css", + "display", + "none" + ); + }); + + it("Edit Mode", () => { + cy.visit("/app/web-form/note"); + + cy.findByRole("tab", { name: "Settings" }).click(); + cy.get('input[data-fieldname="allow_edit"]').check(); + + cy.save(); + + cy.visit("/note"); + cy.url().should("include", "/note/list"); + cy.get(".web-list-table tbody tr:last").click(); + + cy.get(".web-form-actions a").contains("Edit Response").click(); + cy.url().should("include", "/edit"); + + // Editable Field + cy.get_field("title").should("have.value", "Note 1"); + + cy.fill_field("title", " Edited"); + cy.get(".web-form-actions button").contains("Save").click(); + cy.get(".success-page .edit-button").click(); + cy.get_field("title").should("have.value", "Note 1 Edited"); + }); + + it("Allow Multiple Response", () => { + cy.visit("/app/web-form/note"); + + cy.findByRole("tab", { name: "Settings" }).click(); + cy.get('input[data-fieldname="allow_multiple"]').check(); + + cy.save(); + + cy.visit("/note"); + cy.url().should("include", "/note/list"); + + cy.get(".web-list-actions a:visible").contains("New").click(); + cy.url().should("include", "/note/new"); + + cy.fill_field("title", "Note 2"); + cy.get(".web-form-actions button").contains("Save").click(); + }); + + it("Allow Delete", () => { + cy.visit("/app/web-form/note"); + + cy.findByRole("tab", { name: "Settings" }).click(); + cy.get('input[data-fieldname="allow_delete"]').check(); + + cy.save(); + + cy.visit("/note"); + cy.url().should("include", "/note/list"); + + cy.get(".web-list-table tbody tr:nth-child(1) .list-col-checkbox input").click(); + cy.get(".web-list-table tbody tr:nth-child(2) .list-col-checkbox input").click(); + cy.get(".web-list-actions button:visible").contains("Delete").click({ force: true }); + + cy.get(".web-list-actions button").contains("Delete").should("not.be.visible"); + + cy.visit("/note"); + cy.get(".web-list-table tbody tr:nth-child(1)").should("not.exist"); + }); + + it("Navigate and Submit a WebForm", () => { + cy.visit("/update-profile"); + + cy.get(".web-form-actions a").contains("Edit Response").click(); + + cy.fill_field("middle_name", "_Test User"); + + cy.get(".web-form-actions .btn-primary").click(); + cy.url().should("include", "/me"); + }); + + it("Navigate and Submit a MultiStep WebForm", () => { + cy.call("frappe.tests.ui_test_helpers.update_webform_to_multistep").then(() => { + cy.visit("/update-profile-duplicate"); + + cy.get(".web-form-actions a").contains("Edit Response").click(); + + cy.fill_field("middle_name", "_Test User"); + + cy.get(".btn-next").should("be.visible"); + cy.get(".btn-next").click(); + + cy.get(".btn-previous").should("be.visible"); + cy.get(".btn-next").should("not.be.visible"); + + cy.get(".web-form-actions .btn-primary").click(); + cy.url().should("include", "/me"); }); }); }); diff --git a/cypress/integration/workspace.js b/cypress/integration/workspace.js index a12d86b3d6..d52417b234 100644 --- a/cypress/integration/workspace.js +++ b/cypress/integration/workspace.js @@ -1,185 +1,256 @@ -context('Workspace 2.0', () => { +context("Workspace 2.0", () => { before(() => { - cy.visit('/login'); + cy.visit("/login"); cy.login(); }); - it('Navigate to page from sidebar', () => { - cy.visit('/app/build'); - cy.get('.codex-editor__redactor .ce-block'); + it("Navigate to page from sidebar", () => { + cy.visit("/app/build"); + cy.get(".codex-editor__redactor .ce-block"); cy.get('.sidebar-item-container[item-name="Settings"]').first().click(); - cy.location('pathname').should('eq', '/app/settings'); + cy.location("pathname").should("eq", "/app/settings"); }); - it('Create Private Page', () => { + it("Create Private Page", () => { cy.intercept({ - method: 'POST', - url: 'api/method/frappe.desk.doctype.workspace.workspace.new_page' - }).as('new_page'); + method: "POST", + url: "api/method/frappe.desk.doctype.workspace.workspace.new_page", + }).as("new_page"); - cy.get('.codex-editor__redactor .ce-block'); + cy.get(".codex-editor__redactor .ce-block"); cy.get('.custom-actions button[data-label="Create%20Workspace"]').click(); - cy.fill_field('title', 'Test Private Page', 'Data'); - cy.fill_field('icon', 'edit', 'Icon'); - cy.get_open_dialog().find('.modal-header').click(); - cy.get_open_dialog().find('.btn-primary').click(); + cy.fill_field("title", "Test Private Page", "Data"); + cy.fill_field("icon", "edit", "Icon"); + cy.get_open_dialog().find(".modal-header").click(); + cy.get_open_dialog().find(".btn-primary").click(); // check if sidebar item is added in pubic section - cy.get('.sidebar-item-container[item-name="Test Private Page"]').should('have.attr', 'item-public', '0'); + cy.get('.sidebar-item-container[item-name="Test Private Page"]').should( + "have.attr", + "item-public", + "0" + ); cy.get('.standard-actions .btn-primary[data-label="Save"]').click(); cy.wait(300); - cy.get('.sidebar-item-container[item-name="Test Private Page"]').should('have.attr', 'item-public', '0'); + cy.get('.sidebar-item-container[item-name="Test Private Page"]').should( + "have.attr", + "item-public", + "0" + ); - cy.wait('@new_page'); + cy.wait("@new_page"); }); - it('Create Child Page', () => { + it("Create Child Page", () => { cy.intercept({ - method: 'POST', - url: 'api/method/frappe.desk.doctype.workspace.workspace.new_page' - }).as('new_page'); + method: "POST", + url: "api/method/frappe.desk.doctype.workspace.workspace.new_page", + }).as("new_page"); - cy.get('.codex-editor__redactor .ce-block'); + cy.get(".codex-editor__redactor .ce-block"); cy.get('.custom-actions button[data-label="Create%20Workspace"]').click(); - cy.fill_field('title', 'Test Child Page', 'Data'); - cy.fill_field('parent', 'Test Private Page', 'Select'); - cy.fill_field('icon', 'edit', 'Icon'); - cy.get_open_dialog().find('.modal-header').click(); - cy.get_open_dialog().find('.btn-primary').click(); + cy.fill_field("title", "Test Child Page", "Data"); + cy.fill_field("parent", "Test Private Page", "Select"); + cy.fill_field("icon", "edit", "Icon"); + cy.get_open_dialog().find(".modal-header").click(); + cy.get_open_dialog().find(".btn-primary").click(); // check if sidebar item is added in pubic section - cy.get('.sidebar-item-container[item-name="Test Child Page"]').should('have.attr', 'item-public', '0'); + cy.get('.sidebar-item-container[item-name="Test Child Page"]').should( + "have.attr", + "item-public", + "0" + ); cy.get('.standard-actions .btn-primary[data-label="Save"]').click(); cy.wait(300); - cy.get('.sidebar-item-container[item-name="Test Child Page"]').should('have.attr', 'item-public', '0'); + cy.get('.sidebar-item-container[item-name="Test Child Page"]').should( + "have.attr", + "item-public", + "0" + ); - cy.wait('@new_page'); + cy.wait("@new_page"); }); - it('Duplicate Page', () => { + it("Duplicate Page", () => { cy.intercept({ - method: 'POST', - url: 'api/method/frappe.desk.doctype.workspace.workspace.duplicate_page' - }).as('page_duplicated'); + method: "POST", + url: "api/method/frappe.desk.doctype.workspace.workspace.duplicate_page", + }).as("page_duplicated"); - cy.get('.codex-editor__redactor .ce-block'); - cy.get('.standard-actions .btn-secondary[data-label=Edit]').click(); + cy.get(".codex-editor__redactor .ce-block"); + cy.get(".standard-actions .btn-secondary[data-label=Edit]").click(); - cy.get('.sidebar-item-container[item-name="Test Private Page"]').as('sidebar-item'); + cy.get('.sidebar-item-container[item-name="Test Private Page"]').as("sidebar-item"); - cy.get('@sidebar-item').find('.standard-sidebar-item').first().click(); - cy.get('@sidebar-item').find('.dropdown-btn').first().click(); - cy.get('@sidebar-item').find('.dropdown-list .dropdown-item').contains('Duplicate').first().click({force: true}); + cy.get("@sidebar-item").find(".standard-sidebar-item").first().click(); + cy.get("@sidebar-item").find(".dropdown-btn").first().click(); + cy.get("@sidebar-item") + .find(".dropdown-list .dropdown-item") + .contains("Duplicate") + .first() + .click({ force: true }); - cy.get_open_dialog().fill_field('title', 'Duplicate Page', 'Data'); - cy.click_modal_primary_button('Duplicate'); + cy.get_open_dialog().fill_field("title", "Duplicate Page", "Data"); + cy.click_modal_primary_button("Duplicate"); - cy.wait('@page_duplicated'); + cy.wait("@page_duplicated"); }); - it('Drag Sidebar Item', () => { + it("Drag Sidebar Item", () => { cy.intercept({ - method: 'POST', - url: 'api/method/frappe.desk.doctype.workspace.workspace.sort_pages' - }).as('page_sorted'); + method: "POST", + url: "api/method/frappe.desk.doctype.workspace.workspace.sort_pages", + }).as("page_sorted"); - cy.get('.sidebar-item-container[item-name="Duplicate Page"]').as('sidebar-item'); + cy.get('.sidebar-item-container[item-name="Duplicate Page"]').as("sidebar-item"); - cy.get('@sidebar-item').find('.standard-sidebar-item').first().click(); - cy.get('@sidebar-item').find('.drag-handle').first().move({ deltaX: 0, deltaY: 100 }); + cy.get("@sidebar-item").find(".standard-sidebar-item").first().click(); + cy.get("@sidebar-item").find(".drag-handle").first().move({ deltaX: 0, deltaY: 100 }); - cy.get('.sidebar-item-container[item-name="Build"]').as('sidebar-item'); + cy.get('.sidebar-item-container[item-name="Build"]').as("sidebar-item"); - cy.get('@sidebar-item').find('.standard-sidebar-item').first().click(); - cy.get('@sidebar-item').find('.drag-handle').first().move({ deltaX: 0, deltaY: 100 }); + cy.get("@sidebar-item").find(".standard-sidebar-item").first().click(); + cy.get("@sidebar-item").find(".drag-handle").first().move({ deltaX: 0, deltaY: 100 }); - cy.wait('@page_sorted'); + cy.wait("@page_sorted"); }); - it('Edit Page Detail', () => { + it("Edit Page Detail", () => { cy.intercept({ - method: 'POST', - url: 'api/method/frappe.desk.doctype.workspace.workspace.update_page' - }).as('page_updated'); + method: "POST", + url: "api/method/frappe.desk.doctype.workspace.workspace.update_page", + }).as("page_updated"); - cy.get('.sidebar-item-container[item-name="Test Private Page"]').as('sidebar-item'); + cy.get('.sidebar-item-container[item-name="Test Private Page"]').as("sidebar-item"); - cy.get('@sidebar-item').find('.standard-sidebar-item').first().click(); - cy.get('@sidebar-item').find('.dropdown-btn').first().click(); - cy.get('@sidebar-item').find('.dropdown-list .dropdown-item').contains('Edit').first().click({force: true}); + cy.get("@sidebar-item").find(".standard-sidebar-item").first().click(); + cy.get("@sidebar-item").find(".dropdown-btn").first().click(); + cy.get("@sidebar-item") + .find(".dropdown-list .dropdown-item") + .contains("Edit") + .first() + .click({ force: true }); - cy.get_open_dialog().fill_field('title', ' 1', 'Data'); + cy.get_open_dialog().fill_field("title", " 1", "Data"); cy.get_open_dialog().find('input[data-fieldname="is_public"]').check(); - cy.click_modal_primary_button('Update'); + cy.click_modal_primary_button("Update"); - cy.get('.standard-sidebar-section:first .sidebar-item-container[item-name="Test Private Page"]').should('not.exist'); - cy.get('.standard-sidebar-section:last .sidebar-item-container[item-name="Test Private Page 1"]').should('exist'); + cy.get( + '.standard-sidebar-section:first .sidebar-item-container[item-name="Test Private Page"]' + ).should("not.exist"); + cy.get( + '.standard-sidebar-section:last .sidebar-item-container[item-name="Test Private Page 1"]' + ).should("exist"); - cy.wait('@page_updated'); + cy.wait("@page_updated"); }); - it('Add New Block', () => { - cy.get('.sidebar-item-container[item-name="Duplicate Page"]').as('sidebar-item'); + it("Add New Block", () => { + cy.get('.sidebar-item-container[item-name="Duplicate Page"]').as("sidebar-item"); - cy.get('@sidebar-item').find('.standard-sidebar-item').first().click(); + cy.get("@sidebar-item").find(".standard-sidebar-item").first().click(); - cy.get('.ce-block').click().type('{enter}'); - cy.get('.block-list-container .block-list-item').contains('Heading').click(); - cy.get(":focus").type('Header'); - cy.get(".ce-block:last").find('.ce-header').should('exist'); + cy.get(".ce-block").click().type("{enter}"); + cy.get(".block-list-container .block-list-item").contains("Heading").click(); + cy.get(":focus").type("Header"); + cy.get(".ce-block:last").find(".ce-header").should("exist"); - cy.get('.ce-block:last').click().type('{enter}'); - cy.get('.block-list-container .block-list-item').contains('Text').click(); - cy.get(":focus").type('Paragraph text'); - cy.get(".ce-block:last").find('.ce-paragraph').should('exist'); + cy.get(".ce-block:last").click().type("{enter}"); + cy.get(".block-list-container .block-list-item").contains("Text").click(); + cy.get(":focus").type("Paragraph text"); + cy.get(".ce-block:last").find(".ce-paragraph").should("exist"); }); - it('Delete A Block', () => { + it("Delete A Block", () => { cy.get(":focus").click(); - cy.get('.paragraph-control .setting-btn').click(); - cy.get('.paragraph-control .dropdown-item').contains('Delete').click(); - cy.get(".ce-block:last").find('.ce-paragraph').should('not.exist'); + cy.get(".paragraph-control .setting-btn").click(); + cy.get(".paragraph-control .dropdown-item").contains("Delete").click(); + cy.get(".ce-block:last").find(".ce-paragraph").should("not.exist"); }); - it('Shrink and Expand A Block', () => { + it("Shrink and Expand A Block", () => { cy.get(":focus").click(); - cy.get('.ce-block:last .setting-btn').click(); - cy.get('.ce-block:last .dropdown-item').contains('Shrink').click(); - cy.get(".ce-block:last").should('have.class', 'col-xs-11'); - cy.get('.ce-block:last .dropdown-item').contains('Shrink').click(); - cy.get(".ce-block:last").should('have.class', 'col-xs-10'); - cy.get('.ce-block:last .dropdown-item').contains('Shrink').click(); - cy.get(".ce-block:last").should('have.class', 'col-xs-9'); - cy.get('.ce-block:last .dropdown-item').contains('Expand').click(); - cy.get(".ce-block:last").should('have.class', 'col-xs-10'); - cy.get('.ce-block:last .dropdown-item').contains('Expand').click(); - cy.get(".ce-block:last").should('have.class', 'col-xs-11'); - cy.get('.ce-block:last .dropdown-item').contains('Expand').click(); - cy.get(".ce-block:last").should('have.class', 'col-xs-12'); + cy.get(".ce-block:last .setting-btn").click(); + cy.get(".ce-block:last .dropdown-item").contains("Shrink").click(); + cy.get(".ce-block:last").should("have.class", "col-xs-11"); + cy.get(".ce-block:last .dropdown-item").contains("Shrink").click(); + cy.get(".ce-block:last").should("have.class", "col-xs-10"); + cy.get(".ce-block:last .dropdown-item").contains("Shrink").click(); + cy.get(".ce-block:last").should("have.class", "col-xs-9"); + cy.get(".ce-block:last .dropdown-item").contains("Expand").click(); + cy.get(".ce-block:last").should("have.class", "col-xs-10"); + cy.get(".ce-block:last .dropdown-item").contains("Expand").click(); + cy.get(".ce-block:last").should("have.class", "col-xs-11"); + cy.get(".ce-block:last .dropdown-item").contains("Expand").click(); + cy.get(".ce-block:last").should("have.class", "col-xs-12"); cy.get('.standard-actions .btn-primary[data-label="Save"]').click(); }); - it('Delete Duplicate Page', () => { + it("Hide/Unhide Workspaces", () => { + // hide cy.intercept({ - method: 'POST', - url: 'api/method/frappe.desk.doctype.workspace.workspace.delete_page' - }).as('page_deleted'); + method: "POST", + url: "api/method/frappe.desk.doctype.workspace.workspace.hide_page", + }).as("hide_page"); - cy.get('.codex-editor__redactor .ce-block'); - cy.get('.standard-actions .btn-secondary[data-label=Edit]').click(); + cy.get(".codex-editor__redactor .ce-block"); + cy.get(".standard-actions .btn-secondary[data-label=Edit]").click(); cy.get('.sidebar-item-container[item-name="Duplicate Page"]') - .find('.sidebar-item-control .setting-btn').click(); + .find(".sidebar-item-control .setting-btn") + .click(); cy.get('.sidebar-item-container[item-name="Duplicate Page"]') - .find('.dropdown-item[title="Delete Workspace"]').click({force: true}); + .find('.dropdown-item[title="Hide Workspace"]') + .click({ force: true }); cy.wait(300); - cy.get('.modal-footer > .standard-actions > .btn-modal-primary:visible').first().click(); - cy.get('.sidebar-item-container[item-name="Duplicate Page"]').should('not.exist'); + cy.get('.standard-actions .btn-secondary[data-label="Discard"]').click(); + cy.get('.sidebar-item-container[item-name="Duplicate Page"]').should("not.be.visible"); - cy.wait('@page_deleted'); + cy.wait("@hide_page"); + + // unhide + cy.intercept({ + method: "POST", + url: "api/method/frappe.desk.doctype.workspace.workspace.unhide_page", + }).as("unhide_page"); + + cy.get(".codex-editor__redactor .ce-block"); + cy.get(".standard-actions .btn-secondary[data-label=Edit]").click(); + + cy.get('.sidebar-item-container[item-name="Duplicate Page"]') + .find('[title="Unhide Workspace"]') + .click({ force: true }); + cy.wait(300); + + cy.get('.standard-actions .btn-secondary[data-label="Discard"]').click(); + cy.get('.sidebar-item-container[item-name="Duplicate Page"]').should("be.visible"); + + cy.wait("@unhide_page"); }); -}); \ No newline at end of file + it("Delete Duplicate Page", () => { + cy.intercept({ + method: "POST", + url: "api/method/frappe.desk.doctype.workspace.workspace.delete_page", + }).as("page_deleted"); + + cy.get(".codex-editor__redactor .ce-block"); + cy.get(".standard-actions .btn-secondary[data-label=Edit]").click(); + + cy.get('.sidebar-item-container[item-name="Duplicate Page"]') + .find(".sidebar-item-control .setting-btn") + .click(); + cy.get('.sidebar-item-container[item-name="Duplicate Page"]') + .find('.dropdown-item[title="Delete Workspace"]') + .click({ force: true }); + cy.wait(300); + cy.get(".modal-footer > .standard-actions > .btn-modal-primary:visible").first().click(); + cy.get('.sidebar-item-container[item-name="Duplicate Page"]').should("not.exist"); + + cy.wait("@page_deleted"); + }); +}); diff --git a/cypress/integration/workspace_blocks.js b/cypress/integration/workspace_blocks.js index 527cacab93..3b75ffb8c1 100644 --- a/cypress/integration/workspace_blocks.js +++ b/cypress/integration/workspace_blocks.js @@ -1,140 +1,186 @@ -context('Workspace Blocks', () => { +context("Workspace Blocks", () => { before(() => { cy.login(); - cy.visit('/app'); - return cy.window().its('frappe').then(frappe => { - return frappe.xcall("frappe.tests.ui_test_helpers.setup_workflow"); - }); + cy.visit("/app"); + return cy + .window() + .its("frappe") + .then((frappe) => { + return frappe.xcall("frappe.tests.ui_test_helpers.setup_workflow"); + }); }); - it('Create Test Page', () => { + it("Create Test Page", () => { cy.intercept({ - method: 'POST', - url: 'api/method/frappe.desk.doctype.workspace.workspace.new_page' - }).as('new_page'); + method: "POST", + url: "api/method/frappe.desk.doctype.workspace.workspace.new_page", + }).as("new_page"); - cy.visit('/app/website'); - cy.get('.codex-editor__redactor .ce-block'); + cy.visit("/app/website"); + cy.get(".codex-editor__redactor .ce-block"); cy.get('.custom-actions button[data-label="Create%20Workspace"]').click(); - cy.fill_field('title', 'Test Block Page', 'Data'); - cy.fill_field('icon', 'edit', 'Icon'); - cy.get_open_dialog().find('.modal-header').click(); - cy.get_open_dialog().find('.btn-primary').click(); + cy.fill_field("title", "Test Block Page", "Data"); + cy.fill_field("icon", "edit", "Icon"); + cy.get_open_dialog().find(".modal-header").click(); + cy.get_open_dialog().find(".btn-primary").click(); // check if sidebar item is added in private section - cy.get('.sidebar-item-container[item-name="Test Block Page"]').should('have.attr', 'item-public', '0'); + cy.get('.sidebar-item-container[item-name="Test Block Page"]').should( + "have.attr", + "item-public", + "0" + ); cy.get('.standard-actions .btn-primary[data-label="Save"]').click(); cy.wait(300); - cy.get('.sidebar-item-container[item-name="Test Block Page"]').should('have.attr', 'item-public', '0'); + cy.get('.sidebar-item-container[item-name="Test Block Page"]').should( + "have.attr", + "item-public", + "0" + ); - cy.wait('@new_page'); + cy.wait("@new_page"); }); - it('Quick List Block', () => { + it.skip("Quick List Block", () => { cy.create_records([ { - doctype: 'ToDo', - description: 'Quick List ToDo 1', - status: 'Open' + doctype: "ToDo", + description: "Quick List ToDo 1", + status: "Open", }, { - doctype: 'ToDo', - description: 'Quick List ToDo 2', - status: 'Open' + doctype: "ToDo", + description: "Quick List ToDo 2", + status: "Open", }, { - doctype: 'ToDo', - description: 'Quick List ToDo 3', - status: 'Open' + doctype: "ToDo", + description: "Quick List ToDo 3", + status: "Open", }, { - doctype: 'ToDo', - description: 'Quick List ToDo 4', - status: 'Open' - } + doctype: "ToDo", + description: "Quick List ToDo 4", + status: "Open", + }, ]); cy.intercept({ - method: 'GET', - url: 'api/method/frappe.desk.form.load.getdoctype' - }).as('get_doctype'); + method: "GET", + url: "api/method/frappe.desk.form.load.getdoctype?**", + }).as("get_doctype"); - cy.get('.codex-editor__redactor .ce-block'); - cy.get('.standard-actions .btn-secondary[data-label=Edit]').click(); + cy.get(".codex-editor__redactor .ce-block"); + cy.get(".standard-actions .btn-secondary[data-label=Edit]").click(); // test quick list creation - cy.get('.ce-block').first().click({force: true}).type('{enter}'); - cy.get('.block-list-container .block-list-item').contains('Quick List').click(); + cy.get(".ce-block").first().click({ force: true }).type("{enter}"); + cy.get(".block-list-container .block-list-item").contains("Quick List").click(); - cy.get_open_dialog().find('.modal-header').click(); + cy.fill_field("label", "ToDo", "Data"); + cy.fill_field("document_type", "ToDo", "Link").blur(); + cy.wait("@get_doctype"); - cy.fill_field('document_type', 'ToDo', 'Link').blur(); - cy.fill_field('label', 'ToDo', 'Data').blur(); - cy.wait('@get_doctype'); + cy.get_open_dialog().find(".filter-edit-area").should("contain", "No filters selected"); + cy.get_open_dialog().find(".filter-area .add-filter").click(); - cy.get_open_dialog().find('.filter-edit-area').should('contain', 'No filters selected'); - cy.get_open_dialog().find('.filter-area .add-filter').click(); + cy.get_open_dialog() + .find(".fieldname-select-area input") + .type("Workflow State{enter}") + .blur(); + cy.get_open_dialog().find(".filter-field .input-with-feedback").type("Pending"); - cy.get_open_dialog().find('.fieldname-select-area input').type('Workflow State{enter}').blur(); - cy.get_open_dialog().find('.filter-field .input-with-feedback').type('Pending'); - - cy.get_open_dialog().find('.modal-header').click(); - cy.get_open_dialog().find('.btn-primary').click(); + cy.get_open_dialog().find(".modal-header").click(); + cy.get_open_dialog().find(".btn-primary").click(); cy.get('.standard-actions .btn-primary[data-label="Save"]').click(); + cy.get(".codex-editor__redactor .ce-block"); - cy.get('.codex-editor__redactor .ce-block'); + cy.get(".ce-block .quick-list-widget-box").first().as("todo-quick-list"); - cy.get('.ce-block .quick-list-widget-box').first().as('todo-quick-list'); - - cy.get('@todo-quick-list').find('.quick-list-item .status').should('contain', 'Pending'); + cy.get("@todo-quick-list").find(".quick-list-item .status").should("contain", "Pending"); // test quick-list-item - cy.get('@todo-quick-list').find('.quick-list-item .title') + cy.get("@todo-quick-list") + .find(".quick-list-item .title") .first() - .invoke('attr', 'title') - .then(title => { - cy.get('@todo-quick-list').find('.quick-list-item').contains(title).click(); - cy.get_field('description', 'Text Editor').should('contain', title); - cy.click_action_button('Approve'); + .invoke("attr", "title") + .then((title) => { + cy.get("@todo-quick-list").find(".quick-list-item").contains(title).click(); + cy.get_field("description", "Text Editor").should("contain", title); + cy.click_action_button("Approve"); }); - cy.go('back'); + cy.go("back"); // test filter-list - cy.get('@todo-quick-list').realHover().find('.widget-control .filter-list').click(); + cy.get("@todo-quick-list").realHover().find(".widget-control .filter-list").click(); - cy.get_open_dialog().find('.filter-field .input-with-feedback').type('{selectall}Approved'); - cy.get_open_dialog().find('.modal-header').click(); - cy.get_open_dialog().find('.btn-primary').click(); - - cy.get('@todo-quick-list').find('.quick-list-item .status').should('contain', 'Approved'); + cy.get_open_dialog() + .find(".filter-field .input-with-feedback") + .focus() + .type("{selectall}Approved"); + cy.get_open_dialog().find(".modal-header").click(); + cy.get_open_dialog().find(".btn-primary").click(); + cy.get("@todo-quick-list").find(".quick-list-item .status").should("contain", "Approved"); // test refresh-list cy.intercept({ - method: 'POST', - url: 'api/method/frappe.desk.reportview.get' - }).as('refresh-list'); - - cy.get('@todo-quick-list').realHover().find('.widget-control .refresh-list').click(); - cy.wait('@refresh-list'); + method: "POST", + url: "api/method/frappe.desk.reportview.get", + }).as("refresh-list"); + cy.get("@todo-quick-list").realHover().find(".widget-control .refresh-list").click(); + cy.wait("@refresh-list"); // test add-new - cy.get('@todo-quick-list').realHover().find('.widget-control .add-new').click(); - cy.url().should('include', `/todo/new-todo-1`); - cy.go('back'); - + cy.get("@todo-quick-list").realHover().find(".widget-control .add-new").click(); + cy.url().should("include", `/todo/new-todo-1`); + cy.go("back"); // test see-all - cy.get('@todo-quick-list').find('.widget-footer .see-all').click(); + cy.get("@todo-quick-list").find(".widget-footer .see-all").click(); cy.open_list_filter(); cy.get('.filter-field input[data-fieldname="workflow_state"]') - .invoke('val') - .should('eq', 'Pending'); - cy.go('back'); + .invoke("val") + .should("eq", "Pending"); + cy.go("back"); }); -}); \ No newline at end of file + it("Number Card Block", () => { + cy.create_records([ + { + doctype: "Number Card", + label: "Test Number Card", + document_type: "ToDo", + color: "#f74343", + }, + ]); + + cy.get(".codex-editor__redactor .ce-block"); + cy.get(".standard-actions .btn-secondary[data-label=Edit]").click(); + + cy.get(".ce-block").first().click({ force: true }).type("{enter}"); + cy.get(".block-list-container .block-list-item").contains("Number Card").click(); + + // add number card + cy.fill_field("number_card_name", "Test Number Card", "Link"); + cy.get('[data-fieldname="number_card_name"] ul li').contains("Test Number Card").click(); + cy.click_modal_primary_button("Add"); + cy.get(".ce-block .number-widget-box").first().as("number_card"); + cy.get("@number_card").find(".widget-title").should("contain", "Test Number Card"); + cy.get('.standard-actions .btn-primary[data-label="Save"]').click(); + cy.get("@number_card").find(".widget-title").should("contain", "Test Number Card"); + + // edit number card + cy.get(".standard-actions .btn-secondary[data-label=Edit]").click(); + cy.get("@number_card").realHover().find(".widget-control .edit-button").click(); + cy.get_field("label", "Data").invoke("val", "ToDo Count"); + cy.click_modal_primary_button("Save"); + cy.get("@number_card").find(".widget-title").should("contain", "ToDo Count"); + cy.get('.standard-actions .btn-primary[data-label="Save"]').click(); + cy.get("@number_card").find(".widget-title").should("contain", "ToDo Count"); + }); +}); diff --git a/cypress/plugins/index.js b/cypress/plugins/index.js index 9720faa666..b13275373c 100644 --- a/cypress/plugins/index.js +++ b/cypress/plugins/index.js @@ -12,6 +12,6 @@ // the project's config changing) module.exports = (on, config) => { - require('@cypress/code-coverage/task')(on, config); + require("@cypress/code-coverage/task")(on, config); return config; -}; \ No newline at end of file +}; diff --git a/cypress/support/commands.js b/cypress/support/commands.js index 5ee26348e2..4b44a24598 100644 --- a/cypress/support/commands.js +++ b/cypress/support/commands.js @@ -1,6 +1,5 @@ -import 'cypress-file-upload'; -import '@testing-library/cypress/add-commands'; -import '@4tw/cypress-drag-drop'; +import "@testing-library/cypress/add-commands"; +import "@4tw/cypress-drag-drop"; import "cypress-real-events/support"; // *********************************************** // This example commands.js shows you how to @@ -28,291 +27,308 @@ import "cypress-real-events/support"; // -- This is will overwrite an existing command -- // Cypress.Commands.overwrite("visit", (originalFn, url, options) => { ... }); -Cypress.Commands.add('login', (email, password) => { +Cypress.Commands.add("login", (email, password) => { if (!email) { - email = 'Administrator'; + email = Cypress.config("testUser") || "Administrator"; } if (!password) { - password = Cypress.env('adminPassword'); + password = Cypress.env("adminPassword"); } cy.request({ - url: '/api/method/login', - method: 'POST', + url: "/api/method/login", + method: "POST", body: { usr: email, - pwd: password - } + pwd: password, + }, }); }); -Cypress.Commands.add('call', (method, args) => { +Cypress.Commands.add("call", (method, args) => { return cy .window() - .its('frappe.csrf_token') - .then(csrf_token => { + .its("frappe.csrf_token") + .then((csrf_token) => { return cy .request({ url: `/api/method/${method}`, - method: 'POST', + method: "POST", body: args, headers: { - Accept: 'application/json', - 'Content-Type': 'application/json', - 'X-Frappe-CSRF-Token': csrf_token - } + Accept: "application/json", + "Content-Type": "application/json", + "X-Frappe-CSRF-Token": csrf_token, + }, }) - .then(res => { + .then((res) => { expect(res.status).eq(200); return res.body; }); }); }); -Cypress.Commands.add('get_list', (doctype, fields = [], filters = []) => { +Cypress.Commands.add("get_list", (doctype, fields = [], filters = []) => { filters = JSON.stringify(filters); fields = JSON.stringify(fields); let url = `/api/resource/${doctype}?fields=${fields}&filters=${filters}`; return cy .window() - .its('frappe.csrf_token') - .then(csrf_token => { + .its("frappe.csrf_token") + .then((csrf_token) => { return cy .request({ - method: 'GET', + method: "GET", url, headers: { - Accept: 'application/json', - 'X-Frappe-CSRF-Token': csrf_token - } + Accept: "application/json", + "X-Frappe-CSRF-Token": csrf_token, + }, }) - .then(res => { + .then((res) => { expect(res.status).eq(200); return res.body; }); }); }); -Cypress.Commands.add('get_doc', (doctype, name) => { +Cypress.Commands.add("get_doc", (doctype, name) => { return cy .window() - .its('frappe.csrf_token') - .then(csrf_token => { + .its("frappe.csrf_token") + .then((csrf_token) => { return cy .request({ - method: 'GET', + method: "GET", url: `/api/resource/${doctype}/${name}`, headers: { - Accept: 'application/json', - 'X-Frappe-CSRF-Token': csrf_token - } + Accept: "application/json", + "X-Frappe-CSRF-Token": csrf_token, + }, }) - .then(res => { + .then((res) => { expect(res.status).eq(200); return res.body; }); }); }); -Cypress.Commands.add('remove_doc', (doctype, name) => { +Cypress.Commands.add("remove_doc", (doctype, name) => { return cy .window() - .its('frappe.csrf_token') - .then(csrf_token => { + .its("frappe.csrf_token") + .then((csrf_token) => { return cy .request({ - method: 'DELETE', + method: "DELETE", url: `/api/resource/${doctype}/${name}`, headers: { - Accept: 'application/json', - 'X-Frappe-CSRF-Token': csrf_token - } + Accept: "application/json", + "X-Frappe-CSRF-Token": csrf_token, + }, }) - .then(res => { + .then((res) => { expect(res.status).eq(202); return res.body; }); }); }); -Cypress.Commands.add('create_records', doc => { +Cypress.Commands.add("create_records", (doc) => { return cy - .call('frappe.tests.ui_test_helpers.create_if_not_exists', {doc: JSON.stringify(doc)}) - .then(r => r.message); + .call("frappe.tests.ui_test_helpers.create_if_not_exists", { doc: JSON.stringify(doc) }) + .then((r) => r.message); }); -Cypress.Commands.add('set_value', (doctype, name, obj) => { - return cy.call('frappe.client.set_value', { +Cypress.Commands.add("set_value", (doctype, name, obj) => { + return cy.call("frappe.client.set_value", { doctype, name, - fieldname: obj + fieldname: obj, }); }); -Cypress.Commands.add('fill_field', (fieldname, value, fieldtype = 'Data') => { - cy.get_field(fieldname, fieldtype).as('input'); +Cypress.Commands.add("fill_field", (fieldname, value, fieldtype = "Data") => { + cy.get_field(fieldname, fieldtype).as("input"); - if (['Date', 'Time', 'Datetime'].includes(fieldtype)) { - cy.get('@input').click().wait(200); - cy.get('.datepickers-container .datepicker.active').should('exist'); + if (["Date", "Time", "Datetime"].includes(fieldtype)) { + cy.get("@input").click().wait(200); + cy.get(".datepickers-container .datepicker.active").should("exist"); } - if (fieldtype === 'Time') { - cy.get('@input').clear().wait(200); + if (fieldtype === "Time") { + cy.get("@input").clear().wait(200); } - if (fieldtype === 'Select') { - cy.get('@input').select(value); + if (fieldtype === "Select") { + cy.get("@input").select(value); } else { - cy.get('@input').type(value, {waitForAnimations: false, force: true, delay: 100}); + cy.get("@input").type(value, { + waitForAnimations: false, + parseSpecialCharSequences: false, + force: true, + delay: 100, + }); } - return cy.get('@input'); + return cy.get("@input"); }); -Cypress.Commands.add('get_field', (fieldname, fieldtype = 'Data') => { - let field_element = fieldtype === 'Select' ? 'select': 'input'; +Cypress.Commands.add("get_field", (fieldname, fieldtype = "Data") => { + let field_element = fieldtype === "Select" ? "select" : "input"; let selector = `[data-fieldname="${fieldname}"] ${field_element}:visible`; - if (fieldtype === 'Text Editor') { + if (fieldtype === "Text Editor") { selector = `[data-fieldname="${fieldname}"] .ql-editor[contenteditable=true]:visible`; } - if (fieldtype === 'Code') { + if (fieldtype === "Code") { selector = `[data-fieldname="${fieldname}"] .ace_text-input`; } - if (fieldtype === 'Markdown Editor') { + if (fieldtype === "Markdown Editor") { selector = `[data-fieldname="${fieldname}"] .ace-editor-target`; } return cy.get(selector).first(); }); -Cypress.Commands.add('fill_table_field', (tablefieldname, row_idx, fieldname, value, fieldtype = 'Data') => { - cy.get_table_field(tablefieldname, row_idx, fieldname, fieldtype).as('input'); +Cypress.Commands.add( + "fill_table_field", + (tablefieldname, row_idx, fieldname, value, fieldtype = "Data") => { + cy.get_table_field(tablefieldname, row_idx, fieldname, fieldtype).as("input"); - if (['Date', 'Time', 'Datetime'].includes(fieldtype)) { - cy.get('@input').click().wait(200); - cy.get('.datepickers-container .datepicker.active').should('exist'); - } - if (fieldtype === 'Time') { - cy.get('@input').clear().wait(200); - } + if (["Date", "Time", "Datetime"].includes(fieldtype)) { + cy.get("@input").click().wait(200); + cy.get(".datepickers-container .datepicker.active").should("exist"); + } + if (fieldtype === "Time") { + cy.get("@input").clear().wait(200); + } - if (fieldtype === 'Select') { - cy.get('@input').select(value); - } else { - cy.get('@input').type(value, {waitForAnimations: false, force: true}); + if (fieldtype === "Select") { + cy.get("@input").select(value); + } else { + cy.get("@input").type(value, { waitForAnimations: false, force: true }); + } + return cy.get("@input"); } - return cy.get('@input'); +); + +Cypress.Commands.add( + "get_table_field", + (tablefieldname, row_idx, fieldname, fieldtype = "Data") => { + let selector = `.frappe-control[data-fieldname="${tablefieldname}"]`; + selector += ` [data-idx="${row_idx}"]`; + + if (fieldtype === "Text Editor") { + selector += ` [data-fieldname="${fieldname}"] .ql-editor[contenteditable=true]`; + } else if (fieldtype === "Code") { + selector += ` [data-fieldname="${fieldname}"] .ace_text-input`; + } else { + selector += ` [data-fieldname="${fieldname}"]`; + return cy.get(selector).find(".form-control:visible, .static-area:visible").first(); + } + return cy.get(selector); + } +); + +Cypress.Commands.add("awesomebar", (text) => { + cy.get("#navbar-search").type(`${text}{downarrow}{enter}`, { delay: 700 }); }); -Cypress.Commands.add('get_table_field', (tablefieldname, row_idx, fieldname, fieldtype = 'Data') => { - let selector = `.frappe-control[data-fieldname="${tablefieldname}"]`; - selector += ` [data-idx="${row_idx}"]`; - - if (fieldtype === 'Text Editor') { - selector += ` [data-fieldname="${fieldname}"] .ql-editor[contenteditable=true]`; - } else if (fieldtype === 'Code') { - selector += ` [data-fieldname="${fieldname}"] .ace_text-input`; - } else { - selector += ` [data-fieldname="${fieldname}"]`; - return cy.get(selector).find('.form-control:visible, .static-area:visible').first(); - } - return cy.get(selector); -}); - -Cypress.Commands.add('awesomebar', text => { - cy.get('#navbar-search').type(`${text}{downarrow}{enter}`, {delay: 700}); -}); - -Cypress.Commands.add('new_form', doctype => { - let dt_in_route = doctype.toLowerCase().replace(/ /g, '-'); +Cypress.Commands.add("new_form", (doctype) => { + let dt_in_route = doctype.toLowerCase().replace(/ /g, "-"); cy.visit(`/app/${dt_in_route}/new`); - cy.get('body').should('have.attr', 'data-route', `Form/${doctype}/new-${dt_in_route}-1`); - cy.get('body').should('have.attr', 'data-ajax-state', 'complete'); + cy.get("body").should("have.attr", "data-route", `Form/${doctype}/new-${dt_in_route}-1`); + cy.get("body").should("have.attr", "data-ajax-state", "complete"); }); -Cypress.Commands.add('go_to_list', doctype => { - let dt_in_route = doctype.toLowerCase().replace(/ /g, '-'); +Cypress.Commands.add("select_form_tab", (label) => { + cy.get(".form-tabs-list [data-toggle='tab']").contains(label).click().wait(500); +}); + +Cypress.Commands.add("go_to_list", (doctype) => { + let dt_in_route = doctype.toLowerCase().replace(/ /g, "-"); cy.visit(`/app/${dt_in_route}`); }); -Cypress.Commands.add('clear_cache', () => { +Cypress.Commands.add("clear_cache", () => { cy.window() - .its('frappe') - .then(frappe => { + .its("frappe") + .then((frappe) => { frappe.ui.toolbar.clear_cache(); }); }); -Cypress.Commands.add('dialog', opts => { - return cy.window({ log: false }).its('frappe', { log: false }).then(frappe => { - Cypress.log({ - name: "dialog", - displayName: "dialog", - message: 'frappe.ui.Dialog', - consoleProps: () => { - return { - options: opts, - dialog: d - } - } +Cypress.Commands.add("dialog", (opts) => { + return cy + .window({ log: false }) + .its("frappe", { log: false }) + .then((frappe) => { + Cypress.log({ + name: "dialog", + displayName: "dialog", + message: "frappe.ui.Dialog", + consoleProps: () => { + return { + options: opts, + dialog: d, + }; + }, + }); + + var d = new frappe.ui.Dialog(opts); + d.show(); + return d; }); - - var d = new frappe.ui.Dialog(opts); - d.show(); - return d; - }); }); -Cypress.Commands.add('get_open_dialog', () => { - return cy.get('.modal:visible').last(); +Cypress.Commands.add("get_open_dialog", () => { + return cy.get(".modal:visible").last(); }); -Cypress.Commands.add('save', () => { - cy.intercept('/api').as('api'); - cy.get(`button[data-label="Save"]:visible`).click({scrollBehavior: false, force: true}); - cy.wait('@api'); +Cypress.Commands.add("save", () => { + cy.intercept("/api/method/frappe.desk.form.save.savedocs").as("save_call"); + cy.get(`.page-container:visible button[data-label="Save"]`).click({ force: true }); + cy.wait("@save_call"); }); -Cypress.Commands.add('hide_dialog', () => { - cy.wait(300); - cy.get_open_dialog().focus().find('.btn-modal-close').click(); - cy.get('.modal:visible').should('not.exist'); +Cypress.Commands.add("hide_dialog", () => { + cy.wait(500); + cy.get_open_dialog().focus().find(".btn-modal-close").click(); + cy.get(".modal:visible").should("not.exist"); }); -Cypress.Commands.add('clear_dialogs', () => { +Cypress.Commands.add("clear_dialogs", () => { cy.window().then((win) => { - win.$('.modal, .modal-backdrop').remove(); + win.$(".modal, .modal-backdrop").remove(); }); - cy.get('.modal').should('not.exist'); + cy.get(".modal").should("not.exist"); }); -Cypress.Commands.add('clear_datepickers', () => { +Cypress.Commands.add("clear_datepickers", () => { cy.window().then((win) => { - win.$('.datepicker').remove(); + win.$(".datepicker").remove(); }); - cy.get('.datepicker').should('not.exist'); + cy.get(".datepicker").should("not.exist"); }); - -Cypress.Commands.add('insert_doc', (doctype, args, ignore_duplicate) => { +Cypress.Commands.add("insert_doc", (doctype, args, ignore_duplicate) => { if (!args.doctype) { args.doctype = doctype; } return cy .window() - .its('frappe.csrf_token') - .then(csrf_token => { + .its("frappe.csrf_token") + .then((csrf_token) => { return cy .request({ - method: 'POST', + method: "POST", url: `/api/resource/${doctype}`, body: args, headers: { - Accept: 'application/json', - 'Content-Type': 'application/json', - 'X-Frappe-CSRF-Token': csrf_token + Accept: "application/json", + "Content-Type": "application/json", + "X-Frappe-CSRF-Token": csrf_token, }, - failOnStatusCode: !ignore_duplicate + failOnStatusCode: !ignore_duplicate, }) - .then(res => { + .then((res) => { let status_codes = [200]; if (ignore_duplicate) { status_codes.push(409); @@ -320,7 +336,11 @@ Cypress.Commands.add('insert_doc', (doctype, args, ignore_duplicate) => { let message = null; if (ignore_duplicate && !status_codes.includes(res.status)) { - message = `Document insert failed, response: ${JSON.stringify(res, null, '\t')}`; + message = `Document insert failed, response: ${JSON.stringify( + res, + null, + "\t" + )}`; } expect(res.status).to.be.oneOf(status_codes, message); return res.body.data; @@ -328,108 +348,179 @@ Cypress.Commands.add('insert_doc', (doctype, args, ignore_duplicate) => { }); }); -Cypress.Commands.add('update_doc', (doctype, docname, args) => { +Cypress.Commands.add("update_doc", (doctype, docname, args) => { return cy .window() - .its('frappe.csrf_token') - .then(csrf_token => { + .its("frappe.csrf_token") + .then((csrf_token) => { return cy .request({ - method: 'PUT', + method: "PUT", url: `/api/resource/${doctype}/${docname}`, body: args, headers: { - Accept: 'application/json', - 'Content-Type': 'application/json', - 'X-Frappe-CSRF-Token': csrf_token + Accept: "application/json", + "Content-Type": "application/json", + "X-Frappe-CSRF-Token": csrf_token, }, }) - .then(res => { + .then((res) => { expect(res.status).to.eq(200); return res.body.data; }); }); }); - -Cypress.Commands.add('open_list_filter', () => { - cy.get('.filter-section .filter-button').click(); - cy.wait(300); - cy.get('.filter-popover').should('exist'); +Cypress.Commands.add("switch_to_user", (user) => { + cy.call("logout"); + cy.login(user); }); -Cypress.Commands.add('click_action_button', (name) => { - cy.findByRole('button', {name: 'Actions'}).click(); +Cypress.Commands.add("add_role", (user, role) => { + cy.window() + .its("frappe") + .then((frappe) => { + const session_user = frappe.session.user; + add_remove_role("add", user, role, session_user); + }); +}); + +Cypress.Commands.add("remove_role", (user, role) => { + cy.window() + .its("frappe") + .then((frappe) => { + const session_user = frappe.session.user; + add_remove_role("remove", user, role, session_user); + }); +}); + +const add_remove_role = (action, user, role, session_user) => { + if (session_user !== "Administrator") { + cy.switch_to_user("Administrator"); + } + + cy.call("frappe.tests.ui_test_helpers.add_remove_role", { + action: action, + user: user, + role: role, + }); + + if (session_user !== "Administrator") { + cy.switch_to_user(session_user); + } +}; + +Cypress.Commands.add("open_list_filter", () => { + cy.get(".filter-section .filter-button").click(); + cy.wait(300); + cy.get(".filter-popover").should("exist"); +}); + +Cypress.Commands.add("click_custom_action_button", (name) => { + cy.get(`.custom-actions [data-label="${encodeURIComponent(name)}"]`).click(); +}); + +Cypress.Commands.add("click_action_button", (name) => { + cy.findByRole("button", { name: "Actions" }).click(); cy.get(`.actions-btn-group [data-label="${encodeURIComponent(name)}"]`).click(); }); -Cypress.Commands.add('click_menu_button', (name) => { - cy.get('.standard-actions .menu-btn-group > .btn').click(); +Cypress.Commands.add("click_menu_button", (name) => { + cy.get(".standard-actions .menu-btn-group > .btn").click(); cy.get(`.menu-btn-group [data-label="${encodeURIComponent(name)}"]`).click(); }); -Cypress.Commands.add('clear_filters', () => { +Cypress.Commands.add("clear_filters", () => { let has_filter = false; cy.intercept({ - method: 'POST', - url: 'api/method/frappe.model.utils.user_settings.save' - }).as('filter-saved'); - cy.get('.filter-section .filter-button').click({force: true}); + method: "POST", + url: "api/method/frappe.model.utils.user_settings.save", + }).as("filter-saved"); + cy.get(".filter-section .filter-button").click({ force: true }); cy.wait(300); - cy.get('.filter-popover').should('exist'); - cy.get('.filter-popover').then(popover => { - if (popover.find('input.input-with-feedback')[0].value != '') { + cy.get(".filter-popover").should("exist"); + cy.get(".filter-popover").then((popover) => { + if (popover.find("input.input-with-feedback")[0].value != "") { has_filter = true; } }); - cy.get('.filter-popover').find('.clear-filters').click(); - cy.get('.filter-section .filter-button').click(); - cy.window().its('cur_list').then(cur_list => { - cur_list && cur_list.filter_area && cur_list.filter_area.clear(); - has_filter && cy.wait('@filter-saved'); - }); + cy.get(".filter-popover").find(".clear-filters").click(); + cy.get(".filter-section .filter-button").click(); + cy.window() + .its("cur_list") + .then((cur_list) => { + cur_list && cur_list.filter_area && cur_list.filter_area.clear(); + has_filter && cy.wait("@filter-saved"); + }); }); -Cypress.Commands.add('click_modal_primary_button', (btn_name) => { +Cypress.Commands.add("click_modal_primary_button", (btn_name) => { cy.wait(400); - cy.get('.modal-footer > .standard-actions > .btn-primary').contains(btn_name).click({force: true}); + cy.get(".modal-footer > .standard-actions > .btn-primary") + .contains(btn_name) + .click({ force: true }); }); -Cypress.Commands.add('click_sidebar_button', (btn_name) => { - cy.get('.list-group-by-fields .list-link > a').contains(btn_name).click({force: true}); +Cypress.Commands.add("click_sidebar_button", (btn_name) => { + cy.get(".list-group-by-fields .list-link > a").contains(btn_name).click({ force: true }); }); -Cypress.Commands.add('click_listview_row_item', (row_no) => { - cy.get('.list-row > .level-left > .list-subject > .level-item > .ellipsis').eq(row_no).click({force: true}); +Cypress.Commands.add("click_listview_row_item", (row_no) => { + cy.get(".list-row > .level-left > .list-subject > .level-item > .ellipsis") + .eq(row_no) + .click({ force: true }); }); -Cypress.Commands.add('click_listview_row_item_with_text', (text) => { - cy.get('.list-row > .level-left > .list-subject > .level-item > .ellipsis') +Cypress.Commands.add("click_listview_row_item_with_text", (text) => { + cy.get(".list-row > .level-left > .list-subject > .level-item > .ellipsis") .contains(text) .first() - .click({force: true}); + .click({ force: true }); }); -Cypress.Commands.add('click_filter_button', () => { - cy.get('.filter-selector > .btn').click(); +Cypress.Commands.add("click_filter_button", () => { + cy.get(".filter-button").click(); }); -Cypress.Commands.add('click_listview_primary_button', (btn_name) => { - cy.get('.primary-action').contains(btn_name).click({force: true}); +Cypress.Commands.add("click_listview_primary_button", (btn_name) => { + cy.get(".primary-action").contains(btn_name).click({ force: true }); }); -Cypress.Commands.add('click_doc_primary_button', (btn_name) => { - cy.get('.primary-action').contains(btn_name).click({force: true}); +Cypress.Commands.add("click_doc_primary_button", (btn_name) => { + cy.get(".primary-action").contains(btn_name).click({ force: true }); }); -Cypress.Commands.add('click_timeline_action_btn', (btn_name) => { - cy.get('.timeline-message-box .actions .action-btn').contains(btn_name).click(); +Cypress.Commands.add("click_timeline_action_btn", (btn_name) => { + cy.get(".timeline-message-box .actions .action-btn").contains(btn_name).click(); }); -Cypress.Commands.add('select_listview_row_checkbox', (row_no) => { - cy.get('.frappe-list .select-like > .list-row-checkbox').eq(row_no).click(); +Cypress.Commands.add("select_listview_row_checkbox", (row_no) => { + cy.get(".frappe-list .select-like > .list-row-checkbox").eq(row_no).click(); }); -Cypress.Commands.add('click_form_section', (section_name) => { - cy.get('.section-head').contains(section_name).click(); +Cypress.Commands.add("click_form_section", (section_name) => { + cy.get(".section-head").contains(section_name).click(); +}); + +const compare_document = (expected, actual) => { + for (const prop in expected) { + if (expected[prop] instanceof Array) { + // recursively compare child documents. + expected[prop].forEach((item, idx) => { + compare_document(item, actual[prop][idx]); + }); + } else { + assert.equal(expected[prop], actual[prop], `${prop} should be equal.`); + } + } +}; + +Cypress.Commands.add("compare_document", (expected_document) => { + cy.window() + .its("cur_frm") + .then((frm) => { + // Don't remove this, cypress can't magically wait for events it has no control over. + cy.wait(1000); + compare_document(expected_document, frm.doc); + }); }); diff --git a/cypress/support/index.js b/cypress/support/e2e.js similarity index 83% rename from cypress/support/index.js rename to cypress/support/e2e.js index 5980e96677..8ce8317a2f 100644 --- a/cypress/support/index.js +++ b/cypress/support/e2e.js @@ -14,10 +14,10 @@ // *********************************************************** // Import commands.js using ES2015 syntax: -import './commands'; -import '@cypress/code-coverage/support'; +import "./commands"; +import "@cypress/code-coverage/support"; -Cypress.on('uncaught:exception', (err, runnable) => { +Cypress.on("uncaught:exception", (err, runnable) => { return false; }); @@ -25,5 +25,5 @@ Cypress.on('uncaught:exception', (err, runnable) => { // require('./commands') Cypress.Cookies.defaults({ - preserve: 'sid' -}); \ No newline at end of file + preserve: "sid", +}); diff --git a/esbuild/build-cleanup.js b/esbuild/build-cleanup.js index cf03606a34..023fce08c5 100644 --- a/esbuild/build-cleanup.js +++ b/esbuild/build-cleanup.js @@ -4,9 +4,9 @@ const fs = require("fs"); const glob = require("fast-glob"); module.exports = { - name: 'build_cleanup', + name: "build_cleanup", setup(build) { - build.onEnd(result => { + build.onEnd((result) => { if (result.errors.length) return; clean_dist_files(Object.keys(result.metafile.outputs)); }); @@ -14,25 +14,18 @@ module.exports = { }; function clean_dist_files(new_files) { - new_files.forEach( - file => { - if (file.endsWith(".map")) return; + new_files.forEach((file) => { + if (file.endsWith(".map")) return; - const pattern = file.split(".").slice(0, -2).join(".") + "*"; - glob.sync(pattern).forEach( - file_to_delete => { - if (file_to_delete.startsWith(file)) return; + const pattern = file.split(".").slice(0, -2).join(".") + "*"; + glob.sync(pattern).forEach((file_to_delete) => { + if (file_to_delete.startsWith(file)) return; - fs.unlink(path.resolve(file_to_delete), err => { - if (!err) return; + fs.unlink(path.resolve(file_to_delete), (err) => { + if (!err) return; - console.error( - `Error deleting ${file.split(path.sep).pop()}` - ); - }); - } - - ); - } - ); -} \ No newline at end of file + console.error(`Error deleting ${file.split(path.sep).pop()}`); + }); + }); + }); +} diff --git a/esbuild/esbuild.js b/esbuild/esbuild.js index 4aa1ebc824..3c5c305665 100644 --- a/esbuild/esbuild.js +++ b/esbuild/esbuild.js @@ -3,12 +3,13 @@ const path = require("path"); const fs = require("fs"); const glob = require("fast-glob"); const esbuild = require("esbuild"); -const vue = require("esbuild-vue"); +const vue = require("esbuild-plugin-vue3"); const yargs = require("yargs"); const cliui = require("cliui")(); const chalk = require("chalk"); const html_plugin = require("./frappe-html"); -const rtlcss = require('rtlcss'); +const vue_style_plugin = require("./frappe-vue-style"); +const rtlcss = require("rtlcss"); const postCssPlugin = require("@frappe/esbuild-plugin-postcss2").default; const ignore_assets = require("./ignore-assets"); const sass_options = require("./sass_options"); @@ -25,44 +26,41 @@ const { log_warn, log_error, bench_path, - get_redis_subscriber + get_redis_subscriber, } = require("./utils"); const argv = yargs .usage("Usage: node esbuild [options]") .option("apps", { type: "string", - description: "Run build for specific apps" + description: "Run build for specific apps", }) .option("skip_frappe", { type: "boolean", - description: "Skip building frappe assets" + description: "Skip building frappe assets", }) .option("files", { type: "string", - description: "Run build for specified bundles" + description: "Run build for specified bundles", }) .option("watch", { type: "boolean", - description: "Run in watch mode and rebuild on file changes" + description: "Run in watch mode and rebuild on file changes", }) .option("live-reload", { type: "boolean", description: `Automatically reload Desk when assets are rebuilt. - Can only be used with the --watch flag.` + Can only be used with the --watch flag.`, }) .option("production", { type: "boolean", - description: "Run build in production mode" + description: "Run build in production mode", }) .option("run-build-command", { type: "boolean", - description: "Run build command for apps" + description: "Run build command for apps", }) - .example( - "node esbuild --apps frappe,erpnext", - "Run build only for frappe and erpnext" - ) + .example("node esbuild --apps frappe,erpnext", "Run build only for frappe and erpnext") .example( "node esbuild --files frappe/website.bundle.js,frappe/desk.bundle.js", "Run build only for specified bundles" @@ -70,7 +68,7 @@ const argv = yargs .version(false).argv; const APPS = (!argv.apps ? app_list : argv.apps.split(",")).filter( - app => !(argv.skip_frappe && app == "frappe") + (app) => !(argv.skip_frappe && app == "frappe") ); const FILES_TO_BUILD = argv.files ? argv.files.split(",") : []; const WATCH_MODE = Boolean(argv.watch); @@ -81,17 +79,15 @@ const TOTAL_BUILD_TIME = `${chalk.black.bgGreen(" DONE ")} Total Build Time`; const NODE_PATHS = [].concat( // node_modules of apps directly importable app_list - .map(app => path.resolve(get_app_path(app), "../node_modules")) + .map((app) => path.resolve(get_app_path(app), "../node_modules")) .filter(fs.existsSync), // import js file of any app if you provide the full path - app_list - .map(app => path.resolve(get_app_path(app), "..")) - .filter(fs.existsSync) + app_list.map((app) => path.resolve(get_app_path(app), "..")).filter(fs.existsSync) ); execute() .then(() => RUN_BUILD_COMMAND && run_build_command_for_apps(APPS)) - .catch(e => console.error(e)); + .catch((e) => console.error(e)); if (WATCH_MODE) { // listen for open files in editor event @@ -108,7 +104,7 @@ async function execute() { log_error("There were some problems during build"); log(); log(chalk.dim(e.stack)); - if (process.env.CI) { + if (process.env.CI || PRODUCTION) { process.kill(process.pid); } return; @@ -131,7 +127,7 @@ function build_assets_for_apps(apps, files) { ? get_files_to_build(files) : get_all_files_to_build(apps); - return glob(include_patterns, { ignore: ignore_patterns }).then(files => { + return glob(include_patterns, { ignore: ignore_patterns }).then((files) => { let output_path = assets_path; let file_map = {}; @@ -143,39 +139,38 @@ function build_assets_for_apps(apps, files) { let extension = path.extname(file); let output_name = path.basename(file, extension); - if ( - [".css", ".scss", ".less", ".sass", ".styl"].includes(extension) - ) { + if ([".css", ".scss", ".less", ".sass", ".styl"].includes(extension)) { output_name = path.join("css", output_name); } else if ([".js", ".ts"].includes(extension)) { output_name = path.join("js", output_name); } output_name = path.join(app, "dist", output_name); - if (Object.keys(file_map).includes(output_name) || Object.keys(style_file_map).includes(output_name)) { - log_warn( - `Duplicate output file ${output_name} generated from ${file}` - ); + if ( + Object.keys(file_map).includes(output_name) || + Object.keys(style_file_map).includes(output_name) + ) { + log_warn(`Duplicate output file ${output_name} generated from ${file}`); } if ([".css", ".scss", ".less", ".sass", ".styl"].includes(extension)) { style_file_map[output_name] = file; - rtl_style_file_map[output_name.replace('/css/', '/css-rtl/')] = file; + rtl_style_file_map[output_name.replace("/css/", "/css-rtl/")] = file; } else { file_map[output_name] = file; } } let build = build_files({ files: file_map, - outdir: output_path + outdir: output_path, }); let style_build = build_style_files({ files: style_file_map, - outdir: output_path + outdir: output_path, }); let rtl_style_build = build_style_files({ files: rtl_style_file_map, outdir: output_path, - rtl_style: true + rtl_style: true, }); return Promise.all([build, style_build, rtl_style_build]); }); @@ -188,11 +183,7 @@ function get_all_files_to_build(apps) { for (let app of apps) { let public_path = get_public_path(app); include_patterns.push( - path.resolve( - public_path, - "**", - "*.bundle.{js,ts,css,sass,scss,less,styl}" - ) + path.resolve(public_path, "**", "*.bundle.{js,ts,css,sass,scss,less,styl,jsx}") ); ignore_patterns.push( path.resolve(public_path, "node_modules"), @@ -202,7 +193,7 @@ function get_all_files_to_build(apps) { return { include_patterns, - ignore_patterns + ignore_patterns, }; } @@ -223,16 +214,12 @@ function get_files_to_build(files) { return { include_patterns, - ignore_patterns + ignore_patterns, }; } function build_files({ files, outdir }) { - let build_plugins = [ - html_plugin, - build_cleanup_plugin, - vue(), - ]; + let build_plugins = [vue(), html_plugin, build_cleanup_plugin, vue_style_plugin]; return esbuild.build(get_build_options(files, outdir, build_plugins)); } @@ -247,8 +234,8 @@ function build_style_files({ files, outdir, rtl_style = false }) { build_cleanup_plugin, postCssPlugin({ plugins: plugins, - sassOptions: sass_options - }) + sassOptions: sass_options, + }), ]; plugins.push(require("autoprefixer")); @@ -259,7 +246,7 @@ function get_build_options(files, outdir, plugins) { return { entryPoints: files, entryNames: "[dir]/[name].[hash]", - target: ['es2017'], + target: ["es2017"], outdir, sourcemap: true, bundle: true, @@ -267,12 +254,12 @@ function get_build_options(files, outdir, plugins) { minify: PRODUCTION, nodePaths: NODE_PATHS, define: { - "process.env.NODE_ENV": JSON.stringify( - PRODUCTION ? "production" : "development" - ) + "process.env.NODE_ENV": JSON.stringify(PRODUCTION ? "production" : "development"), + __VUE_OPTIONS_API__: JSON.stringify(true), + __VUE_PROD_DEVTOOLS__: JSON.stringify(false), }, plugins: plugins, - watch: get_watch_config() + watch: get_watch_config(), }; } @@ -286,17 +273,13 @@ function get_watch_config() { log(chalk.dim(error.stack)); notify_redis({ error }); } else { - let { - new_assets_json, - prev_assets_json - } = await write_assets_json(result.metafile); + let { new_assets_json, prev_assets_json } = await write_assets_json( + result.metafile + ); let changed_files; if (prev_assets_json) { - changed_files = get_rebuilt_assets( - prev_assets_json, - new_assets_json - ); + changed_files = get_rebuilt_assets(prev_assets_json, new_assets_json); let timestamp = new Date().toLocaleTimeString(); let message = `${timestamp}: Compiled ${changed_files.length} files...`; @@ -309,7 +292,7 @@ function get_watch_config() { } notify_redis({ success: true, changed_files }); } - } + }, }; } return null; @@ -324,11 +307,11 @@ function log_built_assets(results) { cliui.div( { text: chalk.cyan.bold("File"), - width: column_widths[0] + width: column_widths[0], }, { text: chalk.cyan.bold("Size"), - width: column_widths[1] + width: column_widths[1], } ); cliui.div(""); @@ -344,7 +327,7 @@ function log_built_assets(results) { output_by_dist_path[dist_path] = output_by_dist_path[dist_path] || []; output_by_dist_path[dist_path].push({ name: filename, - size: (data.bytes / 1000).toFixed(2) + " Kb" + size: (data.bytes / 1000).toFixed(2) + " Kb", }); } @@ -352,7 +335,7 @@ function log_built_assets(results) { let files = output_by_dist_path[dist_path]; cliui.div({ text: dist_path, - width: column_widths[0] + width: column_widths[0], }); for (let i in files) { @@ -367,11 +350,11 @@ function log_built_assets(results) { cliui.div( { text: branch + chalk[color]("" + file.name), - width: column_widths[0] + width: column_widths[0], }, { text: file.size, - width: column_widths[1] + width: column_widths[1], } ); } @@ -393,7 +376,7 @@ async function write_assets_json(metafile) { let asset_path = "/" + path.relative(sites_path, output); if (info.entryPoint) { let key = path.basename(info.entryPoint); - if (key.endsWith('.css') && asset_path.includes('/css-rtl/')) { + if (key.endsWith(".css") && asset_path.includes("/css-rtl/")) { rtl = true; key = `rtl_${key}`; } @@ -401,7 +384,7 @@ async function write_assets_json(metafile) { } } - let assets_json_path = path.resolve(assets_path, `assets${rtl?'-rtl':''}.json`); + let assets_json_path = path.resolve(assets_path, `assets${rtl ? "-rtl" : ""}.json`); let assets_json; try { assets_json = await fs.promises.readFile(assets_json_path, "utf-8"); @@ -413,26 +396,23 @@ async function write_assets_json(metafile) { let new_assets_json = Object.assign({}, assets_json, out); curr_assets_json = new_assets_json; - await fs.promises.writeFile( - assets_json_path, - JSON.stringify(new_assets_json, null, 4) - ); + await fs.promises.writeFile(assets_json_path, JSON.stringify(new_assets_json, null, 4)); await update_assets_json_in_cache(); return { new_assets_json, - prev_assets_json + prev_assets_json, }; } function update_assets_json_in_cache() { // update assets_json cache in redis, so that it can be read directly by python - return new Promise(resolve => { + return new Promise((resolve) => { let client = get_redis_subscriber("redis_cache"); // handle error event to avoid printing stack traces - client.on("error", _ => { + client.on("error", (_) => { log_warn("Cannot connect to redis_cache to update assets_json"); }); - client.del("assets_json", err => { + client.del("assets_json", (err) => { client.unref(); resolve(); }); @@ -464,7 +444,7 @@ function run_build_command_for_apps(apps) { async function notify_redis({ error, success, changed_files }) { // notify redis which in turns tells socketio to publish this to browser let subscriber = get_redis_subscriber("redis_socketio"); - subscriber.on("error", _ => { + subscriber.on("error", (_) => { log_warn("Cannot connect to redis_socketio for browser events"); }); @@ -472,20 +452,20 @@ async function notify_redis({ error, success, changed_files }) { if (error) { let formatted = await esbuild.formatMessages(error.errors, { kind: "error", - terminalWidth: 100 + terminalWidth: 100, }); let stack = error.stack.replace(new RegExp(bench_path, "g"), ""); payload = { error, formatted, - stack + stack, }; } if (success) { payload = { success: true, changed_files, - live_reload: argv["live-reload"] + live_reload: argv["live-reload"], }; } @@ -493,14 +473,14 @@ async function notify_redis({ error, success, changed_files }) { "events", JSON.stringify({ event: "build_event", - message: payload + message: payload, }) ); } function open_in_editor() { let subscriber = get_redis_subscriber("redis_socketio"); - subscriber.on("error", _ => { + subscriber.on("error", (_) => { log_warn("Cannot connect to redis_socketio for open_in_editor events"); }); subscriber.on("message", (event, file) => { diff --git a/esbuild/frappe-html.js b/esbuild/frappe-html.js index 9a7edb144d..d38a0c23cb 100644 --- a/esbuild/frappe-html.js +++ b/esbuild/frappe-html.js @@ -4,24 +4,24 @@ module.exports = { let path = require("path"); let fs = require("fs/promises"); - build.onResolve({ filter: /\.html$/ }, args => { + build.onResolve({ filter: /\.html$/ }, (args) => { return { path: path.join(args.resolveDir, args.path), - namespace: "frappe-html" + namespace: "frappe-html", }; }); - build.onLoad({ filter: /.*/, namespace: "frappe-html" }, args => { + build.onLoad({ filter: /.*/, namespace: "frappe-html" }, (args) => { let filepath = args.path; let filename = path.basename(filepath).split(".")[0]; return fs .readFile(filepath, "utf-8") - .then(content => { + .then((content) => { content = scrub_html_template(content); return { contents: `\n\tfrappe.templates['${filename}'] = \`${content}\`;\n`, - watchFiles: [filepath] + watchFiles: [filepath], }; }) .catch(() => { @@ -29,13 +29,13 @@ module.exports = { contents: "", warnings: [ { - text: `There was an error importing ${filepath}` - } - ] + text: `There was an error importing ${filepath}`, + }, + ], }; }); }); - } + }, }; function scrub_html_template(content) { diff --git a/esbuild/frappe-vue-style.js b/esbuild/frappe-vue-style.js new file mode 100644 index 0000000000..238a6e92e5 --- /dev/null +++ b/esbuild/frappe-vue-style.js @@ -0,0 +1,59 @@ +const fs = require("fs"); +const path = require("path"); +const { sites_path } = require("./utils"); + +module.exports = { + name: "frappe-vue-style", + setup(build) { + build.initialOptions.write = false; + build.onEnd((result) => { + let files = get_files(result.metafile.outputs); + let keys = Object.keys(files); + for (let out of result.outputFiles) { + let asset_path = "/" + path.relative(sites_path, out.path); + let dir = path.dirname(out.path); + if (out.path.endsWith(".js") && keys.includes(asset_path)) { + let name = out.path.split(".bundle.")[0]; + name = path.basename(name); + + let index = result.outputFiles.findIndex((f) => { + return f.path.endsWith(".css") && f.path.includes(`/${name}.bundle.`); + }); + + let css_data = JSON.stringify(result.outputFiles[index].text); + let modified = `frappe.dom.set_style(${css_data});\n${out.text}`; + out.contents = Buffer.from(modified); + + result.outputFiles.splice(index, 1); + if (result.outputFiles[index - 1].path.endsWith(".css.map")) { + result.outputFiles.splice(index - 1, 1); + } + } + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } + fs.writeFile(out.path, out.contents, (err) => { + err && console.error(err); + }); + } + }); + }, +}; + +function get_files(files) { + let result = {}; + for (let file in files) { + let info = files[file]; + let asset_path = "/" + path.relative(sites_path, file); + if (info && info.entryPoint && Object.keys(info.inputs).length !== 0) { + for (let input in info.inputs) { + if (input.includes(".vue?type=style")) { + let bundle_css = path.basename(info.entryPoint).replace(".js", ".css"); + result[asset_path] = bundle_css; + break; + } + } + } + } + return result; +} diff --git a/esbuild/ignore-assets.js b/esbuild/ignore-assets.js index 5edfef2110..fad7a95e0d 100644 --- a/esbuild/ignore-assets.js +++ b/esbuild/ignore-assets.js @@ -1,11 +1,11 @@ module.exports = { name: "frappe-ignore-asset", setup(build) { - build.onResolve({ filter: /^\/assets\// }, args => { + build.onResolve({ filter: /^\/assets\// }, (args) => { return { path: args.path, - external: true + external: true, }; }); - } + }, }; diff --git a/esbuild/sass_options.js b/esbuild/sass_options.js index 0c1189e90c..92b691cb46 100644 --- a/esbuild/sass_options.js +++ b/esbuild/sass_options.js @@ -1,19 +1,13 @@ let path = require("path"); let { get_app_path, app_list } = require("./utils"); -let node_modules_path = path.resolve( - get_app_path("frappe"), - "..", - "node_modules" -); -let app_paths = app_list - .map(get_app_path) - .map(app_path => path.resolve(app_path, "..")); +let node_modules_path = path.resolve(get_app_path("frappe"), "..", "node_modules"); +let app_paths = app_list.map(get_app_path).map((app_path) => path.resolve(app_path, "..")); module.exports = { includePaths: [node_modules_path, ...app_paths], quietDeps: true, - importer: function(url) { + importer: function (url) { if (url.startsWith("~")) { // strip ~ so that it can resolve from node_modules url = url.slice(1); @@ -24,7 +18,7 @@ module.exports = { } // normal file, let it go return { - file: url + file: url, }; - } + }, }; diff --git a/esbuild/utils.js b/esbuild/utils.js index 82490adb36..3edccfd024 100644 --- a/esbuild/utils.js +++ b/esbuild/utils.js @@ -26,24 +26,20 @@ const bundle_map = app_list.reduce((out, app) => { const public_js_path = public_js_paths[app]; if (fs.existsSync(public_js_path)) { const all_files = fs.readdirSync(public_js_path); - const js_files = all_files.filter(file => file.endsWith(".js")); + const js_files = all_files.filter((file) => file.endsWith(".js")); for (let js_file of js_files) { const filename = path.basename(js_file).split(".")[0]; - out[path.join(app, "js", filename)] = path.resolve( - public_js_path, - js_file - ); + out[path.join(app, "js", filename)] = path.resolve(public_js_path, js_file); } } return out; }, {}); -const get_public_path = app => public_paths[app]; +const get_public_path = (app) => public_paths[app]; -const get_build_json_path = app => - path.resolve(get_public_path(app), "build.json"); +const get_build_json_path = (app) => path.resolve(get_public_path(app), "build.json"); function get_build_json(app) { try { @@ -62,7 +58,7 @@ function delete_file(path) { function run_serially(tasks) { let result = Promise.resolve(); - tasks.forEach(task => { + tasks.forEach((task) => { if (task) { result = result.then ? result.then(task) : Promise.resolve(); } @@ -70,12 +66,12 @@ function run_serially(tasks) { return result; } -const get_app_path = app => app_paths[app]; +const get_app_path = (app) => app_paths[app]; function get_apps_list() { return fs .readFileSync(path.resolve(sites_path, "apps.txt"), { - encoding: "utf-8" + encoding: "utf-8", }) .split("\n") .filter(Boolean); @@ -112,16 +108,21 @@ function log(...args) { function get_redis_subscriber(kind) { // get redis subscriber that aborts after 10 connection attempts - let { get_redis_subscriber: get_redis } = require("../node_utils"); - return get_redis(kind, { - retry_strategy: function(options) { - // abort after 10 connection attempts - if (options.attempt > 10) { + let retry_strategy; + let { get_redis_subscriber: get_redis, get_conf } = require("../node_utils"); + + if (process.env.CI == 1 || get_conf().developer_mode == 0) { + retry_strategy = () => {}; + } else { + retry_strategy = function (options) { + // abort after 5 x 3 connection attempts ~= 3 seconds + if (options.attempt > 4) { return undefined; } - return Math.min(options.attempt * 100, 2000); - } - }); + return options.attempt * 100; + }; + } + return get_redis(kind, { retry_strategy }); } module.exports = { @@ -141,5 +142,5 @@ module.exports = { log, log_warn, log_error, - get_redis_subscriber + get_redis_subscriber, }; diff --git a/frappe/__init__.py b/frappe/__init__.py index 3057eacd3b..5a03438a9e 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -23,7 +23,7 @@ import click from werkzeug.local import Local, release_local from frappe.query_builder import ( - get_qb_engine, + get_query, get_query_builder, patch_query_aggregation, patch_query_execute, @@ -42,7 +42,7 @@ from .utils.jinja import ( ) from .utils.lazy_loader import lazy_import -__version__ = "14.0.0-dev" +__version__ = "15.0.0-dev" __title__ = "Frappe Framework" controllers = {} @@ -89,7 +89,7 @@ def _(msg: str, lang: str | None = None, context: str | None = None) -> str: _('Change') _('Change', context='Coins') """ - from frappe.translate import get_full_dict + from frappe.translate import get_all_translations from frappe.utils import is_html, strip_html_tags if not hasattr(local, "lang"): @@ -107,14 +107,15 @@ def _(msg: str, lang: str | None = None, context: str | None = None) -> str: msg = as_unicode(msg).strip() translated_string = "" + + all_translations = get_all_translations(lang) if context: string_key = f"{msg}:{context}" - translated_string = get_full_dict(lang).get(string_key) + translated_string = all_translations.get(string_key) if not translated_string: - translated_string = get_full_dict(lang).get(msg) + translated_string = all_translations.get(msg) - # return lang_full_dict according to lang passed parameter return translated_string or non_translated_string @@ -181,9 +182,9 @@ if TYPE_CHECKING: # end: static analysis hack -def init(site: str, sites_path: str = ".", new_site: bool = False) -> None: +def init(site: str, sites_path: str = ".", new_site: bool = False, force=False) -> None: """Initialize frappe for the current site. Reset thread locals `frappe.local`""" - if getattr(local, "initialised", None): + if getattr(local, "initialised", None) and not force: return local.error_log = [] @@ -203,6 +204,7 @@ def init(site: str, sites_path: str = ".", new_site: bool = False) -> None: "mute_emails": False, "has_dataurl": False, "new_site": new_site, + "read_only": False, } ) local.rollback_observers = [] @@ -221,7 +223,6 @@ def init(site: str, sites_path: str = ".", new_site: bool = False) -> None: local.conf = _dict(get_site_config()) local.lang = local.conf.lang or "en" - local.lang_full_dict = None local.module_app = None local.app_modules = None @@ -237,13 +238,12 @@ def init(site: str, sites_path: str = ".", new_site: bool = False) -> None: local.jenv = None local.jloader = None local.cache = {} - local.document_cache = {} - local.meta_cache = {} local.form_dict = _dict() + local.preload_assets = {"style": [], "script": []} local.session = _dict() local.dev_server = _dev_server local.qb = get_query_builder(local.conf.db_type or "mariadb") - local.qb.engine = get_qb_engine() + local.qb.get_query = get_query setup_module_map() if not _qb_patched.get(local.conf.db_type): @@ -325,10 +325,9 @@ def get_conf(site: str | None = None) -> dict[str, Any]: if hasattr(local, "conf"): return local.conf - else: - # if no site, get from common_site_config.json - with init_site(site): - return local.conf + # if no site, get from common_site_config.json + with init_site(site): + return local.conf class init_site: @@ -452,8 +451,11 @@ def msgprint( if as_list and type(msg) in (list, tuple): out.as_list = 1 - if sys.stdin.isatty(): - msg = _strip_html_tags(out.message) + if sys.stdin and sys.stdin.isatty(): + if out.as_list: + msg = [_strip_html_tags(msg) for msg in out.message] + else: + msg = _strip_html_tags(out.message) if flags.print_messages and out.message: print(f"Message: {_strip_html_tags(out.message)}") @@ -568,7 +570,7 @@ def get_user(): def get_roles(username=None) -> list[str]: """Returns roles of current user.""" - if not local.session: + if not local.session or not local.session.user: return ["Guest"] import frappe.permissions @@ -620,6 +622,7 @@ def sendmail( header=None, print_letterhead=False, with_container=False, + email_read_tracker_url=None, ): """Send email using user's default **Email Account** or global default **Email Account**. @@ -701,6 +704,7 @@ def sendmail( header=header, print_letterhead=print_letterhead, with_container=with_container, + email_read_tracker_url=email_read_tracker_url, ) # build email queue and send the email if send_now is True. @@ -732,14 +736,21 @@ def whitelist(allow_guest=False, xss_safe=False, methods=None): methods = ["GET", "POST", "PUT", "DELETE"] def innerfn(fn): + from frappe.utils.typing_validations import validate_argument_types + global whitelisted, guest_methods, xss_safe_methods, allowed_http_methods_for_whitelisted_func + # validate argument types only if request is present + in_request_or_test = lambda: getattr(local, "request", None) or local.flags.in_test # noqa: E731 + # get function from the unbound / bound method # this is needed because functions can be compared, but not methods method = None if hasattr(fn, "__func__"): - method = fn + method = validate_argument_types(fn, apply_condition=in_request_or_test) fn = method.__func__ + else: + fn = validate_argument_types(fn, apply_condition=in_request_or_test) whitelisted.append(fn) allowed_http_methods_for_whitelisted_func[fn] = methods @@ -760,7 +771,12 @@ def is_whitelisted(method): is_guest = session["user"] == "Guest" if method not in whitelisted or is_guest and method not in guest_methods: - throw(_("Not permitted"), PermissionError) + summary = _("You are not permitted to access this resource.") + detail = _("Function {0} is not whitelisted.").format( + bold(f"{method.__module__}.{method.__name__}") + ) + msg = f"
{summary}{detail}
" + throw(msg, PermissionError, title="Method Not Allowed") if is_guest and method not in xss_safe_methods: # strictly sanitize form_dict @@ -816,23 +832,30 @@ def write_only(): return innfn -def only_for(roles: list[str] | str, message=False): - """Raise `frappe.PermissionError` if the user does not have any of the given **Roles**. +def only_for(roles: list[str] | tuple[str] | str, message=False): + """ + Raises `frappe.PermissionError` if the user does not have any of the permitted roles. - :param roles: List of roles to check.""" - if local.flags.in_test: + :param roles: Permitted role(s) + """ + + if local.flags.in_test or local.session.user == "Administrator": return - if not isinstance(roles, (tuple, list)): + if isinstance(roles, str): roles = (roles,) - roles = set(roles) - myroles = set(get_roles()) - if not roles.intersection(myroles): - if message: - msgprint( - _("This action is only allowed for {}").format(bold(", ".join(roles))), _("Not Permitted") - ) - raise PermissionError + + if set(roles).isdisjoint(get_roles()): + if not message: + raise PermissionError + + throw( + _("This action is only allowed for {}").format( + ", ".join(bold(_(role)) for role in roles), + ), + PermissionError, + _("Not Permitted"), + ) def get_domain_data(module): @@ -889,29 +912,35 @@ def only_has_select_perm(doctype, user=None, ignore_permissions=False): if ignore_permissions: return False - if not user: - user = local.session.user + from frappe.permissions import get_role_permissions - import frappe.permissions + user = user or local.session.user + permissions = get_role_permissions(doctype, user=user) - permissions = frappe.permissions.get_role_permissions(doctype, user=user) - - if permissions.get("select") and not permissions.get("read"): - return True - else: - return False + return permissions.get("select") and not permissions.get("read") def has_permission( - doctype=None, ptype="read", doc=None, user=None, verbose=False, throw=False, parent_doctype=None + doctype=None, + ptype="read", + doc=None, + user=None, + verbose=False, + throw=False, + *, + parent_doctype=None, ): - """Raises `frappe.PermissionError` if not permitted. + """ + Returns True if the user has permission `ptype` for given `doctype` or `doc` + Raises `frappe.PermissionError` if user isn't permitted and `throw` is truthy :param doctype: DocType for which permission is to be check. :param ptype: Permission type (`read`, `write`, `create`, `submit`, `cancel`, `amend`). Default: `read`. :param doc: [optional] Checks User permissions for given doc. :param user: [optional] Check for given user. Default: current user. - :param parent_doctype: Required when checking permission for a child DocType (unless doc is specified).""" + :param verbose: DEPRECATED, will be removed in a future release. + :param parent_doctype: Required when checking permission for a child DocType (unless doc is specified). + """ import frappe.permissions if not doctype and doc: @@ -921,7 +950,6 @@ def has_permission( doctype, ptype, doc=doc, - verbose=verbose, user=user, raise_exception=throw, parent_doctype=parent_doctype, @@ -1001,19 +1029,15 @@ def get_precision( return get_field_precision(get_meta(doctype).get_field(fieldname), doc, currency) -def generate_hash(txt: str | None = None, length: int | None = None) -> str: - """Generates random hash for given text + current timestamp + random string.""" - import hashlib - import time +def generate_hash(txt: str | None = None, length: int = 56) -> str: + """Generate random hash using best available randomness source.""" + import math + import secrets - from .utils import random_string + if not length: + length = 56 - digest = hashlib.sha224( - ((txt or "") + repr(time.time()) + repr(random_string(8))).encode() - ).hexdigest() - if length: - digest = digest[:length] - return digest + return secrets.token_hex(math.ceil(length / 2))[:length] def reset_metadata_version(): @@ -1046,56 +1070,26 @@ def set_value(doctype, docname, fieldname, value=None): return frappe.client.set_value(doctype, docname, fieldname, value) -@overload -def get_cached_doc(doctype, docname, _allow_dict=True) -> dict: - ... - - -@overload def get_cached_doc(*args, **kwargs) -> "Document": - ... - - -def get_cached_doc(*args, **kwargs): - allow_dict = kwargs.pop("_allow_dict", False) - - def _respond(doc, from_redis=False): - if not allow_dict and isinstance(doc, dict): - local.document_cache[key] = doc = get_doc(doc) - - elif from_redis: - local.document_cache[key] = doc - + if (key := can_cache_doc(args)) and (doc := cache().hget("document_cache", key)): return doc - if key := can_cache_doc(args): - # local cache - has "ready" `Document` objects - if doc := local.document_cache.get(key): - return _respond(doc) - - # redis cache - if doc := cache().hget("document_cache", key): - return _respond(doc, True) - - # Not found in local/redis, fetch from DB + # Not found in cache, fetch from DB doc = get_doc(*args, **kwargs) # Store in cache if not key: key = get_document_cache_key(doc.doctype, doc.name) - local.document_cache[key] = doc - - # Avoid setting in local.cache since we're already using local.document_cache above - # Try pickling the doc object as-is first, else fallback to doc.as_dict() - try: - cache().hset("document_cache", key, doc, cache_locally=False) - except Exception: - cache().hset("document_cache", key, doc.as_dict(), cache_locally=False) + _set_document_in_cache(key, doc) return doc +def _set_document_in_cache(key: str, doc: "Document") -> None: + cache().hset("document_cache", key, doc) + + def can_cache_doc(args) -> str | None: """ Determine if document should be cached based on get_doc params. @@ -1106,7 +1100,7 @@ def can_cache_doc(args) -> str | None: return doctype = args[0] - name = doctype if len(args) == 1 else args[1] + name = doctype if len(args) == 1 or args[1] is None else args[1] # Only cache if both doctype and name are strings if isinstance(doctype, str) and isinstance(name, str): @@ -1119,12 +1113,11 @@ def get_document_cache_key(doctype: str, name: str): def clear_document_cache(doctype, name): cache().hdel("last_modified", doctype) - key = get_document_cache_key(doctype, name) - if key in local.document_cache: - del local.document_cache[key] - cache().hdel("document_cache", key) + cache().hdel("document_cache", get_document_cache_key(doctype, name)) + if doctype == "System Settings" and hasattr(local, "system_settings"): delattr(local, "system_settings") + if doctype == "Website Settings" and hasattr(local, "website_settings"): delattr(local, "website_settings") @@ -1133,7 +1126,7 @@ def get_cached_value( doctype: str, name: str, fieldname: str = "name", as_dict: bool = False ) -> Any: try: - doc = get_cached_doc(doctype, name, _allow_dict=True) + doc = get_cached_doc(doctype, name) except DoesNotExistError: clear_last_message() return @@ -1169,22 +1162,18 @@ def get_doc(*args, **kwargs) -> "Document": doc = frappe.model.document.get_doc(*args, **kwargs) - # Replace cache - if key := can_cache_doc(args): - if key in local.document_cache: - local.document_cache[key] = doc - - if cache().hexists("document_cache", key): - cache().hset("document_cache", key, doc.as_dict()) + # Replace cache if stale one exists + if (key := can_cache_doc(args)) and cache().hexists("document_cache", key): + _set_document_in_cache(key, doc) return doc -def get_last_doc(doctype, filters=None, order_by="creation desc"): +def get_last_doc(doctype, filters=None, order_by="creation desc", *, for_update=False): """Get last created document of this type.""" d = get_all(doctype, filters=filters, limit_page_length=1, order_by=order_by, pluck="name") if d: - return get_doc(doctype, d[0]) + return get_doc(doctype, d[0], for_update=for_update) else: raise DoesNotExistError @@ -1280,7 +1269,7 @@ def reload_doc( return frappe.modules.reload_doc(module, dt, dn, force=force, reset_permissions=reset_permissions) -@whitelist() +@whitelist(methods=["POST", "PUT"]) def rename_doc( doctype: str, old: str, @@ -1388,23 +1377,37 @@ def get_all_apps(with_internal_apps=True, sites_path=None): @request_cache -def get_installed_apps(sort=False, frappe_last=False): - """Get list of installed apps in current site.""" +def get_installed_apps(sort=False, frappe_last=False, *, _ensure_on_bench=False): + """ + Get list of installed apps in current site. + + :param sort: [DEPRECATED] Sort installed apps based on the sequence in sites/apps.txt + :param frappe_last: [DEPRECATED] Keep frappe last. Do not use this, reverse the app list instead. + :param ensure_on_bench: Only return apps that are present on bench. + """ + from frappe.utils.deprecations import deprecation_warning + if getattr(flags, "in_install_db", True): return [] if not db: connect() - if not local.all_apps: - local.all_apps = cache().get_value("all_apps", get_all_apps) - installed = json.loads(db.get_global("installed_apps") or "[]") if sort: + if not local.all_apps: + local.all_apps = cache().get_value("all_apps", get_all_apps) + + deprecation_warning("`sort` argument is deprecated and will be removed in v15.") installed = [app for app in local.all_apps if app in installed] + if _ensure_on_bench: + all_apps = cache().get_value("all_apps", get_all_apps) + installed = [app for app in installed if app in all_apps] + if frappe_last: + deprecation_warning("`frappe_last` argument is deprecated and will be removed in v15.") if "frappe" in installed: installed.remove("frappe") installed.append("frappe") @@ -1431,24 +1434,28 @@ def get_doc_hooks(): @request_cache def _load_app_hooks(app_name: str | None = None): + import types + hooks = {} - apps = [app_name] if app_name else get_installed_apps(sort=True) + apps = [app_name] if app_name else get_installed_apps(_ensure_on_bench=True) for app in apps: try: app_hooks = get_module(f"{app}.hooks") - except ImportError: + except ImportError as e: if local.flags.in_install_app: # if app is not installed while restoring # ignore it pass - print(f'Could not find app "{app}"') - if not request: - raise SystemExit + print(f'Could not find app "{app}": \n{e}') raise - for key in dir(app_hooks): + + def _is_valid_hook(obj): + return not isinstance(obj, (types.ModuleType, types.FunctionType, type)) + + for key, value in inspect.getmembers(app_hooks, predicate=_is_valid_hook): if not key.startswith("_"): - append_hook(hooks, key, getattr(app_hooks, key)) + append_hook(hooks, key, value) return hooks @@ -1556,7 +1563,7 @@ def read_file(path, raise_not_found=False): def get_attr(method_string: str) -> Any: """Get python method object from its name.""" - app_name = method_string.split(".")[0] + app_name = method_string.split(".", 1)[0] if ( not local.flags.in_uninstall and not local.flags.in_install @@ -1717,27 +1724,6 @@ def copy_doc(doc: "Document", ignore_no_copy: bool = True) -> "Document": return newdoc -def compare(val1, condition, val2): - """Compare two values using `frappe.utils.compare` - - `condition` could be: - - "^" - - "in" - - "not in" - - "=" - - "!=" - - ">" - - "<" - - ">=" - - "<=" - - "not None" - - "None" - """ - import frappe.utils - - return frappe.utils.compare(val1, condition, val2) - - def respond_as_web_page( title, html, @@ -1796,6 +1782,14 @@ def respond_as_web_page( local.response["context"] = context +def redirect(url): + """Raise a 301 redirect to url""" + from frappe.exceptions import Redirect + + flags.redirect_location = url + raise Redirect + + def redirect_to_message(title, html, http_status_code=None, context=None, indicator_color=None): """Redirects to /message?id=random Similar to respond_as_web_page, but used to 'redirect' and show message pages like success, failure, etc. with a detailed message @@ -1851,9 +1845,6 @@ def get_list(doctype, *args, **kwargs): # filter as a list of lists frappe.get_list("ToDo", fields="*", filters = [["modified", ">", "2014-01-01"]]) - - # filter as a list of dicts - frappe.get_list("ToDo", fields="*", filters = {"description": ("like", "test%")}) """ import frappe.model.db_query @@ -1878,9 +1869,6 @@ def get_all(doctype, *args, **kwargs): # filter as a list of lists frappe.get_all("ToDo", fields=["*"], filters = [["modified", ">", "2014-01-01"]]) - - # filter as a list of dicts - frappe.get_all("ToDo", fields=["*"], filters = {"description": ("like", "test%")}) """ kwargs["ignore_permissions"] = True if not "limit_page_length" in kwargs: @@ -1903,7 +1891,7 @@ def get_value(*args, **kwargs): return db.get_value(*args, **kwargs) -def as_json(obj: dict | list, indent=1, separators=None) -> str: +def as_json(obj: dict | list, indent=1, separators=None, ensure_ascii=True) -> str: from frappe.utils.response import json_handler if separators is None: @@ -1911,13 +1899,24 @@ def as_json(obj: dict | list, indent=1, separators=None) -> str: try: return json.dumps( - obj, indent=indent, sort_keys=True, default=json_handler, separators=separators + obj, + indent=indent, + sort_keys=True, + default=json_handler, + separators=separators, + ensure_ascii=ensure_ascii, ) except TypeError: # this would break in case the keys are not all os "str" type - as defined in the JSON # adding this to ensure keys are sorted (expected behaviour) sorted_obj = dict(sorted(obj.items(), key=lambda kv: str(kv[0]))) - return json.dumps(sorted_obj, indent=indent, default=json_handler, separators=separators) + return json.dumps( + sorted_obj, + indent=indent, + default=json_handler, + separators=separators, + ensure_ascii=ensure_ascii, + ) def are_emails_muted(): @@ -1965,13 +1964,13 @@ def get_print( name=None, print_format=None, style=None, - html=None, as_pdf=False, doc=None, output=None, no_letterhead=0, password=None, pdf_options=None, + letterhead=None, ): """Get Print Format for given document. @@ -1990,18 +1989,14 @@ def get_print( local.form_dict.style = style local.form_dict.doc = doc local.form_dict.no_letterhead = no_letterhead + local.form_dict.letterhead = letterhead pdf_options = pdf_options or {} if password: pdf_options["password"] = password - if not html: - html = get_response_content("printview") - - if as_pdf: - return get_pdf(html, options=pdf_options, output=output) - else: - return html + html = get_response_content("printview") + return get_pdf(html, options=pdf_options, output=output) if as_pdf else html def attach_print( @@ -2017,6 +2012,7 @@ def attach_print( password=None, ): from frappe.utils import scrub_urls + from frappe.utils.pdf import get_pdf if not file_name: file_name = name @@ -2036,7 +2032,6 @@ def attach_print( kwargs = dict( print_format=print_format, style=style, - html=html, doc=doc, no_letterhead=no_letterhead, password=password, @@ -2046,10 +2041,14 @@ def attach_print( if int(print_settings.send_print_as_pdf or 0): ext = ".pdf" kwargs["as_pdf"] = True - content = get_print(doctype, name, **kwargs) + content = ( + get_pdf(html, options={"password": password} if password else None) + if html + else get_print(doctype, name, **kwargs) + ) else: ext = ".html" - content = scrub_urls(get_print(doctype, name, **kwargs)).encode("utf-8") + content = html or scrub_urls(get_print(doctype, name, **kwargs)).encode("utf-8") out = {"fname": file_name + ext, "fcontent": content} @@ -2197,13 +2196,22 @@ def log_error(title=None, message=None, reference_doctype=None, reference_name=N title = title or "Error" traceback = as_unicode(traceback or get_traceback(with_context=True)) - return get_doc( + if not db: + print(f"Failed to log error in db: {title}") + return + + error_log = get_doc( doctype="Error Log", error=traceback, method=title, reference_doctype=reference_doctype, reference_name=reference_name, - ).insert(ignore_permissions=True) + ) + + if flags.read_only: + error_log.deferred_insert() + else: + return error_log.insert(ignore_permissions=True) def get_desk_link(doctype, name): @@ -2257,14 +2265,22 @@ def safe_eval(code, eval_globals=None, eval_locals=None): def get_website_settings(key): if not hasattr(local, "website_settings"): - local.website_settings = db.get_singles_dict("Website Settings", cast=True) + try: + local.website_settings = get_cached_doc("Website Settings") + except DoesNotExistError: + clear_last_message() + return return local.website_settings.get(key) def get_system_settings(key): if not hasattr(local, "system_settings"): - local.system_settings = db.get_singles_dict("System Settings", cast=True) + try: + local.system_settings = get_cached_doc("System Settings") + except DoesNotExistError: # possible during new install + clear_last_message() + return return local.system_settings.get(key) @@ -2294,7 +2310,7 @@ def get_version(doctype, name, limit=None, head=False, raise_err=True): """ meta = get_meta(doctype) if meta.track_changes: - names = db.get_all( + names = get_all( "Version", filters={ "ref_doctype": doctype, diff --git a/frappe/api.py b/frappe/api.py index 1048468077..084bee060b 100644 --- a/frappe/api.py +++ b/frappe/api.py @@ -3,6 +3,7 @@ import base64 import binascii import json +from typing import Literal from urllib.parse import urlencode, urlparse import frappe @@ -49,106 +50,149 @@ def handle(): if len(parts) > 3: name = parts[3] - if call == "method": - frappe.local.form_dict.cmd = doctype - return frappe.handler.handle() + return _RESTAPIHandler(call, doctype, name).get_response() - elif call == "resource": - if "run_method" in frappe.local.form_dict: - method = frappe.local.form_dict.pop("run_method") - doc = frappe.get_doc(doctype, name) - doc.is_whitelisted(method) - if frappe.local.request.method == "GET": - if not doc.has_permission("read"): - frappe.throw(_("Not permitted"), frappe.PermissionError) - frappe.local.response.update({"data": doc.run_method(method, **frappe.local.form_dict)}) +class _RESTAPIHandler: + def __init__(self, call: Literal["method", "resource"], doctype: str | None, name: str | None): + self.call = call + self.doctype = doctype + self.name = name - if frappe.local.request.method == "POST": - if not doc.has_permission("write"): - frappe.throw(_("Not permitted"), frappe.PermissionError) + def get_response(self): + """Prepare and get response based on URL and form body. - frappe.local.response.update({"data": doc.run_method(method, **frappe.local.form_dict)}) - frappe.db.commit() - - else: - if name: - if frappe.local.request.method == "GET": - doc = frappe.get_doc(doctype, name) - if not doc.has_permission("read"): - raise frappe.PermissionError - frappe.local.response.update({"data": doc}) - - if frappe.local.request.method == "PUT": - data = get_request_form_data() - - doc = frappe.get_doc(doctype, name, for_update=True) - - if "flags" in data: - del data["flags"] - - # Not checking permissions here because it's checked in doc.save - doc.update(data) - - frappe.local.response.update({"data": doc.save().as_dict()}) - - # check for child table doctype - if doc.get("parenttype"): - frappe.get_doc(doc.parenttype, doc.parent).save() - - frappe.db.commit() - - if frappe.local.request.method == "DELETE": - # Not checking permissions here because it's checked in delete_doc - frappe.delete_doc(doctype, name, ignore_missing=False) - frappe.local.response.http_status_code = 202 - frappe.local.response.message = "ok" - frappe.db.commit() - - elif doctype: - if frappe.local.request.method == "GET": - # set fields for frappe.get_list - if frappe.local.form_dict.get("fields"): - frappe.local.form_dict["fields"] = json.loads(frappe.local.form_dict["fields"]) - - # set limit of records for frappe.get_list - frappe.local.form_dict.setdefault( - "limit_page_length", - frappe.local.form_dict.limit or frappe.local.form_dict.limit_page_length or 20, - ) - - # convert strings to native types - only as_dict and debug accept bool - for param in ["as_dict", "debug"]: - param_val = frappe.local.form_dict.get(param) - if param_val is not None: - frappe.local.form_dict[param] = sbool(param_val) - - # evaluate frappe.get_list - data = frappe.call(frappe.client.get_list, doctype, **frappe.local.form_dict) - - # set frappe.get_list result to response - frappe.local.response.update({"data": data}) - - if frappe.local.request.method == "POST": - # fetch data from from dict - data = get_request_form_data() - data.update({"doctype": doctype}) - - # insert document from request data - doc = frappe.get_doc(data).insert() - - # set response data - frappe.local.response.update({"data": doc.as_dict()}) - - # commit for POST requests - frappe.db.commit() - else: + Note: most methods of this class directly operate on the response local. + """ + match self.call: + case "method": + return self.handle_method() + case "resource": + self.handle_resource() + case _: raise frappe.DoesNotExistError - else: - raise frappe.DoesNotExistError + return build_response("json") - return build_response("json") + def handle_method(self): + frappe.local.form_dict.cmd = self.doctype + return frappe.handler.handle() + + def handle_resource(self): + if self.doctype and self.name: + self.handle_document_resource() + elif self.doctype: + self.handle_doctype_resource() + else: + raise frappe.DoesNotExistError + + def handle_document_resource(self): + if "run_method" in frappe.local.form_dict: + self.execute_doc_method() + return + + match frappe.local.request.method: + case "GET": + self.get_doc() + case "PUT": + self.update_doc() + case "DELETE": + self.delete_doc() + case _: + raise frappe.DoesNotExistError + + def handle_doctype_resource(self): + match frappe.local.request.method: + case "GET": + self.get_doc_list() + case "POST": + self.create_doc() + case _: + raise frappe.DoesNotExistError + + def execute_doc_method(self): + method = frappe.local.form_dict.pop("run_method") + doc = frappe.get_doc(self.doctype, self.name) + doc.is_whitelisted(method) + + if frappe.local.request.method == "GET": + if not doc.has_permission("read"): + frappe.throw(_("Not permitted"), frappe.PermissionError) + frappe.local.response.update({"data": doc.run_method(method, **frappe.local.form_dict)}) + + elif frappe.local.request.method == "POST": + if not doc.has_permission("write"): + frappe.throw(_("Not permitted"), frappe.PermissionError) + + frappe.local.response.update({"data": doc.run_method(method, **frappe.local.form_dict)}) + frappe.db.commit() + + def get_doc(self): + doc = frappe.get_doc(self.doctype, self.name) + if not doc.has_permission("read"): + raise frappe.PermissionError + doc.apply_fieldlevel_read_permissions() + frappe.local.response.update({"data": doc}) + + def update_doc(self): + data = get_request_form_data() + + doc = frappe.get_doc(self.doctype, self.name, for_update=True) + + if "flags" in data: + del data["flags"] + + # Not checking permissions here because it's checked in doc.save + doc.update(data) + + frappe.local.response.update({"data": doc.save().as_dict()}) + + # check for child table doctype + if doc.get("parenttype"): + frappe.get_doc(doc.parenttype, doc.parent).save() + frappe.db.commit() + + def delete_doc(self): + # Not checking permissions here because it's checked in delete_doc + frappe.delete_doc(self.doctype, self.name, ignore_missing=False) + frappe.local.response.http_status_code = 202 + frappe.local.response.message = "ok" + frappe.db.commit() + + def get_doc_list(self): + if frappe.local.form_dict.get("fields"): + frappe.local.form_dict["fields"] = json.loads(frappe.local.form_dict["fields"]) + + # set limit of records for frappe.get_list + frappe.local.form_dict.setdefault( + "limit_page_length", + frappe.local.form_dict.limit or frappe.local.form_dict.limit_page_length or 20, + ) + + # convert strings to native types - only as_dict and debug accept bool + for param in ["as_dict", "debug"]: + param_val = frappe.local.form_dict.get(param) + if param_val is not None: + frappe.local.form_dict[param] = sbool(param_val) + + # evaluate frappe.get_list + data = frappe.call(frappe.client.get_list, self.doctype, **frappe.local.form_dict) + + # set frappe.get_list result to response + frappe.local.response.update({"data": data}) + + def create_doc(self): + data = get_request_form_data() + data.update({"doctype": self.doctype}) + + # insert document from request data + doc = frappe.get_doc(data).insert() + + # set response data + frappe.local.response.update({"data": doc.as_dict()}) + + # commit for POST requests + frappe.db.commit() def get_request_form_data(): diff --git a/frappe/app.py b/frappe/app.py index b9db59cdb1..fab8facd3f 100644 --- a/frappe/app.py +++ b/frappe/app.py @@ -12,40 +12,28 @@ from werkzeug.wrappers import Request, Response import frappe import frappe.api -import frappe.auth import frappe.handler import frappe.monitor import frappe.rate_limiter import frappe.recorder import frappe.utils.response from frappe import _ +from frappe.auth import SAFE_HTTP_METHODS, UNSAFE_HTTP_METHODS, HTTPRequest from frappe.core.doctype.comment.comment import update_comments_in_parent_after_request from frappe.middlewares import StaticDataMiddleware -from frappe.utils import get_site_name, sanitize_html +from frappe.utils import cint, get_site_name, sanitize_html from frappe.utils.error import make_error_snapshot from frappe.website.serve import get_response -local_manager = LocalManager([frappe.local]) +local_manager = LocalManager(frappe.local) _site = None _sites_path = os.environ.get("SITES_PATH", ".") -SAFE_HTTP_METHODS = ("GET", "HEAD", "OPTIONS") -UNSAFE_HTTP_METHODS = ("POST", "PUT", "DELETE", "PATCH") - - -class RequestContext: - def __init__(self, environ): - self.request = Request(environ) - - def __enter__(self): - init_request(self.request) - - def __exit__(self, type, value, traceback): - frappe.destroy() +@local_manager.middleware @Request.application -def application(request): +def application(request: Request): response = None try: @@ -53,9 +41,6 @@ def application(request): init_request(request) - frappe.recorder.record() - frappe.monitor.start() - frappe.rate_limiter.apply() frappe.api.validate_auth() if request.method == "OPTIONS": @@ -82,53 +67,89 @@ def application(request): except HTTPException as e: return e - except frappe.SessionStopped as e: - response = frappe.utils.response.handle_session_stopped() - except Exception as e: response = handle_exception(e) else: - rollback = after_request(rollback) + rollback = sync_database(rollback) finally: - if request.method in ("POST", "PUT") and frappe.db and rollback: + # Important note: + # this function *must* always return a response, hence any exception thrown outside of + # try..catch block like this finally block needs to be handled appropriately. + + if request.method in UNSAFE_HTTP_METHODS and frappe.db and rollback: frappe.db.rollback() - frappe.rate_limiter.update() - frappe.monitor.stop(response) - frappe.recorder.dump() + try: + run_after_request_hooks(request, response) + except Exception as e: + # We can not handle exceptions safely here. + frappe.logger().error("Failed to run after request hook", exc_info=True) log_request(request, response) process_response(response) - frappe.destroy() + if frappe.db: + frappe.db.close() return response +def run_after_request_hooks(request, response): + if not getattr(frappe.local, "initialised", False): + return + + for after_request_task in frappe.get_hooks("after_request"): + frappe.call(after_request_task, response=response, request=request) + + def init_request(request): frappe.local.request = request frappe.local.is_ajax = frappe.get_request_header("X-Requested-With") == "XMLHttpRequest" site = _site or request.headers.get("X-Frappe-Site-Name") or get_site_name(request.host) - frappe.init(site=site, sites_path=_sites_path) + frappe.init(site=site, sites_path=_sites_path, force=True) if not (frappe.local.conf and frappe.local.conf.db_name): # site does not exist raise NotFound - if frappe.local.conf.get("maintenance_mode"): + if frappe.local.conf.maintenance_mode: frappe.connect() - raise frappe.SessionStopped("Session Stopped") + if frappe.local.conf.allow_reads_during_maintenance: + setup_read_only_mode() + else: + raise frappe.SessionStopped("Session Stopped") else: frappe.connect(set_admin_as_user=False) - request.max_content_length = frappe.local.conf.get("max_file_size") or 10 * 1024 * 1024 + request.max_content_length = cint(frappe.local.conf.get("max_file_size")) or 10 * 1024 * 1024 make_form_dict(request) if request.method != "OPTIONS": - frappe.local.http_request = frappe.auth.HTTPRequest() + frappe.local.http_request = HTTPRequest() + + for before_request_task in frappe.get_hooks("before_request"): + frappe.call(before_request_task) + + +def setup_read_only_mode(): + """During maintenance_mode reads to DB can still be performed to reduce downtime. This + function sets up read only mode + + - Setting global flag so other pages, desk and database can know that we are in read only mode. + - Setup read only database access either by: + - Connecting to read replica if one exists + - Or setting up read only SQL transactions. + """ + frappe.flags.read_only = True + + # If replica is available then just connect replica, else setup read only transaction. + if frappe.conf.read_from_replica: + frappe.connect_replica() + else: + frappe.db.begin(read_only=True) def log_request(request, response): @@ -159,35 +180,45 @@ def process_response(response): response.headers.extend(frappe.local.rate_limiter.headers()) # CORS headers - if hasattr(frappe.local, "conf") and frappe.conf.allow_cors: + if hasattr(frappe.local, "conf"): set_cors_headers(response) def set_cors_headers(response): - origin = frappe.request.headers.get("Origin") - allow_cors = frappe.conf.allow_cors - if not (origin and allow_cors): + if not ( + (allowed_origins := frappe.conf.allow_cors) + and (request := frappe.local.request) + and (origin := request.headers.get("Origin")) + ): return - if allow_cors != "*": - if not isinstance(allow_cors, list): - allow_cors = [allow_cors] + if allowed_origins != "*": + if not isinstance(allowed_origins, list): + allowed_origins = [allowed_origins] - if origin not in allow_cors: + if origin not in allowed_origins: return - response.headers.extend( - { - "Access-Control-Allow-Origin": origin, - "Access-Control-Allow-Credentials": "true", - "Access-Control-Allow-Methods": "GET, POST, PUT, DELETE, OPTIONS", - "Access-Control-Allow-Headers": ( - "Authorization,DNT,X-Mx-ReqToken," - "Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since," - "Cache-Control,Content-Type" - ), - } - ) + cors_headers = { + "Access-Control-Allow-Credentials": "true", + "Access-Control-Allow-Origin": origin, + "Vary": "Origin", + } + + # only required for preflight requests + if request.method == "OPTIONS": + cors_headers["Access-Control-Allow-Methods"] = request.headers.get( + "Access-Control-Request-Method" + ) + + if allowed_headers := request.headers.get("Access-Control-Request-Headers"): + cors_headers["Access-Control-Allow-Headers"] = allowed_headers + + # allow browsers to cache preflight requests for upto a day + if not frappe.conf.developer_mode: + cors_headers["Access-Control-Max-Age"] = "86400" + + response.headers.extend(cors_headers) def make_form_dict(request): @@ -222,11 +253,20 @@ def handle_exception(e): or (frappe.local.request.path.startswith("/api/") and not accept_header.startswith("text")) ) + if not frappe.session.user: + # If session creation fails then user won't be unset. This causes a lot of code that + # assumes presence of this to fail. Session creation fails => guest or expired login + # usually. + frappe.session.user = "Guest" + if respond_as_json: # handle ajax responses first # if the request is ajax, send back the trace or error message response = frappe.utils.response.report_error(http_status_code) + elif isinstance(e, frappe.SessionStopped): + response = frappe.utils.response.handle_session_stopped() + elif ( http_status_code == 500 and (frappe.db and isinstance(e, frappe.db.InternalError)) @@ -292,19 +332,22 @@ def handle_exception(e): return response -def after_request(rollback): +def sync_database(rollback: bool) -> bool: # if HTTP method would change server state, commit if necessary - if frappe.db and ( - frappe.local.flags.commit or frappe.local.request.method in UNSAFE_HTTP_METHODS + if ( + frappe.db + and (frappe.local.flags.commit or frappe.local.request.method in UNSAFE_HTTP_METHODS) + and frappe.db.transaction_writes ): - if frappe.db.transaction_writes: - frappe.db.commit() - rollback = False + frappe.db.commit() + rollback = False + elif frappe.db: + frappe.db.rollback() + rollback = False # update session - if getattr(frappe.local, "session_obj", None): - updated_in_db = frappe.local.session_obj.update() - if updated_in_db: + if session := getattr(frappe.local, "session_obj", None): + if session.update(): frappe.db.commit() rollback = False @@ -313,9 +356,6 @@ def after_request(rollback): return rollback -application = local_manager.make_middleware(application) - - def serve( port=8000, profile=False, no_reload=False, no_threading=False, site=None, sites_path="." ): @@ -349,6 +389,7 @@ def serve( "0.0.0.0", int(port), application, + exclude_patterns=["test_*"], use_reloader=False if in_test_env else not no_reload, use_debugger=not in_test_env, use_evalex=not in_test_env, diff --git a/frappe/auth.py b/frappe/auth.py index 455e9ee0c5..f1cdac52bd 100644 --- a/frappe/auth.py +++ b/frappe/auth.py @@ -6,9 +6,8 @@ import frappe import frappe.database import frappe.utils import frappe.utils.user -from frappe import _, conf +from frappe import _ from frappe.core.doctype.activity_log.activity_log import add_authentication_log -from frappe.modules.patch_handler import check_session_stopped from frappe.sessions import Session, clear_sessions, delete_session from frappe.translate import get_language from frappe.twofactor import ( @@ -21,6 +20,9 @@ from frappe.utils import cint, date_diff, datetime, get_datetime, today from frappe.utils.password import check_password from frappe.website.utils import get_home_page +SAFE_HTTP_METHODS = frozenset(("GET", "HEAD", "OPTIONS")) +UNSAFE_HTTP_METHODS = frozenset(("POST", "PUT", "DELETE", "PATCH")) + class HTTPRequest: def __init__(self): @@ -30,9 +32,6 @@ class HTTPRequest: # load cookies self.set_cookies() - # set frappe.local.db - self.connect() - # login and start/resume user session self.set_session() @@ -45,9 +44,6 @@ class HTTPRequest: # write out latest cookies frappe.local.cookie_manager.init_cookies() - # check session status - check_session_stopped() - @property def domain(self): if not getattr(self, "_domain", None): @@ -59,7 +55,9 @@ class HTTPRequest: def set_request_ip(self): if frappe.get_request_header("X-Forwarded-For"): - frappe.local.request_ip = (frappe.get_request_header("X-Forwarded-For").split(",")[0]).strip() + frappe.local.request_ip = ( + frappe.get_request_header("X-Forwarded-For").split(",", 1)[0] + ).strip() elif frappe.get_request_header("REMOTE_ADDR"): frappe.local.request_ip = frappe.get_request_header("REMOTE_ADDR") @@ -74,39 +72,25 @@ class HTTPRequest: frappe.local.login_manager = LoginManager() def validate_csrf_token(self): - if frappe.local.request and frappe.local.request.method in ("POST", "PUT", "DELETE"): - if not frappe.local.session: - return - if ( - not frappe.local.session.data.csrf_token - or frappe.local.session.data.device == "mobile" - or frappe.conf.get("ignore_csrf", None) - ): - # not via boot - return + if ( + not frappe.request + or frappe.request.method not in UNSAFE_HTTP_METHODS + or frappe.conf.ignore_csrf + or not frappe.session + or not (saved_token := frappe.session.data.csrf_token) + or ( + (frappe.get_request_header("X-Frappe-CSRF-Token") or frappe.form_dict.pop("csrf_token", None)) + == saved_token + ) + ): + return - csrf_token = frappe.get_request_header("X-Frappe-CSRF-Token") - if not csrf_token and "csrf_token" in frappe.local.form_dict: - csrf_token = frappe.local.form_dict.csrf_token - del frappe.local.form_dict["csrf_token"] - - if frappe.local.session.data.csrf_token != csrf_token: - frappe.local.flags.disable_traceback = True - frappe.throw(_("Invalid Request"), frappe.CSRFTokenError) + frappe.flags.disable_traceback = True + frappe.throw(_("Invalid Request"), frappe.CSRFTokenError) def set_lang(self): frappe.local.lang = get_language() - def get_db_name(self): - """get database name from conf""" - return conf.db_name - - def connect(self): - """connect to db, from ac_name or db_name""" - frappe.local.db = frappe.database.get_db( - user=self.get_db_name(), password=getattr(conf, "db_password", "") - ) - class LoginManager: @@ -140,6 +124,9 @@ class LoginManager: self.set_user_info() def login(self): + if frappe.get_system_settings("disable_user_pass_login"): + frappe.throw(_("Login with username and password is not allowed."), frappe.AuthenticationError) + # clear cache frappe.clear_cache(user=frappe.form_dict.get("usr")) user, pwd = get_cached_user_pass() @@ -156,6 +143,7 @@ class LoginManager: authenticate_for_2factor(self.user) if not confirm_otp_token(self): return False + frappe.form_dict.pop("pwd", None) self.post_login() def post_login(self): @@ -225,14 +213,16 @@ class LoginManager: def clear_active_sessions(self): """Clear other sessions of the current user if `deny_multiple_sessions` is not set""" + if frappe.session.user == "Guest": + return + if not ( cint(frappe.conf.get("deny_multiple_sessions")) or cint(frappe.db.get_system_setting("deny_multiple_sessions")) ): return - if frappe.session.user != "Guest": - clear_sessions(frappe.session.user, keep_current=True) + clear_sessions(frappe.session.user, keep_current=True) def authenticate(self, user: str = None, pwd: str = None): from frappe.core.doctype.user.user import User @@ -242,10 +232,11 @@ class LoginManager: if not (user and pwd): self.fail(_("Incomplete login details"), user=user) + _raw_user_name = user user = User.find_by_credentials(user, pwd) if not user: - self.fail("Invalid login credentials") + self.fail("Invalid login credentials", user=_raw_user_name) # Current login flow uses cached credentials for authentication while checking OTP. # Incase of OTP check, tracker for auth needs to be disabled(If not, it can remove tracker history as it is going to succeed anyway) @@ -316,7 +307,7 @@ class LoginManager: current_hour = int(now_datetime().strftime("%H")) - if login_before and current_hour > login_before: + if login_before and current_hour >= login_before: frappe.throw(_("Login not allowed at this time"), frappe.AuthenticationError) if login_after and current_hour < login_after: @@ -365,10 +356,6 @@ class CookieManager: if not secure and hasattr(frappe.local, "request"): secure = frappe.local.request.scheme == "https" - # Cordova does not work with Lax - if frappe.local.session.data.device == "mobile": - samesite = None - self.cookies[key] = { "value": value, "expires": expires, diff --git a/frappe/automation/doctype/assignment_rule/assignment_rule.js b/frappe/automation/doctype/assignment_rule/assignment_rule.js index 97bed4f8f3..3e029e8444 100644 --- a/frappe/automation/doctype/assignment_rule/assignment_rule.js +++ b/frappe/automation/doctype/assignment_rule/assignment_rule.js @@ -1,82 +1,77 @@ // Copyright (c) 2019, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Assignment Rule', { - refresh: function(frm) { - frm.trigger('setup_assignment_days_buttons'); - frm.trigger('set_options'); +frappe.ui.form.on("Assignment Rule", { + refresh: function (frm) { + frm.trigger("setup_assignment_days_buttons"); + frm.trigger("set_options"); // refresh description frm.events.rule(frm); }, - setup: function(frm) { + setup: function (frm) { frm.set_query("document_type", () => { return { filters: { - name: ["!=", "ToDo"] - } + name: ["!=", "ToDo"], + }, }; }); }, - document_type: function(frm) { - frm.trigger('set_options'); + document_type: function (frm) { + frm.trigger("set_options"); }, - setup_assignment_days_buttons: function(frm) { - const labels = ['Weekends', 'Weekdays', 'All Days']; + setup_assignment_days_buttons: function (frm) { + const labels = ["Weekends", "Weekdays", "All Days"]; let get_days = (label) => { - const weekdays = ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday']; - const weekends = ['Saturday', 'Sunday']; + const weekdays = ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday"]; + const weekends = ["Saturday", "Sunday"]; return { - 'All Days': weekdays.concat(weekends), - 'Weekdays': weekdays, - 'Weekends': weekends, + "All Days": weekdays.concat(weekends), + Weekdays: weekdays, + Weekends: weekends, }[label]; }; let set_days = (e) => { - frm.clear_table('assignment_days'); + frm.clear_table("assignment_days"); const label = $(e.currentTarget).text(); - get_days(label).forEach((day) => - frm.add_child('assignment_days', { day: day }) - ); - frm.refresh_field('assignment_days'); + get_days(label).forEach((day) => frm.add_child("assignment_days", { day: day })); + frm.refresh_field("assignment_days"); }; - labels.forEach(label => - frm.fields_dict['assignment_days'].grid.add_custom_button( - label, - set_days, - 'top' - ) + labels.forEach((label) => + frm.fields_dict["assignment_days"].grid.add_custom_button(label, set_days, "top") ); }, - rule: function(frm) { + rule: function (frm) { const description_map = { - 'Round Robin': __('Assign one by one, in sequence'), - 'Load Balancing': __('Assign to the one who has the least assignments'), - 'Based on Field': __('Assign to the user set in this field'), + "Round Robin": __("Assign one by one, in sequence"), + "Load Balancing": __("Assign to the one who has the least assignments"), + "Based on Field": __("Assign to the user set in this field"), }; - frm.get_field('rule').set_description(description_map[frm.doc.rule]); + frm.get_field("rule").set_description(description_map[frm.doc.rule]); }, set_options(frm) { const doctype = frm.doc.document_type; frm.set_fields_as_options( - 'field', + "field", doctype, - (df) => ['Dynamic Link', 'Data'].includes(df.fieldtype) - || (df.fieldtype == 'Link' && df.options == 'User'), - [{ label: 'Owner', value: 'owner' }] + (df) => + ["Dynamic Link", "Data"].includes(df.fieldtype) || + (df.fieldtype == "Link" && df.options == "User"), + [{ label: "Owner", value: "owner" }] ); if (doctype) { - frm.set_fields_as_options( - 'due_date_based_on', - doctype, - (df) => ['Date', 'Datetime'].includes(df.fieldtype) - ).then(options => frm.set_df_property('due_date_based_on', 'hidden', !options.length)); + frm.set_fields_as_options("due_date_based_on", doctype, (df) => + ["Date", "Datetime"].includes(df.fieldtype) + ).then((options) => + frm.set_df_property("due_date_based_on", "hidden", !options.length) + ); } }, }); diff --git a/frappe/automation/doctype/assignment_rule/test_assignment_rule.py b/frappe/automation/doctype/assignment_rule/test_assignment_rule.py index 2ab2e4d263..2460c40e8a 100644 --- a/frappe/automation/doctype/assignment_rule/test_assignment_rule.py +++ b/frappe/automation/doctype/assignment_rule/test_assignment_rule.py @@ -1,23 +1,26 @@ # Copyright (c) 2021, Frappe Technologies and Contributors # License: MIT. See LICENSE -import unittest - import frappe from frappe.test_runner import make_test_records -from frappe.utils import random_string +from frappe.tests.utils import FrappeTestCase + +TEST_DOCTYPE = "Assignment Test" -class TestAutoAssign(unittest.TestCase): +class TestAutoAssign(FrappeTestCase): @classmethod def setUpClass(cls): + super().setUpClass() frappe.db.delete("Assignment Rule") + create_test_doctype(TEST_DOCTYPE) @classmethod def tearDownClass(cls): frappe.db.rollback() def setUp(self): + frappe.set_user("Administrator") make_test_records("User") days = [ dict(day="Sunday"), @@ -33,45 +36,49 @@ class TestAutoAssign(unittest.TestCase): clear_assignments() def test_round_robin(self): - note = make_note(dict(public=1)) - # check if auto assigned to first user + record = _make_test_record(public=1) self.assertEqual( frappe.db.get_value( - "ToDo", dict(reference_type="Note", reference_name=note.name, status="Open"), "allocated_to" + "ToDo", + dict(reference_type=TEST_DOCTYPE, reference_name=record.name, status="Open"), + "allocated_to", ), "test@example.com", ) - note = make_note(dict(public=1)) - # check if auto assigned to second user + record = _make_test_record(public=1) self.assertEqual( frappe.db.get_value( - "ToDo", dict(reference_type="Note", reference_name=note.name, status="Open"), "allocated_to" + "ToDo", + dict(reference_type=TEST_DOCTYPE, reference_name=record.name, status="Open"), + "allocated_to", ), "test1@example.com", ) clear_assignments() - note = make_note(dict(public=1)) - # check if auto assigned to third user, even if # previous assignments where closed + record = _make_test_record(public=1) self.assertEqual( frappe.db.get_value( - "ToDo", dict(reference_type="Note", reference_name=note.name, status="Open"), "allocated_to" + "ToDo", + dict(reference_type=TEST_DOCTYPE, reference_name=record.name, status="Open"), + "allocated_to", ), "test2@example.com", ) # check loop back to first user - note = make_note(dict(public=1)) - + record = _make_test_record(public=1) self.assertEqual( frappe.db.get_value( - "ToDo", dict(reference_type="Note", reference_name=note.name, status="Open"), "allocated_to" + "ToDo", + dict(reference_type=TEST_DOCTYPE, reference_name=record.name, status="Open"), + "allocated_to", ), "test@example.com", ) @@ -81,29 +88,29 @@ class TestAutoAssign(unittest.TestCase): self.assignment_rule.save() for _ in range(30): - note = make_note(dict(public=1)) + _make_test_record(public=1) # check if each user has 10 assignments (?) for user in ("test@example.com", "test1@example.com", "test2@example.com"): self.assertEqual( - len(frappe.get_all("ToDo", dict(allocated_to=user, reference_type="Note"))), 10 + len(frappe.get_all("ToDo", dict(allocated_to=user, reference_type=TEST_DOCTYPE))), 10 ) # clear 5 assignments for first user # can't do a limit in "delete" since postgres does not support it for d in frappe.get_all( - "ToDo", dict(reference_type="Note", allocated_to="test@example.com"), limit=5 + "ToDo", dict(reference_type=TEST_DOCTYPE, allocated_to="test@example.com"), limit=5 ): frappe.db.delete("ToDo", {"name": d.name}) # add 5 more assignments for i in range(5): - make_note(dict(public=1)) + _make_test_record(public=1) # check if each user still has 10 assignments for user in ("test@example.com", "test1@example.com", "test2@example.com"): self.assertEqual( - len(frappe.get_all("ToDo", dict(allocated_to=user, reference_type="Note"))), 10 + len(frappe.get_all("ToDo", dict(allocated_to=user, reference_type=TEST_DOCTYPE))), 10 ) def test_based_on_field(self): @@ -111,45 +118,36 @@ class TestAutoAssign(unittest.TestCase): self.assignment_rule.field = "owner" self.assignment_rule.save() - frappe.set_user("test1@example.com") - note = make_note(dict(public=1)) - # check if auto assigned to doc owner, test1@example.com - self.assertEqual( - frappe.db.get_value( - "ToDo", dict(reference_type="Note", reference_name=note.name, status="Open"), "owner" - ), - "test1@example.com", - ) - - frappe.set_user("test2@example.com") - note = make_note(dict(public=1)) - # check if auto assigned to doc owner, test2@example.com - self.assertEqual( - frappe.db.get_value( - "ToDo", dict(reference_type="Note", reference_name=note.name, status="Open"), "owner" - ), - "test2@example.com", - ) - - frappe.set_user("Administrator") + for test_user in ("test1@example.com", "test2@example.com"): + frappe.set_user(test_user) + note = _make_test_record(public=1) + # check if auto assigned to doc owner, test1@example.com + self.assertEqual( + frappe.db.get_value( + "ToDo", dict(reference_type=TEST_DOCTYPE, reference_name=note.name, status="Open"), "owner" + ), + test_user, + ) def test_assign_condition(self): # check condition - note = make_note(dict(public=0)) + note = _make_test_record(public=0) self.assertEqual( frappe.db.get_value( - "ToDo", dict(reference_type="Note", reference_name=note.name, status="Open"), "allocated_to" + "ToDo", + dict(reference_type=TEST_DOCTYPE, reference_name=note.name, status="Open"), + "allocated_to", ), None, ) def test_clear_assignment(self): - note = make_note(dict(public=1)) + note = _make_test_record(public=1) # check if auto assigned to first user todo = frappe.get_list( - "ToDo", dict(reference_type="Note", reference_name=note.name, status="Open"), limit=1 + "ToDo", dict(reference_type=TEST_DOCTYPE, reference_name=note.name, status="Open"), limit=1 )[0] todo = frappe.get_doc("ToDo", todo["name"]) @@ -165,11 +163,11 @@ class TestAutoAssign(unittest.TestCase): self.assertEqual(todo.status, "Cancelled") def test_close_assignment(self): - note = make_note(dict(public=1, content="valid")) + note = _make_test_record(public=1, content="valid") # check if auto assigned todo = frappe.get_list( - "ToDo", dict(reference_type="Note", reference_name=note.name, status="Open"), limit=1 + "ToDo", dict(reference_type=TEST_DOCTYPE, reference_name=note.name, status="Open"), limit=1 )[0] todo = frappe.get_doc("ToDo", todo["name"]) @@ -186,12 +184,14 @@ class TestAutoAssign(unittest.TestCase): self.assertEqual(todo.allocated_to, "test@example.com") def check_multiple_rules(self): - note = make_note(dict(public=1, notify_on_login=1)) + note = _make_test_record(public=1, notify_on_login=1) # check if auto assigned to test3 (2nd rule is applied, as it has higher priority) self.assertEqual( frappe.db.get_value( - "ToDo", dict(reference_type="Note", reference_name=note.name, status="Open"), "allocated_to" + "ToDo", + dict(reference_type=TEST_DOCTYPE, reference_name=note.name, status="Open"), + "allocated_to", ), "test@example.com", ) @@ -206,21 +206,25 @@ class TestAutoAssign(unittest.TestCase): get_assignment_rule([days_1, days_2], ["public == 1", "public == 1"]) frappe.flags.assignment_day = "Monday" - note = make_note(dict(public=1)) + note = _make_test_record(public=1) self.assertIn( frappe.db.get_value( - "ToDo", dict(reference_type="Note", reference_name=note.name, status="Open"), "allocated_to" + "ToDo", + dict(reference_type=TEST_DOCTYPE, reference_name=note.name, status="Open"), + "allocated_to", ), ["test@example.com", "test1@example.com", "test2@example.com"], ) frappe.flags.assignment_day = "Friday" - note = make_note(dict(public=1)) + note = _make_test_record(public=1) self.assertIn( frappe.db.get_value( - "ToDo", dict(reference_type="Note", reference_name=note.name, status="Open"), "allocated_to" + "ToDo", + dict(reference_type=TEST_DOCTYPE, reference_name=note.name, status="Open"), + "allocated_to", ), ["test3@example.com"], ) @@ -228,17 +232,11 @@ class TestAutoAssign(unittest.TestCase): def test_assignment_rule_condition(self): frappe.db.delete("Assignment Rule") - # Add expiry_date custom field - from frappe.custom.doctype.custom_field.custom_field import create_custom_field - - df = dict(fieldname="expiry_date", label="Expiry Date", fieldtype="Date") - create_custom_field("Note", df) - assignment_rule = frappe.get_doc( dict( name="Assignment with Due Date", doctype="Assignment Rule", - document_type="Note", + document_type=TEST_DOCTYPE, assign_condition="public == 0", due_date_based_on="expiry_date", assignment_days=self.days, @@ -249,11 +247,11 @@ class TestAutoAssign(unittest.TestCase): ).insert() expiry_date = frappe.utils.add_days(frappe.utils.nowdate(), 2) - note1 = make_note({"expiry_date": expiry_date}) - note2 = make_note({"expiry_date": expiry_date}) + note1 = _make_test_record(expiry_date=expiry_date) + note2 = _make_test_record(expiry_date=expiry_date) note1_todo = frappe.get_all( - "ToDo", filters=dict(reference_type="Note", reference_name=note1.name, status="Open") + "ToDo", filters=dict(reference_type=TEST_DOCTYPE, reference_name=note1.name, status="Open") )[0] note1_todo_doc = frappe.get_doc("ToDo", note1_todo.name) @@ -268,30 +266,31 @@ class TestAutoAssign(unittest.TestCase): # saving one note's expiry should not update other note todo's due date note2_todo = frappe.get_all( "ToDo", - filters=dict(reference_type="Note", reference_name=note2.name, status="Open"), + filters=dict(reference_type=TEST_DOCTYPE, reference_name=note2.name, status="Open"), fields=["name", "date"], )[0] self.assertNotEqual(frappe.utils.get_date_str(note2_todo.date), note1.expiry_date) self.assertEqual(frappe.utils.get_date_str(note2_todo.date), expiry_date) assignment_rule.delete() + frappe.db.commit() # undo changes commited by DDL def clear_assignments(): - frappe.db.delete("ToDo", {"reference_type": "Note"}) + frappe.db.delete("ToDo", {"reference_type": TEST_DOCTYPE}) def get_assignment_rule(days, assign=None): - frappe.delete_doc_if_exists("Assignment Rule", "For Note 1") + frappe.delete_doc_if_exists("Assignment Rule", f"For {TEST_DOCTYPE} 1") if not assign: assign = ["public == 1", "notify_on_login == 1"] assignment_rule = frappe.get_doc( dict( - name="For Note 1", + name=f"For {TEST_DOCTYPE} 1", doctype="Assignment Rule", priority=0, - document_type="Note", + document_type=TEST_DOCTYPE, assign_condition=assign[0], unassign_condition="public == 0 or notify_on_login == 1", close_condition='"Closed" in content', @@ -305,15 +304,15 @@ def get_assignment_rule(days, assign=None): ) ).insert() - frappe.delete_doc_if_exists("Assignment Rule", "For Note 2") + frappe.delete_doc_if_exists("Assignment Rule", f"For {TEST_DOCTYPE} 2") # 2nd rule frappe.get_doc( dict( - name="For Note 2", + name=f"For {TEST_DOCTYPE} 2", doctype="Assignment Rule", priority=1, - document_type="Note", + document_type=TEST_DOCTYPE, assign_condition=assign[1], unassign_condition="notify_on_login == 0", rule="Round Robin", @@ -325,12 +324,60 @@ def get_assignment_rule(days, assign=None): return assignment_rule -def make_note(values=None): - note = frappe.get_doc(dict(doctype="Note", title=random_string(10), content=random_string(20))) +def _make_test_record(**kwargs): + doc = frappe.new_doc(TEST_DOCTYPE) - if values: - note.update(values) + if kwargs: + doc.update(kwargs) - note.insert() + return doc.insert() - return note + +def create_test_doctype(doctype: str): + """Create custom doctype.""" + frappe.db.delete("DocType", doctype) + + frappe.get_doc( + { + "doctype": "DocType", + "name": doctype, + "module": "Custom", + "custom": 1, + "fields": [ + { + "fieldname": "expiry_date", + "label": "Expiry Date", + "fieldtype": "Date", + }, + { + "fieldname": "notify_on_login", + "label": "Notify on Login", + "fieldtype": "Check", + }, + { + "fieldname": "public", + "label": "Public", + "fieldtype": "Check", + }, + { + "fieldname": "content", + "label": "Content", + "fieldtype": "Text", + }, + ], + "permissions": [ + { + "create": 1, + "delete": 1, + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "All", + "share": 1, + "write": 1, + }, + ], + } + ).insert() diff --git a/frappe/automation/doctype/auto_repeat/auto_repeat.js b/frappe/automation/doctype/auto_repeat/auto_repeat.js index 80f2255f47..c0fa2696be 100644 --- a/frappe/automation/doctype/auto_repeat/auto_repeat.js +++ b/frappe/automation/doctype/auto_repeat/auto_repeat.js @@ -2,41 +2,45 @@ // For license information, please see license.txt frappe.provide("frappe.auto_repeat"); -frappe.ui.form.on('Auto Repeat', { - setup: function(frm) { - frm.fields_dict['reference_doctype'].get_query = function() { +frappe.ui.form.on("Auto Repeat", { + setup: function (frm) { + frm.fields_dict["reference_doctype"].get_query = function () { return { - query: "frappe.automation.doctype.auto_repeat.auto_repeat.get_auto_repeat_doctypes" + query: "frappe.automation.doctype.auto_repeat.auto_repeat.get_auto_repeat_doctypes", }; }; - frm.fields_dict['reference_document'].get_query = function() { + frm.fields_dict["reference_document"].get_query = function () { return { filters: { - "auto_repeat": '' - } + auto_repeat: "", + }, }; }; - frm.fields_dict['print_format'].get_query = function() { + frm.fields_dict["print_format"].get_query = function () { return { filters: { - "doc_type": frm.doc.reference_doctype - } + doc_type: frm.doc.reference_doctype, + }, }; }; }, - refresh: function(frm) { + refresh: function (frm) { // auto repeat message if (frm.is_new()) { - let customize_form_link = `${__('Customize Form')}`; - frm.dashboard.set_headline(__('To configure Auto Repeat, enable "Allow Auto Repeat" from {0}.', [customize_form_link])); + let customize_form_link = `${__("Customize Form")}`; + frm.dashboard.set_headline( + __('To configure Auto Repeat, enable "Allow Auto Repeat" from {0}.', [ + customize_form_link, + ]) + ); } // view document button if (!frm.is_dirty()) { - let label = __('View {0}', [__(frm.doc.reference_doctype)]); + let label = __("View {0}", [__(frm.doc.reference_doctype)]); frm.add_custom_button(label, () => frappe.set_route("List", frm.doc.reference_doctype, { auto_repeat: frm.doc.name }) ); @@ -45,24 +49,24 @@ frappe.ui.form.on('Auto Repeat', { // auto repeat schedule frappe.auto_repeat.render_schedule(frm); - frm.trigger('toggle_submit_on_creation'); + frm.trigger("toggle_submit_on_creation"); }, - reference_doctype: function(frm) { - frm.trigger('toggle_submit_on_creation'); + reference_doctype: function (frm) { + frm.trigger("toggle_submit_on_creation"); }, - toggle_submit_on_creation: function(frm) { + toggle_submit_on_creation: function (frm) { // submit on creation checkbox if (frm.doc.reference_doctype) { frappe.model.with_doctype(frm.doc.reference_doctype, () => { let meta = frappe.get_meta(frm.doc.reference_doctype); - frm.toggle_display('submit_on_creation', meta.is_submittable); + frm.toggle_display("submit_on_creation", meta.is_submittable); }); } }, - template: function(frm) { + template: function (frm) { if (frm.doc.template) { frappe.model.with_doc("Email Template", frm.doc.template, () => { let email_template = frappe.get_doc("Email Template", frm.doc.template); @@ -74,11 +78,11 @@ frappe.ui.form.on('Auto Repeat', { } }, - get_contacts: function(frm) { - frm.call('fetch_linked_contacts'); + get_contacts: function (frm) { + frm.call("fetch_linked_contacts"); }, - preview_message: function(frm) { + preview_message: function (frm) { if (frm.doc.message) { frappe.call({ method: "frappe.automation.doctype.auto_repeat.auto_repeat.generate_message_preview", @@ -86,29 +90,29 @@ frappe.ui.form.on('Auto Repeat', { reference_dt: frm.doc.reference_doctype, reference_doc: frm.doc.reference_document, subject: frm.doc.subject, - message: frm.doc.message + message: frm.doc.message, }, - callback: function(r) { + callback: function (r) { if (r.message) { - frappe.msgprint(r.message.message, r.message.subject) + frappe.msgprint(r.message.message, r.message.subject); } - } + }, }); } else { - frappe.msgprint(__("Please setup a message first"), __("Message not setup")) + frappe.msgprint(__("Please setup a message first"), __("Message not setup")); } - } + }, }); -frappe.auto_repeat.render_schedule = function(frm) { - if (!frm.is_dirty() && frm.doc.status !== 'Disabled') { - frm.call("get_auto_repeat_schedule").then(r => { +frappe.auto_repeat.render_schedule = function (frm) { + if (!frm.is_dirty() && frm.doc.status !== "Disabled") { + frm.call("get_auto_repeat_schedule").then((r) => { frm.dashboard.reset(); frm.dashboard.add_section( frappe.render_template("auto_repeat_schedule", { - schedule_details: r.message || [] + schedule_details: r.message || [], }), - __('Auto Repeat Schedule') + __("Auto Repeat Schedule") ); frm.dashboard.show(); }); diff --git a/frappe/automation/doctype/auto_repeat/auto_repeat.py b/frappe/automation/doctype/auto_repeat/auto_repeat.py index 0442be0976..1bb3f7c51f 100644 --- a/frappe/automation/doctype/auto_repeat/auto_repeat.py +++ b/frappe/automation/doctype/auto_repeat/auto_repeat.py @@ -12,8 +12,6 @@ from frappe.contacts.doctype.contact.contact import ( get_contacts_linked_from, get_contacts_linking_to, ) -from frappe.core.doctype.communication.email import make -from frappe.desk.form import assign_to from frappe.model.document import Document from frappe.utils import ( add_days, @@ -241,7 +239,7 @@ class AutoRepeat(Document): def set_auto_repeat_period(self, new_doc): mcount = month_map.get(self.frequency) if mcount and new_doc.meta.get_field("from_date") and new_doc.meta.get_field("to_date"): - last_ref_doc = frappe.db.get_all( + last_ref_doc = frappe.get_all( doctype=self.reference_doctype, fields=["name", "from_date", "to_date"], filters=[ @@ -365,7 +363,7 @@ class AutoRepeat(Document): error_string += _( "{0}: Failed to attach new recurring document. To enable attaching document in the auto repeat notification email, enable {1} in Print Settings" ).format(frappe.bold(_("Note")), frappe.bold(_("Allow Print for Draft"))) - attachments = "[]" + attachments = None if error_string: message = error_string @@ -374,16 +372,14 @@ class AutoRepeat(Document): elif "{" in self.message: message = frappe.render_template(self.message, {"doc": new_doc}) - recipients = self.recipients.split("\n") - - make( - doctype=new_doc.doctype, - name=new_doc.name, - recipients=recipients, + frappe.sendmail( + reference_doctype=new_doc.doctype, + reference_name=new_doc.name, + recipients=self.recipients, subject=subject, content=message, attachments=attachments, - send_email=1, + expose_recipients="header", ) @frappe.whitelist() @@ -470,7 +466,7 @@ def create_repeated_entries(data): def get_auto_repeat_entries(date=None): if not date: date = getdate(today()) - return frappe.db.get_all( + return frappe.get_all( "Auto Repeat", filters=[["next_schedule_date", "<=", date], ["status", "=", "Active"]] ) @@ -504,7 +500,7 @@ def make_auto_repeat(doctype, docname, frequency="Daily", start_date=None, end_d @frappe.whitelist() @frappe.validate_and_sanitize_search_inputs def get_auto_repeat_doctypes(doctype, txt, searchfield, start, page_len, filters): - res = frappe.db.get_all( + res = frappe.get_all( "Property Setter", { "property": "allow_auto_repeat", @@ -514,7 +510,7 @@ def get_auto_repeat_doctypes(doctype, txt, searchfield, start, page_len, filters ) docs = [r.doc_type for r in res] - res = frappe.db.get_all( + res = frappe.get_all( "DocType", { "allow_auto_repeat": 1, diff --git a/frappe/automation/doctype/auto_repeat/auto_repeat_list.js b/frappe/automation/doctype/auto_repeat/auto_repeat_list.js index f906580f7e..f970341fa3 100644 --- a/frappe/automation/doctype/auto_repeat/auto_repeat_list.js +++ b/frappe/automation/doctype/auto_repeat/auto_repeat_list.js @@ -1,11 +1,11 @@ -frappe.listview_settings['Auto Repeat'] = { +frappe.listview_settings["Auto Repeat"] = { add_fields: ["next_schedule_date"], - get_indicator: function(doc) { + get_indicator: function (doc) { var colors = { - "Active": "green", - "Disabled": "red", - "Completed": "blue", + Active: "green", + Disabled: "red", + Completed: "blue", }; return [__(doc.status), colors[doc.status], "status,=," + doc.status]; - } + }, }; diff --git a/frappe/automation/doctype/auto_repeat/test_auto_repeat.py b/frappe/automation/doctype/auto_repeat/test_auto_repeat.py index ee0addf847..969c68fbb8 100644 --- a/frappe/automation/doctype/auto_repeat/test_auto_repeat.py +++ b/frappe/automation/doctype/auto_repeat/test_auto_repeat.py @@ -1,6 +1,6 @@ # Copyright (c) 2018, Frappe Technologies and Contributors # License: MIT. See LICENSE -import unittest +from typing import TYPE_CHECKING import frappe from frappe.automation.doctype.auto_repeat.auto_repeat import ( @@ -9,10 +9,14 @@ from frappe.automation.doctype.auto_repeat.auto_repeat import ( week_map, ) from frappe.custom.doctype.custom_field.custom_field import create_custom_field +from frappe.tests.utils import FrappeTestCase from frappe.utils import add_days, add_months, getdate, today +if TYPE_CHECKING: + from frappe.custom.doctype.custom_field.custom_field import CustomField -def add_custom_fields(): + +def add_custom_fields() -> "CustomField": df = dict( fieldname="auto_repeat", label="Auto Repeat", @@ -23,15 +27,17 @@ def add_custom_fields(): print_hide=1, read_only=1, ) - create_custom_field("ToDo", df) + return create_custom_field("ToDo", df) or frappe.get_doc( + "Custom Field", dict(fieldname=df["fieldname"], dt="ToDo") + ) -class TestAutoRepeat(unittest.TestCase): - def setUp(self): - if not frappe.db.sql( - "SELECT `fieldname` FROM `tabCustom Field` WHERE `fieldname`='auto_repeat' and `dt`=%s", "Todo" - ): - add_custom_fields() +class TestAutoRepeat(FrappeTestCase): + @classmethod + def setUpClass(cls): + cls.custom_field = add_custom_fields() + cls.addClassCleanup(cls.custom_field.delete) + return super().setUpClass() def test_daily_auto_repeat(self): todo = frappe.get_doc( @@ -157,7 +163,7 @@ class TestAutoRepeat(unittest.TestCase): docnames = frappe.get_all(doc.reference_doctype, {"auto_repeat": doc.name}) self.assertEqual(len(docnames), months) - def test_notification_is_attached(self): + def test_email_notification(self): todo = frappe.get_doc( dict( doctype="ToDo", @@ -181,10 +187,10 @@ class TestAutoRepeat(unittest.TestCase): "ToDo", {"auto_repeat": doc.name, "name": ("!=", todo.name)}, "name" ) - linked_comm = frappe.db.exists( - "Communication", dict(reference_doctype="ToDo", reference_name=new_todo) + email_queue = frappe.db.exists( + "Email Queue", dict(reference_doctype="ToDo", reference_name=new_todo) ) - self.assertTrue(linked_comm) + self.assertTrue(email_queue) def test_next_schedule_date(self): current_date = getdate(today()) @@ -228,7 +234,7 @@ class TestAutoRepeat(unittest.TestCase): data = get_auto_repeat_entries(current_date) create_repeated_entries(data) - docnames = frappe.db.get_all( + docnames = frappe.get_all( doc.reference_doctype, filters={"auto_repeat": doc.name}, fields=["docstatus"], limit=1 ) self.assertEqual(docnames[0].docstatus, 1) diff --git a/frappe/automation/doctype/milestone/milestone.js b/frappe/automation/doctype/milestone/milestone.js index 9a1cf577ff..2a5ab04135 100644 --- a/frappe/automation/doctype/milestone/milestone.js +++ b/frappe/automation/doctype/milestone/milestone.js @@ -1,8 +1,7 @@ // Copyright (c) 2019, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Milestone', { +frappe.ui.form.on("Milestone", { // refresh: function(frm) { - // } }); diff --git a/frappe/automation/doctype/milestone/milestone.json b/frappe/automation/doctype/milestone/milestone.json index 8360ce7bf4..aa2dd35891 100644 --- a/frappe/automation/doctype/milestone/milestone.json +++ b/frappe/automation/doctype/milestone/milestone.json @@ -1,230 +1,81 @@ { - "allow_copy": 0, - "allow_events_in_timeline": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "autoname": "", - "beta": 0, + "actions": [], "creation": "2019-04-17 09:39:15.647817", - "custom": 0, - "docstatus": 0, "doctype": "DocType", - "document_type": "", - "editable_grid": 0, "engine": "InnoDB", + "field_order": [ + "reference_type", + "reference_name", + "track_field", + "value", + "milestone_tracker" + ], "fields": [ { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fetch_if_empty": 0, "fieldname": "reference_type", "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, "in_list_view": 1, - "in_standard_filter": 0, "label": "Document Type", - "length": 0, - "no_copy": 0, "options": "DocType", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, "reqd": 1, - "search_index": 1, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "search_index": 1 }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fetch_if_empty": 0, "fieldname": "reference_name", "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, "in_list_view": 1, - "in_standard_filter": 0, "label": "Document", - "length": 0, - "no_copy": 0, - "options": "", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "reqd": 1 }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fetch_if_empty": 0, "fieldname": "track_field", "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, "label": "Track Field", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "reqd": 1 }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fetch_if_empty": 0, "fieldname": "value", "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, "in_list_view": 1, - "in_standard_filter": 0, "label": "Value", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "reqd": 1 }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fetch_if_empty": 0, "fieldname": "milestone_tracker", "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, "label": "Milestone Tracker", - "length": 0, - "no_copy": 0, - "options": "Milestone Tracker", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "options": "Milestone Tracker" } ], - "has_web_view": 0, - "hide_toolbar": 0, - "idx": 0, "in_create": 1, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2019-04-17 16:01:21.430344", + "links": [], + "modified": "2022-08-03 12:20:55.076769", "modified_by": "Administrator", "module": "Automation", "name": "Milestone", - "name_case": "", "owner": "Administrator", "permissions": [ { - "amend": 0, - "cancel": 0, "create": 1, "delete": 1, "email": 1, "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, "print": 1, "read": 1, "report": 1, "role": "System Manager", - "set_user_permissions": 0, "share": 1, - "submit": 0, "write": 1 } ], "quick_entry": 1, - "read_only": 0, - "show_name_in_global_search": 0, "sort_field": "modified", "sort_order": "ASC", + "states": [], "title_field": "reference_type", - "track_changes": 1, - "track_seen": 0, - "track_views": 0 + "track_changes": 1 } \ No newline at end of file diff --git a/frappe/automation/doctype/milestone/test_milestone.py b/frappe/automation/doctype/milestone/test_milestone.py index 5ac0754e5a..5348479809 100644 --- a/frappe/automation/doctype/milestone/test_milestone.py +++ b/frappe/automation/doctype/milestone/test_milestone.py @@ -1,8 +1,8 @@ # Copyright (c) 2019, Frappe Technologies and Contributors # License: MIT. See LICENSE # import frappe -import unittest +from frappe.tests.utils import FrappeTestCase -class TestMilestone(unittest.TestCase): +class TestMilestone(FrappeTestCase): pass diff --git a/frappe/automation/doctype/milestone_tracker/milestone_tracker.js b/frappe/automation/doctype/milestone_tracker/milestone_tracker.js index 2a74bfb070..bf5be880a0 100644 --- a/frappe/automation/doctype/milestone_tracker/milestone_tracker.js +++ b/frappe/automation/doctype/milestone_tracker/milestone_tracker.js @@ -1,14 +1,14 @@ // Copyright (c) 2019, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Milestone Tracker', { - refresh: function(frm) { - frm.trigger('update_options'); +frappe.ui.form.on("Milestone Tracker", { + refresh: function (frm) { + frm.trigger("update_options"); }, - document_type: function(frm) { - frm.trigger('update_options'); + document_type: function (frm) { + frm.trigger("update_options"); }, - update_options: function(frm) { + update_options: function (frm) { // update select options for `track_field` let doctype = frm.doc.document_type; let track_fields = []; @@ -16,18 +16,16 @@ frappe.ui.form.on('Milestone Tracker', { if (doctype) { frappe.model.with_doctype(doctype, () => { // get all date and datetime fields - frappe.get_meta(doctype).fields.map(df => { - if (['Link', 'Select'].includes(df.fieldtype)) { - track_fields.push({label: df.label, value: df.fieldname}); + frappe.get_meta(doctype).fields.map((df) => { + if (["Link", "Select"].includes(df.fieldtype)) { + track_fields.push({ label: df.label, value: df.fieldname }); } }); - frm.set_df_property('track_field', 'options', track_fields); + frm.set_df_property("track_field", "options", track_fields); }); } else { // update select options - frm.set_df_property('track_field', 'options', []); + frm.set_df_property("track_field", "options", []); } - }, - }); diff --git a/frappe/automation/doctype/milestone_tracker/milestone_tracker.json b/frappe/automation/doctype/milestone_tracker/milestone_tracker.json index 8e22e3e199..8d4ed94dcd 100644 --- a/frappe/automation/doctype/milestone_tracker/milestone_tracker.json +++ b/frappe/automation/doctype/milestone_tracker/milestone_tracker.json @@ -1,162 +1,61 @@ { - "allow_copy": 0, - "allow_events_in_timeline": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, + "actions": [], "autoname": "format:{document_type}-{track_field}", - "beta": 0, "creation": "2019-04-17 09:36:41.774774", - "custom": 0, "description": "Track milestones for any document", - "docstatus": 0, "doctype": "DocType", - "document_type": "", - "editable_grid": 0, "engine": "InnoDB", + "field_order": [ + "document_type", + "track_field", + "disabled" + ], "fields": [ { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fetch_if_empty": 0, "fieldname": "document_type", "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, "in_list_view": 1, - "in_standard_filter": 0, "label": "Document Type to Track", - "length": 0, - "no_copy": 0, "options": "DocType", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, "unique": 1 }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fetch_if_empty": 0, "fieldname": "track_field", "fieldtype": "Select", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, "in_list_view": 1, - "in_standard_filter": 0, "label": "Field to Track", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "reqd": 1 }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fetch_if_empty": 0, + "default": "0", "fieldname": "disabled", "fieldtype": "Check", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Disabled", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "label": "Disabled" } ], - "has_web_view": 0, - "hide_toolbar": 0, - "idx": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2019-04-22 16:03:32.848937", + "links": [], + "modified": "2022-08-03 12:20:54.955953", "modified_by": "Administrator", "module": "Automation", "name": "Milestone Tracker", - "name_case": "", "owner": "Administrator", "permissions": [ { - "amend": 0, - "cancel": 0, "create": 1, "delete": 1, "email": 1, "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, "print": 1, "read": 1, "report": 1, "role": "System Manager", - "set_user_permissions": 0, "share": 1, - "submit": 0, "write": 1 } ], - "quick_entry": 0, - "read_only": 0, - "show_name_in_global_search": 0, "sort_field": "modified", "sort_order": "ASC", - "track_changes": 1, - "track_seen": 0, - "track_views": 0 + "states": [], + "track_changes": 1 } \ No newline at end of file diff --git a/frappe/automation/doctype/milestone_tracker/test_milestone_tracker.py b/frappe/automation/doctype/milestone_tracker/test_milestone_tracker.py index 2b48a76805..3242145bc4 100644 --- a/frappe/automation/doctype/milestone_tracker/test_milestone_tracker.py +++ b/frappe/automation/doctype/milestone_tracker/test_milestone_tracker.py @@ -1,12 +1,11 @@ # Copyright (c) 2019, Frappe Technologies and Contributors # License: MIT. See LICENSE -import unittest - import frappe import frappe.cache_manager +from frappe.tests.utils import FrappeTestCase -class TestMilestoneTracker(unittest.TestCase): +class TestMilestoneTracker(FrappeTestCase): def test_milestone(self): frappe.db.delete("Milestone Tracker") diff --git a/frappe/core/doctype/feedback/__init__.py b/frappe/automation/doctype/reminder/__init__.py similarity index 100% rename from frappe/core/doctype/feedback/__init__.py rename to frappe/automation/doctype/reminder/__init__.py diff --git a/frappe/automation/doctype/reminder/reminder.js b/frappe/automation/doctype/reminder/reminder.js new file mode 100644 index 0000000000..6d1a72bab2 --- /dev/null +++ b/frappe/automation/doctype/reminder/reminder.js @@ -0,0 +1,8 @@ +// Copyright (c) 2023, Frappe Technologies and contributors +// For license information, please see license.txt + +// frappe.ui.form.on("Reminder", { +// refresh(frm) { + +// }, +// }); diff --git a/frappe/automation/doctype/reminder/reminder.json b/frappe/automation/doctype/reminder/reminder.json new file mode 100644 index 0000000000..a288f205a2 --- /dev/null +++ b/frappe/automation/doctype/reminder/reminder.json @@ -0,0 +1,90 @@ +{ + "actions": [], + "autoname": "hash", + "creation": "2023-02-22 11:23:58.183276", + "default_view": "List", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "user", + "remind_at", + "description", + "reminder_doctype", + "reminder_docname", + "notified" + ], + "fields": [ + { + "default": "__user", + "fieldname": "user", + "fieldtype": "Link", + "hidden": 1, + "label": "User", + "options": "User", + "reqd": 1, + "search_index": 1 + }, + { + "fieldname": "reminder_doctype", + "fieldtype": "Link", + "in_list_view": 1, + "label": "Document Type", + "options": "DocType", + "read_only": 1 + }, + { + "fieldname": "reminder_docname", + "fieldtype": "Dynamic Link", + "in_list_view": 1, + "label": "Document Name", + "options": "reminder_doctype", + "read_only": 1 + }, + { + "default": "now", + "fieldname": "remind_at", + "fieldtype": "Datetime", + "in_list_view": 1, + "label": "Remind At", + "reqd": 1, + "search_index": 1 + }, + { + "fieldname": "description", + "fieldtype": "Small Text", + "label": "Description", + "reqd": 1 + }, + { + "default": "0", + "fieldname": "notified", + "fieldtype": "Check", + "hidden": 1, + "label": "notified" + } + ], + "index_web_pages_for_search": 1, + "links": [], + "modified": "2023-02-24 13:47:50.419648", + "modified_by": "Administrator", + "module": "Automation", + "name": "Reminder", + "naming_rule": "Random", + "owner": "Administrator", + "permissions": [ + { + "create": 1, + "delete": 1, + "if_owner": 1, + "read": 1, + "role": "All", + "write": 1 + } + ], + "quick_entry": 1, + "sort_field": "modified", + "sort_order": "DESC", + "states": [], + "title_field": "description" +} \ No newline at end of file diff --git a/frappe/automation/doctype/reminder/reminder.py b/frappe/automation/doctype/reminder/reminder.py new file mode 100644 index 0000000000..795cdfda69 --- /dev/null +++ b/frappe/automation/doctype/reminder/reminder.py @@ -0,0 +1,78 @@ +# Copyright (c) 2023, Frappe Technologies and contributors +# For license information, please see license.txt + +import frappe +from frappe import _ +from frappe.model.document import Document +from frappe.utils import cint +from frappe.utils.data import add_to_date, get_datetime, now_datetime + + +class Reminder(Document): + @staticmethod + def clear_old_logs(days=30): + from frappe.query_builder import Interval + from frappe.query_builder.functions import Now + + table = frappe.qb.DocType("Reminder") + frappe.db.delete(table, filters=(table.remind_at < (Now() - Interval(days=days)))) + + def validate(self): + self.user = frappe.session.user + if get_datetime(self.remind_at) < now_datetime(): + frappe.throw(_("Reminder cannot be created in past.")) + + def send_reminder(self): + if self.notified: + return + + self.db_set("notified", 1, update_modified=False) + + try: + notification = frappe.new_doc("Notification Log") + notification.for_user = self.user + notification.set("type", "Alert") + notification.document_type = self.reminder_doctype + notification.document_name = self.reminder_docname + notification.subject = self.description + notification.insert() + except Exception: + self.log_error("Failed to send reminder") + + +@frappe.whitelist() +def create_new_reminder( + remind_at: str, + description: str, + reminder_doctype: str | None = None, + reminder_docname: str | None = None, +): + reminder = frappe.new_doc("Reminder") + + reminder.description = description + reminder.remind_at = remind_at + reminder.reminder_doctype = reminder_doctype + reminder.reminder_docname = reminder_docname + + return reminder.insert() + + +def send_reminders(): + # Ensure that we send all reminders that might be before next job execution. + job_freq = cint(frappe.get_conf().scheduler_interval) or 240 + upper_threshold = add_to_date(now_datetime(), seconds=job_freq, as_string=True, as_datetime=True) + + lower_threshold = add_to_date(now_datetime(), hours=-8, as_string=True, as_datetime=True) + + pending_reminders = frappe.get_all( + "Reminder", + filters=[ + ("remind_at", "<=", upper_threshold), + ("remind_at", ">=", lower_threshold), # dont send too old reminders if failed to send + ("notified", "=", 0), + ], + pluck="name", + ) + + for reminder in pending_reminders: + frappe.get_doc("Reminder", reminder).send_reminder() diff --git a/frappe/automation/doctype/reminder/test_reminder.py b/frappe/automation/doctype/reminder/test_reminder.py new file mode 100644 index 0000000000..84cc258701 --- /dev/null +++ b/frappe/automation/doctype/reminder/test_reminder.py @@ -0,0 +1,28 @@ +# Copyright (c) 2023, Frappe Technologies and Contributors +# See license.txt + +import frappe +from frappe.automation.doctype.reminder.reminder import create_new_reminder, send_reminders +from frappe.desk.doctype.notification_log.notification_log import get_notification_logs +from frappe.tests.utils import FrappeTestCase +from frappe.utils import add_to_date, now_datetime + + +class TestReminder(FrappeTestCase): + def test_reminder(self): + + description = "TEST_REMINDER" + + create_new_reminder( + remind_at=add_to_date(now_datetime(), minutes=1, as_datetime=True, as_string=True), + description=description, + ) + + send_reminders() + + notifications = get_notification_logs()["notification_logs"] + self.assertIn( + description, + [n.subject for n in notifications], + msg=f"Failed to find reminder notification \n{notifications}", + ) diff --git a/frappe/automation/workspace/tools/tools.json b/frappe/automation/workspace/tools/tools.json index 40b265b34f..d0e2c4fcfd 100644 --- a/frappe/automation/workspace/tools/tools.json +++ b/frappe/automation/workspace/tools/tools.json @@ -1,6 +1,6 @@ { "charts": [], - "content": "[{\"type\":\"header\",\"data\":{\"text\":\"Your Shortcuts\",\"col\":12}},{\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"ToDo\",\"col\":3}},{\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"Note\",\"col\":3}},{\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"File\",\"col\":3}},{\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"Assignment Rule\",\"col\":3}},{\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"Auto Repeat\",\"col\":3}},{\"type\":\"spacer\",\"data\":{\"col\":12}},{\"type\":\"header\",\"data\":{\"text\":\"Reports & Masters\",\"col\":12}},{\"type\":\"card\",\"data\":{\"card_name\":\"Tools\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Email\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Automation\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Event Streaming\",\"col\":4}}]", + "content": "[{\"type\":\"header\",\"data\":{\"text\":\"Your Shortcuts\",\"col\":12}},{\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"ToDo\",\"col\":3}},{\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"Note\",\"col\":3}},{\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"File\",\"col\":3}},{\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"Assignment Rule\",\"col\":3}},{\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"Auto Repeat\",\"col\":3}},{\"type\":\"spacer\",\"data\":{\"col\":12}},{\"type\":\"header\",\"data\":{\"text\":\"Reports & Masters\",\"col\":12}},{\"type\":\"card\",\"data\":{\"card_name\":\"Tools\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Email\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Automation\",\"col\":4}}]", "creation": "2020-03-02 14:53:24.980279", "docstatus": 0, "doctype": "Workspace", @@ -13,11 +13,82 @@ { "hidden": 0, "is_query_report": 0, - "label": "Tools", + "label": "Email", "link_count": 0, "onboard": 0, "type": "Card Break" }, + { + "dependencies": "", + "hidden": 0, + "is_query_report": 0, + "label": "Newsletter", + "link_count": 0, + "link_to": "Newsletter", + "link_type": "DocType", + "onboard": 1, + "type": "Link" + }, + { + "dependencies": "", + "hidden": 0, + "is_query_report": 0, + "label": "Email Group", + "link_count": 0, + "link_to": "Email Group", + "link_type": "DocType", + "onboard": 0, + "type": "Link" + }, + { + "hidden": 0, + "is_query_report": 0, + "label": "Automation", + "link_count": 3, + "onboard": 0, + "type": "Card Break" + }, + { + "dependencies": "", + "hidden": 0, + "is_query_report": 0, + "label": "Assignment Rule", + "link_count": 0, + "link_to": "Assignment Rule", + "link_type": "DocType", + "onboard": 0, + "type": "Link" + }, + { + "dependencies": "", + "hidden": 0, + "is_query_report": 0, + "label": "Milestone", + "link_count": 0, + "link_to": "Milestone", + "link_type": "DocType", + "onboard": 0, + "type": "Link" + }, + { + "dependencies": "", + "hidden": 0, + "is_query_report": 0, + "label": "Auto Repeat", + "link_count": 0, + "link_to": "Auto Repeat", + "link_type": "DocType", + "onboard": 0, + "type": "Link" + }, + { + "hidden": 0, + "is_query_report": 0, + "label": "Tools", + "link_count": 4, + "onboard": 0, + "type": "Card Break" + }, { "dependencies": "", "hidden": 0, @@ -61,160 +132,16 @@ "link_type": "DocType", "onboard": 0, "type": "Link" - }, - { - "dependencies": "", - "hidden": 0, - "is_query_report": 0, - "label": "Activity", - "link_count": 0, - "link_to": "activity", - "link_type": "Page", - "onboard": 0, - "type": "Link" - }, - { - "hidden": 0, - "is_query_report": 0, - "label": "Email", - "link_count": 0, - "onboard": 0, - "type": "Card Break" - }, - { - "dependencies": "", - "hidden": 0, - "is_query_report": 0, - "label": "Newsletter", - "link_count": 0, - "link_to": "Newsletter", - "link_type": "DocType", - "onboard": 1, - "type": "Link" - }, - { - "dependencies": "", - "hidden": 0, - "is_query_report": 0, - "label": "Email Group", - "link_count": 0, - "link_to": "Email Group", - "link_type": "DocType", - "onboard": 0, - "type": "Link" - }, - { - "hidden": 0, - "is_query_report": 0, - "label": "Automation", - "link_count": 0, - "onboard": 0, - "type": "Card Break" - }, - { - "dependencies": "", - "hidden": 0, - "is_query_report": 0, - "label": "Assignment Rule", - "link_count": 0, - "link_to": "Assignment Rule", - "link_type": "DocType", - "onboard": 0, - "type": "Link" - }, - { - "dependencies": "", - "hidden": 0, - "is_query_report": 0, - "label": "Milestone", - "link_count": 0, - "link_to": "Milestone", - "link_type": "DocType", - "onboard": 0, - "type": "Link" - }, - { - "dependencies": "", - "hidden": 0, - "is_query_report": 0, - "label": "Auto Repeat", - "link_count": 0, - "link_to": "Auto Repeat", - "link_type": "DocType", - "onboard": 0, - "type": "Link" - }, - { - "hidden": 0, - "is_query_report": 0, - "label": "Event Streaming", - "link_count": 0, - "onboard": 0, - "type": "Card Break" - }, - { - "dependencies": "", - "hidden": 0, - "is_query_report": 0, - "label": "Event Producer", - "link_count": 0, - "link_to": "Event Producer", - "link_type": "DocType", - "onboard": 0, - "type": "Link" - }, - { - "dependencies": "", - "hidden": 0, - "is_query_report": 0, - "label": "Event Consumer", - "link_count": 0, - "link_to": "Event Consumer", - "link_type": "DocType", - "onboard": 0, - "type": "Link" - }, - { - "dependencies": "", - "hidden": 0, - "is_query_report": 0, - "label": "Event Update Log", - "link_count": 0, - "link_to": "Event Update Log", - "link_type": "DocType", - "onboard": 0, - "type": "Link" - }, - { - "dependencies": "", - "hidden": 0, - "is_query_report": 0, - "label": "Event Sync Log", - "link_count": 0, - "link_to": "Event Sync Log", - "link_type": "DocType", - "onboard": 0, - "type": "Link" - }, - { - "dependencies": "", - "hidden": 0, - "is_query_report": 0, - "label": "Document Type Mapping", - "link_count": 0, - "link_to": "Document Type Mapping", - "link_type": "DocType", - "onboard": 0, - "type": "Link" } ], - "modified": "2022-01-13 17:48:48.456763", + "modified": "2022-12-12 14:58:44.733393", "modified_by": "Administrator", "module": "Automation", "name": "Tools", "owner": "Administrator", "parent_page": "", "public": 1, + "quick_lists": [], "restrict_to_domain": "", "roles": [], "sequence_id": 26.0, diff --git a/frappe/boot.py b/frappe/boot.py index ad729746fe..83c9902020 100644 --- a/frappe/boot.py +++ b/frappe/boot.py @@ -12,15 +12,16 @@ from frappe.desk.doctype.route_history.route_history import frequently_visited_l from frappe.desk.form.load import get_meta_bundle from frappe.email.inbox import get_email_accounts from frappe.model.base_document import get_controller +from frappe.permissions import has_permission from frappe.query_builder import DocType from frappe.query_builder.functions import Count -from frappe.query_builder.terms import SubQuery +from frappe.query_builder.terms import ParameterizedValueWrapper, SubQuery from frappe.social.doctype.energy_point_log.energy_point_log import get_energy_points from frappe.social.doctype.energy_point_settings.energy_point_settings import ( is_energy_point_enabled, ) -from frappe.translate import get_lang_dict -from frappe.utils import add_user_info, cstr, get_time_zone +from frappe.translate import get_lang_dict, get_messages_for_boot, get_translated_doctypes +from frappe.utils import add_user_info, cstr, get_system_timezone from frappe.utils.change_log import get_versions from frappe.website.doctype.web_page_view.web_page_view import is_tracking_enabled @@ -100,7 +101,8 @@ def get_bootinfo(): bootinfo.desk_settings = get_desk_settings() bootinfo.app_logo_url = get_app_logo() bootinfo.link_title_doctypes = get_link_title_doctypes() - bootinfo.translatable_doctypes = get_translatable_doctypes() + bootinfo.translated_doctypes = get_translated_doctypes() + bootinfo.subscription_conf = add_subscription_conf() return bootinfo @@ -128,7 +130,7 @@ def load_desktop_data(bootinfo): from frappe.desk.desktop import get_workspace_sidebar_items bootinfo.allowed_workspaces = get_workspace_sidebar_items().get("pages") - bootinfo.module_page_map = get_controller("Workspace").get_module_page_map() + bootinfo.module_wise_workspaces = get_controller("Workspace").get_module_wise_workspaces() bootinfo.dashboards = frappe.get_all("Dashboard") @@ -233,7 +235,10 @@ def get_user_pages_or_reports(parent, cache=False): has_role[p.name] = {"modified": p.modified, "title": p.title} elif parent == "Report": - reports = frappe.get_all( + if not has_permission("Report", raise_exception=False): + return {} + + reports = frappe.get_list( "Report", fields=["name", "report_type"], filters={"name": ("in", has_role.keys())}, @@ -242,24 +247,18 @@ def get_user_pages_or_reports(parent, cache=False): for report in reports: has_role[report.name]["report_type"] = report.report_type + non_permitted_reports = set(has_role.keys()) - {r.name for r in reports} + for r in non_permitted_reports: + has_role.pop(r, None) + # Expire every six hours _cache.set_value("has_role:" + parent, has_role, frappe.session.user, 21600) return has_role def load_translations(bootinfo): - messages = frappe.get_lang_dict("boot") - bootinfo["lang"] = frappe.lang - - # load translated report names - for name in bootinfo.user.all_reports: - messages[name] = frappe._(name) - - # only untranslated - messages = {k: v for k, v in messages.items() if k != v} - - bootinfo["__messages"] = messages + bootinfo["__messages"] = get_messages_for_boot() def get_user_info(): @@ -328,11 +327,11 @@ def get_unseen_notes(): frappe.qb.from_(note) .select(note.name, note.title, note.content, note.notify_on_every_login) .where( - (note.notify_on_every_login == 1) + (note.notify_on_login == 1) & (note.expire_notification_on > frappe.utils.now()) & ( - SubQuery(frappe.qb.from_(nsb).select(nsb.user).where(nsb.parent == note.name)).notin( - [frappe.session.user] + ParameterizedValueWrapper(frappe.session.user).notin( + SubQuery(frappe.qb.from_(nsb).select(nsb.user).where(nsb.parent == note.name)) ) ) ) @@ -346,7 +345,7 @@ def get_success_action(): def get_link_preview_doctypes(): from frappe.utils import cint - link_preview_doctypes = [d.name for d in frappe.db.get_all("DocType", {"show_preview_popup": 1})] + link_preview_doctypes = [d.name for d in frappe.get_all("DocType", {"show_preview_popup": 1})] customizations = frappe.get_all( "Property Setter", fields=["doc_type", "value"], filters={"property": "show_preview_popup"} ) @@ -391,7 +390,6 @@ def get_notification_settings(): return frappe.get_cached_doc("Notification Settings", frappe.session.user) -@frappe.whitelist() def get_link_title_doctypes(): dts = frappe.get_all("DocType", {"show_title_field_in_link": 1}) custom_dts = frappe.get_all( @@ -404,20 +402,12 @@ def get_link_title_doctypes(): def set_time_zone(bootinfo): bootinfo.time_zone = { - "system": get_time_zone(), + "system": get_system_timezone(), "user": bootinfo.get("user_info", {}).get(frappe.session.user, {}).get("time_zone", None) - or get_time_zone(), + or get_system_timezone(), } -def get_translatable_doctypes(): - dts = frappe.get_all("DocType", {"translate_link_fields": 1}, pluck="name") - custom_dts = frappe.get_all( - "Property Setter", {"property": "translate_link_fields", "value": "1"}, pluck="doc_type" - ) - return dts + custom_dts - - def load_country_doc(bootinfo): country = frappe.db.get_default("country") if not country: @@ -447,3 +437,10 @@ def load_currency_docs(bootinfo): ) bootinfo.docs += currency_docs + + +def add_subscription_conf(): + try: + return frappe.conf.subscription + except Exception: + return "" diff --git a/frappe/build.py b/frappe/build.py index e66da4bd79..b74afa5d06 100644 --- a/frappe/build.py +++ b/frappe/build.py @@ -4,7 +4,6 @@ import os import re import shutil import subprocess -from distutils.spawn import find_executable from subprocess import getoutput from tempfile import mkdtemp, mktemp from urllib.parse import urlparse @@ -280,7 +279,7 @@ def check_node_executable(): warn = "⚠️ " if node_version.major < 14: click.echo(f"{warn} Please update your node version to 14") - if not find_executable("yarn"): + if not shutil.which("yarn"): click.echo(f"{warn} Please install yarn using below command and try again.\nnpm install -g yarn") click.echo() diff --git a/frappe/cache_manager.py b/frappe/cache_manager.py index 01ccc03753..12e829ff09 100644 --- a/frappe/cache_manager.py +++ b/frappe/cache_manager.py @@ -8,12 +8,19 @@ from frappe.desk.notifications import clear_notifications, delete_notification_c common_default_keys = ["__default", "__global"] -doctype_map_keys = ( - "energy_point_rule_map", - "assignment_rule_map", - "milestone_tracker_map", - "event_consumer_document_type_map", -) +doctypes_for_mapping = { + "Energy Point Rule", + "Assignment Rule", + "Milestone Tracker", + "Document Naming Rule", +} + + +def get_doctype_map_key(doctype): + return frappe.scrub(doctype) + "_map" + + +doctype_map_keys = tuple(map(get_doctype_map_key, doctypes_for_mapping)) bench_cache_keys = ("assets_json",) @@ -56,18 +63,19 @@ user_cache_keys = ( "has_role:Page", "has_role:Report", "desk_sidebar_items", + "contacts", ) doctype_cache_keys = ( - "meta", - "form_meta", + "doctype_meta", + "doctype_form_meta", "table_columns", "last_modified", "linked_doctypes", "notifications", "workflow", "data_import_column_header_map", -) + doctype_map_keys +) def clear_user_cache(user=None): @@ -117,14 +125,9 @@ def clear_doctype_cache(doctype=None): clear_controller_cache(doctype) cache = frappe.cache() - if getattr(frappe.local, "meta_cache") and (doctype in frappe.local.meta_cache): - del frappe.local.meta_cache[doctype] - for key in ("is_table", "doctype_modules", "document_cache"): cache.delete_value(key) - frappe.local.document_cache = {} - def clear_single(dt): for name in doctype_cache_keys: cache.hdel(name, dt) @@ -133,12 +136,18 @@ def clear_doctype_cache(doctype=None): clear_single(doctype) # clear all parent doctypes - - for dt in frappe.db.get_all( + for dt in frappe.get_all( "DocField", "parent", dict(fieldtype=["in", frappe.model.table_fields], options=doctype) ): clear_single(dt.parent) + # clear all parent doctypes + if not frappe.flags.in_install: + for dt in frappe.get_all( + "Custom Field", "dt", dict(fieldtype=["in", frappe.model.table_fields], options=doctype) + ): + clear_single(dt.dt) + # clear all notifications delete_notification_count_for(doctype) @@ -150,32 +159,19 @@ def clear_doctype_cache(doctype=None): def clear_controller_cache(doctype=None): if not doctype: - del frappe.controllers - frappe.controllers = {} + frappe.controllers.pop(frappe.local.site, None) return - for site_controllers in frappe.controllers.values(): + if site_controllers := frappe.controllers.get(frappe.local.site): site_controllers.pop(doctype, None) def get_doctype_map(doctype, name, filters=None, order_by=None): - cache = frappe.cache() - cache_key = frappe.scrub(doctype) + "_map" - doctype_map = cache.hget(cache_key, name) - - if doctype_map is not None: - # cached, return - items = json.loads(doctype_map) - else: - # non cached, build cache - try: - items = frappe.get_all(doctype, filters=filters, order_by=order_by) - cache.hset(cache_key, name, json.dumps(items)) - except frappe.db.TableMissingError: - # executed from inside patch, ignore - items = [] - - return items + return frappe.cache().hget( + get_doctype_map_key(doctype), + name, + lambda: frappe.get_all(doctype, filters=filters, order_by=order_by, ignore_ddl=True), + ) def clear_doctype_map(doctype, name): diff --git a/frappe/client.py b/frappe/client.py index 6ed40f8344..b09f9168f4 100644 --- a/frappe/client.py +++ b/frappe/client.py @@ -78,16 +78,17 @@ def get(doctype, name=None, filters=None, parent=None): if frappe.is_table(doctype): check_parent_permission(parent, doctype) - if filters and not name: - name = frappe.db.get_value(doctype, frappe.parse_json(filters)) - if not name: - frappe.throw(_("No document found for given filters")) + if name: + doc = frappe.get_doc(doctype, name) + elif filters or filters == {}: + doc = frappe.get_doc(doctype, frappe.parse_json(filters)) + else: + doc = frappe.get_doc(doctype) # single - doc = frappe.get_doc(doctype, name) - if not doc.has_permission("read"): - raise frappe.PermissionError + doc.check_permission() + doc.apply_fieldlevel_read_permissions() - return frappe.get_doc(doctype, name).as_dict() + return doc.as_dict() @frappe.whitelist() @@ -144,8 +145,8 @@ def get_value(doctype, fieldname, filters=None, as_dict=True, debug=False, paren def get_single_value(doctype, field): if not frappe.has_permission(doctype): frappe.throw(_("No permission for {0}").format(_(doctype)), frappe.PermissionError) - value = frappe.db.get_single_value(doctype, field) - return value + + return frappe.db.get_single_value(doctype, field) @frappe.whitelist(methods=["POST", "PUT"]) @@ -207,9 +208,9 @@ def insert_many(docs=None): if len(docs) > 200: frappe.throw(_("Only 200 inserts allowed in one request")) - out = set() + out = [] for doc in docs: - out.add(insert_doc(doc).name) + out.append(insert_doc(doc).name) return out @@ -271,14 +272,7 @@ def delete(doctype, name): :param doctype: DocType of the document to be deleted :param name: name of the document to be deleted""" - frappe.delete_doc(doctype, name, ignore_missing=False) - - -@frappe.whitelist(methods=["POST", "PUT"]) -def set_default(key, value, parent=None): - """set a user default value""" - frappe.db.set_default(key, value, parent or frappe.session.user) - frappe.clear_cache(user=frappe.session.user) + delete_doc(doctype, name) @frappe.whitelist(methods=["POST", "PUT"]) @@ -311,6 +305,17 @@ def has_permission(doctype, docname, perm_type="read"): return {"has_permission": frappe.has_permission(doctype, perm_type.lower(), docname)} +@frappe.whitelist() +def get_doc_permissions(doctype, docname): + """Returns an evaluated document permissions dict like `{"read":1, "write":1}` + + :param doctype: DocType of the document to be evaluated + :param docname: `name` of the document to be evaluated + """ + doc = frappe.get_doc(doctype, docname) + return {"permissions": frappe.permissions.get_doc_permissions(doc)} + + @frappe.whitelist() def get_password(doctype, name, fieldname): """Return a password type property. Only applicable for System Managers @@ -341,11 +346,6 @@ def get_js(items): with open(contentpath) as srcfile: code = frappe.utils.cstr(srcfile.read()) - if frappe.local.lang != "en": - messages = frappe.get_lang_dict("jsfile", contentpath) - messages = json.dumps(messages) - code += f"\n\n$.extend(frappe._messages, {messages})" - out.append(code) return out @@ -470,3 +470,24 @@ def insert_doc(doc) -> "Document": return parent return frappe.get_doc(doc).insert() + + +def delete_doc(doctype, name): + """Deletes document + if doctype is a child table, then deletes the child record using the parent doc + so that the parent doc's `on_update` is called + """ + + if frappe.is_table(doctype): + values = frappe.db.get_value(doctype, name, ["parenttype", "parent", "parentfield"]) + if not values: + raise frappe.DoesNotExistError + parenttype, parent, parentfield = values + parent = frappe.get_doc(parenttype, parent) + for row in parent.get(parentfield): + if row.name == name: + parent.remove(row) + parent.save() + break + else: + frappe.delete_doc(doctype, name, ignore_missing=False) diff --git a/frappe/commands/__init__.py b/frappe/commands/__init__.py index f61b3b9d34..05c3593175 100644 --- a/frappe/commands/__init__.py +++ b/frappe/commands/__init__.py @@ -112,7 +112,7 @@ def get_commands(): from .translate import commands as translate_commands from .utils import commands as utils_commands - clickable_link = "\x1b]8;;https://frappeframework.com/docs\afrappeframework.com\x1b]8;;\a" + clickable_link = "https://frappeframework.com/docs" all_commands = ( scheduler_commands + site_commands + translate_commands + utils_commands + redis_commands ) diff --git a/frappe/commands/scheduler.py b/frappe/commands/scheduler.py index f26180e169..36fa81f8a5 100755 --- a/frappe/commands/scheduler.py +++ b/frappe/commands/scheduler.py @@ -5,22 +5,6 @@ import click import frappe from frappe.commands import get_site, pass_context from frappe.exceptions import SiteNotSpecifiedError -from frappe.utils import cint - - -def _is_scheduler_enabled(): - enable_scheduler = False - try: - frappe.connect() - enable_scheduler = ( - cint(frappe.db.get_single_value("System Settings", "enable_scheduler")) and True or False - ) - except Exception: - pass - finally: - frappe.db.close() - - return enable_scheduler @click.command("trigger-scheduler-event", help="Trigger a scheduler event") @@ -89,35 +73,40 @@ def disable_scheduler(context): @click.command("scheduler") @click.option("--site", help="site name") -@click.argument("state", type=click.Choice(["pause", "resume", "disable", "enable"])) +@click.argument("state", type=click.Choice(["pause", "resume", "disable", "enable", "status"])) +@click.option( + "--format", "-f", default="text", type=click.Choice(["json", "text"]), help="Output format" +) +@click.option("--verbose", "-v", is_flag=True, help="Verbose output") @pass_context -def scheduler(context, state, site=None): - import frappe.utils.scheduler - from frappe.installer import update_site_config +def scheduler(context, state: str, format: str, verbose: bool = False, site: str | None = None): + """Control scheduler state.""" + import frappe + from frappe.utils.scheduler import is_scheduler_inactive, toggle_scheduler - if not site: - site = get_site(context) + site = site or get_site(context) - try: - frappe.init(site=site) + output = { + "text": "Scheduler is {status} for site {site}", + "json": '{{"status": "{status}", "site": "{site}"}}', + } - if state == "pause": - update_site_config("pause_scheduler", 1) - elif state == "resume": - update_site_config("pause_scheduler", 0) - elif state == "disable": - frappe.connect() - frappe.utils.scheduler.disable_scheduler() - frappe.db.commit() - elif state == "enable": - frappe.connect() - frappe.utils.scheduler.enable_scheduler() - frappe.db.commit() + with frappe.init_site(site=site): + match state: + case "status": + frappe.connect() + status = "disabled" if is_scheduler_inactive(verbose=verbose) else "enabled" + return print(output[format].format(status=status, site=site)) + case "pause" | "resume": + from frappe.installer import update_site_config - print(f"Scheduler {state}d for site {site}") + update_site_config("pause_scheduler", state == "pause") + case "enable" | "disable": + frappe.connect() + toggle_scheduler(state == "enable") + frappe.db.commit() - finally: - frappe.destroy() + print(output[format].format(status=f"{state}d", site=site)) @click.command("set-maintenance-mode") @@ -125,6 +114,7 @@ def scheduler(context, state, site=None): @click.argument("state", type=click.Choice(["on", "off"])) @pass_context def set_maintenance_mode(context, state, site=None): + """Put the site in maintenance mode for upgrades.""" from frappe.installer import update_site_config if not site: @@ -187,21 +177,42 @@ def purge_jobs(site=None, queue=None, event=None): @click.command("schedule") def start_scheduler(): + """Start scheduler process which is responsible for enqueueing the scheduled job types.""" from frappe.utils.scheduler import start_scheduler start_scheduler() @click.command("worker") -@click.option("--queue", type=str) +@click.option( + "--queue", + type=str, + help="Queue to consume from. Multiple queues can be specified using comma-separated string. If not specified all queues are consumed.", +) @click.option("--quiet", is_flag=True, default=False, help="Hide Log Outputs") @click.option("-u", "--rq-username", default=None, help="Redis ACL user") @click.option("-p", "--rq-password", default=None, help="Redis ACL user password") -def start_worker(queue, quiet=False, rq_username=None, rq_password=None): - """Site is used to find redis credentals.""" +@click.option("--burst", is_flag=True, default=False, help="Run Worker in Burst mode.") +@click.option( + "--strategy", + required=False, + type=click.Choice(["round_robin", "random"]), + help="Dequeuing strategy to use", +) +def start_worker( + queue, quiet=False, rq_username=None, rq_password=None, burst=False, strategy=None +): + """Start a backgrond worker""" from frappe.utils.background_jobs import start_worker - start_worker(queue, quiet=quiet, rq_username=rq_username, rq_password=rq_password) + start_worker( + queue, + quiet=quiet, + rq_username=rq_username, + rq_password=rq_password, + burst=burst, + strategy=strategy, + ) @click.command("ready-for-migration") diff --git a/frappe/commands/site.py b/frappe/commands/site.py index 626da058c3..25c8c3159d 100644 --- a/frappe/commands/site.py +++ b/frappe/commands/site.py @@ -44,7 +44,7 @@ from frappe.exceptions import SiteNotSpecifiedError @click.option( "--force", help="Force restore if site/database already exists", is_flag=True, default=False ) -@click.option("--source_sql", help="Initiate database with a SQL file") +@click.option("--source-sql", "--source_sql", help="Initiate database with a SQL file") @click.option("--install-app", multiple=True, help="Install app after installation") @click.option( "--set-default", is_flag=True, default=False, help="Set the new site as default site" @@ -67,10 +67,13 @@ def new_site( set_default=False, ): "Create a new site" - from frappe.installer import _new_site + from frappe.installer import _new_site, extract_sql_from_archive frappe.init(site=site, new_site=True) + if source_sql: + source_sql = extract_sql_from_archive(source_sql) + _new_site( db_name, site, @@ -86,7 +89,6 @@ def new_site( db_type=db_type, db_host=db_host, db_port=db_port, - new_site=True, ) if set_default: @@ -110,7 +112,8 @@ def new_site( "--with-public-files", help="Restores the public files of the site, given path to its tar file" ) @click.option( - "--with-private-files", help="Restores the private files of the site, given path to its tar file" + "--with-private-files", + help="Restores the private files of the site, given path to its tar file", ) @click.option( "--force", @@ -135,6 +138,43 @@ def restore( with_private_files=None, ): "Restore site database from an sql file" + + from frappe.utils.synchronization import filelock + + site = get_site(context) + frappe.init(site=site) + + with filelock("site_restore", timeout=1): + _restore( + site=site, + sql_file_path=sql_file_path, + encryption_key=encryption_key, + db_root_username=db_root_username, + db_root_password=db_root_password, + verbose=context.verbose or verbose, + install_app=install_app, + admin_password=admin_password, + force=context.force or force, + with_public_files=with_public_files, + with_private_files=with_private_files, + ) + + +def _restore( + *, + site=None, + sql_file_path=None, + encryption_key=None, + db_root_username=None, + db_root_password=None, + verbose=None, + install_app=None, + admin_password=None, + force=None, + with_public_files=None, + with_private_files=None, +): + from frappe.installer import ( _new_site, extract_files, @@ -143,14 +183,10 @@ def restore( is_partial, validate_database_sql, ) - from frappe.utils.backups import Backup + from frappe.utils.backups import Backup, get_or_generate_backup_encryption_key _backup = Backup(sql_file_path) - site = get_site(context) - frappe.init(site=site) - force = context.force or force - try: decompressed_file_name = extract_sql_from_archive(sql_file_path) if is_partial(decompressed_file_name): @@ -159,7 +195,8 @@ def restore( fg="red", ) click.secho( - "Use `bench partial-restore` to restore a partial backup to an existing site.", fg="yellow" + "Use `bench partial-restore` to restore a partial backup to an existing site.", + fg="yellow", ) _backup.decryption_rollback() sys.exit(1) @@ -172,7 +209,7 @@ def restore( else: click.secho("Encrypted backup file detected. Decrypting using site config.", fg="yellow") - encryption_key = frappe.get_site_config().encryption_key + encryption_key = get_or_generate_backup_encryption_key() _backup.backup_decryption(encryption_key) # Rollback on unsuccessful decryrption @@ -190,7 +227,8 @@ def restore( fg="red", ) click.secho( - "Use `bench partial-restore` to restore a partial backup to an existing site.", fg="yellow" + "Use `bench partial-restore` to restore a partial backup to an existing site.", + fg="yellow", ) _backup.decryption_rollback() sys.exit(1) @@ -212,7 +250,7 @@ def restore( db_root_username=db_root_username, db_root_password=db_root_password, admin_password=admin_password, - verbose=context.verbose, + verbose=verbose, install_apps=install_app, source_sql=decompressed_file_name, force=True, @@ -269,7 +307,7 @@ def restore( @pass_context def partial_restore(context, sql_file_path, verbose, encryption_key=None): from frappe.installer import extract_sql_from_archive, partial_restore - from frappe.utils.backups import Backup + from frappe.utils.backups import Backup, get_or_generate_backup_encryption_key if not os.path.exists(sql_file_path): print("Invalid path", sql_file_path) @@ -292,7 +330,8 @@ def partial_restore(context, sql_file_path, verbose, encryption_key=None): # Check for full backup file if "Partial Backup" not in header: click.secho( - "Full backup file detected.Use `bench restore` to restore a Frappe Site.", fg="red" + "Full backup file detected.Use `bench restore` to restore a Frappe Site.", + fg="red", ) _backup.decryption_rollback() sys.exit(1) @@ -305,7 +344,7 @@ def partial_restore(context, sql_file_path, verbose, encryption_key=None): else: click.secho("Encrypted backup file detected. Decrypting using site config.", fg="yellow") - key = frappe.get_site_config().encryption_key + key = get_or_generate_backup_encryption_key() _backup.backup_decryption(key) @@ -323,7 +362,8 @@ def partial_restore(context, sql_file_path, verbose, encryption_key=None): # Check for Full backup file. if "Partial Backup" not in header: click.secho( - "Full Backup file detected.Use `bench restore` to restore a Frappe Site.", fg="red" + "Full Backup file detected.Use `bench restore` to restore a Frappe Site.", + fg="red", ) _backup.decryption_rollback() sys.exit(1) @@ -359,9 +399,15 @@ def reinstall( def _reinstall( - site, admin_password=None, db_root_username=None, db_root_password=None, yes=False, verbose=False + site, + admin_password=None, + db_root_username=None, + db_root_password=None, + yes=False, + verbose=False, ): from frappe.installer import _new_site + from frappe.utils.synchronization import filelock if not yes: click.confirm("This will wipe your database. Are you sure you want to reinstall?", abort=True) @@ -379,6 +425,7 @@ def _reinstall( frappe.destroy() frappe.init(site=site) + _new_site( frappe.conf.db_name, site, @@ -399,6 +446,7 @@ def _reinstall( def install_app(context, apps, force=False): "Install a new app to site, supports multiple apps" from frappe.installer import install_app as _install_app + from frappe.utils.synchronization import filelock exit_code = 0 @@ -409,17 +457,21 @@ def install_app(context, apps, force=False): frappe.init(site=site) frappe.connect() - for app in apps: - try: - _install_app(app, verbose=context.verbose, force=force) - except frappe.IncompatibleApp as err: - err_msg = f":\n{err}" if str(err) else "" - print(f"App {app} is Incompatible with Site {site}{err_msg}") - exit_code = 1 - except Exception as err: - err_msg = f": {str(err)}\n{frappe.get_traceback()}" - print(f"An error occurred while installing {app}{err_msg}") - exit_code = 1 + with filelock("install_app", timeout=1): + for app in apps: + try: + _install_app(app, verbose=context.verbose, force=force) + except frappe.IncompatibleApp as err: + err_msg = f":\n{err}" if str(err) else "" + print(f"App {app} is Incompatible with Site {site}{err_msg}") + exit_code = 1 + except Exception as err: + err_msg = f": {str(err)}\n{frappe.get_traceback()}" + print(f"An error occurred while installing {app}{err_msg}") + exit_code = 1 + + if not exit_code: + frappe.db.commit() frappe.destroy() @@ -497,10 +549,37 @@ def add_system_manager(context, email, first_name, last_name, send_welcome_email raise SiteNotSpecifiedError +@click.command("add-user") +@click.argument("email") +@click.option("--first-name") +@click.option("--last-name") +@click.option("--password") +@click.option("--user-type") +@click.option("--add-role", multiple=True) +@click.option("--send-welcome-email", default=False, is_flag=True) +@pass_context +def add_user_for_sites( + context, email, first_name, last_name, user_type, send_welcome_email, password, add_role +): + "Add user to a site" + import frappe.utils.user + + for site in context.sites: + frappe.connect(site=site) + try: + add_new_user(email, first_name, last_name, user_type, send_welcome_email, password, add_role) + frappe.db.commit() + finally: + frappe.destroy() + if not context.sites: + raise SiteNotSpecifiedError + + @click.command("disable-user") @click.argument("email") @pass_context def disable_user(context, email): + """Disable a user account on site.""" site = get_site(context) with frappe.init_site(site): frappe.connect() @@ -516,6 +595,8 @@ def disable_user(context, email): @pass_context def migrate(context, skip_failing=False, skip_search_index=False): "Run patches, sync schema and rebuild files/translations" + from traceback_with_variables import activate_by_import + from frappe.migrate import SiteMigration for site in context.sites: @@ -653,7 +734,10 @@ def use(site, sites_path="."): @click.option("--backup-path-private-files", default=None, help="Set path for saving private file") @click.option("--backup-path-conf", default=None, help="Set path for saving config file") @click.option( - "--ignore-backup-conf", default=False, is_flag=True, help="Ignore excludes/includes set in config" + "--ignore-backup-conf", + default=False, + is_flag=True, + help="Ignore excludes/includes set in config", ) @click.option("--verbose", default=False, is_flag=True, help="Add verbosity") @click.option("--compress", default=False, is_flag=True, help="Compress private and public files") @@ -708,7 +792,8 @@ def backup( continue if frappe.get_system_settings("encrypt_backup") and frappe.get_site_config().encryption_key: click.secho( - "Backup encryption is turned on. Please note the backup encryption key.", fg="yellow" + "Backup encryption is turned on. Please note the backup encryption key.", + fg="yellow", ) odb.print_summary() @@ -762,12 +847,14 @@ def remove_from_installed_apps(context, app): def uninstall(context, app, dry_run, yes, no_backup, force): "Remove app and linked modules from site" from frappe.installer import remove_app + from frappe.utils.synchronization import filelock for site in context.sites: try: frappe.init(site=site) frappe.connect() - remove_app(app_name=app, dry_run=dry_run, yes=yes, no_backup=no_backup, force=force) + with filelock("uninstall_app"): + remove_app(app_name=app, dry_run=dry_run, yes=yes, no_backup=no_backup, force=force) finally: frappe.destroy() if not context.sites: @@ -801,6 +888,7 @@ def drop_site( force=False, no_backup=False, ): + """Remove a site from database and filesystem.""" _drop_site(site, db_root_username, db_root_password, archived_sites_path, force, no_backup) @@ -812,7 +900,6 @@ def _drop_site( force=False, no_backup=False, ): - "Remove site from database and filesystem" from frappe.database import drop_user_and_database from frappe.utils.backups import scheduled_backup @@ -844,9 +931,10 @@ def _drop_site( archived_sites_path = archived_sites_path or os.path.join( frappe.get_app_path("frappe"), "..", "..", "..", "archived", "sites" ) + archived_sites_path = os.path.realpath(archived_sites_path) + click.secho(f"Moving site to archive under {archived_sites_path}", fg="green") os.makedirs(archived_sites_path, exist_ok=True) - move(archived_sites_path, site) @@ -1022,6 +1110,7 @@ def browse(context, site, user=None): @click.command("start-recording") @pass_context def start_recording(context): + """Start Frappe Recorder.""" import frappe.recorder for site in context.sites: @@ -1035,6 +1124,7 @@ def start_recording(context): @click.command("stop-recording") @pass_context def stop_recording(context): + """Stop Frappe Recorder.""" import frappe.recorder for site in context.sites: @@ -1049,13 +1139,31 @@ def stop_recording(context): @click.option( "--bind-tls", is_flag=True, default=False, help="Returns a reference to the https tunnel." ) +@click.option( + "--use-default-authtoken", + is_flag=True, + default=False, + help="Use the auth token present in ngrok's config.", +) @pass_context -def start_ngrok(context, bind_tls): +def start_ngrok(context, bind_tls, use_default_authtoken): + """Start a ngrok tunnel to your local development server.""" from pyngrok import ngrok site = get_site(context) frappe.init(site=site) + ngrok_authtoken = frappe.conf.ngrok_authtoken + if not use_default_authtoken: + if not ngrok_authtoken: + click.echo( + f"\n{click.style('ngrok_authtoken', fg='yellow')} not found in site config.\n" + "Please register for a free ngrok account at: https://dashboard.ngrok.com/signup and place the obtained authtoken in the site config.", + ) + sys.exit(1) + + ngrok.set_auth_token(ngrok_authtoken) + port = frappe.conf.http_port or frappe.conf.webserver_port tunnel = ngrok.connect(addr=str(port), host_header=site, bind_tls=bind_tls) print(f"Public URL: {tunnel.public_url}") @@ -1074,6 +1182,7 @@ def start_ngrok(context, bind_tls): @click.command("build-search-index") @pass_context def build_search_index(context): + """Rebuild search index used by global search.""" from frappe.search.website_search import build_index_for_all_routes site = get_site(context) @@ -1090,7 +1199,7 @@ def build_search_index(context): @click.command("clear-log-table") -@click.option("--doctype", default="text", type=click.Choice(LOG_DOCTYPES), help="Log DocType") +@click.option("--doctype", required=True, type=click.Choice(LOG_DOCTYPES), help="Log DocType") @click.option("--days", type=int, help="Keep records for days") @click.option("--no-backup", is_flag=True, default=False, help="Do not backup the table") @pass_context @@ -1140,8 +1249,16 @@ def clear_log_table(context, doctype, days, no_backup): "--format", "-f", default="text", type=click.Choice(["json", "text"]), help="Output format" ) @click.option("--no-backup", is_flag=True, default=False, help="Do not backup the site") +@click.option( + "--yes", + "-y", + help="To bypass confirmation prompt.", + is_flag=True, + default=False, +) @pass_context -def trim_database(context, dry_run, format, no_backup): +def trim_database(context, dry_run, format, no_backup, yes=False): + """Remove database tables for deleted DocTypes.""" if not context.sites: raise SiteNotSpecifiedError @@ -1162,6 +1279,7 @@ def trim_database(context, dry_run, format, no_backup): frappe.qb.from_(information_schema.tables) .select(table_name) .where(information_schema.tables.table_schema == frappe.conf.db_name) + .where(information_schema.tables.table_type == "BASE TABLE") .run() ) @@ -1169,6 +1287,8 @@ def trim_database(context, dry_run, format, no_backup): doctype_tables = frappe.get_all("DocType", pluck="name") for x in database_tables: + if not x.startswith("tab"): + continue doctype = x.replace("tab", "", 1) if not (doctype in doctype_tables or x.startswith("__") or x in STANDARD_TABLES): TABLES_TO_DROP.append(x) @@ -1177,6 +1297,11 @@ def trim_database(context, dry_run, format, no_backup): if format == "text": click.secho(f"No ghost tables found in {frappe.local.site}...Great!", fg="green") else: + if not yes: + print("Following tables will be dropped:") + print("\n".join(f"* {dt}" for dt in TABLES_TO_DROP)) + click.confirm("Do you want to continue?", abort=True) + if not (no_backup or dry_run): if format == "text": print(f"Backing Up Tables: {', '.join(TABLES_TO_DROP)}") @@ -1237,6 +1362,7 @@ def get_standard_tables(): @click.option("--no-backup", is_flag=True, default=False, help="Do not backup the site") @pass_context def trim_tables(context, dry_run, format, no_backup): + """Remove columns from tables where fields are deleted from doctypes.""" if not context.sites: raise SiteNotSpecifiedError @@ -1275,8 +1401,38 @@ def handle_data(data: dict, format="json"): render_table(data) +def add_new_user( + email, + first_name=None, + last_name=None, + user_type="System User", + send_welcome_email=False, + password=None, + role=None, +): + user = frappe.new_doc("User") + user.update( + { + "name": email, + "email": email, + "enabled": 1, + "first_name": first_name or email, + "last_name": last_name, + "user_type": user_type, + "send_welcome_email": 1 if send_welcome_email else 0, + } + ) + user.insert() + user.add_roles(*role) + if password: + from frappe.utils.password import update_password + + update_password(user=user.name, pwd=password) + + commands = [ add_system_manager, + add_user_for_sites, backup, drop_site, install_app, diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py index 585478c0ff..03374986d4 100644 --- a/frappe/commands/utils.py +++ b/frappe/commands/utils.py @@ -2,7 +2,7 @@ import json import os import subprocess import sys -from distutils.spawn import find_executable +from shutil import which import click @@ -12,17 +12,23 @@ from frappe.coverage import CodeCoverage from frappe.exceptions import SiteNotSpecifiedError from frappe.utils import cint, update_progress_bar +find_executable = which # backwards compatibility DATA_IMPORT_DEPRECATION = ( "[DEPRECATED] The `import-csv` command used 'Data Import Legacy' which has been deprecated.\n" "Use `data-import` command instead to import data via 'Data Import'." ) +EXTRA_ARGS_CTX = {"ignore_unknown_options": True, "allow_extra_args": True} @click.command("build") @click.option("--app", help="Build assets for app") @click.option("--apps", help="Build assets for specific apps") @click.option( - "--hard-link", is_flag=True, default=False, help="Copy the files instead of symlinking" + "--hard-link", + is_flag=True, + default=False, + help="Copy the files instead of symlinking", + envvar="FRAPPE_HARD_LINK_ASSETS", ) @click.option( "--make-copy", @@ -53,33 +59,35 @@ def build( ): "Compile JS and CSS source files" from frappe.build import bundle, download_frappe_assets + from frappe.utils.synchronization import filelock frappe.init("") if not apps and app: apps = app - # dont try downloading assets if force used, app specified or running via CI - if not (force or apps or os.environ.get("CI")): - # skip building frappe if assets exist remotely - skip_frappe = download_frappe_assets(verbose=verbose) - else: - skip_frappe = False + with filelock("bench_build", is_global=True, timeout=10): + # dont try downloading assets if force used, app specified or running via CI + if not (force or apps or os.environ.get("CI")): + # skip building frappe if assets exist remotely + skip_frappe = download_frappe_assets(verbose=verbose) + else: + skip_frappe = False - # don't minify in developer_mode for faster builds - development = frappe.local.conf.developer_mode or frappe.local.dev_server - mode = "development" if development else "production" - if production: - mode = "production" + # don't minify in developer_mode for faster builds + development = frappe.local.conf.developer_mode or frappe.local.dev_server + mode = "development" if development else "production" + if production: + mode = "production" - if make_copy or restore: - hard_link = make_copy or restore - click.secho( - "bench build: --make-copy and --restore options are deprecated in favour of --hard-link", - fg="yellow", - ) + if make_copy or restore: + hard_link = make_copy or restore + click.secho( + "bench build: --make-copy and --restore options are deprecated in favour of --hard-link", + fg="yellow", + ) - bundle(mode, apps=apps, hard_link=hard_link, verbose=verbose, skip_frappe=skip_frappe) + bundle(mode, apps=apps, hard_link=hard_link, verbose=verbose, skip_frappe=skip_frappe) @click.command("watch") @@ -482,9 +490,10 @@ def bulk_rename(context, doctype, path): frappe.destroy() -@click.command("db-console") +@click.command("db-console", context_settings=EXTRA_ARGS_CTX) +@click.argument("extra_args", nargs=-1) @pass_context -def database(context): +def database(context, extra_args): """ Enter into the Database console for given site. """ @@ -493,14 +502,18 @@ def database(context): raise SiteNotSpecifiedError frappe.init(site=site) if not frappe.conf.db_type or frappe.conf.db_type == "mariadb": - _mariadb() + _mariadb(extra_args=extra_args) elif frappe.conf.db_type == "postgres": - _psql() + _psql(extra_args=extra_args) -@click.command("mariadb") +@click.command( + "mariadb", + context_settings=EXTRA_ARGS_CTX, +) +@click.argument("extra_args", nargs=-1) @pass_context -def mariadb(context): +def mariadb(context, extra_args): """ Enter into mariadb console for a given site. """ @@ -508,49 +521,64 @@ def mariadb(context): if not site: raise SiteNotSpecifiedError frappe.init(site=site) - _mariadb() + _mariadb(extra_args=extra_args) -@click.command("postgres") +@click.command("postgres", context_settings=EXTRA_ARGS_CTX) +@click.argument("extra_args", nargs=-1) @pass_context -def postgres(context): +def postgres(context, extra_args): """ Enter into postgres console for a given site. """ site = get_site(context) frappe.init(site=site) - _psql() + _psql(extra_args=extra_args) -def _mariadb(): - mysql = find_executable("mysql") - os.execv( +def _mariadb(extra_args=None): + from frappe.database.mariadb.database import MariaDBDatabase + + mysql = which("mysql") + command = [ mysql, - [ - mysql, - "-u", - frappe.conf.db_name, - "-p" + frappe.conf.db_password, - frappe.conf.db_name, - "-h", - frappe.conf.db_host or "localhost", - "--pager=less -SFX", - "--safe-updates", - "-A", - ], - ) + "--port", + str(frappe.conf.db_port or MariaDBDatabase.default_port), + "-u", + frappe.conf.db_name, + f"-p{frappe.conf.db_password}", + frappe.conf.db_name, + "-h", + frappe.conf.db_host or "localhost", + "--pager=less -SFX", + "--safe-updates", + "-A", + ] + if extra_args: + command += list(extra_args) + os.execv(mysql, command) -def _psql(): - psql = find_executable("psql") - subprocess.run([psql, "-d", frappe.conf.db_name]) +def _psql(extra_args=None): + psql = which("psql") + + host = frappe.conf.db_host or "127.0.0.1" + port = frappe.conf.db_port or "5432" + env = os.environ.copy() + env["PGPASSWORD"] = frappe.conf.db_password + conn_string = f"postgresql://{frappe.conf.db_name}@{host}:{port}/{frappe.conf.db_name}" + psql_cmd = [psql, conn_string] + if extra_args: + psql_cmd = psql_cmd + list(extra_args) + subprocess.run(psql_cmd, check=True, env=env) @click.command("jupyter") @pass_context def jupyter(context): + """Start an interactive jupyter notebook""" installed_packages = ( - r.split("==")[0] + r.split("==", 1)[0] for r in subprocess.check_output([sys.executable, "-m", "pip", "freeze"], encoding="utf8") ) @@ -767,6 +795,7 @@ def run_tests( failfast=False, case=None, ): + """Run python unit-tests""" with CodeCoverage(coverage, app): import frappe @@ -817,10 +846,19 @@ def run_tests( @click.option("--total-builds", help="Total number of builds", default=1) @click.option("--with-coverage", is_flag=True, help="Build coverage file") @click.option("--use-orchestrator", is_flag=True, help="Use orchestrator to run parallel tests") +@click.option("--dry-run", is_flag=True, default=False, help="Dont actually run tests") @pass_context def run_parallel_tests( - context, app, build_number, total_builds, with_coverage=False, use_orchestrator=False + context, + app, + build_number, + total_builds, + with_coverage=False, + use_orchestrator=False, + dry_run=False, ): + from traceback_with_variables import activate_by_import + with CodeCoverage(with_coverage, app): site = get_site(context) if use_orchestrator: @@ -830,18 +868,36 @@ def run_parallel_tests( else: from frappe.parallel_test_runner import ParallelTestRunner - ParallelTestRunner(app, site=site, build_number=build_number, total_builds=total_builds) + ParallelTestRunner( + app, + site=site, + build_number=build_number, + total_builds=total_builds, + dry_run=dry_run, + ) -@click.command("run-ui-tests") +@click.command( + "run-ui-tests", + context_settings=dict( + ignore_unknown_options=True, + ), +) @click.argument("app") +@click.argument("cypressargs", nargs=-1, type=click.UNPROCESSED) @click.option("--headless", is_flag=True, help="Run UI Test in headless mode") @click.option("--parallel", is_flag=True, help="Run UI Test in parallel mode") @click.option("--with-coverage", is_flag=True, help="Generate coverage report") @click.option("--ci-build-id") @pass_context def run_ui_tests( - context, app, headless=False, parallel=True, with_coverage=False, ci_build_id=None + context, + app, + headless=False, + parallel=True, + with_coverage=False, + ci_build_id=None, + cypressargs=None, ): "Run UI tests" site = get_site(context) @@ -856,9 +912,8 @@ def run_ui_tests( os.chdir(app_base_path) - node_bin = subprocess.getoutput("npm bin") + node_bin = subprocess.getoutput("(cd ../frappe && yarn bin)") cypress_path = f"{node_bin}/cypress" - plugin_path = f"{node_bin}/../cypress-file-upload" drag_drop_plugin_path = f"{node_bin}/../@4tw/cypress-drag-drop" real_events_plugin_path = f"{node_bin}/../cypress-real-events" testing_library_path = f"{node_bin}/../@testing-library" @@ -867,17 +922,24 @@ def run_ui_tests( # check if cypress in path...if not, install it. if not ( os.path.exists(cypress_path) - and os.path.exists(plugin_path) and os.path.exists(drag_drop_plugin_path) and os.path.exists(real_events_plugin_path) and os.path.exists(testing_library_path) and os.path.exists(coverage_plugin_path) ): - # install cypress + # install cypress & dependent plugins click.secho("Installing Cypress...", fg="yellow") - frappe.commands.popen( - "yarn add cypress@^6 cypress-file-upload@^5 @4tw/cypress-drag-drop@^2 cypress-real-events @testing-library/cypress@^8 @cypress/code-coverage@^3 --no-lockfile" + packages = " ".join( + [ + "cypress@^10", + "@4tw/cypress-drag-drop@^2", + "cypress-real-events", + "@testing-library/cypress@^8", + "@testing-library/dom@8.17.1", + "@cypress/code-coverage@^3", + ] ) + frappe.commands.popen(f"(cd ../frappe && yarn add {packages} --no-lockfile)") # run for headless mode run_or_open = "run --browser chrome --record" if headless else "open" @@ -889,6 +951,9 @@ def run_ui_tests( if ci_build_id: formatted_command += f" --ci-build-id {ci_build_id}" + if cypressargs: + formatted_command += " " + " ".join(cypressargs) + click.secho("Running Cypress...", fg="yellow") frappe.commands.popen(formatted_command, cwd=app_base_path, raise_err=True) @@ -952,7 +1017,7 @@ def request(context, args=None, path=None): frappe.local.form_dict = frappe._dict() if args.startswith("/api/method"): - frappe.local.form_dict.cmd = args.split("?")[0].split("/")[-1] + frappe.local.form_dict.cmd = args.split("?", 1)[0].split("/")[-1] elif path: with open(os.path.join("..", path)) as f: args = json.loads(f.read()) @@ -981,6 +1046,16 @@ def make_app(destination, app_name, no_git=False): make_boilerplate(destination, app_name, no_git=no_git) +@click.command("create-patch") +def create_patch(): + "Creates a new patch interactively" + from frappe.utils.boilerplate import PatchCreator + + pc = PatchCreator() + pc.fetch_user_inputs() + pc.create_patch_file() + + @click.command("set-config") @click.argument("key") @click.argument("value") @@ -1012,6 +1087,8 @@ def set_config(context, key, value, global_=False, parse=False, as_dict=False): common_site_config_path = os.path.join(sites_path, "common_site_config.json") update_site_config(key, value, validate=False, site_config_path=common_site_config_path) else: + if not context.sites: + raise SiteNotSpecifiedError for site in context.sites: frappe.init(site=site) update_site_config(key, value, validate=False) @@ -1127,6 +1204,7 @@ commands = [ data_import, import_doc, make_app, + create_patch, mariadb, postgres, request, diff --git a/frappe/config/__init__.py b/frappe/config/__init__.py index ebd75cd70a..02626aedf5 100644 --- a/frappe/config/__init__.py +++ b/frappe/config/__init__.py @@ -1,17 +1,9 @@ import frappe from frappe import _ -from frappe.desk.moduleview import ( - config_exists, - get_data, - get_module_link_items_from_list, - get_onboard_items, -) -def get_modules_from_all_apps_for_user(user=None): - if not user: - user = frappe.session.user - +def get_modules_from_all_apps_for_user(user: str = None) -> list[dict]: + user = user or frappe.session.user all_modules = get_modules_from_all_apps() global_blocked_modules = frappe.get_doc("User", "Administrator").get_blocked_modules() user_blocked_modules = frappe.get_doc("User", user).get_blocked_modules() @@ -27,9 +19,6 @@ def get_modules_from_all_apps_for_user(user=None): if module_name in empty_tables_by_module: module["onboard_present"] = 1 - # Set defaults links - module["links"] = get_onboard_items(module["app"], frappe.scrub(module_name))[:5] - return allowed_modules_list @@ -61,7 +50,7 @@ def get_all_empty_tables_by_module(): empty_tables_by_module = {} for doctype, module in results: - if "tab" + doctype in empty_tables: + if f"tab{doctype}" in empty_tables: if module in empty_tables_by_module: empty_tables_by_module[module].append(doctype) else: diff --git a/frappe/contacts/address_and_contact.py b/frappe/contacts/address_and_contact.py index 4df32c6705..1d3a5d644c 100644 --- a/frappe/contacts/address_and_contact.py +++ b/frappe/contacts/address_and_contact.py @@ -149,18 +149,26 @@ def get_permitted_and_not_permitted_links(doctype): return {"permitted_links": permitted_links, "not_permitted_links": not_permitted_links} -def delete_contact_and_address(doctype, docname): +def delete_contact_and_address(doctype: str, docname: str) -> None: for parenttype in ("Contact", "Address"): - items = frappe.db.sql_list( - """select parent from `tabDynamic Link` - where parenttype=%s and link_doctype=%s and link_name=%s""", - (parenttype, doctype, docname), - ) - - for name in items: + for name in frappe.get_all( + "Dynamic Link", + filters={ + "parenttype": parenttype, + "link_doctype": doctype, + "link_name": docname, + }, + pluck="parent", + ): doc = frappe.get_doc(parenttype, name) if len(doc.links) == 1: doc.delete() + else: + for link in doc.links: + if link.link_doctype == doctype and link.link_name == docname: + doc.remove(link) + doc.save() + break @frappe.whitelist() diff --git a/frappe/contacts/doctype/address/address.js b/frappe/contacts/doctype/address/address.js index 63574622c0..548dd40060 100644 --- a/frappe/contacts/doctype/address/address.js +++ b/frappe/contacts/doctype/address/address.js @@ -2,60 +2,74 @@ // For license information, please see license.txt frappe.ui.form.on("Address", { - refresh: function(frm) { - if(frm.doc.__islocal) { + refresh: function (frm) { + if (frm.doc.__islocal) { const last_doc = frappe.contacts.get_last_doc(frm); - if(frappe.dynamic_link && frappe.dynamic_link.doc - && frappe.dynamic_link.doc.name == last_doc.docname) { - frm.set_value('links', ''); - frm.add_child('links', { + if ( + frappe.dynamic_link && + frappe.dynamic_link.doc && + frappe.dynamic_link.doc.name == last_doc.docname + ) { + frm.set_value("links", ""); + frm.add_child("links", { link_doctype: frappe.dynamic_link.doctype, - link_name: frappe.dynamic_link.doc[frappe.dynamic_link.fieldname] + link_name: frappe.dynamic_link.doc[frappe.dynamic_link.fieldname], }); } } - frm.set_query('link_doctype', "links", function() { + frm.set_query("link_doctype", "links", function () { return { query: "frappe.contacts.address_and_contact.filter_dynamic_link_doctypes", filters: { fieldtype: "HTML", fieldname: "address_html", - } - } + }, + }; }); frm.refresh_field("links"); if (frm.doc.links) { for (let i in frm.doc.links) { let link = frm.doc.links[i]; - frm.add_custom_button(__("{0}: {1}", [__(link.link_doctype), __(link.link_name)]), function() { - frappe.set_route("Form", link.link_doctype, link.link_name); - }, __("Links")); + frm.add_custom_button( + __("{0}: {1}", [__(link.link_doctype), __(link.link_name)]), + function () { + frappe.set_route("Form", link.link_doctype, link.link_name); + }, + __("Links") + ); } } }, - validate: function(frm) { + validate: function (frm) { // clear linked customer / supplier / sales partner on saving... - if(frm.doc.links) { - frm.doc.links.forEach(function(d) { + if (frm.doc.links) { + frm.doc.links.forEach(function (d) { frappe.model.remove_from_locals(d.link_doctype, d.link_name); }); } }, - after_save: function(frm) { + after_save: function (frm) { frappe.run_serially([ () => frappe.timeout(1), () => { const last_doc = frappe.contacts.get_last_doc(frm); - if (frappe.dynamic_link && frappe.dynamic_link.doc && frappe.dynamic_link.doc.name == last_doc.docname) { + if ( + frappe.dynamic_link && + frappe.dynamic_link.doc && + frappe.dynamic_link.doc.name == last_doc.docname + ) { for (let i in frm.doc.links) { let link = frm.doc.links[i]; - if (last_doc.doctype == link.link_doctype && last_doc.docname == link.link_name) { - frappe.set_route('Form', last_doc.doctype, last_doc.docname); + if ( + last_doc.doctype == link.link_doctype && + last_doc.docname == link.link_name + ) { + frappe.set_route("Form", last_doc.doctype, last_doc.docname); } } } - } + }, ]); - } + }, }); diff --git a/frappe/contacts/doctype/address/address.json b/frappe/contacts/doctype/address/address.json index e85a89ff1a..c30299c7ad 100644 --- a/frappe/contacts/doctype/address/address.json +++ b/frappe/contacts/doctype/address/address.json @@ -152,7 +152,6 @@ "modified_by": "Administrator", "module": "Contacts", "name": "Address", - "name_case": "Title Case", "owner": "Administrator", "permissions": [ { @@ -205,7 +204,6 @@ "read": 1, "report": 1, "role": "System Manager", - "set_user_permissions": 1, "share": 1, "write": 1 } @@ -213,4 +211,4 @@ "search_fields": "country, state", "sort_field": "modified", "sort_order": "DESC" -} \ No newline at end of file +} diff --git a/frappe/contacts/doctype/address/address.py b/frappe/contacts/doctype/address/address.py index 42dbdd6177..94bc65a115 100644 --- a/frappe/contacts/doctype/address/address.py +++ b/frappe/contacts/doctype/address/address.py @@ -1,6 +1,8 @@ # Copyright (c) 2015, Frappe Technologies and contributors # License: MIT. See LICENSE +from typing import Optional + from jinja2 import TemplateSyntaxError import frappe @@ -100,36 +102,30 @@ def get_preferred_address(doctype, name, preferred_key="is_primary_address"): @frappe.whitelist() -def get_default_address(doctype, name, sort_key="is_primary_address"): +def get_default_address( + doctype: str, name: str | None, sort_key: str = "is_primary_address" +) -> str | None: """Returns default Address name for the given doctype, name""" if sort_key not in ["is_shipping_address", "is_primary_address"]: return None - out = frappe.db.sql( - """ SELECT - addr.name, addr.%s - FROM - `tabAddress` addr, `tabDynamic Link` dl - WHERE - dl.parent = addr.name and dl.link_doctype = %s and - dl.link_name = %s and ifnull(addr.disabled, 0) = 0 - """ - % (sort_key, "%s", "%s"), - (doctype, name), - as_dict=True, + addresses = frappe.get_all( + "Address", + filters=[ + ["Dynamic Link", "link_doctype", "=", doctype], + ["Dynamic Link", "link_name", "=", name], + ["disabled", "=", 0], + ], + pluck="name", + order_by=f"{sort_key} DESC", + limit=1, ) - if out: - for contact in out: - if contact.get(sort_key): - return contact.name - return out[0].name - else: - return None + return addresses[0] if addresses else None @frappe.whitelist() -def get_address_display(address_dict): +def get_address_display(address_dict: dict | str | None) -> str | None: if not address_dict: return @@ -217,8 +213,10 @@ def get_address_templates(address): def get_company_address(company): ret = frappe._dict() - ret.company_address = get_default_address("Company", company) - ret.company_address_display = get_address_display(ret.company_address) + + if company: + ret.company_address = get_default_address("Company", company) + ret.company_address_display = get_address_display(ret.company_address) return ret @@ -228,11 +226,12 @@ def get_company_address(company): def address_query(doctype, txt, searchfield, start, page_len, filters): from frappe.desk.reportview import get_match_cond + doctype = "Address" link_doctype = filters.pop("link_doctype") link_name = filters.pop("link_name") condition = "" - meta = frappe.get_meta("Address") + meta = frappe.get_meta(doctype) for fieldname, value in filters.items(): if meta.get_field(fieldname) or fieldname in frappe.db.DEFAULT_COLUMNS: condition += f" and {fieldname}={frappe.db.escape(value)}" @@ -253,19 +252,23 @@ def address_query(doctype, txt, searchfield, start, page_len, filters): """select `tabAddress`.name, `tabAddress`.city, `tabAddress`.country from - `tabAddress`, `tabDynamic Link` + `tabAddress` + join `tabDynamic Link` + on (`tabDynamic Link`.parent = `tabAddress`.name and `tabDynamic Link`.parenttype = 'Address') where - `tabDynamic Link`.parent = `tabAddress`.name and - `tabDynamic Link`.parenttype = 'Address' and `tabDynamic Link`.link_doctype = %(link_doctype)s and `tabDynamic Link`.link_name = %(link_name)s and ifnull(`tabAddress`.disabled, 0) = 0 and ({search_condition}) {mcond} {condition} order by - if(locate(%(_txt)s, `tabAddress`.name), locate(%(_txt)s, `tabAddress`.name), 99999), + case + when locate(%(_txt)s, `tabAddress`.name) != 0 + then locate(%(_txt)s, `tabAddress`.name) + else 99999 + end, `tabAddress`.idx desc, `tabAddress`.name - limit %(start)s, %(page_len)s """.format( + limit %(page_len)s offset %(start)s""".format( mcond=get_match_cond(doctype), search_condition=search_condition, condition=condition or "", diff --git a/frappe/contacts/doctype/address/test_address.py b/frappe/contacts/doctype/address/test_address.py index edcf87f5bc..ecb95f9e0c 100644 --- a/frappe/contacts/doctype/address/test_address.py +++ b/frappe/contacts/doctype/address/test_address.py @@ -1,12 +1,13 @@ # Copyright (c) 2015, Frappe Technologies and Contributors # License: MIT. See LICENSE -import unittest +from functools import partial import frappe -from frappe.contacts.doctype.address.address import get_address_display +from frappe.contacts.doctype.address.address import address_query, get_address_display +from frappe.tests.utils import FrappeTestCase -class TestAddress(unittest.TestCase): +class TestAddress(FrappeTestCase): def test_template_works(self): if not frappe.db.exists("Address Template", "India"): frappe.get_doc({"doctype": "Address Template", "country": "India", "is_default": 1}).insert() @@ -29,3 +30,29 @@ class TestAddress(unittest.TestCase): address = frappe.get_list("Address")[0].name display = get_address_display(frappe.get_doc("Address", address).as_dict()) self.assertTrue(display) + + def test_address_query(self): + def query(doctype="Address", txt="", searchfield="name", start=0, page_len=20, filters=None): + if filters is None: + filters = {"link_doctype": "User", "link_name": "Administrator"} + return address_query(doctype, txt, searchfield, start, page_len, filters) + + frappe.get_doc( + { + "address_type": "Billing", + "address_line1": "1", + "city": "Mumbai", + "state": "Maharashtra", + "country": "India", + "doctype": "Address", + "links": [ + { + "link_doctype": "User", + "link_name": "Administrator", + } + ], + } + ).insert() + + self.assertGreaterEqual(len(query(txt="Admin")), 1) + self.assertEqual(len(query(txt="what_zyx")), 0) diff --git a/frappe/contacts/doctype/address_template/address_template.jinja b/frappe/contacts/doctype/address_template/address_template.jinja new file mode 100644 index 0000000000..65ea58eb21 --- /dev/null +++ b/frappe/contacts/doctype/address_template/address_template.jinja @@ -0,0 +1,10 @@ +{{ address_line1 }}
+{% if address_line2 %}{{ address_line2 }}
{% endif -%} +{{ city }}
+{% if state %}{{ state }}
{% endif -%} +{% if pincode %}{{ pincode }}
{% endif -%} +{{ country }}
+
+{% if phone %}{{ _("Phone") }}: {{ phone }}
{% endif -%} +{% if fax %}{{ _("Fax") }}: {{ fax }}
{% endif -%} +{% if email_id %}{{ _("Email") }}: {{ email_id }}
{% endif -%} diff --git a/frappe/contacts/doctype/address_template/address_template.js b/frappe/contacts/doctype/address_template/address_template.js index 502d02e7f9..bfe139bce8 100644 --- a/frappe/contacts/doctype/address_template/address_template.js +++ b/frappe/contacts/doctype/address_template/address_template.js @@ -1,16 +1,16 @@ // Copyright (c) 2016, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Address Template', { - refresh: function(frm) { - if(frm.is_new() && !frm.doc.template) { +frappe.ui.form.on("Address Template", { + refresh: function (frm) { + if (frm.is_new() && !frm.doc.template) { // set default template via js so that it is translated frappe.call({ - method: 'frappe.contacts.doctype.address_template.address_template.get_default_address_template', - callback: function(r) { - frm.set_value('template', r.message); - } + method: "frappe.contacts.doctype.address_template.address_template.get_default_address_template", + callback: function (r) { + frm.set_value("template", r.message); + }, }); } - } + }, }); diff --git a/frappe/contacts/doctype/address_template/address_template.json b/frappe/contacts/doctype/address_template/address_template.json index e27d97daad..58b8210a49 100644 --- a/frappe/contacts/doctype/address_template/address_template.json +++ b/frappe/contacts/doctype/address_template/address_template.json @@ -1,152 +1,64 @@ { - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 1, - "autoname": "field:country", - "beta": 0, - "creation": "2014-06-05 02:22:36.029850", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "Setup", - "editable_grid": 0, - "engine": "InnoDB", + "actions": [], + "allow_rename": 1, + "autoname": "field:country", + "creation": "2014-06-05 02:22:36.029850", + "doctype": "DocType", + "document_type": "Setup", + "engine": "InnoDB", + "field_order": [ + "country", + "is_default", + "template" + ], "fields": [ { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "country", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 1, - "label": "Country", - "length": 0, - "no_copy": 0, - "options": "Country", - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 1, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "country", + "fieldtype": "Link", + "in_list_view": 1, + "in_standard_filter": 1, + "label": "Country", + "options": "Country", + "reqd": 1, + "search_index": 1, + "unique": 1 + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "description": "This format is used if country specific format is not found", - "fieldname": "is_default", - "fieldtype": "Check", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Is Default", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "default": "0", + "description": "This format is used if country specific format is not found", + "fieldname": "is_default", + "fieldtype": "Check", + "in_list_view": 1, + "label": "Is Default" + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "default": "", - "description": "

Default Template

\n

Uses Jinja Templating and all the fields of Address (including Custom Fields if any) will be available

\n
{{ address_line1 }}<br>\n{% if address_line2 %}{{ address_line2 }}<br>{% endif -%}\n{{ city }}<br>\n{% if state %}{{ state }}<br>{% endif -%}\n{% if pincode %} PIN:  {{ pincode }}<br>{% endif -%}\n{{ country }}<br>\n{% if phone %}Phone: {{ phone }}<br>{% endif -%}\n{% if fax %}Fax: {{ fax }}<br>{% endif -%}\n{% if email_id %}Email: {{ email_id }}<br>{% endif -%}\n
", - "fieldname": "template", - "fieldtype": "Code", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Template", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "description": "

Default Template

\n

Uses Jinja Templating and all the fields of Address (including Custom Fields if any) will be available

\n
{{ address_line1 }}<br>\n{% if address_line2 %}{{ address_line2 }}<br>{% endif -%}\n{{ city }}<br>\n{% if state %}{{ state }}<br>{% endif -%}\n{% if pincode %} PIN:  {{ pincode }}<br>{% endif -%}\n{{ country }}<br>\n{% if phone %}Phone: {{ phone }}<br>{% endif -%}\n{% if fax %}Fax: {{ fax }}<br>{% endif -%}\n{% if email_id %}Email: {{ email_id }}<br>{% endif -%}\n
", + "fieldname": "template", + "fieldtype": "Code", + "label": "Template" } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "icon": "fa fa-map-marker", - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2017-04-10 13:09:53.761009", - "modified_by": "Administrator", - "module": "Contacts", - "name": "Address Template", - "name_case": "", - "owner": "Administrator", + ], + "icon": "fa fa-map-marker", + "links": [], + "modified": "2022-08-03 12:20:49.095228", + "modified_by": "Administrator", + "module": "Contacts", + "name": "Address Template", + "owner": "Administrator", "permissions": [ { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 1, - "delete": 1, - "email": 0, - "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 0, - "read": 1, - "report": 1, - "role": "System Manager", - "set_user_permissions": 1, - "share": 1, - "submit": 0, + "create": 1, + "delete": 1, + "export": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1, "write": 1 } - ], - "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 0, - "track_seen": 0 -} \ No newline at end of file + ], + "quick_entry": 1, + "sort_field": "modified", + "sort_order": "DESC", + "states": [] +} diff --git a/frappe/contacts/doctype/address_template/address_template.py b/frappe/contacts/doctype/address_template/address_template.py index a8806b336b..a33115b105 100644 --- a/frappe/contacts/doctype/address_template/address_template.py +++ b/frappe/contacts/doctype/address_template/address_template.py @@ -4,52 +4,36 @@ import frappe from frappe import _ from frappe.model.document import Document -from frappe.utils import cint from frappe.utils.jinja import validate_template class AddressTemplate(Document): def validate(self): + validate_template(self.template) + if not self.template: self.template = get_default_address_template() - self.defaults = frappe.db.get_values( - "Address Template", {"is_default": 1, "name": ("!=", self.name)} - ) - if not self.is_default: - if not self.defaults: - self.is_default = 1 - if cint(frappe.db.get_single_value("System Settings", "setup_complete")): - frappe.msgprint(_("Setting this Address Template as default as there is no other default")) - - validate_template(self.template) + if not self.is_default and not self._get_previous_default(): + self.is_default = 1 + if frappe.db.get_single_value("System Settings", "setup_complete"): + frappe.msgprint(_("Setting this Address Template as default as there is no other default")) def on_update(self): - if self.is_default and self.defaults: - for d in self.defaults: - frappe.db.set_value("Address Template", d[0], "is_default", 0) + if self.is_default and (previous_default := self._get_previous_default()): + frappe.db.set_value("Address Template", previous_default, "is_default", 0) def on_trash(self): if self.is_default: frappe.throw(_("Default Address Template cannot be deleted")) + def _get_previous_default(self) -> str | None: + return frappe.db.get_value("Address Template", {"is_default": 1, "name": ("!=", self.name)}) + @frappe.whitelist() -def get_default_address_template(): - """Get default address template (translated)""" - return ( - """{{ address_line1 }}
{% if address_line2 %}{{ address_line2 }}
{% endif -%}\ -{{ city }}
-{% if state %}{{ state }}
{% endif -%} -{% if pincode %}{{ pincode }}
{% endif -%} -{{ country }}
-{% if phone %}""" - + _("Phone") - + """: {{ phone }}
{% endif -%} -{% if fax %}""" - + _("Fax") - + """: {{ fax }}
{% endif -%} -{% if email_id %}""" - + _("Email") - + """: {{ email_id }}
{% endif -%}""" - ) +def get_default_address_template() -> str: + """Return the default address template.""" + from pathlib import Path + + return (Path(__file__).parent / "address_template.jinja").read_text() diff --git a/frappe/contacts/doctype/address_template/test_address_template.py b/frappe/contacts/doctype/address_template/test_address_template.py index 8045313c69..c3c5b544d6 100644 --- a/frappe/contacts/doctype/address_template/test_address_template.py +++ b/frappe/contacts/doctype/address_template/test_address_template.py @@ -1,40 +1,39 @@ # Copyright (c) 2015, Frappe Technologies and Contributors # License: MIT. See LICENSE -import unittest - import frappe +from frappe.contacts.doctype.address_template.address_template import get_default_address_template +from frappe.tests.utils import FrappeTestCase +from frappe.utils.jinja import validate_template -class TestAddressTemplate(unittest.TestCase): - def setUp(self): - self.make_default_address_template() +class TestAddressTemplate(FrappeTestCase): + def setUp(self) -> None: + frappe.db.delete("Address Template", {"country": "India"}) + frappe.db.delete("Address Template", {"country": "Brazil"}) + + def test_default_address_template(self): + validate_template(get_default_address_template()) def test_default_is_unset(self): - a = frappe.get_doc("Address Template", "India") - a.is_default = 1 - a.save() + frappe.get_doc({"doctype": "Address Template", "country": "India", "is_default": 1}).insert() - b = frappe.get_doc("Address Template", "Brazil") - b.is_default = 1 - b.save() + self.assertEqual(frappe.db.get_value("Address Template", "India", "is_default"), 1) + + frappe.get_doc({"doctype": "Address Template", "country": "Brazil", "is_default": 1}).insert() self.assertEqual(frappe.db.get_value("Address Template", "India", "is_default"), 0) + self.assertEqual(frappe.db.get_value("Address Template", "Brazil", "is_default"), 1) - def tearDown(self): - a = frappe.get_doc("Address Template", "India") - a.is_default = 1 - a.save() + def test_delete_address_template(self): + india = frappe.get_doc( + {"doctype": "Address Template", "country": "India", "is_default": 0} + ).insert() - @classmethod - def make_default_address_template(self): - template = """{{ address_line1 }}
{% if address_line2 %}{{ address_line2 }}
{% endif -%}{{ city }}
{% if state %}{{ state }}
{% endif -%}{% if pincode %}{{ pincode }}
{% endif -%}{{ country }}
{% if phone %}Phone: {{ phone }}
{% endif -%}{% if fax %}Fax: {{ fax }}
{% endif -%}{% if email_id %}Email: {{ email_id }}
{% endif -%}""" + brazil = frappe.get_doc( + {"doctype": "Address Template", "country": "Brazil", "is_default": 1} + ).insert() - if not frappe.db.exists("Address Template", "India"): - frappe.get_doc( - {"doctype": "Address Template", "country": "India", "is_default": 1, "template": template} - ).insert() + india.reload() # might have been modified by the second template + india.delete() # should not raise an error - if not frappe.db.exists("Address Template", "Brazil"): - frappe.get_doc( - {"doctype": "Address Template", "country": "Brazil", "template": template} - ).insert() + self.assertRaises(frappe.ValidationError, brazil.delete) diff --git a/frappe/contacts/doctype/contact/contact.js b/frappe/contacts/doctype/contact/contact.js index fae6e6515e..d4ae9379fa 100644 --- a/frappe/contacts/doctype/contact/contact.js +++ b/frappe/contacts/doctype/contact/contact.js @@ -5,49 +5,52 @@ frappe.ui.form.on("Contact", { onload(frm) { frm.email_field = "email_id"; }, - refresh: function(frm) { - if(frm.doc.__islocal) { + refresh: function (frm) { + if (frm.doc.__islocal) { const last_doc = frappe.contacts.get_last_doc(frm); - if(frappe.dynamic_link && frappe.dynamic_link.doc - && frappe.dynamic_link.doc.name == last_doc.docname) { - frm.set_value('links', ''); - frm.add_child('links', { + if ( + frappe.dynamic_link && + frappe.dynamic_link.doc && + frappe.dynamic_link.doc.name == last_doc.docname + ) { + frm.set_value("links", ""); + frm.add_child("links", { link_doctype: frappe.dynamic_link.doctype, - link_name: frappe.dynamic_link.doc[frappe.dynamic_link.fieldname] + link_name: frappe.dynamic_link.doc[frappe.dynamic_link.fieldname], }); } } - if(!frm.doc.user && !frm.is_new() && frm.perm[0].write) { - frm.add_custom_button(__("Invite as User"), function() { + if (!frm.doc.user && !frm.is_new() && frm.perm[0].write) { + frm.add_custom_button(__("Invite as User"), function () { return frappe.call({ method: "frappe.contacts.doctype.contact.contact.invite_user", args: { - contact: frm.doc.name + contact: frm.doc.name, }, - callback: function(r) { + callback: function (r) { frm.set_value("user", r.message); - } + }, }); }); } - frm.set_query('link_doctype', "links", function() { + frm.set_query("link_doctype", "links", function () { return { query: "frappe.contacts.address_and_contact.filter_dynamic_link_doctypes", filters: { fieldtype: "HTML", fieldname: "contact_html", - } - } + }, + }; }); frm.refresh_field("links"); let numbers = frm.doc.phone_nos; if (numbers && numbers.length && frappe.phone_call.handler) { - frm.add_custom_button(__('Call'), () => { + frm.add_custom_button(__("Call"), () => { numbers = frm.doc.phone_nos .sort((prev, next) => next.is_primary_mobile_no - prev.is_primary_mobile_no) - .map(d => d.phone); + .map((d) => d.phone); frappe.phone_call.handler(numbers); }); } @@ -55,73 +58,94 @@ frappe.ui.form.on("Contact", { if (frm.doc.links) { frappe.call({ method: "frappe.contacts.doctype.contact.contact.address_query", - args: {links: frm.doc.links}, - callback: function(r) { + args: { links: frm.doc.links }, + callback: function (r) { if (r && r.message) { frm.set_query("address", function () { return { filters: { name: ["in", r.message], - } - } + }, + }; }); } - } + }, }); for (let i in frm.doc.links) { let link = frm.doc.links[i]; - frm.add_custom_button(__("{0}: {1}", [__(link.link_doctype), __(link.link_name)]), function() { - frappe.set_route("Form", link.link_doctype, link.link_name); - }, __("Links")); + frm.add_custom_button( + __("{0}: {1}", [__(link.link_doctype), __(link.link_name)]), + function () { + frappe.set_route("Form", link.link_doctype, link.link_name); + }, + __("Links") + ); } } }, - validate: function(frm) { + validate: function (frm) { // clear linked customer / supplier / sales partner on saving... - if(frm.doc.links) { - frm.doc.links.forEach(function(d) { + if (frm.doc.links) { + frm.doc.links.forEach(function (d) { frappe.model.remove_from_locals(d.link_doctype, d.link_name); }); } }, - after_save: function(frm) { + after_save: function (frm) { frappe.run_serially([ () => frappe.timeout(1), () => { const last_doc = frappe.contacts.get_last_doc(frm); - if (frappe.dynamic_link && frappe.dynamic_link.doc && frappe.dynamic_link.doc.name == last_doc.docname) { + if ( + frappe.dynamic_link && + frappe.dynamic_link.doc && + frappe.dynamic_link.doc.name == last_doc.docname + ) { for (let i in frm.doc.links) { let link = frm.doc.links[i]; - if (last_doc.doctype == link.link_doctype && last_doc.docname == link.link_name) { - frappe.set_route('Form', last_doc.doctype, last_doc.docname); + if ( + last_doc.doctype == link.link_doctype && + last_doc.docname == link.link_name + ) { + frappe.set_route("Form", last_doc.doctype, last_doc.docname); } } } - } + }, ]); }, - sync_with_google_contacts: function(frm) { + sync_with_google_contacts: function (frm) { if (frm.doc.sync_with_google_contacts) { - frappe.db.get_value("Google Contacts", {"email_id": frappe.session.user}, "name", (r) => { - if (r && r.name) { - frm.set_value("google_contacts", r.name); + frappe.db.get_value( + "Google Contacts", + { email_id: frappe.session.user }, + "name", + (r) => { + if (r && r.name) { + frm.set_value("google_contacts", r.name); + } } - }) + ); } - } + }, }); frappe.ui.form.on("Dynamic Link", { - link_name:function(frm, cdt, cdn){ + link_name: function (frm, cdt, cdn) { var child = locals[cdt][cdn]; - if(child.link_name) { + if (child.link_name) { frappe.model.with_doctype(child.link_doctype, function () { - var title_field = frappe.get_meta(child.link_doctype).title_field || "name" - frappe.model.get_value(child.link_doctype, child.link_name, title_field, function (r) { - frappe.model.set_value(cdt, cdn, "link_title", r[title_field]) - }) - }) + var title_field = frappe.get_meta(child.link_doctype).title_field || "name"; + frappe.model.get_value( + child.link_doctype, + child.link_name, + title_field, + function (r) { + frappe.model.set_value(cdt, cdn, "link_title", r[title_field]); + } + ); + }); } - } -}) + }, +}); diff --git a/frappe/contacts/doctype/contact/contact.json b/frappe/contacts/doctype/contact/contact.json index 696cd61d6c..3090746657 100644 --- a/frappe/contacts/doctype/contact/contact.json +++ b/frappe/contacts/doctype/contact/contact.json @@ -12,6 +12,7 @@ "first_name", "middle_name", "last_name", + "full_name", "email_id", "user", "address", @@ -52,8 +53,7 @@ "in_global_search": 1, "label": "First Name", "oldfieldname": "first_name", - "oldfieldtype": "Data", - "reqd": 1 + "oldfieldtype": "Data" }, { "bold": 1, @@ -243,6 +243,13 @@ "fieldname": "company_name", "fieldtype": "Data", "label": "Company Name" + }, + { + "fieldname": "full_name", + "fieldtype": "Data", + "hidden": 1, + "label": "Full Name", + "read_only": 1 } ], "icon": "fa fa-user", @@ -250,11 +257,12 @@ "image_field": "image", "index_web_pages_for_search": 1, "links": [], - "modified": "2020-08-27 14:12:09.906719", + "modified": "2022-10-27 10:40:50.097481", "modified_by": "Administrator", "module": "Contacts", "name": "Contact", "name_case": "Title Case", + "naming_rule": "By script", "owner": "Administrator", "permissions": [ { @@ -267,7 +275,6 @@ "read": 1, "report": 1, "role": "System Manager", - "set_user_permissions": 1, "share": 1, "write": 1 }, @@ -380,6 +387,9 @@ "role": "All" } ], + "show_title_field_in_link": 1, "sort_field": "modified", - "sort_order": "ASC" -} \ No newline at end of file + "sort_order": "ASC", + "states": [], + "title_field": "full_name" +} diff --git a/frappe/contacts/doctype/contact/contact.py b/frappe/contacts/doctype/contact/contact.py index a17f46216b..e7d250148b 100644 --- a/frappe/contacts/doctype/contact/contact.py +++ b/frappe/contacts/doctype/contact/contact.py @@ -11,10 +11,7 @@ from frappe.utils import cstr, has_gravatar class Contact(Document): def autoname(self): - # concat first and last name - self.name = " ".join( - filter(None, [cstr(self.get(f)).strip() for f in ["first_name", "last_name"]]) - ) + self.name = self._get_full_name() if frappe.db.exists("Contact", self.name): self.name = append_number_if_name_exists("Contact", self.name) @@ -25,6 +22,7 @@ class Contact(Document): break def validate(self): + self.full_name = self._get_full_name() self.set_primary_email() self.set_primary("phone") self.set_primary("mobile_no") @@ -128,11 +126,14 @@ class Contact(Document): if not primary_number_exists: setattr(self, fieldname, "") + def _get_full_name(self) -> str: + return get_full_name(self.first_name, self.middle_name, self.last_name, self.company_name) + def get_default_contact(doctype, name): """Returns default contact for the given doctype, name""" out = frappe.db.sql( - '''select parent, + """select parent, IFNULL((select is_primary_contact from tabContact c where c.name = dl.parent), 0) as is_primary_contact from @@ -140,7 +141,7 @@ def get_default_contact(doctype, name): where dl.link_doctype=%s and dl.link_name=%s and - dl.parenttype = "Contact"''', + dl.parenttype = 'Contact' """, (doctype, name), as_dict=True, ) @@ -210,8 +211,9 @@ def update_contact(doc, method): def contact_query(doctype, txt, searchfield, start, page_len, filters): from frappe.desk.reportview import get_match_cond + doctype = "Contact" if ( - not frappe.get_meta("Contact").get_field(searchfield) + not frappe.get_meta(doctype).get_field(searchfield) and searchfield not in frappe.db.DEFAULT_COLUMNS ): return [] @@ -221,7 +223,7 @@ def contact_query(doctype, txt, searchfield, start, page_len, filters): return frappe.db.sql( """select - `tabContact`.name, `tabContact`.first_name, `tabContact`.last_name + `tabContact`.name, `tabContact`.full_name, `tabContact`.company_name from `tabContact`, `tabDynamic Link` where @@ -232,8 +234,8 @@ def contact_query(doctype, txt, searchfield, start, page_len, filters): `tabContact`.`{key}` like %(txt)s {mcond} order by - if(locate(%(_txt)s, `tabContact`.name), locate(%(_txt)s, `tabContact`.name), 99999), - `tabContact`.idx desc, `tabContact`.name + if(locate(%(_txt)s, `tabContact`.full_name), locate(%(_txt)s, `tabContact`.company_name), 99999), + `tabContact`.idx desc, `tabContact`.full_name limit %(start)s, %(page_len)s """.format( mcond=get_match_cond(doctype), key=searchfield ), @@ -326,3 +328,16 @@ def get_contacts_linked_from(doctype, docname, fields=None): return [] return frappe.get_list("Contact", fields=fields, filters={"name": ("in", contact_names)}) + + +def get_full_name( + first: str | None = None, + middle: str | None = None, + last: str | None = None, + company: str | None = None, +) -> str: + full_name = " ".join(filter(None, [cstr(f).strip() for f in [first, middle, last]])) + if not full_name and company: + full_name = company + + return full_name diff --git a/frappe/contacts/doctype/contact/contact_list.js b/frappe/contacts/doctype/contact/contact_list.js index a93b3f0d73..2b3cd8a062 100644 --- a/frappe/contacts/doctype/contact/contact_list.js +++ b/frappe/contacts/doctype/contact/contact_list.js @@ -1,3 +1,3 @@ -frappe.listview_settings['Contact'] = { +frappe.listview_settings["Contact"] = { add_fields: ["image"], -}; \ No newline at end of file +}; diff --git a/frappe/contacts/doctype/contact/test_contact.py b/frappe/contacts/doctype/contact/test_contact.py index bf0d1037db..e91e132258 100644 --- a/frappe/contacts/doctype/contact/test_contact.py +++ b/frappe/contacts/doctype/contact/test_contact.py @@ -1,13 +1,13 @@ # Copyright (c) 2017, Frappe Technologies and Contributors # License: MIT. See LICENSE -import unittest - import frappe +from frappe.contacts.doctype.contact.contact import get_full_name +from frappe.tests.utils import FrappeTestCase test_dependencies = ["Contact", "Salutation"] -class TestContact(unittest.TestCase): +class TestContact(FrappeTestCase): def test_check_default_email(self): emails = [ {"email": "test1@example.com", "is_primary": 0}, @@ -32,6 +32,18 @@ class TestContact(unittest.TestCase): self.assertEqual(contact.phone, "+91 0000000002") self.assertEqual(contact.mobile_no, "+91 0000000003") + def test_get_full_name(self): + self.assertEqual(get_full_name(first="John"), "John") + self.assertEqual(get_full_name(last="Doe"), "Doe") + self.assertEqual(get_full_name(company="Doe Pvt Ltd"), "Doe Pvt Ltd") + self.assertEqual(get_full_name(first="John", last="Doe"), "John Doe") + self.assertEqual(get_full_name(first="John", middle="Jane"), "John Jane") + self.assertEqual(get_full_name(first="John", last="Doe", company="Doe Pvt Ltd"), "John Doe") + self.assertEqual( + get_full_name(first="John", middle="Jane", last="Doe", company="Doe Pvt Ltd"), + "John Jane Doe", + ) + def create_contact(name, salutation, emails=None, phones=None, save=True): doc = frappe.get_doc( diff --git a/frappe/contacts/doctype/gender/gender.js b/frappe/contacts/doctype/gender/gender.js index e2fd2f18eb..3b34b1584e 100644 --- a/frappe/contacts/doctype/gender/gender.js +++ b/frappe/contacts/doctype/gender/gender.js @@ -1,8 +1,6 @@ // Copyright (c) 2017, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Gender', { - refresh: function() { - - } +frappe.ui.form.on("Gender", { + refresh: function () {}, }); diff --git a/frappe/contacts/doctype/gender/gender.json b/frappe/contacts/doctype/gender/gender.json index 86a066cf0f..34e1ddaee6 100644 --- a/frappe/contacts/doctype/gender/gender.json +++ b/frappe/contacts/doctype/gender/gender.json @@ -1,113 +1,48 @@ { - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "autoname": "field:gender", - "beta": 0, - "creation": "2017-04-10 12:11:36.526508", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", + "actions": [], + "autoname": "field:gender", + "creation": "2017-04-10 12:11:36.526508", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "gender" + ], "fields": [ { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "gender", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Gender", - "length": 0, - "no_copy": 0, - "options": "", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "fieldname": "gender", + "fieldtype": "Data", + "label": "Gender", + "unique": 1 } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2017-04-10 12:17:04.848338", - "modified_by": "Administrator", - "module": "Contacts", - "name": "Gender", - "name_case": "", - "owner": "Administrator", + ], + "links": [], + "modified": "2022-08-05 18:33:28.043370", + "modified_by": "Administrator", + "module": "Contacts", + "name": "Gender", + "owner": "Administrator", "permissions": [ { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "set_user_permissions": 0, - "share": 1, - "submit": 0, + "create": 1, + "delete": 1, + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1, "write": 1 - }, + }, { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 0, - "delete": 0, - "email": 0, - "export": 0, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 0, - "read": 1, - "report": 0, - "role": "All", - "set_user_permissions": 0, - "share": 0, - "submit": 0, - "write": 0 + "read": 1, + "role": "All" } - ], - "quick_entry": 0, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1, - "track_seen": 0 + ], + "sort_field": "modified", + "sort_order": "DESC", + "states": [], + "track_changes": 1, + "translated_doctype": 1 } \ No newline at end of file diff --git a/frappe/contacts/doctype/gender/test_gender.py b/frappe/contacts/doctype/gender/test_gender.py index c8df3b566d..1b428521b7 100644 --- a/frappe/contacts/doctype/gender/test_gender.py +++ b/frappe/contacts/doctype/gender/test_gender.py @@ -1,7 +1,7 @@ # Copyright (c) 2017, Frappe Technologies and Contributors # License: MIT. See LICENSE -import unittest +from frappe.tests.utils import FrappeTestCase -class TestGender(unittest.TestCase): +class TestGender(FrappeTestCase): pass diff --git a/frappe/contacts/doctype/salutation/salutation.js b/frappe/contacts/doctype/salutation/salutation.js index 856b72e04c..e7da1f389b 100644 --- a/frappe/contacts/doctype/salutation/salutation.js +++ b/frappe/contacts/doctype/salutation/salutation.js @@ -1,8 +1,6 @@ // Copyright (c) 2017, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Salutation', { - refresh: function() { - - } +frappe.ui.form.on("Salutation", { + refresh: function () {}, }); diff --git a/frappe/contacts/doctype/salutation/salutation.json b/frappe/contacts/doctype/salutation/salutation.json index 579f176aa7..98ed082e69 100644 --- a/frappe/contacts/doctype/salutation/salutation.json +++ b/frappe/contacts/doctype/salutation/salutation.json @@ -1,132 +1,61 @@ { - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 1, - "autoname": "field:salutation", - "beta": 0, - "creation": "2017-04-10 12:17:58.071915", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", + "actions": [], + "allow_rename": 1, + "autoname": "field:salutation", + "creation": "2017-04-10 12:17:58.071915", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "salutation" + ], "fields": [ { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "salutation", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Salutation", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "fieldname": "salutation", + "fieldtype": "Data", + "label": "Salutation", + "unique": 1 } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2020-09-14 12:55:18.855578", - "modified_by": "Administrator", - "module": "Contacts", - "name": "Salutation", - "name_case": "", - "owner": "Administrator", + ], + "links": [], + "modified": "2022-08-05 18:33:28.196387", + "modified_by": "Administrator", + "module": "Contacts", + "name": "Salutation", + "owner": "Administrator", "permissions": [ { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "set_user_permissions": 0, - "share": 1, - "submit": 0, + "create": 1, + "delete": 1, + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1, "write": 1 - }, + }, { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 0, - "delete": 0, - "email": 0, - "export": 0, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 0, - "read": 1, - "report": 0, - "role": "All", - "set_user_permissions": 0, - "share": 0, - "submit": 0, - "write": 0 - }, + "read": 1, + "role": "All" + }, { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 1, - "role": "Administrator", - "set_user_permissions": 0, - "share": 1, - "submit": 0, + "create": 1, + "delete": 1, + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "Administrator", + "share": 1, "write": 1 } - ], - "quick_entry": 0, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1, - "track_seen": 0 -} + ], + "sort_field": "modified", + "sort_order": "DESC", + "states": [], + "track_changes": 1, + "translated_doctype": 1 +} \ No newline at end of file diff --git a/frappe/contacts/doctype/salutation/test_salutation.py b/frappe/contacts/doctype/salutation/test_salutation.py index 2c35e5bd2b..a1d9e044a0 100644 --- a/frappe/contacts/doctype/salutation/test_salutation.py +++ b/frappe/contacts/doctype/salutation/test_salutation.py @@ -1,7 +1,7 @@ # Copyright (c) 2017, Frappe Technologies and Contributors # License: MIT. See LICENSE -import unittest +from frappe.tests.utils import FrappeTestCase -class TestSalutation(unittest.TestCase): +class TestSalutation(FrappeTestCase): pass diff --git a/frappe/contacts/report/addresses_and_contacts/addresses_and_contacts.js b/frappe/contacts/report/addresses_and_contacts/addresses_and_contacts.js index 10137e80d5..9870ee611b 100644 --- a/frappe/contacts/report/addresses_and_contacts/addresses_and_contacts.js +++ b/frappe/contacts/report/addresses_and_contacts/addresses_and_contacts.js @@ -2,32 +2,32 @@ // For license information, please see license.txt frappe.query_reports["Addresses And Contacts"] = { - "filters": [ + filters: [ { - "reqd": 1, - "fieldname":"reference_doctype", - "label": __("Entity Type"), - "fieldtype": "Link", - "options": "DocType", - "get_query": function() { + reqd: 1, + fieldname: "reference_doctype", + label: __("Entity Type"), + fieldtype: "Link", + options: "DocType", + get_query: function () { return { - "filters": { - "name": ["in", "Contact, Address"], - } - } - } + filters: { + name: ["in", "Contact, Address"], + }, + }; + }, }, { - "fieldname":"reference_name", - "label": __("Entity Name"), - "fieldtype": "Dynamic Link", - "get_options": function() { - let reference_doctype = frappe.query_report.get_filter_value('reference_doctype'); - if(!reference_doctype) { + fieldname: "reference_name", + label: __("Entity Name"), + fieldtype: "Dynamic Link", + get_options: function () { + let reference_doctype = frappe.query_report.get_filter_value("reference_doctype"); + if (!reference_doctype) { frappe.throw(__("Please select Entity Type first")); } return reference_doctype; - } - } - ] -} + }, + }, + ], +}; diff --git a/frappe/contacts/report/addresses_and_contacts/test_addresses_and_contacts.py b/frappe/contacts/report/addresses_and_contacts/test_addresses_and_contacts.py index 2ad8bfaba3..74c797ca65 100644 --- a/frappe/contacts/report/addresses_and_contacts/test_addresses_and_contacts.py +++ b/frappe/contacts/report/addresses_and_contacts/test_addresses_and_contacts.py @@ -1,8 +1,7 @@ -import unittest - import frappe import frappe.defaults from frappe.contacts.report.addresses_and_contacts.addresses_and_contacts import get_data +from frappe.tests.utils import FrappeTestCase def get_custom_linked_doctype(): @@ -87,7 +86,7 @@ def create_linked_contact(link_list, address): frappe.flags.test_contact_created = True -class TestAddressesAndContacts(unittest.TestCase): +class TestAddressesAndContacts(FrappeTestCase): def test_get_data(self): linked_docs = [get_custom_doc_for_address_and_contacts()] links_list = [item.name for item in linked_docs] @@ -113,6 +112,3 @@ class TestAddressesAndContacts(unittest.TestCase): 1, ] self.assertListEqual(test_item, report_data[idx]) - - def tearDown(self): - frappe.db.rollback() diff --git a/frappe/core/api/file.py b/frappe/core/api/file.py index ec305aff4f..e3e6a9de08 100644 --- a/frappe/core/api/file.py +++ b/frappe/core/api/file.py @@ -13,7 +13,7 @@ def unzip_file(name: str): @frappe.whitelist() -def get_attached_images(doctype: str, names: list[str]) -> frappe._dict: +def get_attached_images(doctype: str, names: list[str] | str) -> frappe._dict: """get list of image urls attached in form returns {name: ['image.jpg', 'image.png']}""" @@ -40,9 +40,6 @@ def get_attached_images(doctype: str, names: list[str]) -> frappe._dict: @frappe.whitelist() def get_files_in_folder(folder: str, start: int = 0, page_length: int = 20) -> dict: - start = cint(start) - page_length = cint(page_length) - attachment_folder = frappe.db.get_value( "File", "Home/Attachments", @@ -101,10 +98,11 @@ def create_new_folder(file_name: str, folder: str) -> File: @frappe.whitelist() -def move_file(file_list: list[File], new_parent: str, old_parent: str) -> None: +def move_file(file_list: list[File | dict] | str, new_parent: str, old_parent: str) -> None: if isinstance(file_list, str): file_list = json.loads(file_list) + # will check for permission on each file & update parent for file_obj in file_list: setup_folder_path(file_obj.get("name"), new_parent) diff --git a/frappe/core/doctype/access_log/access_log.js b/frappe/core/doctype/access_log/access_log.js index d36d10768b..94f1bf732d 100644 --- a/frappe/core/doctype/access_log/access_log.js +++ b/frappe/core/doctype/access_log/access_log.js @@ -1,17 +1,17 @@ // Copyright (c) 2019, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Access Log', { +frappe.ui.form.on("Access Log", { show_document: function (frm) { - frappe.set_route('Form', frm.doc.export_from, frm.doc.reference_document); + frappe.set_route("Form", frm.doc.export_from, frm.doc.reference_document); }, show_report: function (frm) { - if (frm.doc.report_name.includes('/')) { + if (frm.doc.report_name.includes("/")) { frappe.set_route(frm.doc.report_name); } else { let filters = frm.doc.filters ? JSON.parse(frm.doc.filters) : {}; - frappe.set_route('query-report', frm.doc.report_name, filters); + frappe.set_route("query-report", frm.doc.report_name, filters); } - } + }, }); diff --git a/frappe/core/doctype/access_log/access_log.py b/frappe/core/doctype/access_log/access_log.py index b7a6d77206..c194f5d603 100644 --- a/frappe/core/doctype/access_log/access_log.py +++ b/frappe/core/doctype/access_log/access_log.py @@ -8,7 +8,13 @@ from frappe.utils import cstr class AccessLog(Document): - pass + @staticmethod + def clear_old_logs(days=30): + from frappe.query_builder import Interval + from frappe.query_builder.functions import Now + + table = frappe.qb.DocType("Access Log") + frappe.db.delete(table, filters=(table.modified < (Now() - Interval(days=days)))) @frappe.whitelist() @@ -35,7 +41,11 @@ def make_access_log( @frappe.write_only() -@retry(stop=stop_after_attempt(3), retry=retry_if_exception_type(frappe.DuplicateEntryError)) +@retry( + stop=stop_after_attempt(3), + retry=retry_if_exception_type(frappe.DuplicateEntryError), + reraise=True, +) def _make_access_log( doctype=None, document=None, diff --git a/frappe/core/doctype/access_log/access_log_list.js b/frappe/core/doctype/access_log/access_log_list.js new file mode 100644 index 0000000000..dab5f083cb --- /dev/null +++ b/frappe/core/doctype/access_log/access_log_list.js @@ -0,0 +1,7 @@ +frappe.listview_settings["Access Log"] = { + onload: function (list_view) { + frappe.require("logtypes.bundle.js", () => { + frappe.utils.logtypes.show_log_retention_message(list_view.doctype); + }); + }, +}; diff --git a/frappe/core/doctype/access_log/test_access_log.py b/frappe/core/doctype/access_log/test_access_log.py index ee0422e11a..b3432d60bf 100644 --- a/frappe/core/doctype/access_log/test_access_log.py +++ b/frappe/core/doctype/access_log/test_access_log.py @@ -4,9 +4,6 @@ import base64 import os -# imports - standard imports -import unittest - # imports - third party imports import requests @@ -15,10 +12,13 @@ import frappe from frappe.core.doctype.access_log.access_log import make_access_log from frappe.core.doctype.data_import.data_import import export_csv from frappe.core.doctype.user.user import generate_keys + +# imports - standard imports +from frappe.tests.utils import FrappeTestCase from frappe.utils import cstr, get_site_url -class TestAccessLog(unittest.TestCase): +class TestAccessLog(FrappeTestCase): def setUp(self): # generate keys for current user to send requests for the following tests generate_keys(frappe.session.user) diff --git a/frappe/core/doctype/activity_log/activity_log.js b/frappe/core/doctype/activity_log/activity_log.js index 97e49e4b34..7df644a86a 100644 --- a/frappe/core/doctype/activity_log/activity_log.js +++ b/frappe/core/doctype/activity_log/activity_log.js @@ -1,8 +1,6 @@ // Copyright (c) 2017, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Activity Log', { - refresh: function() { - - } +frappe.ui.form.on("Activity Log", { + refresh: function () {}, }); diff --git a/frappe/core/doctype/activity_log/activity_log.json b/frappe/core/doctype/activity_log/activity_log.json index ad12246a95..910baceb5e 100644 --- a/frappe/core/doctype/activity_log/activity_log.json +++ b/frappe/core/doctype/activity_log/activity_log.json @@ -102,8 +102,7 @@ "fetch_from": "reference_name.owner", "fieldname": "reference_owner", "fieldtype": "Read Only", - "label": "Reference Owner", - "search_index": 1 + "label": "Reference Owner" }, { "fieldname": "column_break_14", @@ -154,7 +153,7 @@ "icon": "fa fa-comment", "index_web_pages_for_search": 1, "links": [], - "modified": "2021-10-25 11:43:57.504565", + "modified": "2022-09-13 15:19:42.474114", "modified_by": "Administrator", "module": "Core", "name": "Activity Log", @@ -181,6 +180,7 @@ "search_fields": "subject", "sort_field": "modified", "sort_order": "DESC", + "states": [], "title_field": "subject", "track_seen": 1 -} +} \ No newline at end of file diff --git a/frappe/core/doctype/activity_log/activity_log.py b/frappe/core/doctype/activity_log/activity_log.py index 468b7f4473..e819683d0a 100644 --- a/frappe/core/doctype/activity_log/activity_log.py +++ b/frappe/core/doctype/activity_log/activity_log.py @@ -37,7 +37,6 @@ def on_doctype_update(): """Add indexes in `tabActivity Log`""" frappe.db.add_index("Activity Log", ["reference_doctype", "reference_name"]) frappe.db.add_index("Activity Log", ["timeline_doctype", "timeline_name"]) - frappe.db.add_index("Activity Log", ["link_doctype", "link_name"]) def add_authentication_log(subject, user, operation="Login", status="Success"): diff --git a/frappe/core/doctype/activity_log/activity_log_list.js b/frappe/core/doctype/activity_log/activity_log_list.js index e3a75a1941..53758a7b62 100644 --- a/frappe/core/doctype/activity_log/activity_log_list.js +++ b/frappe/core/doctype/activity_log/activity_log_list.js @@ -1,13 +1,12 @@ -frappe.listview_settings['Activity Log'] = { - get_indicator: function(doc) { - if(doc.operation == "Login" && doc.status == "Success") - return [__(doc.status), "green"]; - else if(doc.operation == "Login" && doc.status == "Failed") +frappe.listview_settings["Activity Log"] = { + get_indicator: function (doc) { + if (doc.operation == "Login" && doc.status == "Success") return [__(doc.status), "green"]; + else if (doc.operation == "Login" && doc.status == "Failed") return [__(doc.status), "red"]; }, - onload: function(listview) { + onload: function (listview) { frappe.require("logtypes.bundle.js", () => { frappe.utils.logtypes.show_log_retention_message(cur_list.doctype); - }) + }); }, }; diff --git a/frappe/core/doctype/activity_log/feed.py b/frappe/core/doctype/activity_log/feed.py index eb040927e1..b2db5dfe91 100644 --- a/frappe/core/doctype/activity_log/feed.py +++ b/frappe/core/doctype/activity_log/feed.py @@ -8,47 +8,6 @@ from frappe.core.doctype.activity_log.activity_log import add_authentication_log from frappe.utils import get_fullname -def update_feed(doc, method=None): - if frappe.flags.in_patch or frappe.flags.in_install or frappe.flags.in_import: - return - - if doc._action != "save" or doc.flags.ignore_feed: - return - - if doc.doctype == "Activity Log" or doc.meta.issingle: - return - - if hasattr(doc, "get_feed"): - feed = doc.get_feed() - - if feed: - if isinstance(feed, str): - feed = {"subject": feed} - - feed = frappe._dict(feed) - doctype = feed.doctype or doc.doctype - name = feed.name or doc.name - - # delete earlier feed - frappe.db.delete( - "Activity Log", - {"reference_doctype": doctype, "reference_name": name, "link_doctype": feed.link_doctype}, - ) - - frappe.get_doc( - { - "doctype": "Activity Log", - "reference_doctype": doctype, - "reference_name": name, - "subject": feed.subject, - "full_name": get_fullname(doc.owner), - "reference_owner": frappe.db.get_value(doctype, name, "owner"), - "link_doctype": feed.link_doctype, - "link_name": feed.link_name, - } - ).insert(ignore_permissions=True) - - def login_feed(login_manager): if login_manager.user != "Guest": subject = _("{0} logged in").format(get_fullname(login_manager.user)) @@ -59,44 +18,3 @@ def logout_feed(user, reason): if user and user != "Guest": subject = _("{0} logged out: {1}").format(get_fullname(user), frappe.bold(reason)) add_authentication_log(subject, user, operation="Logout") - - -def get_feed_match_conditions(user=None, doctype="Comment"): - if not user: - user = frappe.session.user - - conditions = [ - "`tab{doctype}`.owner={user} or `tab{doctype}`.reference_owner={user}".format( - user=frappe.db.escape(user), doctype=doctype - ) - ] - - user_permissions = frappe.permissions.get_user_permissions(user) - can_read = frappe.get_user().get_can_read() - - can_read_doctypes = [f"'{dt}'" for dt in list(set(can_read) - set(list(user_permissions)))] - - if can_read_doctypes: - conditions += [ - """(`tab{doctype}`.reference_doctype is null - or `tab{doctype}`.reference_doctype = '' - or `tab{doctype}`.reference_doctype - in ({values}))""".format( - doctype=doctype, values=", ".join(can_read_doctypes) - ) - ] - - if user_permissions: - can_read_docs = [] - for dt, obj in user_permissions.items(): - for n in obj: - can_read_docs.append("{}|{}".format(frappe.db.escape(dt), frappe.db.escape(n.get("doc", "")))) - - if can_read_docs: - conditions.append( - "concat_ws('|', `tab{doctype}`.reference_doctype, `tab{doctype}`.reference_name) in ({values})".format( - doctype=doctype, values=", ".join(can_read_docs) - ) - ) - - return "(" + " or ".join(conditions) + ")" diff --git a/frappe/core/doctype/activity_log/test_activity_log.py b/frappe/core/doctype/activity_log/test_activity_log.py index c362fca521..df3f113a85 100644 --- a/frappe/core/doctype/activity_log/test_activity_log.py +++ b/frappe/core/doctype/activity_log/test_activity_log.py @@ -1,24 +1,30 @@ # Copyright (c) 2015, Frappe Technologies and Contributors # License: MIT. See LICENSE import time -import unittest import frappe from frappe.auth import CookieManager, LoginManager +from frappe.tests.utils import FrappeTestCase -class TestActivityLog(unittest.TestCase): +class TestActivityLog(FrappeTestCase): def test_activity_log(self): # test user login log frappe.local.form_dict = frappe._dict( - {"cmd": "login", "sid": "Guest", "pwd": "admin", "usr": "Administrator"} + { + "cmd": "login", + "sid": "Guest", + "pwd": frappe.conf.admin_password or "admin", + "usr": "Administrator", + } ) frappe.local.cookie_manager = CookieManager() frappe.local.login_manager = LoginManager() auth_log = self.get_auth_log() + self.assertFalse(frappe.form_dict.pwd) self.assertEqual(auth_log.status, "Success") # test user logout log @@ -35,7 +41,7 @@ class TestActivityLog(unittest.TestCase): frappe.local.form_dict = frappe._dict() def get_auth_log(self, operation="Login"): - names = frappe.db.get_all( + names = frappe.get_all( "Activity Log", filters={ "user": "Administrator", diff --git a/frappe/core/doctype/block_module/block_module.json b/frappe/core/doctype/block_module/block_module.json index 64deff66ee..9711aaa001 100644 --- a/frappe/core/doctype/block_module/block_module.json +++ b/frappe/core/doctype/block_module/block_module.json @@ -1,71 +1,31 @@ { - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, - "creation": "2015-03-24 14:28:15.882903", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "Other", - "editable_grid": 1, + "actions": [], + "creation": "2015-03-24 14:28:15.882903", + "doctype": "DocType", + "document_type": "Other", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "module" + ], "fields": [ { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "module", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Module", - "length": 0, - "no_copy": 0, - "options": "", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "fieldname": "module", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Module", + "reqd": 1 } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 1, - "max_attachments": 0, - "modified": "2017-10-31 19:36:18.586834", - "modified_by": "Administrator", - "module": "Core", - "name": "Block Module", - "name_case": "", - "owner": "Administrator", - "permissions": [], - "quick_entry": 0, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 0, - "track_seen": 0 + ], + "istable": 1, + "links": [], + "modified": "2022-08-03 12:20:52.738977", + "modified_by": "Administrator", + "module": "Core", + "name": "Block Module", + "owner": "Administrator", + "permissions": [], + "sort_field": "modified", + "sort_order": "DESC", + "states": [] } \ No newline at end of file diff --git a/frappe/core/doctype/comment/comment.js b/frappe/core/doctype/comment/comment.js index a793f766cb..4d227f6f5f 100644 --- a/frappe/core/doctype/comment/comment.js +++ b/frappe/core/doctype/comment/comment.js @@ -1,8 +1,7 @@ // Copyright (c) 2019, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Comment', { +frappe.ui.form.on("Comment", { // refresh: function(frm) { - // } }); diff --git a/frappe/core/doctype/comment/comment.json b/frappe/core/doctype/comment/comment.json index fe465f46bd..9f27e7e7be 100644 --- a/frappe/core/doctype/comment/comment.json +++ b/frappe/core/doctype/comment/comment.json @@ -1,4 +1,5 @@ { + "actions": [], "creation": "2019-02-07 10:10:46.845678", "doctype": "DocType", "editable_grid": 1, @@ -17,7 +18,8 @@ "link_name", "reference_owner", "section_break_10", - "content" + "content", + "ip_address" ], "fields": [ { @@ -102,9 +104,16 @@ "ignore_xss_filter": 1, "in_list_view": 1, "label": "Content" + }, + { + "fieldname": "ip_address", + "fieldtype": "Data", + "hidden": 1, + "label": "IP Address" } ], - "modified": "2019-09-02 21:00:10.784787", + "links": [], + "modified": "2022-07-12 17:35:31.774137", "modified_by": "Administrator", "module": "Core", "name": "Comment", @@ -138,6 +147,7 @@ "quick_entry": 1, "sort_field": "modified", "sort_order": "DESC", + "states": [], "title_field": "comment_type", "track_changes": 1 } \ No newline at end of file diff --git a/frappe/core/doctype/comment/comment.py b/frappe/core/doctype/comment/comment.py index dab9cfbfe4..dff13e1170 100644 --- a/frappe/core/doctype/comment/comment.py +++ b/frappe/core/doctype/comment/comment.py @@ -3,23 +3,17 @@ import json import frappe -from frappe import _ -from frappe.core.doctype.user.user import extract_mentions from frappe.database.schema import add_column -from frappe.desk.doctype.notification_log.notification_log import ( - enqueue_create_notification, - get_title, - get_title_html, -) +from frappe.desk.notifications import notify_mentions from frappe.exceptions import ImplicitCommitError from frappe.model.document import Document -from frappe.utils import get_fullname +from frappe.model.utils import is_virtual_doctype from frappe.website.utils import clear_cache class Comment(Document): def after_insert(self): - self.notify_mentions() + notify_mentions(self.reference_doctype, self.reference_name, self.content) self.notify_change("add") def validate(self): @@ -50,8 +44,10 @@ class Comment(Document): return frappe.publish_realtime( - f"update_docinfo_for_{self.reference_doctype}_{self.reference_name}", + "docinfo_update", {"doc": self.as_dict(), "key": key, "action": action}, + doctype=self.reference_doctype, + docname=self.reference_name, after_commit=True, ) @@ -63,44 +59,9 @@ class Comment(Document): update_comments_in_parent(self.reference_doctype, self.reference_name, _comments) - def notify_mentions(self): - if self.reference_doctype and self.reference_name and self.content: - mentions = extract_mentions(self.content) - - if not mentions: - return - - sender_fullname = get_fullname(frappe.session.user) - title = get_title(self.reference_doctype, self.reference_name) - - recipients = [ - frappe.db.get_value( - "User", - {"enabled": 1, "name": name, "user_type": "System User", "allowed_in_mentions": 1}, - "email", - ) - for name in mentions - ] - - notification_message = _("""{0} mentioned you in a comment in {1} {2}""").format( - frappe.bold(sender_fullname), frappe.bold(self.reference_doctype), get_title_html(title) - ) - - notification_doc = { - "type": "Mention", - "document_type": self.reference_doctype, - "document_name": self.reference_name, - "subject": notification_message, - "from_user": frappe.session.user, - "email_content": self.content, - } - - enqueue_create_notification(recipients, notification_doc) - def on_doctype_update(): frappe.db.add_index("Comment", ["reference_doctype", "reference_name"]) - frappe.db.add_index("Comment", ["link_doctype", "link_name"]) def update_comment_in_doc(doc): @@ -152,7 +113,10 @@ def get_comments_from_parent(doc): `_comments` """ try: - _comments = frappe.db.get_value(doc.reference_doctype, doc.reference_name, "_comments") or "[]" + if is_virtual_doctype(doc.reference_doctype): + _comments = "[]" + else: + _comments = frappe.db.get_value(doc.reference_doctype, doc.reference_name, "_comments") or "[]" except Exception as e: if frappe.db.is_missing_table_or_column(e): @@ -175,7 +139,7 @@ def update_comments_in_parent(reference_doctype, reference_name, _comments): not reference_doctype or not reference_name or frappe.db.get_value("DocType", reference_doctype, "issingle") - or frappe.db.get_value("DocType", reference_doctype, "is_virtual") + or is_virtual_doctype(reference_doctype) ): return @@ -197,13 +161,14 @@ def update_comments_in_parent(reference_doctype, reference_name, _comments): raise frappe.DataTooLongException else: - raise ImplicitCommitError - + raise else: - if not frappe.flags.in_patch: - reference_doc = frappe.get_doc(reference_doctype, reference_name) - if getattr(reference_doc, "route", None): - clear_cache(reference_doc.route) + if frappe.flags.in_patch: + return + + # Clear route cache + if route := frappe.get_cached_value(reference_doctype, reference_name, "route"): + clear_cache(route) def update_comments_in_parent_after_request(): diff --git a/frappe/core/doctype/comment/test_comment.py b/frappe/core/doctype/comment/test_comment.py index bedcea6e7e..ee2d473210 100644 --- a/frappe/core/doctype/comment/test_comment.py +++ b/frappe/core/doctype/comment/test_comment.py @@ -1,12 +1,15 @@ # Copyright (c) 2019, Frappe Technologies and Contributors # License: MIT. See LICENSE import json -import unittest import frappe +from frappe.templates.includes.comments.comments import add_comment +from frappe.tests.test_model_utils import set_user +from frappe.tests.utils import FrappeTestCase, change_settings +from frappe.website.doctype.blog_post.test_blog_post import make_test_blog -class TestComment(unittest.TestCase): +class TestComment(FrappeTestCase): def tearDown(self): frappe.form_dict.comment = None frappe.form_dict.comment_email = None @@ -39,14 +42,10 @@ class TestComment(unittest.TestCase): # test via blog def test_public_comment(self): - from frappe.website.doctype.blog_post.test_blog_post import make_test_blog - test_blog = make_test_blog() frappe.db.delete("Comment", {"reference_doctype": "Blog Post"}) - from frappe.templates.includes.comments.comments import add_comment - frappe.form_dict.comment = "Good comment with 10 chars" frappe.form_dict.comment_email = "test@test.com" frappe.form_dict.comment_by = "Good Tester" @@ -102,3 +101,32 @@ class TestComment(unittest.TestCase): ) test_blog.delete() + + @change_settings("Blog Settings", {"allow_guest_to_comment": 0}) + def test_guest_cannot_comment(self): + test_blog = make_test_blog() + with set_user("Guest"): + frappe.form_dict.comment = "Good comment with 10 chars" + frappe.form_dict.comment_email = "mail@example.org" + frappe.form_dict.comment_by = "Good Tester" + frappe.form_dict.reference_doctype = "Blog Post" + frappe.form_dict.reference_name = test_blog.name + frappe.form_dict.route = test_blog.route + frappe.local.request_ip = "127.0.0.1" + + self.assertEqual(add_comment(), None) + + def test_user_not_logged_in(self): + some_system_user = frappe.db.get_value("User", {}) + + test_blog = make_test_blog() + with set_user("Guest"): + frappe.form_dict.comment = "Good comment with 10 chars" + frappe.form_dict.comment_email = some_system_user + frappe.form_dict.comment_by = "Good Tester" + frappe.form_dict.reference_doctype = "Blog Post" + frappe.form_dict.reference_name = test_blog.name + frappe.form_dict.route = test_blog.route + frappe.local.request_ip = "127.0.0.1" + + self.assertRaises(frappe.ValidationError, add_comment) diff --git a/frappe/core/doctype/communication/communication.js b/frappe/core/doctype/communication/communication.js index 07674d16ae..a36af705a7 100644 --- a/frappe/core/doctype/communication/communication.js +++ b/frappe/core/doctype/communication/communication.js @@ -1,120 +1,158 @@ frappe.ui.form.on("Communication", { - onload: function(frm) { - if(frm.doc.content) { + onload: function (frm) { + if (frm.doc.content) { frm.doc.content = frappe.dom.remove_script_and_style(frm.doc.content); } - frm.set_query("reference_doctype", function() { + frm.set_query("reference_doctype", function () { return { filters: { - "issingle": 0, - "istable": 0 - } - } + issingle: 0, + istable: 0, + }, + }; }); }, - refresh: function(frm) { - if(frm.is_new()) return; + refresh: function (frm) { + if (frm.is_new()) return; frm.convert_to_click && frm.set_convert_button(); frm.subject_field = "subject"; // content field contains weird table html that does not render well in Quill // this field is not to be edited directly anyway, so setting it as read only - frm.set_df_property('content', 'read_only', 1); + frm.set_df_property("content", "read_only", 1); - if(frm.doc.reference_doctype && frm.doc.reference_name) { - frm.add_custom_button(__(frm.doc.reference_name), function() { + if (frm.doc.reference_doctype && frm.doc.reference_name) { + frm.add_custom_button(__(frm.doc.reference_name), function () { frappe.set_route("Form", frm.doc.reference_doctype, frm.doc.reference_name); }); } else { // if an unlinked communication, set email field - if (frm.doc.sent_or_received==="Received") { + if (frm.doc.sent_or_received === "Received") { frm.email_field = "sender"; } else { frm.email_field = "recipients"; } } - if(frm.doc.status==="Open") { - frm.add_custom_button(__("Close"), function() { - frm.trigger('mark_as_closed_open'); + if (frm.doc.status === "Open") { + frm.add_custom_button(__("Close"), function () { + frm.trigger("mark_as_closed_open"); }); } else if (frm.doc.status !== "Linked") { - frm.add_custom_button(__("Reopen"), function() { - frm.trigger('mark_as_closed_open'); + frm.add_custom_button(__("Reopen"), function () { + frm.trigger("mark_as_closed_open"); }); } - frm.add_custom_button(__("Relink"), function() { - frm.trigger('show_relink_dialog'); + frm.add_custom_button(__("Relink"), function () { + frm.trigger("show_relink_dialog"); }); - if(frm.doc.communication_type=="Communication" - && frm.doc.communication_medium == "Email" - && frm.doc.sent_or_received == "Received") { - - frm.add_custom_button(__("Reply"), function() { - frm.trigger('reply'); + if ( + frm.doc.communication_type == "Communication" && + frm.doc.communication_medium == "Email" && + frm.doc.sent_or_received == "Received" + ) { + frm.add_custom_button(__("Reply"), function () { + frm.trigger("reply"); }); - frm.add_custom_button(__("Reply All"), function() { - frm.trigger('reply_all'); - }, __("Actions")); + frm.add_custom_button( + __("Reply All"), + function () { + frm.trigger("reply_all"); + }, + __("Actions") + ); - frm.add_custom_button(__("Forward"), function() { - frm.trigger('forward_mail'); - }, __("Actions")); + frm.add_custom_button( + __("Forward"), + function () { + frm.trigger("forward_mail"); + }, + __("Actions") + ); - frm.add_custom_button(frm.doc.seen ? __("Mark as Unread") : __("Mark as Read"), function() { - frm.trigger('mark_as_read_unread'); - }, __("Actions")); + frm.add_custom_button( + frm.doc.seen ? __("Mark as Unread") : __("Mark as Read"), + function () { + frm.trigger("mark_as_read_unread"); + }, + __("Actions") + ); - frm.add_custom_button(__("Move"), function() { - frm.trigger('show_move_dialog'); - }, __("Actions")); + frm.add_custom_button( + __("Move"), + function () { + frm.trigger("show_move_dialog"); + }, + __("Actions") + ); - if(frm.doc.email_status != "Spam") - frm.add_custom_button(__("Mark as Spam"), function() { - frm.trigger('mark_as_spam'); - }, __("Actions")); + if (frm.doc.email_status != "Spam") + frm.add_custom_button( + __("Mark as Spam"), + function () { + frm.trigger("mark_as_spam"); + }, + __("Actions") + ); - if(frm.doc.email_status != "Trash") { - frm.add_custom_button(__("Move To Trash"), function() { - frm.trigger('move_to_trash'); - }, __("Actions")); + if (frm.doc.email_status != "Trash") { + frm.add_custom_button( + __("Move To Trash"), + function () { + frm.trigger("move_to_trash"); + }, + __("Actions") + ); } - frm.add_custom_button(__("Contact"), function() { - frm.trigger('add_to_contact'); - }, __('Create')); + frm.add_custom_button( + __("Contact"), + function () { + frm.trigger("add_to_contact"); + }, + __("Create") + ); } - if(frm.doc.communication_type=="Communication" - && frm.doc.communication_medium == "Phone" - && frm.doc.sent_or_received == "Received"){ - - frm.add_custom_button(__("Add Contact"), function() { - frm.trigger('add_to_contact'); - }, __("Actions")); + if ( + frm.doc.communication_type == "Communication" && + frm.doc.communication_medium == "Phone" && + frm.doc.sent_or_received == "Received" + ) { + frm.add_custom_button( + __("Add Contact"), + function () { + frm.trigger("add_to_contact"); + }, + __("Actions") + ); } }, - show_relink_dialog: function(frm) { - var d = new frappe.ui.Dialog ({ + show_relink_dialog: function (frm) { + var d = new frappe.ui.Dialog({ title: __("Relink Communication"), - fields: [{ - "fieldtype": "Link", - "options": "DocType", - "label": __("Reference Doctype"), - "fieldname": "reference_doctype", - "get_query": function() {return {"query": "frappe.email.get_communication_doctype"}} - }, - { - "fieldtype": "Dynamic Link", - "options": "reference_doctype", - "label": __("Reference Name"), - "fieldname": "reference_name" - }] + fields: [ + { + fieldtype: "Link", + options: "DocType", + label: __("Reference Doctype"), + fieldname: "reference_doctype", + get_query: function () { + return { query: "frappe.email.get_communication_doctype" }; + }, + }, + { + fieldtype: "Dynamic Link", + options: "reference_doctype", + label: __("Reference Name"), + fieldname: "reference_name", + }, + ], }); d.set_value("reference_doctype", frm.doc.reference_doctype); d.set_value("reference_name", frm.doc.reference_name); @@ -122,24 +160,27 @@ frappe.ui.form.on("Communication", { var values = d.get_values(); if (values) { frappe.confirm( - __('Are you sure you want to relink this communication to {0}?', [values["reference_name"]]), + __("Are you sure you want to relink this communication to {0}?", [ + values["reference_name"], + ]), function () { d.hide(); frappe.call({ method: "frappe.email.relink", args: { - "name": frm.doc.name, - "reference_doctype": values["reference_doctype"], - "reference_name": values["reference_name"] + name: frm.doc.name, + reference_doctype: values["reference_doctype"], + reference_name: values["reference_name"], }, callback: function () { frm.refresh(); - } + }, }); }, - function() { + function () { frappe.show_alert({ - message: __('Document not Relinked'), 'indicator': 'info' + message: __("Document not Relinked"), + indicator: "info", }); } ); @@ -148,24 +189,26 @@ frappe.ui.form.on("Communication", { d.show(); }, - show_move_dialog: function(frm) { - var d = new frappe.ui.Dialog ({ + show_move_dialog: function (frm) { + var d = new frappe.ui.Dialog({ title: __("Move"), - fields: [{ - "fieldtype": "Link", - "options": "Email Account", - "label": __("Email Account"), - "fieldname": "email_account", - "reqd": 1, - "get_query": function() { - return { - "filters": { - "name": ["!=", frm.doc.email_account], - "enable_incoming": ["=", 1] - } - }; - } - }], + fields: [ + { + fieldtype: "Link", + options: "Email Account", + label: __("Email Account"), + fieldname: "email_account", + reqd: 1, + get_query: function () { + return { + filters: { + name: ["!=", frm.doc.email_account], + enable_incoming: ["=", 1], + }, + }; + }, + }, + ], primary_action_label: __("Move"), primary_action(values) { d.hide(); @@ -173,88 +216,88 @@ frappe.ui.form.on("Communication", { method: "frappe.email.inbox.move_email", args: { communication: frm.doc.name, - email_account: values.email_account + email_account: values.email_account, }, freeze: true, - callback: function() { + callback: function () { window.history.back(); - } + }, }); - } + }, }); d.show(); }, - mark_as_read_unread: function(frm) { - var action = frm.doc.seen? "Unread": "Read"; + mark_as_read_unread: function (frm) { + var action = frm.doc.seen ? "Unread" : "Read"; var flag = "(\\SEEN)"; return frappe.call({ method: "frappe.email.inbox.create_email_flag_queue", args: { - 'names': [frm.doc.name], - 'action': action, - 'flag': flag + names: [frm.doc.name], + action: action, + flag: flag, }, freeze: true, - callback: function() { + callback: function () { frm.reload_doc(); - } + }, }); }, - mark_as_closed_open: function(frm) { + mark_as_closed_open: function (frm) { var status = frm.doc.status == "Open" ? "Closed" : "Open"; return frappe.call({ method: "frappe.email.inbox.mark_as_closed_open", args: { communication: frm.doc.name, - status: status + status: status, }, freeze: true, - callback: function() { + callback: function () { frm.reload_doc(); - } + }, }); }, - reply: function(frm) { + reply: function (frm) { var args = frm.events.get_mail_args(frm); $.extend(args, { subject: __("Re: {0}", [frm.doc.subject]), - recipients: frm.doc.sender - }) + recipients: frm.doc.sender, + }); new frappe.views.CommunicationComposer(args); }, - reply_all: function(frm) { - var args = frm.events.get_mail_args(frm) + reply_all: function (frm) { + var args = frm.events.get_mail_args(frm); $.extend(args, { subject: __("Res: {0}", [frm.doc.subject]), recipients: frm.doc.sender, - cc: frm.doc.cc - }) + cc: frm.doc.cc, + }); new frappe.views.CommunicationComposer(args); }, - forward_mail: function(frm) { - var args = frm.events.get_mail_args(frm) + forward_mail: function (frm) { + var args = frm.events.get_mail_args(frm); $.extend(args, { forward: true, subject: __("Fw: {0}", [frm.doc.subject]), - }) + }); new frappe.views.CommunicationComposer(args); }, - get_mail_args: function(frm) { - var sender_email_id = "" - $.each(frappe.boot.email_accounts, function(idx, account) { - if(account.email_account == frm.doc.email_account) { - sender_email_id = account.email_id - return + get_mail_args: function (frm) { + var sender_email_id = ""; + $.each(frappe.boot.email_accounts, function (idx, account) { + if (account.email_account == frm.doc.email_account) { + sender_email_id = account.email_id; + return; } }); @@ -263,51 +306,51 @@ frappe.ui.form.on("Communication", { doc: frm.doc, last_email: frm.doc, sender: sender_email_id, - attachments: frm.doc.attachments - } + attachments: frm.doc.attachments, + }; }, - add_to_contact: function(frm) { + add_to_contact: function (frm) { var me = this; - var fullname = frm.doc.sender_full_name || "" + var fullname = frm.doc.sender_full_name || ""; - var names = fullname.split(" ") - var first_name = names[0] - var last_name = names.length >= 2? names[names.length - 1]: "" + var names = fullname.split(" "); + var first_name = names[0]; + var last_name = names.length >= 2 ? names[names.length - 1] : ""; frappe.route_options = { - "email_id": frm.doc.sender || "", - "first_name": first_name, - "last_name": last_name, - "mobile_no": frm.doc.phone_no || "" - } - frappe.new_doc("Contact") + email_id: frm.doc.sender || "", + first_name: first_name, + last_name: last_name, + mobile_no: frm.doc.phone_no || "", + }; + frappe.new_doc("Contact"); }, - mark_as_spam: function(frm) { + mark_as_spam: function (frm) { frappe.call({ method: "frappe.email.inbox.mark_as_spam", args: { communication: frm.doc.name, - sender: frm.doc.sender + sender: frm.doc.sender, }, freeze: true, - callback: function(r) { - frappe.msgprint(__("Email has been marked as spam")) - } - }) + callback: function (r) { + frappe.msgprint(__("Email has been marked as spam")); + }, + }); }, - move_to_trash: function(frm) { + move_to_trash: function (frm) { frappe.call({ method: "frappe.email.inbox.mark_as_trash", args: { - communication: frm.doc.name + communication: frm.doc.name, }, freeze: true, - callback: function(r) { - frappe.msgprint(__("Email has been moved to trash")) - } - }) - } + callback: function (r) { + frappe.msgprint(__("Email has been moved to trash")); + }, + }); + }, }); diff --git a/frappe/core/doctype/communication/communication.json b/frappe/core/doctype/communication/communication.json index f9d15af483..e5f090e2f7 100644 --- a/frappe/core/doctype/communication/communication.json +++ b/frappe/core/doctype/communication/communication.json @@ -2,6 +2,7 @@ "actions": [], "allow_import": 1, "creation": "2013-01-29 10:47:14", + "default_view": "Inbox", "description": "Keeps track of all communications", "doctype": "DocType", "document_type": "Setup", @@ -198,7 +199,6 @@ "label": "More Information" }, { - "bold": 0, "default": "Now", "fieldname": "communication_date", "fieldtype": "Datetime", @@ -318,7 +318,7 @@ }, { "fieldname": "message_id", - "fieldtype": "Data", + "fieldtype": "Small Text", "ignore_xss_filter": 1, "label": "Message ID", "length": 995, @@ -395,7 +395,7 @@ "icon": "fa fa-comment", "idx": 1, "links": [], - "modified": "2022-03-30 11:24:25.728637", + "modified": "2023-03-16 12:04:18.113817", "modified_by": "Administrator", "module": "Core", "name": "Communication", @@ -454,8 +454,9 @@ "sender_field": "sender", "sort_field": "modified", "sort_order": "DESC", + "states": [], "subject_field": "subject", "title_field": "subject", "track_changes": 1, "track_seen": 1 -} +} \ No newline at end of file diff --git a/frappe/core/doctype/communication/communication.py b/frappe/core/doctype/communication/communication.py index fac8ac74b9..067cba59b2 100644 --- a/frappe/core/doctype/communication/communication.py +++ b/frappe/core/doctype/communication/communication.py @@ -5,7 +5,7 @@ from collections import Counter from email.utils import getaddresses from urllib.parse import unquote -from parse import compile +from bs4 import BeautifulSoup import frappe from frappe import _ @@ -19,6 +19,7 @@ from frappe.core.doctype.communication.mixins import CommunicationEmailMixin from frappe.core.utils import get_parent_doc from frappe.model.document import Document from frappe.utils import ( + cstr, parse_addr, split_emails, strip_html, @@ -144,8 +145,8 @@ class Communication(Document, CommunicationEmailMixin): if not self.content: return - quill_parser = compile('
{}
') - email_body = quill_parser.parse(self.content) + soup = BeautifulSoup(self.content, "html.parser") + email_body = soup.find("div", {"class": "ql-editor read-mode"}) if not email_body: return @@ -169,9 +170,13 @@ class Communication(Document, CommunicationEmailMixin): if not signature: return - _signature = quill_parser.parse(signature)[0] if "ql-editor" in signature else None + soup = BeautifulSoup(signature, "html.parser") + html_signature = soup.find("div", {"class": "ql-editor read-mode"}) + _signature = None + if html_signature: + _signature = html_signature.renderContents() - if (_signature or signature) not in self.content: + if (cstr(_signature) or signature) not in self.content: self.content = f'{self.content}


{signature}' def before_save(self): @@ -228,8 +233,10 @@ class Communication(Document, CommunicationEmailMixin): def notify_change(self, action): frappe.publish_realtime( - f"update_docinfo_for_{self.reference_doctype}_{self.reference_name}", + "docinfo_update", {"doc": self.as_dict(), "key": "communications", "action": action}, + doctype=self.reference_doctype, + docname=self.reference_name, after_commit=True, ) @@ -390,6 +397,7 @@ def on_doctype_update(): """Add indexes in `tabCommunication`""" frappe.db.add_index("Communication", ["reference_doctype", "reference_name"]) frappe.db.add_index("Communication", ["status", "communication_type"]) + frappe.db.add_index("Communication", ["message_id(140)"]) def has_permission(doc, ptype, user): @@ -480,28 +488,32 @@ def parse_email(communication, email_strings): """ Parse email to add timeline links. When automatic email linking is enabled, an email from email_strings can contain - a doctype and docname ie in the format `admin+doctype+docname@example.com`, + a doctype and docname ie in the format `admin+doctype+docname@example.com` or `admin+doctype=docname@example.com`, the email is parsed and doctype and docname is extracted and timeline link is added. """ - if not frappe.get_all("Email Account", filters={"enable_automatic_linking": 1}): + if not frappe.db.get_value("Email Account", filters={"enable_automatic_linking": 1}): return - delimiter = "+" - for email_string in email_strings: if email_string: for email in email_string.split(","): - if delimiter in email: - email = email.split("@")[0] - email_local_parts = email.split(delimiter) - if not len(email_local_parts) == 3: - continue - + email_username = email.split("@", 1)[0] + email_local_parts = email_username.split("+") + docname = doctype = None + if len(email_local_parts) == 3: doctype = unquote(email_local_parts[1]) docname = unquote(email_local_parts[2]) - if doctype and docname and frappe.db.exists(doctype, docname): - communication.add_link(doctype, docname) + elif len(email_local_parts) == 2: + document_parts = email_local_parts[1].split("=", 1) + if len(document_parts) != 2: + continue + + doctype = unquote(document_parts[0]) + docname = unquote(document_parts[1]) + + if doctype and docname and frappe.db.get_value(doctype, docname, ignore=True): + communication.add_link(doctype, docname) def get_email_without_link(email): @@ -514,7 +526,7 @@ def get_email_without_link(email): try: _email = email.split("@") - email_id = _email[0].split("+")[0] + email_id = _email[0].split("+", 1)[0] email_host = _email[1] except IndexError: return email @@ -543,9 +555,6 @@ def update_parent_document_on_communication(doc): parent.db_set("status", "Open") parent.run_method("handle_hold_time", "Replied") apply_assignment_rule(parent) - else: - # update the modified date for document - parent.update_modified() update_first_response_time(parent, doc) set_avg_response_time(parent, doc) diff --git a/frappe/core/doctype/communication/communication_list.js b/frappe/core/doctype/communication/communication_list.js index 315b74a39c..4ef3a384ff 100644 --- a/frappe/core/doctype/communication/communication_list.js +++ b/frappe/core/doctype/communication/communication_list.js @@ -1,25 +1,32 @@ -frappe.listview_settings['Communication'] = { +frappe.listview_settings["Communication"] = { add_fields: [ - "sent_or_received","recipients", "subject", - "communication_medium", "communication_type", - "sender", "seen", "reference_doctype", "reference_name", - "has_attachment", "communication_date" + "sent_or_received", + "recipients", + "subject", + "communication_medium", + "communication_type", + "sender", + "seen", + "reference_doctype", + "reference_name", + "has_attachment", + "communication_date", ], filters: [["status", "=", "Open"]], - onload: function(list_view) { - let method = "frappe.email.inbox.create_email_flag_queue" + onload: function (list_view) { + let method = "frappe.email.inbox.create_email_flag_queue"; - list_view.page.add_menu_item(__("Mark as Read"), function() { + list_view.page.add_menu_item(__("Mark as Read"), function () { list_view.call_for_selected_items(method, { action: "Read" }); }); - list_view.page.add_menu_item(__("Mark as Unread"), function() { + list_view.page.add_menu_item(__("Mark as Unread"), function () { list_view.call_for_selected_items(method, { action: "Unread" }); }); }, - primary_action: function() { + primary_action: function () { new frappe.views.CommunicationComposer(); - } + }, }; diff --git a/frappe/core/doctype/communication/email.py b/frappe/core/doctype/communication/email.py index c5585fd463..1733b7b716 100755 --- a/frappe/core/doctype/communication/email.py +++ b/frappe/core/doctype/communication/email.py @@ -12,6 +12,7 @@ from frappe.utils import ( cint, get_datetime, get_formatted_email, + get_imaginary_pixel_response, get_string_between, list_to_str, split_emails, @@ -92,7 +93,7 @@ def make( send_me_a_copy=cint(send_me_a_copy), cc=cc, bcc=bcc, - read_receipt=read_receipt, + read_receipt=cint(read_receipt), print_letterhead=print_letterhead, email_template=email_template, communication_type=communication_type, @@ -164,7 +165,7 @@ def _make( if not comm.get_outgoing_email_account(): frappe.throw( _( - "Unable to send mail because of a missing email account. Please setup default Email Account from Setup > Email > Email Account" + "Unable to send mail because of a missing email account. Please setup default Email Account from Settings > Email Account" ), exc=frappe.OutgoingEmailError, ) @@ -249,18 +250,7 @@ def mark_email_as_seen(name: str = None): frappe.log_error("Unable to mark as seen", None, "Communication", name) finally: - frappe.response.update( - { - "type": "binary", - "filename": "imaginary_pixel.png", - "filecontent": ( - b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00" - b"\x00\x01\x08\x06\x00\x00\x00\x1f\x15\xc4\x89\x00\x00\x00\r" - b"IDATx\x9cc\xf8\xff\xff?\x03\x00\x08\xfc\x02\xfe\xa7\x9a\xa0" - b"\xa0\x00\x00\x00\x00IEND\xaeB`\x82" - ), - } - ) + frappe.response.update(frappe.utils.get_imaginary_pixel_response()) def update_communication_as_read(name): diff --git a/frappe/core/doctype/communication/mixins.py b/frappe/core/doctype/communication/mixins.py index ad74b47026..73e94fad09 100644 --- a/frappe/core/doctype/communication/mixins.py +++ b/frappe/core/doctype/communication/mixins.py @@ -59,33 +59,35 @@ class CommunicationEmailMixin: * if email copy is requested by sender, then add sender to CC. * If this doc is created through inbound mail, then add doc owner to cc list * remove all the thread_notify disabled users. - * Make sure that all users enabled in the system - * Remove admin from email list - - * FixMe: Removed adding TODO owners to cc list. Check if that is needed. + * Remove standard users from email list """ if hasattr(self, "_final_cc"): return self._final_cc cc = self.cc_list() - # Need to inform parent document owner incase communication is created through inbound mail if include_sender: - cc.append(self.sender_mailid) + sender = self.sender_mailid + # if user has selected send_me_a_copy, use their email as sender + if frappe.session.user not in frappe.STANDARD_USERS: + sender = frappe.db.get_value("User", frappe.session.user, "email") + cc.append(sender) + if is_inbound_mail_communcation: - cc.append(self.get_owner()) + # inform parent document owner incase communication is created through inbound mail + if doc_owner := self.get_owner(): + cc.append(doc_owner) cc = set(cc) - {self.sender_mailid} cc.update(self.get_assignees()) cc = set(cc) - set(self.filter_thread_notification_disbled_users(cc)) cc = cc - set(self.mail_recipients(is_inbound_mail_communcation=is_inbound_mail_communcation)) - cc = cc - set(self.filter_disabled_users(cc)) # # Incase of inbound mail, to and cc already received the mail, no need to send again. if is_inbound_mail_communcation: cc = cc - set(self.cc_list() + self.to_list()) - self._final_cc = list(filter(lambda id: id != "Administrator", cc)) + self._final_cc = [m for m in cc if m and m not in frappe.STANDARD_USERS] return self._final_cc def get_mail_cc_with_displayname(self, is_inbound_mail_communcation=False, include_sender=False): @@ -98,8 +100,7 @@ class CommunicationEmailMixin: """ * Thread_notify check * Email unsubscribe list - * User must be enabled in the system - * remove_administrator_from_email_list + * remove standard users. """ if hasattr(self, "_final_bcc"): return self._final_bcc @@ -109,13 +110,12 @@ class CommunicationEmailMixin: bcc = bcc - {self.sender_mailid} bcc = bcc - set(self.filter_thread_notification_disbled_users(bcc)) bcc = bcc - set(self.mail_recipients(is_inbound_mail_communcation=is_inbound_mail_communcation)) - bcc = bcc - set(self.filter_disabled_users(bcc)) # Incase of inbound mail, to and cc & bcc already received the mail, no need to send again. if is_inbound_mail_communcation: bcc = bcc - set(self.bcc_list() + self.to_list()) - self._final_bcc = list(filter(lambda id: id != "Administrator", bcc)) + self._final_bcc = [m for m in bcc if m not in frappe.STANDARD_USERS] return self._final_bcc def get_mail_bcc_with_displayname(self, is_inbound_mail_communcation=False): @@ -164,7 +164,8 @@ class CommunicationEmailMixin: ) if self.sent_or_received == "Sent" and self._outgoing_email_account: - self.db_set("email_account", self._outgoing_email_account.name) + if frappe.db.exists("Email Account", self._outgoing_email_account.name): + self.db_set("email_account", self._outgoing_email_account.name) return self._outgoing_email_account @@ -220,7 +221,11 @@ class CommunicationEmailMixin: "reference_name": self.reference_name, "reference_type": self.reference_doctype, } - return ToDo.get_owners(filters) + + if self.reference_doctype and self.reference_name: + return ToDo.get_owners(filters) + else: + return [] @staticmethod def filter_thread_notification_disbled_users(emails): @@ -247,7 +252,7 @@ class CommunicationEmailMixin: send_me_a_copy=None, print_letterhead=None, is_inbound_mail_communcation=None, - ): + ) -> dict: outgoing_email_account = self.get_outgoing_email_account() if not outgoing_email_account: @@ -297,13 +302,11 @@ class CommunicationEmailMixin: print_letterhead=None, is_inbound_mail_communcation=None, ): - input_dict = self.sendmail_input_dict( + if input_dict := self.sendmail_input_dict( print_html=print_html, print_format=print_format, send_me_a_copy=send_me_a_copy, print_letterhead=print_letterhead, is_inbound_mail_communcation=is_inbound_mail_communcation, - ) - - if input_dict: + ): frappe.sendmail(**input_dict) diff --git a/frappe/core/doctype/communication/test_communication.py b/frappe/core/doctype/communication/test_communication.py index 77f83b7f91..7f2d36d60a 100644 --- a/frappe/core/doctype/communication/test_communication.py +++ b/frappe/core/doctype/communication/test_communication.py @@ -1,16 +1,21 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors +# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import unittest +from typing import TYPE_CHECKING from urllib.parse import quote import frappe -from frappe.core.doctype.communication.communication import get_emails +from frappe.core.doctype.communication.communication import Communication, get_emails from frappe.email.doctype.email_queue.email_queue import EmailQueue +from frappe.tests.utils import FrappeTestCase + +if TYPE_CHECKING: + from frappe.contacts.doctype.contact.contact import Contact + from frappe.email.doctype.email_account.email_account import EmailAccount test_records = frappe.get_test_records("Communication") -class TestCommunication(unittest.TestCase): +class TestCommunication(FrappeTestCase): def test_email(self): valid_email_list = [ "Full Name ", @@ -32,11 +37,13 @@ class TestCommunication(unittest.TestCase): "[invalid!email].com", ] - for x in valid_email_list: - self.assertTrue(frappe.utils.parse_addr(x)[1]) + for i, x in enumerate(valid_email_list): + with self.subTest(i=i, x=x): + self.assertTrue(frappe.utils.parse_addr(x)[1]) - for x in invalid_email_list: - self.assertFalse(frappe.utils.parse_addr(x)[0]) + for i, x in enumerate(invalid_email_list): + with self.subTest(i=i, x=x): + self.assertFalse(frappe.utils.parse_addr(x)[0]) def test_name(self): valid_email_list = [ @@ -129,7 +136,7 @@ class TestCommunication(unittest.TestCase): self.assertNotEqual(2, len(comm.timeline_links)) def test_contacts_attached(self): - contact_sender = frappe.get_doc( + contact_sender: "Contact" = frappe.get_doc( { "doctype": "Contact", "first_name": "contact_sender", @@ -138,7 +145,7 @@ class TestCommunication(unittest.TestCase): contact_sender.add_email("comm_sender@example.com") contact_sender.insert(ignore_permissions=True) - contact_recipient = frappe.get_doc( + contact_recipient: "Contact" = frappe.get_doc( { "doctype": "Contact", "first_name": "contact_recipient", @@ -147,7 +154,7 @@ class TestCommunication(unittest.TestCase): contact_recipient.add_email("comm_recipient@example.com") contact_recipient.insert(ignore_permissions=True) - contact_cc = frappe.get_doc( + contact_cc: "Contact" = frappe.get_doc( { "doctype": "Contact", "first_name": "contact_cc", @@ -156,7 +163,7 @@ class TestCommunication(unittest.TestCase): contact_cc.add_email("comm_cc@example.com") contact_cc.insert(ignore_permissions=True) - comm = frappe.get_doc( + comm: Communication = frappe.get_doc( { "doctype": "Communication", "communication_medium": "Email", @@ -168,10 +175,7 @@ class TestCommunication(unittest.TestCase): ).insert(ignore_permissions=True) comm = frappe.get_doc("Communication", comm.name) - - contact_links = [] - for timeline_link in comm.timeline_links: - contact_links.append(timeline_link.link_name) + contact_links = [x.link_name for x in comm.timeline_links] self.assertIn(contact_sender.name, contact_links) self.assertIn(contact_recipient.name, contact_links) @@ -210,25 +214,22 @@ class TestCommunication(unittest.TestCase): comms = get_communication_data("Note", note.name, as_dict=True) - data = [] - for comm in comms: - data.append(comm.name) - + data = [comm.name for comm in comms] self.assertIn(comm_note_1.name, data) self.assertIn(comm_note_2.name, data) def test_link_in_email(self): - frappe.delete_doc_if_exists("Note", "test document link in email") - create_email_account() - note = frappe.get_doc( - { - "doctype": "Note", - "title": "test document link in email", - "content": "test document link in email", - } - ).insert(ignore_permissions=True) + notes = {} + for i in range(2): + frappe.delete_doc_if_exists("Note", f"test document link in email {i}") + notes[i] = frappe.get_doc( + { + "doctype": "Note", + "title": f"test document link in email {i}", + } + ).insert(ignore_permissions=True) comm = frappe.get_doc( { @@ -236,15 +237,15 @@ class TestCommunication(unittest.TestCase): "communication_medium": "Email", "subject": "Document Link in Email", "sender": "comm_sender@example.com", - "recipients": "comm_recipient+{}+{}@example.com".format(quote("Note"), quote(note.name)), + "recipients": f'comm_recipient+{quote("Note")}+{quote(notes[0].name)}@example.com,comm_recipient+{quote("Note")}={quote(notes[1].name)}@example.com', } ).insert(ignore_permissions=True) - doc_links = [] - for timeline_link in comm.timeline_links: - doc_links.append((timeline_link.link_doctype, timeline_link.link_name)) - - self.assertIn(("Note", note.name), doc_links) + doc_links = [ + (timeline_link.link_doctype, timeline_link.link_name) for timeline_link in comm.timeline_links + ] + self.assertIn(("Note", notes[0].name), doc_links) + self.assertIn(("Note", notes[1].name), doc_links) def test_parse_emails(self): emails = get_emails( @@ -259,14 +260,46 @@ class TestCommunication(unittest.TestCase): self.assertEqual(emails[1], "first.lastname@email.com") self.assertEqual(emails[2], "test@user.com") + def test_signature_in_email_content(self): + email_account = create_email_account() + signature = email_account.signature + base_communication = { + "doctype": "Communication", + "communication_medium": "Email", + "subject": "Document Link in Email", + "sender": "comm_sender@example.com", + } + comm_with_signature = frappe.get_doc( + base_communication + | { + "content": f"""

+ Hi, + How are you? +


{signature}

""", + } + ).insert(ignore_permissions=True) + comm_without_signature = frappe.get_doc( + base_communication + | { + "content": """
+ Hi, + How are you? +
""" + } + ).insert(ignore_permissions=True) -class TestCommunicationEmailMixin(unittest.TestCase): - def new_communication(self, recipients=None, cc=None, bcc=None): + self.assertEqual(comm_with_signature.content, comm_without_signature.content) + self.assertEqual(comm_with_signature.content.count(signature), 1) + self.assertEqual(comm_without_signature.content.count(signature), 1) + + +class TestCommunicationEmailMixin(FrappeTestCase): + def new_communication(self, recipients=None, cc=None, bcc=None) -> Communication: recipients = ", ".join(recipients or []) cc = ", ".join(cc or []) bcc = ", ".join(bcc or []) - comm = frappe.get_doc( + return frappe.get_doc( { "doctype": "Communication", "communication_type": "Communication", @@ -275,9 +308,9 @@ class TestCommunicationEmailMixin(unittest.TestCase): "recipients": recipients, "cc": cc, "bcc": bcc, + "sender": "sender@test.com", } ).insert(ignore_permissions=True) - return comm def new_user(self, email, **user_data): user_data.setdefault("first_name", "first_name") @@ -295,14 +328,26 @@ class TestCommunicationEmailMixin(unittest.TestCase): comm.delete() def test_cc(self): - to_list = ["to@test.com"] - cc_list = ["cc+1@test.com", "cc ", "to@test.com"] - user = self.new_user(email="cc+1@test.com", thread_notify=0) - comm = self.new_communication(recipients=to_list, cc=cc_list) - res = comm.get_mail_cc_with_displayname() - self.assertCountEqual(res, ["cc "]) - user.delete() - comm.delete() + def test(assertion, cc_list=None, set_user_as=None, include_sender=False, thread_notify=False): + if set_user_as: + frappe.set_user(set_user_as) + + user = self.new_user(email="cc+1@test.com", thread_notify=thread_notify) + comm = self.new_communication(recipients=["to@test.com"], cc=cc_list) + res = comm.get_mail_cc_with_displayname(include_sender=include_sender) + + frappe.set_user("Administrator") + user.delete() + comm.delete() + + self.assertEqual(res, assertion) + + # test filter_thread_notification_disbled_users and filter_mail_recipients + test(["cc "], cc_list=["cc+1@test.com", "cc ", "to@test.com"]) + + # test include_sender + test(["sender@test.com"], include_sender=True, thread_notify=True) + test(["cc+1@test.com"], include_sender=True, thread_notify=True, set_user_as="cc+1@test.com") def test_bcc(self): bcc_list = [ @@ -312,7 +357,7 @@ class TestCommunicationEmailMixin(unittest.TestCase): user = self.new_user(email="bcc+2@test.com", enabled=0) comm = self.new_communication(bcc=bcc_list) res = comm.get_mail_bcc_with_displayname() - self.assertCountEqual(res, ["bcc+1@test.com"]) + self.assertCountEqual(res, bcc_list) user.delete() comm.delete() @@ -330,13 +375,13 @@ class TestCommunicationEmailMixin(unittest.TestCase): comm.delete() -def create_email_account(): +def create_email_account() -> "EmailAccount": frappe.delete_doc_if_exists("Email Account", "_Test Comm Account 1") frappe.flags.mute_emails = False frappe.flags.sent_mail = None - email_account = frappe.get_doc( + return frappe.get_doc( { "is_default": 1, "is_global": 1, @@ -345,6 +390,7 @@ def create_email_account(): "append_to": "ToDo", "email_account_name": "_Test Comm Account 1", "enable_outgoing": 1, + "default_outgoing": 1, "smtp_server": "test.example.com", "email_id": "test_comm@example.com", "password": "password", @@ -358,9 +404,6 @@ def create_email_account(): "send_notification_to": "test_comm@example.com", "pop3_server": "pop.test.example.com", "imap_folder": [{"folder_name": "INBOX", "append_to": "ToDo"}], - "no_remaining": "0", "enable_automatic_linking": 1, } ).insert(ignore_permissions=True) - - return email_account diff --git a/frappe/core/doctype/custom_docperm/custom_docperm.js b/frappe/core/doctype/custom_docperm/custom_docperm.js index 1f04a638a1..0da50217d2 100644 --- a/frappe/core/doctype/custom_docperm/custom_docperm.js +++ b/frappe/core/doctype/custom_docperm/custom_docperm.js @@ -1,8 +1,6 @@ // Copyright (c) 2016, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Custom DocPerm', { - refresh: function(frm) { - - } +frappe.ui.form.on("Custom DocPerm", { + refresh: function (frm) {}, }); diff --git a/frappe/core/doctype/custom_docperm/custom_docperm.json b/frappe/core/doctype/custom_docperm/custom_docperm.json index 93f5431903..208b0beef9 100644 --- a/frappe/core/doctype/custom_docperm/custom_docperm.json +++ b/frappe/core/doctype/custom_docperm/custom_docperm.json @@ -27,7 +27,6 @@ "report", "export", "import", - "set_user_permissions", "column_break_19", "share", "print", @@ -179,13 +178,6 @@ "fieldtype": "Check", "label": "Import" }, - { - "default": "0", - "description": "This role update User Permissions for a user", - "fieldname": "set_user_permissions", - "fieldtype": "Check", - "label": "Set User Permissions" - }, { "fieldname": "column_break_19", "fieldtype": "Column Break" @@ -212,7 +204,8 @@ "fieldname": "parent", "fieldtype": "Data", "label": "Reference Document Type", - "read_only": 1 + "read_only": 1, + "search_index": 1 }, { "default": "0", @@ -222,7 +215,7 @@ } ], "links": [], - "modified": "2020-12-03 15:20:48.296730", + "modified": "2023-02-20 13:19:04.889081", "modified_by": "Administrator", "module": "Core", "name": "Custom DocPerm", diff --git a/frappe/core/doctype/custom_docperm/test_custom_docperm.py b/frappe/core/doctype/custom_docperm/test_custom_docperm.py index 4aa04f0223..bc113f1f8f 100644 --- a/frappe/core/doctype/custom_docperm/test_custom_docperm.py +++ b/frappe/core/doctype/custom_docperm/test_custom_docperm.py @@ -1,9 +1,9 @@ # Copyright (c) 2015, Frappe Technologies and Contributors # License: MIT. See LICENSE -import unittest +from frappe.tests.utils import FrappeTestCase # test_records = frappe.get_test_records('Custom DocPerm') -class TestCustomDocPerm(unittest.TestCase): +class TestCustomDocPerm(FrappeTestCase): pass diff --git a/frappe/core/doctype/custom_role/custom_role.js b/frappe/core/doctype/custom_role/custom_role.js index 85302a48b7..16b86485ed 100644 --- a/frappe/core/doctype/custom_role/custom_role.js +++ b/frappe/core/doctype/custom_role/custom_role.js @@ -1,8 +1,6 @@ // Copyright (c) 2016, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Custom Role', { - refresh: function(frm) { - - } +frappe.ui.form.on("Custom Role", { + refresh: function (frm) {}, }); diff --git a/frappe/core/doctype/custom_role/custom_role.json b/frappe/core/doctype/custom_role/custom_role.json index 55af8e2acd..7504882caf 100644 --- a/frappe/core/doctype/custom_role/custom_role.json +++ b/frappe/core/doctype/custom_role/custom_role.json @@ -1,240 +1,76 @@ { - "allow_copy": 0, - "allow_guest_to_view": 0, + "actions": [], "allow_import": 1, - "allow_rename": 0, "autoname": "hash", - "beta": 0, "creation": "2017-02-13 14:53:36.240122", - "custom": 0, - "docstatus": 0, "doctype": "DocType", - "document_type": "", "editable_grid": 1, "engine": "InnoDB", + "field_order": [ + "page", + "report", + "permission_rules", + "roles", + "response", + "ref_doctype" + ], "fields": [ { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "page", "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, "label": "Page", - "length": 0, - "no_copy": 0, - "options": "Page", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "options": "Page" }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "report", "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, "label": "Report", - "length": 0, - "no_copy": 0, - "options": "Report", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "options": "Report" }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "permission_rules", "fieldtype": "Section Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Permission Rules", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "label": "Permission Rules" }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "roles", "fieldtype": "Table", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, "label": "Role", - "length": 0, - "no_copy": 0, - "options": "Has Role", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "options": "Has Role" }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "response", "fieldtype": "HTML", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "response", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "label": "response" }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "ref_doctype", "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Reference Document Type", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "label": "Reference Document Type" } ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2019-09-05 14:22:27.664645", + "links": [], + "modified": "2022-08-03 12:20:52.985554", "modified_by": "Administrator", "module": "Core", "name": "Custom Role", - "name_case": "", "owner": "Administrator", "permissions": [ { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, "create": 1, "delete": 1, "email": 1, "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, "print": 1, "read": 1, "report": 1, "role": "System Manager", - "set_user_permissions": 0, "share": 1, - "submit": 0, "write": 1 } ], - "quick_entry": 0, "read_only": 1, - "read_only_onload": 0, - "show_name_in_global_search": 0, "sort_field": "modified", "sort_order": "DESC", - "track_changes": 0, - "track_seen": 0 + "states": [] } \ No newline at end of file diff --git a/frappe/core/doctype/custom_role/test_custom_role.py b/frappe/core/doctype/custom_role/test_custom_role.py index 79c66255c0..c81d70d3b5 100644 --- a/frappe/core/doctype/custom_role/test_custom_role.py +++ b/frappe/core/doctype/custom_role/test_custom_role.py @@ -1,9 +1,9 @@ # Copyright (c) 2015, Frappe Technologies and Contributors # License: MIT. See LICENSE -import unittest +from frappe.tests.utils import FrappeTestCase # test_records = frappe.get_test_records('Custom Role') -class TestCustomRole(unittest.TestCase): +class TestCustomRole(FrappeTestCase): pass diff --git a/frappe/core/doctype/data_export/data_export.js b/frappe/core/doctype/data_export/data_export.js index f195e79119..54677b98a6 100644 --- a/frappe/core/doctype/data_export/data_export.js +++ b/frappe/core/doctype/data_export/data_export.js @@ -1,64 +1,69 @@ // Copyright (c) 2018, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Data Export', { - refresh: frm => { +frappe.ui.form.on("Data Export", { + refresh: (frm) => { frm.disable_save(); - frm.page.set_primary_action('Export', () => { + frm.page.set_primary_action("Export", () => { can_export(frm) ? export_data(frm) : null; }); }, onload: (frm) => { frm.set_query("reference_doctype", () => { return { - "filters": { - "issingle": 0, - "istable": 0, - "name": ['in', frappe.boot.user.can_export] - } + filters: { + issingle: 0, + istable: 0, + name: ["in", frappe.boot.user.can_export], + }, }; }); }, - reference_doctype: frm => { + reference_doctype: (frm) => { const doctype = frm.doc.reference_doctype; if (doctype) { frappe.model.with_doctype(doctype, () => set_field_options(frm)); } else { reset_filter_and_field(frm); } - } + }, + export_without_main_header: (frm) => { + frm.refresh(); + }, }); -const can_export = frm => { +const can_export = (frm) => { const doctype = frm.doc.reference_doctype; - const parent_multicheck_options = frm.fields_multicheck[doctype] ? - frm.fields_multicheck[doctype].get_checked_options() : []; + const parent_multicheck_options = frm.fields_multicheck[doctype] + ? frm.fields_multicheck[doctype].get_checked_options() + : []; let is_valid_form = false; if (!doctype) { - frappe.msgprint(__('Please select the Document Type.')); + frappe.msgprint(__("Please select the Document Type.")); } else if (!parent_multicheck_options.length) { - frappe.msgprint(__('Atleast one field of Parent Document Type is mandatory')); + frappe.msgprint(__("Atleast one field of Parent Document Type is mandatory")); } else { is_valid_form = true; } return is_valid_form; }; -const export_data = frm => { - let get_template_url = '/api/method/frappe.core.doctype.data_export.exporter.export_data'; +const export_data = (frm) => { + let get_template_url = "/api/method/frappe.core.doctype.data_export.exporter.export_data"; var export_params = () => { let columns = {}; - Object.keys(frm.fields_multicheck).forEach(dt => { + Object.keys(frm.fields_multicheck).forEach((dt) => { const options = frm.fields_multicheck[dt].get_checked_options(); columns[dt] = options; }); return { doctype: frm.doc.reference_doctype, select_columns: JSON.stringify(columns), - filters: frm.filter_list.get_filters().map(filter => filter.slice(1, 4)), + filters: frm.filter_list.get_filters().map((filter) => filter.slice(1, 4)), file_type: frm.doc.file_type, - template: true, - with_data: 1 + template: !frm.doc.export_without_main_header, + with_data: 1, + export_without_column_meta: frm.doc.export_without_main_header ? true : false, }; }; @@ -86,26 +91,24 @@ const set_field_options = (frm) => { frm.filter_list = new frappe.ui.FilterGroup({ parent: filter_wrapper, doctype: doctype, - on_change: () => { }, + on_change: () => {}, }); // Add 'Select All' and 'Unselect All' button make_multiselect_buttons(parent_wrapper); frm.fields_multicheck = {}; - related_doctypes.forEach(dt => { + related_doctypes.forEach((dt) => { frm.fields_multicheck[dt] = add_doctype_field_multicheck_control(dt, parent_wrapper); }); frm.refresh(); }; -const make_multiselect_buttons = parent_wrapper => { - const button_container = $(parent_wrapper) - .append('
') - .find('.flex'); +const make_multiselect_buttons = (parent_wrapper) => { + const button_container = $(parent_wrapper).append('
').find(".flex"); - ["Select All", "Unselect All"].map(d => { + ["Select All", "Unselect All"].map((d) => { frappe.ui.form.make_control({ parent: $(button_container), df: { @@ -113,59 +116,59 @@ const make_multiselect_buttons = parent_wrapper => { fieldname: frappe.scrub(d), fieldtype: "Button", click: () => { - checkbox_toggle(d !== 'Select All'); - } + checkbox_toggle(d !== "Select All"); + }, }, - render_input: true + render_input: true, }); }); - $(button_container).find('.frappe-control').map((index, button) => { - $(button).css({"margin-right": "1em"}); - }); + $(button_container) + .find(".frappe-control") + .map((index, button) => { + $(button).css({ "margin-right": "1em" }); + }); function checkbox_toggle(checked) { - $(parent_wrapper).find('[data-fieldtype="MultiCheck"]').map((index, element) => { - $(element).find(`:checkbox`).prop("checked", checked).trigger('click'); - }); + $(parent_wrapper) + .find('[data-fieldtype="MultiCheck"]') + .map((index, element) => { + $(element).find(`:checkbox`).prop("checked", checked).trigger("click"); + }); } - }; -const get_doctypes = parentdt => { - return [parentdt].concat( - frappe.meta.get_table_fields(parentdt).map(df => df.options) - ); +const get_doctypes = (parentdt) => { + return [parentdt].concat(frappe.meta.get_table_fields(parentdt).map((df) => df.options)); }; const add_doctype_field_multicheck_control = (doctype, parent_wrapper) => { const fields = get_fields(doctype); - const options = fields - .map(df => { - return { - label: df.label, - value: df.fieldname, - danger: df.reqd, - checked: 1 - }; - }); + const options = fields.map((df) => { + return { + label: df.label, + value: df.fieldname, + danger: df.reqd, + checked: 1, + }; + }); const multicheck_control = frappe.ui.form.make_control({ parent: parent_wrapper, df: { - "label": doctype, - "fieldname": doctype + '_fields', - "fieldtype": "MultiCheck", - "options": options, - "columns": 3, + label: doctype, + fieldname: doctype + "_fields", + fieldtype: "MultiCheck", + options: options, + columns: 3, }, - render_input: true + render_input: true, }); multicheck_control.refresh_input(); return multicheck_control; }; -const filter_fields = df => frappe.model.is_value_type(df) && !df.hidden; -const get_fields = dt => frappe.meta.get_docfields(dt).filter(filter_fields); \ No newline at end of file +const filter_fields = (df) => frappe.model.is_value_type(df) && !df.hidden; +const get_fields = (dt) => frappe.meta.get_docfields(dt).filter(filter_fields); diff --git a/frappe/core/doctype/data_export/data_export.json b/frappe/core/doctype/data_export/data_export.json index 8304430fdb..f63d939499 100644 --- a/frappe/core/doctype/data_export/data_export.json +++ b/frappe/core/doctype/data_export/data_export.json @@ -1,250 +1,84 @@ { - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, - "creation": "2018-03-07 10:09:49.794764", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", + "actions": [], + "creation": "2018-03-07 10:09:49.794764", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "reference_doctype", + "export_without_main_header", + "column_break_2", + "file_type", + "section_break", + "filter_list", + "fields_multicheck" + ], "fields": [ { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "reference_doctype", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Select Doctype", - "length": 0, - "no_copy": 0, - "options": "DocType", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "column_break_2", - "fieldtype": "Column Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "default": "CSV", - "fieldname": "file_type", - "fieldtype": "Select", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "File Type", - "length": 0, - "no_copy": 0, - "options": "Excel\nCSV", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "depends_on": "reference_doctype", - "fieldname": "section_break", - "fieldtype": "Section Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "filter_list", - "fieldtype": "HTML", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Filter List", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "fieldname": "reference_doctype", + "fieldtype": "Link", + "label": "Select Doctype", + "options": "DocType", + "reqd": 1 }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "fields_multicheck", - "fieldtype": "HTML", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Fields Multicheck", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "fieldname": "column_break_2", + "fieldtype": "Column Break" + }, + { + "default": "CSV", + "fieldname": "file_type", + "fieldtype": "Select", + "in_list_view": 1, + "label": "File Type", + "options": "Excel\nCSV", + "reqd": 1 + }, + { + "depends_on": "reference_doctype", + "fieldname": "section_break", + "fieldtype": "Section Break" + }, + { + "fieldname": "filter_list", + "fieldtype": "HTML", + "label": "Filter List" + }, + { + "fieldname": "fields_multicheck", + "fieldtype": "HTML", + "label": "Fields Multicheck" + }, + { + "default": "0", + "description": "Export the data without any header notes and column descriptions", + "fieldname": "export_without_main_header", + "fieldtype": "Check", + "label": "Export without main header" } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 1, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 1, - "istable": 0, - "max_attachments": 0, - "modified": "2018-03-21 13:23:05.623052", - "modified_by": "Administrator", - "module": "Core", - "name": "Data Export", - "name_case": "", - "owner": "Administrator", + ], + "hide_toolbar": 1, + "issingle": 1, + "links": [], + "modified": "2022-09-28 03:51:02.404681", + "modified_by": "Administrator", + "module": "Core", + "name": "Data Export", + "owner": "Administrator", "permissions": [ { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 1, - "delete": 1, - "email": 1, - "export": 0, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 0, - "role": "System Manager", - "set_user_permissions": 0, - "share": 1, - "submit": 0, + "create": 1, + "delete": 1, + "email": 1, + "print": 1, + "read": 1, + "role": "System Manager", + "share": 1, "write": 1 } - ], - "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 0, - "track_seen": 0 + ], + "quick_entry": 1, + "sort_field": "modified", + "sort_order": "DESC", + "states": [] } \ No newline at end of file diff --git a/frappe/core/doctype/data_export/exporter.py b/frappe/core/doctype/data_export/exporter.py index b7f69ab43d..bc67087151 100644 --- a/frappe/core/doctype/data_export/exporter.py +++ b/frappe/core/doctype/data_export/exporter.py @@ -9,6 +9,7 @@ import frappe import frappe.permissions from frappe import _ from frappe.core.doctype.access_log.access_log import make_access_log +from frappe.model.utils import is_virtual_doctype from frappe.utils import cint, cstr, format_datetime, format_duration, formatdate, parse_json from frappe.utils.csvutils import UnicodeWriter @@ -37,6 +38,7 @@ def export_data( file_type="CSV", template=False, filters=None, + export_without_column_meta=False, ): _doctype = doctype if isinstance(_doctype, list): @@ -48,6 +50,15 @@ def export_data( filters=filters, method=parent_doctype, ) + + template_bool = template + if isinstance(template, str): + template_bool = template.lower() == "true" + + export_without_column_meta_bool = export_without_column_meta + if isinstance(export_without_column_meta, str): + export_without_column_meta_bool = export_without_column_meta.lower() == "true" + exporter = DataExporter( doctype=doctype, parent_doctype=parent_doctype, @@ -55,8 +66,9 @@ def export_data( with_data=with_data, select_columns=select_columns, file_type=file_type, - template=template, + template=template_bool, filters=filters, + export_without_column_meta=export_without_column_meta_bool, ) exporter.build_response() @@ -72,6 +84,7 @@ class DataExporter: file_type="CSV", template=False, filters=None, + export_without_column_meta=False, ): self.doctype = doctype self.parent_doctype = parent_doctype @@ -81,6 +94,7 @@ class DataExporter: self.file_type = file_type self.template = template self.filters = filters + self.export_without_column_meta = export_without_column_meta self.data_keys = get_data_keys() self.prepare_args() @@ -117,7 +131,10 @@ class DataExporter: if self.template: self.add_main_header() - self.writer.writerow([""]) + # No need of empty row at the start + if not self.export_without_column_meta: + self.writer.writerow([""]) + self.tablerow = [self.data_keys.doctype] self.labelrow = [_("Column Labels:")] self.fieldrow = [self.data_keys.columns] @@ -310,12 +327,18 @@ class DataExporter: return "" def add_field_headings(self): - self.writer.writerow(self.tablerow) + if not self.export_without_column_meta: + self.writer.writerow(self.tablerow) + + # Just include Labels in the first row self.writer.writerow(self.labelrow) - self.writer.writerow(self.fieldrow) - self.writer.writerow(self.mandatoryrow) - self.writer.writerow(self.typerow) - self.writer.writerow(self.inforow) + + if not self.export_without_column_meta: + self.writer.writerow(self.fieldrow) + self.writer.writerow(self.mandatoryrow) + self.writer.writerow(self.typerow) + self.writer.writerow(self.inforow) + if self.template: self.writer.writerow([self.data_keys.data_separator]) @@ -368,6 +391,8 @@ class DataExporter: if self.all_doctypes: # add child tables for c in self.child_doctypes: + if is_virtual_doctype(c["doctype"]): + continue child_doctype_table = DocType(c["doctype"]) data_row = ( frappe.qb.from_(child_doctype_table) @@ -410,23 +435,20 @@ class DataExporter: row[_column_start_end.start + i + 1] = value def build_response_as_excel(self): - filename = frappe.generate_hash("", 10) + from frappe.desk.utils import provide_binary_file + from frappe.utils.xlsxutils import make_xlsx + + filename = frappe.generate_hash(length=10) with open(filename, "wb") as f: f.write(cstr(self.writer.getvalue()).encode("utf-8")) f = open(filename) reader = csv.reader(f) - - from frappe.utils.xlsxutils import make_xlsx - xlsx_file = make_xlsx(reader, "Data Import Template" if self.template else "Data Export") f.close() os.remove(filename) - # write out response as a xlsx type - frappe.response["filename"] = self.doctype + ".xlsx" - frappe.response["filecontent"] = xlsx_file.getvalue() - frappe.response["type"] = "binary" + provide_binary_file(self.doctype, "xlsx", xlsx_file.getvalue()) def _append_name_column(self, dt=None): self.append_field_column( diff --git a/frappe/core/doctype/data_export/test_data_exporter.py b/frappe/core/doctype/data_export/test_data_exporter.py index 812f65aaad..eb3ebaa80d 100644 --- a/frappe/core/doctype/data_export/test_data_exporter.py +++ b/frappe/core/doctype/data_export/test_data_exporter.py @@ -1,12 +1,11 @@ # Copyright (c) 2019, Frappe Technologies and Contributors # License: MIT. See LICENSE -import unittest - import frappe from frappe.core.doctype.data_export.exporter import DataExporter +from frappe.tests.utils import FrappeTestCase -class TestDataExporter(unittest.TestCase): +class TestDataExporter(FrappeTestCase): def setUp(self): self.doctype_name = "Test DocType for Export Tool" self.doc_name = "Test Data for Export Tool" diff --git a/frappe/core/doctype/data_import/data_import.js b/frappe/core/doctype/data_import/data_import.js index dfc560a98a..7db3aa9629 100644 --- a/frappe/core/doctype/data_import/data_import.js +++ b/frappe/core/doctype/data_import/data_import.js @@ -1,17 +1,17 @@ // Copyright (c) 2019, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Data Import', { +frappe.ui.form.on("Data Import", { setup(frm) { - frappe.realtime.on('data_import_refresh', ({ data_import }) => { + frappe.realtime.on("data_import_refresh", ({ data_import }) => { frm.import_in_progress = false; if (data_import !== frm.doc.name) return; - frappe.model.clear_doc('Data Import', frm.doc.name); - frappe.model.with_doc('Data Import', frm.doc.name).then(() => { + frappe.model.clear_doc("Data Import", frm.doc.name); + frappe.model.with_doc("Data Import", frm.doc.name).then(() => { frm.refresh(); }); }); - frappe.realtime.on('data_import_progress', data => { + frappe.realtime.on("data_import_progress", (data) => { frm.import_in_progress = true; if (data.data_import !== frm.doc.name) { return; @@ -31,20 +31,16 @@ frappe.ui.form.on('Data Import', { if (data.success) { let message_args = [data.current, data.total, eta_message]; message = - frm.doc.import_type === 'Insert New Records' - ? __('Importing {0} of {1}, {2}', message_args) - : __('Updating {0} of {1}, {2}', message_args); + frm.doc.import_type === "Insert New Records" + ? __("Importing {0} of {1}, {2}", message_args) + : __("Updating {0} of {1}, {2}", message_args); } if (data.skipping) { - message = __('Skipping {0} of {1}, {2}', [ - data.current, - data.total, - eta_message - ]); + message = __("Skipping {0} of {1}, {2}", [data.current, data.total, eta_message]); } - frm.dashboard.show_progress(__('Import Progress'), percent, message); - frm.page.set_indicator(__('In Progress'), 'orange'); - frm.trigger('update_primary_action'); + frm.dashboard.show_progress(__("Import Progress"), percent, message); + frm.page.set_indicator(__("In Progress"), "orange"); + frm.trigger("update_primary_action"); // hide progress when complete if (data.current === data.total) { @@ -55,18 +51,18 @@ frappe.ui.form.on('Data Import', { } }); - frm.set_query('reference_doctype', () => { + frm.set_query("reference_doctype", () => { return { filters: { - name: ['in', frappe.boot.user.can_import] - } + name: ["in", frappe.boot.user.can_import], + }, }; }); - frm.get_field('import_file').df.options = { + frm.get_field("import_file").df.options = { restrictions: { - allowed_file_types: ['.csv', '.xls', '.xlsx'] - } + allowed_file_types: [".csv", ".xls", ".xlsx"], + }, }; frm.has_import_file = () => { @@ -76,33 +72,31 @@ frappe.ui.form.on('Data Import', { refresh(frm) { frm.page.hide_icon_group(); - frm.trigger('update_indicators'); - frm.trigger('import_file'); - frm.trigger('show_import_log'); - frm.trigger('show_import_warnings'); - frm.trigger('toggle_submit_after_import'); + frm.trigger("update_indicators"); + frm.trigger("import_file"); + frm.trigger("show_import_log"); + frm.trigger("show_import_warnings"); + frm.trigger("toggle_submit_after_import"); - if (frm.doc.status != 'Pending') - frm.trigger('show_import_status'); + if (frm.doc.status != "Pending") frm.trigger("show_import_status"); - frm.trigger('show_report_error_button'); + frm.trigger("show_report_error_button"); - if (frm.doc.status === 'Partial Success') { - frm.add_custom_button(__('Export Errored Rows'), () => - frm.trigger('export_errored_rows') + if (frm.doc.status === "Partial Success") { + frm.add_custom_button(__("Export Errored Rows"), () => + frm.trigger("export_errored_rows") ); } - if (frm.doc.status.includes('Success')) { - frm.add_custom_button( - __('Go to {0} List', [__(frm.doc.reference_doctype)]), - () => frappe.set_route('List', frm.doc.reference_doctype) + if (frm.doc.status.includes("Success")) { + frm.add_custom_button(__("Go to {0} List", [__(frm.doc.reference_doctype)]), () => + frappe.set_route("List", frm.doc.reference_doctype) ); } }, onload_post_render(frm) { - frm.trigger('update_primary_action'); + frm.trigger("update_primary_action"); }, update_primary_action(frm) { @@ -111,13 +105,12 @@ frappe.ui.form.on('Data Import', { return; } frm.disable_save(); - if (frm.doc.status !== 'Success') { - if (!frm.is_new() && (frm.has_import_file())) { - let label = - frm.doc.status === 'Pending' ? __('Start Import') : __('Retry'); + if (frm.doc.status !== "Success") { + if (!frm.is_new() && frm.has_import_file()) { + let label = frm.doc.status === "Pending" ? __("Start Import") : __("Retry"); frm.page.set_primary_action(label, () => frm.events.start_import(frm)); } else { - frm.page.set_primary_action(__('Save'), () => frm.save()); + frm.page.set_primary_action(__("Save"), () => frm.save()); } } }, @@ -133,11 +126,11 @@ frappe.ui.form.on('Data Import', { show_import_status(frm) { frappe.call({ - 'method': 'frappe.core.doctype.data_import.data_import.get_import_status', - 'args': { - 'data_import_name': frm.doc.name + method: "frappe.core.doctype.data_import.data_import.get_import_status", + args: { + data_import_name: frm.doc.name, }, - 'callback': function(r) { + callback: function (r) { let successful_records = cint(r.message.success); let failed_records = cint(r.message.failed); let total_records = cint(r.message.total_records); @@ -147,52 +140,64 @@ frappe.ui.form.on('Data Import', { let message; if (failed_records === 0) { let message_args = [successful_records]; - if (frm.doc.import_type === 'Insert New Records') { + if (frm.doc.import_type === "Insert New Records") { message = successful_records > 1 - ? __('Successfully imported {0} records.', message_args) - : __('Successfully imported {0} record.', message_args); + ? __("Successfully imported {0} records.", message_args) + : __("Successfully imported {0} record.", message_args); } else { message = successful_records > 1 - ? __('Successfully updated {0} records.', message_args) - : __('Successfully updated {0} record.', message_args); + ? __("Successfully updated {0} records.", message_args) + : __("Successfully updated {0} record.", message_args); } } else { let message_args = [successful_records, total_records]; - if (frm.doc.import_type === 'Insert New Records') { + if (frm.doc.import_type === "Insert New Records") { message = successful_records > 1 - ? __('Successfully imported {0} records out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args) - : __('Successfully imported {0} record out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args); + ? __( + "Successfully imported {0} records out of {1}. Click on Export Errored Rows, fix the errors and import again.", + message_args + ) + : __( + "Successfully imported {0} record out of {1}. Click on Export Errored Rows, fix the errors and import again.", + message_args + ); } else { message = successful_records > 1 - ? __('Successfully updated {0} records out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args) - : __('Successfully updated {0} record out of {1}. Click on Export Errored Rows, fix the errors and import again.', message_args); + ? __( + "Successfully updated {0} records out of {1}. Click on Export Errored Rows, fix the errors and import again.", + message_args + ) + : __( + "Successfully updated {0} record out of {1}. Click on Export Errored Rows, fix the errors and import again.", + message_args + ); } } frm.dashboard.set_headline(message); - } + }, }); }, show_report_error_button(frm) { - if (frm.doc.status === 'Error') { + if (frm.doc.status === "Error") { frappe.db - .get_list('Error Log', { + .get_list("Error Log", { filters: { method: frm.doc.name }, - fields: ['method', 'error'], - order_by: 'creation desc', - limit: 1 + fields: ["method", "error"], + order_by: "creation desc", + limit: 1, }) - .then(result => { + .then((result) => { if (result.length > 0) { - frm.add_custom_button('Report Error', () => { + frm.add_custom_button("Report Error", () => { let fake_xhr = { responseText: JSON.stringify({ - exc: result[0].error - }) + exc: result[0].error, + }), }; frappe.request.report_error(fake_xhr, {}); }); @@ -202,21 +207,19 @@ frappe.ui.form.on('Data Import', { }, start_import(frm) { - frm - .call({ - method: 'form_start_import', - args: { data_import: frm.doc.name }, - btn: frm.page.btn_primary - }) - .then(r => { - if (r.message === true) { - frm.disable_save(); - } - }); + frm.call({ + method: "form_start_import", + args: { data_import: frm.doc.name }, + btn: frm.page.btn_primary, + }).then((r) => { + if (r.message === true) { + frm.disable_save(); + } + }); }, download_template(frm) { - frappe.require('data_import_tools.bundle.js', () => { + frappe.require("data_import_tools.bundle.js", () => { frm.data_exporter = new frappe.data_import.DataExporter( frm.doc.reference_doctype, frm.doc.import_type @@ -225,127 +228,123 @@ frappe.ui.form.on('Data Import', { }, reference_doctype(frm) { - frm.trigger('toggle_submit_after_import'); + frm.trigger("toggle_submit_after_import"); }, toggle_submit_after_import(frm) { - frm.toggle_display('submit_after_import', false); + frm.toggle_display("submit_after_import", false); let doctype = frm.doc.reference_doctype; if (doctype) { frappe.model.with_doctype(doctype, () => { let meta = frappe.get_meta(doctype); - frm.toggle_display('submit_after_import', meta.is_submittable); + frm.toggle_display("submit_after_import", meta.is_submittable); }); } }, google_sheets_url(frm) { if (!frm.is_dirty()) { - frm.trigger('import_file'); + frm.trigger("import_file"); } else { - frm.trigger('update_primary_action'); + frm.trigger("update_primary_action"); } }, refresh_google_sheet(frm) { - frm.trigger('import_file'); + frm.trigger("import_file"); }, import_file(frm) { - frm.toggle_display('section_import_preview', frm.has_import_file()); + frm.toggle_display("section_import_preview", frm.has_import_file()); if (!frm.has_import_file()) { - frm.get_field('import_preview').$wrapper.empty(); + frm.get_field("import_preview").$wrapper.empty(); return; } else { - frm.trigger('update_primary_action'); + frm.trigger("update_primary_action"); } // load import preview - frm.get_field('import_preview').$wrapper.empty(); + frm.get_field("import_preview").$wrapper.empty(); $('') - .html(__('Loading import file...')) - .appendTo(frm.get_field('import_preview').$wrapper); + .html(__("Loading import file...")) + .appendTo(frm.get_field("import_preview").$wrapper); - frm - .call({ - method: 'get_preview_from_template', - args: { - data_import: frm.doc.name, - import_file: frm.doc.import_file, - google_sheets_url: frm.doc.google_sheets_url + frm.call({ + method: "get_preview_from_template", + args: { + data_import: frm.doc.name, + import_file: frm.doc.import_file, + google_sheets_url: frm.doc.google_sheets_url, + }, + error_handlers: { + TimestampMismatchError() { + // ignore this error }, - error_handlers: { - TimestampMismatchError() { - // ignore this error - } - } - }) - .then(r => { - let preview_data = r.message; - frm.events.show_import_preview(frm, preview_data); - frm.events.show_import_warnings(frm, preview_data); - }); + }, + }).then((r) => { + let preview_data = r.message; + frm.events.show_import_preview(frm, preview_data); + frm.events.show_import_warnings(frm, preview_data); + }); }, show_import_preview(frm, preview_data) { let import_log = preview_data.import_log; - if ( - frm.import_preview && - frm.import_preview.doctype === frm.doc.reference_doctype - ) { + if (frm.import_preview && frm.import_preview.doctype === frm.doc.reference_doctype) { frm.import_preview.preview_data = preview_data; frm.import_preview.import_log = import_log; frm.import_preview.refresh(); return; } - frappe.require('data_import_tools.bundle.js', () => { + frappe.require("data_import_tools.bundle.js", () => { frm.import_preview = new frappe.data_import.ImportPreview({ - wrapper: frm.get_field('import_preview').$wrapper, + wrapper: frm.get_field("import_preview").$wrapper, doctype: frm.doc.reference_doctype, preview_data, import_log, frm, events: { remap_column(changed_map) { - let template_options = JSON.parse(frm.doc.template_options || '{}'); - template_options.column_to_field_map = template_options.column_to_field_map || {}; + let template_options = JSON.parse(frm.doc.template_options || "{}"); + template_options.column_to_field_map = + template_options.column_to_field_map || {}; Object.assign(template_options.column_to_field_map, changed_map); - frm.set_value('template_options', JSON.stringify(template_options)); - frm.save().then(() => frm.trigger('import_file')); - } - } + frm.set_value("template_options", JSON.stringify(template_options)); + frm.save().then(() => frm.trigger("import_file")); + }, + }, }); }); }, export_errored_rows(frm) { open_url_post( - '/api/method/frappe.core.doctype.data_import.data_import.download_errored_template', + "/api/method/frappe.core.doctype.data_import.data_import.download_errored_template", { - data_import_name: frm.doc.name + data_import_name: frm.doc.name, } ); }, export_import_log(frm) { open_url_post( - '/api/method/frappe.core.doctype.data_import.data_import.download_import_log', + "/api/method/frappe.core.doctype.data_import.data_import.download_import_log", { - data_import_name: frm.doc.name + data_import_name: frm.doc.name, } ); }, show_import_warnings(frm, preview_data) { let columns = preview_data.columns; - let warnings = JSON.parse(frm.doc.template_warnings || '[]'); + let warnings = JSON.parse(frm.doc.template_warnings || "[]"); warnings = warnings.concat(preview_data.warnings || []); - frm.toggle_display('import_warnings_section', warnings.length > 0); + frm.toggle_display("import_warnings_section", warnings.length > 0); if (warnings.length === 0) { - frm.get_field('import_warnings').$wrapper.html(''); + frm.get_field("import_warnings").$wrapper.html(""); return; } @@ -361,36 +360,38 @@ frappe.ui.form.on('Data Import', { } } - let html = ''; + let html = ""; html += Object.keys(warnings_by_row) - .map(row_number => { + .map((row_number) => { let message = warnings_by_row[row_number] - .map(w => { + .map((w) => { if (w.field) { let label = w.field.label + (w.field.parent !== frm.doc.reference_doctype ? ` (${w.field.parent})` - : ''); + : ""); return `
  • ${label}: ${w.message}
    • `; } return `
  • ${w.message}
    • `; }) - .join(''); + .join(""); return `
    -
    ${__('Row {0}', [row_number])}
    +
    ${__("Row {0}", [row_number])}
      ${message}
    `; }) - .join(''); + .join(""); html += other_warnings - .map(warning => { - let header = ''; + .map((warning) => { + let header = ""; if (warning.col) { - let column_number = `${__('Column {0}', [warning.col])}`; + let column_number = `${__("Column {0}", [ + warning.col, + ])}`; let column_header = columns[warning.col].header_title; header = `${column_number} (${column_header})`; } @@ -401,8 +402,8 @@ frappe.ui.form.on('Data Import', {
    `; }) - .join(''); - frm.get_field('import_warnings').$wrapper.html(` + .join(""); + frm.get_field("import_warnings").$wrapper.html(`
    ${html}
    @@ -410,62 +411,62 @@ frappe.ui.form.on('Data Import', { }, show_failed_logs(frm) { - frm.trigger('show_import_log'); + frm.trigger("show_import_log"); }, render_import_log(frm) { frappe.call({ - 'method': 'frappe.client.get_list', - 'args': { - 'doctype': 'Data Import Log', - 'filters': { - 'data_import': frm.doc.name + method: "frappe.client.get_list", + args: { + doctype: "Data Import Log", + filters: { + data_import: frm.doc.name, }, - 'fields': ['success', 'docname', 'messages', 'exception', 'row_indexes'], - 'limit_page_length': 5000, - 'order_by': 'log_index' + fields: ["success", "docname", "messages", "exception", "row_indexes"], + limit_page_length: 5000, + order_by: "log_index", }, - callback: function(r) { + callback: function (r) { let logs = r.message; if (logs.length === 0) return; - frm.toggle_display('import_log_section', true); + frm.toggle_display("import_log_section", true); let rows = logs - .map(log => { - let html = ''; + .map((log) => { + let html = ""; if (log.success) { - if (frm.doc.import_type === 'Insert New Records') { - html = __('Successfully imported {0}', [ + if (frm.doc.import_type === "Insert New Records") { + html = __("Successfully imported {0}", [ `${frappe.utils.get_form_link( frm.doc.reference_doctype, log.docname, true - )}` + )}`, ]); } else { - html = __('Successfully updated {0}', [ + html = __("Successfully updated {0}", [ `${frappe.utils.get_form_link( frm.doc.reference_doctype, log.docname, true - )}` + )}`, ]); } } else { - let messages = (JSON.parse(log.messages || '[]')) + let messages = JSON.parse(log.messages || "[]") .map(JSON.parse) - .map(m => { - let title = m.title ? `${m.title}` : ''; - let message = m.message ? `
    ${m.message}
    ` : ''; + .map((m) => { + let title = m.title ? `${m.title}` : ""; + let message = m.message ? `
    ${m.message}
    ` : ""; return title + message; }) - .join(''); + .join(""); let id = frappe.dom.get_unique_id(); html = `${messages}
    @@ -473,15 +474,15 @@ frappe.ui.form.on('Data Import', {
    `; } - let indicator_color = log.success ? 'green' : 'red'; - let title = log.success ? __('Success') : __('Failure'); + let indicator_color = log.success ? "green" : "red"; + let title = log.success ? __("Success") : __("Failure"); if (frm.doc.show_failed_logs && log.success) { - return ''; + return ""; } return ` - ${JSON.parse(log.row_indexes).join(', ')} + ${JSON.parse(log.row_indexes).join(", ")}
    ${title}
    @@ -490,54 +491,54 @@ frappe.ui.form.on('Data Import', { `; }) - .join(''); + .join(""); if (!rows && frm.doc.show_failed_logs) { rows = ` - ${__('No failed logs')} + ${__("No failed logs")} `; } - frm.get_field('import_log_preview').$wrapper.html(` + frm.get_field("import_log_preview").$wrapper.html(` - - - + + + ${rows}
    ${__('Row Number')}${__('Status')}${__('Message')}${__("Row Number")}${__("Status")}${__("Message")}
    `); - } + }, }); }, show_import_log(frm) { - frm.toggle_display('import_log_section', false); + frm.toggle_display("import_log_section", false); if (frm.import_in_progress) { return; } frappe.call({ - 'method': 'frappe.client.get_count', - 'args': { - 'doctype': 'Data Import Log', - 'filters': { - 'data_import': frm.doc.name - } + method: "frappe.client.get_count", + args: { + doctype: "Data Import Log", + filters: { + data_import: frm.doc.name, + }, }, - 'callback': function(r) { + callback: function (r) { let count = r.message; if (count < 5000) { - frm.trigger('render_import_log'); + frm.trigger("render_import_log"); } else { - frm.toggle_display('import_log_section', false); - frm.add_custom_button(__('Export Import Log'), () => - frm.trigger('export_import_log') + frm.toggle_display("import_log_section", false); + frm.add_custom_button(__("Export Import Log"), () => + frm.trigger("export_import_log") ); } - } + }, }); }, }); diff --git a/frappe/core/doctype/data_import/data_import.json b/frappe/core/doctype/data_import/data_import.json index 9e948dac8c..faa9a33bf1 100644 --- a/frappe/core/doctype/data_import/data_import.json +++ b/frappe/core/doctype/data_import/data_import.json @@ -94,6 +94,7 @@ "fieldtype": "Select", "hidden": 1, "label": "Status", + "no_copy": 1, "options": "Pending\nSuccess\nPartial Success\nError", "read_only": 1 }, @@ -170,7 +171,7 @@ ], "hide_toolbar": 1, "links": [], - "modified": "2022-02-01 20:08:37.624914", + "modified": "2022-02-14 10:08:37.624914", "modified_by": "Administrator", "module": "Core", "name": "Data Import", @@ -194,4 +195,4 @@ "sort_order": "DESC", "states": [], "track_changes": 1 -} \ No newline at end of file +} diff --git a/frappe/core/doctype/data_import/data_import.py b/frappe/core/doctype/data_import/data_import.py index 5ad603e416..22c47be692 100644 --- a/frappe/core/doctype/data_import/data_import.py +++ b/frappe/core/doctype/data_import/data_import.py @@ -9,7 +9,7 @@ from frappe.core.doctype.data_import.exporter import Exporter from frappe.core.doctype.data_import.importer import Importer from frappe.model.document import Document from frappe.modules.import_file import import_file_by_path -from frappe.utils.background_jobs import enqueue +from frappe.utils.background_jobs import enqueue, is_job_enqueued from frappe.utils.csvutils import validate_google_sheets_url @@ -59,21 +59,20 @@ class DataImport(Document): return i.get_data_for_import_preview() def start_import(self): - from frappe.core.page.background_jobs.background_jobs import get_info from frappe.utils.scheduler import is_scheduler_inactive if is_scheduler_inactive() and not frappe.flags.in_test: frappe.throw(_("Scheduler is inactive. Cannot import data."), title=_("Scheduler Inactive")) - enqueued_jobs = [d.get("job_name") for d in get_info()] + job_id = f"data_import::{self.name}" - if self.name not in enqueued_jobs: + if not is_job_enqueued(job_id): enqueue( start_import, queue="default", timeout=10000, event="data_import", - job_name=self.name, + job_id=job_id, data_import=self.name, now=frappe.conf.developer_mode or frappe.flags.in_test, ) @@ -263,7 +262,7 @@ def export_json(doctype, path, filters=None, or_filters=None, name=None, order_b path = os.path.join("..", path) with open(path, "w") as outfile: - outfile.write(frappe.as_json(out)) + outfile.write(frappe.as_json(out, ensure_ascii=False)) def export_csv(doctype, path): diff --git a/frappe/core/doctype/data_import/data_import_list.js b/frappe/core/doctype/data_import/data_import_list.js index 6ab750ba25..c054655e62 100644 --- a/frappe/core/doctype/data_import/data_import_list.js +++ b/frappe/core/doctype/data_import/data_import_list.js @@ -1,46 +1,44 @@ let imports_in_progress = []; -frappe.listview_settings['Data Import'] = { +frappe.listview_settings["Data Import"] = { onload(listview) { - frappe.realtime.on('data_import_progress', data => { + frappe.realtime.on("data_import_progress", (data) => { if (!imports_in_progress.includes(data.data_import)) { imports_in_progress.push(data.data_import); } }); - frappe.realtime.on('data_import_refresh', data => { - imports_in_progress = imports_in_progress.filter( - d => d !== data.data_import - ); + frappe.realtime.on("data_import_refresh", (data) => { + imports_in_progress = imports_in_progress.filter((d) => d !== data.data_import); listview.refresh(); }); }, - get_indicator: function(doc) { + get_indicator: function (doc) { var colors = { - 'Pending': 'orange', - 'Not Started': 'orange', - 'Partial Success': 'orange', - 'Success': 'green', - 'In Progress': 'orange', - 'Error': 'red' + Pending: "orange", + "Not Started": "orange", + "Partial Success": "orange", + Success: "green", + "In Progress": "orange", + Error: "red", }; let status = doc.status; if (imports_in_progress.includes(doc.name)) { - status = 'In Progress'; + status = "In Progress"; } - if (status == 'Pending') { - status = 'Not Started'; + if (status == "Pending") { + status = "Not Started"; } - return [__(status), colors[status], 'status,=,' + doc.status]; + return [__(status), colors[status], "status,=," + doc.status]; }, formatters: { import_type(value) { return { - 'Insert New Records': __('Insert'), - 'Update Existing Records': __('Update') + "Insert New Records": __("Insert"), + "Update Existing Records": __("Update"), }[value]; - } + }, }, - hide_name_column: true + hide_name_column: true, }; diff --git a/frappe/core/doctype/data_import/exporter.py b/frappe/core/doctype/data_import/exporter.py index 8c73391bd0..88d5b2b4b1 100644 --- a/frappe/core/doctype/data_import/exporter.py +++ b/frappe/core/doctype/data_import/exporter.py @@ -1,8 +1,6 @@ # Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import typing - import frappe from frappe import _ from frappe.model import display_fieldtypes, no_value_fields @@ -183,7 +181,7 @@ class Exporter: child_fields = ["name", "idx", "parent", "parentfield"] + list( {format_column_name(df) for df in self.fields if df.parent == child_table_doctype} ) - data = frappe.db.get_all( + data = frappe.get_all( child_table_doctype, filters={ "parent": ("in", parent_names), @@ -199,7 +197,7 @@ class Exporter: # Group children data by parent name grouped_children_data = self.group_children_data_by_parent(child_data) for doc in parent_data: - related_children_docs = grouped_children_data.get(doc.name, {}) + related_children_docs = grouped_children_data.get(str(doc.name), {}) yield {**doc, **related_children_docs} def add_header(self): @@ -207,9 +205,11 @@ class Exporter: for df in self.fields: is_parent = not df.is_child_table_field if is_parent: - label = _(df.label) + label = _(df.label or df.fieldname) else: - label = f"{_(df.label)} ({_(df.child_table_df.label)})" + label = ( + f"{_(df.label or df.fieldname)} ({_(df.child_table_df.label or df.child_table_df.fieldname)})" + ) if label in header: # this label is already in the header, @@ -241,15 +241,9 @@ class Exporter: def build_response(self): if self.file_type == "CSV": - self.build_csv_response() + build_csv_response(self.get_csv_array_for_export(), _(self.doctype)) elif self.file_type == "Excel": - self.build_xlsx_response() - - def build_csv_response(self): - build_csv_response(self.get_csv_array_for_export(), _(self.doctype)) - - def build_xlsx_response(self): - build_xlsx_response(self.get_csv_array_for_export(), _(self.doctype)) + build_xlsx_response(self.get_csv_array_for_export(), _(self.doctype)) def group_children_data_by_parent(self, children_data: dict[str, list]): return groupby_metric(children_data, key="parent") diff --git a/frappe/core/doctype/data_import/importer.py b/frappe/core/doctype/data_import/importer.py index 9e741ab590..20a8e7db9b 100644 --- a/frappe/core/doctype/data_import/importer.py +++ b/frappe/core/doctype/data_import/importer.py @@ -6,7 +6,7 @@ import json import os import re import timeit -from datetime import date, datetime +from datetime import date, datetime, time import frappe from frappe import _ @@ -50,7 +50,7 @@ class Importer: def get_data_for_import_preview(self): out = self.import_file.get_data_for_import_preview() - out.import_log = frappe.db.get_all( + out.import_log = frappe.get_all( "Data Import Log", fields=["row_indexes", "success"], filters={"data_import": self.data_import.name}, @@ -90,7 +90,7 @@ class Importer: # setup import log import_log = ( - frappe.db.get_all( + frappe.get_all( "Data Import Log", fields=["row_indexes", "success", "log_index"], filters={"data_import": self.data_import.name}, @@ -139,6 +139,7 @@ class Importer: "skipping": True, "data_import": self.data_import.name, }, + user=frappe.session.user, ) continue @@ -166,6 +167,7 @@ class Importer: "row_indexes": row_indexes, "eta": eta, }, + user=frappe.session.user, ) create_import_log( @@ -204,7 +206,7 @@ class Importer: # Logs are db inserted directly so will have to be fetched again import_log = ( - frappe.db.get_all( + frappe.get_all( "Data Import Log", fields=["row_indexes", "success", "log_index"], filters={"data_import": self.data_import.name}, @@ -297,7 +299,7 @@ class Importer: return import_log = ( - frappe.db.get_all( + frappe.get_all( "Data Import Log", fields=["row_indexes", "success"], filters={"data_import": self.data_import.name}, @@ -327,7 +329,7 @@ class Importer: if not self.data_import: return - import_log = frappe.db.get_all( + import_log = frappe.get_all( "Data Import Log", fields=["row_indexes", "success", "messages", "exception", "docname"], filters={"data_import": self.data_import.name}, @@ -449,7 +451,7 @@ class ImportFile: continue if not header: - header = Header(i, row, self.doctype, self.raw_data, self.column_to_field_map) + header = Header(i, row, self.doctype, self.raw_data[1:], self.column_to_field_map) else: row_obj = Row(i, row, self.doctype, header, self.import_type) data.append(row_obj) @@ -572,12 +574,15 @@ class ImportFile: ###### - def read_file(self, file_path): + def read_file(self, file_path: str): extn = os.path.splitext(file_path)[1][1:] file_content = None - with open(file_path, mode="rb") as f: - file_content = f.read() + + file_name = frappe.db.get_value("File", {"file_url": file_path}) + if file_name: + file = frappe.get_doc("File", file_name) + file_content = file.get_content() return file_content, extn @@ -937,11 +942,13 @@ class Column: """ def guess_date_format(d): - if isinstance(d, (datetime, date)): + if isinstance(d, (datetime, date, time)): if self.df.fieldtype == "Date": return "%Y-%m-%d" if self.df.fieldtype == "Datetime": return "%Y-%m-%d %H:%M:%S" + if self.df.fieldtype == "Time": + return "%H:%M:%S" if isinstance(d, str): return frappe.utils.guess_date_format(d) @@ -979,49 +986,60 @@ class Column: if self.skip_import: return + if not any(self.column_values): + return + if self.df.fieldtype == "Link": # find all values that dont exist - values = list({cstr(v) for v in self.column_values[1:] if v}) - exists = [d.name for d in frappe.db.get_all(self.df.options, filters={"name": ("in", values)})] + values = list({cstr(v) for v in self.column_values if v}) + exists = [ + cstr(d.name) for d in frappe.get_all(self.df.options, filters={"name": ("in", values)}) + ] not_exists = list(set(values) - set(exists)) if not_exists: missing_values = ", ".join(not_exists) + message = _("The following values do not exist for {0}: {1}") self.warnings.append( { "col": self.column_number, - "message": (f"The following values do not exist for {self.df.options}: {missing_values}"), + "message": message.format(self.df.options, missing_values), "type": "warning", } ) elif self.df.fieldtype in ("Date", "Time", "Datetime"): - # guess date format + # guess date/time format self.date_format = self.guess_date_format_for_column() if not self.date_format: - self.date_format = "%Y-%m-%d" + if self.df.fieldtype == "Time": + self.date_format = "%H:%M:%S" + date_format = "HH:mm:ss" + else: + self.date_format = "%Y-%m-%d" + date_format = "yyyy-mm-dd" + + message = _( + "{0} format could not be determined from the values in this column. Defaulting to {1}." + ) self.warnings.append( { "col": self.column_number, - "message": _( - "Date format could not be determined from the values in this column. Defaulting to yyyy-mm-dd." - ), + "message": message.format(self.df.fieldtype, date_format), "type": "info", } ) elif self.df.fieldtype == "Select": options = get_select_options(self.df) if options: - values = {cstr(v) for v in self.column_values[1:] if v} + values = {cstr(v) for v in self.column_values if v} invalid = values - set(options) if invalid: valid_values = ", ".join(frappe.bold(o) for o in options) invalid_values = ", ".join(frappe.bold(i) for i in invalid) + message = _("The following values are invalid: {0}. Values must be one of {1}") self.warnings.append( { "col": self.column_number, - "message": ( - "The following values are invalid: {}. Values must be" - " one of {}".format(invalid_values, valid_values) - ), + "message": message.format(invalid_values, valid_values), } ) diff --git a/frappe/core/doctype/data_import/test_data_import.py b/frappe/core/doctype/data_import/test_data_import.py index 253882383c..10f8098b2b 100644 --- a/frappe/core/doctype/data_import/test_data_import.py +++ b/frappe/core/doctype/data_import/test_data_import.py @@ -1,8 +1,8 @@ # Copyright (c) 2020, Frappe Technologies and Contributors # License: MIT. See LICENSE # import frappe -import unittest +from frappe.tests.utils import FrappeTestCase -class TestDataImport(unittest.TestCase): +class TestDataImport(FrappeTestCase): pass diff --git a/frappe/core/doctype/data_import/test_exporter.py b/frappe/core/doctype/data_import/test_exporter.py index ceeac90e36..cd7f91d079 100644 --- a/frappe/core/doctype/data_import/test_exporter.py +++ b/frappe/core/doctype/data_import/test_exporter.py @@ -1,15 +1,14 @@ # Copyright (c) 2019, Frappe Technologies and Contributors # License: MIT. See LICENSE -import unittest - import frappe from frappe.core.doctype.data_import.exporter import Exporter from frappe.core.doctype.data_import.test_importer import create_doctype_if_not_exists +from frappe.tests.utils import FrappeTestCase doctype_name = "DocType for Export" -class TestExporter(unittest.TestCase): +class TestExporter(FrappeTestCase): def setUp(self): create_doctype_if_not_exists(doctype_name) diff --git a/frappe/core/doctype/data_import/test_importer.py b/frappe/core/doctype/data_import/test_importer.py index 2c250a4e87..978f5792dd 100644 --- a/frappe/core/doctype/data_import/test_importer.py +++ b/frappe/core/doctype/data_import/test_importer.py @@ -1,18 +1,18 @@ # Copyright (c) 2019, Frappe Technologies and Contributors # License: MIT. See LICENSE -import unittest - import frappe from frappe.core.doctype.data_import.importer import Importer from frappe.tests.test_query_builder import db_type_is, run_only_if +from frappe.tests.utils import FrappeTestCase from frappe.utils import format_duration, getdate doctype_name = "DocType for Import" -class TestImporter(unittest.TestCase): +class TestImporter(FrappeTestCase): @classmethod def setUpClass(cls): + super().setUpClass() create_doctype_if_not_exists( doctype_name, ) @@ -67,7 +67,7 @@ class TestImporter(unittest.TestCase): data_import.start_import() data_import.reload() - import_log = frappe.db.get_all( + import_log = frappe.get_all( "Data Import Log", fields=["row_indexes", "success", "messages", "exception", "docname"], filters={"data_import": data_import.name}, @@ -97,7 +97,7 @@ class TestImporter(unittest.TestCase): def test_data_import_update(self): existing_doc = frappe.get_doc( doctype=doctype_name, - title=frappe.generate_hash(doctype_name, 8), + title=frappe.generate_hash(length=8), table_field_1=[{"child_title": "child title to update"}], ) existing_doc.save() diff --git a/frappe/core/doctype/data_import_log/data_import_log.js b/frappe/core/doctype/data_import_log/data_import_log.js index c376edeec9..19ba0eb727 100644 --- a/frappe/core/doctype/data_import_log/data_import_log.js +++ b/frappe/core/doctype/data_import_log/data_import_log.js @@ -1,8 +1,7 @@ // Copyright (c) 2021, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Data Import Log', { +frappe.ui.form.on("Data Import Log", { // refresh: function(frm) { - // } }); diff --git a/frappe/core/doctype/data_import_log/test_data_import_log.py b/frappe/core/doctype/data_import_log/test_data_import_log.py index 6ae7c532bf..6db1b87b9b 100644 --- a/frappe/core/doctype/data_import_log/test_data_import_log.py +++ b/frappe/core/doctype/data_import_log/test_data_import_log.py @@ -2,8 +2,8 @@ # See license.txt # import frappe -import unittest +from frappe.tests.utils import FrappeTestCase -class TestDataImportLog(unittest.TestCase): +class TestDataImportLog(FrappeTestCase): pass diff --git a/frappe/core/doctype/defaultvalue/defaultvalue.json b/frappe/core/doctype/defaultvalue/defaultvalue.json index 35b08c2dca..22e2583774 100644 --- a/frappe/core/doctype/defaultvalue/defaultvalue.json +++ b/frappe/core/doctype/defaultvalue/defaultvalue.json @@ -1,90 +1,47 @@ { - "allow_copy": 0, - "allow_import": 0, - "allow_rename": 0, - "autoname": "hash", - "beta": 0, - "creation": "2013-02-22 01:27:32", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "editable_grid": 1, + "actions": [], + "autoname": "hash", + "creation": "2013-02-22 01:27:32", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "defkey", + "defvalue" + ], "fields": [ { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "fieldname": "defkey", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 1, - "label": "Key", - "length": 0, - "no_copy": 0, - "oldfieldname": "defkey", - "oldfieldtype": "Data", - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "print_width": "200px", - "read_only": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0, + "fieldname": "defkey", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Key", + "oldfieldname": "defkey", + "oldfieldtype": "Data", + "print_width": "200px", + "reqd": 1, "width": "200px" - }, + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "fieldname": "defvalue", - "fieldtype": "Text", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 1, - "label": "Value", - "length": 0, - "no_copy": 0, - "oldfieldname": "defvalue", - "oldfieldtype": "Text", - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "print_width": "200px", - "read_only": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0, + "fieldname": "defvalue", + "fieldtype": "Text", + "in_list_view": 1, + "label": "Value", + "oldfieldname": "defvalue", + "oldfieldtype": "Text", + "print_width": "200px", "width": "200px" } - ], - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 1, - "image_view": 0, - "in_create": 0, - - "is_submittable": 0, - "issingle": 0, - "istable": 1, - "max_attachments": 0, - "modified": "2016-07-11 03:27:59.126216", - "modified_by": "Administrator", - "module": "Core", - "name": "DefaultValue", - "owner": "Administrator", - "permissions": [], - "quick_entry": 0, - "read_only": 0, - "read_only_onload": 0, - "track_seen": 0 + ], + "idx": 1, + "istable": 1, + "links": [], + "modified": "2022-08-03 12:20:54.832785", + "modified_by": "Administrator", + "module": "Core", + "name": "DefaultValue", + "owner": "Administrator", + "permissions": [], + "sort_field": "modified", + "sort_order": "DESC", + "states": [] } \ No newline at end of file diff --git a/frappe/core/doctype/deleted_document/deleted_document.js b/frappe/core/doctype/deleted_document/deleted_document.js index 3125cb2f1c..34e13c01ab 100644 --- a/frappe/core/doctype/deleted_document/deleted_document.js +++ b/frappe/core/doctype/deleted_document/deleted_document.js @@ -1,22 +1,22 @@ // Copyright (c) 2016, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Deleted Document', { - refresh: function(frm) { - if(frm.doc.restored) { - frm.add_custom_button(__('Open'), function() { - frappe.set_route('Form', frm.doc.deleted_doctype, frm.doc.new_name); +frappe.ui.form.on("Deleted Document", { + refresh: function (frm) { + if (frm.doc.restored) { + frm.add_custom_button(__("Open"), function () { + frappe.set_route("Form", frm.doc.deleted_doctype, frm.doc.new_name); }); } else { - frm.add_custom_button(__('Restore'), function() { + frm.add_custom_button(__("Restore"), function () { frappe.call({ - method: 'frappe.core.doctype.deleted_document.deleted_document.restore', - args: {name: frm.doc.name}, - callback: function(r) { + method: "frappe.core.doctype.deleted_document.deleted_document.restore", + args: { name: frm.doc.name }, + callback: function (r) { frm.reload_doc(); - } + }, }); }); } - } + }, }); diff --git a/frappe/core/doctype/deleted_document/deleted_document.py b/frappe/core/doctype/deleted_document/deleted_document.py index 14b9bb5c11..9aa8e41708 100644 --- a/frappe/core/doctype/deleted_document/deleted_document.py +++ b/frappe/core/doctype/deleted_document/deleted_document.py @@ -7,6 +7,7 @@ import frappe from frappe import _ from frappe.desk.doctype.bulk_update.bulk_update import show_progress from frappe.model.document import Document +from frappe.model.workflow import get_workflow_name class DeletedDocument(Document): @@ -27,6 +28,11 @@ def restore(name, alert=True): except frappe.DocstatusTransitionError: frappe.msgprint(_("Cancelled Document restored as Draft")) doc.docstatus = 0 + active_workflow = get_workflow_name(doc.doctype) + if active_workflow: + workflow_state_fieldname = frappe.get_value("Workflow", active_workflow, "workflow_state_field") + if doc.get(workflow_state_fieldname): + doc.set(workflow_state_fieldname, None) doc.insert() doc.add_comment("Edit", _("restored {0} as {1}").format(deleted.deleted_name, doc.name)) diff --git a/frappe/core/doctype/deleted_document/deleted_document_list.js b/frappe/core/doctype/deleted_document/deleted_document_list.js index 92413bfdf4..6a271f5ae9 100644 --- a/frappe/core/doctype/deleted_document/deleted_document_list.js +++ b/frappe/core/doctype/deleted_document/deleted_document_list.js @@ -3,27 +3,36 @@ frappe.listview_settings["Deleted Document"] = { const action = () => { const selected_docs = doclist.get_checked_items(); if (selected_docs.length > 0) { - let docnames = selected_docs.map(doc => doc.name); + let docnames = selected_docs.map((doc) => doc.name); frappe.call({ method: "frappe.core.doctype.deleted_document.deleted_document.bulk_restore", args: { docnames }, callback: function (r) { if (r.message) { let body = (docnames) => { - const html = docnames.map(docname => { + const html = docnames.map((docname) => { return `
  • ${docname}
    • `; }); return "
      " + html.join(""); }; let message = (title, docnames) => { - return (docnames.length > 0) ? title + body(docnames) + "
    ": ""; + return docnames.length > 0 ? title + body(docnames) + "" : ""; }; const { restored, invalid, failed } = r.message; - const restored_summary = message(__("Documents restored successfully"), restored); - const invalid_summary = message(__("Documents that were already restored"), invalid); - const failed_summary = message(__("Documents that failed to restore"), failed); + const restored_summary = message( + __("Documents restored successfully"), + restored + ); + const invalid_summary = message( + __("Documents that were already restored"), + invalid + ); + const failed_summary = message( + __("Documents that failed to restore"), + failed + ); const summary = restored_summary + invalid_summary + failed_summary; frappe.msgprint(summary, __("Document Restoration Summary"), true); diff --git a/frappe/core/doctype/deleted_document/test_deleted_document.py b/frappe/core/doctype/deleted_document/test_deleted_document.py index f696689664..28d494556f 100644 --- a/frappe/core/doctype/deleted_document/test_deleted_document.py +++ b/frappe/core/doctype/deleted_document/test_deleted_document.py @@ -1,9 +1,9 @@ # Copyright (c) 2015, Frappe Technologies and Contributors # License: MIT. See LICENSE -import unittest +from frappe.tests.utils import FrappeTestCase # test_records = frappe.get_test_records('Deleted Document') -class TestDeletedDocument(unittest.TestCase): +class TestDeletedDocument(FrappeTestCase): pass diff --git a/frappe/core/doctype/docfield/docfield.json b/frappe/core/doctype/docfield/docfield.json index 803ad3c140..90b1c6cb77 100644 --- a/frappe/core/doctype/docfield/docfield.json +++ b/frappe/core/doctype/docfield/docfield.json @@ -70,6 +70,7 @@ "columns", "column_break_22", "description", + "documentation_url", "oldfieldname", "oldfieldtype" ], @@ -303,6 +304,7 @@ }, { "default": "0", + "depends_on": "eval:!in_list(['Section Break', 'Column Break', 'Tab Break'], doc.fieldtype)", "fieldname": "permlevel", "fieldtype": "Int", "label": "Perm Level", @@ -541,13 +543,20 @@ "fieldname": "is_virtual", "fieldtype": "Check", "label": "Virtual" + }, + { + "depends_on": "eval:!in_list([\"Tab Break\", \"Section Break\", \"Column Break\", \"Button\", \"HTML\"], doc.fieldtype)", + "fieldname": "documentation_url", + "fieldtype": "Data", + "label": "Documentation URL", + "options": "URL" } ], "idx": 1, "index_web_pages_for_search": 1, "istable": 1, "links": [], - "modified": "2022-04-19 12:27:28.641580", + "modified": "2023-02-20 12:07:29.552523", "modified_by": "Administrator", "module": "Core", "name": "DocField", @@ -557,4 +566,4 @@ "sort_field": "modified", "sort_order": "ASC", "states": [] -} +} \ No newline at end of file diff --git a/frappe/core/doctype/docfield/docfield.py b/frappe/core/doctype/docfield/docfield.py index 9fe55df5fe..96dbfee4cb 100644 --- a/frappe/core/doctype/docfield/docfield.py +++ b/frappe/core/doctype/docfield/docfield.py @@ -29,3 +29,12 @@ class DocField(Document): if self.fieldtype == "Select": options = self.options or "" return [d for d in options.split("\n") if d] + + def __repr__(self): + unsaved = "unsaved" if not self.name else "" + doctype = self.__class__.__name__ + + docstatus = f" docstatus={self.docstatus}" if self.docstatus else "" + parent = f" parent={self.parent}" if getattr(self, "parent", None) else "" + + return f"<{self.fieldtype}{doctype}: {self.fieldname}{docstatus}{parent}{unsaved}>" diff --git a/frappe/core/doctype/docperm/docperm.json b/frappe/core/doctype/docperm/docperm.json index 4411a67435..3ce49c4d6b 100644 --- a/frappe/core/doctype/docperm/docperm.json +++ b/frappe/core/doctype/docperm/docperm.json @@ -26,7 +26,6 @@ "report", "export", "import", - "set_user_permissions", "column_break_19", "share", "print", @@ -178,13 +177,6 @@ "fieldtype": "Check", "label": "Import" }, - { - "default": "0", - "description": "This role update User Permissions for a user", - "fieldname": "set_user_permissions", - "fieldtype": "Check", - "label": "Set User Permissions" - }, { "fieldname": "column_break_19", "fieldtype": "Column Break" @@ -218,7 +210,7 @@ "idx": 1, "istable": 1, "links": [], - "modified": "2020-12-03 15:15:30.488212", + "modified": "2023-02-20 13:21:45.071310", "modified_by": "Administrator", "module": "Core", "name": "DocPerm", diff --git a/frappe/core/doctype/docshare/docshare.js b/frappe/core/doctype/docshare/docshare.js index 48db47a8cc..4d68c65cff 100644 --- a/frappe/core/doctype/docshare/docshare.js +++ b/frappe/core/doctype/docshare/docshare.js @@ -1,8 +1,6 @@ // Copyright (c) 2016, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('DocShare', { - refresh: function(frm) { - - } +frappe.ui.form.on("DocShare", { + refresh: function (frm) {}, }); diff --git a/frappe/core/doctype/docshare/test_docshare.py b/frappe/core/doctype/docshare/test_docshare.py index e374e4069d..e080b0d4ff 100644 --- a/frappe/core/doctype/docshare/test_docshare.py +++ b/frappe/core/doctype/docshare/test_docshare.py @@ -1,16 +1,15 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import unittest - import frappe import frappe.share from frappe.automation.doctype.auto_repeat.test_auto_repeat import create_submittable_doctype +from frappe.tests.utils import FrappeTestCase, change_settings test_dependencies = ["User"] -class TestDocShare(unittest.TestCase): +class TestDocShare(FrappeTestCase): def setUp(self): self.user = "test@example.com" self.event = frappe.get_doc( @@ -126,3 +125,75 @@ class TestDocShare(unittest.TestCase): ) frappe.share.remove(doctype, submittable_doc.name, self.user) + + def test_share_int_pk(self): + test_doc = frappe.new_doc("Console Log") + + test_doc.insert() + frappe.share.add("Console Log", test_doc.name, self.user) + + frappe.set_user(self.user) + self.assertIn( + str(test_doc.name), [str(name) for name in frappe.get_list("Console Log", pluck="name")] + ) + + test_doc.reload() + self.assertTrue(test_doc.has_permission("read")) + + @change_settings("System Settings", {"disable_document_sharing": 1}) + def test_share_disabled_add(self): + "Test if user loses share access on disabling share globally." + frappe.share.add("Event", self.event.name, self.user, share=1) # Share as admin + frappe.set_user(self.user) + + # User does not have share access although given to them + self.assertFalse(self.event.has_permission("share")) + self.assertRaises( + frappe.PermissionError, frappe.share.add, "Event", self.event.name, "test1@example.com" + ) + + @change_settings("System Settings", {"disable_document_sharing": 1}) + def test_share_disabled_add_with_ignore_permissions(self): + frappe.share.add("Event", self.event.name, self.user, share=1) + frappe.set_user(self.user) + + # User does not have share access although given to them + self.assertFalse(self.event.has_permission("share")) + + # Test if behaviour is consistent for developer overrides + frappe.share.add_docshare( + "Event", self.event.name, "test1@example.com", flags={"ignore_share_permission": True} + ) + + @change_settings("System Settings", {"disable_document_sharing": 1}) + def test_share_disabled_set_permission(self): + frappe.share.add("Event", self.event.name, self.user, share=1) + frappe.set_user(self.user) + + # User does not have share access although given to them + self.assertFalse(self.event.has_permission("share")) + self.assertRaises( + frappe.PermissionError, + frappe.share.set_permission, + "Event", + self.event.name, + "test1@example.com", + "read", + ) + + @change_settings("System Settings", {"disable_document_sharing": 1}) + def test_share_disabled_assign_to(self): + """ + Assigning a document to a user without access must not share the document, + if sharing disabled. + """ + from frappe.desk.form.assign_to import add + + frappe.share.add("Event", self.event.name, self.user, share=1) + frappe.set_user(self.user) + + self.assertRaises( + frappe.ValidationError, + add, + {"doctype": "Event", "name": self.event.name, "assign_to": ["test1@example.com"]}, + ) diff --git a/frappe/core/doctype/doctype/boilerplate/controller._py b/frappe/core/doctype/doctype/boilerplate/controller._py index 6db99def55..d8f02bf09c 100644 --- a/frappe/core/doctype/doctype/boilerplate/controller._py +++ b/frappe/core/doctype/doctype/boilerplate/controller._py @@ -4,5 +4,6 @@ # import frappe {base_class_import} + class {classname}({base_class}): {custom_controller} diff --git a/frappe/core/doctype/doctype/boilerplate/controller.js b/frappe/core/doctype/doctype/boilerplate/controller.js index 6d9fb2a514..0e3dcd2e26 100644 --- a/frappe/core/doctype/doctype/boilerplate/controller.js +++ b/frappe/core/doctype/doctype/boilerplate/controller.js @@ -1,8 +1,8 @@ // Copyright (c) {year}, {app_publisher} and contributors // For license information, please see license.txt -frappe.ui.form.on('{doctype}', {{ - // refresh: function(frm) {{ +// frappe.ui.form.on("{doctype}", {{ +// refresh(frm) {{ - // }} -}}); +// }}, +// }}); diff --git a/frappe/core/doctype/doctype/boilerplate/controller_list.js b/frappe/core/doctype/doctype/boilerplate/controller_list.js index b1f6d12008..3740cfa85d 100644 --- a/frappe/core/doctype/doctype/boilerplate/controller_list.js +++ b/frappe/core/doctype/doctype/boilerplate/controller_list.js @@ -1,5 +1,5 @@ /* eslint-disable */ -frappe.listview_settings['{doctype}'] = {{ - // add_fields: ["status"], - // filters:[["status","=", "Open"]] -}}; +// frappe.listview_settings["{doctype}"] = {{ +// add_fields: ["status"], +// filters: [["status","=", "Open"]], +// }}; diff --git a/frappe/core/doctype/doctype/doctype.js b/frappe/core/doctype/doctype/doctype.js index 514e3a9455..bb2af5cec0 100644 --- a/frappe/core/doctype/doctype/doctype.js +++ b/frappe/core/doctype/doctype/doctype.js @@ -1,19 +1,19 @@ // Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors // MIT License. See license.txt -frappe.ui.form.on('DocType', { - refresh: function(frm) { - frm.set_query('role', 'permissions', function(doc) { - if (doc.custom && frappe.session.user != 'Administrator') { +frappe.ui.form.on("DocType", { + refresh: function (frm) { + frm.set_query("role", "permissions", function (doc) { + if (doc.custom && frappe.session.user != "Administrator") { return { query: "frappe.core.doctype.role.role.role_query", - filters: [['Role', 'name', '!=', 'All']] + filters: [["Role", "name", "!=", "All"]], }; } }); - if(frappe.session.user !== "Administrator" || !frappe.boot.developer_mode) { - if(frm.is_new()) { + if (frappe.session.user !== "Administrator" || !frappe.boot.developer_mode) { + if (frm.is_new()) { frm.set_value("custom", 1); } frm.toggle_enable("custom", 0); @@ -21,143 +21,125 @@ frappe.ui.form.on('DocType', { frm.toggle_enable("beta", 0); } + render_form_builder_message(frm); + if (!frm.is_new() && !frm.doc.istable) { if (frm.doc.issingle) { - frm.add_custom_button(__('Go to {0}', [__(frm.doc.name)]), () => { + frm.add_custom_button(__("Go to {0}", [__(frm.doc.name)]), () => { window.open(`/app/${frappe.router.slug(frm.doc.name)}`); }); } else { - frm.add_custom_button(__('Go to {0} List', [__(frm.doc.name)]), () => { + frm.add_custom_button(__("Go to {0} List", [__(frm.doc.name)]), () => { window.open(`/app/${frappe.router.slug(frm.doc.name)}`); }); } } const customize_form_link = "Customize Form"; - if(!frappe.boot.developer_mode && !frm.doc.custom) { + if (!frappe.boot.developer_mode && !frm.doc.custom) { // make the document read-only frm.set_read_only(); - frm.dashboard.add_comment(__("DocTypes can not be modified, please use {0} instead", [customize_form_link]), "blue", true); + frm.dashboard.clear_comment(); + frm.dashboard.add_comment( + __("DocTypes can not be modified, please use {0} instead", [customize_form_link]), + "blue", + true + ); } else if (frappe.boot.developer_mode) { - let msg = __("This site is running in developer mode. Any change made here will be updated in code."); + frm.dashboard.clear_comment(); + let msg = __( + "This site is running in developer mode. Any change made here will be updated in code." + ); msg += "
    "; - msg += __("If you just want to customize for your site, use {0} instead.", [customize_form_link]); - frm.dashboard.add_comment(msg, "yellow"); + msg += __("If you just want to customize for your site, use {0} instead.", [ + customize_form_link, + ]); + frm.dashboard.add_comment(msg, "yellow", true); } - if(frm.is_new()) { - if (!(frm.doc.permissions && frm.doc.permissions.length)) { - frm.add_child('permissions', {role: 'System Manager'}); - } + if (frm.is_new()) { + frm.events.set_default_permission(frm); + frm.set_value("default_view", "List"); } else { frm.toggle_enable("engine", 0); } // set label for "In List View" for child tables - frm.get_docfield('fields', 'in_list_view').label = frm.doc.istable ? - __('In Grid View') : __('In List View'); + frm.get_docfield("fields", "in_list_view").label = frm.doc.istable + ? __("In Grid View") + : __("In List View"); frm.cscript.autoname(frm); frm.cscript.set_naming_rule_description(frm); + frm.trigger("setup_default_views"); }, istable: (frm) => { if (frm.doc.istable && frm.is_new()) { - frm.set_value('autoname', 'autoincrement'); - frm.set_value('allow_rename', 0); + frm.set_value("default_view", null); + } else if (!frm.doc.istable && !frm.is_new()) { + frm.events.set_default_permission(frm); } }, + + set_default_permission: (frm) => { + if (!(frm.doc.permissions && frm.doc.permissions.length)) { + frm.add_child("permissions", { role: "System Manager" }); + } + }, + + is_tree: (frm) => { + frm.trigger("setup_default_views"); + }, + + is_calendar_and_gantt: (frm) => { + frm.trigger("setup_default_views"); + }, + + setup_default_views: (frm) => { + frappe.model.set_default_views_for_doctype(frm.doc.name, frm); + }, }); frappe.ui.form.on("DocField", { form_render(frm, doctype, docname) { - // Render two select fields for Fetch From instead of Small Text for better UX - let field = frm.cur_grid.grid_form.fields_dict.fetch_from; - $(field.input_area).hide(); - - let $doctype_select = $(``); - let $wrapper = $('
    '); - $wrapper.append($doctype_select, $field_select); - field.$input_wrapper.append($wrapper); - $doctype_select.wrap('
    '); - $field_select.wrap('
    '); - - let row = frappe.get_doc(doctype, docname); - let curr_value = { doctype: null, fieldname: null }; - if (row.fetch_from) { - let [doctype, fieldname] = row.fetch_from.split("."); - curr_value.doctype = doctype; - curr_value.fieldname = fieldname; - } - - let doctypes = frm.doc.fields - .filter(df => df.fieldtype == "Link") - .filter(df => df.options && df.fieldname != row.fieldname) - .map(df => ({ - label: `${df.options} (${df.fieldname})`, - value: df.fieldname - })); - $doctype_select.add_options([ - { label: __("Select DocType"), value: "", selected: true }, - ...doctypes - ]); - - $doctype_select.on("change", () => { - row.fetch_from = ""; - frm.dirty(); - update_fieldname_options(); - }); - - function update_fieldname_options() { - $field_select.find("option").remove(); - - let link_fieldname = $doctype_select.val(); - if (!link_fieldname) return; - let link_field = frm.doc.fields.find( - df => df.fieldname === link_fieldname - ); - let link_doctype = link_field.options; - frappe.model.with_doctype(link_doctype, () => { - let fields = frappe.meta - .get_docfields(link_doctype, null, { - fieldtype: ["not in", frappe.model.no_value_type] - }) - .map(df => ({ - label: `${df.label} (${df.fieldtype})`, - value: df.fieldname - })); - $field_select.add_options([ - { - label: __("Select Field"), - value: "", - selected: true, - disabled: true - }, - ...fields - ]); - - if (curr_value.fieldname) { - $field_select.val(curr_value.fieldname); - } - }); - } - - $field_select.on("change", () => { - let fetch_from = `${$doctype_select.val()}.${$field_select.val()}`; - row.fetch_from = fetch_from; - frm.dirty(); - }); - - if (curr_value.doctype) { - $doctype_select.val(curr_value.doctype); - update_fieldname_options(); - } + frm.trigger("setup_fetch_from_fields", doctype, docname); }, - fieldtype: function(frm) { + fieldtype: function (frm) { frm.trigger("max_attachments"); - } + }, + + fields_add: (frm) => { + frm.trigger("setup_default_views"); + }, }); -extend_cscript(cur_frm.cscript, new frappe.model.DocTypeController({frm: cur_frm})); +function render_form_builder_message(frm) { + $(frm.fields_dict["try_form_builder_html"].wrapper).empty(); + if (!frm.is_new() && frm.fields_dict["try_form_builder_html"]) { + let title = __("Use Form Builder to visually edit your form layout"); + let msg = __( + "You can drag and drop fields to create your form layout, add tabs, sections and columns to organize your form and update field properties all from one screen." + ); + + let message = ` +
    +
    + +
    + `; + + $(frm.fields_dict["try_form_builder_html"].wrapper).html(message); + } +} + +extend_cscript(cur_frm.cscript, new frappe.model.DocTypeController({ frm: cur_frm })); diff --git a/frappe/core/doctype/doctype/doctype.json b/frappe/core/doctype/doctype/doctype.json index 4e110202d2..842898d064 100644 --- a/frappe/core/doctype/doctype/doctype.json +++ b/frappe/core/doctype/doctype/doctype.json @@ -14,6 +14,7 @@ "istable", "issingle", "is_tree", + "is_calendar_and_gantt", "editable_grid", "quick_entry", "cb01", @@ -23,12 +24,13 @@ "custom", "beta", "is_virtual", + "queue_in_background", "fields_section_break", + "try_form_builder_html", "fields", "sb1", "naming_rule", "autoname", - "name_case", "allow_rename", "column_break_15", "description", @@ -44,14 +46,17 @@ "allow_import", "allow_events_in_timeline", "allow_auto_repeat", + "make_attachments_public", "view_settings", "title_field", "show_title_field_in_link", - "translate_link_fields", + "translated_doctype", "search_fields", "default_print_format", "sort_field", "sort_order", + "default_view", + "force_re_route_to_default_view", "column_break_29", "document_type", "icon", @@ -185,6 +190,7 @@ }, { "default": "0", + "depends_on": "eval:!doc.istable", "fieldname": "beta", "fieldtype": "Check", "label": "Beta" @@ -216,15 +222,6 @@ "oldfieldname": "autoname", "oldfieldtype": "Data" }, - { - "depends_on": "eval:doc.naming_rule !== \"Autoincrement\"", - "fieldname": "name_case", - "fieldtype": "Select", - "label": "Name Case", - "oldfieldname": "name_case", - "oldfieldtype": "Select", - "options": "\nTitle Case\nUPPER CASE" - }, { "fieldname": "column_break_15", "fieldtype": "Column Break" @@ -270,7 +267,7 @@ "default": "0", "fieldname": "hide_toolbar", "fieldtype": "Check", - "label": "Hide Sidebar and Menu", + "label": "Hide Sidebar, Menu, and Comments", "oldfieldname": "hide_toolbar", "oldfieldtype": "Check" }, @@ -319,7 +316,8 @@ "depends_on": "eval:!doc.istable", "fieldname": "title_field", "fieldtype": "Data", - "label": "Title Field" + "label": "Title Field", + "mandatory_depends_on": "eval:doc.show_title_field_in_link" }, { "depends_on": "eval:!doc.istable", @@ -467,6 +465,7 @@ }, { "default": "0", + "depends_on": "eval:!doc.istable", "description": "Tree structures are implemented using Nested Set", "fieldname": "is_tree", "fieldtype": "Check", @@ -543,6 +542,7 @@ }, { "default": "0", + "depends_on": "eval:!doc.istable", "fieldname": "is_virtual", "fieldtype": "Check", "label": "Is Virtual" @@ -595,9 +595,47 @@ }, { "default": "0", - "fieldname": "translate_link_fields", + "fieldname": "translated_doctype", "fieldtype": "Check", "label": "Translate Link Fields" + }, + { + "default": "0", + "fieldname": "make_attachments_public", + "fieldtype": "Check", + "label": "Make Attachments Public by Default" + }, + { + "default": "0", + "depends_on": "eval: doc.is_submittable", + "description": "Enabling this will submit documents in background", + "fieldname": "queue_in_background", + "fieldtype": "Check", + "label": "Queue in Background" + }, + { + "fieldname": "default_view", + "fieldtype": "Select", + "label": "Default View" + }, + { + "default": "0", + "fieldname": "force_re_route_to_default_view", + "fieldtype": "Check", + "label": "Force Re-route to Default View" + }, + { + "default": "0", + "depends_on": "eval:!doc.istable", + "description": "Enables Calendar and Gantt views.", + "fieldname": "is_calendar_and_gantt", + "fieldtype": "Check", + "label": "Is Calendar and Gantt" + }, + { + "fieldname": "try_form_builder_html", + "fieldtype": "HTML", + "label": "Try Form Builder HTML" } ], "icon": "fa fa-bolt", @@ -680,7 +718,7 @@ "link_fieldname": "reference_doctype" } ], - "modified": "2022-02-28 21:56:52.116915", + "modified": "2023-05-15 14:07:51.526257", "modified_by": "Administrator", "module": "Core", "name": "DocType", @@ -716,5 +754,5 @@ "sort_order": "DESC", "states": [], "track_changes": 1, - "translate_link_fields": 1 + "translated_doctype": 1 } \ No newline at end of file diff --git a/frappe/core/doctype/doctype/doctype.py b/frappe/core/doctype/doctype/doctype.py index dbbbbc521a..9a0613e6ca 100644 --- a/frappe/core/doctype/doctype/doctype.py +++ b/frappe/core/doctype/doctype/doctype.py @@ -17,7 +17,7 @@ from frappe.cache_manager import clear_controller_cache, clear_user_cache from frappe.custom.doctype.custom_field.custom_field import create_custom_field from frappe.custom.doctype.property_setter.property_setter import make_property_setter from frappe.database.schema import validate_column_length, validate_column_name -from frappe.desk.notifications import delete_notification_count_for +from frappe.desk.notifications import delete_notification_count_for, get_filters_for from frappe.desk.utils import validate_route_conflict from frappe.model import ( child_table_fields, @@ -33,7 +33,7 @@ from frappe.model.meta import Meta from frappe.modules import get_doc_path, make_boilerplate from frappe.modules.import_file import get_file_path from frappe.query_builder.functions import Concat -from frappe.utils import cint +from frappe.utils import cint, random_string from frappe.website.utils import clear_cache if TYPE_CHECKING: @@ -86,9 +86,6 @@ form_grid_templates = {"fields": "templates/form_grid/fields.html"} class DocType(Document): - def get_feed(self): - return self.name - def validate(self): """Validate DocType before saving. @@ -122,6 +119,7 @@ class DocType(Document): self.validate_nestedset() self.validate_child_table() self.validate_website() + self.validate_virtual_doctype_methods() self.ensure_minimum_max_attachment_limit() validate_links_table_fieldnames(self) @@ -181,10 +179,6 @@ class DocType(Document): ) ) - def after_insert(self): - # clear user cache so that on the next reload this doctype is included in boot - clear_user_cache(frappe.session.user) - def set_defaults_for_single_and_table(self): if self.issingle: self.allow_import = 0 @@ -201,10 +195,12 @@ class DocType(Document): def set_default_in_list_view(self): """Set default in-list-view for first 4 mandatory fields""" + not_allowed_in_list_view = get_fields_not_allowed_in_list_view(self.meta) + if not [d.fieldname for d in self.fields if d.in_list_view]: cnt = 0 for d in self.fields: - if d.reqd and not d.hidden and not d.fieldtype in table_fields: + if d.reqd and not d.hidden and not d.fieldtype in not_allowed_in_list_view: d.in_list_view = 1 cnt += 1 if cnt == 4: @@ -306,6 +302,14 @@ class DocType(Document): # clear website cache clear_cache() + def validate_virtual_doctype_methods(self): + if not self.get("is_virtual") or self.is_new(): + return + + from frappe.model.virtual_doctype import validate_controller + + validate_controller(self.name) + def ensure_minimum_max_attachment_limit(self): """Ensure that max_attachments is *at least* bigger than number of attach fields.""" from frappe.model import attachment_fieldtypes @@ -329,11 +333,11 @@ class DocType(Document): """Change the timestamp of parent DocType if the current one is a child to clear caches.""" if frappe.flags.in_import: return - parent_list = frappe.db.get_all( + parent_list = frappe.get_all( "DocField", "parent", dict(fieldtype=["in", frappe.model.table_fields], options=self.name) ) for p in parent_list: - frappe.db.update("DocType", p.parent, {}, for_update=False) + frappe.db.set_value("DocType", p.parent, {}, for_update=False) def scrub_field_names(self): """Sluggify fieldnames if not set from Label.""" @@ -341,6 +345,7 @@ class DocType(Document): "name", "parent", "creation", + "owner", "modified", "modified_by", "parentfield", @@ -362,8 +367,10 @@ class DocType(Document): d.fieldname = d.fieldname + "_column" elif d.fieldtype == "Tab Break": d.fieldname = d.fieldname + "_tab" + elif d.fieldtype in ("Section Break", "Column Break", "Tab Break"): + d.fieldname = d.fieldtype.lower().replace(" ", "_") + "_" + str(random_string(4)) else: - d.fieldname = d.fieldtype.lower().replace(" ", "_") + "_" + str(d.idx) + frappe.throw(_("Row #{}: Fieldname is required").format(d.idx), title="Missing Fieldname") else: if d.fieldname in restricted: frappe.throw(_("Fieldname {0} is restricted").format(d.fieldname), InvalidFieldNameError) @@ -412,13 +419,12 @@ class DocType(Document): delete_notification_count_for(doctype=self.name) frappe.clear_cache(doctype=self.name) + # clear user cache so that on the next reload this doctype is included in boot + clear_user_cache(frappe.session.user) + if not frappe.flags.in_install and hasattr(self, "before_update"): self.sync_global_search() - # clear from local cache - if self.name in frappe.local.meta_cache: - del frappe.local.meta_cache[self.name] - clear_linked_doctype_cache() def setup_autoincrement_and_sequence(self): @@ -880,7 +886,7 @@ def validate_series(dt, autoname=None, name=None): if not autoname and dt.get("fields", {"fieldname": "naming_series"}): dt.autoname = "naming_series:" elif dt.autoname and dt.autoname.startswith("naming_series:"): - fieldname = dt.autoname.split("naming_series:")[0] or "naming_series" + fieldname = dt.autoname.split("naming_series:", 1)[0] or "naming_series" if not dt.get("fields", {"fieldname": fieldname}): frappe.throw( _("Fieldname called {0} must exist to enable autonaming").format(frappe.bold(fieldname)), @@ -908,7 +914,7 @@ def validate_series(dt, autoname=None, name=None): and (not autoname.startswith("format:")) ): - prefix = autoname.split(".")[0] + prefix = autoname.split(".", 1)[0] doctype = frappe.qb.DocType("DocType") used_in = ( frappe.qb.from_(doctype) @@ -978,16 +984,12 @@ def change_name_column_type(doctype_name: str, type: str) -> None: def validate_links_table_fieldnames(meta): """Validate fieldnames in Links table""" - if not meta.links or frappe.flags.in_patch or frappe.flags.in_fixtures: + if not meta.links or frappe.flags.in_patch or frappe.flags.in_fixtures or frappe.flags.in_migrate: return fieldnames = tuple(field.fieldname for field in meta.fields) for index, link in enumerate(meta.links, 1): - if not frappe.get_meta(link.link_doctype).has_field(link.link_fieldname): - message = _("Document Links Row #{0}: Could not find field {1} in {2} DocType").format( - index, frappe.bold(link.link_fieldname), frappe.bold(link.link_doctype) - ) - frappe.throw(message, InvalidFieldNameError, _("Invalid Fieldname")) + _test_connection_query(doctype=link.link_doctype, field=link.link_fieldname, idx=index) if not link.is_child_table: continue @@ -1016,6 +1018,25 @@ def validate_links_table_fieldnames(meta): frappe.throw(message, frappe.ValidationError, _("Invalid Table Fieldname")) +def _test_connection_query(doctype, field, idx): + """Make sure that connection can be queried. + + This function executes query similar to one that would be executed for + finding count on dashboard and hence validates if fieldname/doctype are + correct. + """ + filters = get_filters_for(doctype) or {} + filters[field] = "" + + try: + frappe.get_all(doctype, filters=filters, limit=1, distinct=True, ignore_ifnull=True) + except Exception as e: + frappe.clear_last_message() + msg = _("Document Links Row #{0}: Invalid doctype or fieldname.").format(idx) + msg += "
    " + str(e) + frappe.throw(msg, InvalidFieldNameError) + + def validate_fields_for_doctype(doctype): meta = frappe.get_meta(doctype, cached=False) validate_links_table_fieldnames(meta) @@ -1078,10 +1099,7 @@ def validate_fields(meta): ) def check_link_table_options(docname, d): - if frappe.flags.in_patch: - return - - if frappe.flags.in_fixtures: + if frappe.flags.in_patch or frappe.flags.in_fixtures: return if d.fieldtype in ("Link",) + table_fields: @@ -1115,7 +1133,7 @@ def validate_fields(meta): d.options = options def check_hidden_and_mandatory(docname, d): - if d.hidden and d.reqd and not d.default: + if d.hidden and d.reqd and not d.default and not frappe.flags.in_migrate: frappe.throw( _("{0}: Field {1} in row {2} cannot be hidden and mandatory without default").format( docname, d.label, d.idx @@ -1184,6 +1202,9 @@ def validate_fields(meta): frappe.throw(_("Precision should be between 1 and 6")) def check_unique_and_text(docname, d): + if meta.is_virtual: + return + if meta.issingle: d.unique = 0 d.search_index = 0 @@ -1325,7 +1346,7 @@ def validate_fields(meta): if meta.sort_field: sort_fields = [meta.sort_field] if "," in meta.sort_field: - sort_fields = [d.split()[0] for d in meta.sort_field.split(",")] + sort_fields = [d.split(maxsplit=1)[0] for d in meta.sort_field.split(",")] for fieldname in sort_fields: if fieldname not in (fieldname_list + list(default_fields) + list(child_table_fields)): @@ -1395,10 +1416,9 @@ def validate_fields(meta): ) df_options_str = "
    • " + "
    • ".join(_(x) for x in data_field_options) + "
    " - frappe.msgprint(text_str + df_options_str, title="Invalid Data Field", raise_exception=True) + frappe.msgprint(text_str + df_options_str, title="Invalid Data Field", alert=True) def check_child_table_option(docfield): - if frappe.flags.in_fixtures: return if docfield.fieldtype not in ["Table MultiSelect", "Table"]: @@ -1427,10 +1447,7 @@ def validate_fields(meta): fields = meta.get("fields") fieldname_list = [d.fieldname for d in fields] - not_allowed_in_list_view = list(copy.copy(no_value_fields)) - not_allowed_in_list_view.append("Attach Image") - if meta.istable: - not_allowed_in_list_view.remove("Button") + not_allowed_in_list_view = get_fields_not_allowed_in_list_view(meta) for d in fields: if not d.permlevel: @@ -1444,31 +1461,42 @@ def validate_fields(meta): check_invalid_fieldnames(meta.get("name"), d.fieldname) check_unique_fieldname(meta.get("name"), d.fieldname) check_fieldname_length(d.fieldname) - check_illegal_mandatory(meta.get("name"), d) - check_link_table_options(meta.get("name"), d) - check_dynamic_link_options(d) check_hidden_and_mandatory(meta.get("name"), d) - check_in_list_view(meta.get("istable"), d) - check_in_global_search(d) - check_illegal_default(d) check_unique_and_text(meta.get("name"), d) - check_illegal_depends_on_conditions(d) - check_child_table_option(d) check_table_multiselect_option(d) scrub_options_in_select(d) scrub_fetch_from(d) validate_data_field_type(d) - check_max_height(d) - check_no_of_ratings(d) - check_fold(fields) - check_search_fields(meta, fields) - check_title_field(meta) - check_timeline_field(meta) - check_is_published_field(meta) - check_website_search_field(meta) - check_sort_field(meta) - check_image_field(meta) + if not frappe.flags.in_migrate: + check_link_table_options(meta.get("name"), d) + check_illegal_mandatory(meta.get("name"), d) + check_dynamic_link_options(d) + check_in_list_view(meta.get("istable"), d) + check_in_global_search(d) + check_illegal_depends_on_conditions(d) + check_illegal_default(d) + check_child_table_option(d) + check_max_height(d) + check_no_of_ratings(d) + + if not frappe.flags.in_migrate: + check_fold(fields) + check_search_fields(meta, fields) + check_title_field(meta) + check_timeline_field(meta) + check_is_published_field(meta) + check_website_search_field(meta) + check_sort_field(meta) + check_image_field(meta) + + +def get_fields_not_allowed_in_list_view(meta) -> list[str]: + not_allowed_in_list_view = list(copy.copy(no_value_fields)) + not_allowed_in_list_view.append("Attach Image") + if meta.istable: + not_allowed_in_list_view.remove("Button") + return not_allowed_in_list_view def validate_permissions_for_doctype(doctype, for_remove=False, alert=False): @@ -1577,11 +1605,6 @@ def validate_permissions(doctype, for_remove=False, alert=False): d.set("import", 0) d.set("export", 0) - for ptype, label in [["set_user_permissions", _("Set User Permissions")]]: - if d.get(ptype): - d.set(ptype, 0) - frappe.msgprint(_("{0} cannot be set for Single types").format(label)) - def check_if_submittable(d): if d.submit and not issubmittable: frappe.throw(_("{0}: Cannot set Assign Submit if not Submittable").format(get_txt(d))) diff --git a/frappe/core/doctype/doctype/test_doctype.py b/frappe/core/doctype/doctype/test_doctype.py index 3a5ca4329f..fead7672fe 100644 --- a/frappe/core/doctype/doctype/test_doctype.py +++ b/frappe/core/doctype/doctype/test_doctype.py @@ -2,7 +2,6 @@ # License: MIT. See LICENSE import random import string -import unittest from unittest.mock import patch import frappe @@ -19,9 +18,10 @@ from frappe.core.doctype.doctype.doctype import ( ) from frappe.custom.doctype.custom_field.custom_field import create_custom_fields from frappe.desk.form.load import getdoc +from frappe.tests.utils import FrappeTestCase -class TestDocType(unittest.TestCase): +class TestDocType(FrappeTestCase): def tearDown(self): frappe.db.rollback() @@ -172,32 +172,6 @@ class TestDocType(unittest.TestCase): if condition: self.assertFalse(re.match(pattern, condition)) - def test_data_field_options(self): - doctype_name = "Test Data Fields" - valid_data_field_options = frappe.model.data_field_options + ("",) - invalid_data_field_options = ("Invalid Option 1", frappe.utils.random_string(5)) - - for field_option in valid_data_field_options + invalid_data_field_options: - test_doctype = frappe.get_doc( - { - "doctype": "DocType", - "name": doctype_name, - "module": "Core", - "custom": 1, - "fields": [ - {"fieldname": f"{field_option}_field", "fieldtype": "Data", "options": field_option} - ], - } - ) - - if field_option in invalid_data_field_options: - # assert that only data options in frappe.model.data_field_options are valid - self.assertRaises(frappe.ValidationError, test_doctype.insert) - else: - test_doctype.insert() - self.assertEqual(test_doctype.name, doctype_name) - test_doctype.delete() - def test_sync_field_order(self): import os @@ -543,7 +517,7 @@ class TestDocType(unittest.TestCase): # check invalid doctype doc.append("links", {"link_doctype": "User2", "link_fieldname": "first_name"}) - self.assertRaises(frappe.DoesNotExistError, validate_links_table_fieldnames, doc) + self.assertRaises(InvalidFieldNameError, validate_links_table_fieldnames, doc) doc.links = [] # reset links table # check invalid fieldname @@ -552,13 +526,14 @@ class TestDocType(unittest.TestCase): self.assertRaises(InvalidFieldNameError, validate_links_table_fieldnames, doc) def test_create_virtual_doctype(self): - """Test virtual DOcTYpe.""" + """Test virtual DocType.""" virtual_doc = new_doctype("Test Virtual Doctype") virtual_doc.is_virtual = 1 - virtual_doc.insert() - virtual_doc.save() + virtual_doc.insert(ignore_if_duplicate=True) + virtual_doc.reload() doc = frappe.get_doc("DocType", "Test Virtual Doctype") + self.assertDictEqual(doc.as_dict(), virtual_doc.as_dict()) self.assertEqual(doc.is_virtual, 1) self.assertFalse(frappe.db.table_exists("Test Virtual Doctype")) @@ -670,6 +645,21 @@ class TestDocType(unittest.TestCase): self.assertEqual(test_json.test_json_field["hello"], "world") + def test_no_delete_doc(self): + self.assertRaises(frappe.ValidationError, frappe.delete_doc, "DocType", "Address") + + @patch.dict(frappe.conf, {"developer_mode": 1}) + def test_custom_field_deletion(self): + """Custom child tables whose doctype doesn't exist should be auto deleted.""" + doctype = new_doctype(custom=0).insert().name + child = new_doctype(custom=0, istable=1).insert().name + + field = "abc" + create_custom_fields({doctype: [{"fieldname": field, "fieldtype": "Table", "options": child}]}) + + frappe.delete_doc("DocType", child) + self.assertFalse(frappe.get_meta(doctype).get_field(field)) + @patch.dict(frappe.conf, {"developer_mode": 1}) def test_delete_doctype_with_customization(self): from frappe.custom.doctype.property_setter.property_setter import make_property_setter @@ -707,6 +697,28 @@ class TestDocType(unittest.TestCase): self.assertEqual(frappe.get_meta(doctype).get_field(field).default, "DELETETHIS") frappe.delete_doc("DocType", doctype) + def test_not_in_list_view_for_not_allowed_mandatory_field(self): + doctype = new_doctype( + fields=[ + { + "fieldname": "cover_image", + "fieldtype": "Attach Image", + "label": "Cover Image", + "reqd": 1, # mandatory + }, + { + "fieldname": "book_name", + "fieldtype": "Data", + "label": "Book Name", + "reqd": 1, # mandatory + }, + ], + ).insert() + + self.assertFalse(doctype.fields[0].in_list_view) + self.assertTrue(doctype.fields[1].in_list_view) + frappe.delete_doc("DocType", doctype.name) + def new_doctype( name: str | None = None, @@ -744,8 +756,7 @@ def new_doctype( } ) - if fields: - for f in fields: - doc.append("fields", f) + if fields and len(fields) > 0: + doc.set("fields", fields) return doc diff --git a/frappe/core/doctype/document_naming_rule/document_naming_rule.js b/frappe/core/doctype/document_naming_rule/document_naming_rule.js index 097a4e9a6e..f6ea996107 100644 --- a/frappe/core/doctype/document_naming_rule/document_naming_rule.js +++ b/frappe/core/doctype/document_naming_rule/document_naming_rule.js @@ -1,64 +1,46 @@ // Copyright (c) 2020, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Document Naming Rule', { - refresh: function(frm) { - frm.trigger('document_type'); - if (!frm.doc.__islocal) frm.trigger("add_update_counter_button"); +frappe.ui.form.on("Document Naming Rule", { + refresh: function (frm) { + frm.trigger("document_type"); + frm.last_counter_value = frm.doc.counter; + frm.skip_before_save = false; + }, + before_save: function (frm) { + if (frm.is_new() || frm.skip_before_save || frm.last_counter_value === frm.doc.counter) + return; + + frappe.validated = false; + frappe.warn( + __("Are you sure?"), + __("Updating counter may lead to document name conflicts if not done properly"), + () => { + frm.skip_before_save = true; + frm.save(); + }, + __("Proceed"), + false + ); }, document_type: (frm) => { // update the select field options with fieldnames if (frm.doc.document_type) { frappe.model.with_doctype(frm.doc.document_type, () => { - let fieldnames = frappe.get_meta(frm.doc.document_type).fields - .filter((d) => { + let fieldnames = frappe + .get_meta(frm.doc.document_type) + .fields.filter((d) => { return frappe.model.no_value_type.indexOf(d.fieldtype) === -1; - }).map((d) => { - return {label: `${d.label} (${d.fieldname})`, value: d.fieldname}; + }) + .map((d) => { + return { label: `${d.label} (${d.fieldname})`, value: d.fieldname }; }); frm.fields_dict.conditions.grid.update_docfield_property( - 'field', 'options', fieldnames + "field", + "options", + fieldnames ); }); } }, - add_update_counter_button: (frm) => { - frm.add_custom_button(__('Update Counter'), function() { - - const fields = [{ - fieldtype: 'Data', - fieldname: 'new_counter', - label: __('New Counter'), - default: frm.doc.counter, - reqd: 1, - description: __('Warning: Updating counter may lead to document name conflicts if not done properly') - }]; - - let primary_action_label = __('Save'); - - let primary_action = (fields) => { - frappe.call({ - method: 'frappe.core.doctype.document_naming_rule.document_naming_rule.update_current', - args: { - name: frm.doc.name, - new_counter: fields.new_counter - }, - callback: function() { - frm.set_value("counter", fields.new_counter); - dialog.hide(); - } - }); - }; - - const dialog = new frappe.ui.Dialog({ - title: __('Update Counter Value for Prefix: {0}', [frm.doc.prefix]), - fields, - primary_action_label, - primary_action - }); - - dialog.show(); - - }); - } }); diff --git a/frappe/core/doctype/document_naming_rule/document_naming_rule.json b/frappe/core/doctype/document_naming_rule/document_naming_rule.json index 4e6f3f3fd1..1e2247c250 100644 --- a/frappe/core/doctype/document_naming_rule/document_naming_rule.json +++ b/frappe/core/doctype/document_naming_rule/document_naming_rule.json @@ -12,8 +12,9 @@ "conditions", "naming_section", "prefix", - "prefix_digits", - "counter" + "counter", + "column_break_xfqa", + "prefix_digits" ], "fields": [ { @@ -22,7 +23,8 @@ "in_list_view": 1, "in_standard_filter": 1, "label": "Document Type", - "options": "DocType" + "options": "DocType", + "reqd": 1 }, { "default": "0", @@ -38,11 +40,12 @@ "reqd": 1 }, { + "default": "0", + "description": "Warning: Updating counter may lead to document name conflicts if not done properly", "fieldname": "counter", "fieldtype": "Int", "label": "Counter", - "no_copy": 1, - "read_only": 1 + "no_copy": 1 }, { "default": "5", @@ -76,11 +79,15 @@ "fieldname": "priority", "fieldtype": "Int", "label": "Priority" + }, + { + "fieldname": "column_break_xfqa", + "fieldtype": "Column Break" } ], "index_web_pages_for_search": 1, "links": [], - "modified": "2021-09-13 20:07:47.617615", + "modified": "2023-04-24 15:14:32.054272", "modified_by": "Administrator", "module": "Core", "name": "Document Naming Rule", @@ -102,6 +109,7 @@ "quick_entry": 1, "sort_field": "modified", "sort_order": "DESC", + "states": [], "title_field": "document_type", "track_changes": 1 } \ No newline at end of file diff --git a/frappe/core/doctype/document_naming_rule/document_naming_rule.py b/frappe/core/doctype/document_naming_rule/document_naming_rule.py index 3fecf26ade..46bc1c5ee2 100644 --- a/frappe/core/doctype/document_naming_rule/document_naming_rule.py +++ b/frappe/core/doctype/document_naming_rule/document_naming_rule.py @@ -12,6 +12,15 @@ class DocumentNamingRule(Document): def validate(self): self.validate_fields_in_conditions() + def clear_doctype_map(self): + frappe.cache_manager.clear_doctype_map(self.doctype, self.document_type) + + def on_update(self): + self.clear_doctype_map() + + def on_trash(self): + self.clear_doctype_map() + def validate_fields_in_conditions(self): if self.has_value_changed("document_type"): docfields = [x.fieldname for x in frappe.get_meta(self.document_type).fields] @@ -38,9 +47,3 @@ class DocumentNamingRule(Document): doc.name = naming_series + ("%0" + str(self.prefix_digits) + "d") % (counter + 1) frappe.db.set_value(self.doctype, self.name, "counter", counter + 1) - - -@frappe.whitelist() -def update_current(name, new_counter): - frappe.only_for("System Manager") - frappe.db.set_value("Document Naming Rule", name, "counter", new_counter) diff --git a/frappe/core/doctype/document_naming_rule/test_document_naming_rule.py b/frappe/core/doctype/document_naming_rule/test_document_naming_rule.py index cc406ed5cd..60e6803a1a 100644 --- a/frappe/core/doctype/document_naming_rule/test_document_naming_rule.py +++ b/frappe/core/doctype/document_naming_rule/test_document_naming_rule.py @@ -1,11 +1,10 @@ # Copyright (c) 2020, Frappe Technologies and Contributors # License: MIT. See LICENSE -import unittest - import frappe +from frappe.tests.utils import FrappeTestCase -class TestDocumentNamingRule(unittest.TestCase): +class TestDocumentNamingRule(FrappeTestCase): def test_naming_rule_by_series(self): naming_rule = frappe.get_doc( dict(doctype="Document Naming Rule", document_type="ToDo", prefix="test-todo-", prefix_digits=5) diff --git a/frappe/core/doctype/document_naming_rule_condition/document_naming_rule_condition.js b/frappe/core/doctype/document_naming_rule_condition/document_naming_rule_condition.js index 8ef39c7b70..fdf46e82e0 100644 --- a/frappe/core/doctype/document_naming_rule_condition/document_naming_rule_condition.js +++ b/frappe/core/doctype/document_naming_rule_condition/document_naming_rule_condition.js @@ -1,8 +1,7 @@ // Copyright (c) 2020, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Document Naming Rule Condition', { +frappe.ui.form.on("Document Naming Rule Condition", { // refresh: function(frm) { - // } }); diff --git a/frappe/core/doctype/document_naming_rule_condition/test_document_naming_rule_condition.py b/frappe/core/doctype/document_naming_rule_condition/test_document_naming_rule_condition.py index 68f0677f2c..edf52fc0c4 100644 --- a/frappe/core/doctype/document_naming_rule_condition/test_document_naming_rule_condition.py +++ b/frappe/core/doctype/document_naming_rule_condition/test_document_naming_rule_condition.py @@ -1,8 +1,8 @@ # Copyright (c) 2020, Frappe Technologies and Contributors # License: MIT. See LICENSE # import frappe -import unittest +from frappe.tests.utils import FrappeTestCase -class TestDocumentNamingRuleCondition(unittest.TestCase): +class TestDocumentNamingRuleCondition(FrappeTestCase): pass diff --git a/frappe/core/doctype/document_naming_settings/document_naming_settings.js b/frappe/core/doctype/document_naming_settings/document_naming_settings.js index 2dc5fc4d58..2a9ec4aae5 100644 --- a/frappe/core/doctype/document_naming_settings/document_naming_settings.js +++ b/frappe/core/doctype/document_naming_settings/document_naming_settings.js @@ -2,28 +2,28 @@ // For license information, please see license.txt frappe.ui.form.on("Document Naming Settings", { - refresh: function(frm) { + refresh: function (frm) { frm.trigger("setup_transaction_autocomplete"); frm.disable_save(); }, - setup_transaction_autocomplete: function(frm) { + setup_transaction_autocomplete: function (frm) { frappe.call({ method: "get_transactions_and_prefixes", doc: frm.doc, - callback: function(r) { + callback: function (r) { frm.fields_dict.transaction_type.set_data(r.message.transactions); frm.fields_dict.prefix.set_data(r.message.prefixes); }, }); }, - transaction_type: function(frm) { + transaction_type: function (frm) { frm.set_value("user_must_always_select", 0); frappe.call({ method: "get_options", doc: frm.doc, - callback: function(r) { + callback: function (r) { frm.set_value("naming_series_options", r.message); if (r.message && r.message.split("\n")[0] == "") frm.set_value("user_must_always_select", 1); @@ -31,23 +31,23 @@ frappe.ui.form.on("Document Naming Settings", { }); }, - prefix: function(frm) { + prefix: function (frm) { frappe.call({ method: "get_current", doc: frm.doc, - callback: function(r) { + callback: function (r) { frm.refresh_field("current_value"); }, }); }, - update: function(frm) { + update: function (frm) { frappe.call({ method: "update_series", doc: frm.doc, freeze: true, freeze_msg: __("Updating naming series options"), - callback: function(r) { + callback: function (r) { frm.trigger("setup_transaction_autocomplete"); frm.trigger("transaction_type"); }, @@ -58,14 +58,11 @@ frappe.ui.form.on("Document Naming Settings", { frappe.call({ method: "preview_series", doc: frm.doc, - callback: function(r) { + callback: function (r) { if (!r.exc) { frm.set_value("series_preview", r.message); } else { - frm.set_value( - "series_preview", - __("Failed to generate preview of series") - ); + frm.set_value("series_preview", __("Failed to generate preview of series")); } }, }); diff --git a/frappe/core/doctype/document_naming_settings/document_naming_settings.json b/frappe/core/doctype/document_naming_settings/document_naming_settings.json index 4c86b2ec1d..9a12f3f77e 100644 --- a/frappe/core/doctype/document_naming_settings/document_naming_settings.json +++ b/frappe/core/doctype/document_naming_settings/document_naming_settings.json @@ -81,10 +81,10 @@ }, { "depends_on": "transaction_type", - "description": "Generate 3 preview of names generate by any valid series.", + "description": "Get a preview of generated names with a series.", "fieldname": "try_naming_series", "fieldtype": "Data", - "label": "Try a naming Series" + "label": "Try a Naming Series" }, { "fieldname": "transaction_type", @@ -111,7 +111,7 @@ "icon": "fa fa-sort-by-order", "issingle": 1, "links": [], - "modified": "2022-05-30 23:51:36.136535", + "modified": "2023-02-20 13:11:56.662100", "modified_by": "Administrator", "module": "Core", "name": "Document Naming Settings", @@ -130,4 +130,4 @@ "sort_field": "modified", "sort_order": "DESC", "states": [] -} +} \ No newline at end of file diff --git a/frappe/core/doctype/document_naming_settings/document_naming_settings.py b/frappe/core/doctype/document_naming_settings/document_naming_settings.py index 6fc4d9b23e..f8647bd74a 100644 --- a/frappe/core/doctype/document_naming_settings/document_naming_settings.py +++ b/frappe/core/doctype/document_naming_settings/document_naming_settings.py @@ -113,6 +113,8 @@ class DocumentNamingSettings(Document): option_string = "\n".join(options) + # Erase default first, it might not be in new options. + self.update_naming_series_property_setter(doctype, "default", "") self.update_naming_series_property_setter(doctype, "options", option_string) self.update_naming_series_property_setter(doctype, "default", default) @@ -163,7 +165,7 @@ class DocumentNamingSettings(Document): @frappe.whitelist() def get_current(self): """get series current""" - if self.prefix: + if self.prefix is not None: self.current_value = NamingSeries(self.prefix).get_current_value() return self.current_value @@ -171,7 +173,7 @@ class DocumentNamingSettings(Document): def update_series_start(self): frappe.only_for("System Manager") - if not self.prefix: + if self.prefix is None: frappe.throw(_("Please select prefix first")) naming_series = NamingSeries(self.prefix) @@ -191,7 +193,7 @@ class DocumentNamingSettings(Document): def create_version_log_for_change(self, series, old, new): version = frappe.new_doc("Version") version.ref_doctype = "Series" - version.docname = series + version.docname = series or ".#" version.data = frappe.as_json({"changed": [["current", old, new]]}) version.flags.ignore_links = True # series is not a "real" doctype version.flags.ignore_permissions = True diff --git a/frappe/core/doctype/document_share_key/document_share_key.js b/frappe/core/doctype/document_share_key/document_share_key.js index c51233e10f..7e1712beff 100644 --- a/frappe/core/doctype/document_share_key/document_share_key.js +++ b/frappe/core/doctype/document_share_key/document_share_key.js @@ -1,8 +1,7 @@ // Copyright (c) 2021, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Document Share Key', { +frappe.ui.form.on("Document Share Key", { // refresh: function(frm) { - // } }); diff --git a/frappe/core/doctype/document_share_key/test_document_share_key.py b/frappe/core/doctype/document_share_key/test_document_share_key.py index 10499fcc5d..23f91157e6 100644 --- a/frappe/core/doctype/document_share_key/test_document_share_key.py +++ b/frappe/core/doctype/document_share_key/test_document_share_key.py @@ -2,8 +2,8 @@ # See license.txt # import frappe -import unittest +from frappe.tests.utils import FrappeTestCase -class TestDocumentShareKey(unittest.TestCase): +class TestDocumentShareKey(FrappeTestCase): pass diff --git a/frappe/core/doctype/domain/domain.js b/frappe/core/doctype/domain/domain.js index 397ed4b19c..9b51c10d77 100644 --- a/frappe/core/doctype/domain/domain.js +++ b/frappe/core/doctype/domain/domain.js @@ -1,8 +1,6 @@ // Copyright (c) 2017, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Domain', { - refresh: function(frm) { - - } +frappe.ui.form.on("Domain", { + refresh: function (frm) {}, }); diff --git a/frappe/core/doctype/domain/domain.py b/frappe/core/doctype/domain/domain.py index e0b0e80982..1f5dbfa22e 100644 --- a/frappe/core/doctype/domain/domain.py +++ b/frappe/core/doctype/domain/domain.py @@ -85,8 +85,8 @@ class Domain(Document): def set_default_portal_role(self): """Set default portal role based on domain""" if self.data.get("default_portal_role"): - frappe.db.set_value( - "Portal Settings", None, "default_role", self.data.get("default_portal_role") + frappe.db.set_single_value( + "Portal Settings", "default_role", self.data.get("default_portal_role") ) def setup_properties(self): diff --git a/frappe/core/doctype/domain/test_domain.py b/frappe/core/doctype/domain/test_domain.py index 32592705b4..92540cda6b 100644 --- a/frappe/core/doctype/domain/test_domain.py +++ b/frappe/core/doctype/domain/test_domain.py @@ -1,7 +1,7 @@ # Copyright (c) 2017, Frappe Technologies and Contributors # License: MIT. See LICENSE -import unittest +from frappe.tests.utils import FrappeTestCase -class TestDomain(unittest.TestCase): +class TestDomain(FrappeTestCase): pass diff --git a/frappe/core/doctype/domain_settings/domain_settings.js b/frappe/core/doctype/domain_settings/domain_settings.js index 7178cb4cd6..87386ce9bd 100644 --- a/frappe/core/doctype/domain_settings/domain_settings.js +++ b/frappe/core/doctype/domain_settings/domain_settings.js @@ -1,66 +1,69 @@ // Copyright (c) 2017, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Domain Settings', { - before_load: function(frm) { - if(!frm.domains_multicheck) { +frappe.ui.form.on("Domain Settings", { + before_load: function (frm) { + if (!frm.domains_multicheck) { frm.domains_multicheck = frappe.ui.form.make_control({ parent: frm.fields_dict.domains_html.$wrapper, df: { fieldname: "domains_multicheck", fieldtype: "MultiCheck", get_data: () => { - let active_domains = (frm.doc.active_domains || []).map(row => row.domain); - return frappe.boot.all_domains.map(domain => { + let active_domains = (frm.doc.active_domains || []).map( + (row) => row.domain + ); + return frappe.boot.all_domains.map((domain) => { return { label: domain, value: domain, - checked: active_domains.includes(domain) + checked: active_domains.includes(domain), }; }); }, on_change: () => { frm.dirty(); - } + }, }, - render_input: true + render_input: true, }); frm.domains_multicheck.refresh_input(); } }, - validate: function(frm) { - frm.trigger('set_options_in_table'); + validate: function (frm) { + frm.trigger("set_options_in_table"); }, - set_options_in_table: function(frm) { + set_options_in_table: function (frm) { let selected_options = frm.domains_multicheck.get_value(); let unselected_options = frm.domains_multicheck.options - .map(option => option.value) - .filter(value => { + .map((option) => option.value) + .filter((value) => { return !selected_options.includes(value); }); - let map = {}, list = []; - (frm.doc.active_domains || []).map(row => { + let map = {}, + list = []; + (frm.doc.active_domains || []).map((row) => { map[row.domain] = row.name; list.push(row.domain); }); - unselected_options.map(option => { - if(list.includes(option)) { + unselected_options.map((option) => { + if (list.includes(option)) { frappe.model.clear_doc("Has Domain", map[option]); } }); - selected_options.map(option => { - if(!list.includes(option)) { + selected_options.map((option) => { + if (!list.includes(option)) { frappe.model.clear_doc("Has Domain", map[option]); let row = frappe.model.add_child(frm.doc, "Has Domain", "active_domains"); row.domain = option; } }); - refresh_field('active_domains'); - } + refresh_field("active_domains"); + }, }); diff --git a/frappe/core/doctype/domain_settings/domain_settings.json b/frappe/core/doctype/domain_settings/domain_settings.json index 8efd296da6..c363529cbd 100644 --- a/frappe/core/doctype/domain_settings/domain_settings.json +++ b/frappe/core/doctype/domain_settings/domain_settings.json @@ -1,153 +1,56 @@ { - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, - "creation": "2017-05-03 16:28:11.295095", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", + "actions": [], + "creation": "2017-05-03 16:28:11.295095", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "active_domains_sb", + "domains_html", + "active_domains" + ], "fields": [ { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "active_domains_sb", - "fieldtype": "Section Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Active Domains", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "active_domains_sb", + "fieldtype": "Section Break", + "label": "Active Domains" + }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "domains_html", - "fieldtype": "HTML", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Domains HTML", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "domains_html", + "fieldtype": "HTML", + "label": "Domains HTML" + }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "active_domains", - "fieldtype": "Table", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Active Domains", - "length": 0, - "no_copy": 0, - "options": "Has Domain", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "fieldname": "active_domains", + "fieldtype": "Table", + "hidden": 1, + "label": "Active Domains", + "options": "Has Domain", + "read_only": 1 } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 1, - "istable": 0, - "max_attachments": 0, - "modified": "2017-12-05 17:36:46.842134", - "modified_by": "Administrator", - "module": "Core", - "name": "Domain Settings", - "name_case": "", - "owner": "Administrator", + ], + "issingle": 1, + "links": [], + "modified": "2022-08-03 12:20:53.256607", + "modified_by": "Administrator", + "module": "Core", + "name": "Domain Settings", + "owner": "Administrator", "permissions": [ { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 1, - "delete": 1, - "email": 1, - "export": 0, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 0, - "role": "System Manager", - "set_user_permissions": 0, - "share": 1, - "submit": 0, + "create": 1, + "delete": 1, + "email": 1, + "print": 1, + "read": 1, + "role": "System Manager", + "share": 1, "write": 1 } - ], - "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1, - "track_seen": 0 + ], + "quick_entry": 1, + "sort_field": "modified", + "sort_order": "DESC", + "states": [], + "track_changes": 1 } \ No newline at end of file diff --git a/frappe/core/doctype/error_log/error_log.js b/frappe/core/doctype/error_log/error_log.js index 1262002b04..85b1c8b60a 100644 --- a/frappe/core/doctype/error_log/error_log.js +++ b/frappe/core/doctype/error_log/error_log.js @@ -2,11 +2,11 @@ // For license information, please see license.txt frappe.ui.form.on("Error Log", { - refresh: function(frm) { + refresh: function (frm) { frm.disable_save(); if (frm.doc.reference_doctype && frm.doc.reference_name) { - frm.add_custom_button(__("Show Related Errors"), function() { + frm.add_custom_button(__("Show Related Errors"), function () { frappe.set_route("List", "Error Log", { reference_doctype: frm.doc.reference_doctype, reference_name: frm.doc.reference_name, diff --git a/frappe/core/doctype/error_log/error_log.py b/frappe/core/doctype/error_log/error_log.py index c7ab98e034..871ec8ebdd 100644 --- a/frappe/core/doctype/error_log/error_log.py +++ b/frappe/core/doctype/error_log/error_log.py @@ -9,7 +9,7 @@ from frappe.query_builder.functions import Now class ErrorLog(Document): def onload(self): - if not self.seen: + if not self.seen and not frappe.flags.read_only: self.db_set("seen", 1, update_modified=0) frappe.db.commit() diff --git a/frappe/core/doctype/error_log/error_log_list.js b/frappe/core/doctype/error_log/error_log_list.js index e92773a9de..dabe95d6d7 100644 --- a/frappe/core/doctype/error_log/error_log_list.js +++ b/frappe/core/doctype/error_log/error_log_list.js @@ -1,6 +1,6 @@ frappe.listview_settings["Error Log"] = { add_fields: ["seen"], - get_indicator: function(doc) { + get_indicator: function (doc) { if (cint(doc.seen)) { return [__("Seen"), "green", "seen,=,1"]; } else { @@ -8,11 +8,11 @@ frappe.listview_settings["Error Log"] = { } }, order_by: "seen asc, modified desc", - onload: function(listview) { - listview.page.add_menu_item(__("Clear Error Logs"), function() { + onload: function (listview) { + listview.page.add_menu_item(__("Clear Error Logs"), function () { frappe.call({ method: "frappe.core.doctype.error_log.error_log.clear_error_logs", - callback: function() { + callback: function () { listview.refresh(); }, }); @@ -20,6 +20,6 @@ frappe.listview_settings["Error Log"] = { frappe.require("logtypes.bundle.js", () => { frappe.utils.logtypes.show_log_retention_message(cur_list.doctype); - }) + }); }, }; diff --git a/frappe/core/doctype/error_log/test_error_log.py b/frappe/core/doctype/error_log/test_error_log.py index c7cf26a0cf..59670de8d2 100644 --- a/frappe/core/doctype/error_log/test_error_log.py +++ b/frappe/core/doctype/error_log/test_error_log.py @@ -1,13 +1,12 @@ # Copyright (c) 2015, Frappe Technologies and Contributors # License: MIT. See LICENSE -import unittest - import frappe +from frappe.tests.utils import FrappeTestCase # test_records = frappe.get_test_records('Error Log') -class TestErrorLog(unittest.TestCase): +class TestErrorLog(FrappeTestCase): def test_error_log(self): """let's do an error log on error log?""" doc = frappe.new_doc("Error Log") diff --git a/frappe/core/doctype/error_snapshot/error_snapshot.js b/frappe/core/doctype/error_snapshot/error_snapshot.js index c1b2d996a1..f8a7e3ded5 100644 --- a/frappe/core/doctype/error_snapshot/error_snapshot.js +++ b/frappe/core/doctype/error_snapshot/error_snapshot.js @@ -1,14 +1,18 @@ -frappe.ui.form.on("Error Snapshot", "load", function(frm){ +frappe.ui.form.on("Error Snapshot", "load", function (frm) { frm.set_read_only(true); }); -frappe.ui.form.on("Error Snapshot", "refresh", function(frm){ - frm.set_df_property("view", "options", frappe.render_template("error_snapshot", {"doc": frm.doc})); +frappe.ui.form.on("Error Snapshot", "refresh", function (frm) { + frm.set_df_property( + "view", + "options", + frappe.render_template("error_snapshot", { doc: frm.doc }) + ); if (frm.doc.relapses) { - frm.add_custom_button(__('Show Relapses'), function() { + frm.add_custom_button(__("Show Relapses"), function () { frappe.route_options = { - parent_error_snapshot: frm.doc.name + parent_error_snapshot: frm.doc.name, }; frappe.set_route("List", "Error Snapshot"); }); diff --git a/frappe/core/doctype/error_snapshot/error_snapshot.json b/frappe/core/doctype/error_snapshot/error_snapshot.json index 1333fe0d5b..b92db8f99a 100644 --- a/frappe/core/doctype/error_snapshot/error_snapshot.json +++ b/frappe/core/doctype/error_snapshot/error_snapshot.json @@ -1,398 +1,130 @@ { - "allow_copy": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, - "creation": "2015-11-28 00:57:39.766888", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "System", - "editable_grid": 0, + "actions": [], + "creation": "2015-11-28 00:57:39.766888", + "doctype": "DocType", + "document_type": "System", + "engine": "InnoDB", + "field_order": [ + "view", + "seen", + "evalue", + "timestamp", + "relapses", + "etype", + "traceback", + "parent_error_snapshot", + "pyver", + "exception", + "locals", + "frames" + ], "fields": [ { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "view", - "fieldtype": "HTML", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Snapshot View", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "view", + "fieldtype": "HTML", + "label": "Snapshot View" + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "seen", - "fieldtype": "Check", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 1, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Seen", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "default": "0", + "fieldname": "seen", + "fieldtype": "Check", + "hidden": 1, + "in_filter": 1, + "label": "Seen" + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "evalue", - "fieldtype": "Code", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Friendly Title", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "evalue", + "fieldtype": "Code", + "hidden": 1, + "in_list_view": 1, + "label": "Friendly Title", + "read_only": 1 + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "timestamp", - "fieldtype": "Datetime", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Timestamp", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "timestamp", + "fieldtype": "Datetime", + "hidden": 1, + "label": "Timestamp", + "read_only": 1 + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "default": "1", - "fieldname": "relapses", - "fieldtype": "Int", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Relapses", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "default": "1", + "fieldname": "relapses", + "fieldtype": "Int", + "hidden": 1, + "in_list_view": 1, + "label": "Relapses", + "read_only": 1 + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "etype", - "fieldtype": "Data", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Exception Type", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "etype", + "fieldtype": "Data", + "hidden": 1, + "label": "Exception Type", + "read_only": 1 + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "traceback", - "fieldtype": "Code", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Traceback", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "traceback", + "fieldtype": "Code", + "hidden": 1, + "label": "Traceback", + "read_only": 1 + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "parent_error_snapshot", - "fieldtype": "Data", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Parent Error Snapshot", - "length": 0, - "no_copy": 0, - "options": "", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "parent_error_snapshot", + "fieldtype": "Data", + "hidden": 1, + "label": "Parent Error Snapshot" + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "pyver", - "fieldtype": "Code", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Pyver", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "pyver", + "fieldtype": "Code", + "hidden": 1, + "label": "Pyver", + "read_only": 1 + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "exception", - "fieldtype": "Code", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Exception", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "exception", + "fieldtype": "Code", + "hidden": 1, + "label": "Exception" + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "locals", - "fieldtype": "Code", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Locals", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "locals", + "fieldtype": "Code", + "hidden": 1, + "label": "Locals" + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "frames", - "fieldtype": "Code", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Frames", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "fieldname": "frames", + "fieldtype": "Code", + "hidden": 1, + "label": "Frames" } - ], - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 1, - - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2021-10-25 14:40:38.619106", - "modified_by": "Administrator", - "module": "Core", - "name": "Error Snapshot", - "name_case": "", - "owner": "Administrator", + ], + "in_create": 1, + "links": [], + "modified": "2022-08-03 12:20:53.504160", + "modified_by": "Administrator", + "module": "Core", + "name": "Error Snapshot", + "owner": "Administrator", "permissions": [ { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "if_owner": 0, - "import": 0, - "is_custom": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 1, - "role": "Administrator", - "set_user_permissions": 0, - "share": 1, - "submit": 0, + "create": 1, + "delete": 1, + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "Administrator", + "share": 1, "write": 1 } - ], - "quick_entry": 0, - "read_only": 0, - "read_only_onload": 0, - "sort_field": "timestamp", - "sort_order": "DESC", - "title_field": "evalue", - "track_seen": 0 -} + ], + "sort_field": "timestamp", + "sort_order": "DESC", + "states": [], + "title_field": "evalue" +} \ No newline at end of file diff --git a/frappe/core/doctype/error_snapshot/error_snapshot.py b/frappe/core/doctype/error_snapshot/error_snapshot.py index 6966cf0aca..acc49c78cd 100644 --- a/frappe/core/doctype/error_snapshot/error_snapshot.py +++ b/frappe/core/doctype/error_snapshot/error_snapshot.py @@ -12,10 +12,10 @@ class ErrorSnapshot(Document): def onload(self): if not self.parent_error_snapshot: - self.db_set("seen", True, update_modified=False) + self.db_set("seen", 1, update_modified=False) for relapsed in frappe.get_all("Error Snapshot", filters={"parent_error_snapshot": self.name}): - frappe.db.set_value("Error Snapshot", relapsed.name, "seen", True, update_modified=False) + frappe.db.set_value("Error Snapshot", relapsed.name, "seen", 1, update_modified=False) frappe.local.flags.commit = True @@ -32,7 +32,7 @@ class ErrorSnapshot(Document): self.update({"parent_error_snapshot": parent["name"]}) frappe.db.set_value("Error Snapshot", parent["name"], "relapses", parent["relapses"] + 1) if parent["seen"]: - frappe.db.set_value("Error Snapshot", parent["name"], "seen", False) + frappe.db.set_value("Error Snapshot", parent["name"], "seen", 0) @staticmethod def clear_old_logs(days=30): diff --git a/frappe/core/doctype/error_snapshot/error_snapshot_list.js b/frappe/core/doctype/error_snapshot/error_snapshot_list.js index 553495beb1..b331788852 100644 --- a/frappe/core/doctype/error_snapshot/error_snapshot_list.js +++ b/frappe/core/doctype/error_snapshot/error_snapshot_list.js @@ -1,19 +1,19 @@ frappe.listview_settings["Error Snapshot"] = { add_fields: ["parent_error_snapshot", "relapses", "seen"], - filters:[ - ["parent_error_snapshot","=",null], - ["seen", "=", false] + filters: [ + ["parent_error_snapshot", "=", null], + ["seen", "=", false], ], - get_indicator: function(doc){ - if (doc.parent_error_snapshot && doc.parent_error_snapshot.length){ + get_indicator: function (doc) { + if (doc.parent_error_snapshot && doc.parent_error_snapshot.length) { return [__("Relapsed"), !doc.seen ? "orange" : "blue", "parent_error_snapshot,!=,"]; } else { return [__("First Level"), !doc.seen ? "red" : "green", "parent_error_snapshot,=,"]; } }, - onload: function(listview) { + onload: function (listview) { frappe.require("logtypes.bundle.js", () => { frappe.utils.logtypes.show_log_retention_message(cur_list.doctype); - }) + }); }, -} +}; diff --git a/frappe/core/doctype/error_snapshot/test_error_snapshot.py b/frappe/core/doctype/error_snapshot/test_error_snapshot.py index 0c1f861b43..4779d56c7b 100644 --- a/frappe/core/doctype/error_snapshot/test_error_snapshot.py +++ b/frappe/core/doctype/error_snapshot/test_error_snapshot.py @@ -1,9 +1,11 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import unittest +from frappe.tests.utils import FrappeTestCase +from frappe.utils.logger import sanitized_dict # test_records = frappe.get_test_records('Error Snapshot') -class TestErrorSnapshot(unittest.TestCase): - pass +class TestErrorSnapshot(FrappeTestCase): + def test_form_dict_sanitization(self): + self.assertNotEqual(sanitized_dict({"pwd": "SECRET", "usr": "WHAT"}).get("pwd"), "SECRET") diff --git a/frappe/core/doctype/feedback/feedback.js b/frappe/core/doctype/feedback/feedback.js deleted file mode 100644 index 131f0e19d8..0000000000 --- a/frappe/core/doctype/feedback/feedback.js +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright (c) 2021, Frappe Technologies and contributors -// For license information, please see license.txt - -frappe.ui.form.on('Feedback', { - // refresh: function(frm) { - - // } -}); diff --git a/frappe/core/doctype/feedback/feedback.json b/frappe/core/doctype/feedback/feedback.json deleted file mode 100644 index f8380cfda6..0000000000 --- a/frappe/core/doctype/feedback/feedback.json +++ /dev/null @@ -1,73 +0,0 @@ -{ - "actions": [], - "creation": "2021-06-03 19:02:55.328423", - "doctype": "DocType", - "editable_grid": 1, - "engine": "InnoDB", - "field_order": [ - "reference_doctype", - "reference_name", - "column_break_3", - "like", - "ip_address" - ], - "fields": [ - { - "fieldname": "column_break_3", - "fieldtype": "Column Break" - }, - { - "fieldname": "reference_doctype", - "fieldtype": "Select", - "in_list_view": 1, - "label": "Reference Document Type", - "options": "\nBlog Post" - }, - { - "fieldname": "reference_name", - "fieldtype": "Dynamic Link", - "in_list_view": 1, - "label": "Reference Name", - "options": "reference_doctype", - "reqd": 1 - }, - { - "fieldname": "ip_address", - "fieldtype": "Data", - "hidden": 1, - "label": "IP Address", - "read_only": 1 - }, - { - "default": "0", - "fieldname": "like", - "fieldtype": "Check", - "label": "Like" - } - ], - "index_web_pages_for_search": 1, - "links": [], - "modified": "2021-11-10 20:53:21.255593", - "modified_by": "Administrator", - "module": "Core", - "name": "Feedback", - "owner": "Administrator", - "permissions": [ - { - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "share": 1, - "write": 1 - } - ], - "sort_field": "modified", - "sort_order": "DESC", - "title_field": "reference_name", - "track_changes": 1 -} \ No newline at end of file diff --git a/frappe/core/doctype/feedback/feedback.py b/frappe/core/doctype/feedback/feedback.py deleted file mode 100644 index c616787e4b..0000000000 --- a/frappe/core/doctype/feedback/feedback.py +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright (c) 2021, Frappe Technologies and contributors -# License: MIT. See LICENSE - -# import frappe -from frappe.model.document import Document - - -class Feedback(Document): - pass diff --git a/frappe/core/doctype/feedback/test_feedback.py b/frappe/core/doctype/feedback/test_feedback.py deleted file mode 100644 index e8e29e75ae..0000000000 --- a/frappe/core/doctype/feedback/test_feedback.py +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright (c) 2021, Frappe Technologies and Contributors -# License: MIT. See LICENSE - -import unittest - -import frappe - - -class TestFeedback(unittest.TestCase): - def tearDown(self): - frappe.form_dict.reference_doctype = None - frappe.form_dict.reference_name = None - frappe.form_dict.like = None - frappe.local.request_ip = None - - def test_feedback_creation_updation(self): - from frappe.website.doctype.blog_post.test_blog_post import make_test_blog - - test_blog = make_test_blog() - - frappe.db.delete("Feedback", {"reference_doctype": "Blog Post"}) - - from frappe.templates.includes.feedback.feedback import give_feedback - - frappe.form_dict.reference_doctype = "Blog Post" - frappe.form_dict.reference_name = test_blog.name - frappe.form_dict.like = True - frappe.local.request_ip = "127.0.0.1" - - feedback = give_feedback() - - self.assertEqual(feedback.like, True) - - frappe.form_dict.like = False - - updated_feedback = give_feedback() - - self.assertEqual(updated_feedback.like, False) - - frappe.db.delete("Feedback", {"reference_doctype": "Blog Post"}) - - test_blog.delete() diff --git a/frappe/core/doctype/file/file.js b/frappe/core/doctype/file/file.js index ecad8d884a..159cf1ce39 100644 --- a/frappe/core/doctype/file/file.js +++ b/frappe/core/doctype/file/file.js @@ -1,41 +1,97 @@ -frappe.ui.form.on("File", "refresh", function(frm) { - if(!frm.doc.is_folder) { - frm.add_custom_button(__('Download'), function() { - var file_url = frm.doc.file_url; - if (frm.doc.file_name) { - file_url = file_url.replace(/#/g, '%23'); - } - window.open(file_url); - }, "fa fa-download"); - } +frappe.ui.form.on("File", { + refresh: function (frm) { + if (!frm.doc.is_folder) { + // add download button + frm.add_custom_button(__("Download"), () => frm.trigger("download"), "fa fa-download"); + } - frm.get_field("preview_html").$wrapper.html(`
    - -
    `); + frm.toggle_display("preview", false); - var is_raster_image = (/\.(gif|jpg|jpeg|tiff|png)$/i).test(frm.doc.file_url); - var is_optimizable = !frm.doc.is_folder && is_raster_image && frm.doc.file_size > 0; + // preview different file types + frm.trigger("preview_file"); - if (is_optimizable) { - frm.add_custom_button(__("Optimize"), function() { - frappe.show_alert(__("Optimizing image...")); - frm.call("optimize_file").then(() => { - frappe.show_alert(__("Image optimized")); - }); + let is_raster_image = /\.(gif|jpg|jpeg|tiff|png)$/i.test(frm.doc.file_url); + let is_optimizable = !frm.doc.is_folder && is_raster_image && frm.doc.file_size > 0; + + // add optimize button + is_optimizable && frm.add_custom_button(__("Optimize"), () => frm.trigger("optimize")); + + // add unzip button + if (frm.doc.file_name && frm.doc.file_name.split(".").splice(-1)[0] === "zip") { + frm.add_custom_button(__("Unzip"), () => frm.trigger("unzip")); + } + }, + + preview_file: function (frm) { + let $preview = ""; + let file_name = frm.doc.file_name.split("?")[0]; + let file_extension = file_name.split(".").pop()?.toLowerCase(); + + if (frappe.utils.is_image_file(frm.doc.file_url)) { + $preview = $(`
    + +
    `); + } else if (frappe.utils.is_video_file(frm.doc.file_url)) { + $preview = $(`
    + +
    `); + } else if (file_extension === "pdf") { + $preview = $(`
    + + + +
    `); + } else if (file_extension === "mp3") { + $preview = $(`
    + +
    `); + } + + if ($preview) { + frm.toggle_display("preview", true); + frm.get_field("preview_html").$wrapper.html($preview); + } + }, + + download: function (frm) { + let file_url = frm.doc.file_url; + if (frm.doc.file_name) { + file_url = file_url.replace(/#/g, "%23"); + } + window.open(file_url); + }, + + optimize: function (frm) { + frappe.show_alert(__("Optimizing image...")); + frm.call("optimize_file").then(() => { + frappe.show_alert(__("Image optimized")); }); - } + }, - if(frm.doc.file_name && frm.doc.file_name.split('.').splice(-1)[0]==='zip') { - frm.add_custom_button(__('Unzip'), function() { - frappe.call({ - method: "frappe.core.api.file.unzip_file", - args: { - name: frm.doc.name, - }, - callback: function() { - frappe.set_route('List', 'File'); - } - }); + unzip: function (frm) { + frappe.call({ + method: "frappe.core.api.file.unzip_file", + args: { + name: frm.doc.name, + }, + callback: function () { + frappe.set_route("List", "File"); + }, }); - } + }, }); diff --git a/frappe/core/doctype/file/file.json b/frappe/core/doctype/file/file.json index 3008e27aa0..d6c4a99bc3 100644 --- a/frappe/core/doctype/file/file.json +++ b/frappe/core/doctype/file/file.json @@ -2,6 +2,7 @@ "actions": [], "allow_import": 1, "creation": "2012-12-12 11:19:22", + "default_view": "File", "doctype": "DocType", "engine": "InnoDB", "field_order": [ @@ -64,8 +65,7 @@ "fieldname": "is_home_folder", "fieldtype": "Check", "hidden": 1, - "label": "Is Home Folder", - "search_index": 1 + "label": "Is Home Folder" }, { "default": "0", @@ -125,8 +125,7 @@ "in_standard_filter": 1, "label": "Attached To DocType", "options": "DocType", - "read_only": 1, - "search_index": 1 + "read_only": 1 }, { "fieldname": "column_break_10", @@ -136,8 +135,7 @@ "fieldname": "attached_to_name", "fieldtype": "Data", "label": "Attached To Name", - "read_only": 1, - "search_index": 1 + "read_only": 1 }, { "fieldname": "attached_to_field", @@ -172,10 +170,11 @@ "read_only": 1 } ], + "force_re_route_to_default_view": 1, "icon": "fa fa-file", "idx": 1, "links": [], - "modified": "2020-06-28 12:21:30.772386", + "modified": "2022-09-13 15:50:15.508251", "modified_by": "Administrator", "module": "Core", "name": "File", @@ -210,6 +209,7 @@ ], "sort_field": "modified", "sort_order": "ASC", + "states": [], "title_field": "file_name", "track_changes": 1 } \ No newline at end of file diff --git a/frappe/core/doctype/file/file.py b/frappe/core/doctype/file/file.py index 7a66d45c73..3728bd0af0 100755 --- a/frappe/core/doctype/file/file.py +++ b/frappe/core/doctype/file/file.py @@ -14,6 +14,7 @@ from requests.exceptions import HTTPError, SSLError import frappe from frappe import _ +from frappe.database.schema import SPECIAL_CHAR_PATTERN from frappe.model.document import Document from frappe.utils import call_hook_method, cint, get_files_path, get_hook_method, get_url from frappe.utils.file_manager import is_safe_path @@ -78,21 +79,35 @@ class File(Document): self.validate_duplicate_entry() def validate(self): + if self.is_folder: + return + # Ensure correct formatting and type self.file_url = unquote(self.file_url) if self.file_url else "" + self.validate_attachment_references() + # when dict is passed to get_doc for creation of new_doc, is_new returns None # this case is handled inside handle_is_private_changed if not self.is_new() and self.has_value_changed("is_private"): self.handle_is_private_changed() - if not self.is_folder: - self.validate_file_path() - self.validate_file_url() - self.validate_file_on_disk() + self.validate_file_path() + self.validate_file_url() + self.validate_file_on_disk() self.file_size = frappe.form_dict.file_size or self.file_size + def validate_attachment_references(self): + if not self.attached_to_doctype: + return + + if not self.attached_to_name or not isinstance(self.attached_to_name, (str, int)): + frappe.throw(_("Attached To Name must be a string or an integer"), frappe.ValidationError) + + if self.attached_to_field and SPECIAL_CHAR_PATTERN.search(self.attached_to_field): + frappe.throw(_("The fieldname you've specified in Attached To Field is invalid")) + def after_rename(self, *args, **kwargs): for successor in self.get_successors(): setup_folder_path(successor, self.name) @@ -262,7 +277,7 @@ class File(Document): def validate_remote_file(self): """Validates if file uploaded using URL already exist""" site_url = get_url() - if "/files/" in self.file_url and self.file_url.startswith(site_url): + if self.file_url and "/files/" in self.file_url and self.file_url.startswith(site_url): self.file_url = self.file_url.split(site_url, 1)[1] def set_folder_name(self): @@ -314,7 +329,11 @@ class File(Document): self.file_url = duplicate_file.file_url def set_file_name(self): - if not self.file_name and self.file_url: + if not self.file_name and not self.file_url: + frappe.throw( + _("Fields `file_name` or `file_url` must be set for File"), exc=frappe.MandatoryError + ) + elif not self.file_name and self.file_url: self.file_name = self.file_url.split("/")[-1] else: self.file_name = re.sub(r"/", "", self.file_name) @@ -406,7 +425,10 @@ class File(Document): continue file_doc = frappe.new_doc("File") - file_doc.content = z.read(file.filename) + try: + file_doc.content = z.read(file.filename) + except zipfile.BadZipFile: + frappe.throw(_("{0} is a not a valid zip file").format(self.file_name)) file_doc.file_name = filename file_doc.folder = self.folder file_doc.is_private = self.is_private @@ -422,7 +444,6 @@ class File(Document): return os.path.exists(self.get_full_path()) def get_content(self) -> bytes: - """Returns [`file_name`, `content`] for given file name `fname`""" if self.is_folder: frappe.throw(_("Cannot get file contents of a Folder")) @@ -617,7 +638,9 @@ class File(Document): def create_attachment_record(self): icon = ' ' if self.is_private else "" - file_url = quote(frappe.safe_encode(self.file_url)) if self.file_url else self.file_name + file_url = ( + quote(frappe.safe_encode(self.file_url), safe="/:") if self.file_url else self.file_name + ) file_name = self.file_name or self.file_url self.add_comment_in_reference_doc( diff --git a/frappe/core/doctype/file/test_file.py b/frappe/core/doctype/file/test_file.py index a9d40b35b1..51e065f710 100644 --- a/frappe/core/doctype/file/test_file.py +++ b/frappe/core/doctype/file/test_file.py @@ -3,7 +3,8 @@ import base64 import json import os -import unittest +import shutil +import tempfile from contextlib import contextmanager from typing import TYPE_CHECKING @@ -16,6 +17,7 @@ from frappe.core.api.file import ( move_file, unzip_file, ) +from frappe.core.doctype.file.utils import get_extension from frappe.exceptions import ValidationError from frappe.tests.utils import FrappeTestCase from frappe.utils import get_files_path @@ -84,7 +86,7 @@ class TestBase64File(FrappeTestCase): "doctype": "File", "file_name": "test_base64.txt", "attached_to_doctype": self.attached_to_doctype, - "attached_to_docname": self.attached_to_docname, + "attached_to_name": self.attached_to_docname, "content": self.test_content, "decode": True, } @@ -239,7 +241,7 @@ class TestFile(FrappeTestCase): pass def delete_test_data(self): - test_file_data = frappe.db.get_all( + test_file_data = frappe.get_all( "File", pluck="name", filters={"is_home_folder": 0, "is_attachments_folder": 0}, @@ -460,7 +462,7 @@ class TestFile(FrappeTestCase): ).insert(ignore_permissions=True) test_file.make_thumbnail() - self.assertTrue(test_file.thumbnail_url.endswith("_small.jpeg")) + self.assertTrue(test_file.thumbnail_url.endswith("_small.jpg")) # test local image test_file.db_set("thumbnail_url", None) @@ -510,12 +512,51 @@ class TestFile(FrappeTestCase): ).insert(ignore_permissions=True) self.assertRaisesRegex(ValidationError, "not a zip file", test_file.unzip) + def test_create_file_without_file_url(self): + test_file = frappe.get_doc( + { + "doctype": "File", + "file_name": "logo", + "content": "frappe", + } + ).insert() + assert test_file is not None -class TestAttachment(unittest.TestCase): + def test_symlinked_files_folder(self): + files_dir = os.path.abspath(get_files_path()) + with convert_to_symlink(files_dir): + file = frappe.get_doc( + { + "doctype": "File", + "file_name": "symlinked_folder_test.txt", + "content": "42", + } + ) + file.save() + file.content = "" + file._content = "" + file.save().reload() + self.assertIn("42", file.get_content()) + + +@contextmanager +def convert_to_symlink(directory): + """Moves a directory to temp directory and symlinks original path for testing""" + try: + new_directory = shutil.move(directory, tempfile.mkdtemp()) + os.symlink(new_directory, directory) + yield + finally: + os.unlink(directory) + shutil.move(new_directory, directory) + + +class TestAttachment(FrappeTestCase): test_doctype = "Test For Attachment" @classmethod def setUpClass(cls): + super().setUpClass() frappe.get_doc( doctype="DocType", name=cls.test_doctype, @@ -699,3 +740,10 @@ class TestFileOptimization(FrappeTestCase): size_after_rollback = os.stat(image_path).st_size self.assertEqual(size_before_optimization, size_after_rollback) + + def test_image_header_guessing(self): + file_path = frappe.get_app_path("frappe", "tests/data/sample_image_for_optimization.jpg") + with open(file_path, "rb") as f: + file_content = f.read() + + self.assertEqual(get_extension("", None, file_content), "jpg") diff --git a/frappe/core/doctype/file/utils.py b/frappe/core/doctype/file/utils.py index e59ec2aede..1d0d145303 100644 --- a/frappe/core/doctype/file/utils.py +++ b/frappe/core/doctype/file/utils.py @@ -1,5 +1,4 @@ import hashlib -import imghdr import mimetypes import os import re @@ -7,6 +6,7 @@ from io import BytesIO from typing import TYPE_CHECKING, Optional from urllib.parse import unquote +import filetype import requests import requests.exceptions from PIL import Image @@ -76,9 +76,11 @@ def get_extension( mimetype = mimetypes.guess_type(filename + "." + extn)[0] - if mimetype is None or not mimetype.startswith("image/") and content: - # detect file extension by reading image header properties - extn = imghdr.what(filename + "." + (extn or ""), h=content) + if mimetype is None and extn is None and content: + # detect file extension by using filetype matchers + _type_info = filetype.match(content) + if _type_info: + extn = _type_info.extension return extn @@ -225,7 +227,7 @@ def extract_images_from_html(doc: "Document", content: str, is_private: bool = F def _save_file(match): data = match.group(1).split("data:")[1] headers, content = data.split(",") - mtype = headers.split(";")[0] + mtype = headers.split(";", 1)[0] if isinstance(content, str): content = content.encode("utf-8") @@ -237,7 +239,7 @@ def extract_images_from_html(doc: "Document", content: str, is_private: bool = F if "filename=" in headers: filename = headers.split("filename=")[-1] - filename = safe_decode(filename).split(";")[0] + filename = safe_decode(filename).split(";", 1)[0] else: filename = get_random_filename(content_type=mtype) @@ -327,7 +329,7 @@ def attach_files_to_document(doc: "File", event) -> None: folder="Home/Attachments", ) try: - file.insert() + file.insert(ignore_permissions=True) except Exception: doc.log_error("Error Attaching File") diff --git a/frappe/core/doctype/has_domain/has_domain.json b/frappe/core/doctype/has_domain/has_domain.json index e2b646b457..c34626b269 100644 --- a/frappe/core/doctype/has_domain/has_domain.json +++ b/frappe/core/doctype/has_domain/has_domain.json @@ -1,72 +1,32 @@ { - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, - "creation": "2017-05-03 15:20:22.326623", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", + "actions": [], + "creation": "2017-05-03 15:20:22.326623", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "domain" + ], "fields": [ { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "domain", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Domain", - "length": 0, - "no_copy": 0, - "options": "Domain", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "fieldname": "domain", + "fieldtype": "Link", + "in_list_view": 1, + "label": "Domain", + "options": "Domain" } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 1, - "max_attachments": 0, - "modified": "2020-09-18 17:26:09.703215", - "modified_by": "Administrator", - "module": "Core", - "name": "Has Domain", - "name_case": "", - "owner": "Administrator", - "permissions": [], - "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1, - "track_seen": 0 + ], + "istable": 1, + "links": [], + "modified": "2022-08-03 12:20:53.381248", + "modified_by": "Administrator", + "module": "Core", + "name": "Has Domain", + "owner": "Administrator", + "permissions": [], + "quick_entry": 1, + "sort_field": "modified", + "sort_order": "DESC", + "states": [], + "track_changes": 1 } \ No newline at end of file diff --git a/frappe/core/doctype/has_role/has_role.json b/frappe/core/doctype/has_role/has_role.json index e0759dcd7e..689e80480e 100644 --- a/frappe/core/doctype/has_role/has_role.json +++ b/frappe/core/doctype/has_role/has_role.json @@ -1,64 +1,34 @@ { - "allow_copy": 0, - "allow_import": 0, - "allow_rename": 0, - "autoname": "hash", - "beta": 0, - "creation": "2013-02-22 01:27:34", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "editable_grid": 1, + "actions": [], + "autoname": "hash", + "creation": "2013-02-22 01:27:34", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "role" + ], "fields": [ { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "role", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Role", - "length": 0, - "no_copy": 0, - "oldfieldname": "role", - "oldfieldtype": "Link", - "options": "Role", - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "fieldname": "role", + "fieldtype": "Link", + "in_list_view": 1, + "label": "Role", + "oldfieldname": "role", + "oldfieldtype": "Link", + "options": "Role" } - ], - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 1, - "image_view": 0, - "in_create": 0, - - "is_submittable": 0, - "issingle": 0, - "istable": 1, - "max_attachments": 0, - "modified": "2017-02-13 14:00:08.116312", - "modified_by": "Administrator", - "module": "Core", - "name": "Has Role", - "owner": "Administrator", - "permissions": [], - "quick_entry": 0, - "read_only": 0, - "read_only_onload": 0, - "track_changes": 0, - "track_seen": 0 + ], + "idx": 1, + "istable": 1, + "links": [], + "modified": "2022-08-03 12:20:54.382064", + "modified_by": "Administrator", + "module": "Core", + "name": "Has Role", + "owner": "Administrator", + "permissions": [], + "sort_field": "modified", + "sort_order": "DESC", + "states": [] } \ No newline at end of file diff --git a/frappe/core/doctype/installed_applications/installed_applications.js b/frappe/core/doctype/installed_applications/installed_applications.js index 9a1fd5ac18..507cf76875 100644 --- a/frappe/core/doctype/installed_applications/installed_applications.js +++ b/frappe/core/doctype/installed_applications/installed_applications.js @@ -1,8 +1,67 @@ // Copyright (c) 2020, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Installed Applications', { - // refresh: function(frm) { +frappe.ui.form.on("Installed Applications", { + refresh: function (frm) { + frm.add_custom_button(__("Update Hooks Resolution Order"), () => { + frm.trigger("show_update_order_dialog"); + }); + }, - // } + show_update_order_dialog() { + const dialog = new frappe.ui.Dialog({ + title: __("Update Hooks Resolution Order"), + fields: [ + { + fieldname: "apps", + fieldtype: "Table", + label: __("Installed Apps"), + cannot_add_rows: true, + cannot_delete_rows: true, + in_place_edit: true, + data: [], + fields: [ + { + fieldtype: "Data", + fieldname: "app_name", + label: __("App Name"), + in_list_view: 1, + read_only: 1, + }, + ], + }, + ], + primary_action: function () { + const new_order = this.get_values()["apps"].map((row) => row.app_name); + frappe.call({ + method: "frappe.core.doctype.installed_applications.installed_applications.update_installed_apps_order", + freeze: true, + args: { + new_order: new_order, + }, + }); + this.hide(); + }, + primary_action_label: __("Update Order"), + }); + + frappe + .xcall( + "frappe.core.doctype.installed_applications.installed_applications.get_installed_app_order" + ) + .then((data) => { + data.forEach((app) => { + dialog.fields_dict.apps.df.data.push({ + app_name: app, + }); + }); + + dialog.fields_dict.apps.grid.refresh(); + // hack: change checkboxes to drag handles. + let grid = $(dialog.fields_dict.apps.grid.parent); + grid.find(".grid-row-check:first").remove() && + grid.find(".grid-row-check").replaceWith(frappe.utils.icon("menu")); + dialog.show(); + }); + }, }); diff --git a/frappe/core/doctype/installed_applications/installed_applications.py b/frappe/core/doctype/installed_applications/installed_applications.py index 07b20db153..e3f63e8bb2 100644 --- a/frappe/core/doctype/installed_applications/installed_applications.py +++ b/frappe/core/doctype/installed_applications/installed_applications.py @@ -1,10 +1,17 @@ # Copyright (c) 2020, Frappe Technologies and contributors # License: MIT. See LICENSE +import json + import frappe +from frappe import _ from frappe.model.document import Document +class InvalidAppOrder(frappe.ValidationError): + pass + + class InstalledApplications(Document): def update_versions(self): self.delete_key("installed_applications") @@ -18,3 +25,51 @@ class InstalledApplications(Document): }, ) self.save() + + +@frappe.whitelist() +def update_installed_apps_order(new_order: list[str] | str): + """Change the ordering of `installed_apps` global + + This list is used to resolve hooks and by default it's order of installation on site. + + Sometimes it might not be the ordering you want, so thie function is provided to override it. + """ + frappe.only_for("System Manager") + + if isinstance(new_order, str): + new_order = json.loads(new_order) + + frappe.local.request_cache and frappe.local.request_cache.clear() + existing_order = frappe.get_installed_apps(_ensure_on_bench=True) + + if set(existing_order) != set(new_order) or not isinstance(new_order, list): + frappe.throw( + _("You are only allowed to update order, do not remove or add apps."), exc=InvalidAppOrder + ) + + # Ensure frappe is always first regardless of user's preference. + if "frappe" in new_order: + new_order.remove("frappe") + new_order.insert(0, "frappe") + + frappe.db.set_global("installed_apps", json.dumps(new_order)) + + _create_version_log_for_change(existing_order, new_order) + + +def _create_version_log_for_change(old, new): + version = frappe.new_doc("Version") + version.ref_doctype = "DefaultValue" + version.docname = "installed_apps" + version.data = frappe.as_json({"changed": [["current", json.dumps(old), json.dumps(new)]]}) + version.flags.ignore_links = True # This is a fake doctype + version.flags.ignore_permissions = True + version.insert() + + +@frappe.whitelist() +def get_installed_app_order() -> list[str]: + frappe.only_for("System Manager") + + return frappe.get_installed_apps(_ensure_on_bench=True) diff --git a/frappe/core/doctype/installed_applications/test_installed_applications.py b/frappe/core/doctype/installed_applications/test_installed_applications.py index b67cc4c8c7..1ee1c99b86 100644 --- a/frappe/core/doctype/installed_applications/test_installed_applications.py +++ b/frappe/core/doctype/installed_applications/test_installed_applications.py @@ -1,8 +1,16 @@ # Copyright (c) 2020, Frappe Technologies and Contributors # License: MIT. See LICENSE -# import frappe -import unittest + +import frappe +from frappe.core.doctype.installed_applications.installed_applications import ( + InvalidAppOrder, + update_installed_apps_order, +) +from frappe.tests.utils import FrappeTestCase -class TestInstalledApplications(unittest.TestCase): - pass +class TestInstalledApplications(FrappeTestCase): + def test_order_change(self): + update_installed_apps_order(["frappe"]) + self.assertRaises(InvalidAppOrder, update_installed_apps_order, []) + self.assertRaises(InvalidAppOrder, update_installed_apps_order, ["frappe", "deepmind"]) diff --git a/frappe/core/doctype/language/language.js b/frappe/core/doctype/language/language.js index e60282ebbf..9c7852f9e0 100644 --- a/frappe/core/doctype/language/language.js +++ b/frappe/core/doctype/language/language.js @@ -1,8 +1,6 @@ // Copyright (c) 2016, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Language', { - refresh: function(frm) { - - } +frappe.ui.form.on("Language", { + refresh: function (frm) {}, }); diff --git a/frappe/core/doctype/language/language.json b/frappe/core/doctype/language/language.json index 9ab8f55f6b..c9110bb998 100644 --- a/frappe/core/doctype/language/language.json +++ b/frappe/core/doctype/language/language.json @@ -51,7 +51,7 @@ "icon": "fa fa-globe", "in_create": 1, "links": [], - "modified": "2021-10-18 14:02:06.818219", + "modified": "2023-04-13 13:48:38.127995", "modified_by": "Administrator", "module": "Core", "name": "Language", @@ -66,18 +66,15 @@ "write": 1 }, { - "email": 1, - "export": 1, - "print": 1, - "read": 1, - "report": 1, - "role": "Guest", - "share": 1 + "role": "All", + "read": 1 } ], "search_fields": "language_name", + "show_title_field_in_link": 1, "sort_field": "modified", "sort_order": "DESC", + "states": [], "title_field": "language_name", "track_changes": 1 } \ No newline at end of file diff --git a/frappe/core/doctype/language/language.py b/frappe/core/doctype/language/language.py index efac7b0d77..0c5ee2d840 100644 --- a/frappe/core/doctype/language/language.py +++ b/frappe/core/doctype/language/language.py @@ -30,7 +30,7 @@ def validate_with_regex(name, label): def export_languages_json(): """Export list of all languages""" - languages = frappe.db.get_all("Language", fields=["name", "language_name"]) + languages = frappe.get_all("Language", fields=["name", "language_name"]) languages = [{"name": d.language_name, "code": d.name} for d in languages] languages.sort(key=lambda a: a["code"]) diff --git a/frappe/core/doctype/language/test_language.py b/frappe/core/doctype/language/test_language.py index 1f9c96a5d8..1f1e9cb913 100644 --- a/frappe/core/doctype/language/test_language.py +++ b/frappe/core/doctype/language/test_language.py @@ -1,9 +1,9 @@ # Copyright (c) 2015, Frappe Technologies and Contributors # License: MIT. See LICENSE -import unittest +from frappe.tests.utils import FrappeTestCase # test_records = frappe.get_test_records('Language') -class TestLanguage(unittest.TestCase): +class TestLanguage(FrappeTestCase): pass diff --git a/frappe/core/doctype/log_setting_user/log_setting_user.js b/frappe/core/doctype/log_setting_user/log_setting_user.js index a1eb824e22..61f3aa67c5 100644 --- a/frappe/core/doctype/log_setting_user/log_setting_user.js +++ b/frappe/core/doctype/log_setting_user/log_setting_user.js @@ -1,8 +1,7 @@ // Copyright (c) 2020, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Log Setting User', { +frappe.ui.form.on("Log Setting User", { // refresh: function(frm) { - // } }); diff --git a/frappe/core/doctype/log_setting_user/test_log_setting_user.py b/frappe/core/doctype/log_setting_user/test_log_setting_user.py index 9ea56e8ec4..556dc36dc9 100644 --- a/frappe/core/doctype/log_setting_user/test_log_setting_user.py +++ b/frappe/core/doctype/log_setting_user/test_log_setting_user.py @@ -1,8 +1,8 @@ # Copyright (c) 2020, Frappe Technologies and Contributors # License: MIT. See LICENSE # import frappe -import unittest +from frappe.tests.utils import FrappeTestCase -class TestLogSettingUser(unittest.TestCase): +class TestLogSettingUser(FrappeTestCase): pass diff --git a/frappe/core/doctype/log_settings/log_settings.js b/frappe/core/doctype/log_settings/log_settings.js index dc7cc7eac2..b72cd5285a 100644 --- a/frappe/core/doctype/log_settings/log_settings.js +++ b/frappe/core/doctype/log_settings/log_settings.js @@ -7,9 +7,7 @@ frappe.ui.form.on("Log Settings", { const added_doctypes = frm.doc.logs_to_clear.map((r) => r.ref_doctype); return { query: "frappe.core.doctype.log_settings.log_settings.get_log_doctypes", - filters: [ - ["name", "not in", added_doctypes], - ], + filters: [["name", "not in", added_doctypes]], }; }); }, diff --git a/frappe/core/doctype/log_settings/log_settings.py b/frappe/core/doctype/log_settings/log_settings.py index 8009324e70..832be49f3c 100644 --- a/frappe/core/doctype/log_settings/log_settings.py +++ b/frappe/core/doctype/log_settings/log_settings.py @@ -16,6 +16,12 @@ DEFAULT_LOGTYPES_RETENTION = { "Email Queue": 30, "Error Snapshot": 30, "Scheduled Job Log": 90, + "Route History": 90, + "Submission Queue": 30, + "Prepared Report": 30, + "Webhook Request Log": 30, + "Integration Request": 90, + "Reminder": 30, } @@ -39,27 +45,26 @@ def _supports_log_clearing(doctype: str) -> bool: class LogSettings(Document): def validate(self): - self.validate_supported_doctypes() - self.validate_duplicates() + self._remove_unsupported_doctypes() + self._deduplicate_entries() self.add_default_logtypes() - def validate_supported_doctypes(self): - for entry in self.logs_to_clear: + def _remove_unsupported_doctypes(self): + for entry in list(self.logs_to_clear): if _supports_log_clearing(entry.ref_doctype): continue msg = _("{} does not support automated log clearing.").format(frappe.bold(entry.ref_doctype)) if frappe.conf.developer_mode: msg += "
    " + _("Implement `clear_old_logs` method to enable auto error clearing.") - frappe.throw(msg, title=_("DocType not supported by Log Settings.")) + frappe.msgprint(msg, title=_("DocType not supported by Log Settings.")) + self.remove(entry) - def validate_duplicates(self): + def _deduplicate_entries(self): seen = set() - for entry in self.logs_to_clear: + for entry in list(self.logs_to_clear): if entry.ref_doctype in seen: - frappe.throw( - _("{} appears more than once in configured log doctypes.").format(entry.ref_doctype) - ) + self.remove(entry) seen.add(entry.ref_doctype) def add_default_logtypes(self): @@ -67,6 +72,9 @@ class LogSettings(Document): added_logtypes = set() for logtype, retention in DEFAULT_LOGTYPES_RETENTION.items(): if logtype not in existing_logtypes and _supports_log_clearing(logtype): + if not frappe.db.exists("DocType", logtype): + continue + self.append("logs_to_clear", {"ref_doctype": logtype, "days": cint(retention)}) added_logtypes.add(logtype) diff --git a/frappe/core/doctype/module_def/module_def.js b/frappe/core/doctype/module_def/module_def.js index 73d2d6562c..aa26c92c8b 100644 --- a/frappe/core/doctype/module_def/module_def.js +++ b/frappe/core/doctype/module_def/module_def.js @@ -1,13 +1,20 @@ // Copyright (c) 2016, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Module Def', { - refresh: function(frm) { - frappe.xcall('frappe.core.doctype.module_def.module_def.get_installed_apps').then(r => { - frm.set_df_property('app_name', 'options', JSON.parse(r)); +frappe.ui.form.on("Module Def", { + refresh: function (frm) { + frappe.xcall("frappe.core.doctype.module_def.module_def.get_installed_apps").then((r) => { + frm.set_df_property("app_name", "options", JSON.parse(r)); if (!frm.doc.app_name) { - frm.set_value('app_name', 'frappe'); + frm.set_value("app_name", "frappe"); } }); - } + + if (!frappe.boot.developer_mode) { + frm.set_df_property("custom", "read_only", 1); + if (frm.is_new()) { + frm.set_value("custom", 1); + } + } + }, }); diff --git a/frappe/core/doctype/module_def/test_module_def.py b/frappe/core/doctype/module_def/test_module_def.py index 914ba07949..e44741c894 100644 --- a/frappe/core/doctype/module_def/test_module_def.py +++ b/frappe/core/doctype/module_def/test_module_def.py @@ -1,9 +1,9 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import unittest +from frappe.tests.utils import FrappeTestCase # test_records = frappe.get_test_records('Module Def') -class TestModuleDef(unittest.TestCase): +class TestModuleDef(FrappeTestCase): pass diff --git a/frappe/core/doctype/module_profile/module_profile.js b/frappe/core/doctype/module_profile/module_profile.js index 3714d31ade..7860577a6c 100644 --- a/frappe/core/doctype/module_profile/module_profile.js +++ b/frappe/core/doctype/module_profile/module_profile.js @@ -19,5 +19,5 @@ frappe.ui.form.on("Module Profile", { if (frm.module_editor) { frm.module_editor.set_modules_in_table(); } - } + }, }); diff --git a/frappe/core/doctype/module_profile/module_profile.py b/frappe/core/doctype/module_profile/module_profile.py index 46c09827f3..c5c67d3c72 100644 --- a/frappe/core/doctype/module_profile/module_profile.py +++ b/frappe/core/doctype/module_profile/module_profile.py @@ -8,4 +8,4 @@ class ModuleProfile(Document): def onload(self): from frappe.config import get_modules_from_all_apps - self.set_onload("all_modules", [m.get("module_name") for m in get_modules_from_all_apps()]) + self.set_onload("all_modules", sorted(m.get("module_name") for m in get_modules_from_all_apps())) diff --git a/frappe/core/doctype/module_profile/test_module_profile.py b/frappe/core/doctype/module_profile/test_module_profile.py index 099d1371fb..a92adb0e74 100644 --- a/frappe/core/doctype/module_profile/test_module_profile.py +++ b/frappe/core/doctype/module_profile/test_module_profile.py @@ -1,11 +1,10 @@ # Copyright (c) 2020, Frappe Technologies and Contributors # License: MIT. See LICENSE -import unittest - import frappe +from frappe.tests.utils import FrappeTestCase -class TestModuleProfile(unittest.TestCase): +class TestModuleProfile(FrappeTestCase): def test_make_new_module_profile(self): if not frappe.db.get_value("Module Profile", "_Test Module Profile"): frappe.get_doc( diff --git a/frappe/core/doctype/navbar_item/navbar_item.js b/frappe/core/doctype/navbar_item/navbar_item.js index bd4274db49..b14d0a5670 100644 --- a/frappe/core/doctype/navbar_item/navbar_item.js +++ b/frappe/core/doctype/navbar_item/navbar_item.js @@ -1,8 +1,7 @@ // Copyright (c) 2020, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Navbar Item', { +frappe.ui.form.on("Navbar Item", { // refresh: function(frm) { - // } }); diff --git a/frappe/core/doctype/navbar_item/test_navbar_item.py b/frappe/core/doctype/navbar_item/test_navbar_item.py index 7eeb4f642b..7ad92a3ae8 100644 --- a/frappe/core/doctype/navbar_item/test_navbar_item.py +++ b/frappe/core/doctype/navbar_item/test_navbar_item.py @@ -1,8 +1,8 @@ # Copyright (c) 2020, Frappe Technologies and Contributors # License: MIT. See LICENSE # import frappe -import unittest +from frappe.tests.utils import FrappeTestCase -class TestNavbarItem(unittest.TestCase): +class TestNavbarItem(FrappeTestCase): pass diff --git a/frappe/core/doctype/navbar_settings/navbar_settings.js b/frappe/core/doctype/navbar_settings/navbar_settings.js index e2c157fe6a..c0e1113087 100644 --- a/frappe/core/doctype/navbar_settings/navbar_settings.js +++ b/frappe/core/doctype/navbar_settings/navbar_settings.js @@ -1,8 +1,7 @@ // Copyright (c) 2020, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Navbar Settings', { +frappe.ui.form.on("Navbar Settings", { // refresh: function(frm) { - // } }); diff --git a/frappe/core/doctype/navbar_settings/test_navbar_settings.py b/frappe/core/doctype/navbar_settings/test_navbar_settings.py index 76fb3d298a..f6c2cf69cd 100644 --- a/frappe/core/doctype/navbar_settings/test_navbar_settings.py +++ b/frappe/core/doctype/navbar_settings/test_navbar_settings.py @@ -1,8 +1,8 @@ # Copyright (c) 2020, Frappe Technologies and Contributors # License: MIT. See LICENSE # import frappe -import unittest +from frappe.tests.utils import FrappeTestCase -class TestNavbarSettings(unittest.TestCase): +class TestNavbarSettings(FrappeTestCase): pass diff --git a/frappe/core/doctype/package/package.js b/frappe/core/doctype/package/package.js index 90e2eed1e3..97d9c32c85 100644 --- a/frappe/core/doctype/package/package.js +++ b/frappe/core/doctype/package/package.js @@ -1,17 +1,20 @@ // Copyright (c) 2021, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Package', { - validate: function(frm) { +frappe.ui.form.on("Package", { + validate: function (frm) { if (!frm.doc.package_name) { - frm.set_value('package_name', frm.doc.name.toLowerCase().replace(' ', '-')); + frm.set_value("package_name", frm.doc.name.toLowerCase().replace(" ", "-")); } }, - license_type: function(frm) { - frappe.call('frappe.core.doctype.package.package.get_license_text', - {'license_type': frm.doc.license_type}).then(r => { - frm.set_value('license', r.message); - }); - } + license_type: function (frm) { + frappe + .call("frappe.core.doctype.package.package.get_license_text", { + license_type: frm.doc.license_type, + }) + .then((r) => { + frm.set_value("license", r.message); + }); + }, }); diff --git a/frappe/core/doctype/package/test_package.py b/frappe/core/doctype/package/test_package.py index 0bd52a9e3c..8af076f1be 100644 --- a/frappe/core/doctype/package/test_package.py +++ b/frappe/core/doctype/package/test_package.py @@ -3,12 +3,12 @@ import json import os -import unittest import frappe +from frappe.tests.utils import FrappeTestCase -class TestPackage(unittest.TestCase): +class TestPackage(FrappeTestCase): def test_package_release(self): make_test_package() make_test_module() diff --git a/frappe/core/doctype/package_import/package_import.js b/frappe/core/doctype/package_import/package_import.js index c01a6266cc..72f5bbf681 100644 --- a/frappe/core/doctype/package_import/package_import.js +++ b/frappe/core/doctype/package_import/package_import.js @@ -1,8 +1,7 @@ // Copyright (c) 2021, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Package Import', { +frappe.ui.form.on("Package Import", { // refresh: function(frm) { - // } }); diff --git a/frappe/core/doctype/package_import/package_import.py b/frappe/core/doctype/package_import/package_import.py index 19762eae4a..4939b357b0 100644 --- a/frappe/core/doctype/package_import/package_import.py +++ b/frappe/core/doctype/package_import/package_import.py @@ -26,7 +26,7 @@ class PackageImport(Document): attachment = attachment[0] # get package_name from file (package_name-0.0.0.tar.gz) - package_name = attachment.file_name.split(".")[0].rsplit("-", 1)[0] + package_name = attachment.file_name.split(".", 1)[0].rsplit("-", 1)[0] if not os.path.exists(frappe.get_site_path("packages")): os.makedirs(frappe.get_site_path("packages")) diff --git a/frappe/core/doctype/package_import/test_package_import.py b/frappe/core/doctype/package_import/test_package_import.py index 7e8008cc44..e4bb3d6715 100644 --- a/frappe/core/doctype/package_import/test_package_import.py +++ b/frappe/core/doctype/package_import/test_package_import.py @@ -2,8 +2,8 @@ # See license.txt # import frappe -import unittest +from frappe.tests.utils import FrappeTestCase -class TestPackageImport(unittest.TestCase): +class TestPackageImport(FrappeTestCase): pass diff --git a/frappe/core/doctype/package_release/package_release.js b/frappe/core/doctype/package_release/package_release.js index 9eabe36839..af482fc4a0 100644 --- a/frappe/core/doctype/package_release/package_release.js +++ b/frappe/core/doctype/package_release/package_release.js @@ -1,8 +1,7 @@ // Copyright (c) 2021, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Package Release', { +frappe.ui.form.on("Package Release", { // refresh: function(frm) { - // } }); diff --git a/frappe/core/doctype/package_release/package_release.py b/frappe/core/doctype/package_release/package_release.py index 58fdc2ab86..d56023ca0d 100644 --- a/frappe/core/doctype/package_release/package_release.py +++ b/frappe/core/doctype/package_release/package_release.py @@ -60,7 +60,7 @@ class PackageRelease(Document): self.make_tarfile(package) def export_modules(self): - for m in frappe.db.get_all("Module Def", dict(package=self.package)): + for m in frappe.get_all("Module Def", dict(package=self.package)): module = frappe.get_doc("Module Def", m.name) for l in module.meta.links: if l.link_doctype == "Module Def": diff --git a/frappe/core/doctype/package_release/test_package_release.py b/frappe/core/doctype/package_release/test_package_release.py index e7b680463d..8cea4d0aff 100644 --- a/frappe/core/doctype/package_release/test_package_release.py +++ b/frappe/core/doctype/package_release/test_package_release.py @@ -2,8 +2,8 @@ # See license.txt # import frappe -import unittest +from frappe.tests.utils import FrappeTestCase -class TestPackageRelease(unittest.TestCase): +class TestPackageRelease(FrappeTestCase): pass diff --git a/frappe/core/doctype/page/page.js b/frappe/core/doctype/page/page.js index d1d9600e59..295dca1aae 100644 --- a/frappe/core/doctype/page/page.js +++ b/frappe/core/doctype/page/page.js @@ -1,16 +1,16 @@ // Copyright (c) 2016, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Page', { - refresh: function(frm) { - if (!frappe.boot.developer_mode && frappe.session.user != 'Administrator') { +frappe.ui.form.on("Page", { + refresh: function (frm) { + if (!frappe.boot.developer_mode && frappe.session.user != "Administrator") { // make the document read-only frm.set_read_only(); } if (!frm.is_new() && !frm.doc.istable) { - frm.add_custom_button(__('Go to {0} Page', [frm.doc.title || frm.doc.name]), () => { + frm.add_custom_button(__("Go to {0} Page", [frm.doc.title || frm.doc.name]), () => { frappe.set_route(frm.doc.name); }); } - } + }, }); diff --git a/frappe/core/doctype/page/page.json b/frappe/core/doctype/page/page.json index 0c586643d4..e913f126af 100644 --- a/frappe/core/doctype/page/page.json +++ b/frappe/core/doctype/page/page.json @@ -1,415 +1,133 @@ { - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 1, - "autoname": "field:page_name", - "beta": 0, - "creation": "2012-12-20 17:16:49", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "System", - "editable_grid": 0, - "engine": "InnoDB", + "actions": [], + "allow_rename": 1, + "autoname": "field:page_name", + "creation": "2012-12-20 17:16:49", + "doctype": "DocType", + "document_type": "System", + "engine": "InnoDB", + "field_order": [ + "system_page", + "page_html", + "page_name", + "title", + "icon", + "column_break0", + "module", + "restrict_to_domain", + "standard", + "section_break0", + "roles" + ], "fields": [ { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "system_page", - "fieldtype": "Check", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "System Page", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "default": "0", + "fieldname": "system_page", + "fieldtype": "Check", + "label": "System Page" + }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "page_html", - "fieldtype": "Section Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Page HTML", - "length": 0, - "no_copy": 0, - "oldfieldtype": "Section Break", - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "page_html", + "fieldtype": "Section Break", + "label": "Page HTML", + "oldfieldtype": "Section Break" + }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "page_name", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Page Name", - "length": 0, - "no_copy": 0, - "oldfieldname": "page_name", - "oldfieldtype": "Data", - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "page_name", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Page Name", + "oldfieldname": "page_name", + "oldfieldtype": "Data", + "reqd": 1, + "unique": 1 + }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "title", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Title", - "length": 0, - "no_copy": 1, - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "title", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Title", + "no_copy": 1 + }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "icon", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "icon", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "icon", + "fieldtype": "Data", + "in_list_view": 1, + "label": "icon" + }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "column_break0", - "fieldtype": "Column Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "column_break0", + "fieldtype": "Column Break" + }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "module", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 1, - "label": "Module", - "length": 0, - "no_copy": 0, - "oldfieldname": "module", - "oldfieldtype": "Select", - "options": "Module Def", - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "module", + "fieldtype": "Link", + "in_standard_filter": 1, + "label": "Module", + "oldfieldname": "module", + "oldfieldtype": "Select", + "options": "Module Def", + "reqd": 1 + }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "restrict_to_domain", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Restrict To Domain", - "length": 0, - "no_copy": 0, - "options": "Domain", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "restrict_to_domain", + "fieldtype": "Link", + "label": "Restrict To Domain", + "options": "Domain" + }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "standard", - "fieldtype": "Select", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Standard", - "length": 0, - "no_copy": 0, - "oldfieldname": "standard", - "oldfieldtype": "Select", - "options": "Yes\nNo", - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 1, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "standard", + "fieldtype": "Select", + "label": "Standard", + "oldfieldname": "standard", + "oldfieldtype": "Select", + "options": "Yes\nNo", + "reqd": 1, + "search_index": 1 + }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "section_break0", - "fieldtype": "Section Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "section_break0", + "fieldtype": "Section Break" + }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "depends_on": "eval:doc.standard == 'Yes'", - "fieldname": "roles", - "fieldtype": "Table", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Roles", - "length": 0, - "no_copy": 0, - "oldfieldname": "roles", - "oldfieldtype": "Table", - "options": "Has Role", - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "depends_on": "eval:doc.standard == 'Yes'", + "fieldname": "roles", + "fieldtype": "Table", + "label": "Roles", + "oldfieldname": "roles", + "oldfieldtype": "Table", + "options": "Has Role" } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "icon": "fa fa-file", - "idx": 1, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2017-11-13 16:37:04.422547", - "modified_by": "Administrator", - "module": "Core", - "name": "Page", - "owner": "Administrator", + ], + "icon": "fa fa-file", + "idx": 1, + "links": [], + "modified": "2022-08-03 12:20:54.219236", + "modified_by": "Administrator", + "module": "Core", + "name": "Page", + "owner": "Administrator", "permissions": [ { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 1, - "delete": 0, - "email": 1, - "export": 0, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 0, - "role": "Administrator", - "set_user_permissions": 0, - "share": 1, - "submit": 0, + "create": 1, + "email": 1, + "print": 1, + "read": 1, + "role": "Administrator", + "share": 1, "write": 1 - }, + }, { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 0, - "delete": 0, - "email": 1, - "export": 0, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 0, - "role": "System Manager", - "set_user_permissions": 0, - "share": 1, - "submit": 0, + "email": 1, + "print": 1, + "read": 1, + "role": "System Manager", + "share": 1, "write": 1 } - ], - "quick_entry": 0, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_order": "ASC", - "track_changes": 1, - "track_seen": 0 + ], + "sort_field": "modified", + "sort_order": "ASC", + "states": [], + "track_changes": 1 } \ No newline at end of file diff --git a/frappe/core/doctype/page/test_page.py b/frappe/core/doctype/page/test_page.py index 1eff54cad7..edf7f7c9b8 100644 --- a/frappe/core/doctype/page/test_page.py +++ b/frappe/core/doctype/page/test_page.py @@ -1,13 +1,12 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import unittest - import frappe +from frappe.tests.utils import FrappeTestCase test_records = frappe.get_test_records("Page") -class TestPage(unittest.TestCase): +class TestPage(FrappeTestCase): def test_naming(self): self.assertRaises( frappe.NameError, diff --git a/frappe/core/doctype/patch_log/patch_log.js b/frappe/core/doctype/patch_log/patch_log.js index b52876ac97..171a1d3a0f 100644 --- a/frappe/core/doctype/patch_log/patch_log.js +++ b/frappe/core/doctype/patch_log/patch_log.js @@ -1,8 +1,8 @@ // Copyright (c) 2016, Frappe Technologies Pvt. Ltd. and contributors // For license information, please see license.txt -frappe.ui.form.on('Patch Log', { - refresh: function(frm) { +frappe.ui.form.on("Patch Log", { + refresh: function (frm) { frm.disable_save(); - } + }, }); diff --git a/frappe/core/doctype/patch_log/patch_log.json b/frappe/core/doctype/patch_log/patch_log.json index 9750c51279..53e85b99d3 100644 --- a/frappe/core/doctype/patch_log/patch_log.json +++ b/frappe/core/doctype/patch_log/patch_log.json @@ -1,13 +1,15 @@ { "actions": [], - "autoname": "PATCHLOG.#####", + "autoname": "hash", "creation": "2013-01-17 11:36:45", "description": "List of patches executed", "doctype": "DocType", "document_type": "System", "engine": "InnoDB", "field_order": [ - "patch" + "patch", + "skipped", + "traceback" ], "fields": [ { @@ -15,16 +17,30 @@ "fieldtype": "Code", "label": "Patch", "read_only": 1 + }, + { + "default": "0", + "fieldname": "skipped", + "fieldtype": "Check", + "label": "Skipped", + "read_only": 1 + }, + { + "depends_on": "eval:doc.skipped == 1", + "fieldname": "traceback", + "fieldtype": "Code", + "label": "Traceback", + "read_only": 1 } ], "icon": "fa fa-cog", "idx": 1, "links": [], - "modified": "2022-06-13 05:34:37.845368", + "modified": "2023-05-10 19:27:10.883330", "modified_by": "Administrator", "module": "Core", "name": "Patch Log", - "naming_rule": "Expression (old style)", + "naming_rule": "Random", "owner": "Administrator", "permissions": [ { @@ -33,6 +49,14 @@ "read": 1, "report": 1, "role": "Administrator" + }, + { + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager" } ], "quick_entry": 1, diff --git a/frappe/core/doctype/patch_log/patch_log.py b/frappe/core/doctype/patch_log/patch_log.py index eee57af4c2..c7d619017e 100644 --- a/frappe/core/doctype/patch_log/patch_log.py +++ b/frappe/core/doctype/patch_log/patch_log.py @@ -3,8 +3,13 @@ # License: MIT. See LICENSE +import frappe from frappe.model.document import Document class PatchLog(Document): pass + + +def before_migrate(): + frappe.reload_doc("core", "doctype", "patch_log") diff --git a/frappe/core/doctype/patch_log/test_patch_log.py b/frappe/core/doctype/patch_log/test_patch_log.py index 0c8a2ae4d4..849c3b30a2 100644 --- a/frappe/core/doctype/patch_log/test_patch_log.py +++ b/frappe/core/doctype/patch_log/test_patch_log.py @@ -1,9 +1,9 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import unittest +from frappe.tests.utils import FrappeTestCase # test_records = frappe.get_test_records('Patch Log') -class TestPatchLog(unittest.TestCase): +class TestPatchLog(FrappeTestCase): pass diff --git a/frappe/core/doctype/payment_gateway/payment_gateway.js b/frappe/core/doctype/payment_gateway/payment_gateway.js deleted file mode 100644 index 0eff5a5608..0000000000 --- a/frappe/core/doctype/payment_gateway/payment_gateway.js +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright (c) 2016, Frappe Technologies Pvt. Ltd. and contributors -// For license information, please see license.txt - -frappe.ui.form.on('Payment Gateway', { - refresh: function(frm) { - - } -}); diff --git a/frappe/core/doctype/payment_gateway/payment_gateway.json b/frappe/core/doctype/payment_gateway/payment_gateway.json deleted file mode 100644 index 7195b3949e..0000000000 --- a/frappe/core/doctype/payment_gateway/payment_gateway.json +++ /dev/null @@ -1,55 +0,0 @@ -{ - "actions": [], - "autoname": "field:gateway", - "creation": "2022-01-24 21:09:47.229371", - "doctype": "DocType", - "editable_grid": 1, - "engine": "InnoDB", - "field_order": [ - "gateway", - "gateway_settings", - "gateway_controller" - ], - "fields": [ - { - "fieldname": "gateway", - "fieldtype": "Data", - "in_list_view": 1, - "label": "Gateway", - "reqd": 1, - "unique": 1 - }, - { - "fieldname": "gateway_settings", - "fieldtype": "Link", - "label": "Gateway Settings", - "options": "DocType" - }, - { - "fieldname": "gateway_controller", - "fieldtype": "Dynamic Link", - "label": "Gateway Controller", - "options": "gateway_settings" - } - ], - "links": [], - "modified": "2022-01-24 21:17:03.864719", - "modified_by": "Administrator", - "module": "Core", - "name": "Payment Gateway", - "naming_rule": "By fieldname", - "owner": "Administrator", - "permissions": [ - { - "create": 1, - "delete": 1, - "read": 1, - "role": "System Manager", - "write": 1 - } - ], - "quick_entry": 1, - "sort_field": "modified", - "sort_order": "DESC", - "states": [] -} \ No newline at end of file diff --git a/frappe/core/doctype/payment_gateway/payment_gateway.py b/frappe/core/doctype/payment_gateway/payment_gateway.py deleted file mode 100644 index 74306ae4ad..0000000000 --- a/frappe/core/doctype/payment_gateway/payment_gateway.py +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors -# License: MIT. See LICENSE - -from frappe.model.document import Document - - -class PaymentGateway(Document): - pass diff --git a/frappe/core/doctype/payment_gateway/test_payment_gateway.py b/frappe/core/doctype/payment_gateway/test_payment_gateway.py deleted file mode 100644 index 6900e79434..0000000000 --- a/frappe/core/doctype/payment_gateway/test_payment_gateway.py +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors -# License: MIT. See LICENSE -import unittest - -# test_records = frappe.get_test_records('Payment Gateway') - - -class TestPaymentGateway(unittest.TestCase): - pass diff --git a/frappe/core/doctype/prepared_report/prepared_report.js b/frappe/core/doctype/prepared_report/prepared_report.js index 6a7cf2728c..abdf418165 100644 --- a/frappe/core/doctype/prepared_report/prepared_report.js +++ b/frappe/core/doctype/prepared_report/prepared_report.js @@ -1,45 +1,55 @@ // Copyright (c) 2018, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Prepared Report', { - onload: function(frm) { +frappe.ui.form.on("Prepared Report", { + render_filter_values: function (frm, filters) { var wrapper = $(frm.fields_dict["filter_values"].wrapper).empty(); let filter_table = $(` - - + +
    ${ __("Filter") }${ __("Value") }${__("Filter")}${__("Value")}
    `); - const filters = JSON.parse(frm.doc.filters); - - Object.keys(filters).forEach(key => { + Object.keys(filters).forEach((key) => { const filter_row = $(` ${frappe.model.unscrub(key)} ${filters[key]} `); - filter_table.find('tbody').append(filter_row); + filter_table.find("tbody").append(filter_row); }); wrapper.append(filter_table); }, - refresh: function(frm) { + refresh: function (frm) { frm.disable_save(); - if (frm.doc.status == 'Completed') { + + const filters = JSON.parse(frm.doc.filters); + if (!$.isEmptyObject(filters)) { + frm.toggle_display(["filter_values"], 1); + frm.events.render_filter_values(frm, filters); + } + + // always keep report_name hidden - we do this as we can't set mandatory and hidden + // property on a docfield at the same time + frm.toggle_display(["report_name"], 0); + + if (frm.doc.status == "Completed") { frm.page.set_primary_action(__("Show Report"), () => { + frappe.route_options = filters; frappe.set_route( "query-report", frm.doc.report_name, frappe.utils.make_query_string({ - prepared_report_name: frm.doc.name + prepared_report_name: frm.doc.name, }) ); }); } - } + }, }); diff --git a/frappe/core/doctype/prepared_report/prepared_report.json b/frappe/core/doctype/prepared_report/prepared_report.json index cafe323519..fb3809e481 100644 --- a/frappe/core/doctype/prepared_report/prepared_report.json +++ b/frappe/core/doctype/prepared_report/prepared_report.json @@ -1,38 +1,31 @@ { "actions": [], - "autoname": "REP.#####", + "autoname": "hash", "creation": "2018-06-25 18:39:11.152960", "doctype": "DocType", "editable_grid": 1, "engine": "InnoDB", "field_order": [ - "report_name", - "ref_report_doctype", "status", + "report_name", + "queued_by", + "job_id", "column_break_4", - "report_start_time", + "queued_at", "report_end_time", "section_break_7", "error_message", "filters_sb", "filters", - "filter_values", - "columns" + "filter_values" ], "fields": [ { "fieldname": "report_name", "fieldtype": "Data", "label": "Report Name", - "read_only": 1 - }, - { - "fieldname": "ref_report_doctype", - "fieldtype": "Link", - "in_standard_filter": 1, - "label": "Report Type", - "options": "Report", - "read_only": 1 + "read_only": 1, + "reqd": 1 }, { "default": "Queued", @@ -49,16 +42,10 @@ "fieldname": "column_break_4", "fieldtype": "Column Break" }, - { - "fieldname": "report_start_time", - "fieldtype": "Datetime", - "label": "Report Start Time", - "read_only": 1 - }, { "fieldname": "report_end_time", "fieldtype": "Datetime", - "label": "Report End Time", + "label": "Finished At", "read_only": 1 }, { @@ -89,25 +76,39 @@ { "fieldname": "filter_values", "fieldtype": "HTML", + "hidden": 1, "label": "Filter Values" }, { - "fieldname": "columns", - "fieldtype": "Code", - "hidden": 1, - "label": "Columns", + "fieldname": "job_id", + "fieldtype": "Link", + "label": "Job ID", "no_copy": 1, - "print_hide": 1, + "options": "RQ Job", + "read_only": 1 + }, + { + "fieldname": "queued_by", + "fieldtype": "Data", + "is_virtual": 1, + "label": "Queued By", + "read_only": 1 + }, + { + "fieldname": "queued_at", + "fieldtype": "Datetime", + "is_virtual": 1, + "label": "Queued At", "read_only": 1 } ], "in_create": 1, "links": [], - "modified": "2022-06-13 06:20:34.496412", + "modified": "2023-05-19 15:41:03.428589", "modified_by": "Administrator", "module": "Core", "name": "Prepared Report", - "naming_rule": "Expression (old style)", + "naming_rule": "Random", "owner": "Administrator", "permissions": [ { @@ -134,7 +135,19 @@ ], "sort_field": "modified", "sort_order": "DESC", - "states": [], - "title_field": "ref_report_doctype", - "track_changes": 1 + "states": [ + { + "color": "Blue", + "title": "Queued" + }, + { + "color": "Red", + "title": "Error" + }, + { + "color": "Green", + "title": "Completed" + } + ], + "title_field": "report_name" } \ No newline at end of file diff --git a/frappe/core/doctype/prepared_report/prepared_report.py b/frappe/core/doctype/prepared_report/prepared_report.py index 0ff4ce3070..ed7f4711aa 100644 --- a/frappe/core/doctype/prepared_report/prepared_report.py +++ b/frappe/core/doctype/prepared_report/prepared_report.py @@ -3,27 +3,67 @@ import json +from typing import Any + +from rq import get_current_job import frappe from frappe.desk.form.load import get_attachments from frappe.desk.query_report import generate_report_result from frappe.model.document import Document +from frappe.monitor import add_data_to_monitor from frappe.utils import gzip_compress, gzip_decompress from frappe.utils.background_jobs import enqueue class PreparedReport(Document): + @property + def queued_by(self): + return self.owner + + @property + def queued_at(self): + return self.creation + + @staticmethod + def clear_old_logs(days=30): + prepared_reports_to_delete = frappe.get_all( + "Prepared Report", + filters={"modified": ["<", frappe.utils.add_days(frappe.utils.now(), -days)]}, + ) + + for batch in frappe.utils.create_batch(prepared_reports_to_delete, 100): + enqueue(method=delete_prepared_reports, reports=batch) + def before_insert(self): self.status = "Queued" - self.report_start_time = frappe.utils.now() - def enqueue_report(self): - enqueue(run_background, prepared_report=self.name, timeout=6000) + def after_insert(self): + enqueue( + generate_report, + queue="long", + prepared_report=self.name, + timeout=1500, + enqueue_after_commit=True, + ) + + def get_prepared_data(self, with_file_name=False): + if attachments := get_attachments(self.doctype, self.name): + attachment = attachments[0] + attached_file = frappe.get_doc("File", attachment.name) + + if with_file_name: + return (gzip_decompress(attached_file.get_content()), attachment.file_name) + return gzip_decompress(attached_file.get_content()) -def run_background(prepared_report): +def generate_report(prepared_report): + update_job_id(prepared_report, get_current_job().id) + instance = frappe.get_doc("Prepared Report", prepared_report) - report = frappe.get_doc("Report", instance.ref_report_doctype) + report = frappe.get_doc("Report", instance.report_name) + + add_data_to_monitor(report=instance.report_name) try: report.custom_columns = [] @@ -38,19 +78,15 @@ def run_background(prepared_report): report.custom_columns = data["columns"] result = generate_report_result(report=report, filters=instance.filters, user=instance.owner) - create_json_gz_file(result["result"], "Prepared Report", instance.name) + create_json_gz_file(result, instance.doctype, instance.name) instance.status = "Completed" - instance.columns = json.dumps(result["columns"]) - instance.report_end_time = frappe.utils.now() - instance.save(ignore_permissions=True) - except Exception: - report.log_error("Prepared report failed") - instance = frappe.get_doc("Prepared Report", prepared_report) instance.status = "Error" instance.error_message = frappe.get_traceback() - instance.save(ignore_permissions=True) + + instance.report_end_time = frappe.utils.now() + instance.save(ignore_permissions=True) frappe.publish_realtime( "report_generated", @@ -59,35 +95,60 @@ def run_background(prepared_report): ) +def update_job_id(prepared_report, job_id): + frappe.db.set_value("Prepared Report", prepared_report, "job_id", job_id, update_modified=False) + frappe.db.commit() + + +@frappe.whitelist() +def make_prepared_report(report_name, filters=None): + """run reports in background""" + prepared_report = frappe.get_doc( + { + "doctype": "Prepared Report", + "report_name": report_name, + "filters": process_filters_for_prepared_report(filters), + } + ).insert(ignore_permissions=True) + + return {"name": prepared_report.name} + + +def process_filters_for_prepared_report(filters: dict[str, Any] | str) -> str: + if isinstance(filters, str): + filters = json.loads(filters) + + # This looks like an insanity but, without this it'd be very hard to find Prepared Reports matching given condition + # We're ensuring that spacing is consistent. e.g. JS seems to put no spaces after ":", Python on the other hand does. + # We are also ensuring that order of keys is same so generated JSON string will be identical too. + # PS: frappe.as_json sorts keys + return frappe.as_json(filters, indent=None, separators=(",", ":")) + + @frappe.whitelist() def get_reports_in_queued_state(report_name, filters): reports = frappe.get_all( "Prepared Report", filters={ "report_name": report_name, - "filters": json.dumps(json.loads(filters)), + "filters": process_filters_for_prepared_report(filters), "status": "Queued", + "owner": frappe.session.user, }, ) return reports -def delete_expired_prepared_reports(): - system_settings = frappe.get_single("System Settings") - enable_auto_deletion = system_settings.enable_prepared_report_auto_deletion - if enable_auto_deletion: - expiry_period = system_settings.prepared_report_expiry_period - prepared_reports_to_delete = frappe.get_all( - "Prepared Report", - filters={"creation": ["<", frappe.utils.add_days(frappe.utils.now(), -expiry_period)]}, - ) - - batches = frappe.utils.create_batch(prepared_reports_to_delete, 100) - for batch in batches: - args = { - "reports": batch, - } - enqueue(method=delete_prepared_reports, job_name="delete_prepared_reports", **args) +def get_completed_prepared_report(filters, user, report_name): + return frappe.db.get_value( + "Prepared Report", + filters={ + "status": "Completed", + "filters": process_filters_for_prepared_report(filters), + "owner": user, + "report_name": report_name, + }, + ) @frappe.whitelist() @@ -124,10 +185,13 @@ def create_json_gz_file(data, dt, dn): @frappe.whitelist() def download_attachment(dn): - attachment = get_attachments("Prepared Report", dn)[0] - frappe.local.response.filename = attachment.file_name[:-2] - attached_file = frappe.get_doc("File", attachment.name) - frappe.local.response.filecontent = gzip_decompress(attached_file.get_content()) + pr = frappe.get_doc("Prepared Report", dn) + if not pr.has_permission("read"): + frappe.throw(frappe._("Cannot Download Report due to insufficient permissions")) + + data, file_name = pr.get_prepared_data(with_file_name=True) + frappe.local.response.filename = file_name[:-3] + frappe.local.response.filecontent = data frappe.local.response.type = "binary" @@ -146,9 +210,7 @@ def get_permission_query_condition(user): reports = [frappe.db.escape(report) for report in user.get_all_reports().keys()] - return """`tabPrepared Report`.ref_report_doctype in ({reports})""".format( - reports=",".join(reports) - ) + return """`tabPrepared Report`.report_name in ({reports})""".format(reports=",".join(reports)) def has_permission(doc, user): @@ -164,4 +226,4 @@ def has_permission(doc, user): if "System Manager" in user.roles: return True - return doc.ref_report_doctype in user.get_all_reports().keys() + return doc.report_name in user.get_all_reports().keys() diff --git a/frappe/core/doctype/prepared_report/prepared_report_list.js b/frappe/core/doctype/prepared_report/prepared_report_list.js index 8acb3bc75a..d0565fe826 100644 --- a/frappe/core/doctype/prepared_report/prepared_report_list.js +++ b/frappe/core/doctype/prepared_report/prepared_report_list.js @@ -1,12 +1,7 @@ -frappe.listview_settings['Prepared Report'] = { - add_fields: ["status"], - get_indicator: function(doc) { - if(doc.status==="Completed"){ - return [__("Completed"), "green", "status,=,Completed"]; - } else if(doc.status ==="Error"){ - return [__("Error"), "red", "status,=,Error"]; - } else if(doc.status ==="Queued"){ - return [__("Queued"), "orange", "status,=,Queued"]; - } - } -}; \ No newline at end of file +frappe.listview_settings["Prepared Report"] = { + onload: function (list_view) { + frappe.require("logtypes.bundle.js", () => { + frappe.utils.logtypes.show_log_retention_message(list_view.doctype); + }); + }, +}; diff --git a/frappe/core/doctype/prepared_report/test_prepared_report.py b/frappe/core/doctype/prepared_report/test_prepared_report.py index 6d0c809a01..a864ea73f8 100644 --- a/frappe/core/doctype/prepared_report/test_prepared_report.py +++ b/frappe/core/doctype/prepared_report/test_prepared_report.py @@ -1,28 +1,53 @@ # Copyright (c) 2018, Frappe Technologies and Contributors # License: MIT. See LICENSE import json -import unittest +import time import frappe +from frappe.desk.query_report import generate_report_result, get_report_doc +from frappe.tests.utils import FrappeTestCase -class TestPreparedReport(unittest.TestCase): - def setUp(self): - self.report = frappe.get_doc({"doctype": "Report", "name": "Permitted Documents For User"}) - self.filters = {"user": "Administrator", "doctype": "Role"} - self.prepared_report_doc = frappe.get_doc( +class TestPreparedReport(FrappeTestCase): + @classmethod + def tearDownClass(cls): + for r in frappe.get_all("Prepared Report", pluck="name"): + frappe.delete_doc("Prepared Report", r, force=True, delete_permanently=True) + + frappe.db.commit() + + def create_prepared_report(self, commit=False): + doc = frappe.get_doc( { "doctype": "Prepared Report", - "report_name": self.report.name, - "filters": json.dumps(self.filters), - "ref_report_doctype": self.report.name, + "report_name": "Database Storage Usage By Tables", } ).insert() - def tearDown(self): - frappe.set_user("Administrator") - self.prepared_report_doc.delete() + if commit: + frappe.db.commit() - def test_for_creation(self): - self.assertTrue("QUEUED" == self.prepared_report_doc.status.upper()) - self.assertTrue(self.prepared_report_doc.report_start_time) + return doc + + def test_queueing(self): + doc_ = self.create_prepared_report() + self.assertEqual("Queued", doc_.status) + self.assertTrue(doc_.queued_at) + + frappe.db.commit() + time.sleep(5) + + doc_ = frappe.get_last_doc("Prepared Report") + self.assertEqual("Completed", doc_.status) + self.assertTrue(doc_.job_id) + self.assertTrue(doc_.report_end_time) + + def test_prepared_data(self): + doc_ = self.create_prepared_report(commit=True) + time.sleep(5) + + prepared_data = json.loads(doc_.get_prepared_data().decode("utf-8")) + generated_data = generate_report_result(get_report_doc("Database Storage Usage By Tables")) + self.assertEqual(len(prepared_data["columns"]), len(generated_data["columns"])) + self.assertEqual(len(prepared_data["result"]), len(generated_data["result"])) + self.assertEqual(len(prepared_data), len(generated_data)) diff --git a/frappe/core/doctype/report/report.js b/frappe/core/doctype/report/report.js index 71ed0dac64..fdbda8de9c 100644 --- a/frappe/core/doctype/report/report.js +++ b/frappe/core/doctype/report/report.js @@ -1,5 +1,5 @@ -frappe.ui.form.on('Report', { - refresh: function(frm) { +frappe.ui.form.on("Report", { + refresh: function (frm) { if (frm.doc.is_standard === "Yes" && !frappe.boot.developer_mode) { // make the document read-only frm.disable_form(); @@ -8,43 +8,61 @@ frappe.ui.form.on('Report', { } let doc = frm.doc; - frm.add_custom_button(__("Show Report"), function() { - switch(doc.report_type) { - case "Report Builder": - frappe.set_route('List', doc.ref_doctype, 'Report', doc.name); - break; - case "Query Report": - frappe.set_route("query-report", doc.name); - break; - case "Script Report": - frappe.set_route("query-report", doc.name); - break; - case "Custom Report": - frappe.set_route("query-report", doc.name); - break; - } - }, "fa fa-table"); + if (!doc.__islocal) { + frm.add_custom_button( + __("Show Report"), + function () { + switch (doc.report_type) { + case "Report Builder": + frappe.set_route("List", doc.ref_doctype, "Report", doc.name); + break; + case "Query Report": + frappe.set_route("query-report", doc.name); + break; + case "Script Report": + frappe.set_route("query-report", doc.name); + break; + case "Custom Report": + frappe.set_route("query-report", doc.name); + break; + } + }, + "fa fa-table" + ); + } if (doc.is_standard === "Yes" && frm.perm[0].write) { - frm.add_custom_button(doc.disabled ? __("Enable Report") : __("Disable Report"), function() { - frm.call('toggle_disable', { - disable: doc.disabled ? 0 : 1 - }).then(() => { - frm.reload_doc(); - }); - }, doc.disabled ? "fa fa-check" : "fa fa-off"); + frm.add_custom_button( + doc.disabled ? __("Enable Report") : __("Disable Report"), + function () { + frm.call("toggle_disable", { + disable: doc.disabled ? 0 : 1, + }).then(() => { + frm.reload_doc(); + }); + }, + doc.disabled ? "fa fa-check" : "fa fa-off" + ); } + + frm.set_query("ref_doctype", () => { + return { + filters: { + istable: 0, + }, + }; + }); }, - ref_doctype: function(frm) { - if(frm.doc.ref_doctype) { + ref_doctype: function (frm) { + if (frm.doc.ref_doctype) { frm.trigger("set_doctype_roles"); } }, - set_doctype_roles: function(frm) { - return frm.call('set_doctype_roles').then(() => { - frm.refresh_field('roles'); + set_doctype_roles: function (frm) { + return frm.call("set_doctype_roles").then(() => { + frm.refresh_field("roles"); }); - } -}) + }, +}); diff --git a/frappe/core/doctype/report/report.json b/frappe/core/doctype/report/report.json index 5b3593e658..9b6b04afcc 100644 --- a/frappe/core/doctype/report/report.json +++ b/frappe/core/doctype/report/report.json @@ -16,7 +16,6 @@ "letter_head", "add_total_row", "disabled", - "disable_prepared_report", "prepared_report", "filters_section", "filters", @@ -128,25 +127,16 @@ "fieldtype": "Section Break" }, { - "depends_on": "eval:doc.is_standard == 'Yes'", "fieldname": "roles", "fieldtype": "Table", "label": "Roles", "options": "Has Role" }, - { - "default": "0", - "fieldname": "disable_prepared_report", - "fieldtype": "Check", - "label": "Disable Prepared Report" - }, { "default": "0", "fieldname": "prepared_report", "fieldtype": "Check", - "hidden": 1, - "label": "Prepared Report", - "read_only": 1 + "label": "Prepared Report" }, { "depends_on": "eval:(doc.report_type===\"Script Report\" \n|| doc.report_type==\"Query Report\") \n&& doc.is_standard===\"No\"", @@ -158,11 +148,13 @@ { "collapsible": 1, "collapsible_depends_on": "filters", + "depends_on": "eval:doc.report_type != \"Custom Report\"", "fieldname": "filters_section", "fieldtype": "Section Break", "label": "Filters" }, { + "depends_on": "eval:doc.report_type != \"Custom Report\"", "fieldname": "filters", "fieldtype": "Table", "label": "Filters", @@ -171,11 +163,13 @@ { "collapsible": 1, "collapsible_depends_on": "columns", + "depends_on": "eval:doc.report_type != \"Custom Report\"", "fieldname": "columns_section", "fieldtype": "Section Break", "label": "Columns" }, { + "depends_on": "eval:doc.report_type != \"Custom Report\"", "fieldname": "columns", "fieldtype": "Table", "label": "Columns", @@ -192,10 +186,11 @@ "idx": 1, "index_web_pages_for_search": 1, "links": [], - "modified": "2020-08-17 16:49:28.474274", + "modified": "2023-04-07 18:18:11.782178", "modified_by": "Administrator", "module": "Core", "name": "Report", + "naming_rule": "By fieldname", "owner": "Administrator", "permissions": [ { @@ -242,5 +237,6 @@ "show_name_in_global_search": 1, "sort_field": "modified", "sort_order": "DESC", + "states": [], "track_changes": 1 } \ No newline at end of file diff --git a/frappe/core/doctype/report/report.py b/frappe/core/doctype/report/report.py index 7fe3cadf9c..9b2a2ccc18 100644 --- a/frappe/core/doctype/report/report.py +++ b/frappe/core/doctype/report/report.py @@ -57,17 +57,6 @@ class Report(Document): ): frappe.throw(_("You are not allowed to delete Standard Report")) delete_custom_role("report", self.name) - self.delete_prepared_reports() - - def delete_prepared_reports(self): - prepared_reports = frappe.get_all( - "Prepared Report", filters={"ref_report_doctype": self.name}, pluck="name" - ) - - for report in prepared_reports: - frappe.delete_doc( - "Prepared Report", report, ignore_missing=True, force=True, delete_permanently=True - ) def get_columns(self): return [d.as_dict(no_default_fields=True, no_child_table_fields=True) for d in self.columns] @@ -168,10 +157,18 @@ class Report(Document): return self.get_columns(), loc["result"] def get_data( - self, filters=None, limit=None, user=None, as_dict=False, ignore_prepared_report=False + self, + filters=None, + limit=None, + user=None, + as_dict=False, + ignore_prepared_report=False, + are_default_filters=True, ): if self.report_type in ("Query Report", "Script Report", "Custom Report"): - columns, result = self.run_query_report(filters, user, ignore_prepared_report) + columns, result = self.run_query_report( + filters, user, ignore_prepared_report, are_default_filters + ) else: columns, result = self.run_standard_report(filters, limit, user) @@ -180,10 +177,16 @@ class Report(Document): return columns, result - def run_query_report(self, filters, user, ignore_prepared_report=False): + def run_query_report( + self, filters=None, user=None, ignore_prepared_report=False, are_default_filters=True + ): columns, result = [], [] data = frappe.desk.query_report.run( - self.name, filters=filters, user=user, ignore_prepared_report=ignore_prepared_report + self.name, + filters=filters, + user=user, + ignore_prepared_report=ignore_prepared_report, + are_default_filters=are_default_filters, ) for d in data.get("columns"): @@ -337,16 +340,15 @@ class Report(Document): return data @frappe.whitelist() - def toggle_disable(self, disable): + def toggle_disable(self, disable: bool): if not self.has_permission("write"): frappe.throw(_("You are not allowed to edit the report.")) self.db_set("disabled", cint(disable)) -@frappe.whitelist() -def is_prepared_report_disabled(report): - return frappe.db.get_value("Report", report, "disable_prepared_report") or 0 +def is_prepared_report_enabled(report): + return cint(frappe.db.get_value("Report", report, "prepared_report")) or 0 def get_report_module_dotted_path(module, report_name): diff --git a/frappe/core/doctype/report/test_report.py b/frappe/core/doctype/report/test_report.py index 0e1ed80eda..670b6b7410 100644 --- a/frappe/core/doctype/report/test_report.py +++ b/frappe/core/doctype/report/test_report.py @@ -118,11 +118,10 @@ class TestReport(FrappeTestCase): } ] ), + json.dumps({"user": "Administrator", "doctype": "User"}), ) custom_report = frappe.get_doc("Report", custom_report_name) - columns, result = custom_report.run_query_report( - filters={"user": "Administrator", "doctype": "User"}, user=frappe.session.user - ) + columns, result = custom_report.run_query_report(user=frappe.session.user) self.assertListEqual(["email"], [column.get("fieldname") for column in columns]) admin_dict = frappe.core.utils.find(result, lambda d: d["name"] == "Administrator") diff --git a/frappe/core/doctype/role/patches/v13_set_default_desk_properties.py b/frappe/core/doctype/role/patches/v13_set_default_desk_properties.py index 87de6ac79a..22854fa5f8 100644 --- a/frappe/core/doctype/role/patches/v13_set_default_desk_properties.py +++ b/frappe/core/doctype/role/patches/v13_set_default_desk_properties.py @@ -4,8 +4,6 @@ from ..role import desk_properties def execute(): - frappe.reload_doctype("user") - frappe.reload_doctype("role") for role in frappe.get_all("Role", ["name", "desk_access"]): role_doc = frappe.get_doc("Role", role.name) for key in desk_properties: diff --git a/frappe/core/doctype/role/role.js b/frappe/core/doctype/role/role.js index 595e857d02..c0a65bcf58 100644 --- a/frappe/core/doctype/role/role.js +++ b/frappe/core/doctype/role/role.js @@ -1,8 +1,8 @@ // Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors // MIT License. See LICENSE -frappe.ui.form.on('Role', { - refresh: function(frm) { +frappe.ui.form.on("Role", { + refresh: function (frm) { if (frm.doc.name === "All") { frm.dashboard.add_comment( __("Role 'All' will be given to all System Users."), @@ -10,15 +10,15 @@ frappe.ui.form.on('Role', { ); } - frm.set_df_property('is_custom', 'read_only', frappe.session.user !== 'Administrator'); + frm.set_df_property("is_custom", "read_only", frappe.session.user !== "Administrator"); - frm.add_custom_button("Role Permissions Manager", function() { - frappe.route_options = {"role": frm.doc.name}; + frm.add_custom_button("Role Permissions Manager", function () { + frappe.route_options = { role: frm.doc.name }; frappe.set_route("permission-manager"); }); - frm.add_custom_button("Show Users", function() { - frappe.route_options = {"role": frm.doc.name}; + frm.add_custom_button("Show Users", function () { + frappe.route_options = { role: frm.doc.name }; frappe.set_route("List", "User", "Report"); }); - } + }, }); diff --git a/frappe/core/doctype/role/role.json b/frappe/core/doctype/role/role.json index e370082fb5..2039d3889d 100644 --- a/frappe/core/doctype/role/role.json +++ b/frappe/core/doctype/role/role.json @@ -148,7 +148,7 @@ "idx": 1, "index_web_pages_for_search": 1, "links": [], - "modified": "2022-01-12 20:18:18.496230", + "modified": "2022-08-05 18:33:27.694065", "modified_by": "Administrator", "module": "Core", "name": "Role", @@ -171,5 +171,6 @@ "sort_field": "modified", "sort_order": "ASC", "states": [], - "track_changes": 1 + "track_changes": 1, + "translated_doctype": 1 } \ No newline at end of file diff --git a/frappe/core/doctype/role/role.py b/frappe/core/doctype/role/role.py index 97a0e9b581..31b82501cb 100644 --- a/frappe/core/doctype/role/role.py +++ b/frappe/core/doctype/role/role.py @@ -64,13 +64,14 @@ class Role(Document): user.save() -def get_info_based_on_role(role, field="email"): +def get_info_based_on_role(role, field="email", ignore_permissions=False): """Get information of all users that have been assigned this role""" users = frappe.get_list( "Has Role", filters={"role": role, "parenttype": "User"}, parent_doctype="User", fields=["parent as user_name"], + ignore_permissions=ignore_permissions, ) return get_user_info(users, field) diff --git a/frappe/core/doctype/role/test_role.py b/frappe/core/doctype/role/test_role.py index 44b9b1cdee..58aadfcbca 100644 --- a/frappe/core/doctype/role/test_role.py +++ b/frappe/core/doctype/role/test_role.py @@ -1,14 +1,14 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import unittest import frappe from frappe.core.doctype.role.role import get_info_based_on_role +from frappe.tests.utils import FrappeTestCase test_records = frappe.get_test_records("Role") -class TestUser(unittest.TestCase): +class TestUser(FrappeTestCase): def test_disable_role(self): frappe.get_doc("User", "test@example.com").add_roles("_Test Role 3") diff --git a/frappe/core/doctype/role_permission_for_page_and_report/role_permission_for_page_and_report.js b/frappe/core/doctype/role_permission_for_page_and_report/role_permission_for_page_and_report.js index 5048d24077..86d09bef27 100644 --- a/frappe/core/doctype/role_permission_for_page_and_report/role_permission_for_page_and_report.js +++ b/frappe/core/doctype/role_permission_for_page_and_report/role_permission_for_page_and_report.js @@ -1,22 +1,22 @@ // Copyright (c) 2016, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Role Permission for Page and Report', { - setup: function(frm) { +frappe.ui.form.on("Role Permission for Page and Report", { + setup: function (frm) { frm.trigger("set_queries"); }, - refresh: function(frm) { + refresh: function (frm) { frm.disable_save(); frm.role_area.hide(); frm.events.setup_buttons(frm); }, - setup_buttons: function(frm) { + setup_buttons: function (frm) { frm.clear_custom_buttons(); frm.page.clear_actions(); if (frm.doc.set_role_for && frm.doc[frappe.model.scrub(frm.doc.set_role_for)]) { - frm.add_custom_button(__("Reset to defaults"), function() { + frm.add_custom_button(__("Reset to defaults"), function () { frm.trigger("reset_roles"); }); @@ -26,34 +26,34 @@ frappe.ui.form.on('Role Permission for Page and Report', { } }, - onload: function(frm) { + onload: function (frm) { if (!frm.roles_editor) { frm.role_area = $(frm.fields_dict.roles_html.wrapper); frm.roles_editor = new frappe.RoleEditor(frm.role_area, frm); } }, - set_queries: function(frm) { - frm.set_query("page", function() { + set_queries: function (frm) { + frm.set_query("page", function () { return { filters: { - system_page: 0 - } - } + system_page: 0, + }, + }; }); }, - set_role_for: function(frm) { - frm.trigger("clear_fields") - frm.toggle_display('roles_html', false) + set_role_for: function (frm) { + frm.trigger("clear_fields"); + frm.toggle_display("roles_html", false); }, - clear_fields: function(frm) { - var field = (frm.doc.set_role_for == 'Report') ? 'page' : 'report'; - frm.set_value(field, ''); + clear_fields: function (frm) { + var field = frm.doc.set_role_for == "Report" ? "page" : "report"; + frm.set_value(field, ""); }, - page: function(frm) { + page: function (frm) { frm.events.setup_buttons(frm); if (frm.doc.page) { frm.trigger("set_report_page_data"); @@ -62,7 +62,7 @@ frappe.ui.form.on('Role Permission for Page and Report', { } }, - report: function(frm) { + report: function (frm) { frm.events.setup_buttons(frm); if (frm.doc.report) { frm.trigger("set_report_page_data"); @@ -71,57 +71,57 @@ frappe.ui.form.on('Role Permission for Page and Report', { } }, - set_report_page_data: function(frm) { - frm.toggle_display('roles_html', true) + set_report_page_data: function (frm) { + frm.toggle_display("roles_html", true); frm.role_area.show(); return frm.call({ - method:"set_report_page_data", + method: "set_report_page_data", doc: frm.doc, - callback: function(r) { - refresh_field('roles') - frm.roles_editor.show() - } - }) + callback: function (r) { + refresh_field("roles"); + frm.roles_editor.show(); + }, + }); }, - update_report_page_data: function(frm) { - frm.trigger("validate_mandatory_fields") - if(frm.roles_editor) { - frm.roles_editor.set_roles_in_table() + update_report_page_data: function (frm) { + frm.trigger("validate_mandatory_fields"); + if (frm.roles_editor) { + frm.roles_editor.set_roles_in_table(); } return frm.call({ - method:"update_report_page_data", + method: "update_report_page_data", doc: frm.doc, - callback: function(r) { - refresh_field('roles') - frm.roles_editor.show() - frappe.msgprint(__("Successfully Updated")) - } - }) + callback: function (r) { + refresh_field("roles"); + frm.roles_editor.show(); + frappe.msgprint(__("Successfully Updated")); + }, + }); }, - reset_roles: function(frm) { - frm.trigger("validate_mandatory_fields") + reset_roles: function (frm) { + frm.trigger("validate_mandatory_fields"); return frm.call({ - method:"reset_roles", + method: "reset_roles", doc: frm.doc, - callback: function(r) { - refresh_field('roles') - frm.roles_editor.show() - frappe.msgprint(__("Successfully Updated")) - } - }) + callback: function (r) { + refresh_field("roles"); + frm.roles_editor.show(); + frappe.msgprint(__("Successfully Updated")); + }, + }); }, - validate_mandatory_fields: function(frm) { - if(!frm.doc.set_role_for){ - frappe.throw(__("Mandatory field: set role for")) + validate_mandatory_fields: function (frm) { + if (!frm.doc.set_role_for) { + frappe.throw(__("Mandatory field: set role for")); } - if(frm.doc.set_role_for && !frm.doc[frm.doc.set_role_for.toLocaleLowerCase()]) { - frappe.throw(__("Mandatory field: {0}", [frm.doc.set_role_for])) + if (frm.doc.set_role_for && !frm.doc[frm.doc.set_role_for.toLocaleLowerCase()]) { + frappe.throw(__("Mandatory field: {0}", [frm.doc.set_role_for])); } - } + }, }); diff --git a/frappe/core/doctype/role_permission_for_page_and_report/role_permission_for_page_and_report.json b/frappe/core/doctype/role_permission_for_page_and_report/role_permission_for_page_and_report.json index 8a5393b872..52ecc5d38f 100644 --- a/frappe/core/doctype/role_permission_for_page_and_report/role_permission_for_page_and_report.json +++ b/frappe/core/doctype/role_permission_for_page_and_report/role_permission_for_page_and_report.json @@ -1,327 +1,95 @@ { - "allow_copy": 1, - "allow_events_in_timeline": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, - "creation": "2017-02-13 17:33:25.157332", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", + "actions": [], + "allow_copy": 1, + "creation": "2017-02-13 17:33:25.157332", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "set_role_for", + "page", + "report", + "column_break_4", + "enable_prepared_report", + "roles_permission", + "roles_html", + "roles" + ], "fields": [ { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "set_role_for", - "fieldtype": "Select", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Set Role For", - "length": 0, - "no_copy": 0, - "options": "\nPage\nReport", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "set_role_for", + "fieldtype": "Select", + "in_list_view": 1, + "label": "Set Role For", + "options": "\nPage\nReport", + "reqd": 1 + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "depends_on": "eval:doc.set_role_for == 'Page'", - "fieldname": "page", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Page", - "length": 0, - "no_copy": 0, - "options": "Page", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "depends_on": "eval:doc.set_role_for == 'Page'", + "fieldname": "page", + "fieldtype": "Link", + "label": "Page", + "options": "Page" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "depends_on": "eval:doc.set_role_for == 'Report'", - "fieldname": "report", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Report", - "length": 0, - "no_copy": 0, - "options": "Report", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "depends_on": "eval:doc.set_role_for == 'Report'", + "fieldname": "report", + "fieldtype": "Link", + "label": "Report", + "options": "Report" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "column_break_4", - "fieldtype": "Column Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "column_break_4", + "fieldtype": "Column Break" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "depends_on": "report", - "fetch_from": "", - "fieldname": "disable_prepared_report", - "fieldtype": "Check", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Disable Prepared Report", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "roles_permission", + "fieldtype": "Section Break", + "label": "Allow Roles" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "roles_permission", - "fieldtype": "Section Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Allow Roles", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "roles_html", + "fieldtype": "HTML", + "label": "Roles Html" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "depends_on": "", - "fieldname": "roles_html", - "fieldtype": "HTML", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Roles Html", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "roles", + "fieldtype": "Table", + "hidden": 1, + "label": "Roles", + "options": "Has Role", + "read_only": 1 + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "roles", - "fieldtype": "Table", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Roles", - "length": 0, - "no_copy": 0, - "options": "Has Role", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "default": "0", + "depends_on": "report", + "fieldname": "enable_prepared_report", + "fieldtype": "Check", + "label": "Enable Prepared Report" } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 1, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 1, - "istable": 0, - "max_attachments": 0, - "modified": "2019-01-25 12:08:57.250719", - "modified_by": "Administrator", - "module": "Core", - "name": "Role Permission for Page and Report", - "name_case": "", - "owner": "Administrator", + ], + "hide_toolbar": 1, + "issingle": 1, + "links": [], + "modified": "2022-11-23 12:39:05.750386", + "modified_by": "Administrator", + "module": "Core", + "name": "Role Permission for Page and Report", + "owner": "Administrator", "permissions": [ { - "amend": 0, - "cancel": 0, - "create": 1, - "delete": 1, - "email": 1, - "export": 0, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 0, - "role": "System Manager", - "set_user_permissions": 0, - "share": 1, - "submit": 0, + "create": 1, + "delete": 1, + "email": 1, + "print": 1, + "read": 1, + "role": "System Manager", + "share": 1, "write": 1 } - ], - "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 0, - "track_seen": 0, - "track_views": 0 + ], + "quick_entry": 1, + "sort_field": "modified", + "sort_order": "DESC", + "states": [] } \ No newline at end of file diff --git a/frappe/core/doctype/role_permission_for_page_and_report/role_permission_for_page_and_report.py b/frappe/core/doctype/role_permission_for_page_and_report/role_permission_for_page_and_report.py index bd61995ba3..9a3511184d 100644 --- a/frappe/core/doctype/role_permission_for_page_and_report/role_permission_for_page_and_report.py +++ b/frappe/core/doctype/role_permission_for_page_and_report/role_permission_for_page_and_report.py @@ -2,8 +2,9 @@ # License: MIT. See LICENSE import frappe -from frappe.core.doctype.report.report import is_prepared_report_disabled +from frappe.core.doctype.report.report import is_prepared_report_enabled from frappe.model.document import Document +from frappe.utils import cint class RolePermissionforPageandReport(Document): @@ -27,7 +28,7 @@ class RolePermissionforPageandReport(Document): def check_prepared_report_disabled(self): if self.report: - self.disable_prepared_report = is_prepared_report_disabled(self.report) + self.enable_prepared_report = is_prepared_report_enabled(self.report) def get_standard_roles(self): doctype = self.set_role_for @@ -67,9 +68,9 @@ class RolePermissionforPageandReport(Document): if self.report: # intentionally written update query in frappe.db.sql instead of frappe.db.set_value frappe.db.sql( - """ update `tabReport` set disable_prepared_report = %s + """update `tabReport` set prepared_report = %s where name = %s""", - (self.disable_prepared_report, self.report), + (self.enable_prepared_report, self.report), ) def get_args(self, row=None): diff --git a/frappe/core/doctype/role_permission_for_page_and_report/test_role_permission_for_page_and_report.py b/frappe/core/doctype/role_permission_for_page_and_report/test_role_permission_for_page_and_report.py new file mode 100644 index 0000000000..b05da325e6 --- /dev/null +++ b/frappe/core/doctype/role_permission_for_page_and_report/test_role_permission_for_page_and_report.py @@ -0,0 +1,9 @@ +# Copyright (c) 2022, Frappe Technologies and Contributors +# See license.txt + +# import frappe +from frappe.tests.utils import FrappeTestCase + + +class TestRolePermissionforPageandReport(FrappeTestCase): + pass diff --git a/frappe/core/doctype/role_profile/role_profile.js b/frappe/core/doctype/role_profile/role_profile.js index e43980770a..1a5ed95287 100644 --- a/frappe/core/doctype/role_profile/role_profile.js +++ b/frappe/core/doctype/role_profile/role_profile.js @@ -1,21 +1,20 @@ // Copyright (c) 2017, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Role Profile', { - refresh: function(frm) { +frappe.ui.form.on("Role Profile", { + refresh: function (frm) { if (has_common(frappe.user_roles, ["Administrator", "System Manager"])) { if (!frm.roles_editor) { const role_area = $(frm.fields_dict.roles_html.wrapper); frm.roles_editor = new frappe.RoleEditor(role_area, frm); } frm.roles_editor.show(); - } }, - validate: function(frm) { + validate: function (frm) { if (frm.roles_editor) { frm.roles_editor.set_roles_in_table(); } - } + }, }); diff --git a/frappe/core/doctype/role_profile/test_role_profile.py b/frappe/core/doctype/role_profile/test_role_profile.py index 726a5fc83e..34fb9741f9 100644 --- a/frappe/core/doctype/role_profile/test_role_profile.py +++ b/frappe/core/doctype/role_profile/test_role_profile.py @@ -1,13 +1,12 @@ # Copyright (c) 2017, Frappe Technologies and Contributors # License: MIT. See LICENSE -import unittest - import frappe +from frappe.tests.utils import FrappeTestCase test_dependencies = ["Role"] -class TestRoleProfile(unittest.TestCase): +class TestRoleProfile(FrappeTestCase): def test_make_new_role_profile(self): frappe.delete_doc_if_exists("Role Profile", "Test 1", force=1) new_role_profile = frappe.get_doc(dict(doctype="Role Profile", role_profile="Test 1")).insert() diff --git a/frappe/core/doctype/payment_gateway/__init__.py b/frappe/core/doctype/rq_job/__init__.py similarity index 100% rename from frappe/core/doctype/payment_gateway/__init__.py rename to frappe/core/doctype/rq_job/__init__.py diff --git a/frappe/core/doctype/rq_job/rq_job.js b/frappe/core/doctype/rq_job/rq_job.js new file mode 100644 index 0000000000..3f7a1a15b7 --- /dev/null +++ b/frappe/core/doctype/rq_job/rq_job.js @@ -0,0 +1,30 @@ +// Copyright (c) 2022, Frappe Technologies and contributors +// For license information, please see license.txt + +frappe.ui.form.on("RQ Job", { + refresh: function (frm) { + // Nothing in this form is supposed to be editable. + frm.disable_form(); + frm.dashboard.set_headline_alert( + "This is a virtual doctype and data is cleared periodically." + ); + + if (["started", "queued"].includes(frm.doc.status)) { + frm.add_custom_button(__("Force Stop job"), () => { + frappe.confirm( + "This will terminate the job immediately and might be dangerous, are you sure? ", + () => { + frappe + .xcall("frappe.core.doctype.rq_job.rq_job.stop_job", { + job_id: frm.doc.name, + }) + .then((r) => { + frappe.show_alert("Job Stopped Succefully"); + frm.reload_doc(); + }); + } + ); + }); + } + }, +}); diff --git a/frappe/core/doctype/rq_job/rq_job.json b/frappe/core/doctype/rq_job/rq_job.json new file mode 100644 index 0000000000..7cae15cf59 --- /dev/null +++ b/frappe/core/doctype/rq_job/rq_job.json @@ -0,0 +1,162 @@ +{ + "actions": [], + "allow_copy": 1, + "autoname": "field:job_id", + "creation": "2022-09-10 16:19:37.934903", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "job_info_section", + "job_id", + "job_name", + "queue", + "timeout", + "column_break_5", + "arguments", + "job_status_section", + "status", + "time_taken", + "column_break_11", + "started_at", + "ended_at", + "exception_section", + "exc_info" + ], + "fields": [ + { + "fieldname": "queue", + "fieldtype": "Select", + "in_list_view": 1, + "in_standard_filter": 1, + "label": "Queue", + "options": "default\nshort\nlong" + }, + { + "fieldname": "status", + "fieldtype": "Select", + "in_list_view": 1, + "in_standard_filter": 1, + "label": "Status", + "options": "queued\nstarted\nfinished\nfailed\ndeferred\nscheduled\ncanceled" + }, + { + "fieldname": "job_id", + "fieldtype": "Data", + "label": "Job ID", + "unique": 1 + }, + { + "fieldname": "exc_info", + "fieldtype": "Code", + "label": "Exception" + }, + { + "fieldname": "job_name", + "fieldtype": "Data", + "label": "Job Name" + }, + { + "fieldname": "arguments", + "fieldtype": "Code", + "label": "Arguments" + }, + { + "fieldname": "timeout", + "fieldtype": "Duration", + "label": "Timeout" + }, + { + "fieldname": "time_taken", + "fieldtype": "Duration", + "label": "Time Taken" + }, + { + "fieldname": "started_at", + "fieldtype": "Datetime", + "label": "Started At" + }, + { + "fieldname": "ended_at", + "fieldtype": "Datetime", + "label": "Ended At" + }, + { + "fieldname": "job_info_section", + "fieldtype": "Section Break", + "label": "Job Info" + }, + { + "fieldname": "job_status_section", + "fieldtype": "Section Break", + "label": "Job Status" + }, + { + "fieldname": "column_break_5", + "fieldtype": "Column Break" + }, + { + "fieldname": "column_break_11", + "fieldtype": "Column Break" + }, + { + "fieldname": "exception_section", + "fieldtype": "Section Break" + } + ], + "in_create": 1, + "is_virtual": 1, + "links": [], + "modified": "2022-09-11 05:27:50.878534", + "modified_by": "Administrator", + "module": "Core", + "name": "RQ Job", + "naming_rule": "By fieldname", + "owner": "Administrator", + "permissions": [ + { + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1 + }, + { + "delete": 1, + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "Administrator", + "share": 1 + } + ], + "sort_field": "modified", + "sort_order": "DESC", + "states": [ + { + "color": "Yellow", + "title": "queued" + }, + { + "color": "Blue", + "title": "started" + }, + { + "color": "Red", + "title": "failed" + }, + { + "color": "Green", + "title": "finished" + }, + { + "color": "Orange", + "title": "cancelled" + } + ], + "title_field": "job_name" +} \ No newline at end of file diff --git a/frappe/core/doctype/rq_job/rq_job.py b/frappe/core/doctype/rq_job/rq_job.py new file mode 100644 index 0000000000..391ccd8dfd --- /dev/null +++ b/frappe/core/doctype/rq_job/rq_job.py @@ -0,0 +1,205 @@ +# Copyright (c) 2022, Frappe Technologies and contributors +# For license information, please see license.txt + +import functools +import re + +from rq.command import send_stop_job_command +from rq.exceptions import InvalidJobOperation, NoSuchJobError +from rq.job import Job +from rq.queue import Queue + +import frappe +from frappe import _ +from frappe.model.document import Document +from frappe.utils import ( + cint, + compare, + convert_utc_to_system_timezone, + create_batch, + make_filter_dict, +) +from frappe.utils.background_jobs import get_queues, get_redis_conn + +QUEUES = ["default", "long", "short"] +JOB_STATUSES = ["queued", "started", "failed", "finished", "deferred", "scheduled", "canceled"] + + +def check_permissions(method): + @functools.wraps(method) + def wrapper(*args, **kwargs): + frappe.only_for("System Manager") + job = args[0].job + if not for_current_site(job): + raise frappe.PermissionError + + return method(*args, **kwargs) + + return wrapper + + +class RQJob(Document): + def load_from_db(self): + try: + job = Job.fetch(self.name, connection=get_redis_conn()) + except NoSuchJobError: + raise frappe.DoesNotExistError + + if not for_current_site(job): + raise frappe.PermissionError + + super(Document, self).__init__(serialize_job(job)) + self._job_obj = job + + @property + def job(self): + return self._job_obj + + @staticmethod + def get_list(args): + + start = cint(args.get("start")) or 0 + page_length = cint(args.get("page_length")) or 20 + + order_desc = "desc" in args.get("order_by", "") + + matched_job_ids = RQJob.get_matching_job_ids(args) + + jobs = [] + for job_ids in create_batch(matched_job_ids, 100): + jobs.extend( + serialize_job(job) + for job in Job.fetch_many(job_ids=job_ids, connection=get_redis_conn()) + if job and for_current_site(job) + ) + if len(jobs) > start + page_length: + # we have fetched enough. This is inefficient but because of site filtering TINA + break + + return sorted(jobs, key=lambda j: j.modified, reverse=order_desc)[start : start + page_length] + + @staticmethod + def get_matching_job_ids(args): + filters = make_filter_dict(args.get("filters")) + + queues = _eval_filters(filters.get("queue"), QUEUES) + statuses = _eval_filters(filters.get("status"), JOB_STATUSES) + + matched_job_ids = [] + for queue in get_queues(): + if not queue.name.endswith(tuple(queues)): + continue + for status in statuses: + matched_job_ids.extend(fetch_job_ids(queue, status)) + + return matched_job_ids + + @check_permissions + def delete(self): + self.job.delete() + + @check_permissions + def stop_job(self): + try: + send_stop_job_command(connection=get_redis_conn(), job_id=self.job_id) + except InvalidJobOperation: + frappe.msgprint(_("Job is not running."), title=_("Invalid Operation")) + + @staticmethod + def get_count(args) -> int: + # Can not be implemented efficiently due to site filtering hence ignored. + return 0 + + # None of these methods apply to virtual job doctype, overriden for sanity. + @staticmethod + def get_stats(args): + return {} + + def db_insert(self, *args, **kwargs): + pass + + def db_update(self, *args, **kwargs): + pass + + +def serialize_job(job: Job) -> frappe._dict: + modified = job.last_heartbeat or job.ended_at or job.started_at or job.created_at + job_name = job.kwargs.get("kwargs", {}).get("job_type") or str(job.kwargs.get("job_name")) + + # function objects have this repr: '' + # This regex just removes unnecessary things around it. + if matches := re.match(r".*) at 0x.*>", job_name): + job_name = matches.group("func_name") + + return frappe._dict( + name=job.id, + job_id=job.id, + queue=job.origin.rsplit(":", 1)[1], + job_name=job_name, + status=job.get_status(), + started_at=convert_utc_to_system_timezone(job.started_at) if job.started_at else "", + ended_at=convert_utc_to_system_timezone(job.ended_at) if job.ended_at else "", + time_taken=(job.ended_at - job.started_at).total_seconds() if job.ended_at else "", + exc_info=job.exc_info, + arguments=frappe.as_json(job.kwargs), + timeout=job.timeout, + creation=convert_utc_to_system_timezone(job.created_at), + modified=convert_utc_to_system_timezone(modified), + _comment_count=0, + owner=job.kwargs.get("user"), + modified_by=job.kwargs.get("user"), + ) + + +def for_current_site(job: Job) -> bool: + return job.kwargs.get("site") == frappe.local.site + + +def _eval_filters(filter, values: list[str]) -> list[str]: + if filter: + operator, operand = filter + return [val for val in values if compare(val, operator, operand)] + return values + + +def fetch_job_ids(queue: Queue, status: str) -> list[str]: + registry_map = { + "queued": queue, # self + "started": queue.started_job_registry, + "finished": queue.finished_job_registry, + "failed": queue.failed_job_registry, + "deferred": queue.deferred_job_registry, + "scheduled": queue.scheduled_job_registry, + "canceled": queue.canceled_job_registry, + } + + registry = registry_map.get(status) + if registry is not None: + job_ids = registry.get_job_ids() + return [j for j in job_ids if j] + + return [] + + +@frappe.whitelist() +def remove_failed_jobs(): + frappe.only_for("System Manager") + for queue in get_queues(): + fail_registry = queue.failed_job_registry + for job_ids in create_batch(fail_registry.get_job_ids(), 100): + for job in Job.fetch_many(job_ids=job_ids, connection=get_redis_conn()): + if job and for_current_site(job): + fail_registry.remove(job, delete_job=True) + + +def get_all_queued_jobs(): + jobs = [] + for q in get_queues(): + jobs.extend(q.get_jobs()) + + return [job for job in jobs if for_current_site(job)] + + +@frappe.whitelist() +def stop_job(job_id): + frappe.get_doc("RQ Job", job_id).stop_job() diff --git a/frappe/core/doctype/rq_job/rq_job_list.js b/frappe/core/doctype/rq_job/rq_job_list.js new file mode 100644 index 0000000000..aa05b411ba --- /dev/null +++ b/frappe/core/doctype/rq_job/rq_job_list.js @@ -0,0 +1,62 @@ +frappe.listview_settings["RQ Job"] = { + hide_name_column: true, + + onload(listview) { + if (!has_common(frappe.user_roles, ["Administrator", "System Manager"])) return; + + listview.page.add_inner_button( + __("Remove Failed Jobs"), + () => { + frappe.confirm(__("Are you sure you want to remove all failed jobs?"), () => { + frappe.xcall("frappe.core.doctype.rq_job.rq_job.remove_failed_jobs"); + }); + }, + __("Actions") + ); + + if (listview.list_view_settings) { + listview.list_view_settings.disable_count = 1; + listview.list_view_settings.disable_sidebar_stats = 1; + } + + frappe.xcall("frappe.utils.scheduler.get_scheduler_status").then(({ status }) => { + if (status === "active") { + listview.page.set_indicator(__("Scheduler: Active"), "green"); + } else { + listview.page.set_indicator(__("Scheduler: Inactive"), "red"); + listview.page.add_inner_button( + __("Enable Scheduler"), + () => { + frappe.confirm(__("Are you sure you want to re-enable scheduler?"), () => { + frappe + .xcall("frappe.utils.scheduler.activate_scheduler") + .then(() => { + frappe.show_alert(__("Enabled Scheduler")); + }) + .catch((e) => { + frappe.show_alert({ + message: __("Failed to enable scheduler: {0}", e), + indicator: "error", + }); + }); + }); + }, + __("Actions") + ); + } + }); + + setInterval(() => { + if (listview.list_view_settings.disable_auto_refresh) { + return; + } + + const route = frappe.get_route() || []; + if (route[0] != "List" || "RQ Job" != route[1]) { + return; + } + + listview.refresh(); + }, 5000); + }, +}; diff --git a/frappe/core/doctype/rq_job/test_rq_job.py b/frappe/core/doctype/rq_job/test_rq_job.py new file mode 100644 index 0000000000..265583fe83 --- /dev/null +++ b/frappe/core/doctype/rq_job/test_rq_job.py @@ -0,0 +1,114 @@ +# Copyright (c) 2022, Frappe Technologies and Contributors + +# See license.txt + +import time + +from rq import exceptions as rq_exc +from rq.job import Job + +import frappe +from frappe.core.doctype.rq_job.rq_job import RQJob, remove_failed_jobs, stop_job +from frappe.tests.utils import FrappeTestCase, timeout +from frappe.utils import cstr, execute_in_shell +from frappe.utils.background_jobs import is_job_enqueued + + +class TestRQJob(FrappeTestCase): + + BG_JOB = "frappe.core.doctype.rq_job.test_rq_job.test_func" + + @timeout(seconds=20) + def check_status(self, job: Job, status, wait=True): + while wait: + if not (job.is_queued or job.is_started): + break + time.sleep(0.2) + + self.assertEqual(frappe.get_doc("RQ Job", job.id).status, status) + + def test_serialization(self): + + job = frappe.enqueue(method=self.BG_JOB, queue="short") + rq_job = frappe.get_doc("RQ Job", job.id) + + self.assertEqual(job, rq_job.job) + self.assertDocumentEqual( + { + "name": job.id, + "queue": "short", + "job_name": self.BG_JOB, + "exc_info": None, + }, + rq_job, + ) + self.check_status(job, "finished") + + def test_configurable_ttl(self): + frappe.conf.rq_job_failure_ttl = 600 + job = frappe.enqueue(method=self.BG_JOB, queue="short") + + self.assertEqual(job.failure_ttl, 600) + + def test_func_obj_serialization(self): + job = frappe.enqueue(method=test_func, queue="short") + rq_job = frappe.get_doc("RQ Job", job.id) + self.assertEqual(rq_job.job_name, "test_func") + + def test_get_list_filtering(self): + + # Check failed job clearning and filtering + remove_failed_jobs() + jobs = RQJob.get_list({"filters": [["RQ Job", "status", "=", "failed"]]}) + self.assertEqual(jobs, []) + + # Fail a job + job = frappe.enqueue(method=self.BG_JOB, queue="short", fail=True) + self.check_status(job, "failed") + jobs = RQJob.get_list({"filters": [["RQ Job", "status", "=", "failed"]]}) + self.assertEqual(len(jobs), 1) + self.assertTrue(jobs[0].exc_info) + + # Assert that non-failed job still exists + non_failed_jobs = RQJob.get_list({"filters": [["RQ Job", "status", "!=", "failed"]]}) + self.assertGreaterEqual(len(non_failed_jobs), 1) + + # Create a slow job and check if it's stuck in "Started" + job = frappe.enqueue(method=self.BG_JOB, queue="short", sleep=10) + time.sleep(3) + self.check_status(job, "started", wait=False) + stop_job(job_id=job.id) + self.check_status(job, "stopped") + + def test_delete_doc(self): + job = frappe.enqueue(method=self.BG_JOB, queue="short") + frappe.get_doc("RQ Job", job.id).delete() + + with self.assertRaises(rq_exc.NoSuchJobError): + job.refresh() + + @timeout(20) + def test_multi_queue_burst_consumption(self): + for _ in range(3): + for q in ["default", "short"]: + frappe.enqueue(self.BG_JOB, sleep=1, queue=q) + + _, stderr = execute_in_shell("bench worker --queue short,default --burst", check_exit_code=True) + self.assertIn("quitting", cstr(stderr)) + + @timeout(20) + def test_job_id_dedup(self): + job_id = "test_dedup" + job = frappe.enqueue(self.BG_JOB, sleep=5, job_id=job_id) + self.assertTrue(is_job_enqueued(job_id)) + self.check_status(job, "finished") + self.assertFalse(is_job_enqueued(job_id)) + + +def test_func(fail=False, sleep=0): + if fail: + 42 / 0 + if sleep: + time.sleep(sleep) + + return True diff --git a/frappe/core/doctype/test/__init__.py b/frappe/core/doctype/rq_worker/__init__.py similarity index 100% rename from frappe/core/doctype/test/__init__.py rename to frappe/core/doctype/rq_worker/__init__.py diff --git a/frappe/core/doctype/rq_worker/rq_worker.js b/frappe/core/doctype/rq_worker/rq_worker.js new file mode 100644 index 0000000000..622cb30cb9 --- /dev/null +++ b/frappe/core/doctype/rq_worker/rq_worker.js @@ -0,0 +1,9 @@ +// Copyright (c) 2022, Frappe Technologies and contributors +// For license information, please see license.txt + +frappe.ui.form.on("RQ Worker", { + refresh: function (frm) { + // Nothing in this form is supposed to be editable. + frm.disable_form(); + }, +}); diff --git a/frappe/core/doctype/rq_worker/rq_worker.json b/frappe/core/doctype/rq_worker/rq_worker.json new file mode 100644 index 0000000000..18441377c9 --- /dev/null +++ b/frappe/core/doctype/rq_worker/rq_worker.json @@ -0,0 +1,144 @@ +{ + "actions": [], + "allow_copy": 1, + "creation": "2022-09-10 14:54:57.342170", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "worker_information_section", + "queue", + "queue_type", + "column_break_4", + "worker_name", + "statistics_section", + "status", + "pid", + "current_job_id", + "successful_job_count", + "failed_job_count", + "column_break_12", + "birth_date", + "last_heartbeat", + "total_working_time", + "utilization_percent" + ], + "fields": [ + { + "fieldname": "worker_name", + "fieldtype": "Data", + "label": "Worker Name", + "unique": 1 + }, + { + "fieldname": "status", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Status" + }, + { + "fieldname": "current_job_id", + "fieldtype": "Link", + "label": "Current Job ID", + "options": "RQ Job" + }, + { + "fieldname": "pid", + "fieldtype": "Data", + "label": "PID" + }, + { + "fieldname": "last_heartbeat", + "fieldtype": "Datetime", + "label": "Last Heartbeat" + }, + { + "fieldname": "birth_date", + "fieldtype": "Datetime", + "label": "Start Time" + }, + { + "fieldname": "successful_job_count", + "fieldtype": "Int", + "label": "Successful Job Count" + }, + { + "fieldname": "failed_job_count", + "fieldtype": "Int", + "in_list_view": 1, + "label": "Failed Job Count" + }, + { + "fieldname": "total_working_time", + "fieldtype": "Duration", + "label": "Total Working Time" + }, + { + "fieldname": "queue", + "fieldtype": "Data", + "label": "Queue(s)" + }, + { + "fieldname": "queue_type", + "fieldtype": "Select", + "in_list_view": 1, + "label": "Queue Type(s)", + "options": "default\nlong\nshort" + }, + { + "fieldname": "worker_information_section", + "fieldtype": "Section Break", + "label": "Worker Information" + }, + { + "fieldname": "statistics_section", + "fieldtype": "Section Break", + "label": "Statistics" + }, + { + "fieldname": "column_break_4", + "fieldtype": "Column Break" + }, + { + "fieldname": "column_break_12", + "fieldtype": "Column Break" + }, + { + "fieldname": "utilization_percent", + "fieldtype": "Percent", + "in_list_view": 1, + "label": "Utilization %" + } + ], + "in_create": 1, + "is_virtual": 1, + "links": [], + "modified": "2022-11-24 14:50:48.511706", + "modified_by": "Administrator", + "module": "Core", + "name": "RQ Worker", + "owner": "Administrator", + "permissions": [ + { + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1 + } + ], + "sort_field": "modified", + "sort_order": "DESC", + "states": [ + { + "color": "Blue", + "title": "idle" + }, + { + "color": "Yellow", + "title": "busy" + } + ] +} \ No newline at end of file diff --git a/frappe/core/doctype/rq_worker/rq_worker.py b/frappe/core/doctype/rq_worker/rq_worker.py new file mode 100644 index 0000000000..ce2f4ca8b2 --- /dev/null +++ b/frappe/core/doctype/rq_worker/rq_worker.py @@ -0,0 +1,84 @@ +# Copyright (c) 2022, Frappe Technologies and contributors +# For license information, please see license.txt + +import datetime +from contextlib import suppress + +from rq import Worker + +import frappe +from frappe.model.document import Document +from frappe.utils import cint, convert_utc_to_system_timezone +from frappe.utils.background_jobs import get_workers + + +class RQWorker(Document): + def load_from_db(self): + + all_workers = get_workers() + workers = [w for w in all_workers if w.pid == cint(self.name)] + if not workers: + raise frappe.DoesNotExistError + d = serialize_worker(workers[0]) + + super(Document, self).__init__(d) + + @staticmethod + def get_list(args): + start = cint(args.get("start")) or 0 + page_length = cint(args.get("page_length")) or 20 + + workers = get_workers() + + valid_workers = [w for w in workers if w.pid][start : start + page_length] + return [serialize_worker(worker) for worker in valid_workers] + + @staticmethod + def get_count(args) -> int: + return len(get_workers()) + + # None of these methods apply to virtual workers, overriden for sanity. + @staticmethod + def get_stats(args): + return {} + + def db_insert(self, *args, **kwargs): + pass + + def db_update(self, *args, **kwargs): + pass + + def delete(self): + pass + + +def serialize_worker(worker: Worker) -> frappe._dict: + queue_names = worker.queue_names() + + queue = ", ".join(queue_names) + queue_types = ",".join(q.rsplit(":", 1)[1] for q in queue_names) + + return frappe._dict( + name=worker.pid, + queue=queue, + queue_type=queue_types, + worker_name=worker.name, + status=worker.get_state(), + pid=worker.pid, + current_job_id=worker.get_current_job_id(), + last_heartbeat=convert_utc_to_system_timezone(worker.last_heartbeat), + birth_date=convert_utc_to_system_timezone(worker.birth_date), + successful_job_count=worker.successful_job_count, + failed_job_count=worker.failed_job_count, + total_working_time=worker.total_working_time, + _comment_count=0, + modified=convert_utc_to_system_timezone(worker.last_heartbeat), + creation=convert_utc_to_system_timezone(worker.birth_date), + utilization_percent=compute_utilization(worker), + ) + + +def compute_utilization(worker: Worker) -> float: + with suppress(Exception): + total_time = (datetime.datetime.utcnow() - worker.birth_date).total_seconds() + return worker.total_working_time / total_time * 100 diff --git a/frappe/core/doctype/rq_worker/test_rq_worker.py b/frappe/core/doctype/rq_worker/test_rq_worker.py new file mode 100644 index 0000000000..f07338d630 --- /dev/null +++ b/frappe/core/doctype/rq_worker/test_rq_worker.py @@ -0,0 +1,17 @@ +# Copyright (c) 2022, Frappe Technologies and Contributors +# See license.txt + +import frappe +from frappe.core.doctype.rq_worker.rq_worker import RQWorker +from frappe.tests.utils import FrappeTestCase + + +class TestRQWorker(FrappeTestCase): + def test_get_worker_list(self): + workers = RQWorker.get_list({}) + self.assertGreaterEqual(len(workers), 1) + self.assertTrue(any("short" in w.queue_type for w in workers)) + + def test_worker_serialization(self): + workers = RQWorker.get_list({}) + frappe.get_doc("RQ Worker", workers[0].pid) diff --git a/frappe/core/doctype/scheduled_job_log/scheduled_job_log.js b/frappe/core/doctype/scheduled_job_log/scheduled_job_log.js index d43160c658..dd9691854d 100644 --- a/frappe/core/doctype/scheduled_job_log/scheduled_job_log.js +++ b/frappe/core/doctype/scheduled_job_log/scheduled_job_log.js @@ -1,8 +1,7 @@ // Copyright (c) 2019, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Scheduled Job Log', { +frappe.ui.form.on("Scheduled Job Log", { // refresh: function(frm) { - // } }); diff --git a/frappe/core/doctype/scheduled_job_log/scheduled_job_log_list.js b/frappe/core/doctype/scheduled_job_log/scheduled_job_log_list.js index 5ddccb5d44..1edd718651 100644 --- a/frappe/core/doctype/scheduled_job_log/scheduled_job_log_list.js +++ b/frappe/core/doctype/scheduled_job_log/scheduled_job_log_list.js @@ -1,7 +1,7 @@ frappe.listview_settings["Scheduled Job Log"] = { - onload: function(listview) { + onload: function (listview) { frappe.require("logtypes.bundle.js", () => { frappe.utils.logtypes.show_log_retention_message(cur_list.doctype); - }) + }); }, }; diff --git a/frappe/core/doctype/scheduled_job_log/test_scheduled_job_log.py b/frappe/core/doctype/scheduled_job_log/test_scheduled_job_log.py index 11d60e35d8..6fd187b4e4 100644 --- a/frappe/core/doctype/scheduled_job_log/test_scheduled_job_log.py +++ b/frappe/core/doctype/scheduled_job_log/test_scheduled_job_log.py @@ -1,8 +1,8 @@ # Copyright (c) 2019, Frappe Technologies and Contributors # License: MIT. See LICENSE # import frappe -import unittest +from frappe.tests.utils import FrappeTestCase -class TestScheduledJobLog(unittest.TestCase): +class TestScheduledJobLog(FrappeTestCase): pass diff --git a/frappe/core/doctype/scheduled_job_type/scheduled_job_type.js b/frappe/core/doctype/scheduled_job_type/scheduled_job_type.js index 55907b17fc..238754277b 100644 --- a/frappe/core/doctype/scheduled_job_type/scheduled_job_type.js +++ b/frappe/core/doctype/scheduled_job_type/scheduled_job_type.js @@ -1,8 +1,7 @@ // Copyright (c) 2019, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Scheduled Job Type', { +frappe.ui.form.on("Scheduled Job Type", { // refresh: function(frm) { - // } }); diff --git a/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py b/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py index 1c178fcee2..1dd1f07e9f 100644 --- a/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py +++ b/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py @@ -10,7 +10,7 @@ from croniter import croniter import frappe from frappe.model.document import Document from frappe.utils import get_datetime, now_datetime -from frappe.utils.background_jobs import enqueue, get_jobs +from frappe.utils.background_jobs import enqueue, is_job_enqueued class ScheduledJobType(Document): @@ -22,22 +22,21 @@ class ScheduledJobType(Document): # force logging for all events other than continuous ones (ALL) self.create_log = 1 - def enqueue(self, force=False): + def enqueue(self, force=False) -> bool: # enqueue event if last execution is done if self.is_event_due() or force: - if frappe.flags.enqueued_jobs: - frappe.flags.enqueued_jobs.append(self.method) - - if frappe.flags.execute_job: - self.execute() + if not self.is_job_in_queue(): + enqueue( + "frappe.core.doctype.scheduled_job_type.scheduled_job_type.run_scheduled_job", + queue=self.get_queue_name(), + job_type=self.method, + job_id=self.rq_job_id, + ) + return True else: - if not self.is_job_in_queue(): - enqueue( - "frappe.core.doctype.scheduled_job_type.scheduled_job_type.run_scheduled_job", - queue=self.get_queue_name(), - job_type=self.method, - ) - return True + frappe.logger("scheduler").error( + f"Skipped queueing {self.method} because it was found in queue for {frappe.local.site}" + ) return False @@ -46,9 +45,13 @@ class ScheduledJobType(Document): # if the next scheduled event is before NOW, then its due! return self.get_next_execution() <= (current_time or now_datetime()) - def is_job_in_queue(self): - queued_jobs = get_jobs(site=frappe.local.site, key="job_type")[frappe.local.site] - return self.method in queued_jobs + def is_job_in_queue(self) -> bool: + return is_job_enqueued(self.rq_job_id) + + @property + def rq_job_id(self): + """Unique ID created to deduplicate jobs with single RQ call.""" + return f"scheduled_job::{self.method}" @property def next_execution(self): diff --git a/frappe/core/doctype/scheduled_job_type/test_scheduled_job_type.py b/frappe/core/doctype/scheduled_job_type/test_scheduled_job_type.py index 5448bda91f..bbc92dfbc9 100644 --- a/frappe/core/doctype/scheduled_job_type/test_scheduled_job_type.py +++ b/frappe/core/doctype/scheduled_job_type/test_scheduled_job_type.py @@ -1,13 +1,12 @@ # Copyright (c) 2019, Frappe Technologies and Contributors # License: MIT. See LICENSE -import unittest - import frappe from frappe.core.doctype.scheduled_job_type.scheduled_job_type import sync_jobs +from frappe.tests.utils import FrappeTestCase from frappe.utils import get_datetime -class TestScheduledJobType(unittest.TestCase): +class TestScheduledJobType(FrappeTestCase): def setUp(self): frappe.db.rollback() frappe.db.truncate("Scheduled Job Type") diff --git a/frappe/core/doctype/server_script/server_script.js b/frappe/core/doctype/server_script/server_script.js index ca34af11ab..ca5b8d721b 100644 --- a/frappe/core/doctype/server_script/server_script.js +++ b/frappe/core/doctype/server_script/server_script.js @@ -1,31 +1,30 @@ // Copyright (c) 2019, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Server Script', { - setup: function(frm) { - frm.trigger('setup_help'); +frappe.ui.form.on("Server Script", { + setup: function (frm) { + frm.trigger("setup_help"); }, - refresh: function(frm) { - if (frm.doc.script_type != 'Scheduler Event') { + refresh: function (frm) { + if (frm.doc.script_type != "Scheduler Event") { frm.dashboard.hide(); } if (!frm.is_new()) { - frm.add_custom_button(__('Compare Versions'), () => { + frm.add_custom_button(__("Compare Versions"), () => { new frappe.ui.DiffView("Server Script", "script", frm.doc.name); }); } - - frm.call('get_autocompletion_items') - .then(r => r.message) - .then(items => { - frm.set_df_property('script', 'autocompletions', items); + frm.call("get_autocompletion_items") + .then((r) => r.message) + .then((items) => { + frm.set_df_property("script", "autocompletions", items); }); }, setup_help(frm) { - frm.get_field('help_html').html(` + frm.get_field("help_html").html(`

    DocType Event

    Add logic for standard doctype events like Before Insert, After Submit, etc.

    @@ -77,6 +76,5 @@ where tenant_id = 2
 order by creation desc
    `); - } - + }, }); diff --git a/frappe/core/doctype/server_script/server_script.json b/frappe/core/doctype/server_script/server_script.json index 5446cc1a39..3aedd4f542 100644 --- a/frappe/core/doctype/server_script/server_script.json +++ b/frappe/core/doctype/server_script/server_script.json @@ -17,6 +17,10 @@ "disabled", "section_break_8", "script", + "rate_limiting_section", + "enable_rate_limit", + "rate_limit_count", + "rate_limit_seconds", "help_section", "help_html" ], @@ -102,6 +106,32 @@ "fieldtype": "Link", "label": "Module (for export)", "options": "Module Def" + }, + { + "depends_on": "eval:doc.script_type==='API'", + "fieldname": "rate_limiting_section", + "fieldtype": "Section Break", + "label": "Rate Limiting" + }, + { + "default": "0", + "fieldname": "enable_rate_limit", + "fieldtype": "Check", + "label": "Enable Rate Limit" + }, + { + "default": "5", + "depends_on": "enable_rate_limit", + "fieldname": "rate_limit_count", + "fieldtype": "Int", + "label": "Request Limit" + }, + { + "default": "86400", + "depends_on": "enable_rate_limit", + "fieldname": "rate_limit_seconds", + "fieldtype": "Int", + "label": "Time Window (Seconds)" } ], "index_web_pages_for_search": 1, @@ -111,7 +141,7 @@ "link_fieldname": "server_script" } ], - "modified": "2022-06-13 06:04:20.937969", + "modified": "2023-05-16 11:03:58.282680", "modified_by": "Administrator", "module": "Core", "name": "Server Script", diff --git a/frappe/core/doctype/server_script/server_script.py b/frappe/core/doctype/server_script/server_script.py index fda5ca8591..a9b870e240 100644 --- a/frappe/core/doctype/server_script/server_script.py +++ b/frappe/core/doctype/server_script/server_script.py @@ -1,11 +1,13 @@ # Copyright (c) 2019, Frappe Technologies and contributors # License: MIT. See LICENSE +from functools import partial from types import FunctionType, MethodType, ModuleType import frappe from frappe import _ from frappe.model.document import Document +from frappe.rate_limiter import rate_limit from frappe.utils.safe_exec import NamespaceDict, get_safe_globals, safe_exec @@ -77,17 +79,17 @@ class ServerScript(Document): Returns: dict: Evaluates self.script with frappe.utils.safe_exec.safe_exec and returns the flags set in it's safe globals """ - # wrong report type! - if self.script_type != "API": - raise frappe.DoesNotExistError - # validate if guest is allowed - if frappe.session.user == "Guest" and not self.allow_guest: - raise frappe.PermissionError + if self.enable_rate_limit: + # Wrap in rate limiter, required for specifying custom limits for each script + # Note that rate limiter works on `cmd` which is script name + limit = self.rate_limit_count or 5 + seconds = self.rate_limit_seconds or 24 * 60 * 60 - # output can be stored in flags - _globals, _locals = safe_exec(self.script) - return _globals.frappe.flags + _fn = partial(execute_api_server_script, script=self) + return rate_limit(limit=limit, seconds=seconds)(_fn)() + else: + return execute_api_server_script(self) def execute_doc(self, doc: Document): """Specific to Document Event triggered Server Scripts @@ -129,7 +131,7 @@ class ServerScript(Document): Returns: list: Returns list of autocompletion items. - For e.g., ["frappe.utils.cint", "frappe.db.get_all", ...] + For e.g., ["frappe.utils.cint", "frappe.get_all", ...] """ def get_keys(obj): @@ -169,7 +171,6 @@ class ServerScript(Document): return items -@frappe.whitelist() def setup_scheduler_events(script_name, frequency): """Creates or Updates Scheduled Job Type documents based on the specified script name and frequency @@ -202,3 +203,21 @@ def setup_scheduler_events(script_name, frequency): doc.save() frappe.msgprint(_("Scheduled execution for script {0} has updated").format(script_name)) + + +def execute_api_server_script(script=None, *args, **kwargs): + # These are only added for compatibility with rate limiter. + del args + del kwargs + + if script.script_type != "API": + raise frappe.DoesNotExistError + + # validate if guest is allowed + if frappe.session.user == "Guest" and not script.allow_guest: + raise frappe.PermissionError + + # output can be stored in flags + _globals, _locals = safe_exec(script.script) + + return _globals.frappe.flags diff --git a/frappe/core/doctype/server_script/test_server_script.py b/frappe/core/doctype/server_script/test_server_script.py index 4c1c12b7f2..3d11a02ca4 100644 --- a/frappe/core/doctype/server_script/test_server_script.py +++ b/frappe/core/doctype/server_script/test_server_script.py @@ -1,10 +1,10 @@ # Copyright (c) 2019, Frappe Technologies and Contributors # License: MIT. See LICENSE -import unittest - import requests import frappe +from frappe.frappeclient import FrappeClient, FrappeException +from frappe.tests.utils import FrappeTestCase from frappe.utils import get_site_url scripts = [ @@ -86,10 +86,10 @@ frappe.db.add_index("Todo", ["color", "date"]) ] -class TestServerScript(unittest.TestCase): +class TestServerScript(FrappeTestCase): @classmethod def setUpClass(cls): - frappe.db.commit() + super().setUpClass() frappe.db.truncate("Server Script") frappe.get_doc("User", "Administrator").add_roles("Script Manager") for script in scripts: @@ -212,3 +212,74 @@ frappe.qb.from_(todo).select(todo.name).where(todo.name == "{todo.name}").run() """ script.save() script.execute_method() + + def test_scripts_all_the_way_down(self): + # why not + script = frappe.get_doc( + doctype="Server Script", + name="test_nested_scripts_1", + script_type="API", + api_method="test_nested_scripts_1", + script=f"""log("nothing")""", + ) + script.insert() + script.execute_method() + + script = frappe.get_doc( + doctype="Server Script", + name="test_nested_scripts_2", + script_type="API", + api_method="test_nested_scripts_2", + script=f"""frappe.call("test_nested_scripts_1")""", + ) + script.insert() + script.execute_method() + + def test_server_script_rate_limiting(self): + # why not + script1 = frappe.get_doc( + doctype="Server Script", + name="rate_limited_server_script", + script_type="API", + enable_rate_limit=1, + allow_guest=1, + rate_limit_count=5, + api_method="rate_limited_endpoint", + script="""frappe.flags = {"test": True}""", + ) + + script1.insert() + + script2 = frappe.get_doc( + doctype="Server Script", + name="rate_limited_server_script2", + script_type="API", + enable_rate_limit=1, + allow_guest=1, + rate_limit_count=5, + api_method="rate_limited_endpoint2", + script="""frappe.flags = {"test": False}""", + ) + + script2.insert() + + frappe.db.commit() + + site = frappe.utils.get_site_url(frappe.local.site) + client = FrappeClient(site) + + # Exhaust rate limti + for _ in range(5): + client.get_api(script1.api_method) + + self.assertRaises(FrappeException, client.get_api, script1.api_method) + + # Exhaust rate limti + for _ in range(5): + client.get_api(script2.api_method) + + self.assertRaises(FrappeException, client.get_api, script2.api_method) + + script1.delete() + script2.delete() + frappe.db.commit() diff --git a/frappe/core/doctype/session_default_settings/session_default_settings.js b/frappe/core/doctype/session_default_settings/session_default_settings.js index f7cce14809..af333e29a3 100644 --- a/frappe/core/doctype/session_default_settings/session_default_settings.js +++ b/frappe/core/doctype/session_default_settings/session_default_settings.js @@ -1,15 +1,15 @@ // Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors // MIT License. See license.txt -frappe.ui.form.on('Session Default Settings', { - refresh: function(frm) { - frm.set_query('ref_doctype', 'session_defaults', function() { +frappe.ui.form.on("Session Default Settings", { + refresh: function (frm) { + frm.set_query("ref_doctype", "session_defaults", function () { return { filters: { issingle: 0, - istable: 0 - } + istable: 0, + }, }; }); - } + }, }); diff --git a/frappe/core/doctype/session_default_settings/test_session_default_settings.py b/frappe/core/doctype/session_default_settings/test_session_default_settings.py index aa60085ce9..532e805141 100644 --- a/frappe/core/doctype/session_default_settings/test_session_default_settings.py +++ b/frappe/core/doctype/session_default_settings/test_session_default_settings.py @@ -1,15 +1,14 @@ # Copyright (c) 2019, Frappe Technologies and Contributors # License: MIT. See LICENSE -import unittest - import frappe from frappe.core.doctype.session_default_settings.session_default_settings import ( clear_session_defaults, set_session_default_values, ) +from frappe.tests.utils import FrappeTestCase -class TestSessionDefaultSettings(unittest.TestCase): +class TestSessionDefaultSettings(FrappeTestCase): def test_set_session_default_settings(self): frappe.set_user("Administrator") settings = frappe.get_single("Session Default Settings") diff --git a/frappe/core/doctype/sms_parameter/sms_parameter.json b/frappe/core/doctype/sms_parameter/sms_parameter.json index 43b93ed182..98972f9e7d 100755 --- a/frappe/core/doctype/sms_parameter/sms_parameter.json +++ b/frappe/core/doctype/sms_parameter/sms_parameter.json @@ -1,128 +1,51 @@ { - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, - "creation": "2013-02-22 01:27:58", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "editable_grid": 1, + "actions": [], + "creation": "2013-02-22 01:27:58", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "parameter", + "value", + "header" + ], "fields": [ { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "parameter", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Parameter", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "print_width": "150px", - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0, + "fieldname": "parameter", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Parameter", + "print_width": "150px", + "reqd": 1, "width": "150px" - }, + }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "value", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Value", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "print_width": "150px", - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0, + "fieldname": "value", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Value", + "print_width": "150px", + "reqd": 1, "width": "150px" - }, + }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "header", - "fieldtype": "Check", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Header", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "default": "0", + "fieldname": "header", + "fieldtype": "Check", + "in_list_view": 1, + "label": "Header" } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 1, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 1, - "max_attachments": 0, - "modified": "2017-10-13 16:48:00.518463", - "modified_by": "Administrator", - "module": "Core", - "name": "SMS Parameter", - "owner": "Administrator", - "permissions": [], - "quick_entry": 0, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "track_changes": 0, - "track_seen": 0 + ], + "idx": 1, + "istable": 1, + "links": [], + "modified": "2022-08-03 12:20:53.129765", + "modified_by": "Administrator", + "module": "Core", + "name": "SMS Parameter", + "owner": "Administrator", + "permissions": [], + "sort_field": "modified", + "sort_order": "DESC", + "states": [] } \ No newline at end of file diff --git a/frappe/core/doctype/sms_settings/sms_settings.py b/frappe/core/doctype/sms_settings/sms_settings.py index 686890514a..0a5536eb9b 100644 --- a/frappe/core/doctype/sms_settings/sms_settings.py +++ b/frappe/core/doctype/sms_settings/sms_settings.py @@ -63,7 +63,7 @@ def send_sms(receiver_list, msg, sender_name="", success_msg=True): "success_msg": success_msg, } - if frappe.db.get_value("SMS Settings", None, "sms_gateway_url"): + if frappe.db.get_single_value("SMS Settings", "sms_gateway_url"): send_via_gateway(arg) else: msgprint(_("Please Update SMS Settings")) diff --git a/frappe/core/doctype/sms_settings/test_sms_settings.py b/frappe/core/doctype/sms_settings/test_sms_settings.py index 61be20ff66..56cbe9b163 100644 --- a/frappe/core/doctype/sms_settings/test_sms_settings.py +++ b/frappe/core/doctype/sms_settings/test_sms_settings.py @@ -1,7 +1,7 @@ # Copyright (c) 2017, Frappe Technologies and Contributors # License: MIT. See LICENSE -import unittest +from frappe.tests.utils import FrappeTestCase -class TestSMSSettings(unittest.TestCase): +class TestSMSSettings(FrappeTestCase): pass diff --git a/frappe/core/page/background_jobs/__init__.py b/frappe/core/doctype/submission_queue/__init__.py similarity index 100% rename from frappe/core/page/background_jobs/__init__.py rename to frappe/core/doctype/submission_queue/__init__.py diff --git a/frappe/core/doctype/submission_queue/submission_queue.js b/frappe/core/doctype/submission_queue/submission_queue.js new file mode 100644 index 0000000000..6e64be780a --- /dev/null +++ b/frappe/core/doctype/submission_queue/submission_queue.js @@ -0,0 +1,20 @@ +// Copyright (c) 2022, Frappe Technologies and contributors +// For license information, please see license.txt + +frappe.ui.form.on("Submission Queue", { + refresh: function (frm) { + if (frm.doc.status === "Queued" && frappe.boot.user.roles.includes("System Manager")) { + frm.add_custom_button(__("Unlock Reference Document"), () => { + frappe.confirm( + ` + Are you sure you want to go ahead with this action? + Doing this could unlock other submissions of this document which are in queue (if present) + and could lead to non-ideal conditions.`, + () => { + frm.call("unlock_doc"); + } + ); + }); + } + }, +}); diff --git a/frappe/core/doctype/submission_queue/submission_queue.json b/frappe/core/doctype/submission_queue/submission_queue.json new file mode 100644 index 0000000000..04668e1c76 --- /dev/null +++ b/frappe/core/doctype/submission_queue/submission_queue.json @@ -0,0 +1,129 @@ +{ + "actions": [], + "autoname": "hash", + "creation": "2022-10-04 00:41:00.028163", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "status", + "created_at", + "enqueued_by", + "job_id", + "column_break_5", + "ended_at", + "ref_doctype", + "ref_docname", + "section_break_8", + "exception" + ], + "fields": [ + { + "fieldname": "job_id", + "fieldtype": "Link", + "label": "Job Id", + "options": "RQ Job", + "read_only": 1 + }, + { + "fieldname": "ref_doctype", + "fieldtype": "Link", + "in_list_view": 1, + "label": "Reference DocType", + "options": "DocType", + "read_only": 1 + }, + { + "fieldname": "ref_docname", + "fieldtype": "Dynamic Link", + "in_list_view": 1, + "label": "Reference Docname", + "options": "ref_doctype", + "read_only": 1, + "search_index": 1 + }, + { + "fieldname": "status", + "fieldtype": "Select", + "hidden": 1, + "in_list_view": 1, + "label": "Status", + "options": "Queued\nFinished\nFailed", + "read_only": 1 + }, + { + "fieldname": "column_break_5", + "fieldtype": "Column Break" + }, + { + "fieldname": "section_break_8", + "fieldtype": "Section Break" + }, + { + "fieldname": "enqueued_by", + "fieldtype": "Data", + "is_virtual": 1, + "label": "Enqueued By", + "read_only": 1 + }, + { + "fieldname": "ended_at", + "fieldtype": "Datetime", + "label": "Ended At", + "read_only": 1 + }, + { + "fieldname": "created_at", + "fieldtype": "Datetime", + "is_virtual": 1, + "label": "Created At", + "read_only": 1 + }, + { + "fieldname": "exception", + "fieldtype": "Long Text", + "label": "Exception", + "read_only": 1 + } + ], + "index_web_pages_for_search": 1, + "links": [], + "modified": "2023-01-23 12:45:53.997708", + "modified_by": "Administrator", + "module": "Core", + "name": "Submission Queue", + "naming_rule": "Random", + "owner": "Administrator", + "permissions": [ + { + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1 + }, + { + "if_owner": 1, + "read": 1, + "role": "All" + } + ], + "sort_field": "modified", + "sort_order": "DESC", + "states": [ + { + "color": "Blue", + "title": "Queued" + }, + { + "color": "Red", + "title": "Failed" + }, + { + "color": "Green", + "title": "Finished" + } + ] +} \ No newline at end of file diff --git a/frappe/core/doctype/submission_queue/submission_queue.py b/frappe/core/doctype/submission_queue/submission_queue.py new file mode 100644 index 0000000000..be0c20fc32 --- /dev/null +++ b/frappe/core/doctype/submission_queue/submission_queue.py @@ -0,0 +1,190 @@ +# Copyright (c) 2022, Frappe Technologies and contributors +# For license information, please see license.txt + +from urllib.parse import quote + +from rq import get_current_job + +import frappe +from frappe import _ +from frappe.desk.doctype.notification_log.notification_log import enqueue_create_notification +from frappe.model.document import Document +from frappe.monitor import add_data_to_monitor +from frappe.utils import now, time_diff_in_seconds +from frappe.utils.data import cint + + +class SubmissionQueue(Document): + @property + def created_at(self): + return self.creation + + @property + def enqueued_by(self): + return self.owner + + @property + def queued_doc(self): + return getattr(self, "to_be_queued_doc", frappe.get_doc(self.ref_doctype, self.ref_docname)) + + @staticmethod + def clear_old_logs(days=30): + from frappe.query_builder import Interval + from frappe.query_builder.functions import Now + + table = frappe.qb.DocType("Submission Queue") + frappe.db.delete(table, filters=(table.modified < (Now() - Interval(days=days)))) + + def insert(self, to_be_queued_doc: Document, action: str): + self.status = "Queued" + self.to_be_queued_doc = to_be_queued_doc + self.action_for_queuing = action + super().insert(ignore_permissions=True) + + def lock(self): + self.queued_doc.lock() + + def unlock(self): + self.queued_doc.unlock() + + def update_job_id(self, job_id): + frappe.db.set_value( + self.doctype, + self.name, + {"job_id": job_id}, + update_modified=False, + ) + frappe.db.commit() + + def after_insert(self): + self.queue_action( + "background_submission", + to_be_queued_doc=self.queued_doc, + action_for_queuing=self.action_for_queuing, + timeout=600, + enqueue_after_commit=True, + ) + + def background_submission(self, to_be_queued_doc: Document, action_for_queuing: str): + # Set the job id for that submission doctype + self.update_job_id(get_current_job().id) + + _action = action_for_queuing.lower() + if _action == "update": + _action = "submit" + + try: + getattr(to_be_queued_doc, _action)() + add_data_to_monitor( + doctype=to_be_queued_doc.doctype, + docname=to_be_queued_doc.name, + action=_action, + execution_time=time_diff_in_seconds(now(), self.created_at), + enqueued_by=self.enqueued_by, + ) + values = {"status": "Finished"} + except Exception: + values = {"status": "Failed", "exception": frappe.get_traceback(with_context=True)} + frappe.db.rollback() + + values["ended_at"] = now() + frappe.db.set_value(self.doctype, self.name, values, update_modified=False) + self.notify(values["status"], action_for_queuing) + + def notify(self, submission_status: str, action: str): + if submission_status == "Failed": + doctype = self.doctype + docname = self.name + message = _("Action {0} failed on {1} {2}. View it {3}") + else: + doctype = self.ref_doctype + docname = self.ref_docname + message = _("Action {0} completed successfully on {1} {2}. View it {3}") + + message_replacements = ( + frappe.bold(action), + frappe.bold(str(self.ref_doctype)), + frappe.bold(str(self.ref_docname)), + ) + + time_diff = time_diff_in_seconds(now(), self.created_at) + if cint(time_diff) <= 60: + frappe.publish_realtime( + "msgprint", + { + "message": message.format( + *message_replacements, + f"here", + ), + "alert": True, + "indicator": "red" if submission_status == "Failed" else "green", + }, + user=self.enqueued_by, + ) + else: + notification_doc = { + "type": "Alert", + "document_type": doctype, + "document_name": docname, + "subject": message.format(*message_replacements, "here"), + } + + notify_to = frappe.db.get_value("User", self.enqueued_by, fieldname="email") + enqueue_create_notification([notify_to], notification_doc) + + @frappe.whitelist() + def unlock_doc(self): + # NOTE: this can lead to some weird unlocking/locking behaviours. + # for example: hitting unlock on a submission could lead to unlocking of another submission + # of the same reference document. + + if self.status != "Queued": + return + + self.queued_doc.unlock() + frappe.msgprint(_("Document Unlocked")) + + +def queue_submission(doc: Document, action: str, alert: bool = True): + queue = frappe.new_doc("Submission Queue") + queue.ref_doctype = doc.doctype + queue.ref_docname = doc.name + queue.insert(doc, action) + + if alert: + frappe.msgprint( + _("Queued for Submission. You can track the progress over {0}.").format( + f"here" + ), + indicator="green", + alert=True, + ) + + +@frappe.whitelist() +def get_latest_submissions(doctype, docname): + # NOTE: not used creation as orderby intentianlly as we have used update_modified=False everywhere + # hence assuming modified will be equal to creation for submission queue documents + + latest_submission = frappe.db.get_value( + "Submission Queue", + filters={"ref_doctype": doctype, "ref_docname": docname}, + fieldname=["name", "exception", "status"], + ) + + out = None + if latest_submission: + out = { + "latest_submission": latest_submission[0], + "exc": format_tb(latest_submission[1]), + "status": latest_submission[2], + } + + return out + + +def format_tb(traceback: str | None = None): + if not traceback: + return + + return traceback.strip().split("\n")[-1] diff --git a/frappe/core/doctype/submission_queue/test_submission_queue.py b/frappe/core/doctype/submission_queue/test_submission_queue.py new file mode 100644 index 0000000000..c057bd22e1 --- /dev/null +++ b/frappe/core/doctype/submission_queue/test_submission_queue.py @@ -0,0 +1,51 @@ +# Copyright (c) 2022, Frappe Technologies and Contributors +# See license.txt + +import time +import typing + +import frappe +from frappe.tests.utils import FrappeTestCase, timeout +from frappe.utils.background_jobs import get_queue + +if typing.TYPE_CHECKING: + from rq.job import Job + + +class TestSubmissionQueue(FrappeTestCase): + queue = get_queue(qtype="default") + + @timeout(seconds=20) + def check_status(self, job: "Job", status, wait=True): + if wait: + while True: + if job.is_queued or job.is_started: + time.sleep(0.2) + else: + break + self.assertEqual(frappe.get_doc("RQ Job", job.id).status, status) + + def test_queue_operation(self): + from frappe.core.doctype.doctype.test_doctype import new_doctype + from frappe.core.doctype.submission_queue.submission_queue import queue_submission + + if not frappe.db.table_exists("Test Submission Queue", cached=False): + doc = new_doctype("Test Submission Queue", is_submittable=True, queue_in_background=True) + doc.insert() + + d = frappe.new_doc("Test Submission Queue") + d.update({"some_fieldname": "Random"}) + d.insert() + + frappe.db.commit() + queue_submission(d, "submit") + frappe.db.commit() + + # Waiting for execution + time.sleep(4) + submission_queue = frappe.get_last_doc("Submission Queue") + + # Test queueing / starting + job = self.queue.fetch_job(submission_queue.job_id) + # Test completion + self.check_status(job, status="finished") diff --git a/frappe/core/doctype/success_action/success_action.js b/frappe/core/doctype/success_action/success_action.js index 50ddb3b66a..993f6eabf4 100644 --- a/frappe/core/doctype/success_action/success_action.js +++ b/frappe/core/doctype/success_action/success_action.js @@ -1,28 +1,28 @@ // Copyright (c) 2018, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Success Action', { +frappe.ui.form.on("Success Action", { on_load: (frm) => { if (!frm.action_multicheck) { - frm.trigger('set_next_action_multicheck'); + frm.trigger("set_next_action_multicheck"); } }, refresh: (frm) => { if (!frm.action_multicheck) { - frm.trigger('set_next_action_multicheck'); + frm.trigger("set_next_action_multicheck"); } }, validate: (frm) => { const checked_actions = frm.action_multicheck.get_checked_options(); if (checked_actions.length < 2) { - frappe.msgprint(__('Select atleast 2 actions')); + frappe.msgprint(__("Select atleast 2 actions")); } else { return true; } }, before_save: (frm) => { const checked_actions = frm.action_multicheck.get_checked_options(); - frm.doc.next_actions = checked_actions.join('\n'); + frm.doc.next_actions = checked_actions.join("\n"); }, after_save: (frm) => { frappe.boot.success_action.push(frm.doc); @@ -30,31 +30,31 @@ frappe.ui.form.on('Success Action', { }, set_next_action_multicheck: (frm) => { const next_actions_wrapper = frm.fields_dict.next_actions_html.$wrapper; - const checked_actions = frm.doc.next_actions ? frm.doc.next_actions.split('\n') : []; - const action_multicheck_options = get_default_next_actions().map(action => { + const checked_actions = frm.doc.next_actions ? frm.doc.next_actions.split("\n") : []; + const action_multicheck_options = get_default_next_actions().map((action) => { return { label: action.label, value: action.value, - checked: checked_actions.length ? checked_actions.includes(action.value) : 1 + checked: checked_actions.length ? checked_actions.includes(action.value) : 1, }; }); frm.action_multicheck = frappe.ui.form.make_control({ parent: next_actions_wrapper, df: { - 'label': 'Next Actions', - 'fieldname': 'next_actions_multicheck', - 'fieldtype': 'MultiCheck', - 'options': action_multicheck_options, + label: "Next Actions", + fieldname: "next_actions_multicheck", + fieldtype: "MultiCheck", + options: action_multicheck_options, }, }); - } + }, }); const get_default_next_actions = () => { return [ - { label: __('New'), value: 'new' }, - { label: __('Print'), value: 'print' }, - { label: __('Email'), value: 'email' }, - { label: __('View All'), value: 'list' } + { label: __("New"), value: "new" }, + { label: __("Print"), value: "print" }, + { label: __("Email"), value: "email" }, + { label: __("View All"), value: "list" }, ]; -}; \ No newline at end of file +}; diff --git a/frappe/core/doctype/success_action/success_action.json b/frappe/core/doctype/success_action/success_action.json index 25c8e79a05..749fa6764f 100644 --- a/frappe/core/doctype/success_action/success_action.json +++ b/frappe/core/doctype/success_action/success_action.json @@ -1,259 +1,84 @@ { - "allow_copy": 0, - "allow_events_in_timeline": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, + "actions": [], "autoname": "field:ref_doctype", - "beta": 0, "creation": "2018-04-15 18:07:35.316870", - "custom": 0, - "docstatus": 0, "doctype": "DocType", - "document_type": "", "editable_grid": 1, "engine": "InnoDB", + "field_order": [ + "ref_doctype", + "first_success_message", + "message", + "next_actions_html", + "next_actions", + "action_timeout" + ], "fields": [ { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "ref_doctype", "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, "in_list_view": 1, - "in_standard_filter": 0, "label": "Reference Document Type", - "length": 0, - "no_copy": 0, "options": "DocType", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "unique": 1 }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "default": "Congratulations on first creations", "fieldname": "first_success_message", "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, "in_list_view": 1, - "in_standard_filter": 0, "label": "First Success Message", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "reqd": 1 }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "default": "Successfully created", "fieldname": "message", "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, "in_list_view": 1, - "in_standard_filter": 0, "label": "Message", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "reqd": 1 }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "next_actions_html", "fieldtype": "HTML", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Next Actions HTML", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "label": "Next Actions HTML" }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "next_actions", "fieldtype": "Data", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "hidden": 1 }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, + "default": "7", "fieldname": "action_timeout", "fieldtype": "Int", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Action Timeout (Seconds)", - "default": 7, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "label": "Action Timeout (Seconds)" } ], - "has_web_view": 0, - "hide_heading": 0, "hide_toolbar": 1, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2019-09-05 14:22:27.664645", + "links": [], + "modified": "2022-08-03 12:20:54.532708", "modified_by": "Administrator", "module": "Core", "name": "Success Action", - "name_case": "", "owner": "Administrator", "permissions": [ { - "amend": 0, - "cancel": 0, "create": 1, "delete": 1, "email": 1, "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, "print": 1, "read": 1, "report": 1, "role": "System Manager", - "set_user_permissions": 0, "share": 1, - "submit": 0, "write": 1 } ], - "quick_entry": 0, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, "sort_field": "modified", "sort_order": "DESC", - "track_changes": 1, - "track_seen": 0, - "track_views": 0 + "states": [], + "track_changes": 1 } \ No newline at end of file diff --git a/frappe/core/doctype/system_settings/system_settings.js b/frappe/core/doctype/system_settings/system_settings.js index 5128ae24cb..bf8988d64c 100644 --- a/frappe/core/doctype/system_settings/system_settings.js +++ b/frappe/core/doctype/system_settings/system_settings.js @@ -1,43 +1,38 @@ frappe.ui.form.on("System Settings", { - refresh: function(frm) { + refresh: function (frm) { frappe.call({ method: "frappe.core.doctype.system_settings.system_settings.load", - callback: function(data) { + callback: function (data) { frappe.all_timezones = data.message.timezones; frm.set_df_property("time_zone", "options", frappe.all_timezones); - $.each(data.message.defaults, function(key, val) { - frm.set_value(key, val); + $.each(data.message.defaults, function (key, val) { + frm.set_value(key, val, null, true); frappe.sys_defaults[key] = val; }); if (frm.re_setup_moment) { frappe.app.setup_moment(); delete frm.re_setup_moment; } - } + }, }); + + frm.trigger("set_rounding_method_options"); }, - enable_password_policy: function(frm) { + enable_password_policy: function (frm) { if (frm.doc.enable_password_policy == 0) { frm.set_value("minimum_password_score", ""); } else { frm.set_value("minimum_password_score", "2"); } }, - enable_two_factor_auth: function(frm) { + enable_two_factor_auth: function (frm) { if (frm.doc.enable_two_factor_auth == 0) { frm.set_value("bypass_2fa_for_retricted_ip_users", 0); frm.set_value("bypass_restrict_ip_check_if_2fa_enabled", 0); } }, - enable_prepared_report_auto_deletion: function(frm) { - if (frm.doc.enable_prepared_report_auto_deletion) { - if (!frm.doc.prepared_report_expiry_period) { - frm.set_value('prepared_report_expiry_period', 7); - } - } - }, - on_update: function(frm) { + on_update: function (frm) { if (frappe.boot.time_zone && frappe.boot.time_zone.system !== frm.doc.time_zone) { // Clear cache after saving to refresh the values of boot. frappe.ui.toolbar.clear_cache(); @@ -46,4 +41,34 @@ frappe.ui.form.on("System Settings", { first_day_of_the_week(frm) { frm.re_setup_moment = true; }, + + rounding_method: function (frm) { + if (frm.doc.rounding_method == frappe.boot.sysdefaults.rounding_method) return; + let msg = __( + "Changing rounding method on site with data can result in unexpected behaviour." + ); + msg += "
    "; + msg += __("Do you still want to proceed?"); + + frappe.confirm( + msg, + () => {}, + () => { + frm.set_value("rounding_method", frappe.boot.sysdefaults.rounding_method); + } + ); + }, + + set_rounding_method_options: function (frm) { + if (frm.doc.rounding_method != "Banker's Rounding (legacy)") { + let field = frm.fields_dict.rounding_method; + + field.df.options = field.df.options + .split("\n") + .filter((o) => o != "Banker's Rounding (legacy)") + .join("\n"); + + field.refresh(); + } + }, }); diff --git a/frappe/core/doctype/system_settings/system_settings.json b/frappe/core/doctype/system_settings/system_settings.json index a444062b5a..091dc1df1e 100644 --- a/frappe/core/doctype/system_settings/system_settings.json +++ b/frappe/core/doctype/system_settings/system_settings.json @@ -13,14 +13,16 @@ "time_zone", "enable_onboarding", "setup_complete", + "disable_document_sharing", "date_and_number_format", "date_format", "time_format", "number_format", + "first_day_of_the_week", "column_break_7", "float_precision", "currency_precision", - "first_day_of_the_week", + "rounding_method", "sec_backup_limit", "backup_limit", "encrypt_backup", @@ -33,12 +35,14 @@ "allow_guests_to_upload_files", "security", "session_expiry", - "session_expiry_mobile", "document_share_key_expiry", "column_break_13", "deny_multiple_sessions", "allow_login_using_mobile_number", "allow_login_using_user_name", + "disable_user_pass_login", + "login_with_email_link", + "login_with_email_link_expiry", "allow_error_traceback", "strip_exif_metadata_from_uploaded_images", "allow_older_web_view_links", @@ -69,13 +73,12 @@ "hide_footer_in_auto_email_reports", "attach_view_link", "prepared_report_section", - "enable_prepared_report_auto_deletion", - "prepared_report_expiry_period", - "column_break_64", "max_auto_email_report_per_user", "system_updates_section", "disable_system_update_notification", - "disable_change_log_notification" + "disable_change_log_notification", + "telemetry_section", + "enable_telemetry" ], "fields": [ { @@ -207,18 +210,11 @@ "label": "Security" }, { - "default": "06:00", - "description": "Session Expiry in Hours e.g. 06:00", + "default": "60:00", + "description": "Example: Setting this to 24:00 will log out a user if they are not active for 24:00 hours.", "fieldname": "session_expiry", "fieldtype": "Data", - "label": "Session Expiry" - }, - { - "default": "720:00", - "description": "In Hours", - "fieldname": "session_expiry_mobile", - "fieldtype": "Data", - "label": "Session Expiry Mobile" + "label": "Session Expiry (idle timeout)" }, { "fieldname": "column_break_13", @@ -425,20 +421,6 @@ "fieldtype": "Check", "label": "Send document Web View link in email" }, - { - "default": "30", - "depends_on": "enable_prepared_report_auto_deletion", - "description": "System will auto-delete Prepared Reports permanently after these many days since creation", - "fieldname": "prepared_report_expiry_period", - "fieldtype": "Int", - "label": "Prepared Report Expiry Period (Days)" - }, - { - "default": "1", - "fieldname": "enable_prepared_report_auto_deletion", - "fieldtype": "Check", - "label": "Enable Auto-deletion of Prepared Reports" - }, { "collapsible": 1, "fieldname": "prepared_report_section", @@ -497,10 +479,6 @@ "fieldtype": "Check", "label": "Allow Older Web View Links (Insecure)" }, - { - "fieldname": "column_break_64", - "fieldtype": "Column Break" - }, { "default": "20", "fieldname": "max_auto_email_report_per_user", @@ -525,12 +503,58 @@ "fieldname": "email_retry_limit", "fieldtype": "Int", "label": "Email Retry Limit" + }, + { + "default": "0", + "description": "Make sure to configure a Social Login Key before disabling to prevent lockout", + "fieldname": "disable_user_pass_login", + "fieldtype": "Check", + "label": "Disable Username/Password Login" + }, + { + "default": "1", + "description": "Allow users to log in without a password, using a login link sent to their email", + "fieldname": "login_with_email_link", + "fieldtype": "Check", + "label": "Login with email link" + }, + { + "default": "10", + "depends_on": "login_with_email_link", + "fieldname": "login_with_email_link_expiry", + "fieldtype": "Int", + "label": "Login with email link expiry (in minutes)" + }, + { + "default": "Banker's Rounding (legacy)", + "fieldname": "rounding_method", + "fieldtype": "Select", + "label": "Rounding Method", + "options": "Banker's Rounding (legacy)\nBanker's Rounding\nCommercial Rounding" + }, + { + "default": "0", + "fieldname": "disable_document_sharing", + "fieldtype": "Check", + "label": "Disable Document Sharing" + }, + { + "collapsible": 1, + "fieldname": "telemetry_section", + "fieldtype": "Section Break", + "label": "Telemetry" + }, + { + "default": "1", + "fieldname": "enable_telemetry", + "fieldtype": "Check", + "label": "Allow Sending Usage Data for Improving Applications" } ], "icon": "fa fa-cog", "issingle": 1, "links": [], - "modified": "2022-06-21 13:55:04.796152", + "modified": "2023-04-23 11:14:59.302851", "modified_by": "Administrator", "module": "Core", "name": "System Settings", diff --git a/frappe/core/doctype/system_settings/system_settings.py b/frappe/core/doctype/system_settings/system_settings.py index e4d36b7fc7..c4f35f3cc0 100644 --- a/frappe/core/doctype/system_settings/system_settings.py +++ b/frappe/core/doctype/system_settings/system_settings.py @@ -20,15 +20,14 @@ class SystemSettings(Document): elif not enable_password_policy: self.minimum_password_score = "" - for key in ("session_expiry", "session_expiry_mobile"): - if self.get(key): - parts = self.get(key).split(":") - if len(parts) != 2 or not (cint(parts[0]) or cint(parts[1])): - frappe.throw(_("Session Expiry must be in format {0}").format("hh:mm")) + if self.session_expiry: + parts = self.session_expiry.split(":") + if len(parts) != 2 or not (cint(parts[0]) or cint(parts[1])): + frappe.throw(_("Session Expiry must be in format {0}").format("hh:mm")) if self.enable_two_factor_auth: if self.two_factor_method == "SMS": - if not frappe.db.get_value("SMS Settings", None, "sms_gateway_url"): + if not frappe.db.get_single_value("SMS Settings", "sms_gateway_url"): frappe.throw( _("Please setup SMS before setting it as an authentication method, via SMS Settings") ) @@ -43,13 +42,27 @@ class SystemSettings(Document): ): frappe.flags.update_last_reset_password_date = True - def on_update(self): - for df in self.meta.get("fields"): - if df.fieldtype not in no_value_fields and self.has_value_changed(df.fieldname): - frappe.db.set_default(df.fieldname, self.get(df.fieldname)) + self.validate_user_pass_login() - if self.language: - set_default_language(self.language) + def validate_user_pass_login(self): + if not self.disable_user_pass_login: + return + + social_login_enabled = frappe.db.exists("Social Login Key", {"enable_social_login": 1}) + ldap_enabled = frappe.db.get_single_value("LDAP Settings", "enabled") + login_with_email_link_enabled = frappe.db.get_single_value( + "System Settings", "login_with_email_link" + ) + + if not (social_login_enabled or ldap_enabled or login_with_email_link_enabled): + frappe.throw( + _( + "Please enable atleast one Social Login Key or LDAP or Login With Email Link before disabling username/password based login." + ) + ) + + def on_update(self): + self.set_defaults() frappe.cache().delete_value("system_settings") frappe.cache().delete_value("time_zone") @@ -57,6 +70,14 @@ class SystemSettings(Document): if frappe.flags.update_last_reset_password_date: update_last_reset_password_date() + def set_defaults(self): + for df in self.meta.get("fields"): + if df.fieldtype not in no_value_fields and self.has_value_changed(df.fieldname): + frappe.db.set_default(df.fieldname, self.get(df.fieldname)) + + if self.language: + set_default_language(self.language) + def update_last_reset_password_date(): frappe.db.sql( diff --git a/frappe/core/doctype/system_settings/test_system_settings.py b/frappe/core/doctype/system_settings/test_system_settings.py index b126976eeb..a876e8301d 100644 --- a/frappe/core/doctype/system_settings/test_system_settings.py +++ b/frappe/core/doctype/system_settings/test_system_settings.py @@ -1,7 +1,7 @@ # Copyright (c) 2017, Frappe Technologies and Contributors # License: MIT. See LICENSE -import unittest +from frappe.tests.utils import FrappeTestCase -class TestSystemSettings(unittest.TestCase): +class TestSystemSettings(FrappeTestCase): pass diff --git a/frappe/core/doctype/test/test.js b/frappe/core/doctype/test/test.js deleted file mode 100644 index e423c58686..0000000000 --- a/frappe/core/doctype/test/test.js +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright (c) 2021, Frappe Technologies and contributors -// For license information, please see license.txt - -frappe.ui.form.on('test', { - // refresh: function(frm) { - - // } -}); diff --git a/frappe/core/doctype/test/test.json b/frappe/core/doctype/test/test.json deleted file mode 100644 index 31a57c9964..0000000000 --- a/frappe/core/doctype/test/test.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "actions": [], - "creation": "2021-03-31 10:06:57.919697", - "doctype": "DocType", - "editable_grid": 1, - "engine": "InnoDB", - "field_order": [ - "test" - ], - "fields": [ - { - "fieldname": "test", - "fieldtype": "Data", - "label": "Test" - } - ], - "index_web_pages_for_search": 1, - "is_virtual": 1, - "links": [], - "modified": "2021-03-31 10:06:57.919697", - "modified_by": "Administrator", - "module": "Core", - "name": "test", - "owner": "Administrator", - "permissions": [ - { - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "share": 1, - "write": 1 - } - ], - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1 -} \ No newline at end of file diff --git a/frappe/core/doctype/test/test.py b/frappe/core/doctype/test/test.py deleted file mode 100644 index 664d06ac84..0000000000 --- a/frappe/core/doctype/test/test.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright (c) 2021, Frappe Technologies and contributors -# License: MIT. See LICENSE - -import json - -# import frappe -from frappe.model.document import Document - - -class test(Document): - def db_insert(self): - d = self.get_valid_dict(convert_dates_to_str=True) - with open("data_file.json", "w+") as read_file: - json.dump(d, read_file) - - def load_from_db(self): - with open("data_file.json") as read_file: - d = json.load(read_file) - super(Document, self).__init__(d) - - def db_update(self): - d = self.get_valid_dict(convert_dates_to_str=True) - with open("data_file.json", "w+") as read_file: - json.dump(d, read_file) - - def get_list(self, args): - with open("data_file.json") as read_file: - return [json.load(read_file)] - - def get_value(self, fields, filters, **kwargs): - # return [] - with open("data_file.json") as read_file: - return [json.load(read_file)] - - def get_count(self, args): - # return [] - with open("data_file.json") as read_file: - return [json.load(read_file)] - - def get_stats(self, args): - # return [] - with open("data_file.json") as read_file: - return [json.load(read_file)] diff --git a/frappe/core/doctype/test/test_test.py b/frappe/core/doctype/test/test_test.py deleted file mode 100644 index 6080c200c1..0000000000 --- a/frappe/core/doctype/test/test_test.py +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright (c) 2021, Frappe Technologies and Contributors -# License: MIT. See LICENSE -# import frappe -import unittest - - -class Testtest(unittest.TestCase): - pass diff --git a/frappe/core/doctype/transaction_log/test_transaction_log.py b/frappe/core/doctype/transaction_log/test_transaction_log.py index 8b179f8d85..0a76e5ac65 100644 --- a/frappe/core/doctype/transaction_log/test_transaction_log.py +++ b/frappe/core/doctype/transaction_log/test_transaction_log.py @@ -1,14 +1,14 @@ # Copyright (c) 2018, Frappe Technologies and Contributors # License: MIT. See LICENSE import hashlib -import unittest import frappe +from frappe.tests.utils import FrappeTestCase test_records = [] -class TestTransactionLog(unittest.TestCase): +class TestTransactionLog(FrappeTestCase): def test_validate_chaining(self): frappe.get_doc( { diff --git a/frappe/core/doctype/transaction_log/transaction_log.js b/frappe/core/doctype/transaction_log/transaction_log.js index 569cd9bf61..8f22b859f7 100644 --- a/frappe/core/doctype/transaction_log/transaction_log.js +++ b/frappe/core/doctype/transaction_log/transaction_log.js @@ -1,6 +1,4 @@ // Copyright (c) 2018, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Transaction Log', { - -}); +frappe.ui.form.on("Transaction Log", {}); diff --git a/frappe/core/doctype/transaction_log/transaction_log.json b/frappe/core/doctype/transaction_log/transaction_log.json index 5c6aa5bc8b..2135976add 100644 --- a/frappe/core/doctype/transaction_log/transaction_log.json +++ b/frappe/core/doctype/transaction_log/transaction_log.json @@ -1,476 +1,124 @@ { - "allow_copy": 0, - "allow_events_in_timeline": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, + "actions": [], "creation": "2018-02-06 11:48:51.270524", - "custom": 0, - "docstatus": 0, "doctype": "DocType", - "document_type": "", "editable_grid": 1, "engine": "InnoDB", + "field_order": [ + "row_index", + "section_break_2", + "reference_doctype", + "document_name", + "column_break_5", + "timestamp", + "checksum_version", + "section_break_8", + "previous_hash", + "transaction_hash", + "chaining_hash", + "data", + "amended_from" + ], "fields": [ { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "row_index", "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, "label": "Row Index", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "read_only": 1 }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "section_break_2", - "fieldtype": "Section Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "fieldtype": "Section Break" }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "reference_doctype", "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, "label": "Reference Document Type", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "read_only": 1 }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "document_name", "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, "label": "Document Name", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "read_only": 1 }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "column_break_5", - "fieldtype": "Column Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "fieldtype": "Column Break" }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "timestamp", "fieldtype": "Datetime", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, "label": "Timestamp", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "read_only": 1 }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "checksum_version", "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, "label": "Checksum Version", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "read_only": 1 }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "section_break_8", - "fieldtype": "Section Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "fieldtype": "Section Break" }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "previous_hash", "fieldtype": "Small Text", "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, "label": "Previous Hash", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "read_only": 1 }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "transaction_hash", "fieldtype": "Small Text", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, "label": "Transaction Hash", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "read_only": 1 }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "chaining_hash", "fieldtype": "Small Text", "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, "label": "Chaining Hash", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "read_only": 1 }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "data", "fieldtype": "Long Text", "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, "label": "Data", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "read_only": 1 }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "amended_from", "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, "label": "Amended From", - "length": 0, "no_copy": 1, "options": "Transaction Log", - "permlevel": 0, "print_hide": 1, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "read_only": 1 } ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, "in_create": 1, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2019-09-05 14:22:27.664645", + "links": [], + "modified": "2022-08-03 12:20:54.684305", "modified_by": "Administrator", "module": "Core", "name": "Transaction Log", - "name_case": "", "owner": "Administrator", "permissions": [ { - "amend": 0, - "cancel": 0, - "create": 0, - "delete": 0, "email": 1, "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, "print": 1, "read": 1, "report": 1, "role": "Administrator", - "set_user_permissions": 0, - "share": 1, - "submit": 0, - "write": 0 + "share": 1 } ], - "quick_entry": 0, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, "sort_field": "modified", "sort_order": "DESC", - "track_changes": 1, - "track_seen": 0, - "track_views": 0 + "states": [], + "track_changes": 1 } \ No newline at end of file diff --git a/frappe/core/doctype/translation/test_translation.py b/frappe/core/doctype/translation/test_translation.py index c9f4e85086..a64715a32e 100644 --- a/frappe/core/doctype/translation/test_translation.py +++ b/frappe/core/doctype/translation/test_translation.py @@ -1,30 +1,28 @@ # Copyright (c) 2015, Frappe Technologies and Contributors # License: MIT. See LICENSE -import unittest - import frappe from frappe import _ +from frappe.tests.utils import FrappeTestCase +from frappe.translate import clear_cache -class TestTranslation(unittest.TestCase): +class TestTranslation(FrappeTestCase): def setUp(self): frappe.db.delete("Translation") def tearDown(self): frappe.local.lang = "en" - frappe.local.lang_full_dict = None + clear_cache() def test_doctype(self): translation_data = get_translation_data() for key, val in translation_data.items(): frappe.local.lang = key - frappe.local.lang_full_dict = None + translation = create_translation(key, val) self.assertEqual(_(val[0]), val[1]) frappe.delete_doc("Translation", translation.name) - frappe.local.lang_full_dict = None - self.assertEqual(_(val[0]), val[0]) def test_parent_language(self): @@ -39,22 +37,22 @@ class TestTranslation(unittest.TestCase): frappe.local.lang = "es" - frappe.local.lang_full_dict = None self.assertTrue(_(data[0][0]), data[0][1]) - frappe.local.lang_full_dict = None self.assertTrue(_(data[1][0]), data[1][1]) frappe.local.lang = "es-MX" # different translation for es-MX - frappe.local.lang_full_dict = None self.assertTrue(_(data[2][0]), data[2][1]) # from spanish (general) - frappe.local.lang_full_dict = None self.assertTrue(_(data[1][0]), data[1][1]) + def test_multi_language_translations(self): + source = "User" + self.assertNotEqual(_(source, lang="de"), _(source, lang="es")) + def test_html_content_data_translation(self): source = """ ') - .appendTo(frm.fields_dict.roles_html.wrapper); + const role_area = $('
    ').appendTo( + frm.fields_dict.roles_html.wrapper + ); - frm.roles_editor = new frappe.RoleEditor(role_area, frm, frm.doc.role_profile_name ? 1 : 0); + frm.roles_editor = new frappe.RoleEditor( + role_area, + frm, + frm.doc.role_profile_name ? 1 : 0 + ); - if (frm.doc.user_type == 'System User') { - var module_area = $('
    ') - .appendTo(frm.fields_dict.modules_html.wrapper); + if (frm.doc.user_type == "System User") { + var module_area = $("
    ").appendTo(frm.fields_dict.modules_html.wrapper); frm.module_editor = new frappe.ModuleEditor(frm, module_area); } } else { @@ -80,111 +97,147 @@ frappe.ui.form.on('User', { } } }, - refresh: function(frm) { + refresh: function (frm) { let doc = frm.doc; if (frm.is_new()) { frm.set_value("time_zone", frappe.sys_defaults.time_zone); } - if (in_list(['System User', 'Website User'], frm.doc.user_type) - && !frm.is_new() && !frm.roles_editor && frm.can_edit_roles) { + if ( + in_list(["System User", "Website User"], frm.doc.user_type) && + !frm.is_new() && + !frm.roles_editor && + frm.can_edit_roles + ) { frm.reload_doc(); return; } - if(doc.name===frappe.session.user && !doc.__unsaved - && frappe.all_timezones - && (doc.language || frappe.boot.user.language) - && doc.language !== frappe.boot.user.language) { + const hasChanged = (doc_attr, boot_attr) => { + return doc_attr && boot_attr && doc_attr !== boot_attr; + }; + + if ( + doc.name === frappe.session.user && + !doc.__unsaved && + frappe.all_timezones && + (hasChanged(doc.language, frappe.boot.user.language) || + hasChanged(doc.time_zone, frappe.boot.time_zone.user)) + ) { frappe.msgprint(__("Refreshing...")); window.location.reload(); } - frm.toggle_display(['sb1', 'sb3', 'modules_access'], false); + frm.toggle_display(["sb1", "sb3", "modules_access"], false); - if(!frm.is_new()) { - if(has_access_to_edit_user()) { + if (!frm.is_new()) { + if (has_access_to_edit_user()) { + frm.add_custom_button( + __("Set User Permissions"), + function () { + frappe.route_options = { + user: doc.name, + }; + frappe.set_route("List", "User Permission"); + }, + __("Permissions") + ); - frm.add_custom_button(__("Set User Permissions"), function() { - frappe.route_options = { - "user": doc.name - }; - frappe.set_route('List', 'User Permission'); - }, __("Permissions")); + frm.add_custom_button( + __("View Permitted Documents"), + () => + frappe.set_route("query-report", "Permitted Documents For User", { + user: frm.doc.name, + }), + __("Permissions") + ); - frm.add_custom_button(__('View Permitted Documents'), - () => frappe.set_route('query-report', 'Permitted Documents For User', - {user: frm.doc.name}), __("Permissions")); - - frm.toggle_display(['sb1', 'sb3', 'modules_access'], true); + frm.toggle_display(["sb1", "sb3", "modules_access"], true); } - frm.add_custom_button(__("Reset Password"), function() { - frappe.call({ - method: "frappe.core.doctype.user.user.reset_password", - args: { - "user": frm.doc.name - } - }); - }, __("Password")); + frm.add_custom_button( + __("Reset Password"), + function () { + frappe.call({ + method: "frappe.core.doctype.user.user.reset_password", + args: { + user: frm.doc.name, + }, + }); + }, + __("Password") + ); if (frappe.user.has_role("System Manager")) { frappe.db.get_single_value("LDAP Settings", "enabled").then((value) => { if (value === 1 && frm.doc.name != "Administrator") { - frm.add_custom_button(__("Reset LDAP Password"), function() { - const d = new frappe.ui.Dialog({ - title: __("Reset LDAP Password"), - fields: [ - { - label: __("New Password"), - fieldtype: "Password", - fieldname: "new_password", - reqd: 1 + frm.add_custom_button( + __("Reset LDAP Password"), + function () { + const d = new frappe.ui.Dialog({ + title: __("Reset LDAP Password"), + fields: [ + { + label: __("New Password"), + fieldtype: "Password", + fieldname: "new_password", + reqd: 1, + }, + { + label: __("Confirm New Password"), + fieldtype: "Password", + fieldname: "confirm_password", + reqd: 1, + }, + { + label: __("Logout All Sessions"), + fieldtype: "Check", + fieldname: "logout_sessions", + }, + ], + primary_action: (values) => { + d.hide(); + if (values.new_password !== values.confirm_password) { + frappe.throw(__("Passwords do not match!")); + } + frappe.call( + "frappe.integrations.doctype.ldap_settings.ldap_settings.reset_password", + { + user: frm.doc.email, + password: values.new_password, + logout: values.logout_sessions, + } + ); }, - { - label: __("Confirm New Password"), - fieldtype: "Password", - fieldname: "confirm_password", - reqd: 1 - }, - { - label: __("Logout All Sessions"), - fieldtype: "Check", - fieldname: "logout_sessions" - } - ], - primary_action: (values) => { - d.hide(); - if (values.new_password !== values.confirm_password) { - frappe.throw(__("Passwords do not match!")); - } - frappe.call( - "frappe.integrations.doctype.ldap_settings.ldap_settings.reset_password", { - user: frm.doc.email, - password: values.new_password, - logout: values.logout_sessions - }); - } - }); - d.show(); - }, __("Password")); + }); + d.show(); + }, + __("Password") + ); } }); } - if (frappe.session.user == doc.name || frappe.user.has_role("System Manager")) { - frm.add_custom_button(__("Reset OTP Secret"), function() { - frappe.call({ - method: "frappe.twofactor.reset_otp_secret", - args: { - "user": frm.doc.name - } - }); - }, __("Password")); + if ( + cint(frappe.boot.sysdefaults.enable_two_factor_auth) && + (frappe.session.user == doc.name || frappe.user.has_role("System Manager")) + ) { + frm.add_custom_button( + __("Reset OTP Secret"), + function () { + frappe.call({ + method: "frappe.twofactor.reset_otp_secret", + args: { + user: frm.doc.name, + }, + }); + }, + __("Password") + ); } - frm.trigger('enabled'); + frm.trigger("enabled"); if (frm.roles_editor && frm.can_edit_roles) { frm.roles_editor.disable = frm.doc.role_profile_name ? 1 : 0; @@ -193,10 +246,12 @@ frappe.ui.form.on('User', { frm.module_editor && frm.module_editor.show(); - if(frappe.session.user==doc.name) { + if (frappe.session.user == doc.name) { // update display settings - if(doc.user_image) { - frappe.boot.user_info[frappe.session.user].image = frappe.utils.get_file_link(doc.user_image); + if (doc.user_image) { + frappe.boot.user_info[frappe.session.user].image = frappe.utils.get_file_link( + doc.user_image + ); } } } @@ -208,51 +263,51 @@ frappe.ui.form.on('User', { } } if (!found) { - frm.add_custom_button(__("Create User Email"), function() { + frm.add_custom_button(__("Create User Email"), function () { frm.events.create_user_email(frm); }); } } - if (frappe.route_flags.unsaved===1){ + if (frappe.route_flags.unsaved === 1) { delete frappe.route_flags.unsaved; - for ( var i=0;i { + child_row.used_oauth = value.auth_method === "OAuth"; + frm.refresh_field("user_emails", cdn, "used_oauth"); + } + ); + }, }); function has_access_to_edit_user() { @@ -292,7 +362,10 @@ function has_access_to_edit_user() { } function get_roles_for_editing_user() { - return frappe.get_meta('User').permissions - .filter(perm => perm.permlevel >= 1 && perm.write) - .map(perm => perm.role) || ['System Manager']; + return ( + frappe + .get_meta("User") + .permissions.filter((perm) => perm.permlevel >= 1 && perm.write) + .map((perm) => perm.role) || ["System Manager"] + ); } diff --git a/frappe/core/doctype/user/user.json b/frappe/core/doctype/user/user.json index 82e3fa71f3..00e1cffa88 100644 --- a/frappe/core/doctype/user/user.json +++ b/frappe/core/doctype/user/user.json @@ -7,6 +7,7 @@ "doctype": "DocType", "engine": "InnoDB", "field_order": [ + "user_details_tab", "enabled", "section_break_3", "email", @@ -22,23 +23,31 @@ "send_welcome_email", "unsubscribed", "user_image", + "roles_permissions_tab", "sb1", "role_profile_name", "roles_html", "roles", + "sb_allow_modules", + "module_profile", + "modules_html", + "block_modules", + "home_settings", "short_bio", "gender", "birth_date", "interest", - "banner_image", - "desk_theme", "column_break_26", "phone", "location", "bio", - "mute_sounds", "column_break_22", "mobile_no", + "settings_tab", + "desk_settings_section", + "mute_sounds", + "desk_theme", + "banner_image", "change_password", "new_password", "logout_all_sessions", @@ -61,11 +70,6 @@ "send_me_a_copy", "allowed_in_mentions", "user_emails", - "sb_allow_modules", - "module_profile", - "modules_html", - "block_modules", - "home_settings", "sb2", "defaults", "sb3", @@ -87,7 +91,8 @@ "api_key", "generate_keys", "column_break_65", - "api_secret" + "api_secret", + "connections_tab" ], "fields": [ { @@ -126,7 +131,7 @@ { "fieldname": "middle_name", "fieldtype": "Data", - "label": "Middle Name (Optional)", + "label": "Middle Name", "oldfieldname": "middle_name", "oldfieldtype": "Data" }, @@ -232,7 +237,7 @@ "collapsible": 1, "depends_on": "enabled", "fieldname": "short_bio", - "fieldtype": "Section Break", + "fieldtype": "Tab Break", "label": "More Information" }, { @@ -398,7 +403,6 @@ "permlevel": 1 }, { - "collapsible": 1, "depends_on": "eval:in_list(['System User'], doc.user_type)", "fieldname": "sb_allow_modules", "fieldtype": "Section Break", @@ -492,7 +496,7 @@ { "description": "Restrict user from this IP address only. Multiple IP addresses can be added by separating with commas. Also accepts partial IP addresses like (111.111.111)", "fieldname": "restrict_ip", - "fieldtype": "Data", + "fieldtype": "Small Text", "label": "Restrict IP", "permlevel": 1 }, @@ -615,13 +619,13 @@ "options": "Module Profile" }, { - "description": "Stores the datetime when the last reset password key was generated.", - "fieldname": "last_reset_password_key_generated_on", - "fieldtype": "Datetime", - "hidden": 1, - "label": "Last Reset Password Key Generated On", - "read_only": 1 - }, + "description": "Stores the datetime when the last reset password key was generated.", + "fieldname": "last_reset_password_key_generated_on", + "fieldtype": "Datetime", + "hidden": 1, + "label": "Last Reset Password Key Generated On", + "read_only": 1 + }, { "fieldname": "column_break_75", "fieldtype": "Column Break" @@ -648,18 +652,45 @@ "label": "Auto follow documents that you Like" }, { - "default": "0", - "depends_on": "eval:(doc.document_follow_notify== 1)", - "fieldname": "follow_shared_documents", - "fieldtype": "Check", - "label": "Auto follow documents that are shared with you" - }, + "default": "0", + "depends_on": "eval:(doc.document_follow_notify== 1)", + "fieldname": "follow_shared_documents", + "fieldtype": "Check", + "label": "Auto follow documents that are shared with you" + }, { "default": "0", "depends_on": "eval:(doc.document_follow_notify== 1)", "fieldname": "follow_assigned_documents", "fieldtype": "Check", "label": "Auto follow documents that are assigned to you" + }, + { + "fieldname": "user_details_tab", + "fieldtype": "Tab Break", + "label": "User Details" + }, + { + "fieldname": "roles_permissions_tab", + "fieldtype": "Tab Break", + "label": "Roles & Permissions" + }, + { + "fieldname": "settings_tab", + "fieldtype": "Tab Break", + "label": "Settings" + }, + { + "fieldname": "connections_tab", + "fieldtype": "Tab Break", + "label": "Connections", + "show_dashboard": 1 + }, + { + "collapsible": 1, + "fieldname": "desk_settings_section", + "fieldtype": "Section Break", + "label": "Desk Settings" } ], "icon": "fa fa-user", @@ -722,7 +753,7 @@ "link_fieldname": "user" } ], - "modified": "2022-05-25 01:00:51.345319", + "modified": "2022-09-19 16:05:46.485242", "modified_by": "Administrator", "module": "Core", "name": "User", @@ -738,7 +769,6 @@ "read": 1, "report": 1, "role": "System Manager", - "set_user_permissions": 1, "share": 1, "write": 1 }, diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py index 12a48afe7e..14266e4cd8 100644 --- a/frappe/core/doctype/user/user.py +++ b/frappe/core/doctype/user/user.py @@ -1,14 +1,13 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE from datetime import timedelta - -from bs4 import BeautifulSoup +from typing import Optional, Sequence import frappe import frappe.defaults import frappe.permissions import frappe.share -from frappe import _, msgprint, throw +from frappe import STANDARD_USERS, _, msgprint, throw from frappe.core.doctype.user_type.user_type import user_linked_with_permission_on_doctype from frappe.desk.doctype.notification_settings.notification_settings import ( create_notification_settings, @@ -24,7 +23,7 @@ from frappe.utils import ( flt, format_datetime, get_formatted_email, - get_time_zone, + get_system_timezone, has_gravatar, now_datetime, today, @@ -34,8 +33,6 @@ from frappe.utils.password import update_password as _update_password from frappe.utils.user import get_system_managers from frappe.website.utils import is_signup_disabled -STANDARD_USERS = frappe.STANDARD_USERS - class User(Document): __new_password = None @@ -55,7 +52,7 @@ class User(Document): def onload(self): from frappe.config import get_modules_from_all_apps - self.set_onload("all_modules", [m.get("module_name") for m in get_modules_from_all_apps()]) + self.set_onload("all_modules", sorted(m.get("module_name") for m in get_modules_from_all_apps())) def before_insert(self): self.flags.in_insert = True @@ -125,10 +122,20 @@ class User(Document): now = frappe.flags.in_test or frappe.flags.in_install self.send_password_notification(self.__new_password) frappe.enqueue( - "frappe.core.doctype.user.user.create_contact", user=self, ignore_mandatory=True, now=now + "frappe.core.doctype.user.user.create_contact", + user=self, + ignore_mandatory=True, + now=now, + enqueue_after_commit=True, ) - if self.name not in ("Administrator", "Guest") and not self.user_image: - frappe.enqueue("frappe.core.doctype.user.user.update_gravatar", name=self.name, now=now) + + if self.name not in STANDARD_USERS and not self.user_image: + frappe.enqueue( + "frappe.core.doctype.user.user.update_gravatar", + name=self.name, + now=now, + enqueue_after_commit=True, + ) # Set user selected timezone if self.time_zone: @@ -239,7 +246,10 @@ class User(Document): ) def share_with_self(self): - frappe.share.add( + if self.name in STANDARD_USERS: + return + + frappe.share.add_docshare( self.doctype, self.name, self.name, write=1, share=1, flags={"ignore_share_permission": True} ) @@ -304,12 +314,10 @@ class User(Document): .from_(user_role_doctype) .select(user_doctype.name) .where(user_role_doctype.role == "System Manager") - .where(user_doctype.docstatus < 2) .where(user_doctype.enabled == 1) .where(user_role_doctype.parent == user_doctype.name) .where(user_role_doctype.parent.notin(["Administrator", self.name])) .limit(1) - .distinct() ).run() def get_fullname(self): @@ -473,7 +481,7 @@ class User(Document): frappe.rename_doc("Notification Settings", old_name, new_name, force=True, show_alert=False) # set email - frappe.db.update("User", new_name, "email", new_name) + frappe.db.set_value("User", new_name, "email", new_name) def append_roles(self, *roles): """Add roles to user""" @@ -538,11 +546,11 @@ class User(Document): if self.__new_password: user_data = (self.first_name, self.middle_name, self.last_name, self.email, self.birth_date) - result = test_password_strength(self.__new_password, "", None, user_data) + result = test_password_strength(self.__new_password, user_data=user_data) feedback = result.get("feedback", None) if feedback and not feedback.get("password_policy_validation_passed", False): - handle_password_test_fail(result) + handle_password_test_fail(feedback) def suggest_username(self): def _check_suggestion(suggestion): @@ -581,7 +589,7 @@ class User(Document): if len(email_accounts) != len(set(email_accounts)): frappe.throw(_("Email Account added multiple times")) - def get_social_login_userid(self, provider): + def get_social_login_userid(self, provider: str): try: for p in self.social_logins: if p.provider == provider: @@ -611,10 +619,10 @@ class User(Document): """ login_with_mobile = cint( - frappe.db.get_value("System Settings", "System Settings", "allow_login_using_mobile_number") + frappe.db.get_single_value("System Settings", "allow_login_using_mobile_number") ) login_with_username = cint( - frappe.db.get_value("System Settings", "System Settings", "allow_login_using_user_name") + frappe.db.get_single_value("System Settings", "allow_login_using_user_name") ) or_filters = [{"name": user_name}] @@ -623,7 +631,7 @@ class User(Document): if login_with_username: or_filters.append({"username": user_name}) - users = frappe.db.get_all("User", fields=["name", "enabled"], or_filters=or_filters, limit=1) + users = frappe.get_all("User", fields=["name", "enabled"], or_filters=or_filters, limit=1) if not users: return @@ -639,7 +647,7 @@ class User(Document): def set_time_zone(self): if not self.time_zone: - self.time_zone = get_time_zone() + self.time_zone = get_system_timezone() @frappe.whitelist() @@ -679,16 +687,23 @@ def get_perm_info(role): @frappe.whitelist(allow_guest=True) -def update_password(new_password, logout_all_sessions=0, key=None, old_password=None): - # validate key to avoid key input like ['like', '%'], '', ['in', ['']] - if key and not isinstance(key, str): - frappe.throw(_("Invalid key type")) +def update_password( + new_password: str, logout_all_sessions: int = 0, key: str = None, old_password: str = None +): + """Update password for the current user. - result = test_password_strength(new_password, key, old_password) + Args: + new_password (str): New password. + logout_all_sessions (int, optional): If set to 1, all other sessions will be logged out. Defaults to 0. + key (str, optional): Password reset key. Defaults to None. + old_password (str, optional): Old password. Defaults to None. + """ + + result = test_password_strength(new_password) feedback = result.get("feedback", None) if feedback and not feedback.get("password_policy_validation_passed", False): - handle_password_test_fail(result) + handle_password_test_fail(feedback) res = _get_user_for_update_password(key, old_password) if res.get("message"): @@ -718,22 +733,22 @@ def update_password(new_password, logout_all_sessions=0, key=None, old_password= if user_doc.user_type == "System User": return "/app" else: - return redirect_url if redirect_url else "/" + return redirect_url or "/" @frappe.whitelist(allow_guest=True) -def test_password_strength(new_password, key=None, old_password=None, user_data=None): +def test_password_strength( + new_password: str, key=None, old_password=None, user_data: tuple | None = None +): + from frappe.utils.deprecations import deprecation_warning from frappe.utils.password_strength import test_password_strength as _test_password_strength - password_policy = ( - frappe.db.get_value( - "System Settings", None, ["enable_password_policy", "minimum_password_score"], as_dict=True + if key is not None or old_password is not None: + deprecation_warning( + "Arguments `key` and `old_password` are deprecated in function `test_password_strength`." ) - or {} - ) - enable_password_policy = cint(password_policy.get("enable_password_policy", 0)) - minimum_password_score = cint(password_policy.get("minimum_password_score", 0)) + enable_password_policy = frappe.get_system_settings("enable_password_policy") or 0 if not enable_password_policy: return {} @@ -746,6 +761,7 @@ def test_password_strength(new_password, key=None, old_password=None, user_data= if new_password: result = _test_password_strength(new_password, user_inputs=user_data) password_policy_validation_passed = False + minimum_password_score = cint(frappe.get_system_settings("minimum_password_score")) or 0 # score should be greater than 0 and minimum_password_score if result.get("score") and result.get("score") >= minimum_password_score: @@ -755,9 +771,8 @@ def test_password_strength(new_password, key=None, old_password=None, user_data= return result -# for login @frappe.whitelist() -def has_email_account(email): +def has_email_account(email: str): return frappe.get_list("Email Account", filters={"email_id": email}) @@ -824,7 +839,7 @@ def verify_password(password): @frappe.whitelist(allow_guest=True) -def sign_up(email, full_name, redirect_to): +def sign_up(email: str, full_name: str, redirect_to: str) -> tuple[int, str]: if is_signup_disabled(): frappe.throw(_("Sign Up is disabled"), title=_("Not Allowed")) @@ -861,7 +876,7 @@ def sign_up(email, full_name, redirect_to): user.insert() # set default signup role as per Portal Settings - default_role = frappe.db.get_value("Portal Settings", None, "default_role") + default_role = frappe.db.get_single_value("Portal Settings", "default_role") if default_role: user.add_roles(default_role) @@ -876,12 +891,12 @@ def sign_up(email, full_name, redirect_to): @frappe.whitelist(allow_guest=True) @rate_limit(limit=get_password_reset_limit, seconds=24 * 60 * 60, methods=["POST"]) -def reset_password(user): +def reset_password(user: str) -> str: if user == "Administrator": return "not allowed" try: - user = frappe.get_doc("User", user) + user: User = frappe.get_doc("User", user) if not user.enabled: return "disabled" @@ -893,7 +908,7 @@ def reset_password(user): title=_("Password Email Sent"), ) except frappe.DoesNotExistError: - frappe.local.response["http_status_code"] = 400 + frappe.local.response["http_status_code"] = 404 frappe.clear_messages() return "not found" @@ -903,6 +918,7 @@ def reset_password(user): def user_query(doctype, txt, searchfield, start, page_len, filters): from frappe.desk.reportview import get_filters_cond, get_match_cond + doctype = "User" conditions = [] user_type_condition = "and user_type != 'Website User'" @@ -1044,31 +1060,15 @@ def notify_admin_access_to_system_manager(login_manager=None): ) -def extract_mentions(txt): - """Find all instances of @mentions in the html.""" - soup = BeautifulSoup(txt, "html.parser") - emails = [] - for mention in soup.find_all(class_="mention"): - if mention.get("data-is-group") == "true": - try: - user_group = frappe.get_cached_doc("User Group", mention["data-id"]) - emails += [d.user for d in user_group.user_group_members] - except frappe.DoesNotExistError: - pass - continue - email = mention["data-id"] - emails.append(email) +def handle_password_test_fail(feedback: dict): + # Backward compatibility + if "feedback" in feedback: + feedback = feedback["feedback"] - return emails + suggestions = feedback.get("suggestions", []) + warning = feedback.get("warning", "") - -def handle_password_test_fail(result): - suggestions = result["feedback"]["suggestions"][0] if result["feedback"]["suggestions"] else "" - warning = result["feedback"]["warning"] if "warning" in result["feedback"] else "" - suggestions += ( - "
    " + _("Hint: Include symbols, numbers and capital letters in the password") + "
    " - ) - frappe.throw(" ".join([_("Invalid Password:"), warning, suggestions])) + frappe.throw(msg=" ".join([warning] + suggestions), title=_("Invalid Password")) def update_gravatar(name): @@ -1086,13 +1086,12 @@ def throttle_user_creation(): @frappe.whitelist() -def get_role_profile(role_profile): - roles = frappe.get_doc("Role Profile", {"role_profile": role_profile}) - return roles.roles +def get_role_profile(role_profile: str): + return frappe.get_doc("Role Profile", {"role_profile": role_profile}).roles @frappe.whitelist() -def get_module_profile(module_profile): +def get_module_profile(module_profile: str): module_profile = frappe.get_doc("Module Profile", {"module_profile_name": module_profile}) return module_profile.get("block_modules") @@ -1165,14 +1164,14 @@ def get_restricted_ip_list(user): @frappe.whitelist() -def generate_keys(user): +def generate_keys(user: str): """ generate api key and api secret :param user: str """ frappe.only_for("System Manager") - user_details = frappe.get_doc("User", user) + user_details: User = frappe.get_doc("User", user) api_secret = frappe.generate_hash(length=15) # if api key is not set generate api key if not user_details.api_key: diff --git a/frappe/core/doctype/user/user_list.js b/frappe/core/doctype/user/user_list.js index 5632edf0cc..334ed0b370 100644 --- a/frappe/core/doctype/user/user_list.js +++ b/frappe/core/doctype/user/user_list.js @@ -1,19 +1,19 @@ // Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors // MIT License. See license.txt -frappe.listview_settings['User'] = { +frappe.listview_settings["User"] = { add_fields: ["enabled", "user_type", "user_image"], - filters: [["enabled","=",1]], - prepare_data: function(data) { + filters: [["enabled", "=", 1]], + prepare_data: function (data) { data["user_for_avatar"] = data["name"]; }, - get_indicator: function(doc) { - if(doc.enabled) { + get_indicator: function (doc) { + if (doc.enabled) { return [__("Active"), "green", "enabled,=,1"]; } else { return [__("Disabled"), "grey", "enabled,=,0"]; } - } + }, }; frappe.help.youtube_id["User"] = "8Slw1hsTmUI"; diff --git a/frappe/core/doctype/user_group/test_user_group.py b/frappe/core/doctype/user_group/test_user_group.py index 368f4eaef2..79e013672d 100644 --- a/frappe/core/doctype/user_group/test_user_group.py +++ b/frappe/core/doctype/user_group/test_user_group.py @@ -1,8 +1,8 @@ # Copyright (c) 2021, Frappe Technologies and Contributors # License: MIT. See LICENSE # import frappe -import unittest +from frappe.tests.utils import FrappeTestCase -class TestUserGroup(unittest.TestCase): +class TestUserGroup(FrappeTestCase): pass diff --git a/frappe/core/doctype/user_group/user_group.js b/frappe/core/doctype/user_group/user_group.js index 2aa9b68658..cab1f5dff1 100644 --- a/frappe/core/doctype/user_group/user_group.js +++ b/frappe/core/doctype/user_group/user_group.js @@ -1,8 +1,7 @@ // Copyright (c) 2021, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('User Group', { +frappe.ui.form.on("User Group", { // refresh: function(frm) { - // } }); diff --git a/frappe/core/doctype/user_group_member/test_user_group_member.py b/frappe/core/doctype/user_group_member/test_user_group_member.py index 5d709d0bec..a2eb5c7bfc 100644 --- a/frappe/core/doctype/user_group_member/test_user_group_member.py +++ b/frappe/core/doctype/user_group_member/test_user_group_member.py @@ -1,8 +1,8 @@ # Copyright (c) 2021, Frappe Technologies and Contributors # License: MIT. See LICENSE # import frappe -import unittest +from frappe.tests.utils import FrappeTestCase -class TestUserGroupMember(unittest.TestCase): +class TestUserGroupMember(FrappeTestCase): pass diff --git a/frappe/core/doctype/user_group_member/user_group_member.js b/frappe/core/doctype/user_group_member/user_group_member.js index 0b2dbe0d46..4c4011c8b4 100644 --- a/frappe/core/doctype/user_group_member/user_group_member.js +++ b/frappe/core/doctype/user_group_member/user_group_member.js @@ -1,8 +1,7 @@ // Copyright (c) 2021, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('User Group Member', { +frappe.ui.form.on("User Group Member", { // refresh: function(frm) { - // } }); diff --git a/frappe/core/doctype/user_permission/test_user_permission.py b/frappe/core/doctype/user_permission/test_user_permission.py index b963da2f49..8742d2e040 100644 --- a/frappe/core/doctype/user_permission/test_user_permission.py +++ b/frappe/core/doctype/user_permission/test_user_permission.py @@ -1,7 +1,5 @@ # Copyright (c) 2021, Frappe Technologies and Contributors # See LICENSE -import unittest - import frappe from frappe.core.doctype.doctype.test_doctype import new_doctype from frappe.core.doctype.user_permission.user_permission import ( @@ -9,10 +7,11 @@ from frappe.core.doctype.user_permission.user_permission import ( remove_applicable, ) from frappe.permissions import has_user_permission +from frappe.tests.utils import FrappeTestCase from frappe.website.doctype.blog_post.test_blog_post import make_test_blog -class TestUserPermission(unittest.TestCase): +class TestUserPermission(FrappeTestCase): def setUp(self): test_users = ( "test_bulk_creation_update@example.com", @@ -278,7 +277,7 @@ def create_user(email, *roles): user = frappe.new_doc("User") user.email = email - user.first_name = email.split("@")[0] + user.first_name = email.split("@", 1)[0] if not roles: roles = ("System Manager",) diff --git a/frappe/core/doctype/user_permission/user_permission.js b/frappe/core/doctype/user_permission/user_permission.js index f6989db5d8..39ee4348b9 100644 --- a/frappe/core/doctype/user_permission/user_permission.js +++ b/frappe/core/doctype/user_permission/user_permission.js @@ -1,59 +1,58 @@ // Copyright (c) 2017, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('User Permission', { - setup: frm => { +frappe.ui.form.on("User Permission", { + setup: (frm) => { frm.set_query("allow", () => { return { - "filters": { + filters: { issingle: 0, - istable: 0 - } + istable: 0, + }, }; }); - frm.set_query('applicable_for', () => { + frm.set_query("applicable_for", () => { return { - 'query': 'frappe.core.doctype.user_permission.user_permission.get_applicable_for_doctype_list', - 'doctype': frm.doc.allow + query: "frappe.core.doctype.user_permission.user_permission.get_applicable_for_doctype_list", + doctype: frm.doc.allow, }; }); - }, - refresh: frm => { - frm.add_custom_button(__('View Permitted Documents'), - () => frappe.set_route('query-report', 'Permitted Documents For User', - { user: frm.doc.user })); - frm.trigger('set_applicable_for_constraint'); - frm.trigger('toggle_hide_descendants'); + refresh: (frm) => { + frm.add_custom_button(__("View Permitted Documents"), () => + frappe.set_route("query-report", "Permitted Documents For User", { + user: frm.doc.user, + }) + ); + frm.trigger("set_applicable_for_constraint"); + frm.trigger("toggle_hide_descendants"); }, - allow: frm => { + allow: (frm) => { if (frm.doc.allow) { if (frm.doc.for_value) { - frm.set_value('for_value', null); + frm.set_value("for_value", null); } - frm.trigger('toggle_hide_descendants'); + frm.trigger("toggle_hide_descendants"); } }, - apply_to_all_doctypes: frm => { - frm.trigger('set_applicable_for_constraint'); + apply_to_all_doctypes: (frm) => { + frm.trigger("set_applicable_for_constraint"); }, - set_applicable_for_constraint: frm => { - frm.toggle_reqd('applicable_for', !frm.doc.apply_to_all_doctypes); + set_applicable_for_constraint: (frm) => { + frm.toggle_reqd("applicable_for", !frm.doc.apply_to_all_doctypes); if (frm.doc.apply_to_all_doctypes && frm.doc.applicable_for) { - frm.set_value('applicable_for', null, null, true); + frm.set_value("applicable_for", null, null, true); } }, - toggle_hide_descendants: frm => { + toggle_hide_descendants: (frm) => { let show = frappe.boot.nested_set_doctypes.includes(frm.doc.allow); - frm.toggle_display('hide_descendants', show); - } - - + frm.toggle_display("hide_descendants", show); + }, }); diff --git a/frappe/core/doctype/user_permission/user_permission.py b/frappe/core/doctype/user_permission/user_permission.py index 2dfd7863b1..63c1f40512 100644 --- a/frappe/core/doctype/user_permission/user_permission.py +++ b/frappe/core/doctype/user_permission/user_permission.py @@ -18,16 +18,16 @@ class UserPermission(Document): def on_update(self): frappe.cache().hdel("user_permissions", self.user) - frappe.publish_realtime("update_user_permissions") + frappe.publish_realtime("update_user_permissions", user=self.user, after_commit=True) - def on_trash(self): # pylint: disable=no-self-use + def on_trash(self): frappe.cache().hdel("user_permissions", self.user) - frappe.publish_realtime("update_user_permissions") + frappe.publish_realtime("update_user_permissions", user=self.user, after_commit=True) def validate_user_permission(self): """checks for duplicate user permission records""" - duplicate_exists = frappe.db.get_all( + duplicate_exists = frappe.get_all( self.doctype, filters={ "allow": self.allow, diff --git a/frappe/core/doctype/user_permission/user_permission_list.js b/frappe/core/doctype/user_permission/user_permission_list.js index 0ce66fa8e3..ce5e624403 100644 --- a/frappe/core/doctype/user_permission/user_permission_list.js +++ b/frappe/core/doctype/user_permission/user_permission_list.js @@ -1,18 +1,17 @@ -frappe.listview_settings['User Permission'] = { - - onload: function(list_view) { +frappe.listview_settings["User Permission"] = { + onload: function (list_view) { var me = this; - list_view.page.add_inner_button( __("Add / Update"), function() { - let dialog =new frappe.ui.Dialog({ - title : __('Add User Permissions'), + list_view.page.add_inner_button(__("Add / Update"), function () { + let dialog = new frappe.ui.Dialog({ + title: __("Add User Permissions"), fields: [ { - fieldname: 'user', - label: __('For User'), - fieldtype: 'Link', - options: 'User', + fieldname: "user", + label: __("For User"), + fieldtype: "Link", + options: "User", reqd: 1, - onchange: function() { + onchange: function () { dialog.fields_dict.doctype.set_input(undefined); dialog.fields_dict.docname.set_input(undefined); dialog.set_df_property("docname", "hidden", 1); @@ -20,77 +19,87 @@ frappe.listview_settings['User Permission'] = { dialog.set_df_property("apply_to_all_doctypes", "hidden", 1); dialog.set_df_property("applicable_doctypes", "hidden", 1); dialog.set_df_property("hide_descendants", "hidden", 1); - } + }, }, { - fieldname: 'doctype', - label: __('Document Type'), - fieldtype: 'Link', - options: 'DocType', + fieldname: "doctype", + label: __("Document Type"), + fieldtype: "Link", + options: "DocType", reqd: 1, - onchange: function() { + onchange: function () { me.on_doctype_change(dialog); - } + }, }, { - fieldname: 'docname', - label: __('Document Name'), - fieldtype: 'Dynamic Link', - options: 'doctype', + fieldname: "docname", + label: __("Document Name"), + fieldtype: "Dynamic Link", + options: "doctype", hidden: 1, - onchange: function() { + onchange: function () { let field = dialog.fields_dict["docname"]; - if(field.value != field.last_value) { - if(dialog.fields_dict.doctype.value && dialog.fields_dict.docname.value && dialog.fields_dict.user.value){ - me.get_applicable_doctype(dialog).then(applicable => { - me.get_multi_select_options(dialog, applicable).then(options => { - me.applicable_options = options; - me.on_docname_change(dialog, options, applicable); - if(options.length > 5){ - dialog.fields_dict.applicable_doctypes.setup_select_all(); + if (field.value != field.last_value) { + if ( + dialog.fields_dict.doctype.value && + dialog.fields_dict.docname.value && + dialog.fields_dict.user.value + ) { + me.get_applicable_doctype(dialog).then((applicable) => { + me.get_multi_select_options(dialog, applicable).then( + (options) => { + me.applicable_options = options; + me.on_docname_change(dialog, options, applicable); + if (options.length > 5) { + dialog.fields_dict.applicable_doctypes.setup_select_all(); + } } - }); + ); }); } } - } + }, }, { fieldtype: "Section Break", - hide_border: 1 + hide_border: 1, }, { - fieldname: 'is_default', - label: __('Is Default'), - fieldtype: 'Check', - hidden: 1 - }, - { - fieldname: 'apply_to_all_doctypes', - label: __('Apply to all Documents Types'), - fieldtype: 'Check', + fieldname: "is_default", + label: __("Is Default"), + fieldtype: "Check", hidden: 1, - onchange: function() { - if(dialog.fields_dict.doctype.value && dialog.fields_dict.docname.value && dialog.fields_dict.user.value){ + }, + { + fieldname: "apply_to_all_doctypes", + label: __("Apply to all Documents Types"), + fieldtype: "Check", + hidden: 1, + onchange: function () { + if ( + dialog.fields_dict.doctype.value && + dialog.fields_dict.docname.value && + dialog.fields_dict.user.value + ) { me.on_apply_to_all_doctypes_change(dialog, me.applicable_options); - if(me.applicable_options.length > 5){ + if (me.applicable_options.length > 5) { dialog.fields_dict.applicable_doctypes.setup_select_all(); } } - } + }, }, { - fieldtype: "Column Break" + fieldtype: "Column Break", }, { - fieldname: 'hide_descendants', - label: __('Hide Descendants'), - fieldtype: 'Check', - hidden: 1 + fieldname: "hide_descendants", + label: __("Hide Descendants"), + fieldtype: "Check", + hidden: 1, }, { fieldtype: "Section Break", - hide_border: 1 + hide_border: 1, }, { label: __("Applicable Document Types"), @@ -98,7 +107,7 @@ frappe.listview_settings['User Permission'] = { fieldtype: "MultiCheck", options: [], columns: 2, - hidden: 1 + hidden: 1, }, ], primary_action: (data) => { @@ -107,126 +116,137 @@ frappe.listview_settings['User Permission'] = { async: false, method: "frappe.core.doctype.user_permission.user_permission.add_user_permissions", args: { - data : data + data: data, }, - callback: function(r) { - if(r.message === 1) { - frappe.show_alert({message:__("User Permissions created sucessfully"), indicator:'blue'}); + callback: function (r) { + if (r.message === 1) { + frappe.show_alert({ + message: __("User Permissions created sucessfully"), + indicator: "blue", + }); } else { - frappe.show_alert({message:__("Nothing to update"), indicator:'red'}); - + frappe.show_alert({ + message: __("Nothing to update"), + indicator: "red", + }); } - } + }, }); dialog.hide(); list_view.refresh(); }, - primary_action_label: __('Submit') + primary_action_label: __("Submit"), }); dialog.show(); }); - list_view.page.add_inner_button( __("Bulk Delete"), function() { + list_view.page.add_inner_button(__("Bulk Delete"), function () { const dialog = new frappe.ui.Dialog({ - title: __('Clear User Permissions'), + title: __("Clear User Permissions"), fields: [ { - fieldname: 'user', - label: __('For User'), - fieldtype: 'Link', - options: 'User', - reqd: 1 + fieldname: "user", + label: __("For User"), + fieldtype: "Link", + options: "User", + reqd: 1, }, { - fieldname: 'for_doctype', - label: __('For Document Type'), - fieldtype: 'Link', - options: 'DocType', - reqd: 1 + fieldname: "for_doctype", + label: __("For Document Type"), + fieldtype: "Link", + options: "DocType", + reqd: 1, }, ], primary_action: (data) => { // mandatory not filled if (!data) return; - frappe.confirm(__('Are you sure?'), () => { + frappe.confirm(__("Are you sure?"), () => { frappe - .xcall('frappe.core.doctype.user_permission.user_permission.clear_user_permissions', data) - .then(data => { + .xcall( + "frappe.core.doctype.user_permission.user_permission.clear_user_permissions", + data + ) + .then((data) => { dialog.hide(); - let message = ''; + let message = ""; if (data === 0) { - message = __('No records deleted'); - } else if(data === 1) { - message = __('{0} record deleted', [data]); + message = __("No records deleted"); + } else if (data === 1) { + message = __("{0} record deleted", [data]); } else { - message = __('{0} records deleted', [data]); + message = __("{0} records deleted", [data]); } frappe.show_alert({ message, - indicator: 'info' + indicator: "info", }); list_view.refresh(); }); }); - }, - primary_action_label: __('Delete') + primary_action_label: __("Delete"), }); dialog.show(); }); }, - validate: function(dialog, data) { - if(dialog.fields_dict.applicable_doctypes.get_unchecked_options().length == 0) { + validate: function (dialog, data) { + if (dialog.fields_dict.applicable_doctypes.get_unchecked_options().length == 0) { data.apply_to_all_doctypes = 1; data.applicable_doctypes = []; return data; } - if(data.apply_to_all_doctypes == 0 && !("applicable_doctypes" in data)) { + if (data.apply_to_all_doctypes == 0 && !("applicable_doctypes" in data)) { frappe.throw(__("Please select applicable Doctypes")); } return data; }, - get_applicable_doctype: function(dialog) { - return new Promise(resolve => { - frappe.call({ - method: 'frappe.core.doctype.user_permission.user_permission.check_applicable_doc_perm', - async: false, - args:{ - user: dialog.fields_dict.user.value, - doctype: dialog.fields_dict.doctype.value, - docname: dialog.fields_dict.docname.value - } - }).then(r => { - resolve(r.message); - }); + get_applicable_doctype: function (dialog) { + return new Promise((resolve) => { + frappe + .call({ + method: "frappe.core.doctype.user_permission.user_permission.check_applicable_doc_perm", + async: false, + args: { + user: dialog.fields_dict.user.value, + doctype: dialog.fields_dict.doctype.value, + docname: dialog.fields_dict.docname.value, + }, + }) + .then((r) => { + resolve(r.message); + }); }); }, - get_multi_select_options: function(dialog, applicable){ - return new Promise(resolve => { - frappe.call({ - method: 'frappe.desk.form.linked_with.get_linked_doctypes', - async: false, - args:{ - user: dialog.fields_dict.user.value, - doctype: dialog.fields_dict.doctype.value, - docname: dialog.fields_dict.docname.value - } - }).then(r => { - var options = []; - for(var d in r.message){ - var checked = ($.inArray(d, applicable) != -1) ? 1 : 0; - options.push({ "label":d, "value": d , "checked": checked}); - } - resolve(options); - }); + get_multi_select_options: function (dialog, applicable) { + return new Promise((resolve) => { + frappe + .call({ + method: "frappe.desk.form.linked_with.get_linked_doctypes", + async: false, + args: { + user: dialog.fields_dict.user.value, + doctype: dialog.fields_dict.doctype.value, + docname: dialog.fields_dict.docname.value, + }, + }) + .then((r) => { + var options = []; + for (var d in r.message) { + var checked = $.inArray(d, applicable) != -1 ? 1 : 0; + options.push({ label: d, value: d, checked: checked }); + } + resolve(options); + }); }); }, - on_doctype_change: function(dialog) { + on_doctype_change: function (dialog) { dialog.set_df_property("docname", "hidden", 0); dialog.set_df_property("docname", "reqd", 1); dialog.set_df_property("is_default", "hidden", 0); @@ -237,12 +257,15 @@ frappe.listview_settings['User Permission'] = { dialog.refresh(); }, - on_docname_change: function(dialog, options, applicable) { - if(applicable.length != 0 ) { + on_docname_change: function (dialog, options, applicable) { + if (applicable.length != 0) { dialog.set_primary_action("Update"); dialog.set_title("Update User Permissions"); dialog.set_df_property("applicable_doctypes", "options", options); - if(dialog.fields_dict.applicable_doctypes.get_checked_options().length == options.length) { + if ( + dialog.fields_dict.applicable_doctypes.get_checked_options().length == + options.length + ) { dialog.set_df_property("applicable_doctypes", "hidden", 1); } else { dialog.set_df_property("applicable_doctypes", "hidden", 0); @@ -257,8 +280,8 @@ frappe.listview_settings['User Permission'] = { dialog.refresh(); }, - on_apply_to_all_doctypes_change: function(dialog, options) { - if(dialog.fields_dict.apply_to_all_doctypes.get_value() == 0) { + on_apply_to_all_doctypes_change: function (dialog, options) { + if (dialog.fields_dict.apply_to_all_doctypes.get_value() == 0) { dialog.set_df_property("applicable_doctypes", "hidden", 0); dialog.set_df_property("applicable_doctypes", "options", options); } else { @@ -266,5 +289,5 @@ frappe.listview_settings['User Permission'] = { dialog.set_df_property("applicable_doctypes", "hidden", 1); } dialog.refresh_sections(); - } + }, }; diff --git a/frappe/core/doctype/user_social_login/user_social_login.json b/frappe/core/doctype/user_social_login/user_social_login.json index 3cac838016..6b4b1822d1 100644 --- a/frappe/core/doctype/user_social_login/user_social_login.json +++ b/frappe/core/doctype/user_social_login/user_social_login.json @@ -1,189 +1,58 @@ { - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, - "creation": "2017-12-02 13:01:20.507112", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", + "actions": [], + "creation": "2017-12-02 13:01:20.507112", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "provider", + "section_break_0", + "username", + "column_break_0", + "userid" + ], "fields": [ { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "provider", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Provider", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "provider", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Provider", + "read_only": 1 + }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "section_break_0", - "fieldtype": "Section Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "section_break_0", + "fieldtype": "Section Break" + }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "username", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Username", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "username", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Username", + "read_only": 1 + }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "column_break_0", - "fieldtype": "Column Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "column_break_0", + "fieldtype": "Column Break" + }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "userid", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "User ID", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "fieldname": "userid", + "fieldtype": "Data", + "in_list_view": 1, + "label": "User ID", + "read_only": 1 } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 1, - "max_attachments": 0, - "modified": "2017-12-02 15:37:58.397062", - "modified_by": "Administrator", - "module": "Core", - "name": "User Social Login", - "name_case": "", - "owner": "Administrator", - "permissions": [], - "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1, - "track_seen": 0 + ], + "istable": 1, + "links": [], + "modified": "2022-08-03 12:20:53.800689", + "modified_by": "Administrator", + "module": "Core", + "name": "User Social Login", + "owner": "Administrator", + "permissions": [], + "quick_entry": 1, + "sort_field": "modified", + "sort_order": "DESC", + "states": [], + "track_changes": 1 } \ No newline at end of file diff --git a/frappe/core/doctype/user_type/test_user_type.py b/frappe/core/doctype/user_type/test_user_type.py index 235881517a..ec1a5e3bfc 100644 --- a/frappe/core/doctype/user_type/test_user_type.py +++ b/frappe/core/doctype/user_type/test_user_type.py @@ -1,12 +1,11 @@ # Copyright (c) 2021, Frappe Technologies and Contributors # License: MIT. See LICENSE -import unittest - import frappe from frappe.installer import update_site_config +from frappe.tests.utils import FrappeTestCase -class TestUserType(unittest.TestCase): +class TestUserType(FrappeTestCase): def setUp(self): create_role() diff --git a/frappe/core/doctype/user_type/user_type.js b/frappe/core/doctype/user_type/user_type.js index 6b53248fd4..5cf0dbb25f 100644 --- a/frappe/core/doctype/user_type/user_type.js +++ b/frappe/core/doctype/user_type/user_type.js @@ -1,72 +1,71 @@ // Copyright (c) 2021, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('User Type', { - refresh: function(frm) { - if (frm.is_new() && !frappe.boot.developer_mode) - frm.set_value('is_standard', 1); +frappe.ui.form.on("User Type", { + refresh: function (frm) { + if (frm.is_new() && !frappe.boot.developer_mode) frm.set_value("is_standard", 1); - frm.set_query('document_type', 'user_doctypes', function() { + frm.set_query("document_type", "user_doctypes", function () { return { filters: { - istable: 0 - } + istable: 0, + }, }; }); - frm.set_query('document_type', 'select_doctypes', function() { + frm.set_query("document_type", "select_doctypes", function () { return { filters: { - istable: 0 - } + istable: 0, + }, }; }); - frm.set_query('document_type', 'custom_select_doctypes', function() { + frm.set_query("document_type", "custom_select_doctypes", function () { return { filters: { - istable: 0 - } + istable: 0, + }, }; }); - frm.set_query('role', function() { + frm.set_query("role", function () { return { filters: { is_custom: 1, disabled: 0, - desk_access: 1 - } + desk_access: 1, + }, }; }); - frm.set_query('apply_user_permission_on', function() { + frm.set_query("apply_user_permission_on", function () { return { - query: "frappe.core.doctype.user_type.user_type.get_user_linked_doctypes" + query: "frappe.core.doctype.user_type.user_type.get_user_linked_doctypes", }; }); }, - onload: function(frm) { - frm.trigger('get_user_id_fields'); + onload: function (frm) { + frm.trigger("get_user_id_fields"); }, - apply_user_permission_on: function(frm) { - frm.set_value('user_id_field', ''); - frm.trigger('get_user_id_fields'); + apply_user_permission_on: function (frm) { + frm.set_value("user_id_field", ""); + frm.trigger("get_user_id_fields"); }, - get_user_id_fields: function(frm) { + get_user_id_fields: function (frm) { if (frm.doc.apply_user_permission_on) { frappe.call({ - method: 'frappe.core.doctype.user_type.user_type.get_user_id', + method: "frappe.core.doctype.user_type.user_type.get_user_id", args: { - parent: frm.doc.apply_user_permission_on + parent: frm.doc.apply_user_permission_on, + }, + callback: function (r) { + set_field_options("user_id_field", [""].concat(r.message)); }, - callback: function(r) { - set_field_options('user_id_field', [""].concat(r.message)); - } }); } - } + }, }); diff --git a/frappe/core/doctype/user_type/user_type.py b/frappe/core/doctype/user_type/user_type.py index 369e70bf56..39d9133412 100644 --- a/frappe/core/doctype/user_type/user_type.py +++ b/frappe/core/doctype/user_type/user_type.py @@ -14,6 +14,12 @@ class UserType(Document): self.set_modules() self.add_select_perm_doctypes() + def clear_cache(self): + super().clear_cache() + + if not self.is_standard: + frappe.cache().delete_value("non_standard_user_types") + def on_update(self): if self.is_standard: return @@ -24,7 +30,6 @@ class UserType(Document): self.add_role_permissions_for_select_doctypes() self.add_role_permissions_for_file() self.update_users() - get_non_standard_user_type_details() self.remove_permission_for_deleted_doctypes() def on_trash(self): @@ -184,19 +189,14 @@ def add_role_permissions(doctype, role): return name -def get_non_standard_user_type_details(): +def get_non_standard_user_types(): user_types = frappe.get_all( "User Type", fields=["apply_user_permission_on", "name", "user_id_field"], filters={"is_standard": 0}, ) - if user_types: - user_type_details = {d.name: [d.apply_user_permission_on, d.user_id_field] for d in user_types} - - frappe.cache().set_value("non_standard_user_types", user_type_details) - - return user_type_details + return {d.name: [d.apply_user_permission_on, d.user_id_field] for d in user_types} @frappe.whitelist() @@ -287,13 +287,13 @@ def user_linked_with_permission_on_doctype(doc, user): def apply_permissions_for_non_standard_user_type(doc, method=None): """Create user permission for the non standard user type""" - if not frappe.db.table_exists("User Type"): + if not frappe.db.table_exists("User Type") or frappe.flags.in_migrate: return - user_types = frappe.cache().get_value("non_standard_user_types") - - if not user_types: - user_types = get_non_standard_user_type_details() + user_types = frappe.cache().get_value( + "non_standard_user_types", + get_non_standard_user_types, + ) if not user_types: return diff --git a/frappe/core/doctype/user_type/user_type_list.js b/frappe/core/doctype/user_type/user_type_list.js index 9a9ef417ac..856fe8985e 100644 --- a/frappe/core/doctype/user_type/user_type_list.js +++ b/frappe/core/doctype/user_type/user_type_list.js @@ -1,4 +1,4 @@ -frappe.listview_settings['User Type'] = { +frappe.listview_settings["User Type"] = { add_fields: ["is_standard"], get_indicator: function (doc) { if (doc.is_standard) { @@ -6,5 +6,5 @@ frappe.listview_settings['User Type'] = { } else { return [__("Custom"), "blue", "is_standard,=,0"]; } - } + }, }; diff --git a/frappe/core/doctype/version/test_version.py b/frappe/core/doctype/version/test_version.py index 3e82f30f06..ce8e0e8b89 100644 --- a/frappe/core/doctype/version/test_version.py +++ b/frappe/core/doctype/version/test_version.py @@ -1,14 +1,14 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE import copy -import unittest import frappe from frappe.core.doctype.version.version import get_diff from frappe.test_runner import make_test_objects +from frappe.tests.utils import FrappeTestCase -class TestVersion(unittest.TestCase): +class TestVersion(FrappeTestCase): def test_get_diff(self): frappe.set_user("Administrator") test_records = make_test_objects("Event", reset=True) diff --git a/frappe/core/doctype/version/version.js b/frappe/core/doctype/version/version.js index d39d2eac03..1e26e5f748 100644 --- a/frappe/core/doctype/version/version.js +++ b/frappe/core/doctype/version/version.js @@ -1,9 +1,12 @@ -frappe.ui.form.on("Version", "refresh", function(frm) { - $(frappe.render_template('version_view', {doc:frm.doc, data:JSON.parse(frm.doc.data)})) - .appendTo(frm.fields_dict.table_html.$wrapper.empty()); +frappe.ui.form.on("Version", "refresh", function (frm) { + $( + frappe.render_template("version_view", { doc: frm.doc, data: JSON.parse(frm.doc.data) }) + ).appendTo(frm.fields_dict.table_html.$wrapper.empty()); - frm.add_custom_button(__('Show all Versions'), function() { - frappe.set_route('List', 'Version', - {ref_doctype: frm.doc.ref_doctype, docname: frm.doc.docname}); + frm.add_custom_button(__("Show all Versions"), function () { + frappe.set_route("List", "Version", { + ref_doctype: frm.doc.ref_doctype, + docname: frm.doc.docname, + }); }); }); diff --git a/frappe/core/doctype/version/version.json b/frappe/core/doctype/version/version.json index 463a7d3cba..13c82fa2b2 100644 --- a/frappe/core/doctype/version/version.json +++ b/frappe/core/doctype/version/version.json @@ -1,247 +1,81 @@ { - "allow_copy": 0, - "allow_import": 0, - "allow_rename": 0, - "autoname": "hash", - "beta": 0, - "creation": "2014-02-20 17:22:37", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "Setup", - "editable_grid": 0, - "engine": "InnoDB", + "actions": [], + "autoname": "hash", + "creation": "2014-02-20 17:22:37", + "doctype": "DocType", + "document_type": "Setup", + "engine": "InnoDB", + "field_order": [ + "ref_doctype", + "column_break_3", + "docname", + "data", + "section_break_4", + "table_html" + ], "fields": [ { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "ref_doctype", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 1, - "in_standard_filter": 1, - "label": "DocType", - "length": 0, - "no_copy": 0, - "options": "DocType", - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "ref_doctype", + "fieldtype": "Link", + "in_list_view": 1, + "in_standard_filter": 1, + "label": "DocType", + "options": "DocType", + "reqd": 1 + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "column_break_3", - "fieldtype": "Column Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "column_break_3", + "fieldtype": "Column Break" + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "docname", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Document Name", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "docname", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Document Name", + "reqd": 1 + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "data", - "fieldtype": "Code", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Data", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "data", + "fieldtype": "Code", + "hidden": 1, + "label": "Data" + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "section_break_4", - "fieldtype": "Section Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "section_break_4", + "fieldtype": "Section Break" + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "table_html", - "fieldtype": "HTML", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Table HTML", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "fieldname": "table_html", + "fieldtype": "HTML", + "label": "Table HTML" } - ], - "hide_heading": 0, - "hide_toolbar": 0, - "icon": "fa fa-copy", - "idx": 1, - "image_view": 0, - "in_create": 1, - - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2018-04-10 14:39:45.926836", - "modified_by": "Administrator", - "module": "Core", - "name": "Version", - "owner": "Administrator", + ], + "icon": "fa fa-copy", + "idx": 1, + "in_create": 1, + "links": [], + "modified": "2022-08-03 12:20:53.929691", + "modified_by": "Administrator", + "module": "Core", + "name": "Version", + "owner": "Administrator", "permissions": [ { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 0, - "delete": 0, - "email": 0, - "export": 1, - "if_owner": 0, - "import": 0, - "is_custom": 0, - "permlevel": 0, - "print": 0, - "read": 1, - "report": 1, - "role": "System Manager", - "set_user_permissions": 0, - "share": 0, - "submit": 0, - "write": 0 - }, + "export": 1, + "read": 1, + "report": 1, + "role": "System Manager" + }, { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 0, - "delete": 1, - "email": 0, - "export": 0, - "if_owner": 0, - "import": 0, - "is_custom": 0, - "permlevel": 0, - "print": 0, - "read": 1, - "report": 0, - "role": "Administrator", - "set_user_permissions": 0, - "share": 0, - "submit": 0, - "write": 0 + "delete": 1, + "read": 1, + "role": "Administrator" } - ], - "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "sort_order": "ASC", - "title_field": "docname", - "track_changes": 1, - "track_seen": 0 + ], + "quick_entry": 1, + "sort_field": "modified", + "sort_order": "ASC", + "states": [], + "title_field": "docname", + "track_changes": 1 } \ No newline at end of file diff --git a/frappe/core/doctype/version/version_view.html b/frappe/core/doctype/version/version_view.html index a17460ccc7..c6473b6a42 100644 --- a/frappe/core/doctype/version/version_view.html +++ b/frappe/core/doctype/version/version_view.html @@ -18,8 +18,8 @@ {% for item in data.changed %} {{ frappe.meta.get_label(doc.ref_doctype, item[0]) }} - {{ item[1] }} - {{ item[2] }} + {{ frappe.utils.escape_html(item[1]) }} + {{ frappe.utils.escape_html(item[2]) }} {% endfor %} @@ -50,7 +50,7 @@ {% for row_key in item_keys %} {{ row_key }} - {{ item[1][row_key] }} + {{ frappe.utils.escape_html(item[1][row_key]) }} {% endfor %} @@ -85,8 +85,8 @@ {{ frappe.meta.get_label(doc.ref_doctype, table_info[0]) }} {{ table_info[1] }} {{ item[0] }} - {{ item[1] }} - {{ item[2] }} + {{ frappe.utils.escape_html(item[1]) }} + {{ frappe.utils.escape_html(item[2]) }} {% endfor %} {% endfor %} diff --git a/frappe/core/doctype/view_log/test_view_log.py b/frappe/core/doctype/view_log/test_view_log.py index 5a88269028..d1596d84a4 100644 --- a/frappe/core/doctype/view_log/test_view_log.py +++ b/frappe/core/doctype/view_log/test_view_log.py @@ -1,11 +1,10 @@ # Copyright (c) 2018, Frappe Technologies and Contributors # License: MIT. See LICENSE -import unittest - import frappe +from frappe.tests.utils import FrappeTestCase -class TestViewLog(unittest.TestCase): +class TestViewLog(FrappeTestCase): def tearDown(self): frappe.set_user("Administrator") diff --git a/frappe/core/doctype/view_log/view_log.js b/frappe/core/doctype/view_log/view_log.js index a8c95b01e8..06d23802be 100644 --- a/frappe/core/doctype/view_log/view_log.js +++ b/frappe/core/doctype/view_log/view_log.js @@ -1,8 +1,6 @@ // Copyright (c) 2018, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('View Log', { - refresh: function(frm) { - - } +frappe.ui.form.on("View Log", { + refresh: function (frm) {}, }); diff --git a/frappe/core/doctype/view_log/view_log.json b/frappe/core/doctype/view_log/view_log.json index 3c4486c944..a350ae835c 100644 --- a/frappe/core/doctype/view_log/view_log.json +++ b/frappe/core/doctype/view_log/view_log.json @@ -1,163 +1,56 @@ { - "allow_copy": 0, - "allow_events_in_timeline": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, + "actions": [], "creation": "2018-05-27 02:20:11.193944", - "custom": 0, - "docstatus": 0, "doctype": "DocType", - "document_type": "", "editable_grid": 1, "engine": "InnoDB", + "field_order": [ + "viewed_by", + "reference_doctype", + "reference_name" + ], "fields": [ { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fetch_if_empty": 0, "fieldname": "viewed_by", "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, "label": "Viewed By", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 1, - "set_only_once": 1, - "translatable": 0, - "unique": 0 + "set_only_once": 1 }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fetch_if_empty": 0, "fieldname": "reference_doctype", "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, "label": "Reference Document Type", - "length": 0, - "no_copy": 0, "options": "DocType", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 1, - "set_only_once": 1, - "translatable": 0, - "unique": 0 + "set_only_once": 1 }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fetch_if_empty": 0, "fieldname": "reference_name", "fieldtype": "Dynamic Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, "label": "Reference name", - "length": 0, - "no_copy": 0, "options": "reference_doctype", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, "search_index": 1, - "set_only_once": 1, - "translatable": 0, - "unique": 0 + "set_only_once": 1 } ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2021-10-25 14:22:27.664645", + "links": [], + "modified": "2022-09-07 05:16:14.587628", "modified_by": "Administrator", "module": "Core", "name": "View Log", - "name_case": "", "owner": "Administrator", "permissions": [ { - "amend": 0, - "cancel": 0, - "create": 0, - "delete": 0, "email": 1, "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, "print": 1, "read": 1, "report": 1, "role": "System Manager", - "set_user_permissions": 0, - "share": 1, - "submit": 0, - "write": 0 + "share": 1 } ], "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, "sort_field": "modified", "sort_order": "DESC", - "track_seen": 0, - "track_views": 0 -} + "states": [] +} \ No newline at end of file diff --git a/frappe/core/notifications.py b/frappe/core/notifications.py index 9a127e567e..093418e345 100644 --- a/frappe/core/notifications.py +++ b/frappe/core/notifications.py @@ -32,11 +32,10 @@ def get_things_todo(as_list=False): if as_list: return data - else: - return data[0][0] + return data[0][0] -def get_todays_events(as_list=False): +def get_todays_events(as_list: bool = False): """Returns a count of todays events in calendar""" from frappe.desk.doctype.event.event import get_events from frappe.utils import nowdate diff --git a/frappe/core/page/background_jobs/background_jobs.css b/frappe/core/page/background_jobs/background_jobs.css deleted file mode 100644 index 7716519113..0000000000 --- a/frappe/core/page/background_jobs/background_jobs.css +++ /dev/null @@ -1,47 +0,0 @@ - -.table-background-jobs { - margin-bottom: 0px; - margin-top: 0px; - font-size: var(--text-md); - table-layout: fixed; -} - -.table-background-jobs th { - font-weight: normal; - color: var(--text-muted); -} - -.table-background-jobs td { - color: var(--text-light); -} - -.table-background-jobs th, .table-background-jobs td { - padding: var(--padding-sm) var(--padding-md); -} - -.table-background-jobs tbody tr:hover { - background-color: var(--highlight-color); -} - -.job-name { - font-size: var(--text-md); - font-family: var(--font-family-monospace); - word-break: break-word; -} - -.no-background-jobs { - min-height: 320px; - display: flex; - align-items: center; - justify-content: center; - flex-direction: column; -} - -.no-background-jobs > img { - margin-bottom: var(--margin-md); - max-height: 100px; -} - -.footer { - padding: var(--padding-md); -} diff --git a/frappe/core/page/background_jobs/background_jobs.html b/frappe/core/page/background_jobs/background_jobs.html deleted file mode 100644 index e0c1a8f633..0000000000 --- a/frappe/core/page/background_jobs/background_jobs.html +++ /dev/null @@ -1,58 +0,0 @@ -{% if jobs.length %} - - - - - - - - - - - {% for j in jobs %} - - - - - - - {% endfor %} - -
    {{ __("Queue") }}{{ __("Job") }}{{ __("Status") }}{{ __("Created") }}
    - {{ toTitle(j.queue.split(":").slice(-1)[0]) }} - -
    - - {{ frappe.utils.encode_tags(j.job_name) }} - -
    - {% if j.exc_info %} -
    - {{ __("Exception") }} -
    - 
    {{ frappe.utils.encode_tags(j.exc_info) }}
    -
    -
    - {% endif %} -     		- - {{ toTitle(j.status) }} - - - {{ frappe.datetime.prettyDate(j.creation) }} -
    -{% else %} -
    - Empty State -

    {{ __("No jobs found on this site") }}

    -
    -{% endif %} -
    -
    - {{ __("Last refreshed") }} - {{ frappe.datetime.now_datetime(true).toLocaleString() }} -
    -
    diff --git a/frappe/core/page/background_jobs/background_jobs.js b/frappe/core/page/background_jobs/background_jobs.js deleted file mode 100644 index 7334bfd5dd..0000000000 --- a/frappe/core/page/background_jobs/background_jobs.js +++ /dev/null @@ -1,149 +0,0 @@ -frappe.pages["background_jobs"].on_page_load = wrapper => { - const background_job = new BackgroundJobs(wrapper); - - $(wrapper).bind("show", () => { - background_job.show(); - }); - - window.background_jobs = background_job; -}; - -class BackgroundJobs { - constructor(wrapper) { - this.page = frappe.ui.make_app_page({ - parent: wrapper, - title: __("Background Jobs"), - single_column: true - }); - - this.page.add_inner_button(__("Remove Failed Jobs"), () => { - frappe.confirm( - __("Are you sure you want to remove all failed jobs?"), - () => { - frappe - .call( - "frappe.core.page.background_jobs.background_jobs.remove_failed_jobs" - ) - .then(() => this.refresh_jobs()); - } - ); - }); - - this.page.main.addClass("frappe-card"); - this.page.body.append('
    '); - this.$content = $(this.page.body).find(".table-area"); - - this.make_filters(); - this.refresh_jobs = frappe.utils.throttle( - this.refresh_jobs.bind(this), - 1000 - ); - } - - make_filters() { - this.view = this.page.add_field({ - label: __("View"), - fieldname: "view", - fieldtype: "Select", - options: ["Jobs", "Workers"], - default: "Jobs", - change: () => { - this.queue_timeout.toggle(this.view.get_value() === "Jobs"); - this.job_status.toggle(this.view.get_value() === "Jobs"); - } - }); - this.queue_timeout = this.page.add_field({ - label: __("Queue"), - fieldname: "queue_timeout", - fieldtype: "Select", - options: [ - { label: "All Queues", value: "all" }, - { label: "Default", value: "default" }, - { label: "Short", value: "short" }, - { label: "Long", value: "long" } - ], - default: "all" - }); - this.job_status = this.page.add_field({ - label: __("Job Status"), - fieldname: "job_status", - fieldtype: "Select", - options: [ - { label: "All Jobs", value: "all" }, - { label: "Queued", value: "queued" }, - { label: "Deferred", value: "deferred" }, - { label: "Started", value: "started" }, - { label: "Finished", value: "finished" }, - { label: "Failed", value: "failed" } - ], - default: "all" - }); - this.auto_refresh = this.page.add_field({ - label: __("Auto Refresh"), - fieldname: "auto_refresh", - fieldtype: "Check", - default: 1, - change: () => { - if (this.auto_refresh.get_value()) { - this.refresh_jobs(); - } - } - }); - } - - show() { - this.refresh_jobs(); - this.update_scheduler_status(); - } - - update_scheduler_status() { - frappe.call({ - method: - "frappe.core.page.background_jobs.background_jobs.get_scheduler_status", - callback: r => { - let { status } = r.message; - if (status === "active") { - this.page.set_indicator(__("Scheduler: Active"), "green"); - } else { - this.page.set_indicator(__("Scheduler: Inactive"), "red"); - } - } - }); - } - - refresh_jobs() { - let view = this.view.get_value(); - let args; - let { queue_timeout, job_status } = this.page.get_form_values(); - if (view === "Jobs") { - args = { view, queue_timeout, job_status }; - } else { - args = { view }; - } - - this.page.add_inner_message(__("Refreshing...")); - frappe.call({ - method: "frappe.core.page.background_jobs.background_jobs.get_info", - args, - callback: res => { - this.page.add_inner_message(""); - - let template = - view === "Jobs" ? "background_jobs" : "background_workers"; - this.$content.html( - frappe.render_template(template, { - jobs: res.message || [] - }) - ); - - let auto_refresh = this.auto_refresh.get_value(); - if ( - frappe.get_route()[0] === "background_jobs" && - auto_refresh - ) { - setTimeout(() => this.refresh_jobs(), 2000); - } - } - }); - } -} diff --git a/frappe/core/page/background_jobs/background_jobs.json b/frappe/core/page/background_jobs/background_jobs.json deleted file mode 100644 index 6701cc54bc..0000000000 --- a/frappe/core/page/background_jobs/background_jobs.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "content": null, - "creation": "2016-08-18 16:44:14.322642", - "docstatus": 0, - "doctype": "Page", - "idx": 0, - "modified": "2016-08-18 16:48:11.577611", - "modified_by": "Administrator", - "module": "Core", - "name": "background_jobs", - "owner": "Administrator", - "page_name": "background_jobs", - "roles": [ - { - "role": "System Manager" - } - ], - "script": null, - "standard": "Yes", - "style": null, - "title": "Background Jobs" -} \ No newline at end of file diff --git a/frappe/core/page/background_jobs/background_jobs.py b/frappe/core/page/background_jobs/background_jobs.py deleted file mode 100644 index 8ef15b65eb..0000000000 --- a/frappe/core/page/background_jobs/background_jobs.py +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors -# License: MIT. See LICENSE - -from typing import TYPE_CHECKING - -import frappe -from frappe.utils import convert_utc_to_user_timezone -from frappe.utils.background_jobs import get_queues, get_workers -from frappe.utils.scheduler import is_scheduler_inactive - -if TYPE_CHECKING: - from rq.job import Job - -JOB_COLORS = {"queued": "orange", "failed": "red", "started": "blue", "finished": "green"} - - -@frappe.whitelist() -def get_info(view=None, queue_timeout=None, job_status=None) -> list[dict]: - jobs = [] - - def add_job(job: "Job", queue: str) -> None: - - if job.kwargs.get("site") == frappe.local.site: - job_info = { - "job_name": job.kwargs.get("kwargs", {}).get("playbook_method") - or job.kwargs.get("kwargs", {}).get("job_type") - or str(job.kwargs.get("job_name")), - "status": job.get_status(), - "queue": queue, - "creation": convert_utc_to_user_timezone(job.created_at), - "color": JOB_COLORS[job.get_status()], - } - - if job.exc_info: - job_info["exc_info"] = job.exc_info - - jobs.append(job_info) - - if view == "Jobs": - queues = get_queues() - for queue in queues: - for job in queue.jobs: - if job_status != "all" and job.get_status() != job_status: - return - if queue_timeout != "all" and not queue.name.endswith(f":{queue_timeout}"): - return - add_job(job, queue.name) - - elif view == "Workers": - workers = get_workers() - for worker in workers: - current_job = worker.get_current_job() - if current_job: - if hasattr(current_job, "kwargs") and current_job.kwargs.get("site") == frappe.local.site: - add_job(current_job, current_job.origin) - else: - jobs.append({"queue": worker.name, "job_name": "busy", "status": "", "creation": ""}) - else: - jobs.append({"queue": worker.name, "job_name": "idle", "status": "", "creation": ""}) - - return jobs - - -@frappe.whitelist() -def remove_failed_jobs(): - queues = get_queues() - for queue in queues: - fail_registry = queue.failed_job_registry - for job_id in fail_registry.get_job_ids(): - job = queue.fetch_job(job_id) - fail_registry.remove(job, delete_job=True) - - -@frappe.whitelist() -def get_scheduler_status(): - if is_scheduler_inactive(): - return {"status": "inactive"} - return {"status": "active"} diff --git a/frappe/core/page/background_jobs/background_workers.html b/frappe/core/page/background_jobs/background_workers.html deleted file mode 100644 index 1647cea4b4..0000000000 --- a/frappe/core/page/background_jobs/background_workers.html +++ /dev/null @@ -1,51 +0,0 @@ -{% if jobs.length %} - - - - - - - - - - - {% for j in jobs %} - - - - - - - {% endfor %} - -
    {{ __("Worker") }}{{ __("Current Job") }}{{ __("Status") }}{{ __("Created") }}
    - {{ j.queue }} - -
    - - {{ frappe.utils.encode_tags(j.job_name) }} - -
    - {% if j.exc_info %} -
    - {{ __("Exception") }} -
    - 
    {{ frappe.utils.encode_tags(j.exc_info) }}
    -
    -
    - {% endif %} -     		- {{ toTitle(j.status) }} - {{ frappe.datetime.prettyDate(j.creation) }}
    -{% else %} -
    - Empty State -

    {{ __("No workers online on this site") }}

    -
    -{% endif %} -
    -
    - {{ __("Last refreshed") }} - {{ frappe.datetime.now_datetime(true).toLocaleString() }} -
    -
    \ No newline at end of file diff --git a/frappe/core/page/dashboard_view/dashboard_view.js b/frappe/core/page/dashboard_view/dashboard_view.js index bf9fb2a286..0ee697dfc1 100644 --- a/frappe/core/page/dashboard_view/dashboard_view.js +++ b/frappe/core/page/dashboard_view/dashboard_view.js @@ -1,19 +1,18 @@ // Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and Contributors // MIT License. See license.txt -frappe.provide('frappe.dashboards'); -frappe.provide('frappe.dashboards.chart_sources'); +frappe.provide("frappe.dashboards"); +frappe.provide("frappe.dashboards.chart_sources"); - -frappe.pages['dashboard-view'].on_page_load = function(wrapper) { +frappe.pages["dashboard-view"].on_page_load = function (wrapper) { frappe.ui.make_app_page({ parent: wrapper, title: __("Dashboard"), - single_column: true + single_column: true, }); frappe.dashboard = new Dashboard(wrapper); - $(wrapper).bind('show', function() { + $(wrapper).bind("show", function () { frappe.dashboard.show(); }); }; @@ -37,20 +36,20 @@ class Dashboard { } else { // last opened if (frappe.last_dashboard) { - frappe.set_re_route('dashboard-view', frappe.last_dashboard); + frappe.set_re_route("dashboard-view", frappe.last_dashboard); } else { // default dashboard - frappe.db.get_list('Dashboard', {filters: {is_default: 1}}).then(data => { + frappe.db.get_list("Dashboard", { filters: { is_default: 1 } }).then((data) => { if (data && data.length) { - frappe.set_re_route('dashboard-view', data[0].name); + frappe.set_re_route("dashboard-view", data[0].name); } else { // no default, get the latest one - frappe.db.get_list('Dashboard', {limit: 1}).then(data => { + frappe.db.get_list("Dashboard", { limit: 1 }).then((data) => { if (data && data.length) { - frappe.set_re_route('dashboard-view', data[0].name); + frappe.set_re_route("dashboard-view", data[0].name); } else { // create a new dashboard! - frappe.new_doc('Dashboard'); + frappe.new_doc("Dashboard"); } }); } @@ -63,11 +62,11 @@ class Dashboard { if (this.dashboard_name !== current_dashboard_name) { this.dashboard_name = current_dashboard_name; let title = this.dashboard_name; - if (!this.dashboard_name.toLowerCase().includes(__('dashboard'))) { + if (!this.dashboard_name.toLowerCase().includes(__("dashboard"))) { // ensure dashboard title has "dashboard" - title = __('{0} Dashboard', [title]); + title = __("{0} Dashboard", [__(title)]); } - this.page.set_title(title); + this.page.set_title(__(title)); this.set_dropdown(); this.container.empty(); this.refresh(); @@ -81,31 +80,30 @@ class Dashboard { } refresh() { - frappe.run_serially([ - () => this.render_cards(), - () => this.render_charts() - ]); + frappe.run_serially([() => this.render_cards(), () => this.render_charts()]); } render_charts() { return this.get_permitted_items( - 'frappe.desk.doctype.dashboard.dashboard.get_permitted_charts' - ).then(charts => { + "frappe.desk.doctype.dashboard.dashboard.get_permitted_charts" + ).then((charts) => { if (!charts.length) { - frappe.msgprint(__('No Permitted Charts on this Dashboard'), __('No Permitted Charts')) + frappe.msgprint( + __("No Permitted Charts on this Dashboard"), + __("No Permitted Charts") + ); } frappe.dashboard_utils.get_dashboard_settings().then((settings) => { - let chart_config = settings.chart_config? JSON.parse(settings.chart_config): {}; - this.charts = - charts.map(chart => { - return { - chart_name: chart.chart, - label: chart.chart, - chart_settings: chart_config[chart.chart] || {}, - ...chart - } - }); + let chart_config = settings.chart_config ? JSON.parse(settings.chart_config) : {}; + this.charts = charts.map((chart) => { + return { + chart_name: chart.chart, + label: chart.chart, + chart_settings: chart_config[chart.chart] || {}, + ...chart, + }; + }); this.chart_group = new frappe.widget.WidgetGroup({ title: null, @@ -121,24 +119,23 @@ class Dashboard { }, widgets: this.charts, }); - }) + }); }); } render_cards() { return this.get_permitted_items( - 'frappe.desk.doctype.dashboard.dashboard.get_permitted_cards' - ).then(cards => { + "frappe.desk.doctype.dashboard.dashboard.get_permitted_cards" + ).then((cards) => { if (!cards.length) { return; } - this.number_cards = - cards.map(card => { - return { - name: card.card, - }; - }); + this.number_cards = cards.map((card) => { + return { + name: card.card, + }; + }); this.number_card_group = new frappe.widget.WidgetGroup({ container: this.container, @@ -157,41 +154,43 @@ class Dashboard { } get_permitted_items(method) { - return frappe.xcall( - method, - { - dashboard_name: this.dashboard_name - } - ).then(items => { - return items; - }); + return frappe + .xcall(method, { + dashboard_name: this.dashboard_name, + }) + .then((items) => { + return items; + }); } set_dropdown() { this.page.clear_menu(); - this.page.add_menu_item(__('Edit'), () => { - frappe.set_route('Form', 'Dashboard', frappe.dashboard.dashboard_name); + this.page.add_menu_item(__("Edit"), () => { + frappe.set_route("Form", "Dashboard", frappe.dashboard.dashboard_name); }); - this.page.add_menu_item(__('New'), () => { - frappe.new_doc('Dashboard'); + this.page.add_menu_item(__("New"), () => { + frappe.new_doc("Dashboard"); }); - this.page.add_menu_item(__('Refresh All'), () => { - this.chart_group && - this.chart_group.widgets_list.forEach(chart => chart.refresh()); + this.page.add_menu_item(__("Refresh All"), () => { + this.chart_group && this.chart_group.widgets_list.forEach((chart) => chart.refresh()); this.number_card_group && - this.number_card_group.widgets_list.forEach(card => card.render_card()); + this.number_card_group.widgets_list.forEach((card) => card.render_card()); }); - frappe.db.get_list('Dashboard').then(dashboards => { - dashboards.map(dashboard => { + frappe.db.get_list("Dashboard").then((dashboards) => { + dashboards.map((dashboard) => { let name = dashboard.name; if (name != this.dashboard_name) { - this.page.add_menu_item(name, () => frappe.set_route("dashboard-view", name), 1); + this.page.add_menu_item( + name, + () => frappe.set_route("dashboard-view", name), + 1 + ); } }); }); } -} \ No newline at end of file +} diff --git a/frappe/core/page/permission_manager/permission_manager.js b/frappe/core/page/permission_manager/permission_manager.js index 8a06a9aac5..bc27106068 100644 --- a/frappe/core/page/permission_manager/permission_manager.js +++ b/frappe/core/page/permission_manager/permission_manager.js @@ -1,20 +1,21 @@ -frappe.pages['permission-manager'].on_page_load = (wrapper) => { +frappe.pages["permission-manager"].on_page_load = (wrapper) => { let page = frappe.ui.make_app_page({ parent: wrapper, - title: __('Role Permissions Manager'), + title: __("Role Permissions Manager"), card_layout: true, - single_column: true + single_column: true, }); frappe.breadcrumbs.add("Setup"); - $("
    ").appendTo(page.main); + $("
    ").appendTo( + page.main + ); $(frappe.render_template("permission_manager_help", {})).appendTo(page.main); wrapper.permission_engine = new frappe.PermissionEngine(wrapper); - }; -frappe.pages['permission-manager'].refresh = function (wrapper) { +frappe.pages["permission-manager"].refresh = function (wrapper) { wrapper.permission_engine.set_from_route(); }; @@ -30,33 +31,38 @@ frappe.PermissionEngine = class PermissionEngine { make() { this.make_reset_button(); - frappe.call({ - module: "frappe.core", - page: "permission_manager", - method: "get_roles_and_doctypes" - }).then((res) => { - this.options = res.message; - this.setup_page(); - }); + frappe + .call({ + module: "frappe.core", + page: "permission_manager", + method: "get_roles_and_doctypes", + }) + .then((res) => { + this.options = res.message; + this.setup_page(); + }); } setup_page() { - this.doctype_select - = this.wrapper.page.add_select(__("Document Type"), - [{ value: "", label: __("Select Document Type") + "..." }].concat(this.options.doctypes)) - .change(function () { - frappe.set_route("permission-manager", $(this).val()); - }); + this.doctype_select = this.wrapper.page + .add_select( + __("Document Type"), + [{ value: "", label: __("Select Document Type") + "..." }].concat( + this.options.doctypes + ) + ) + .change(function () { + frappe.set_route("permission-manager", $(this).val()); + }); - this.role_select - = this.wrapper.page.add_select(__("Roles"), - [__("Select Role") + "..."].concat(this.options.roles)) - .change(() => { - this.refresh(); - }); + this.role_select = this.wrapper.page + .add_select(__("Roles"), [__("Select Role") + "..."].concat(this.options.roles)) + .change(() => { + this.refresh(); + }); - this.page.add_inner_button(__('Set User Permissions'), () => { - return frappe.set_route('List', 'User Permission'); + this.page.add_inner_button(__("Set User Permissions"), () => { + return frappe.set_route("List", "User Permission"); }); this.set_from_route(); } @@ -91,7 +97,7 @@ frappe.PermissionEngine = class PermissionEngine { page: "permission_manager", method: "get_standard_permissions", args: { doctype: doctype }, - callback: callback + callback: callback, }); } return false; @@ -100,18 +106,22 @@ frappe.PermissionEngine = class PermissionEngine { reset_std_permissions(data) { let doctype = this.get_doctype(); let d = frappe.confirm(__("Reset Permissions for {0}?", [doctype]), () => { - return frappe.call({ - module: "frappe.core", - page: "permission_manager", - method: "reset", - args: { doctype } - }).then(() => { - this.refresh(); - }); + return frappe + .call({ + module: "frappe.core", + page: "permission_manager", + method: "reset", + args: { doctype }, + }) + .then(() => { + this.refresh(); + }); }); // show standard permissions - let $d = $(d.wrapper).find(".frappe-confirm-message").append("
    Standard Permissions:

    "); + let $d = $(d.wrapper) + .find(".frappe-confirm-message") + .append("
    Standard Permissions:

    "); let $wrapper = $("

    ").appendTo($d); data.message.forEach((d) => { let rights = this.rights @@ -164,14 +174,16 @@ frappe.PermissionEngine = class PermissionEngine { } // get permissions - frappe.call({ - module: "frappe.core", - page: "permission_manager", - method: "get_permissions", - args: { doctype, role } - }).then((r) => { - this.render(r.message); - }); + frappe + .call({ + module: "frappe.core", + page: "permission_manager", + method: "get_permissions", + args: { doctype, role }, + }) + .then((r) => { + this.render(r.message); + }); } render(perm_list) { @@ -187,19 +199,21 @@ frappe.PermissionEngine = class PermissionEngine { } show_permission_table(perm_list) { - this.table = $("
    \ + this.table = $( + "
    \ \ \ \
    \ -
    ").appendTo(this.body); +
    " + ).appendTo(this.body); const table_columns = [ [__("Document Type"), 150], [__("Role"), 170], [__("Level"), 40], [__("Permissions"), 350], - ["", 40] + ["", 40], ]; table_columns.forEach((col) => { @@ -236,9 +250,9 @@ frappe.PermissionEngine = class PermissionEngine { let perm_cell = this.add_cell(row, d, "permissions"); let perm_container = $("
    ").appendTo(perm_cell); - this.rights.forEach(r => { - if (!d.is_submittable && ['submit', 'cancel', 'amend'].includes(r)) return; - if (d.in_create && ['create', 'write', 'delete'].includes(r)) return; + this.rights.forEach((r) => { + if (!d.is_submittable && ["submit", "cancel", "amend"].includes(r)) return; + if (d.in_create && ["create", "write", "delete"].includes(r)) return; this.add_check(perm_container, d, r); }); @@ -248,7 +262,8 @@ frappe.PermissionEngine = class PermissionEngine { } add_cell(row, d, fieldname) { - return $("").appendTo(row) + return $("") + .appendTo(row) .attr("data-fieldname", fieldname) .addClass("pt-4") .html(__(d[fieldname])); @@ -266,19 +281,20 @@ frappe.PermissionEngine = class PermissionEngine {

    ${__(description)}

    -
    `) +
    ` + ) .appendTo(cell) .attr("data-fieldname", fieldname); - checkbox.find("input") + checkbox + .find("input") .prop("checked", d[fieldname] ? true : false) .attr("data-ptype", fieldname) .attr("data-role", d.role) .attr("data-permlevel", d.permlevel) .attr("data-doctype", d.parent); - checkbox.find("label") - .css("text-transform", "capitalize"); + checkbox.find("label").css("text-transform", "capitalize"); return checkbox; } @@ -290,8 +306,22 @@ frappe.PermissionEngine = class PermissionEngine { } get rights() { - return ["select", "read", "write", "create", "delete", "submit", "cancel", "amend", - "print", "email", "report", "import", "export", "set_user_permissions", "share"]; + return [ + "select", + "read", + "write", + "create", + "delete", + "submit", + "cancel", + "amend", + "print", + "email", + "report", + "import", + "export", + "share", + ]; } set_show_users(cell, role) { @@ -305,22 +335,29 @@ frappe.PermissionEngine = class PermissionEngine { page: "permission_manager", method: "get_users_with_role", args: { - role: role + role: role, }, callback: function (r) { r.message = $.map(r.message, function (p) { return $.format('{1}', [p, p]); }); - frappe.msgprint(__("Users with role {0}:", [__(role)]) - + "
    " + r.message.join("
    ")); - } + frappe.msgprint( + __("Users with role {0}:", [__(role)]) + + "
    " + + r.message.join("
    ") + ); + }, }); return false; }); } add_delete_button(row, d) { - $(``) + $( + `` + ) .appendTo($(``).appendTo(row)) .attr("data-doctype", d.parent) .attr("data-role", d.role) @@ -333,7 +370,7 @@ frappe.PermissionEngine = class PermissionEngine { args: { doctype: d.parent, role: d.role, - permlevel: d.permlevel + permlevel: d.permlevel, }, callback: (r) => { if (r.exc) { @@ -341,7 +378,7 @@ frappe.PermissionEngine = class PermissionEngine { } else { this.refresh(); } - } + }, }); }); } @@ -350,7 +387,7 @@ frappe.PermissionEngine = class PermissionEngine { let me = this; this.body.on("click", ".show-user-permissions", () => { frappe.route_options = { allow: this.get_doctype() || "" }; - frappe.set_route('List', 'User Permission'); + frappe.set_route("List", "User Permission"); }); this.body.on("click", "input[type='checkbox']", function () { @@ -361,7 +398,7 @@ frappe.PermissionEngine = class PermissionEngine { permlevel: chk.attr("data-permlevel"), doctype: chk.attr("data-doctype"), ptype: chk.attr("data-ptype"), - value: chk.prop("checked") ? 1 : 0 + value: chk.prop("checked") ? 1 : 0, }; return frappe.call({ module: "frappe.core", @@ -376,7 +413,7 @@ frappe.PermissionEngine = class PermissionEngine { } else { me.get_perm(args.role)[args.ptype] = args.value; } - } + }, }); }); } @@ -389,19 +426,30 @@ frappe.PermissionEngine = class PermissionEngine { title: __("Add New Permission Rule"), fields: [ { - fieldtype: "Select", label: __("Document Type"), - options: this.options.doctypes, reqd: 1, fieldname: "parent" + fieldtype: "Select", + label: __("Document Type"), + options: this.options.doctypes, + reqd: 1, + fieldname: "parent", }, { - fieldtype: "Select", label: __("Role"), - options: this.options.roles, reqd: 1, fieldname: "role" + fieldtype: "Select", + label: __("Role"), + options: this.options.roles, + reqd: 1, + fieldname: "role", }, { - fieldtype: "Select", label: __("Permission Level"), - options: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], reqd: 1, fieldname: "permlevel", - description: __("Level 0 is for document level permissions, higher levels for field level permissions.") - } - ] + fieldtype: "Select", + label: __("Permission Level"), + options: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], + reqd: 1, + fieldname: "permlevel", + description: __( + "Level 0 is for document level permissions, higher levels for field level permissions." + ), + }, + ], }); if (this.get_doctype()) { d.set_value("parent", this.get_doctype()); @@ -412,7 +460,7 @@ frappe.PermissionEngine = class PermissionEngine { d.get_input("role").prop("disabled", true); } d.set_value("permlevel", "0"); - d.set_primary_action(__('Add'), () => { + d.set_primary_action(__("Add"), () => { let args = d.get_values(); if (!args) { return; @@ -428,7 +476,7 @@ frappe.PermissionEngine = class PermissionEngine { } else { this.refresh(); } - } + }, }); d.hide(); }); @@ -439,13 +487,11 @@ frappe.PermissionEngine = class PermissionEngine { } make_reset_button() { - this.page.set_secondary_action( - __("Restore Original Permissions"), - () => { - this.get_standard_permissions((data) => { - this.reset_std_permissions(data); - }); + this.page.set_secondary_action(__("Restore Original Permissions"), () => { + this.get_standard_permissions((data) => { + this.reset_std_permissions(data); }); + }); } get_perm(role) { @@ -455,7 +501,9 @@ frappe.PermissionEngine = class PermissionEngine { } get_link_fields(doctype) { - return frappe.get_children("DocType", doctype, "fields", - { fieldtype: "Link", options: ["not in", ["User", '[Select]']] }); + return frappe.get_children("DocType", doctype, "fields", { + fieldtype: "Link", + options: ["not in", ["User", "[Select]"]], + }); } }; diff --git a/frappe/core/page/permission_manager/permission_manager.py b/frappe/core/page/permission_manager/permission_manager.py index 46c9e0aca2..5ed3014778 100644 --- a/frappe/core/page/permission_manager/permission_manager.py +++ b/frappe/core/page/permission_manager/permission_manager.py @@ -19,7 +19,6 @@ from frappe.permissions import ( setup_custom_perms, update_permission_property, ) -from frappe.translate import send_translations from frappe.utils.user import get_users_with_role as _get_user_with_role not_allowed_in_permission_manager = ["DocType", "Patch Log", "Module Def", "Transaction Log"] @@ -28,7 +27,6 @@ not_allowed_in_permission_manager = ["DocType", "Patch Log", "Module Def", "Tran @frappe.whitelist() def get_roles_and_doctypes(): frappe.only_for("System Manager") - send_translations(frappe.get_lang_dict("doctype", "DocPerm")) active_domains = frappe.get_active_domains() @@ -64,8 +62,8 @@ def get_roles_and_doctypes(): roles_list = [{"label": _(d.get("name")), "value": d.get("name")} for d in roles] return { - "doctypes": sorted(doctypes_list, key=lambda d: d["label"]), - "roles": sorted(roles_list, key=lambda d: d["label"]), + "doctypes": sorted(doctypes_list, key=lambda d: d["label"].casefold()), + "roles": sorted(roles_list, key=lambda d: d["label"].casefold()), } diff --git a/frappe/core/page/recorder/recorder.js b/frappe/core/page/recorder/recorder.js index f1f74daf71..1f004915fe 100644 --- a/frappe/core/page/recorder/recorder.js +++ b/frappe/core/page/recorder/recorder.js @@ -1,28 +1,28 @@ -frappe.pages['recorder'].on_page_load = function(wrapper) { +frappe.pages["recorder"].on_page_load = function (wrapper) { frappe.ui.make_app_page({ parent: wrapper, - title: __('Recorder'), + title: __("Recorder"), single_column: true, - card_layout: true + card_layout: true, }); frappe.recorder = new Recorder(wrapper); - $(wrapper).bind('show', function() { + $(wrapper).bind("show", function () { frappe.recorder.show(); }); - frappe.require('recorder.bundle.js'); + frappe.require("recorder.bundle.js"); }; class Recorder { constructor(wrapper) { this.wrapper = $(wrapper); - this.container = this.wrapper.find('.layout-main-section'); + this.container = this.wrapper.find(".layout-main-section"); this.container.append($('
    ')); } show() { - if (!this.view || this.view.$route.name == "recorder-detail") return; - this.view.$router.replace({name: "recorder-detail"}); + if (!this.route || this.route.name == "RecorderDetail") return; + this.router?.replace({ name: "RecorderDetail" }); } } diff --git a/frappe/custom/doctype/test_rename_new/__init__.py b/frappe/core/report/database_storage_usage_by_tables/__init__.py similarity index 100% rename from frappe/custom/doctype/test_rename_new/__init__.py rename to frappe/core/report/database_storage_usage_by_tables/__init__.py diff --git a/frappe/core/report/database_storage_usage_by_tables/database_storage_usage_by_tables.js b/frappe/core/report/database_storage_usage_by_tables/database_storage_usage_by_tables.js new file mode 100644 index 0000000000..b2cf268b36 --- /dev/null +++ b/frappe/core/report/database_storage_usage_by_tables/database_storage_usage_by_tables.js @@ -0,0 +1,7 @@ +// Copyright (c) 2022, Frappe Technologies and contributors +// For license information, please see license.txt +/* eslint-disable */ + +frappe.query_reports["Database Storage Usage By Tables"] = { + filters: [], +}; diff --git a/frappe/core/report/database_storage_usage_by_tables/database_storage_usage_by_tables.json b/frappe/core/report/database_storage_usage_by_tables/database_storage_usage_by_tables.json new file mode 100644 index 0000000000..20deb78ad6 --- /dev/null +++ b/frappe/core/report/database_storage_usage_by_tables/database_storage_usage_by_tables.json @@ -0,0 +1,28 @@ +{ + "add_total_row": 1, + "columns": [], + "creation": "2022-10-19 02:25:24.326791", + "disable_prepared_report": 0, + "disabled": 0, + "docstatus": 0, + "doctype": "Report", + "filters": [], + "idx": 0, + "is_standard": "Yes", + "letter_head": "abc", + "modified": "2022-10-19 02:59:00.365307", + "modified_by": "Administrator", + "module": "Core", + "name": "Database Storage Usage By Tables", + "owner": "Administrator", + "prepared_report": 0, + "query": "", + "ref_doctype": "Error Log", + "report_name": "Database Storage Usage By Tables", + "report_type": "Script Report", + "roles": [ + { + "role": "System Manager" + } + ] +} \ No newline at end of file diff --git a/frappe/core/report/database_storage_usage_by_tables/database_storage_usage_by_tables.py b/frappe/core/report/database_storage_usage_by_tables/database_storage_usage_by_tables.py new file mode 100644 index 0000000000..c88262552e --- /dev/null +++ b/frappe/core/report/database_storage_usage_by_tables/database_storage_usage_by_tables.py @@ -0,0 +1,40 @@ +# Copyright (c) 2022, Frappe Technologies and contributors +# For license information, please see license.txt + +import frappe + +COLUMNS = [ + {"label": "Table", "fieldname": "table", "fieldtype": "Data", "width": 200}, + {"label": "Size (MB)", "fieldname": "size", "fieldtype": "Float"}, + {"label": "Data (MB)", "fieldname": "data_size", "fieldtype": "Float"}, + {"label": "Index (MB)", "fieldname": "index_size", "fieldtype": "Float"}, +] + + +def execute(filters=None): + frappe.only_for("System Manager") + + data = frappe.db.multisql( + { + "mariadb": """ + SELECT table_name AS `table`, + round(((data_length + index_length) / 1024 / 1024), 2) `size`, + round((data_length / 1024 / 1024), 2) as data_size, + round((index_length / 1024 / 1024), 2) as index_size + FROM information_schema.TABLES + ORDER BY (data_length + index_length) DESC; + """, + "postgres": """ + SELECT + table_name as "table", + round(pg_total_relation_size(quote_ident(table_name)) / 1024 / 1024, 2) as "size", + round(pg_relation_size(quote_ident(table_name)) / 1024 / 1024, 2) as "data_size", + round(pg_indexes_size(quote_ident(table_name)) / 1024 / 1024, 2) as "index_size" + FROM information_schema.tables + WHERE table_schema = 'public' + ORDER BY 2 DESC; + """, + }, + as_dict=1, + ) + return COLUMNS, data diff --git a/frappe/core/report/database_storage_usage_by_tables/test_database_storage_usage_by_tables.py b/frappe/core/report/database_storage_usage_by_tables/test_database_storage_usage_by_tables.py new file mode 100644 index 0000000000..e82cbe9caf --- /dev/null +++ b/frappe/core/report/database_storage_usage_by_tables/test_database_storage_usage_by_tables.py @@ -0,0 +1,15 @@ +# Copyright (c) 2022, Frappe Technologies and contributors +# For license information, please see license.txt + + +from frappe.core.report.database_storage_usage_by_tables.database_storage_usage_by_tables import ( + execute, +) +from frappe.tests.utils import FrappeTestCase + + +class TestDBUsageReport(FrappeTestCase): + def test_basic_query(self): + _, data = execute() + tables = [d.table for d in data] + self.assertFalse({"tabUser", "tabDocField"}.difference(tables)) diff --git a/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.js b/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.js index 195f25f533..f840a49c92 100644 --- a/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.js +++ b/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.js @@ -2,33 +2,33 @@ // MIT License. See license.txt frappe.query_reports["Permitted Documents For User"] = { - "filters": [ + filters: [ { - "fieldname": "user", - "label": __("User"), - "fieldtype": "Link", - "options": "User", - "reqd": 1 + fieldname: "user", + label: __("User"), + fieldtype: "Link", + options: "User", + reqd: 1, }, { - "fieldname": "doctype", - "label": __("DocType"), - "fieldtype": "Link", - "options": "DocType", - "reqd": 1, - "get_query": function () { + fieldname: "doctype", + label: __("DocType"), + fieldtype: "Link", + options: "DocType", + reqd: 1, + get_query: function () { return { - "query": "frappe.core.report.permitted_documents_for_user.permitted_documents_for_user.query_doctypes", - "filters": { - "user": frappe.query_report.get_filter_value('user') - } - } - } + query: "frappe.core.report.permitted_documents_for_user.permitted_documents_for_user.query_doctypes", + filters: { + user: frappe.query_report.get_filter_value("user"), + }, + }; + }, }, { - "fieldname": "show_permissions", - "label": __("Show Permissions"), - "fieldtype": "Check" - } - ] -} + fieldname: "show_permissions", + label: __("Show Permissions"), + fieldtype: "Check", + }, + ], +}; diff --git a/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.py b/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.py index 362cc6b105..2c92a72ab3 100644 --- a/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.py +++ b/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.py @@ -4,19 +4,18 @@ import frappe import frappe.utils.user from frappe.model import data_fieldtypes -from frappe.permissions import check_admin_or_system_manager, rights +from frappe.permissions import rights def execute(filters=None): + frappe.only_for("System Manager") + user, doctype, show_permissions = ( filters.get("user"), filters.get("doctype"), filters.get("show_permissions"), ) - if not validate(user, doctype): - return [], [] - columns, fields = get_columns_and_fields(doctype) data = frappe.get_list(doctype, fields=fields, as_list=True, user=user) @@ -30,15 +29,9 @@ def execute(filters=None): return columns, data -def validate(user, doctype): - # check if current user is System Manager - check_admin_or_system_manager() - return user and doctype - - def get_columns_and_fields(doctype): columns = [f"Name:Link/{doctype}:200"] - fields = ["`name`"] + fields = ["name"] for df in frappe.get_meta(doctype).fields: if df.in_list_view and df.fieldtype in data_fieldtypes: fields.append(f"`{df.fieldname}`") diff --git a/frappe/core/report/transaction_log_report/transaction_log_report.js b/frappe/core/report/transaction_log_report/transaction_log_report.js index 54ecf3fcf1..3c7261306d 100644 --- a/frappe/core/report/transaction_log_report/transaction_log_report.js +++ b/frappe/core/report/transaction_log_report/transaction_log_report.js @@ -3,9 +3,9 @@ /* eslint-disable */ frappe.query_reports["Transaction Log Report"] = { - onload: function(query_report) { - query_report.add_make_chart_button = function() { + onload: function (query_report) { + query_report.add_make_chart_button = function () { // }; - } -} + }, +}; diff --git a/frappe/core/web_form/edit_profile/edit_profile.js b/frappe/core/web_form/edit_profile/edit_profile.js index 699703c579..8f56ebb353 100644 --- a/frappe/core/web_form/edit_profile/edit_profile.js +++ b/frappe/core/web_form/edit_profile/edit_profile.js @@ -1,3 +1,3 @@ -frappe.ready(function() { +frappe.ready(function () { // bind events here -}) \ No newline at end of file +}); diff --git a/frappe/core/web_form/edit_profile/edit_profile.json b/frappe/core/web_form/edit_profile/edit_profile.json index c04e705820..9a38b29b68 100644 --- a/frappe/core/web_form/edit_profile/edit_profile.json +++ b/frappe/core/web_form/edit_profile/edit_profile.json @@ -1,37 +1,33 @@ { - "accept_payment": 0, "allow_comments": 0, "allow_delete": 0, "allow_edit": 1, "allow_incomplete": 0, "allow_multiple": 0, "allow_print": 0, - "amount": 0.0, - "amount_based_on_field": 0, "apply_document_permissions": 0, "breadcrumbs": "[{\"title\": _(\"My Account\"), \"route\": \"me\"}]", + "client_script": "frappe.web_form.after_load = () => {\n if (window.location.pathname.endsWith(\"/new\") && frappe.session.user) {\n let current_path = window.location.href;\n window.location.href = current_path.replace(\"/new\", \"/\" + frappe.session.user);\n }\n}", "creation": "2016-09-19 05:16:59.242754", "doc_type": "User", "docstatus": 0, "doctype": "Web Form", "idx": 0, "introduction_text": "", - "is_multi_step_form": 0, "is_standard": 1, + "list_columns": [], "login_required": 1, "max_attachment_size": 0, - "modified": "2022-03-22 15:00:43.456738", + "modified": "2023-01-18 10:26:26.766414", "modified_by": "Administrator", "module": "Core", "name": "edit-profile", "owner": "Administrator", "published": 1, "route": "update-profile", - "route_to_success_link": 0, "show_attachments": 0, - "show_in_grid": 0, + "show_list": 0, "show_sidebar": 0, - "sidebar_items": [], "success_message": "Profile updated successfully.", "success_url": "/me", "title": "Update Profile", diff --git a/frappe/core/workspace/build/build.json b/frappe/core/workspace/build/build.json index c1c506ae3a..67dfae650f 100644 --- a/frappe/core/workspace/build/build.json +++ b/frappe/core/workspace/build/build.json @@ -1,6 +1,6 @@ { "charts": [], - "content": "[{\"type\":\"header\",\"data\":{\"text\":\"Your Shortcuts\",\"col\":12}},{\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"DocType\",\"col\":3}},{\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"Workspace\",\"col\":3}},{\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"Report\",\"col\":3}},{\"type\":\"spacer\",\"data\":{\"col\":12}},{\"type\":\"header\",\"data\":{\"text\":\"Elements\",\"col\":12}},{\"type\":\"card\",\"data\":{\"card_name\":\"Modules\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Models\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Views\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Scripting\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Packages\",\"col\":4}}]", + "content": "[{\"type\":\"header\",\"data\":{\"text\":\"Your Shortcuts\",\"col\":12}},{\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"DocType\",\"col\":3}},{\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"Workspace\",\"col\":3}},{\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"Report\",\"col\":3}},{\"type\":\"spacer\",\"data\":{\"col\":12}},{\"type\":\"header\",\"data\":{\"text\":\"Elements\",\"col\":12}},{\"type\":\"card\",\"data\":{\"card_name\":\"Modules\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Models\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Views\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Scripting\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Packages\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"System Logs\",\"col\":4}}]", "creation": "2021-01-02 10:51:16.579957", "docstatus": 0, "doctype": "Workspace", @@ -10,60 +10,6 @@ "idx": 0, "label": "Build", "links": [ - { - "hidden": 0, - "is_query_report": 0, - "label": "Modules", - "link_count": 0, - "link_type": "DocType", - "onboard": 0, - "only_for": "", - "type": "Card Break" - }, - { - "hidden": 0, - "is_query_report": 0, - "label": "Module Def", - "link_count": 0, - "link_to": "Module Def", - "link_type": "DocType", - "onboard": 0, - "only_for": "", - "type": "Link" - }, - { - "hidden": 0, - "is_query_report": 0, - "label": "Workspace", - "link_count": 0, - "link_to": "Workspace", - "link_type": "DocType", - "onboard": 0, - "only_for": "", - "type": "Link" - }, - { - "hidden": 0, - "is_query_report": 0, - "label": "Module Onboarding", - "link_count": 0, - "link_to": "Module Onboarding", - "link_type": "DocType", - "onboard": 0, - "only_for": "", - "type": "Link" - }, - { - "hidden": 0, - "is_query_report": 0, - "label": "Block Module", - "link_count": 0, - "link_to": "Block Module", - "link_type": "DocType", - "onboard": 0, - "only_for": "", - "type": "Link" - }, { "hidden": 0, "is_query_report": 0, @@ -96,60 +42,6 @@ "only_for": "", "type": "Link" }, - { - "hidden": 0, - "is_query_report": 0, - "label": "Views", - "link_count": 0, - "link_type": "DocType", - "onboard": 0, - "only_for": "", - "type": "Card Break" - }, - { - "hidden": 0, - "is_query_report": 0, - "label": "Report", - "link_count": 0, - "link_to": "Report", - "link_type": "DocType", - "onboard": 0, - "only_for": "", - "type": "Link" - }, - { - "hidden": 0, - "is_query_report": 0, - "label": "Print Format", - "link_count": 0, - "link_to": "Print Format", - "link_type": "DocType", - "onboard": 0, - "only_for": "", - "type": "Link" - }, - { - "hidden": 0, - "is_query_report": 0, - "label": "Workspace", - "link_count": 0, - "link_to": "Workspace", - "link_type": "DocType", - "onboard": 0, - "only_for": "", - "type": "Link" - }, - { - "hidden": 0, - "is_query_report": 0, - "label": "Dashboard", - "link_count": 0, - "link_to": "Dashboard", - "link_type": "DocType", - "onboard": 0, - "only_for": "", - "type": "Link" - }, { "hidden": 0, "is_query_report": 0, @@ -220,15 +112,175 @@ "link_type": "DocType", "onboard": 0, "type": "Link" + }, + { + "hidden": 0, + "is_query_report": 0, + "label": "Modules", + "link_count": 3, + "onboard": 0, + "type": "Card Break" + }, + { + "hidden": 0, + "is_query_report": 0, + "label": "Module Def", + "link_count": 0, + "link_to": "Module Def", + "link_type": "DocType", + "onboard": 0, + "only_for": "", + "type": "Link" + }, + { + "hidden": 0, + "is_query_report": 0, + "label": "Module Onboarding", + "link_count": 0, + "link_to": "Module Onboarding", + "link_type": "DocType", + "onboard": 0, + "only_for": "", + "type": "Link" + }, + { + "hidden": 0, + "is_query_report": 0, + "label": "Module Profile", + "link_count": 0, + "link_to": "Module Profile", + "link_type": "DocType", + "onboard": 0, + "type": "Link" + }, + { + "hidden": 0, + "is_query_report": 0, + "label": "Views", + "link_count": 4, + "onboard": 0, + "type": "Card Break" + }, + { + "hidden": 0, + "is_query_report": 0, + "label": "Report", + "link_count": 0, + "link_to": "Report", + "link_type": "DocType", + "onboard": 0, + "only_for": "", + "type": "Link" + }, + { + "hidden": 0, + "is_query_report": 0, + "label": "Print Format", + "link_count": 0, + "link_to": "Print Format", + "link_type": "DocType", + "onboard": 0, + "only_for": "", + "type": "Link" + }, + { + "hidden": 0, + "is_query_report": 0, + "label": "Dashboard", + "link_count": 0, + "link_to": "Dashboard", + "link_type": "DocType", + "onboard": 0, + "only_for": "", + "type": "Link" + }, + { + "hidden": 0, + "is_query_report": 0, + "label": "Workspace", + "link_count": 0, + "link_to": "Workspace", + "link_type": "DocType", + "onboard": 0, + "type": "Link" + }, + { + "hidden": 0, + "is_query_report": 0, + "label": "System Logs", + "link_count": 6, + "onboard": 0, + "type": "Card Break" + }, + { + "hidden": 0, + "is_query_report": 0, + "label": "Background Jobs", + "link_count": 0, + "link_to": "RQ Job", + "link_type": "DocType", + "onboard": 0, + "type": "Link" + }, + { + "hidden": 0, + "is_query_report": 0, + "label": "Scheduled Jobs Logs", + "link_count": 0, + "link_to": "Scheduled Job Log", + "link_type": "DocType", + "onboard": 0, + "type": "Link" + }, + { + "hidden": 0, + "is_query_report": 0, + "label": "Error Logs", + "link_count": 0, + "link_to": "Error Log", + "link_type": "DocType", + "onboard": 0, + "type": "Link" + }, + { + "hidden": 0, + "is_query_report": 0, + "label": "Error Snapshot", + "link_count": 0, + "link_to": "Error Snapshot", + "link_type": "DocType", + "onboard": 0, + "type": "Link" + }, + { + "hidden": 0, + "is_query_report": 0, + "label": "Communication Logs", + "link_count": 0, + "link_to": "Communication", + "link_type": "DocType", + "onboard": 0, + "type": "Link" + }, + { + "hidden": 0, + "is_query_report": 0, + "label": "Activity Log", + "link_count": 0, + "link_to": "Activity Log", + "link_type": "DocType", + "onboard": 0, + "type": "Link" } ], - "modified": "2022-01-13 17:26:02.736366", + "modified": "2022-09-11 06:41:31.095300", "modified_by": "Administrator", "module": "Core", "name": "Build", "owner": "Administrator", "parent_page": "", "public": 1, + "quick_lists": [], "restrict_to_domain": "", "roles": [], "sequence_id": 5.0, diff --git a/frappe/core/workspace/settings/settings.json b/frappe/core/workspace/settings/settings.json index 5aadbc42d5..1469892bd8 100644 --- a/frappe/core/workspace/settings/settings.json +++ b/frappe/core/workspace/settings/settings.json @@ -1,6 +1,6 @@ { "charts": [], - "content": "[{\"type\":\"header\",\"data\":{\"text\":\"Settings\",\"col\":12}},{\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"System Settings\",\"col\":3}},{\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"Print Settings\",\"col\":3}},{\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"Website Settings\",\"col\":3}},{\"type\":\"spacer\",\"data\":{\"col\":12}},{\"type\":\"header\",\"data\":{\"text\":\"Reports & Masters\",\"col\":12}},{\"type\":\"card\",\"data\":{\"card_name\":\"Data\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Email / Notifications\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Website\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Core\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Printing\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Workflow\",\"col\":4}}]", + "content": "[{\"type\":\"header\",\"data\":{\"text\":\"Settings\",\"col\":12}},{\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"System Settings\",\"col\":3}},{\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"Print Settings\",\"col\":3}},{\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"Website Settings\",\"col\":3}},{\"type\":\"spacer\",\"data\":{\"col\":12}},{\"type\":\"header\",\"data\":{\"text\":\"Reports & Masters\",\"col\":12}},{\"type\":\"card\",\"data\":{\"card_name\":\"Data\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Email / Notifications\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Website\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Core\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Printing\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Workflow\",\"col\":4}}]", "creation": "2020-03-02 15:09:40.527211", "docstatus": 0, "doctype": "Workspace", @@ -221,58 +221,6 @@ "onboard": 0, "type": "Link" }, - { - "hidden": 0, - "is_query_report": 0, - "label": "Core", - "link_count": 0, - "onboard": 0, - "type": "Card Break" - }, - { - "dependencies": "", - "hidden": 0, - "is_query_report": 0, - "label": "System Settings", - "link_count": 0, - "link_to": "System Settings", - "link_type": "DocType", - "onboard": 0, - "type": "Link" - }, - { - "dependencies": "", - "hidden": 0, - "is_query_report": 0, - "label": "Error Log", - "link_count": 0, - "link_to": "Error Log", - "link_type": "DocType", - "onboard": 0, - "type": "Link" - }, - { - "dependencies": "", - "hidden": 0, - "is_query_report": 0, - "label": "Error Snapshot", - "link_count": 0, - "link_to": "Error Snapshot", - "link_type": "DocType", - "onboard": 0, - "type": "Link" - }, - { - "dependencies": "", - "hidden": 0, - "is_query_report": 0, - "label": "Domain Settings", - "link_count": 0, - "link_to": "Domain Settings", - "link_type": "DocType", - "onboard": 0, - "type": "Link" - }, { "hidden": 0, "is_query_report": 0, @@ -365,15 +313,46 @@ "link_type": "DocType", "onboard": 0, "type": "Link" + }, + { + "hidden": 0, + "is_query_report": 0, + "label": "Core", + "link_count": 2, + "onboard": 0, + "type": "Card Break" + }, + { + "dependencies": "", + "hidden": 0, + "is_query_report": 0, + "label": "System Settings", + "link_count": 0, + "link_to": "System Settings", + "link_type": "DocType", + "onboard": 0, + "type": "Link" + }, + { + "dependencies": "", + "hidden": 0, + "is_query_report": 0, + "label": "Domain Settings", + "link_count": 0, + "link_to": "Domain Settings", + "link_type": "DocType", + "onboard": 0, + "type": "Link" } ], - "modified": "2022-01-13 17:49:59.586909", + "modified": "2022-08-28 21:41:28.065190", "modified_by": "Administrator", "module": "Core", "name": "Settings", "owner": "Administrator", "parent_page": "", "public": 1, + "quick_lists": [], "restrict_to_domain": "", "roles": [], "sequence_id": 29.0, diff --git a/frappe/coverage.py b/frappe/coverage.py index ffa3576818..a1f3e26585 100644 --- a/frappe/coverage.py +++ b/frappe/coverage.py @@ -24,6 +24,15 @@ STANDARD_EXCLUSIONS = [ "*/patches/*", ] +# tested via commands' test suite +TESTED_VIA_CLI = [ + "*/frappe/installer.py", + "*/frappe/build.py", + "*/frappe/database/__init__.py", + "*/frappe/database/db_manager.py", + "*/frappe/database/**/setup_db.py", +] + FRAPPE_EXCLUSIONS = [ "*/tests/*", "*/commands/*", @@ -33,7 +42,7 @@ FRAPPE_EXCLUSIONS = [ "*frappe/setup.py", "*/doctype/*/*_dashboard.py", "*/patches/*", -] +] + TESTED_VIA_CLI class CodeCoverage: @@ -64,3 +73,4 @@ class CodeCoverage: self.coverage.stop() self.coverage.save() self.coverage.xml_report() + print("Saved Coverage") diff --git a/frappe/custom/doctype/client_script/client_script.js b/frappe/custom/doctype/client_script/client_script.js index 18786c62cf..67bb0083c8 100644 --- a/frappe/custom/doctype/client_script/client_script.js +++ b/frappe/custom/doctype/client_script/client_script.js @@ -1,46 +1,49 @@ // Copyright (c) 2016, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Client Script', { +frappe.ui.form.on("Client Script", { setup(frm) { frm.get_field("sample").html(SAMPLE_HTML); }, refresh(frm) { if (frm.doc.dt && frm.doc.script) { - frm.add_custom_button(__('Go to {0}', [frm.doc.dt]), - () => frappe.set_route('List', frm.doc.dt, 'List')); + frm.add_custom_button(__("Go to {0}", [frm.doc.dt]), () => + frappe.set_route("List", frm.doc.dt, "List") + ); } - if (frm.doc.view == 'Form') { - frm.add_custom_button(__('Add script for Child Table'), () => { + if (frm.doc.view == "Form") { + frm.add_custom_button(__("Add script for Child Table"), () => { frappe.model.with_doctype(frm.doc.dt, () => { - const child_tables = frappe.meta.get_docfields(frm.doc.dt, null, { - fieldtype: 'Table' - }).map(df => df.options); + const child_tables = frappe.meta + .get_docfields(frm.doc.dt, null, { + fieldtype: "Table", + }) + .map((df) => df.options); const d = new frappe.ui.Dialog({ - title: __('Select Child Table'), + title: __("Select Child Table"), fields: [ { - label: __('Select Child Table'), - fieldtype: 'Link', - fieldname: 'cdt', - options: 'DocType', + label: __("Select Child Table"), + fieldtype: "Link", + fieldname: "cdt", + options: "DocType", get_query: () => { return { filters: { istable: 1, - name: ['in', child_tables] - } + name: ["in", child_tables], + }, }; - } - } + }, + }, ], primary_action: ({ cdt }) => { - cdt = d.get_field('cdt').value; + cdt = d.get_field("cdt").value; frm.events.add_script_for_doctype(frm, cdt); d.hide(); - } + }, }); d.show(); @@ -48,39 +51,39 @@ frappe.ui.form.on('Client Script', { }); if (!frm.is_new()) { - frm.add_custom_button(__('Compare Versions'), () => { + frm.add_custom_button(__("Compare Versions"), () => { new frappe.ui.DiffView("Client Script", "script", frm.doc.name); }); } } - frm.set_query('dt', { + frm.set_query("dt", { filters: { - istable: 0 - } + istable: 0, + }, }); }, dt(frm) { - frm.toggle_display('view', !frappe.boot.single_types.includes(frm.doc.dt)); + frm.toggle_display("view", !frappe.boot.single_types.includes(frm.doc.dt)); if (!frm.doc.script) { frm.events.add_script_for_doctype(frm, frm.doc.dt); } if (frm.doc.script && !frm.doc.script.includes(frm.doc.dt)) { - frm.doc.script = ''; + frm.doc.script = ""; frm.events.add_script_for_doctype(frm, frm.doc.dt); } }, view(frm) { - let has_form_boilerplate = frm.doc.script.includes('frappe.ui.form.on') - if (frm.doc.view === 'List' && has_form_boilerplate) { - frm.set_value('script', ''); + let has_form_boilerplate = frm.doc.script.includes("frappe.ui.form.on"); + if (frm.doc.view === "List" && has_form_boilerplate) { + frm.set_value("script", ""); } - if (frm.doc.view === 'Form' && !has_form_boilerplate) { - frm.trigger('dt'); + if (frm.doc.view === "Form" && !has_form_boilerplate) { + frm.trigger("dt"); } }, @@ -93,12 +96,12 @@ frappe.ui.form.on('${doctype}', { } }) `.trim(); - let script = (frm.doc.script || ''); + let script = frm.doc.script || ""; if (script) { - script += '\n\n'; + script += "\n\n"; } - frm.set_value('script', script + boilerplate); - } + frm.set_value("script", script + boilerplate); + }, }); const SAMPLE_HTML = `

    Client Script Help

    diff --git a/frappe/custom/doctype/client_script/test_client_script.py b/frappe/custom/doctype/client_script/test_client_script.py index c93df04c98..9b50a3d0b0 100644 --- a/frappe/custom/doctype/client_script/test_client_script.py +++ b/frappe/custom/doctype/client_script/test_client_script.py @@ -1,9 +1,9 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import unittest +from frappe.tests.utils import FrappeTestCase # test_records = frappe.get_test_records('Client Script') -class TestClientScript(unittest.TestCase): +class TestClientScript(FrappeTestCase): pass diff --git a/frappe/custom/doctype/client_script/ui_test_client_script.js b/frappe/custom/doctype/client_script/ui_test_client_script.js index 022f677151..0d202d697c 100644 --- a/frappe/custom/doctype/client_script/ui_test_client_script.js +++ b/frappe/custom/doctype/client_script/ui_test_client_script.js @@ -12,14 +12,14 @@ context("Client Script", () => { dt: "ToDo", view: "Form", enabled: 1, - script: `console.log('todo form script')` + script: `console.log('todo form script')`, }, true ); cy.visit("/app/todo/new", { onBeforeLoad(win) { cy.spy(win.console, "log").as("consoleLog"); - } + }, }); cy.get("@consoleLog").should("be.calledWith", "todo form script"); }); @@ -32,14 +32,14 @@ context("Client Script", () => { dt: "ToDo", view: "List", enabled: 1, - script: `console.log('todo list script')` + script: `console.log('todo list script')`, }, true ); cy.visit("/app/todo", { onBeforeLoad(win) { cy.spy(win.console, "log").as("consoleLog"); - } + }, }); cy.get("@consoleLog").should("be.calledWith", "todo list script"); }); @@ -52,19 +52,16 @@ context("Client Script", () => { dt: "ToDo", view: "List", enabled: 0, - script: `console.log('todo disabled script')` + script: `console.log('todo disabled script')`, }, true ); cy.visit("/app/todo", { onBeforeLoad(win) { cy.spy(win.console, "log").as("consoleLog"); - } + }, }); - cy.get("@consoleLog").should( - "not.be.calledWith", - "todo disabled script" - ); + cy.get("@consoleLog").should("not.be.calledWith", "todo disabled script"); }); it("should run multiple scripts", () => { @@ -75,7 +72,7 @@ context("Client Script", () => { dt: "ToDo", view: "Form", enabled: 1, - script: `console.log('todo form script 1')` + script: `console.log('todo form script 1')`, }, true ); @@ -86,14 +83,14 @@ context("Client Script", () => { dt: "ToDo", view: "Form", enabled: 1, - script: `console.log('todo form script 2')` + script: `console.log('todo form script 2')`, }, true ); cy.visit("/app/todo/new", { onBeforeLoad(win) { cy.spy(win.console, "log").as("consoleLog"); - } + }, }); cy.get("@consoleLog").should("be.calledWith", "todo form script 1"); cy.get("@consoleLog").should("be.calledWith", "todo form script 2"); diff --git a/frappe/custom/doctype/custom_field/custom_field.js b/frappe/custom/doctype/custom_field/custom_field.js index c59fabeaa6..be416cb49a 100644 --- a/frappe/custom/doctype/custom_field/custom_field.js +++ b/frappe/custom/doctype/custom_field/custom_field.js @@ -4,88 +4,120 @@ // Refresh // -------- -frappe.ui.form.on('Custom Field', { - setup: function(frm) { - frm.set_query('dt', function(doc) { +frappe.ui.form.on("Custom Field", { + setup: function (frm) { + frm.set_query("dt", function (doc) { var filters = [ - ['DocType', 'issingle', '=', 0], - ['DocType', 'custom', '=', 0], - ['DocType', 'name', 'not in', frappe.model.core_doctypes_list], - ['DocType', 'restrict_to_domain', 'in', frappe.boot.active_domains] + ["DocType", "issingle", "=", 0], + ["DocType", "custom", "=", 0], + ["DocType", "name", "not in", frappe.model.core_doctypes_list], + ["DocType", "restrict_to_domain", "in", frappe.boot.active_domains], ]; - if(frappe.session.user!=="Administrator") { - filters.push(['DocType', 'module', 'not in', ['Core', 'Custom']]) + if (frappe.session.user !== "Administrator") { + filters.push(["DocType", "module", "not in", ["Core", "Custom"]]); } return { - "filters": filters - } + filters: filters, + }; }); }, - refresh: function(frm) { - frm.toggle_enable('dt', frm.doc.__islocal); - frm.trigger('dt'); - frm.toggle_reqd('label', !frm.doc.fieldname); + refresh: function (frm) { + frm.toggle_enable("dt", frm.doc.__islocal); + frm.trigger("dt"); + frm.toggle_reqd("label", !frm.doc.fieldname); + + if (frm.doc.is_system_generated) { + frm.dashboard.add_comment( + __( + "Warning: This field is system generated and may be overwritten by a future update. Modify it using {0} instead.", + [ + frappe.utils.get_form_link( + "Customize Form", + "Customize Form", + true, + __("Customize Form"), + { + doc_type: frm.doc.dt, + } + ), + ] + ), + "yellow", + true + ); + } }, - dt: function(frm) { - if(!frm.doc.dt) { - set_field_options('insert_after', ''); + dt: function (frm) { + if (!frm.doc.dt) { + set_field_options("insert_after", ""); return; } var insert_after = frm.doc.insert_after || null; return frappe.call({ - method: 'frappe.custom.doctype.custom_field.custom_field.get_fields_label', + method: "frappe.custom.doctype.custom_field.custom_field.get_fields_label", args: { doctype: frm.doc.dt, fieldname: frm.doc.fieldname }, - callback: function(r) { - if(r) { - if(r._server_messages && r._server_messages.length) { + callback: function (r) { + if (r) { + if (r._server_messages && r._server_messages.length) { frm.set_value("dt", ""); } else { - set_field_options('insert_after', r.message); - var fieldnames = $.map(r.message, function(v) { return v.value; }); + set_field_options("insert_after", r.message); + var fieldnames = $.map(r.message, function (v) { + return v.value; + }); - if(insert_after==null || !in_list(fieldnames, insert_after)) { + if (insert_after == null || !in_list(fieldnames, insert_after)) { insert_after = fieldnames[-1]; } - frm.set_value('insert_after', insert_after); + frm.set_value("insert_after", insert_after); } } - } + }, }); - }, - label: function(frm) { - if(frm.doc.label && frappe.utils.has_special_chars(frm.doc.label)) { - frm.fields_dict['label_help'].disp_area.innerHTML = - ''+__('Special Characters are not allowed')+''; - frm.set_value('label', ''); + label: function (frm) { + if (frm.doc.label && frappe.utils.has_special_chars(frm.doc.label)) { + frm.fields_dict["label_help"].disp_area.innerHTML = + '' + __("Special Characters are not allowed") + ""; + frm.set_value("label", ""); } else { - frm.fields_dict['label_help'].disp_area.innerHTML = ''; + frm.fields_dict["label_help"].disp_area.innerHTML = ""; } }, - fieldtype: function(frm) { - if(frm.doc.fieldtype == 'Link') { - frm.fields_dict['options_help'].disp_area.innerHTML = - __('Name of the Document Type (DocType) you want this field to be linked to. e.g. Customer'); - } else if(frm.doc.fieldtype == 'Select') { - frm.fields_dict['options_help'].disp_area.innerHTML = - __('Options for select. Each option on a new line.')+' '+__('e.g.:')+'
    '+__('Option 1')+'
    '+__('Option 2')+'
    '+__('Option 3')+'
    '; - } else if(frm.doc.fieldtype == 'Dynamic Link') { - frm.fields_dict['options_help'].disp_area.innerHTML = - __('Fieldname which will be the DocType for this link field.'); + fieldtype: function (frm) { + if (frm.doc.fieldtype == "Link") { + frm.fields_dict["options_help"].disp_area.innerHTML = __( + "Name of the Document Type (DocType) you want this field to be linked to. e.g. Customer" + ); + } else if (frm.doc.fieldtype == "Select") { + frm.fields_dict["options_help"].disp_area.innerHTML = + __("Options for select. Each option on a new line.") + + " " + + __("e.g.:") + + "
    " + + __("Option 1") + + "
    " + + __("Option 2") + + "
    " + + __("Option 3") + + "
    "; + } else if (frm.doc.fieldtype == "Dynamic Link") { + frm.fields_dict["options_help"].disp_area.innerHTML = __( + "Fieldname which will be the DocType for this link field." + ); } else { - frm.fields_dict['options_help'].disp_area.innerHTML = ''; + frm.fields_dict["options_help"].disp_area.innerHTML = ""; } - } + }, }); - -frappe.utils.has_special_chars = function(t) { - var iChars = "!@#$%^&*()+=-[]\\\';,./{}|\":<>?"; +frappe.utils.has_special_chars = function (t) { + var iChars = "!@#$%^&*()+=-[]\\';,./{}|\":<>?"; for (var i = 0; i < t.length; i++) { if (iChars.indexOf(t.charAt(i)) != -1) { return true; } } return false; -} +}; diff --git a/frappe/custom/doctype/custom_field/custom_field.py b/frappe/custom/doctype/custom_field/custom_field.py index 8a2a2663de..8953153be6 100644 --- a/frappe/custom/doctype/custom_field/custom_field.py +++ b/frappe/custom/doctype/custom_field/custom_field.py @@ -9,7 +9,7 @@ from frappe.model import core_doctypes_list from frappe.model.docfield import supports_translation from frappe.model.document import Document from frappe.query_builder.functions import IfNull -from frappe.utils import cstr +from frappe.utils import cstr, random_string class CustomField(Document): @@ -18,11 +18,23 @@ class CustomField(Document): self.name = self.dt + "-" + self.fieldname def set_fieldname(self): + restricted = ( + "name", + "parent", + "creation", + "modified", + "modified_by", + "parentfield", + "parenttype", + "file_list", + "flags", + "docstatus", + ) if not self.fieldname: label = self.label if not label: if self.fieldtype in ["Section Break", "Column Break", "Tab Break"]: - label = self.fieldtype + "_" + str(self.idx) + label = self.fieldtype + "_" + str(random_string(5)) else: frappe.throw(_("Label is mandatory")) @@ -34,6 +46,9 @@ class CustomField(Document): # fieldnames should be lowercase self.fieldname = self.fieldname.lower() + if self.fieldname in restricted: + self.fieldname = self.fieldname + "1" + def before_insert(self): self.set_fieldname() @@ -80,17 +95,15 @@ class CustomField(Document): check_fieldname_conflicts(self) def on_update(self): - if not frappe.flags.in_setup_wizard: - frappe.clear_cache(doctype=self.dt) - + # validate field if not self.flags.ignore_validate: - # validate field from frappe.core.doctype.doctype.doctype import validate_fields_for_doctype validate_fields_for_doctype(self.dt) - # update the schema - if not frappe.db.get_value("DocType", self.dt, "issingle") and not frappe.flags.in_setup_wizard: + # clear cache and update the schema + if not frappe.flags.in_create_custom_fields: + frappe.clear_cache(doctype=self.dt) frappe.db.updatedb(self.dt) def on_trash(self): @@ -104,6 +117,20 @@ class CustomField(Document): # delete property setter entries frappe.db.delete("Property Setter", {"doc_type": self.dt, "field_name": self.fieldname}) + + # update doctype layouts + doctype_layouts = frappe.get_all( + "DocType Layout", filters={"document_type": self.dt}, pluck="name" + ) + + for layout in doctype_layouts: + layout_doc = frappe.get_doc("DocType Layout", layout) + for field in layout_doc.fields: + if field.fieldname == self.fieldname: + layout_doc.remove(field) + layout_doc.save() + break + frappe.clear_cache(doctype=self.dt) def validate_insert_after(self, meta): @@ -130,7 +157,7 @@ def get_fields_label(doctype=None): return frappe.msgprint(_("Custom Fields can only be added to a standard DocType.")) return [ - {"value": df.fieldname or "", "label": _(df.label or "")} + {"value": df.fieldname or "", "label": _(df.label) if df.label else ""} for df in frappe.get_meta(doctype).get("fields") ] @@ -169,38 +196,45 @@ def create_custom_fields(custom_fields, ignore_validate=False, update=True): :param custom_fields: example `{'Sales Invoice': [dict(fieldname='test')]}`""" - if not ignore_validate and frappe.flags.in_setup_wizard: - ignore_validate = True + try: + frappe.flags.in_create_custom_fields = True + doctypes_to_update = set() - for doctypes, fields in custom_fields.items(): - if isinstance(fields, dict): - # only one field - fields = [fields] + if frappe.flags.in_setup_wizard: + ignore_validate = True - if isinstance(doctypes, str): - # only one doctype - doctypes = (doctypes,) + for doctypes, fields in custom_fields.items(): + if isinstance(fields, dict): + # only one field + fields = [fields] - for doctype in doctypes: - for df in fields: - field = frappe.db.get_value("Custom Field", {"dt": doctype, "fieldname": df["fieldname"]}) - if not field: - try: - df["owner"] = "Administrator" - create_custom_field(doctype, df, ignore_validate=ignore_validate) - except frappe.exceptions.DuplicateEntryError: - pass - elif update: - custom_field = frappe.get_doc("Custom Field", field) - custom_field.flags.ignore_validate = ignore_validate - custom_field.update(df) - custom_field.save() + if isinstance(doctypes, str): + # only one doctype + doctypes = (doctypes,) - frappe.clear_cache(doctype=doctype) - frappe.db.updatedb(doctype) + for doctype in doctypes: + doctypes_to_update.add(doctype) + for df in fields: + field = frappe.db.get_value("Custom Field", {"dt": doctype, "fieldname": df["fieldname"]}) + if not field: + try: + df = df.copy() + df["owner"] = "Administrator" + create_custom_field(doctype, df, ignore_validate=ignore_validate) -@frappe.whitelist() -def add_custom_field(doctype, df): - df = json.loads(df) - return create_custom_field(doctype, df) + except frappe.exceptions.DuplicateEntryError: + pass + + elif update: + custom_field = frappe.get_doc("Custom Field", field) + custom_field.flags.ignore_validate = ignore_validate + custom_field.update(df) + custom_field.save() + + for doctype in doctypes_to_update: + frappe.clear_cache(doctype=doctype) + frappe.db.updatedb(doctype) + + finally: + frappe.flags.in_create_custom_fields = False diff --git a/frappe/custom/doctype/custom_field/test_custom_field.py b/frappe/custom/doctype/custom_field/test_custom_field.py index 34223315c5..cf64e4495b 100644 --- a/frappe/custom/doctype/custom_field/test_custom_field.py +++ b/frappe/custom/doctype/custom_field/test_custom_field.py @@ -1,17 +1,15 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import unittest - import frappe +from frappe.custom.doctype.custom_field.custom_field import create_custom_fields +from frappe.tests.utils import FrappeTestCase test_records = frappe.get_test_records("Custom Field") -class TestCustomField(unittest.TestCase): +class TestCustomField(FrappeTestCase): def test_create_custom_fields(self): - from .custom_field import create_custom_fields - create_custom_fields( { "Address": [ @@ -38,3 +36,48 @@ class TestCustomField(unittest.TestCase): self.assertTrue(frappe.db.exists("Custom Field", "Address-_test_custom_field_1")) self.assertTrue(frappe.db.exists("Custom Field", "Address-_test_custom_field_2")) self.assertTrue(frappe.db.exists("Custom Field", "Contact-_test_custom_field_2")) + + def test_custom_field_sorting(self): + try: + custom_fields = { + "ToDo": [ + {"fieldname": "a_test_field", "insert_after": "b_test_field"}, + {"fieldname": "b_test_field", "insert_after": "status"}, + {"fieldname": "c_test_field", "insert_after": "unknown_custom_field"}, + {"fieldname": "d_test_field", "insert_after": "status"}, + ] + } + + create_custom_fields(custom_fields, ignore_validate=True) + + fields = frappe.get_meta("ToDo", cached=False).fields + + for i, field in enumerate(fields): + if field.fieldname == "b_test_field": + self.assertEqual(fields[i - 1].fieldname, "status") + + if field.fieldname == "d_test_field": + self.assertEqual(fields[i - 1].fieldname, "a_test_field") + + self.assertEqual(fields[-1].fieldname, "c_test_field") + + finally: + frappe.db.delete( + "Custom Field", + { + "dt": "ToDo", + "fieldname": ( + "in", + ( + "a_test_field", + "b_test_field", + "c_test_field", + "d_test_field", + ), + ), + }, + ) + + # undo changes commited by DDL + # nosemgrep + frappe.db.commit() diff --git a/frappe/custom/doctype/customize_form/customize_form.js b/frappe/custom/doctype/customize_form/customize_form.js index 3ec6795f0e..8549c239e5 100644 --- a/frappe/custom/doctype/customize_form/customize_form.js +++ b/frappe/custom/doctype/customize_form/customize_form.js @@ -4,7 +4,7 @@ frappe.provide("frappe.customize_form"); frappe.ui.form.on("Customize Form", { - setup: function(frm) { + setup: function (frm) { // save the last setting if refreshing window.addEventListener("beforeunload", () => { if (frm.doc.doc_type && frm.doc.doc_type != "undefined") { @@ -13,93 +13,90 @@ frappe.ui.form.on("Customize Form", { }); }, - onload: function(frm) { - frm.set_query("doc_type", function() { + onload: function (frm) { + frm.set_query("doc_type", function () { return { filters: [ ["DocType", "issingle", "=", 0], ["DocType", "custom", "=", 0], - [ - "DocType", - "name", - "not in", - frappe.model.core_doctypes_list - ], - [ - "DocType", - "restrict_to_domain", - "in", - frappe.boot.active_domains - ] - ] + ["DocType", "name", "not in", frappe.model.core_doctypes_list], + ["DocType", "restrict_to_domain", "in", frappe.boot.active_domains], + ], }; }); - frm.set_query("default_print_format", function() { + frm.set_query("default_print_format", function () { return { filters: { print_format_type: ["!=", "JS"], - doc_type: ["=", frm.doc.doc_type] - } + doc_type: ["=", frm.doc.doc_type], + }, }; }); - $(frm.wrapper).on("grid-row-render", function(e, grid_row) { + $(frm.wrapper).on("grid-row-render", function (e, grid_row) { if (grid_row.doc && grid_row.doc.fieldtype == "Section Break") { $(grid_row.row).css({ "font-weight": "bold" }); } grid_row.row.removeClass("highlight"); - if (grid_row.doc.is_custom_field && - !grid_row.row.hasClass('highlight') && - !grid_row.doc.is_system_generated) { + if ( + grid_row.doc.is_custom_field && + !grid_row.row.hasClass("highlight") && + !grid_row.doc.is_system_generated + ) { grid_row.row.addClass("highlight"); } }); - $(frm.wrapper).on("grid-make-sortable", function(e, frm) { + $(frm.wrapper).on("grid-make-sortable", function (e, frm) { frm.trigger("setup_sortable"); }); - $(frm.wrapper).on("grid-move-row", function(e, frm) { + $(frm.wrapper).on("grid-move-row", function (e, frm) { frm.trigger("setup_sortable"); }); }, - doc_type: function(frm) { + doc_type: function (frm) { if (frm.doc.doc_type) { return frm.call({ method: "fetch_to_customize", doc: frm.doc, freeze: true, - callback: function(r) { + callback: function (r) { if (r) { if (r._server_messages && r._server_messages.length) { frm.set_value("doc_type", ""); } else { frm.refresh(); frm.trigger("setup_sortable"); + frm.trigger("setup_default_views"); } } localStorage["customize_doctype"] = frm.doc.doc_type; - } + }, }); } else { frm.refresh(); } }, - setup_sortable: function(frm) { - frm.doc.fields.forEach(function(f, i) { - if (!f.is_custom_field) { + is_calendar_and_gantt: function (frm) { + frm.trigger("setup_default_views"); + }, + + setup_sortable: function (frm) { + frm.doc.fields.forEach(function (f) { + if (!f.is_custom_field || f.is_system_generated) { f._sortable = false; } if (f.fieldtype == "Table") { frm.add_custom_button( f.options, - function() { + function () { frm.set_value("doc_type", f.options); }, __("Customize Child Table") @@ -109,52 +106,57 @@ frappe.ui.form.on("Customize Form", { frm.fields_dict.fields.grid.refresh(); }, - refresh: function(frm) { + refresh: function (frm) { frm.disable_save(true); frm.page.clear_icons(); if (frm.doc.doc_type) { - frm.page.set_title(__('Customize Form - {0}', [frm.doc.doc_type])); - frappe.customize_form.set_primary_action(frm); + frappe.model.with_doctype(frm.doc.doc_type).then(() => { + frm.page.set_title(__("Customize Form - {0}", [frm.doc.doc_type])); + frappe.customize_form.set_primary_action(frm); - frm.add_custom_button( - __("Go to {0} List", [__(frm.doc.doc_type)]), - function() { - frappe.set_route("List", frm.doc.doc_type); - }, - __("Actions") - ); + render_form_builder_message(frm); - frm.add_custom_button( - __("Reload"), - function() { - frm.script_manager.trigger("doc_type"); - }, - __("Actions") - ); + frm.add_custom_button( + __("Go to {0} List", [__(frm.doc.doc_type)]), + function () { + frappe.set_route("List", frm.doc.doc_type); + }, + __("Actions") + ); - frm.add_custom_button( - __("Reset to defaults"), - function() { - frappe.customize_form.confirm( - __("Remove all customizations?"), - frm - ); - }, - __("Actions") - ); + frm.add_custom_button( + __("Reload"), + function () { + frm.script_manager.trigger("doc_type"); + }, + __("Actions") + ); - frm.add_custom_button( - __("Set Permissions"), - function() { - frappe.set_route("permission-manager", frm.doc.doc_type); - }, - __("Actions") - ); + frm.add_custom_button( + __("Reset to defaults"), + function () { + frappe.customize_form.confirm(__("Remove all customizations?"), frm); + }, + __("Actions") + ); - const is_autoname_autoincrement = frm.doc.autoname === 'autoincrement'; - frm.set_df_property("naming_rule", "hidden", is_autoname_autoincrement); - frm.set_df_property("autoname", "read_only", is_autoname_autoincrement); + frm.add_custom_button( + __("Set Permissions"), + function () { + frappe.set_route("permission-manager", frm.doc.doc_type); + }, + __("Actions") + ); + + const is_autoname_autoincrement = frm.doc.autoname === "autoincrement"; + frm.set_df_property("naming_rule", "hidden", is_autoname_autoincrement); + frm.set_df_property("autoname", "read_only", is_autoname_autoincrement); + frm.toggle_display( + ["queue_in_background"], + frappe.get_meta(frm.doc.doc_type).is_submittable || 0 + ); + }); } frm.events.setup_export(frm); @@ -181,37 +183,38 @@ frappe.ui.form.on("Customize Form", { if (frappe.boot.developer_mode) { frm.add_custom_button( __("Export Customizations"), - function() { + function () { frappe.prompt( [ { fieldtype: "Link", fieldname: "module", options: "Module Def", - label: __("Module to Export") + label: __("Module to Export"), + reqd: 1, }, { fieldtype: "Check", fieldname: "sync_on_migrate", label: __("Sync on Migrate"), - default: 1 + default: 1, }, { fieldtype: "Check", fieldname: "with_permissions", label: __("Export Custom Permissions"), - default: 1 - } + default: 1, + }, ], - function(data) { + function (data) { frappe.call({ method: "frappe.modules.utils.export_customizations", args: { doctype: frm.doc.doc_type, module: data.module, sync_on_migrate: data.sync_on_migrate, - with_permissions: data.with_permissions - } + with_permissions: data.with_permissions, + }, }); }, __("Select Module") @@ -225,127 +228,190 @@ frappe.ui.form.on("Customize Form", { setup_sort_order(frm) { // sort order select if (frm.doc.doc_type) { - var fields = $.map(frm.doc.fields, function(df) { - return frappe.model.is_value_type(df.fieldtype) - ? df.fieldname - : null; + var fields = $.map(frm.doc.fields, function (df) { + return frappe.model.is_value_type(df.fieldtype) ? df.fieldname : null; }); fields = ["", "name", "modified"].concat(fields); frm.set_df_property("sort_field", "options", fields); } - } + }, + + setup_default_views(frm) { + frappe.model.set_default_views_for_doctype(frm.doc.doc_type, frm); + }, }); // can't delete standard fields frappe.ui.form.on("Customize Form Field", { - before_fields_remove: function(frm, doctype, name) { - var row = frappe.get_doc(doctype, name); + before_fields_remove: function (frm, doctype, name) { + const row = frappe.get_doc(doctype, name); + + if (row.is_system_generated) { + frappe.throw( + __( + "Cannot delete system generated field {0}. You can hide it instead.", + [__(row.label) || row.fieldname] + ) + ); + } + if (!(row.is_custom_field || row.__islocal)) { - frappe.msgprint(__("Cannot delete standard field. You can hide it if you want")); - throw "cannot delete standard field"; + frappe.throw( + __("Cannot delete standard field {0}. You can hide it instead.", [ + __(row.label) || row.fieldname, + ]) + ); } }, - fields_add: function(frm, cdt, cdn) { + fields_add: function (frm, cdt, cdn) { var f = frappe.model.get_doc(cdt, cdn); f.is_system_generated = false; f.is_custom_field = true; - } + frm.trigger("setup_default_views"); + }, + + form_render(frm, doctype, docname) { + frm.trigger("setup_fetch_from_fields", doctype, docname); + }, }); // can't delete standard links frappe.ui.form.on("DocType Link", { - before_links_remove: function(frm, doctype, name) { + before_links_remove: function (frm, doctype, name) { let row = frappe.get_doc(doctype, name); if (!(row.custom || row.__islocal)) { frappe.msgprint(__("Cannot delete standard link. You can hide it if you want")); throw "cannot delete standard link"; } }, - links_add: function(frm, cdt, cdn) { + links_add: function (frm, cdt, cdn) { let f = frappe.model.get_doc(cdt, cdn); f.custom = 1; - } + }, }); // can't delete standard actions frappe.ui.form.on("DocType Action", { - before_actions_remove: function(frm, doctype, name) { + before_actions_remove: function (frm, doctype, name) { let row = frappe.get_doc(doctype, name); if (!(row.custom || row.__islocal)) { frappe.msgprint(__("Cannot delete standard action. You can hide it if you want")); throw "cannot delete standard action"; } }, - actions_add: function(frm, cdt, cdn) { + actions_add: function (frm, cdt, cdn) { let f = frappe.model.get_doc(cdt, cdn); f.custom = 1; - } + }, }); // can't delete standard states frappe.ui.form.on("DocType State", { - before_states_remove: function(frm, doctype, name) { + before_states_remove: function (frm, doctype, name) { let row = frappe.get_doc(doctype, name); if (!(row.custom || row.__islocal)) { frappe.msgprint(__("Cannot delete standard document state.")); throw "cannot delete standard document state"; } }, - states_add: function(frm, cdt, cdn) { + states_add: function (frm, cdt, cdn) { let f = frappe.model.get_doc(cdt, cdn); f.custom = 1; - } + }, }); -frappe.customize_form.set_primary_action = function(frm) { - frm.page.set_primary_action(__("Update"), function() { - if (frm.doc.doc_type) { - return frm.call({ - doc: frm.doc, - freeze: true, - btn: frm.page.btn_primary, - method: "save_customization", - callback: function(r) { - if (!r.exc) { - frappe.customize_form.clear_locals_and_refresh(frm); - frm.script_manager.trigger("doc_type"); - } - } - }); +frappe.customize_form.validate_fieldnames = async function (frm) { + for (let i = 0; i < frm.doc.fields.length; i++) { + let field = frm.doc.fields[i]; + + let fieldname = field.label && frappe.model.scrub(field.label).toLowerCase(); + if ( + field.label && + !field.fieldname && + in_list(frappe.model.restricted_fields, fieldname) + ) { + let message = __( + "For field {0} in row {1}, fieldname {2} is restricted it will be renamed as {2}1. Do you want to continue?", + [field.label, field.idx, fieldname] + ); + await pause_to_confirm(message); } + } + + function pause_to_confirm(message) { + return new Promise((resolve) => { + frappe.confirm( + message, + () => resolve(), + () => { + frm.page.btn_primary.prop("disabled", false); + } + ); + }); + } +}; + +frappe.customize_form.save_customization = function (frm) { + if (frm.doc.doc_type) { + return frm.call({ + doc: frm.doc, + freeze: true, + freeze_message: __("Saving Customization..."), + btn: frm.page.btn_primary, + method: "save_customization", + callback: function (r) { + if (!r.exc) { + frappe.customize_form.clear_locals_and_refresh(frm); + frm.script_manager.trigger("doc_type"); + } + }, + }); + } +}; + +frappe.customize_form.set_primary_action = function (frm) { + frm.page.set_primary_action(__("Update"), async () => { + await this.validate_fieldnames(frm); + this.save_customization(frm); }); }; -frappe.customize_form.confirm = function(msg, frm) { +frappe.customize_form.confirm = function (msg, frm) { if (!frm.doc.doc_type) return; var d = new frappe.ui.Dialog({ - title: 'Reset To Defaults', + title: "Reset To Defaults", fields: [ - {fieldtype:"HTML", options:__("All customizations will be removed. Please confirm.")}, + { + fieldtype: "HTML", + options: __("All customizations will be removed. Please confirm."), + }, ], - primary_action: function() { + primary_action: function () { return frm.call({ doc: frm.doc, method: "reset_to_defaults", - callback: function(r) { + callback: function (r) { if (r.exc) { frappe.msgprint(r.exc); } else { d.hide(); - frappe.show_alert({message:__('Customizations Reset'), indicator:'green'}); + frappe.show_alert({ + message: __("Customizations Reset"), + indicator: "green", + }); frappe.customize_form.clear_locals_and_refresh(frm); } - } + }, }); - } + }, }); frappe.customize_form.confirm.dialog = d; d.show(); -} +}; -frappe.customize_form.clear_locals_and_refresh = function(frm) { +frappe.customize_form.clear_locals_and_refresh = function (frm) { delete frm.doc.__unsaved; // clear doctype from locals frappe.model.clear_doc("DocType", frm.doc.doc_type); @@ -353,4 +419,31 @@ frappe.customize_form.clear_locals_and_refresh = function(frm) { frm.refresh(); }; -extend_cscript(cur_frm.cscript, new frappe.model.DocTypeController({frm: cur_frm})); +function render_form_builder_message(frm) { + $(frm.fields_dict["try_form_builder_html"].wrapper).empty(); + if (!frm.is_new() && frm.fields_dict["try_form_builder_html"]) { + let title = __("Use Form Builder to visually customize your form layout"); + let msg = __( + "You can drag and drop fields to create your form layout, add tabs, sections and columns to organize your form and update field properties all from one screen." + ); + + let message = ` +
    +
    + +
    + `; + + $(frm.fields_dict["try_form_builder_html"].wrapper).html(message); + } +} + +extend_cscript(cur_frm.cscript, new frappe.model.DocTypeController({ frm: cur_frm })); diff --git a/frappe/custom/doctype/customize_form/customize_form.json b/frappe/custom/doctype/customize_form/customize_form.json index 0011f51af4..e0d822eb61 100644 --- a/frappe/custom/doctype/customize_form/customize_form.json +++ b/frappe/custom/doctype/customize_form/customize_form.json @@ -10,28 +10,36 @@ "doc_type", "properties", "label", - "max_attachments", "search_fields", "column_break_5", - "allow_copy", "istable", + "is_calendar_and_gantt", "editable_grid", "quick_entry", "track_changes", "track_views", "allow_auto_repeat", "allow_import", + "queue_in_background", "fields_section_break", + "try_form_builder_html", "fields", "naming_section", "naming_rule", "autoname", + "form_settings_section", + "image_field", + "max_attachments", + "column_break_21", + "allow_copy", + "make_attachments_public", "view_settings_section", "title_field", "show_title_field_in_link", - "translate_link_fields", - "image_field", + "translated_doctype", "default_print_format", + "default_view", + "force_re_route_to_default_view", "column_break_29", "show_preview_popup", "email_settings_section", @@ -315,9 +323,55 @@ }, { "default": "0", - "fieldname": "translate_link_fields", + "fieldname": "translated_doctype", "fieldtype": "Check", "label": "Translate Link Fields" + }, + { + "collapsible": 1, + "fieldname": "form_settings_section", + "fieldtype": "Section Break", + "label": "Form Settings" + }, + { + "fieldname": "column_break_21", + "fieldtype": "Column Break" + }, + { + "default": "0", + "fieldname": "make_attachments_public", + "fieldtype": "Check", + "label": "Make Attachments Public by Default" + }, + { + "default": "0", + "fieldname": "queue_in_background", + "fieldtype": "Check", + "label": "Queue in Background" + }, + { + "fieldname": "default_view", + "fieldtype": "Select", + "label": "Default View" + }, + { + "default": "0", + "depends_on": "default_view", + "fieldname": "force_re_route_to_default_view", + "fieldtype": "Check", + "label": "Force Re-route to Default View" + }, + { + "default": "0", + "description": "Enables Calendar and Gantt views.", + "fieldname": "is_calendar_and_gantt", + "fieldtype": "Check", + "label": "Is Calendar and Gantt" + }, + { + "fieldname": "try_form_builder_html", + "fieldtype": "HTML", + "label": "Try Form Builder HTML" } ], "hide_toolbar": 1, @@ -326,7 +380,7 @@ "index_web_pages_for_search": 1, "issingle": 1, "links": [], - "modified": "2022-05-13 15:36:16.772277", + "modified": "2023-05-15 16:03:19.872532", "modified_by": "Administrator", "module": "Custom", "name": "Customize Form", diff --git a/frappe/custom/doctype/customize_form/customize_form.py b/frappe/custom/doctype/customize_form/customize_form.py index 4923bfc525..42cbf33f4f 100644 --- a/frappe/custom/doctype/customize_form/customize_form.py +++ b/frappe/custom/doctype/customize_form/customize_form.py @@ -193,8 +193,9 @@ class CustomizeForm(Document): # docfield for df in self.get("fields"): meta_df = meta.get("fields", {"fieldname": df.fieldname}) - if not meta_df or meta_df[0].get("is_custom_field"): + if not meta_df or not is_standard_or_system_generated_field(meta_df[0]): continue + self.set_property_setters_for_docfield(meta, df, meta_df) # action and links @@ -350,12 +351,14 @@ class CustomizeForm(Document): def update_custom_fields(self): for i, df in enumerate(self.get("fields")): - if df.get("is_custom_field"): - if not frappe.db.exists("Custom Field", {"dt": self.doc_type, "fieldname": df.fieldname}): - self.add_custom_field(df, i) - self.flags.update_db = True - else: - self.update_in_custom_field(df, i) + if is_standard_or_system_generated_field(df): + continue + + if not frappe.db.exists("Custom Field", {"dt": self.doc_type, "fieldname": df.fieldname}): + self.add_custom_field(df, i) + self.flags.update_db = True + else: + self.update_in_custom_field(df, i) self.delete_custom_fields() @@ -374,10 +377,13 @@ class CustomizeForm(Document): d.insert() df.fieldname = d.fieldname + if df.get("in_global_search"): + self.flags.rebuild_doctype_for_global_search = True + def update_in_custom_field(self, df, i): meta = frappe.get_meta(self.doc_type) meta_df = meta.get("fields", {"fieldname": df.fieldname}) - if not (meta_df and meta_df[0].get("is_custom_field")): + if not meta_df or is_standard_or_system_generated_field(meta_df[0]): # not a custom field return @@ -387,6 +393,8 @@ class CustomizeForm(Document): if df.get(prop) != custom_field.get(prop): if prop == "fieldtype": self.validate_fieldtype_change(df, meta_df[0].get(prop), df.get(prop)) + if prop == "in_global_search": + self.flags.rebuild_doctype_for_global_search = True custom_field.set(prop, df.get(prop)) changed = True @@ -411,7 +419,7 @@ class CustomizeForm(Document): } for fieldname in fields_to_remove: df = meta.get("fields", {"fieldname": fieldname})[0] - if df.get("is_custom_field"): + if not is_standard_or_system_generated_field(df): frappe.delete_doc("Custom Field", df.name) def make_property_setter( @@ -556,6 +564,10 @@ def reset_customization(doctype): frappe.clear_cache(doctype=doctype) +def is_standard_or_system_generated_field(df): + return not df.get("is_custom_field") or df.get("is_system_generated") + + doctype_properties = { "search_fields": "Data", "title_field": "Data", @@ -566,8 +578,10 @@ doctype_properties = { "allow_copy": "Check", "istable": "Check", "quick_entry": "Check", + "queue_in_background": "Check", "editable_grid": "Check", "max_attachments": "Int", + "make_attachments_public": "Check", "track_changes": "Check", "track_views": "Check", "allow_auto_repeat": "Check", @@ -581,6 +595,10 @@ doctype_properties = { "autoname": "Data", "show_title_field_in_link": "Check", "translate_link_fields": "Check", + "is_calendar_and_gantt": "Check", + "default_view": "Select", + "force_re_route_to_default_view": "Check", + "translated_doctype": "Check", } docfield_properties = { diff --git a/frappe/custom/doctype/customize_form/test_customize_form.py b/frappe/custom/doctype/customize_form/test_customize_form.py index b00f45f5d2..8d98dc4149 100644 --- a/frappe/custom/doctype/customize_form/test_customize_form.py +++ b/frappe/custom/doctype/customize_form/test_customize_form.py @@ -2,17 +2,17 @@ # License: MIT. See LICENSE import json -import unittest import frappe from frappe.core.doctype.doctype.doctype import InvalidFieldNameError from frappe.core.doctype.doctype.test_doctype import new_doctype from frappe.test_runner import make_test_records_for_doctype +from frappe.tests.utils import FrappeTestCase test_dependencies = ["Custom Field", "Property Setter"] -class TestCustomizeForm(unittest.TestCase): +class TestCustomizeForm(FrappeTestCase): def insert_custom_field(self): frappe.delete_doc_if_exists("Custom Field", "Event-test_custom_field") frappe.get_doc( @@ -54,7 +54,7 @@ class TestCustomizeForm(unittest.TestCase): d = self.get_customize_form("Event") self.assertEqual(d.doc_type, "Event") - self.assertEqual(len(d.get("fields")), 36) + self.assertEqual(len(d.get("fields")), 38) d = self.get_customize_form("Event") self.assertEqual(d.doc_type, "Event") @@ -403,3 +403,25 @@ class TestCustomizeForm(unittest.TestCase): with self.assertRaises(frappe.ValidationError): d.run_method("save_customization") + + def test_system_generated_fields(self): + doctype = "Event" + custom_field_name = "test_custom_field" + + custom_field = frappe.get_doc("Custom Field", {"dt": doctype, "fieldname": custom_field_name}) + custom_field.is_system_generated = 1 + custom_field.save() + + d = self.get_customize_form(doctype) + custom_field = d.getone("fields", {"fieldname": custom_field_name}) + custom_field.description = "Test Description" + d.run_method("save_customization") + + property_setter_filters = { + "doc_type": doctype, + "field_name": custom_field_name, + "property": "description", + } + self.assertEqual( + frappe.db.get_value("Property Setter", property_setter_filters, "value"), "Test Description" + ) diff --git a/frappe/custom/doctype/customize_form_field/customize_form_field.json b/frappe/custom/doctype/customize_form_field/customize_form_field.json index 8fa054894f..d8da44101b 100644 --- a/frappe/custom/doctype/customize_form_field/customize_form_field.json +++ b/frappe/custom/doctype/customize_form_field/customize_form_field.json @@ -212,6 +212,7 @@ }, { "default": "0", + "depends_on": "eval:!in_list(['Section Break', 'Column Break', 'Tab Break'], doc.fieldtype)", "fieldname": "permlevel", "fieldtype": "Int", "in_list_view": 1, @@ -348,7 +349,7 @@ "width": "50px" }, { - "depends_on": "eval:cur_frm.doc.istable", + "depends_on": "eval:parent.istable", "description": "Number of columns for a field in a Grid (Total Columns in a grid should be less than 11)", "fieldname": "columns", "fieldtype": "Int", @@ -467,7 +468,7 @@ "index_web_pages_for_search": 1, "istable": 1, "links": [], - "modified": "2022-04-13 22:31:14.162661", + "modified": "2023-02-20 12:07:40.242470", "modified_by": "Administrator", "module": "Custom", "name": "Customize Form Field", @@ -477,4 +478,4 @@ "sort_field": "modified", "sort_order": "ASC", "states": [] -} +} \ No newline at end of file diff --git a/frappe/custom/doctype/doctype_layout/doctype_layout.js b/frappe/custom/doctype/doctype_layout/doctype_layout.js index 533efea9b8..b212b79a5b 100644 --- a/frappe/custom/doctype/doctype_layout/doctype_layout.js +++ b/frappe/custom/doctype/doctype_layout/doctype_layout.js @@ -1,30 +1,105 @@ // Copyright (c) 2020, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('DocType Layout', { - refresh: function(frm) { - frm.trigger('document_type'); - frm.events.set_button(frm); - }, +frappe.ui.form.on("DocType Layout", { + onload_post_render(frm) { + // disallow users from manually adding/deleting rows; this doctype should only + // be used for managing layout, and docfields and custom fields should be used + // to manage other field metadata (hidden, etc.) + frm.set_df_property("fields", "cannot_add_rows", true); + frm.set_df_property("fields", "cannot_delete_rows", true); - document_type(frm) { - frm.set_fields_as_options('fields', frm.doc.document_type, null, [], 'fieldname').then(() => { - // child table empty? then show all fields as default - if (frm.doc.document_type) { - if (!(frm.doc.fields || []).length) { - for (let f of frappe.get_doc('DocType', frm.doc.document_type).fields) { - frm.add_child('fields', { fieldname: f.fieldname, label: f.label }); - } - } - } + $(frm.wrapper).on("grid-move-row", (e, frm) => { + // refresh the layout after moving a row + frm.dirty(); }); }, - set_button(frm) { + refresh(frm) { + frm.events.add_buttons(frm); + }, + + async document_type(frm) { + if (frm.doc.document_type) { + // refreshing the doctype fields resets the new name input field; + // once the fields are set, reset the name to the original input + if (frm.is_new()) { + const document_name = frm.doc.__newname || frm.doc.name; + } + + frm.set_value("fields", []); + await frm.events.sync_fields(frm, false); + + if (frm.is_new()) { + frm.doc.__newname = document_name; + frm.refresh_field("__newname"); + } + } + }, + + add_buttons(frm) { if (!frm.is_new()) { - frm.add_custom_button(__('Go to {0} List', [frm.doc.name]), () => { + frm.add_custom_button(__("Go to {0} List", [frm.doc.name]), () => { window.open(`/app/${frappe.router.slug(frm.doc.name)}`); }); + + frm.add_custom_button(__("Sync {0} Fields", [frm.doc.name]), async () => { + await frm.events.sync_fields(frm, true); + }); } - } + }, + + async sync_fields(frm, notify) { + frappe.dom.freeze("Fetching fields..."); + const response = await frm.call({ doc: frm.doc, method: "sync_fields" }); + frm.refresh_field("fields"); + frappe.dom.unfreeze(); + + if (!response.message) { + frappe.msgprint(__("No changes to sync")); + return; + } + + frm.dirty(); + if (notify) { + const addedFields = response.message.added; + const removedFields = response.message.removed; + + const getChangedMessage = (fields) => { + let changes = ""; + for (const field of fields) { + if (field.label) { + changes += `
  • Row #${field.idx}: ${field.fieldname.bold()} (${ + field.label + })
    • `; + } else { + changes += `
  • Row #${field.idx}: ${field.fieldname.bold()}
    • `; + } + } + return changes; + }; + + let message = ""; + + if (addedFields.length) { + message += `The following fields have been added:

      ${getChangedMessage( + addedFields + )}
    `; + } + + if (removedFields.length) { + message += `The following fields have been removed:

      ${getChangedMessage( + removedFields + )}
    `; + } + + if (message) { + frappe.msgprint({ + message: __(message), + indicator: "green", + title: __("Synced Fields"), + }); + } + } + }, }); diff --git a/frappe/custom/doctype/doctype_layout/doctype_layout.json b/frappe/custom/doctype/doctype_layout/doctype_layout.json index e47c9e03e0..ffb5cdae31 100644 --- a/frappe/custom/doctype/doctype_layout/doctype_layout.json +++ b/frappe/custom/doctype/doctype_layout/doctype_layout.json @@ -1,7 +1,7 @@ { "actions": [], "allow_rename": 1, - "autoname": "Prompt", + "autoname": "prompt", "creation": "2020-11-16 17:05:35.306846", "doctype": "DocType", "editable_grid": 1, @@ -19,7 +19,8 @@ "in_list_view": 1, "label": "Document Type", "options": "DocType", - "reqd": 1 + "reqd": 1, + "set_only_once": 1 }, { "fieldname": "fields", @@ -42,10 +43,11 @@ ], "index_web_pages_for_search": 1, "links": [], - "modified": "2020-12-10 15:01:04.352184", + "modified": "2023-02-14 17:53:24.486171", "modified_by": "Administrator", "module": "Custom", "name": "DocType Layout", + "naming_rule": "Set by user", "owner": "Administrator", "permissions": [ { @@ -62,11 +64,12 @@ }, { "read": 1, - "role": "Guest" + "role": "All" } ], "route": "doctype-layout", "sort_field": "modified", "sort_order": "DESC", + "states": [], "track_changes": 1 } \ No newline at end of file diff --git a/frappe/custom/doctype/doctype_layout/doctype_layout.py b/frappe/custom/doctype/doctype_layout/doctype_layout.py index ea8e9acc99..f712853ccd 100644 --- a/frappe/custom/doctype/doctype_layout/doctype_layout.py +++ b/frappe/custom/doctype/doctype_layout/doctype_layout.py @@ -1,11 +1,77 @@ # Copyright (c) 2020, Frappe Technologies and contributors # License: MIT. See LICENSE +from typing import TYPE_CHECKING + +import frappe from frappe.desk.utils import slug from frappe.model.document import Document +if TYPE_CHECKING: + from frappe.core.doctype.docfield.docfield import DocField + class DocTypeLayout(Document): def validate(self): if not self.route: self.route = slug(self.name) + + @frappe.whitelist() + def sync_fields(self): + doctype_fields = frappe.get_meta(self.document_type).fields + + if self.is_new(): + added_fields = [field.fieldname for field in doctype_fields] + removed_fields = [] + else: + doctype_fieldnames = {field.fieldname for field in doctype_fields} + layout_fieldnames = {field.fieldname for field in self.fields} + added_fields = list(doctype_fieldnames - layout_fieldnames) + removed_fields = list(layout_fieldnames - doctype_fieldnames) + + if not (added_fields or removed_fields): + return + + added = self.add_fields(added_fields, doctype_fields) + removed = self.remove_fields(removed_fields) + + for index, field in enumerate(self.fields): + field.idx = index + 1 + + return {"added": added, "removed": removed} + + def add_fields(self, added_fields: list[str], doctype_fields: list["DocField"]) -> list[dict]: + added = [] + for field in added_fields: + field_details = next((f for f in doctype_fields if f.fieldname == field), None) + if not field_details: + continue + + # remove 'doctype' data from the DocField to allow adding it to the layout + row = self.append("fields", field_details.as_dict(no_default_fields=True)) + row_data = row.as_dict() + + if field_details.get("insert_after"): + insert_after = next( + (f for f in self.fields if f.fieldname == field_details.insert_after), + None, + ) + + # initialize new row to just after the insert_after field + if insert_after: + self.fields.insert(insert_after.idx, row) + self.fields.pop() + + row_data = {"idx": insert_after.idx + 1, "fieldname": row.fieldname, "label": row.label} + + added.append(row_data) + return added + + def remove_fields(self, removed_fields: list[str]) -> list[dict]: + removed = [] + for field in removed_fields: + field_details = next((f for f in self.fields if f.fieldname == field), None) + if field_details: + self.remove(field_details) + removed.append(field_details.as_dict()) + return removed diff --git a/frappe/custom/doctype/doctype_layout/patches/convert_web_forms_to_doctype_layout.py b/frappe/custom/doctype/doctype_layout/patches/convert_web_forms_to_doctype_layout.py index 59cdfffb21..7d22ee3c7d 100644 --- a/frappe/custom/doctype/doctype_layout/patches/convert_web_forms_to_doctype_layout.py +++ b/frappe/custom/doctype/doctype_layout/patches/convert_web_forms_to_doctype_layout.py @@ -2,7 +2,7 @@ import frappe def execute(): - for web_form_name in frappe.db.get_all("Web Form", pluck="name"): + for web_form_name in frappe.get_all("Web Form", pluck="name"): web_form = frappe.get_doc("Web Form", web_form_name) doctype_layout = frappe.get_doc( dict( diff --git a/frappe/custom/doctype/doctype_layout/test_doctype_layout.py b/frappe/custom/doctype/doctype_layout/test_doctype_layout.py index 0e64a9e727..c568cd4df7 100644 --- a/frappe/custom/doctype/doctype_layout/test_doctype_layout.py +++ b/frappe/custom/doctype/doctype_layout/test_doctype_layout.py @@ -1,8 +1,8 @@ # Copyright (c) 2020, Frappe Technologies and Contributors # License: MIT. See LICENSE # import frappe -import unittest +from frappe.tests.utils import FrappeTestCase -class TestDocTypeLayout(unittest.TestCase): +class TestDocTypeLayout(FrappeTestCase): pass diff --git a/frappe/custom/doctype/property_setter/property_setter.js b/frappe/custom/doctype/property_setter/property_setter.js index bff5ad0e63..955e01c33e 100644 --- a/frappe/custom/doctype/property_setter/property_setter.js +++ b/frappe/custom/doctype/property_setter/property_setter.js @@ -1,10 +1,10 @@ // Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors // MIT License. See license.txt -frappe.ui.form.on('Property Setter', { - validate: function(frm) { - if(frm.doc.property_type=='Check' && !in_list(['0','1'], frm.doc.value)) { - frappe.throw(__('Value for a check field can be either 0 or 1')); +frappe.ui.form.on("Property Setter", { + validate: function (frm) { + if (frm.doc.property_type == "Check" && !in_list(["0", "1"], frm.doc.value)) { + frappe.throw(__("Value for a check field can be either 0 or 1")); } - } + }, }); diff --git a/frappe/custom/doctype/property_setter/property_setter.py b/frappe/custom/doctype/property_setter/property_setter.py index 3034904381..bac616356d 100644 --- a/frappe/custom/doctype/property_setter/property_setter.py +++ b/frappe/custom/doctype/property_setter/property_setter.py @@ -21,6 +21,9 @@ class PropertySetter(Document): delete_property_setter(self.doc_type, self.property, self.field_name, self.row_name) frappe.clear_cache(doctype=self.doc_type) + def on_trash(self): + frappe.clear_cache(doctype=self.doc_type) + def validate_fieldtype_change(self): if self.property == "fieldtype" and self.field_name in not_allowed_fieldtype_change: frappe.throw(_("Field type cannot be changed for {0}").format(self.field_name)) diff --git a/frappe/custom/doctype/property_setter/test_property_setter.py b/frappe/custom/doctype/property_setter/test_property_setter.py index 1fa2d2cefb..dc74a69161 100644 --- a/frappe/custom/doctype/property_setter/test_property_setter.py +++ b/frappe/custom/doctype/property_setter/test_property_setter.py @@ -1,9 +1,9 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import unittest +from frappe.tests.utils import FrappeTestCase # test_records = frappe.get_test_records('Property Setter') -class TestPropertySetter(unittest.TestCase): +class TestPropertySetter(FrappeTestCase): pass diff --git a/frappe/custom/doctype/test_rename_new/test_rename_new.json b/frappe/custom/doctype/test_rename_new/test_rename_new.json deleted file mode 100644 index 0b089091a1..0000000000 --- a/frappe/custom/doctype/test_rename_new/test_rename_new.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "actions": [], - "creation": "2021-01-13 12:47:03.572640", - "doctype": "DocType", - "editable_grid": 1, - "engine": "InnoDB", - "field_order": [ - "random" - ], - "fields": [ - { - "fieldname": "random", - "fieldtype": "Data", - "label": "random" - } - ], - "index_web_pages_for_search": 1, - "links": [], - "modified": "2021-01-13 12:47:03.572640", - "modified_by": "Administrator", - "module": "Custom", - "name": "Test rename new", - "owner": "Administrator", - "permissions": [ - { - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "share": 1, - "write": 1 - } - ], - "route": "test-rename", - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1 -} \ No newline at end of file diff --git a/frappe/custom/doctype/test_rename_new/test_rename_new.py b/frappe/custom/doctype/test_rename_new/test_rename_new.py deleted file mode 100644 index ed89d1fad1..0000000000 --- a/frappe/custom/doctype/test_rename_new/test_rename_new.py +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright (c) 2021, Frappe Technologies and contributors -# License: MIT. See LICENSE - -# import frappe -from frappe.model.document import Document - - -class Testrenamenew(Document): - pass diff --git a/frappe/custom/doctype/test_rename_new/test_test_rename_new.py b/frappe/custom/doctype/test_rename_new/test_test_rename_new.py deleted file mode 100644 index f1ccf42ede..0000000000 --- a/frappe/custom/doctype/test_rename_new/test_test_rename_new.py +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright (c) 2021, Frappe Technologies and Contributors -# License: MIT. See LICENSE -# import frappe -import unittest - - -class Testrenamenew(unittest.TestCase): - pass diff --git a/frappe/custom/fixtures/temp_doctype.json b/frappe/custom/fixtures/temp_doctype.json index 343aa2cb37..20b3d9caa9 100644 --- a/frappe/custom/fixtures/temp_doctype.json +++ b/frappe/custom/fixtures/temp_doctype.json @@ -18,7 +18,6 @@ "beta": 0, "is_virtual": 0, "naming_rule": "", - "name_case": "", "allow_rename": 1, "hide_toolbar": 0, "allow_copy": 0, @@ -58,7 +57,6 @@ "report": 1, "export": 1, "import": 0, - "set_user_permissions": 0, "share": 1, "print": 1, "email": 1, diff --git a/frappe/custom/fixtures/temp_singles.json b/frappe/custom/fixtures/temp_singles.json index b7e2536f25..723f47d7ac 100644 --- a/frappe/custom/fixtures/temp_singles.json +++ b/frappe/custom/fixtures/temp_singles.json @@ -18,7 +18,6 @@ "beta": 0, "is_virtual": 0, "naming_rule": "", - "name_case": "", "allow_rename": 1, "hide_toolbar": 0, "allow_copy": 0, @@ -58,7 +57,6 @@ "report": 1, "export": 1, "import": 0, - "set_user_permissions": 0, "share": 1, "print": 1, "email": 1, diff --git a/frappe/desk/page/activity/__init__.py b/frappe/custom/report/__init__.py similarity index 100% rename from frappe/desk/page/activity/__init__.py rename to frappe/custom/report/__init__.py diff --git a/frappe/desk/page/translation_tool/__init__.py b/frappe/custom/report/audit_system_hooks/__init__.py similarity index 100% rename from frappe/desk/page/translation_tool/__init__.py rename to frappe/custom/report/audit_system_hooks/__init__.py diff --git a/frappe/custom/report/audit_system_hooks/audit_system_hooks.js b/frappe/custom/report/audit_system_hooks/audit_system_hooks.js new file mode 100644 index 0000000000..a78464f3da --- /dev/null +++ b/frappe/custom/report/audit_system_hooks/audit_system_hooks.js @@ -0,0 +1,7 @@ +// Copyright (c) 2023, Frappe Technologies and contributors +// For license information, please see license.txt +/* eslint-disable */ + +frappe.query_reports["Audit System Hooks"] = { + filters: [], +}; diff --git a/frappe/custom/report/audit_system_hooks/audit_system_hooks.json b/frappe/custom/report/audit_system_hooks/audit_system_hooks.json new file mode 100644 index 0000000000..b13a43a0c5 --- /dev/null +++ b/frappe/custom/report/audit_system_hooks/audit_system_hooks.json @@ -0,0 +1,27 @@ +{ + "add_total_row": 0, + "columns": [], + "creation": "2023-01-25 15:02:21.896117", + "disabled": 0, + "docstatus": 0, + "doctype": "Report", + "filters": [], + "idx": 0, + "is_standard": "Yes", + "letter_head": "", + "modified": "2023-01-31 14:53:37.778576", + "modified_by": "Administrator", + "module": "Custom", + "name": "Audit System Hooks", + "owner": "Administrator", + "prepared_report": 0, + "query": "", + "ref_doctype": "Property Setter", + "report_name": "Audit System Hooks", + "report_type": "Script Report", + "roles": [ + { + "role": "System Manager" + } + ] +} \ No newline at end of file diff --git a/frappe/custom/report/audit_system_hooks/audit_system_hooks.py b/frappe/custom/report/audit_system_hooks/audit_system_hooks.py new file mode 100644 index 0000000000..a42c5c361a --- /dev/null +++ b/frappe/custom/report/audit_system_hooks/audit_system_hooks.py @@ -0,0 +1,70 @@ +# Copyright (c) 2023, Frappe Technologies and contributors +# For license information, please see license.txt + +import frappe + + +def execute(filters=None): + return get_columns(), get_data() + + +def get_columns(): + values_field_type = "Data" # TODO: better text wrapping in reportview + columns = [ + {"label": "Hook name", "fieldname": "hook_name", "fieldtype": "Data", "width": 200}, + {"label": "Hook key (optional)", "fieldname": "hook_key", "fieldtype": "Data", "width": 200}, + {"label": "Hook Values (resolved)", "fieldname": "hook_values", "fieldtype": values_field_type}, + ] + + # Each app is shown in order as a column + installed_apps = frappe.get_installed_apps(_ensure_on_bench=True) + columns += [ + {"label": app, "fieldname": app, "fieldtype": values_field_type} for app in installed_apps + ] + + return columns + + +def get_data(): + hooks = frappe.get_hooks() + installed_apps = frappe.get_installed_apps(_ensure_on_bench=True) + + def fmt_hook_values(v): + """Improve readability by discarding falsy values and removing containers when only 1 + value is in container""" + if not v: + return "" + + v = delist(v) + + if isinstance(v, (dict, list)): + try: + return frappe.as_json(v) + except Exception: + pass + + return str(v) + + data = [] + for hook, values in hooks.items(): + if isinstance(values, dict): + for k, v in values.items(): + row = {"hook_name": hook, "hook_key": fmt_hook_values(k), "hook_values": fmt_hook_values(v)} + for app in installed_apps: + if app_hooks := delist(frappe.get_hooks(hook, app_name=app)): + row[app] = fmt_hook_values(app_hooks.get(k)) + data.append(row) + else: + row = {"hook_name": hook, "hook_values": fmt_hook_values(values)} + for app in installed_apps: + row[app] = fmt_hook_values(frappe.get_hooks(hook, app_name=app)) + + data.append(row) + + return data + + +def delist(val): + if isinstance(val, list) and len(val) == 1: + return val[0] + return val diff --git a/frappe/custom/report/audit_system_hooks/test_audit_system_hooks.py b/frappe/custom/report/audit_system_hooks/test_audit_system_hooks.py new file mode 100644 index 0000000000..cd3edffc77 --- /dev/null +++ b/frappe/custom/report/audit_system_hooks/test_audit_system_hooks.py @@ -0,0 +1,17 @@ +# Copyright (c) 2022, Frappe Technologies and contributors +# For license information, please see license.txt + + +from frappe.custom.report.audit_system_hooks.audit_system_hooks import execute +from frappe.tests.utils import FrappeTestCase + + +class TestAuditSystemHooksReport(FrappeTestCase): + def test_basic_query(self): + _, data = execute() + for row in data: + if row.get("hook_name") == "app_name": + self.assertEqual(row.get("hook_values"), "frappe") + break + else: + self.fail("Failed to generate hooks report") diff --git a/frappe/custom/workspace/customization/customization.json b/frappe/custom/workspace/customization/customization.json index 1756abcb1d..8985bf54ed 100644 --- a/frappe/custom/workspace/customization/customization.json +++ b/frappe/custom/workspace/customization/customization.json @@ -1,6 +1,6 @@ { "charts": [], - "content": "[{\"type\":\"onboarding\",\"data\":{\"onboarding_name\":\"Customization\",\"col\":12}},{\"type\":\"header\",\"data\":{\"text\":\"Your Shortcuts\",\"col\":12}},{\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"Customize Form\",\"col\":3}},{\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"Custom Role\",\"col\":3}},{\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"Client Script\",\"col\":3}},{\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"Server Script\",\"col\":3}},{\"type\":\"spacer\",\"data\":{\"col\":12}},{\"type\":\"header\",\"data\":{\"text\":\"Reports & Masters\",\"col\":12}},{\"type\":\"card\",\"data\":{\"card_name\":\"Dashboards\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Form Customization\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Other\",\"col\":4}}]", + "content": "[{\"type\":\"onboarding\",\"data\":{\"onboarding_name\":\"Customization\",\"col\":12}},{\"type\":\"header\",\"data\":{\"text\":\"Your Shortcuts\",\"col\":12}},{\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"Customize Form\",\"col\":3}},{\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"Custom Role\",\"col\":3}},{\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"Client Script\",\"col\":3}},{\"type\":\"shortcut\",\"data\":{\"shortcut_name\":\"Server Script\",\"col\":3}},{\"type\":\"spacer\",\"data\":{\"col\":12}},{\"type\":\"header\",\"data\":{\"text\":\"Reports & Masters\",\"col\":12}},{\"type\":\"card\",\"data\":{\"card_name\":\"Dashboards\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Form Customization\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Other\",\"col\":4}}]", "creation": "2020-03-02 15:15:03.839594", "docstatus": 0, "doctype": "Workspace", @@ -107,7 +107,7 @@ "hidden": 0, "is_query_report": 0, "label": "Other", - "link_count": 0, + "link_count": 2, "onboard": 0, "type": "Card Break" }, @@ -121,15 +121,26 @@ "link_type": "DocType", "onboard": 0, "type": "Link" + }, + { + "hidden": 0, + "is_query_report": 0, + "label": "Navbar Settings", + "link_count": 0, + "link_to": "Navbar Settings", + "link_type": "DocType", + "onboard": 0, + "type": "Link" } ], - "modified": "2022-01-13 17:28:08.345794", + "modified": "2022-08-28 20:56:24.980719", "modified_by": "Administrator", "module": "Custom", "name": "Customization", "owner": "Administrator", "parent_page": "", "public": 1, + "quick_lists": [], "restrict_to_domain": "", "roles": [], "sequence_id": 8.0, diff --git a/frappe/database/__init__.py b/frappe/database/__init__.py index 7de3fabf01..76ad24b6e6 100644 --- a/frappe/database/__init__.py +++ b/frappe/database/__init__.py @@ -50,16 +50,3 @@ def get_db(host=None, user=None, password=None, port=None): import frappe.database.mariadb.database return frappe.database.mariadb.database.MariaDBDatabase(host, user, password, port=port) - - -def setup_help_database(help_db_name): - import frappe - - if frappe.conf.db_type == "postgres": - import frappe.database.postgres.setup_db - - return frappe.database.postgres.setup_db.setup_help_database(help_db_name) - else: - import frappe.database.mariadb.setup_db - - return frappe.database.mariadb.setup_db.setup_help_database(help_db_name) diff --git a/frappe/database/database.py b/frappe/database/database.py index a7906bc3fb..2d38a6dea8 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -1,28 +1,39 @@ # Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -# Database Module -# -------------------- - import datetime +import itertools +import json import random import re import string -from contextlib import contextmanager +import traceback +from contextlib import contextmanager, suppress from time import time +from typing import Any, Iterable, Sequence +from pypika.dialects import MySQLQueryBuilder, PostgreSQLQueryBuilder from pypika.terms import Criterion, NullValue import frappe import frappe.defaults import frappe.model.meta from frappe import _ -from frappe.exceptions import DoesNotExistError +from frappe.database.utils import ( + DefaultOrderBy, + EmptyQueryValues, + FallBackDateTimeStr, + LazyMogrify, + Query, + QueryValues, + is_query_type, +) +from frappe.exceptions import DoesNotExistError, ImplicitCommitError from frappe.model.utils.link_count import flush_local_link_count from frappe.query_builder.functions import Count -from frappe.query_builder.utils import DocType from frappe.utils import cast as cast_fieldtype -from frappe.utils import get_datetime, get_table_name, getdate, now, sbool +from frappe.utils import cint, get_datetime, get_table_name, getdate, now, sbool +from frappe.utils.deprecations import deprecated, deprecation_warning IFNULL_PATTERN = re.compile(r"ifnull\(", flags=re.IGNORECASE) INDEX_PATTERN = re.compile(r"\s*\([^)]+\)\s*") @@ -30,10 +41,6 @@ SINGLE_WORD_PATTERN = re.compile(r'([`"]?)(tab([A-Z]\w+))\1') MULTI_WORD_PATTERN = re.compile(r'([`"])(tab([A-Z]\w+)( [A-Z]\w+)+)\1') -def is_query_type(query: str, query_type: str | tuple[str]) -> bool: - return query.lstrip().split(maxsplit=1)[0].lower().startswith(query_type) - - class Database: """ Open a database connection with the given parmeters, if use_default is True, use the @@ -51,10 +58,35 @@ class Database: CHILD_TABLE_COLUMNS = ("parent", "parenttype", "parentfield") MAX_WRITES_PER_TRANSACTION = 200_000 + # NOTE: + # FOR MARIADB - using no cache - as during backup, if the sequence was used in anyform, + # it drops the cache and uses the next non cached value in setval query and + # puts that in the backup file, which will start the counter + # from that value when inserting any new record in the doctype. + # By default the cache is 1000 which will mess up the sequence when + # using the system after a restore. + # + # Another case could be if the cached values expire then also there is a chance of + # the cache being skipped. + # + # FOR POSTGRES - The sequence cache for postgres is per connection. + # Since we're opening and closing connections for every request this results in skipping the cache + # to the next non-cached value hence not using cache in postgres. + # ref: https://stackoverflow.com/questions/21356375/postgres-9-0-4-sequence-skipping-numbers + SEQUENCE_CACHE = 0 + class InvalidColumnName(frappe.ValidationError): pass - def __init__(self, host=None, user=None, password=None, ac_name=None, use_default=0, port=None): + def __init__( + self, + host=None, + user=None, + password=None, + ac_name=None, + use_default=0, + port=None, + ): self.setup_type_map() self.host = host or frappe.conf.db_host or "127.0.0.1" self.port = port or frappe.conf.db_port or "" @@ -73,6 +105,8 @@ class Database: self.password = password or frappe.conf.db_password self.value_cache = {} + # self.db_type: str + # self.last_query (lazy) attribute of last sql query executed def setup_type_map(self): pass @@ -84,26 +118,43 @@ class Database: self._cursor = self._conn.cursor() frappe.local.rollback_observers = [] + try: + if execution_timeout := get_query_execution_timeout(): + self.set_execution_timeout(execution_timeout) + except Exception as e: + frappe.logger("database").warning(f"Couldn't set execution timeout {e}") + + def set_execution_timeout(self, seconds: int): + """Set session speicifc timeout on exeuction of statements. + If any statement takes more time it will be killed along with entire transaction.""" + raise NotImplementedError + def use(self, db_name): """`USE` db_name.""" self._conn.select_db(db_name) def get_connection(self): - pass + """Returns a Database connection object that conforms with https://peps.python.org/pep-0249/#connection-objects""" + raise NotImplementedError def get_database_size(self): - pass + raise NotImplementedError + + def _transform_query(self, query: Query, values: QueryValues) -> tuple: + return query, values + + def _transform_result(self, result: list[tuple]) -> list[tuple]: + return result def sql( self, - query, - values=(), + query: Query, + values: QueryValues = EmptyQueryValues, + *, as_dict=0, as_list=0, - formatted=0, debug=0, ignore_ddl=0, - as_utf8=0, auto_commit=0, update=None, explain=False, @@ -113,13 +164,11 @@ class Database: """Execute a SQL query and fetch all rows. :param query: SQL query. - :param values: List / dict of values to be escaped and substituted in the query. + :param values: Tuple / List / Dict of values to be escaped and substituted in the query. :param as_dict: Return as a dictionary. :param as_list: Always return as a list. - :param formatted: Format values like date etc. :param debug: Print query and `EXPLAIN` in debug log. :param ignore_ddl: Catch exception if table, column missing. - :param as_utf8: Encode values as UTF 8. :param auto_commit: Commit after executing the query. :param update: Update this dict to all rows (if returned `as_dict`). :param run: Returns query without executing it if False. @@ -136,6 +185,9 @@ class Database: {"name": "a%", "owner":"test@example.com"}) """ + if isinstance(query, (MySQLQueryBuilder, PostgreSQLQueryBuilder)): + frappe.errprint("Use run method to execute SQL queries generated by Query Engine") + debug = debug or getattr(self, "debug", False) query = str(query) if not run: @@ -152,132 +204,147 @@ class Database: # in transaction validations self.check_transaction_status(query) - self.clear_db_table_cache(query) - # autocommit if auto_commit: self.commit() - # execute + if debug: + time_start = time() + + if values == EmptyQueryValues: + values = None + elif not isinstance(values, (tuple, dict, list)): + values = (values,) + query, values = self._transform_query(query, values) + try: - if debug: - time_start = time() - - self.log_query(query, values, debug, explain) - - if values != (): - - # MySQL-python==1.2.5 hack! - if not isinstance(values, (dict, tuple, list)): - values = (values,) - - self._cursor.execute(query, values) - - if frappe.flags.in_migrate: - self.log_touched_tables(query, values) - - else: - self._cursor.execute(query) - - if frappe.flags.in_migrate: - self.log_touched_tables(query) - - if debug: - time_end = time() - frappe.errprint(("Execution time: {} sec").format(round(time_end - time_start, 2))) - + self._cursor.execute(query, values) except Exception as e: if self.is_syntax_error(e): - # only for mariadb - frappe.errprint("Syntax error in query:") - frappe.errprint(query) + frappe.errprint(f"Syntax error in query:\n{query} {values or ''}") elif self.is_deadlocked(e): - raise frappe.QueryDeadlockError(e) + raise frappe.QueryDeadlockError(e) from e elif self.is_timedout(e): - raise frappe.QueryTimeoutError(e) + raise frappe.QueryTimeoutError(e) from e - elif frappe.conf.db_type == "postgres": - # TODO: added temporarily - import traceback + elif self.is_read_only_mode_error(e): + frappe.throw( + _( + "Site is running in read only mode for maintenance or site update, this action can not be performed right now. Please try again later." + ), + title=_("In Read Only Mode"), + exc=frappe.InReadOnlyMode, + ) + # TODO: added temporarily + elif self.db_type == "postgres": traceback.print_stack() - print(e) + frappe.errprint(f"Error in query:\n{e}") raise - if ignore_ddl and ( - self.is_missing_column(e) or self.is_table_missing(e) or self.cant_drop_field_or_key(e) - ): - pass - else: + elif isinstance(e, self.ProgrammingError): + if frappe.conf.developer_mode: + traceback.print_stack() + frappe.errprint(f"Error in query:\n{query, values}") raise + if not ( + ignore_ddl + and (self.is_missing_column(e) or self.is_table_missing(e) or self.cant_drop_field_or_key(e)) + ): + raise + + if debug: + time_end = time() + frappe.errprint(f"Execution time: {time_end - time_start:.2f} sec") + + self.log_query(query, values, debug, explain) + if auto_commit: self.commit() if not self._cursor.description: return () + self.last_result = self._transform_result(self._cursor.fetchall()) + if pluck: - return [r[0] for r in self._cursor.fetchall()] + return [r[0] for r in self.last_result] # scrub output if required if as_dict: - ret = self.fetch_as_dict(formatted, as_utf8) + ret = self.fetch_as_dict() if update: for r in ret: r.update(update) return ret elif as_list: - return self.convert_to_lists(self._cursor.fetchall(), formatted, as_utf8) - elif as_utf8: - return self.convert_to_lists(self._cursor.fetchall(), formatted, as_utf8) - else: - return self._cursor.fetchall() + return self.convert_to_lists(self.last_result) + return self.last_result - def log_query(self, query, values, debug, explain): - # for debugging in tests - if frappe.conf.get("allow_tests") and frappe.cache().get_value("flag_print_sql"): - print(self.mogrify(query, values)) + def _log_query(self, mogrified_query: str, debug: bool = False, explain: bool = False) -> None: + """Takes the query and logs it to various interfaces according to the settings.""" + _query = None + + if frappe.conf.allow_tests and frappe.cache().get_value("flag_print_sql"): + _query = _query or str(mogrified_query) + print(_query) - # debug if debug: - if explain and is_query_type(query, "select"): - self.explain_query(query, values) - frappe.errprint(self.mogrify(query, values)) + _query = _query or str(mogrified_query) + if explain and is_query_type(_query, "select"): + self.explain_query(_query) + frappe.errprint(_query) - # info - if (frappe.conf.get("logging") or False) == 2: - frappe.log("<<<< query") - frappe.log(self.mogrify(query, values)) - frappe.log(">>>>") + if frappe.conf.logging == 2: + _query = _query or str(mogrified_query) + frappe.log(f"<<<< query\n{_query}\n>>>>") - def mogrify(self, query, values): + if frappe.flags.in_migrate: + _query = _query or str(mogrified_query) + self.log_touched_tables(_query) + + def log_query( + self, query: str, values: QueryValues = None, debug: bool = False, explain: bool = False + ) -> str: + # TODO: Use mogrify until MariaDB Connector/C 1.1 is released and we can fetch something + # like cursor._transformed_statement from the cursor object. We can also avoid setting + # mogrified_query if we don't need to log it. + mogrified_query = self.lazy_mogrify(query, values) + self._log_query(mogrified_query, debug, explain) + return mogrified_query + + def mogrify(self, query: Query, values: QueryValues): """build the query string with values""" if not values: return query - else: - try: - return self._cursor.mogrify(query, values) - except Exception: - return (query, values) + + try: + return self._cursor.mogrify(query, values) + except AttributeError: + if isinstance(values, dict): + return query % {k: frappe.db.escape(v) if isinstance(v, str) else v for k, v in values.items()} + elif isinstance(values, (list, tuple)): + return query % tuple(frappe.db.escape(v) if isinstance(v, str) else v for v in values) + return query, values + + def lazy_mogrify(self, query: Query, values: QueryValues) -> LazyMogrify: + """Wrap the object with str to generate mogrified query.""" + return LazyMogrify(query, values) def explain_query(self, query, values=None): """Print `EXPLAIN` in error log.""" + frappe.errprint("--- query explain ---") try: - frappe.errprint("--- query explain ---") - if values is None: - self._cursor.execute("explain " + query) - else: - self._cursor.execute("explain " + query, values) - import json - + self._cursor.execute(f"EXPLAIN {query}", values) + except Exception as e: + frappe.errprint(f"error in query explain: {e}") + else: frappe.errprint(json.dumps(self.fetch_as_dict(), indent=1)) frappe.errprint("--- query explain end ---") - except Exception: - frappe.errprint("error in query explain") def sql_list(self, query, values=(), debug=False, **kwargs): """Return data as list of single elements (first column). @@ -296,7 +363,7 @@ class Database: self.sql(query, debug=debug) def check_transaction_status(self, query): - """Raises exception if more than 20,000 `INSERT`, `UPDATE` queries are + """Raises exception if more than 200,000 `INSERT`, `UPDATE` queries are executed in one transaction. This is to ensure that writes are always flushed otherwise this could cause the system to hang.""" self.check_implicit_commit(query) @@ -320,58 +387,29 @@ class Database: and query and is_query_type(query, ("start", "alter", "drop", "create", "begin", "truncate")) ): - raise Exception("This statement can cause implicit commit") + raise ImplicitCommitError("This statement can cause implicit commit") - def fetch_as_dict(self, formatted=0, as_utf8=0): + def fetch_as_dict(self) -> list[frappe._dict]: """Internal. Converts results to dict.""" - result = self._cursor.fetchall() - ret = [] + result = self.last_result if result: keys = [column[0] for column in self._cursor.description] - for r in result: - values = [] - for value in r: - if as_utf8 and isinstance(value, str): - value = value.encode("utf-8") - values.append(value) - - ret.append(frappe._dict(zip(keys, values))) - return ret + return [frappe._dict(zip(keys, row)) for row in result] @staticmethod def clear_db_table_cache(query): if query and is_query_type(query, ("drop", "create")): frappe.cache().delete_key("db_tables") - @staticmethod - def needs_formatting(result, formatted): - """Returns true if the first row in the result has a Date, Datetime, Long Int.""" - if result and result[0]: - for v in result[0]: - if isinstance(v, (datetime.date, datetime.timedelta, datetime.datetime, int)): - return True - if formatted and isinstance(v, (int, float)): - return True - - return False - def get_description(self): """Returns result metadata.""" return self._cursor.description @staticmethod - def convert_to_lists(res, formatted=0, as_utf8=0): + def convert_to_lists(res): """Convert tuple output to lists (internal).""" - nres = [] - for r in res: - nr = [] - for val in r: - if as_utf8 and isinstance(val, str): - val = val.encode("utf-8") - nr.append(val) - nres.append(nr) - return nres + return [[value for value in row] for row in res] def get(self, doctype, filters=None, as_dict=True, cache=False): """Returns `get_value` with fieldname='*'""" @@ -385,7 +423,7 @@ class Database: ignore=None, as_dict=False, debug=False, - order_by="KEEP_DEFAULT_ORDERING", + order_by=DefaultOrderBy, cache=False, for_update=False, *, @@ -444,9 +482,8 @@ class Database: if len(row) > 1 or as_dict: return row - else: - # single field is requested, send it without wrapping in containers - return row[0] + # single field is requested, send it without wrapping in containers + return row[0] def get_values( self, @@ -456,7 +493,7 @@ class Database: ignore=None, as_dict=False, debug=False, - order_by="KEEP_DEFAULT_ORDERING", + order_by=DefaultOrderBy, update=None, cache=False, for_update=False, @@ -515,7 +552,7 @@ class Database: if (filters is not None) and (filters != doctype or doctype == "DocType"): try: if order_by: - order_by = "modified" if order_by == "KEEP_DEFAULT_ORDERING" else order_by + order_by = "modified" if order_by == DefaultOrderBy else order_by out = self._get_values_from_table( fields=fields, filters=filters, @@ -571,10 +608,6 @@ class Database: :param filters: Filters (dict). :param doctype: DocType name. """ - # TODO - # if not frappe.model.meta.is_single(doctype): - # raise frappe.DoesNotExistError("DocType", doctype) - if fields == "*" or isinstance(filters, dict): # check if single doc matches with filters values = self.get_singles_dict(doctype) @@ -590,7 +623,7 @@ class Database: return [map(values.get, fields)] else: - r = frappe.qb.engine.get_query( + r = frappe.qb.get_query( "Singles", filters={"field": ("in", tuple(fields)), "doctype": doctype}, fields=["field", "value"], @@ -623,7 +656,7 @@ class Database: # Get coulmn and value of the single doctype Accounts Settings account_settings = frappe.db.get_singles_dict("Accounts Settings") """ - queried_result = frappe.qb.engine.get_query( + queried_result = frappe.qb.get_query( "Singles", filters={"doctype": doctype}, fields=["field", "value"], @@ -657,13 +690,30 @@ class Database: def get_list(*args, **kwargs): return frappe.get_list(*args, **kwargs) + @staticmethod + def _get_update_dict( + fieldname: str | dict, value: Any, *, modified: str, modified_by: str, update_modified: bool + ) -> dict[str, Any]: + """Create update dict that represents column-values to be updated.""" + update_dict = fieldname if isinstance(fieldname, dict) else {fieldname: value} + + if update_modified: + modified = modified or now() + modified_by = modified_by or frappe.session.user + update_dict.update({"modified": modified, "modified_by": modified_by}) + + return update_dict + def set_single_value( self, doctype: str, fieldname: str | dict, value: str | int | None = None, - *args, - **kwargs, + *, + modified=None, + modified_by=None, + update_modified=True, + debug=False, ): """Set field value of Single DocType. @@ -676,7 +726,23 @@ class Database: # Update the `deny_multiple_sessions` field in System Settings DocType. company = frappe.db.set_single_value("System Settings", "deny_multiple_sessions", True) """ - return self.set_value(doctype, doctype, fieldname, value, *args, **kwargs) + + to_update = self._get_update_dict( + fieldname, value, modified=modified, modified_by=modified_by, update_modified=update_modified + ) + + frappe.db.delete( + "Singles", filters={"field": ("in", tuple(to_update)), "doctype": doctype}, debug=debug + ) + + singles_data = ((doctype, key, sbool(value)) for key, value in to_update.items()) + frappe.qb.into("Singles").columns("doctype", "field", "value").insert(*singles_data).run( + debug=debug + ) + frappe.clear_document_cache(doctype, doctype) + + if doctype in self.value_cache: + del self.value_cache[doctype] def get_single_value(self, doctype, fieldname, cache=True): """Get property of Single DocType. Cache locally by default @@ -696,7 +762,7 @@ class Database: if cache and fieldname in self.value_cache[doctype]: return self.value_cache[doctype][fieldname] - val = frappe.qb.engine.get_query( + val = frappe.qb.get_query( table="Singles", filters={"doctype": doctype, "field": fieldname}, fields="value", @@ -707,7 +773,9 @@ class Database: if not df: frappe.throw( - _("Invalid field name: {0}").format(frappe.bold(fieldname)), self.InvalidColumnName + _("Field {0} does not exist on {1}").format( + frappe.bold(fieldname), frappe.bold(doctype), self.InvalidColumnName + ) ) val = cast_fieldtype(df.fieldtype, val) @@ -736,23 +804,19 @@ class Database: distinct=False, limit=None, ): - field_objects = [] - - query = frappe.qb.engine.get_query( + query = frappe.qb.get_query( table=doctype, filters=filters, - orderby=order_by, + order_by=order_by, for_update=for_update, - field_objects=field_objects, fields=fields, distinct=distinct, limit=limit, ) - if fields == "*" and not isinstance(fields, (list, tuple)) and not isinstance(fields, Criterion): + if isinstance(fields, str) and fields == "*": as_dict = True - r = self.sql(query, as_dict=as_dict, debug=debug, update=update, run=run, pluck=pluck) - return r + return query.run(as_dict=as_dict, debug=debug, update=update, run=run, pluck=pluck) def _get_value_for_many_names( self, @@ -768,26 +832,16 @@ class Database: limit=None, as_dict=False, ): - names = list(filter(None, names)) - if names: - return self.get_all( + if names := list(filter(None, names)): + return frappe.qb.get_query( doctype, fields=field, filters=names, order_by=order_by, - pluck=pluck, - debug=debug, - as_list=not as_dict, - run=run, distinct=distinct, - limit_page_length=limit, - ) - else: - return {} - - def update(self, *args, **kwargs): - """Update multiple values. Alias for `set_value`.""" - return self.set_value(*args, **kwargs) + limit=limit, + ).run(debug=debug, run=run, as_dict=as_dict, pluck=pluck) + return {} def set_value( self, @@ -807,90 +861,52 @@ class Database: **Warning:** this function will not call Document events and should be avoided in normal cases. :param dt: DocType name. - :param dn: Document name. + :param dn: Document name for updating single record or filters for updating many records. :param field: Property / field name or dictionary of values to be updated :param value: Value to be updated. :param modified: Use this as the `modified` timestamp. :param modified_by: Set this user as `modified_by`. :param update_modified: default True. Set as false, if you don't want to update the timestamp. :param debug: Print the query in the developer / js console. - :param for_update: Will add a row-level lock to the value that is being set so that it can be released on commit. """ - is_single_doctype = not (dn and dt != dn) - to_update = field if isinstance(field, dict) else {field: val} - if update_modified: - modified = modified or now() - modified_by = modified_by or frappe.session.user - to_update.update({"modified": modified, "modified_by": modified_by}) - - if is_single_doctype: - frappe.db.delete( - "Singles", filters={"field": ("in", tuple(to_update)), "doctype": dt}, debug=debug + if _is_single_doctype := not (dn and dt != dn): + deprecation_warning( + "Calling db.set_value on single doctype is deprecated. This behaviour will be removed in version 15. Use db.set_single_value instead." ) + self.set_single_value( + doctype=dt, + fieldname=field, + value=val, + debug=debug, + update_modified=update_modified, + modified=modified, + modified_by=modified_by, + ) + return - singles_data = ((dt, key, sbool(value)) for key, value in to_update.items()) - query = ( - frappe.qb.into("Singles").columns("doctype", "field", "value").insert(*singles_data) - ).run(debug=debug) - frappe.clear_document_cache(dt, dt) + to_update = self._get_update_dict( + field, val, modified=modified, modified_by=modified_by, update_modified=update_modified + ) + query = frappe.qb.get_query(table=dt, filters=dn, update=True) + + if isinstance(dn, str): + frappe.clear_document_cache(dt, dn) else: - table = DocType(dt) + # TODO: Fix this; doesn't work rn - gavin@frappe.io + # frappe.cache().hdel_keys(dt, "document_cache") + # Workaround: clear all document caches + frappe.cache().delete_value("document_cache") - if for_update: - docnames = tuple( - self.get_values(dt, dn, "name", debug=debug, for_update=for_update, pluck=True) - ) or (NullValue(),) - query = frappe.qb.update(table).where(table.name.isin(docnames)) + for column, value in to_update.items(): + query = query.set(column, value) - for docname in docnames: - frappe.clear_document_cache(dt, docname) - - else: - query = frappe.qb.engine.build_conditions(table=dt, filters=dn, update=True) - # TODO: Fix this; doesn't work rn - gavin@frappe.io - # frappe.cache().hdel_keys(dt, "document_cache") - # Workaround: clear all document caches - frappe.cache().delete_value("document_cache") - - for column, value in to_update.items(): - query = query.set(column, value) - - query.run(debug=debug) + query.run(debug=debug) if dt in self.value_cache: del self.value_cache[dt] - @staticmethod - def set(doc, field, val): - """Set value in document. **Avoid**""" - doc.db_set(field, val) - - def touch(self, doctype, docname): - """Update the modified timestamp of this document.""" - modified = now() - self.sql( - """update `tab{doctype}` set `modified`=%s - where name=%s""".format( - doctype=doctype - ), - (modified, docname), - ) - return modified - - @staticmethod - def set_temp(value): - """Set a temperory value and return a key.""" - key = frappe.generate_hash() - frappe.cache().hset("temp", key, value) - return key - - @staticmethod - def get_temp(key): - """Return the temperory value and delete it.""" - return frappe.cache().hget("temp", key) - def set_global(self, key, val, user="__global"): """Save a global key value. Global values will be automatically set if they match fieldname.""" self.set_default(key, val, user) @@ -926,8 +942,10 @@ class Database: return defaults.get(frappe.scrub(key)) - def begin(self): - self.sql("START TRANSACTION") + def begin(self, *, read_only=False): + read_only = read_only or frappe.flags.read_only + mode = "READ ONLY" if read_only else "" + self.sql(f"START TRANSACTION {mode}") def commit(self): """Commit current transaction. Calls SQL `COMMIT`.""" @@ -935,9 +953,7 @@ class Database: frappe.call(method[0], *(method[1] or []), **(method[2] or {})) self.sql("commit") - if frappe.conf.db_type == "postgres": - # Postgres requires explicitly starting new transaction - self.begin() + self.begin() # explicitly start a new transaction frappe.local.rollback_observers = [] self.flush_realtime_log() @@ -979,34 +995,26 @@ class Database: obj.on_rollback() frappe.local.rollback_observers = [] + frappe.local.realtime_log = [] + frappe.flags.enqueue_after_commit = [] + def field_exists(self, dt, fn): """Return true of field exists.""" return self.exists("DocField", {"fieldname": fn, "parent": dt}) def table_exists(self, doctype, cached=True): """Returns True if table for given doctype exists.""" - return ("tab" + doctype) in self.get_tables(cached=cached) + return f"tab{doctype}" in self.get_tables(cached=cached) def has_table(self, doctype): return self.table_exists(doctype) def get_tables(self, cached=True): - tables = frappe.cache().get_value("db_tables") - if not tables or not cached: - table_rows = self.sql( - """ - SELECT table_name - FROM information_schema.tables - WHERE table_schema NOT IN ('pg_catalog', 'information_schema') - """ - ) - tables = {d[0] for d in table_rows} - frappe.cache().set_value("db_tables", tables) - return tables + raise NotImplementedError def a_row_exists(self, doctype): """Returns True if atleast one row exists.""" - return self.sql(f"select name from `tab{doctype}` limit 1") + return frappe.get_all(doctype, limit=1, order_by=None, as_list=True) def exists(self, dt, dn=None, cache=False): """Return the document name of a matching document, or None. @@ -1041,7 +1049,7 @@ class Database: dt = dt.copy() # don't modify the original dict dt, dn = dt.pop("doctype"), dt - return self.get_value(dt, dn, ignore=True, cache=cache) + return self.get_value(dt, dn, ignore=True, cache=cache, order_by=None) def count(self, dt, filters=None, debug=False, cache=False, distinct: bool = True): """Returns `COUNT(*)` for given DocType and filters.""" @@ -1049,10 +1057,9 @@ class Database: cache_count = frappe.cache().get_value(f"doctype:count:{dt}") if cache_count is not None: return cache_count - query = frappe.qb.engine.get_query( - table=dt, filters=filters, fields=Count("*"), distinct=distinct - ) - count = self.sql(query, debug=debug)[0][0] + count = frappe.qb.get_query(table=dt, filters=filters, fields=Count("*"), distinct=distinct).run( + debug=debug + )[0][0] if not filters and cache: frappe.cache().set_value(f"doctype:count:{dt}", count, expires_in_sec=86400) return count @@ -1062,17 +1069,11 @@ class Database: return getdate(date).strftime("%Y-%m-%d") @staticmethod - def format_datetime(datetime): + def format_datetime(datetime): # noqa: F811 if not datetime: - return "0001-01-01 00:00:00.000000" + return FallBackDateTimeStr - if isinstance(datetime, str): - if ":" not in datetime: - datetime = datetime + " 00:00:00.000000" - else: - datetime = datetime.strftime("%Y-%m-%d %H:%M:%S.%f") - - return datetime + return get_datetime(datetime).strftime("%Y-%m-%d %H:%M:%S.%f") def get_creation_count(self, doctype, minutes): """Get count of records created in the last x minutes""" @@ -1080,28 +1081,27 @@ class Database: from frappe.utils import now_datetime - return self.sql( - """select count(name) from `tab{doctype}` - where creation >= %s""".format( - doctype=doctype - ), - now_datetime() - relativedelta(minutes=minutes), - )[0][0] + Table = frappe.qb.DocType(doctype) + + return ( + frappe.qb.from_(Table) + .select(Count(Table.name)) + .where(Table.creation >= now_datetime() - relativedelta(minutes=minutes)) + .run()[0][0] + ) def get_db_table_columns(self, table) -> list[str]: """Returns list of column names from given table.""" columns = frappe.cache().hget("table_columns", table) if columns is None: - columns = [ - r[0] - for r in self.sql( - """ - select column_name - from information_schema.columns - where table_name = %s """, - table, - ) - ] + information_schema = frappe.qb.Schema("information_schema") + + columns = ( + frappe.qb.from_(information_schema.columns) + .select(information_schema.columns.column_name) + .where(information_schema.columns.table_name == table) + .run(pluck=True) + ) if columns: frappe.cache().hset("table_columns", table, columns) @@ -1120,12 +1120,19 @@ class Database: return column in self.get_table_columns(doctype) def get_column_type(self, doctype, column): - return self.sql( - """SELECT column_type FROM INFORMATION_SCHEMA.COLUMNS - WHERE table_name = 'tab{}' AND column_name = '{}' """.format( - doctype, column + """Returns column type from database.""" + information_schema = frappe.qb.Schema("information_schema") + table = get_table_name(doctype) + + return ( + frappe.qb.from_(information_schema.columns) + .select(information_schema.columns.column_type) + .where( + (information_schema.columns.table_name == table) + & (information_schema.columns.column_name == column) ) - )[0][0] + .run(pluck=True)[0] + ) def has_index(self, table_name, index_name): raise NotImplementedError @@ -1148,7 +1155,6 @@ class Database: def close(self): """Close database connection.""" if self._conn: - # self._cursor.close() self._conn.close() self._cursor = None self._conn = None @@ -1177,7 +1183,7 @@ class Database: return self.is_missing_column(e) or self.is_table_missing(e) def multisql(self, sql_dict, values=(), **kwargs): - current_dialect = frappe.db.db_type or "mariadb" + current_dialect = self.db_type or "mariadb" query = sql_dict.get(current_dialect) return self.sql(query, values, **kwargs) @@ -1188,7 +1194,7 @@ class Database: Doctype name can be passed directly, it will be pre-pended with `tab`. """ filters = filters or kwargs.get("conditions") - query = frappe.qb.engine.build_conditions(table=doctype, filters=filters).delete() + query = frappe.qb.get_query(table=doctype, filters=filters, delete=True) if "debug" not in kwargs: kwargs["debug"] = debug return query.run(**kwargs) @@ -1201,9 +1207,6 @@ class Database: """ return self.sql_ddl(f"truncate `{get_table_name(doctype)}`") - def clear_table(self, doctype): - return self.truncate(doctype) - def get_last_created(self, doctype): last_record = self.get_all(doctype, ("creation"), limit=1, order_by="creation desc") if last_record: @@ -1211,9 +1214,7 @@ class Database: else: return None - def log_touched_tables(self, query, values=None): - if values: - query = frappe.safe_decode(self._cursor.mogrify(query, values)) + def log_touched_tables(self, query): if is_query_type(query, ("insert", "delete", "update", "alter", "drop", "rename")): # single_word_regex is designed to match following patterns # `tabXxx`, tabXxx and "tabXxx" @@ -1221,7 +1222,7 @@ class Database: # multi_word_regex is designed to match following patterns # `tabXxx Xxx` and "tabXxx Xxx" - # ([`"]?) Captures " or ` at the begining of the table name (if provided) + # ([`"]?) Captures " or ` at the beginning of the table name (if provided) # \1 matches the first captured group (quote character) at the end of the table name # multi word table name must have surrounding quotes. @@ -1237,28 +1238,36 @@ class Database: frappe.flags.touched_tables = set() frappe.flags.touched_tables.update(tables) - def bulk_insert(self, doctype, fields, values, ignore_duplicates=False, *, chunk_size=10_000): + def bulk_insert( + self, + doctype: str, + fields: list[str], + values: Iterable[Sequence[Any]], + ignore_duplicates=False, + *, + chunk_size=10_000, + ): """ Insert multiple records at a time :param doctype: Doctype name :param fields: list of fields - :params values: list of list of values + :params values: iterable of values """ - values = list(values) table = frappe.qb.DocType(doctype) - for start_index in range(0, len(values), chunk_size): - query = frappe.qb.into(table) - if ignore_duplicates: - # Pypika does not have same api for ignoring duplicates - if frappe.conf.db_type == "mariadb": - query = query.ignore() - elif frappe.conf.db_type == "postgres": - query = query.on_conflict().do_nothing() + query = frappe.qb.into(table).columns(fields) - values_to_insert = values[start_index : start_index + chunk_size] - query.columns(fields).insert(*values_to_insert).run() + if ignore_duplicates: + # Pypika does not have same api for ignoring duplicates + if frappe.conf.db_type == "mariadb": + query = query.ignore() + elif frappe.conf.db_type == "postgres": + query = query.on_conflict().do_nothing() + + value_iterator = iter(values) + while value_chunk := tuple(itertools.islice(value_iterator, chunk_size)): + query.insert(*value_chunk).run() def create_sequence(self, *args, **kwargs): from frappe.database.sequence import create_sequence @@ -1277,12 +1286,24 @@ class Database: def enqueue_jobs_after_commit(): - from frappe.utils.background_jobs import execute_job, get_queue + from frappe.utils.background_jobs import ( + RQ_JOB_FAILURE_TTL, + RQ_RESULTS_TTL, + execute_job, + get_queue, + ) if frappe.flags.enqueue_after_commit and len(frappe.flags.enqueue_after_commit) > 0: for job in frappe.flags.enqueue_after_commit: q = get_queue(job.get("queue"), is_async=job.get("is_async")) - q.enqueue_call(execute_job, timeout=job.get("timeout"), kwargs=job.get("queue_args")) + q.enqueue_call( + execute_job, + timeout=job.get("timeout"), + kwargs=job.get("queue_args"), + failure_ttl=frappe.conf.get("rq_job_failure_ttl") or RQ_JOB_FAILURE_TTL, + result_ttl=frappe.conf.get("rq_results_ttl") or RQ_RESULTS_TTL, + job_id=job.get("job_id"), + ) frappe.flags.enqueue_after_commit = [] @@ -1310,3 +1331,28 @@ def savepoint(catch: type | tuple[type, ...] = Exception): frappe.db.rollback(save_point=savepoint) else: frappe.db.release_savepoint(savepoint) + + +def get_query_execution_timeout() -> int: + """Get execution timeout based on current timeout in different contexts. + + HTTP requests: HTTP timeout or a default (300) + Background jobs: Job timeout + Console/Commands: No timeout = 0. + + Note: Timeout adds 1.5x as "safety factor" + """ + from rq import get_current_job + + if not frappe.conf.get("enable_db_statement_timeout"): + return 0 + + # Zero means no timeout, which is the default value in db. + timeout = 0 + with suppress(Exception): + if getattr(frappe.local, "request", None): + timeout = frappe.conf.http_timeout or 300 + elif job := get_current_job(): + timeout = job.timeout + + return int(cint(timeout) * 1.5) diff --git a/frappe/database/db_manager.py b/frappe/database/db_manager.py index 796f23a054..5840158fa1 100644 --- a/frappe/database/db_manager.py +++ b/frappe/database/db_manager.py @@ -1,5 +1,3 @@ -import os - import frappe @@ -15,63 +13,51 @@ class DbManager: return self.db.sql("select user()")[0][0].split("@")[1] def create_user(self, user, password, host=None): - # Create user if it doesn't exist. - if not host: - host = self.get_current_host() - - if password: - self.db.sql(f"CREATE USER '{user}'@'{host}' IDENTIFIED BY '{password}';") - else: - self.db.sql(f"CREATE USER '{user}'@'{host}';") + host = host or self.get_current_host() + password_predicate = f" IDENTIFIED BY '{password}'" if password else "" + self.db.sql(f"CREATE USER '{user}'@'{host}'{password_predicate}") def delete_user(self, target, host=None): - if not host: - host = self.get_current_host() - try: - self.db.sql(f"DROP USER '{target}'@'{host}';") - except Exception as e: - if e.args[0] == 1396: - pass - else: - raise + host = host or self.get_current_host() + self.db.sql(f"DROP USER IF EXISTS '{target}'@'{host}'") def create_database(self, target): if target in self.get_database_list(): self.drop_database(target) - - self.db.sql("CREATE DATABASE `%s` ;" % target) + self.db.sql(f"CREATE DATABASE `{target}`") def drop_database(self, target): - self.db.sql("DROP DATABASE IF EXISTS `%s`;" % target) + self.db.sql_ddl(f"DROP DATABASE IF EXISTS `{target}`") def grant_all_privileges(self, target, user, host=None): - if not host: - host = self.get_current_host() - - if frappe.conf.get("rds_db", 0) == 1: - self.db.sql( - "GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, INDEX, ALTER, CREATE TEMPORARY TABLES, CREATE VIEW, EVENT, TRIGGER, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EXECUTE, LOCK TABLES ON `%s`.* TO '%s'@'%s';" - % (target, user, host) + host = host or self.get_current_host() + permissions = ( + ( + "SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, INDEX, ALTER, " + "CREATE TEMPORARY TABLES, CREATE VIEW, EVENT, TRIGGER, SHOW VIEW, " + "CREATE ROUTINE, ALTER ROUTINE, EXECUTE, LOCK TABLES" ) - else: - self.db.sql(f"GRANT ALL PRIVILEGES ON `{target}`.* TO '{user}'@'{host}';") + if frappe.conf.rds_db + else "ALL PRIVILEGES" + ) + self.db.sql(f"GRANT {permissions} ON `{target}`.* TO '{user}'@'{host}'") def flush_privileges(self): self.db.sql("FLUSH PRIVILEGES") def get_database_list(self): - """get list of databases""" - return [d[0] for d in self.db.sql("SHOW DATABASES")] + return self.db.sql("SHOW DATABASES", pluck=True) @staticmethod def restore_database(target, source, user, password): + import os + from shutil import which + from frappe.utils import make_esc esc = make_esc("$ ") + pv = which("pv") - from distutils.spawn import find_executable - - pv = find_executable("pv") if pv: pipe = f"{pv} {source} |" source = "" diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index 047039b0df..43540956e0 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -1,3 +1,5 @@ +import re + import pymysql from pymysql.constants import ER, FIELD_TYPE from pymysql.converters import conversions, escape_string @@ -7,24 +9,142 @@ from frappe.database.database import Database from frappe.database.mariadb.schema import MariaDBTable from frappe.utils import UnicodeWithAttrs, cstr, get_datetime, get_table_name +_PARAM_COMP = re.compile(r"%\([\w]*\)s") -class MariaDBDatabase(Database): - ProgrammingError = pymysql.err.ProgrammingError - TableMissingError = pymysql.err.ProgrammingError - OperationalError = pymysql.err.OperationalError - InternalError = pymysql.err.InternalError - SQLError = pymysql.err.ProgrammingError - DataError = pymysql.err.DataError + +class MariaDBExceptionUtil: + ProgrammingError = pymysql.ProgrammingError + TableMissingError = pymysql.ProgrammingError + OperationalError = pymysql.OperationalError + InternalError = pymysql.InternalError + SQLError = pymysql.ProgrammingError + DataError = pymysql.DataError + + # match ER_SEQUENCE_RUN_OUT - https://mariadb.com/kb/en/mariadb-error-codes/ + SequenceGeneratorLimitExceeded = pymysql.OperationalError + SequenceGeneratorLimitExceeded.errno = 4084 + + @staticmethod + def is_deadlocked(e: pymysql.Error) -> bool: + return e.args[0] == ER.LOCK_DEADLOCK + + @staticmethod + def is_timedout(e: pymysql.Error) -> bool: + return e.args[0] == ER.LOCK_WAIT_TIMEOUT + + @staticmethod + def is_read_only_mode_error(e: pymysql.Error) -> bool: + return e.args[0] == 1792 + + @staticmethod + def is_table_missing(e: pymysql.Error) -> bool: + return e.args[0] == ER.NO_SUCH_TABLE + + @staticmethod + def is_missing_table(e: pymysql.Error) -> bool: + return MariaDBDatabase.is_table_missing(e) + + @staticmethod + def is_missing_column(e: pymysql.Error) -> bool: + return e.args[0] == ER.BAD_FIELD_ERROR + + @staticmethod + def is_duplicate_fieldname(e: pymysql.Error) -> bool: + return e.args[0] == ER.DUP_FIELDNAME + + @staticmethod + def is_duplicate_entry(e: pymysql.Error) -> bool: + return e.args[0] == ER.DUP_ENTRY + + @staticmethod + def is_access_denied(e: pymysql.Error) -> bool: + return e.args[0] == ER.ACCESS_DENIED_ERROR + + @staticmethod + def cant_drop_field_or_key(e: pymysql.Error) -> bool: + return e.args[0] == ER.CANT_DROP_FIELD_OR_KEY + + @staticmethod + def is_syntax_error(e: pymysql.Error) -> bool: + return e.args[0] == ER.PARSE_ERROR + + @staticmethod + def is_statement_timeout(e: pymysql.Error) -> bool: + return e.args[0] == 1969 + + @staticmethod + def is_data_too_long(e: pymysql.Error) -> bool: + return e.args[0] == ER.DATA_TOO_LONG + + @staticmethod + def is_primary_key_violation(e: pymysql.Error) -> bool: + return ( + MariaDBDatabase.is_duplicate_entry(e) + and "PRIMARY" in cstr(e.args[1]) + and isinstance(e, pymysql.IntegrityError) + ) + + @staticmethod + def is_unique_key_violation(e: pymysql.Error) -> bool: + return ( + MariaDBDatabase.is_duplicate_entry(e) + and "Duplicate" in cstr(e.args[1]) + and isinstance(e, pymysql.IntegrityError) + ) + + +class MariaDBConnectionUtil: + def get_connection(self): + conn = self._get_connection() + conn.auto_reconnect = True + return conn + + def _get_connection(self): + """Return MariaDB connection object.""" + return self.create_connection() + + def create_connection(self): + return pymysql.connect(**self.get_connection_settings()) + + def set_execution_timeout(self, seconds: int): + self.sql("set session max_statement_time = %s", int(seconds)) + + def get_connection_settings(self) -> dict: + conn_settings = { + "host": self.host, + "user": self.user, + "password": self.password, + "conv": self.CONVERSION_MAP, + "charset": "utf8mb4", + "use_unicode": True, + } + + if self.user not in (frappe.flags.root_login, "root"): + conn_settings["database"] = self.user + + if self.port: + conn_settings["port"] = int(self.port) + + if frappe.conf.local_infile: + conn_settings["local_infile"] = frappe.conf.local_infile + + if frappe.conf.db_ssl_ca and frappe.conf.db_ssl_cert and frappe.conf.db_ssl_key: + conn_settings["ssl"] = { + "ca": frappe.conf.db_ssl_ca, + "cert": frappe.conf.db_ssl_cert, + "key": frappe.conf.db_ssl_key, + } + return conn_settings + + +class MariaDBDatabase(MariaDBConnectionUtil, MariaDBExceptionUtil, Database): REGEX_CHARACTER = "regexp" - - # NOTE: using a very small cache - as during backup, if the sequence was used in anyform, - # it drops the cache and uses the next non cached value in setval query and - # puts that in the backup file, which will start the counter - # from that value when inserting any new record in the doctype. - # By default the cache is 1000 which will mess up the sequence when - # using the system after a restore. - # issue link: https://jira.mariadb.org/browse/MDEV-21786 - SEQUENCE_CACHE = 50 + CONVERSION_MAP = conversions | { + FIELD_TYPE.NEWDECIMAL: float, + FIELD_TYPE.DATETIME: get_datetime, + UnicodeWithAttrs: escape_string, + } + default_port = "3306" def setup_type_map(self): self.db_type = "mariadb" @@ -65,44 +185,6 @@ class MariaDBDatabase(Database): "JSON": ("json", ""), } - def get_connection(self): - usessl = 0 - if frappe.conf.db_ssl_ca and frappe.conf.db_ssl_cert and frappe.conf.db_ssl_key: - usessl = 1 - ssl_params = { - "ca": frappe.conf.db_ssl_ca, - "cert": frappe.conf.db_ssl_cert, - "key": frappe.conf.db_ssl_key, - } - - conversions.update( - { - FIELD_TYPE.NEWDECIMAL: float, - FIELD_TYPE.DATETIME: get_datetime, - UnicodeWithAttrs: conversions[str], - } - ) - - conn = pymysql.connect( - user=self.user or "", - password=self.password or "", - host=self.host, - port=self.port, - charset="utf8mb4", - use_unicode=True, - ssl=ssl_params if usessl else None, - conv=conversions, - local_infile=frappe.conf.local_infile, - ) - - # MYSQL_OPTION_MULTI_STATEMENTS_OFF = 1 - # # self._conn.set_server_option(MYSQL_OPTION_MULTI_STATEMENTS_OFF) - - if self.user != "root": - conn.select_db(self.user) - - return conn - def get_database_size(self): """'Returns database size in MB""" db_size = self.sql( @@ -117,9 +199,18 @@ class MariaDBDatabase(Database): return db_size[0].get("database_size") + def log_query(self, query, values, debug, explain): + self.last_query = query = self._cursor._executed + self._log_query(query, debug, explain) + return self.last_query + @staticmethod def escape(s, percent=True): """Excape quotes and percent in given string.""" + # Update: We've scrapped PyMySQL in favour of MariaDB's official Python client + # Also, given we're promoting use of the PyPika builder via frappe.qb, the use + # of this method should be limited. + # pymysql expects unicode argument to escape_string with Python 3 s = frappe.as_unicode(escape_string(frappe.as_unicode(s)), "utf-8").replace("`", "\\`") @@ -140,7 +231,7 @@ class MariaDBDatabase(Database): @staticmethod def is_type_datetime(code): - return code in (pymysql.DATE, pymysql.DATETIME) + return code == pymysql.DATETIME def rename_table(self, old_name: str, new_name: str) -> list | tuple: old_name = get_table_name(old_name) @@ -158,57 +249,6 @@ class MariaDBDatabase(Database): null_constraint = "NOT NULL" if not nullable else "" return self.sql_ddl(f"ALTER TABLE `{table_name}` MODIFY `{column}` {type} {null_constraint}") - # exception types - @staticmethod - def is_deadlocked(e): - return e.args[0] == ER.LOCK_DEADLOCK - - @staticmethod - def is_timedout(e): - return e.args[0] == ER.LOCK_WAIT_TIMEOUT - - @staticmethod - def is_table_missing(e): - return e.args[0] == ER.NO_SUCH_TABLE - - @staticmethod - def is_missing_table(e): - return MariaDBDatabase.is_table_missing(e) - - @staticmethod - def is_missing_column(e): - return e.args[0] == ER.BAD_FIELD_ERROR - - @staticmethod - def is_duplicate_fieldname(e): - return e.args[0] == ER.DUP_FIELDNAME - - @staticmethod - def is_duplicate_entry(e): - return e.args[0] == ER.DUP_ENTRY - - @staticmethod - def is_access_denied(e): - return e.args[0] == ER.ACCESS_DENIED_ERROR - - @staticmethod - def cant_drop_field_or_key(e): - return e.args[0] == ER.CANT_DROP_FIELD_OR_KEY - - @staticmethod - def is_syntax_error(e): - return e.args[0] == ER.PARSE_ERROR - - @staticmethod - def is_data_too_long(e): - return e.args[0] == ER.DATA_TOO_LONG - - def is_primary_key_violation(self, e): - return self.is_duplicate_entry(e) and "PRIMARY" in cstr(e.args[1]) - - def is_unique_key_violation(self, e): - return self.is_duplicate_entry(e) and "Duplicate" in cstr(e.args[1]) - def create_auth_table(self): self.sql_ddl( """create table if not exists `__Auth` ( @@ -250,22 +290,6 @@ class MariaDBDatabase(Database): ) ENGINE=InnoDB DEFAULT CHARSET=utf8""" ) - def create_help_table(self): - self.sql( - """create table help( - path varchar(255), - content text, - title text, - intro text, - full_path text, - fulltext(title), - fulltext(content), - index (path)) - COLLATE=utf8mb4_unicode_ci - ENGINE=MyISAM - CHARACTER SET=utf8mb4""" - ) - @staticmethod def get_on_duplicate_update(key=None): return "ON DUPLICATE key UPDATE " @@ -283,6 +307,7 @@ class MariaDBDatabase(Database): where table_name="{table_name}" and column_name=columns.column_name and NON_UNIQUE=1 + and Seq_in_index = 1 limit 1 ), 0) as 'index', column_key = 'UNI' as 'unique' @@ -301,6 +326,37 @@ class MariaDBDatabase(Database): ) ) + def get_column_index( + self, table_name: str, fieldname: str, unique: bool = False + ) -> frappe._dict | None: + """Check if column exists for a specific fields in specified order. + + This differs from db.has_index because it doesn't rely on index name but columns inside an + index. + """ + + indexes = self.sql( + f"""SHOW INDEX FROM `{table_name}` + WHERE Column_name = "{fieldname}" + AND Seq_in_index = 1 + AND Non_unique={int(not unique)} + """, + as_dict=True, + ) + + # Same index can be part of clustered index which contains more fields + # We don't want those. + for index in indexes: + clustered_index = self.sql( + f"""SHOW INDEX FROM `{table_name}` + WHERE Key_name = "{index.Key_name}" + AND Seq_in_index = 2 + """, + as_dict=True, + ) + if not clustered_index: + return index + def add_index(self, doctype: str, fields: list, index_name: str = None): """Creates an index with given fields if not already created. Index name will be `fieldname1_fieldname2_index`""" @@ -351,5 +407,26 @@ class MariaDBDatabase(Database): db_table.sync() self.begin() - def get_database_list(self, target): - return [d[0] for d in self.sql("SHOW DATABASES;")] + def get_database_list(self): + return self.sql("SHOW DATABASES", pluck=True) + + def get_tables(self, cached=True): + """Returns list of tables""" + to_query = not cached + + if cached: + tables = frappe.cache().get_value("db_tables") + to_query = not tables + + if to_query: + information_schema = frappe.qb.Schema("information_schema") + + tables = ( + frappe.qb.from_(information_schema.tables) + .select(information_schema.tables.table_name) + .where(information_schema.tables.table_schema != "information_schema") + .run(pluck=True) + ) + frappe.cache().set_value("db_tables", tables) + + return tables diff --git a/frappe/database/mariadb/framework_mariadb.sql b/frappe/database/mariadb/framework_mariadb.sql index dc91873a82..9507a48b91 100644 --- a/frappe/database/mariadb/framework_mariadb.sql +++ b/frappe/database/mariadb/framework_mariadb.sql @@ -116,6 +116,7 @@ CREATE TABLE `tabDocPerm` ( -- Table structure for table `tabDocType Action` -- +DROP TABLE IF EXISTS `tabDocType Action`; CREATE TABLE `tabDocType Action` ( `name` varchar(140) COLLATE utf8mb4_unicode_ci NOT NULL, `creation` datetime(6) DEFAULT NULL, @@ -137,9 +138,10 @@ CREATE TABLE `tabDocType Action` ( ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci ROW_FORMAT=DYNAMIC; -- --- Table structure for table `tabDocType Action` +-- Table structure for table `tabDocType Link` -- +DROP TABLE IF EXISTS `tabDocType Link`; CREATE TABLE `tabDocType Link` ( `name` varchar(140) COLLATE utf8mb4_unicode_ci NOT NULL, `creation` datetime(6) DEFAULT NULL, @@ -183,7 +185,6 @@ CREATE TABLE `tabDocType` ( `app` varchar(255) DEFAULT NULL, `autoname` varchar(255) DEFAULT NULL, `naming_rule` varchar(40) DEFAULT NULL, - `name_case` varchar(255) DEFAULT NULL, `title_field` varchar(255) DEFAULT NULL, `image_field` varchar(255) DEFAULT NULL, `timeline_field` varchar(255) DEFAULT NULL, @@ -226,7 +227,7 @@ CREATE TABLE `tabDocType` ( `sender_field` varchar(255) DEFAULT NULL, `show_title_field_in_link` int(1) NOT NULL DEFAULT 0, `migration_hash` varchar(255) DEFAULT NULL, - `translate_link_fields` int(1) NOT NULL DEFAULT 0, + `translated_doctype` int(1) NOT NULL DEFAULT 0, PRIMARY KEY (`name`) ) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; @@ -253,7 +254,6 @@ CREATE TABLE `tabSessions` ( `sessiondata` longtext, `ipaddress` varchar(16) DEFAULT NULL, `lastupdate` datetime(6) DEFAULT NULL, - `device` varchar(255) DEFAULT 'desktop', `status` varchar(20) DEFAULT NULL, KEY `sid` (`sid`) ) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci; diff --git a/frappe/database/mariadb/schema.py b/frappe/database/mariadb/schema.py index 24a78012e1..bbdd95d921 100644 --- a/frappe/database/mariadb/schema.py +++ b/frappe/database/mariadb/schema.py @@ -1,3 +1,5 @@ +from pymysql.constants.ER import DUP_ENTRY + import frappe from frappe import _ from frappe.database.schema import DBTable @@ -6,7 +8,7 @@ from frappe.model import log_types class MariaDBTable(DBTable): def create(self): - additional_definitions = "" + additional_definitions = [] engine = self.meta.get("engine") or "InnoDB" varchar_len = frappe.db.VARCHAR_LEN name_column = f"name varchar({varchar_len}) primary key" @@ -14,26 +16,24 @@ class MariaDBTable(DBTable): # columns column_defs = self.get_column_definitions() if column_defs: - additional_definitions += ",\n".join(column_defs) + ",\n" + additional_definitions += column_defs # index index_defs = self.get_index_definitions() if index_defs: - additional_definitions += ",\n".join(index_defs) + ",\n" + additional_definitions += index_defs # child table columns if self.meta.get("istable") or 0: - additional_definitions += ( - ",\n".join( - ( - f"parent varchar({varchar_len})", - f"parentfield varchar({varchar_len})", - f"parenttype varchar({varchar_len})", - "index parent(parent)", - ) - ) - + ",\n" - ) + additional_definitions += [ + f"parent varchar({varchar_len})", + f"parentfield varchar({varchar_len})", + f"parenttype varchar({varchar_len})", + "index parent(parent)", + ] + else: + # parent types + additional_definitions.append("index modified(modified)") # creating sequence(s) if ( @@ -44,9 +44,11 @@ class MariaDBTable(DBTable): # NOTE: not used nextval func as default as the ability to restore # database with sequences has bugs in mariadb and gives a scary error. - # issue link: https://jira.mariadb.org/browse/MDEV-21786 + # issue link: https://jira.mariadb.org/browse/MDEV-20070 name_column = "name bigint primary key" + additional_definitions = ",\n".join(additional_definitions) + # create table query = f"""create table `{self.table_name}` ( {name_column}, @@ -56,8 +58,7 @@ class MariaDBTable(DBTable): owner varchar({varchar_len}), docstatus int(1) not null default '0', idx int(8) not null default '0', - {additional_definitions} - index modified(modified)) + {additional_definitions}) ENGINE={engine} ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 @@ -74,55 +75,39 @@ class MariaDBTable(DBTable): add_index_query = [] drop_index_query = [] - columns_to_modify = set(self.change_type + self.add_unique + self.set_default) - for col in self.add_column: add_column_query.append(f"ADD COLUMN `{col.fieldname}` {col.get_definition()}") + columns_to_modify = set(self.change_type + self.set_default) for col in columns_to_modify: - modify_column_query.append(f"MODIFY `{col.fieldname}` {col.get_definition()}") + modify_column_query.append( + f"MODIFY `{col.fieldname}` {col.get_definition(for_modification=True)}" + ) + + for col in self.add_unique: + modify_column_query.append( + f"ADD UNIQUE INDEX IF NOT EXISTS {col.fieldname} (`{col.fieldname}`)" + ) for col in self.add_index: # if index key does not exists - if not frappe.db.has_index(self.table_name, col.fieldname + "_index"): + if not frappe.db.get_column_index(self.table_name, col.fieldname, unique=False): add_index_query.append(f"ADD INDEX `{col.fieldname}_index`(`{col.fieldname}`)") - for col in self.drop_index + self.drop_unique: - if col.fieldname != "name": # primary key - current_column = self.current_columns.get(col.fieldname.lower()) - unique_constraint_changed = current_column.unique != col.unique - if unique_constraint_changed and not col.unique: - # nosemgrep - unique_index_record = frappe.db.sql( - """ - SHOW INDEX FROM `{}` - WHERE Key_name=%s - AND Non_unique=0 - """.format( - self.table_name - ), - (col.fieldname), - as_dict=1, - ) - if unique_index_record: - drop_index_query.append(f"DROP INDEX `{unique_index_record[0].Key_name}`") - index_constraint_changed = current_column.index != col.set_index - # if index key exists - if index_constraint_changed and not col.set_index: - # nosemgrep - index_record = frappe.db.sql( - """ - SHOW INDEX FROM `{}` - WHERE Key_name=%s - AND Non_unique=1 - """.format( - self.table_name - ), - (col.fieldname + "_index"), - as_dict=1, - ) - if index_record: - drop_index_query.append(f"DROP INDEX `{index_record[0].Key_name}`") + for col in {*self.drop_index, *self.drop_unique}: + if col.fieldname == "name": + continue + + current_column = self.current_columns.get(col.fieldname.lower()) + unique_constraint_changed = current_column.unique != col.unique + if unique_constraint_changed and not col.unique: + if unique_index := frappe.db.get_column_index(self.table_name, col.fieldname, unique=True): + drop_index_query.append(f"DROP INDEX `{unique_index.Key_name}`") + + index_constraint_changed = current_column.index != col.set_index + if index_constraint_changed and not col.set_index: + if index_record := frappe.db.get_column_index(self.table_name, col.fieldname, unique=False): + drop_index_query.append(f"DROP INDEX `{index_record.Key_name}`") try: for query_parts in [add_column_query, modify_column_query, add_index_query, drop_index_query]: @@ -132,17 +117,15 @@ class MariaDBTable(DBTable): frappe.db.sql(query) except Exception as e: - # sanitize - if e.args[0] == 1060: - frappe.throw(str(e)) - elif e.args[0] == 1062: + if query := locals().get("query"): # this weirdness is to avoid potentially unbounded vars + print(f"Failed to alter schema using query: {query}") + + if e.args[0] == DUP_ENTRY: fieldname = str(e).split("'")[-2] frappe.throw( _("{0} field cannot be set as unique in {1}, as there are non-unique existing values").format( fieldname, self.table_name ) ) - elif e.args[0] == 1067: - frappe.throw(str(e.args[1])) - else: - raise e + + raise diff --git a/frappe/database/mariadb/setup_db.py b/frappe/database/mariadb/setup_db.py index 5eef0ef2c6..add7fa373f 100644 --- a/frappe/database/mariadb/setup_db.py +++ b/frappe/database/mariadb/setup_db.py @@ -1,31 +1,26 @@ import os +import click + import frappe from frappe.database.db_manager import DbManager -expected_settings_10_2_earlier = { - "innodb_file_format": "Barracuda", - "innodb_file_per_table": "ON", - "innodb_large_prefix": "ON", - "character_set_server": "utf8mb4", - "collation_server": "utf8mb4_unicode_ci", -} - -expected_settings_10_3_later = { +REQUIRED_MARIADB_CONFIG = { "character_set_server": "utf8mb4", "collation_server": "utf8mb4_unicode_ci", } -def get_mariadb_versions(): +def get_mariadb_variables(): + return frappe._dict(frappe.db.sql("show variables")) + + +def get_mariadb_version(version_string: str = ""): # MariaDB classifies their versions as Major (1st and 2nd number), and Minor (3rd number) # Example: Version 10.3.13 is Major Version = 10.3, Minor Version = 13 - mariadb_variables = frappe._dict(frappe.db.sql("""show variables""")) - version_string = mariadb_variables.get("version").split("-")[0] - versions = {} - versions["major"] = version_string.split(".")[0] + "." + version_string.split(".")[1] - versions["minor"] = version_string.split(".")[2] - return versions + version_string = version_string or get_mariadb_variables().get("version") + version = version_string.split("-", 1)[0] + return version.rsplit(".", 1) def setup_database(force, source_sql, verbose, no_mariadb_socket=False): @@ -63,29 +58,12 @@ def setup_database(force, source_sql, verbose, no_mariadb_socket=False): bootstrap_database(db_name, verbose, source_sql) -def setup_help_database(help_db_name): - dbman = DbManager(get_root_connection(frappe.flags.root_login, frappe.flags.root_password)) - dbman.drop_database(help_db_name) - - # make database - if not help_db_name in dbman.get_database_list(): - try: - dbman.create_user(help_db_name, help_db_name) - except Exception as e: - # user already exists - if e.args[0] != 1396: - raise - dbman.create_database(help_db_name) - dbman.grant_all_privileges(help_db_name, help_db_name) - dbman.flush_privileges() - - def drop_user_and_database(db_name, root_login, root_password): frappe.local.db = get_root_connection(root_login, root_password) dbman = DbManager(frappe.local.db) + dbman.drop_database(db_name) dbman.delete_user(db_name, host="%") dbman.delete_user(db_name) - dbman.drop_database(db_name) def bootstrap_database(db_name, verbose, source_sql=None): @@ -125,36 +103,55 @@ def import_db_from_sql(source_sql=None, verbose=False): def check_database_settings(): - versions = get_mariadb_versions() - if versions["major"] <= "10.2": - expected_variables = expected_settings_10_2_earlier - else: - expected_variables = expected_settings_10_3_later - mariadb_variables = frappe._dict(frappe.db.sql("""show variables""")) + check_compatible_versions() + # Check each expected value vs. actuals: + mariadb_variables = get_mariadb_variables() result = True - for key, expected_value in expected_variables.items(): + for key, expected_value in REQUIRED_MARIADB_CONFIG.items(): if mariadb_variables.get(key) != expected_value: print( "For key %s. Expected value %s, found value %s" % (key, expected_value, mariadb_variables.get(key)) ) result = False + if not result: - site = frappe.local.site - msg = ( - "Creation of your site - {x} failed because MariaDB is not properly {sep}" - "configured. If using version 10.2.x or earlier, make sure you use the {sep}" - "the Barracuda storage engine. {sep}{sep}" - "Please verify the settings above in MariaDB's my.cnf. Restart MariaDB. And {sep}" - "then run `bench new-site {x}` again.{sep2}" - "" - ).format(x=site, sep2="\n" * 2, sep="\n") - print_db_config(msg) + print( + ( + "{sep2}Creation of your site - {site} failed because MariaDB is not properly {sep}" + "configured.{sep2}" + "Please verify the above settings in MariaDB's my.cnf. Restart MariaDB.{sep}" + "And then run `bench new-site {site}` again.{sep2}" + ).format(site=frappe.local.site, sep2="\n\n", sep="\n") + ) + return result +def check_compatible_versions(): + try: + version = get_mariadb_version() + version_tuple = tuple(int(v) for v in version[0].split(".")) + + if version_tuple < (10, 6): + click.secho( + f"Warning: MariaDB version {version} is less than 10.6 which is not supported by Frappe", + fg="yellow", + ) + elif version_tuple >= (10, 9): + click.secho( + f"Warning: MariaDB version {version} is more than 10.8 which is not yet tested with Frappe Framework.", + fg="yellow", + ) + except Exception: + click.secho( + "MariaDB version compatibility checks failed, make sure you're running a supported version.", + fg="yellow", + ) + + def get_root_connection(root_login, root_password): import getpass @@ -173,9 +170,3 @@ def get_root_connection(root_login, root_password): ) return frappe.local.flags.root_connection - - -def print_db_config(explanation): - print("=" * 80) - print(explanation) - print("=" * 80) diff --git a/frappe/database/operator_map.py b/frappe/database/operator_map.py new file mode 100644 index 0000000000..2c8b53dae3 --- /dev/null +++ b/frappe/database/operator_map.py @@ -0,0 +1,138 @@ +# Copyright (c) 2023, Frappe Technologies Pvt. Ltd. and Contributors +# MIT License. See license.txt + +import operator +from typing import Callable + +import frappe +from frappe.database.utils import NestedSetHierarchy +from frappe.model.db_query import get_timespan_date_range +from frappe.query_builder import Field + + +def like(key: Field, value: str) -> frappe.qb: + """Wrapper method for `LIKE` + + Args: + key (str): field + value (str): criterion + + Returns: + frappe.qb: `frappe.qb object with `LIKE` + """ + return key.like(value) + + +def func_in(key: Field, value: list | tuple) -> frappe.qb: + """Wrapper method for `IN` + + Args: + key (str): field + value (Union[int, str]): criterion + + Returns: + frappe.qb: `frappe.qb object with `IN` + """ + if isinstance(value, str): + value = value.split(",") + return key.isin(value) + + +def not_like(key: Field, value: str) -> frappe.qb: + """Wrapper method for `NOT LIKE` + + Args: + key (str): field + value (str): criterion + + Returns: + frappe.qb: `frappe.qb object with `NOT LIKE` + """ + return key.not_like(value) + + +def func_not_in(key: Field, value: list | tuple | str): + """Wrapper method for `NOT IN` + + Args: + key (str): field + value (Union[int, str]): criterion + + Returns: + frappe.qb: `frappe.qb object with `NOT IN` + """ + if isinstance(value, str): + value = value.split(",") + return key.notin(value) + + +def func_regex(key: Field, value: str) -> frappe.qb: + """Wrapper method for `REGEX` + + Args: + key (str): field + value (str): criterion + + Returns: + frappe.qb: `frappe.qb object with `REGEX` + """ + return key.regex(value) + + +def func_between(key: Field, value: list | tuple) -> frappe.qb: + """Wrapper method for `BETWEEN` + + Args: + key (str): field + value (Union[int, str]): criterion + + Returns: + frappe.qb: `frappe.qb object with `BETWEEN` + """ + return key[slice(*value)] + + +def func_is(key, value): + "Wrapper for IS" + return key.isnotnull() if value.lower() == "set" else key.isnull() + + +def func_timespan(key: Field, value: str) -> frappe.qb: + """Wrapper method for `TIMESPAN` + + Args: + key (str): field + value (str): criterion + + Returns: + frappe.qb: `frappe.qb object with `TIMESPAN` + """ + + return func_between(key, get_timespan_date_range(value)) + + +# default operators +OPERATOR_MAP: dict[str, Callable] = { + "+": operator.add, + "=": operator.eq, + "-": operator.sub, + "!=": operator.ne, + "<": operator.lt, + ">": operator.gt, + "<=": operator.le, + "=<": operator.le, + ">=": operator.ge, + "=>": operator.ge, + "/": operator.truediv, + "*": operator.mul, + "in": func_in, + "not in": func_not_in, + "like": like, + "not like": not_like, + "regex": func_regex, + "between": func_between, + "is": func_is, + "timespan": func_timespan, + "nested_set": NestedSetHierarchy, + # TODO: Add support for custom operators (WIP) - via filters_config hooks +} diff --git a/frappe/database/postgres/database.py b/frappe/database/postgres/database.py index 2553ebaa26..d082afceaf 100644 --- a/frappe/database/postgres/database.py +++ b/frappe/database/postgres/database.py @@ -2,12 +2,23 @@ import re import psycopg2 import psycopg2.extensions -from psycopg2.errorcodes import STRING_DATA_RIGHT_TRUNCATION +from psycopg2.errorcodes import ( + CLASS_INTEGRITY_CONSTRAINT_VIOLATION, + DEADLOCK_DETECTED, + DUPLICATE_COLUMN, + INSUFFICIENT_PRIVILEGE, + STRING_DATA_RIGHT_TRUNCATION, + UNDEFINED_COLUMN, + UNDEFINED_TABLE, + UNIQUE_VIOLATION, +) +from psycopg2.errors import ReadOnlySqlTransaction, SequenceGeneratorLimitExceeded, SyntaxError from psycopg2.extensions import ISOLATION_LEVEL_REPEATABLE_READ import frappe from frappe.database.database import Database from frappe.database.postgres.schema import PostgresTable +from frappe.database.utils import EmptyQueryValues, LazyDecode from frappe.utils import cstr, get_table_name # cast decimals as floats @@ -25,7 +36,7 @@ PG_TRANSFORM_PATTERN = re.compile(r"([=><]+)\s*([+-]?\d+)(\.0)?(?![a-zA-Z\.\d])" FROM_TAB_PATTERN = re.compile(r"from tab([\w-]*)", flags=re.IGNORECASE) -class PostgresDatabase(Database): +class PostgresExceptionUtil: ProgrammingError = psycopg2.ProgrammingError TableMissingError = psycopg2.ProgrammingError OperationalError = psycopg2.OperationalError @@ -33,13 +44,73 @@ class PostgresDatabase(Database): SQLError = psycopg2.ProgrammingError DataError = psycopg2.DataError InterfaceError = psycopg2.InterfaceError - REGEX_CHARACTER = "~" + SequenceGeneratorLimitExceeded = SequenceGeneratorLimitExceeded - # NOTE; The sequence cache for postgres is per connection. - # Since we're opening and closing connections for every transaction this results in skipping the cache - # to the next non-cached value hence not using cache in postgres. - # ref: https://stackoverflow.com/questions/21356375/postgres-9-0-4-sequence-skipping-numbers - SEQUENCE_CACHE = 0 + @staticmethod + def is_deadlocked(e): + return getattr(e, "pgcode", None) == DEADLOCK_DETECTED + + @staticmethod + def is_timedout(e): + # http://initd.org/psycopg/docs/extensions.html?highlight=datatype#psycopg2.extensions.QueryCanceledError + return isinstance(e, psycopg2.extensions.QueryCanceledError) + + @staticmethod + def is_read_only_mode_error(e) -> bool: + return isinstance(e, ReadOnlySqlTransaction) + + @staticmethod + def is_syntax_error(e): + return isinstance(e, SyntaxError) + + @staticmethod + def is_table_missing(e): + return getattr(e, "pgcode", None) == UNDEFINED_TABLE + + @staticmethod + def is_missing_table(e): + return PostgresDatabase.is_table_missing(e) + + @staticmethod + def is_missing_column(e): + return getattr(e, "pgcode", None) == UNDEFINED_COLUMN + + @staticmethod + def is_access_denied(e): + return getattr(e, "pgcode", None) == INSUFFICIENT_PRIVILEGE + + @staticmethod + def cant_drop_field_or_key(e): + return getattr(e, "pgcode", None) == CLASS_INTEGRITY_CONSTRAINT_VIOLATION + + @staticmethod + def is_duplicate_entry(e): + return getattr(e, "pgcode", None) == UNIQUE_VIOLATION + + @staticmethod + def is_primary_key_violation(e): + return getattr(e, "pgcode", None) == UNIQUE_VIOLATION and "_pkey" in cstr(e.args[0]) + + @staticmethod + def is_unique_key_violation(e): + return getattr(e, "pgcode", None) == UNIQUE_VIOLATION and "_key" in cstr(e.args[0]) + + @staticmethod + def is_duplicate_fieldname(e): + return getattr(e, "pgcode", None) == DUPLICATE_COLUMN + + @staticmethod + def is_statement_timeout(e): + return PostgresDatabase.is_timedout(e) or isinstance(e, frappe.QueryTimeoutError) + + @staticmethod + def is_data_too_long(e): + return getattr(e, "pgcode", None) == STRING_DATA_RIGHT_TRUNCATION + + +class PostgresDatabase(PostgresExceptionUtil, Database): + REGEX_CHARACTER = "~" + default_port = "5432" def setup_type_map(self): self.db_type = "postgres" @@ -80,6 +151,10 @@ class PostgresDatabase(Database): "JSON": ("json", ""), } + @property + def last_query(self): + return LazyDecode(self._cursor.query) + def get_connection(self): conn = psycopg2.connect( "host='{}' dbname='{}' user='{}' password='{}' port={}".format( @@ -90,6 +165,10 @@ class PostgresDatabase(Database): return conn + def set_execution_timeout(self, seconds: int): + # Postgres expects milliseconds as input + self.sql("set local statement_timeout = %s", int(seconds) * 1000) + def escape(self, s, percent=True): """Escape quotes and percent in given string.""" if isinstance(s, bytes): @@ -116,9 +195,12 @@ class PostgresDatabase(Database): return db_size[0].get("database_size") # pylint: disable=W0221 - def sql(self, query, values=(), *args, **kwargs): + def sql(self, query, values=EmptyQueryValues, *args, **kwargs): return super().sql(modify_query(query), modify_values(values), *args, **kwargs) + def lazy_mogrify(self, *args, **kwargs) -> str: + return self.last_query + def get_tables(self, cached=True): return [ d[0] @@ -151,60 +233,6 @@ class PostgresDatabase(Database): def is_type_datetime(code): return code == psycopg2.DATETIME - # exception type - @staticmethod - def is_deadlocked(e): - return e.pgcode == "40P01" - - @staticmethod - def is_timedout(e): - # http://initd.org/psycopg/docs/extensions.html?highlight=datatype#psycopg2.extensions.QueryCanceledError - return isinstance(e, psycopg2.extensions.QueryCanceledError) - - @staticmethod - def is_syntax_error(e): - return isinstance(e, psycopg2.errors.SyntaxError) - - @staticmethod - def is_table_missing(e): - return getattr(e, "pgcode", None) == "42P01" - - @staticmethod - def is_missing_table(e): - return PostgresDatabase.is_table_missing(e) - - @staticmethod - def is_missing_column(e): - return getattr(e, "pgcode", None) == "42703" - - @staticmethod - def is_access_denied(e): - return e.pgcode == "42501" - - @staticmethod - def cant_drop_field_or_key(e): - return e.pgcode.startswith("23") - - @staticmethod - def is_duplicate_entry(e): - return e.pgcode == "23505" - - @staticmethod - def is_primary_key_violation(e): - return getattr(e, "pgcode", None) == "23505" and "_pkey" in cstr(e.args[0]) - - @staticmethod - def is_unique_key_violation(e): - return getattr(e, "pgcode", None) == "23505" and "_key" in cstr(e.args[0]) - - @staticmethod - def is_duplicate_fieldname(e): - return e.pgcode == "42701" - - @staticmethod - def is_data_too_long(e): - return e.pgcode == STRING_DATA_RIGHT_TRUNCATION - def rename_table(self, old_name: str, new_name: str) -> list | tuple: old_name = get_table_name(old_name) new_name = get_table_name(new_name) @@ -269,17 +297,6 @@ class PostgresDatabase(Database): )""" ) - def create_help_table(self): - self.sql( - """CREATE TABLE "help"( - "path" varchar(255), - "content" text, - "title" text, - "intro" text, - "full_path" text)""" - ) - self.sql("""CREATE INDEX IF NOT EXISTS "help_index" ON "help" ("path")""") - def updatedb(self, doctype, meta=None): """ Syncs a `DocType` to the table @@ -377,8 +394,8 @@ class PostgresDatabase(Database): as_dict=1, ) - def get_database_list(self, target): - return [d[0] for d in self.sql("SELECT datname FROM pg_database;")] + def get_database_list(self): + return self.sql("SELECT datname FROM pg_database", pluck=True) def modify_query(query): @@ -409,7 +426,7 @@ def modify_values(values): return value - if not values: + if not values or values == EmptyQueryValues: return values if isinstance(values, dict): diff --git a/frappe/database/postgres/framework_postgres.sql b/frappe/database/postgres/framework_postgres.sql index 99e94a226f..37605be0f6 100644 --- a/frappe/database/postgres/framework_postgres.sql +++ b/frappe/database/postgres/framework_postgres.sql @@ -188,7 +188,6 @@ CREATE TABLE "tabDocType" ( "app" varchar(255) DEFAULT NULL, "autoname" varchar(255) DEFAULT NULL, "naming_rule" varchar(40) DEFAULT NULL, - "name_case" varchar(255) DEFAULT NULL, "title_field" varchar(255) DEFAULT NULL, "image_field" varchar(255) DEFAULT NULL, "timeline_field" varchar(255) DEFAULT NULL, @@ -231,7 +230,7 @@ CREATE TABLE "tabDocType" ( "sender_field" varchar(255) DEFAULT NULL, "show_title_field_in_link" smallint NOT NULL DEFAULT 0, "migration_hash" varchar(255) DEFAULT NULL, - "translate_link_fields" smallint NOT NULL DEFAULT 0, + "translated_doctype" smallint NOT NULL DEFAULT 0, PRIMARY KEY ("name") ) ; @@ -257,7 +256,6 @@ CREATE TABLE "tabSessions" ( "sessiondata" text, "ipaddress" varchar(16) DEFAULT NULL, "lastupdate" timestamp(6) DEFAULT NULL, - "device" varchar(255) DEFAULT 'desktop', "status" varchar(20) DEFAULT NULL ); diff --git a/frappe/database/postgres/setup_db.py b/frappe/database/postgres/setup_db.py index 7eee8081c0..ff14510c9c 100644 --- a/frappe/database/postgres/setup_db.py +++ b/frappe/database/postgres/setup_db.py @@ -75,15 +75,6 @@ def import_db_from_sql(source_sql=None, verbose=False): ) -def setup_help_database(help_db_name): - root_conn = get_root_connection(frappe.flags.root_login, frappe.flags.root_password) - root_conn.sql(f"DROP DATABASE IF EXISTS `{help_db_name}`") - root_conn.sql(f"DROP USER IF EXISTS {help_db_name}") - root_conn.sql(f"CREATE DATABASE `{help_db_name}`") - root_conn.sql(f"CREATE user {help_db_name} password '{help_db_name}'") - root_conn.sql("GRANT ALL PRIVILEGES ON DATABASE `{0}` TO {0}".format(help_db_name)) - - def get_root_connection(root_login=None, root_password=None): if not frappe.local.flags.root_connection: if not root_login: diff --git a/frappe/database/query.py b/frappe/database/query.py index 9bb5383b24..595bd5a3ff 100644 --- a/frappe/database/query.py +++ b/frappe/database/query.py @@ -1,15 +1,23 @@ -import operator +import itertools import re from ast import literal_eval -from functools import cached_property from types import BuiltinFunctionType -from typing import TYPE_CHECKING, Any, Callable +from typing import TYPE_CHECKING + +import sqlparse +from pypika.queries import QueryBuilder, Table import frappe from frappe import _ -from frappe.model.db_query import get_timespan_date_range -from frappe.query_builder import Criterion, Field, Order, Table, functions +from frappe.database.operator_map import OPERATOR_MAP +from frappe.database.utils import DefaultOrderBy, get_doctype_name +from frappe.query_builder import Criterion, Field, Order, functions from frappe.query_builder.functions import Function, SqlFunctions +from frappe.query_builder.utils import PseudoColumnMapper +from frappe.utils.data import MARIADB_SPECIFIC_COMMENT + +if TYPE_CHECKING: + from frappe.query_builder import DocType TAB_PATTERN = re.compile("^tab") WORDS_PATTERN = re.compile(r"\w+") @@ -18,364 +26,181 @@ SQL_FUNCTIONS = [sql_function.value for sql_function in SqlFunctions] COMMA_PATTERN = re.compile(r",\s*(?![^()]*\))") -def like(key: Field, value: str) -> frappe.qb: - """Wrapper method for `LIKE` - - Args: - key (str): field - value (str): criterion - - Returns: - frappe.qb: `frappe.qb object with `LIKE` - """ - return key.like(value) - - -def func_in(key: Field, value: list | tuple) -> frappe.qb: - """Wrapper method for `IN` - - Args: - key (str): field - value (Union[int, str]): criterion - - Returns: - frappe.qb: `frappe.qb object with `IN` - """ - return key.isin(value) - - -def not_like(key: Field, value: str) -> frappe.qb: - """Wrapper method for `NOT LIKE` - - Args: - key (str): field - value (str): criterion - - Returns: - frappe.qb: `frappe.qb object with `NOT LIKE` - """ - return key.not_like(value) - - -def func_not_in(key: Field, value: list | tuple): - """Wrapper method for `NOT IN` - - Args: - key (str): field - value (Union[int, str]): criterion - - Returns: - frappe.qb: `frappe.qb object with `NOT IN` - """ - return key.notin(value) - - -def func_regex(key: Field, value: str) -> frappe.qb: - """Wrapper method for `REGEX` - - Args: - key (str): field - value (str): criterion - - Returns: - frappe.qb: `frappe.qb object with `REGEX` - """ - return key.regex(value) - - -def func_between(key: Field, value: list | tuple) -> frappe.qb: - """Wrapper method for `BETWEEN` - - Args: - key (str): field - value (Union[int, str]): criterion - - Returns: - frappe.qb: `frappe.qb object with `BETWEEN` - """ - return key[slice(*value)] - - -def func_is(key, value): - "Wrapper for IS" - return key.isnotnull() if value.lower() == "set" else key.isnull() - - -def func_timespan(key: Field, value: str) -> frappe.qb: - """Wrapper method for `TIMESPAN` - - Args: - key (str): field - value (str): criterion - - Returns: - frappe.qb: `frappe.qb object with `TIMESPAN` - """ - - return func_between(key, get_timespan_date_range(value)) - - -def make_function(key: Any, value: int | str): - """returns fucntion query - - Args: - key (Any): field - value (Union[int, str]): criterion - - Returns: - frappe.qb: frappe.qb object - """ - return OPERATOR_MAP[value[0].casefold()](key, value[1]) - - -def change_orderby(order: str): - """Convert orderby to standart Order object - - Args: - order (str): Field, order - - Returns: - tuple: field, order - """ - order = order.split() - - try: - if order[1].lower() == "asc": - return order[0], Order.asc - except IndexError: - pass - - return order[0], Order.desc - - -def literal_eval_(literal): - try: - return literal_eval(literal) - except (ValueError, SyntaxError): - return literal - - -# default operators -OPERATOR_MAP: dict[str, Callable] = { - "+": operator.add, - "=": operator.eq, - "-": operator.sub, - "!=": operator.ne, - "<": operator.lt, - ">": operator.gt, - "<=": operator.le, - "=<": operator.le, - ">=": operator.ge, - "=>": operator.ge, - "/": operator.truediv, - "*": operator.mul, - "in": func_in, - "not in": func_not_in, - "like": like, - "not like": not_like, - "regex": func_regex, - "between": func_between, - "is": func_is, - "timespan": func_timespan, - # TODO: Add support for nested set - # TODO: Add support for custom operators (WIP) - via filters_config hooks -} - - class Engine: - tables: dict[str, str] = {} + def get_query( + self, + table: str | Table, + fields: list | tuple | None = None, + filters: dict[str, str | int] | str | int | list[list | str | int] | None = None, + order_by: str | None = None, + group_by: str | None = None, + limit: int | None = None, + offset: int | None = None, + distinct: bool = False, + for_update: bool = False, + update: bool = False, + into: bool = False, + delete: bool = False, + ) -> QueryBuilder: + self.is_mariadb = frappe.db.db_type == "mariadb" + self.is_postgres = frappe.db.db_type == "postgres" - @cached_property - def OPERATOR_MAP(self): - from frappe.boot import get_additional_filters_from_hooks + if isinstance(table, Table): + self.table = table + self.doctype = get_doctype_name(table.get_sql()) + else: + self.doctype = table + self.table = frappe.qb.DocType(table) - # default operators - all_operators = OPERATOR_MAP.copy() + if update: + self.query = frappe.qb.update(self.table) + elif into: + self.query = frappe.qb.into(self.table) + elif delete: + self.query = frappe.qb.from_(self.table).delete() + else: + self.query = frappe.qb.from_(self.table) + self.apply_fields(fields) - # update with site-specific custom operators - additional_filters_config = get_additional_filters_from_hooks() + self.apply_filters(filters) + self.apply_order_by(order_by) - if additional_filters_config: - from frappe.utils.commands import warn + if limit: + self.query = self.query.limit(limit) - warn("'filters_config' hook is not completely implemented yet in frappe.db.query engine") + if offset: + self.query = self.query.offset(offset) - for _operator, function in additional_filters_config.items(): - if callable(function): - all_operators.update({_operator.casefold(): function}) - elif isinstance(function, dict): - all_operators[_operator.casefold()] = frappe.get_attr(function.get("get_field"))()["operator"] + if distinct: + self.query = self.query.distinct() - return all_operators + if for_update: + self.query = self.query.for_update() - def get_condition(self, table: str | Table, **kwargs) -> frappe.qb: - """Get initial table object + if group_by: + self.query = self.query.groupby(group_by) - Args: - table (str): DocType + return self.query - Returns: - frappe.qb: DocType with initial condition - """ - table_object = self.get_table(table) - if kwargs.get("update"): - return frappe.qb.update(table_object) - if kwargs.get("into"): - return frappe.qb.into(table_object) - return frappe.qb.from_(table_object) + def apply_fields(self, fields): + # add fields + self.fields = self.parse_fields(fields) + if not self.fields: + self.fields = [getattr(self.table, "name")] - def get_table(self, table_name: str | Table) -> Table: - if isinstance(table_name, Table): - return table_name - table_name = table_name.strip('"').strip("'") - if table_name not in self.tables: - self.tables[table_name] = frappe.qb.DocType(table_name) - return self.tables[table_name] - - def criterion_query(self, table: str, criterion: Criterion, **kwargs) -> frappe.qb: - """Generate filters from Criterion objects - - Args: - table (str): DocType - criterion (Criterion): Filters - - Returns: - frappe.qb: condition object - """ - condition = self.add_conditions(self.get_condition(table, **kwargs), **kwargs) - return condition.where(criterion) - - def add_conditions(self, conditions: frappe.qb, **kwargs): - """Adding additional conditions - - Args: - conditions (frappe.qb): built conditions - - Returns: - conditions (frappe.qb): frappe.qb object - """ - if kwargs.get("orderby") and kwargs.get("orderby") != "KEEP_DEFAULT_ORDERING": - orderby = kwargs.get("orderby") - if isinstance(orderby, str) and len(orderby.split()) > 1: - for ordby in orderby.split(","): - if ordby := ordby.strip(): - orderby, order = change_orderby(ordby) - conditions = conditions.orderby(orderby, order=order) + self.query._child_queries = [] + for field in self.fields: + if isinstance(field, DynamicTableField): + self.query = field.apply_select(self.query) + elif isinstance(field, ChildQuery): + self.query._child_queries.append(field) else: - conditions = conditions.orderby(orderby, order=kwargs.get("order") or Order.desc) + self.query = self.query.select(field) - if kwargs.get("limit"): - conditions = conditions.limit(kwargs.get("limit")) - conditions = conditions.offset(kwargs.get("offset", 0)) + def apply_filters( + self, + filters: dict[str, str | int] | str | int | list[list | str | int] | None = None, + ): + if filters is None: + return - if kwargs.get("distinct"): - conditions = conditions.distinct() - - if kwargs.get("for_update"): - conditions = conditions.for_update() - - if kwargs.get("groupby"): - conditions = conditions.groupby(kwargs.get("groupby")) - - return conditions - - def misc_query(self, table: str, filters: list | tuple = None, **kwargs): - """Build conditions using the given Lists or Tuple filters - - Args: - table (str): DocType - filters (Union[List, Tuple], optional): Filters. Defaults to None. - """ - conditions = self.get_condition(table, **kwargs) - if not filters: - return conditions - if isinstance(filters, list): - for f in filters: - if isinstance(f, (list, tuple)): - _operator = self.OPERATOR_MAP[f[-2].casefold()] - if len(f) == 4: - table_object = self.get_table(f[0]) - _field = table_object[f[1]] - else: - _field = Field(f[0]) - conditions = conditions.where(_operator(_field, f[-1])) - elif isinstance(f, dict): - conditions = self.dict_query(table, f, **kwargs) - else: - _operator = self.OPERATOR_MAP[filters[1].casefold()] - if not isinstance(filters[0], str): - conditions = make_function(filters[0], filters[2]) - break - conditions = conditions.where(_operator(Field(filters[0]), filters[2])) - break - - return self.add_conditions(conditions, **kwargs) - - def dict_query(self, table: str, filters: dict[str, str | int] = None, **kwargs) -> frappe.qb: - """Build conditions using the given dictionary filters - - Args: - table (str): DocType - filters (Dict[str, Union[str, int]], optional): Filters. Defaults to None. - - Returns: - frappe.qb: conditions object - """ - conditions = self.get_condition(table, **kwargs) - if not filters: - conditions = self.add_conditions(conditions, **kwargs) - return conditions - - for key, value in filters.items(): - if isinstance(value, bool): - filters.update({key: str(int(value))}) - - for key in filters: - value = filters.get(key) - _operator = self.OPERATOR_MAP["="] - - if not isinstance(key, str): - conditions = conditions.where(make_function(key, value)) - continue - if isinstance(value, (list, tuple)): - _operator = self.OPERATOR_MAP[value[0].casefold()] - _value = value[1] if value[1] else ("",) - conditions = conditions.where(_operator(Field(key), _value)) - else: - if value is not None: - conditions = conditions.where(_operator(Field(key), value)) - else: - _table = conditions._from[0] - field = getattr(_table, key) - conditions = conditions.where(field.isnull()) - - return self.add_conditions(conditions, **kwargs) - - def build_conditions( - self, table: str, filters: dict[str, str | int] | str | int = None, **kwargs - ) -> frappe.qb: - """Build conditions for sql query - - Args: - filters (Union[Dict[str, Union[str, int]], str, int]): conditions in Dict - table (str): DocType - - Returns: - frappe.qb: frappe.qb conditions object - """ - if isinstance(filters, int) or isinstance(filters, str): + if isinstance(filters, (str, int)): filters = {"name": str(filters)} if isinstance(filters, Criterion): - criterion = self.criterion_query(table, filters, **kwargs) + self.query = self.query.where(filters) + + elif isinstance(filters, dict): + self.apply_dict_filters(filters) elif isinstance(filters, (list, tuple)): - criterion = self.misc_query(table, filters, **kwargs) + if all(isinstance(d, (str, int)) for d in filters) and len(filters) > 0: + self.apply_dict_filters({"name": ("in", filters)}) + else: + for filter in filters: + if isinstance(filter, (str, int, Criterion, dict)): + self.apply_filters(filter) + elif isinstance(filter, (list, tuple)): + self.apply_list_filters(filter) + def apply_list_filters(self, filter: list): + if len(filter) == 2: + field, value = filter + self._apply_filter(field, value) + elif len(filter) == 3: + field, operator, value = filter + self._apply_filter(field, value, operator) + elif len(filter) == 4: + doctype, field, operator, value = filter + self._apply_filter(field, value, operator, doctype) + + def apply_dict_filters(self, filters: dict[str, str | int | list]): + for key in filters: + value = filters.get(key) + self._apply_filter(key, value) + + def _apply_filter( + self, field: str, value: str | int | list | None, operator: str = "=", doctype: str | None = None + ): + _field = field + _value = value + _operator = operator + + if isinstance(_field, Field): + pass + elif dynamic_field := DynamicTableField.parse(field, self.doctype): + # apply implicit join if link field's field is referenced + self.query = dynamic_field.apply_join(self.query) + _field = dynamic_field.field + elif has_function(field): + _field = self.get_function_object(field) + elif not doctype or doctype == self.doctype: + _field = self.table[field] + elif doctype: + _field = frappe.qb.DocType(doctype)[field] + + # apply implicit join if child table is referenced + if doctype and doctype != self.doctype: + meta = frappe.get_meta(doctype) + table = frappe.qb.DocType(doctype) + if meta.istable and not self.query.is_joined(table): + self.query = self.query.left_join(table).on( + (table.parent == self.table.name) & (table.parenttype == self.doctype) + ) + + if isinstance(_value, (list, tuple)): + _operator, _value = _value + elif isinstance(_value, bool): + _value = int(_value) + + if isinstance(_value, str) and has_function(_value): + _value = self.get_function_object(_value) + + if isinstance(_value, (list, tuple)) and not _value: + _value = ("",) + + # Nested set + if _operator in OPERATOR_MAP["nested_set"]: + hierarchy = _operator + docname = _value + result = get_nested_set_hierarchy_result(self.doctype, docname, hierarchy) + operator_fn = ( + OPERATOR_MAP["not in"] + if hierarchy in ("not ancestors of", "not descendants of") + else OPERATOR_MAP["in"] + ) + if result: + result = list(itertools.chain.from_iterable(result)) + self.query = self.query.where(operator_fn(_field, result)) + else: + self.query = self.query.where(operator_fn(_field, ("",))) + return + + operator_fn = OPERATOR_MAP[_operator.casefold()] + if _value is None and isinstance(_field, Field): + self.query = self.query.where(_field.isnull()) else: - criterion = self.dict_query(filters=filters, table=table, **kwargs) - - return criterion + self.query = self.query.where(operator_fn(_field, _value)) def get_function_object(self, field: str) -> "Function": """Expects field to look like 'SUM(*)' or 'name' or something similar. Returns PyPika Function object""" @@ -399,147 +224,110 @@ class Engine: if isinstance(operator_mapping, BuiltinFunctionType): has_primitive_operator = True field = operator_mapping( - *map(lambda field: Field(field.strip()), arg.split(_operator)), + *map( + lambda field: Field(field.strip()) + if "`" not in field + else PseudoColumnMapper(field.strip()), + arg.split(_operator), + ), ) - field = Field(initial_fields) if not has_primitive_operator else field + field = ( + (Field(initial_fields) if "`" not in initial_fields else PseudoColumnMapper(initial_fields)) + if not has_primitive_operator + else field + ) else: field = initial_fields _args.append(field) + + if alias and "`" in alias: + alias = alias.replace("`", "") try: + if func.casefold() == "now": + return getattr(functions, func)() return getattr(functions, func)(*_args, alias=alias or None) except AttributeError: # Fall back for functions not present in `SqlFunctions`` return Function(func, *_args, alias=alias or None) - def function_objects_from_string(self, fields): - fields = list(map(lambda str: str.strip(), COMMA_PATTERN.split(fields))) - return self.function_objects_from_list(fields=fields) - - def function_objects_from_list(self, fields): - functions = [] - for field in fields: - field = field.casefold() if isinstance(field, str) else field - if not issubclass(type(field), Criterion): - if any([f"{func}(" in field for func in SQL_FUNCTIONS]) or "(" in field: - functions.append(field) - - return [self.get_function_object(function) for function in functions] - - def remove_string_functions(self, fields, function_objects): - """Remove string functions from fields which have already been converted to function objects""" - for function in function_objects: - if isinstance(fields, str): - if function.alias: - fields = fields.replace(" as " + function.alias.casefold(), "") - fields = BRACKETS_PATTERN.sub("", fields.replace(function.name.casefold(), "")) - # Check if only comma is left in fields after stripping functions. - if "," in fields and (len(fields.strip()) == 1): - fields = "" - else: - updated_fields = [] - for field in fields: - if isinstance(field, str): - if function.alias: - field = field.replace(" as " + function.alias.casefold(), "") - field = ( - BRACKETS_PATTERN.sub("", field).strip().casefold().replace(function.name.casefold(), "") - ) - updated_fields.append(field) - - fields = [field for field in updated_fields if field] - - return fields - - def set_fields(self, fields, **kwargs): - fields = kwargs.get("pluck") if kwargs.get("pluck") else fields or "name" - if isinstance(fields, list) and None in fields and Field not in fields: - return None - - function_objects = [] - - is_list = isinstance(fields, (list, tuple, set)) - if is_list and len(fields) == 1: - fields = fields[0] - is_list = False - - if is_list: - function_objects += self.function_objects_from_list(fields=fields) - - is_str = isinstance(fields, str) - if is_str: - fields = fields.casefold() - function_objects += self.function_objects_from_string(fields=fields) - - fields = self.remove_string_functions(fields, function_objects) - - if is_str and "," in fields: - fields = [field.replace(" ", "") if "as" not in field else field for field in fields.split(",")] - is_list, is_str = True, False - - if is_str: - if fields == "*": - return fields - if " as " in fields: - fields, reference = fields.split(" as ") - fields = Field(fields).as_(reference) - - if not is_str and fields: - if issubclass(type(fields), Criterion): - return fields - updated_fields = [] - if "*" in fields: - return fields - for field in fields: - if not isinstance(field, Criterion) and field: - if " as " in field: - field, reference = field.split(" as ") - updated_fields.append(Field(field.strip()).as_(reference)) - else: - updated_fields.append(Field(field)) - - fields = updated_fields - - # Need to check instance again since fields modified. - if not isinstance(fields, (list, tuple, set)): - fields = [fields] if fields else [] - - fields.extend(function_objects) - return fields - - def get_query( - self, - table: str, - fields: list | tuple, - filters: dict[str, str | int] | str | int | list[list | str | int] = None, - **kwargs, - ): - # Clean up state before each query - self.tables = {} - criterion = self.build_conditions(table, filters, **kwargs) - fields = self.set_fields(kwargs.get("field_objects") or fields, **kwargs) - - join = kwargs.get("join").replace(" ", "_") if kwargs.get("join") else "left_join" - - if len(self.tables) > 1: - primary_table = self.tables[table] - del self.tables[table] - for table_object in self.tables.values(): - criterion = getattr(criterion, join)(table_object).on( - table_object.parent == primary_table.name - ) + def sanitize_fields(self, fields: str | list | tuple): + def _sanitize_field(field: str): + if not isinstance(field, str): + return field + stripped_field = sqlparse.format(field, strip_comments=True, keyword_case="lower") + if self.is_mariadb: + return MARIADB_SPECIFIC_COMMENT.sub("", stripped_field) + return stripped_field if isinstance(fields, (list, tuple)): - query = criterion.select(*fields) + return [_sanitize_field(field) for field in fields] + elif isinstance(fields, str): + return _sanitize_field(fields) - elif isinstance(fields, Criterion): - query = criterion.select(fields) + return fields - else: - query = criterion.select(fields) + def parse_string_field(self, field: str): + if field == "*": + return self.table.star + alias = None + if " as " in field: + field, alias = field.split(" as ") + if "`" in field: + if alias: + return PseudoColumnMapper(f"{field} {alias}") + return PseudoColumnMapper(field) + if alias: + return self.table[field].as_(alias) + return self.table[field] - return query + def parse_fields(self, fields: str | list | tuple | None) -> list: + if not fields: + return [] + fields = self.sanitize_fields(fields) + if isinstance(fields, (list, tuple, set)) and None in fields and Field not in fields: + return [] + + if not isinstance(fields, (list, tuple)): + fields = [fields] + + def parse_field(field: str): + if has_function(field): + return self.get_function_object(field) + elif parsed := DynamicTableField.parse(field, self.doctype): + return parsed + else: + return self.parse_string_field(field) + + _fields = [] + for field in fields: + if isinstance(field, Criterion): + _fields.append(field) + elif isinstance(field, dict): + for child_field, fields in field.items(): + _fields.append(ChildQuery(child_field, fields, self.doctype)) + elif isinstance(field, str): + if "," in field: + field = field.casefold() if "`" not in field else field + field_list = COMMA_PATTERN.split(field) + for field in field_list: + if _field := field.strip(): + _fields.append(parse_field(_field)) + else: + _fields.append(parse_field(field)) + + return _fields + + def apply_order_by(self, order_by: str | None): + if not order_by or order_by == DefaultOrderBy: + return + for declaration in order_by.split(","): + if _order_by := declaration.strip(): + parts = _order_by.split(" ") + order_field, order_direction = parts[0], parts[1] if len(parts) > 1 else "desc" + order_direction = Order.asc if order_direction.lower() == "asc" else Order.desc + self.query = self.query.orderby(order_field, order=order_direction) class Permission: @@ -570,3 +358,178 @@ class Permission: @staticmethod def get_tables_from_query(query: str): return [table for table in WORDS_PATTERN.findall(query) if table.startswith("tab")] + + +class DynamicTableField: + def __init__( + self, + doctype: str, + fieldname: str, + parent_doctype: str, + alias: str | None = None, + ) -> None: + self.doctype = doctype + self.fieldname = fieldname + self.alias = alias + self.parent_doctype = parent_doctype + + def __str__(self) -> str: + table_name = f"`tab{self.doctype}`" + fieldname = f"`{self.fieldname}`" + if frappe.db.db_type == "postgres": + table_name = table_name.replace("`", '"') + fieldname = fieldname.replace("`", '"') + alias = f"AS {self.alias}" if self.alias else "" + return f"{table_name}.{fieldname} {alias}".strip() + + @staticmethod + def parse(field: str, doctype: str): + if "." in field: + alias = None + if " as " in field: + field, alias = field.split(" as ") + if field.startswith("`tab") or field.startswith('"tab'): + _, child_doctype, child_field = re.search(r'([`"])tab(.+?)\1.\1(.+)\1', field).groups() + if child_doctype == doctype: + return + return ChildTableField(child_doctype, child_field, doctype, alias=alias) + else: + linked_fieldname, fieldname = field.split(".") + linked_field = frappe.get_meta(doctype).get_field(linked_fieldname) + linked_doctype = linked_field.options + if linked_field.fieldtype == "Link": + return LinkTableField(linked_doctype, fieldname, doctype, linked_fieldname, alias=alias) + elif linked_field.fieldtype in frappe.model.table_fields: + return ChildTableField(linked_doctype, fieldname, doctype, alias=alias) + + def apply_select(self, query: QueryBuilder) -> QueryBuilder: + raise NotImplementedError + + +class ChildTableField(DynamicTableField): + def __init__( + self, + doctype: str, + fieldname: str, + parent_doctype: str, + alias: str | None = None, + ) -> None: + self.doctype = doctype + self.fieldname = fieldname + self.alias = alias + self.parent_doctype = parent_doctype + self.table = frappe.qb.DocType(self.doctype) + self.field = self.table[self.fieldname] + + def apply_select(self, query: QueryBuilder) -> QueryBuilder: + table = frappe.qb.DocType(self.doctype) + query = self.apply_join(query) + return query.select(getattr(table, self.fieldname).as_(self.alias or None)) + + def apply_join(self, query: QueryBuilder) -> QueryBuilder: + table = frappe.qb.DocType(self.doctype) + main_table = frappe.qb.DocType(self.parent_doctype) + if not query.is_joined(table): + query = query.left_join(table).on( + (table.parent == main_table.name) & (table.parenttype == self.parent_doctype) + ) + return query + + +class LinkTableField(DynamicTableField): + def __init__( + self, + doctype: str, + fieldname: str, + parent_doctype: str, + link_fieldname: str, + alias: str | None = None, + ) -> None: + super().__init__(doctype, fieldname, parent_doctype, alias=alias) + self.link_fieldname = link_fieldname + self.table = frappe.qb.DocType(self.doctype) + self.field = self.table[self.fieldname] + + def apply_select(self, query: QueryBuilder) -> QueryBuilder: + table = frappe.qb.DocType(self.doctype) + query = self.apply_join(query) + return query.select(getattr(table, self.fieldname).as_(self.alias or None)) + + def apply_join(self, query: QueryBuilder) -> QueryBuilder: + table = frappe.qb.DocType(self.doctype) + main_table = frappe.qb.DocType(self.parent_doctype) + if not query.is_joined(table): + query = query.left_join(table).on(table.name == getattr(main_table, self.link_fieldname)) + return query + + +class ChildQuery: + def __init__( + self, + fieldname: str, + fields: list, + parent_doctype: str, + ) -> None: + field = frappe.get_meta(parent_doctype).get_field(fieldname) + if field.fieldtype not in frappe.model.table_fields: + return + self.fieldname = fieldname + self.fields = fields + self.parent_doctype = parent_doctype + self.doctype = field.options + + def get_query(self, parent_names=None) -> QueryBuilder: + filters = { + "parenttype": self.parent_doctype, + "parentfield": self.fieldname, + "parent": ["in", parent_names], + } + return frappe.qb.get_query( + self.doctype, + fields=self.fields + ["parent", "parentfield"], + filters=filters, + order_by="idx asc", + ) + + +def literal_eval_(literal): + try: + return literal_eval(literal) + except (ValueError, SyntaxError): + return literal + + +def has_function(field): + _field = field.casefold() if (isinstance(field, str) and "`" not in field) else field + if not issubclass(type(_field), Criterion): + if any([f"{func}(" in _field for func in SQL_FUNCTIONS]): + return True + + +def get_nested_set_hierarchy_result(doctype: str, name: str, hierarchy: str): + table = frappe.qb.DocType(doctype) + try: + lft, rgt = frappe.qb.from_(table).select("lft", "rgt").where(table.name == name).run()[0] + except IndexError: + lft, rgt = None, None + + if hierarchy in ("descendants of", "not descendants of"): + result = ( + frappe.qb.from_(table) + .select(table.name) + .where(table.lft > lft) + .where(table.rgt < rgt) + .orderby(table.lft, order=Order.asc) + .run() + ) + else: + # Get ancestor elements of a DocType with a tree structure + result = ( + frappe.qb.from_(table) + .select(table.name) + .where(table.lft < lft) + .where(table.rgt > rgt) + .orderby(table.lft, order=Order.desc) + .run() + ) + return result diff --git a/frappe/database/schema.py b/frappe/database/schema.py index 5920d14c3d..7a8330595e 100644 --- a/frappe/database/schema.py +++ b/frappe/database/schema.py @@ -17,18 +17,18 @@ class DBTable: self.doctype = doctype self.table_name = f"tab{doctype}" self.meta = meta or frappe.get_meta(doctype, False) - self.columns = {} + self.columns: dict[str, DbColumn] = {} self.current_columns = {} # lists for change - self.add_column = [] - self.change_type = [] - self.change_name = [] - self.add_unique = [] - self.add_index = [] - self.drop_unique = [] - self.drop_index = [] - self.set_default = [] + self.add_column: list[DbColumn] = [] + self.change_type: list[DbColumn] = [] + self.change_name: list[DbColumn] = [] + self.add_unique: list[DbColumn] = [] + self.add_index: list[DbColumn] = [] + self.drop_unique: list[DbColumn] = [] + self.drop_index: list[DbColumn] = [] + self.set_default: list[DbColumn] = [] # load self.get_columns_from_docfields() @@ -187,7 +187,7 @@ class DbColumn: self.unique = unique self.precision = precision - def get_definition(self, with_default=1): + def get_definition(self, for_modification=False): column_def = get_definition(self.fieldtype, precision=self.precision, length=self.length) if not column_def: @@ -209,7 +209,7 @@ class DbColumn: ): column_def += f" default {frappe.db.escape(self.default)}" - if self.unique and (column_def not in ("text", "longtext")): + if self.unique and not for_modification and (column_def not in ("text", "longtext")): column_def += " unique" return column_def diff --git a/frappe/database/sequence.py b/frappe/database/sequence.py index 6a352d20d1..54362a5895 100644 --- a/frappe/database/sequence.py +++ b/frappe/database/sequence.py @@ -57,12 +57,17 @@ def create_sequence( def get_next_val(doctype_name: str, slug: str = "_id_seq") -> int: - return db.multisql( - { - "postgres": f"select nextval('\"{scrub(doctype_name + slug)}\"')", - "mariadb": f"select nextval(`{scrub(doctype_name + slug)}`)", - } - )[0][0] + sequence_name = scrub(f"{doctype_name}{slug}") + + if db.db_type == "postgres": + sequence_name = f"'\"{sequence_name}\"'" + elif db.db_type == "mariadb": + sequence_name = f"`{sequence_name}`" + + try: + return db.sql(f"SELECT nextval({sequence_name})")[0][0] + except IndexError: + raise db.SequenceGeneratorLimitExceeded def set_next_val( diff --git a/frappe/database/utils.py b/frappe/database/utils.py new file mode 100644 index 0000000000..d1030ca6d7 --- /dev/null +++ b/frappe/database/utils.py @@ -0,0 +1,78 @@ +# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors +# License: MIT. See LICENSE + +import typing +from functools import cached_property +from types import NoneType + +import frappe +from frappe.query_builder.builder import MariaDB, Postgres +from frappe.query_builder.functions import Function + +if typing.TYPE_CHECKING: + from frappe.query_builder import DocType + +Query = str | MariaDB | Postgres +QueryValues = tuple | list | dict | NoneType + +EmptyQueryValues = object() +FallBackDateTimeStr = "0001-01-01 00:00:00.000000" +DefaultOrderBy = "KEEP_DEFAULT_ORDERING" +NestedSetHierarchy = ( + "ancestors of", + "descendants of", + "not ancestors of", + "not descendants of", +) + + +def is_query_type(query: str, query_type: str | tuple[str]) -> bool: + return query.lstrip().split(maxsplit=1)[0].lower().startswith(query_type) + + +def is_pypika_function_object(field: str) -> bool: + return getattr(field, "__module__", None) == "pypika.functions" or isinstance(field, Function) + + +def get_doctype_name(table_name: str) -> str: + if table_name.startswith(("tab", "`tab", '"tab')): + table_name = table_name.replace("tab", "", 1) + table_name = table_name.replace("`", "") + table_name = table_name.replace('"', "") + return table_name + + +class LazyString: + def _setup(self) -> None: + raise NotImplementedError + + @cached_property + def value(self) -> str: + return self._setup() + + def __str__(self) -> str: + return self.value + + def __repr__(self) -> str: + return f"'{self.value}'" + + +class LazyDecode(LazyString): + __slots__ = () + + def __init__(self, value: str) -> None: + self._value = value + + def _setup(self) -> None: + return self._value.decode() + + +class LazyMogrify(LazyString): + __slots__ = () + + def __init__(self, query, values) -> None: + self.query = query + self.values = values + + def _setup(self) -> str: + return frappe.db.mogrify(self.query, self.values) diff --git a/frappe/defaults.py b/frappe/defaults.py index 02076b1fda..edbf784200 100644 --- a/frappe/defaults.py +++ b/frappe/defaults.py @@ -6,8 +6,7 @@ from frappe.cache_manager import clear_defaults_cache, common_default_keys from frappe.desk.notifications import clear_notifications from frappe.query_builder import DocType -# Note: DefaultValue records are identified by parenttype -# __default, __global or 'User Permission' +# Note: DefaultValue records are identified by parent (e.g. __default, __global) def set_user_default(key, value, user=None, parenttype=None): @@ -26,9 +25,12 @@ def get_user_default(key, user=None): if d and isinstance(d, (list, tuple)) and len(d) == 1: # Use User Permission value when only when it has a single value d = d[0] - else: d = user_defaults.get(frappe.scrub(key), None) + user_permission_default = get_user_permission_default(key, user_defaults) + if not d: + # If no default value is found, use the User Permission value + d = user_permission_default value = isinstance(d, (list, tuple)) and d[0] or d if not_in_user_permission(key, value, user): @@ -37,6 +39,24 @@ def get_user_default(key, user=None): return value +def get_user_permission_default(key, defaults): + permissions = get_user_permissions() + user_default = "" + if permissions.get(key): + # global default in user permission + for item in permissions.get(key): + doc = item.get("doc") + if defaults.get(key) == doc: + user_default = doc + + for item in permissions.get(key): + if item.get("is_default"): + user_default = item.get("doc") + break + + return user_default + + def get_user_default_as_list(key, user=None): user_defaults = get_defaults(user or frappe.session.user) d = user_defaults.get(key, None) @@ -242,4 +262,6 @@ def get_defaults_for(parent="__default"): def _clear_cache(parent): + if frappe.flags.in_install: + return frappe.clear_cache(user=parent if parent not in common_default_keys else None) diff --git a/frappe/desk/desk_page.py b/frappe/desk/desk_page.py index ad0bd549d8..182c2f9ef7 100644 --- a/frappe/desk/desk_page.py +++ b/frappe/desk/desk_page.py @@ -2,7 +2,6 @@ # License: MIT. See LICENSE import frappe -from frappe.translate import send_translations @frappe.whitelist() @@ -31,28 +30,4 @@ def getpage(): page = frappe.form_dict.get("name") doc = get(page) - # load translations - if frappe.lang != "en": - send_translations(frappe.get_lang_dict("page", page)) - frappe.response.docs.append(doc) - - -def has_permission(page): - if frappe.session.user == "Administrator" or "System Manager" in frappe.get_roles(): - return True - - page_roles = [d.role for d in page.get("roles")] - if page_roles: - if frappe.session.user == "Guest" and "Guest" not in page_roles: - return False - elif not set(page_roles).intersection(set(frappe.get_roles())): - # check if roles match - return False - - if not frappe.has_permission("Page", ptype="read", doc=page): - # check if there are any user_permissions - return False - else: - # hack for home pages! if no Has Roles, allow everyone to see! - return True diff --git a/frappe/desk/desktop.py b/frappe/desk/desktop.py index e2be2656a9..46cda8fe5d 100644 --- a/frappe/desk/desktop.py +++ b/frappe/desk/desktop.py @@ -40,7 +40,6 @@ class Workspace: self.allowed_modules = self.get_cached("user_allowed_modules", self.get_allowed_modules) self.doc = frappe.get_cached_doc("Workspace", self.page_name) - if ( self.doc and self.doc.module @@ -154,26 +153,26 @@ class Workspace: return True if item_type == "dashboard": return True + if item_type == "url": + return True return False def build_workspace(self): self.cards = {"items": self.get_links()} - self.charts = {"items": self.get_charts()} - self.shortcuts = {"items": self.get_shortcuts()} - self.onboardings = {"items": self.get_onboardings()} - self.quick_lists = {"items": self.get_quick_lists()} + self.number_cards = {"items": self.get_number_cards()} + self.custom_blocks = {"items": self.get_custom_blocks()} def _doctype_contains_a_record(self, name): exists = self.table_counts.get(name, False) if not exists and frappe.db.exists(name): if not frappe.db.get_value("DocType", name, "issingle"): - exists = bool(frappe.db.get_all(name, limit=1)) + exists = bool(frappe.get_all(name, limit=1)) else: exists = True self.table_counts[name] = exists @@ -205,6 +204,24 @@ class Workspace: return item + def is_custom_block_permitted(self, custom_block_name): + from frappe.utils import has_common + + allowed = [ + d.role + for d in frappe.get_all("Has Role", fields=["role"], filters={"parent": custom_block_name}) + ] + + if not allowed: + return True + + roles = frappe.get_roles() + + if has_common(roles, allowed): + return True + + return False + @handle_not_exist def get_links(self): cards = self.doc.get_link_groups() @@ -292,12 +309,13 @@ class Workspace: quick_lists = self.doc.quick_lists for item in quick_lists: - new_item = item.as_dict().copy() + if self.is_item_allowed(item.document_type, "doctype"): + new_item = item.as_dict().copy() - # Translate label - new_item["label"] = _(item.label) if item.label else _(item.document_type) + # Translate label + new_item["label"] = _(item.label) if item.label else _(item.document_type) - items.append(new_item) + items.append(new_item) return items @@ -332,6 +350,40 @@ class Workspace: return steps + @handle_not_exist + def get_number_cards(self): + all_number_cards = [] + if frappe.has_permission("Number Card", throw=False): + number_cards = self.doc.number_cards + for number_card in number_cards: + if frappe.has_permission("Number Card", doc=number_card.number_card_name): + # Translate label + number_card.label = ( + _(number_card.label) if number_card.label else _(number_card.number_card_name) + ) + all_number_cards.append(number_card) + + return all_number_cards + + @handle_not_exist + def get_custom_blocks(self): + all_custom_blocks = [] + if frappe.has_permission("Custom HTML Block", throw=False): + custom_blocks = self.doc.custom_blocks + + for custom_block in custom_blocks: + if frappe.has_permission("Custom HTML Block", doc=custom_block.custom_block_name): + if not self.is_custom_block_permitted(custom_block.custom_block_name): + continue + + # Translate label + custom_block.label = ( + _(custom_block.label) if custom_block.label else _(custom_block.custom_block_name) + ) + all_custom_blocks.append(custom_block) + + return all_custom_blocks + @frappe.whitelist() @frappe.read_only() @@ -354,6 +406,8 @@ def get_desktop_page(page): "cards": workspace.cards, "onboardings": workspace.onboardings, "quick_lists": workspace.quick_lists, + "number_cards": workspace.number_cards, + "custom_blocks": workspace.custom_blocks, } except DoesNotExistError: frappe.log_error("Workspace Missing") @@ -379,7 +433,17 @@ def get_workspace_sidebar_items(): # pages sorted based on sequence id order_by = "sequence_id asc" - fields = ["name", "title", "for_user", "parent_page", "content", "public", "module", "icon"] + fields = [ + "name", + "title", + "for_user", + "parent_page", + "content", + "public", + "module", + "icon", + "is_hidden", + ] all_pages = frappe.get_all( "Workspace", fields=fields, filters=filters, order_by=order_by, ignore_permissions=True ) @@ -391,7 +455,7 @@ def get_workspace_sidebar_items(): try: workspace = Workspace(page, True) if has_access or workspace.is_permitted(): - if page.public: + if page.public and (has_access or not page.is_hidden): pages.append(page) elif page.for_user == frappe.session.user: private_pages.append(page) @@ -472,6 +536,14 @@ def save_new_widget(doc, page, blocks, new_widgets): doc.shortcuts.extend(new_widget(widgets.shortcut, "Workspace Shortcut", "shortcuts")) if widgets.quick_list: doc.quick_lists.extend(new_widget(widgets.quick_list, "Workspace Quick List", "quick_lists")) + if widgets.custom_block: + doc.custom_blocks.extend( + new_widget(widgets.custom_block, "Workspace Custom Block", "custom_blocks") + ) + if widgets.number_card: + doc.number_cards.extend( + new_widget(widgets.number_card, "Workspace Number Card", "number_cards") + ) if widgets.card: doc.build_links_table_from_card(widgets.card) @@ -501,12 +573,12 @@ def save_new_widget(doc, page, blocks, new_widgets): def clean_up(original_page, blocks): page_widgets = {} - for wid in ["shortcut", "card", "chart", "quick_list"]: + for wid in ["shortcut", "card", "chart", "quick_list", "number_card", "custom_block"]: # get list of widget's name from blocks page_widgets[wid] = [x["data"][wid + "_name"] for x in loads(blocks) if x["type"] == wid] - # shortcut, chart & quick_list cleanup - for wid in ["shortcut", "chart", "quick_list"]: + # shortcut, chart, quick_list, number_card & custom_block cleanup + for wid in ["shortcut", "chart", "quick_list", "number_card", "custom_block"]: updated_widgets = [] original_page.get(wid + "s").reverse() @@ -588,4 +660,8 @@ def update_onboarding_step(name, field, value): value: Value to be updated """ + from frappe.utils.telemetry import capture + frappe.db.set_value("Onboarding Step", name, field, value) + + capture(frappe.scrub(name), app="frappe_onboarding", properties={field: value}) diff --git a/frappe/desk/doctype/bulk_update/bulk_update.js b/frappe/desk/doctype/bulk_update/bulk_update.js index bb9cf2af51..d8a2b89cf3 100644 --- a/frappe/desk/doctype/bulk_update/bulk_update.js +++ b/frappe/desk/doctype/bulk_update/bulk_update.js @@ -1,41 +1,36 @@ // Copyright (c) 2016, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Bulk Update', { - refresh: function(frm) { - frm.set_query("document_type", function() { +frappe.ui.form.on("Bulk Update", { + refresh: function (frm) { + frm.set_query("document_type", function () { return { filters: [ - ['DocType', 'issingle', '=', 0], - ['DocType', 'name', 'not in', frappe.model.core_doctypes_list] - ] + ["DocType", "issingle", "=", 0], + ["DocType", "name", "not in", frappe.model.core_doctypes_list], + ], }; }); - frm.page.set_primary_action(__('Update'), function() { + frm.page.set_primary_action(__("Update"), function () { if (!frm.doc.update_value) { frappe.throw(__('Field "value" is mandatory. Please specify value to be updated')); } else { - frappe.call({ - method: 'frappe.desk.doctype.bulk_update.bulk_update.update', - args: { - doctype: frm.doc.document_type, - field: frm.doc.field, - value: frm.doc.update_value, - condition: frm.doc.condition, - limit: frm.doc.limit - }, - }).then(r => { + frm.call("bulk_update").then((r) => { let failed = r.message; if (!failed) failed = []; if (failed.length && !r._server_messages) { - frappe.throw(__('Cannot update {0}', [failed.map(f => f.bold ? f.bold(): f).join(', ')])); + frappe.throw( + __("Cannot update {0}", [ + failed.map((f) => (f.bold ? f.bold() : f)).join(", "), + ]) + ); } else { frappe.msgprint({ - title: __('Success'), - message: __('Updated Successfully'), - indicator: 'green' + title: __("Success"), + message: __("Updated Successfully"), + indicator: "green", }); } @@ -46,20 +41,18 @@ frappe.ui.form.on('Bulk Update', { }); }, - document_type: function(frm) { + document_type: function (frm) { // set field options - if(!frm.doc.document_type) return; + if (!frm.doc.document_type) return; - frappe.model.with_doctype(frm.doc.document_type, function() { - var options = $.map(frappe.get_meta(frm.doc.document_type).fields, - function(d) { - if(d.fieldname && frappe.model.no_value_type.indexOf(d.fieldtype)===-1) { - return d.fieldname; - } - return null; + frappe.model.with_doctype(frm.doc.document_type, function () { + var options = $.map(frappe.get_meta(frm.doc.document_type).fields, function (d) { + if (d.fieldname && frappe.model.no_value_type.indexOf(d.fieldtype) === -1) { + return d.fieldname; } - ); - frm.set_df_property('field', 'options', options); + return null; + }); + frm.set_df_property("field", "options", options); }); - } + }, }); diff --git a/frappe/desk/doctype/bulk_update/bulk_update.json b/frappe/desk/doctype/bulk_update/bulk_update.json index 0ec29a0dda..93458516fd 100644 --- a/frappe/desk/doctype/bulk_update/bulk_update.json +++ b/frappe/desk/doctype/bulk_update/bulk_update.json @@ -1,204 +1,77 @@ { - "allow_copy": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, - "creation": "2016-07-15 05:51:29.224123", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, + "actions": [], + "creation": "2016-07-15 05:51:29.224123", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "document_type", + "field", + "update_value", + "condition", + "limit" + ], "fields": [ { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "document_type", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Document Type", - "length": 0, - "no_copy": 0, - "options": "DocType", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "document_type", + "fieldtype": "Link", + "in_list_view": 1, + "label": "Document Type", + "options": "DocType", + "reqd": 1 + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "field", - "fieldtype": "Select", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Field", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "field", + "fieldtype": "Select", + "in_list_view": 1, + "label": "Field", + "reqd": 1 + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "update_value", - "fieldtype": "Small Text", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Update Value", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "update_value", + "fieldtype": "Small Text", + "in_list_view": 1, + "label": "Update Value", + "reqd": 1 + }, { - "allow_on_submit": 0, - "bold": 1, - "collapsible": 0, - "columns": 0, - "description": "SQL Conditions. Example: status=\"Open\"", - "fieldname": "condition", - "fieldtype": "Small Text", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Condition", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "bold": 1, + "description": "SQL Conditions. Example: status=\"Open\"", + "fieldname": "condition", + "fieldtype": "Small Text", + "label": "Condition" + }, { - "allow_on_submit": 0, - "bold": 1, - "collapsible": 0, - "columns": 0, - "default": "500", - "description": "Max 500 records at a time", - "fieldname": "limit", - "fieldtype": "Int", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Limit", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "bold": 1, + "default": "500", + "description": "Max 500 records at a time", + "fieldname": "limit", + "fieldtype": "Int", + "label": "Limit" } - ], - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - - "is_submittable": 0, - "issingle": 1, - "istable": 0, - "max_attachments": 0, - "modified": "2016-12-29 14:40:31.929701", - "modified_by": "Administrator", - "module": "Desk", - "name": "Bulk Update", - "name_case": "", - "owner": "Administrator", + ], + "issingle": 1, + "links": [], + "modified": "2022-08-03 12:20:50.742376", + "modified_by": "Administrator", + "module": "Desk", + "name": "Bulk Update", + "owner": "Administrator", "permissions": [ { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 1, - "delete": 1, - "email": 1, - "export": 0, - "if_owner": 0, - "import": 0, - "is_custom": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 0, - "role": "System Manager", - "set_user_permissions": 0, - "share": 1, - "submit": 0, + "create": 1, + "delete": 1, + "email": 1, + "print": 1, + "read": 1, + "role": "System Manager", + "share": 1, "write": 1 } - ], - "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1, - "track_seen": 0 + ], + "quick_entry": 1, + "sort_field": "modified", + "sort_order": "DESC", + "states": [], + "track_changes": 1 } \ No newline at end of file diff --git a/frappe/desk/doctype/bulk_update/bulk_update.py b/frappe/desk/doctype/bulk_update/bulk_update.py index 1e515bbc47..535be8155f 100644 --- a/frappe/desk/doctype/bulk_update/bulk_update.py +++ b/frappe/desk/doctype/bulk_update/bulk_update.py @@ -3,31 +3,31 @@ import frappe from frappe import _ +from frappe.core.doctype.submission_queue.submission_queue import queue_submission from frappe.model.document import Document from frappe.utils import cint +from frappe.utils.scheduler import is_scheduler_inactive class BulkUpdate(Document): - pass + @frappe.whitelist() + def bulk_update(self): + self.check_permission("write") + limit = self.limit if self.limit and cint(self.limit) < 500 else 500 + condition = "" + if self.condition: + if ";" in self.condition: + frappe.throw(_("; not allowed in condition")) -@frappe.whitelist() -def update(doctype, field, value, condition="", limit=500): - if not limit or cint(limit) > 500: - limit = 500 + condition = f" where {self.condition}" - if condition: - condition = " where " + condition - - if ";" in condition: - frappe.throw(_("; not allowed in condition")) - - docnames = frappe.db.sql_list( - f"""select name from `tab{doctype}`{condition} limit {limit} offset 0""" - ) - data = {} - data[field] = value - return submit_cancel_or_update_docs(doctype, docnames, "update", data) + docnames = frappe.db.sql_list( + f"""select name from `tab{self.document_type}`{condition} limit {limit} offset 0""" + ) + return submit_cancel_or_update_docs( + self.document_type, docnames, "update", {self.field: self.update_value} + ) @frappe.whitelist() @@ -44,8 +44,12 @@ def submit_cancel_or_update_docs(doctype, docnames, action="submit", data=None): try: message = "" if action == "submit" and doc.docstatus.is_draft(): - doc.submit() - message = _("Submitting {0}").format(doctype) + if doc.meta.queue_in_background and not is_scheduler_inactive(): + queue_submission(doc, action) + message = _("Queuing {0} for Submission").format(doctype) + else: + doc.submit() + message = _("Submitting {0}").format(doctype) elif action == "cancel" and doc.docstatus.is_submitted(): doc.cancel() message = _("Cancelling {0}").format(doctype) diff --git a/frappe/desk/doctype/calendar_view/calendar_view.js b/frappe/desk/doctype/calendar_view/calendar_view.js index a58a9555db..c302c1a4d8 100644 --- a/frappe/desk/doctype/calendar_view/calendar_view.js +++ b/frappe/desk/doctype/calendar_view/calendar_view.js @@ -1,35 +1,36 @@ // Copyright (c) 2017, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Calendar View', { - onload: function(frm) { - frm.trigger('reference_doctype'); +frappe.ui.form.on("Calendar View", { + onload: function (frm) { + frm.trigger("reference_doctype"); }, - refresh: function(frm) { + refresh: function (frm) { if (!frm.is_new()) { - frm.add_custom_button(__('Show Calendar'), - () => frappe.set_route('List', frm.doc.reference_doctype, 'Calendar', frm.doc.name)); + frm.add_custom_button(__("Show Calendar"), () => + frappe.set_route("List", frm.doc.reference_doctype, "Calendar", frm.doc.name) + ); } }, - reference_doctype: function(frm) { + reference_doctype: function (frm) { const { reference_doctype } = frm.doc; if (!reference_doctype) return; frappe.model.with_doctype(reference_doctype, () => { const meta = frappe.get_meta(reference_doctype); - const subject_options = meta.fields.filter( - df => !frappe.model.no_value_type.includes(df.fieldtype) - ).map(df => df.fieldname); + const subject_options = meta.fields + .filter((df) => !frappe.model.no_value_type.includes(df.fieldtype)) + .map((df) => df.fieldname); - const date_options = meta.fields.filter( - df => ['Date', 'Datetime'].includes(df.fieldtype) - ).map(df => df.fieldname); + const date_options = meta.fields + .filter((df) => ["Date", "Datetime"].includes(df.fieldtype)) + .map((df) => df.fieldname); - frm.set_df_property('subject_field', 'options', subject_options); - frm.set_df_property('start_date_field', 'options', date_options); - frm.set_df_property('end_date_field', 'options', date_options); + frm.set_df_property("subject_field", "options", subject_options); + frm.set_df_property("start_date_field", "options", date_options); + frm.set_df_property("end_date_field", "options", date_options); frm.refresh(); }); - } + }, }); diff --git a/frappe/desk/doctype/calendar_view/calendar_view_list.js b/frappe/desk/doctype/calendar_view/calendar_view_list.js new file mode 100644 index 0000000000..aa55a8ebbb --- /dev/null +++ b/frappe/desk/doctype/calendar_view/calendar_view_list.js @@ -0,0 +1,16 @@ +frappe.listview_settings["Calendar View"] = { + button: { + show(doc) { + return doc.name; + }, + get_label() { + return frappe.utils.icon("calendar", "sm"); + }, + get_description(doc) { + return __("View {0}", [`${doc.name}`]); + }, + action(doc) { + frappe.set_route("List", doc.reference_doctype, "Calendar", doc.name); + }, + }, +}; diff --git a/frappe/desk/doctype/console_log/console_log.js b/frappe/desk/doctype/console_log/console_log.js index 1ef4fdce59..9a980667ac 100644 --- a/frappe/desk/doctype/console_log/console_log.js +++ b/frappe/desk/doctype/console_log/console_log.js @@ -1,8 +1,7 @@ // Copyright (c) 2020, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Console Log', { +frappe.ui.form.on("Console Log", { // refresh: function(frm) { - // } }); diff --git a/frappe/desk/doctype/console_log/test_console_log.py b/frappe/desk/doctype/console_log/test_console_log.py index 0579098382..9ae8d1f1e8 100644 --- a/frappe/desk/doctype/console_log/test_console_log.py +++ b/frappe/desk/doctype/console_log/test_console_log.py @@ -1,8 +1,8 @@ # Copyright (c) 2020, Frappe Technologies and Contributors # License: MIT. See LICENSE # import frappe -import unittest +from frappe.tests.utils import FrappeTestCase -class TestConsoleLog(unittest.TestCase): +class TestConsoleLog(FrappeTestCase): pass diff --git a/frappe/event_streaming/__init__.py b/frappe/desk/doctype/custom_html_block/__init__.py similarity index 100% rename from frappe/event_streaming/__init__.py rename to frappe/desk/doctype/custom_html_block/__init__.py diff --git a/frappe/desk/doctype/custom_html_block/custom_html_block.js b/frappe/desk/doctype/custom_html_block/custom_html_block.js new file mode 100644 index 0000000000..727a73c92f --- /dev/null +++ b/frappe/desk/doctype/custom_html_block/custom_html_block.js @@ -0,0 +1,49 @@ +// Copyright (c) 2023, Frappe Technologies and contributors +// For license information, please see license.txt + +frappe.ui.form.on("Custom HTML Block", { + refresh(frm) { + render_custom_html_block(frm); + }, +}); + +function render_custom_html_block(frm) { + let wrapper = frm.fields_dict["preview"].wrapper; + wrapper.classList.add("mb-3"); + + let random_id = "custom-block-" + frappe.utils.get_random(5).toLowerCase(); + + class CustomBlock extends HTMLElement { + constructor() { + super(); + + // html + let div = document.createElement("div"); + div.innerHTML = frappe.dom.remove_script_and_style(frm.doc.html); + + // css + let style = document.createElement("style"); + style.textContent = frm.doc.style; + + // javascript + let script = document.createElement("script"); + script.textContent = ` + (function() { + let cname = ${JSON.stringify(random_id)}; + let root_element = document.querySelector(cname).shadowRoot; + ${frm.doc.script} + })(); + `; + + this.attachShadow({ mode: "open" }); + this.shadowRoot?.appendChild(div); + this.shadowRoot?.appendChild(style); + this.shadowRoot?.appendChild(script); + } + } + + if (!customElements.get(random_id)) { + customElements.define(random_id, CustomBlock); + } + wrapper.innerHTML = `<${random_id}>`; +} diff --git a/frappe/desk/doctype/custom_html_block/custom_html_block.json b/frappe/desk/doctype/custom_html_block/custom_html_block.json new file mode 100644 index 0000000000..6c3d80fba9 --- /dev/null +++ b/frappe/desk/doctype/custom_html_block/custom_html_block.json @@ -0,0 +1,152 @@ +{ + "actions": [], + "allow_rename": 1, + "autoname": "prompt", + "creation": "2023-05-17 13:58:37.311045", + "doctype": "DocType", + "engine": "InnoDB", + "field_order": [ + "preview_section", + "preview", + "html_section", + "html_message", + "html", + "javascript_section", + "js_message", + "script", + "css_section", + "style", + "roles_section", + "roles" + ], + "fields": [ + { + "collapsible": 1, + "collapsible_depends_on": "eval:true;", + "fieldname": "html_section", + "fieldtype": "Section Break", + "label": "HTML" + }, + { + "fieldname": "html", + "fieldtype": "Code", + "options": "HTML" + }, + { + "fieldname": "preview_section", + "fieldtype": "Section Break", + "label": "Preview" + }, + { + "fieldname": "preview", + "fieldtype": "HTML" + }, + { + "collapsible": 1, + "collapsible_depends_on": "eval:true;", + "fieldname": "javascript_section", + "fieldtype": "Section Break", + "label": "Javascript" + }, + { + "fieldname": "script", + "fieldtype": "Code", + "options": "JS" + }, + { + "collapsible": 1, + "collapsible_depends_on": "eval:true;", + "fieldname": "css_section", + "fieldtype": "Section Break", + "label": "CSS" + }, + { + "fieldname": "style", + "fieldtype": "Code", + "options": "CSS" + }, + { + "fieldname": "js_message", + "fieldtype": "HTML", + "label": "JS Message", + "options": "

    To interact with above HTML you will have to use `root_element` as a parent selector.

    For example:

    // here root_element is provided by default\nlet some_class_element = root_element.querySelector('.some-class');\nsome_class_element.textContent = \"New content\";\n
    " + }, + { + "fieldname": "html_message", + "fieldtype": "HTML", + "label": "HTML Message", + "options": "

    You cannot use global class on elements. The css for those classes will not be applied on this HTML, you will have to rewrite styles again in CSS field

    For Example:

    \n
    // style for class m-3 will not work\n
    <div class=\"m-3\"></div>
    \n
    // You will have to add style of m-3 in CSS field below like\n
    .m-3 {\n
        margin: 14px!important\n
    }\n
    " + }, + { + "fieldname": "roles_section", + "fieldtype": "Section Break", + "label": "Roles" + }, + { + "fieldname": "roles", + "fieldtype": "Table", + "label": "Roles", + "options": "Has Role" + } + ], + "index_web_pages_for_search": 1, + "links": [], + "modified": "2023-05-17 17:17:04.232519", + "modified_by": "Administrator", + "module": "Desk", + "name": "Custom HTML Block", + "naming_rule": "Set by user", + "owner": "Administrator", + "permissions": [ + { + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "All", + "share": 1 + }, + { + "create": 1, + "delete": 1, + "email": 1, + "export": 1, + "if_owner": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "All", + "share": 1, + "write": 1 + }, + { + "create": 1, + "delete": 1, + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1, + "write": 1 + }, + { + "create": 1, + "delete": 1, + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "Workspace Manager", + "share": 1, + "write": 1 + } + ], + "sort_field": "modified", + "sort_order": "DESC", + "states": [], + "track_changes": 1 +} \ No newline at end of file diff --git a/frappe/desk/doctype/custom_html_block/custom_html_block.py b/frappe/desk/doctype/custom_html_block/custom_html_block.py new file mode 100644 index 0000000000..7f85c2db5f --- /dev/null +++ b/frappe/desk/doctype/custom_html_block/custom_html_block.py @@ -0,0 +1,9 @@ +# Copyright (c) 2023, Frappe Technologies and contributors +# For license information, please see license.txt + +# import frappe +from frappe.model.document import Document + + +class CustomHTMLBlock(Document): + pass diff --git a/frappe/desk/doctype/custom_html_block/test_custom_html_block.py b/frappe/desk/doctype/custom_html_block/test_custom_html_block.py new file mode 100644 index 0000000000..fbef9af45b --- /dev/null +++ b/frappe/desk/doctype/custom_html_block/test_custom_html_block.py @@ -0,0 +1,9 @@ +# Copyright (c) 2023, Frappe Technologies and Contributors +# See license.txt + +# import frappe +from frappe.tests.utils import FrappeTestCase + + +class TestCustomHTMLBlock(FrappeTestCase): + pass diff --git a/frappe/desk/doctype/dashboard/dashboard.js b/frappe/desk/doctype/dashboard/dashboard.js index c640259cf2..9f584ca552 100644 --- a/frappe/desk/doctype/dashboard/dashboard.js +++ b/frappe/desk/doctype/dashboard/dashboard.js @@ -1,30 +1,30 @@ // Copyright (c) 2019, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Dashboard', { - refresh: function(frm) { - frm.add_custom_button(__("Show Dashboard"), - () => frappe.set_route('dashboard-view', frm.doc.name) +frappe.ui.form.on("Dashboard", { + refresh: function (frm) { + frm.add_custom_button(__("Show Dashboard"), () => + frappe.set_route("dashboard-view", frm.doc.name) ); if (!frappe.boot.developer_mode && frm.doc.is_standard) { frm.disable_form(); } - frm.set_query("chart", "charts", function() { + frm.set_query("chart", "charts", function () { return { filters: { is_public: 1, - } + }, }; }); - frm.set_query("card", "cards", function() { + frm.set_query("card", "cards", function () { return { filters: { is_public: 1, - } + }, }; }); - } + }, }); diff --git a/frappe/desk/doctype/dashboard/dashboard_list.js b/frappe/desk/doctype/dashboard/dashboard_list.js index d60a324048..80ebbd4355 100644 --- a/frappe/desk/doctype/dashboard/dashboard_list.js +++ b/frappe/desk/doctype/dashboard/dashboard_list.js @@ -1,4 +1,4 @@ -frappe.listview_settings['Dashboard'] = { +frappe.listview_settings["Dashboard"] = { button: { show(doc) { return doc.name; @@ -7,10 +7,10 @@ frappe.listview_settings['Dashboard'] = { return frappe.utils.icon("dashboard-list", "sm"); }, get_description(doc) { - return __('View {0}', [`${doc.name}`]); + return __("View {0}", [`${doc.name}`]); }, action(doc) { - frappe.set_route('dashboard-view', doc.name); - } + frappe.set_route("dashboard-view", doc.name); + }, }, -}; \ No newline at end of file +}; diff --git a/frappe/desk/doctype/dashboard/test_dashboard.py b/frappe/desk/doctype/dashboard/test_dashboard.py index d2ba871509..99aeecaee6 100644 --- a/frappe/desk/doctype/dashboard/test_dashboard.py +++ b/frappe/desk/doctype/dashboard/test_dashboard.py @@ -1,7 +1,7 @@ # Copyright (c) 2019, Frappe Technologies and Contributors # License: MIT. See LICENSE -import unittest +from frappe.tests.utils import FrappeTestCase -class TestDashboard(unittest.TestCase): +class TestDashboard(FrappeTestCase): pass diff --git a/frappe/desk/doctype/dashboard_chart/dashboard_chart.js b/frappe/desk/doctype/dashboard_chart/dashboard_chart.js index 0b93786e8e..6d23be79d7 100644 --- a/frappe/desk/doctype/dashboard_chart/dashboard_chart.js +++ b/frappe/desk/doctype/dashboard_chart/dashboard_chart.js @@ -1,42 +1,44 @@ // Copyright (c) 2019, Frappe Technologies and contributors // For license information, please see license.txt -frappe.provide('frappe.dashboards.chart_sources'); +frappe.provide("frappe.dashboards.chart_sources"); -frappe.ui.form.on('Dashboard Chart', { - setup: function(frm) { +frappe.ui.form.on("Dashboard Chart", { + setup: function (frm) { // fetch timeseries from source - frm.add_fetch('source', 'timeseries', 'timeseries'); + frm.add_fetch("source", "timeseries", "timeseries"); }, - before_save: function(frm) { - let dynamic_filters = JSON.parse(frm.doc.dynamic_filters_json || 'null'); - let static_filters = JSON.parse(frm.doc.filters_json || 'null'); - static_filters = - frappe.dashboard_utils.remove_common_static_filter_values(static_filters, dynamic_filters); + before_save: function (frm) { + let dynamic_filters = JSON.parse(frm.doc.dynamic_filters_json || "null"); + let static_filters = JSON.parse(frm.doc.filters_json || "null"); + static_filters = frappe.dashboard_utils.remove_common_static_filter_values( + static_filters, + dynamic_filters + ); - frm.set_value('filters_json', JSON.stringify(static_filters)); - frm.trigger('show_filters'); + frm.set_value("filters_json", JSON.stringify(static_filters)); + frm.trigger("show_filters"); }, - refresh: function(frm) { + refresh: function (frm) { frm.chart_filters = null; frm.is_disabled = !frappe.boot.developer_mode && frm.doc.is_standard; if (frm.is_disabled) { - !frm.doc.custom_options && frm.set_df_property('chart_options_section', 'hidden', 1); + !frm.doc.custom_options && frm.set_df_property("chart_options_section", "hidden", 1); frm.disable_form(); } - frm.add_custom_button('Add Chart to Dashboard', () => { + frm.add_custom_button("Add Chart to Dashboard", () => { const dialog = frappe.dashboard_utils.get_add_to_dashboard_dialog( frm.doc.name, - 'Dashboard Chart', - 'frappe.desk.doctype.dashboard_chart.dashboard_chart.add_chart_to_dashboard' + "Dashboard Chart", + "frappe.desk.doctype.dashboard_chart.dashboard_chart.add_chart_to_dashboard" ); if (!frm.doc.chart_name) { - frappe.msgprint(__('Please create chart first')); + frappe.msgprint(__("Please create chart first")); } else { dialog.show(); } @@ -45,199 +47,227 @@ frappe.ui.form.on('Dashboard Chart', { frm.set_df_property("filters_section", "hidden", 1); frm.set_df_property("dynamic_filters_section", "hidden", 1); - frm.trigger('set_parent_document_type'); - frm.trigger('set_time_series'); - frm.set_query('document_type', function() { + frm.trigger("set_parent_document_type"); + frm.trigger("set_time_series"); + frm.set_query("document_type", function () { return { filters: { - 'issingle': false - } - } + issingle: false, + }, + }; }); - frm.trigger('update_options'); - frm.trigger('set_heatmap_year_options'); + frm.trigger("update_options"); + frm.trigger("set_heatmap_year_options"); if (frm.doc.report_name) { - frm.trigger('set_chart_report_filters'); + frm.trigger("set_chart_report_filters"); } }, - is_standard: function(frm) { + is_standard: function (frm) { if (frappe.boot.developer_mode && frm.doc.is_standard) { - frm.trigger('render_dynamic_filters_table'); + frm.trigger("render_dynamic_filters_table"); } else { frm.set_df_property("dynamic_filters_section", "hidden", 1); } }, - source: function(frm) { + source: function (frm) { frm.trigger("show_filters"); }, - set_heatmap_year_options: function(frm) { - if (frm.doc.type == 'Heatmap') { - frappe.db.get_doc('System Settings').then(doc => { + set_heatmap_year_options: function (frm) { + if (frm.doc.type == "Heatmap") { + frappe.db.get_doc("System Settings").then((doc) => { const creation_date = doc.creation; - frm.set_df_property('heatmap_year', 'options', frappe.dashboard_utils.get_years_since_creation(creation_date)); + frm.set_df_property( + "heatmap_year", + "options", + frappe.dashboard_utils.get_years_since_creation(creation_date) + ); }); } }, - chart_type: function(frm) { - frm.trigger('set_time_series'); - if (frm.doc.chart_type == 'Report') { - frm.set_query('report_name', () => { + chart_type: function (frm) { + frm.trigger("set_time_series"); + if (frm.doc.chart_type == "Report") { + frm.set_query("report_name", () => { return { filters: { - 'report_type': ['!=', 'Report Builder'] - } - } + report_type: ["!=", "Report Builder"], + }, + }; }); } else { - frm.set_value('document_type', ''); + frm.set_value("document_type", ""); } }, - set_time_series: function(frm) { + set_time_series: function (frm) { // set timeseries based on chart type - if (['Count', 'Average', 'Sum'].includes(frm.doc.chart_type)) { - frm.set_value('timeseries', 1); + if (["Count", "Average", "Sum"].includes(frm.doc.chart_type)) { + frm.set_value("timeseries", 1); } else { - frm.set_value('timeseries', 0); + frm.set_value("timeseries", 0); } }, - document_type: function(frm) { + document_type: function (frm) { // update `based_on` options based on date / datetime fields - frm.set_value('source', ''); - frm.set_value('based_on', ''); - frm.set_value('value_based_on', ''); - frm.set_value('parent_document_type', ''); - frm.set_value('filters_json', '[]'); - frm.set_value('dynamic_filters_json', '[]'); - frm.trigger('update_options'); - frm.trigger('set_parent_document_type'); + frm.set_value("source", ""); + frm.set_value("based_on", ""); + frm.set_value("value_based_on", ""); + frm.set_value("parent_document_type", ""); + frm.set_value("filters_json", "[]"); + frm.set_value("dynamic_filters_json", "[]"); + frm.trigger("update_options"); + frm.trigger("set_parent_document_type"); }, - report_name: function(frm) { - frm.set_value('x_field', ''); - frm.set_value('y_axis', []); - frm.set_df_property('x_field', 'options', []); - frm.set_value('filters_json', '{}'); - frm.set_value('dynamic_filters_json', '{}'); - frm.set_value('use_report_chart', 0); - frm.trigger('set_chart_report_filters'); + report_name: function (frm) { + frm.set_value("x_field", ""); + frm.set_value("y_axis", []); + frm.set_df_property("x_field", "options", []); + frm.set_value("filters_json", "{}"); + frm.set_value("dynamic_filters_json", "{}"); + frm.set_value("use_report_chart", 0); + frm.trigger("set_chart_report_filters"); }, - set_chart_report_filters: function(frm) { + set_chart_report_filters: function (frm) { let report_name = frm.doc.report_name; if (report_name) { if (frm.doc.filters_json.length > 2) { - frm.trigger('show_filters'); - frm.trigger('set_chart_field_options'); + frm.trigger("show_filters"); + frm.trigger("set_chart_field_options"); } else { - frappe.report_utils.get_report_filters(report_name).then(filters => { + frappe.report_utils.get_report_filters(report_name).then((filters) => { if (filters) { frm.chart_filters = filters; let filter_values = frappe.report_utils.get_filter_values(filters); - frm.set_value('filters_json', JSON.stringify(filter_values)); + frm.set_value("filters_json", JSON.stringify(filter_values)); } - frm.trigger('show_filters'); - frm.trigger('set_chart_field_options'); + frm.trigger("show_filters"); + frm.trigger("set_chart_field_options"); }); } - } }, - use_report_chart: function(frm) { - !frm.doc.use_report_chart && frm.trigger('set_chart_field_options'); + use_report_chart: function (frm) { + !frm.doc.use_report_chart && frm.trigger("set_chart_field_options"); }, - set_chart_field_options: function(frm) { + set_chart_field_options: function (frm) { let filters = frm.doc.filters_json.length > 2 ? JSON.parse(frm.doc.filters_json) : null; if (frm.doc.dynamic_filters_json && frm.doc.dynamic_filters_json.length > 2) { filters = frappe.dashboard_utils.get_all_filters(frm.doc); } - frappe.xcall( - 'frappe.desk.query_report.run', - { + frappe + .xcall("frappe.desk.query_report.run", { report_name: frm.doc.report_name, filters: filters, - ignore_prepared_report: 1 - } - ).then(data => { - frm.report_data = data; - let report_has_chart = Boolean(data.chart); + ignore_prepared_report: 1, + }) + .then((data) => { + frm.report_data = data; + let report_has_chart = Boolean(data.chart); - frm.set_df_property('use_report_chart', 'hidden', !report_has_chart); + frm.set_df_property("use_report_chart", "hidden", !report_has_chart); - if (!frm.doc.use_report_chart) { - if (data.result.length) { - frm.field_options = frappe.report_utils.get_field_options_from_report(data.columns, data); - frm.set_df_property('x_field', 'options', frm.field_options.non_numeric_fields); - if (!frm.field_options.numeric_fields.length) { - frappe.msgprint(__("Report has no numeric fields, please change the Report Name")); + if (!frm.doc.use_report_chart) { + if (data.result.length) { + frm.field_options = frappe.report_utils.get_field_options_from_report( + data.columns, + data + ); + frm.set_df_property( + "x_field", + "options", + frm.field_options.non_numeric_fields + ); + if (!frm.field_options.numeric_fields.length) { + frappe.msgprint( + __("Report has no numeric fields, please change the Report Name") + ); + } else { + let y_field_df = frappe.meta.get_docfield( + "Dashboard Chart Field", + "y_field", + frm.doc.name + ); + y_field_df.options = frm.field_options.numeric_fields; + } } else { - let y_field_df = frappe.meta.get_docfield('Dashboard Chart Field', 'y_field', frm.doc.name); - y_field_df.options = frm.field_options.numeric_fields; + frappe.msgprint( + __( + "Report has no data, please modify the filters or change the Report Name" + ) + ); } } else { - frappe.msgprint(__('Report has no data, please modify the filters or change the Report Name')); + frm.set_value("use_report_chart", 1); + frm.set_df_property("use_report_chart", "hidden", false); } - } else { - frm.set_value('use_report_chart', 1); - frm.set_df_property('use_report_chart', 'hidden', false); - } - }); + }); }, - timespan: function(frm) { + timespan: function (frm) { const time_interval_options = { "Select Date Range": ["Quarterly", "Monthly", "Weekly", "Daily"], "All Time": ["Yearly", "Monthly"], "Last Year": ["Quarterly", "Monthly", "Weekly", "Daily"], "Last Quarter": ["Monthly", "Weekly", "Daily"], "Last Month": ["Weekly", "Daily"], - "Last Week": ["Daily"] + "Last Week": ["Daily"], }; if (frm.doc.timespan) { - frm.set_df_property('time_interval', 'options', time_interval_options[frm.doc.timespan]); + frm.set_df_property( + "time_interval", + "options", + time_interval_options[frm.doc.timespan] + ); } }, - update_options: function(frm) { + update_options: function (frm) { let doctype = frm.doc.document_type; let date_fields = [ - {label: __('Created On'), value: 'creation'}, - {label: __('Last Modified On'), value: 'modified'} + { label: __("Created On"), value: "creation" }, + { label: __("Last Modified On"), value: "modified" }, ]; let value_fields = []; - let group_by_fields = [{label: 'Created By', value: 'owner'}]; + let group_by_fields = [{ label: "Created By", value: "owner" }]; let aggregate_function_fields = []; - let update_form = function() { + let update_form = function () { // update select options - frm.set_df_property('based_on', 'options', date_fields); - frm.set_df_property('value_based_on', 'options', value_fields); - frm.set_df_property('group_by_based_on', 'options', group_by_fields); - frm.set_df_property('aggregate_function_based_on', 'options', aggregate_function_fields); + frm.set_df_property("based_on", "options", date_fields); + frm.set_df_property("value_based_on", "options", value_fields); + frm.set_df_property("group_by_based_on", "options", group_by_fields); + frm.set_df_property( + "aggregate_function_based_on", + "options", + aggregate_function_fields + ); frm.trigger("show_filters"); - } - + }; if (doctype) { frappe.model.with_doctype(doctype, () => { // get all date and datetime fields - frappe.get_meta(doctype).fields.map(df => { - if (['Date', 'Datetime'].includes(df.fieldtype)) { - date_fields.push({label: df.label, value: df.fieldname}); + frappe.get_meta(doctype).fields.map((df) => { + if (["Date", "Datetime"].includes(df.fieldtype)) { + date_fields.push({ label: df.label, value: df.fieldname }); } - if (['Int', 'Float', 'Currency', 'Percent', 'Duration'].includes(df.fieldtype)) { - value_fields.push({label: df.label, value: df.fieldname}); - aggregate_function_fields.push({label: df.label, value: df.fieldname}); + if ( + ["Int", "Float", "Currency", "Percent", "Duration"].includes(df.fieldtype) + ) { + value_fields.push({ label: df.label, value: df.fieldname }); + aggregate_function_fields.push({ label: df.label, value: df.fieldname }); } - if (['Link', 'Select'].includes(df.fieldtype)) { - group_by_fields.push({label: df.label, value: df.fieldname}); + if (["Link", "Select"].includes(df.fieldtype)) { + group_by_fields.push({ label: df.label, value: df.fieldname }); } }); update_form(); @@ -246,92 +276,89 @@ frappe.ui.form.on('Dashboard Chart', { // update select options update_form(); } - }, - show_filters: function(frm) { + show_filters: function (frm) { frm.chart_filters = []; - frappe.dashboard_utils.get_filters_for_chart_type(frm.doc).then(filters => { + frappe.dashboard_utils.get_filters_for_chart_type(frm.doc).then((filters) => { if (filters) { frm.chart_filters = filters; } - frm.trigger('render_filters_table'); + frm.trigger("render_filters_table"); if (frappe.boot.developer_mode && frm.doc.is_standard) { - frm.trigger('render_dynamic_filters_table'); + frm.trigger("render_dynamic_filters_table"); } }); }, - render_filters_table: function(frm) { + render_filters_table: function (frm) { frm.set_df_property("filters_section", "hidden", 0); - let is_document_type = frm.doc.chart_type!== 'Report' && frm.doc.chart_type!=='Custom'; - let is_dynamic_filter = f => ['Date', 'DateRange'].includes(f.fieldtype) && f.default; + let is_document_type = frm.doc.chart_type !== "Report" && frm.doc.chart_type !== "Custom"; + let is_dynamic_filter = (f) => ["Date", "DateRange"].includes(f.fieldtype) && f.default; - let wrapper = $(frm.get_field('filters_json').wrapper).empty(); + let wrapper = $(frm.get_field("filters_json").wrapper).empty(); let table = $(` - - - + + +
    ${__('Filter')}${__('Condition')}${__('Value')}${__("Filter")}${__("Condition")}${__("Value")}
    `).appendTo(wrapper); $(`

    ${__("Click table to edit")}

    `).appendTo(wrapper); - let filters = JSON.parse(frm.doc.filters_json || '[]'); + let filters = JSON.parse(frm.doc.filters_json || "[]"); var filters_set = false; // Set dynamic filters for reports - if (frm.doc.chart_type == 'Report') { + if (frm.doc.chart_type == "Report") { let set_filters = false; - frm.chart_filters.forEach(f => { + frm.chart_filters.forEach((f) => { if (is_dynamic_filter(f)) { filters[f.fieldname] = f.default; set_filters = true; } }); - set_filters && frm.set_value('filters_json', JSON.stringify(filters)); + set_filters && frm.set_value("filters_json", JSON.stringify(filters)); } let fields = []; if (is_document_type) { fields = [ { - fieldtype: 'HTML', - fieldname: 'filter_area', - } + fieldtype: "HTML", + fieldname: "filter_area", + }, ]; if (filters.length > 0) { - filters.forEach( filter => { - const filter_row = - $(` + filters.forEach((filter) => { + const filter_row = $(` ${filter[1]} ${filter[2] || ""} ${filter[3]} `); - table.find('tbody').append(filter_row); + table.find("tbody").append(filter_row); filters_set = true; }); } } else if (frm.chart_filters.length) { - fields = frm.chart_filters.filter(f => f.fieldname); + fields = frm.chart_filters.filter((f) => f.fieldname); - fields.map(f => { + fields.map((f) => { if (filters[f.fieldname]) { - let condition = '='; - const filter_row = - $(` + let condition = "="; + const filter_row = $(` ${f.label} ${condition} ${filters[f.fieldname] || ""} `); - table.find('tbody').append(filter_row); + table.find("tbody").append(filter_row); filters_set = true; } }); @@ -340,39 +367,39 @@ frappe.ui.form.on('Dashboard Chart', { if (!filters_set) { const filter_row = $(` ${__("Click to Set Filters")}`); - table.find('tbody').append(filter_row); + table.find("tbody").append(filter_row); } - table.on('click', () => { - frm.is_disabled && frappe.throw(__('Cannot edit filters for standard charts')); + table.on("click", () => { + frm.is_disabled && frappe.throw(__("Cannot edit filters for standard charts")); let dialog = new frappe.ui.Dialog({ - title: __('Set Filters'), - fields: fields.filter(f => !is_dynamic_filter(f)), - primary_action: function() { + title: __("Set Filters"), + fields: fields.filter((f) => !is_dynamic_filter(f)), + primary_action: function () { let values = this.get_values(); if (values) { this.hide(); if (is_document_type) { let filters = frm.filter_group.get_filters(); - frm.set_value('filters_json', JSON.stringify(filters)); + frm.set_value("filters_json", JSON.stringify(filters)); } else { - frm.set_value('filters_json', JSON.stringify(values)); + frm.set_value("filters_json", JSON.stringify(values)); } - frm.trigger('show_filters'); - if (frm.doc.chart_type == 'Report') { - frm.trigger('set_chart_report_filters'); + frm.trigger("show_filters"); + if (frm.doc.chart_type == "Report") { + frm.trigger("set_chart_report_filters"); } } }, - primary_action_label: "Set" + primary_action_label: "Set", }); frappe.dashboards.filters_dialog = dialog; if (is_document_type) { frm.filter_group = new frappe.ui.FilterGroup({ - parent: dialog.get_field('filter_area').$wrapper, + parent: dialog.get_field("filter_area").$wrapper, doctype: frm.doc.document_type, parent_doctype: frm.doc.parent_document_type, on_change: () => {}, @@ -383,12 +410,14 @@ frappe.ui.form.on('Dashboard Chart', { dialog.show(); - if (frm.doc.chart_type == 'Report') { + if (frm.doc.chart_type == "Report") { //Set query report object so that it can be used while fetching filter values in the report - frappe.query_report = new frappe.views.QueryReport({'filters': dialog.fields_list}); - frappe.query_reports[frm.doc.report_name] - && frappe.query_reports[frm.doc.report_name].onload - && frappe.query_reports[frm.doc.report_name].onload(frappe.query_report); + frappe.query_report = new frappe.views.QueryReport({ + filters: dialog.fields_list, + }); + frappe.query_reports[frm.doc.report_name] && + frappe.query_reports[frm.doc.report_name].onload && + frappe.query_reports[frm.doc.report_name].onload(frappe.query_report); } dialog.set_values(filters); @@ -398,37 +427,40 @@ frappe.ui.form.on('Dashboard Chart', { render_dynamic_filters_table(frm) { frm.set_df_property("dynamic_filters_section", "hidden", 0); - let is_document_type = frm.doc.chart_type !== 'Report' - && frm.doc.chart_type !== 'Custom'; + let is_document_type = frm.doc.chart_type !== "Report" && frm.doc.chart_type !== "Custom"; - let wrapper = $(frm.get_field('dynamic_filters_json').wrapper).empty(); + let wrapper = $(frm.get_field("dynamic_filters_json").wrapper).empty(); - frm.dynamic_filter_table = $(` + frm.dynamic_filter_table = + $(`
    - - - + + +
    ${__('Filter')}${__('Condition')}${__('Value')}${__("Filter")}${__("Condition")}${__("Value")}
    `).appendTo(wrapper); - frm.dynamic_filters = frm.doc.dynamic_filters_json && frm.doc.dynamic_filters_json.length > 2 - ? JSON.parse(frm.doc.dynamic_filters_json) - : null; + frm.dynamic_filters = + frm.doc.dynamic_filters_json && frm.doc.dynamic_filters_json.length > 2 + ? JSON.parse(frm.doc.dynamic_filters_json) + : null; - frm.trigger('set_dynamic_filters_in_table'); + frm.trigger("set_dynamic_filters_in_table"); - let filters = JSON.parse(frm.doc.filters_json || '[]'); + let filters = JSON.parse(frm.doc.filters_json || "[]"); let fields = frappe.dashboard_utils.get_fields_for_dynamic_filter_dialog( - is_document_type, filters, frm.dynamic_filters + is_document_type, + filters, + frm.dynamic_filters ); - frm.dynamic_filter_table.on('click', () => { + frm.dynamic_filter_table.on("click", () => { let dialog = new frappe.ui.Dialog({ - title: __('Set Dynamic Filters'), + title: __("Set Dynamic Filters"), fields: fields, primary_action: () => { let values = dialog.get_values(); @@ -436,19 +468,19 @@ frappe.ui.form.on('Dashboard Chart', { let dynamic_filters = []; for (let key of Object.keys(values)) { if (is_document_type) { - let [doctype, fieldname] = key.split(':'); - dynamic_filters.push([doctype, fieldname, '=', values[key]]); + let [doctype, fieldname] = key.split(":"); + dynamic_filters.push([doctype, fieldname, "=", values[key]]); } } if (is_document_type) { - frm.set_value('dynamic_filters_json', JSON.stringify(dynamic_filters)); + frm.set_value("dynamic_filters_json", JSON.stringify(dynamic_filters)); } else { - frm.set_value('dynamic_filters_json', JSON.stringify(values)); + frm.set_value("dynamic_filters_json", JSON.stringify(values)); } - frm.trigger('set_dynamic_filters_in_table'); + frm.trigger("set_dynamic_filters_in_table"); }, - primary_action_label: "Set" + primary_action_label: "Set", }); dialog.show(); @@ -456,71 +488,66 @@ frappe.ui.form.on('Dashboard Chart', { }); }, - set_dynamic_filters_in_table: function(frm) { - frm.dynamic_filters = frm.doc.dynamic_filters_json && frm.doc.dynamic_filters_json.length > 2 - ? JSON.parse(frm.doc.dynamic_filters_json) - : null; + set_dynamic_filters_in_table: function (frm) { + frm.dynamic_filters = + frm.doc.dynamic_filters_json && frm.doc.dynamic_filters_json.length > 2 + ? JSON.parse(frm.doc.dynamic_filters_json) + : null; if (!frm.dynamic_filters) { const filter_row = $(` ${__("Click to Set Dynamic Filters")}`); - frm.dynamic_filter_table.find('tbody').html(filter_row); + frm.dynamic_filter_table.find("tbody").html(filter_row); } else { - let filter_rows = ''; + let filter_rows = ""; if ($.isArray(frm.dynamic_filters)) { - frm.dynamic_filters.forEach(filter => { - filter_rows += - ` + frm.dynamic_filters.forEach((filter) => { + filter_rows += ` ${filter[1]} ${filter[2] || ""} ${filter[3]} `; }); } else { - let condition = '='; + let condition = "="; for (let [key, val] of Object.entries(frm.dynamic_filters)) { - filter_rows += - ` + filter_rows += ` ${key} ${condition} ${val || ""} - ` - ; + `; } } - frm.dynamic_filter_table.find('tbody').html(filter_rows); + frm.dynamic_filter_table.find("tbody").html(filter_rows); } }, - set_parent_document_type: async function(frm) { + set_parent_document_type: async function (frm) { let document_type = frm.doc.document_type; - let doc_is_table = document_type && - (await frappe.db.get_value('DocType', document_type, 'istable')).message.istable; + let doc_is_table = + document_type && + (await frappe.db.get_value("DocType", document_type, "istable")).message.istable; - frm.set_df_property('parent_document_type', 'hidden', !doc_is_table); + frm.set_df_property("parent_document_type", "hidden", !doc_is_table); if (document_type && doc_is_table) { - let parent = await frappe.db.get_list('DocField', { - filters: { - 'fieldtype': 'Table', - 'options': document_type - }, - fields: ['parent'] - }); + let parents = await frappe.xcall( + "frappe.desk.doctype.dashboard_chart.dashboard_chart.get_parent_doctypes", + { child_type: document_type } + ); - parent && frm.set_query('parent_document_type', function() { + frm.set_query("parent_document_type", function () { return { filters: { - "name": ['in', parent.map(({ parent }) => parent)] - } + name: ["in", parents], + }, }; }); - if (parent.length === 1) { - frm.set_value('parent_document_type', parent[0].parent); + if (parents.length === 1) { + frm.set_value("parent_document_type", parents[0]); } } - } - + }, }); diff --git a/frappe/desk/doctype/dashboard_chart/dashboard_chart.json b/frappe/desk/doctype/dashboard_chart/dashboard_chart.json index a5d30c10e5..a5aa6cc20a 100644 --- a/frappe/desk/doctype/dashboard_chart/dashboard_chart.json +++ b/frappe/desk/doctype/dashboard_chart/dashboard_chart.json @@ -42,7 +42,8 @@ "column_break_2", "color", "section_break_10", - "last_synced_on" + "last_synced_on", + "roles" ], "fields": [ { @@ -277,13 +278,20 @@ "fieldtype": "Link", "label": "Parent Document Type", "options": "DocType" + }, + { + "fieldname": "roles", + "fieldtype": "Table", + "label": "Roles", + "options": "Has Role" } ], "links": [], - "modified": "2021-11-09 17:18:11.456145", + "modified": "2022-07-27 11:09:09.203236", "modified_by": "Administrator", "module": "Desk", "name": "Dashboard Chart", + "naming_rule": "By fieldname", "owner": "Administrator", "permissions": [ { @@ -323,5 +331,6 @@ ], "sort_field": "modified", "sort_order": "DESC", + "states": [], "track_changes": 1 } \ No newline at end of file diff --git a/frappe/desk/doctype/dashboard_chart/dashboard_chart.py b/frappe/desk/doctype/dashboard_chart/dashboard_chart.py index 75f230a901..5cbeb06e33 100644 --- a/frappe/desk/doctype/dashboard_chart/dashboard_chart.py +++ b/frappe/desk/doctype/dashboard_chart/dashboard_chart.py @@ -11,7 +11,7 @@ from frappe.config import get_modules_from_all_apps_for_user from frappe.model.document import Document from frappe.model.naming import append_number_if_name_exists from frappe.modules.export_file import export_to_files -from frappe.utils import cint, get_datetime, getdate, now_datetime, nowdate +from frappe.utils import cint, get_datetime, getdate, has_common, now_datetime, nowdate from frappe.utils.dashboard import cache_source from frappe.utils.data import format_date from frappe.utils.dateutils import ( @@ -87,6 +87,11 @@ def has_permission(doc, ptype, user): if doc.document_type in allowed_doctypes: return True + if doc.roles: + allowed = [d.role for d in doc.roles] + if has_common(roles, allowed): + return True + return False @@ -210,14 +215,13 @@ def get_chart_config(chart, filters, timespan, timegrain, from_date, to_date): group_by="_unit", order_by="_unit asc", as_list=True, - ignore_ifnull=True, ) result = get_result(data, timegrain, from_date, to_date, chart.chart_type) return { "labels": [ - format_date(get_period(r[0], timegrain)) + format_date(get_period(r[0], timegrain), parse_day_first=True) if timegrain in ("Daily", "Weekly") else get_period(r[0], timegrain) for r in result @@ -244,7 +248,7 @@ def get_heatmap_chart_config(chart, filters, heatmap_year): timestamp_field = f"extract(epoch from timestamp {datefield})" data = dict( - frappe.db.get_all( + frappe.get_all( doctype, fields=[ timestamp_field, @@ -274,28 +278,20 @@ def get_group_by_chart_config(chart, filters): group_by_field = chart.group_by_based_on doctype = chart.document_type - data = frappe.db.get_list( + data = frappe.get_list( doctype, fields=[ f"{group_by_field} as name", - "{aggregate_function}({value_field}) as count".format( - aggregate_function=aggregate_function, value_field=value_field - ), + f"{aggregate_function}({value_field}) as count", ], filters=filters, + parent_doctype=chart.parent_document_type, group_by=group_by_field, order_by="count desc", ignore_ifnull=True, ) if data: - if chart.number_of_groups and chart.number_of_groups < len(data): - other_count = 0 - for i in range(chart.number_of_groups - 1, len(data)): - other_count += data[i]["count"] - data = data[0 : chart.number_of_groups - 1] - data.append({"name": "Other", "count": other_count}) - chart_config = { "labels": [item["name"] if item["name"] else "Not Specified" for item in data], "datasets": [{"name": chart.name, "values": [item["count"] for item in data]}], @@ -387,3 +383,25 @@ class DashboardChart(Document): json.loads(self.custom_options) except ValueError as error: frappe.throw(_("Invalid json added in the custom options: {0}").format(error)) + + +@frappe.whitelist() +def get_parent_doctypes(child_type: str) -> list[str]: + """Get all parent doctypes that have the child doctype.""" + assert isinstance(child_type, str) + + standard = frappe.get_all( + "DocField", + fields="parent", + filters={"fieldtype": "Table", "options": child_type}, + pluck="parent", + ) + + custom = frappe.get_all( + "Custom Field", + fields="dt", + filters={"fieldtype": "Table", "options": child_type}, + pluck="dt", + ) + + return standard + custom diff --git a/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py b/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py index 820f3c0555..ddbabedcb4 100644 --- a/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py +++ b/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py @@ -173,7 +173,7 @@ class TestDashboardChart(FrappeTestCase): self.assertEqual(result.get("datasets")[0].get("values"), [200.0, 400.0, 300.0, 0.0, 100.0, 0.0]) self.assertEqual( result.get("labels"), - ["06-01-2019", "07-01-2019", "08-01-2019", "09-01-2019", "10-01-2019", "11-01-2019"], + ["01-06-2019", "01-07-2019", "01-08-2019", "01-09-2019", "01-10-2019", "01-11-2019"], ) def test_weekly_dashboard_chart(self): @@ -203,7 +203,7 @@ class TestDashboardChart(FrappeTestCase): result = get(chart_name="Test Weekly Dashboard Chart", refresh=1) self.assertEqual(result.get("datasets")[0].get("values"), [50.0, 300.0, 800.0, 0.0]) - self.assertEqual(result.get("labels"), ["12-30-2018", "06-01-2019", "01-13-2019", "01-20-2019"]) + self.assertEqual(result.get("labels"), ["12-30-2018", "01-06-2019", "01-13-2019", "01-20-2019"]) def test_avg_dashboard_chart(self): insert_test_records() @@ -230,7 +230,7 @@ class TestDashboardChart(FrappeTestCase): with patch.object(frappe.utils.data, "get_first_day_of_the_week", return_value="Monday"): result = get(chart_name="Test Average Dashboard Chart", refresh=1) - self.assertEqual(result.get("labels"), ["12-30-2018", "06-01-2019", "01-13-2019", "01-20-2019"]) + self.assertEqual(result.get("labels"), ["12-30-2018", "01-06-2019", "01-13-2019", "01-20-2019"]) self.assertEqual(result.get("datasets")[0].get("values"), [50.0, 150.0, 266.6666666666667, 0.0]) def test_user_date_label_dashboard_chart(self): @@ -255,13 +255,13 @@ class TestDashboardChart(FrappeTestCase): with patch.object(frappe.utils.data, "get_user_date_format", return_value="dd.mm.yyyy"): result = get(chart_name="Test Dashboard Chart Date Label") self.assertEqual( - sorted(result.get("labels")), sorted(["01.05.2019", "01.12.2019", "19.01.2019"]) + sorted(result.get("labels")), sorted(["05.01.2019", "12.01.2019", "19.01.2019"]) ) with patch.object(frappe.utils.data, "get_user_date_format", return_value="mm-dd-yyyy"): result = get(chart_name="Test Dashboard Chart Date Label") self.assertEqual( - sorted(result.get("labels")), sorted(["01-19-2019", "05-01-2019", "12-01-2019"]) + sorted(result.get("labels")), sorted(["01-19-2019", "01-05-2019", "01-12-2019"]) ) diff --git a/frappe/desk/doctype/dashboard_chart_source/dashboard_chart_source.js b/frappe/desk/doctype/dashboard_chart_source/dashboard_chart_source.js index 96dd40d3c1..6f1fa36ffd 100644 --- a/frappe/desk/doctype/dashboard_chart_source/dashboard_chart_source.js +++ b/frappe/desk/doctype/dashboard_chart_source/dashboard_chart_source.js @@ -1,5 +1,4 @@ // Copyright (c) 2019, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Dashboard Chart Source', { -}); +frappe.ui.form.on("Dashboard Chart Source", {}); diff --git a/frappe/desk/doctype/dashboard_chart_source/test_dashboard_chart_source.py b/frappe/desk/doctype/dashboard_chart_source/test_dashboard_chart_source.py index 457487bb6d..5b4c114287 100644 --- a/frappe/desk/doctype/dashboard_chart_source/test_dashboard_chart_source.py +++ b/frappe/desk/doctype/dashboard_chart_source/test_dashboard_chart_source.py @@ -1,7 +1,7 @@ # Copyright (c) 2019, Frappe Technologies and Contributors # License: MIT. See LICENSE -import unittest +from frappe.tests.utils import FrappeTestCase -class TestDashboardChartSource(unittest.TestCase): +class TestDashboardChartSource(FrappeTestCase): pass diff --git a/frappe/desk/doctype/dashboard_settings/dashboard_settings.js b/frappe/desk/doctype/dashboard_settings/dashboard_settings.js index 8e7966366d..aa5be2b1a5 100644 --- a/frappe/desk/doctype/dashboard_settings/dashboard_settings.js +++ b/frappe/desk/doctype/dashboard_settings/dashboard_settings.js @@ -1,8 +1,7 @@ // Copyright (c) 2020, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Dashboard Settings', { +frappe.ui.form.on("Dashboard Settings", { // refresh: function(frm) { - // } }); diff --git a/frappe/desk/doctype/desktop_icon/desktop_icon.js b/frappe/desk/doctype/desktop_icon/desktop_icon.js index 58ea09e732..72ef1f7a12 100644 --- a/frappe/desk/doctype/desktop_icon/desktop_icon.js +++ b/frappe/desk/doctype/desktop_icon/desktop_icon.js @@ -1,8 +1,6 @@ // Copyright (c) 2016, Frappe Technologies Pvt. Ltd. and contributors // For license information, please see license.txt -frappe.ui.form.on('Desktop Icon', { - refresh: function(frm) { - - } +frappe.ui.form.on("Desktop Icon", { + refresh: function (frm) {}, }); diff --git a/frappe/desk/doctype/desktop_icon/desktop_icon.json b/frappe/desk/doctype/desktop_icon/desktop_icon.json index 59c95953ad..ef88346f53 100644 --- a/frappe/desk/doctype/desktop_icon/desktop_icon.json +++ b/frappe/desk/doctype/desktop_icon/desktop_icon.json @@ -1,736 +1,175 @@ { - "allow_copy": 0, - "allow_events_in_timeline": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, - "creation": "2016-02-22 03:47:45.387068", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 0, - "engine": "InnoDB", + "actions": [], + "creation": "2016-02-22 03:47:45.387068", + "doctype": "DocType", + "engine": "InnoDB", + "field_order": [ + "module_name", + "label", + "standard", + "custom", + "column_break_3", + "app", + "description", + "category", + "hidden", + "blocked", + "force_show", + "section_break_7", + "type", + "_doctype", + "_report", + "link", + "column_break_10", + "color", + "icon", + "reverse", + "idx" + ], "fields": [ { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "module_name", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Module Name", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "module_name", + "fieldtype": "Data", + "label": "Module Name" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "label", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Label", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "label", + "fieldtype": "Data", + "label": "Label" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "standard", - "fieldtype": "Check", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Standard", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "default": "0", + "fieldname": "standard", + "fieldtype": "Check", + "in_list_view": 1, + "label": "Standard" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "custom", - "fieldtype": "Check", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Custom", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "default": "0", + "fieldname": "custom", + "fieldtype": "Check", + "label": "Custom", + "read_only": 1 + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "column_break_3", - "fieldtype": "Column Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "column_break_3", + "fieldtype": "Column Break" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "app", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "App", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "app", + "fieldtype": "Data", + "label": "App", + "read_only": 1 + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "description", - "fieldtype": "Small Text", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Description", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "description", + "fieldtype": "Small Text", + "label": "Description" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "category", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Category", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "category", + "fieldtype": "Data", + "label": "Category" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "hidden", - "fieldtype": "Check", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Hidden", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "default": "0", + "fieldname": "hidden", + "fieldtype": "Check", + "label": "Hidden" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "blocked", - "fieldtype": "Check", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Blocked", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "default": "0", + "fieldname": "blocked", + "fieldtype": "Check", + "label": "Blocked" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "force_show", - "fieldtype": "Check", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Force Show", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "default": "0", + "fieldname": "force_show", + "fieldtype": "Check", + "label": "Force Show", + "read_only": 1 + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "section_break_7", - "fieldtype": "Section Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "section_break_7", + "fieldtype": "Section Break" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "type", - "fieldtype": "Select", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 1, - "label": "Type", - "length": 0, - "no_copy": 0, - "options": "module\nlist\nlink\npage\nquery-report", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "type", + "fieldtype": "Select", + "in_list_view": 1, + "in_standard_filter": 1, + "label": "Type", + "options": "module\nlist\nlink\npage\nquery-report" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "_doctype", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "_doctype", - "length": 0, - "no_copy": 0, - "options": "DocType", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "_doctype", + "fieldtype": "Link", + "label": "_doctype", + "options": "DocType" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "_report", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "_report", - "length": 0, - "no_copy": 0, - "options": "Report", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "_report", + "fieldtype": "Link", + "label": "_report", + "options": "Report" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "link", - "fieldtype": "Small Text", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Link", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "link", + "fieldtype": "Small Text", + "label": "Link" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "column_break_10", - "fieldtype": "Column Break", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "column_break_10", + "fieldtype": "Column Break" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "color", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Color", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "color", + "fieldtype": "Data", + "label": "Color" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "icon", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Icon", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "fieldname": "icon", + "fieldtype": "Data", + "label": "Icon" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "reverse", - "fieldtype": "Check", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Reverse Icon Color", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, + "default": "0", + "fieldname": "reverse", + "fieldtype": "Check", + "label": "Reverse Icon Color" + }, { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "idx", - "fieldtype": "Int", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Idx", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "fieldname": "idx", + "fieldtype": "Int", + "label": "Idx" } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 1, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2019-01-24 04:58:58.720618", - "modified_by": "Administrator", - "module": "Desk", - "name": "Desktop Icon", - "name_case": "", - "owner": "Administrator", + ], + "in_create": 1, + "links": [], + "modified": "2022-08-03 12:20:50.577580", + "modified_by": "Administrator", + "module": "Desk", + "name": "Desktop Icon", + "owner": "Administrator", "permissions": [ { - "amend": 0, - "cancel": 0, - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "set_user_permissions": 0, - "share": 1, - "submit": 0, + "create": 1, + "delete": 1, + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1, "write": 1 } - ], - "quick_entry": 0, - "read_only": 1, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "title_field": "module_name", - "track_changes": 1, - "track_seen": 0, - "track_views": 0 + ], + "read_only": 1, + "sort_field": "modified", + "sort_order": "DESC", + "states": [], + "title_field": "module_name", + "track_changes": 1 } \ No newline at end of file diff --git a/frappe/desk/doctype/desktop_icon/desktop_icon.py b/frappe/desk/doctype/desktop_icon/desktop_icon.py index 5602f4da24..63fa12b8fb 100644 --- a/frappe/desk/doctype/desktop_icon/desktop_icon.py +++ b/frappe/desk/doctype/desktop_icon/desktop_icon.py @@ -61,7 +61,7 @@ def get_desktop_icons(user=None): blocked_doctypes = [d.get("name") for d in blocked_doctypes] - standard_icons = frappe.db.get_all("Desktop Icon", fields=fields, filters={"standard": 1}) + standard_icons = frappe.get_all("Desktop Icon", fields=fields, filters={"standard": 1}) standard_map = {} for icon in standard_icons: @@ -69,7 +69,7 @@ def get_desktop_icons(user=None): icon.blocked = 1 standard_map[icon.module_name] = icon - user_icons = frappe.db.get_all( + user_icons = frappe.get_all( "Desktop Icon", fields=fields, filters={"standard": 0, "owner": user} ) @@ -430,7 +430,7 @@ def get_context(context): context.user = frappe.session.user if "System Manager" in frappe.get_roles(): - context.users = frappe.db.get_all( + context.users = frappe.get_all( "User", filters={"user_type": "System User", "enabled": 1}, fields=["name", "first_name", "last_name"], @@ -443,7 +443,7 @@ def get_module_icons(user=None): frappe.only_for("System Manager") if not user: - icons = frappe.db.get_all("Desktop Icon", fields="*", filters={"standard": 1}, order_by="idx") + icons = frappe.get_all("Desktop Icon", fields="*", filters={"standard": 1}, order_by="idx") else: frappe.cache().hdel("desktop_icons", user) icons = get_user_icons(user) diff --git a/frappe/desk/doctype/event/event.js b/frappe/desk/doctype/event/event.js index 87d78bae94..299cbe5cc3 100644 --- a/frappe/desk/doctype/event/event.js +++ b/frappe/desk/doctype/event/event.js @@ -3,70 +3,91 @@ frappe.provide("frappe.desk"); frappe.ui.form.on("Event", { - onload: function(frm) { - frm.set_query('reference_doctype', "event_participants", function() { - return { - "filters": { - "issingle": 0, - } - }; - }); - frm.set_query('google_calendar', function() { + onload: function (frm) { + frm.set_query("reference_doctype", "event_participants", function () { return { filters: { - "owner": frappe.session.user - } + issingle: 0, + }, + }; + }); + frm.set_query("google_calendar", function () { + return { + filters: { + owner: frappe.session.user, + }, }; }); }, - refresh: function(frm) { - if(frm.doc.event_participants) { - frm.doc.event_participants.forEach(value => { - frm.add_custom_button(__(value.reference_docname), function() { - frappe.set_route("Form", value.reference_doctype, value.reference_docname); - }, __("Participants")); - }) + refresh: function (frm) { + if (frm.doc.event_participants) { + frm.doc.event_participants.forEach((value) => { + frm.add_custom_button( + __(value.reference_docname), + function () { + frappe.set_route("Form", value.reference_doctype, value.reference_docname); + }, + __("Participants") + ); + }); } frm.page.set_inner_btn_group_as_primary(__("Add Participants")); - frm.add_custom_button(__('Add Contacts'), function() { - new frappe.desk.eventParticipants(frm, "Contact"); - }, __("Add Participants")); - }, - repeat_on: function(frm) { - if(frm.doc.repeat_on==="Every Day") { - ["monday", "tuesday", "wednesday", "thursday", - "friday", "saturday", "sunday"].map(function(v) { - frm.set_value(v, 1); - }); + frm.add_custom_button( + __("Add Contacts"), + function () { + new frappe.desk.eventParticipants(frm, "Contact"); + }, + __("Add Participants") + ); + + const [ends_on_date] = frm.doc.ends_on + ? frm.doc.ends_on.split(" ") + : frm.doc.starts_on.split(" "); + + if (frm.doc.google_meet_link && frappe.datetime.now_date() <= ends_on_date) { + frm.dashboard.set_headline( + __("Join video conference with {0}", [ + `Google Meet`, + ]) + ); } - } + }, + repeat_on: function (frm) { + if (frm.doc.repeat_on === "Every Day") { + ["monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday"].map( + function (v) { + frm.set_value(v, 1); + } + ); + } + }, }); frappe.ui.form.on("Event Participants", { - event_participants_remove: function(frm, cdt, cdn) { - if (cdt&&!cdn.includes("New Event Participants")){ + event_participants_remove: function (frm, cdt, cdn) { + if (cdt && !cdn.includes("New Event Participants")) { frappe.call({ type: "POST", method: "frappe.desk.doctype.event.event.delete_communication", args: { - "event": frm.doc, - "reference_doctype": cdt, - "reference_docname": cdn + event: frm.doc, + reference_doctype: cdt, + reference_docname: cdn, }, freeze: true, - callback: function(r) { - if(r.exc) { + callback: function (r) { + if (r.exc) { frappe.show_alert({ message: __("{0}", [r.exc]), - indicator: 'orange' + indicator: "orange", }); } - } + }, }); } - } + }, }); frappe.desk.eventParticipants = class eventParticipants { @@ -86,7 +107,7 @@ frappe.desk.eventParticipants = class eventParticipants { dynamic_link_reference: me.doctype, fieldname: "reference_docname", target: table, - txt: "" + txt: "", }); } }; diff --git a/frappe/desk/doctype/event/event.json b/frappe/desk/doctype/event/event.json index bce3b1e65a..5ca49f3831 100644 --- a/frappe/desk/doctype/event/event.json +++ b/frappe/desk/doctype/event/event.json @@ -22,12 +22,14 @@ "sender", "all_day", "sync_with_google_calendar", + "add_video_conferencing", "sb_00", "google_calendar", - "pulled_from_google_calendar", - "cb_00", "google_calendar_id", + "cb_00", "google_calendar_event_id", + "google_meet_link", + "pulled_from_google_calendar", "section_break_13", "repeat_on", "repeat_till", @@ -221,11 +223,11 @@ "in_list_view": 1, "in_standard_filter": 1, "label": "Status", - "options": "Open\nClosed" + "options": "Open\nCompleted\nClosed" }, { "collapsible": 1, - "depends_on": "eval:doc.sync_with_google_calendar", + "depends_on": "eval:doc.sync_with_google_calendar || doc.pulled_from_google_calendar", "fieldname": "sb_00", "fieldtype": "Section Break", "label": "Google Calendar" @@ -245,6 +247,7 @@ "fieldname": "google_calendar_event_id", "fieldtype": "Data", "label": "Google Calendar Event ID", + "no_copy": 1, "read_only": 1 }, { @@ -272,12 +275,27 @@ "label": "Sender", "options": "Email", "read_only": 1 + }, + { + "default": "0", + "depends_on": "eval:doc.sync_with_google_calendar", + "description": "via Google Meet", + "fieldname": "add_video_conferencing", + "fieldtype": "Check", + "label": "Add Video Conferencing" + }, + { + "fieldname": "google_meet_link", + "fieldtype": "Data", + "label": "Google Meet Link", + "no_copy": 1, + "read_only": 1 } ], "icon": "fa fa-calendar", "idx": 1, "links": [], - "modified": "2022-05-12 05:43:27.935510", + "modified": "2022-08-12 19:24:34.794098", "modified_by": "Administrator", "module": "Desk", "name": "Event", @@ -318,4 +336,4 @@ "track_changes": 1, "track_seen": 1, "track_views": 1 -} \ No newline at end of file +} diff --git a/frappe/desk/doctype/event/event.py b/frappe/desk/doctype/event/event.py index e9104ef897..5013c4fb3d 100644 --- a/frappe/desk/doctype/event/event.py +++ b/frappe/desk/doctype/event/event.py @@ -6,6 +6,7 @@ import json import frappe from frappe import _ +from frappe.contacts.doctype.contact.contact import get_default_contact from frappe.desk.doctype.notification_settings.notification_settings import ( is_email_notifications_enabled_for_type, ) @@ -55,6 +56,12 @@ class Event(Document): if self.sync_with_google_calendar and not self.google_calendar: frappe.throw(_("Select Google Calendar to which event should be synced.")) + if not self.sync_with_google_calendar: + self.add_video_conferencing = 0 + + def before_save(self): + self.set_participants_email() + def on_update(self): self.sync_communication() @@ -131,6 +138,22 @@ class Event(Document): for participant in participants: self.add_participant(participant["doctype"], participant["docname"]) + def set_participants_email(self): + for participant in self.event_participants: + if participant.email: + continue + + if participant.reference_doctype != "Contact": + participant_contact = get_default_contact( + participant.reference_doctype, participant.reference_docname + ) + else: + participant_contact = participant.reference_docname + + participant.email = ( + frappe.get_value("Contact", participant_contact, "email_id") if participant_contact else None + ) + @frappe.whitelist() def delete_communication(event, reference_doctype, reference_docname): @@ -205,7 +228,7 @@ def send_event_digest(): @frappe.whitelist() -def get_events(start, end, user=None, for_reminder=False, filters=None): +def get_events(start, end, user=None, for_reminder=False, filters=None) -> list[frappe._dict]: if not user: user = frappe.session.user @@ -283,8 +306,8 @@ def get_events(start, end, user=None, for_reminder=False, filters=None): ) # process recurring events - start = start.split(" ")[0] - end = end.split(" ")[0] + start = start.split(" ", 1)[0] + end = end.split(" ", 1)[0] add_events = [] remove_events = [] @@ -292,7 +315,7 @@ def get_events(start, end, user=None, for_reminder=False, filters=None): new_event = e.copy() enddate = ( - add_days(date, int(date_diff(e.ends_on.split(" ")[0], e.starts_on.split(" ")[0]))) + add_days(date, int(date_diff(e.ends_on.split(" ", 1)[0], e.starts_on.split(" ", 1)[0]))) if (e.starts_on and e.ends_on) else date ) @@ -314,8 +337,8 @@ def get_events(start, end, user=None, for_reminder=False, filters=None): repeat = "3000-01-01" if cstr(e.repeat_till) == "" else e.repeat_till if e.repeat_on == "Yearly": - start_year = cint(start.split("-")[0]) - end_year = cint(end.split("-")[0]) + start_year = cint(start.split("-", 1)[0]) + end_year = cint(end.split("-", 1)[0]) # creates a string with date (27) and month (07) eg: 07-27 event_start = "-".join(event_start.split("-")[1:]) @@ -334,12 +357,13 @@ def get_events(start, end, user=None, for_reminder=False, filters=None): if e.repeat_on == "Monthly": # creates a string with date (27) and month (07) and year (2019) eg: 2019-07-27 - date = start.split("-")[0] + "-" + start.split("-")[1] + "-" + event_start.split("-")[2] + year, month = start.split("-", maxsplit=2)[:2] + date = f"{year}-{month}-" + event_start.split("-", maxsplit=3)[2] # last day of month issue, start from prev month! try: getdate(date) - except ValueError: + except Exception: date = date.split("-") date = date[0] + "-" + str(cint(date[1]) - 1) + "-" + date[2] diff --git a/frappe/desk/doctype/event/event_calendar.js b/frappe/desk/doctype/event/event_calendar.js index df474a0258..bfdd09b864 100644 --- a/frappe/desk/doctype/event/event_calendar.js +++ b/frappe/desk/doctype/event/event_calendar.js @@ -1,16 +1,16 @@ frappe.views.calendar["Event"] = { field_map: { - "start": "starts_on", - "end": "ends_on", - "id": "name", - "allDay": "all_day", - "title": "subject", - "status": "event_type", - "color": "color" + start: "starts_on", + end: "ends_on", + id: "name", + allDay: "all_day", + title: "subject", + status: "event_type", + color: "color", }, style_map: { - "Public": "success", - "Private": "info" + Public: "success", + Private: "info", }, - get_events_method: "frappe.desk.doctype.event.event.get_events" -} \ No newline at end of file + get_events_method: "frappe.desk.doctype.event.event.get_events", +}; diff --git a/frappe/desk/doctype/event/event_list.js b/frappe/desk/doctype/event/event_list.js index 5d73d9dd1a..f6460288d8 100644 --- a/frappe/desk/doctype/event/event_list.js +++ b/frappe/desk/doctype/event/event_list.js @@ -1,8 +1,8 @@ -frappe.listview_settings['Event'] = { +frappe.listview_settings["Event"] = { add_fields: ["starts_on", "ends_on"], - onload: function() { + onload: function () { frappe.route_options = { - "status": "Open" + status: "Open", }; - } -} \ No newline at end of file + }, +}; diff --git a/frappe/desk/doctype/event/test_event.py b/frappe/desk/doctype/event/test_event.py index efbd54fb09..72eab8f416 100644 --- a/frappe/desk/doctype/event/test_event.py +++ b/frappe/desk/doctype/event/test_event.py @@ -3,17 +3,17 @@ """Use blog post test to test user permissions logic""" import json -import unittest import frappe import frappe.defaults from frappe.desk.doctype.event.event import get_events from frappe.test_runner import make_test_objects +from frappe.tests.utils import FrappeTestCase test_records = frappe.get_test_records("Event") -class TestEvent(unittest.TestCase): +class TestEvent(FrappeTestCase): def setUp(self): frappe.db.delete("Event") make_test_objects("Event", reset=True) diff --git a/frappe/desk/doctype/event_participants/event_participants.json b/frappe/desk/doctype/event_participants/event_participants.json index 86cf2670c9..bbb0a24f3e 100644 --- a/frappe/desk/doctype/event_participants/event_participants.json +++ b/frappe/desk/doctype/event_participants/event_participants.json @@ -1,108 +1,49 @@ { - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, - "creation": "2018-09-21 15:44:58.836156", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", - "fields": [ - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "reference_doctype", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Reference Document Type", - "length": 0, - "no_copy": 0, - "options": "DocType", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - }, - { - "allow_bulk_edit": 0, - "allow_in_quick_entry": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "reference_docname", - "fieldtype": "Dynamic Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Reference Name", - "length": 0, - "no_copy": 0, - "options": "reference_doctype", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 - } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 1, - "max_attachments": 0, - "modified": "2019-09-05 14:22:27.664645", - "modified_by": "Administrator", - "module": "Desk", - "name": "Event Participants", - "name_case": "", - "owner": "Administrator", - "permissions": [], - "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1, - "track_seen": 0, - "track_views": 0 - } \ No newline at end of file + "actions": [], + "creation": "2018-09-21 15:44:58.836156", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "reference_doctype", + "reference_docname", + "email" + ], + "fields": [ + { + "fieldname": "reference_doctype", + "fieldtype": "Link", + "in_list_view": 1, + "label": "Reference Document Type", + "options": "DocType", + "reqd": 1 + }, + { + "fieldname": "reference_docname", + "fieldtype": "Dynamic Link", + "in_list_view": 1, + "label": "Reference Name", + "options": "reference_doctype", + "reqd": 1 + }, + { + "fieldname": "email", + "fieldtype": "Data", + "label": "Email", + "options": "Email" + } + ], + "istable": 1, + "links": [], + "modified": "2022-10-18 17:49:33.549459", + "modified_by": "Administrator", + "module": "Desk", + "name": "Event Participants", + "owner": "Administrator", + "permissions": [], + "quick_entry": 1, + "sort_field": "modified", + "sort_order": "DESC", + "states": [], + "track_changes": 1 +} \ No newline at end of file diff --git a/frappe/desk/doctype/form_tour/form_tour.js b/frappe/desk/doctype/form_tour/form_tour.js index 3f3fc0ff8a..1e67e10779 100644 --- a/frappe/desk/doctype/form_tour/form_tour.js +++ b/frappe/desk/doctype/form_tour/form_tour.js @@ -1,10 +1,10 @@ // Copyright (c) 2021, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Form Tour', { - setup: function(frm) { +frappe.ui.form.on("Form Tour", { + setup: function (frm) { if (!frm.doc.is_standard || frappe.boot.developer_mode) { - frm.trigger('setup_queries'); + frm.trigger("setup_queries"); } }, @@ -13,28 +13,26 @@ frappe.ui.form.on('Form Tour', { frm.trigger("disable_form"); } - frm.add_custom_button(__('Show Tour'), async () => { + frm.add_custom_button(__("Show Tour"), async () => { const issingle = await check_if_single(frm.doc.reference_doctype); let route_changed = null; if (issingle) { - route_changed = frappe.set_route('Form', frm.doc.reference_doctype); + route_changed = frappe.set_route("Form", frm.doc.reference_doctype); } else if (frm.doc.first_document) { const name = await get_first_document(frm.doc.reference_doctype); - route_changed = frappe.set_route('Form', frm.doc.reference_doctype, name); + route_changed = frappe.set_route("Form", frm.doc.reference_doctype, name); } else { - route_changed = frappe.set_route('Form', frm.doc.reference_doctype, 'new'); + route_changed = frappe.set_route("Form", frm.doc.reference_doctype, "new"); } route_changed.then(() => { const tour_name = frm.doc.name; - cur_frm.tour - .init({ tour_name }) - .then(() => cur_frm.tour.start()); + cur_frm.tour.init({ tour_name }).then(() => cur_frm.tour.start()); }); }); }, - disable_form: function(frm) { + disable_form: function (frm) { frm.set_read_only(); frm.fields .filter((field) => field.has_input) @@ -45,51 +43,48 @@ frappe.ui.form.on('Form Tour', { }, setup_queries(frm) { - frm.set_query("reference_doctype", function() { + frm.set_query("reference_doctype", function () { return { filters: { - istable: 0 - } + istable: 0, + }, }; }); - frm.trigger('reference_doctype'); + frm.trigger("reference_doctype"); }, reference_doctype(frm) { if (!frm.doc.reference_doctype) return; - frm.set_fields_as_options( - "fieldname", - frm.doc.reference_doctype, - df => !df.hidden - ).then(options => { - frm.fields_dict.steps.grid.update_docfield_property( - "fieldname", - "options", - [""].concat(options) - ); - }); + frm.set_fields_as_options("fieldname", frm.doc.reference_doctype, (df) => !df.hidden).then( + (options) => { + frm.fields_dict.steps.grid.update_docfield_property( + "fieldname", + "options", + [""].concat(options) + ); + } + ); frm.set_fields_as_options( - 'parent_fieldname', + "parent_fieldname", frm.doc.reference_doctype, - (df) => df.fieldtype == "Table" && !df.hidden, - ).then(options => { + (df) => df.fieldtype == "Table" && !df.hidden + ).then((options) => { frm.fields_dict.steps.grid.update_docfield_property( "parent_fieldname", "options", [""].concat(options) ); }); - - } + }, }); -frappe.ui.form.on('Form Tour Step', { +frappe.ui.form.on("Form Tour Step", { form_render(frm, cdt, cdn) { if (locals[cdt][cdn].is_table_field) { - frm.trigger('parent_fieldname', cdt, cdn); + frm.trigger("parent_fieldname", cdt, cdn); } }, parent_fieldname(frm, cdt, cdn) { @@ -97,37 +92,36 @@ frappe.ui.form.on('Form Tour Step', { const parent_fieldname_df = frappe .get_meta(frm.doc.reference_doctype) - .fields.find(df => df.fieldname == child_row.parent_fieldname); + .fields.find((df) => df.fieldname == child_row.parent_fieldname); frm.set_fields_as_options( - 'fieldname', + "fieldname", parent_fieldname_df.options, - (df) => !df.hidden, - ).then(options => { + (df) => !df.hidden + ).then((options) => { frm.fields_dict.steps.grid.update_docfield_property( "fieldname", "options", [""].concat(options) ); if (child_row.fieldname) { - frappe.model.set_value(cdt, cdn, 'fieldname', child_row.fieldname); + frappe.model.set_value(cdt, cdn, "fieldname", child_row.fieldname); } }); - } + }, }); async function check_if_single(doctype) { - const { message } = await frappe.db.get_value('DocType', doctype, 'issingle'); + const { message } = await frappe.db.get_value("DocType", doctype, "issingle"); return message.issingle || 0; } async function get_first_document(doctype) { let docname; - await frappe.db.get_list(doctype, { order_by: "creation" }).then(res => { - if (Array.isArray(res) && res.length) - docname = res[0].name; + await frappe.db.get_list(doctype, { order_by: "creation" }).then((res) => { + if (Array.isArray(res) && res.length) docname = res[0].name; }); - return docname || 'new'; + return docname || "new"; } diff --git a/frappe/desk/doctype/form_tour/test_form_tour.py b/frappe/desk/doctype/form_tour/test_form_tour.py index cb0c4ef33a..3fdcdf95a6 100644 --- a/frappe/desk/doctype/form_tour/test_form_tour.py +++ b/frappe/desk/doctype/form_tour/test_form_tour.py @@ -2,8 +2,8 @@ # License: MIT. See LICENSE # import frappe -import unittest +from frappe.tests.utils import FrappeTestCase -class TestFormTour(unittest.TestCase): +class TestFormTour(FrappeTestCase): pass diff --git a/frappe/desk/doctype/global_search_settings/global_search_settings.js b/frappe/desk/doctype/global_search_settings/global_search_settings.js index c333f83585..147a72eef1 100644 --- a/frappe/desk/doctype/global_search_settings/global_search_settings.js +++ b/frappe/desk/doctype/global_search_settings/global_search_settings.js @@ -1,14 +1,17 @@ // Copyright (c) 2019, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Global Search Settings', { - refresh: function(frm) { - - frappe.realtime.on('global_search_settings', (data) => { +frappe.ui.form.on("Global Search Settings", { + refresh: function (frm) { + frappe.realtime.on("global_search_settings", (data) => { if (data.progress) { - frm.dashboard.show_progress('Setting up Global Search', data.progress / data.total * 100, data.msg); + frm.dashboard.show_progress( + "Setting up Global Search", + (data.progress / data.total) * 100, + data.msg + ); if (data.progress === data.total) { - frm.dashboard.hide_progress('Setting up Global Search'); + frm.dashboard.hide_progress("Setting up Global Search"); } } }); @@ -16,14 +19,14 @@ frappe.ui.form.on('Global Search Settings', { frm.add_custom_button(__("Reset"), function () { frappe.call({ method: "frappe.desk.doctype.global_search_settings.global_search_settings.reset_global_search_settings_doctypes", - callback: function() { + callback: function () { frappe.show_alert({ message: __("Global Search Document Types Reset."), - indicator: "green" + indicator: "green", }); frm.refresh(); - } + }, }); }); - } + }, }); diff --git a/frappe/desk/doctype/kanban_board/kanban_board.js b/frappe/desk/doctype/kanban_board/kanban_board.js index ff80a58fa0..3b815fef0e 100644 --- a/frappe/desk/doctype/kanban_board/kanban_board.js +++ b/frappe/desk/doctype/kanban_board/kanban_board.js @@ -1,43 +1,45 @@ // Copyright (c) 2016, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Kanban Board', { - onload: function(frm) { - frm.trigger('reference_doctype'); +frappe.ui.form.on("Kanban Board", { + onload: function (frm) { + frm.trigger("reference_doctype"); }, - refresh: function(frm) { - if(frm.is_new()) return; - frm.add_custom_button("Show Board", function() { + refresh: function (frm) { + if (frm.is_new()) return; + frm.add_custom_button("Show Board", function () { frappe.set_route("List", frm.doc.reference_doctype, "Kanban", frm.doc.name); }); }, - reference_doctype: function(frm) { - + reference_doctype: function (frm) { // set field options - if(!frm.doc.reference_doctype) return; + if (!frm.doc.reference_doctype) return; - frappe.model.with_doctype(frm.doc.reference_doctype, function() { - var options = $.map(frappe.get_meta(frm.doc.reference_doctype).fields, - function(d) { - if(d.fieldname && d.fieldtype === 'Select' && - frappe.model.no_value_type.indexOf(d.fieldtype)===-1) { - return d.fieldname; - } - return null; - }); - frm.set_df_property('field_name', 'options', options); - frm.get_field('field_name').refresh(); + frappe.model.with_doctype(frm.doc.reference_doctype, function () { + var options = $.map(frappe.get_meta(frm.doc.reference_doctype).fields, function (d) { + if ( + d.fieldname && + d.fieldtype === "Select" && + frappe.model.no_value_type.indexOf(d.fieldtype) === -1 + ) { + return d.fieldname; + } + return null; + }); + frm.set_df_property("field_name", "options", options); + frm.get_field("field_name").refresh(); }); }, - field_name: function(frm) { + field_name: function (frm) { var field = frappe.meta.get_field(frm.doc.reference_doctype, frm.doc.field_name); frm.doc.columns = []; - field.options && field.options.split('\n').forEach(function(o) { - o = o.trim(); - if(!o) return; - var d = frm.add_child('columns'); - d.column_name = o; - }); + field.options && + field.options.split("\n").forEach(function (o) { + o = o.trim(); + if (!o) return; + var d = frm.add_child("columns"); + d.column_name = o; + }); frm.refresh(); - } + }, }); diff --git a/frappe/desk/doctype/kanban_board/kanban_board.py b/frappe/desk/doctype/kanban_board/kanban_board.py index 83f0f46df0..e3257e25be 100644 --- a/frappe/desk/doctype/kanban_board/kanban_board.py +++ b/frappe/desk/doctype/kanban_board/kanban_board.py @@ -88,10 +88,15 @@ def update_order(board_name, order): """Save the order of cards in columns""" board = frappe.get_doc("Kanban Board", board_name) doctype = board.reference_doctype + updated_cards = [] + + if not frappe.has_permission(doctype, "write"): + # Return board data from db + return board, updated_cards + fieldname = board.field_name order_dict = json.loads(order) - updated_cards = [] for col_name, cards in order_dict.items(): for card in cards: column = frappe.get_value(doctype, {"name": card}, fieldname) @@ -103,8 +108,7 @@ def update_order(board_name, order): if column.column_name == col_name: column.order = json.dumps(cards) - board.save() - return board, updated_cards + return board.save(ignore_permissions=True), updated_cards @frappe.whitelist() @@ -114,6 +118,9 @@ def update_order_for_single_card( """Save the order of cards in columns""" board = frappe.get_doc("Kanban Board", board_name) doctype = board.reference_doctype + + frappe.has_permission(doctype, "write", throw=True) + fieldname = board.field_name old_index = frappe.parse_json(old_index) new_index = frappe.parse_json(new_index) @@ -130,7 +137,7 @@ def update_order_for_single_card( # save updated order board.columns[from_col_idx].order = frappe.as_json(from_col_order) board.columns[to_col_idx].order = frappe.as_json(to_col_order) - board.save() + board.save(ignore_permissions=True) # update changed value in doc frappe.set_value(doctype, docname, fieldname, to_colname) @@ -151,13 +158,14 @@ def get_kanban_column_order_and_index(board, colname): def add_card(board_name, docname, colname): board = frappe.get_doc("Kanban Board", board_name) + frappe.has_permission(board.reference_doctype, "write", throw=True) + col_order, col_idx = get_kanban_column_order_and_index(board, colname) col_order.insert(0, docname) board.columns[col_idx].order = frappe.as_json(col_order) - board.save() - return board + return board.save(ignore_permissions=True) @frappe.whitelist() diff --git a/frappe/desk/doctype/kanban_board/test_kanban_board.py b/frappe/desk/doctype/kanban_board/test_kanban_board.py index 73f566b906..05cd5723c4 100644 --- a/frappe/desk/doctype/kanban_board/test_kanban_board.py +++ b/frappe/desk/doctype/kanban_board/test_kanban_board.py @@ -1,9 +1,9 @@ # Copyright (c) 2015, Frappe Technologies and Contributors # License: MIT. See LICENSE -import unittest +from frappe.tests.utils import FrappeTestCase # test_records = frappe.get_test_records('Kanban Board') -class TestKanbanBoard(unittest.TestCase): +class TestKanbanBoard(FrappeTestCase): pass diff --git a/frappe/desk/doctype/list_filter/list_filter.json b/frappe/desk/doctype/list_filter/list_filter.json index dad62bf8d6..257bbc6d45 100644 --- a/frappe/desk/doctype/list_filter/list_filter.json +++ b/frappe/desk/doctype/list_filter/list_filter.json @@ -1,188 +1,62 @@ { - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, + "actions": [], "creation": "2018-02-22 15:10:24.401801", - "custom": 0, - "docstatus": 0, "doctype": "DocType", - "document_type": "", "editable_grid": 1, "engine": "InnoDB", + "field_order": [ + "filter_name", + "reference_doctype", + "for_user", + "filters" + ], "fields": [ { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "filter_name", "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Filter Name", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "label": "Filter Name" }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "reference_doctype", "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, "label": "Reference Document Type", - "length": 0, - "no_copy": 0, - "options": "DocType", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "options": "DocType" }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "for_user", "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, "label": "For User", - "length": 0, - "no_copy": 0, - "options": "User", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "options": "User" }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, "fieldname": "filters", "fieldtype": "Long Text", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Filters", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "translatable": 0, - "unique": 0 + "label": "Filters" } ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, "in_create": 1, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2019-09-05 14:22:27.664645", + "links": [], + "modified": "2022-08-03 12:20:50.889979", "modified_by": "Administrator", "module": "Desk", "name": "List Filter", - "name_case": "", "owner": "Administrator", "permissions": [ { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, "create": 1, "delete": 1, "email": 1, "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, "print": 1, "read": 1, "report": 1, "role": "All", - "set_user_permissions": 0, "share": 1, - "submit": 0, "write": 1 } ], - "quick_entry": 0, "read_only": 1, - "read_only_onload": 0, - "show_name_in_global_search": 0, "sort_field": "modified", "sort_order": "DESC", - "track_changes": 0, - "track_seen": 0 + "states": [] } \ No newline at end of file diff --git a/frappe/desk/doctype/list_view_settings/list_view_settings.js b/frappe/desk/doctype/list_view_settings/list_view_settings.js index db33f71675..007a242dd3 100644 --- a/frappe/desk/doctype/list_view_settings/list_view_settings.js +++ b/frappe/desk/doctype/list_view_settings/list_view_settings.js @@ -1,8 +1,7 @@ // Copyright (c) 2020, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('List View Settings', { +frappe.ui.form.on("List View Settings", { // refresh: function(frm) { - // } -}); \ No newline at end of file +}); diff --git a/frappe/desk/doctype/list_view_settings/list_view_settings.json b/frappe/desk/doctype/list_view_settings/list_view_settings.json index 44761992f1..69ea379e61 100644 --- a/frappe/desk/doctype/list_view_settings/list_view_settings.json +++ b/frappe/desk/doctype/list_view_settings/list_view_settings.json @@ -7,6 +7,7 @@ "engine": "InnoDB", "field_order": [ "disable_count", + "disable_comment_count", "disable_sidebar_stats", "disable_auto_refresh", "total_fields", @@ -49,13 +50,20 @@ "hidden": 1, "label": "Fields", "read_only": 1 + }, + { + "default": "0", + "fieldname": "disable_comment_count", + "fieldtype": "Check", + "label": "Disable Comment Count" } ], "links": [], - "modified": "2020-05-12 18:27:15.568199", + "modified": "2023-02-14 14:46:43.764229", "modified_by": "Administrator", "module": "Desk", "name": "List View Settings", + "naming_rule": "Set by user", "owner": "Administrator", "permissions": [ { @@ -72,5 +80,6 @@ "read_only": 1, "sort_field": "modified", "sort_order": "DESC", + "states": [], "track_changes": 1 } \ No newline at end of file diff --git a/frappe/desk/doctype/list_view_settings/test_list_view_settings.py b/frappe/desk/doctype/list_view_settings/test_list_view_settings.py index 0eab9cd7a6..8e27385230 100644 --- a/frappe/desk/doctype/list_view_settings/test_list_view_settings.py +++ b/frappe/desk/doctype/list_view_settings/test_list_view_settings.py @@ -1,8 +1,8 @@ # Copyright (c) 2019, Frappe Technologies and Contributors # License: MIT. See LICENSE # import frappe -import unittest +from frappe.tests.utils import FrappeTestCase -class TestListViewSettings(unittest.TestCase): +class TestListViewSettings(FrappeTestCase): pass diff --git a/frappe/desk/doctype/module_onboarding/module_onboarding.js b/frappe/desk/doctype/module_onboarding/module_onboarding.js index d95920e2ca..0e312025bf 100644 --- a/frappe/desk/doctype/module_onboarding/module_onboarding.js +++ b/frappe/desk/doctype/module_onboarding/module_onboarding.js @@ -2,7 +2,7 @@ // For license information, please see license.txt frappe.ui.form.on("Module Onboarding", { - refresh: function(frm) { + refresh: function (frm) { frappe.boot.developer_mode && frm.set_intro( __( @@ -15,7 +15,7 @@ frappe.ui.form.on("Module Onboarding", { } }, - disable_form: function(frm) { + disable_form: function (frm) { frm.set_read_only(); frm.fields .filter((field) => field.has_input) diff --git a/frappe/desk/doctype/module_onboarding/test_module_onboarding.py b/frappe/desk/doctype/module_onboarding/test_module_onboarding.py index fa19794c1e..e84a5c4d2b 100644 --- a/frappe/desk/doctype/module_onboarding/test_module_onboarding.py +++ b/frappe/desk/doctype/module_onboarding/test_module_onboarding.py @@ -1,8 +1,8 @@ # Copyright (c) 2020, Frappe Technologies and Contributors # License: MIT. See LICENSE # import frappe -import unittest +from frappe.tests.utils import FrappeTestCase -class TestModuleOnboarding(unittest.TestCase): +class TestModuleOnboarding(FrappeTestCase): pass diff --git a/frappe/desk/doctype/note/note.js b/frappe/desk/doctype/note/note.js index 5718180b70..567e7534ba 100644 --- a/frappe/desk/doctype/note/note.js +++ b/frappe/desk/doctype/note/note.js @@ -1,45 +1,45 @@ frappe.ui.form.on("Note", { - refresh: function(frm) { - if (frm.doc.__islocal) { - frm.events.set_editable(frm, true); - } else { - if (!frm.doc.content) { - frm.doc.content = ""; - } - - // toggle edit - frm.add_custom_button("Edit", function() { - frm.events.set_editable(frm, !frm.is_note_editable); - }); - frm.events.set_editable(frm, false); + refresh: function (frm) { + if (!frm.is_new()) { + frm.is_note_editable = false; + frm.events.set_editable(frm); } }, - set_editable: function(frm, editable) { - // hide all fields other than content - - // no permission - if (editable && !frm.perm[0].write) return; + set_editable: function (frm) { + if (frm.has_perm("write")) { + const read_label = __("Read mode"); + const edit_label = __("Edit mode"); + frm.remove_custom_button(frm.is_note_editable ? edit_label : read_label); + frm.add_custom_button(frm.is_note_editable ? read_label : edit_label, function () { + frm.is_note_editable = !frm.is_note_editable; + frm.events.set_editable(frm); + }); + } + // toggle "read_only" for content and "hidden" of all other fields // content read_only - frm.set_df_property("content", "read_only", editable ? 0 : 1); + frm.set_df_property("content", "read_only", frm.is_note_editable ? 0 : 1); // hide all other fields - $.each(frm.fields_dict, function(fieldname) { - if (fieldname !== "content") { - frm.set_df_property(fieldname, "hidden", editable ? 0 : 1); + for (const field of frm.meta.fields) { + if (field.fieldname !== "content") { + frm.set_df_property( + field.fieldname, + "hidden", + frm.is_note_editable && !field.hidden && frm.get_perm(field.permlevel, "write") + ? 0 + : 1 + ); } - }); + } // no label, description for content either - frm.get_field("content").toggle_label(editable); - frm.get_field("content").toggle_description(editable); - - // set flag for toggle - frm.is_note_editable = editable; - } + frm.get_field("content").toggle_label(frm.is_note_editable); + frm.get_field("content").toggle_description(frm.is_note_editable); + }, }); -frappe.tour['Note'] = [ +frappe.tour["Note"] = [ { fieldname: "title", title: "Title of the Note", @@ -48,6 +48,7 @@ frappe.tour['Note'] = [ { fieldname: "public", title: "Sets the Note to Public", - description: "You can change the visibility of the note with this, setting it to public will allow other users to view it.", + description: + "You can change the visibility of the note with this, setting it to public will allow other users to view it.", }, -]; \ No newline at end of file +]; diff --git a/frappe/desk/doctype/note/note.json b/frappe/desk/doctype/note/note.json index 69a9518ac4..b297e2ada6 100644 --- a/frappe/desk/doctype/note/note.json +++ b/frappe/desk/doctype/note/note.json @@ -1,6 +1,6 @@ { "actions": [], - "allow_rename": 1, + "autoname": "hash", "creation": "2013-05-24 13:41:00", "doctype": "DocType", "document_type": "Document", @@ -19,11 +19,8 @@ { "fieldname": "title", "fieldtype": "Data", - "in_global_search": 1, "in_list_view": 1, "label": "Title", - "no_copy": 1, - "print_hide": 1, "reqd": 1 }, { @@ -32,6 +29,7 @@ "fieldname": "public", "fieldtype": "Check", "label": "Public", + "permlevel": 1, "print_hide": 1 }, { @@ -40,7 +38,8 @@ "depends_on": "public", "fieldname": "notify_on_login", "fieldtype": "Check", - "label": "Notify users with a popup when they log in" + "label": "Notify users with a popup when they log in", + "permlevel": 1 }, { "bold": 1, @@ -49,13 +48,15 @@ "description": "If enabled, users will be notified every time they login. If not enabled, users will only be notified once.", "fieldname": "notify_on_every_login", "fieldtype": "Check", - "label": "Notify Users On Every Login" + "label": "Notify Users On Every Login", + "permlevel": 1 }, { "depends_on": "eval:doc.notify_on_login && doc.public", "fieldname": "expire_notification_on", "fieldtype": "Date", "label": "Expire Notification On", + "permlevel": 1, "search_index": 1 }, { @@ -68,39 +69,80 @@ }, { "collapsible": 1, + "depends_on": "notify_on_login", "fieldname": "seen_by_section", "fieldtype": "Section Break", - "label": "Seen By" + "label": "Seen By", + "permlevel": 1 }, { "fieldname": "seen_by", "fieldtype": "Table", "label": "Seen By Table", - "options": "Note Seen By" + "options": "Note Seen By", + "permlevel": 1 } ], "icon": "fa fa-file-text", "idx": 1, "links": [], - "modified": "2021-09-18 10:57:51.352643", + "modified": "2023-04-24 16:07:03.281184", "modified_by": "Administrator", "module": "Desk", "name": "Note", + "naming_rule": "Random", "owner": "Administrator", "permissions": [ { "create": 1, "delete": 1, "email": 1, + "export": 1, "print": 1, "read": 1, + "report": 1, + "role": "System Manager", + "share": 1, + "write": 1 + }, + { + "permlevel": 1, + "read": 1, + "role": "System Manager", + "write": 1 + }, + { + "permlevel": 2, + "read": 1, + "role": "System Manager", + "write": 1 + }, + { + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "All" + }, + { + "create": 1, + "delete": 1, + "email": 1, + "if_owner": 1, "role": "All", "share": 1, "write": 1 + }, + { + "permlevel": 1, + "read": 1, + "role": "All" } ], "quick_entry": 1, "sort_field": "modified", "sort_order": "ASC", + "states": [], + "title_field": "title", "track_changes": 1 } \ No newline at end of file diff --git a/frappe/desk/doctype/note/note.py b/frappe/desk/doctype/note/note.py index c0a37d5f44..6aba6391cb 100644 --- a/frappe/desk/doctype/note/note.py +++ b/frappe/desk/doctype/note/note.py @@ -1,53 +1,39 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE -import re - import frappe from frappe.model.document import Document -NAME_PATTERN = re.compile("[%'\"#*?`]") - class Note(Document): - def autoname(self): - # replace forbidden characters - self.name = NAME_PATTERN.sub("", self.title.strip()) - def validate(self): if self.notify_on_login and not self.expire_notification_on: - # expire this notification in a week (default) self.expire_notification_on = frappe.utils.add_days(self.creation, 7) + if not self.content: + self.content = "" + def before_print(self, settings=None): self.print_heading = self.name self.sub_heading = "" + def mark_seen_by(self, user: str) -> None: + if user in [d.user for d in self.seen_by]: + return + + self.append("seen_by", {"user": user}) + @frappe.whitelist() -def mark_as_seen(note): - note = frappe.get_doc("Note", note) - if frappe.session.user not in [d.user for d in note.seen_by]: - note.append("seen_by", {"user": frappe.session.user}) - note.save(ignore_version=True) +def mark_as_seen(note: str): + note: Note = frappe.get_doc("Note", note) + note.mark_seen_by(frappe.session.user) + note.save(ignore_permissions=True, ignore_version=True) def get_permission_query_conditions(user): if not user: user = frappe.session.user - if user == "Administrator": - return "" - - return f"""(`tabNote`.public=1 or `tabNote`.owner={frappe.db.escape(user)})""" - - -def has_permission(doc, ptype, user): - if doc.public == 1 or user == "Administrator": - return True - - if user == doc.owner: - return True - - return False + return f"(`tabNote`.owner = {frappe.db.escape(user)} or `tabNote`.public = 1)" diff --git a/frappe/desk/doctype/note/note_list.js b/frappe/desk/doctype/note/note_list.js index f7f8d37dcf..a8e948bc94 100644 --- a/frappe/desk/doctype/note/note_list.js +++ b/frappe/desk/doctype/note/note_list.js @@ -1,13 +1,11 @@ -frappe.listview_settings['Note'] = { - onload: function(me) { - me.page.set_title(__("Notes")); - }, - add_fields: ["title", "public"], - get_indicator: function(doc) { - if(doc.public) { +frappe.listview_settings["Note"] = { + hide_name_column: true, + add_fields: ["public"], + get_indicator: function (doc) { + if (doc.public) { return [__("Public"), "green", "public,=,Yes"]; } else { return [__("Private"), "gray", "public,=,No"]; } - } -} + }, +}; diff --git a/frappe/desk/doctype/note/test_note.py b/frappe/desk/doctype/note/test_note.py index d8bdb9efc4..426fb5a16e 100644 --- a/frappe/desk/doctype/note/test_note.py +++ b/frappe/desk/doctype/note/test_note.py @@ -1,14 +1,13 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors and Contributors # License: MIT. See LICENSE -import unittest - import frappe +from frappe.tests.utils import FrappeTestCase test_records = frappe.get_test_records("Note") -class TestNote(unittest.TestCase): +class TestNote(FrappeTestCase): def insert_note(self): frappe.db.delete("Version") frappe.db.delete("Note") diff --git a/frappe/desk/doctype/note_seen_by/note_seen_by.json b/frappe/desk/doctype/note_seen_by/note_seen_by.json index 7ee423e347..905a043284 100644 --- a/frappe/desk/doctype/note_seen_by/note_seen_by.json +++ b/frappe/desk/doctype/note_seen_by/note_seen_by.json @@ -1,64 +1,32 @@ { - "allow_copy": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, - "creation": "2016-08-29 05:29:16.726172", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, + "actions": [], + "creation": "2016-08-29 05:29:16.726172", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "user" + ], "fields": [ { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "user", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 1, - "label": "User", - "length": 0, - "no_copy": 0, - "options": "User", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "fieldname": "user", + "fieldtype": "Link", + "in_list_view": 1, + "label": "User", + "options": "User", + "permlevel": 2 } - ], - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - - "is_submittable": 0, - "issingle": 0, - "istable": 1, - "max_attachments": 0, - "modified": "2016-08-29 06:02:41.531341", - "modified_by": "Administrator", - "module": "Desk", - "name": "Note Seen By", - "name_case": "", - "owner": "Administrator", - "permissions": [], - "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_seen": 0 + ], + "istable": 1, + "links": [], + "modified": "2023-04-24 16:14:53.684098", + "modified_by": "Administrator", + "module": "Desk", + "name": "Note Seen By", + "owner": "Administrator", + "permissions": [], + "quick_entry": 1, + "sort_field": "modified", + "sort_order": "DESC", + "states": [] } \ No newline at end of file diff --git a/frappe/desk/doctype/notification_log/notification_log.js b/frappe/desk/doctype/notification_log/notification_log.js index 1f381d115b..ea5fdc6400 100644 --- a/frappe/desk/doctype/notification_log/notification_log.js +++ b/frappe/desk/doctype/notification_log/notification_log.js @@ -1,25 +1,25 @@ // Copyright (c) 2019, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Notification Log', { - refresh: function(frm) { +frappe.ui.form.on("Notification Log", { + refresh: function (frm) { if (frm.doc.attached_file) { - frm.trigger('set_attachment'); + frm.trigger("set_attachment"); } else { - frm.get_field('attachment_link').$wrapper.empty(); + frm.get_field("attachment_link").$wrapper.empty(); } }, - open_reference_document: function(frm) { + open_reference_document: function (frm) { const dt = frm.doc.document_type; const dn = frm.doc.document_name; - frappe.set_route('Form', dt, dn); + frappe.set_route("Form", dt, dn); }, - set_attachment: function(frm) { + set_attachment: function (frm) { const attachment = JSON.parse(frm.doc.attached_file); - const $wrapper = frm.get_field('attachment_link').$wrapper; + const $wrapper = frm.get_field("attachment_link").$wrapper; $wrapper.html(`
    @@ -41,5 +41,5 @@ frappe.ui.form.on('Notification Log', { frappe.msgprint(__("Please enable pop-ups")); } }); - } + }, }); diff --git a/frappe/desk/doctype/notification_log/notification_log.json b/frappe/desk/doctype/notification_log/notification_log.json index e188708277..f24a6447b4 100644 --- a/frappe/desk/doctype/notification_log/notification_log.json +++ b/frappe/desk/doctype/notification_log/notification_log.json @@ -22,18 +22,14 @@ "fieldname": "subject", "fieldtype": "Text", "in_list_view": 1, - "label": "Subject", - "show_days": 1, - "show_seconds": 1 + "label": "Subject" }, { "fieldname": "for_user", "fieldtype": "Link", "hidden": 1, "label": "For User", - "options": "User", - "show_days": 1, - "show_seconds": 1 + "options": "User" }, { "fieldname": "type", @@ -42,36 +38,26 @@ "in_list_view": 1, "in_standard_filter": 1, "label": "Type", - "options": "Mention\nEnergy Point\nAssignment\nShare\nAlert", - "search_index": 1, - "show_days": 1, - "show_seconds": 1 + "options": "Mention\nEnergy Point\nAssignment\nShare\nAlert" }, { "fieldname": "email_content", "fieldtype": "Text Editor", - "label": "Message", - "show_days": 1, - "show_seconds": 1 + "label": "Message" }, { "fieldname": "document_type", "fieldtype": "Link", "hidden": 1, "label": "Document Type", - "options": "DocType", - "search_index": 1, - "show_days": 1, - "show_seconds": 1 + "options": "DocType" }, { "fieldname": "document_name", "fieldtype": "Data", "hidden": 1, "label": "Document Link", - "search_index": 1, - "show_days": 1, - "show_seconds": 1 + "search_index": 1 }, { "fieldname": "from_user", @@ -79,9 +65,7 @@ "hidden": 1, "label": "From User", "options": "User", - "search_index": 1, - "show_days": 1, - "show_seconds": 1 + "search_index": 1 }, { "default": "0", @@ -89,38 +73,30 @@ "fieldtype": "Check", "hidden": 1, "ignore_user_permissions": 1, - "label": "Read", - "show_days": 1, - "show_seconds": 1 + "label": "Read" }, { "fieldname": "open_reference_document", "fieldtype": "Button", - "label": "Open Reference Document", - "show_days": 1, - "show_seconds": 1 + "label": "Open Reference Document" }, { "fieldname": "attached_file", "fieldtype": "Code", "hidden": 1, "label": "Attached File", - "options": "JSON", - "show_days": 1, - "show_seconds": 1 + "options": "JSON" }, { "fieldname": "attachment_link", "fieldtype": "HTML", - "label": "Attachment Link", - "show_days": 1, - "show_seconds": 1 + "label": "Attachment Link" } ], "hide_toolbar": 1, "in_create": 1, "links": [], - "modified": "2021-10-25 17:26:09.703215", + "modified": "2022-09-13 16:08:48.153934", "modified_by": "Administrator", "module": "Desk", "name": "Notification Log", @@ -138,6 +114,7 @@ ], "sort_field": "modified", "sort_order": "DESC", + "states": [], "title_field": "subject", "track_seen": 1 -} +} \ No newline at end of file diff --git a/frappe/desk/doctype/notification_log/notification_log.py b/frappe/desk/doctype/notification_log/notification_log.py index 17734635dd..4d82932555 100644 --- a/frappe/desk/doctype/notification_log/notification_log.py +++ b/frappe/desk/doctype/notification_log/notification_log.py @@ -20,6 +20,14 @@ class NotificationLog(Document): except frappe.OutgoingEmailError: self.log_error(_("Failed to send notification email")) + @staticmethod + def clear_old_logs(days=180): + from frappe.query_builder import Interval + from frappe.query_builder.functions import Now + + table = frappe.qb.DocType("Notification Log") + frappe.db.delete(table, filters=(table.modified < (Now() - Interval(days=days)))) + def get_permission_query_conditions(for_user): if not for_user: @@ -133,9 +141,25 @@ def get_email_header(doc): return header_map[doc.type or "Default"] +@frappe.whitelist() +def get_notification_logs(limit=20): + notification_logs = frappe.db.get_list( + "Notification Log", fields=["*"], limit=limit, order_by="modified desc" + ) + + users = [log.from_user for log in notification_logs] + users = [*set(users)] # remove duplicates + user_info = frappe._dict() + + for user in users: + frappe.utils.add_user_info(user, user_info) + + return {"notification_logs": notification_logs, "user_info": user_info} + + @frappe.whitelist() def mark_all_as_read(): - unread_docs_list = frappe.db.get_all( + unread_docs_list = frappe.get_all( "Notification Log", filters={"read": 0, "for_user": frappe.session.user} ) unread_docnames = [doc.name for doc in unread_docs_list] diff --git a/frappe/desk/doctype/notification_log/notification_log_list.js b/frappe/desk/doctype/notification_log/notification_log_list.js new file mode 100644 index 0000000000..150ffabfa7 --- /dev/null +++ b/frappe/desk/doctype/notification_log/notification_log_list.js @@ -0,0 +1,7 @@ +frappe.listview_settings["Notification Log"] = { + onload: function (listview) { + frappe.require("logtypes.bundle.js", () => { + frappe.utils.logtypes.show_log_retention_message(cur_list.doctype); + }); + }, +}; diff --git a/frappe/desk/doctype/notification_log/test_notification_log.py b/frappe/desk/doctype/notification_log/test_notification_log.py index 532f05ab57..a43455149f 100644 --- a/frappe/desk/doctype/notification_log/test_notification_log.py +++ b/frappe/desk/doctype/notification_log/test_notification_log.py @@ -1,13 +1,12 @@ # Copyright (c) 2019, Frappe Technologies and Contributors # License: MIT. See LICENSE -import unittest - import frappe from frappe.core.doctype.user.user import get_system_users from frappe.desk.form.assign_to import add as assign_task +from frappe.tests.utils import FrappeTestCase -class TestNotificationLog(unittest.TestCase): +class TestNotificationLog(FrappeTestCase): def test_assignment(self): todo = get_todo() user = get_user() @@ -38,7 +37,7 @@ class TestNotificationLog(unittest.TestCase): def get_last_email_queue(): - res = frappe.db.get_all("Email Queue", fields=["message"], order_by="creation desc", limit=1) + res = frappe.get_all("Email Queue", fields=["message"], order_by="creation desc", limit=1) return res[0] diff --git a/frappe/desk/doctype/notification_settings/notification_settings.js b/frappe/desk/doctype/notification_settings/notification_settings.js index cc2fd95204..ba72369273 100644 --- a/frappe/desk/doctype/notification_settings/notification_settings.js +++ b/frappe/desk/doctype/notification_settings/notification_settings.js @@ -1,28 +1,27 @@ // Copyright (c) 2019, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Notification Settings', { +frappe.ui.form.on("Notification Settings", { onload: (frm) => { frappe.breadcrumbs.add({ - label: __('Settings'), - route: '#modules/Settings', - type: 'Custom' + label: __("Settings"), + route: "#modules/Settings", + type: "Custom", }); - frm.set_query('subscribed_documents', () => { + frm.set_query("subscribed_documents", () => { return { filters: { - istable: 0 - } + istable: 0, + }, }; }); }, refresh: (frm) => { - if (frappe.user.has_role('System Manager')) { - frm.add_custom_button(__('Go to Notification Settings List'), () => { - frappe.set_route('List', 'Notification Settings'); + if (frappe.user.has_role("System Manager")) { + frm.add_custom_button(__("Go to Notification Settings List"), () => { + frappe.set_route("List", "Notification Settings"); }); } - } - + }, }); diff --git a/frappe/desk/doctype/notification_settings/notification_settings.py b/frappe/desk/doctype/notification_settings/notification_settings.py index 801d512fe7..cd85c7d06d 100644 --- a/frappe/desk/doctype/notification_settings/notification_settings.py +++ b/frappe/desk/doctype/notification_settings/notification_settings.py @@ -92,4 +92,7 @@ def get_permission_query_conditions(user): @frappe.whitelist() def set_seen_value(value, user): + if frappe.flags.read_only: + return + frappe.db.set_value("Notification Settings", user, "seen", value, update_modified=False) diff --git a/frappe/desk/doctype/notification_settings/test_notification_settings.py b/frappe/desk/doctype/notification_settings/test_notification_settings.py index 966b923567..e88f98443b 100644 --- a/frappe/desk/doctype/notification_settings/test_notification_settings.py +++ b/frappe/desk/doctype/notification_settings/test_notification_settings.py @@ -2,8 +2,8 @@ # See license.txt # import frappe -import unittest +from frappe.tests.utils import FrappeTestCase -class TestNotificationSettings(unittest.TestCase): +class TestNotificationSettings(FrappeTestCase): pass diff --git a/frappe/desk/doctype/number_card/number_card.js b/frappe/desk/doctype/number_card/number_card.js index 79ddb71187..b0c5456268 100644 --- a/frappe/desk/doctype/number_card/number_card.js +++ b/frappe/desk/doctype/number_card/number_card.js @@ -1,73 +1,75 @@ // Copyright (c) 2020, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Number Card', { - refresh: function(frm) { +frappe.ui.form.on("Number Card", { + refresh: function (frm) { if (!frappe.boot.developer_mode && frm.doc.is_standard) { frm.disable_form(); } frm.set_df_property("filters_section", "hidden", 1); frm.set_df_property("dynamic_filters_section", "hidden", 1); - frm.trigger('set_options'); + frm.trigger("set_options"); if (!frm.doc.type) { - frm.set_value('type', 'Document Type'); + frm.set_value("type", "Document Type"); } - if (frm.doc.type == 'Report' && frm.doc.report_name) { - frm.trigger('set_report_filters'); + if (frm.doc.type == "Report" && frm.doc.report_name) { + frm.trigger("set_report_filters"); } - if (frm.doc.type == 'Custom') { + if (frm.doc.type == "Custom") { if (!frappe.boot.developer_mode) { frm.disable_form(); } frm.filters = eval(frm.doc.filters_config); - frm.trigger('set_filters_description'); - frm.trigger('set_method_description'); - frm.trigger('render_filters_table'); + frm.trigger("set_filters_description"); + frm.trigger("set_method_description"); + frm.trigger("render_filters_table"); } - frm.trigger('set_parent_document_type'); + frm.trigger("set_parent_document_type"); if (!frm.is_new()) { - frm.trigger('create_add_to_dashboard_button'); + frm.trigger("create_add_to_dashboard_button"); } }, - create_add_to_dashboard_button: function(frm) { - frm.add_custom_button('Add Card to Dashboard', () => { + create_add_to_dashboard_button: function (frm) { + frm.add_custom_button("Add Card to Dashboard", () => { const dialog = frappe.dashboard_utils.get_add_to_dashboard_dialog( frm.doc.name, - 'Number Card', - 'frappe.desk.doctype.number_card.number_card.add_card_to_dashboard' + "Number Card", + "frappe.desk.doctype.number_card.number_card.add_card_to_dashboard" ); if (!frm.doc.name) { - frappe.msgprint(__('Please create Card first')); + frappe.msgprint(__("Please create Card first")); } else { dialog.show(); } }); }, - before_save: function(frm) { - let dynamic_filters = JSON.parse(frm.doc.dynamic_filters_json || 'null'); - let static_filters = JSON.parse(frm.doc.filters_json || 'null'); - static_filters = - frappe.dashboard_utils.remove_common_static_filter_values(static_filters, dynamic_filters); + before_save: function (frm) { + let dynamic_filters = JSON.parse(frm.doc.dynamic_filters_json || "null"); + let static_filters = JSON.parse(frm.doc.filters_json || "null"); + static_filters = frappe.dashboard_utils.remove_common_static_filter_values( + static_filters, + dynamic_filters + ); - frm.set_value('filters_json', JSON.stringify(static_filters)); - frm.trigger('render_filters_table'); - frm.trigger('render_dynamic_filters_table'); + frm.set_value("filters_json", JSON.stringify(static_filters)); + frm.trigger("render_filters_table"); + frm.trigger("render_dynamic_filters_table"); }, - is_standard: function(frm) { - frm.trigger('render_dynamic_filters_table'); + is_standard: function (frm) { + frm.trigger("render_dynamic_filters_table"); frm.set_df_property("dynamic_filters_section", "hidden", 1); }, - set_filters_description: function(frm) { - if (frm.doc.type == 'Custom') { + set_filters_description: function (frm) { + if (frm.doc.type == "Custom") { frm.fields_dict.filters_config.set_description(` Set the filters here. For example: 
    @@ -91,8 +93,8 @@ frappe.ui.form.on('Number Card', {
 		}
 	},
 
-	set_method_description: function(frm) {
-		if (frm.doc.type == 'Custom') {
+	set_method_description: function (frm) {
+		if (frm.doc.type == "Custom") {
 			frm.fields_dict.method.set_description(`
 		Set the path to a whitelisted function that will return the number on the card in the format:
 
    @@ -105,53 +107,52 @@ frappe.ui.form.on('Number Card', {
 		}
 	},
 
-	type: function(frm) {
-		frm.trigger('set_filters_description');
-		if (frm.doc.type == 'Report') {
-			frm.set_query('report_name', () => {
+	type: function (frm) {
+		frm.trigger("set_filters_description");
+		if (frm.doc.type == "Report") {
+			frm.set_query("report_name", () => {
 				return {
 					filters: {
-						'report_type': ['!=', 'Report Builder']
-					}
+						report_type: ["!=", "Report Builder"],
+					},
 				};
 			});
 		}
-
 	},
 
-	report_name: function(frm) {
+	report_name: function (frm) {
 		frm.filters = [];
-		frm.set_value('filters_json', '{}');
-		frm.set_value('dynamic_filters_json', '{}');
-		frm.set_df_property('report_field', 'options', []);
-		frm.trigger('set_report_filters');
+		frm.set_value("filters_json", "{}");
+		frm.set_value("dynamic_filters_json", "{}");
+		frm.set_df_property("report_field", "options", []);
+		frm.trigger("set_report_filters");
 	},
 
-	filters_config: function(frm) {
+	filters_config: function (frm) {
 		frm.filters = eval(frm.doc.filters_config);
 		const filter_values = frappe.report_utils.get_filter_values(frm.filters);
-		frm.set_value('filters_json', JSON.stringify(filter_values));
-		frm.trigger('render_filters_table');
+		frm.set_value("filters_json", JSON.stringify(filter_values));
+		frm.trigger("render_filters_table");
 	},
 
-	document_type: function(frm) {
-		frm.set_query('document_type', function() {
+	document_type: function (frm) {
+		frm.set_query("document_type", function () {
 			return {
 				filters: {
-					'issingle': false
-				}
+					issingle: false,
+				},
 			};
 		});
-		frm.set_value('filters_json', '[]');
-		frm.set_value('dynamic_filters_json', '[]');
-		frm.set_value('aggregate_function_based_on', '');
-		frm.set_value('parent_document_type', '');
-		frm.trigger('set_options');
-		frm.trigger('set_parent_document_type');
+		frm.set_value("filters_json", "[]");
+		frm.set_value("dynamic_filters_json", "[]");
+		frm.set_value("aggregate_function_based_on", "");
+		frm.set_value("parent_document_type", "");
+		frm.trigger("set_options");
+		frm.trigger("set_parent_document_type");
 	},
 
-	set_options: function(frm) {
-		if (frm.doc.type !== 'Document Type') {
+	set_options: function (frm) {
+		if (frm.doc.type !== "Document Type") {
 			return;
 		}
 
@@ -160,134 +161,148 @@ frappe.ui.form.on('Number Card', {
 
 		if (doctype) {
 			frappe.model.with_doctype(doctype, () => {
-				frappe.get_meta(doctype).fields.map(df => {
+				frappe.get_meta(doctype).fields.map((df) => {
 					if (frappe.model.numeric_fieldtypes.includes(df.fieldtype)) {
-						if (df.fieldtype == 'Currency') {
-							if (!df.options || df.options !== 'Company:company:default_currency') {
+						if (df.fieldtype == "Currency") {
+							if (!df.options || df.options !== "Company:company:default_currency") {
 								return;
 							}
 						}
-						aggregate_based_on_fields.push({label: df.label, value: df.fieldname});
+						aggregate_based_on_fields.push({ label: df.label, value: df.fieldname });
 					}
 				});
 
-				frm.set_df_property('aggregate_function_based_on', 'options', aggregate_based_on_fields);
+				frm.set_df_property(
+					"aggregate_function_based_on",
+					"options",
+					aggregate_based_on_fields
+				);
 			});
-			frm.trigger('render_filters_table');
-			frm.trigger('render_dynamic_filters_table');
+			frm.trigger("render_filters_table");
+			frm.trigger("render_dynamic_filters_table");
 		}
 	},
 
-	set_report_filters: function(frm) {
+	set_report_filters: function (frm) {
 		const report_name = frm.doc.report_name;
 		if (report_name) {
-			frappe.report_utils.get_report_filters(report_name).then(filters => {
+			frappe.report_utils.get_report_filters(report_name).then((filters) => {
 				if (filters) {
 					frm.filters = filters;
 					const filter_values = frappe.report_utils.get_filter_values(filters);
 					if (frm.doc.filters_json.length <= 2) {
-						frm.set_value('filters_json', JSON.stringify(filter_values));
+						frm.set_value("filters_json", JSON.stringify(filter_values));
 					}
 				}
-				frm.trigger('render_filters_table');
-				frm.trigger('set_report_field_options');
-				frm.trigger('render_dynamic_filters_table');
+				frm.trigger("render_filters_table");
+				frm.trigger("set_report_field_options");
+				frm.trigger("render_dynamic_filters_table");
 			});
 		}
 	},
 
-	set_report_field_options: function(frm) {
+	set_report_field_options: function (frm) {
 		let filters = frm.doc.filters_json.length > 2 ? JSON.parse(frm.doc.filters_json) : null;
 		if (frm.doc.dynamic_filters_json && frm.doc.dynamic_filters_json.length > 2) {
 			filters = frappe.dashboard_utils.get_all_filters(frm.doc);
 		}
-		frappe.xcall(
-			'frappe.desk.query_report.run',
-			{
+		frappe
+			.xcall("frappe.desk.query_report.run", {
 				report_name: frm.doc.report_name,
 				filters: filters,
-				ignore_prepared_report: 1
-			}
-		).then(data => {
-			if (data.result.length) {
-				frm.field_options = frappe.report_utils.get_field_options_from_report(data.columns, data);
-				frm.set_df_property('report_field', 'options', frm.field_options.numeric_fields);
-				if (!frm.field_options.numeric_fields.length) {
-					frappe.msgprint(__("Report has no numeric fields, please change the Report Name"));
+				ignore_prepared_report: 1,
+			})
+			.then((data) => {
+				if (data.result.length) {
+					frm.field_options = frappe.report_utils.get_field_options_from_report(
+						data.columns,
+						data
+					);
+					frm.set_df_property(
+						"report_field",
+						"options",
+						frm.field_options.numeric_fields
+					);
+					if (!frm.field_options.numeric_fields.length) {
+						frappe.msgprint(
+							__("Report has no numeric fields, please change the Report Name")
+						);
+					}
+				} else {
+					frappe.msgprint(
+						__(
+							"Report has no data, please modify the filters or change the Report Name"
+						)
+					);
 				}
-			} else {
-				frappe.msgprint(__('Report has no data, please modify the filters or change the Report Name'));
-			}
-		});
+			});
 	},
 
-	render_filters_table: function(frm) {
+	render_filters_table: function (frm) {
 		frm.set_df_property("filters_section", "hidden", 0);
-		let is_document_type = frm.doc.type == 'Document Type';
-		let is_dynamic_filter = f => ['Date', 'DateRange'].includes(f.fieldtype) && f.default;
+		let is_document_type = frm.doc.type == "Document Type";
+		let is_dynamic_filter = (f) => ["Date", "DateRange"].includes(f.fieldtype) && f.default;
 
-		let wrapper = $(frm.get_field('filters_json').wrapper).empty();
+		let wrapper = $(frm.get_field("filters_json").wrapper).empty();
 		let table = $(`
-					
-					
-					
+					
+					
+					
    
 			
 				
    ${__('Filter')}${__('Condition')}${__('Value')}${__("Filter")}${__("Condition")}${__("Value")}
 				
    
 			    
 			
 		
    `).appendTo(wrapper);
 		$(`
    ${__("Click table to edit")}
    `).appendTo(wrapper);
 
-		let filters = JSON.parse(frm.doc.filters_json || '[]');
+		let filters = JSON.parse(frm.doc.filters_json || "[]");
 		let filters_set = false;
 
 		// Set dynamic filters for reports
-		if (frm.doc.type == 'Report') {
+		if (frm.doc.type == "Report") {
 			let set_filters = false;
-			frm.filters.forEach(f => {
+			frm.filters.forEach((f) => {
 				if (is_dynamic_filter(f)) {
 					filters[f.fieldname] = f.default;
 					set_filters = true;
 				}
 			});
-			set_filters && frm.set_value('filters_json', JSON.stringify(filters));
+			set_filters && frm.set_value("filters_json", JSON.stringify(filters));
 		}
 
 		let fields = [];
 		if (is_document_type) {
 			fields = [
 				{
-					fieldtype: 'HTML',
-					fieldname: 'filter_area',
-				}
+					fieldtype: "HTML",
+					fieldname: "filter_area",
+				},
 			];
 
 			if (filters.length) {
-				filters.forEach(filter => {
-					const filter_row =
-						$(`
+				filters.forEach((filter) => {
+					const filter_row = $(`
 							${filter[1]}
 							${filter[2] || ""}
 							${filter[3]}
 						`);
 
-					table.find('tbody').append(filter_row);
+					table.find("tbody").append(filter_row);
 				});
 				filters_set = true;
 			}
 		} else if (frm.filters.length) {
-			fields = frm.filters.filter(f => f.fieldname);
-			fields.map(f => {
+			fields = frm.filters.filter((f) => f.fieldname);
+			fields.map((f) => {
 				if (filters[f.fieldname]) {
-					let condition = '=';
-					const filter_row =
-						$(`
+					let condition = "=";
+					const filter_row = $(`
 							${f.label}
 							${condition}
 							${filters[f.fieldname] || ""}
 						`);
-					table.find('tbody').append(filter_row);
+					table.find("tbody").append(filter_row);
 					if (!filters_set) filters_set = true;
 				}
 			});
@@ -296,32 +311,32 @@ frappe.ui.form.on('Number Card', {
 		if (!filters_set) {
 			const filter_row = $(`
 				${__("Click to Set Filters")}`);
-			table.find('tbody').append(filter_row);
+			table.find("tbody").append(filter_row);
 		}
 
-		table.on('click', () => {
+		table.on("click", () => {
 			let dialog = new frappe.ui.Dialog({
-				title: __('Set Filters'),
-				fields: fields.filter(f => !is_dynamic_filter(f)),
-				primary_action: function() {
+				title: __("Set Filters"),
+				fields: fields.filter((f) => !is_dynamic_filter(f)),
+				primary_action: function () {
 					let values = this.get_values();
 					if (values) {
 						this.hide();
 						if (is_document_type) {
 							let filters = frm.filter_group.get_filters();
-							frm.set_value('filters_json', JSON.stringify(filters));
+							frm.set_value("filters_json", JSON.stringify(filters));
 						} else {
-							frm.set_value('filters_json', JSON.stringify(values));
+							frm.set_value("filters_json", JSON.stringify(values));
 						}
-						frm.trigger('render_filters_table');
+						frm.trigger("render_filters_table");
 					}
 				},
-				primary_action_label: "Set"
+				primary_action_label: "Set",
 			});
 
 			if (is_document_type) {
 				frm.filter_group = new frappe.ui.FilterGroup({
-					parent: dialog.get_field('filter_area').$wrapper,
+					parent: dialog.get_field("filter_area").$wrapper,
 					doctype: frm.doc.document_type,
 					parent_doctype: frm.doc.parent_document_type,
 					on_change: () => {},
@@ -331,56 +346,61 @@ frappe.ui.form.on('Number Card', {
 
 			dialog.show();
 
-			if (frm.doc.type == 'Report') {
+			if (frm.doc.type == "Report") {
 				//Set query report object so that it can be used while fetching filter values in the report
-				frappe.query_report = new frappe.views.QueryReport({'filters': dialog.fields_list});
-				frappe.query_reports[frm.doc.report_name]
-					&& frappe.query_reports[frm.doc.report_name].onload
-						&& frappe.query_reports[frm.doc.report_name].onload(frappe.query_report);
+				frappe.query_report = new frappe.views.QueryReport({
+					filters: dialog.fields_list,
+				});
+				frappe.query_reports[frm.doc.report_name] &&
+					frappe.query_reports[frm.doc.report_name].onload &&
+					frappe.query_reports[frm.doc.report_name].onload(frappe.query_report);
 			}
 
 			dialog.set_values(filters);
 		});
-
 	},
 
 	render_dynamic_filters_table(frm) {
-		if (!frappe.boot.developer_mode || !frm.doc.is_standard || frm.doc.type == 'Custom') {
+		if (!frappe.boot.developer_mode || !frm.doc.is_standard || frm.doc.type == "Custom") {
 			return;
 		}
 
 		frm.set_df_property("dynamic_filters_section", "hidden", 0);
 
-		let is_document_type = frm.doc.type == 'Document Type';
+		let is_document_type = frm.doc.type == "Document Type";
 
-		let wrapper = $(frm.get_field('dynamic_filters_json').wrapper).empty();
+		let wrapper = $(frm.get_field("dynamic_filters_json").wrapper).empty();
 
-		frm.dynamic_filter_table = $(`
+		frm.dynamic_filter_table =
+			$(`
    
-					
-					
-					
+					
+					
+					
    
 			
 				
    ${__('Filter')}${__('Condition')}${__('Value')}${__("Filter")}${__("Condition")}${__("Value")}
 				
    
 			    
 			
 		
    `).appendTo(wrapper);
 
-		frm.dynamic_filters = frm.doc.dynamic_filters_json && frm.doc.dynamic_filters_json.length > 2
-			? JSON.parse(frm.doc.dynamic_filters_json)
-			: null;
+		frm.dynamic_filters =
+			frm.doc.dynamic_filters_json && frm.doc.dynamic_filters_json.length > 2
+				? JSON.parse(frm.doc.dynamic_filters_json)
+				: null;
 
-		frm.trigger('set_dynamic_filters_in_table');
+		frm.trigger("set_dynamic_filters_in_table");
 
-		let filters = JSON.parse(frm.doc.filters_json || '[]');
+		let filters = JSON.parse(frm.doc.filters_json || "[]");
 
 		let fields = frappe.dashboard_utils.get_fields_for_dynamic_filter_dialog(
-			is_document_type, filters, frm.dynamic_filters
+			is_document_type,
+			filters,
+			frm.dynamic_filters
 		);
 
-		frm.dynamic_filter_table.on('click', () => {
+		frm.dynamic_filter_table.on("click", () => {
 			let dialog = new frappe.ui.Dialog({
-				title: __('Set Dynamic Filters'),
+				title: __("Set Dynamic Filters"),
 				fields: fields,
 				primary_action: () => {
 					let values = dialog.get_values();
@@ -388,19 +408,19 @@ frappe.ui.form.on('Number Card', {
 					let dynamic_filters = [];
 					for (let key of Object.keys(values)) {
 						if (is_document_type) {
-							let [doctype, fieldname] = key.split(':');
-							dynamic_filters.push([doctype, fieldname, '=', values[key]]);
+							let [doctype, fieldname] = key.split(":");
+							dynamic_filters.push([doctype, fieldname, "=", values[key]]);
 						}
 					}
 
 					if (is_document_type) {
-						frm.set_value('dynamic_filters_json', JSON.stringify(dynamic_filters));
+						frm.set_value("dynamic_filters_json", JSON.stringify(dynamic_filters));
 					} else {
-						frm.set_value('dynamic_filters_json', JSON.stringify(values));
+						frm.set_value("dynamic_filters_json", JSON.stringify(values));
 					}
-					frm.trigger('set_dynamic_filters_in_table');
+					frm.trigger("set_dynamic_filters_in_table");
 				},
-				primary_action_label: "Set"
+				primary_action_label: "Set",
 			});
 
 			dialog.show();
@@ -408,71 +428,66 @@ frappe.ui.form.on('Number Card', {
 		});
 	},
 
-	set_dynamic_filters_in_table: function(frm) {
-		frm.dynamic_filters =  frm.doc.dynamic_filters_json && frm.doc.dynamic_filters_json.length > 2
-			? JSON.parse(frm.doc.dynamic_filters_json)
-			: null;
+	set_dynamic_filters_in_table: function (frm) {
+		frm.dynamic_filters =
+			frm.doc.dynamic_filters_json && frm.doc.dynamic_filters_json.length > 2
+				? JSON.parse(frm.doc.dynamic_filters_json)
+				: null;
 
 		if (!frm.dynamic_filters) {
 			const filter_row = $(`
 				${__("Click to Set Dynamic Filters")}`);
-			frm.dynamic_filter_table.find('tbody').html(filter_row);
+			frm.dynamic_filter_table.find("tbody").html(filter_row);
 		} else {
-			let filter_rows = '';
+			let filter_rows = "";
 			if ($.isArray(frm.dynamic_filters)) {
-				frm.dynamic_filters.forEach(filter => {
-					filter_rows +=
-						`
+				frm.dynamic_filters.forEach((filter) => {
+					filter_rows += `
 							${filter[1]}
 							${filter[2] || ""}
 							${filter[3]}
 						`;
 				});
 			} else {
-				let condition = '=';
+				let condition = "=";
 				for (let [key, val] of Object.entries(frm.dynamic_filters)) {
-					filter_rows +=
-						`
+					filter_rows += `
 							${key}
 							${condition}
 							${val || ""}
-						`
-					;
+						`;
 				}
 			}
 
-			frm.dynamic_filter_table.find('tbody').html(filter_rows);
+			frm.dynamic_filter_table.find("tbody").html(filter_rows);
 		}
 	},
 
-	set_parent_document_type: async function(frm) {
+	set_parent_document_type: async function (frm) {
 		let document_type = frm.doc.document_type;
-		let doc_is_table = document_type &&
-			(await frappe.db.get_value('DocType', document_type, 'istable')).message.istable;
+		let doc_is_table =
+			document_type &&
+			(await frappe.db.get_value("DocType", document_type, "istable")).message.istable;
 
-		frm.set_df_property('parent_document_type', 'hidden', !doc_is_table);
+		frm.set_df_property("parent_document_type", "hidden", !doc_is_table);
 
 		if (document_type && doc_is_table) {
-			let parent = await frappe.db.get_list('DocField', {
-				filters: {
-					'fieldtype': 'Table',
-					'options': document_type
-				},
-				fields: ['parent']
-			});
+			let parents = await frappe.xcall(
+				"frappe.desk.doctype.dashboard_chart.dashboard_chart.get_parent_doctypes",
+				{ child_type: document_type }
+			);
 
-			parent && frm.set_query('parent_document_type', function() {
+			frm.set_query("parent_document_type", function () {
 				return {
 					filters: {
-						"name": ['in', parent.map(({ parent }) => parent)]
-					}
+						name: ["in", parents],
+					},
 				};
 			});
 
-			if (parent.length === 1) {
-				frm.set_value('parent_document_type', parent[0].parent);
+			if (parents.length === 1) {
+				frm.set_value("parent_document_type", parents[0]);
 			}
 		}
-	}
-
+	},
 });
diff --git a/frappe/desk/doctype/number_card/number_card.py b/frappe/desk/doctype/number_card/number_card.py
index 8e808ff635..451dc699fe 100644
--- a/frappe/desk/doctype/number_card/number_card.py
+++ b/frappe/desk/doctype/number_card/number_card.py
@@ -124,15 +124,17 @@ def get_result(doc, filters, to_date=None):
 			)
 		]
 
-	filters = frappe.parse_json(filters)
-
 	if not filters:
 		filters = []
+	elif isinstance(filters, str):
+		filters = frappe.parse_json(filters)
 
 	if to_date:
 		filters.append([doc.document_type, "creation", "<", to_date])
 
-	res = frappe.db.get_list(doc.document_type, fields=fields, filters=filters)
+	res = frappe.get_list(
+		doc.document_type, fields=fields, filters=filters, parent_doctype=doc.parent_document_type
+	)
 	number = res[0]["result"] if res else 0
 
 	return cint(number)
@@ -200,7 +202,7 @@ def get_cards_for_user(doctype, txt, searchfield, start, page_len, filters):
 	if txt:
 		search_conditions = [numberCard[field].like(f"%{txt}%") for field in searchfields]
 
-	condition_query = frappe.qb.engine.build_conditions(doctype, filters)
+	condition_query = frappe.qb.get_query(doctype, filters=filters)
 
 	return (
 		condition_query.select(numberCard.name, numberCard.label, numberCard.document_type)
diff --git a/frappe/desk/doctype/number_card/test_number_card.py b/frappe/desk/doctype/number_card/test_number_card.py
index c0dda40104..a002b8aad3 100644
--- a/frappe/desk/doctype/number_card/test_number_card.py
+++ b/frappe/desk/doctype/number_card/test_number_card.py
@@ -1,8 +1,8 @@
 # Copyright (c) 2020, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
 # import frappe
-import unittest
+from frappe.tests.utils import FrappeTestCase
 
 
-class TestNumberCard(unittest.TestCase):
+class TestNumberCard(FrappeTestCase):
 	pass
diff --git a/frappe/desk/doctype/onboarding_permission/onboarding_permission.js b/frappe/desk/doctype/onboarding_permission/onboarding_permission.js
index 752b8a02cc..ec2c8a03b0 100644
--- a/frappe/desk/doctype/onboarding_permission/onboarding_permission.js
+++ b/frappe/desk/doctype/onboarding_permission/onboarding_permission.js
@@ -1,8 +1,7 @@
 // Copyright (c) 2020, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Onboarding Permission', {
+frappe.ui.form.on("Onboarding Permission", {
 	// refresh: function(frm) {
-
 	// }
 });
diff --git a/frappe/desk/doctype/onboarding_permission/test_onboarding_permission.py b/frappe/desk/doctype/onboarding_permission/test_onboarding_permission.py
index cdfe0d7890..d82cb3c346 100644
--- a/frappe/desk/doctype/onboarding_permission/test_onboarding_permission.py
+++ b/frappe/desk/doctype/onboarding_permission/test_onboarding_permission.py
@@ -1,8 +1,8 @@
 # Copyright (c) 2020, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
 # import frappe
-import unittest
+from frappe.tests.utils import FrappeTestCase
 
 
-class TestOnboardingPermission(unittest.TestCase):
+class TestOnboardingPermission(FrappeTestCase):
 	pass
diff --git a/frappe/desk/doctype/onboarding_step/onboarding_step.js b/frappe/desk/doctype/onboarding_step/onboarding_step.js
index 3c9bbab9ac..67b2ed0501 100644
--- a/frappe/desk/doctype/onboarding_step/onboarding_step.js
+++ b/frappe/desk/doctype/onboarding_step/onboarding_step.js
@@ -2,18 +2,17 @@
 // For license information, please see license.txt
 
 frappe.ui.form.on("Onboarding Step", {
-
-	setup: function(frm) {
-		frm.set_query("form_tour", function() {
+	setup: function (frm) {
+		frm.set_query("form_tour", function () {
 			return {
 				filters: {
-					reference_doctype: frm.doc.reference_document
-				}
+					reference_doctype: frm.doc.reference_document,
+				},
 			};
 		});
 	},
 
-	refresh: function(frm) {
+	refresh: function (frm) {
 		frappe.boot.developer_mode &&
 			frm.set_intro(
 				__(
@@ -30,15 +29,16 @@ frappe.ui.form.on("Onboarding Step", {
 		}
 	},
 
-	reference_document: function(frm) {
+	reference_document: function (frm) {
 		if (frm.doc.reference_document && frm.doc.action == "Update Settings") {
 			setup_fields(frm);
 		}
 	},
 
-	action: function(frm) {
+	action: function (frm) {
 		if (frm.doc.action == "Show Form Tour") {
-			frm.fields_dict.reference_document.set_description(`You need to add the steps in the contoller JS file. For example: note.js
+			frm.fields_dict.reference_document
+				.set_description(`You need to add the steps in the contoller JS file. For example: note.js
 
    
 frappe.tour['Note'] = [
 	{
@@ -54,7 +54,7 @@ frappe.tour['Note'] = [
 		}
 	},
 
-	disable_form: function(frm) {
+	disable_form: function (frm) {
 		frm.set_read_only();
 		frm.fields
 			.filter((field) => field.has_input)
@@ -71,9 +71,7 @@ function setup_fields(frm) {
 			let fields = frappe
 				.get_meta(frm.doc.reference_document)
 				.fields.filter((df) => {
-					return ["Data", "Check", "Int", "Link", "Select"].includes(
-						df.fieldtype
-					);
+					return ["Data", "Check", "Int", "Link", "Select"].includes(df.fieldtype);
 				})
 				.map((df) => {
 					return {
diff --git a/frappe/desk/doctype/onboarding_step/test_onboarding_step.py b/frappe/desk/doctype/onboarding_step/test_onboarding_step.py
index d8bf55584c..73b9ab4ac3 100644
--- a/frappe/desk/doctype/onboarding_step/test_onboarding_step.py
+++ b/frappe/desk/doctype/onboarding_step/test_onboarding_step.py
@@ -1,8 +1,8 @@
 # Copyright (c) 2020, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
 # import frappe
-import unittest
+from frappe.tests.utils import FrappeTestCase
 
 
-class TestOnboardingStep(unittest.TestCase):
+class TestOnboardingStep(FrappeTestCase):
 	pass
diff --git a/frappe/desk/doctype/route_history/route_history.js b/frappe/desk/doctype/route_history/route_history.js
index 19689e406b..c68d4e2b54 100644
--- a/frappe/desk/doctype/route_history/route_history.js
+++ b/frappe/desk/doctype/route_history/route_history.js
@@ -1,8 +1,6 @@
 // Copyright (c) 2018, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Route History', {
-	refresh: function() {
-
-	}
+frappe.ui.form.on("Route History", {
+	refresh: function () {},
 });
diff --git a/frappe/desk/doctype/route_history/route_history_list.js b/frappe/desk/doctype/route_history/route_history_list.js
index 84a441852c..03bf86b9fd 100644
--- a/frappe/desk/doctype/route_history/route_history_list.js
+++ b/frappe/desk/doctype/route_history/route_history_list.js
@@ -1,7 +1,7 @@
 frappe.listview_settings["Route History"] = {
-	onload: function(listview) {
+	onload: function (listview) {
 		frappe.require("logtypes.bundle.js", () => {
 			frappe.utils.logtypes.show_log_retention_message(cur_list.doctype);
-		})
+		});
 	},
 };
diff --git a/frappe/desk/doctype/system_console/system_console.js b/frappe/desk/doctype/system_console/system_console.js
index 7751ffe860..dc73f33b67 100644
--- a/frappe/desk/doctype/system_console/system_console.js
+++ b/frappe/desk/doctype/system_console/system_console.js
@@ -1,21 +1,21 @@
 // Copyright (c) 2020, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('System Console', {
-	onload: function(frm) {
+frappe.ui.form.on("System Console", {
+	onload: function (frm) {
 		frappe.ui.keys.add_shortcut({
-			shortcut: 'shift+enter',
-			action: () => frm.page.btn_primary.trigger('click'),
+			shortcut: "shift+enter",
+			action: () => frm.page.btn_primary.trigger("click"),
 			page: frm.page,
-			description: __('Execute Console script'),
+			description: __("Execute Console script"),
 			ignore_inputs: true,
 		});
 		frm.set_value("type", "Python");
 	},
 
-	refresh: function(frm) {
+	refresh: function (frm) {
 		frm.disable_save();
-		frm.page.set_primary_action(__("Execute"), $btn => {
+		frm.page.set_primary_action(__("Execute"), ($btn) => {
 			$btn.text(__("Executing..."));
 			return frm
 				.execute_action("Execute")
@@ -24,7 +24,7 @@ frappe.ui.form.on('System Console', {
 		});
 	},
 
-	type: function(frm) {
+	type: function (frm) {
 		if (frm.doc.type == "Python") {
 			frm.set_value("output", "");
 			if (frm.sql_output) {
@@ -34,7 +34,7 @@ frappe.ui.form.on('System Console', {
 		}
 	},
 
-	render_sql_output: function(frm) {
+	render_sql_output: function (frm) {
 		if (frm.doc.type !== "SQL") return;
 		if (frm.sql_output) {
 			frm.sql_output.destroy();
@@ -46,50 +46,51 @@ frappe.ui.form.on('System Console', {
 		}
 
 		let result = JSON.parse(frm.doc.output);
-		frm.set_value("output", `${result.length} ${result.length == 1 ? 'row' : 'rows'}`);
+		frm.set_value("output", `${result.length} ${result.length == 1 ? "row" : "rows"}`);
 
 		if (result.length) {
 			let columns = Object.keys(result[0]);
-			frm.sql_output = new DataTable(
-				frm.get_field("sql_output").$wrapper.get(0),
-				{
-					columns,
-					data: result
-				}
-			);
+			frm.sql_output = new DataTable(frm.get_field("sql_output").$wrapper.get(0), {
+				columns,
+				data: result,
+			});
 		}
 	},
 
-	show_processlist: function(frm) {
+	show_processlist: function (frm) {
 		if (frm.doc.show_processlist) {
 			// keep refreshing every 5 seconds
 			frm.events.refresh_processlist(frm);
-			frm.processlist_interval = setInterval(() => frm.events.refresh_processlist(frm), 5000);
+			frm.processlist_interval = setInterval(
+				() => frm.events.refresh_processlist(frm),
+				5000
+			);
 		} else {
 			if (frm.processlist_interval) {
-
 				// end it
 				clearInterval(frm.processlist_interval);
-				frm.get_field("processlist").html('');
+				frm.get_field("processlist").html("");
 			}
 		}
 	},
 
-	refresh_processlist: function(frm) {
+	refresh_processlist: function (frm) {
 		let timestamp = new Date();
-		frappe.call('frappe.desk.doctype.system_console.system_console.show_processlist').then(r => {
-			let rows = '';
-			for (let row of r.message) {
-				rows += `
+		frappe
+			.call("frappe.desk.doctype.system_console.system_console.show_processlist")
+			.then((r) => {
+				let rows = "";
+				for (let row of r.message) {
+					rows += `
 					${row.Id}
 					${row.Time}
 					${row.State}
 					${row.Info}
 					${row.Progress}
-				`
-			}
+				`;
+				}
 
-			frm.get_field('processlist').html(`
+				frm.get_field("processlist").html(`
 				
    Requested on: ${timestamp}
    
 				
@@ -100,6 +101,6 @@ frappe.ui.form.on('System Console', {
 					${rows}`);
-		});
+			});
 	},
 });
diff --git a/frappe/desk/doctype/system_console/test_system_console.py b/frappe/desk/doctype/system_console/test_system_console.py
index 96bf555f59..2664f7c925 100644
--- a/frappe/desk/doctype/system_console/test_system_console.py
+++ b/frappe/desk/doctype/system_console/test_system_console.py
@@ -1,11 +1,10 @@
 # Copyright (c) 2020, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
-import unittest
-
 import frappe
+from frappe.tests.utils import FrappeTestCase
 
 
-class TestSystemConsole(unittest.TestCase):
+class TestSystemConsole(FrappeTestCase):
 	def test_system_console(self):
 		system_console = frappe.get_doc("System Console")
 		system_console.console = 'log("hello")'
diff --git a/frappe/desk/doctype/tag/tag.js b/frappe/desk/doctype/tag/tag.js
index f55f98c3d0..1c60f417e0 100644
--- a/frappe/desk/doctype/tag/tag.js
+++ b/frappe/desk/doctype/tag/tag.js
@@ -1,8 +1,7 @@
 // Copyright (c) 2019, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Tag', {
+frappe.ui.form.on("Tag", {
 	// refresh: function(frm) {
-
 	// }
 });
diff --git a/frappe/desk/doctype/tag/tag.py b/frappe/desk/doctype/tag/tag.py
index ca167c148e..c5fe6407b7 100644
--- a/frappe/desk/doctype/tag/tag.py
+++ b/frappe/desk/doctype/tag/tag.py
@@ -59,7 +59,7 @@ def get_tags(doctype, txt):
 	tag = frappe.get_list("Tag", filters=[["name", "like", f"%{txt}%"]])
 	tags = [t.name for t in tag]
 
-	return sorted(filter(lambda t: t and txt.lower() in t.lower(), list(set(tags))))
+	return sorted(filter(lambda t: t and txt.casefold() in t.casefold(), list(set(tags))))
 
 
 class DocTags:
@@ -192,4 +192,4 @@ def get_documents_for_tag(tag):
 
 @frappe.whitelist()
 def get_tags_list_for_awesomebar():
-	return [t.name for t in frappe.get_list("Tag")]
+	return frappe.get_list("Tag", pluck="name", order_by=None)
diff --git a/frappe/desk/doctype/tag/test_tag.py b/frappe/desk/doctype/tag/test_tag.py
index 8719da8c21..0c746e67ac 100644
--- a/frappe/desk/doctype/tag/test_tag.py
+++ b/frappe/desk/doctype/tag/test_tag.py
@@ -1,11 +1,10 @@
-import unittest
-
 import frappe
 from frappe.desk.doctype.tag.tag import add_tag
 from frappe.desk.reportview import get_stats
+from frappe.tests.utils import FrappeTestCase
 
 
-class TestTag(unittest.TestCase):
+class TestTag(FrappeTestCase):
 	def setUp(self) -> None:
 		frappe.db.delete("Tag")
 		frappe.db.sql("UPDATE `tabDocType` set _user_tags=''")
diff --git a/frappe/desk/doctype/tag_link/tag_link.js b/frappe/desk/doctype/tag_link/tag_link.js
index d85655bb90..e2cb4fcd7f 100644
--- a/frappe/desk/doctype/tag_link/tag_link.js
+++ b/frappe/desk/doctype/tag_link/tag_link.js
@@ -1,8 +1,7 @@
 // Copyright (c) 2019, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Tag Link', {
+frappe.ui.form.on("Tag Link", {
 	// refresh: function(frm) {
-
 	// }
 });
diff --git a/frappe/desk/doctype/tag_link/test_tag_link.py b/frappe/desk/doctype/tag_link/test_tag_link.py
index 59d7bcd2bc..10edb2859e 100644
--- a/frappe/desk/doctype/tag_link/test_tag_link.py
+++ b/frappe/desk/doctype/tag_link/test_tag_link.py
@@ -1,8 +1,8 @@
 # Copyright (c) 2019, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
 # import frappe
-import unittest
+from frappe.tests.utils import FrappeTestCase
 
 
-class TestTagLink(unittest.TestCase):
+class TestTagLink(FrappeTestCase):
 	pass
diff --git a/frappe/desk/doctype/todo/test_todo.py b/frappe/desk/doctype/todo/test_todo.py
index 56ca1f30e7..4880fad9d5 100644
--- a/frappe/desk/doctype/todo/test_todo.py
+++ b/frappe/desk/doctype/todo/test_todo.py
@@ -1,16 +1,15 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
-import unittest
-
 import frappe
 from frappe.core.doctype.doctype.doctype import clear_permissions_cache
 from frappe.model.db_query import DatabaseQuery
 from frappe.permissions import add_permission, reset_perms
+from frappe.tests.utils import FrappeTestCase
 
 test_dependencies = ["User"]
 
 
-class TestToDo(unittest.TestCase):
+class TestToDo(FrappeTestCase):
 	def test_delete(self):
 		todo = frappe.get_doc(
 			dict(doctype="ToDo", description="test todo", assigned_by="Administrator")
diff --git a/frappe/desk/doctype/todo/todo.js b/frappe/desk/doctype/todo/todo.js
index 0317281371..6c1af67d2b 100644
--- a/frappe/desk/doctype/todo/todo.js
+++ b/frappe/desk/doctype/todo/todo.js
@@ -1,40 +1,55 @@
 // bind events
 
 frappe.ui.form.on("ToDo", {
-	onload: function(frm) {
-		frm.set_query("reference_type", function(txt) {
+	onload: function (frm) {
+		frm.set_query("reference_type", function (txt) {
 			return {
-				"filters": {
-					"issingle": 0,
-				}
+				filters: {
+					issingle: 0,
+				},
 			};
 		});
 	},
-	refresh: function(frm) {
-		if(frm.doc.reference_type && frm.doc.reference_name) {
-			frm.add_custom_button(__(frm.doc.reference_name), function() {
+	refresh: function (frm) {
+		if (frm.doc.reference_type && frm.doc.reference_name) {
+			frm.add_custom_button(__(frm.doc.reference_name), function () {
 				frappe.set_route("Form", frm.doc.reference_type, frm.doc.reference_name);
 			});
 		}
 
 		if (!frm.doc.__islocal) {
-			if(frm.doc.status!=="Closed") {
-				frm.add_custom_button(__("Close"), function() {
-					frm.set_value("status", "Closed");
-					frm.save(null, function() {
-						// back to list
-						frappe.set_route("List", "ToDo");
-					});
-				}, "fa fa-check", "btn-success");
+			if (frm.doc.status !== "Closed") {
+				frm.add_custom_button(
+					__("Close"),
+					function () {
+						frm.set_value("status", "Closed");
+						frm.save(null, function () {
+							// back to list
+							frappe.set_route("List", "ToDo");
+						});
+					},
+					"fa fa-check",
+					"btn-success"
+				);
 			} else {
-				frm.add_custom_button(__("Reopen"), function() {
-					frm.set_value("status", "Open");
-					frm.save();
-				}, null, "btn-default");
+				frm.add_custom_button(
+					__("Reopen"),
+					function () {
+						frm.set_value("status", "Open");
+						frm.save();
+					},
+					null,
+					"btn-default"
+				);
 			}
-			frm.add_custom_button(__("New"), function() {
-				frappe.new_doc("ToDo")
-			}, null, "btn-default");
+			frm.add_custom_button(
+				__("New"),
+				function () {
+					frappe.new_doc("ToDo");
+				},
+				null,
+				"btn-default"
+			);
 		}
-	}
+	},
 });
diff --git a/frappe/desk/doctype/todo/todo_calendar.js b/frappe/desk/doctype/todo/todo_calendar.js
index 8ba020fac1..f79243a86e 100644
--- a/frappe/desk/doctype/todo/todo_calendar.js
+++ b/frappe/desk/doctype/todo/todo_calendar.js
@@ -3,29 +3,27 @@
 
 frappe.views.calendar["ToDo"] = {
 	field_map: {
-		"start": "date",
-		"end": "date",
-		"id": "name",
-		"title": "description",
-		"allDay": "allDay",
-		"progress": "progress"
+		start: "date",
+		end: "date",
+		id: "name",
+		title: "description",
+		allDay: "allDay",
+		progress: "progress",
 	},
 	gantt: true,
 	filters: [
 		{
-			"fieldtype": "Link",
-			"fieldname": "reference_type",
-			"options": "Task",
-			"label": __("Task")
+			fieldtype: "Link",
+			fieldname: "reference_type",
+			options: "Task",
+			label: __("Task"),
 		},
 		{
-			"fieldtype": "Dynamic Link",
-			"fieldname": "reference_name",
-			"options": "reference_type",
-			"label": __("Task")
-		}
-
+			fieldtype: "Dynamic Link",
+			fieldname: "reference_name",
+			options: "reference_type",
+			label: __("Task"),
+		},
 	],
-	get_events_method: "frappe.desk.calendar.get_events"
+	get_events_method: "frappe.desk.calendar.get_events",
 };
-
diff --git a/frappe/desk/doctype/todo/todo_list.js b/frappe/desk/doctype/todo/todo_list.js
index 53564cc017..bf62088f1d 100644
--- a/frappe/desk/doctype/todo/todo_list.js
+++ b/frappe/desk/doctype/todo/todo_list.js
@@ -1,40 +1,44 @@
-frappe.listview_settings['ToDo'] = {
+frappe.listview_settings["ToDo"] = {
 	hide_name_column: true,
 	add_fields: ["reference_type", "reference_name"],
 
-	onload: function(me) {
+	onload: function (me) {
 		if (!frappe.route_options) {
 			frappe.route_options = {
-				"owner": frappe.session.user,
-				"status": "Open"
+				owner: frappe.session.user,
+				status: "Open",
 			};
 		}
 		me.page.set_title(__("To Do"));
 	},
 
 	button: {
-		show: function(doc) {
+		show: function (doc) {
 			return doc.reference_name;
 		},
-		get_label: function() {
-			return __('Open');
+		get_label: function () {
+			return __("Open", null, "Access");
 		},
-		get_description: function(doc) {
-			return __('Open {0}', [`${doc.reference_type} ${doc.reference_name}`])
+		get_description: function (doc) {
+			return __("Open {0}", [`${__(doc.reference_type)}: ${doc.reference_name}`]);
+		},
+		action: function (doc) {
+			frappe.set_route("Form", doc.reference_type, doc.reference_name);
 		},
-		action: function(doc) {
-			frappe.set_route('Form', doc.reference_type, doc.reference_name);
-		}
 	},
 
-	refresh: function(me) {
+	refresh: function (me) {
 		if (me.todo_sidebar_setup) return;
 
 		// add assigned by me
-		me.page.add_sidebar_item(__("Assigned By Me"), function() {
-			me.filter_area.add([[me.doctype, "assigned_by", '=', frappe.session.user]]);
-		}, ('.list-link[data-view="Kanban"]'));
+		me.page.add_sidebar_item(
+			__("Assigned By Me"),
+			function () {
+				me.filter_area.add([[me.doctype, "assigned_by", "=", frappe.session.user]]);
+			},
+			'.list-link[data-view="Kanban"]'
+		);
 
 		me.todo_sidebar_setup = true;
 	},
-}
\ No newline at end of file
+};
diff --git a/frappe/desk/doctype/workspace/test_workspace.py b/frappe/desk/doctype/workspace/test_workspace.py
index d0b0eba9e2..22665e1dd2 100644
--- a/frappe/desk/doctype/workspace/test_workspace.py
+++ b/frappe/desk/doctype/workspace/test_workspace.py
@@ -1,11 +1,10 @@
 # Copyright (c) 2020, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
-import unittest
-
 import frappe
+from frappe.tests.utils import FrappeTestCase
 
 
-class TestWorkspace(unittest.TestCase):
+class TestWorkspace(FrappeTestCase):
 	def setUp(self):
 		create_module("Test Module")
 
diff --git a/frappe/desk/doctype/workspace/workspace.js b/frappe/desk/doctype/workspace/workspace.js
index 3f912127fc..b5f298c477 100644
--- a/frappe/desk/doctype/workspace/workspace.js
+++ b/frappe/desk/doctype/workspace/workspace.js
@@ -1,26 +1,59 @@
 // Copyright (c) 2020, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Workspace', {
-	setup: function() {
-		frappe.meta.get_field('Workspace Link', 'only_for').no_default = true;
+frappe.ui.form.on("Workspace", {
+	setup: function () {
+		frappe.meta.get_field("Workspace Link", "only_for").no_default = true;
 	},
 
-	refresh: function(frm) {
+	refresh: function (frm) {
 		frm.enable_save();
 
-		if (frm.doc.for_user || (frm.doc.public && !frm.has_perm('write') &&
-			!frappe.user.has_role('Workspace Manager'))) {
-			frm.trigger('disable_form');
+		let url = `/app/${
+			frm.doc.public
+				? frappe.router.slug(frm.doc.title)
+				: "private/" + frappe.router.slug(frm.doc.title)
+		}`;
+		frm.sidebar
+			.add_user_action(__("Go to Workspace"))
+			.attr("href", url)
+			.attr("target", "_blank");
+
+		frm.layout.message.empty();
+		let message = __(
+			"This document allows you to edit limited fields. For all kinds of workspace customization, use the Edit button located on the workspace page"
+		);
+
+		if (
+			frm.doc.for_user ||
+			(frm.doc.public &&
+				!frm.has_perm("write") &&
+				!frappe.user.has_role("Workspace Manager"))
+		) {
+			frm.trigger("disable_form");
+
+			if (frm.doc.public) {
+				message = __("Only Workspace Manager can edit public workspaces");
+			} else {
+				message = __(
+					"We do not allow editing of this document. Simply click the Edit button on the workspace page to make your workspace editable and customize it as you wish"
+				);
+			}
 		}
+
+		if (frappe.boot.developer_mode) {
+			frm.set_df_property("module", "read_only", 0);
+		}
+
+		frm.layout.show_message(message);
 	},
 
-	disable_form: function(frm) {
+	disable_form: function (frm) {
 		frm.fields
-			.filter(field => field.has_input)
-			.forEach(field => {
+			.filter((field) => field.has_input)
+			.forEach((field) => {
 				frm.set_df_property(field.df.fieldname, "read_only", "1");
 			});
 		frm.disable_save();
-	}
-});
\ No newline at end of file
+	},
+});
diff --git a/frappe/desk/doctype/workspace/workspace.json b/frappe/desk/doctype/workspace/workspace.json
index 032de9de4e..2759acd228 100644
--- a/frappe/desk/doctype/workspace/workspace.json
+++ b/frappe/desk/doctype/workspace/workspace.json
@@ -19,7 +19,10 @@
   "restrict_to_domain",
   "hide_custom",
   "public",
+  "is_hidden",
   "content",
+  "number_cards_tab",
+  "number_cards",
   "tab_break_2",
   "charts",
   "tab_break_15",
@@ -28,6 +31,8 @@
   "links",
   "quick_lists_tab",
   "quick_lists",
+  "custom_blocks_tab",
+  "custom_blocks",
   "roles_tab",
   "roles"
  ],
@@ -71,7 +76,8 @@
    "in_list_view": 1,
    "in_standard_filter": 1,
    "label": "Module",
-   "options": "Module Def"
+   "options": "Module Def",
+   "read_only": 1
   },
   {
    "fieldname": "column_break_3",
@@ -107,7 +113,8 @@
   {
    "fieldname": "icon",
    "fieldtype": "Icon",
-   "label": "Icon"
+   "label": "Icon",
+   "read_only": 1
   },
   {
    "fieldname": "links",
@@ -122,18 +129,21 @@
    "in_list_view": 1,
    "in_standard_filter": 1,
    "label": "Public",
+   "read_only": 1,
    "search_index": 1
   },
   {
    "fieldname": "title",
    "fieldtype": "Data",
    "label": "Title",
+   "read_only": 1,
    "reqd": 1
   },
   {
    "fieldname": "parent_page",
    "fieldtype": "Data",
-   "label": "Parent Page"
+   "label": "Parent Page",
+   "read_only": 1
   },
   {
    "default": "[]",
@@ -145,7 +155,8 @@
   {
    "fieldname": "sequence_id",
    "fieldtype": "Float",
-   "label": "Sequence Id"
+   "label": "Sequence Id",
+   "read_only": 1
   },
   {
    "fieldname": "roles",
@@ -168,11 +179,39 @@
    "fieldtype": "Table",
    "label": "Quick Lists",
    "options": "Workspace Quick List"
+  },
+  {
+   "default": "0",
+   "fieldname": "is_hidden",
+   "fieldtype": "Check",
+   "label": "Is Hidden"
+  },
+  {
+   "fieldname": "number_cards_tab",
+   "fieldtype": "Tab Break",
+   "label": "Number Cards"
+  },
+  {
+   "fieldname": "number_cards",
+   "fieldtype": "Table",
+   "label": "Number Cards",
+   "options": "Workspace Number Card"
+  },
+  {
+   "fieldname": "custom_blocks_tab",
+   "fieldtype": "Tab Break",
+   "label": "Custom Blocks"
+  },
+  {
+   "fieldname": "custom_blocks",
+   "fieldtype": "Table",
+   "label": "Custom Blocks",
+   "options": "Workspace Custom Block"
   }
  ],
  "in_create": 1,
  "links": [],
- "modified": "2022-05-12 13:00:03.925605",
+ "modified": "2023-05-17 14:52:38.110224",
  "modified_by": "Administrator",
  "module": "Desk",
  "name": "Workspace",
@@ -190,18 +229,10 @@
    "role": "Workspace Manager",
    "share": 1,
    "write": 1
-  },
-  {
-   "email": 1,
-   "export": 1,
-   "print": 1,
-   "read": 1,
-   "report": 1,
-   "role": "All",
-   "share": 1
   }
  ],
  "sort_field": "modified",
  "sort_order": "DESC",
- "states": []
+ "states": [],
+ "track_changes": 1
 }
\ No newline at end of file
diff --git a/frappe/desk/doctype/workspace/workspace.py b/frappe/desk/doctype/workspace/workspace.py
index 9fa99884fb..7b5970a7d6 100644
--- a/frappe/desk/doctype/workspace/workspace.py
+++ b/frappe/desk/doctype/workspace/workspace.py
@@ -1,6 +1,7 @@
 # Copyright (c) 2020, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
+from collections import defaultdict
 from json import loads
 
 import frappe
@@ -9,14 +10,17 @@ from frappe.desk.desktop import save_new_widget
 from frappe.desk.utils import validate_route_conflict
 from frappe.model.document import Document
 from frappe.model.rename_doc import rename_doc
-from frappe.modules.export_file import export_to_files
+from frappe.modules.export_file import delete_folder, export_to_files
 
 
 class Workspace(Document):
 	def validate(self):
 		if self.public and not is_workspace_manager() and not disable_saving_as_public():
 			frappe.throw(_("You need to be Workspace Manager to edit this document"))
-		validate_route_conflict(self.doctype, self.name)
+		if self.has_value_changed("title"):
+			validate_route_conflict(self.doctype, self.title)
+		else:
+			validate_route_conflict(self.doctype, self.name)
 
 		try:
 			if not isinstance(loads(self.content), list):
@@ -28,16 +32,43 @@ class Workspace(Document):
 		if disable_saving_as_public():
 			return
 
-		if frappe.conf.developer_mode and self.module and self.public:
-			export_to_files(record_list=[["Workspace", self.name]], record_module=self.module)
+		if frappe.conf.developer_mode and self.public:
+			if self.module:
+				export_to_files(record_list=[["Workspace", self.name]], record_module=self.module)
+
+			if self.has_value_changed("title") or self.has_value_changed("module"):
+				previous = self.get_doc_before_save()
+				if previous and previous.get("module") and previous.get("title"):
+					delete_folder(previous.get("module"), "Workspace", previous.get("title"))
+
+	def before_export(self, doc):
+		if doc.title != doc.label and doc.label == doc.name:
+			self.name = doc.name = doc.label = doc.title
+
+	def after_delete(self):
+		if disable_saving_as_public():
+			return
+
+		if self.module and frappe.conf.developer_mode:
+			delete_folder(self.module, "Workspace", self.title)
 
 	@staticmethod
-	def get_module_page_map():
-		pages = frappe.get_all(
-			"Workspace", fields=["name", "module"], filters={"for_user": ""}, as_list=1
+	def get_module_wise_workspaces():
+		workspaces = frappe.get_all(
+			"Workspace",
+			fields=["name", "module"],
+			filters={"for_user": "", "public": 1},
+			order_by="creation",
 		)
 
-		return {page[1]: page[0] for page in pages if page[1]}
+		module_workspaces = defaultdict(list)
+
+		for workspace in workspaces:
+			if not workspace.module:
+				continue
+			module_workspaces[workspace.module].append(workspace.name)
+
+		return module_workspaces
 
 	def get_link_groups(self):
 		cards = []
@@ -121,6 +152,7 @@ class Workspace(Document):
 def disable_saving_as_public():
 	return (
 		frappe.flags.in_install
+		or frappe.flags.in_uninstall
 		or frappe.flags.in_patch
 		or frappe.flags.in_test
 		or frappe.flags.in_fixtures
@@ -176,7 +208,7 @@ def save_page(title, public, new_widgets, blocks):
 
 	if not public:
 		filters = {"for_user": frappe.session.user, "label": title + "-" + frappe.session.user}
-	pages = frappe.get_list("Workspace", filters=filters)
+	pages = frappe.get_all("Workspace", filters=filters)
 	if pages:
 		doc = frappe.get_doc("Workspace", pages[0])
 
@@ -191,12 +223,8 @@ def save_page(title, public, new_widgets, blocks):
 @frappe.whitelist()
 def update_page(name, title, icon, parent, public):
 	public = frappe.parse_json(public)
-
 	doc = frappe.get_doc("Workspace", name)
 
-	filters = {"parent_page": doc.title, "public": doc.public}
-	child_docs = frappe.get_list("Workspace", filters=filters)
-
 	if doc:
 		doc.title = title
 		doc.icon = icon
@@ -212,6 +240,9 @@ def update_page(name, title, icon, parent, public):
 			rename_doc("Workspace", name, new_name, force=True, ignore_permissions=True)
 
 		# update new name and public in child pages
+		child_docs = frappe.get_all(
+			"Workspace", filters={"parent_page": doc.title, "public": doc.public}
+		)
 		if child_docs:
 			for child in child_docs:
 				child_doc = frappe.get_doc("Workspace", child.name)
@@ -230,6 +261,32 @@ def update_page(name, title, icon, parent, public):
 	return {"name": title, "public": public, "label": new_name}
 
 
+def hide_unhide_page(page_name: str, is_hidden: bool):
+	page = frappe.get_doc("Workspace", page_name)
+
+	if page.get("public") and not is_workspace_manager():
+		frappe.throw(
+			_("Need Workspace Manager role to hide/unhide public workspaces"), frappe.PermissionError
+		)
+
+	if not page.get("public") and page.get("for_user") != frappe.session.user:
+		frappe.throw(_("Cannot update private workspace of other users"), frappe.PermissionError)
+
+	page.is_hidden = int(is_hidden)
+	page.save(ignore_permissions=True)
+	return True
+
+
+@frappe.whitelist()
+def hide_page(page_name: str):
+	return hide_unhide_page(page_name, 1)
+
+
+@frappe.whitelist()
+def unhide_page(page_name: str):
+	return hide_unhide_page(page_name, 0)
+
+
 @frappe.whitelist()
 def duplicate_page(page_name, new_page):
 	if not loads(new_page):
@@ -248,6 +305,7 @@ def duplicate_page(page_name, new_page):
 	doc.public = new_page.get("is_public")
 	doc.for_user = ""
 	doc.label = doc.title
+	doc.module = ""
 	if not doc.public:
 		doc.for_user = doc.for_user or frappe.session.user
 		doc.label = f"{doc.title}-{doc.for_user}"
@@ -319,7 +377,7 @@ def last_sequence_id(doc):
 	if not doc_exists:
 		return 0
 
-	return frappe.db.get_list(
+	return frappe.get_all(
 		"Workspace",
 		fields=["sequence_id"],
 		filters={"public": doc.public, "for_user": doc.for_user},
@@ -328,7 +386,7 @@ def last_sequence_id(doc):
 
 
 def get_page_list(fields, filters):
-	return frappe.get_list("Workspace", fields=fields, filters=filters, order_by="sequence_id asc")
+	return frappe.get_all("Workspace", fields=fields, filters=filters, order_by="sequence_id asc")
 
 
 def is_workspace_manager():
diff --git a/frappe/event_streaming/doctype/__init__.py b/frappe/desk/doctype/workspace_custom_block/__init__.py
similarity index 100%
rename from frappe/event_streaming/doctype/__init__.py
rename to frappe/desk/doctype/workspace_custom_block/__init__.py
diff --git a/frappe/desk/doctype/workspace_custom_block/workspace_custom_block.json b/frappe/desk/doctype/workspace_custom_block/workspace_custom_block.json
new file mode 100644
index 0000000000..090719de3a
--- /dev/null
+++ b/frappe/desk/doctype/workspace_custom_block/workspace_custom_block.json
@@ -0,0 +1,39 @@
+{
+ "actions": [],
+ "allow_rename": 1,
+ "creation": "2023-05-17 14:49:19.454932",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "custom_block_name",
+  "label"
+ ],
+ "fields": [
+  {
+   "fieldname": "custom_block_name",
+   "fieldtype": "Link",
+   "in_list_view": 1,
+   "label": "Custom Block Name",
+   "options": "Custom HTML Block"
+  },
+  {
+   "fieldname": "label",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Label"
+  }
+ ],
+ "index_web_pages_for_search": 1,
+ "istable": 1,
+ "links": [],
+ "modified": "2023-05-17 14:50:45.575609",
+ "modified_by": "Administrator",
+ "module": "Desk",
+ "name": "Workspace Custom Block",
+ "owner": "Administrator",
+ "permissions": [],
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "states": []
+}
\ No newline at end of file
diff --git a/frappe/desk/doctype/workspace_custom_block/workspace_custom_block.py b/frappe/desk/doctype/workspace_custom_block/workspace_custom_block.py
new file mode 100644
index 0000000000..745844f5c1
--- /dev/null
+++ b/frappe/desk/doctype/workspace_custom_block/workspace_custom_block.py
@@ -0,0 +1,9 @@
+# Copyright (c) 2023, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+# import frappe
+from frappe.model.document import Document
+
+
+class WorkspaceCustomBlock(Document):
+	pass
diff --git a/frappe/event_streaming/doctype/document_type_field_mapping/__init__.py b/frappe/desk/doctype/workspace_number_card/__init__.py
similarity index 100%
rename from frappe/event_streaming/doctype/document_type_field_mapping/__init__.py
rename to frappe/desk/doctype/workspace_number_card/__init__.py
diff --git a/frappe/desk/doctype/workspace_number_card/workspace_number_card.json b/frappe/desk/doctype/workspace_number_card/workspace_number_card.json
new file mode 100644
index 0000000000..f9e3865d74
--- /dev/null
+++ b/frappe/desk/doctype/workspace_number_card/workspace_number_card.json
@@ -0,0 +1,40 @@
+{
+ "actions": [],
+ "allow_rename": 1,
+ "creation": "2023-02-15 01:16:26.216201",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "number_card_name",
+  "label"
+ ],
+ "fields": [
+  {
+   "fieldname": "number_card_name",
+   "fieldtype": "Link",
+   "in_list_view": 1,
+   "label": "Number Card Name",
+   "options": "Number Card",
+   "reqd": 1
+  },
+  {
+   "fieldname": "label",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Label"
+  }
+ ],
+ "index_web_pages_for_search": 1,
+ "istable": 1,
+ "links": [],
+ "modified": "2023-02-15 01:16:26.216201",
+ "modified_by": "Administrator",
+ "module": "Desk",
+ "name": "Workspace Number Card",
+ "owner": "Administrator",
+ "permissions": [],
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "states": []
+}
\ No newline at end of file
diff --git a/frappe/desk/doctype/workspace_number_card/workspace_number_card.py b/frappe/desk/doctype/workspace_number_card/workspace_number_card.py
new file mode 100644
index 0000000000..e972f3f525
--- /dev/null
+++ b/frappe/desk/doctype/workspace_number_card/workspace_number_card.py
@@ -0,0 +1,9 @@
+# Copyright (c) 2023, Frappe Technologies and contributors
+# For license information, please see license.txt
+
+# import frappe
+from frappe.model.document import Document
+
+
+class WorkspaceNumberCard(Document):
+	pass
diff --git a/frappe/desk/doctype/workspace_shortcut/workspace_shortcut.json b/frappe/desk/doctype/workspace_shortcut/workspace_shortcut.json
index 8673e93cf7..8832d9e1f4 100644
--- a/frappe/desk/doctype/workspace_shortcut/workspace_shortcut.json
+++ b/frappe/desk/doctype/workspace_shortcut/workspace_shortcut.json
@@ -7,6 +7,7 @@
  "field_order": [
   "type",
   "link_to",
+  "url",
   "doc_view",
   "column_break_4",
   "label",
@@ -24,16 +25,16 @@
    "fieldtype": "Select",
    "in_list_view": 1,
    "label": "Type",
-   "options": "DocType\nReport\nPage\nDashboard",
+   "options": "DocType\nReport\nPage\nDashboard\nURL",
    "reqd": 1
   },
   {
+   "depends_on": "eval:doc.type != \"URL\"",
    "fieldname": "link_to",
    "fieldtype": "Dynamic Link",
    "in_list_view": 1,
    "label": "Link To",
-   "options": "type",
-   "reqd": 1
+   "options": "type"
   },
   {
    "depends_on": "eval:doc.type == \"DocType\"",
@@ -94,12 +95,20 @@
    "fieldname": "format",
    "fieldtype": "Data",
    "label": "Format"
+  },
+  {
+   "depends_on": "eval:doc.type == \"URL\"",
+   "fieldname": "url",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "URL",
+   "options": "URL"
   }
  ],
  "index_web_pages_for_search": 1,
  "istable": 1,
  "links": [],
- "modified": "2021-01-12 13:13:17.571324",
+ "modified": "2023-04-19 13:32:31.005443",
  "modified_by": "Administrator",
  "module": "Desk",
  "name": "Workspace Shortcut",
diff --git a/frappe/desk/form/assign_to.py b/frappe/desk/form/assign_to.py
index 7853e807b8..ce8bb444a1 100644
--- a/frappe/desk/form/assign_to.py
+++ b/frappe/desk/form/assign_to.py
@@ -93,10 +93,17 @@ def add(args=None):
 
 			doc = frappe.get_doc(args["doctype"], args["name"])
 
-			# if assignee does not have permissions, share
+			# if assignee does not have permissions, share or inform
 			if not frappe.has_permission(doc=doc, user=assign_to):
-				frappe.share.add(doc.doctype, doc.name, assign_to)
-				shared_with_users.append(assign_to)
+				if frappe.get_system_settings("disable_document_sharing"):
+					msg = _("User {0} is not permitted to access this document.").format(frappe.bold(assign_to))
+					msg += "
    " + _(
+						"As document sharing is disabled, please give them the required permissions before assigning."
+					)
+					frappe.throw(msg, title=_("Missing Permission"))
+				else:
+					frappe.share.add(doc.doctype, doc.name, assign_to)
+					shared_with_users.append(assign_to)
 
 			# make this document followed by assigned user
 			if frappe.get_cached_value("User", assign_to, "follow_assigned_documents"):
@@ -138,37 +145,38 @@ def add_multiple(args=None):
 
 
 def close_all_assignments(doctype, name):
-	assignments = frappe.db.get_all(
+	assignments = frappe.get_all(
 		"ToDo",
-		fields=["allocated_to"],
+		fields=["allocated_to", "name"],
 		filters=dict(reference_type=doctype, reference_name=name, status=("!=", "Cancelled")),
 	)
 	if not assignments:
 		return False
 
 	for assign_to in assignments:
-		set_status(doctype, name, assign_to.allocated_to, status="Closed")
+		set_status(doctype, name, todo=assign_to.name, assign_to=assign_to.allocated_to, status="Closed")
 
 	return True
 
 
 @frappe.whitelist()
 def remove(doctype, name, assign_to):
-	return set_status(doctype, name, assign_to, status="Cancelled")
+	return set_status(doctype, name, "", assign_to, status="Cancelled")
 
 
-def set_status(doctype, name, assign_to, status="Cancelled"):
+def set_status(doctype, name, todo=None, assign_to=None, status="Cancelled"):
 	"""remove from todo"""
 	try:
-		todo = frappe.db.get_value(
-			"ToDo",
-			{
-				"reference_type": doctype,
-				"reference_name": name,
-				"allocated_to": assign_to,
-				"status": ("!=", status),
-			},
-		)
+		if not todo:
+			todo = frappe.db.get_value(
+				"ToDo",
+				{
+					"reference_type": doctype,
+					"reference_name": name,
+					"allocated_to": assign_to,
+					"status": ("!=", status),
+				},
+			)
 		if todo:
 			todo = frappe.get_doc("ToDo", todo)
 			todo.status = status
@@ -189,14 +197,18 @@ def clear(doctype, name):
 	"""
 	Clears assignments, return False if not assigned.
 	"""
-	assignments = frappe.db.get_all(
-		"ToDo", fields=["allocated_to"], filters=dict(reference_type=doctype, reference_name=name)
+	assignments = frappe.get_all(
+		"ToDo",
+		fields=["allocated_to", "name"],
+		filters=dict(reference_type=doctype, reference_name=name),
 	)
 	if not assignments:
 		return False
 
 	for assign_to in assignments:
-		set_status(doctype, name, assign_to.allocated_to, "Cancelled")
+		set_status(
+			doctype, name, todo=assign_to.name, assign_to=assign_to.allocated_to, status="Cancelled"
+		)
 
 	return True
 
diff --git a/frappe/desk/form/document_follow.py b/frappe/desk/form/document_follow.py
index 2e4bcedf5a..f12e44fe61 100644
--- a/frappe/desk/form/document_follow.py
+++ b/frappe/desk/form/document_follow.py
@@ -161,9 +161,14 @@ def get_document_followed_by_user(user):
 
 def get_version(doctype, doc_name, frequency, user):
 	timeline = []
-	filters = get_filters("docname", doc_name, frequency, user)
 	version = frappe.get_all(
-		"Version", filters=filters, fields=["ref_doctype", "data", "modified", "modified", "modified_by"]
+		"Version",
+		filters=[
+			["ref_doctype", "=", doctype],
+			["docname", "=", doc_name],
+			*_get_filters(frequency, user),
+		],
+		fields=["data", "modified", "modified_by"],
 	)
 	if version:
 		for v in version:
@@ -186,9 +191,14 @@ def get_comments(doctype, doc_name, frequency, user):
 	from frappe.core.utils import html2text
 
 	timeline = []
-	filters = get_filters("reference_name", doc_name, frequency, user)
 	comments = frappe.get_all(
-		"Comment", filters=filters, fields=["content", "modified", "modified_by", "comment_type"]
+		"Comment",
+		filters=[
+			["reference_doctype", "=", doctype],
+			["reference_name", "=", doc_name],
+			*_get_filters(frequency, user),
+		],
+		fields=["content", "modified", "modified_by", "comment_type"],
 	)
 	for comment in comments:
 		if comment.comment_type == "Like":
@@ -306,29 +316,27 @@ def send_weekly_updates():
 	send_document_follow_mails("Weekly")
 
 
-def get_filters(search_by, name, frequency, user):
-	filters = []
+def _get_filters(frequency, user):
+	filters = [
+		["modified_by", "!=", user],
+	]
 
 	if frequency == "Weekly":
-		filters = [
-			[search_by, "=", name],
+		filters += [
 			["modified", ">", frappe.utils.add_days(frappe.utils.nowdate(), -7)],
 			["modified", "<", frappe.utils.nowdate()],
-			["modified_by", "!=", user],
 		]
+
 	elif frequency == "Daily":
-		filters = [
-			[search_by, "=", name],
+		filters += [
 			["modified", ">", frappe.utils.add_days(frappe.utils.nowdate(), -1)],
 			["modified", "<", frappe.utils.nowdate()],
-			["modified_by", "!=", user],
 		]
+
 	elif frequency == "Hourly":
-		filters = [
-			[search_by, "=", name],
+		filters += [
 			["modified", ">", frappe.utils.add_to_date(frappe.utils.now_datetime(), hours=-1)],
 			["modified", "<", frappe.utils.now_datetime()],
-			["modified_by", "!=", user],
 		]
 
 	return filters
diff --git a/frappe/desk/form/linked_with.py b/frappe/desk/form/linked_with.py
index b60c11774f..9bc7b138dd 100644
--- a/frappe/desk/form/linked_with.py
+++ b/frappe/desk/form/linked_with.py
@@ -136,7 +136,9 @@ class SubmittableDocumentTree:
 	def get_submittable_doctypes(self) -> list[str]:
 		"""Returns list of submittable doctypes."""
 		if not self._submittable_doctypes:
-			self._submittable_doctypes = frappe.db.get_all("DocType", {"is_submittable": 1}, pluck="name")
+			self._submittable_doctypes = frappe.get_all(
+				"DocType", {"is_submittable": 1}, pluck="name", order_by=None
+			)
 		return self._submittable_doctypes
 
 
@@ -155,6 +157,7 @@ def get_child_tables_of_doctypes(doctypes: list[str] = None):
 		fields=["parent", "fieldname", "options as child_table"],
 		filters=filters_for_docfield,
 		as_list=1,
+		order_by=None,
 	)
 
 	links += frappe.get_all(
@@ -162,6 +165,7 @@ def get_child_tables_of_doctypes(doctypes: list[str] = None):
 		fields=["dt as parent", "fieldname", "options as child_table"],
 		filters=filters_for_customfield,
 		as_list=1,
+		order_by=None,
 	)
 
 	child_tables_by_doctype = defaultdict(list)
@@ -275,6 +279,7 @@ def get_references_across_doctypes_by_dynamic_link_field(
 		fields=["parent as doctype", "fieldname", "options as doctype_fieldname"],
 		filters=filters_for_docfield,
 		as_list=1,
+		order_by=None,
 	)
 
 	links += frappe.get_all(
@@ -282,15 +287,14 @@ def get_references_across_doctypes_by_dynamic_link_field(
 		fields=["dt as doctype", "fieldname", "options as doctype_fieldname"],
 		filters=filters_for_customfield,
 		as_list=1,
+		order_by=None,
 	)
 
 	links_by_doctype = defaultdict(list)
 	for doctype, fieldname, doctype_fieldname in links:
 		try:
 			filters = [[doctype_fieldname, "in", to_doctypes]] if to_doctypes else []
-			for linked_to in frappe.db.get_all(
-				doctype, pluck=doctype_fieldname, filters=filters, distinct=1
-			):
+			for linked_to in frappe.get_all(doctype, pluck=doctype_fieldname, filters=filters, distinct=1):
 				if linked_to:
 					links_by_doctype[linked_to].append(
 						{"doctype": doctype, "fieldname": fieldname, "doctype_fieldname": doctype_fieldname}
@@ -330,17 +334,21 @@ def get_referencing_documents(
 
 	if not link_info.get("is_child"):
 		filters.extend(parent_filters or [])
-		return {from_table: frappe.db.get_all(from_table, filters, pluck="name")}
+		return {from_table: frappe.get_all(from_table, filters, pluck="name", order_by=None)}
 
 	filters.extend(child_filters or [])
-	res = frappe.db.get_all(from_table, filters=filters, fields=["name", "parenttype", "parent"])
+	res = frappe.get_all(
+		from_table, filters=filters, fields=["name", "parenttype", "parent"], order_by=None
+	)
 	documents = defaultdict(list)
 
 	for parent, rows in itertools.groupby(res, key=lambda row: row["parenttype"]):
 		if allowed_parents and parent not in allowed_parents:
 			continue
 		filters = (parent_filters or []) + [["name", "in", tuple(row.parent for row in rows)]]
-		documents[parent].extend(frappe.db.get_all(parent, filters=filters, pluck="name") or [])
+		documents[parent].extend(
+			frappe.get_all(parent, filters=filters, pluck="name", order_by=None) or []
+		)
 	return documents
 
 
@@ -405,7 +413,6 @@ def get_exempted_doctypes():
 	return auto_cancel_exempt_doctypes
 
 
-@frappe.whitelist()
 def get_linked_docs(doctype: str, name: str, linkinfo: dict | None = None) -> dict[str, list]:
 	if isinstance(linkinfo, str):
 		# additional fields are added in linkinfo
@@ -428,9 +435,6 @@ def get_linked_docs(doctype: str, name: str, linkinfo: dict | None = None) -> di
 			continue
 		linkmeta = link_meta_bundle[0]
 
-		if not linkmeta.has_permission():
-			continue
-
 		if not linkmeta.get("issingle"):
 			fields = [
 				d.fieldname
@@ -450,7 +454,7 @@ def get_linked_docs(doctype: str, name: str, linkinfo: dict | None = None) -> di
 
 			try:
 				if link.get("filters"):
-					ret = frappe.get_all(doctype=dt, fields=fields, filters=link.get("filters"))
+					ret = frappe.get_all(doctype=dt, fields=fields, filters=link.get("filters"), order_by=None)
 
 				elif link.get("get_parent"):
 					ret = None
@@ -459,9 +463,11 @@ def get_linked_docs(doctype: str, name: str, linkinfo: dict | None = None) -> di
 					if not frappe.get_meta(doctype).istable:
 						continue
 
-					me = frappe.db.get_value(doctype, name, ["parenttype", "parent"], as_dict=True)
+					me = frappe.db.get_value(doctype, name, ["parenttype", "parent"], as_dict=True, order_by=None)
 					if me and me.parenttype == dt:
-						ret = frappe.get_all(doctype=dt, fields=fields, filters=[[dt, "name", "=", me.parent]])
+						ret = frappe.get_all(
+							doctype=dt, fields=fields, filters=[[dt, "name", "=", me.parent]], order_by=None
+						)
 
 				elif link.get("child_doctype"):
 					or_filters = [
@@ -474,7 +480,12 @@ def get_linked_docs(doctype: str, name: str, linkinfo: dict | None = None) -> di
 						filters.append([link.get("child_doctype"), link.get("doctype_fieldname"), "=", doctype])
 
 					ret = frappe.get_all(
-						doctype=dt, fields=fields, filters=filters, or_filters=or_filters, distinct=True
+						doctype=dt,
+						fields=fields,
+						filters=filters,
+						or_filters=or_filters,
+						distinct=True,
+						order_by=None,
 					)
 
 				else:
@@ -486,7 +497,9 @@ def get_linked_docs(doctype: str, name: str, linkinfo: dict | None = None) -> di
 						# dynamic link
 						if link.get("doctype_fieldname"):
 							filters.append([dt, link.get("doctype_fieldname"), "=", doctype])
-						ret = frappe.get_all(doctype=dt, fields=fields, filters=filters, or_filters=or_filters)
+						ret = frappe.get_all(
+							doctype=dt, fields=fields, filters=filters, or_filters=or_filters, order_by=None
+						)
 
 					else:
 						ret = None
diff --git a/frappe/desk/form/load.py b/frappe/desk/form/load.py
index 898c6461e9..42109f8863 100644
--- a/frappe/desk/form/load.py
+++ b/frappe/desk/form/load.py
@@ -11,6 +11,7 @@ import frappe.share
 import frappe.utils
 from frappe import _, _dict
 from frappe.desk.form.document_follow import is_document_followed
+from frappe.model.utils import is_virtual_doctype
 from frappe.model.utils.user_settings import get_user_settings
 from frappe.permissions import get_doc_permissions
 from frappe.utils.data import cstr
@@ -27,14 +28,10 @@ def getdoc(doctype, name, user=None):
 	if not (doctype and name):
 		raise Exception("doctype and name required!")
 
-	if not name:
-		name = doctype
-
-	if not frappe.db.exists(doctype, name):
+	if not is_virtual_doctype(doctype) and not frappe.db.exists(doctype, name):
 		return []
 
 	doc = frappe.get_doc(doctype, name)
-	run_onload(doc)
 
 	if not doc.has_permission("read"):
 		frappe.flags.error_message = _("Insufficient Permission for {0}").format(
@@ -42,6 +39,7 @@ def getdoc(doctype, name, user=None):
 		)
 		raise frappe.PermissionError(("read", doctype, name))
 
+	run_onload(doc)
 	doc.apply_fieldlevel_read_permissions()
 
 	# add file list
@@ -63,11 +61,9 @@ def getdoctype(doctype, with_parent=False, cached_timestamp=None):
 	parent_dt = None
 
 	# with parent (called from report builder)
-	if with_parent:
-		parent_dt = frappe.model.meta.get_parent_dt(doctype)
-		if parent_dt:
-			docs = get_meta_bundle(parent_dt)
-			frappe.response["parent_dt"] = parent_dt
+	if with_parent and (parent_dt := frappe.model.meta.get_parent_dt(doctype)):
+		docs = get_meta_bundle(parent_dt)
+		frappe.response["parent_dt"] = parent_dt
 
 	if not docs:
 		docs = get_meta_bundle(doctype)
@@ -109,6 +105,8 @@ def get_docinfo(doc=None, doctype=None, name=None):
 
 	docinfo.update(
 		{
+			"doctype": doc.doctype,
+			"name": doc.name,
 			"attachments": get_attachments(doc.doctype, doc.name),
 			"communications": communications_except_auto_messages,
 			"automated_messages": automated_messages,
@@ -177,7 +175,7 @@ def add_comments(doc, docinfo):
 
 
 def get_milestones(doctype, name):
-	return frappe.db.get_all(
+	return frappe.get_all(
 		"Milestone",
 		fields=["creation", "owner", "track_field", "value"],
 		filters=dict(reference_type=doctype, reference_name=name),
@@ -248,7 +246,7 @@ def get_comments(
 
 
 def get_point_logs(doctype, docname):
-	return frappe.db.get_all(
+	return frappe.get_all(
 		"Energy Point Log",
 		filters={"reference_doctype": doctype, "reference_name": docname, "type": ["!=", "Review"]},
 		fields=["*"],
@@ -372,7 +370,7 @@ def run_onload(doc):
 def get_view_logs(doctype, docname):
 	"""get and return the latest view logs if available"""
 	logs = []
-	if hasattr(frappe.get_meta(doctype), "track_views") and frappe.get_meta(doctype).track_views:
+	if getattr(frappe.get_meta(doctype), "track_views", None):
 		view_logs = frappe.get_all(
 			"View Log",
 			filters={
@@ -405,7 +403,7 @@ def get_document_email(doctype, name):
 		return None
 
 	email = email.split("@")
-	return f"{email[0]}+{quote(doctype)}+{quote(cstr(name))}@{email[1]}"
+	return f"{email[0]}+{quote(doctype)}={quote(cstr(name))}@{email[1]}"
 
 
 def get_automatic_email_link():
@@ -451,7 +449,9 @@ def get_title_values_for_link_and_dynamic_link_fields(doc, link_fields=None):
 		if not meta or not (meta.title_field and meta.show_title_field_in_link):
 			continue
 
-		link_title = frappe.db.get_value(doctype, doc.get(field.fieldname), meta.title_field, cache=True)
+		link_title = frappe.db.get_value(
+			doctype, doc.get(field.fieldname), meta.title_field, cache=True, order_by=None
+		)
 		link_titles.update({doctype + "::" + doc.get(field.fieldname): link_title})
 
 	return link_titles
diff --git a/frappe/desk/form/meta.py b/frappe/desk/form/meta.py
index 5a426b4c63..62a9c89c81 100644
--- a/frappe/desk/form/meta.py
+++ b/frappe/desk/form/meta.py
@@ -4,12 +4,14 @@ import io
 import os
 
 import frappe
+from frappe import _
 from frappe.build import scrub_html_template
 from frappe.model.meta import Meta
 from frappe.model.utils import render_include
 from frappe.modules import get_module_path, load_doctype_module, scrub
 from frappe.translate import extract_messages_from_code, make_dict_from_messages
 from frappe.utils import get_html_format
+from frappe.utils.data import get_link_to_form
 
 ASSET_KEYS = (
 	"__js",
@@ -35,12 +37,10 @@ ASSET_KEYS = (
 def get_meta(doctype, cached=True):
 	# don't cache for developer mode as js files, templates may be edited
 	if cached and not frappe.conf.developer_mode:
-		meta = frappe.cache().hget("form_meta", doctype)
-		if meta:
-			meta = FormMeta(meta)
-		else:
+		meta = frappe.cache().hget("doctype_form_meta", doctype)
+		if not meta:
 			meta = FormMeta(doctype)
-			frappe.cache().hset("form_meta", doctype, meta.as_dict())
+			frappe.cache().hset("doctype_form_meta", doctype, meta)
 	else:
 		meta = FormMeta(doctype)
 
@@ -52,7 +52,7 @@ def get_meta(doctype, cached=True):
 
 class FormMeta(Meta):
 	def __init__(self, doctype):
-		super().__init__(doctype)
+		self.__dict__.update(frappe.get_meta(doctype).__dict__)
 		self.load_assets()
 
 	def load_assets(self):
@@ -134,7 +134,7 @@ class FormMeta(Meta):
 		for fname in os.listdir(path):
 			if fname.endswith(".html"):
 				with open(os.path.join(path, fname), encoding="utf-8") as f:
-					templates[fname.split(".")[0]] = scrub_html_template(f.read())
+					templates[fname.split(".", 1)[0]] = scrub_html_template(f.read())
 
 		self.set("__templates", templates or None)
 
@@ -146,7 +146,7 @@ class FormMeta(Meta):
 		"""embed all require files"""
 		# custom script
 		client_scripts = (
-			frappe.db.get_all(
+			frappe.get_all(
 				"Client Script",
 				filters={"dt": self.name, "enabled": 1},
 				fields=["name", "script", "view"],
@@ -158,6 +158,9 @@ class FormMeta(Meta):
 		list_script = ""
 		form_script = ""
 		for script in client_scripts:
+			if not script.script:
+				continue
+
 			if script.view == "List":
 				list_script += f"""
 // {script.name}
@@ -165,7 +168,7 @@ class FormMeta(Meta):
 
 """
 
-			if script.view == "Form":
+			elif script.view == "Form":
 				form_script += f"""
 // {script.name}
 {script.script}
@@ -183,19 +186,40 @@ class FormMeta(Meta):
 		"""add search fields found in the doctypes indicated by link fields' options"""
 		for df in self.get("fields", {"fieldtype": "Link", "options": ["!=", "[Select]"]}):
 			if df.options:
-				search_fields = frappe.get_meta(df.options).search_fields
+				try:
+					search_fields = frappe.get_meta(df.options).search_fields
+				except frappe.DoesNotExistError:
+					self._show_missing_doctype_msg(df)
+
 				if search_fields:
 					search_fields = search_fields.split(",")
 					df.search_fields = [sf.strip() for sf in search_fields]
 
+	def _show_missing_doctype_msg(self, df):
+		# A link field is referring to non-existing doctype, this usually happens when
+		# customizations are removed or some custom app is removed but hasn't cleaned
+		# up after itself.
+		frappe.clear_last_message()
+
+		msg = _("Field {0} is referring to non-existing doctype {1}.").format(
+			frappe.bold(df.fieldname), frappe.bold(df.options)
+		)
+
+		if df.get("is_custom_field"):
+			custom_field_link = get_link_to_form("Custom Field", df.name)
+			msg += " " + _("Please delete the field from {0} or add the required doctype.").format(
+				custom_field_link
+			)
+
+		frappe.throw(msg, title=_("Missing DocType"))
+
 	def add_linked_document_type(self):
 		for df in self.get("fields", {"fieldtype": "Link"}):
 			if df.options:
 				try:
 					df.linked_document_type = frappe.get_meta(df.options).document_type
 				except frappe.DoesNotExistError:
-					# edge case where options="[Select]"
-					pass
+					self._show_missing_doctype_msg(df)
 
 	def load_print_formats(self):
 		print_formats = frappe.db.sql(
@@ -225,7 +249,7 @@ class FormMeta(Meta):
 	def load_templates(self):
 		if not self.custom:
 			module = load_doctype_module(self.name)
-			app = module.__name__.split(".")[0]
+			app = module.__name__.split(".", 1)[0]
 			templates = {}
 			if hasattr(module, "form_grid_templates"):
 				for key, path in module.form_grid_templates.items():
diff --git a/frappe/desk/form/save.py b/frappe/desk/form/save.py
index f3e7b6294f..9ee2541a90 100644
--- a/frappe/desk/form/save.py
+++ b/frappe/desk/form/save.py
@@ -4,7 +4,11 @@
 import json
 
 import frappe
+from frappe.core.doctype.submission_queue.submission_queue import queue_submission
 from frappe.desk.form.load import run_onload
+from frappe.model.docstatus import DocStatus
+from frappe.monitor import add_data_to_monitor
+from frappe.utils.scheduler import is_scheduler_inactive
 
 
 @frappe.whitelist()
@@ -14,9 +18,17 @@ def savedocs(doc, action):
 	set_local_name(doc)
 
 	# action
-	doc.docstatus = {"Save": 0, "Submit": 1, "Update": 1, "Cancel": 2}[action]
+	doc.docstatus = {
+		"Save": DocStatus.draft(),
+		"Submit": DocStatus.submitted(),
+		"Update": DocStatus.submitted(),
+		"Cancel": DocStatus.cancelled(),
+	}[action]
 
-	if doc.docstatus == 1:
+	if doc.docstatus.is_submitted():
+		if action == "Submit" and doc.meta.queue_in_background and not is_scheduler_inactive():
+			queue_submission(doc, action)
+			return
 		doc.submit()
 	else:
 		doc.save()
@@ -25,6 +37,7 @@ def savedocs(doc, action):
 	run_onload(doc)
 	send_updated_docs(doc)
 
+	add_data_to_monitor(doctype=doc.doctype, action=action)
 	frappe.msgprint(frappe._("Saved"), indicator="green", alert=True)
 
 
diff --git a/frappe/desk/form/test_form.py b/frappe/desk/form/test_form.py
index c05a932241..f256b03d27 100644
--- a/frappe/desk/form/test_form.py
+++ b/frappe/desk/form/test_form.py
@@ -1,13 +1,12 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
 
-import unittest
-
 import frappe
 from frappe.desk.form.linked_with import get_linked_docs, get_linked_doctypes
+from frappe.tests.utils import FrappeTestCase
 
 
-class TestForm(unittest.TestCase):
+class TestForm(FrappeTestCase):
 	def test_linked_with(self):
 		results = get_linked_docs("Role", "System Manager", linkinfo=get_linked_doctypes("Role"))
 		self.assertTrue("User" in results)
@@ -15,5 +14,7 @@ class TestForm(unittest.TestCase):
 
 
 if __name__ == "__main__":
+	import unittest
+
 	frappe.connect()
 	unittest.main()
diff --git a/frappe/desk/form/utils.py b/frappe/desk/form/utils.py
index 9e10ced912..28377572c3 100644
--- a/frappe/desk/form/utils.py
+++ b/frappe/desk/form/utils.py
@@ -57,7 +57,14 @@ def update_comment(name, content):
 	if frappe.session.user not in ["Administrator", doc.owner]:
 		frappe.throw(_("Comment can only be edited by the owner"), frappe.PermissionError)
 
-	doc.content = content
+	if doc.reference_doctype and doc.reference_name:
+		reference_doc = frappe.get_doc(doc.reference_doctype, doc.reference_name)
+		reference_doc.check_permission()
+
+		doc.content = extract_images_from_html(reference_doc, content, is_private=True)
+	else:
+		doc.content = content
+
 	doc.save(ignore_permissions=True)
 
 
diff --git a/frappe/desk/leaderboard.py b/frappe/desk/leaderboard.py
index a5f5de3117..65d6aaf785 100644
--- a/frappe/desk/leaderboard.py
+++ b/frappe/desk/leaderboard.py
@@ -16,7 +16,7 @@ def get_leaderboards():
 
 @frappe.whitelist()
 def get_energy_point_leaderboard(date_range, company=None, field=None, limit=None):
-	all_users = frappe.db.get_all(
+	all_users = frappe.get_all(
 		"User",
 		filters={
 			"name": ["not in", ["Administrator", "Guest"]],
@@ -31,7 +31,7 @@ def get_energy_point_leaderboard(date_range, company=None, field=None, limit=Non
 	if date_range:
 		date_range = frappe.parse_json(date_range)
 		filters.append(["creation", "between", [date_range[0], date_range[1]]])
-	energy_point_users = frappe.db.get_all(
+	energy_point_users = frappe.get_all(
 		"Energy Point Log",
 		fields=["user as name", "sum(points) as value"],
 		filters=filters,
diff --git a/frappe/desk/listview.py b/frappe/desk/listview.py
index ea6eb6259c..05d45ad9ac 100644
--- a/frappe/desk/listview.py
+++ b/frappe/desk/listview.py
@@ -36,7 +36,7 @@ def get_group_by_count(doctype: str, current_filters: str, field: str) -> list[d
 		ToDo = DocType("ToDo")
 		User = DocType("User")
 		count = Count("*").as_("count")
-		filtered_records = frappe.qb.engine.build_conditions(doctype, current_filters).select("name")
+		filtered_records = frappe.qb.get_query(doctype, filters=current_filters, fields=["name"])
 
 		return (
 			frappe.qb.from_(ToDo)
diff --git a/frappe/desk/moduleview.py b/frappe/desk/moduleview.py
deleted file mode 100644
index 913b3406e3..0000000000
--- a/frappe/desk/moduleview.py
+++ /dev/null
@@ -1,615 +0,0 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# License: MIT. See LICENSE
-
-import json
-
-import frappe
-from frappe import _
-from frappe.boot import get_allowed_pages, get_allowed_reports
-from frappe.cache_manager import (
-	build_domain_restriced_doctype_cache,
-	build_domain_restriced_page_cache,
-	build_table_count_cache,
-)
-from frappe.desk.doctype.desktop_icon.desktop_icon import clear_desktop_icons_cache, set_hidden
-
-
-@frappe.whitelist()
-def get(module):
-	"""Returns data (sections, list of reports, counts) to render module view in desk:
-	`/desk/#Module/[name]`."""
-	data = get_data(module)
-
-	out = {"data": data}
-
-	return out
-
-
-@frappe.whitelist()
-def hide_module(module):
-	set_hidden(module, frappe.session.user, 1)
-	clear_desktop_icons_cache()
-
-
-def get_table_with_counts():
-	counts = frappe.cache().get_value("information_schema:counts")
-	if counts:
-		return counts
-	else:
-		return build_table_count_cache()
-
-
-def get_data(module, build=True):
-	"""Get module data for the module view `desk/#Module/[name]`"""
-	doctype_info = get_doctype_info(module)
-	data = build_config_from_file(module)
-
-	if not data:
-		data = build_standard_config(module, doctype_info)
-	else:
-		add_custom_doctypes(data, doctype_info)
-
-	add_section(data, _("Custom Reports"), "fa fa-list-alt", get_report_list(module))
-
-	data = combine_common_sections(data)
-	data = apply_permissions(data)
-
-	# set_last_modified(data)
-
-	if build:
-		exists_cache = get_table_with_counts()
-
-		def doctype_contains_a_record(name):
-			exists = exists_cache.get(name)
-			if not exists:
-				if not frappe.db.get_value("DocType", name, "issingle"):
-					exists = frappe.db.count(name)
-				else:
-					exists = True
-				exists_cache[name] = exists
-			return exists
-
-		for section in data:
-			for item in section["items"]:
-				# Onboarding
-
-				# First disable based on exists of depends_on list
-				doctype = item.get("doctype")
-				dependencies = item.get("dependencies") or None
-				if not dependencies and doctype:
-					item["dependencies"] = [doctype]
-
-				dependencies = item.get("dependencies")
-				if dependencies:
-					incomplete_dependencies = [d for d in dependencies if not doctype_contains_a_record(d)]
-					if len(incomplete_dependencies):
-						item["incomplete_dependencies"] = incomplete_dependencies
-
-				if item.get("onboard"):
-					# Mark Spotlights for initial
-					if item.get("type") == "doctype":
-						name = item.get("name")
-						count = doctype_contains_a_record(name)
-
-						item["count"] = count
-
-	return data
-
-
-def build_config_from_file(module):
-	"""Build module info from `app/config/desktop.py` files."""
-	data = []
-	module = frappe.scrub(module)
-
-	for app in frappe.get_installed_apps():
-		try:
-			data += get_config(app, module)
-		except ImportError:
-			pass
-
-	return filter_by_restrict_to_domain(data)
-
-
-def filter_by_restrict_to_domain(data):
-	"""filter Pages and DocType depending on the Active Module(s)"""
-	doctypes = (
-		frappe.cache().get_value("domain_restricted_doctypes") or build_domain_restriced_doctype_cache()
-	)
-	pages = frappe.cache().get_value("domain_restricted_pages") or build_domain_restriced_page_cache()
-
-	for d in data:
-		_items = []
-		for item in d.get("items", []):
-
-			item_type = item.get("type")
-			item_name = item.get("name")
-
-			if (item_name in pages) or (item_name in doctypes) or item_type == "report":
-				_items.append(item)
-
-		d.update({"items": _items})
-
-	return data
-
-
-def build_standard_config(module, doctype_info):
-	"""Build standard module data from DocTypes."""
-	if not frappe.db.get_value("Module Def", module):
-		frappe.throw(_("Module Not Found"))
-
-	data = []
-
-	add_section(
-		data,
-		_("Documents"),
-		"fa fa-star",
-		[d for d in doctype_info if d.document_type in ("Document", "Transaction")],
-	)
-
-	add_section(
-		data,
-		_("Setup"),
-		"fa fa-cog",
-		[d for d in doctype_info if d.document_type in ("Master", "Setup", "")],
-	)
-
-	add_section(data, _("Standard Reports"), "fa fa-list", get_report_list(module, is_standard="Yes"))
-
-	return data
-
-
-def add_section(data, label, icon, items):
-	"""Adds a section to the module data."""
-	if not items:
-		return
-	data.append({"label": label, "icon": icon, "items": items})
-
-
-def add_custom_doctypes(data, doctype_info):
-	"""Adds Custom DocTypes to modules setup via `config/desktop.py`."""
-	add_section(
-		data,
-		_("Documents"),
-		"fa fa-star",
-		[d for d in doctype_info if (d.custom and d.document_type in ("Document", "Transaction"))],
-	)
-
-	add_section(
-		data,
-		_("Setup"),
-		"fa fa-cog",
-		[d for d in doctype_info if (d.custom and d.document_type in ("Setup", "Master", ""))],
-	)
-
-
-def get_doctype_info(module):
-	"""Returns list of non child DocTypes for given module."""
-	active_domains = frappe.get_active_domains()
-
-	doctype_info = frappe.get_all(
-		"DocType",
-		filters={"module": module, "istable": 0},
-		or_filters={"ifnull(restrict_to_domain, '')": "", "restrict_to_domain": ("in", active_domains)},
-		fields=["'doctype' as type", "name", "description", "document_type", "custom", "issingle"],
-		order_by="custom asc, document_type desc, name asc",
-	)
-
-	for d in doctype_info:
-		d.document_type = d.document_type or ""
-		d.description = _(d.description or "")
-
-	return doctype_info
-
-
-def combine_common_sections(data):
-	"""Combine sections declared in separate apps."""
-	sections = []
-	sections_dict = {}
-	for each in data:
-		if each["label"] not in sections_dict:
-			sections_dict[each["label"]] = each
-			sections.append(each)
-		else:
-			sections_dict[each["label"]]["items"] += each["items"]
-
-	return sections
-
-
-def apply_permissions(data):
-	default_country = frappe.db.get_default("country")
-
-	user = frappe.get_user()
-	user.build_permissions()
-
-	allowed_pages = get_allowed_pages()
-	allowed_reports = get_allowed_reports()
-
-	new_data = []
-	for section in data:
-		new_items = []
-
-		for item in section.get("items") or []:
-			item = frappe._dict(item)
-
-			if item.country and item.country != default_country:
-				continue
-
-			if (
-				(item.type == "doctype" and item.name in user.can_read)
-				or (item.type == "page" and item.name in allowed_pages)
-				or (item.type == "report" and item.name in allowed_reports)
-				or item.type == "help"
-			):
-
-				new_items.append(item)
-
-		if new_items:
-			new_section = section.copy()
-			new_section["items"] = new_items
-			new_data.append(new_section)
-
-	return new_data
-
-
-def get_disabled_reports():
-	if not hasattr(frappe.local, "disabled_reports"):
-		frappe.local.disabled_reports = {r.name for r in frappe.get_all("Report", {"disabled": 1})}
-	return frappe.local.disabled_reports
-
-
-def get_config(app, module):
-	"""Load module info from `[app].config.[module]`."""
-	config = frappe.get_module(f"{app}.config.{module}")
-	config = config.get_data()
-
-	sections = [s for s in config if s.get("condition", True)]
-
-	disabled_reports = get_disabled_reports()
-	for section in sections:
-		items = []
-		for item in section["items"]:
-			if item["type"] == "report" and item["name"] in disabled_reports:
-				continue
-			# some module links might not have name
-			if not item.get("name"):
-				item["name"] = item.get("label")
-			if not item.get("label"):
-				item["label"] = _(item.get("name"))
-			items.append(item)
-		section["items"] = items
-
-	return sections
-
-
-def config_exists(app, module):
-	try:
-		frappe.get_module(f"{app}.config.{module}")
-		return True
-	except ImportError:
-		return False
-
-
-def add_setup_section(config, app, module, label, icon):
-	"""Add common sections to `/desk#Module/Setup`"""
-	try:
-		setup_section = get_setup_section(app, module, label, icon)
-		if setup_section:
-			config.append(setup_section)
-	except ImportError:
-		pass
-
-
-def get_setup_section(app, module, label, icon):
-	"""Get the setup section from each module (for global Setup page)."""
-	config = get_config(app, module)
-	for section in config:
-		if section.get("label") == _("Setup"):
-			return {"label": label, "icon": icon, "items": section["items"]}
-
-
-def get_onboard_items(app, module):
-	try:
-		sections = get_config(app, module)
-	except ImportError:
-		return []
-
-	onboard_items = []
-	fallback_items = []
-
-	if not sections:
-		doctype_info = get_doctype_info(module)
-		sections = build_standard_config(module, doctype_info)
-
-	for section in sections:
-		for item in section["items"]:
-			if item.get("onboard", 0) == 1:
-				onboard_items.append(item)
-
-			# in case onboard is not set
-			fallback_items.append(item)
-
-			if len(onboard_items) > 5:
-				return onboard_items
-
-	return onboard_items or fallback_items
-
-
-@frappe.whitelist()
-def get_links_for_module(app, module):
-	return [{"value": l.get("name"), "label": l.get("label")} for l in get_links(app, module)]
-
-
-def get_links(app, module):
-	try:
-		sections = get_config(app, frappe.scrub(module))
-	except ImportError:
-		return []
-
-	links = []
-	for section in sections:
-		for item in section["items"]:
-			links.append(item)
-	return links
-
-
-@frappe.whitelist()
-def get_desktop_settings():
-	from frappe.config import get_modules_from_all_apps_for_user
-
-	all_modules = get_modules_from_all_apps_for_user()
-	home_settings = get_home_settings()
-
-	modules_by_name = {}
-	for m in all_modules:
-		modules_by_name[m["module_name"]] = m
-
-	module_categories = ["Modules", "Domains", "Places", "Administration"]
-	user_modules_by_category = {}
-
-	user_saved_modules_by_category = home_settings.modules_by_category or {}
-	user_saved_links_by_module = home_settings.links_by_module or {}
-
-	def apply_user_saved_links(module):
-		module = frappe._dict(module)
-		all_links = get_links(module.app, module.module_name)
-		module_links_by_name = {}
-		for link in all_links:
-			module_links_by_name[link["name"]] = link
-
-		if module.module_name in user_saved_links_by_module:
-			user_links = frappe.parse_json(user_saved_links_by_module[module.module_name])
-			module.links = [module_links_by_name[l] for l in user_links if l in module_links_by_name]
-
-		return module
-
-	for category in module_categories:
-		if category in user_saved_modules_by_category:
-			user_modules = user_saved_modules_by_category[category]
-			user_modules_by_category[category] = [
-				apply_user_saved_links(modules_by_name[m]) for m in user_modules if modules_by_name.get(m)
-			]
-		else:
-			user_modules_by_category[category] = [
-				apply_user_saved_links(m) for m in all_modules if m.get("category") == category
-			]
-
-	# filter out hidden modules
-	if home_settings.hidden_modules:
-		for category in user_modules_by_category:
-			hidden_modules = home_settings.hidden_modules or []
-			modules = user_modules_by_category[category]
-			user_modules_by_category[category] = [
-				module for module in modules if module.module_name not in hidden_modules
-			]
-
-	return user_modules_by_category
-
-
-@frappe.whitelist()
-def update_hidden_modules(category_map):
-	category_map = frappe.parse_json(category_map)
-	home_settings = get_home_settings()
-
-	saved_hidden_modules = home_settings.hidden_modules or []
-
-	for category in category_map:
-		config = frappe._dict(category_map[category])
-		saved_hidden_modules += config.removed or []
-		saved_hidden_modules = [d for d in saved_hidden_modules if d not in (config.added or [])]
-
-		if home_settings.get("modules_by_category") and home_settings.modules_by_category.get(category):
-			module_placement = [
-				d for d in (config.added or []) if d not in home_settings.modules_by_category[category]
-			]
-			home_settings.modules_by_category[category] += module_placement
-
-	home_settings.hidden_modules = saved_hidden_modules
-	set_home_settings(home_settings)
-
-	return get_desktop_settings()
-
-
-@frappe.whitelist()
-def update_global_hidden_modules(modules):
-	modules = frappe.parse_json(modules)
-	frappe.only_for("System Manager")
-
-	doc = frappe.get_doc("User", "Administrator")
-	doc.set("block_modules", [])
-	for module in modules:
-		doc.append("block_modules", {"module": module})
-
-	doc.save(ignore_permissions=True)
-
-	return get_desktop_settings()
-
-
-@frappe.whitelist()
-def update_modules_order(module_category, modules):
-	modules = frappe.parse_json(modules)
-	home_settings = get_home_settings()
-
-	home_settings.modules_by_category = home_settings.modules_by_category or {}
-	home_settings.modules_by_category[module_category] = modules
-
-	set_home_settings(home_settings)
-
-
-@frappe.whitelist()
-def update_links_for_module(module_name, links):
-	links = frappe.parse_json(links)
-	home_settings = get_home_settings()
-
-	home_settings.setdefault("links_by_module", {})
-	home_settings["links_by_module"].setdefault(module_name, None)
-	home_settings["links_by_module"][module_name] = links
-
-	set_home_settings(home_settings)
-
-	return get_desktop_settings()
-
-
-@frappe.whitelist()
-def get_options_for_show_hide_cards():
-	global_options = []
-
-	if "System Manager" in frappe.get_roles():
-		global_options = get_options_for_global_modules()
-
-	return {"user_options": get_options_for_user_blocked_modules(), "global_options": global_options}
-
-
-@frappe.whitelist()
-def get_options_for_global_modules():
-	from frappe.config import get_modules_from_all_apps
-
-	all_modules = get_modules_from_all_apps()
-
-	blocked_modules = frappe.get_doc("User", "Administrator").get_blocked_modules()
-
-	options = []
-	for module in all_modules:
-		module = frappe._dict(module)
-		options.append(
-			{
-				"category": module.category,
-				"label": module.label,
-				"value": module.module_name,
-				"checked": module.module_name not in blocked_modules,
-			}
-		)
-
-	return options
-
-
-@frappe.whitelist()
-def get_options_for_user_blocked_modules():
-	from frappe.config import get_modules_from_all_apps_for_user
-
-	all_modules = get_modules_from_all_apps_for_user()
-	home_settings = get_home_settings()
-
-	hidden_modules = home_settings.hidden_modules or []
-
-	options = []
-	for module in all_modules:
-		module = frappe._dict(module)
-		options.append(
-			{
-				"category": module.category,
-				"label": module.label,
-				"value": module.module_name,
-				"checked": module.module_name not in hidden_modules,
-			}
-		)
-
-	return options
-
-
-def set_home_settings(home_settings):
-	frappe.cache().hset("home_settings", frappe.session.user, home_settings)
-	frappe.db.set_value("User", frappe.session.user, "home_settings", json.dumps(home_settings))
-
-
-@frappe.whitelist()
-def get_home_settings():
-	def get_from_db():
-		settings = frappe.db.get_value("User", frappe.session.user, "home_settings")
-		return frappe.parse_json(settings or "{}")
-
-	home_settings = frappe.cache().hget("home_settings", frappe.session.user, get_from_db)
-	return home_settings
-
-
-def get_module_link_items_from_list(app, module, list_of_link_names):
-	try:
-		sections = get_config(app, frappe.scrub(module))
-	except ImportError:
-		return []
-
-	links = []
-	for section in sections:
-		for item in section["items"]:
-			if item.get("label", "") in list_of_link_names:
-				links.append(item)
-
-	return links
-
-
-def set_last_modified(data):
-	for section in data:
-		for item in section["items"]:
-			if item["type"] == "doctype":
-				item["last_modified"] = get_last_modified(item["name"])
-
-
-def get_last_modified(doctype):
-	def _get():
-		try:
-			last_modified = frappe.get_all(
-				doctype, fields=["max(modified)"], as_list=True, limit_page_length=1
-			)[0][0]
-		except Exception as e:
-			if frappe.db.is_table_missing(e):
-				last_modified = None
-			else:
-				raise
-
-		# hack: save as -1 so that it is cached
-		if last_modified is None:
-			last_modified = -1
-
-		return last_modified
-
-	last_modified = frappe.cache().hget("last_modified", doctype, _get)
-
-	if last_modified == -1:
-		last_modified = None
-
-	return last_modified
-
-
-def get_report_list(module, is_standard="No"):
-	"""Returns list on new style reports for modules."""
-	reports = frappe.get_list(
-		"Report",
-		fields=["name", "ref_doctype", "report_type"],
-		filters={"is_standard": is_standard, "disabled": 0, "module": module},
-		order_by="name",
-	)
-
-	out = []
-	for r in reports:
-		out.append(
-			{
-				"type": "report",
-				"doctype": r.ref_doctype,
-				"is_query_report": 1
-				if r.report_type in ("Query Report", "Script Report", "Custom Report")
-				else 0,
-				"label": _(r.name),
-				"name": r.name,
-			}
-		)
-
-	return out
diff --git a/frappe/desk/notifications.py b/frappe/desk/notifications.py
index 2a987f5539..271f2b4074 100644
--- a/frappe/desk/notifications.py
+++ b/frappe/desk/notifications.py
@@ -3,10 +3,19 @@
 
 import json
 
+from bs4 import BeautifulSoup
+
 import frappe
+from frappe import _
+from frappe.desk.doctype.notification_log.notification_log import (
+	enqueue_create_notification,
+	get_title,
+	get_title_html,
+)
 from frappe.desk.doctype.notification_settings.notification_settings import (
 	get_subscribed_documents,
 )
+from frappe.utils import get_fullname
 
 
 @frappe.whitelist()
@@ -146,8 +155,6 @@ def clear_notifications(user=None):
 		else:
 			cache.delete_key("notification_count:" + name)
 
-	frappe.publish_realtime("clear_notifications")
-
 
 def clear_notification_config(user):
 	frappe.cache().hdel("notification_config", user)
@@ -155,7 +162,6 @@ def clear_notification_config(user):
 
 def delete_notification_count_for(doctype):
 	frappe.cache().delete_key("notification_count:" + doctype)
-	frappe.publish_realtime("clear_notifications")
 
 
 def clear_doctype_notifications(doc, method=None, *args, **kwargs):
@@ -298,3 +304,56 @@ def get_open_count(doctype, name, items=None):
 			out["timeline_data"] = module.get_timeline_data(doctype, name)
 
 	return out
+
+
+def notify_mentions(ref_doctype, ref_name, content):
+	if ref_doctype and ref_name and content:
+		mentions = extract_mentions(content)
+
+		if not mentions:
+			return
+
+		sender_fullname = get_fullname(frappe.session.user)
+		title = get_title(ref_doctype, ref_name)
+
+		recipients = [
+			frappe.db.get_value(
+				"User",
+				{"enabled": 1, "name": name, "user_type": "System User", "allowed_in_mentions": 1},
+				"email",
+			)
+			for name in mentions
+		]
+
+		notification_message = _("""{0} mentioned you in a comment in {1} {2}""").format(
+			frappe.bold(sender_fullname), frappe.bold(ref_doctype), get_title_html(title)
+		)
+
+		notification_doc = {
+			"type": "Mention",
+			"document_type": ref_doctype,
+			"document_name": ref_name,
+			"subject": notification_message,
+			"from_user": frappe.session.user,
+			"email_content": content,
+		}
+
+		enqueue_create_notification(recipients, notification_doc)
+
+
+def extract_mentions(txt):
+	"""Find all instances of @mentions in the html."""
+	soup = BeautifulSoup(txt, "html.parser")
+	emails = []
+	for mention in soup.find_all(class_="mention"):
+		if mention.get("data-is-group") == "true":
+			try:
+				user_group = frappe.get_cached_doc("User Group", mention["data-id"])
+				emails += [d.user for d in user_group.user_group_members]
+			except frappe.DoesNotExistError:
+				pass
+			continue
+		email = mention["data-id"]
+		emails.append(email)
+
+	return emails
diff --git a/frappe/desk/page/activity/README.md b/frappe/desk/page/activity/README.md
deleted file mode 100644
index 59e0352d12..0000000000
--- a/frappe/desk/page/activity/README.md
+++ /dev/null
@@ -1 +0,0 @@
-List of latest activities based on Feed.
\ No newline at end of file
diff --git a/frappe/desk/page/activity/activity.css b/frappe/desk/page/activity/activity.css
deleted file mode 100644
index b2387135c7..0000000000
--- a/frappe/desk/page/activity/activity.css
+++ /dev/null
@@ -1,74 +0,0 @@
-#page-activity .label {
-	display: inline-block;
-	margin-right: 7px;
-}
-
-#page-activity .list-row {
-	border: none;
-	padding: 0px;
-	height: auto;
-	cursor: pointer;
-}
-
-#page-activity hr {
-	border-top: 1px solid var(--dark-border-color);
-}
-
-.activity-label {
-	max-width: 100px;
-	margin-bottom: -4px;
-}
-
-.date-indicator {
-	background: none;
-	font-size: 12px;
-	vertical-align: middle;
-	font-weight: bold;
-	color: var(--text-muted);
-}
-.date-indicator::after {
-	margin: 0 -4px 0 12px;
-	content: "";
-	display: inline-block;
-	height: 8px;
-	width: 8px;
-	border-radius: 8px;
-	background: var(--dark-border-color);
-}
-
-.date-indicator.blue {
-	color: var(--primary);
-}
-
-.date-indicator.blue::after {
-	background: var(--primary);
-}
-
-.activity-message {
-	border-left: 1px solid var(--dark-border-color);
-	padding: 15px;
-	padding-right: 30px;
-}
-
-.activity-date {
-	padding: 15px;
-	padding-right: 0px;
-	z-index: 1;
-}
-
-#page-activity .list-filters {
-	display: none !important;
-}
-
-#page-activity .octicon-heart {
-	color: var(--red-500);
-	margin: 0px 5px;
-}
-
-.heatmap {
-	padding-top: 30px;
-}
-
-.heatmap svg {
-	margin: auto;
-}
diff --git a/frappe/desk/page/activity/activity.js b/frappe/desk/page/activity/activity.js
deleted file mode 100644
index 7b4e8ddc1a..0000000000
--- a/frappe/desk/page/activity/activity.js
+++ /dev/null
@@ -1,221 +0,0 @@
-// Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-// License: See license.txt
-
-frappe.provide("frappe.activity");
-
-frappe.pages['activity'].on_page_load = function (wrapper) {
-	var me = this;
-
-	frappe.ui.make_app_page({
-		parent: wrapper,
-		single_column: true
-	});
-
-	me.page = wrapper.page;
-	me.page.set_title(__("Activity"));
-
-	frappe.model.with_doctype("Communication", function () {
-		me.page.list = new frappe.views.Activity({
-			doctype: 'Communication',
-			parent: wrapper
-		});
-	});
-
-	frappe.activity.render_heatmap(me.page);
-
-	me.page.main.on("click", ".activity-message", function () {
-		var link_doctype = $(this).attr("data-link-doctype"),
-			link_name = $(this).attr("data-link-name"),
-			doctype = $(this).attr("data-doctype"),
-			docname = $(this).attr("data-docname");
-
-		[link_doctype, link_name, doctype, docname] =
-			[link_doctype, link_name, doctype, docname].map(decodeURIComponent);
-
-		link_doctype = link_doctype && link_doctype !== 'null' ? link_doctype : null;
-		link_name = link_name && link_name !== 'null' ? link_name : null;
-
-		if (doctype && docname) {
-			if (link_doctype && link_name) {
-				frappe.route_options = {
-					scroll_to: { "doctype": doctype, "name": docname }
-				}
-			}
-
-			frappe.set_route(["Form", link_doctype || doctype, link_name || docname]);
-		}
-	});
-
-	// Build Report Button
-	if (frappe.boot.user.can_get_report.indexOf("Feed") != -1) {
-		this.page.add_menu_item(__('Build Report'), function () {
-			frappe.set_route("List", "Feed", "Report");
-		}, 'fa fa-th')
-	}
-
-	this.page.add_menu_item(__('Activity Log'), function () {
-		frappe.route_options = {
-			"user": frappe.session.user
-		}
-
-		frappe.set_route("List", "Activity Log", "Report");
-	}, 'fa fa-th');
-};
-
-frappe.pages['activity'].on_page_show = function () {
-	frappe.breadcrumbs.add("Desk");
-}
-
-frappe.activity.last_feed_date = false;
-frappe.activity.Feed = class Feed {
-	constructor(row, data) {
-		this.scrub_data(data);
-		this.add_date_separator(row, data);
-		if (!data.add_class)
-			data.add_class = "label-default";
-
-		data.link = "";
-		if (data.link_doctype && data.link_name) {
-			data.link = frappe.format(data.link_name, { fieldtype: "Link", options: data.link_doctype },
-				{ label: __(data.link_doctype) + " " + __(data.link_name) });
-
-		} else if (data.feed_type === "Comment" && data.comment_type === "Comment") {
-			// hack for backward compatiblity
-			data.link_doctype = data.reference_doctype;
-			data.link_name = data.reference_name;
-			data.reference_doctype = "Communication";
-			data.reference_name = data.name;
-
-			data.link = frappe.format(data.link_name, { fieldtype: "Link", options: data.link_doctype },
-				{ label: __(data.link_doctype) + " " + __(data.link_name) });
-
-		} else if (data.reference_doctype && data.reference_name) {
-			data.link = frappe.format(data.reference_name, { fieldtype: "Link", options: data.reference_doctype },
-				{ label: __(data.reference_doctype) + " " + __(data.reference_name) });
-		}
-
-		$(row)
-			.append(frappe.render_template("activity_row", data))
-			.find("a").addClass("grey");
-	}
-
-	scrub_data(data) {
-		data.by = frappe.user.full_name(data.owner);
-		data.avatar = frappe.avatar(data.owner);
-
-		data.icon = "fa fa-flag";
-
-		// color for comment
-		data.add_class = {
-			"Comment": "label-danger",
-			"Assignment": "label-warning",
-			"Login": "label-default"
-		}[data.comment_type || data.communication_medium] || "label-info"
-
-		data.when = comment_when(data.creation);
-		data.feed_type = data.comment_type || data.communication_medium;
-	}
-
-	add_date_separator(row, data) {
-		var date = frappe.datetime.str_to_obj(data.creation);
-		var last = frappe.activity.last_feed_date;
-
-		if ((last && frappe.datetime.obj_to_str(last) != frappe.datetime.obj_to_str(date)) || (!last)) {
-			var diff = frappe.datetime.get_day_diff(frappe.datetime.get_today(), frappe.datetime.obj_to_str(date));
-			var pdate;
-			if (diff < 1) {
-				pdate = 'Today';
-			} else if (diff < 2) {
-				pdate = 'Yesterday';
-			} else {
-				pdate = frappe.datetime.global_date_format(date);
-			}
-			data.date_sep = pdate;
-			data.date_class = pdate == 'Today' ? "date-indicator blue" : "date-indicator";
-		} else {
-			data.date_sep = null;
-			data.date_class = "";
-		}
-		frappe.activity.last_feed_date = date;
-	}
-};
-
-frappe.activity.render_heatmap = function (page) {
-	$('
    \
-		
    \
-		
    ').prependTo(page.main);
-
-	frappe.call({
-		method: "frappe.desk.page.activity.activity.get_heatmap_data",
-		callback: function (r) {
-			if (r.message) {
-				new frappe.Chart(".heatmap", {
-					type: 'heatmap',
-					start: new Date(moment().subtract(1, 'year').toDate()),
-					countLabel: "actions",
-					discreteDomains: 1,
-					radius: 3, // default 0
-					data: {
-						'dataPoints': r.message
-					}
-				});
-			}
-		}
-	});
-};
-
-frappe.views.Activity = class Activity extends frappe.views.BaseList {
-	constructor(opts) {
-		super(opts);
-		this.show();
-	}
-
-	setup_defaults() {
-		super.setup_defaults();
-
-		this.page_title = __('Activity');
-		this.doctype = 'Communication';
-		this.method = 'frappe.desk.page.activity.activity.get_feed';
-
-	}
-
-	setup_filter_area() {
-		//
-	}
-
-	setup_view_menu() {
-		//
-	}
-
-	setup_sort_selector() {
-
-	}
-
-	setup_side_bar() {
-
-	}
-
-	get_args() {
-		return {
-			start: this.start,
-			page_length: this.page_length
-		};
-	}
-
-	update_data(r) {
-		let data = r.message || [];
-
-		if (this.start === 0) {
-			this.data = data;
-		} else {
-			this.data = this.data.concat(data);
-		}
-	}
-
-	render() {
-		this.data.map(value => {
-			const row = $('
    ').data("data", value).appendTo(this.$result).get(0);
-			new frappe.activity.Feed(row, value);
-		});
-	}
-};
diff --git a/frappe/desk/page/activity/activity.json b/frappe/desk/page/activity/activity.json
deleted file mode 100644
index aa195d9323..0000000000
--- a/frappe/desk/page/activity/activity.json
+++ /dev/null
@@ -1,20 +0,0 @@
-{
- "creation": "2013-04-09 11:45:31.000000",
- "docstatus": 0,
- "doctype": "Page",
- "icon": "fa fa-play",
- "idx": 1,
- "modified": "2013-07-11 14:40:20.000001",
- "modified_by": "Administrator",
- "module": "Desk",
- "name": "activity",
- "owner": "Administrator",
- "page_name": "activity",
- "roles": [
-  {
-   "role": "All"
-  }
- ],
- "standard": "Yes",
- "title": "Activity"
-}
diff --git a/frappe/desk/page/activity/activity.py b/frappe/desk/page/activity/activity.py
deleted file mode 100644
index d22fa006a4..0000000000
--- a/frappe/desk/page/activity/activity.py
+++ /dev/null
@@ -1,65 +0,0 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# License: MIT. See LICENSE
-
-import frappe
-from frappe.core.doctype.activity_log.feed import get_feed_match_conditions
-from frappe.utils import cint
-
-
-@frappe.whitelist()
-def get_feed(start, page_length):
-	"""get feed"""
-	match_conditions_communication = get_feed_match_conditions(frappe.session.user, "Communication")
-	match_conditions_comment = get_feed_match_conditions(frappe.session.user, "Comment")
-
-	result = frappe.db.sql(
-		"""select X.*
-		from (select name, owner, modified, creation, seen, comment_type,
-				reference_doctype, reference_name, '' as link_doctype, '' as link_name, subject,
-				communication_type, communication_medium, content
-			from
-				`tabCommunication`
-			where
-				communication_type = 'Communication'
-				and communication_medium != 'Email'
-				and {match_conditions_communication}
-		UNION
-			select name, owner, modified, creation, '0', 'Updated',
-				reference_doctype, reference_name, link_doctype, link_name, subject,
-				'Comment', '', content
-			from
-				`tabActivity Log`
-		UNION
-			select name, owner, modified, creation, '0', comment_type,
-				reference_doctype, reference_name, link_doctype, link_name, '',
-				'Comment', '', content
-			from
-				`tabComment`
-			where
-				{match_conditions_comment}
-		) X
-		order by X.creation DESC
-		LIMIT %(page_length)s
-		OFFSET %(start)s""".format(
-			match_conditions_comment=match_conditions_comment,
-			match_conditions_communication=match_conditions_communication,
-		),
-		{"user": frappe.session.user, "start": cint(start), "page_length": cint(page_length)},
-		as_dict=True,
-	)
-
-	return result
-
-
-@frappe.whitelist()
-def get_heatmap_data():
-	return dict(
-		frappe.db.sql(
-			"""select unix_timestamp(date(creation)), count(name)
-		from `tabActivity Log`
-		where
-			date(creation) > subdate(curdate(), interval 1 year)
-		group by date(creation)
-		order by creation asc"""
-		)
-	)
diff --git a/frappe/desk/page/activity/activity_row.html b/frappe/desk/page/activity/activity_row.html
deleted file mode 100644
index 4a15d3d9cd..0000000000
--- a/frappe/desk/page/activity/activity_row.html
+++ /dev/null
@@ -1,42 +0,0 @@
-
    
-	
    
-		{%= date_sep || "" %}
    
-	
    
-		{{ avatar }}
-		
-		{% if (feed_type==="Login") { %}
-			{%= __("Logged in") %}
-		{% } else if (feed_type==="Label") { %}
-			{%= __("{0} {1}", ["" + subject + "", link]) %}
-		{% } else if (reference_doctype && feed_type==="Comment") { %}
-			{%= __("Commented on {0}: {1}", [link, "" + content + ""]) %}
-		{% } else if (reference_doctype && communication_type==="Communication") { %}
-			{%= __("Communicated via {0} on {1}: {2}", [__(feed_type), link, "" + subject + ""]) %}
-		{% } else if (reference_doctype && !feed_type) { %}
-			{%= __("Updated {0}: {1}", [link, "" + subject + ""]) %}
-		{% } else if (feed_type==="Like" && reference_doctype) { %}
-			{%= by %} 
-			{% if (in_list(["Comment", "Communication"], reference_doctype)) { %}
-				{%= content %}
-			{% } else { %}
-				{%= link %}
-			{% } %}
-		{% } else if (in_list(["Created", "Submitted", "Cancelled", "Deleted"], feed_type)) { %}
-			{%= __("{0} {1}", ["" + __(feed_type) + "", feed_type==="Deleted" ? subject : link ]) %}
-		{% } else if (feed_type==="Updated") { %}
-			{%= __("Updated {0}: {1}", [link, "" + subject + ""]) %}
-		{% } else if (feed_type==="Relinked") { %}
-            		 {%= __("{0} {1} to {2}", [by, content,link]) %}
-		{% } else if (reference_doctype && reference_name) { %}
-			{%= __("{0}: {1}", [link, "" + content + ""]) %}
-		{% } else { %}
-			{%= subject %}
-		{% } %}
-	
-	
    
-
    
diff --git a/frappe/desk/page/backups/backups.js b/frappe/desk/page/backups/backups.js
index d6cab750f0..08289cab2d 100644
--- a/frappe/desk/page/backups/backups.js
+++ b/frappe/desk/page/backups/backups.js
@@ -1,18 +1,18 @@
-frappe.pages['backups'].on_page_load = function (wrapper) {
+frappe.pages["backups"].on_page_load = function (wrapper) {
 	var page = frappe.ui.make_app_page({
 		parent: wrapper,
-		title: __('Download Backups'),
-		single_column: true
+		title: __("Download Backups"),
+		single_column: true,
 	});
 
 	page.add_inner_button(__("Set Number of Backups"), function () {
-		frappe.set_route('Form', 'System Settings');
+		frappe.set_route("Form", "System Settings");
 	});
 
 	page.add_inner_button(__("Download Files Backup"), function () {
 		frappe.call({
 			method: "frappe.desk.page.backups.backups.schedule_files_backup",
-			args: { "user_email": frappe.session.user_email }
+			args: { user_email: frappe.session.user_email },
 		});
 	});
 
@@ -23,18 +23,18 @@ frappe.pages['backups'].on_page_load = function (wrapper) {
 					method: "frappe.utils.backups.get_backup_encryption_key",
 					callback: function (r) {
 						frappe.msgprint({
-							title: __('Backup Encryption Key'),
+							title: __("Backup Encryption Key"),
 							message: __(r.message),
-							indicator: 'blue'
+							indicator: "blue",
 						});
-					}
+					},
 				});
 			});
 		} else {
 			frappe.msgprint({
-				title: __('Error'),
-				message: __('System Manager privileges required.'),
-				indicator: 'red'
+				title: __("Error"),
+				message: __("System Manager privileges required."),
+				indicator: "red",
 			});
 		}
 	});
diff --git a/frappe/desk/page/backups/backups.py b/frappe/desk/page/backups/backups.py
index 2ef09df900..9554c7b9b7 100644
--- a/frappe/desk/page/backups/backups.py
+++ b/frappe/desk/page/backups/backups.py
@@ -4,13 +4,13 @@ import os
 import frappe
 from frappe import _
 from frappe.utils import cint, get_site_path, get_url
-from frappe.utils.data import convert_utc_to_user_timezone
+from frappe.utils.data import convert_utc_to_system_timezone
 
 
 def get_context(context):
 	def get_time(path):
 		dt = os.path.getmtime(path)
-		return convert_utc_to_user_timezone(datetime.datetime.utcfromtimestamp(dt)).strftime(
+		return convert_utc_to_system_timezone(datetime.datetime.utcfromtimestamp(dt)).strftime(
 			"%a %b %d %H:%M %Y"
 		)
 
diff --git a/frappe/event_streaming/doctype/document_type_mapping/__init__.py b/frappe/desk/page/form_builder/__init__.py
similarity index 100%
rename from frappe/event_streaming/doctype/document_type_mapping/__init__.py
rename to frappe/desk/page/form_builder/__init__.py
diff --git a/frappe/desk/page/form_builder/form_builder.js b/frappe/desk/page/form_builder/form_builder.js
new file mode 100644
index 0000000000..cc29084b69
--- /dev/null
+++ b/frappe/desk/page/form_builder/form_builder.js
@@ -0,0 +1,202 @@
+frappe.pages["form-builder"].on_page_load = function (wrapper) {
+	frappe.ui.make_app_page({
+		parent: wrapper,
+		title: __("Form Builder"),
+		single_column: true,
+	});
+
+	// hot reload in development
+	if (frappe.boot.developer_mode) {
+		frappe.hot_update = frappe.hot_update || [];
+		frappe.hot_update.push(() => load_form_builder(wrapper));
+	}
+};
+
+frappe.pages["form-builder"].on_page_show = function (wrapper) {
+	load_form_builder(wrapper);
+};
+
+function load_form_builder(wrapper) {
+	let route = frappe.get_route();
+	route = route.filter((a) => a);
+	if (route.length > 1) {
+		let doctype = route[1];
+		let is_customize_form = route[2] === "customize";
+
+		if (frappe.form_builder?.doctype) {
+			frappe.form_builder.doctype = frappe.form_builder.store.doctype = doctype;
+			frappe.form_builder.customize = frappe.form_builder.store.is_customize_form =
+				is_customize_form;
+			frappe.form_builder.init(true);
+			frappe.form_builder.store.fetch();
+			return;
+		}
+
+		let $parent = $(wrapper).find(".layout-main-section");
+		$parent.empty();
+
+		frappe.require("form_builder.bundle.js").then(() => {
+			frappe.form_builder = new frappe.ui.FormBuilder({
+				wrapper: $parent,
+				page: wrapper.page,
+				doctype: doctype,
+				customize: is_customize_form,
+			});
+		});
+	} else {
+		let d = new frappe.ui.Dialog({
+			title: __("Select DocType"),
+			fields: [
+				{
+					label: __("Select DocType"),
+					fieldname: "doctype",
+					fieldtype: "Link",
+					options: "DocType",
+					only_select: 1,
+				},
+				{
+					label: __("Customize"),
+					fieldname: "customize",
+					fieldtype: "Check",
+				},
+			],
+			primary_action_label: __("Edit"),
+			primary_action({ doctype, customize }) {
+				if (customize) {
+					frappe.model.with_doctype(doctype).then(() => {
+						let meta = frappe.get_meta(doctype);
+						if (in_list(frappe.model.core_doctypes_list, this.doctype))
+							frappe.throw(__("Core DocTypes cannot be customized."));
+
+						if (meta.issingle)
+							frappe.throw(__("Single DocTypes cannot be customized."));
+
+						if (meta.custom)
+							frappe.throw(
+								__(
+									"Only standard DocTypes are allowed to be customized from Customize Form."
+								)
+							);
+						frappe.set_route("form-builder", doctype, "customize");
+					});
+				} else {
+					frappe.set_route("form-builder", doctype);
+				}
+			},
+			secondary_action_label: __("Create New DocType"),
+			secondary_action() {
+				let doctype = d.get_value("doctype") || "";
+				let non_developer =
+					frappe.session.user !== "Administrator" || !frappe.boot.developer_mode;
+				d.hide();
+				let new_d = new frappe.ui.Dialog({
+					title: __("Create New DocType"),
+					fields: [
+						{
+							label: __("DocType Name"),
+							fieldname: "doctype_name",
+							fieldtype: "Data",
+							default: doctype,
+							reqd: 1,
+						},
+						{ fieldtype: "Column Break" },
+						{
+							label: __("Module"),
+							fieldname: "module",
+							fieldtype: "Link",
+							options: "Module Def",
+							reqd: 1,
+						},
+						{ fieldtype: "Section Break" },
+						{
+							label: __("Is Submittable"),
+							fieldname: "is_submittable",
+							fieldtype: "Check",
+							description: __(
+								"Once submitted, submittable documents cannot be changed. They can only be Cancelled and Amended."
+							),
+							depends_on: "eval:!doc.istable && !doc.issingle",
+						},
+						{
+							label: __("Is Child Table"),
+							fieldname: "istable",
+							fieldtype: "Check",
+							description: __("Child Tables are shown as a Grid in other DocTypes"),
+							depends_on: "eval:!doc.is_submittable && !doc.issingle",
+						},
+						{
+							label: __("Editable Grid"),
+							fieldname: "editable_grid",
+							fieldtype: "Check",
+							depends_on: "istable",
+							default: 1,
+						},
+						{
+							label: __("Is Single"),
+							fieldname: "issingle",
+							fieldtype: "Check",
+							description: __(
+								"Single Types have only one record no tables associated. Values are stored in tabSingles"
+							),
+							depends_on: "eval:!doc.istable && !doc.is_submittable",
+						},
+						{
+							label: __("Custom?"),
+							fieldname: "custom",
+							fieldtype: "Check",
+							default: non_developer,
+							read_only: non_developer,
+						},
+					],
+					primary_action_label: __("Create & Continue"),
+					primary_action(values) {
+						if (!values.istable) values.editable_grid = 0;
+						frappe.db
+							.insert({
+								doctype: "DocType",
+								name: values.doctype_name,
+								module: values.module,
+								istable: values.istable,
+								editable_grid: values.editable_grid,
+								issingle: values.issingle,
+								custom: values.custom,
+								is_submittable: values.is_submittable,
+								permissions: [
+									{
+										create: 1,
+										delete: 1,
+										email: 1,
+										export: 1,
+										print: 1,
+										read: 1,
+										report: 1,
+										role: "System Manager",
+										share: 1,
+										write: 1,
+									},
+								],
+								fields: [
+									{
+										label: "Title",
+										fieldname: "title",
+										fieldtype: "Data",
+									},
+								],
+							})
+							.then((doc) => {
+								frappe.set_route("form-builder", doc.name);
+							});
+					},
+					secondary_action_label: __("Back"),
+					secondary_action() {
+						new_d.hide();
+						d.show();
+					},
+				});
+				new_d.show();
+			},
+		});
+
+		d.show();
+	}
+}
diff --git a/frappe/desk/page/form_builder/form_builder.json b/frappe/desk/page/form_builder/form_builder.json
new file mode 100644
index 0000000000..afeacecd90
--- /dev/null
+++ b/frappe/desk/page/form_builder/form_builder.json
@@ -0,0 +1,19 @@
+{
+ "content": null,
+ "creation": "2022-10-10 22:42:53.597423",
+ "docstatus": 0,
+ "doctype": "Page",
+ "idx": 0,
+ "modified": "2022-10-10 22:42:53.597423",
+ "modified_by": "Administrator",
+ "module": "Desk",
+ "name": "form-builder",
+ "owner": "Administrator",
+ "page_name": "form-builder",
+ "roles": [],
+ "script": null,
+ "standard": "Yes",
+ "style": null,
+ "system_page": 0,
+ "title": "Form Builder"
+}
\ No newline at end of file
diff --git a/frappe/desk/page/leaderboard/leaderboard.js b/frappe/desk/page/leaderboard/leaderboard.js
index aa1678af37..9f689b461e 100644
--- a/frappe/desk/page/leaderboard/leaderboard.js
+++ b/frappe/desk/page/leaderboard/leaderboard.js
@@ -1,7 +1,7 @@
 frappe.pages["leaderboard"].on_page_load = (wrapper) => {
 	frappe.leaderboard = new Leaderboard(wrapper);
 
-	$(wrapper).bind('show', ()=> {
+	$(wrapper).bind("show", () => {
 		// Get which leaderboard to show
 		let doctype = frappe.get_route()[1];
 		frappe.leaderboard.show_leaderboard(doctype);
@@ -9,7 +9,6 @@ frappe.pages["leaderboard"].on_page_load = (wrapper) => {
 };
 
 class Leaderboard {
-
 	constructor(parent) {
 		frappe.ui.make_app_page({
 			parent: parent,
@@ -20,11 +19,12 @@ class Leaderboard {
 
 		this.parent = parent;
 		this.page = this.parent.page;
-		this.page.sidebar.html(`
      `);
-		this.$sidebar_list = this.page.sidebar.find('ul');
+		this.page.sidebar.html(
+			`
        `
+		);
+		this.$sidebar_list = this.page.sidebar.find("ul");
 
 		this.get_leaderboard_config();
-
 	}
 
 	get_leaderboard_config() {
@@ -32,47 +32,57 @@ class Leaderboard {
 		this.filters = {};
 		this.leaderboard_limit = 20;
 
-		frappe.xcall("frappe.desk.page.leaderboard.leaderboard.get_leaderboard_config").then(config => {
-			this.leaderboard_config = config;
-			for (let doctype in this.leaderboard_config) {
-				this.doctypes.push(doctype);
-				this.filters[doctype] = this.leaderboard_config[doctype].fields.map(field => {
-					if (typeof field ==='object') {
-						return field.label || field.fieldname;
-					}
-					return field;
-				});
-			}
+		frappe
+			.xcall("frappe.desk.page.leaderboard.leaderboard.get_leaderboard_config")
+			.then((config) => {
+				this.leaderboard_config = config;
+				for (let doctype in this.leaderboard_config) {
+					this.doctypes.push(doctype);
+					this.filters[doctype] = this.leaderboard_config[doctype].fields.map(
+						(field) => {
+							if (typeof field === "object") {
+								return field.label || field.fieldname;
+							}
+							return field;
+						}
+					);
+				}
 
-			// For translation. Do not remove this
-			// __("This Week"), __("This Month"), __("This Quarter"), __("This Year"),
-			//	__("Last Week"), __("Last Month"), __("Last Quarter"), __("Last Year"),
-			//	__("All Time"), __("Select From Date")
-			this.timespans = [
-				"This Week", "This Month", "This Quarter", "This Year",
-				"Last Week", "Last Month", "Last Quarter", "Last Year",
-				"All Time", "Select Date Range"
-			];
+				// For translation. Do not remove this
+				// __("This Week"), __("This Month"), __("This Quarter"), __("This Year"),
+				//	__("Last Week"), __("Last Month"), __("Last Quarter"), __("Last Year"),
+				//	__("All Time"), __("Select From Date")
+				this.timespans = [
+					"This Week",
+					"This Month",
+					"This Quarter",
+					"This Year",
+					"Last Week",
+					"Last Month",
+					"Last Quarter",
+					"Last Year",
+					"All Time",
+					"Select Date Range",
+				];
 
-			// for saving current selected filters
-			const _initial_doctype = frappe.get_route()[1] || this.doctypes[0];
-			const _initial_timespan = this.timespans[0];
-			const _initial_filter = this.filters[_initial_doctype];
+				// for saving current selected filters
+				const _initial_doctype = frappe.get_route()[1] || this.doctypes[0];
+				const _initial_timespan = this.timespans[0];
+				const _initial_filter = this.filters[_initial_doctype];
 
-			this.options = {
-				selected_doctype: _initial_doctype,
-				selected_filter: _initial_filter,
-				selected_filter_item: _initial_filter[0],
-				selected_timespan: _initial_timespan,
-			};
+				this.options = {
+					selected_doctype: _initial_doctype,
+					selected_filter: _initial_filter,
+					selected_filter_item: _initial_filter[0],
+					selected_timespan: _initial_timespan,
+				};
 
-			this.message = null;
-			this.make();
-		});
+				this.message = null;
+				this.make();
+			});
 	}
 
 	make() {
-
 		this.$container = $(`
        
 			
        
 			
        
@@ -80,7 +90,7 @@ class Leaderboard {
 
 		this.$graph_area = this.$container.find(".leaderboard-graph");
 
-		this.doctypes.map(doctype => {
+		this.doctypes.map((doctype) => {
 			const icon = this.leaderboard_config[doctype].icon;
 			this.get_sidebar_item(doctype, icon).appendTo(this.$sidebar_list);
 		});
@@ -94,7 +104,6 @@ class Leaderboard {
 		// Get which leaderboard to show
 		let doctype = frappe.get_route()[1];
 		this.show_leaderboard(doctype);
-
 	}
 
 	setup_leaderboard_fields() {
@@ -106,27 +115,28 @@ class Leaderboard {
 			default: frappe.defaults.get_default("company"),
 			reqd: 1,
 			change: (e) => {
-				this.options.selected_company = e.currentTarget.value;
 				this.make_request();
-			}
+			},
 		});
 
-		this.timespan_select = this.page.add_select(__("Timespan"),
-			this.timespans.map(d => {
-				return {"label": __(d), value: d };
+		this.timespan_select = this.page.add_select(
+			__("Timespan"),
+			this.timespans.map((d) => {
+				return { label: __(d), value: d };
 			})
 		);
 		this.create_date_range_field();
 
-		this.type_select = this.page.add_select(__("Field"),
-			this.options.selected_filter.map(d => {
-				return {"label": __(frappe.model.unscrub(d)), value: d };
+		this.type_select = this.page.add_select(
+			__("Field"),
+			this.options.selected_filter.map((d) => {
+				return { label: __(frappe.model.unscrub(d)), value: d };
 			})
 		);
 
 		this.timespan_select.on("change", (e) => {
 			this.options.selected_timespan = e.currentTarget.value;
-			if (this.options.selected_timespan === 'Select Date Range') {
+			if (this.options.selected_timespan === "Select Date Range") {
 				this.date_range_field.show();
 			} else {
 				this.date_range_field.hide();
@@ -141,41 +151,46 @@ class Leaderboard {
 	}
 
 	create_date_range_field() {
-		let timespan_field = $(this.parent).find(`.frappe-control[data-original-title="${__('Timespan')}"]`);
-		this.date_range_field = $(`
        `).insertAfter(timespan_field).hide();
+		let timespan_field = $(this.parent).find(
+			`.frappe-control[data-original-title="${__("Timespan")}"]`
+		);
+		this.date_range_field = $(`
        `)
+			.insertAfter(timespan_field)
+			.hide();
 
 		let date_field = frappe.ui.form.make_control({
 			df: {
-				fieldtype: 'DateRange',
-				fieldname: 'selected_date_range',
+				fieldtype: "DateRange",
+				fieldname: "selected_date_range",
 				placeholder: __("Date Range"),
 				default: [frappe.datetime.month_start(), frappe.datetime.now_date()],
-				input_class: 'input-xs',
+				input_class: "input-xs",
 				reqd: 1,
 				change: () => {
 					this.selected_date_range = date_field.get_value();
 					if (this.selected_date_range) this.make_request();
-				}
+				},
 			},
-			parent: $(this.parent).find('.from-date-field'),
-			render_input: 1
+			parent: $(this.parent).find(".from-date-field"),
+			render_input: 1,
 		});
 	}
 
 	render_selected_doctype() {
-
-		this.$sidebar_list.on("click", "li", (e)=> {
+		this.$sidebar_list.on("click", "li", (e) => {
 			let $li = $(e.currentTarget);
 			let doctype = $li.find(".doctype-text").attr("doctype-value");
 
-			this.options.selected_company = frappe.defaults.get_default("company");
+			this.company_select.set_value(
+				frappe.defaults.get_default("company") || this.company_select.get_value()
+			);
 			this.options.selected_doctype = doctype;
 			this.options.selected_filter = this.filters[doctype];
 			this.options.selected_filter_item = this.filters[doctype][0];
 
 			this.type_select.empty().add_options(
-				this.options.selected_filter.map(d => {
-					return {"label": __(frappe.model.unscrub(d)), value: d };
+				this.options.selected_filter.map((d) => {
+					return { label: __(frappe.model.unscrub(d)), value: d };
 				})
 			);
 			if (this.leaderboard_config[this.options.selected_doctype].company_disabled) {
@@ -193,10 +208,10 @@ class Leaderboard {
 	}
 
 	render_search_box() {
-
-		this.$search_box =
-			$(`
        
-				
+		this.$search_box = $(`
        
+				
 			
        `);
 
 		$(this.parent).find(".page-form").append(this.$search_box);
@@ -206,7 +221,9 @@ class Leaderboard {
 		if (this.doctypes.length) {
 			if (this.doctypes.includes(doctype)) {
 				this.options.selected_doctype = doctype;
-				this.$sidebar_list.find(`[doctype-value = "${this.options.selected_doctype}"]`).trigger("click");
+				this.$sidebar_list
+					.find(`[doctype-value = "${this.options.selected_doctype}"]`)
+					.trigger("click");
 			}
 
 			this.$search_box.find(".leaderboard-search-input").val("");
@@ -215,43 +232,44 @@ class Leaderboard {
 	}
 
 	make_request() {
-
-		frappe.model.with_doctype(this.options.selected_doctype, ()=> {
+		frappe.model.with_doctype(this.options.selected_doctype, () => {
 			this.get_leaderboard(this.get_leaderboard_data);
 		});
 	}
 
 	get_leaderboard(notify) {
-		if (!this.options.selected_company) {
-			frappe.throw(__("Please select Company"));
+		let company = this.company_select.get_value();
+		if (!company && !this.leaderboard_config[this.options.selected_doctype].company_disabled) {
+			notify(this, null);
+			frappe.show_alert(__("Please select Company"));
+			return;
 		}
-		frappe.call(
-			this.leaderboard_config[this.options.selected_doctype].method,
-			{
-				'date_range': this.get_date_range(),
-				'company': this.options.selected_company,
-				'field': this.options.selected_filter_item,
-				'limit': this.leaderboard_limit,
-			}
-		).then(r => {
-			let results = r.message || [];
+		frappe
+			.call(this.leaderboard_config[this.options.selected_doctype].method, {
+				date_range: this.get_date_range(),
+				company: company,
+				field: this.options.selected_filter_item,
+				limit: this.leaderboard_limit,
+			})
+			.then((r) => {
+				let results = r.message || [];
 
-			let graph_items = results.slice(0, 10);
+				let graph_items = results.slice(0, 10);
 
-			this.$graph_area.show().empty();
+				this.$graph_area.show().empty();
 
-			const custom_options = {
-				data: {
-					datasets: [{ values: graph_items.map(d => d.value) }],
-					labels: graph_items.map(d => d.name)
-				},
-				format_tooltip_x: d => d[this.options.selected_filter_item],
-				height: 140
-			};
-			frappe.utils.make_chart('.leaderboard-graph', custom_options);
+				const custom_options = {
+					data: {
+						datasets: [{ values: graph_items.map((d) => d.value) }],
+						labels: graph_items.map((d) => d.name),
+					},
+					format_tooltip_x: (d) => d[this.options.selected_filter_item],
+					height: 140,
+				};
+				frappe.utils.make_chart(".leaderboard-graph", custom_options);
 
-			notify(this, r);
-		});
+				notify(this, r);
+			});
 	}
 
 	get_leaderboard_data(me, res) {
@@ -267,9 +285,7 @@ class Leaderboard {
 	}
 
 	render_list_view(items = []) {
-
-		var html =
-			`${this.render_message()}
+		var html = `${this.render_message()}
 			
        
 				${this.render_result(items)}
 			
        `;
@@ -278,47 +294,43 @@ class Leaderboard {
 	}
 
 	render_result(items) {
-
-		var html =
-			`${this.render_list_header()}
+		var html = `${this.render_list_header()}
 			${this.render_list_result(items)}`;
 		return html;
 	}
 
 	render_list_header() {
-		const _selected_filter = this.options.selected_filter
-			.map(i => frappe.model.unscrub(i));
+		const _selected_filter = this.options.selected_filter.map((i) => frappe.model.unscrub(i));
 		const fields = ["rank", "name", this.options.selected_filter_item];
-		const filters = fields.map(filter => {
-			const col = __(frappe.model.unscrub(filter));
-			return (
-				`
        
+					${col && _selected_filter.indexOf(col) !== -1 ? "text-right" : ""}">
 					
 						${col}
 					
-				
        `
-			);
-		}).join("");
+				
        `;
+			})
+			.join("");
 
-		const html =
-			`
        
+		const html = `
        
 				
        ${filters}
        
 			
        `;
 		return html;
 	}
 
 	render_list_result(items) {
+		let _html = items
+			.map((item, index) => {
+				const $value = $(this.get_item_html(item, index + 1));
+				const $item_container = $(`
        `).append($value);
+				return $item_container[0].outerHTML;
+			})
+			.join("");
 
-		let _html = items.map((item, index) => {
-			const $value = $(this.get_item_html(item, index+1));
-			const $item_container = $(`
        `).append($value);
-			return $item_container[0].outerHTML;
-		}).join("");
-
-		let html =
-			`
        
+		let html = `
        
 				
        
 					${_html}
 				
        
@@ -328,7 +340,7 @@ class Leaderboard {
 	}
 
 	render_message() {
-		const display_class = this.message ? '' : 'hide';
+		const display_class = this.message ? "" : "hide";
 		let html = `
        
 			
        
 				 {
-			let fieldname = field.fieldname || field;
-			return fieldname === this.options.selected_filter_item;
-		}));
+		const value = frappe.format(
+			item.value,
+			fields.find((field) => {
+				let fieldname = field.fieldname || field;
+				return fieldname === this.options.selected_filter_item;
+			})
+		);
 
 		const link = `/app/${frappe.router.slug(this.options.selected_doctype)}/${item.name}`;
-		const name_html = item.formatted_name ?
-			`${item.formatted_name}`
+		const name_html = item.formatted_name
+			? `${item.formatted_name}`
 			: ` ${item.name} `;
-		const html =
-			`
        
+		const html = `
        
 				
        
 					${index}
 				
        
@@ -369,11 +383,11 @@ class Leaderboard {
 	}
 
 	get_sidebar_item(item, icon) {
-		let icon_html = icon ? frappe.utils.icon(icon, 'md') : '';
+		let icon_html = icon ? frappe.utils.icon(icon, "md") : "";
 		return $(`
      • 
 			${icon_html}
 			
-				${ __(item) }
+				${__(item)}
 			
 		
        • `);
 	}
@@ -391,9 +405,11 @@ class Leaderboard {
 			"last quarter": [frappe.datetime.add_months(current_date, -3), current_date],
 			"last year": [frappe.datetime.add_months(current_date, -12), current_date],
 			"all time": null,
-			"select date range": this.selected_date_range || [frappe.datetime.month_start(), current_date]
-		}
+			"select date range": this.selected_date_range || [
+				frappe.datetime.month_start(),
+				current_date,
+			],
+		};
 		return date_range_map[timespan];
 	}
-
 }
diff --git a/frappe/desk/page/setup_wizard/setup_wizard.js b/frappe/desk/page/setup_wizard/setup_wizard.js
index c683655dd5..862ac8c14d 100644
--- a/frappe/desk/page/setup_wizard/setup_wizard.js
+++ b/frappe/desk/page/setup_wizard/setup_wizard.js
@@ -27,11 +27,11 @@ frappe.setup = {
 		$.each(frappe.setup.events[event] || [], function (i, fn) {
 			fn();
 		});
-	}
-}
+	},
+};
 
-frappe.pages['setup-wizard'].on_page_load = function (wrapper) {
-	let requires = (frappe.boot.setup_wizard_requires || []);
+frappe.pages["setup-wizard"].on_page_load = function (wrapper) {
+	let requires = frappe.boot.setup_wizard_requires || [];
 	frappe.require(requires, function () {
 		frappe.call({
 			method: "frappe.desk.page.setup_wizard.setup_wizard.load_languages",
@@ -46,19 +46,19 @@ frappe.pages['setup-wizard'].on_page_load = function (wrapper) {
 					slide_class: frappe.setup.SetupWizardSlide,
 					unidirectional: 1,
 					done_state: 1,
-				}
+				};
 				frappe.wizard = new frappe.setup.SetupWizard(wizard_settings);
 				frappe.setup.run_event("after_load");
 				let route = frappe.get_route();
 				if (route) {
 					frappe.wizard.show_slide(route[1]);
 				}
-			}
+			},
 		});
 	});
 };
 
-frappe.pages['setup-wizard'].on_page_show = function () {
+frappe.pages["setup-wizard"].on_page_show = function () {
 	if (frappe.get_route()[1]) {
 		frappe.wizard && frappe.wizard.show_slide(frappe.get_route()[1]);
 	}
@@ -67,7 +67,7 @@ frappe.pages['setup-wizard'].on_page_show = function () {
 frappe.setup.on("before_load", function () {
 	// load slides
 	frappe.setup.slides_settings.forEach((s) => {
-		if (!(s.name === 'user' && frappe.boot.developer_mode)) {
+		if (!(s.name === "user" && frappe.boot.developer_mode)) {
 			// if not user slide with developer mode
 			frappe.setup.add_slide(s);
 		}
@@ -87,26 +87,26 @@ frappe.setup.SetupWizard = class SetupWizard extends frappe.ui.Slides {
 	make() {
 		super.make();
 		this.container.addClass("container setup-wizard-slide with-form");
-		this.$next_btn.addClass('action');
-		this.$complete_btn.addClass('action');
+		this.$next_btn.addClass("action");
+		this.$complete_btn.addClass("action");
 		this.setup_keyboard_nav();
 	}
 
 	setup_keyboard_nav() {
-		$('body').on('keydown', this.handle_enter_press.bind(this));
+		$("body").on("keydown", this.handle_enter_press.bind(this));
 	}
 
 	disable_keyboard_nav() {
-		$('body').off('keydown', this.handle_enter_press.bind(this));
+		$("body").off("keydown", this.handle_enter_press.bind(this));
 	}
 
 	handle_enter_press(e) {
 		if (e.which === frappe.ui.keyCode.ENTER) {
 			var $target = $(e.target);
-			if ($target.hasClass('prev-btn')) {
-				$target.trigger('click');
+			if ($target.hasClass("prev-btn")) {
+				$target.trigger("click");
 			} else {
-				this.container.find('.next-btn').trigger('click');
+				this.container.find(".next-btn").trigger("click");
 				e.preventDefault();
 			}
 		}
@@ -122,8 +122,6 @@ frappe.setup.SetupWizard = class SetupWizard extends frappe.ui.Slides {
 
 	show_slide(id) {
 		if (id === this.slides.length) {
-			// show_slide called on last slide
-			this.action_on_complete();
 			return;
 		}
 		super.show_slide(id);
@@ -134,8 +132,10 @@ frappe.setup.SetupWizard = class SetupWizard extends frappe.ui.Slides {
 		super.show_hide_prev_next(id);
 		if (id + 1 === this.slides.length) {
 			this.$next_btn.removeClass("btn-primary").hide();
-			this.$complete_btn.addClass("btn-primary").show()
-				.on('click', () => this.action_on_complete());
+			this.$complete_btn
+				.addClass("btn-primary")
+				.show()
+				.on("click", () => this.action_on_complete());
 		} else {
 			this.$next_btn.addClass("btn-primary").show();
 			this.$complete_btn.removeClass("btn-primary").hide();
@@ -170,12 +170,13 @@ frappe.setup.SetupWizard = class SetupWizard extends frappe.ui.Slides {
 		this.show_slide(this.current_id);
 		this.refresh(this.current_id);
 		setTimeout(() => {
-			this.container.find('.form-control').first().focus();
+			this.container.find(".form-control").first().focus();
 		}, 200);
 		this.in_refresh_slides = false;
 	}
 
 	action_on_complete() {
+		frappe.telemetry.capture("initated_client_side", "setup");
 		if (!this.current_slide.set_values()) return;
 		this.update_values();
 		this.show_working_state();
@@ -186,15 +187,15 @@ frappe.setup.SetupWizard = class SetupWizard extends frappe.ui.Slides {
 			method: "frappe.desk.page.setup_wizard.setup_wizard.setup_complete",
 			args: { args: this.values },
 			callback: (r) => {
-				if (r.message.status === 'ok') {
+				if (r.message.status === "ok") {
 					this.post_setup_success();
-				} else if (r.message.status === 'registered') {
+				} else if (r.message.status === "registered") {
 					this.update_setup_message(__("starting the setup..."));
 				} else if (r.message.fail !== undefined) {
 					this.abort_setup(r.message.fail);
 				}
 			},
-			error: () => this.abort_setup("Error in setup")
+			error: () => this.abort_setup("Error in setup"),
 		});
 	}
 
@@ -205,17 +206,17 @@ frappe.setup.SetupWizard = class SetupWizard extends frappe.ui.Slides {
 		}
 		setTimeout(function () {
 			// Reload
-			window.location.href = '/app';
+			window.location.href = "/app";
 		}, 2000);
 	}
 
 	abort_setup(fail_msg) {
-		this.$working_state.find('.state-icon-container').html('');
+		this.$working_state.find(".state-icon-container").html("");
 		fail_msg = fail_msg ? fail_msg : __("Failed to complete setup");
 
-		this.update_setup_message('Could not start up: ' + fail_msg);
+		this.update_setup_message("Could not start up: " + fail_msg);
 
-		this.$working_state.find('.title').html('Setup failed');
+		this.$working_state.find(".title").html("Setup failed");
 
 		this.$abort_btn.show();
 	}
@@ -226,34 +227,36 @@ frappe.setup.SetupWizard = class SetupWizard extends frappe.ui.Slides {
 			if (data.stage_status) {
 				// .html('Process '+ data.progress[0] + ' of ' + data.progress[1] + ': ' + data.stage_status);
 				this.update_setup_message(data.stage_status);
-				this.set_setup_load_percent((data.progress[0] + 1) / data.progress[1] * 100);
+				this.set_setup_load_percent(((data.progress[0] + 1) / data.progress[1]) * 100);
 			}
 			if (data.fail_msg) {
 				this.abort_setup(data.fail_msg);
 			}
-			if (data.status === 'ok') {
+			if (data.status === "ok") {
 				this.post_setup_success();
 			}
-		})
+		});
 	}
 
 	update_setup_message(message) {
-		this.$working_state.find('.setup-message').html(message);
+		this.$working_state.find(".setup-message").html(message);
 	}
 
 	get_setup_slides_filtered_by_domain() {
-		var filtered_slides = [];
+		let filtered_slides = [];
 		frappe.setup.slides.forEach(function (slide) {
 			if (frappe.setup.domains) {
 				let active_domains = frappe.setup.domains;
-				if (!slide.domains ||
-					slide.domains.filter(d => active_domains.includes(d)).length > 0) {
+				if (
+					!slide.domains ||
+					slide.domains.filter((d) => active_domains.includes(d)).length > 0
+				) {
 					filtered_slides.push(slide);
 				}
 			} else {
 				filtered_slides.push(slide);
 			}
-		})
+		});
 		return filtered_slides;
 	}
 
@@ -263,7 +266,8 @@ frappe.setup.SetupWizard = class SetupWizard extends frappe.ui.Slides {
 
 		this.$working_state = this.get_message(
 			__("Setting up your system"),
-			__("Starting Frappe ...")).appendTo(this.parent);
+			__("Starting Frappe ...")
+		).appendTo(this.parent);
 
 		this.attach_abort_button();
 
@@ -272,11 +276,13 @@ frappe.setup.SetupWizard = class SetupWizard extends frappe.ui.Slides {
 	}
 
 	attach_abort_button() {
-		this.$abort_btn = $(``);
-		this.$working_state.find('.content').append(this.$abort_btn);
+		this.$abort_btn = $(
+			``
+		);
+		this.$working_state.find(".content").append(this.$abort_btn);
 
-		this.$abort_btn.on('click', () => {
-			$(this.parent).find('.setup-in-progress').remove();
+		this.$abort_btn.on("click", () => {
+			$(this.parent).find(".setup-in-progress").remove();
 			this.container.show();
 			frappe.set_route(this.page_name, this.slides.length - 1);
 		});
@@ -301,12 +307,12 @@ frappe.setup.SetupWizard = class SetupWizard extends frappe.ui.Slides {
 	}
 
 	set_setup_complete_message(title, message) {
-		this.$working_state.find('.title').html(title);
-		this.$working_state.find('.setup-message').html(message);
+		this.$working_state.find(".title").html(title);
+		this.$working_state.find(".setup-message").html(message);
 	}
 
 	set_setup_load_percent(percent) {
-		this.$working_state.find('.progress-bar').css({ "width": percent + "%" });
+		this.$working_state.find(".progress-bar").css({ width: percent + "%" });
 	}
 };
 
@@ -318,11 +324,12 @@ frappe.setup.SetupWizardSlide = class SetupWizardSlide extends frappe.ui.Slide {
 	make() {
 		super.make();
 		this.set_init_values();
+		this.setup_telemetry_events();
 		this.reset_action_button_state();
 	}
 
 	set_init_values() {
-		var me = this;
+		let me = this;
 		// set values from frappe.setup.values
 		if (frappe.wizard.values && this.fields) {
 			this.fields.forEach(function (f) {
@@ -334,6 +341,17 @@ frappe.setup.SetupWizardSlide = class SetupWizardSlide extends frappe.ui.Slide {
 		}
 	}
 
+	setup_telemetry_events() {
+		let me = this;
+		this.fields.filter(frappe.model.is_value_type).forEach((field) => {
+			me.get_input(field.fieldname).on("change", function () {
+				frappe.telemetry.capture(`${field.fieldname}_set`, "setup");
+				if (field.fieldname == "enable_telemetry" && !me.get_value("enable_telemetry")) {
+					frappe.telemetry.disable();
+				}
+			});
+		});
+	}
 };
 
 // Frappe slides settings
@@ -342,14 +360,14 @@ frappe.setup.slides_settings = [
 	{
 		// Welcome (language) slide
 		name: "welcome",
-		title: __("Hello!"),
+		title: __("Welcome"),
 
 		fields: [
 			{
 				fieldname: "language",
 				label: __("Your Language"),
 				fieldtype: "Autocomplete",
-				placeholder: __('Select Language'),
+				placeholder: __("Select Language"),
 				default: "English",
 				reqd: 1,
 			},
@@ -357,16 +375,16 @@ frappe.setup.slides_settings = [
 				fieldname: "country",
 				label: __("Your Country"),
 				fieldtype: "Autocomplete",
-				placeholder: __('Select Country'),
+				placeholder: __("Select Country"),
 				reqd: 1,
 			},
 			{
-				fieldtype: "Section Break"
+				fieldtype: "Section Break",
 			},
 			{
 				fieldname: "timezone",
 				label: __("Time Zone"),
-				placeholder: __('Select Time Zone'),
+				placeholder: __("Select Time Zone"),
 				fieldtype: "Select",
 				reqd: 1,
 			},
@@ -374,10 +392,19 @@ frappe.setup.slides_settings = [
 			{
 				fieldname: "currency",
 				label: __("Currency"),
-				placeholder: __('Select Currency'),
+				placeholder: __("Select Currency"),
 				fieldtype: "Select",
 				reqd: 1,
-			}
+			},
+			{
+				fieldtype: "Section Break",
+			},
+			{
+				fieldname: "enable_telemetry",
+				label: __("Allow Sending Usage Data for Improving Applications"),
+				fieldtype: "Check",
+				default: 1,
+			},
 		],
 
 		onload: function (slide) {
@@ -387,7 +414,10 @@ frappe.setup.slides_settings = [
 				frappe.setup.utils.load_regional_data(slide, this.setup_fields);
 			}
 			if (!slide.get_value("language")) {
-				let session_language = frappe.setup.utils.get_language_name_from_code(frappe.boot.lang || navigator.language) || "English";
+				let session_language =
+					frappe.setup.utils.get_language_name_from_code(
+						frappe.boot.lang || navigator.language
+					) || "English";
 				let language_field = slide.get_field("language");
 
 				language_field.set_input(session_language);
@@ -408,23 +438,23 @@ frappe.setup.slides_settings = [
 	},
 	{
 		// Profile slide
-		name: 'user',
-		title: __("The First User: You"),
+		name: "user",
+		title: __("Let's set up your account"),
 		icon: "fa fa-user",
 		fields: [
 			{
-				"fieldtype": "Attach Image", "fieldname": "attach_user_image",
-				label: __("Attach Your Picture"), is_private: 0, align: 'center'
+				fieldname: "full_name",
+				label: __("Full Name"),
+				fieldtype: "Data",
+				reqd: 1,
 			},
 			{
-				"fieldname": "full_name", "label": __("Full Name"), "fieldtype": "Data",
-				reqd: 1
+				fieldname: "email",
+				label: __("Email Address") + " (" + __("Will be your login ID") + ")",
+				fieldtype: "Data",
+				options: "Email",
 			},
-			{
-				"fieldname": "email", "label": __("Email Address") + ' (' + __("Will be your login ID") + ')',
-				"fieldtype": "Data", "options": "Email"
-			},
-			{ "fieldname": "password", "label": __("Password"), "fieldtype": "Password" }
+			{ fieldname: "password", label: __("Password"), fieldtype: "Password" },
 		],
 
 		onload: function (slide) {
@@ -437,19 +467,10 @@ frappe.setup.slides_settings = [
 
 				if (frappe.boot.user.first_name || frappe.boot.user.last_name) {
 					slide.form.fields_dict.full_name.set_input(
-						[frappe.boot.user.first_name, frappe.boot.user.last_name].join(' ').trim());
-				}
-
-				var user_image = frappe.get_cookie("user_image");
-				var $attach_user_image = slide.form.fields_dict.attach_user_image.$wrapper;
-
-				if (user_image) {
-					$attach_user_image.find(".missing-image").toggle(false);
-					$attach_user_image.find("img").attr("src", decodeURIComponent(user_image));
-					$attach_user_image.find(".img-container").toggle(true);
+						[frappe.boot.user.first_name, frappe.boot.user.last_name].join(" ").trim()
+					);
 				}
 				delete slide.form.fields_dict.email;
-
 			} else {
 				slide.form.fields_dict.email.df.reqd = 1;
 				slide.form.fields_dict.email.refresh();
@@ -467,15 +488,9 @@ frappe.setup.slides_settings = [
 			if (frappe.setup.data.email) {
 				let email = frappe.setup.data.email;
 				slide.form.fields_dict.email.set_input(email);
-				if (frappe.get_gravatar(email, 200)) {
-					var $attach_user_image = slide.form.fields_dict.attach_user_image.$wrapper;
-					$attach_user_image.find(".missing-image").toggle(false);
-					$attach_user_image.find("img").attr("src", frappe.get_gravatar(email, 200));
-					$attach_user_image.find(".img-container").toggle(true);
-				}
 			}
 		},
-	}
+	},
 ];
 
 frappe.setup.utils = {
@@ -485,7 +500,7 @@ frappe.setup.utils = {
 			callback: function (data) {
 				frappe.setup.data.regional_data = data.message;
 				callback(slide);
-			}
+			},
 		});
 	},
 
@@ -497,7 +512,7 @@ frappe.setup.utils = {
 				frappe.setup.data.full_name = r.message.full_name;
 				frappe.setup.data.email = r.message.email;
 				callback(slide);
-			}
+			},
 		});
 	},
 
@@ -512,28 +527,28 @@ frappe.setup.utils = {
 			Set a slide's country, timezone and currency fields
 		*/
 		let data = frappe.setup.data.regional_data;
-		let country_field = slide.get_field('country');
+		let country_field = slide.get_field("country");
 		let translated_countries = [];
 
-		Object.keys(data.country_info).sort().forEach(country => {
-			translated_countries.push({
-				label: __(country),
-				value: country
+		Object.keys(data.country_info)
+			.sort()
+			.forEach((country) => {
+				translated_countries.push({
+					label: __(country),
+					value: country,
+				});
 			});
-		});
 
 		country_field.set_data(translated_countries);
 
-		slide.get_input("currency")
+		slide
+			.get_input("currency")
 			.empty()
 			.add_options(
-				frappe.utils.unique(
-					$.map(data.country_info, opts => opts.currency).sort()
-				)
+				frappe.utils.unique($.map(data.country_info, (opts) => opts.currency).sort())
 			);
 
-		slide.get_input("timezone").empty()
-			.add_options(data.all_timezones);
+		slide.get_input("timezone").empty().add_options(data.all_timezones);
 
 		// set values if present
 		if (frappe.wizard.values.country) {
@@ -547,24 +562,27 @@ frappe.setup.utils = {
 	},
 
 	bind_language_events: function (slide) {
-		slide.get_input("language").unbind("change").on("change", function () {
-			clearTimeout(slide.language_call_timeout);
-			slide.language_call_timeout = setTimeout(() => {
-				var lang = $(this).val() || "English";
-				frappe._messages = {};
-				frappe.call({
-					method: "frappe.desk.page.setup_wizard.setup_wizard.load_messages",
-					freeze: true,
-					args: {
-						language: lang
-					},
-					callback: function () {
-						frappe.setup._from_load_messages = true;
-						frappe.wizard.refresh_slides();
-					}
-				});
-			}, 500);
-		});
+		slide
+			.get_input("language")
+			.unbind("change")
+			.on("change", function () {
+				clearTimeout(slide.language_call_timeout);
+				slide.language_call_timeout = setTimeout(() => {
+					let lang = $(this).val() || "English";
+					frappe._messages = {};
+					frappe.call({
+						method: "frappe.desk.page.setup_wizard.setup_wizard.load_messages",
+						freeze: true,
+						args: {
+							language: lang,
+						},
+						callback: function () {
+							frappe.setup._from_load_messages = true;
+							frappe.wizard.refresh_slides();
+						},
+					});
+				}, 500);
+			});
 	},
 
 	get_language_name_from_code: function (language_code) {
@@ -576,9 +594,9 @@ frappe.setup.utils = {
 			Bind a slide's country, timezone and currency fields
 		*/
 		slide.get_input("country").on("change", function () {
-			var country = slide.get_input("country").val();
-			var $timezone = slide.get_input("timezone");
-			var data = frappe.setup.data.regional_data;
+			let country = slide.get_input("country").val();
+			let $timezone = slide.get_input("timezone");
+			let data = frappe.setup.data.regional_data;
 
 			$timezone.empty();
 
@@ -594,17 +612,17 @@ frappe.setup.utils = {
 			slide.get_field("timezone").set_input($timezone.val());
 
 			// temporarily set date format
-			frappe.boot.sysdefaults.date_format = (data.country_info[country].date_format
-				|| "dd-mm-yyyy");
+			frappe.boot.sysdefaults.date_format =
+				data.country_info[country].date_format || "dd-mm-yyyy";
 		});
 
 		slide.get_input("currency").on("change", function () {
-			var currency = slide.get_input("currency").val();
+			let currency = slide.get_input("currency").val();
 			if (!currency) return;
 			frappe.model.with_doc("Currency", currency, function () {
 				frappe.provide("locals.:Currency." + currency);
-				var currency_doc = frappe.model.get_doc("Currency", currency);
-				var number_format = currency_doc.number_format;
+				let currency_doc = frappe.model.get_doc("Currency", currency);
+				let number_format = currency_doc.number_format;
 				if (number_format === "#.###") {
 					number_format = "#.###,##";
 				} else if (number_format === "#,###") {
diff --git a/frappe/desk/page/setup_wizard/setup_wizard.py b/frappe/desk/page/setup_wizard/setup_wizard.py
index 3422602720..cb869fb5fc 100755
--- a/frappe/desk/page/setup_wizard/setup_wizard.py
+++ b/frappe/desk/page/setup_wizard/setup_wizard.py
@@ -2,11 +2,10 @@
 # License: MIT. See LICENSE
 
 import json
-import os
 
 import frappe
 from frappe.geo.country_info import get_country_info
-from frappe.translate import get_dict, send_translations, set_default_language
+from frappe.translate import get_messages_for_boot, send_translations, set_default_language
 from frappe.utils import cint, strip
 from frappe.utils.password import update_password
 
@@ -65,6 +64,9 @@ def setup_complete(args):
 
 @frappe.task()
 def process_setup_stages(stages, user_input, is_background_task=False):
+	from frappe.utils.telemetry import capture
+
+	capture("initated_server_side", "setup")
 	try:
 		frappe.flags.in_setup_wizard = True
 		current_task = None
@@ -89,6 +91,7 @@ def process_setup_stages(stages, user_input, is_background_task=False):
 		)
 	else:
 		run_setup_success(user_input)
+		capture("completed_server_side", "setup")
 		if not is_background_task:
 			return {"status": "ok"}
 		frappe.publish_realtime("setup_task", {"status": "ok"}, user=frappe.session.user)
@@ -166,11 +169,13 @@ def update_system_settings(args):
 			"language": get_language_code(args.get("language")) or "en",
 			"time_zone": args.get("timezone"),
 			"float_precision": 3,
+			"rounding_method": "Banker's Rounding",
 			"date_format": frappe.db.get_value("Country", args.get("country"), "date_format"),
 			"time_format": frappe.db.get_value("Country", args.get("country"), "time_format"),
 			"number_format": number_format,
 			"enable_scheduler": 1 if not frappe.flags.in_test else 0,
 			"backup_limit": 3,  # Default for downloadable backups
+			"enable_telemetry": cint(args.get("enable_telemetry")),
 		}
 	)
 	system_settings.save()
@@ -268,10 +273,10 @@ def add_all_roles_to(name):
 
 def disable_future_access():
 	frappe.db.set_default("desktop:home_page", "workspace")
-	frappe.db.set_value("System Settings", "System Settings", "setup_complete", 1)
+	frappe.db.set_single_value("System Settings", "setup_complete", 1)
 
 	# Enable onboarding after install
-	frappe.db.set_value("System Settings", "System Settings", "enable_onboarding", 1)
+	frappe.db.set_single_value("System Settings", "enable_onboarding", 1)
 
 	if not frappe.flags.in_test:
 		# remove all roles and add 'Administrator' to prevent future access
@@ -290,15 +295,7 @@ def load_messages(language):
 	frappe.clear_cache()
 	set_default_language(get_language_code(language))
 	frappe.db.commit()
-	m = get_dict("page", "setup-wizard")
-
-	for path in frappe.get_hooks("setup_wizard_requires"):
-		# common folder `assets` served from `sites/`
-		js_file_path = os.path.abspath(frappe.get_site_path("..", *path.strip("/").split("/")))
-		m.update(get_dict("jsfile", js_file_path))
-
-	m.update(get_dict("boot"))
-	send_translations(m)
+	send_translations(get_messages_for_boot())
 	return frappe.local.lang
 
 
diff --git a/frappe/desk/page/translation_tool/translation_tool.css b/frappe/desk/page/translation_tool/translation_tool.css
deleted file mode 100644
index 9603a4ce35..0000000000
--- a/frappe/desk/page/translation_tool/translation_tool.css
+++ /dev/null
@@ -1,37 +0,0 @@
-.translation-item {
-	font-size: 12px;
-	padding: 12px 15px;
-	min-height: 40px;
-	cursor: pointer;
-	overflow: hidden;
-}
-
-.translation-item:hover {
-	background-color: #fafbfc;
-}
-.translation-item.active {
-	background-color: #fffce7;
-}
-
-.translation-edit-section {
-	height: 100%;
-	overflow-y: scroll;
-	padding: 0px;
-}
-
-.translation-tool {
-	display: flex;
-	width: 100%;
-	padding: 0;
-	height: 72vh;
-}
-
-.left-side {
-	padding: 0px;
-	height: 100%;
-	overflow-y: scroll;
-}
-
-.contributed-translation {
-	padding: 0.5rem 0;
-}
diff --git a/frappe/desk/page/translation_tool/translation_tool.html b/frappe/desk/page/translation_tool/translation_tool.html
deleted file mode 100644
index a88f698584..0000000000
--- a/frappe/desk/page/translation_tool/translation_tool.html
+++ /dev/null
@@ -1,20 +0,0 @@
-
        
-	
        
-		
        
-			
        
-				{%= __("Contributed Translations") %}
-			
        
-		
        
-		
        
-		
        
-			
        
-				{%= __("Source Text") %}
-			
        
-		
        
-		
        
-	
        
-	
        
-		
        
-		
        
-	
        
-
        
diff --git a/frappe/desk/page/translation_tool/translation_tool.js b/frappe/desk/page/translation_tool/translation_tool.js
deleted file mode 100644
index 13f68e647a..0000000000
--- a/frappe/desk/page/translation_tool/translation_tool.js
+++ /dev/null
@@ -1,465 +0,0 @@
-frappe.pages['translation-tool'].on_page_load = function(wrapper) {
-	var page = frappe.ui.make_app_page({
-		parent: wrapper,
-		title: __('Translation Tool'),
-		single_column: true,
-		card_layout: true,
-	});
-
-	frappe.translation_tool = new TranslationTool(page);
-};
-
-class TranslationTool {
-	constructor(page) {
-		this.page = page;
-		this.wrapper = $(page.body);
-		this.wrapper.append(frappe.render_template('translation_tool'));
-		frappe.utils.bind_actions_with_object(this.wrapper, this);
-		this.active_translation = null;
-		this.edited_translations = {};
-		this.setup_search_box();
-		this.setup_language_filter();
-		this.page.set_primary_action(
-			__('Contribute Translations'),
-			this.show_confirmation_dialog.bind(this)
-		);
-		this.page.set_secondary_action(
-			__('Refresh'),
-			this.fetch_messages_then_render.bind(this)
-		);
-		this.update_header();
-	}
-
-	setup_language_filter() {
-		let languages = Object.keys(frappe.boot.lang_dict).map(language_label => {
-			let value = frappe.boot.lang_dict[language_label];
-			return {
-				label: `${language_label} (${value})`,
-				value: value
-			};
-		});
-
-		let language_selector = this.page.add_field({
-			fieldname: 'language',
-			fieldtype: 'Select',
-			options: languages,
-			change: () => {
-				let language = language_selector.get_value();
-				localStorage.setItem('translation_language', language);
-				this.language = language;
-				this.fetch_messages_then_render();
-			}
-		});
-		let translation_language = localStorage.getItem('translation_language');
-		if (translation_language || frappe.boot.lang !== 'en') {
-			language_selector.set_value(translation_language || frappe.boot.lang);
-		} else {
-			frappe.prompt(
-				{
-					label: __('Please select target language for translation'),
-					fieldname: 'language',
-					fieldtype: 'Select',
-					options: languages,
-					reqd: 1
-				},
-				values => {
-					language_selector.set_value(values.language);
-				},
-				__('Select Language')
-			);
-		}
-	}
-
-	setup_search_box() {
-		let search_box = this.page.add_field({
-			fieldname: 'search',
-			fieldtype: 'Data',
-			label: __('Search Source Text'),
-			change: () => {
-				this.search_text = search_box.get_value();
-				this.fetch_messages_then_render();
-			}
-		});
-	}
-
-	fetch_messages_then_render() {
-		this.fetch_messages().then(messages => {
-			this.messages = messages;
-			this.render_messages(messages);
-		});
-		this.setup_local_contributions();
-	}
-
-	fetch_messages() {
-		frappe.dom.freeze(__('Fetching...'));
-		return frappe
-			.xcall('frappe.translate.get_messages', {
-				language: this.language,
-				search_text: this.search_text
-			})
-			.then(messages => {
-				return messages;
-			})
-			.finally(() => {
-				frappe.dom.unfreeze();
-			});
-	}
-
-	render_messages(messages) {
-		let template = message => `
-			
        
-				
        
-					
-						${frappe.utils.escape_html(message.source_text)}
-					
-				
        
-			
        
-		`;
-
-		let html = messages.map(template).join('');
-		this.wrapper.find('.translation-item-container').html(html);
-	}
-
-	on_translation_click(e, $el) {
-		let message_id = decodeURIComponent($el.data('message-id'));
-		this.wrapper.find('.translation-item').removeClass('active');
-		$el.addClass('active');
-		this.active_translation = this.messages.find(m => m.id === message_id);
-		this.edit_translation(this.active_translation);
-	}
-
-	edit_translation(translation) {
-		if (this.form) {
-			this.form.set_values({});
-		}
-		this.get_additional_info(translation.id).then(data => {
-			this.make_edit_form(translation, data);
-		});
-	}
-
-	get_additional_info(source_id) {
-		frappe.dom.freeze('Fetching...');
-		return frappe.xcall('frappe.translate.get_source_additional_info', {
-			source: source_id,
-			language: this.page.fields_dict['language'].get_value()
-		}).finally(frappe.dom.unfreeze);
-	}
-
-	make_edit_form(translation, { contributions, positions }) {
-		if (!this.form) {
-			this.form = new frappe.ui.FieldGroup({
-				fields: [
-					{
-						fieldtype: 'HTML',
-						fieldname: 'header',
-						read_only: 1
-					},
-					{
-						fieldtype: 'Data',
-						fieldname: 'id',
-						hidden: 1
-					},
-					{
-						label: 'Source Text',
-						fieldtype: 'Code',
-						fieldname: 'source_text',
-						read_only: 1,
-						enable_copy_button: 1
-					},
-					{
-						label: 'Context',
-						fieldtype: 'Code',
-						fieldname: 'context',
-						read_only: 1
-					},
-					{
-						label: 'DocType',
-						fieldtype: 'Data',
-						fieldname: 'doctype',
-						read_only: 1
-					},
-					{
-						label: 'Translated Text',
-						fieldtype: 'Small Text',
-						fieldname: 'translated_text',
-					},
-					{
-						label: 'Suggest',
-						fieldtype: 'Button',
-						click: () => {
-							let { id, translated_text, source_text } = this.form.get_values();
-							let existing_value = this.form.translation_dict.translated_text;
-							if (
-								is_null(translated_text) ||
-								existing_value === translated_text
-							) {
-								delete this.edited_translations[id];
-							} else if (existing_value !== translated_text) {
-								this.edited_translations[id] = {
-									id,
-									translated_text,
-									source_text
-								};
-							}
-							this.update_header();
-						}
-					},
-					{
-						fieldtype: 'Section Break',
-						fieldname: 'contributed_translations_section',
-						label: 'Contributed Translations'
-					},
-					{
-						fieldtype: 'HTML',
-						fieldname: 'contributed_translations'
-					},
-					{
-						fieldtype: 'Section Break',
-						collapsible: 1,
-						label: 'Occurences in source code'
-					},
-					{
-						fieldtype: 'HTML',
-						fieldname: 'positions'
-					},
-				],
-				body: this.wrapper.find('.translation-edit-form')
-			});
-
-			this.form.make();
-			this.setup_header();
-		}
-
-		this.form.set_values(translation);
-		this.form.translation_dict = translation;
-		this.form.set_df_property('doctype', 'hidden', !translation.doctype);
-		this.form.set_df_property('context', 'hidden', !translation.context);
-		this.set_status(translation);
-
-		this.setup_contributions(contributions);
-		this.setup_positions(positions);
-	}
-
-	setup_header() {
-		this.form.get_field('header').$wrapper.html(`
        
-			
-		
        `);
-	}
-
-	set_status(translation) {
-		this.form.get_field('header').$wrapper.find('.translation-status').html(`
-			
-				${this.get_indicator_status_text(translation)}
-			
-		`);
-	}
-
-	setup_positions(positions) {
-		let position_dom = '';
-		if (positions && positions.length) {
-			position_dom = positions.map(position => {
-				if (position.path.startsWith('DocType: ')) {
-					return `
        
-						${position.path}
-					
        `;
-				} else {
-					return ``;
-				}
-			}).join('');
-		}
-		this.form.get_field('positions').$wrapper.html(position_dom);
-	}
-
-	setup_contributions(contributions) {
-		const contributions_exists = contributions && contributions.length;
-		if (contributions_exists) {
-			let contributions_html = contributions.map(c => {
-				return `
-					
        
-						
        ${c.translated}
        
-						
        
-							${comment_when(c.creation)}
-						
        
-					
        
-				`;
-			});
-			this.form.get_field('contributed_translations').html(contributions_html);
-		}
-		this.form.set_df_property('contributed_translations_section', 'hidden', !contributions_exists);
-	}
-	show_confirmation_dialog() {
-		this.confirmation_dialog = new frappe.ui.Dialog({
-			fields: [
-				{
-					label: __('Language'),
-					fieldname: 'language',
-					fieldtype: 'Data',
-					read_only: 1,
-					bold: 1,
-					default: this.language
-				},
-				{
-					fieldtype: 'HTML',
-					fieldname: 'edited_translations'
-				}
-			],
-			title: __('Confirm Translations'),
-			no_submit_on_enter: true,
-			primary_action_label: __('Submit'),
-			primary_action: values => {
-				this.create_translations(values).then(this.confirmation_dialog.hide());
-			}
-		});
-		this.confirmation_dialog.get_field('edited_translations').html(`
-			
        
 				
        Progress / Wait Event
 				
        
 				
        
-				
-					
-					
-				
-				${Object.values(this.edited_translations).map(t => `
-					
-						
-						
-					
-				`).join('')}
-			
        ${__('Source Text')}${__('Translated Text')}
        ${t.source_text}${t.translated_text}
        
-		`);
-		this.confirmation_dialog.show();
-	}
-	create_translations() {
-		frappe.dom.freeze(__('Submitting...'));
-		return frappe
-			.xcall(
-				'frappe.core.doctype.translation.translation.create_translations',
-				{
-					translation_map: this.edited_translations,
-					language: this.language
-				}
-			)
-			.then(() => {
-				frappe.dom.unfreeze();
-				frappe.show_alert({ message: __('Successfully Submitted!'), indicator: 'success'});
-				this.edited_translations = {};
-				this.update_header();
-				this.fetch_messages_then_render();
-			})
-			.finally(() => frappe.dom.unfreeze());
-	}
-
-	setup_local_contributions() {
-		// TODO: Refactor
-		frappe
-			.xcall('frappe.translate.get_contributions', {
-				language: this.language
-			})
-			.then(messages => {
-				let template = message => `
-					
        
-						
        
-							
-								${frappe.utils.escape_html(message.source_text)}
-							
-						
        
-					
        
-				`;
-
-				let html = messages.map(template).join('');
-				this.wrapper.find('.translation-item-tr').html(html);
-			});
-	}
-
-	show_translation_status_modal(e, $el) {
-		let message_id = decodeURIComponent($el.data('message-id'));
-
-		frappe.xcall('frappe.translate.get_contribution_status', { message_id })
-			.then(doc => {
-				let d = new frappe.ui.Dialog({
-					title: __('Contribution Status'),
-					fields: [
-						{
-							fieldname: 'source_message',
-							label: __('Source Message'),
-							fieldtype: 'Data',
-							read_only: 1
-						},
-						{
-							fieldname: 'translated',
-							label: __('Translated Message'),
-							fieldtype: 'Data',
-							read_only: 1
-						},
-						{
-							fieldname: 'contribution_status',
-							label: __('Contribution Status'),
-							fieldtype: 'Data',
-							read_only: 1
-						},
-						{
-							fieldname: 'modified_by',
-							label: __('Verified By'),
-							fieldtype: 'Data',
-							read_only: 1,
-							depends_on: doc => {
-								return doc.contribution_status == 'Verified';
-							}
-						},
-					]
-				});
-				d.set_values(doc);
-				d.show();
-			});
-	}
-
-	update_header() {
-		let edited_translations_count = Object.keys(this.edited_translations)
-			.length;
-		if (edited_translations_count) {
-			let message = '';
-			if (edited_translations_count == 1) {
-				message = __('{0} translation pending', [edited_translations_count]);
-			} else {
-				message = __('{0} translations pending', [edited_translations_count]);
-			}
-			this.page.set_indicator(message, 'orange');
-		} else {
-			this.page.set_indicator('');
-		}
-		this.page.btn_primary.prop('disabled', !edited_translations_count);
-	}
-
-	get_indicator_color(message_obj) {
-		return !message_obj.translated ? 'red' : message_obj.translated_by_google ? 'orange' : 'blue';
-	}
-
-	get_indicator_status_text(message_obj) {
-		if (!message_obj.translated) {
-			return __('Untranslated');
-		} else if (message_obj.translated_by_google) {
-			return __('Google Translation');
-		} else {
-			return __('Community Contribution');
-		}
-	}
-
-	get_contribution_indicator_color(message_obj) {
-		return message_obj.contribution_status == 'Pending' ? 'orange' : 'green';
-	}
-
-	get_code_url(path, line_no, app) {
-		const code_path = path.substring(`apps/${app}`.length);
-		return `https://github.com/frappe/${app}/blob/develop/${code_path}#L${line_no}`;
-	}
-}
diff --git a/frappe/desk/page/translation_tool/translation_tool.json b/frappe/desk/page/translation_tool/translation_tool.json
deleted file mode 100644
index a54b2a4724..0000000000
--- a/frappe/desk/page/translation_tool/translation_tool.json
+++ /dev/null
@@ -1,26 +0,0 @@
-{
- "content": null,
- "creation": "2020-01-30 15:16:12.136323",
- "docstatus": 0,
- "doctype": "Page",
- "idx": 0,
- "modified": "2020-01-30 15:16:23.273733",
- "modified_by": "Administrator",
- "module": "Desk",
- "name": "translation-tool",
- "owner": "Administrator",
- "page_name": "Translation Tool",
- "roles": [
-  {
-   "role": "System Manager"
-  },
-  {
-   "role": "Translator"
-  }
- ],
- "script": null,
- "standard": "Yes",
- "style": null,
- "system_page": 1,
- "title": "Translation Tool"
-}
\ No newline at end of file
diff --git a/frappe/desk/page/user_profile/user_profile.js b/frappe/desk/page/user_profile/user_profile.js
index 5890975e69..e4cef59cc7 100644
--- a/frappe/desk/page/user_profile/user_profile.js
+++ b/frappe/desk/page/user_profile/user_profile.js
@@ -1,5 +1,5 @@
-frappe.pages['user-profile'].on_page_load = function (wrapper) {
-	frappe.require('user_profile_controller.bundle.js', () => {
+frappe.pages["user-profile"].on_page_load = function (wrapper) {
+	frappe.require("user_profile_controller.bundle.js", () => {
 		let user_profile = new frappe.ui.UserProfile(wrapper);
 		user_profile.show();
 	});
diff --git a/frappe/desk/page/user_profile/user_profile.py b/frappe/desk/page/user_profile/user_profile.py
index f9c8d98869..3013df54a5 100644
--- a/frappe/desk/page/user_profile/user_profile.py
+++ b/frappe/desk/page/user_profile/user_profile.py
@@ -1,6 +1,8 @@
 from datetime import datetime
 
 import frappe
+from frappe.query_builder import Interval, Order
+from frappe.query_builder.functions import Date, Sum, UnixTimestamp
 from frappe.utils import getdate
 
 
@@ -11,27 +13,24 @@ def get_energy_points_heatmap_data(user, date):
 	except Exception:
 		date = getdate()
 
+	eps_log = frappe.qb.DocType("Energy Point Log")
+
 	return dict(
-		frappe.db.sql(
-			"""select unix_timestamp(date(creation)), sum(points)
-		from `tabEnergy Point Log`
-		where
-			date(creation) > subdate('{date}', interval 1 year) and
-			date(creation) < subdate('{date}', interval -1 year) and
-			user = %s and
-			type != 'Review'
-		group by date(creation)
-		order by creation asc""".format(
-				date=date
-			),
-			user,
-		)
+		frappe.qb.from_(eps_log)
+		.select(UnixTimestamp(Date(eps_log.creation)), Sum(eps_log.points))
+		.where(eps_log.user == user)
+		.where(eps_log["type"] != "Review")
+		.where(Date(eps_log.creation) > Date(date) - Interval(years=1))
+		.where(Date(eps_log.creation) < Date(date) + Interval(years=1))
+		.groupby(Date(eps_log.creation))
+		.orderby(Date(eps_log.creation), order=Order.asc)
+		.run()
 	)
 
 
 @frappe.whitelist()
 def get_energy_points_percentage_chart_data(user, field):
-	result = frappe.db.get_all(
+	result = frappe.get_all(
 		"Energy Point Log",
 		filters={"user": user, "type": ["!=", "Review"]},
 		group_by=field,
@@ -49,18 +48,18 @@ def get_energy_points_percentage_chart_data(user, field):
 @frappe.whitelist()
 def get_user_rank(user):
 	month_start = datetime.today().replace(day=1)
-	monthly_rank = frappe.db.get_all(
+	monthly_rank = frappe.get_all(
 		"Energy Point Log",
-		group_by="user",
+		group_by="`tabEnergy Point Log`.`user`",
 		filters={"creation": [">", month_start], "type": ["!=", "Review"]},
 		fields=["user", "sum(points)"],
 		order_by="sum(points) desc",
 		as_list=True,
 	)
 
-	all_time_rank = frappe.db.get_all(
+	all_time_rank = frappe.get_all(
 		"Energy Point Log",
-		group_by="user",
+		group_by="`tabEnergy Point Log`.`user`",
 		filters={"type": ["!=", "Review"]},
 		fields=["user", "sum(points)"],
 		order_by="sum(points) desc",
diff --git a/frappe/desk/page/user_profile/user_profile_controller.js b/frappe/desk/page/user_profile/user_profile_controller.js
index 40b542d5c3..5103bd8a19 100644
--- a/frappe/desk/page/user_profile/user_profile_controller.js
+++ b/frappe/desk/page/user_profile/user_profile_controller.js
@@ -1,5 +1,5 @@
 import BaseTimeline from "../../../public/js/frappe/form/footer/base_timeline";
-frappe.provide('frappe.energy_points');
+frappe.provide("frappe.energy_points");
 
 class UserProfile {
 	constructor(wrapper) {
@@ -7,9 +7,9 @@ class UserProfile {
 		this.page = frappe.ui.make_app_page({
 			parent: wrapper,
 		});
-		this.sidebar = this.wrapper.find('.layout-side-section');
-		this.main_section = this.wrapper.find('.layout-main-section');
-		this.wrapper.bind('show', () => {
+		this.sidebar = this.wrapper.find(".layout-side-section");
+		this.main_section = this.wrapper.find(".layout-main-section");
+		this.wrapper.bind("show", () => {
 			this.show();
 		});
 	}
@@ -17,13 +17,13 @@ class UserProfile {
 	show() {
 		let route = frappe.get_route();
 		this.user_id = route[1] || frappe.session.user;
-		frappe.dom.freeze(__('Loading user profile') + '...');
-		frappe.db.exists('User', this.user_id).then(exists => {
+		frappe.dom.freeze(__("Loading user profile") + "...");
+		frappe.db.exists("User", this.user_id).then((exists) => {
 			frappe.dom.unfreeze();
 			if (exists) {
 				this.make_user_profile();
 			} else {
-				frappe.msgprint(__('User does not exist'));
+				frappe.msgprint(__("User does not exist"));
 			}
 		});
 	}
@@ -32,7 +32,7 @@ class UserProfile {
 		this.user = frappe.user_info(this.user_id);
 		this.page.set_title(this.user.fullname);
 		this.setup_user_search();
-		this.main_section.empty().append(frappe.render_template('user_profile'));
+		this.main_section.empty().append(frappe.render_template("user_profile"));
 		this.energy_points = 0;
 		this.review_points = 0;
 		this.rank = 0;
@@ -41,91 +41,92 @@ class UserProfile {
 		this.render_points_and_rank();
 		this.render_heatmap();
 		this.render_line_chart();
-		this.render_percentage_chart('type', 'Type Distribution');
+		this.render_percentage_chart("type", "Type Distribution");
 		this.create_percentage_chart_filters();
 		this.setup_user_activity_timeline();
 	}
 
 	setup_user_search() {
 		this.$user_search_button = this.page.set_secondary_action(
-			__('Change User'),
+			__("Change User"),
 			() => this.show_user_search_dialog(),
-			{ icon: 'change', size: 'sm' }
+			{ icon: "change", size: "sm" }
 		);
 	}
 
 	show_user_search_dialog() {
 		let dialog = new frappe.ui.Dialog({
-			title: __('Change User'),
+			title: __("Change User"),
 			fields: [
 				{
-					fieldtype: 'Link',
-					fieldname: 'user',
-					options: 'User',
-					label: __('User'),
-				}
+					fieldtype: "Link",
+					fieldname: "user",
+					options: "User",
+					label: __("User"),
+				},
 			],
-			primary_action_label: __('Go'),
+			primary_action_label: __("Go"),
 			primary_action: ({ user }) => {
 				dialog.hide();
-				frappe.set_route('user-profile', user);
-			}
+				frappe.set_route("user-profile", user);
+			},
 		});
 		dialog.show();
 	}
 
 	render_heatmap() {
-		this.heatmap = new frappe.Chart('.performance-heatmap', {
-			type: 'heatmap',
-			countLabel: 'Energy Points',
+		this.heatmap = new frappe.Chart(".performance-heatmap", {
+			type: "heatmap",
+			countLabel: "Energy Points",
 			data: {},
 			discreteDomains: 1,
 			radius: 3,
-			height: 150
+			height: 150,
 		});
 		this.update_heatmap_data();
 		this.create_heatmap_chart_filters();
 	}
 
 	update_heatmap_data(date_from) {
-		frappe.xcall('frappe.desk.page.user_profile.user_profile.get_energy_points_heatmap_data', {
-			user: this.user_id,
-			date: date_from || frappe.datetime.year_start(),
-		}).then((r) => {
-			this.heatmap.update({ dataPoints: r });
-		});
+		frappe
+			.xcall("frappe.desk.page.user_profile.user_profile.get_energy_points_heatmap_data", {
+				user: this.user_id,
+				date: date_from || frappe.datetime.year_start(),
+			})
+			.then((r) => {
+				this.heatmap.update({ dataPoints: r });
+			});
 	}
 
-
 	render_line_chart() {
 		this.line_chart_filters = [
-			['Energy Point Log', 'user', '=', this.user_id, false],
-			['Energy Point Log', 'type', '!=', 'Review', false]
+			["Energy Point Log", "user", "=", this.user_id, false],
+			["Energy Point Log", "type", "!=", "Review", false],
 		];
 
 		this.line_chart_config = {
-			timespan: 'Last Month',
-			time_interval: 'Daily',
-			type: 'Line',
-			value_based_on: 'points',
-			chart_type: 'Sum',
-			document_type: 'Energy Point Log',
-			name: 'Energy Points',
-			width: 'half',
-			based_on: 'creation'
+			timespan: "Last Month",
+			time_interval: "Daily",
+			type: "Line",
+			value_based_on: "points",
+			chart_type: "Sum",
+			document_type: "Energy Point Log",
+			name: "Energy Points",
+			width: "half",
+			based_on: "creation",
 		};
 
-		this.line_chart = new frappe.Chart('.performance-line-chart', {
-			type: 'line',
+		this.line_chart = new frappe.Chart(".performance-line-chart", {
+			type: "line",
 			height: 200,
 			data: {
 				labels: [],
-				datasets: [{}]
+				datasets: [{}],
 			},
-			colors: ['purple'],
+			colors: ["purple"],
 			axisOptions: {
-				xIsSeries: 1
-			}
+				xIsSeries: 1,
+			},
 		});
 		this.update_line_chart_data();
 		this.create_line_chart_filters();
@@ -134,217 +135,258 @@ class UserProfile {
 	update_line_chart_data() {
 		this.line_chart_config.filters_json = JSON.stringify(this.line_chart_filters);
 
-		frappe.xcall('frappe.desk.doctype.dashboard_chart.dashboard_chart.get', {
-			chart: this.line_chart_config,
-			no_cache: 1,
-		}).then(chart => {
-			this.line_chart.update(chart);
-		});
+		frappe
+			.xcall("frappe.desk.doctype.dashboard_chart.dashboard_chart.get", {
+				chart: this.line_chart_config,
+				no_cache: 1,
+			})
+			.then((chart) => {
+				this.line_chart.update(chart);
+			});
 	}
 
 	// eslint-disable-next-line no-unused-vars
 	render_percentage_chart(field, title) {
-		frappe.xcall('frappe.desk.page.user_profile.user_profile.get_energy_points_percentage_chart_data', {
-			user: this.user_id,
-			field: field
-		}).then(chart => {
-			if (chart.labels.length) {
-				this.percentage_chart = new frappe.Chart('.performance-percentage-chart', {
-					type: 'percentage',
-					data: {
-						labels: chart.labels,
-						datasets: chart.datasets
-					},
-					truncateLegends: 1,
-					barOptions: {
-						height: 11,
-						depth: 1
-					},
-					height: 200,
-					maxSlices: 8,
-					colors: ['purple', 'blue', 'cyan', 'teal', 'pink', 'red', 'orange', 'yellow'],
-				});
-			} else {
-				this.wrapper.find('.percentage-chart-container').hide();
-			}
-		});
+		frappe
+			.xcall(
+				"frappe.desk.page.user_profile.user_profile.get_energy_points_percentage_chart_data",
+				{
+					user: this.user_id,
+					field: field,
+				}
+			)
+			.then((chart) => {
+				if (chart.labels.length) {
+					this.percentage_chart = new frappe.Chart(".performance-percentage-chart", {
+						type: "percentage",
+						data: {
+							labels: chart.labels,
+							datasets: chart.datasets,
+						},
+						truncateLegends: 1,
+						barOptions: {
+							height: 11,
+							depth: 1,
+						},
+						height: 200,
+						maxSlices: 8,
+						colors: [
+							"purple",
+							"blue",
+							"cyan",
+							"teal",
+							"pink",
+							"red",
+							"orange",
+							"yellow",
+						],
+					});
+				} else {
+					this.wrapper.find(".percentage-chart-container").hide();
+				}
+			});
 	}
 
 	create_line_chart_filters() {
 		let filters = [
 			{
-				label: 'All',
-				options: ['All', 'Auto', 'Criticism', 'Appreciation', 'Revert'],
+				label: "All",
+				options: ["All", "Auto", "Criticism", "Appreciation", "Revert"],
 				action: (selected_item) => {
-					if (selected_item === 'All') {
+					if (selected_item === "All") {
 						this.line_chart_filters = [
-							['Energy Point Log', 'user', '=', this.user_id, false],
-							['Energy Point Log', 'type', '!=', 'Review', false]
+							["Energy Point Log", "user", "=", this.user_id, false],
+							["Energy Point Log", "type", "!=", "Review", false],
 						];
 					} else {
-						this.line_chart_filters[1] = ['Energy Point Log', 'type', '=', selected_item, false];
+						this.line_chart_filters[1] = [
+							"Energy Point Log",
+							"type",
+							"=",
+							selected_item,
+							false,
+						];
 					}
 					this.update_line_chart_data();
-				}
+				},
 			},
 			{
-				label: 'Last Month',
-				options: ['Last Week', 'Last Month', 'Last Quarter', 'Last Year'],
+				label: "Last Month",
+				options: ["Last Week", "Last Month", "Last Quarter", "Last Year"],
 				action: (selected_item) => {
 					this.line_chart_config.timespan = selected_item;
 					this.update_line_chart_data();
-				}
+				},
 			},
 			{
-				label: 'Daily',
-				options: ['Daily', 'Weekly', 'Monthly'],
+				label: "Daily",
+				options: ["Daily", "Weekly", "Monthly"],
 				action: (selected_item) => {
 					this.line_chart_config.time_interval = selected_item;
 					this.update_line_chart_data();
-				}
+				},
 			},
 		];
-		frappe.dashboard_utils.render_chart_filters(filters, 'chart-filter', '.line-chart-options', 1);
+		frappe.dashboard_utils.render_chart_filters(
+			filters,
+			"chart-filter",
+			".line-chart-options",
+			1
+		);
 	}
 
 	create_percentage_chart_filters() {
 		let filters = [
 			{
-				label: 'Type',
-				options: ['Type', 'Reference Doctype', 'Rule'],
-				fieldnames: ['type', 'reference_doctype', 'rule'],
+				label: "Type",
+				options: ["Type", "Reference Doctype", "Rule"],
+				fieldnames: ["type", "reference_doctype", "rule"],
 				action: (selected_item, fieldname) => {
-					let title = selected_item + ' Distribution';
+					let title = selected_item + " Distribution";
 					this.render_percentage_chart(fieldname, title);
-				}
+				},
 			},
 		];
-		frappe.dashboard_utils.render_chart_filters(filters, 'chart-filter', '.percentage-chart-options');
+		frappe.dashboard_utils.render_chart_filters(
+			filters,
+			"chart-filter",
+			".percentage-chart-options"
+		);
 	}
 
 	create_heatmap_chart_filters() {
 		let filters = [
 			{
 				label: frappe.dashboard_utils.get_year(frappe.datetime.now_date()),
-				options: frappe.dashboard_utils.get_years_since_creation(frappe.boot.user.creation),
+				options: frappe.dashboard_utils.get_years_since_creation(
+					frappe.boot.user.creation
+				),
 				action: (selected_item) => {
 					this.update_heatmap_data(frappe.datetime.obj_to_str(selected_item));
-				}
+				},
 			},
 		];
-		frappe.dashboard_utils.render_chart_filters(filters, 'chart-filter', '.heatmap-options');
+		frappe.dashboard_utils.render_chart_filters(filters, "chart-filter", ".heatmap-options");
 	}
 
-
 	edit_profile() {
 		let edit_profile_dialog = new frappe.ui.Dialog({
-			title: __('Edit Profile'),
+			title: __("Edit Profile"),
 			fields: [
 				{
-					fieldtype: 'Attach Image',
-					fieldname: 'user_image',
-					label: 'Profile Image',
+					fieldtype: "Attach Image",
+					fieldname: "user_image",
+					label: "Profile Image",
 				},
 				{
-					fieldtype: 'Data',
-					fieldname: 'interest',
-					label: 'Interests',
+					fieldtype: "Data",
+					fieldname: "interest",
+					label: "Interests",
 				},
 				{
-					fieldtype: 'Column Break'
+					fieldtype: "Column Break",
 				},
 				{
-					fieldtype: 'Data',
-					fieldname: 'location',
-					label: 'Location',
+					fieldtype: "Data",
+					fieldname: "location",
+					label: "Location",
 				},
 				{
-					fieldtype: 'Section Break',
-					fieldname: 'Interest',
+					fieldtype: "Section Break",
+					fieldname: "Interest",
 				},
 				{
-					fieldtype: 'Small Text',
-					fieldname: 'bio',
-					label: 'Bio',
-				}
+					fieldtype: "Small Text",
+					fieldname: "bio",
+					label: "Bio",
+				},
 			],
-			primary_action: values => {
+			primary_action: (values) => {
 				edit_profile_dialog.disable_primary_action();
-				frappe.xcall('frappe.desk.page.user_profile.user_profile.update_profile_info', {
-					profile_info: values
-				}).then(user => {
-					user.image = user.user_image;
-					this.user = Object.assign(values, user);
-					edit_profile_dialog.hide();
-					this.render_user_details();
-				}).finally(() => {
-					edit_profile_dialog.enable_primary_action();
-				});
+				frappe
+					.xcall("frappe.desk.page.user_profile.user_profile.update_profile_info", {
+						profile_info: values,
+					})
+					.then((user) => {
+						user.image = user.user_image;
+						this.user = Object.assign(values, user);
+						edit_profile_dialog.hide();
+						this.render_user_details();
+					})
+					.finally(() => {
+						edit_profile_dialog.enable_primary_action();
+					});
 			},
-			primary_action_label: __('Save')
+			primary_action_label: __("Save"),
 		});
 
 		edit_profile_dialog.set_values({
 			user_image: this.user.image,
 			location: this.user.location,
 			interest: this.user.interest,
-			bio: this.user.bio
+			bio: this.user.bio,
 		});
 		edit_profile_dialog.show();
 	}
 
 	render_user_details() {
-		this.sidebar.empty().append(frappe.render_template('user_profile_sidebar', {
-			user_image: this.user.image,
-			user_abbr: this.user.abbr,
-			user_location: this.user.location,
-			user_interest: this.user.interest,
-			user_bio: this.user.bio,
-		}));
+		this.sidebar.empty().append(
+			frappe.render_template("user_profile_sidebar", {
+				user_image: this.user.image,
+				user_abbr: this.user.abbr,
+				user_location: this.user.location,
+				user_interest: this.user.interest,
+				user_bio: this.user.bio,
+			})
+		);
 
 		this.setup_user_profile_links();
 	}
 
 	setup_user_profile_links() {
 		if (this.user_id !== frappe.session.user) {
-			this.wrapper.find('.profile-links').hide();
+			this.wrapper.find(".profile-links").hide();
 		} else {
-			this.wrapper.find('.edit-profile-link').on('click', () => {
+			this.wrapper.find(".edit-profile-link").on("click", () => {
 				this.edit_profile();
 			});
 
-			this.wrapper.find('.user-settings-link').on('click', () => {
+			this.wrapper.find(".user-settings-link").on("click", () => {
 				this.go_to_user_settings();
 			});
 		}
 	}
 
 	get_user_rank() {
-		return frappe.xcall('frappe.desk.page.user_profile.user_profile.get_user_rank', {
-			user: this.user_id,
-		}).then(r => {
-			if (r.monthly_rank.length) this.month_rank = r.monthly_rank[0];
-			if (r.all_time_rank.length) this.rank = r.all_time_rank[0];
-		});
+		return frappe
+			.xcall("frappe.desk.page.user_profile.user_profile.get_user_rank", {
+				user: this.user_id,
+			})
+			.then((r) => {
+				if (r.monthly_rank.length) this.month_rank = r.monthly_rank[0];
+				if (r.all_time_rank.length) this.rank = r.all_time_rank[0];
+			});
 	}
 
 	get_user_points() {
-		return frappe.xcall(
-			'frappe.social.doctype.energy_point_log.energy_point_log.get_user_energy_and_review_points',
-			{
-				user: this.user_id,
-			}
-		).then(r => {
-			if (r[this.user_id]) {
-				this.energy_points = r[this.user_id].energy_points;
-				this.review_points = r[this.user_id].review_points;
-			}
-		});
+		return frappe
+			.xcall(
+				"frappe.social.doctype.energy_point_log.energy_point_log.get_user_energy_and_review_points",
+				{
+					user: this.user_id,
+				}
+			)
+			.then((r) => {
+				if (r[this.user_id]) {
+					this.energy_points = r[this.user_id].energy_points;
+					this.review_points = r[this.user_id].review_points;
+				}
+			});
 	}
 
 	render_points_and_rank() {
-		let $profile_details = this.wrapper.find('.user-stats');
-		let $profile_details_wrapper = this.wrapper.find('.user-stats-detail');
+		let $profile_details = this.wrapper.find(".user-stats");
+		let $profile_details_wrapper = this.wrapper.find(".user-stats-detail");
 
 		const _get_stat_dom = (value, label, icon) => {
 			return `
        
@@ -359,10 +401,10 @@ class UserProfile {
 		this.get_user_rank().then(() => {
 			this.get_user_points().then(() => {
 				let html = $(`
-					${_get_stat_dom(this.energy_points, __('Energy Points'), "color-energy-points")}
-					${_get_stat_dom(this.review_points, __('Review Points'), "color-review-points")}
-					${_get_stat_dom(this.rank, __('Rank'), "color-rank")}
-					${_get_stat_dom(this.month_rank, __('Monthly Rank'), "color-monthly-rank")}
+					${_get_stat_dom(this.energy_points, __("Energy Points"), "color-energy-points")}
+					${_get_stat_dom(this.review_points, __("Review Points"), "color-review-points")}
+					${_get_stat_dom(this.rank, __("Rank"), "color-rank")}
+					${_get_stat_dom(this.month_rank, __("Monthly Rank"), "color-monthly-rank")}
 				`);
 
 				$profile_details.append(html);
@@ -372,14 +414,14 @@ class UserProfile {
 	}
 
 	go_to_user_settings() {
-		frappe.set_route('Form', 'User', this.user_id);
+		frappe.set_route("Form", "User", this.user_id);
 	}
 
 	setup_user_activity_timeline() {
 		this.user_activity_timeline = new UserProfileTimeline({
-			parent: this.wrapper.find('.recent-activity-list'),
-			footer: this.wrapper.find('.recent-activity-footer'),
-			user: this.user_id
+			parent: this.wrapper.find(".recent-activity-list"),
+			footer: this.wrapper.find(".recent-activity-footer"),
+			user: this.user_id,
 		});
 
 		this.user_activity_timeline.refresh();
@@ -397,24 +439,27 @@ class UserProfileTimeline extends BaseTimeline {
 		return this.get_user_activity_data().then((activities) => {
 			if (!activities.length) {
 				this.show_more_button.hide();
-				this.timeline_wrapper.html(`
        ${__('No activities to show')}
        `);
+				this.timeline_wrapper.html(`
        ${__("No activities to show")}
        `);
 				return;
 			}
 			this.show_more_button.toggle(activities.length === this.activity_limit);
-			this.timeline_items = activities.map((activity) => this.get_activity_timeline_item(activity));
+			this.timeline_items = activities.map((activity) =>
+				this.get_activity_timeline_item(activity)
+			);
 		});
 	}
 
 	get_user_activity_data() {
-		return frappe.xcall('frappe.desk.page.user_profile.user_profile.get_energy_points_list', {
+		return frappe.xcall("frappe.desk.page.user_profile.user_profile.get_energy_points_list", {
 			start: this.activity_start,
 			limit: this.activity_limit,
-			user: this.user
+			user: this.user,
 		});
 	}
 
 	get_activity_timeline_item(data) {
-		let icon = data.type == 'Appreciation' ? 'clap': data.type == 'Criticism' ? 'criticize': null;
+		let icon =
+			data.type == "Appreciation" ? "clap" : data.type == "Criticism" ? "criticize" : null;
 		return {
 			icon: icon,
 			creation: data.creation,
@@ -424,23 +469,27 @@ class UserProfileTimeline extends BaseTimeline {
 	}
 
 	setup_show_more_activity() {
-		this.show_more_button = $(`${__('Show More Activity')}`);
+		this.show_more_button = $(
+			`${__("Show More Activity")}`
+		);
 		this.show_more_button.hide();
 		this.footer.append(this.show_more_button);
-		this.show_more_button.on('click', () => this.show_more_activity());
+		this.show_more_button.on("click", () => this.show_more_activity());
 	}
 
 	show_more_activity() {
 		this.activity_start += this.activity_limit;
-		this.get_user_activity_data().then(activities => {
+		this.get_user_activity_data().then((activities) => {
 			if (!activities.length || activities.length < this.activity_limit) {
 				this.show_more_button.hide();
 			}
-			let timeline_items = activities.map((activity) => this.get_activity_timeline_item(activity));
+			let timeline_items = activities.map((activity) =>
+				this.get_activity_timeline_item(activity)
+			);
 			timeline_items.map((item) => this.add_timeline_item(item, true));
 		});
 	}
 }
 
-frappe.provide('frappe.ui');
+frappe.provide("frappe.ui");
 frappe.ui.UserProfile = UserProfile;
diff --git a/frappe/desk/query_report.py b/frappe/desk/query_report.py
index e88a453e64..69cdecb6dd 100644
--- a/frappe/desk/query_report.py
+++ b/frappe/desk/query_report.py
@@ -10,24 +10,18 @@ import frappe
 import frappe.desk.reportview
 from frappe import _
 from frappe.core.utils import ljust_list
+from frappe.desk.reportview import clean_params, parse_json
 from frappe.model.utils import render_include
 from frappe.modules import get_module_path, scrub
+from frappe.monitor import add_data_to_monitor
 from frappe.permissions import get_role_permissions
-from frappe.translate import send_translations
-from frappe.utils import (
-	cint,
-	cstr,
-	flt,
-	format_duration,
-	get_html_format,
-	get_url_to_form,
-	gzip_decompress,
-)
+from frappe.utils import cint, cstr, flt, format_duration, get_html_format, sbool
 
 
 def get_report_doc(report_name):
 	doc = frappe.get_doc("Report", report_name)
 	doc.custom_columns = []
+	doc.custom_filters = []
 
 	if doc.report_type == "Custom Report":
 		custom_report_doc = doc
@@ -37,7 +31,8 @@ def get_report_doc(report_name):
 		if custom_report_doc.json:
 			data = json.loads(custom_report_doc.json)
 			if data:
-				doc.custom_columns = data["columns"]
+				doc.custom_columns = data.get("columns")
+				doc.custom_filters = data.get("filters")
 		doc.is_custom_report = True
 
 	if not doc.is_permitted():
@@ -144,35 +139,6 @@ def normalize_result(result, columns):
 	return data
 
 
-@frappe.whitelist()
-def background_enqueue_run(report_name, filters=None, user=None):
-	"""run reports in background"""
-	if not user:
-		user = frappe.session.user
-	report = get_report_doc(report_name)
-	track_instance = frappe.get_doc(
-		{
-			"doctype": "Prepared Report",
-			"report_name": report_name,
-			# This looks like an insanity but, without this it'd be very hard to find Prepared Reports matching given condition
-			# We're ensuring that spacing is consistent. e.g. JS seems to put no spaces after ":", Python on the other hand does.
-			"filters": json.dumps(json.loads(filters)),
-			"ref_report_doctype": report_name,
-			"report_type": report.report_type,
-			"query": report.query,
-			"module": report.module,
-		}
-	)
-	track_instance.insert(ignore_permissions=True)
-	frappe.db.commit()
-	track_instance.enqueue_report()
-
-	return {
-		"name": track_instance.name,
-		"redirect_url": get_url_to_form("Prepared Report", track_instance.name),
-	}
-
-
 @frappe.whitelist()
 def get_script(report_name):
 	report = get_report_doc(report_name)
@@ -201,10 +167,6 @@ def get_script(report_name):
 	if not script:
 		script = "frappe.query_reports['%s']={}" % report_name
 
-	# load translations
-	if frappe.lang != "en":
-		send_translations(frappe.get_lang_dict("report", report_name))
-
 	return {
 		"script": render_include(script),
 		"html_format": html_format,
@@ -222,6 +184,7 @@ def run(
 	custom_columns=None,
 	is_tree=False,
 	parent_field=None,
+	are_default_filters=True,
 ):
 	report = get_report_doc(report_name)
 	if not user:
@@ -234,26 +197,27 @@ def run(
 
 	result = None
 
-	if (
-		report.prepared_report
-		and not report.disable_prepared_report
-		and not ignore_prepared_report
-		and not custom_columns
-	):
+	if sbool(are_default_filters) and report.custom_filters:
+		filters = report.custom_filters
+
+	if report.prepared_report and not ignore_prepared_report and not custom_columns:
 		if filters:
 			if isinstance(filters, str):
 				filters = json.loads(filters)
 
-			dn = filters.get("prepared_report_name")
-			filters.pop("prepared_report_name", None)
+			dn = filters.pop("prepared_report_name", None)
 		else:
 			dn = ""
 		result = get_prepared_report_result(report, filters, dn, user)
 	else:
 		result = generate_report_result(report, filters, user, custom_columns, is_tree, parent_field)
+		add_data_to_monitor(report=report.reference_report or report.name)
 
 	result["add_total_row"] = report.add_total_row and not result.get("skip_total_row", False)
 
+	if sbool(are_default_filters) and report.custom_filters:
+		result["custom_filters"] = report.custom_filters
+
 	return result
 
 
@@ -274,103 +238,90 @@ def add_custom_column_data(custom_columns, result):
 
 
 def get_prepared_report_result(report, filters, dn="", user=None):
-	latest_report_data = {}
-	doc = None
-	if dn:
-		# Get specified dn
-		doc = frappe.get_doc("Prepared Report", dn)
-	else:
-		# Only look for completed prepared reports with given filters.
-		doc_list = frappe.get_all(
-			"Prepared Report",
-			filters={
-				"status": "Completed",
-				"filters": json.dumps(filters),
-				"owner": user,
-				"report_name": report.get("custom_report") or report.get("report_name"),
-			},
-			order_by="creation desc",
+	from frappe.core.doctype.prepared_report.prepared_report import get_completed_prepared_report
+
+	def get_report_data(doc, data):
+		# backwards compatibility - prepared report used to have a columns field,
+		# we now directly fetch it from the result file
+		if doc.get("columns") or isinstance(data, list):
+			columns = (doc.get("columns") and json.loads(doc.columns)) or data[0]
+			data = {"result": data}
+		else:
+			columns = data.get("columns")
+
+		for column in columns:
+			if isinstance(column, dict) and column.get("label"):
+				column["label"] = _(column["label"])
+
+		return data | {"columns": columns}
+
+	report_data = {}
+	if not dn:
+		dn = get_completed_prepared_report(
+			filters, user, report.get("custom_report") or report.get("report_name")
 		)
 
-		if doc_list:
-			# Get latest
-			doc = frappe.get_doc("Prepared Report", doc_list[0])
-
+	doc = frappe.get_doc("Prepared Report", dn) if dn else None
 	if doc:
 		try:
-			# Prepared Report data is stored in a GZip compressed JSON file
-			attached_file_name = frappe.db.get_value(
-				"File",
-				{"attached_to_doctype": doc.doctype, "attached_to_name": doc.name},
-				"name",
-			)
-			attached_file = frappe.get_doc("File", attached_file_name)
-			compressed_content = attached_file.get_content()
-			uncompressed_content = gzip_decompress(compressed_content)
-			data = json.loads(uncompressed_content.decode("utf-8"))
-			if data:
-				columns = json.loads(doc.columns) if doc.columns else data[0]
-
-				for column in columns:
-					if isinstance(column, dict) and column.get("label"):
-						column["label"] = _(column["label"])
-
-				latest_report_data = {"columns": columns, "result": data}
+			if data := json.loads(doc.get_prepared_data().decode("utf-8")):
+				report_data = get_report_data(doc, data)
 		except Exception:
-			doc.log_error("Prepared report failed")
-			frappe.delete_doc("Prepared Report", doc.name)
-			frappe.db.commit()
+			doc.log_error("Prepared report render failed")
+			frappe.msgprint(_("Prepared report render failed"))
 			doc = None
 
-	latest_report_data.update({"prepared_report": True, "doc": doc})
-
-	return latest_report_data
+	return report_data | {"prepared_report": True, "doc": doc}
 
 
 @frappe.whitelist()
 def export_query():
 	"""export from query reports"""
-	data = frappe._dict(frappe.local.form_dict)
-	data.pop("cmd", None)
-	data.pop("csrf_token", None)
+	from frappe.desk.utils import get_csv_bytes, pop_csv_params, provide_binary_file
 
-	if isinstance(data.get("filters"), str):
-		filters = json.loads(data["filters"])
+	form_params = frappe._dict(frappe.local.form_dict)
+	csv_params = pop_csv_params(form_params)
+	clean_params(form_params)
+	parse_json(form_params)
 
-	if data.get("report_name"):
-		report_name = data["report_name"]
-		frappe.permissions.can_export(
-			frappe.get_cached_value("Report", report_name, "ref_doctype"),
-			raise_exception=True,
-		)
+	report_name = form_params.report_name
+	frappe.permissions.can_export(
+		frappe.get_cached_value("Report", report_name, "ref_doctype"),
+		raise_exception=True,
+	)
 
-	file_format_type = data.get("file_format_type")
-	custom_columns = frappe.parse_json(data.get("custom_columns", "[]"))
-	include_indentation = data.get("include_indentation")
-	visible_idx = data.get("visible_idx")
+	file_format_type = form_params.file_format_type
+	custom_columns = frappe.parse_json(form_params.custom_columns or "[]")
+	include_indentation = form_params.include_indentation
+	visible_idx = form_params.visible_idx
 
 	if isinstance(visible_idx, str):
 		visible_idx = json.loads(visible_idx)
 
-	if file_format_type == "Excel":
-		data = run(report_name, filters, custom_columns=custom_columns)
-		data = frappe._dict(data)
-		if not data.columns:
-			frappe.respond_as_web_page(
-				_("No data to export"),
-				_("You can try changing the filters of your report."),
-			)
-			return
+	data = run(
+		report_name, form_params.filters, custom_columns=custom_columns, are_default_filters=False
+	)
+	data = frappe._dict(data)
+	if not data.columns:
+		frappe.respond_as_web_page(
+			_("No data to export"),
+			_("You can try changing the filters of your report."),
+		)
+		return
 
+	format_duration_fields(data)
+	xlsx_data, column_widths = build_xlsx_data(data, visible_idx, include_indentation)
+
+	if file_format_type == "CSV":
+		content = get_csv_bytes(xlsx_data, csv_params)
+		file_extension = "csv"
+	elif file_format_type == "Excel":
 		from frappe.utils.xlsxutils import make_xlsx
 
-		format_duration_fields(data)
-		xlsx_data, column_widths = build_xlsx_data(data, visible_idx, include_indentation)
-		xlsx_file = make_xlsx(xlsx_data, "Query Report", column_widths=column_widths)
+		file_extension = "xlsx"
+		content = make_xlsx(xlsx_data, "Query Report", column_widths=column_widths).getvalue()
 
-		frappe.response["filename"] = report_name + ".xlsx"
-		frappe.response["filecontent"] = xlsx_file.getvalue()
-		frappe.response["type"] = "binary"
+	provide_binary_file(report_name, file_extension, content)
 
 
 def format_duration_fields(data: frappe._dict) -> None:
@@ -379,7 +330,7 @@ def format_duration_fields(data: frappe._dict) -> None:
 			continue
 
 		for row in data.result:
-			index = col.fieldname if isinstance(row, dict) else i
+			index = col.get("fieldname") if isinstance(row, dict) else i
 			if row[index]:
 				row[index] = format_duration(row[index])
 
@@ -488,7 +439,7 @@ def add_total_row(result, columns, meta=None, is_tree=False, parent_field=None):
 	if isinstance(columns[0], str):
 		first_col = columns[0].split(":")
 		if len(first_col) > 1:
-			first_col_fieldtype = first_col[1].split("/")[0]
+			first_col_fieldtype = first_col[1].split("/", 1)[0]
 	else:
 		first_col_fieldtype = columns[0].get("fieldtype")
 
@@ -503,7 +454,7 @@ def add_total_row(result, columns, meta=None, is_tree=False, parent_field=None):
 def get_data_for_custom_field(doctype, field):
 
 	if not frappe.has_permission(doctype, "read"):
-		frappe.throw(_("Not Permitted"), frappe.PermissionError)
+		frappe.throw(_("Not Permitted to read {0}").format(doctype), frappe.PermissionError)
 
 	value_map = frappe._dict(frappe.get_all(doctype, fields=["name", field], as_list=1))
 
@@ -523,7 +474,7 @@ def get_data_for_custom_report(columns):
 
 
 @frappe.whitelist()
-def save_report(reference_report, report_name, columns):
+def save_report(reference_report, report_name, columns, filters):
 	report_doc = get_report_doc(reference_report)
 
 	docname = frappe.db.exists(
@@ -539,6 +490,7 @@ def save_report(reference_report, report_name, columns):
 		report = frappe.get_doc("Report", docname)
 		existing_jd = json.loads(report.json)
 		existing_jd["columns"] = json.loads(columns)
+		existing_jd["filters"] = json.loads(filters)
 		report.update({"json": json.dumps(existing_jd, separators=(",", ":"))})
 		report.save()
 		frappe.msgprint(_("Report updated successfully"))
@@ -549,7 +501,7 @@ def save_report(reference_report, report_name, columns):
 			{
 				"doctype": "Report",
 				"report_name": report_name,
-				"json": f'{{"columns":{columns}}}',
+				"json": f'{{"columns":{columns},"filters":{filters}}}',
 				"ref_doctype": report_doc.ref_doctype,
 				"is_standard": "No",
 				"report_type": "Custom Report",
diff --git a/frappe/desk/report/todo/todo.js b/frappe/desk/report/todo/todo.js
index bb2e0f7846..52fee62afd 100644
--- a/frappe/desk/report/todo/todo.js
+++ b/frappe/desk/report/todo/todo.js
@@ -3,7 +3,5 @@
 /* eslint-disable */
 
 frappe.query_reports["ToDo"] = {
-	"filters": [
-
-	]
-}
+	filters: [],
+};
diff --git a/frappe/desk/report_dump.py b/frappe/desk/report_dump.py
deleted file mode 100644
index 6650d24757..0000000000
--- a/frappe/desk/report_dump.py
+++ /dev/null
@@ -1,107 +0,0 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# License: MIT. See LICENSE
-
-
-import copy
-import json
-
-import frappe
-
-
-@frappe.whitelist()
-def get_data(doctypes, last_modified):
-	data_map = {}
-	for dump_report_map in frappe.get_hooks().dump_report_map:
-		data_map.update(frappe.get_attr(dump_report_map))
-
-	out = {}
-
-	doctypes = json.loads(doctypes)
-	last_modified = json.loads(last_modified)
-
-	for d in doctypes:
-		args = copy.deepcopy(data_map[d])
-		dt = d.find("[") != -1 and d[: d.find("[")] or d
-		out[dt] = {}
-
-		if args.get("from"):
-			modified_table = "item."
-		else:
-			modified_table = ""
-
-		conditions = order_by = ""
-		table = args.get("from") or ("`tab%s`" % dt)
-
-		if d in last_modified:
-			if not args.get("conditions"):
-				args["conditions"] = []
-			args["conditions"].append(modified_table + "modified > '" + last_modified[d] + "'")
-			out[dt]["modified_names"] = frappe.db.sql_list(
-				"""select %sname from %s
-				where %smodified > %s"""
-				% (modified_table, table, modified_table, "%s"),
-				last_modified[d],
-			)
-
-		if args.get("force_index"):
-			conditions = " force index (%s) " % args["force_index"]
-		if args.get("conditions"):
-			conditions += " where " + " and ".join(args["conditions"])
-		if args.get("order_by"):
-			order_by = " order by " + args["order_by"]
-
-		out[dt]["data"] = [
-			list(t)
-			for t in frappe.db.sql(
-				"""select {} from {} {} {}""".format(",".join(args["columns"]), table, conditions, order_by)
-			)
-		]
-
-		# last modified
-		modified_table = table
-		if "," in table:
-			modified_table = " ".join(table.split(",")[0].split(" ")[:-1])
-
-		tmp = frappe.db.sql(
-			"""select `modified`
-			from %s order by modified desc limit 1"""
-			% modified_table
-		)
-		out[dt]["last_modified"] = tmp and tmp[0][0] or ""
-		out[dt]["columns"] = list(map(lambda c: c.split(" as ")[-1], args["columns"]))
-
-		if args.get("links"):
-			out[dt]["links"] = args["links"]
-
-	for d in out:
-		unused_links = []
-		# only compress full dumps (not partial)
-		if out[d].get("links") and (d not in last_modified):
-			for link_key in out[d]["links"]:
-				link = out[d]["links"][link_key]
-				if link[0] in out and (link[0] not in last_modified):
-
-					# make a map of link ids
-					# to index
-					link_map = {}
-					doctype_data = out[link[0]]
-
-					col_idx = doctype_data["columns"].index(link[1])
-					for row_idx in range(len(doctype_data["data"])):
-						row = doctype_data["data"][row_idx]
-						link_map[row[col_idx]] = row_idx
-
-					for row in out[d]["data"]:
-						columns = list(out[d]["columns"])
-						if link_key in columns:
-							col_idx = columns.index(link_key)
-							# replace by id
-							if row[col_idx]:
-								row[col_idx] = link_map.get(row[col_idx])
-				else:
-					unused_links.append(link_key)
-
-		for link in unused_links:
-			del out[d]["links"][link]
-
-	return out
diff --git a/frappe/desk/reportview.py b/frappe/desk/reportview.py
index caba0212b9..326e9bb864 100644
--- a/frappe/desk/reportview.py
+++ b/frappe/desk/reportview.py
@@ -4,27 +4,26 @@
 """build query for doclistview and return results"""
 
 import json
-from io import StringIO
 
 import frappe
 import frappe.permissions
 from frappe import _
 from frappe.core.doctype.access_log.access_log import make_access_log
-from frappe.model import child_table_fields, default_fields, optional_fields
+from frappe.model import child_table_fields, default_fields, get_permitted_fields, optional_fields
 from frappe.model.base_document import get_controller
 from frappe.model.db_query import DatabaseQuery
-from frappe.utils import add_user_info, cstr, format_duration
-from frappe.utils.caching import site_cache
+from frappe.model.utils import is_virtual_doctype
+from frappe.utils import add_user_info, format_duration
 
 
 @frappe.whitelist()
 @frappe.read_only()
 def get():
 	args = get_form_params()
-	# If virtual doctype get data from controller het_list method
+	# If virtual doctype, get data from controller get_list method
 	if is_virtual_doctype(args.doctype):
 		controller = get_controller(args.doctype)
-		data = compress(controller(args.doctype).get_list(args))
+		data = compress(controller.get_list(args))
 	else:
 		data = compress(execute(**args), args=args)
 	return data
@@ -37,7 +36,7 @@ def get_list():
 
 	if is_virtual_doctype(args.doctype):
 		controller = get_controller(args.doctype)
-		data = controller(args.doctype).get_list(args)
+		data = controller.get_list(args)
 	else:
 		# uncompressed (refactored from frappe.model.db_query.get_list)
 		data = execute(**args)
@@ -52,7 +51,7 @@ def get_count():
 
 	if is_virtual_doctype(args.doctype):
 		controller = get_controller(args.doctype)
-		data = controller(args.doctype).get_count(args)
+		data = controller.get_count(args)
 	else:
 		distinct = "distinct " if args.distinct == "true" else ""
 		args.fields = [f"count({distinct}`tab{args.doctype}`.name) as total_count"]
@@ -91,7 +90,7 @@ def validate_args(data):
 def validate_fields(data):
 	wildcard = update_wildcard_field_param(data)
 
-	for field in data.fields or []:
+	for field in list(data.fields or []):
 		fieldname = extract_fieldname(field)
 		if is_standard(fieldname):
 			continue
@@ -152,12 +151,8 @@ def setup_group_by(data):
 
 		if frappe.db.has_column(data.aggregate_on_doctype, data.aggregate_on_field):
 			data.fields.append(
-				"{aggregate_function}(`tab{aggregate_on_doctype}`.`{aggregate_on_field}`) AS _aggregate_column".format(
-					**data
-				)
+				f"{data.aggregate_function}(`tab{data.aggregate_on_doctype}`.`{data.aggregate_on_field}`) AS _aggregate_column"
 			)
-			if data.aggregate_on_field:
-				data.fields.append(f"`tab{data.aggregate_on_doctype}`.`{data.aggregate_on_field}`")
 		else:
 			raise_invalid_field(data.aggregate_on_field)
 
@@ -186,7 +181,7 @@ def extract_fieldname(field):
 	fieldname = field
 	for sep in (" as ", " AS "):
 		if sep in fieldname:
-			fieldname = fieldname.split(sep)[0]
+			fieldname = fieldname.split(sep, 1)[0]
 
 	# certain functions allowed, extract the fieldname from the function
 	if fieldname.startswith("count(") or fieldname.startswith("sum(") or fieldname.startswith("avg("):
@@ -208,7 +203,7 @@ def update_wildcard_field_param(data):
 	if (isinstance(data.fields, str) and data.fields == "*") or (
 		isinstance(data.fields, (list, tuple)) and len(data.fields) == 1 and data.fields[0] == "*"
 	):
-		data.fields = frappe.db.get_table_columns(data.doctype)
+		data.fields = get_permitted_fields(data.doctype, parenttype=data.parenttype)
 		return True
 
 	return False
@@ -225,7 +220,7 @@ def parse_json(data):
 	if isinstance(data.get("or_filters"), str):
 		data["or_filters"] = json.loads(data["or_filters"])
 	if isinstance(data.get("fields"), str):
-		data["fields"] = json.loads(data["fields"])
+		data["fields"] = ["*"] if data["fields"] == "*" else json.loads(data["fields"])
 	if isinstance(data.get("docstatus"), str):
 		data["docstatus"] = json.loads(data["docstatus"])
 	if isinstance(data.get("save_user_settings"), str):
@@ -272,7 +267,7 @@ def compress(data, args=None):
 		values.append(new_row)
 
 		# add user info for assignments (avatar)
-		if row._assign:
+		if row.get("_assign", ""):
 			for user in json.loads(row._assign):
 				add_user_info(user, user_info)
 
@@ -295,7 +290,7 @@ def save_report(name, doctype, report_settings):
 		if report.report_type != "Report Builder":
 			frappe.throw(_("Only reports of type Report Builder can be edited"))
 
-		if report.owner != frappe.session.user and not frappe.has_permission("Report", "write"):
+		if report.owner != frappe.session.user and not report.has_permission("write"):
 			frappe.throw(_("Insufficient Permissions for editing Report"), frappe.PermissionError)
 	else:
 		report = frappe.new_doc("Report")
@@ -324,7 +319,7 @@ def delete_report(name):
 	if report.report_type != "Report Builder":
 		frappe.throw(_("Only reports of type Report Builder can be deleted"))
 
-	if report.owner != frappe.session.user and not frappe.has_permission("Report", "delete"):
+	if report.owner != frappe.session.user and not report.has_permission("delete"):
 		frappe.throw(_("Insufficient Permissions for deleting Report"), frappe.PermissionError)
 
 	report.delete(ignore_permissions=True)
@@ -339,30 +334,21 @@ def delete_report(name):
 @frappe.read_only()
 def export_query():
 	"""export from report builder"""
-	title = frappe.form_dict.title
-	frappe.form_dict.pop("title", None)
+	from frappe.desk.utils import get_csv_bytes, pop_csv_params, provide_binary_file
 
 	form_params = get_form_params()
 	form_params["limit_page_length"] = None
 	form_params["as_list"] = True
-	doctype = form_params.doctype
-	add_totals_row = None
-	file_format_type = form_params["file_format_type"]
-	title = title or doctype
-
-	del form_params["doctype"]
-	del form_params["file_format_type"]
-
-	if "add_totals_row" in form_params and form_params["add_totals_row"] == "1":
-		add_totals_row = 1
-		del form_params["add_totals_row"]
+	doctype = form_params.pop("doctype")
+	file_format_type = form_params.pop("file_format_type")
+	title = form_params.pop("title", doctype)
+	csv_params = pop_csv_params(form_params)
+	add_totals_row = 1 if form_params.pop("add_totals_row", None) == "1" else None
 
 	frappe.permissions.can_export(doctype, raise_exception=True)
 
-	if "selected_items" in form_params:
-		si = json.loads(frappe.form_dict.get("selected_items"))
-		form_params["filters"] = {"name": ("in", si)}
-		del form_params["selected_items"]
+	if selection := form_params.pop("selected_items", None):
+		form_params["filters"] = {"name": ("in", json.loads(selection))}
 
 	make_access_log(
 		doctype=doctype,
@@ -378,38 +364,24 @@ def export_query():
 		ret = append_totals_row(ret)
 
 	data = [[_("Sr")] + get_labels(db_query.fields, doctype)]
-	for i, row in enumerate(ret):
-		data.append([i + 1] + list(row))
-
+	data.extend([i + 1] + list(row) for i, row in enumerate(ret))
 	data = handle_duration_fieldtype_values(doctype, data, db_query.fields)
 
 	if file_format_type == "CSV":
-
-		# convert to csv
-		import csv
-
 		from frappe.utils.xlsxutils import handle_html
 
-		f = StringIO()
-		writer = csv.writer(f)
-		for r in data:
-			# encode only unicode type strings and not int, floats etc.
-			writer.writerow([handle_html(frappe.as_unicode(v)) if isinstance(v, str) else v for v in r])
-
-		f.seek(0)
-		frappe.response["result"] = cstr(f.read())
-		frappe.response["type"] = "csv"
-		frappe.response["doctype"] = title
-
+		file_extension = "csv"
+		content = get_csv_bytes(
+			[[handle_html(frappe.as_unicode(v)) if isinstance(v, str) else v for v in r] for r in data],
+			csv_params,
+		)
 	elif file_format_type == "Excel":
-
 		from frappe.utils.xlsxutils import make_xlsx
 
-		xlsx_file = make_xlsx(data, doctype)
+		file_extension = "xlsx"
+		content = make_xlsx(data, doctype).getvalue()
 
-		frappe.response["filename"] = title + ".xlsx"
-		frappe.response["filecontent"] = xlsx_file.getvalue()
-		frappe.response["type"] = "binary"
+	provide_binary_file(title, file_extension, content)
 
 
 def append_totals_row(data):
@@ -436,16 +408,12 @@ def get_labels(fields, doctype):
 	"""get column labels based on column names"""
 	labels = []
 	for key in fields:
-		key = key.split(" as ")[0]
-
-		if key.startswith(("count(", "sum(", "avg(")):
+		try:
+			parenttype, fieldname = parse_field(key)
+		except ValueError:
 			continue
 
-		if "." in key:
-			parenttype, fieldname = key.split(".")[0][4:-1], key.split(".")[1].strip("`")
-		else:
-			parenttype = doctype
-			fieldname = fieldname.strip("`")
+		parenttype = parenttype or doctype
 
 		if parenttype == doctype and fieldname == "name":
 			label = _("ID", context="Label of name column in report")
@@ -464,17 +432,12 @@ def get_labels(fields, doctype):
 
 def handle_duration_fieldtype_values(doctype, data, fields):
 	for field in fields:
-		key = field.split(" as ")[0]
-
-		if key.startswith(("count(", "sum(", "avg(")):
+		try:
+			parenttype, fieldname = parse_field(field)
+		except ValueError:
 			continue
 
-		if "." in key:
-			parenttype, fieldname = key.split(".")[0][4:-1], key.split(".")[1].strip("`")
-		else:
-			parenttype = doctype
-			fieldname = field.strip("`")
-
+		parenttype = parenttype or doctype
 		df = frappe.get_meta(parenttype).get_field(fieldname)
 
 		if df and df.fieldtype == "Duration":
@@ -487,6 +450,20 @@ def handle_duration_fieldtype_values(doctype, data, fields):
 	return data
 
 
+def parse_field(field: str) -> tuple[str | None, str]:
+	"""Parse a field into parenttype and fieldname."""
+	key = field.split(" as ", 1)[0]
+
+	if key.startswith(("count(", "sum(", "avg(")):
+		raise ValueError
+
+	if "." in key:
+		table, column = key.split(".", 2)[:2]
+		return table[4:-1], column.strip("`")
+
+	return None, key.strip("`")
+
+
 @frappe.whitelist()
 def delete_items():
 	"""delete selected items"""
@@ -528,7 +505,7 @@ def get_sidebar_stats(stats, doctype, filters=None):
 	if is_virtual_doctype(doctype):
 		controller = get_controller(doctype)
 		args = {"stats": stats, "filters": filters}
-		data = controller(doctype).get_stats(args)
+		data = controller.get_stats(args)
 	else:
 		data = get_stats(stats, doctype, filters)
 
@@ -684,8 +661,7 @@ def build_match_conditions(doctype, user=None, as_condition=True):
 	)
 	if as_condition:
 		return match_conditions.replace("%", "%%")
-	else:
-		return match_conditions
+	return match_conditions
 
 
 def get_filters_cond(
@@ -702,7 +678,8 @@ def get_filters_cond(
 			for f in filters:
 				if isinstance(f[1], str) and f[1][0] == "!":
 					flt.append([doctype, f[0], "!=", f[1][1:]])
-				elif isinstance(f[1], (list, tuple)) and f[1][0] in (
+				elif isinstance(f[1], (list, tuple)) and f[1][0].lower() in (
+					"=",
 					">",
 					"<",
 					">=",
@@ -713,6 +690,7 @@ def get_filters_cond(
 					"in",
 					"not in",
 					"between",
+					"is",
 				):
 
 					flt.append([doctype, f[0], f[1][0], f[1][1]])
@@ -732,8 +710,3 @@ def get_filters_cond(
 	else:
 		cond = ""
 	return cond
-
-
-@site_cache(maxsize=128)
-def is_virtual_doctype(doctype):
-	return frappe.db.get_value("DocType", doctype, "is_virtual")
diff --git a/frappe/desk/search.py b/frappe/desk/search.py
index 8ae635093c..67695e4e73 100644
--- a/frappe/desk/search.py
+++ b/frappe/desk/search.py
@@ -7,45 +7,18 @@ import re
 
 import frappe
 from frappe import _, is_whitelisted
+from frappe.database.schema import SPECIAL_CHAR_PATTERN
 from frappe.permissions import has_permission
 from frappe.utils import cint, cstr, unique
 
 
 def sanitize_searchfield(searchfield):
-	blacklisted_keywords = ["select", "delete", "drop", "update", "case", "and", "or", "like"]
+	if not searchfield:
+		return
 
-	def _raise_exception(searchfield):
+	if SPECIAL_CHAR_PATTERN.search(searchfield):
 		frappe.throw(_("Invalid Search Field {0}").format(searchfield), frappe.DataError)
 
-	if len(searchfield) == 1:
-		# do not allow special characters to pass as searchfields
-		regex = re.compile(r'^.*[=;*,\'"$\-+%#@()_].*')
-		if regex.match(searchfield):
-			_raise_exception(searchfield)
-
-	if len(searchfield) >= 3:
-
-		# to avoid 1=1
-		if "=" in searchfield:
-			_raise_exception(searchfield)
-
-		# in mysql -- is used for commenting the query
-		elif " --" in searchfield:
-			_raise_exception(searchfield)
-
-		# to avoid and, or and like
-		elif any(f" {keyword} " in searchfield.split() for keyword in blacklisted_keywords):
-			_raise_exception(searchfield)
-
-		# to avoid select, delete, drop, update and case
-		elif any(keyword in searchfield.split() for keyword in blacklisted_keywords):
-			_raise_exception(searchfield)
-
-		else:
-			regex = re.compile(r'^.*[=;*,\'"$\-+%#@()].*')
-			if any(regex.match(f) for f in searchfield.split()):
-				_raise_exception(searchfield)
-
 
 # this is called by the Link Field
 @frappe.whitelist()
@@ -103,19 +76,30 @@ def search_widget(
 
 	standard_queries = frappe.get_hooks().standard_queries or {}
 
-	if query and query.split()[0].lower() != "select":
+	if query and query.split(maxsplit=1)[0].lower() != "select":
 		# by method
 		try:
 			is_whitelisted(frappe.get_attr(query))
 			frappe.response["values"] = frappe.call(
-				query, doctype, txt, searchfield, start, page_length, filters, as_dict=as_dict
+				query,
+				doctype,
+				txt,
+				searchfield,
+				start,
+				page_length,
+				filters,
+				as_dict=as_dict,
+				reference_doctype=reference_doctype,
 			)
 		except frappe.exceptions.PermissionError as e:
 			if frappe.local.conf.developer_mode:
 				raise e
 			else:
 				frappe.respond_as_web_page(
-					title="Invalid Method", html="Method not found", indicator_color="red", http_status_code=404
+					title="Invalid Method",
+					html="Method not found",
+					indicator_color="red",
+					http_status_code=404,
 				)
 			return
 		except Exception as e:
@@ -146,9 +130,18 @@ def search_widget(
 				filters = []
 			or_filters = []
 
-			translated_search_doctypes = frappe.get_hooks("translated_search_doctypes")
 			# build from doctype
 			if txt:
+				field_types = [
+					"Data",
+					"Text",
+					"Small Text",
+					"Long Text",
+					"Link",
+					"Select",
+					"Read Only",
+					"Text Editor",
+				]
 				search_fields = ["name"]
 				if meta.title_field:
 					search_fields.append(meta.title_field)
@@ -158,13 +151,8 @@ def search_widget(
 
 				for f in search_fields:
 					fmeta = meta.get_field(f.strip())
-					if (doctype not in translated_search_doctypes) and (
-						f == "name"
-						or (
-							fmeta
-							and fmeta.fieldtype
-							in ["Data", "Text", "Small Text", "Long Text", "Link", "Select", "Read Only", "Text Editor"]
-						)
+					if not meta.translated_doctype and (
+						f == "name" or (fmeta and fmeta.fieldtype in field_types)
 					):
 						or_filters.append([doctype, f.strip(), "like", f"%{txt}%"])
 
@@ -179,43 +167,45 @@ def search_widget(
 				fields = list(set(fields + json.loads(filter_fields)))
 			formatted_fields = [f"`tab{meta.name}`.`{f.strip()}`" for f in fields]
 
-			title_field_query = get_title_field_query(meta)
-
 			# Insert title field query after name
-			if title_field_query:
-				formatted_fields.insert(1, title_field_query)
-
-			# find relevance as location of search term from the beginning of string `name`. used for sorting results.
-			formatted_fields.append(
-				"""locate({_txt}, `tab{doctype}`.`name`) as `_relevance`""".format(
-					_txt=frappe.db.escape((txt or "").replace("%", "").replace("@", "")), doctype=doctype
-				)
-			)
+			if meta.show_title_field_in_link:
+				formatted_fields.insert(1, f"`tab{meta.name}`.{meta.title_field} as `label`")
 
 			# In order_by, `idx` gets second priority, because it stores link count
 			from frappe.model.db_query import get_order_by
 
 			order_by_based_on_meta = get_order_by(doctype, meta)
 			# 2 is the index of _relevance column
-			order_by = f"_relevance, {order_by_based_on_meta}, `tab{doctype}`.idx desc"
+			order_by = f"{order_by_based_on_meta}, `tab{doctype}`.idx desc"
+
+			if not meta.translated_doctype:
+				formatted_fields.append(
+					"""locate({_txt}, `tab{doctype}`.`name`) as `_relevance`""".format(
+						_txt=frappe.db.escape((txt or "").replace("%", "").replace("@", "")),
+						doctype=doctype,
+					)
+				)
+				order_by = f"_relevance, {order_by}"
 
-			ptype = "select" if frappe.only_has_select_perm(doctype) else "read"
 			ignore_permissions = (
 				True
 				if doctype == "DocType"
-				else (cint(ignore_user_permissions) and has_permission(doctype, ptype=ptype))
+				else (
+					cint(ignore_user_permissions)
+					and has_permission(
+						doctype,
+						ptype="select" if frappe.only_has_select_perm(doctype) else "read",
+					)
+				)
 			)
 
-			if doctype in translated_search_doctypes:
-				page_length = None
-
 			values = frappe.get_list(
 				doctype,
 				filters=filters,
 				fields=formatted_fields,
 				or_filters=or_filters,
 				limit_start=start,
-				limit_page_length=page_length,
+				limit_page_length=None if meta.translated_doctype else page_length,
 				order_by=order_by,
 				ignore_permissions=ignore_permissions,
 				reference_doctype=reference_doctype,
@@ -223,12 +213,15 @@ def search_widget(
 				strict=False,
 			)
 
-			if doctype in translated_search_doctypes:
+			if meta.translated_doctype:
 				# Filtering the values array so that query is included in very element
 				values = (
-					v
-					for v in values
-					if re.search(f"{re.escape(txt)}.*", _(v.name if as_dict else v[0]), re.IGNORECASE)
+					result
+					for result in values
+					if any(
+						re.search(f"{re.escape(txt)}.*", _(cstr(value)) or "", re.IGNORECASE)
+						for value in (result.values() if as_dict else result)
+					)
 				)
 
 			# Sorting the values array so that relevant results always come first
@@ -237,12 +230,14 @@ def search_widget(
 			values = sorted(values, key=lambda x: relevance_sorter(x, txt, as_dict))
 
 			# remove _relevance from results
-			if as_dict:
-				for r in values:
-					r.pop("_relevance")
-				frappe.response["values"] = values
-			else:
-				frappe.response["values"] = [r[:-1] for r in values]
+			if not meta.translated_doctype:
+				if as_dict:
+					for r in values:
+						r.pop("_relevance")
+				else:
+					values = [r[:-1] for r in values]
+
+			frappe.response["values"] = values
 
 
 def get_std_fields_list(meta, key):
@@ -262,39 +257,25 @@ def get_std_fields_list(meta, key):
 	return sflist
 
 
-def get_title_field_query(meta):
-	title_field = meta.title_field if meta.title_field else None
-	show_title_field_in_link = (
-		meta.show_title_field_in_link if meta.show_title_field_in_link else None
-	)
-	field = None
+def build_for_autosuggest(res: list[tuple], doctype: str) -> list[dict]:
+	def to_string(parts):
+		return ", ".join(
+			unique(_(cstr(part)) if meta.translated_doctype else cstr(part) for part in parts if part)
+		)
 
-	if title_field and show_title_field_in_link:
-		field = f"`tab{meta.name}`.{title_field} as `label`"
-
-	return field
-
-
-def build_for_autosuggest(res, doctype):
 	results = []
 	meta = frappe.get_meta(doctype)
-	if not (meta.title_field and meta.show_title_field_in_link):
-		for r in res:
-			r = list(r)
-			results.append({"value": r[0], "description": ", ".join(unique(cstr(d) for d in r[1:] if d))})
-
+	if meta.show_title_field_in_link:
+		for item in res:
+			item = list(item)
+			label = item[1]  # use title as label
+			item[1] = item[0]  # show name in description instead of title
+			if len(item) >= 3 and item[2] == label:
+				# remove redundant title ("label") value
+				del item[2]
+			results.append({"value": item[0], "label": label, "description": to_string(item[1:])})
 	else:
-		title_field_exists = meta.title_field and meta.show_title_field_in_link
-		_from = 2 if title_field_exists else 1  # to exclude title from description if title_field_exists
-		for r in res:
-			r = list(r)
-			results.append(
-				{
-					"value": r[0],
-					"label": r[1] if title_field_exists else None,
-					"description": ", ".join(unique(cstr(d) for d in r[_from:] if d)),
-				}
-			)
+		results.extend({"value": item[0], "description": to_string(item[1:])} for item in res)
 
 	return results
 
@@ -309,7 +290,7 @@ def scrub_custom_query(query, key, txt):
 
 def relevance_sorter(key, query, as_dict):
 	value = _(key.name if as_dict else key[0])
-	return (cstr(value).lower().startswith(query.lower()) is not True, value)
+	return (cstr(value).casefold().startswith(query.casefold()) is not True, value)
 
 
 def validate_and_sanitize_search_inputs(fn):
@@ -370,7 +351,7 @@ def get_user_groups():
 def get_link_title(doctype, docname):
 	meta = frappe.get_meta(doctype)
 
-	if meta.title_field and meta.show_title_field_in_link:
+	if meta.show_title_field_in_link:
 		return frappe.db.get_value(doctype, docname, meta.title_field)
 
 	return docname
diff --git a/frappe/desk/utils.py b/frappe/desk/utils.py
index 72cba79963..77edf88d7a 100644
--- a/frappe/desk/utils.py
+++ b/frappe/desk/utils.py
@@ -9,16 +9,14 @@ def validate_route_conflict(doctype, name):
 	Raises exception if name clashes with routes from other documents for /app routing
 	"""
 
+	if frappe.flags.in_migrate:
+		return
+
 	all_names = []
 	for _doctype in ["Page", "Workspace", "DocType"]:
-		try:
-			all_names.extend(
-				[
-					slug(d) for d in frappe.get_all(_doctype, pluck="name") if (doctype != _doctype and d != name)
-				]
-			)
-		except frappe.db.TableMissingError:
-			pass
+		all_names.extend(
+			[slug(d) for d in frappe.get_all(_doctype, pluck="name") if (doctype != _doctype and d != name)]
+		)
 
 	if slug(name) in all_names:
 		frappe.msgprint(frappe._("Name already taken, please set a new name"))
@@ -27,3 +25,34 @@ def validate_route_conflict(doctype, name):
 
 def slug(name):
 	return name.lower().replace(" ", "-")
+
+
+def pop_csv_params(form_dict):
+	"""Pop csv params from form_dict and return them as a dict."""
+	from csv import QUOTE_NONNUMERIC
+
+	from frappe.utils.data import cint, cstr
+
+	return {
+		"delimiter": cstr(form_dict.pop("csv_delimiter", ","))[0],
+		"quoting": cint(form_dict.pop("csv_quoting", QUOTE_NONNUMERIC)),
+	}
+
+
+def get_csv_bytes(data: list[list], csv_params: dict) -> bytes:
+	"""Convert data to csv bytes."""
+	from csv import writer
+	from io import StringIO
+
+	file = StringIO()
+	csv_writer = writer(file, **csv_params)
+	csv_writer.writerows(data)
+
+	return file.getvalue().encode("utf-8")
+
+
+def provide_binary_file(filename: str, extension: str, content: bytes) -> None:
+	"""Provide a binary file to the client."""
+	frappe.response["type"] = "binary"
+	frappe.response["filecontent"] = content
+	frappe.response["filename"] = f"{filename}.{extension}"
diff --git a/frappe/email/__init__.py b/frappe/email/__init__.py
index 0f0259653a..486db2a784 100644
--- a/frappe/email/__init__.py
+++ b/frappe/email/__init__.py
@@ -10,27 +10,29 @@ def sendmail_to_system_managers(subject, content):
 
 
 @frappe.whitelist()
-def get_contact_list(txt, page_length=20):
-	"""Returns contacts (from autosuggest)"""
+def get_contact_list(txt, page_length=20) -> list[dict]:
+	"""Return email ids for a multiselect field."""
 
-	cached_contacts = get_cached_contacts(txt)
-	if cached_contacts:
+	if cached_contacts := get_cached_contacts(txt):
 		return cached_contacts[:page_length]
 
-	match_conditions = build_match_conditions("Contact")
-	match_conditions = f"and {match_conditions}" if match_conditions else ""
+	reportview_conditions = build_match_conditions("Contact")
+	match_conditions = f"and {reportview_conditions}" if reportview_conditions else ""
 
+	# The multiselect field will store the `label` as the selected value.
+	# The `value` is just used as a unique key to distinguish between the options.
+	# https://github.com/frappe/frappe/blob/6c6a89bcdd9454060a1333e23b855d0505c9ebc2/frappe/public/js/frappe/form/controls/autocomplete.js#L29-L35
 	out = frappe.db.sql(
-		"""select email_id as value,
+		f"""select name as value, email_id as label,
 		concat(first_name, ifnull(concat(' ',last_name), '' )) as description
 		from tabContact
-		where name like %(txt)s or email_id like %(txt)s
-		%(condition)s
+		where (name like %(txt)s or email_id like %(txt)s) and email_id != ''
+		{match_conditions}
 		limit %(page_length)s""",
-		{"txt": "%" + txt + "%", "condition": match_conditions, "page_length": page_length},
+		{"txt": f"%{txt}%", "page_length": page_length},
 		as_dict=True,
 	)
-	out = filter(None, out)
+	out = list(filter(None, out))
 
 	update_contact_cache(out)
 
diff --git a/frappe/email/doctype/auto_email_report/auto_email_report.js b/frappe/email/doctype/auto_email_report/auto_email_report.js
index 3423c3ccba..62b562b97d 100644
--- a/frappe/email/doctype/auto_email_report/auto_email_report.js
+++ b/frappe/email/doctype/auto_email_report/auto_email_report.js
@@ -1,139 +1,176 @@
 // Copyright (c) 2016, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Auto Email Report', {
-	refresh: function(frm) {
-		frm.trigger('fetch_report_filters');
-		if(!frm.is_new()) {
-			frm.add_custom_button(__('Download'), function() {
+frappe.ui.form.on("Auto Email Report", {
+	refresh: function (frm) {
+		frm.trigger("fetch_report_filters");
+		if (!frm.is_new()) {
+			frm.add_custom_button(__("Download"), function () {
 				var w = window.open(
 					frappe.urllib.get_full_url(
-						"/api/method/frappe.email.doctype.auto_email_report.auto_email_report.download?"
-						+"name="+encodeURIComponent(frm.doc.name)));
-				if(!w) {
-					frappe.msgprint(__("Please enable pop-ups")); return;
+						"/api/method/frappe.email.doctype.auto_email_report.auto_email_report.download?" +
+							"name=" +
+							encodeURIComponent(frm.doc.name)
+					)
+				);
+				if (!w) {
+					frappe.msgprint(__("Please enable pop-ups"));
+					return;
 				}
 			});
-			frm.add_custom_button(__('Send Now'), function() {
+			frm.add_custom_button(__("Send Now"), function () {
 				frappe.call({
-					method: 'frappe.email.doctype.auto_email_report.auto_email_report.send_now',
-					args: {name: frm.doc.name},
-					callback: function() {
-						frappe.msgprint(__('Scheduled to send'));
-					}
+					method: "frappe.email.doctype.auto_email_report.auto_email_report.send_now",
+					args: { name: frm.doc.name },
+					callback: function () {
+						frappe.msgprint(__("Scheduled to send"));
+					},
 				});
 			});
 		} else {
-			if(!frm.doc.user) {
-				frm.set_value('user', frappe.session.user);
+			if (!frm.doc.user) {
+				frm.set_value("user", frappe.session.user);
 			}
-			if(!frm.doc.email_to) {
-				frm.set_value('email_to', frappe.session.user);
+			if (!frm.doc.email_to) {
+				frm.set_value("email_to", frappe.session.user);
 			}
 		}
+
+		frm.set_query("sender", function () {
+			return {
+				filters: {
+					enable_outgoing: 1,
+					awaiting_password: 0,
+				},
+			};
+		});
 	},
-	report: function(frm) {
-		frm.set_value('filters', '');
-		frm.trigger('fetch_report_filters');
+	report: function (frm) {
+		frm.set_value("filters", "");
+		frm.trigger("fetch_report_filters");
 	},
 	fetch_report_filters(frm) {
-		if (frm.doc.report
-			&& frm.doc.report_type !== 'Report Builder'
-			&& frm.script_setup_for !== frm.doc.report
+		if (
+			frm.doc.report &&
+			frm.doc.report_type !== "Report Builder" &&
+			frm.script_setup_for !== frm.doc.report
 		) {
 			frappe.call({
 				method: "frappe.desk.query_report.get_script",
 				args: {
-					report_name: frm.doc.report
+					report_name: frm.doc.report,
 				},
-				callback: function(r) {
+				callback: function (r) {
 					frappe.dom.eval(r.message.script || "");
 					frm.script_setup_for = frm.doc.report;
-					frm.trigger('show_filters');
-				}
+					frm.trigger("show_filters");
+				},
 			});
 		} else {
-			frm.trigger('show_filters');
+			frm.trigger("show_filters");
 		}
 	},
-	show_filters: function(frm) {
-		var wrapper = $(frm.get_field('filters_display').wrapper);
+	show_filters: function (frm) {
+		var wrapper = $(frm.get_field("filters_display").wrapper);
 		wrapper.empty();
-		if(frm.doc.report_type === 'Custom Report' || (frm.doc.report_type !== 'Report Builder'
-			&& frappe.query_reports[frm.doc.report]
-			&& frappe.query_reports[frm.doc.report].filters)) {
-
+		if (
+			frm.doc.report_type === "Custom Report" ||
+			(frm.doc.report_type !== "Report Builder" &&
+				frappe.query_reports[frm.doc.report] &&
+				frappe.query_reports[frm.doc.report].filters)
+		) {
 			// make a table to show filters
-			var table = $('\
-				\
-				
        '+__('Filter')+''+__('Value')+'
        ').appendTo(wrapper);
-			$('
        ' + __("Click table to edit") + '
        ').appendTo(wrapper);
+			var table = $(
+				'\
+				\
+				
        ' +
+					__("Filter") +
+					"" +
+					__("Value") +
+					"
        "
+			).appendTo(wrapper);
+			$('
        ' + __("Click table to edit") + "
        ").appendTo(
+				wrapper
+			);
 
-			var filters = JSON.parse(frm.doc.filters || '{}');
+			var filters = {};
 
 			let report_filters;
 
-			if (frm.doc.report_type === 'Custom Report'
-				&& frappe.query_reports[frm.doc.reference_report]
-				&& frappe.query_reports[frm.doc.reference_report].filters) {
+			if (
+				frm.doc.report_type === "Custom Report" &&
+				frappe.query_reports[frm.doc.reference_report] &&
+				frappe.query_reports[frm.doc.reference_report].filters
+			) {
+				if (frm.doc.filters) {
+					filters = JSON.parse(frm.doc.filters);
+				} else {
+					frappe.db.get_value("Report", frm.doc.report, "json", (r) => {
+						if (r && r.json) {
+							filters = JSON.parse(r.json).filters || {};
+						}
+					});
+				}
+
 				report_filters = frappe.query_reports[frm.doc.reference_report].filters;
 			} else {
+				filters = JSON.parse(frm.doc.filters || "{}");
 				report_filters = frappe.query_reports[frm.doc.report].filters;
 			}
 
-			if(report_filters && report_filters.length > 0) {
-				frm.set_value('filter_meta', JSON.stringify(report_filters));
+			if (report_filters && report_filters.length > 0) {
+				frm.set_value("filter_meta", JSON.stringify(report_filters));
 				if (frm.is_dirty()) {
 					frm.save();
 				}
 			}
 
-			var report_filters_list = []
-			$.each(report_filters, function(key, val){
+			var report_filters_list = [];
+			$.each(report_filters, function (key, val) {
 				// Remove break fieldtype from the filters
-				if(val.fieldtype != 'Break') {
-					report_filters_list.push(val)
+				if (val.fieldtype != "Break") {
+					report_filters_list.push(val);
 				}
-			})
+			});
 			report_filters = report_filters_list;
 
 			const mandatory_css = {
 				"background-color": "var(--error-bg)",
-				"font-weight": "bold"
+				"font-weight": "bold",
 			};
 
-			report_filters.forEach(f => {
+			report_filters.forEach((f) => {
 				const css = f.reqd ? mandatory_css : {};
 				const row = $("").appendTo(table.find("tbody"));
 				$("" + f.label + "").appendTo(row);
-				$("" + frappe.format(filters[f.fieldname], f) +"")
+				$("" + frappe.format(filters[f.fieldname], f) + "")
 					.css(css)
 					.appendTo(row);
 			});
 
-			table.on('click', function() {
+			table.on("click", function () {
 				var dialog = new frappe.ui.Dialog({
 					fields: report_filters,
-					primary_action: function() {
+					primary_action: function () {
 						var values = this.get_values();
-						if(values) {
+						if (values) {
 							this.hide();
-							frm.set_value('filters', JSON.stringify(values));
-							frm.trigger('show_filters');
+							frm.set_value("filters", JSON.stringify(values));
+							frm.trigger("show_filters");
 						}
-					}
+					},
 				});
 				dialog.show();
 				dialog.set_values(filters);
-			})
+			});
 
 			// populate dynamic date field selection
 			let date_fields = report_filters
-				.filter(df => df.fieldtype === 'Date')
-				.map(df => ({ label: df.label, value: df.fieldname }));
-			frm.set_df_property('from_date_field', 'options', date_fields);
-			frm.set_df_property('to_date_field', 'options', date_fields);
-			frm.toggle_display('dynamic_report_filters_section', date_fields.length > 0);
+				.filter((df) => df.fieldtype === "Date")
+				.map((df) => ({ label: df.label, value: df.fieldname }));
+			frm.set_df_property("from_date_field", "options", date_fields);
+			frm.set_df_property("to_date_field", "options", date_fields);
+			frm.toggle_display("dynamic_report_filters_section", date_fields.length > 0);
 		}
-	}
+	},
 });
diff --git a/frappe/email/doctype/auto_email_report/auto_email_report.json b/frappe/email/doctype/auto_email_report/auto_email_report.json
index 211e2e9662..75a9e99c96 100644
--- a/frappe/email/doctype/auto_email_report/auto_email_report.json
+++ b/frappe/email/doctype/auto_email_report/auto_email_report.json
@@ -1,238 +1,248 @@
 {
-   "allow_rename": 1,
-   "creation": "2016-09-01 01:34:34.985457",
-   "doctype": "DocType",
-   "editable_grid": 1,
-   "engine": "InnoDB",
-   "field_order": [
-    "report",
-    "user",
-    "enabled",
-    "column_break_4",
-    "report_type",
-    "reference_report",
-    "filter_data",
-    "send_if_data",
-    "data_modified_till",
-    "no_of_rows",
-    "report_filters",
-    "filters_display",
-    "filters",
-    "filter_meta",
-    "dynamic_report_filters_section",
-    "from_date_field",
-    "to_date_field",
-    "column_break_17",
-    "dynamic_date_period",
-    "email_settings",
-    "email_to",
-    "day_of_week",
-    "column_break_13",
-    "frequency",
-    "format",
-    "section_break_15",
-    "description"
-   ],
-   "fields": [
-    {
-     "fieldname": "report",
-     "fieldtype": "Link",
-     "label": "Report",
-     "options": "Report",
-     "reqd": 1
-    },
-    {
-     "default": "User",
-     "fieldname": "user",
-     "fieldtype": "Link",
-     "label": "Based on Permissions For User",
-     "options": "User",
-     "reqd": 1
-    },
-    {
-     "default": "1",
-     "fieldname": "enabled",
-     "fieldtype": "Check",
-     "label": "Enabled"
-    },
-    {
-     "fieldname": "column_break_4",
-     "fieldtype": "Column Break"
-    },
-    {
-     "fetch_from": "report.report_type",
-     "fieldname": "report_type",
-     "fieldtype": "Read Only",
-     "label": "Report Type"
-    },
-    {
-     "fieldname": "filter_data",
-     "fieldtype": "Section Break",
-     "label": "Filter Data"
-    },
-    {
-     "default": "1",
-     "fieldname": "send_if_data",
-     "fieldtype": "Check",
-     "label": "Send only if there is any data"
-    },
-    {
-     "depends_on": "eval:doc.report_type=='Report Builder'",
-     "description": "Zero means send records updated at anytime",
-     "fieldname": "data_modified_till",
-     "fieldtype": "Int",
-     "label": "Only Send Records Updated in Last X Hours"
-    },
-    {
-     "default": "100",
-     "fieldname": "no_of_rows",
-     "fieldtype": "Int",
-     "label": "No of Rows (Max 500)"
-    },
-    {
-     "collapsible": 1,
-     "depends_on": "eval:doc.report_type !== 'Report Builder'",
-     "fieldname": "report_filters",
-     "fieldtype": "Section Break",
-     "label": "Report Filters"
-    },
-    {
-     "fieldname": "filters_display",
-     "fieldtype": "HTML",
-     "label": "Filters Display"
-    },
-    {
-     "fieldname": "filters",
-     "fieldtype": "Text",
-     "hidden": 1,
-     "label": "Filters"
-    },
-    {
-     "fieldname": "filter_meta",
-     "fieldtype": "Text",
-     "hidden": 1,
-     "label": "Filter Meta",
-     "read_only": 1
-    },
-    {
-     "collapsible": 1,
-     "depends_on": "eval:doc.report_type !== 'Report Builder'",
-     "fieldname": "dynamic_report_filters_section",
-     "fieldtype": "Section Break",
-     "label": "Dynamic Report Filters"
-    },
-    {
-     "fieldname": "from_date_field",
-     "fieldtype": "Select",
-     "label": "From Date Field"
-    },
-    {
-     "fieldname": "to_date_field",
-     "fieldtype": "Select",
-     "label": "To Date Field"
-    },
-    {
-     "fieldname": "column_break_17",
-     "fieldtype": "Column Break"
-    },
-    {
-     "fieldname": "dynamic_date_period",
-     "fieldtype": "Select",
-     "label": "Period",
-     "options": "\nDaily\nWeekly\nMonthly\nQuarterly\nHalf Yearly\nYearly"
-    },
-    {
-     "fieldname": "email_settings",
-     "fieldtype": "Section Break",
-     "label": "Email Settings"
-    },
-    {
-     "description": "For multiple addresses, enter the address on different line. e.g. test@test.com \u23ce test1@test.com",
-     "fieldname": "email_to",
-     "fieldtype": "Small Text",
-     "label": "Email To",
-     "reqd": 1
-    },
-    {
-     "default": "Monday",
-     "depends_on": "eval:doc.frequency=='Weekly'",
-     "fieldname": "day_of_week",
-     "fieldtype": "Select",
-     "label": "Day of Week",
-     "options": "Monday\nTuesday\nWednesday\nThursday\nFriday\nSaturday\nSunday"
-    },
-    {
-     "fieldname": "column_break_13",
-     "fieldtype": "Column Break"
-    },
-    {
-     "fieldname": "frequency",
-     "fieldtype": "Select",
-     "in_list_view": 1,
-     "in_standard_filter": 1,
-     "label": "Frequency",
-     "options": "Daily\nWeekdays\nWeekly\nMonthly",
-     "reqd": 1
-    },
-    {
-     "fieldname": "format",
-     "fieldtype": "Select",
-     "label": "Format",
-     "options": "HTML\nXLSX\nCSV",
-     "reqd": 1
-    },
-    {
-     "collapsible": 1,
-     "fieldname": "section_break_15",
-     "fieldtype": "Section Break",
-     "label": "Message"
-    },
-    {
-     "fieldname": "description",
-     "fieldtype": "Text Editor",
-     "label": "Message"
-    },
-    {
-     "fetch_from": "report.reference_report",
-     "fieldname": "reference_report",
-     "fieldtype": "Data",
-     "hidden": 1,
-     "label": "Reference Report",
-     "read_only": 1
-    }
-   ],
-   "modified": "2021-01-28 15:59:43.151995",
-   "modified_by": "Administrator",
-   "module": "Email",
-   "name": "Auto Email Report",
-   "owner": "Administrator",
-   "permissions": [
-    {
-     "create": 1,
-     "delete": 1,
-     "email": 1,
-     "export": 1,
-     "print": 1,
-     "read": 1,
-     "report": 1,
-     "role": "System Manager",
-     "share": 1,
-     "write": 1
-    },
-    {
-     "create": 1,
-     "delete": 1,
-     "email": 1,
-     "export": 1,
-     "print": 1,
-     "read": 1,
-     "report": 1,
-     "role": "Report Manager",
-     "share": 1,
-     "write": 1
-    }
-   ],
-   "sort_field": "modified",
-   "sort_order": "DESC",
-   "track_changes": 1
-  }
\ No newline at end of file
+ "actions": [],
+ "allow_rename": 1,
+ "creation": "2016-09-01 01:34:34.985457",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "report",
+  "user",
+  "enabled",
+  "column_break_4",
+  "report_type",
+  "reference_report",
+  "filter_data",
+  "send_if_data",
+  "data_modified_till",
+  "no_of_rows",
+  "report_filters",
+  "filters_display",
+  "filters",
+  "filter_meta",
+  "dynamic_report_filters_section",
+  "from_date_field",
+  "to_date_field",
+  "column_break_17",
+  "dynamic_date_period",
+  "email_settings",
+  "email_to",
+  "day_of_week",
+  "column_break_13",
+  "sender",
+  "frequency",
+  "format",
+  "section_break_15",
+  "description"
+ ],
+ "fields": [
+  {
+   "fieldname": "report",
+   "fieldtype": "Link",
+   "label": "Report",
+   "options": "Report",
+   "reqd": 1
+  },
+  {
+   "default": "User",
+   "fieldname": "user",
+   "fieldtype": "Link",
+   "label": "Based on Permissions For User",
+   "options": "User",
+   "reqd": 1
+  },
+  {
+   "default": "1",
+   "fieldname": "enabled",
+   "fieldtype": "Check",
+   "label": "Enabled"
+  },
+  {
+   "fieldname": "column_break_4",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fetch_from": "report.report_type",
+   "fieldname": "report_type",
+   "fieldtype": "Read Only",
+   "label": "Report Type"
+  },
+  {
+   "fieldname": "filter_data",
+   "fieldtype": "Section Break",
+   "label": "Filter Data"
+  },
+  {
+   "default": "1",
+   "fieldname": "send_if_data",
+   "fieldtype": "Check",
+   "label": "Send only if there is any data"
+  },
+  {
+   "depends_on": "eval:doc.report_type=='Report Builder'",
+   "description": "Zero means send records updated at anytime",
+   "fieldname": "data_modified_till",
+   "fieldtype": "Int",
+   "label": "Only Send Records Updated in Last X Hours"
+  },
+  {
+   "default": "100",
+   "fieldname": "no_of_rows",
+   "fieldtype": "Int",
+   "label": "No of Rows (Max 500)"
+  },
+  {
+   "collapsible": 1,
+   "depends_on": "eval:doc.report_type !== 'Report Builder'",
+   "fieldname": "report_filters",
+   "fieldtype": "Section Break",
+   "label": "Report Filters"
+  },
+  {
+   "fieldname": "filters_display",
+   "fieldtype": "HTML",
+   "label": "Filters Display"
+  },
+  {
+   "fieldname": "filters",
+   "fieldtype": "Text",
+   "hidden": 1,
+   "label": "Filters"
+  },
+  {
+   "fieldname": "filter_meta",
+   "fieldtype": "Text",
+   "hidden": 1,
+   "label": "Filter Meta",
+   "read_only": 1
+  },
+  {
+   "collapsible": 1,
+   "depends_on": "eval:doc.report_type !== 'Report Builder'",
+   "fieldname": "dynamic_report_filters_section",
+   "fieldtype": "Section Break",
+   "label": "Dynamic Report Filters"
+  },
+  {
+   "fieldname": "from_date_field",
+   "fieldtype": "Select",
+   "label": "From Date Field"
+  },
+  {
+   "fieldname": "to_date_field",
+   "fieldtype": "Select",
+   "label": "To Date Field"
+  },
+  {
+   "fieldname": "column_break_17",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "dynamic_date_period",
+   "fieldtype": "Select",
+   "label": "Period",
+   "options": "\nDaily\nWeekly\nMonthly\nQuarterly\nHalf Yearly\nYearly"
+  },
+  {
+   "fieldname": "email_settings",
+   "fieldtype": "Section Break",
+   "label": "Email Settings"
+  },
+  {
+   "description": "For multiple addresses, enter the address on different line. e.g. test@test.com \u23ce test1@test.com",
+   "fieldname": "email_to",
+   "fieldtype": "Small Text",
+   "label": "Email To",
+   "reqd": 1
+  },
+  {
+   "default": "Monday",
+   "depends_on": "eval:doc.frequency=='Weekly'",
+   "fieldname": "day_of_week",
+   "fieldtype": "Select",
+   "label": "Day of Week",
+   "options": "Monday\nTuesday\nWednesday\nThursday\nFriday\nSaturday\nSunday"
+  },
+  {
+   "fieldname": "column_break_13",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "frequency",
+   "fieldtype": "Select",
+   "in_list_view": 1,
+   "in_standard_filter": 1,
+   "label": "Frequency",
+   "options": "Daily\nWeekdays\nWeekly\nMonthly",
+   "reqd": 1
+  },
+  {
+   "fieldname": "format",
+   "fieldtype": "Select",
+   "label": "Format",
+   "options": "HTML\nXLSX\nCSV",
+   "reqd": 1
+  },
+  {
+   "collapsible": 1,
+   "fieldname": "section_break_15",
+   "fieldtype": "Section Break",
+   "label": "Message"
+  },
+  {
+   "fieldname": "description",
+   "fieldtype": "Text Editor",
+   "label": "Message"
+  },
+  {
+   "fetch_from": "report.reference_report",
+   "fieldname": "reference_report",
+   "fieldtype": "Data",
+   "hidden": 1,
+   "label": "Reference Report",
+   "read_only": 1
+  },
+  {
+   "fieldname": "sender",
+   "fieldtype": "Link",
+   "label": "Sender",
+   "options": "Email Account"
+  }
+ ],
+ "links": [],
+ "modified": "2022-09-08 15:31:55.031023",
+ "modified_by": "Administrator",
+ "module": "Email",
+ "name": "Auto Email Report",
+ "owner": "Administrator",
+ "permissions": [
+  {
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "export": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "System Manager",
+   "share": 1,
+   "write": 1
+  },
+  {
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "export": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "Report Manager",
+   "share": 1,
+   "write": 1
+  }
+ ],
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "states": [],
+ "track_changes": 1
+}
\ No newline at end of file
diff --git a/frappe/email/doctype/auto_email_report/auto_email_report.py b/frappe/email/doctype/auto_email_report/auto_email_report.py
index b9b5e4e8d7..24c69dab7c 100644
--- a/frappe/email/doctype/auto_email_report/auto_email_report.py
+++ b/frappe/email/doctype/auto_email_report/auto_email_report.py
@@ -3,6 +3,7 @@
 
 import calendar
 from datetime import timedelta
+from email.utils import formataddr
 
 import frappe
 from frappe import _
@@ -37,6 +38,10 @@ class AutoEmailReport(Document):
 		self.validate_report_format()
 		self.validate_mandatory_fields()
 
+	@property
+	def sender_email(self):
+		return frappe.db.get_value("Email Account", self.sender, "email_id")
+
 	def validate_emails(self):
 		"""Cleanup list of emails"""
 		if "," in self.email_to:
@@ -109,6 +114,7 @@ class AutoEmailReport(Document):
 			filters=self.filters,
 			as_dict=True,
 			ignore_prepared_report=True,
+			are_default_filters=False,
 		)
 
 		# add serial numbers
@@ -203,6 +209,7 @@ class AutoEmailReport(Document):
 
 		frappe.sendmail(
 			recipients=self.email_to.split(),
+			sender=formataddr((self.sender, self.sender_email)) if self.sender else "",
 			subject=self.name,
 			message=message,
 			attachments=attachments,
diff --git a/frappe/email/doctype/auto_email_report/test_auto_email_report.py b/frappe/email/doctype/auto_email_report/test_auto_email_report.py
index ee0a363bd9..1f806f22b5 100644
--- a/frappe/email/doctype/auto_email_report/test_auto_email_report.py
+++ b/frappe/email/doctype/auto_email_report/test_auto_email_report.py
@@ -1,16 +1,16 @@
 # Copyright (c) 2015, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
 import json
-import unittest
 
 import frappe
+from frappe.tests.utils import FrappeTestCase
 from frappe.utils import add_to_date, get_link_to_form, today
 from frappe.utils.data import is_html
 
 # test_records = frappe.get_test_records('Auto Email Report')
 
 
-class TestAutoEmailReport(unittest.TestCase):
+class TestAutoEmailReport(FrappeTestCase):
 	def test_auto_email(self):
 		frappe.delete_doc("Auto Email Report", "Permitted Documents For User")
 
diff --git a/frappe/email/doctype/document_follow/document_follow.js b/frappe/email/doctype/document_follow/document_follow.js
index 3fdf055d36..59efb37341 100644
--- a/frappe/email/doctype/document_follow/document_follow.js
+++ b/frappe/email/doctype/document_follow/document_follow.js
@@ -1,6 +1,4 @@
 // Copyright (c) 2019, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Document Follow', {
-
-});
+frappe.ui.form.on("Document Follow", {});
diff --git a/frappe/email/doctype/document_follow/test_document_follow.py b/frappe/email/doctype/document_follow/test_document_follow.py
index 159a399ee8..845f6e93bb 100644
--- a/frappe/email/doctype/document_follow/test_document_follow.py
+++ b/frappe/email/doctype/document_follow/test_document_follow.py
@@ -1,6 +1,5 @@
 # Copyright (c) 2019, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
-import unittest
 from dataclasses import dataclass
 
 import frappe
@@ -12,9 +11,10 @@ from frappe.desk.like import toggle_like
 from frappe.query_builder import DocType
 from frappe.query_builder.functions import Cast_
 from frappe.share import add as share
+from frappe.tests.utils import FrappeTestCase
 
 
-class TestDocumentFollow(unittest.TestCase):
+class TestDocumentFollow(FrappeTestCase):
 	def test_document_follow_version(self):
 		user = get_user()
 		event_doc = get_event()
diff --git a/frappe/email/doctype/email_account/email_account.js b/frappe/email/doctype/email_account/email_account.js
index d22009963d..90f71bf88f 100644
--- a/frappe/email/doctype/email_account/email_account.js
+++ b/frappe/email/doctype/email_account/email_account.js
@@ -1,94 +1,95 @@
 frappe.email_defaults = {
-	"GMail": {
-		"email_server": "imap.gmail.com",
-		"incoming_port": 993,
-		"use_ssl": 1,
-		"enable_outgoing": 1,
-		"smtp_server": "smtp.gmail.com",
-		"smtp_port": 587,
-		"use_tls": 1,
-		"use_imap": 1
+	GMail: {
+		email_server: "imap.gmail.com",
+		incoming_port: 993,
+		use_ssl: 1,
+		enable_outgoing: 1,
+		smtp_server: "smtp.gmail.com",
+		smtp_port: 587,
+		use_tls: 1,
+		use_imap: 1,
 	},
 	"Outlook.com": {
-		"email_server": "imap-mail.outlook.com",
-		"use_ssl": 1,
-		"enable_outgoing": 1,
-		"smtp_server": "smtp-mail.outlook.com",
-		"smtp_port": 587,
-		"use_tls": 1,
-		"use_imap": 1
+		email_server: "imap-mail.outlook.com",
+		use_ssl: 1,
+		enable_outgoing: 1,
+		smtp_server: "smtp-mail.outlook.com",
+		smtp_port: 587,
+		use_tls: 1,
+		use_imap: 1,
 	},
-	"Sendgrid": {
-		"enable_outgoing": 1,
-		"smtp_server": "smtp.sendgrid.net",
-		"smtp_port": 587,
-		"use_tls": 1,
+	Sendgrid: {
+		enable_outgoing: 1,
+		smtp_server: "smtp.sendgrid.net",
+		smtp_port: 587,
+		use_tls: 1,
 	},
-	"SparkPost": {
-		"enable_incoming": 0,
-		"enable_outgoing": 1,
-		"smtp_server": "smtp.sparkpostmail.com",
-		"smtp_port": 587,
-		"use_tls": 1
+	SparkPost: {
+		enable_incoming: 0,
+		enable_outgoing: 1,
+		smtp_server: "smtp.sparkpostmail.com",
+		smtp_port: 587,
+		use_tls: 1,
 	},
 	"Yahoo Mail": {
-		"email_server": "imap.mail.yahoo.com",
-		"use_ssl": 1,
-		"enable_outgoing": 1,
-		"smtp_server": "smtp.mail.yahoo.com",
-		"smtp_port": 587,
-		"use_tls": 1,
-		"use_imap": 1
+		email_server: "imap.mail.yahoo.com",
+		use_ssl: 1,
+		enable_outgoing: 1,
+		smtp_server: "smtp.mail.yahoo.com",
+		smtp_port: 587,
+		use_tls: 1,
+		use_imap: 1,
 	},
 	"Yandex.Mail": {
-		"email_server": "imap.yandex.com",
-		"use_ssl": 1,
-		"enable_outgoing": 1,
-		"smtp_server": "smtp.yandex.com",
-		"smtp_port": 587,
-		"use_tls": 1,
-		"use_imap": 1
+		email_server: "imap.yandex.com",
+		use_ssl: 1,
+		enable_outgoing: 1,
+		smtp_server: "smtp.yandex.com",
+		smtp_port: 587,
+		use_tls: 1,
+		use_imap: 1,
 	},
 };
 
 frappe.email_defaults_pop = {
-	"GMail": {
-		"email_server": "pop.gmail.com"
+	GMail: {
+		email_server: "pop.gmail.com",
 	},
 	"Outlook.com": {
-		"email_server": "pop3-mail.outlook.com"
+		email_server: "pop3-mail.outlook.com",
 	},
 	"Yahoo Mail": {
-		"email_server": "pop.mail.yahoo.com"
+		email_server: "pop.mail.yahoo.com",
 	},
 	"Yandex.Mail": {
-		"email_server": "pop.yandex.com"
+		email_server: "pop.yandex.com",
 	},
-
 };
 
 function oauth_access(frm) {
-	return frappe.call({
-		method: "frappe.email.oauth.oauth_access",
-		args: {
-			"email_account": frm.doc.name,
-			"service": frm.doc.service || ""
-		},
-		callback: function(r) {
-			if (!r.exc) {
-				window.open(r.message.url, "_self");
-			}
-		}
+	frappe.model.with_doc("Connected App", frm.doc.connected_app, () => {
+		const connected_app = frappe.get_doc("Connected App", frm.doc.connected_app);
+		return frappe.call({
+			doc: connected_app,
+			method: "initiate_web_application_flow",
+			args: {
+				success_uri: window.location.pathname,
+				user: frm.doc.connected_user,
+			},
+			callback: function (r) {
+				window.open(r.message, "_self");
+			},
+		});
 	});
 }
 
-function set_default_max_attachment_size(frm, field) {
-	if (frm.doc.__islocal && !frm.doc[field]) {
+function set_default_max_attachment_size(frm) {
+	if (frm.doc.__islocal && !frm.doc["attachment_limit"]) {
 		frappe.call({
 			method: "frappe.core.api.file.get_max_file_size",
-			callback: function(r) {
+			callback: function (r) {
 				if (!r.exc) {
-					frm.set_value(field, Number(r.message)/(1024*1024));
+					frm.set_value("attachment_limit", Number(r.message) / (1024 * 1024));
 				}
 			},
 		});
@@ -96,173 +97,136 @@ function set_default_max_attachment_size(frm, field) {
 }
 
 frappe.ui.form.on("Email Account", {
-	service: function(frm) {
-		$.each(frappe.email_defaults[frm.doc.service], function(key, value) {
+	service: function (frm) {
+		$.each(frappe.email_defaults[frm.doc.service], function (key, value) {
 			frm.set_value(key, value);
 		});
 		if (!frm.doc.use_imap) {
-			$.each(frappe.email_defaults_pop[frm.doc.service], function(key, value) {
+			$.each(frappe.email_defaults_pop[frm.doc.service], function (key, value) {
 				frm.set_value(key, value);
 			});
 		}
-		frm.events.show_gmail_message_for_less_secure_apps(frm);
-		frm.events.toggle_auth_method(frm);
 	},
 
-	use_imap: function(frm) {
+	use_imap: function (frm) {
 		if (!frm.doc.use_imap) {
-			$.each(frappe.email_defaults_pop[frm.doc.service], function(key, value) {
+			$.each(frappe.email_defaults_pop[frm.doc.service], function (key, value) {
 				frm.set_value(key, value);
 			});
-		}
-		else{
-			$.each(frappe.email_defaults[frm.doc.service], function(key, value) {
+		} else {
+			$.each(frappe.email_defaults[frm.doc.service], function (key, value) {
 				frm.set_value(key, value);
 			});
 		}
 	},
 
-	enable_incoming: function(frm) {
-		frm.doc.no_remaining = null; //perform full sync
-		//frm.set_df_property("append_to", "reqd", frm.doc.enable_incoming);
+	enable_incoming: function (frm) {
 		frm.trigger("warn_autoreply_on_incoming");
 	},
 
-	enable_auto_reply: function(frm) {
+	enable_auto_reply: function (frm) {
 		frm.trigger("warn_autoreply_on_incoming");
 	},
 
-	notify_if_unreplied: function(frm) {
+	notify_if_unreplied: function (frm) {
 		frm.set_df_property("send_notification_to", "reqd", frm.doc.notify_if_unreplied);
 	},
 
-	onload: function(frm) {
-		if (frappe.utils.get_query_params().successful_authorization === '1') {
-			frappe.show_alert(__("Successfully Authorized"));
-			// FIXME: find better alternative
-			window.history.replaceState(null, "", window.location.pathname);
-		}
-
+	onload: function (frm) {
 		frm.set_df_property("append_to", "only_select", true);
-		frm.set_query("append_to", "frappe.email.doctype.email_account.email_account.get_append_to");
-		frm.set_query("append_to", "imap_folder", function() {
+		frm.set_query(
+			"append_to",
+			"frappe.email.doctype.email_account.email_account.get_append_to"
+		);
+		frm.set_query("append_to", "imap_folder", function () {
 			return {
-				query: "frappe.email.doctype.email_account.email_account.get_append_to"
+				query: "frappe.email.doctype.email_account.email_account.get_append_to",
 			};
 		});
 		if (frm.doc.__islocal) {
-			frm.add_child("imap_folder", {"folder_name": "INBOX"});
+			frm.add_child("imap_folder", { folder_name: "INBOX" });
 			frm.refresh_field("imap_folder");
 		}
-		frm.toggle_display(['auth_method'], frm.doc.service === "GMail");
-		set_default_max_attachment_size(frm, "attachment_limit");
+		set_default_max_attachment_size(frm);
+		frm.events.show_oauth_authorization_message(frm);
 	},
 
-	refresh: function(frm) {
-		frm.events.set_domain_fields(frm);
+	refresh: function (frm) {
 		frm.events.enable_incoming(frm);
 		frm.events.notify_if_unreplied(frm);
-		frm.events.show_gmail_message_for_less_secure_apps(frm);
-		frm.events.show_oauth_authorization_message(frm);
 
 		if (frappe.route_flags.delete_user_from_locals && frappe.route_flags.linked_user) {
 			delete frappe.route_flags.delete_user_from_locals;
-			delete locals['User'][frappe.route_flags.linked_user];
+			delete locals["User"][frappe.route_flags.linked_user];
 		}
 	},
 
-	after_save(frm) {
-		if (frm.doc.auth_method === "OAuth" && !frm.doc.refresh_token) {
-			oauth_access(frm);
-		}
-	},
-
-	toggle_auth_method: function(frm) {
-		if (frm.doc.service !== "GMail") {
-			frm.toggle_display(['auth_method'], false);
-			frm.doc.auth_method = "Basic";
-		} else {
-			frm.toggle_display(['auth_method'], true);
-		}
-	},
-
-	show_gmail_message_for_less_secure_apps: function(frm) {
-		frm.dashboard.clear_headline();
-		let msg = __("GMail will only work if you enable 2-step authentication and use app-specific password OR use OAuth.");
-		let cta = __("Read the step by step guide here.");
-		msg += ` ${cta}`;
-		if (frm.doc.service==="GMail") {
-			frm.dashboard.set_headline_alert(msg);
-		}
-	},
-
-	show_oauth_authorization_message(frm) {
-		if (frm.doc.auth_method === "OAuth" && !frm.doc.refresh_token) {
-			let msg = __('OAuth has been enabled but not authorised. Please use "Authorise API Access" button to do the same.')
-			frm.dashboard.clear_headline();
-			frm.dashboard.set_headline_alert(msg, "yellow");
-		}
-	},
-
-	authorize_api_access: function(frm) {
+	authorize_api_access: function (frm) {
 		oauth_access(frm);
 	},
 
-	email_id:function(frm) {
-		//pull domain and if no matching domain go create one
-		frm.events.update_domain(frm);
+	show_oauth_authorization_message(frm) {
+		if (frm.doc.auth_method === "OAuth" && frm.doc.connected_app) {
+			frappe.call({
+				method: "frappe.integrations.doctype.connected_app.connected_app.has_token",
+				args: {
+					connected_app: frm.doc.connected_app,
+					connected_user: frm.doc.connected_user,
+				},
+				callback: (r) => {
+					if (!r.message) {
+						let msg = __(
+							'OAuth has been enabled but not authorised. Please use "Authorise API Access" button to do the same.'
+						);
+						frm.dashboard.clear_headline();
+						frm.dashboard.set_headline_alert(msg, "yellow");
+					}
+				},
+			});
+		}
 	},
 
-	update_domain: function(frm) {
-		if (!frm.doc.email_id && !frm.doc.service) {
-			return;
+	domain: frappe.utils.debounce((frm) => {
+		if (frm.doc.domain) {
+			frappe.call({
+				method: "get_domain_values",
+				doc: frm.doc,
+				args: {
+					domain: frm.doc.domain,
+				},
+				callback: function (r) {
+					if (!r.exc) {
+						for (let field in r.message) {
+							frm.set_value(field, r.message[field]);
+						}
+					}
+				},
+			});
 		}
+	}),
 
-		frappe.call({
-			method: 'get_domain',
-			doc: frm.doc,
-			args: {
-				"email_id": frm.doc.email_id
-			},
-			callback: function(r) {
-				if (r.message) {
-					frm.events.set_domain_fields(frm, r.message);
-				}
-			}
-		});
-	},
-
-	set_domain_fields: function(frm, args) {
-		if (!args) {
-			args = frappe.route_flags.set_domain_values? frappe.route_options: {};
-		}
-
-		for(var field in args) {
-			frm.set_value(field, args[field]);
-		}
-
-		delete frappe.route_flags.set_domain_values;
-		frappe.route_options = {};
-	},
-
-	email_sync_option: function(frm) {
+	email_sync_option: function (frm) {
 		// confirm if the ALL sync option is selected
 
 		if (frm.doc.email_sync_option == "ALL") {
-			var msg = __("You are selecting Sync Option as ALL, It will resync all read as well as unread message from server. This may also cause the duplication of Communication (emails).");
-			frappe.confirm(msg, null, function() {
+			var msg = __(
+				"You are selecting Sync Option as ALL, It will resync all read as well as unread message from server. This may also cause the duplication of Communication (emails)."
+			);
+			frappe.confirm(msg, null, function () {
 				frm.set_value("email_sync_option", "UNSEEN");
 			});
 		}
 	},
 
-	warn_autoreply_on_incoming: function(frm) {
+	warn_autoreply_on_incoming: function (frm) {
 		if (frm.doc.enable_incoming && frm.doc.enable_auto_reply && frm.doc.__islocal) {
-			var msg = __("Enabling auto reply on an incoming email account will send automated replies to all the synchronized emails. Do you wish to continue?");
-			frappe.confirm(msg, null, function() {
+			var msg = __(
+				"Enabling auto reply on an incoming email account will send automated replies to all the synchronized emails. Do you wish to continue?"
+			);
+			frappe.confirm(msg, null, function () {
 				frm.set_value("enable_auto_reply", 0);
-				frappe.show_alert({message: __("Disabled Auto Reply"), indicator: "blue"});
+				frappe.show_alert({ message: __("Disabled Auto Reply"), indicator: "blue" });
 			});
 		}
-	}
+	},
 });
diff --git a/frappe/email/doctype/email_account/email_account.json b/frappe/email/doctype/email_account/email_account.json
index ecb5af7378..85241b8194 100644
--- a/frappe/email/doctype/email_account/email_account.json
+++ b/frappe/email/doctype/email_account/email_account.json
@@ -20,8 +20,8 @@
   "awaiting_password",
   "ascii_encode_password",
   "column_break_10",
-  "refresh_token",
-  "access_token",
+  "connected_app",
+  "connected_user",
   "login_id_is_different",
   "login_id",
   "mailbox_settings",
@@ -29,6 +29,7 @@
   "default_incoming",
   "use_imap",
   "use_ssl",
+  "use_starttls",
   "email_server",
   "incoming_port",
   "column_break_18",
@@ -70,7 +71,6 @@
   "brand_logo",
   "uidvalidity",
   "uidnext",
-  "no_remaining",
   "no_failed"
  ],
  "fields": [
@@ -145,7 +145,7 @@
    "hide_seconds": 1,
    "in_list_view": 1,
    "in_standard_filter": 1,
-   "label": "Domain (optional)",
+   "label": "Domain",
    "options": "Email Domain"
   },
   {
@@ -154,7 +154,7 @@
    "fieldtype": "Select",
    "hide_days": 1,
    "hide_seconds": 1,
-   "label": "Service (optional)",
+   "label": "Service",
    "options": "\nGMail\nSendgrid\nSparkPost\nYahoo Mail\nOutlook.com\nYandex.Mail"
   },
   {
@@ -203,7 +203,6 @@
    "label": "Use SSL"
   },
   {
-   "default": "1",
    "depends_on": "eval:!doc.domain && doc.enable_incoming",
    "description": "Ignore attachments over this size",
    "fetch_from": "domain.attachment_limit",
@@ -215,7 +214,7 @@
   },
   {
    "depends_on": "eval: doc.enable_incoming && !doc.use_imap",
-   "description": "Append as communication against this DocType (must have fields, \"Status\", \"Subject\")",
+   "description": "Append as communication against this DocType (must have fields: \"Sender\" and \"Subject\"). These fields can be defined in the email settings section of the appended doctype.",
    "fieldname": "append_to",
    "fieldtype": "Link",
    "hide_days": 1,
@@ -472,15 +471,6 @@
    "label": "UIDNEXT",
    "no_copy": 1
   },
-  {
-   "fieldname": "no_remaining",
-   "fieldtype": "Data",
-   "hidden": 1,
-   "hide_days": 1,
-   "hide_seconds": 1,
-   "label": "No of emails remaining to be synced",
-   "no_copy": 1
-  },
   {
    "fieldname": "no_failed",
    "fieldtype": "Int",
@@ -586,37 +576,47 @@
    "label": "IMAP Details"
   },
   {
-   "depends_on": "eval: doc.service === \"GMail\" && doc.auth_method === \"OAuth\" && !doc.__islocal && !doc.__unsaved",
+   "depends_on": "eval: doc.auth_method === \"OAuth\" && !doc.__islocal && !doc.__unsaved",
    "fieldname": "authorize_api_access",
    "fieldtype": "Button",
    "label": "Authorize API Access"
   },
-  {
-   "fieldname": "refresh_token",
-   "fieldtype": "Small Text",
-   "hidden": 1,
-   "label": "Refresh Token",
-   "read_only": 1
-  },
-  {
-   "fieldname": "access_token",
-   "fieldtype": "Small Text",
-   "hidden": 1,
-   "label": "Access Token",
-   "read_only": 1
-  },
   {
    "default": "Basic",
    "fieldname": "auth_method",
    "fieldtype": "Select",
    "label": "Method",
    "options": "Basic\nOAuth"
+  },
+  {
+   "default": "0",
+   "depends_on": "eval:!doc.domain && doc.enable_incoming && doc.use_imap && !doc.use_ssl",
+   "fetch_from": "domain.use_starttls",
+   "fieldname": "use_starttls",
+   "fieldtype": "Check",
+   "label": "Use STARTTLS"
+  },
+  {
+   "depends_on": "eval: doc.auth_method === \"OAuth\"",
+   "fieldname": "connected_app",
+   "fieldtype": "Link",
+   "label": "Connected App",
+   "mandatory_depends_on": "eval: doc.auth_method === \"OAuth\"",
+   "options": "Connected App"
+  },
+  {
+   "depends_on": "eval: doc.auth_method === \"OAuth\"",
+   "fieldname": "connected_user",
+   "fieldtype": "Link",
+   "label": "Connected User",
+   "mandatory_depends_on": "eval: doc.auth_method === \"OAuth\"",
+   "options": "User"
   }
  ],
  "icon": "fa fa-inbox",
  "index_web_pages_for_search": 1,
  "links": [],
- "modified": "2022-07-11 18:34:06.945668",
+ "modified": "2022-12-28 14:56:18.754804",
  "modified_by": "Administrator",
  "module": "Email",
  "name": "Email Account",
@@ -628,7 +628,6 @@
    "delete": 1,
    "read": 1,
    "role": "System Manager",
-   "set_user_permissions": 1,
    "write": 1
   },
   {
@@ -640,4 +639,4 @@
  "sort_order": "DESC",
  "states": [],
  "track_changes": 1
-}
\ No newline at end of file
+}
diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py
index 589ddf42f0..faf28afdb3 100755
--- a/frappe/email/doctype/email_account/email_account.py
+++ b/frappe/email/doctype/email_account/email_account.py
@@ -12,15 +12,14 @@ from poplib import error_proto
 import frappe
 from frappe import _, are_emails_muted, safe_encode
 from frappe.desk.form import assign_to
+from frappe.email.doctype.email_domain.email_domain import EMAIL_DOMAIN_FIELDS
 from frappe.email.receive import EmailServer, InboundMail, SentEmailInInboxError
 from frappe.email.smtp import SMTPServer
 from frappe.email.utils import get_port
 from frappe.model.document import Document
 from frappe.utils import cint, comma_or, cstr, parse_addr, validate_email_address
 from frappe.utils.background_jobs import enqueue, get_jobs
-from frappe.utils.error import raise_error_on_no_output
 from frappe.utils.jinja import render_template
-from frappe.utils.password import decrypt, encrypt
 from frappe.utils.user import get_system_managers
 
 
@@ -82,21 +81,16 @@ class EmailAccount(Document):
 			return
 
 		use_oauth = self.auth_method == "OAuth"
-
-		if getattr(self, "service", "") != "GMail" and use_oauth:
-			self.auth_method = "Basic"
-			use_oauth = False
+		validate_oauth = use_oauth and not (self.is_new() and not self.get_oauth_token())
+		self.use_starttls = cint(self.use_imap and self.use_starttls and not self.use_ssl)
 
 		if use_oauth:
 			# no need for awaiting password for oauth
 			self.awaiting_password = 0
-
-		elif self.refresh_token:
-			# clear access & refresh token
-			self.refresh_token = self.access_token = None
+			self.password = None
 
 		if not frappe.local.flags.in_install and not self.awaiting_password:
-			if self.refresh_token or self.password or self.smtp_server in ("127.0.0.1", "localhost"):
+			if validate_oauth or self.password or self.smtp_server in ("127.0.0.1", "localhost"):
 				if self.enable_incoming:
 					self.get_incoming_server()
 					self.no_failed = 0
@@ -177,46 +171,29 @@ class EmailAccount(Document):
 				email_account.save()
 
 	@frappe.whitelist()
-	def get_domain(self, email_id):
-		"""look-up the domain and then full"""
-		try:
-			domain = email_id.split("@")
-			fields = [
-				"name as domain",
-				"use_imap",
-				"email_server",
-				"use_ssl",
-				"smtp_server",
-				"use_tls",
-				"smtp_port",
-				"incoming_port",
-				"append_emails_to_sent_folder",
-				"use_ssl_for_outgoing",
-			]
-			return frappe.db.get_value("Email Domain", domain[1], fields, as_dict=True)
-		except Exception:
-			pass
+	def get_domain_values(self, domain: str):
+		return frappe.db.get_value("Email Domain", domain, EMAIL_DOMAIN_FIELDS, as_dict=True)
 
 	def get_incoming_server(self, in_receive=False, email_sync_rule="UNSEEN"):
 		"""Returns logged in POP3/IMAP connection object."""
 		if frappe.cache().get_value("workers:no-internet") == True:
 			return None
 
+		oauth_token = self.get_oauth_token()
 		args = frappe._dict(
 			{
 				"email_account_name": self.email_account_name,
 				"email_account": self.name,
 				"host": self.email_server,
 				"use_ssl": self.use_ssl,
+				"use_starttls": self.use_starttls,
 				"username": getattr(self, "login_id", None) or self.email_id,
-				"service": getattr(self, "service", ""),
 				"use_imap": self.use_imap,
 				"email_sync_rule": email_sync_rule,
 				"incoming_port": get_port(self),
 				"initial_sync_count": self.initial_sync_count or 100,
 				"use_oauth": self.auth_method == "OAuth",
-				"refresh_token": decrypt(self.refresh_token) if self.refresh_token else None,
-				"access_token": decrypt(self.access_token) if self.access_token else None,
+				"access_token": oauth_token.get_password("access_token") if oauth_token else None,
 			}
 		)
 
@@ -323,11 +300,6 @@ class EmailAccount(Document):
 		return cls.from_record({"sender": "notifications@example.com"})
 
 	@classmethod
-	@raise_error_on_no_output(
-		keep_quiet=lambda: not cint(frappe.get_system_settings("setup_complete")),
-		error_message=_("Please setup default Email Account from Setup > Email > Email Account"),
-		error_type=frappe.OutgoingEmailError,
-	)  # noqa
 	@cache_email_account("outgoing_email_account")
 	def find_outgoing(cls, match_by_email=None, match_by_doctype=None, _raise_error=False):
 		"""Find the outgoing Email account to use.
@@ -351,6 +323,12 @@ class EmailAccount(Document):
 		if doc:
 			return {"default": doc}
 
+		if _raise_error:
+			frappe.throw(
+				_("Please setup default Email Account from Settings > Email Account"),
+				frappe.OutgoingEmailError,
+			)
+
 	@classmethod
 	def find_default_outgoing(cls):
 		"""Find default outgoing account."""
@@ -405,8 +383,6 @@ class EmailAccount(Document):
 			},
 			"name": {"conf_names": ("email_sender_name",), "default": "Frappe"},
 			"auth_method": {"conf_names": ("auth_method"), "default": "Basic"},
-			"access_token": {"conf_names": ("mail_access_token")},
-			"refresh_token": {"conf_names": ("mail_refresh_token")},
 			"from_site_config": {"default": True},
 		}
 
@@ -414,15 +390,13 @@ class EmailAccount(Document):
 		for doc_field_name, d in field_to_conf_name_map.items():
 			conf_names, default = d.get("conf_names") or [], d.get("default")
 			value = [frappe.conf.get(k) for k in conf_names if frappe.conf.get(k)]
-
-			if doc_field_name in ("refresh_token", "access_token"):
-				account_details[doc_field_name] = value and encrypt(value[0])
-			else:
-				account_details[doc_field_name] = (value and value[0]) or default
+			account_details[doc_field_name] = (value and value[0]) or default
 
 		return account_details
 
 	def sendmail_config(self):
+		oauth_token = self.get_oauth_token()
+
 		return {
 			"email_account": self.name,
 			"server": self.smtp_server,
@@ -431,10 +405,8 @@ class EmailAccount(Document):
 			"password": self._password,
 			"use_ssl": cint(self.use_ssl_for_outgoing),
 			"use_tls": cint(self.use_tls),
-			"service": getattr(self, "service", ""),
 			"use_oauth": self.auth_method == "OAuth",
-			"refresh_token": decrypt(self.refresh_token) if self.refresh_token else None,
-			"access_token": decrypt(self.access_token) if self.access_token else None,
+			"access_token": oauth_token.get_password("access_token") if oauth_token else None,
 		}
 
 	def get_smtp_server(self):
@@ -499,12 +471,6 @@ class EmailAccount(Document):
 			else:
 				frappe.db.commit()
 
-		# notify if user is linked to account
-		if len(inbound_mails) > 0 and not frappe.local.flags.in_test:
-			frappe.publish_realtime(
-				"new_email", {"account": self.email_account_name, "number": len(inbound_mails)}
-			)
-
 		if exceptions:
 			raise Exception(frappe.as_json(exceptions))
 
@@ -691,8 +657,6 @@ class EmailAccount(Document):
 		if not email_server:
 			return
 
-		email_server.connect()
-
 		if email_server.imap:
 			try:
 				message = safe_encode(message)
@@ -700,6 +664,11 @@ class EmailAccount(Document):
 			except Exception:
 				self.log_error("Unable to add to Sent folder")
 
+	def get_oauth_token(self):
+		if self.auth_method == "OAuth":
+			connected_app = frappe.get_doc("Connected App", self.connected_app)
+			return connected_app.get_active_token(self.connected_user)
+
 
 @frappe.whitelist()
 def get_append_to(
@@ -795,25 +764,29 @@ def notify_unreplied():
 
 def pull(now=False):
 	"""Will be called via scheduler, pull emails from all enabled Email accounts."""
+	from frappe.integrations.doctype.connected_app.connected_app import has_token
 
 	if frappe.cache().get_value("workers:no-internet") == True:
 		if test_internet():
 			frappe.cache().set_value("workers:no-internet", False)
-		else:
-			return
+		return
 
 	doctype = frappe.qb.DocType("Email Account")
 	email_accounts = (
 		frappe.qb.from_(doctype)
-		.select(doctype.name)
+		.select(doctype.name, doctype.auth_method, doctype.connected_app, doctype.connected_user)
 		.where(doctype.enable_incoming == 1)
-		.where(
-			(doctype.awaiting_password == 0)
-			| ((doctype.auth_method == "OAuth") & (doctype.refresh_token.isnotnull()))
-		)
+		.where(doctype.awaiting_password == 0)
 		.run(as_dict=1)
 	)
+
 	for email_account in email_accounts:
+		if email_account.auth_method == "OAuth" and not has_token(
+			email_account.connected_app, email_account.connected_user
+		):
+			# don't try to pull from accounts which dont have access token (for Oauth)
+			continue
+
 		if now:
 			pull_from_email_account(email_account.name)
 
@@ -842,7 +815,7 @@ def get_max_email_uid(email_account):
 	# get maximum uid of emails
 	max_uid = 1
 
-	result = frappe.db.get_all(
+	result = frappe.get_all(
 		"Communication",
 		filters={
 			"communication_medium": "Email",
@@ -936,7 +909,7 @@ def remove_user_email_inbox(email_account):
 @frappe.whitelist()
 def set_email_password(email_account, password):
 	account = frappe.get_doc("Email Account", email_account)
-	if account.awaiting_password and not account.auth_method == "OAuth":
+	if account.awaiting_password and account.auth_method != "OAuth":
 		account.awaiting_password = 0
 		account.password = password
 		try:
diff --git a/frappe/email/doctype/email_account/email_account_list.js b/frappe/email/doctype/email_account/email_account_list.js
index 5ec56fb3db..5913706cbf 100644
--- a/frappe/email/doctype/email_account/email_account_list.js
+++ b/frappe/email/doctype/email_account/email_account_list.js
@@ -1,23 +1,24 @@
 frappe.listview_settings["Email Account"] = {
 	add_fields: ["default_incoming", "default_outgoing", "enable_incoming", "enable_outgoing"],
-	get_indicator: function(doc) {
-		if(doc.default_incoming && doc.default_outgoing) {
-			var color = (doc.enable_incoming && doc.enable_outgoing) ? "blue" : "gray";
-			return [__("Default Sending and Inbox"), color, "default_incoming,=,Yes|default_outgoing,=,Yes"]
-		}
-		else if(doc.default_incoming) {
+	get_indicator: function (doc) {
+		if (doc.default_incoming && doc.default_outgoing) {
+			var color = doc.enable_incoming && doc.enable_outgoing ? "blue" : "gray";
+			return [
+				__("Default Sending and Inbox"),
+				color,
+				"default_incoming,=,Yes|default_outgoing,=,Yes",
+			];
+		} else if (doc.default_incoming) {
 			color = doc.enable_incoming ? "blue" : "gray";
 			return [__("Default Inbox"), color, "default_incoming,=,Yes"];
-		}
-		else if(doc.default_outgoing) {
+		} else if (doc.default_outgoing) {
 			color = doc.enable_outgoing ? "blue" : "gray";
 			return [__("Default Sending"), color, "default_outgoing,=,Yes"];
-		}
-		else {
+		} else {
 			color = doc.enable_incoming ? "blue" : "gray";
 			return [__("Inbox"), color, "is_global,=,No|is_default=No"];
 		}
-	}
-}
+	},
+};
 
 frappe.help.youtube_id["Email Account"] = "YFYe0DrB95o";
diff --git a/frappe/email/doctype/email_account/test_email_account.py b/frappe/email/doctype/email_account/test_email_account.py
index 9ab004cdd0..a1740b0b0a 100644
--- a/frappe/email/doctype/email_account/test_email_account.py
+++ b/frappe/email/doctype/email_account/test_email_account.py
@@ -14,14 +14,16 @@ from frappe.email.doctype.email_account.email_account import notify_unreplied
 from frappe.email.email_body import get_message_id
 from frappe.email.receive import Email, InboundMail, SentEmailInInboxError
 from frappe.test_runner import make_test_records
+from frappe.tests.utils import FrappeTestCase
 
 make_test_records("User")
 make_test_records("Email Account")
 
 
-class TestEmailAccount(unittest.TestCase):
+class TestEmailAccount(FrappeTestCase):
 	@classmethod
 	def setUpClass(cls):
+		super().setUpClass()
 		email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
 		email_account.db_set("enable_incoming", 1)
 		email_account.db_set("enable_auto_reply", 1)
@@ -362,6 +364,7 @@ class TestEmailAccount(unittest.TestCase):
 		self.assertTrue(communication.reference_name)
 		self.assertTrue(frappe.db.exists(communication.reference_doctype, communication.reference_name))
 
+	@unittest.skip("poorly written and flaky")
 	def test_append_to_with_imap_folders(self):
 		mail_content_1 = self.get_test_mail(fname="incoming-1.raw")
 		mail_content_2 = self.get_test_mail(fname="incoming-2.raw")
@@ -434,9 +437,10 @@ class TestEmailAccount(unittest.TestCase):
 			email_account.receive()
 
 
-class TestInboundMail(unittest.TestCase):
+class TestInboundMail(FrappeTestCase):
 	@classmethod
 	def setUpClass(cls):
+		super().setUpClass()
 		email_account = frappe.get_doc("Email Account", "_Test Email Account 1")
 		email_account.db_set("enable_incoming", 1)
 
diff --git a/frappe/email/doctype/email_account/test_records.json b/frappe/email/doctype/email_account/test_records.json
index 66eb5a9b2e..2e204e5277 100644
--- a/frappe/email/doctype/email_account/test_records.json
+++ b/frappe/email/doctype/email_account/test_records.json
@@ -18,7 +18,6 @@
 		"unreplied_for_mins": 20,
 		"send_notification_to": "test_unreplied@example.com",
 		"pop3_server": "pop.test.example.com",
-		"no_remaining":"0",
 		"append_to": "ToDo",
 		"imap_folder": [{"folder_name": "INBOX", "append_to": "ToDo"}, {"folder_name": "Test Folder", "append_to": "Communication"}],
 		"track_email_status": 1
diff --git a/frappe/email/doctype/email_domain/email_domain.js b/frappe/email/doctype/email_domain/email_domain.js
index 1716bf9900..04b8581466 100644
--- a/frappe/email/doctype/email_domain/email_domain.js
+++ b/frappe/email/doctype/email_domain/email_domain.js
@@ -1,29 +1,22 @@
-
 frappe.ui.form.on("Email Domain", {
-	email_id:function(frm){
-		frm.set_value("domain_name",frm.doc.email_id.split("@")[1])
+	onload: function (frm) {
+		if (!frm.doc.__islocal) {
+			frm.dashboard.clear_headline();
+			let msg = __(
+				"Changing any setting will reflect on all the email accounts associated with this domain."
+			);
+			frm.dashboard.set_headline_alert(msg);
+		} else {
+			if (!frm.doc.attachment_limit) {
+				frappe.call({
+					method: "frappe.core.api.file.get_max_file_size",
+					callback: function (r) {
+						if (!r.exc) {
+							frm.set_value("attachment_limit", Number(r.message) / (1024 * 1024));
+						}
+					},
+				});
+			}
+		}
 	},
-
-	refresh:function(frm){
-		if (frm.doc.email_id){
-			frm.set_value("domain_name",frm.doc.email_id.split("@")[1])
-		}
-
-		if (frm.doc.__islocal != 1 && frappe.route_flags.return_to_email_account) {
-			var route = frappe.get_prev_route();
-			delete frappe.route_flags.return_to_email_account;
-			frappe.route_flags.set_domain_values = true
-
-			frappe.route_options = {
-				domain: frm.doc.name,
-				use_imap: frm.doc.use_imap,
-				email_server: frm.doc.email_server,
-				use_ssl: frm.doc.use_ssl,
-				smtp_server: frm.doc.smtp_server,
-				use_tls: frm.doc.use_tls,
-				smtp_port: frm.doc.smtp_port
-			},
-			frappe.set_route(route);
-		}
-	}
-})
\ No newline at end of file
+});
diff --git a/frappe/email/doctype/email_domain/email_domain.json b/frappe/email/doctype/email_domain/email_domain.json
index a4ca19a0bd..c162060436 100644
--- a/frappe/email/doctype/email_domain/email_domain.json
+++ b/frappe/email/doctype/email_domain/email_domain.json
@@ -8,20 +8,21 @@
  "field_order": [
   "email_settings",
   "domain_name",
-  "email_id",
   "mailbox_settings",
   "email_server",
   "use_imap",
   "use_ssl",
+  "use_starttls",
+  "column_break_9",
   "incoming_port",
   "attachment_limit",
-  "append_to",
   "outgoing_mail_settings",
   "smtp_server",
   "use_tls",
   "use_ssl_for_outgoing",
-  "append_emails_to_sent_folder",
-  "smtp_port"
+  "column_break_18",
+  "smtp_port",
+  "append_emails_to_sent_folder"
  ],
  "fields": [
   {
@@ -31,26 +32,21 @@
   {
    "fieldname": "domain_name",
    "fieldtype": "Data",
-   "label": "domain name",
-   "read_only": 1,
+   "label": "Domain Name",
+   "reqd": 1,
    "unique": 1
   },
-  {
-   "fieldname": "email_id",
-   "fieldtype": "Data",
-   "label": "Example Email Address",
-   "options": "Email",
-   "reqd": 1
-  },
   {
    "fieldname": "mailbox_settings",
-   "fieldtype": "Section Break"
+   "fieldtype": "Section Break",
+   "label": "Incoming Settings"
   },
   {
    "description": "e.g. pop.gmail.com / imap.gmail.com",
    "fieldname": "email_server",
    "fieldtype": "Data",
-   "label": "Email Server",
+   "in_list_view": 1,
+   "label": "Incoming Server",
    "reqd": 1
   },
   {
@@ -66,30 +62,29 @@
    "label": "Use SSL"
   },
   {
-   "default": "1",
+   "default": "0",
+   "depends_on": "eval:doc.use_imap && !doc.use_ssl",
+   "fieldname": "use_starttls",
+   "fieldtype": "Check",
+   "label": "Use STARTTLS"
+  },
+  {
    "description": "Ignore attachments over this size",
    "fieldname": "attachment_limit",
    "fieldtype": "Int",
    "label": "Attachment Limit (MB)"
   },
-  {
-   "description": "Append as communication against this DocType (must have fields, \"Status\", \"Subject\")",
-   "fieldname": "append_to",
-   "fieldtype": "Link",
-   "hidden": 1,
-   "in_list_view": 1,
-   "label": "Append To",
-   "options": "DocType"
-  },
   {
    "fieldname": "outgoing_mail_settings",
-   "fieldtype": "Section Break"
+   "fieldtype": "Section Break",
+   "label": "Outgoing Settings"
   },
   {
    "description": "e.g. smtp.gmail.com",
    "fieldname": "smtp_server",
    "fieldtype": "Data",
-   "label": "SMTP Server",
+   "in_list_view": 1,
+   "label": "Outgoing Server",
    "reqd": 1
   },
   {
@@ -120,15 +115,29 @@
    "default": "0",
    "fieldname": "use_ssl_for_outgoing",
    "fieldtype": "Check",
-   "label": "Use SSL for Outgoing"
+   "label": "Use SSL"
+  },
+  {
+   "fieldname": "column_break_9",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "column_break_18",
+   "fieldtype": "Column Break"
   }
  ],
  "icon": "icon-inbox",
- "links": [],
- "modified": "2019-12-18 15:57:34.445308",
+ "links": [
+  {
+   "link_doctype": "Email Account",
+   "link_fieldname": "domain"
+  }
+ ],
+ "modified": "2022-08-19 12:55:06.434541",
  "modified_by": "Administrator",
  "module": "Email",
  "name": "Email Domain",
+ "naming_rule": "By fieldname",
  "owner": "Administrator",
  "permissions": [
   {
@@ -136,11 +145,12 @@
    "delete": 1,
    "read": 1,
    "role": "System Manager",
-   "set_user_permissions": 1,
    "share": 1,
    "write": 1
   }
  ],
  "sort_field": "modified",
- "sort_order": "DESC"
-}
\ No newline at end of file
+ "sort_order": "DESC",
+ "states": [],
+ "track_changes": 1
+}
diff --git a/frappe/email/doctype/email_domain/email_domain.py b/frappe/email/doctype/email_domain/email_domain.py
index ab6546e4e6..528407916a 100644
--- a/frappe/email/doctype/email_domain/email_domain.py
+++ b/frappe/email/doctype/email_domain/email_domain.py
@@ -1,119 +1,72 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors
 # License: MIT. See LICENSE
 
-import imaplib
-import poplib
 import smtplib
+from functools import wraps
 
 import frappe
 from frappe import _
+from frappe.email.receive import Timed_IMAP4, Timed_IMAP4_SSL, Timed_POP3, Timed_POP3_SSL
 from frappe.email.utils import get_port
 from frappe.model.document import Document
-from frappe.utils import cint, cstr, validate_email_address
+from frappe.utils import cint
+
+EMAIL_DOMAIN_FIELDS = [
+	"email_server",
+	"use_imap",
+	"use_ssl",
+	"use_starttls",
+	"use_tls",
+	"attachment_limit",
+	"smtp_server",
+	"smtp_port",
+	"use_ssl_for_outgoing",
+	"append_emails_to_sent_folder",
+	"incoming_port",
+]
+
+
+def get_error_message(event):
+	return {
+		"incoming": (_("Incoming email account not correct"), _("Error connecting via IMAP/POP3: {e}")),
+		"outgoing": (_("Outgoing email account not correct"), _("Error connecting via SMTP: {e}")),
+	}[event]
+
+
+def handle_error(event):
+	def decorator(fn):
+		@wraps(fn)
+		def wrapper(*args, **kwargs):
+			err_title, err_message = get_error_message(event)
+			try:
+				fn(*args, **kwargs)
+			except Exception as e:
+				frappe.throw(
+					title=err_title,
+					msg=err_message.format(e=e),
+				)
+
+		return wrapper
+
+	return decorator
 
 
 class EmailDomain(Document):
-	def autoname(self):
-		if self.domain_name:
-			self.name = self.domain_name
-
 	def validate(self):
-		"""Validate email id and check POP3/IMAP and SMTP connections is enabled."""
-		logger = frappe.logger()
+		"""Validate POP3/IMAP and SMTP connections."""
 
-		if self.email_id:
-			validate_email_address(self.email_id, True)
-
-		if frappe.local.flags.in_patch or frappe.local.flags.in_test:
+		if frappe.local.flags.in_patch or frappe.local.flags.in_test or frappe.local.flags.in_install:
 			return
 
-		if not frappe.local.flags.in_install and not frappe.local.flags.in_patch:
-			try:
-				if self.use_imap:
-					logger.info(
-						"Checking incoming IMAP email server {host}:{port} ssl={ssl}...".format(
-							host=self.email_server, port=get_port(self), ssl=self.use_ssl
-						)
-					)
-					if self.use_ssl:
-						test = imaplib.IMAP4_SSL(self.email_server, port=get_port(self))
-					else:
-						test = imaplib.IMAP4(self.email_server, port=get_port(self))
-
-				else:
-					logger.info(
-						"Checking incoming POP3 email server {host}:{port} ssl={ssl}...".format(
-							host=self.email_server, port=get_port(self), ssl=self.use_ssl
-						)
-					)
-					if self.use_ssl:
-						test = poplib.POP3_SSL(self.email_server, port=get_port(self))
-					else:
-						test = poplib.POP3(self.email_server, port=get_port(self))
-
-			except Exception as e:
-				logger.warn(f'Incoming email account "{self.email_server}" not correct', exc_info=e)
-				frappe.throw(
-					title=_("Incoming email account not correct"),
-					msg=f'Error connecting IMAP/POP3 "{self.email_server}": {e}',
-				)
-
-			finally:
-				try:
-					if self.use_imap:
-						test.logout()
-					else:
-						test.quit()
-				except Exception:
-					pass
-
-			try:
-				if self.get("use_ssl_for_outgoing"):
-					if not self.get("smtp_port"):
-						self.smtp_port = 465
-
-					logger.info(
-						"Checking outgoing SMTPS email server {host}:{port}...".format(
-							host=self.smtp_server, port=self.smtp_port
-						)
-					)
-					sess = smtplib.SMTP_SSL(
-						(self.smtp_server or "").encode("utf-8"), cint(self.smtp_port) or None
-					)
-				else:
-					if self.use_tls and not self.smtp_port:
-						self.smtp_port = 587
-					logger.info(
-						"Checking outgoing SMTP email server {host}:{port} STARTTLS={tls}...".format(
-							host=self.smtp_server, port=self.get("smtp_port"), tls=self.use_tls
-						)
-					)
-					sess = smtplib.SMTP(cstr(self.smtp_server or ""), cint(self.smtp_port) or None)
-				sess.quit()
-			except Exception as e:
-				logger.warn(f'Outgoing email account "{self.smtp_server}" not correct', exc_info=e)
-				frappe.throw(
-					title=_("Outgoing email account not correct"),
-					msg=f'Error connecting SMTP "{self.smtp_server}": {e}',
-				)
+		self.validate_incoming_server_conn()
+		self.validate_outgoing_server_conn()
 
 	def on_update(self):
 		"""update all email accounts using this domain"""
 		for email_account in frappe.get_all("Email Account", filters={"domain": self.name}):
 			try:
 				email_account = frappe.get_doc("Email Account", email_account.name)
-				for attr in [
-					"email_server",
-					"use_imap",
-					"use_ssl",
-					"use_tls",
-					"attachment_limit",
-					"smtp_server",
-					"smtp_port",
-					"use_ssl_for_outgoing",
-					"append_emails_to_sent_folder",
-					"incoming_port",
-				]:
+				for attr in EMAIL_DOMAIN_FIELDS:
 					email_account.set(attr, self.get(attr, default=0))
 				email_account.save()
 
@@ -121,3 +74,28 @@ class EmailDomain(Document):
 				frappe.msgprint(
 					_("Error has occurred in {0}").format(email_account.name), raise_exception=e.__class__
 				)
+
+	@handle_error("incoming")
+	def validate_incoming_server_conn(self):
+		self.incoming_port = get_port(self)
+
+		if self.use_imap:
+			conn_method = Timed_IMAP4_SSL if self.use_ssl else Timed_IMAP4
+		else:
+			conn_method = Timed_POP3_SSL if self.use_ssl else Timed_POP3
+
+		self.use_starttls = cint(self.use_imap and self.use_starttls and not self.use_ssl)
+		incoming_conn = conn_method(self.email_server, port=self.incoming_port)
+		incoming_conn.logout() if self.use_imap else incoming_conn.quit()
+
+	@handle_error("outgoing")
+	def validate_outgoing_server_conn(self):
+		conn_method = smtplib.SMTP
+
+		if self.use_ssl_for_outgoing:
+			self.smtp_port = self.smtp_port or 465
+			conn_method = smtplib.SMTP_SSL
+		elif self.use_tls:
+			self.smtp_port = self.smtp_port or 587
+
+		conn_method((self.smtp_server or ""), cint(self.smtp_port) or 0).quit()
diff --git a/frappe/email/doctype/email_domain/test_email_domain.py b/frappe/email/doctype/email_domain/test_email_domain.py
index 55a8d620a8..85ba6450b5 100644
--- a/frappe/email/doctype/email_domain/test_email_domain.py
+++ b/frappe/email/doctype/email_domain/test_email_domain.py
@@ -1,14 +1,13 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
-import unittest
-
 import frappe
 from frappe.test_runner import make_test_objects
+from frappe.tests.utils import FrappeTestCase
 
 test_records = frappe.get_test_records("Email Domain")
 
 
-class TestDomain(unittest.TestCase):
+class TestDomain(FrappeTestCase):
 	def setUp(self):
 		make_test_objects("Email Domain", reset=True)
 
@@ -33,6 +32,7 @@ class TestDomain(unittest.TestCase):
 		# Also make sure that the other attributes match
 		self.assertEqual(mail_account.use_imap, mail_domain.use_imap)
 		self.assertEqual(mail_account.use_ssl, mail_domain.use_ssl)
+		self.assertEqual(mail_account.use_starttls, mail_domain.use_starttls)
 		self.assertEqual(mail_account.use_tls, mail_domain.use_tls)
 		self.assertEqual(mail_account.attachment_limit, mail_domain.attachment_limit)
 		self.assertEqual(mail_account.smtp_server, mail_domain.smtp_server)
diff --git a/frappe/email/doctype/email_flag_queue/email_flag_queue.js b/frappe/email/doctype/email_flag_queue/email_flag_queue.js
index 19c4d4b0c1..ee062815f5 100644
--- a/frappe/email/doctype/email_flag_queue/email_flag_queue.js
+++ b/frappe/email/doctype/email_flag_queue/email_flag_queue.js
@@ -1,8 +1,6 @@
 // Copyright (c) 2016, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Email Flag Queue', {
-	refresh: function(frm) {
-
-	}
+frappe.ui.form.on("Email Flag Queue", {
+	refresh: function (frm) {},
 });
diff --git a/frappe/email/doctype/email_flag_queue/test_email_flag_queue.py b/frappe/email/doctype/email_flag_queue/test_email_flag_queue.py
index 8bfc9230a4..0d731c37da 100644
--- a/frappe/email/doctype/email_flag_queue/test_email_flag_queue.py
+++ b/frappe/email/doctype/email_flag_queue/test_email_flag_queue.py
@@ -1,9 +1,9 @@
 # Copyright (c) 2015, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
-import unittest
+from frappe.tests.utils import FrappeTestCase
 
 # test_records = frappe.get_test_records('Email Flag Queue')
 
 
-class TestEmailFlagQueue(unittest.TestCase):
+class TestEmailFlagQueue(FrappeTestCase):
 	pass
diff --git a/frappe/email/doctype/email_group/email_group.js b/frappe/email/doctype/email_group/email_group.js
index 404600c97d..5ad4a39dd9 100644
--- a/frappe/email/doctype/email_group/email_group.js
+++ b/frappe/email/doctype/email_group/email_group.js
@@ -1,46 +1,74 @@
 // Copyright (c) 2016, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on("Email Group", "refresh", function(frm) {
-	if(!frm.is_new()) {
-		frm.add_custom_button(__("Import Subscribers"), function() {
-			frappe.prompt({fieldtype:"Select", options: frm.doc.__onload.import_types,
-				label:__("Import Email From"), fieldname:"doctype", reqd:1},
-				function(data) {
-					frappe.call({
-						method: "frappe.email.doctype.email_group.email_group.import_from",
-						args: {
-							"name": frm.doc.name,
-							"doctype": data.doctype
-						},
-						callback: function(r) {
-							frm.set_value("total_subscribers", r.message);
-						}
-					})
-				}, __("Import Subscribers"), __("Import"));
-		}, __("Action"));
+frappe.ui.form.on("Email Group", "refresh", function (frm) {
+	if (!frm.is_new()) {
+		frm.add_custom_button(
+			__("Import Subscribers"),
+			function () {
+				frappe.prompt(
+					{
+						fieldtype: "Select",
+						options: frm.doc.__onload.import_types,
+						label: __("Import Email From"),
+						fieldname: "doctype",
+						reqd: 1,
+					},
+					function (data) {
+						frappe.call({
+							method: "frappe.email.doctype.email_group.email_group.import_from",
+							args: {
+								name: frm.doc.name,
+								doctype: data.doctype,
+							},
+							callback: function (r) {
+								frm.set_value("total_subscribers", r.message);
+							},
+						});
+					},
+					__("Import Subscribers"),
+					__("Import")
+				);
+			},
+			__("Action")
+		);
 
-		frm.add_custom_button(__("Add Subscribers"), function() {
-			frappe.prompt({fieldtype:"Text",
-				label:__("Email Addresses"), fieldname:"email_list", reqd:1},
-				function(data) {
-					frappe.call({
-						method: "frappe.email.doctype.email_group.email_group.add_subscribers",
-						args: {
-							"name": frm.doc.name,
-							"email_list": data.email_list
-						},
-						callback: function(r) {
-							frm.set_value("total_subscribers", r.message);
-						}
-					})
-				}, __("Add Subscribers"), __("Add"));
-		}, __("Action"));
-
-		frm.add_custom_button(__("New Newsletter"), function() {
-			frappe.route_options = {"email_group": frm.doc.name};
-			frappe.new_doc("Newsletter");
-		}, __("Action"));
+		frm.add_custom_button(
+			__("Add Subscribers"),
+			function () {
+				frappe.prompt(
+					{
+						fieldtype: "Text",
+						label: __("Email Addresses"),
+						fieldname: "email_list",
+						reqd: 1,
+					},
+					function (data) {
+						frappe.call({
+							method: "frappe.email.doctype.email_group.email_group.add_subscribers",
+							args: {
+								name: frm.doc.name,
+								email_list: data.email_list,
+							},
+							callback: function (r) {
+								frm.set_value("total_subscribers", r.message);
+							},
+						});
+					},
+					__("Add Subscribers"),
+					__("Add")
+				);
+			},
+			__("Action")
+		);
 
+		frm.add_custom_button(
+			__("New Newsletter"),
+			function () {
+				frappe.route_options = { email_group: frm.doc.name };
+				frappe.new_doc("Newsletter");
+			},
+			__("Action")
+		);
 	}
 });
diff --git a/frappe/email/doctype/email_group/email_group.py b/frappe/email/doctype/email_group/email_group.py
index ea62a6e9ec..cbe47eaa81 100755
--- a/frappe/email/doctype/email_group/email_group.py
+++ b/frappe/email/doctype/email_group/email_group.py
@@ -9,10 +9,10 @@ from frappe.utils import parse_addr, validate_email_address
 
 class EmailGroup(Document):
 	def onload(self):
-		singles = [d.name for d in frappe.db.get_all("DocType", "name", {"issingle": 1})]
+		singles = [d.name for d in frappe.get_all("DocType", "name", {"issingle": 1})]
 		self.get("__onload").import_types = [
 			{"value": d.parent, "label": f"{d.parent} ({d.label})"}
-			for d in frappe.db.get_all("DocField", ("parent", "label"), {"options": "Email"})
+			for d in frappe.get_all("DocField", ("parent", "label"), {"options": "Email"})
 			if d.parent not in singles
 		]
 
@@ -27,7 +27,7 @@ class EmailGroup(Document):
 		unsubscribed_field = "unsubscribed" if meta.get_field("unsubscribed") else None
 		added = 0
 
-		for user in frappe.db.get_all(doctype, [email_field, unsubscribed_field or "name"]):
+		for user in frappe.get_all(doctype, [email_field, unsubscribed_field or "name"]):
 			try:
 				email = parse_addr(user.get(email_field))[1] if user.get(email_field) else None
 				if email:
diff --git a/frappe/email/doctype/email_group/test_email_group.py b/frappe/email/doctype/email_group/test_email_group.py
index aa41285f90..ffc325a6bd 100644
--- a/frappe/email/doctype/email_group/test_email_group.py
+++ b/frappe/email/doctype/email_group/test_email_group.py
@@ -1,9 +1,9 @@
 # Copyright (c) 2015, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
-import unittest
+from frappe.tests.utils import FrappeTestCase
 
 # test_records = frappe.get_test_records('Email Group')
 
 
-class TestEmailGroup(unittest.TestCase):
+class TestEmailGroup(FrappeTestCase):
 	pass
diff --git a/frappe/email/doctype/email_group_member/email_group_member.js b/frappe/email/doctype/email_group_member/email_group_member.js
index 417eb70119..ca2c17ad81 100644
--- a/frappe/email/doctype/email_group_member/email_group_member.js
+++ b/frappe/email/doctype/email_group_member/email_group_member.js
@@ -1,8 +1,6 @@
 // Copyright (c) 2016, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Email Group Member', {
-	refresh: function(frm) {
-
-	}
+frappe.ui.form.on("Email Group Member", {
+	refresh: function (frm) {},
 });
diff --git a/frappe/email/doctype/email_group_member/test_email_group_member.py b/frappe/email/doctype/email_group_member/test_email_group_member.py
index 749792fe52..6432cadf54 100644
--- a/frappe/email/doctype/email_group_member/test_email_group_member.py
+++ b/frappe/email/doctype/email_group_member/test_email_group_member.py
@@ -1,9 +1,9 @@
 # Copyright (c) 2015, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
-import unittest
+from frappe.tests.utils import FrappeTestCase
 
 # test_records = frappe.get_test_records('Email Group Member')
 
 
-class TestEmailGroupMember(unittest.TestCase):
+class TestEmailGroupMember(FrappeTestCase):
 	pass
diff --git a/frappe/email/doctype/email_queue/email_queue.js b/frappe/email/doctype/email_queue/email_queue.js
index b6ef0ec082..2ac4b6f7fe 100644
--- a/frappe/email/doctype/email_queue/email_queue.js
+++ b/frappe/email/doctype/email_queue/email_queue.js
@@ -2,37 +2,37 @@
 // For license information, please see license.txt
 
 frappe.ui.form.on("Email Queue", {
-	refresh: function(frm) {
-		if (["Not Sent","Partially Sent"].indexOf(frm.doc.status)!=-1) {
-			let button = frm.add_custom_button("Send Now", function() {
+	refresh: function (frm) {
+		if (["Not Sent", "Partially Sent"].indexOf(frm.doc.status) != -1) {
+			let button = frm.add_custom_button("Send Now", function () {
 				frappe.call({
-					method: 'frappe.email.doctype.email_queue.email_queue.send_now',
+					method: "frappe.email.doctype.email_queue.email_queue.send_now",
 					args: {
-						name: frm.doc.name
+						name: frm.doc.name,
 					},
 					btn: button,
-					callback: function() {
+					callback: function () {
 						frm.reload_doc();
-					}
+					},
 				});
 			});
 		}
 
-		if (["Error","Partially Errored"].indexOf(frm.doc.status)!=-1) {
-			let button = frm.add_custom_button("Retry Sending", function() {
+		if (["Error", "Partially Errored"].indexOf(frm.doc.status) != -1) {
+			let button = frm.add_custom_button("Retry Sending", function () {
 				frm.call({
 					method: "retry_sending",
 					args: {
-						name: frm.doc.name
+						name: frm.doc.name,
 					},
 					btn: button,
-					callback: function(r) {
+					callback: function (r) {
 						if (!r.exc) {
 							frm.set_value("status", "Not Sent");
 						}
-					}
-				})
+					},
+				});
 			});
 		}
-	}
+	},
 });
diff --git a/frappe/email/doctype/email_queue/email_queue.json b/frappe/email/doctype/email_queue/email_queue.json
index c9ec374687..ac8d656678 100644
--- a/frappe/email/doctype/email_queue/email_queue.json
+++ b/frappe/email/doctype/email_queue/email_queue.json
@@ -67,10 +67,9 @@
   },
   {
    "fieldname": "message_id",
-   "fieldtype": "Data",
+   "fieldtype": "Small Text",
    "label": "Message ID",
-   "read_only": 1,
-   "search_index": 1
+   "read_only": 1
   },
   {
    "fieldname": "reference_doctype",
@@ -153,7 +152,7 @@
  "idx": 1,
  "in_create": 1,
  "links": [],
- "modified": "2022-07-12 15:17:37.934316",
+ "modified": "2023-03-16 12:15:17.850292",
  "modified_by": "Administrator",
  "module": "Email",
  "name": "Email Queue",
diff --git a/frappe/email/doctype/email_queue/email_queue.py b/frappe/email/doctype/email_queue/email_queue.py
index eb07be0b38..d254c87a0a 100644
--- a/frappe/email/doctype/email_queue/email_queue.py
+++ b/frappe/email/doctype/email_queue/email_queue.py
@@ -5,6 +5,7 @@ import json
 import quopri
 import smtplib
 import traceback
+from contextlib import suppress
 from email.parser import Parser
 from email.policy import SMTPUTF8
 
@@ -26,9 +27,12 @@ from frappe.utils import (
 	cstr,
 	get_hook_method,
 	get_string_between,
+	get_url,
 	nowdate,
+	sbool,
 	split_emails,
 )
+from frappe.utils.verified_command import get_signed_params
 
 
 class EmailQueue(Document):
@@ -37,7 +41,7 @@ class EmailQueue(Document):
 	def set_recipients(self, recipients):
 		self.set("recipients", [])
 		for r in recipients:
-			self.append("recipients", {"recipient": r, "status": "Not Sent"})
+			self.append("recipients", {"recipient": r.strip(), "status": "Not Sent"})
 
 	def on_trash(self):
 		self.prevent_email_queue_delete()
@@ -110,8 +114,11 @@ class EmailQueue(Document):
 		return self.status in ["Not Sent", "Partially Sent"]
 
 	def can_send_now(self):
-		hold_queue = cint(frappe.defaults.get_defaults().get("hold_queue")) == 1
-		if frappe.are_emails_muted() or not self.is_to_be_sent() or hold_queue:
+		if (
+			frappe.are_emails_muted()
+			or not self.is_to_be_sent()
+			or cint(frappe.db.get_default("suspend_email_queue")) == 1
+		):
 			return False
 
 		return True
@@ -196,7 +203,7 @@ class SendMailContext:
 		# Note: smtp session will have to be manually closed
 		self.retain_smtp_session = bool(smtp_server_instance)
 
-		self.sent_to = [rec.recipient for rec in self.queue_doc.recipients if rec.is_main_sent()]
+		self.sent_to = [rec.recipient for rec in self.queue_doc.recipients if rec.is_mail_sent()]
 
 	def __enter__(self):
 		self.queue_doc.update_status(status="Sending", commit=True)
@@ -206,7 +213,6 @@ class SendMailContext:
 		exceptions = [
 			smtplib.SMTPServerDisconnected,
 			smtplib.SMTPAuthenticationError,
-			smtplib.SMTPRecipientsRefused,
 			smtplib.SMTPConnectError,
 			smtplib.SMTPHeloError,
 			JobTimeoutException,
@@ -278,7 +284,9 @@ class SendMailContext:
 		if not message:
 			return ""
 
-		message = message.replace(self.message_placeholder("tracker"), self.get_tracker_str())
+		message = message.replace(
+			self.message_placeholder("tracker"), self.get_tracker_str(recipient_email)
+		)
 		message = message.replace(
 			self.message_placeholder("unsubscribe_url"), self.get_unsubscribe_str(recipient_email)
 		)
@@ -289,15 +297,25 @@ class SendMailContext:
 		message = self.include_attachments(message)
 		return message
 
-	def get_tracker_str(self):
-		tracker_url_html = ''
+	def get_tracker_str(self, recipient_email) -> str:
+		tracker_url = ""
+		if self.queue_doc.get("email_read_tracker_url"):
+			email_read_tracker_url = self.queue_doc.email_read_tracker_url
+			params = {
+				"recipient_email": recipient_email,
+				"reference_name": self.queue_doc.reference_name,
+				"reference_doctype": self.queue_doc.reference_doctype,
+			}
+			tracker_url = get_url(f"{email_read_tracker_url}?{get_signed_params(params)}")
 
-		message = ""
-		if frappe.conf.use_ssl and self.email_account_doc.track_email_status:
-			message = quopri.encodestring(
-				tracker_url_html.format(frappe.local.site, self.queue_doc.communication).encode()
-			).decode()
-		return message
+		elif frappe.conf.use_ssl and self.email_account_doc.track_email_status:
+			tracker_url = f"{get_url()}/api/method/frappe.core.doctype.communication.email.mark_email_as_seen?name={self.queue_doc.communication}"
+
+		if tracker_url:
+			tracker_url_html = f''
+			return quopri.encodestring(tracker_url_html.encode()).decode()
+
+		return ""
 
 	def get_unsubscribe_str(self, recipient_email: str) -> str:
 		unsubscribe_url = ""
@@ -359,6 +377,8 @@ class SendMailContext:
 @frappe.whitelist()
 def retry_sending(name):
 	doc = frappe.get_doc("Email Queue", name)
+	doc.check_permission()
+
 	if doc and (doc.status == "Error" or doc.status == "Partially Errored"):
 		doc.status = "Not Sent"
 		for d in doc.recipients:
@@ -371,15 +391,24 @@ def retry_sending(name):
 def send_now(name):
 	record = EmailQueue.find(name)
 	if record:
+		record.check_permission()
 		record.send()
 
 
+@frappe.whitelist()
+def toggle_sending(enable):
+	frappe.only_for("System Manager")
+	frappe.db.set_default("suspend_email_queue", 0 if sbool(enable) else 1)
+
+
 def on_doctype_update():
 	"""Add index in `tabCommunication` for `(reference_doctype, reference_name)`"""
 	frappe.db.add_index(
 		"Email Queue", ("status", "send_after", "priority", "creation"), "index_bulk_flush"
 	)
 
+	frappe.db.add_index("Email Queue", ["message_id(140)"])
+
 
 def get_email_retry_limit():
 	return cint(frappe.db.get_system_setting("email_retry_limit")) or 3
@@ -418,6 +447,7 @@ class QueueBuilder:
 		header=None,
 		print_letterhead=False,
 		with_container=False,
+		email_read_tracker_url=None,
 	):
 		"""Add email to sending queue (Email Queue)
 
@@ -442,6 +472,7 @@ class QueueBuilder:
 		:param inline_images: List of inline images as {"filename", "filecontent"}. All src properties will be replaced with random Content-Id
 		:param header: Append header in email (boolean)
 		:param with_container: Wraps email inside styled container
+		:param email_read_tracker_url: A URL for tracking whether an email is read by the recipient.
 		"""
 
 		self._unsubscribe_method = unsubscribe_method
@@ -476,6 +507,7 @@ class QueueBuilder:
 		self.is_notification = is_notification
 		self.inline_images = inline_images
 		self.print_letterhead = print_letterhead
+		self.email_read_tracker_url = email_read_tracker_url
 
 	@property
 	def unsubscribe_method(self):
@@ -674,7 +706,10 @@ class QueueBuilder:
 			if not smtp_server_instance:
 				email_account = q.get_email_account()
 				smtp_server_instance = email_account.get_smtp_server()
-			q.send(smtp_server_instance=smtp_server_instance)
+
+			with suppress(Exception):
+				q.send(smtp_server_instance=smtp_server_instance)
+
 		smtp_server_instance.quit()
 
 	def as_dict(self, include_recipients=True):
@@ -701,7 +736,7 @@ class QueueBuilder:
 			"attachments": json.dumps(self.get_attachments()),
 			"message_id": get_string_between("<", mail.msg_root["Message-Id"], ">"),
 			"message": mail_to_string,
-			"sender": self.sender,
+			"sender": mail.sender,
 			"reference_doctype": self.reference_doctype,
 			"reference_name": self.reference_name,
 			"add_unsubscribe_link": self._add_unsubscribe_link,
@@ -713,6 +748,7 @@ class QueueBuilder:
 			"show_as_cc": ",".join(self.final_cc()),
 			"show_as_bcc": ",".join(self.bcc),
 			"email_account": email_account_name or None,
+			"email_read_tracker_url": self.email_read_tracker_url,
 		}
 
 		if include_recipients:
diff --git a/frappe/email/doctype/email_queue/email_queue_list.js b/frappe/email/doctype/email_queue/email_queue_list.js
index edc6250714..b00503b6f8 100644
--- a/frappe/email/doctype/email_queue/email_queue_list.js
+++ b/frappe/email/doctype/email_queue/email_queue_list.js
@@ -1,29 +1,41 @@
-frappe.listview_settings['Email Queue'] = {
-	get_indicator: function(doc) {
-		var colour = {'Sent': 'green', 'Sending': 'blue', 'Not Sent': 'grey', 'Error': 'red', 'Expired': 'orange'};
+frappe.listview_settings["Email Queue"] = {
+	get_indicator: function (doc) {
+		var colour = {
+			Sent: "green",
+			Sending: "blue",
+			"Not Sent": "grey",
+			Error: "red",
+			Expired: "orange",
+		};
 		return [__(doc.status), colour[doc.status], "status,=," + doc.status];
 	},
-	refresh: function(doclist){
-		if (has_common(frappe.user_roles, ["Administrator", "System Manager"])){
-			if (cint(frappe.defaults.get_default("hold_queue"))){
-				doclist.page.clear_inner_toolbar()
-				doclist.page.add_inner_button(__("Resume Sending"), function() {
-					frappe.defaults.set_default("hold_queue", 0);
-					cur_list.refresh();
-				})
-			} else {
-				doclist.page.clear_inner_toolbar()
-				doclist.page.add_inner_button(__("Suspend Sending"), function() {
-					frappe.defaults.set_default("hold_queue", 1)
-					cur_list.refresh();
-				})
-			}
-		}
-	},
-
-	onload: function(listview) {
+	refresh: show_toggle_sending_button,
+	onload: function (list_view) {
 		frappe.require("logtypes.bundle.js", () => {
-			frappe.utils.logtypes.show_log_retention_message(cur_list.doctype);
-		})
-	}
+			frappe.utils.logtypes.show_log_retention_message(list_view.doctype);
+		});
+	},
+};
+
+function show_toggle_sending_button(list_view) {
+	if (!has_common(frappe.user_roles, ["Administrator", "System Manager"])) return;
+
+	const sending_disabled = cint(frappe.sys_defaults.suspend_email_queue);
+	const label = sending_disabled ? __("Resume Sending") : __("Suspend Sending");
+
+	list_view.page.add_inner_button(label, async () => {
+		await frappe.xcall(
+			"frappe.email.doctype.email_queue.email_queue.toggle_sending",
+
+			// enable if disabled
+			{ enable: sending_disabled }
+		);
+
+		// set new value for suspend_email_queue in sys_defaults
+		frappe.sys_defaults.suspend_email_queue = sending_disabled ? 0 : 1;
+
+		// clear the button and show one with the opposite label
+		list_view.page.remove_inner_button(label);
+		show_toggle_sending_button(list_view);
+	});
 }
diff --git a/frappe/event_streaming/doctype/event_consumer/__init__.py b/frappe/email/doctype/email_queue/patches/__init__.py
similarity index 100%
rename from frappe/event_streaming/doctype/event_consumer/__init__.py
rename to frappe/email/doctype/email_queue/patches/__init__.py
diff --git a/frappe/email/doctype/email_queue/patches/drop_search_index_on_message_id.py b/frappe/email/doctype/email_queue/patches/drop_search_index_on_message_id.py
new file mode 100644
index 0000000000..7c4baf5a2a
--- /dev/null
+++ b/frappe/email/doctype/email_queue/patches/drop_search_index_on_message_id.py
@@ -0,0 +1,11 @@
+import frappe
+
+
+def execute():
+	"""Drop search index on message_id"""
+
+	if frappe.db.get_column_type("Email Queue", "message_id") == "text":
+		return
+
+	if index := frappe.db.get_column_index("tabEmail Queue", "message_id", unique=False):
+		frappe.db.sql(f"ALTER TABLE `tabEmail Queue` DROP INDEX `{index.Key_name}`")
diff --git a/frappe/email/doctype/email_queue_recipient/email_queue_recipient.json b/frappe/email/doctype/email_queue_recipient/email_queue_recipient.json
index c217886ce6..406449e26d 100644
--- a/frappe/email/doctype/email_queue_recipient/email_queue_recipient.json
+++ b/frappe/email/doctype/email_queue_recipient/email_queue_recipient.json
@@ -22,7 +22,7 @@
    "fieldtype": "Select",
    "in_list_view": 1,
    "label": "Status",
-   "options": "\nNot Sent\nSending\nSent\nError\nExpired",
+   "options": "\nNot Sent\nSent",
    "search_index": 1
   },
   {
@@ -33,7 +33,7 @@
  ],
  "istable": 1,
  "links": [],
- "modified": "2022-07-11 16:38:10.644417",
+ "modified": "2022-09-06 13:38:10.644417",
  "modified_by": "Administrator",
  "module": "Email",
  "name": "Email Queue Recipient",
diff --git a/frappe/email/doctype/email_queue_recipient/email_queue_recipient.py b/frappe/email/doctype/email_queue_recipient/email_queue_recipient.py
index bcb8d9b05d..705075a862 100644
--- a/frappe/email/doctype/email_queue_recipient/email_queue_recipient.py
+++ b/frappe/email/doctype/email_queue_recipient/email_queue_recipient.py
@@ -11,7 +11,7 @@ class EmailQueueRecipient(Document):
 	def is_mail_to_be_sent(self):
 		return self.status == "Not Sent"
 
-	def is_main_sent(self):
+	def is_mail_sent(self):
 		return self.status == "Sent"
 
 	def update_db(self, commit=False, **kwargs):
diff --git a/frappe/email/doctype/email_rule/email_rule.js b/frappe/email/doctype/email_rule/email_rule.js
index 974bcd4e51..2670ef3e1c 100644
--- a/frappe/email/doctype/email_rule/email_rule.js
+++ b/frappe/email/doctype/email_rule/email_rule.js
@@ -1,8 +1,6 @@
 // Copyright (c) 2017, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Email Rule', {
-	refresh: function(frm) {
-
-	}
+frappe.ui.form.on("Email Rule", {
+	refresh: function (frm) {},
 });
diff --git a/frappe/email/doctype/email_rule/email_rule.json b/frappe/email/doctype/email_rule/email_rule.json
index b4e505b8c6..20e296290d 100644
--- a/frappe/email/doctype/email_rule/email_rule.json
+++ b/frappe/email/doctype/email_rule/email_rule.json
@@ -1,128 +1,49 @@
 {
+ "actions": [],
  "allow_copy": 1,
- "allow_guest_to_view": 0,
- "allow_import": 0,
- "allow_rename": 0,
  "autoname": "field:email_id",
- "beta": 0,
  "creation": "2017-03-13 09:20:56.387135",
- "custom": 0,
- "docstatus": 0,
  "doctype": "DocType",
- "document_type": "",
  "editable_grid": 1,
  "engine": "InnoDB",
+ "field_order": [
+  "email_id",
+  "is_spam"
+ ],
  "fields": [
   {
-   "allow_bulk_edit": 0,
-   "allow_in_quick_entry": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
    "fieldname": "email_id",
    "fieldtype": "Data",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
    "label": "Email ID",
-   "length": 0,
-   "no_copy": 0,
    "options": "Email",
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "translatable": 0,
    "unique": 1
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_in_quick_entry": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
+   "default": "0",
    "fieldname": "is_spam",
    "fieldtype": "Check",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
-   "label": "Is Spam",
-   "length": 0,
-   "no_copy": 0,
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "translatable": 0,
-   "unique": 0
+   "label": "Is Spam"
   }
  ],
- "has_web_view": 0,
- "hide_heading": 0,
- "hide_toolbar": 0,
- "idx": 0,
- "image_view": 0,
- "in_create": 0,
- "is_submittable": 0,
- "issingle": 0,
- "istable": 0,
- "max_attachments": 0,
- "modified": "2018-08-31 06:08:12.645682",
+ "links": [],
+ "modified": "2022-08-03 12:20:51.443237",
  "modified_by": "Administrator",
  "module": "Email",
  "name": "Email Rule",
- "name_case": "",
  "owner": "Administrator",
  "permissions": [
   {
-   "amend": 0,
-   "cancel": 0,
-   "create": 0,
-   "delete": 0,
    "email": 1,
    "export": 1,
-   "if_owner": 0,
-   "import": 0,
-   "permlevel": 0,
    "print": 1,
    "read": 1,
    "report": 1,
    "role": "System Manager",
-   "set_user_permissions": 0,
-   "share": 1,
-   "submit": 0,
-   "write": 0
+   "share": 1
   }
  ],
  "quick_entry": 1,
- "read_only": 0,
- "read_only_onload": 0,
- "show_name_in_global_search": 0,
  "sort_field": "modified",
  "sort_order": "DESC",
- "track_changes": 0,
- "track_seen": 0,
- "track_views": 0
+ "states": []
 }
\ No newline at end of file
diff --git a/frappe/email/doctype/email_rule/test_email_rule.py b/frappe/email/doctype/email_rule/test_email_rule.py
index 1da5d34d6b..489379451d 100644
--- a/frappe/email/doctype/email_rule/test_email_rule.py
+++ b/frappe/email/doctype/email_rule/test_email_rule.py
@@ -1,7 +1,7 @@
 # Copyright (c) 2017, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
-import unittest
+from frappe.tests.utils import FrappeTestCase
 
 
-class TestEmailRule(unittest.TestCase):
+class TestEmailRule(FrappeTestCase):
 	pass
diff --git a/frappe/email/doctype/email_template/email_template.js b/frappe/email/doctype/email_template/email_template.js
index 62ce4d94ad..33327005a5 100644
--- a/frappe/email/doctype/email_template/email_template.js
+++ b/frappe/email/doctype/email_template/email_template.js
@@ -1,8 +1,6 @@
 // Copyright (c) 2018, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Email Template', {
-	refresh: function() {
-
-	}
+frappe.ui.form.on("Email Template", {
+	refresh: function () {},
 });
diff --git a/frappe/email/doctype/email_template/email_template.json b/frappe/email/doctype/email_template/email_template.json
index c6ec971da4..00f1428475 100644
--- a/frappe/email/doctype/email_template/email_template.json
+++ b/frappe/email/doctype/email_template/email_template.json
@@ -57,18 +57,16 @@
  ],
  "icon": "fa fa-comment",
  "links": [],
- "modified": "2022-01-04 14:12:50.321633",
+ "modified": "2023-01-02 03:56:48.437280",
  "modified_by": "Administrator",
  "module": "Email",
  "name": "Email Template",
+ "naming_rule": "Set by user",
  "owner": "Administrator",
  "permissions": [
   {
-   "create": 1,
    "read": 1,
-   "role": "All",
-   "share": 1,
-   "write": 1
+   "role": "All"
   },
   {
    "create": 1,
@@ -85,5 +83,6 @@
  ],
  "sort_field": "modified",
  "sort_order": "DESC",
+ "states": [],
  "track_changes": 1
 }
\ No newline at end of file
diff --git a/frappe/email/doctype/email_template/email_template.py b/frappe/email/doctype/email_template/email_template.py
index fcc6ce5010..1ef8ec062b 100644
--- a/frappe/email/doctype/email_template/email_template.py
+++ b/frappe/email/doctype/email_template/email_template.py
@@ -9,33 +9,33 @@ from frappe.utils.jinja import validate_template
 
 
 class EmailTemplate(Document):
+	@property
+	def response_(self):
+		return self.response_html if self.use_html else self.response
+
 	def validate(self):
-		if self.use_html:
-			validate_template(self.response_html)
-		else:
-			validate_template(self.response)
+		validate_template(self.subject)
+		validate_template(self.response_)
 
 	def get_formatted_subject(self, doc):
 		return frappe.render_template(self.subject, doc)
 
 	def get_formatted_response(self, doc):
-		if self.use_html:
-			return frappe.render_template(self.response_html, doc)
-
-		return frappe.render_template(self.response, doc)
+		return frappe.render_template(self.response_, doc)
 
 	def get_formatted_email(self, doc):
 		if isinstance(doc, str):
 			doc = json.loads(doc)
 
-		return {"subject": self.get_formatted_subject(doc), "message": self.get_formatted_response(doc)}
+		return {
+			"subject": self.get_formatted_subject(doc),
+			"message": self.get_formatted_response(doc),
+		}
 
 
 @frappe.whitelist()
 def get_email_template(template_name, doc):
 	"""Returns the processed HTML of a email template with the given doc"""
-	if isinstance(doc, str):
-		doc = json.loads(doc)
 
 	email_template = frappe.get_doc("Email Template", template_name)
 	return email_template.get_formatted_email(doc)
diff --git a/frappe/email/doctype/email_template/test_email_template.py b/frappe/email/doctype/email_template/test_email_template.py
index 291f7e1df0..f2c53647bc 100644
--- a/frappe/email/doctype/email_template/test_email_template.py
+++ b/frappe/email/doctype/email_template/test_email_template.py
@@ -1,7 +1,7 @@
 # Copyright (c) 2018, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
-import unittest
+from frappe.tests.utils import FrappeTestCase
 
 
-class TestEmailTemplate(unittest.TestCase):
+class TestEmailTemplate(FrappeTestCase):
 	pass
diff --git a/frappe/email/doctype/email_unsubscribe/email_unsubscribe.js b/frappe/email/doctype/email_unsubscribe/email_unsubscribe.js
index 9a022cf4ca..5f1e28e9b9 100644
--- a/frappe/email/doctype/email_unsubscribe/email_unsubscribe.js
+++ b/frappe/email/doctype/email_unsubscribe/email_unsubscribe.js
@@ -1,8 +1,6 @@
 // Copyright (c) 2016, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Email Unsubscribe', {
-	refresh: function(frm) {
-
-	}
+frappe.ui.form.on("Email Unsubscribe", {
+	refresh: function (frm) {},
 });
diff --git a/frappe/email/doctype/email_unsubscribe/email_unsubscribe.json b/frappe/email/doctype/email_unsubscribe/email_unsubscribe.json
index bf633ead4b..38df531c35 100644
--- a/frappe/email/doctype/email_unsubscribe/email_unsubscribe.json
+++ b/frappe/email/doctype/email_unsubscribe/email_unsubscribe.json
@@ -1,175 +1,70 @@
 {
- "allow_copy": 0,
- "allow_import": 0,
- "allow_rename": 0,
- "beta": 0,
+ "actions": [],
  "creation": "2015-03-18 09:41:20.216320",
- "custom": 0,
- "docstatus": 0,
  "doctype": "DocType",
  "document_type": "System",
- "editable_grid": 0,
  "engine": "InnoDB",
+ "field_order": [
+  "email",
+  "reference_doctype",
+  "reference_name",
+  "global_unsubscribe"
+ ],
  "fields": [
   {
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
    "fieldname": "email",
    "fieldtype": "Data",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
    "in_list_view": 1,
-   "in_standard_filter": 0,
    "label": "Email",
-   "length": 0,
-   "no_copy": 0,
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
    "reqd": 1,
-   "search_index": 1,
-   "set_only_once": 0,
-   "unique": 0
+   "search_index": 1
   },
   {
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
    "fieldname": "reference_doctype",
    "fieldtype": "Link",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
    "in_list_view": 1,
    "in_standard_filter": 1,
    "label": "Reference Document Type",
-   "length": 0,
-   "no_copy": 0,
-   "options": "DocType",
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "unique": 0
+   "options": "DocType"
   },
   {
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
    "fieldname": "reference_name",
    "fieldtype": "Dynamic Link",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
    "in_list_view": 1,
-   "in_standard_filter": 0,
    "label": "Reference Name",
-   "length": 0,
-   "no_copy": 0,
-   "options": "reference_doctype",
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "unique": 0
+   "options": "reference_doctype"
   },
   {
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
+   "default": "0",
    "fieldname": "global_unsubscribe",
    "fieldtype": "Check",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
    "in_list_view": 1,
-   "in_standard_filter": 0,
-   "label": "Global Unsubscribe",
-   "length": 0,
-   "no_copy": 0,
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "unique": 0
+   "label": "Global Unsubscribe"
   }
  ],
- "hide_heading": 0,
- "hide_toolbar": 0,
- "idx": 0,
- "image_view": 0,
- "in_create": 0,
-
- "is_submittable": 0,
- "issingle": 0,
- "istable": 0,
- "max_attachments": 0,
- "modified": "2019-09-05 14:22:27.664645",
+ "links": [],
+ "modified": "2022-08-03 12:20:51.694626",
  "modified_by": "Administrator",
  "module": "Email",
  "name": "Email Unsubscribe",
- "name_case": "",
  "owner": "Administrator",
  "permissions": [
   {
-   "amend": 0,
-   "apply_user_permissions": 0,
-   "cancel": 0,
    "create": 1,
    "delete": 1,
    "email": 1,
    "export": 1,
-   "if_owner": 0,
-   "import": 0,
-   "is_custom": 0,
-   "permlevel": 0,
    "print": 1,
    "read": 1,
    "report": 1,
    "role": "System Manager",
-   "set_user_permissions": 0,
    "share": 1,
-   "submit": 0,
    "write": 1
   }
  ],
  "quick_entry": 1,
- "read_only": 0,
- "read_only_onload": 0,
  "sort_field": "modified",
  "sort_order": "DESC",
- "track_changes": 1,
- "track_seen": 0
-}
+ "states": [],
+ "track_changes": 1
+}
\ No newline at end of file
diff --git a/frappe/email/doctype/email_unsubscribe/test_email_unsubscribe.py b/frappe/email/doctype/email_unsubscribe/test_email_unsubscribe.py
index 9ba99a6690..d0b1731ad9 100644
--- a/frappe/email/doctype/email_unsubscribe/test_email_unsubscribe.py
+++ b/frappe/email/doctype/email_unsubscribe/test_email_unsubscribe.py
@@ -1,9 +1,9 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
-import unittest
+from frappe.tests.utils import FrappeTestCase
 
 # test_records = frappe.get_test_records('Email Unsubscribe')
 
 
-class TestEmailUnsubscribe(unittest.TestCase):
+class TestEmailUnsubscribe(FrappeTestCase):
 	pass
diff --git a/frappe/email/doctype/newsletter/newsletter.js b/frappe/email/doctype/newsletter/newsletter.js
index 3c52e61cbb..8a81bbaab3 100644
--- a/frappe/email/doctype/newsletter/newsletter.js
+++ b/frappe/email/doctype/newsletter/newsletter.js
@@ -1,113 +1,146 @@
 // Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 // License: GNU General Public License v3. See license.txt
 
-frappe.ui.form.on('Newsletter', {
+frappe.ui.form.on("Newsletter", {
 	refresh(frm) {
 		let doc = frm.doc;
 		let can_write = in_list(frappe.boot.user.can_write, doc.doctype);
 		if (!frm.is_new() && !frm.is_dirty() && !doc.email_sent && can_write) {
-			frm.add_custom_button(__('Send a test email'), () => {
-				frm.events.send_test_email(frm);
-			}, __('Preview'));
+			frm.add_custom_button(
+				__("Send a test email"),
+				() => {
+					frm.events.send_test_email(frm);
+				},
+				__("Preview")
+			);
 
-			frm.add_custom_button(__('Check broken links'), () => {
-				frm.dashboard.set_headline(__('Checking broken links...'));
-				frm.call('find_broken_links').then(r => {
-					frm.dashboard.set_headline('');
-					let links = r.message;
-					if (links && links.length) {
-						let html = '
          ' + links.map(link => `
        • ${link}
          • `).join('') + '
        ';
-						frm.dashboard.set_headline(__("Following links are broken in the email content: {0}", [html]));
-					} else {
-						frm.dashboard.set_headline(__("No broken links found in the email content"));
-						setTimeout(() => {
-							frm.dashboard.set_headline('');
-						}, 3000);
-					}
-				});
-			}, __('Preview'));
-
-			frm.add_custom_button(__('Send now'), () => {
-				if (frm.doc.schedule_send) {
-					frappe.confirm(__("This newsletter was scheduled to send on a later date. Are you sure you want to send it now?"), function () {
-						frm.events.send_emails(frm);
+			frm.add_custom_button(
+				__("Check broken links"),
+				() => {
+					frm.dashboard.set_headline(__("Checking broken links..."));
+					frm.call("find_broken_links").then((r) => {
+						frm.dashboard.set_headline("");
+						let links = r.message;
+						if (links && links.length) {
+							let html =
+								"
          " +
+								links.map((link) => `
        • ${link}
          • `).join("") +
+								"
        ";
+							frm.dashboard.set_headline(
+								__("Following links are broken in the email content: {0}", [html])
+							);
+						} else {
+							frm.dashboard.set_headline(
+								__("No broken links found in the email content")
+							);
+							setTimeout(() => {
+								frm.dashboard.set_headline("");
+							}, 3000);
+						}
 					});
-					return;
-				}
-				frappe.confirm(__("Are you sure you want to send this newsletter now?"), () => {
-					frm.events.send_emails(frm);
-				});
-			}, __('Send'));
+				},
+				__("Preview")
+			);
 
-			frm.add_custom_button(__('Schedule sending'), () => {
-				frm.events.schedule_send_dialog(frm);
-			}, __('Send'));
+			frm.add_custom_button(
+				__("Send now"),
+				() => {
+					if (frm.doc.schedule_send) {
+						frappe.confirm(
+							__(
+								"This newsletter was scheduled to send on a later date. Are you sure you want to send it now?"
+							),
+							function () {
+								frm.events.send_emails(frm);
+							}
+						);
+						return;
+					}
+					frappe.confirm(
+						__("Are you sure you want to send this newsletter now?"),
+						() => {
+							frm.events.send_emails(frm);
+						}
+					);
+				},
+				__("Send")
+			);
+
+			frm.add_custom_button(
+				__("Schedule sending"),
+				() => {
+					frm.events.schedule_send_dialog(frm);
+				},
+				__("Send")
+			);
 		}
 
 		frm.events.update_sending_status(frm);
 
 		if (frm.is_new() && !doc.sender_email) {
 			let { fullname, email } = frappe.user_info(doc.owner);
-			frm.set_value('sender_email', email);
-			frm.set_value('sender_name', fullname);
+			frm.set_value("sender_email", email);
+			frm.set_value("sender_name", fullname);
 		}
 
-		frm.trigger('update_schedule_message');
+		frm.trigger("update_schedule_message");
 	},
 
 	send_emails(frm) {
 		frappe.dom.freeze(__("Queuing emails..."));
-		frm.call('send_emails').then(() => {
+		frm.call("send_emails").then(() => {
 			frm.refresh();
 			frappe.dom.unfreeze();
-			frappe.show_alert(__("Queued {0} emails", [frappe.utils.shorten_number(frm.doc.total_recipients)]));
+			frappe.show_alert(
+				__("Queued {0} emails", [frappe.utils.shorten_number(frm.doc.total_recipients)])
+			);
 		});
 	},
 
 	schedule_send_dialog(frm) {
 		let hours = frappe.utils.range(24);
-		let time_slots = hours.map(hour => {
-			return `${(hour + '').padStart(2, '0')}:00`;
+		let time_slots = hours.map((hour) => {
+			return `${(hour + "").padStart(2, "0")}:00`;
 		});
 		let d = new frappe.ui.Dialog({
-			title: __('Schedule Newsletter'),
+			title: __("Schedule Newsletter"),
 			fields: [
 				{
-					label: __('Date'),
-					fieldname: 'date',
-					fieldtype: 'Date',
+					label: __("Date"),
+					fieldname: "date",
+					fieldtype: "Date",
 					options: {
-						minDate: new Date()
-					}
+						minDate: new Date(),
+					},
 				},
 				{
-					label: __('Time'),
-					fieldname: 'time',
-					fieldtype: 'Select',
+					label: __("Time"),
+					fieldname: "time",
+					fieldtype: "Select",
 					options: time_slots,
 				},
 			],
-			primary_action_label: __('Schedule'),
+			primary_action_label: __("Schedule"),
 			primary_action({ date, time }) {
-				frm.set_value('schedule_sending', 1);
-				frm.set_value('schedule_send', `${date} ${time}:00`);
+				frm.set_value("schedule_sending", 1);
+				frm.set_value("schedule_send", `${date} ${time}:00`);
 				d.hide();
 				frm.save();
 			},
-			secondary_action_label: __('Cancel Scheduling'),
+			secondary_action_label: __("Cancel Scheduling"),
 			secondary_action() {
-				frm.set_value('schedule_sending', 0);
-				frm.set_value('schedule_send', '');
+				frm.set_value("schedule_sending", 0);
+				frm.set_value("schedule_send", "");
 				d.hide();
 				frm.save();
-			}
+			},
 		});
 		if (frm.doc.schedule_sending) {
-			let parts = frm.doc.schedule_send.split(' ');
+			let parts = frm.doc.schedule_send.split(" ");
 			if (parts.length === 2) {
 				let [date, time] = parts;
-				d.set_value('date', date);
-				d.set_value('time', time.slice(0, 5));
+				d.set_value("date", date);
+				d.set_value("time", time.slice(0, 5));
 			}
 		}
 		d.show();
@@ -115,35 +148,37 @@ frappe.ui.form.on('Newsletter', {
 
 	send_test_email(frm) {
 		let d = new frappe.ui.Dialog({
-			title: __('Send Test Email'),
+			title: __("Send Test Email"),
 			fields: [
 				{
-					label: __('Email'),
-					fieldname: 'email',
-					fieldtype: 'Data',
-					options: 'Email',
-				}
+					label: __("Email"),
+					fieldname: "email",
+					fieldtype: "Data",
+					options: "Email",
+				},
 			],
-			primary_action_label: __('Send'),
+			primary_action_label: __("Send"),
 			primary_action({ email }) {
-				d.get_primary_btn().text(__('Sending...')).prop('disabled', true);
-				frm.call('send_test_email', { email })
-					.then(() => {
-						d.get_primary_btn().text(__('Send again')).prop('disabled', false);
-					});
-			}
+				d.get_primary_btn().text(__("Sending...")).prop("disabled", true);
+				frm.call("send_test_email", { email }).then(() => {
+					d.get_primary_btn().text(__("Send again")).prop("disabled", false);
+				});
+			},
 		});
 		d.show();
 	},
 
 	async update_sending_status(frm) {
-		if (frm.doc.email_sent && frm.$wrapper.is(':visible') && !frm.waiting_for_request) {
+		if (frm.doc.email_sent && frm.$wrapper.is(":visible") && !frm.waiting_for_request) {
 			frm.waiting_for_request = true;
-			let res = await frm.call('get_sending_status');
+			let res = await frm.call("get_sending_status");
 			frm.waiting_for_request = false;
 			let stats = res.message;
 			stats && frm.events.update_sending_progress(frm, stats);
-			if (stats.sent + stats.error >= frm.doc.total_recipients || (!stats.total && !stats.emails_queued)) {
+			if (
+				stats.sent + stats.error >= frm.doc.total_recipients ||
+				(!stats.total && !stats.emails_queued)
+			) {
 				frm.sending_status && clearInterval(frm.sending_status);
 				frm.sending_status = null;
 				return;
@@ -162,7 +197,11 @@ frappe.ui.form.on('Newsletter', {
 		}
 		if (stats.total) {
 			frm.page.set_indicator(__("Sending"), "blue");
-			frm.dashboard.show_progress(__('Sending emails'), stats.sent * 100 / frm.doc.total_recipients, __("{0} of {1} sent", [stats.sent, frm.doc.total_recipients]));
+			frm.dashboard.show_progress(
+				__("Sending emails"),
+				(stats.sent * 100) / frm.doc.total_recipients,
+				__("{0} of {1} sent", [stats.sent, frm.doc.total_recipients])
+			);
 		} else if (stats.emails_queued) {
 			frm.page.set_indicator(__("Queued"), "blue");
 		}
@@ -178,9 +217,11 @@ frappe.ui.form.on('Newsletter', {
 	update_schedule_message(frm) {
 		if (!frm.doc.email_sent && frm.doc.schedule_send) {
 			let datetime = frappe.datetime.global_date_format(frm.doc.schedule_send);
-			frm.dashboard.set_headline_alert(__('This newsletter is scheduled to be sent on {0}', [datetime.bold()]));
+			frm.dashboard.set_headline_alert(
+				__("This newsletter is scheduled to be sent on {0}", [datetime.bold()])
+			);
 		} else {
 			frm.dashboard.clear_headline();
 		}
-	}
+	},
 });
diff --git a/frappe/email/doctype/newsletter/newsletter.json b/frappe/email/doctype/newsletter/newsletter.json
index b42f4755cb..7ac6203ada 100644
--- a/frappe/email/doctype/newsletter/newsletter.json
+++ b/frappe/email/doctype/newsletter/newsletter.json
@@ -12,6 +12,7 @@
   "column_break_3",
   "total_recipients",
   "column_break_12",
+  "total_views",
   "email_sent",
   "from_section",
   "sender_name",
@@ -28,6 +29,7 @@
   "message",
   "message_md",
   "message_html",
+  "campaign",
   "attachments",
   "send_unsubscribe_link",
   "send_webview_link",
@@ -228,6 +230,21 @@
   {
    "fieldname": "column_break_3",
    "fieldtype": "Column Break"
+  },
+  {
+   "default": "0",
+   "fieldname": "total_views",
+   "fieldtype": "Int",
+   "label": "Total Views",
+   "no_copy": 1,
+   "read_only": 1
+  },
+  {
+   "fieldname": "campaign",
+   "fieldtype": "Link",
+   "label": "Campaign",
+   "options": "Marketing Campaign",
+   "reqd": 0
   }
  ],
  "has_web_view": 1,
@@ -236,7 +253,7 @@
  "index_web_pages_for_search": 1,
  "is_published_field": "published",
  "links": [],
- "modified": "2022-03-09 01:48:16.741603",
+ "modified": "2023-03-20 22:45:59.129630",
  "modified_by": "Administrator",
  "module": "Email",
  "name": "Newsletter",
@@ -251,7 +268,6 @@
    "read": 1,
    "report": 1,
    "role": "Newsletter Manager",
-   "set_user_permissions": 1,
    "share": 1,
    "write": 1
   }
@@ -259,6 +275,7 @@
  "route": "newsletters",
  "sort_field": "modified",
  "sort_order": "ASC",
+ "states": [],
  "title_field": "subject",
  "track_changes": 1
 }
\ No newline at end of file
diff --git a/frappe/email/doctype/newsletter/newsletter.py b/frappe/email/doctype/newsletter/newsletter.py
index b0cbb1993d..4a2f69a44c 100644
--- a/frappe/email/doctype/newsletter/newsletter.py
+++ b/frappe/email/doctype/newsletter/newsletter.py
@@ -6,6 +6,7 @@ import frappe
 import frappe.utils
 from frappe import _
 from frappe.email.doctype.email_group.email_group import add_subscribers
+from frappe.rate_limiter import rate_limit
 from frappe.utils.safe_exec import is_job_queued
 from frappe.utils.verified_command import get_signed_params, verify_request
 from frappe.website.website_generator import WebsiteGenerator
@@ -53,7 +54,7 @@ class Newsletter(WebsiteGenerator):
 	@frappe.whitelist()
 	def send_test_email(self, email):
 		test_emails = frappe.utils.validate_email_address(email, throw=True)
-		self.send_newsletter(emails=test_emails)
+		self.send_newsletter(emails=test_emails, test_email=True)
 		frappe.msgprint(_("Test email sent to {0}").format(email), alert=True)
 
 	@frappe.whitelist()
@@ -128,15 +129,11 @@ class Newsletter(WebsiteGenerator):
 			pluck="name",
 		)
 
-	def get_success_recipients(self) -> list[str]:
-		"""Recipients who have already received the newsletter.
-
-		Couldn't think of a better name ;)
-		"""
+	def get_queued_recipients(self) -> list[str]:
+		"""Recipients who have already been queued for receiving the newsletter."""
 		return frappe.get_all(
 			"Email Queue Recipient",
 			filters={
-				"status": ("in", ["Not Sent", "Sending", "Sent"]),
 				"parent": ("in", self.get_linked_email_queue()),
 			},
 			pluck="recipient",
@@ -146,8 +143,7 @@ class Newsletter(WebsiteGenerator):
 		"""Get list of pending recipients of the newsletter. These
 		recipients may not have receive the newsletter in the previous iteration.
 		"""
-		success_recipients = set(self.get_success_recipients())
-		return [x for x in self.newsletter_recipients if x not in success_recipients]
+		return [x for x in self.newsletter_recipients if x not in self.get_queued_recipients()]
 
 	def queue_all(self):
 		"""Queue Newsletter to all the recipients generated from the `Email Group` table"""
@@ -166,12 +162,12 @@ class Newsletter(WebsiteGenerator):
 		"""Get list of attachments on current Newsletter"""
 		return [{"file_url": row.attachment} for row in self.attachments]
 
-	def send_newsletter(self, emails: list[str]):
+	def send_newsletter(self, emails: list[str], test_email: bool = False):
 		"""Trigger email generation for `emails` and add it in Email Queue."""
 		attachments = self.get_newsletter_attachments()
 		sender = self.send_from or frappe.utils.get_formatted_email(self.owner)
 		args = self.as_dict()
-		args["message"] = self.get_message()
+		args["message"] = self.get_message(medium="email")
 
 		is_auto_commit_set = bool(frappe.db.auto_commit_on_many_writes)
 		frappe.db.auto_commit_on_many_writes = not frappe.flags.in_test
@@ -190,18 +186,42 @@ class Newsletter(WebsiteGenerator):
 			queue_separately=True,
 			send_priority=0,
 			args=args,
+			email_read_tracker_url=None
+			if test_email
+			else "/api/method/frappe.email.doctype.newsletter.newsletter.newsletter_email_read",
 		)
 
 		frappe.db.auto_commit_on_many_writes = is_auto_commit_set
 
-	def get_message(self) -> str:
+	def get_message(self, medium=None) -> str:
 		message = self.message
 		if self.content_type == "Markdown":
 			message = frappe.utils.md_to_html(self.message_md)
 		if self.content_type == "HTML":
 			message = self.message_html
 
-		return frappe.render_template(message, {"doc": self.as_dict()})
+		html = frappe.render_template(message, {"doc": self.as_dict()})
+
+		return self.add_source(html, medium=medium)
+
+	def add_source(self, html: str, medium="None") -> str:
+		"""Add source to the site links in the newsletter content."""
+		from bs4 import BeautifulSoup
+
+		soup = BeautifulSoup(html, "html.parser")
+
+		links = soup.find_all("a")
+		for link in links:
+			href = link.get("href")
+			if href and not href.startswith("#"):
+				if not frappe.utils.is_site_link(href):
+					continue
+				new_href = frappe.utils.add_trackers_to_url(
+					href, source="Newsletter", campaign=self.campaign, medium=medium
+				)
+				link["href"] = new_href
+
+		return str(soup)
 
 	def get_recipients(self) -> list[str]:
 		"""Get recipients from Email Group"""
@@ -232,7 +252,6 @@ class Newsletter(WebsiteGenerator):
 		)
 
 
-@frappe.whitelist(allow_guest=True)
 def confirmed_unsubscribe(email, group):
 	"""unsubscribe the email(user) from the mailing list(email_group)"""
 	frappe.flags.ignore_permissions = True
@@ -243,9 +262,13 @@ def confirmed_unsubscribe(email, group):
 
 
 @frappe.whitelist(allow_guest=True)
-def subscribe(email, email_group=_("Website")):  # noqa
+@rate_limit(limit=10, seconds=60 * 60)
+def subscribe(email, email_group=None):  # noqa
 	"""API endpoint to subscribe an email to a particular email group. Triggers a confirmation email."""
 
+	if email_group is None:
+		email_group = _("Website")
+
 	# build subscription confirmation URL
 	api_endpoint = frappe.utils.get_url(
 		"/api/method/frappe.email.doctype.newsletter.newsletter.confirm_subscription"
@@ -283,7 +306,6 @@ def subscribe(email, email_group=_("Website")):  # noqa
 		email,
 		subject=email_subject,
 		content=content,
-		now=True,
 	)
 
 
@@ -349,3 +371,23 @@ def send_scheduled_email():
 
 		if not frappe.flags.in_test:
 			frappe.db.commit()
+
+
+@frappe.whitelist(allow_guest=True)
+def newsletter_email_read(recipient_email, reference_doctype, reference_name):
+	verify_request()
+	try:
+		doc = frappe.get_cached_doc("Newsletter", reference_name)
+		if doc.add_viewed(recipient_email, force=True, unique_views=True):
+			newsletter = frappe.qb.DocType("Newsletter")
+			(
+				frappe.qb.update(newsletter)
+				.set(newsletter.total_views, newsletter.total_views + 1)
+				.where(newsletter.name == doc.name)
+			).run()
+
+	except Exception:
+		doc.log_error(f"Unable to mark as viewed for {recipient_email}")
+
+	finally:
+		frappe.response.update(frappe.utils.get_imaginary_pixel_response())
diff --git a/frappe/email/doctype/newsletter/newsletter_list.js b/frappe/email/doctype/newsletter/newsletter_list.js
index 0b82f1c9e4..0921de02b4 100644
--- a/frappe/email/doctype/newsletter/newsletter_list.js
+++ b/frappe/email/doctype/newsletter/newsletter_list.js
@@ -1,6 +1,6 @@
-frappe.listview_settings['Newsletter'] = {
+frappe.listview_settings["Newsletter"] = {
 	add_fields: ["subject", "email_sent", "schedule_sending"],
-	get_indicator: function(doc) {
+	get_indicator: function (doc) {
 		if (doc.email_sent) {
 			return [__("Sent"), "green", "email_sent,=,Yes"];
 		} else if (doc.schedule_sending) {
@@ -8,5 +8,5 @@ frappe.listview_settings['Newsletter'] = {
 		} else {
 			return [__("Not Sent"), "gray", "email_sent,=,No"];
 		}
-	}
+	},
 };
diff --git a/frappe/email/doctype/newsletter/templates/newsletter.html b/frappe/email/doctype/newsletter/templates/newsletter.html
index 1244f4c49a..05f3560648 100644
--- a/frappe/email/doctype/newsletter/templates/newsletter.html
+++ b/frappe/email/doctype/newsletter/templates/newsletter.html
@@ -36,7 +36,7 @@
                 
        
             
        
             
        
-                {{ doc.get_message() }}
+                {{ doc.get_message(medium="web_page") }}
             
        
         
 
diff --git a/frappe/email/doctype/newsletter/test_newsletter.py b/frappe/email/doctype/newsletter/test_newsletter.py
index 524289db7f..a25b6bda02 100644
--- a/frappe/email/doctype/newsletter/test_newsletter.py
+++ b/frappe/email/doctype/newsletter/test_newsletter.py
@@ -74,7 +74,7 @@ class TestNewsletterMixin:
 				).insert(ignore_if_duplicate=True)
 			except Exception:
 				frappe.db.rollback(save_point=savepoint)
-				frappe.db.update(doctype, email_filters, "unsubscribed", 0)
+				frappe.db.set_value(doctype, email_filters, "unsubscribed", 0)
 
 			frappe.db.release_savepoint(savepoint)
 
@@ -107,7 +107,7 @@ class TestNewsletterMixin:
 			"content_type": "Rich Text",
 			"message": "Testing my news.",
 		}
-		similar_newsletters = frappe.db.get_all(doctype, newsletter_content, pluck="name")
+		similar_newsletters = frappe.get_all(doctype, newsletter_content, pluck="name")
 
 		for similar_newsletter in similar_newsletters:
 			frappe.delete_doc(doctype, similar_newsletter)
@@ -180,7 +180,7 @@ class TestNewsletter(TestNewsletterMixin, FrappeTestCase):
 		newsletter.save = MagicMock()
 		self.assertFalse(newsletter.save.called)
 		# check if the test email is in the queue
-		email_queue = frappe.db.get_all(
+		email_queue = frappe.get_all(
 			"Email Queue",
 			filters=[
 				["reference_doctype", "=", "Newsletter"],
@@ -236,13 +236,15 @@ class TestNewsletter(TestNewsletterMixin, FrappeTestCase):
 		email_queue_list = [frappe.get_doc("Email Queue", e.name) for e in frappe.get_all("Email Queue")]
 		self.assertEqual(len(email_queue_list), 4)
 
-		# emulate partial send
-		email_queue_list[0].status = "Error"
-		email_queue_list[0].recipients[0].status = "Error"
-		email_queue_list[0].save()
+		# delete a queue document to emulate partial send
+		queue_recipient_name = email_queue_list[0].recipients[0].recipient
+		email_queue_list[0].delete()
 		newsletter.email_sent = False
 
+		# make sure the pending recipient is only the one which has been deleted
+		self.assertEqual(newsletter.get_pending_recipients(), [queue_recipient_name])
+
 		# retry
 		newsletter.send_emails()
-		email_queue_list = [frappe.get_doc("Email Queue", e.name) for e in frappe.get_all("Email Queue")]
-		self.assertEqual(len(email_queue_list), 5)
+		self.assertEqual(frappe.db.count("Email Queue"), 4)
+		self.assertTrue(newsletter.email_sent)
diff --git a/frappe/email/doctype/notification/notification.js b/frappe/email/doctype/notification/notification.js
index f14447707f..eb3e2e8634 100644
--- a/frappe/email/doctype/notification/notification.js
+++ b/frappe/email/doctype/notification/notification.js
@@ -1,100 +1,88 @@
 // Copyright (c) 2018, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-this.frm.add_fetch('sender', 'email_id', 'sender_email');
-
-this.frm.fields_dict.sender.get_query = function() {
-	return {
-		filters: {
-			enable_outgoing: 1
-		}
-	};
-};
-
 frappe.notification = {
-	setup_fieldname_select: function(frm) {
+	setup_fieldname_select: function (frm) {
 		// get the doctype to update fields
 		if (!frm.doc.document_type) {
 			return;
 		}
 
-		frappe.model.with_doctype(frm.doc.document_type, function() {
-			let get_select_options = function(df, parent_field) {
+		frappe.model.with_doctype(frm.doc.document_type, function () {
+			let get_select_options = function (df, parent_field) {
 				// Append parent_field name along with fieldname for child table fields
-				let select_value = parent_field ? df.fieldname + ',' + parent_field : df.fieldname;
+				let select_value = parent_field ? df.fieldname + "," + parent_field : df.fieldname;
 
 				return {
 					value: select_value,
-					label: df.fieldname + ' (' + __(df.label) + ')'
+					label: df.fieldname + " (" + __(df.label) + ")",
 				};
 			};
 
-			let get_date_change_options = function() {
-				let date_options = $.map(fields, function(d) {
-					return d.fieldtype == 'Date' || d.fieldtype == 'Datetime'
+			let get_date_change_options = function () {
+				let date_options = $.map(fields, function (d) {
+					return d.fieldtype == "Date" || d.fieldtype == "Datetime"
 						? get_select_options(d)
 						: null;
 				});
 				// append creation and modified date to Date Change field
 				return date_options.concat([
-					{ value: 'creation', label: `creation (${__('Created On')})` },
-					{ value: 'modified', label: `modified (${__('Last Modified Date')})` }
+					{ value: "creation", label: `creation (${__("Created On")})` },
+					{ value: "modified", label: `modified (${__("Last Modified Date")})` },
 				]);
 			};
 
-			let fields = frappe.get_doc('DocType', frm.doc.document_type).fields;
-			let options = $.map(fields, function(d) {
+			let fields = frappe.get_doc("DocType", frm.doc.document_type).fields;
+			let options = $.map(fields, function (d) {
 				return in_list(frappe.model.no_value_type, d.fieldtype)
-					? null : get_select_options(d);
+					? null
+					: get_select_options(d);
 			});
 
 			// set value changed options
-			frm.set_df_property('value_changed', 'options', [''].concat(options));
-			frm.set_df_property(
-				'set_property_after_alert',
-				'options',
-				[''].concat(options)
-			);
+			frm.set_df_property("value_changed", "options", [""].concat(options));
+			frm.set_df_property("set_property_after_alert", "options", [""].concat(options));
 
 			// set date changed options
-			frm.set_df_property('date_changed', 'options', get_date_change_options());
+			frm.set_df_property("date_changed", "options", get_date_change_options());
 
 			let receiver_fields = [];
-			if (frm.doc.channel === 'Email') {
-				receiver_fields = $.map(fields, function(d) {
-
+			if (frm.doc.channel === "Email") {
+				receiver_fields = $.map(fields, function (d) {
 					// Add User and Email fields from child into select dropdown
-					if (d.fieldtype == 'Table') {
-						let child_fields = frappe.get_doc('DocType', d.options).fields;
-						return $.map(child_fields, function(df) {
-							return df.options == 'Email' ||
-								(df.options == 'User' && df.fieldtype == 'Link')
-								? get_select_options(df, d.fieldname) : null;
+					if (d.fieldtype == "Table") {
+						let child_fields = frappe.get_doc("DocType", d.options).fields;
+						return $.map(child_fields, function (df) {
+							return df.options == "Email" ||
+								(df.options == "User" && df.fieldtype == "Link")
+								? get_select_options(df, d.fieldname)
+								: null;
 						});
-					// Add User and Email fields from parent into select dropdown
+						// Add User and Email fields from parent into select dropdown
 					} else {
-						return d.options == 'Email' ||
-							(d.options == 'User' && d.fieldtype == 'Link')
-							? get_select_options(d) : null;
+						return d.options == "Email" ||
+							(d.options == "User" && d.fieldtype == "Link")
+							? get_select_options(d)
+							: null;
 					}
 				});
-			} else if (in_list(['WhatsApp', 'SMS'], frm.doc.channel)) {
-				receiver_fields = $.map(fields, function(d) {
-					return d.options == 'Phone' ? get_select_options(d) : null;
+			} else if (in_list(["WhatsApp", "SMS"], frm.doc.channel)) {
+				receiver_fields = $.map(fields, function (d) {
+					return d.options == "Phone" ? get_select_options(d) : null;
 				});
 			}
 
 			// set email recipient options
 			frm.fields_dict.recipients.grid.update_docfield_property(
-				'receiver_by_document_field',
-				'options',
-				[''].concat(["owner"]).concat(receiver_fields)
+				"receiver_by_document_field",
+				"options",
+				[""].concat(["owner"]).concat(receiver_fields)
 			);
 		});
 	},
-	setup_example_message: function(frm) {
-		let template = '';
-		if (frm.doc.channel === 'Email') {
+	setup_example_message: function (frm) {
+		let template = "";
+		if (frm.doc.channel === "Email") {
 			template = `
        Message Example
        
 
 
        <h3>Order Overdue</h3>
@@ -114,7 +102,7 @@ Last comment: {{ comments[-1].comment }} by {{ comments[-1].by }}
 </ul>
 
        
 			`;
-		} else if (in_list(['Slack', 'System Notification', 'SMS'], frm.doc.channel)) {
+		} else if (in_list(["Slack", "System Notification", "SMS"], frm.doc.channel)) {
 			template = `
        Message Example
        
 
 
        *Order Overdue*
@@ -133,71 +121,81 @@ Last comment: {{ comments[-1].comment }} by {{ comments[-1].by }}
 
        `;
 		}
 		if (template) {
-			frm.set_df_property('message_examples', 'options', template);
+			frm.set_df_property("message_examples", "options", template);
 		}
-
-	}
+	},
 };
 
-frappe.ui.form.on('Notification', {
-	onload: function(frm) {
-		frm.set_query('document_type', function() {
+frappe.ui.form.on("Notification", {
+	onload: function (frm) {
+		frm.set_query("document_type", function () {
 			return {
 				filters: {
-					istable: 0
-				}
+					istable: 0,
+				},
 			};
 		});
-		frm.set_query('print_format', function() {
+		frm.set_query("print_format", function () {
 			return {
 				filters: {
-					doc_type: frm.doc.document_type
-				}
+					doc_type: frm.doc.document_type,
+				},
 			};
 		});
 	},
-	refresh: function(frm) {
+	refresh: function (frm) {
 		frappe.notification.setup_fieldname_select(frm);
 		frappe.notification.setup_example_message(frm);
-		frm.get_field('is_standard').toggle(frappe.boot.developer_mode);
-		frm.trigger('event');
+
+		frm.add_fetch("sender", "email_id", "sender_email");
+		frm.set_query("sender", () => {
+			return {
+				filters: {
+					enable_outgoing: 1,
+				},
+			};
+		});
+		frm.get_field("is_standard").toggle(frappe.boot.developer_mode);
+		frm.trigger("event");
 	},
-	document_type: function(frm) {
+	document_type: function (frm) {
 		frappe.notification.setup_fieldname_select(frm);
 	},
-	view_properties: function(frm) {
+	view_properties: function (frm) {
 		frappe.route_options = { doc_type: frm.doc.document_type };
-		frappe.set_route('Form', 'Customize Form');
+		frappe.set_route("Form", "Customize Form");
 	},
-	event: function(frm) {
-		if (in_list(['Days Before', 'Days After'], frm.doc.event)) {
-			frm.add_custom_button(__('Get Alerts for Today'), function() {
+	event: function (frm) {
+		if (in_list(["Days Before", "Days After"], frm.doc.event)) {
+			frm.add_custom_button(__("Get Alerts for Today"), function () {
 				frappe.call({
-					method:
-						'frappe.email.doctype.notification.notification.get_documents_for_today',
+					method: "frappe.email.doctype.notification.notification.get_documents_for_today",
 					args: {
-						notification: frm.doc.name
+						notification: frm.doc.name,
 					},
-					callback: function(r) {
-						if (r.message) {
-							frappe.msgprint(r.message);
+					callback: function (r) {
+						if (r.message && r.message.length > 0) {
+							frappe.msgprint(r.message.toString());
 						} else {
-							frappe.msgprint(__('No alerts for today'));
+							frappe.msgprint(__("No alerts for today"));
 						}
-					}
+					},
 				});
 			});
 		}
 	},
-	channel: function(frm) {
-		frm.toggle_reqd('recipients', frm.doc.channel == 'Email');
+	channel: function (frm) {
+		frm.toggle_reqd("recipients", frm.doc.channel == "Email");
 		frappe.notification.setup_fieldname_select(frm);
 		frappe.notification.setup_example_message(frm);
-		if (frm.doc.channel === 'SMS' && frm.doc.__islocal) {
-			frm.set_df_property('channel',
-				'description', `To use SMS Channel, initialize SMS Settings.`);
+		if (frm.doc.channel === "SMS" && frm.doc.__islocal) {
+			frm.set_df_property(
+				"channel",
+				"description",
+				`To use SMS Channel, initialize SMS Settings.`
+			);
 		} else {
-			frm.set_df_property('channel', 'description', ` `);
+			frm.set_df_property("channel", "description", ` `);
 		}
-	}
+	},
 });
diff --git a/frappe/email/doctype/notification/notification.py b/frappe/email/doctype/notification/notification.py
index 8d0857ac60..2efbf597ec 100644
--- a/frappe/email/doctype/notification/notification.py
+++ b/frappe/email/doctype/notification/notification.py
@@ -45,6 +45,7 @@ class Notification(Document):
 		frappe.cache().hdel("notifications", self.document_type)
 
 	def on_update(self):
+		frappe.cache().hdel("notifications", self.document_type)
 		path = export_module_json(self, self.is_standard, self.module)
 		if path:
 			# js
@@ -297,7 +298,7 @@ def get_context(context):
 
 			# For sending emails to specified role
 			if recipient.receiver_by_role:
-				emails = get_info_based_on_role(recipient.receiver_by_role, "email")
+				emails = get_info_based_on_role(recipient.receiver_by_role, "email", ignore_permissions=True)
 
 				for email in emails:
 					recipients = recipients + email.split("\n")
@@ -387,6 +388,9 @@ def get_context(context):
 		if not is_html(self.message):
 			self.message = frappe.utils.md_to_html(self.message)
 
+	def on_trash(self):
+		frappe.cache().hdel("notifications", self.document_type)
+
 
 @frappe.whitelist()
 def get_documents_for_today(notification):
@@ -449,16 +453,17 @@ def evaluate_alert(doc: Document, alert, event):
 			doc.reload()
 		alert.send(doc)
 	except TemplateError:
-		frappe.throw(
-			_("Error while evaluating Notification {0}. Please fix your template.").format(alert)
+		message = _("Error while evaluating Notification {0}. Please fix your template.").format(
+			frappe.utils.get_link_to_form("Notification", alert.name)
 		)
+		frappe.throw(message, title=_("Error in Notification"))
 	except Exception as e:
-		error_log = frappe.log_error(message=frappe.get_traceback(), title=str(e))
-		frappe.throw(
-			_("Error in Notification: {}").format(
-				frappe.utils.get_link_to_form("Error Log", error_log.name)
-			)
-		)
+		title = str(e)
+		message = frappe.get_traceback()
+		frappe.log_error(message=message, title=title)
+
+		msg = f"
        {title}{message}
        "
+		frappe.throw(msg, title=_("Error in Notification"))
 
 
 def get_context(doc):
diff --git a/frappe/email/doctype/notification/test_notification.py b/frappe/email/doctype/notification/test_notification.py
index 4adaeae37e..576d9e9a66 100644
--- a/frappe/email/doctype/notification/test_notification.py
+++ b/frappe/email/doctype/notification/test_notification.py
@@ -1,12 +1,13 @@
 # Copyright (c) 2018, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
-import unittest
+
 from contextlib import contextmanager
 
 import frappe
 import frappe.utils
 import frappe.utils.scheduler
 from frappe.desk.form import assign_to
+from frappe.tests.utils import FrappeTestCase
 
 test_dependencies = ["User", "Notification"]
 
@@ -20,7 +21,7 @@ def get_test_notification(config):
 		notification.delete()
 
 
-class TestNotification(unittest.TestCase):
+class TestNotification(FrappeTestCase):
 	def setUp(self):
 		frappe.db.delete("Email Queue")
 		frappe.set_user("test@example.com")
@@ -93,7 +94,7 @@ class TestNotification(unittest.TestCase):
 	def test_condition(self):
 		"""Check notification is triggered based on a condition."""
 		event = frappe.new_doc("Event")
-		event.subject = ("test",)
+		event.subject = "test"
 		event.event_type = "Private"
 		event.starts_on = "2014-06-06 12:00:00"
 		event.insert()
@@ -146,7 +147,7 @@ class TestNotification(unittest.TestCase):
 
 	def test_value_changed(self):
 		event = frappe.new_doc("Event")
-		event.subject = ("test",)
+		event.subject = "test"
 		event.event_type = "Private"
 		event.starts_on = "2014-06-06 12:00:00"
 		event.insert()
@@ -195,7 +196,7 @@ class TestNotification(unittest.TestCase):
 		frappe.db.commit()
 
 		event = frappe.new_doc("Event")
-		event.subject = ("test-2",)
+		event.subject = "test-2"
 		event.event_type = "Private"
 		event.starts_on = "2014-06-06 12:00:00"
 		event.insert()
@@ -209,9 +210,8 @@ class TestNotification(unittest.TestCase):
 		event.delete()
 
 	def test_date_changed(self):
-
 		event = frappe.new_doc("Event")
-		event.subject = ("test",)
+		event.subject = "test"
 		event.event_type = "Private"
 		event.starts_on = "2014-01-01 12:00:00"
 		event.insert()
diff --git a/frappe/email/doctype/unhandled_email/test_unhandled_email.py b/frappe/email/doctype/unhandled_email/test_unhandled_email.py
index debc52d685..0d36b6b091 100644
--- a/frappe/email/doctype/unhandled_email/test_unhandled_email.py
+++ b/frappe/email/doctype/unhandled_email/test_unhandled_email.py
@@ -1,9 +1,9 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
-import unittest
+from frappe.tests.utils import FrappeTestCase
 
 # test_records = frappe.get_test_records('Unhandled Emails')
 
 
-class TestUnhandledEmail(unittest.TestCase):
+class TestUnhandledEmail(FrappeTestCase):
 	pass
diff --git a/frappe/email/doctype/unhandled_email/unhandled_email.json b/frappe/email/doctype/unhandled_email/unhandled_email.json
index de4407f38f..d904536936 100644
--- a/frappe/email/doctype/unhandled_email/unhandled_email.json
+++ b/frappe/email/doctype/unhandled_email/unhandled_email.json
@@ -1,212 +1,60 @@
 {
- "allow_copy": 0, 
- "allow_guest_to_view": 0, 
- "allow_import": 0, 
- "allow_rename": 0, 
- "beta": 0, 
- "creation": "2016-04-14 09:41:45.892975", 
- "custom": 0, 
- "docstatus": 0, 
- "doctype": "DocType", 
- "document_type": "Setup", 
- "editable_grid": 0, 
+ "actions": [],
+ "creation": "2016-04-14 09:41:45.892975",
+ "doctype": "DocType",
+ "document_type": "Setup",
+ "engine": "InnoDB",
+ "field_order": [
+  "email_account",
+  "uid",
+  "reason",
+  "message_id",
+  "raw"
+ ],
  "fields": [
   {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "email_account", 
-   "fieldtype": "Link", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Email Account", 
-   "length": 0, 
-   "no_copy": 0, 
-   "options": "Email Account", 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "fieldname": "email_account",
+   "fieldtype": "Link",
+   "in_list_view": 1,
+   "label": "Email Account",
+   "options": "Email Account"
+  },
   {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "uid", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "UID", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "fieldname": "uid",
+   "fieldtype": "Data",
+   "label": "UID"
+  },
   {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "reason", 
-   "fieldtype": "Long Text", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Reason", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "fieldname": "reason",
+   "fieldtype": "Long Text",
+   "in_list_view": 1,
+   "label": "Reason"
+  },
   {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "message_id", 
-   "fieldtype": "Code", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Message-id", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "fieldname": "message_id",
+   "fieldtype": "Code",
+   "label": "Message-id"
+  },
   {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "raw", 
-   "fieldtype": "Code", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Raw Email", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
+   "fieldname": "raw",
+   "fieldtype": "Code",
+   "label": "Raw Email"
   }
- ], 
- "has_web_view": 0, 
- "hide_heading": 0, 
- "hide_toolbar": 0, 
- "idx": 0, 
- "image_view": 0, 
- "in_create": 1, 
- "is_submittable": 0, 
- "issingle": 0, 
- "istable": 0, 
- "max_attachments": 0, 
- "modified": "2017-09-19 16:28:00.042256", 
- "modified_by": "Administrator", 
- "module": "Email", 
- "name": "Unhandled Email", 
- "name_case": "", 
- "owner": "Administrator", 
+ ],
+ "in_create": 1,
+ "links": [],
+ "modified": "2022-08-03 12:20:51.822287",
+ "modified_by": "Administrator",
+ "module": "Email",
+ "name": "Unhandled Email",
+ "owner": "Administrator",
  "permissions": [
   {
-   "amend": 0, 
-   "apply_user_permissions": 0, 
-   "cancel": 0, 
-   "create": 0, 
-   "delete": 0, 
-   "email": 0, 
-   "export": 0, 
-   "if_owner": 0, 
-   "import": 0, 
-   "permlevel": 0, 
-   "print": 0, 
-   "read": 1, 
-   "report": 0, 
-   "role": "System Manager", 
-   "set_user_permissions": 0, 
-   "share": 0, 
-   "submit": 0, 
-   "write": 0
+   "read": 1,
+   "role": "System Manager"
   }
- ], 
- "quick_entry": 0, 
- "read_only": 0, 
- "read_only_onload": 0, 
- "show_name_in_global_search": 0, 
- "sort_field": "modified", 
- "sort_order": "DESC", 
- "track_changes": 0, 
- "track_seen": 0
+ ],
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "states": []
 }
\ No newline at end of file
diff --git a/frappe/email/oauth.py b/frappe/email/oauth.py
index 46d1565275..87feb8ca11 100644
--- a/frappe/email/oauth.py
+++ b/frappe/email/oauth.py
@@ -2,15 +2,8 @@ import base64
 from imaplib import IMAP4
 from poplib import POP3
 from smtplib import SMTP
-from urllib.parse import quote
 
 import frappe
-from frappe.integrations.google_oauth import GoogleOAuth
-from frappe.utils.password import encrypt
-
-
-class OAuthenticationError(Exception):
-	pass
 
 
 class Oauth:
@@ -20,46 +13,32 @@ class Oauth:
 		email_account: str,
 		email: str,
 		access_token: str,
-		refresh_token: str,
-		service: str,
 		mechanism: str = "XOAUTH2",
 	) -> None:
 
 		self.email_account = email_account
 		self.email = email
-		self.service = service
 		self._mechanism = mechanism
 		self._conn = conn
 		self._access_token = access_token
-		self._refresh_token = refresh_token
 
 		self._validate()
 
 	def _validate(self) -> None:
-		if self.service != "GMail":
-			raise NotImplementedError(
-				f"Service {self.service} currently doesn't have oauth implementation."
-			)
-
-		if not self._refresh_token:
+		if not self._access_token:
 			frappe.throw(
-				frappe._("Please Authorize OAuth."),
-				OAuthenticationError,
-				frappe._("OAuth Error"),
+				frappe._("Please Authorize OAuth for Email Account {}").format(self.email_account),
+				title=frappe._("OAuth Error"),
 			)
 
 	@property
 	def _auth_string(self) -> str:
 		return f"user={self.email}\1auth=Bearer {self._access_token}\1\1"
 
-	def connect(self, _retry: int = 0) -> None:
-		"""Connection method with retry on exception for Oauth"""
+	def connect(self) -> None:
 		try:
 			if isinstance(self._conn, POP3):
-				res = self._connect_pop()
-
-				if not res.startswith(b"+OK"):
-					raise
+				self._connect_pop()
 
 			elif isinstance(self._conn, IMAP4):
 				self._connect_imap()
@@ -68,96 +47,29 @@ class Oauth:
 				# SMTP
 				self._connect_smtp()
 
-		except Exception as e:
-			# maybe the access token expired - refreshing
-			access_token = self._refresh_access_token()
+		except Exception:
+			frappe.log_error(
+				"Email Connection Error - Authentication Failed",
+				reference_doctype="Email Account",
+				reference_name=self.email_account,
+			)
+			# raising a bare exception here as we have a lot of exception handling present
+			# where the connect method is called from - hence just logging and raising.
+			raise
 
-			if not access_token or _retry > 0:
-				frappe.log_error(
-					"OAuth Error - Authentication Failed", str(e), "Email Account", self.email_account
-				)
-				# raising a bare exception here as we have a lot of exception handling present
-				# where the connect method is called from - hence just logging and raising.
-				raise
-
-			self._access_token = access_token
-			self.connect(_retry + 1)
-
-	def _connect_pop(self) -> bytes:
-		# poplib doesn't have AUTH command implementation
+	def _connect_pop(self) -> None:
+		# NOTE: poplib doesn't have AUTH command implementation
 		res = self._conn._shortcmd(
 			"AUTH {} {}".format(
 				self._mechanism, base64.b64encode(bytes(self._auth_string, "utf-8")).decode("utf-8")
 			)
 		)
 
-		return res
+		if not res.startswith(b"+OK"):
+			raise
 
 	def _connect_imap(self) -> None:
 		self._conn.authenticate(self._mechanism, lambda x: self._auth_string)
 
 	def _connect_smtp(self) -> None:
 		self._conn.auth(self._mechanism, lambda x: self._auth_string, initial_response_ok=False)
-
-	def _refresh_access_token(self) -> str:
-		"""Refreshes access token via calling `refresh_access_token` method of oauth service object"""
-		service_obj = self._get_service_object()
-		access_token = service_obj.refresh_access_token(self._refresh_token).get("access_token", None)
-
-		# set the new access token in db
-		frappe.db.set_value(
-			"Email Account", self.email_account, "access_token", access_token, update_modified=False
-		)
-		return access_token
-
-	def _get_service_object(self):
-		"""Get Oauth service object"""
-
-		return {
-			"GMail": GoogleOAuth("mail", validate=False),
-		}[self.service]
-
-
-@frappe.whitelist(methods=["POST"])
-def oauth_access(email_account: str, service: str):
-	"""Used as a default endpoint/caller for all oauth services.
-	Returns authorization url for redirection"""
-
-	if not service:
-		frappe.throw(frappe._("No Service is selected. Please select one and try again!"))
-
-	doctype = "Email Account"
-
-	if service == "GMail":
-		return authorize_google_access(email_account, doctype)
-
-	raise NotImplementedError(f"Service {service} currently doesn't have oauth implementation.")
-
-
-def authorize_google_access(email_account, doctype: str = "Email Account", code: str = None):
-	"""Facilitates google oauth for email.
-	This is invoked 2 times - first time when user clicks `Authorze API Access` for getting the authorization url
-	and second time for setting the refresh and access token in db when google redirects back with oauth code."""
-
-	oauth_obj = GoogleOAuth("mail")
-
-	if not code:
-		return oauth_obj.get_authentication_url(
-			{
-				"method": "frappe.email.oauth.authorize_google_access",
-				"redirect": f"/app/Form/{quote(doctype)}/{quote(email_account)}",
-				"success_query_param": "successful_authorization=1",
-				"email_account": email_account,
-			},
-		)
-
-	res = oauth_obj.authorize(code)
-	frappe.db.set_value(
-		doctype,
-		email_account,
-		{
-			"refresh_token": encrypt(res.get("refresh_token")),
-			"access_token": encrypt(res.get("access_token")),
-		},
-		update_modified=False,
-	)
diff --git a/frappe/email/queue.py b/frappe/email/queue.py
index 2c3e0ee011..7d4b92baf1 100755
--- a/frappe/email/queue.py
+++ b/frappe/email/queue.py
@@ -3,9 +3,8 @@
 
 import frappe
 from frappe import _, msgprint
-from frappe.query_builder import DocType, Interval
-from frappe.query_builder.functions import Now
-from frappe.utils import cint, get_url, now_datetime
+from frappe.utils import cint, cstr, get_url, now_datetime
+from frappe.utils.data import getdate
 from frappe.utils.verified_command import get_signed_params, verify_request
 
 
@@ -16,26 +15,17 @@ def get_emails_sent_this_month(email_account=None):
 
 	if email_account=None, email account filter is not applied while counting
 	"""
-	q = """
-		SELECT
-			COUNT(*)
-		FROM
-			`tabEmail Queue`
-		WHERE
-			`status`='Sent'
-			AND
-			EXTRACT(YEAR_MONTH FROM `creation`) = EXTRACT(YEAR_MONTH FROM NOW())
-	"""
+	today = getdate()
+	month_start = today.replace(day=1)
 
-	q_args = {}
-	if email_account is not None:
-		if email_account:
-			q += " AND email_account = %(email_account)s"
-			q_args["email_account"] = email_account
-		else:
-			q += " AND (email_account is null OR email_account='')"
+	filters = {
+		"status": "Sent",
+		"creation": [">=", str(month_start)],
+	}
+	if email_account:
+		filters["email_account"] = email_account
 
-	return frappe.db.sql(q, q_args)[0][0]
+	return frappe.db.count("Email Queue", filters=filters)
 
 
 def get_emails_sent_today(email_account=None):
@@ -91,9 +81,9 @@ def get_unsubcribed_url(
 	reference_doctype, reference_name, email, unsubscribe_method, unsubscribe_params
 ):
 	params = {
-		"email": email.encode("utf-8"),
-		"doctype": reference_doctype.encode("utf-8"),
-		"name": reference_name.encode("utf-8"),
+		"email": cstr(email),
+		"doctype": cstr(reference_doctype),
+		"name": cstr(reference_name),
 	}
 	if unsubscribe_params:
 		params.update(unsubscribe_params)
@@ -109,7 +99,7 @@ def get_unsubcribed_url(
 @frappe.whitelist(allow_guest=True)
 def unsubscribe(doctype, name, email):
 	# unsubsribe from comments and communications
-	if not verify_request():
+	if not frappe.flags.in_test and not verify_request():
 		return
 
 	try:
@@ -142,20 +132,35 @@ def return_unsubscribed_page(email, doctype, name):
 def flush(from_test=False):
 	"""flush email queue, every time: called from scheduler"""
 	from frappe.email.doctype.email_queue.email_queue import send_mail
+	from frappe.utils.background_jobs import get_jobs
 
 	# To avoid running jobs inside unit tests
 	if frappe.are_emails_muted():
 		msgprint(_("Emails are muted"))
 		from_test = True
 
-	if cint(frappe.defaults.get_defaults().get("hold_queue")) == 1:
+	if cint(frappe.db.get_default("suspend_email_queue")) == 1:
 		return
 
+	try:
+		queued_jobs = set(get_jobs(site=frappe.local.site, key="job_name")[frappe.local.site])
+	except Exception:
+		queued_jobs = set()
+
 	for row in get_queue():
 		try:
-			func = send_mail if from_test else send_mail.enqueue
-			is_background_task = not from_test
-			func(email_queue_name=row.name, is_background_task=is_background_task)
+			job_name = f"email_queue_sendmail_{row.name}"
+			if job_name not in queued_jobs:
+				frappe.enqueue(
+					method=send_mail,
+					email_queue_name=row.name,
+					is_background_task=not from_test,
+					now=from_test,
+					job_name=job_name,
+					queue="short",
+				)
+			else:
+				frappe.logger().debug(f"Not queueing job {job_name} because it is in queue already")
 		except Exception:
 			frappe.get_doc("Email Queue", row.name).log_error()
 
diff --git a/frappe/email/receive.py b/frappe/email/receive.py
index e26748dd07..382fd2ac99 100644
--- a/frappe/email/receive.py
+++ b/frappe/email/receive.py
@@ -9,6 +9,7 @@ import json
 import poplib
 import re
 import time
+from contextlib import suppress
 from email.header import decode_header
 
 import _socket
@@ -22,7 +23,7 @@ from frappe.email.oauth import Oauth
 from frappe.utils import (
 	add_days,
 	cint,
-	convert_utc_to_user_timezone,
+	convert_utc_to_system_timezone,
 	cstr,
 	extract_email_id,
 	get_datetime,
@@ -37,7 +38,7 @@ from frappe.utils.html_utils import clean_email_html
 from frappe.utils.user import is_system_user
 
 # fix due to a python bug in poplib that limits it to 2048
-poplib._MAXLINE = 20480
+poplib._MAXLINE = 1_00_000
 
 THREAD_ID_PATTERN = re.compile(r"(?<=\[)[\w/-]+")
 WORDS_PATTERN = re.compile(r"\w+")
@@ -51,10 +52,6 @@ class EmailTimeoutError(frappe.ValidationError):
 	pass
 
 
-class TotalSizeExceededError(frappe.ValidationError):
-	pass
-
-
 class LoginLimitExceeded(frappe.ValidationError):
 	pass
 
@@ -67,26 +64,11 @@ class EmailServer:
 	"""Wrapper for POP server to pull emails."""
 
 	def __init__(self, args=None):
-		self.setup(args)
-
-	def setup(self, args=None):
-		# overrride
 		self.settings = args or frappe._dict()
 
-	def check_mails(self):
-		# overrride
-		return True
-
-	def process_message(self, mail):
-		# overrride
-		pass
-
 	def connect(self):
 		"""Connect to **Email Account**."""
-		if cint(self.settings.use_imap):
-			return self.connect_imap()
-		else:
-			return self.connect_pop()
+		return self.connect_imap() if cint(self.settings.use_imap) else self.connect_pop()
 
 	def connect_imap(self):
 		"""Connect to IMAP"""
@@ -100,14 +82,15 @@ class EmailServer:
 					self.settings.host, self.settings.incoming_port, timeout=frappe.conf.get("pop_timeout")
 				)
 
+				if cint(self.settings.use_starttls):
+					self.imap.starttls()
+
 			if self.settings.use_oauth:
 				Oauth(
 					self.imap,
 					self.settings.email_account,
 					self.settings.username,
 					self.settings.access_token,
-					self.settings.refresh_token,
-					self.settings.service,
 				).connect()
 
 			else:
@@ -139,8 +122,6 @@ class EmailServer:
 					self.settings.email_account,
 					self.settings.username,
 					self.settings.access_token,
-					self.settings.refresh_token,
-					self.settings.service,
 				).connect()
 
 			else:
@@ -151,8 +132,7 @@ class EmailServer:
 			return True
 
 		except _socket.error:
-			# log performs rollback and logs error in Error Log
-			self.log_error("POP: Unable to connect")
+			frappe.log_error("POP: Unable to connect")
 
 			# Invalid mail server -- due to refusing connection
 			frappe.msgprint(_("Invalid Mail Server. Please rectify and try again."))
@@ -178,68 +158,33 @@ class EmailServer:
 		return
 
 	def get_messages(self, folder="INBOX"):
-		"""Returns new email messages in a list."""
-		if not (self.check_mails() or self.connect()):
-			return []
+		"""Returns new email messages."""
 
-		frappe.db.commit()
+		self.latest_messages = []
+		self.seen_status = {}
+		self.uid_reindexed = False
 
-		uid_list = []
+		email_list = self.get_new_mails(folder)
 
-		try:
-			# track if errors arised
-			self.errors = False
-			self.latest_messages = []
-			self.seen_status = {}
-			self.uid_reindexed = False
-
-			uid_list = email_list = self.get_new_mails(folder)
-
-			if not email_list:
-				return
-
-			num = num_copy = len(email_list)
-
-			# WARNING: Hard coded max no. of messages to be popped
-			if num > 50:
-				num = 50
-
-			# size limits
-			self.total_size = 0
-			self.max_email_size = cint(frappe.local.conf.get("max_email_size"))
-			self.max_total_size = 5 * self.max_email_size
-
-			for i, message_meta in enumerate(email_list[:num]):
-				try:
-					self.retrieve_message(message_meta, i + 1)
-				except (TotalSizeExceededError, EmailTimeoutError, LoginLimitExceeded):
-					break
-			# WARNING: Mark as read - message number 101 onwards from the pop list
-			# This is to avoid having too many messages entering the system
-			num = num_copy
-			if not cint(self.settings.use_imap):
-				if num > 100 and not self.errors:
-					for m in range(101, num + 1):
-						self.pop.dele(m)
-
-		except Exception as e:
-			if self.has_login_limit_exceeded(e):
-				pass
-			else:
-				raise
+		for i, uid in enumerate(email_list[:100]):
+			try:
+				self.retrieve_message(uid, i + 1)
+			except (EmailTimeoutError, LoginLimitExceeded):
+				# get whatever messages were retrieved
+				break
 
 		out = {"latest_messages": self.latest_messages}
 		if self.settings.use_imap:
 			out.update(
-				{"uid_list": uid_list, "seen_status": self.seen_status, "uid_reindexed": self.uid_reindexed}
+				{"uid_list": email_list, "seen_status": self.seen_status, "uid_reindexed": self.uid_reindexed}
 			)
 
 		return out
 
 	def get_new_mails(self, folder):
 		"""Return list of new mails"""
+		email_list = []
 		if cint(self.settings.use_imap):
-			email_list = []
 			self.check_imap_uidvalidity(folder)
 
 			readonly = False if self.settings.email_sync_rule == "UNSEEN" else True
@@ -297,9 +242,6 @@ class EmailServer:
 			self.settings.email_sync_rule = f"UID {from_uid}:{uidnext}"
 			self.uid_reindexed = True
 
-		elif uid_validity == current_uid_validity:
-			return
-
 	def parse_imap_response(self, cmd, response):
 		pattern = rf"(?<={cmd} )[0-9]*"
 		match = re.search(pattern, response.decode("utf-8"), re.U | re.I)
@@ -309,49 +251,28 @@ class EmailServer:
 		else:
 			return None
 
-	def retrieve_message(self, message_meta, msg_num=None):
-		incoming_mail = None
+	def retrieve_message(self, uid, msg_num):
 		try:
-			self.validate_message_limits(message_meta)
-
 			if cint(self.settings.use_imap):
-				status, message = self.imap.uid("fetch", message_meta, "(BODY.PEEK[] BODY.PEEK[HEADER] FLAGS)")
+				status, message = self.imap.uid("fetch", uid, "(BODY.PEEK[] BODY.PEEK[HEADER] FLAGS)")
 				raw = message[0]
 
-				self.get_email_seen_status(message_meta, raw[0])
+				self.get_email_seen_status(uid, raw[0])
 				self.latest_messages.append(raw[1])
 			else:
 				msg = self.pop.retr(msg_num)
 				self.latest_messages.append(b"\n".join(msg[1]))
-		except (TotalSizeExceededError, EmailTimeoutError):
+		except EmailTimeoutError:
 			# propagate this error to break the loop
-			self.errors = True
 			raise
 
 		except Exception as e:
 			if self.has_login_limit_exceeded(e):
-				self.errors = True
 				raise LoginLimitExceeded(e)
 
-			else:
-				# log performs rollback and logs error in Error Log
-				self.log_error("Unable to fetch email", self.make_error_msg(msg_num, incoming_mail))
-				self.errors = True
-				frappe.db.rollback()
+			frappe.log_error("Unable to fetch email", self.make_error_msg(uid, msg_num))
 
-				if not cint(self.settings.use_imap):
-					self.pop.dele(msg_num)
-				else:
-					# mark as seen if email sync rule is UNSEEN (syncing only unseen mails)
-					if self.settings.email_sync_rule == "UNSEEN":
-						self.imap.uid("STORE", message_meta, "+FLAGS", "(\\SEEN)")
-		else:
-			if not cint(self.settings.use_imap):
-				self.pop.dele(msg_num)
-			else:
-				# mark as seen if email sync rule is UNSEEN (syncing only unseen mails)
-				if self.settings.email_sync_rule == "UNSEEN":
-					self.imap.uid("STORE", message_meta, "+FLAGS", "(\\SEEN)")
+		self._post_retrieve_cleanup(uid, msg_num)
 
 	def get_email_seen_status(self, uid, flag_string):
 		"""parse the email FLAGS response"""
@@ -369,7 +290,16 @@ class EmailServer:
 			self.seen_status.update({uid: "UNSEEN"})
 
 	def has_login_limit_exceeded(self, e):
-		return "-ERR Exceeded the login limit" in strip(cstr(e.message))
+		return "-ERR Exceeded the login limit" in strip(cstr(e))
+
+	def _post_retrieve_cleanup(self, uid, msg_num):
+		with suppress(Exception):
+			if not cint(self.settings.use_imap):
+				self.pop.dele(msg_num)
+			else:
+				# mark as seen if email sync rule is UNSEEN (syncing only unseen mails)
+				if self.settings.email_sync_rule == "UNSEEN":
+					self.imap.uid("STORE", uid, "+FLAGS", "(\\SEEN)")
 
 	def is_temporary_system_problem(self, e):
 		messages = (
@@ -381,36 +311,28 @@ class EmailServer:
 				return True
 		return False
 
-	def validate_message_limits(self, message_meta):
-		# throttle based on email size
-		if not self.max_email_size:
-			return
+	def make_error_msg(self, uid, msg_num):
+		partial_mail = None
+		traceback = frappe.get_traceback(with_context=True)
+		with suppress(Exception):
+			# retrieve headers
+			if not cint(self.settings.use_imap):
+				headers = b"\n".join(self.pop.top(msg_num, 5)[1])
+			else:
+				headers = self.imap.uid("fetch", uid, "(BODY.PEEK[HEADER])")[1][0][1]
 
-		m, size = message_meta.split()
-		size = cint(size)
+			partial_mail = Email(headers)
 
-		if size < self.max_email_size:
-			self.total_size += size
-			if self.total_size > self.max_total_size:
-				raise TotalSizeExceededError
-		else:
-			raise EmailSizeExceededError
-
-	def make_error_msg(self, msg_num, incoming_mail):
-		error_msg = "Error in retrieving email."
-		if not incoming_mail:
-			try:
-				# retrieve headers
-				incoming_mail = Email(b"\n".join(self.pop.top(msg_num, 5)[1]))
-			except Exception:
-				pass
-
-		if incoming_mail:
-			error_msg += "\nDate: {date}\nFrom: {from_email}\nSubject: {subject}\n".format(
-				date=incoming_mail.date, from_email=incoming_mail.from_email, subject=incoming_mail.subject
+		if partial_mail:
+			return (
+				"\nDate: {date}\nFrom: {from_email}\nSubject: {subject}\n\n\nTraceback: \n{traceback}".format(
+					date=partial_mail.date,
+					from_email=partial_mail.from_email,
+					subject=partial_mail.subject,
+					traceback=traceback,
+				)
 			)
-
-		return error_msg
+		return traceback
 
 	def update_flag(self, folder, uid_list=None):
 		"""set all uids mails the flag as seen"""
@@ -461,7 +383,7 @@ class Email:
 			try:
 				utc = email.utils.mktime_tz(email.utils.parsedate_tz(self.mail["Date"]))
 				utc_dt = datetime.datetime.utcfromtimestamp(utc)
-				self.date = convert_utc_to_user_timezone(utc_dt).strftime("%Y-%m-%d %H:%M:%S")
+				self.date = convert_utc_to_system_timezone(utc_dt).strftime("%Y-%m-%d %H:%M:%S")
 			except Exception:
 				self.date = now()
 		else:
@@ -601,6 +523,9 @@ class Email:
 					fname = get_random_filename(content_type=content_type)
 			else:
 				fname = get_random_filename(content_type=content_type)
+			# Don't clobber existing filename
+			while fname in self.cid_map:
+				fname = get_random_filename(content_type=content_type)
 
 			self.attachments.append(
 				{
@@ -724,6 +649,9 @@ class InboundMail(Email):
 		communication.flags.in_receive = True
 		communication.insert(ignore_permissions=True)
 
+		# Communication might have been modified by some hooks, reload before saving
+		communication.reload()
+
 		# save attachments
 		communication._attachments = self.save_attachments_in_doc(communication)
 		communication.content = sanitize_html(self.replace_inline_images(communication._attachments))
@@ -782,7 +710,7 @@ class InboundMail(Email):
 
 		Here are the cases to handle:
 		1. If mail is a reply to already sent mail, then we can get parent communicaion from
-		        Email Queue record.
+		        Email Queue record or message_id on communication.
 		2. Sometimes we send communication name in message-ID directly, use that to get parent communication.
 		3. Sender sent a reply but reply is on top of what (s)he sent before,
 		        then parent record exists directly in communication.
@@ -795,17 +723,15 @@ class InboundMail(Email):
 		if not self.is_reply():
 			return ""
 
-		if not self.is_reply_to_system_sent_mail():
-			communication = Communication.find_one_by_filters(
-				message_id=self.in_reply_to, creation=[">=", self.get_relative_dt(-30)]
-			)
-		elif self.parent_email_queue() and self.parent_email_queue().communication:
-			communication = Communication.find(self.parent_email_queue().communication, ignore_error=True)
-		else:
-			reference = self.in_reply_to
-			if "@" in self.in_reply_to:
-				reference, _ = self.in_reply_to.split("@", 1)
-			communication = Communication.find(reference, ignore_error=True)
+		communication = Communication.find_one_by_filters(message_id=self.in_reply_to)
+		if not communication:
+			if self.parent_email_queue() and self.parent_email_queue().communication:
+				communication = Communication.find(self.parent_email_queue().communication, ignore_error=True)
+			else:
+				reference = self.in_reply_to
+				if "@" in self.in_reply_to:
+					reference, _ = self.in_reply_to.split("@", 1)
+				communication = Communication.find(reference, ignore_error=True)
 
 		self._parent_communication = communication or ""
 		return self._parent_communication
@@ -981,10 +907,10 @@ class TimerMixin:
 			self.sock.settimeout(self.timeout / 5.0)
 
 	def _getline(self, *args, **kwargs):
-		start_time = time.time()
+		start_time = time.monotonic()
 		ret = self._super._getline(self, *args, **kwargs)
 
-		self.elapsed_time += time.time() - start_time
+		self.elapsed_time += time.monotonic() - start_time
 		if self.timeout and self.elapsed_time > self.timeout:
 			raise EmailTimeoutError
 
diff --git a/frappe/email/smtp.py b/frappe/email/smtp.py
index 10eb2f7681..3b22bc4ce4 100644
--- a/frappe/email/smtp.py
+++ b/frappe/email/smtp.py
@@ -54,9 +54,7 @@ class SMTPServer:
 		use_tls=None,
 		use_ssl=None,
 		use_oauth=0,
-		refresh_token=None,
 		access_token=None,
-		service=None,
 	):
 		self.login = login
 		self.email_account = email_account
@@ -66,16 +64,12 @@ class SMTPServer:
 		self.use_tls = use_tls
 		self.use_ssl = use_ssl
 		self.use_oauth = use_oauth
-		self.refresh_token = refresh_token
 		self.access_token = access_token
-		self.service = service
 		self._session = None
 
 		if not self.server:
 			frappe.msgprint(
-				_(
-					"Email Account not setup. Please create a new Email Account from Setup > Email > Email Account"
-				),
+				_("Email Account not setup. Please create a new Email Account from Settings > Email Account"),
 				raise_exception=frappe.OutgoingEmailError,
 			)
 
@@ -112,9 +106,7 @@ class SMTPServer:
 			self.secure_session(_session)
 
 			if self.use_oauth:
-				Oauth(
-					_session, self.email_account, self.login, self.access_token, self.refresh_token, self.service
-				).connect()
+				Oauth(_session, self.email_account, self.login, self.access_token).connect()
 
 			elif self.password:
 				res = _session.login(str(self.login or ""), str(self.password or ""))
diff --git a/frappe/email/test_email_body.py b/frappe/email/test_email_body.py
index d5b1013a73..b9128bc979 100644
--- a/frappe/email/test_email_body.py
+++ b/frappe/email/test_email_body.py
@@ -3,7 +3,6 @@
 
 import base64
 import os
-import unittest
 
 import frappe
 from frappe import safe_decode
@@ -15,9 +14,10 @@ from frappe.email.email_body import (
 	replace_filename_with_cid,
 )
 from frappe.email.receive import Email
+from frappe.tests.utils import FrappeTestCase
 
 
-class TestEmailBody(unittest.TestCase):
+class TestEmailBody(FrappeTestCase):
 	def setUp(self):
 		email_html = """
 
        
diff --git a/frappe/email/test_smtp.py b/frappe/email/test_smtp.py
index 448f61925c..b101f610a1 100644
--- a/frappe/email/test_smtp.py
+++ b/frappe/email/test_smtp.py
@@ -1,14 +1,13 @@
 # Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors
 # License: The MIT License
 
-import unittest
-
 import frappe
 from frappe.email.doctype.email_account.email_account import EmailAccount
 from frappe.email.smtp import SMTPServer
+from frappe.tests.utils import FrappeTestCase
 
 
-class TestSMTP(unittest.TestCase):
+class TestSMTP(FrappeTestCase):
 	def test_smtp_ssl_session(self):
 		for port in [None, 0, 465, "465"]:
 			make_server(port, 1, 0)
diff --git a/frappe/event_streaming/doctype/document_type_field_mapping/document_type_field_mapping.json b/frappe/event_streaming/doctype/document_type_field_mapping/document_type_field_mapping.json
deleted file mode 100644
index bba0a98237..0000000000
--- a/frappe/event_streaming/doctype/document_type_field_mapping/document_type_field_mapping.json
+++ /dev/null
@@ -1,73 +0,0 @@
-{
- "actions": [],
- "creation": "2019-09-27 12:46:50.165135",
- "doctype": "DocType",
- "editable_grid": 1,
- "engine": "InnoDB",
- "field_order": [
-  "local_fieldname",
-  "mapping_type",
-  "mapping",
-  "remote_value_filters",
-  "column_break_5",
-  "remote_fieldname",
-  "default_value"
- ],
- "fields": [
-  {
-   "fieldname": "remote_fieldname",
-   "fieldtype": "Data",
-   "in_list_view": 1,
-   "label": "Remote Fieldname"
-  },
-  {
-   "fieldname": "local_fieldname",
-   "fieldtype": "Data",
-   "in_list_view": 1,
-   "label": "Local Fieldname",
-   "reqd": 1
-  },
-  {
-   "fieldname": "column_break_5",
-   "fieldtype": "Column Break"
-  },
-  {
-   "fieldname": "default_value",
-   "fieldtype": "Data",
-   "label": "Default Value"
-  },
-  {
-   "fieldname": "mapping_type",
-   "fieldtype": "Select",
-   "label": "Mapping Type",
-   "options": "\nChild Table\nDocument"
-  },
-  {
-   "depends_on": "eval:doc.mapping_type;",
-   "fieldname": "mapping",
-   "fieldtype": "Link",
-   "label": "Mapping",
-   "options": "Document Type Mapping"
-  },
-  {
-   "depends_on": "eval:doc.mapping_type==\"Document\";",
-   "fieldname": "remote_value_filters",
-   "fieldtype": "Code",
-   "label": "Remote Value Filters",
-   "mandatory_depends_on": "eval:doc.mapping_type===\"Document\";",
-   "options": "JSON"
-  }
- ],
- "istable": 1,
- "links": [],
- "modified": "2020-03-19 13:56:36.223799",
- "modified_by": "Administrator",
- "module": "Event Streaming",
- "name": "Document Type Field Mapping",
- "owner": "Administrator",
- "permissions": [],
- "quick_entry": 1,
- "sort_field": "modified",
- "sort_order": "DESC",
- "track_changes": 1
-}
\ No newline at end of file
diff --git a/frappe/event_streaming/doctype/document_type_field_mapping/document_type_field_mapping.py b/frappe/event_streaming/doctype/document_type_field_mapping/document_type_field_mapping.py
deleted file mode 100644
index 96d9e0fcb3..0000000000
--- a/frappe/event_streaming/doctype/document_type_field_mapping/document_type_field_mapping.py
+++ /dev/null
@@ -1,9 +0,0 @@
-# Copyright (c) 2019, Frappe Technologies and contributors
-# License: MIT. See LICENSE
-
-# import frappe
-from frappe.model.document import Document
-
-
-class DocumentTypeFieldMapping(Document):
-	pass
diff --git a/frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.js b/frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.js
deleted file mode 100644
index 22b7f2ef4c..0000000000
--- a/frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.js
+++ /dev/null
@@ -1,28 +0,0 @@
-// Copyright (c) 2019, Frappe Technologies and contributors
-// For license information, please see license.txt
-
-frappe.ui.form.on('Document Type Mapping', {
-	local_doctype: function(frm) {
-		if (frm.doc.local_doctype) {
-			frappe.model.clear_table(frm.doc, 'field_mapping');
-			let fields = frm.events.get_fields(frm);
-			$.each(fields, function(i, data) {
-				let row = frappe.model.add_child(frm.doc, 'Document Type Field Mapping', 'field_mapping');
-				row.local_fieldname = data;
-			});
-			refresh_field('field_mapping');
-		}
-	},
-
-	get_fields: function(frm) {
-		let filtered_fields = [];
-		frappe.model.with_doctype(frm.doc.local_doctype, ()=> {
-			frappe.get_meta(frm.doc.local_doctype).fields.map( field => {
-				if (field.fieldname !== 'remote_docname' && field.fieldname !== 'remote_site_name' && frappe.model.is_value_type(field) && !field.hidden) {
-					filtered_fields.push(field.fieldname);
-				}
-			});
-		});
-		return filtered_fields;
-	}
-});
diff --git a/frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.json b/frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.json
deleted file mode 100644
index 6a59cf3b70..0000000000
--- a/frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.json
+++ /dev/null
@@ -1,71 +0,0 @@
-{
- "autoname": "field:mapping_name",
- "creation": "2019-09-27 12:45:56.529124",
- "doctype": "DocType",
- "editable_grid": 1,
- "engine": "InnoDB",
- "field_order": [
-  "mapping_name",
-  "local_doctype",
-  "remote_doctype",
-  "section_break_3",
-  "field_mapping"
- ],
- "fields": [
-  {
-   "fieldname": "local_doctype",
-   "fieldtype": "Link",
-   "in_list_view": 1,
-   "label": "Local Document Type",
-   "options": "DocType",
-   "reqd": 1
-  },
-  {
-   "fieldname": "remote_doctype",
-   "fieldtype": "Data",
-   "in_list_view": 1,
-   "label": "Remote Document Type",
-   "reqd": 1
-  },
-  {
-   "fieldname": "section_break_3",
-   "fieldtype": "Section Break"
-  },
-  {
-   "fieldname": "field_mapping",
-   "fieldtype": "Table",
-   "label": "Field Mapping",
-   "options": "Document Type Field Mapping"
-  },
-  {
-   "fieldname": "mapping_name",
-   "fieldtype": "Data",
-   "label": "Mapping Name",
-   "reqd": 1,
-   "unique": 1
-  }
- ],
- "modified": "2019-10-09 08:36:04.621397",
- "modified_by": "Administrator",
- "module": "Event Streaming",
- "name": "Document Type Mapping",
- "owner": "Administrator",
- "permissions": [
-  {
-   "create": 1,
-   "delete": 1,
-   "email": 1,
-   "export": 1,
-   "print": 1,
-   "read": 1,
-   "report": 1,
-   "role": "System Manager",
-   "share": 1,
-   "write": 1
-  }
- ],
- "quick_entry": 1,
- "sort_field": "modified",
- "sort_order": "DESC",
- "track_changes": 1
-}
\ No newline at end of file
diff --git a/frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.py b/frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.py
deleted file mode 100644
index 04b5015296..0000000000
--- a/frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.py
+++ /dev/null
@@ -1,181 +0,0 @@
-# Copyright (c) 2019, Frappe Technologies and contributors
-# License: MIT. See LICENSE
-import json
-
-import frappe
-from frappe import _
-from frappe.model import child_table_fields, default_fields
-from frappe.model.document import Document
-
-
-class DocumentTypeMapping(Document):
-	def validate(self):
-		self.validate_inner_mapping()
-
-	def validate_inner_mapping(self):
-		meta = frappe.get_meta(self.local_doctype)
-		for field_map in self.field_mapping:
-			if field_map.local_fieldname not in (default_fields + child_table_fields):
-				field = meta.get_field(field_map.local_fieldname)
-				if not field:
-					frappe.throw(_("Row #{0}: Invalid Local Fieldname").format(field_map.idx))
-
-			fieldtype = field.get("fieldtype")
-			if fieldtype in ["Link", "Dynamic Link", "Table"]:
-				if not field_map.mapping and not field_map.default_value:
-					msg = _(
-						"Row #{0}: Please set Mapping or Default Value for the field {1} since its a dependency field"
-					).format(field_map.idx, frappe.bold(field_map.local_fieldname))
-					frappe.throw(msg, title="Inner Mapping Missing")
-
-			if field_map.mapping_type == "Document" and not field_map.remote_value_filters:
-				msg = _(
-					"Row #{0}: Please set remote value filters for the field {1} to fetch the unique remote dependency document"
-				).format(field_map.idx, frappe.bold(field_map.remote_fieldname))
-				frappe.throw(msg, title="Remote Value Filters Missing")
-
-	def get_mapping(self, doc, producer_site, update_type):
-		remote_fields = []
-		# list of tuples (local_fieldname, dependent_doc)
-		dependencies = []
-
-		for mapping in self.field_mapping:
-			if doc.get(mapping.remote_fieldname):
-				if mapping.mapping_type == "Document":
-					if not mapping.default_value:
-						dependency = self.get_mapped_dependency(mapping, producer_site, doc)
-						if dependency:
-							dependencies.append((mapping.local_fieldname, dependency))
-					else:
-						doc[mapping.local_fieldname] = mapping.default_value
-
-				if mapping.mapping_type == "Child Table" and update_type != "Update":
-					doc[mapping.local_fieldname] = get_mapped_child_table_docs(
-						mapping.mapping, doc[mapping.remote_fieldname], producer_site
-					)
-				else:
-					# copy value into local fieldname key and remove remote fieldname key
-					doc[mapping.local_fieldname] = doc[mapping.remote_fieldname]
-
-				if mapping.local_fieldname != mapping.remote_fieldname:
-					remote_fields.append(mapping.remote_fieldname)
-
-			if not doc.get(mapping.remote_fieldname) and mapping.default_value and update_type != "Update":
-				doc[mapping.local_fieldname] = mapping.default_value
-
-		# remove the remote fieldnames
-		for field in remote_fields:
-			doc.pop(field, None)
-
-		if update_type != "Update":
-			doc["doctype"] = self.local_doctype
-
-		mapping = {"doc": frappe.as_json(doc)}
-		if len(dependencies):
-			mapping["dependencies"] = dependencies
-		return mapping
-
-	def get_mapped_update(self, update, producer_site):
-		update_diff = frappe._dict(json.loads(update.data))
-		mapping = update_diff
-		dependencies = []
-		if update_diff.changed:
-			doc_map = self.get_mapping(update_diff.changed, producer_site, "Update")
-			mapped_doc = doc_map.get("doc")
-			mapping.changed = json.loads(mapped_doc)
-			if doc_map.get("dependencies"):
-				dependencies += doc_map.get("dependencies")
-
-		if update_diff.removed:
-			mapping = self.map_rows_removed(update_diff, mapping)
-		if update_diff.added:
-			mapping = self.map_rows(update_diff, mapping, producer_site, operation="added")
-		if update_diff.row_changed:
-			mapping = self.map_rows(update_diff, mapping, producer_site, operation="row_changed")
-
-		update = {"doc": frappe.as_json(mapping)}
-		if len(dependencies):
-			update["dependencies"] = dependencies
-		return update
-
-	def get_mapped_dependency(self, mapping, producer_site, doc):
-		inner_mapping = frappe.get_doc("Document Type Mapping", mapping.mapping)
-		filters = json.loads(mapping.remote_value_filters)
-		for key, value in filters.items():
-			if value.startswith("eval:"):
-				val = frappe.safe_eval(value[5:], None, dict(doc=doc))
-				filters[key] = val
-			if doc.get(value):
-				filters[key] = doc.get(value)
-		matching_docs = producer_site.get_doc(inner_mapping.remote_doctype, filters=filters)
-		if len(matching_docs):
-			remote_docname = matching_docs[0].get("name")
-			remote_doc = producer_site.get_doc(inner_mapping.remote_doctype, remote_docname)
-			doc = inner_mapping.get_mapping(remote_doc, producer_site, "Insert").get("doc")
-			return doc
-		return
-
-	def map_rows_removed(self, update_diff, mapping):
-		removed = []
-		mapping["removed"] = update_diff.removed
-		for key, value in update_diff.removed.copy().items():
-			local_table_name = frappe.db.get_value(
-				"Document Type Field Mapping",
-				{"remote_fieldname": key, "parent": self.name},
-				"local_fieldname",
-			)
-			mapping.removed[local_table_name] = value
-			if local_table_name != key:
-				removed.append(key)
-
-		# remove the remote fieldnames
-		for field in removed:
-			mapping.removed.pop(field, None)
-		return mapping
-
-	def map_rows(self, update_diff, mapping, producer_site, operation):
-		remote_fields = []
-		for tablename, entries in update_diff.get(operation).copy().items():
-			local_table_name = frappe.db.get_value(
-				"Document Type Field Mapping", {"remote_fieldname": tablename}, "local_fieldname"
-			)
-			table_map = frappe.db.get_value(
-				"Document Type Field Mapping",
-				{"local_fieldname": local_table_name, "parent": self.name},
-				"mapping",
-			)
-			table_map = frappe.get_doc("Document Type Mapping", table_map)
-			docs = []
-			for entry in entries:
-				mapped_doc = table_map.get_mapping(entry, producer_site, "Update").get("doc")
-				docs.append(json.loads(mapped_doc))
-			mapping.get(operation)[local_table_name] = docs
-			if local_table_name != tablename:
-				remote_fields.append(tablename)
-
-		# remove the remote fieldnames
-		for field in remote_fields:
-			mapping.get(operation).pop(field, None)
-
-		return mapping
-
-
-def get_mapped_child_table_docs(child_map, table_entries, producer_site):
-	"""Get mapping for child doctypes"""
-	child_map = frappe.get_doc("Document Type Mapping", child_map)
-	mapped_entries = []
-	remote_fields = []
-	for child_doc in table_entries:
-		for mapping in child_map.field_mapping:
-			if child_doc.get(mapping.remote_fieldname):
-				child_doc[mapping.local_fieldname] = child_doc[mapping.remote_fieldname]
-				if mapping.local_fieldname != mapping.remote_fieldname:
-					child_doc.pop(mapping.remote_fieldname, None)
-		mapped_entries.append(child_doc)
-
-	# remove the remote fieldnames
-	for field in remote_fields:
-		child_doc.pop(field, None)
-
-	child_doc["doctype"] = child_map.local_doctype
-	return mapped_entries
diff --git a/frappe/event_streaming/doctype/document_type_mapping/test_document_type_mapping.py b/frappe/event_streaming/doctype/document_type_mapping/test_document_type_mapping.py
deleted file mode 100644
index 676d5040ff..0000000000
--- a/frappe/event_streaming/doctype/document_type_mapping/test_document_type_mapping.py
+++ /dev/null
@@ -1,8 +0,0 @@
-# Copyright (c) 2019, Frappe Technologies and Contributors
-# License: MIT. See LICENSE
-# import frappe
-import unittest
-
-
-class TestDocumentTypeMapping(unittest.TestCase):
-	pass
diff --git a/frappe/event_streaming/doctype/event_consumer/event_consumer.js b/frappe/event_streaming/doctype/event_consumer/event_consumer.js
deleted file mode 100644
index 66d92699fa..0000000000
--- a/frappe/event_streaming/doctype/event_consumer/event_consumer.js
+++ /dev/null
@@ -1,19 +0,0 @@
-// Copyright (c) 2019, Frappe Technologies and contributors
-// For license information, please see license.txt
-
-frappe.ui.form.on('Event Consumer', {
-	refresh: function(frm) {
-		// formatter for subscribed doctype approval status
-		frm.set_indicator_formatter('status',
-			function(doc) {
-				let indicator = 'orange';
-				if (doc.status == 'Approved') {
-					indicator = 'green';
-				} else if (doc.status == 'Rejected') {
-					indicator = 'red';
-				}
-				return indicator;
-			}
-		);
-	}
-});
diff --git a/frappe/event_streaming/doctype/event_consumer/event_consumer.json b/frappe/event_streaming/doctype/event_consumer/event_consumer.json
deleted file mode 100644
index 42b47ce949..0000000000
--- a/frappe/event_streaming/doctype/event_consumer/event_consumer.json
+++ /dev/null
@@ -1,97 +0,0 @@
-{
- "actions": [],
- "autoname": "field:callback_url",
- "creation": "2019-08-26 17:45:15.479530",
- "doctype": "DocType",
- "editable_grid": 1,
- "engine": "InnoDB",
- "field_order": [
-  "consumer_doctypes",
-  "callback_url",
-  "section_break_3",
-  "api_key",
-  "api_secret",
-  "column_break_6",
-  "user",
-  "incoming_change"
- ],
- "fields": [
-  {
-   "fieldname": "callback_url",
-   "fieldtype": "Data",
-   "in_list_view": 1,
-   "label": "Callback URL",
-   "read_only": 1,
-   "reqd": 1,
-   "unique": 1
-  },
-  {
-   "fieldname": "api_key",
-   "fieldtype": "Data",
-   "label": "API Key",
-   "reqd": 1
-  },
-  {
-   "fieldname": "api_secret",
-   "fieldtype": "Password",
-   "label": "API Secret",
-   "reqd": 1
-  },
-  {
-   "fieldname": "user",
-   "fieldtype": "Link",
-   "label": "Event Subscriber",
-   "options": "User",
-   "read_only": 1,
-   "reqd": 1
-  },
-  {
-   "fieldname": "section_break_3",
-   "fieldtype": "Section Break"
-  },
-  {
-   "fieldname": "column_break_6",
-   "fieldtype": "Column Break"
-  },
-  {
-   "default": "0",
-   "fieldname": "incoming_change",
-   "fieldtype": "Check",
-   "hidden": 1,
-   "label": "Incoming Change",
-   "read_only": 1
-  },
-  {
-   "fieldname": "consumer_doctypes",
-   "fieldtype": "Table",
-   "label": "Event Consumer Document Types",
-   "options": "Event Consumer Document Type",
-   "reqd": 1
-  }
- ],
- "in_create": 1,
- "links": [],
- "modified": "2020-09-08 16:42:39.828085",
- "modified_by": "Administrator",
- "module": "Event Streaming",
- "name": "Event Consumer",
- "owner": "Administrator",
- "permissions": [
-  {
-   "create": 1,
-   "delete": 1,
-   "email": 1,
-   "export": 1,
-   "print": 1,
-   "read": 1,
-   "report": 1,
-   "role": "System Manager",
-   "share": 1,
-   "write": 1
-  }
- ],
- "quick_entry": 1,
- "sort_field": "modified",
- "sort_order": "DESC",
- "track_changes": 1
-}
\ No newline at end of file
diff --git a/frappe/event_streaming/doctype/event_consumer/event_consumer.py b/frappe/event_streaming/doctype/event_consumer/event_consumer.py
deleted file mode 100644
index a2ae6f6651..0000000000
--- a/frappe/event_streaming/doctype/event_consumer/event_consumer.py
+++ /dev/null
@@ -1,216 +0,0 @@
-# Copyright (c) 2019, Frappe Technologies and contributors
-# License: MIT. See LICENSE
-
-import json
-import os
-
-import requests
-
-import frappe
-from frappe import _
-from frappe.frappeclient import FrappeClient
-from frappe.model.document import Document
-from frappe.utils.background_jobs import get_jobs
-from frappe.utils.data import get_url
-
-
-class EventConsumer(Document):
-	def validate(self):
-		# approve subscribed doctypes for tests
-		# frappe.flags.in_test won't work here as tests are running on the consumer site
-		if os.environ.get("CI"):
-			for entry in self.consumer_doctypes:
-				entry.status = "Approved"
-
-	def on_update(self):
-		if not self.incoming_change:
-			doc_before_save = self.get_doc_before_save()
-			if doc_before_save.api_key != self.api_key or doc_before_save.api_secret != self.api_secret:
-				return
-
-			self.update_consumer_status()
-		else:
-			frappe.db.set_value(self.doctype, self.name, "incoming_change", 0)
-
-		frappe.cache().delete_value("event_consumer_document_type_map")
-
-	def on_trash(self):
-		for i in frappe.get_all("Event Update Log Consumer", {"consumer": self.name}):
-			frappe.delete_doc("Event Update Log Consumer", i.name)
-		frappe.cache().delete_value("event_consumer_document_type_map")
-
-	def update_consumer_status(self):
-		consumer_site = get_consumer_site(self.callback_url)
-		event_producer = consumer_site.get_doc("Event Producer", get_url())
-		event_producer = frappe._dict(event_producer)
-		config = event_producer.producer_doctypes
-		event_producer.producer_doctypes = []
-		for entry in config:
-			if entry.get("has_mapping"):
-				ref_doctype = consumer_site.get_value(
-					"Document Type Mapping", "remote_doctype", entry.get("mapping")
-				).get("remote_doctype")
-			else:
-				ref_doctype = entry.get("ref_doctype")
-
-			entry["status"] = frappe.db.get_value(
-				"Event Consumer Document Type", {"parent": self.name, "ref_doctype": ref_doctype}, "status"
-			)
-
-		event_producer.producer_doctypes = config
-		# when producer doc is updated it updates the consumer doc
-		# set flag to avoid deadlock
-		event_producer.incoming_change = True
-		consumer_site.update(event_producer)
-
-	def get_consumer_status(self):
-		response = requests.get(self.callback_url)
-		if response.status_code != 200:
-			return "offline"
-		return "online"
-
-
-@frappe.whitelist()
-def register_consumer(data):
-	"""create an event consumer document for registering a consumer"""
-	data = json.loads(data)
-	# to ensure that consumer is created only once
-	if frappe.db.exists("Event Consumer", data["event_consumer"]):
-		return None
-
-	user = data["user"]
-	if not frappe.db.exists("User", user):
-		frappe.throw(_("User {0} not found on the producer site").format(user))
-
-	if "System Manager" not in frappe.get_roles(user):
-		frappe.throw(_("Event Subscriber has to be a System Manager."))
-
-	consumer = frappe.new_doc("Event Consumer")
-	consumer.callback_url = data["event_consumer"]
-	consumer.user = data["user"]
-	consumer.api_key = data["api_key"]
-	consumer.api_secret = data["api_secret"]
-	consumer.incoming_change = True
-	consumer_doctypes = json.loads(data["consumer_doctypes"])
-
-	for entry in consumer_doctypes:
-		consumer.append(
-			"consumer_doctypes",
-			{"ref_doctype": entry.get("doctype"), "status": "Pending", "condition": entry.get("condition")},
-		)
-
-	consumer.insert()
-
-	# consumer's 'last_update' field should point to the latest update
-	# in producer's update log when subscribing
-	# so that, updates after subscribing are consumed and not the old ones.
-	last_update = str(get_last_update())
-	return json.dumps({"last_update": last_update})
-
-
-def get_consumer_site(consumer_url):
-	"""create a FrappeClient object for event consumer site"""
-	consumer_doc = frappe.get_doc("Event Consumer", consumer_url)
-	consumer_site = FrappeClient(
-		url=consumer_url,
-		api_key=consumer_doc.api_key,
-		api_secret=consumer_doc.get_password("api_secret"),
-	)
-	return consumer_site
-
-
-def get_last_update():
-	"""get the creation timestamp of last update consumed"""
-	updates = frappe.get_list(
-		"Event Update Log", "creation", ignore_permissions=True, limit=1, order_by="creation desc"
-	)
-	if updates:
-		return updates[0].creation
-	return frappe.utils.now_datetime()
-
-
-@frappe.whitelist()
-def notify_event_consumers(doctype):
-	"""get all event consumers and set flag for notification status"""
-	event_consumers = frappe.get_all(
-		"Event Consumer Document Type", ["parent"], {"ref_doctype": doctype, "status": "Approved"}
-	)
-	for entry in event_consumers:
-		consumer = frappe.get_doc("Event Consumer", entry.parent)
-		consumer.flags.notified = False
-		notify(consumer)
-
-
-@frappe.whitelist()
-def notify(consumer):
-	"""notify individual event consumers about a new update"""
-	consumer_status = consumer.get_consumer_status()
-	if consumer_status == "online":
-		try:
-			client = get_consumer_site(consumer.callback_url)
-			client.post_request(
-				{
-					"cmd": "frappe.event_streaming.doctype.event_producer.event_producer.new_event_notification",
-					"producer_url": get_url(),
-				}
-			)
-			consumer.flags.notified = True
-		except Exception:
-			consumer.flags.notified = False
-	else:
-		consumer.flags.notified = False
-
-	# enqueue another job if the site was not notified
-	if not consumer.flags.notified:
-		enqueued_method = "frappe.event_streaming.doctype.event_consumer.event_consumer.notify"
-		jobs = get_jobs()
-		if not jobs or enqueued_method not in jobs[frappe.local.site] and not consumer.flags.notifed:
-			frappe.enqueue(
-				enqueued_method, queue="long", enqueue_after_commit=True, **{"consumer": consumer}
-			)
-
-
-def has_consumer_access(consumer, update_log):
-	"""Checks if consumer has completely satisfied all the conditions on the doc"""
-
-	if isinstance(consumer, str):
-		consumer = frappe.get_doc("Event Consumer", consumer)
-
-	if not frappe.db.exists(update_log.ref_doctype, update_log.docname):
-		# Delete Log
-		# Check if the last Update Log of this document was read by this consumer
-		last_update_log = frappe.get_all(
-			"Event Update Log",
-			filters={
-				"ref_doctype": update_log.ref_doctype,
-				"docname": update_log.docname,
-				"creation": ["<", update_log.creation],
-			},
-			order_by="creation desc",
-			limit_page_length=1,
-		)
-		if not len(last_update_log):
-			return False
-
-		last_update_log = frappe.get_doc("Event Update Log", last_update_log[0].name)
-		return len([x for x in last_update_log.consumers if x.consumer == consumer.name])
-
-	doc = frappe.get_doc(update_log.ref_doctype, update_log.docname)
-	try:
-		for dt_entry in consumer.consumer_doctypes:
-			if dt_entry.ref_doctype != update_log.ref_doctype:
-				continue
-
-			if not dt_entry.condition:
-				return True
-
-			condition: str = dt_entry.condition
-			if condition.startswith("cmd:"):
-				cmd = condition.split("cmd:")[1].strip()
-				args = {"consumer": consumer, "doc": doc, "update_log": update_log}
-				return frappe.call(cmd, **args)
-			else:
-				return frappe.safe_eval(condition, frappe._dict(doc=doc))
-	except Exception as e:
-		consumer.log_error("has_consumer_access error")
-	return False
diff --git a/frappe/event_streaming/doctype/event_consumer/test_event_consumer.py b/frappe/event_streaming/doctype/event_consumer/test_event_consumer.py
deleted file mode 100644
index 6f04af643e..0000000000
--- a/frappe/event_streaming/doctype/event_consumer/test_event_consumer.py
+++ /dev/null
@@ -1,8 +0,0 @@
-# Copyright (c) 2019, Frappe Technologies and Contributors
-# License: MIT. See LICENSE
-# import frappe
-import unittest
-
-
-class TestEventConsumer(unittest.TestCase):
-	pass
diff --git a/frappe/event_streaming/doctype/event_consumer_document_type/event_consumer_document_type.json b/frappe/event_streaming/doctype/event_consumer_document_type/event_consumer_document_type.json
deleted file mode 100644
index c243334a09..0000000000
--- a/frappe/event_streaming/doctype/event_consumer_document_type/event_consumer_document_type.json
+++ /dev/null
@@ -1,61 +0,0 @@
-{
- "actions": [],
- "creation": "2019-10-03 21:10:54.754651",
- "doctype": "DocType",
- "editable_grid": 1,
- "engine": "InnoDB",
- "field_order": [
-  "ref_doctype",
-  "status",
-  "unsubscribed",
-  "condition"
- ],
- "fields": [
-  {
-   "columns": 4,
-   "fieldname": "ref_doctype",
-   "fieldtype": "Link",
-   "in_list_view": 1,
-   "label": "Document Type",
-   "options": "DocType",
-   "read_only": 1,
-   "reqd": 1
-  },
-  {
-   "columns": 4,
-   "default": "Pending",
-   "fieldname": "status",
-   "fieldtype": "Select",
-   "in_list_view": 1,
-   "label": "Approval Status",
-   "options": "Pending\nApproved\nRejected"
-  },
-  {
-   "columns": 2,
-   "default": "0",
-   "fieldname": "unsubscribed",
-   "fieldtype": "Check",
-   "in_list_view": 1,
-   "label": "Unsubscribed",
-   "read_only": 1
-  },
-  {
-   "fieldname": "condition",
-   "fieldtype": "Code",
-   "label": "Condition",
-   "read_only": 1
-  }
- ],
- "istable": 1,
- "links": [],
- "modified": "2020-11-07 09:26:49.894294",
- "modified_by": "Administrator",
- "module": "Event Streaming",
- "name": "Event Consumer Document Type",
- "owner": "Administrator",
- "permissions": [],
- "quick_entry": 1,
- "sort_field": "modified",
- "sort_order": "DESC",
- "track_changes": 1
-}
\ No newline at end of file
diff --git a/frappe/event_streaming/doctype/event_consumer_document_type/event_consumer_document_type.py b/frappe/event_streaming/doctype/event_consumer_document_type/event_consumer_document_type.py
deleted file mode 100644
index 1ed15c5a75..0000000000
--- a/frappe/event_streaming/doctype/event_consumer_document_type/event_consumer_document_type.py
+++ /dev/null
@@ -1,9 +0,0 @@
-# Copyright (c) 2019, Frappe Technologies and contributors
-# License: MIT. See LICENSE
-
-# import frappe
-from frappe.model.document import Document
-
-
-class EventConsumerDocumentType(Document):
-	pass
diff --git a/frappe/event_streaming/doctype/event_producer/event_producer.js b/frappe/event_streaming/doctype/event_producer/event_producer.js
deleted file mode 100644
index c2c3389e92..0000000000
--- a/frappe/event_streaming/doctype/event_producer/event_producer.js
+++ /dev/null
@@ -1,27 +0,0 @@
-// Copyright (c) 2019, Frappe Technologies and contributors
-// For license information, please see license.txt
-
-frappe.ui.form.on('Event Producer', {
-	refresh: function(frm) {
-		frm.set_query('ref_doctype', 'producer_doctypes', function() {
-			return {
-				filters: {
-					issingle: 0,
-					istable: 0
-				}
-			};
-		});
-
-		frm.set_indicator_formatter('status',
-			function(doc) {
-				let indicator = 'orange';
-				if (doc.status == 'Approved') {
-					indicator = 'green';
-				} else if (doc.status == 'Rejected') {
-					indicator = 'red';
-				}
-				return indicator;
-			}
-		);
-	}
-});
diff --git a/frappe/event_streaming/doctype/event_producer/event_producer.json b/frappe/event_streaming/doctype/event_producer/event_producer.json
deleted file mode 100644
index d868f6c123..0000000000
--- a/frappe/event_streaming/doctype/event_producer/event_producer.json
+++ /dev/null
@@ -1,96 +0,0 @@
-{
- "actions": [],
- "autoname": "field:producer_url",
- "creation": "2019-08-26 19:17:24.919196",
- "doctype": "DocType",
- "editable_grid": 1,
- "engine": "InnoDB",
- "field_order": [
-  "producer_url",
-  "producer_doctypes",
-  "section_break_3",
-  "api_key",
-  "api_secret",
-  "column_break_6",
-  "user",
-  "incoming_change"
- ],
- "fields": [
-  {
-   "fieldname": "producer_url",
-   "fieldtype": "Data",
-   "in_list_view": 1,
-   "label": "Producer URL",
-   "reqd": 1,
-   "unique": 1
-  },
-  {
-   "description": "API Key of the user(Event Subscriber) on the producer site",
-   "fieldname": "api_key",
-   "fieldtype": "Data",
-   "label": "API Key",
-   "reqd": 1
-  },
-  {
-   "description": "API Secret of the user(Event Subscriber) on the producer site",
-   "fieldname": "api_secret",
-   "fieldtype": "Password",
-   "label": "API Secret",
-   "reqd": 1
-  },
-  {
-   "fieldname": "user",
-   "fieldtype": "Link",
-   "label": "Event Subscriber",
-   "options": "User",
-   "reqd": 1,
-   "set_only_once": 1
-  },
-  {
-   "fieldname": "column_break_6",
-   "fieldtype": "Column Break"
-  },
-  {
-   "fieldname": "section_break_3",
-   "fieldtype": "Section Break"
-  },
-  {
-   "default": "0",
-   "fieldname": "incoming_change",
-   "fieldtype": "Check",
-   "hidden": 1,
-   "label": "Incoming Change"
-  },
-  {
-   "fieldname": "producer_doctypes",
-   "fieldtype": "Table",
-   "label": "Event Producer Document Types",
-   "options": "Event Producer Document Type",
-   "reqd": 1
-  }
- ],
- "links": [],
- "modified": "2020-10-26 13:00:15.361316",
- "modified_by": "Administrator",
- "module": "Event Streaming",
- "name": "Event Producer",
- "owner": "Administrator",
- "permissions": [
-  {
-   "create": 1,
-   "delete": 1,
-   "email": 1,
-   "export": 1,
-   "print": 1,
-   "read": 1,
-   "report": 1,
-   "role": "System Manager",
-   "share": 1,
-   "write": 1
-  }
- ],
- "quick_entry": 1,
- "sort_field": "modified",
- "sort_order": "DESC",
- "track_changes": 1
-}
\ No newline at end of file
diff --git a/frappe/event_streaming/doctype/event_producer/event_producer.py b/frappe/event_streaming/doctype/event_producer/event_producer.py
deleted file mode 100644
index f91c8a4fd4..0000000000
--- a/frappe/event_streaming/doctype/event_producer/event_producer.py
+++ /dev/null
@@ -1,569 +0,0 @@
-# Copyright (c) 2019, Frappe Technologies and contributors
-# License: MIT. See LICENSE
-
-import json
-import time
-
-import requests
-
-import frappe
-from frappe import _
-from frappe.custom.doctype.custom_field.custom_field import create_custom_field
-from frappe.frappeclient import FrappeClient
-from frappe.model.document import Document
-from frappe.utils.background_jobs import get_jobs
-from frappe.utils.data import get_link_to_form, get_url
-from frappe.utils.password import get_decrypted_password
-
-
-class EventProducer(Document):
-	def before_insert(self):
-		self.check_url()
-		self.validate_event_subscriber()
-		self.incoming_change = True
-		self.create_event_consumer()
-		self.create_custom_fields()
-
-	def validate(self):
-		self.validate_event_subscriber()
-		if frappe.flags.in_test:
-			for entry in self.producer_doctypes:
-				entry.status = "Approved"
-
-	def validate_event_subscriber(self):
-		if not frappe.db.get_value("User", self.user, "api_key"):
-			frappe.throw(
-				_("Please generate keys for the Event Subscriber User {0} first.").format(
-					frappe.bold(get_link_to_form("User", self.user))
-				)
-			)
-
-	def on_update(self):
-		if not self.incoming_change:
-			if frappe.db.exists("Event Producer", self.name):
-				if not self.api_key or not self.api_secret:
-					frappe.throw(_("Please set API Key and Secret on the producer and consumer sites first."))
-				else:
-					doc_before_save = self.get_doc_before_save()
-					if doc_before_save.api_key != self.api_key or doc_before_save.api_secret != self.api_secret:
-						return
-
-					self.update_event_consumer()
-					self.create_custom_fields()
-		else:
-			# when producer doc is updated it updates the consumer doc, set flag to avoid deadlock
-			self.db_set("incoming_change", 0)
-			self.reload()
-
-	def on_trash(self):
-		last_update = frappe.db.get_value("Event Producer Last Update", dict(event_producer=self.name))
-		if last_update:
-			frappe.delete_doc("Event Producer Last Update", last_update)
-
-	def check_url(self):
-		valid_url_schemes = ("http", "https")
-		frappe.utils.validate_url(self.producer_url, throw=True, valid_schemes=valid_url_schemes)
-
-		# remove '/' from the end of the url like http://test_site.com/
-		# to prevent mismatch in get_url() results
-		if self.producer_url.endswith("/"):
-			self.producer_url = self.producer_url[:-1]
-
-	def create_event_consumer(self):
-		"""register event consumer on the producer site"""
-		if self.is_producer_online():
-			producer_site = FrappeClient(
-				url=self.producer_url, api_key=self.api_key, api_secret=self.get_password("api_secret")
-			)
-
-			response = producer_site.post_api(
-				"frappe.event_streaming.doctype.event_consumer.event_consumer.register_consumer",
-				params={"data": json.dumps(self.get_request_data())},
-			)
-			if response:
-				response = json.loads(response)
-				self.set_last_update(response["last_update"])
-			else:
-				frappe.throw(
-					_(
-						"Failed to create an Event Consumer or an Event Consumer for the current site is already registered."
-					)
-				)
-
-	def set_last_update(self, last_update):
-		last_update_doc_name = frappe.db.get_value(
-			"Event Producer Last Update", dict(event_producer=self.name)
-		)
-		if not last_update_doc_name:
-			frappe.get_doc(
-				dict(
-					doctype="Event Producer Last Update",
-					event_producer=self.producer_url,
-					last_update=last_update,
-				)
-			).insert(ignore_permissions=True)
-		else:
-			frappe.db.set_value(
-				"Event Producer Last Update", last_update_doc_name, "last_update", last_update
-			)
-
-	def get_last_update(self):
-		return frappe.db.get_value(
-			"Event Producer Last Update", dict(event_producer=self.name), "last_update"
-		)
-
-	def get_request_data(self):
-		consumer_doctypes = []
-		for entry in self.producer_doctypes:
-			if entry.has_mapping:
-				# if mapping, subscribe to remote doctype on consumer's site
-				dt = frappe.db.get_value("Document Type Mapping", entry.mapping, "remote_doctype")
-			else:
-				dt = entry.ref_doctype
-			consumer_doctypes.append({"doctype": dt, "condition": entry.condition})
-
-		user_key = frappe.db.get_value("User", self.user, "api_key")
-		user_secret = get_decrypted_password("User", self.user, "api_secret")
-		return {
-			"event_consumer": get_url(),
-			"consumer_doctypes": json.dumps(consumer_doctypes),
-			"user": self.user,
-			"api_key": user_key,
-			"api_secret": user_secret,
-		}
-
-	def create_custom_fields(self):
-		"""create custom field to store remote docname and remote site url"""
-		for entry in self.producer_doctypes:
-			if not entry.use_same_name:
-				if not frappe.db.exists(
-					"Custom Field", {"fieldname": "remote_docname", "dt": entry.ref_doctype}
-				):
-					df = dict(
-						fieldname="remote_docname",
-						label="Remote Document Name",
-						fieldtype="Data",
-						read_only=1,
-						print_hide=1,
-					)
-					create_custom_field(entry.ref_doctype, df)
-				if not frappe.db.exists(
-					"Custom Field", {"fieldname": "remote_site_name", "dt": entry.ref_doctype}
-				):
-					df = dict(
-						fieldname="remote_site_name",
-						label="Remote Site",
-						fieldtype="Data",
-						read_only=1,
-						print_hide=1,
-					)
-					create_custom_field(entry.ref_doctype, df)
-
-	def update_event_consumer(self):
-		if self.is_producer_online():
-			producer_site = get_producer_site(self.producer_url)
-			event_consumer = producer_site.get_doc("Event Consumer", get_url())
-			event_consumer = frappe._dict(event_consumer)
-			if event_consumer:
-				config = event_consumer.consumer_doctypes
-				event_consumer.consumer_doctypes = []
-				for entry in self.producer_doctypes:
-					if entry.has_mapping:
-						# if mapping, subscribe to remote doctype on consumer's site
-						ref_doctype = frappe.db.get_value("Document Type Mapping", entry.mapping, "remote_doctype")
-					else:
-						ref_doctype = entry.ref_doctype
-
-					event_consumer.consumer_doctypes.append(
-						{
-							"ref_doctype": ref_doctype,
-							"status": get_approval_status(config, ref_doctype),
-							"unsubscribed": entry.unsubscribe,
-							"condition": entry.condition,
-						}
-					)
-				event_consumer.user = self.user
-				event_consumer.incoming_change = True
-				producer_site.update(event_consumer)
-
-	def is_producer_online(self):
-		"""check connection status for the Event Producer site"""
-		retry = 3
-		while retry > 0:
-			res = requests.get(self.producer_url)
-			if res.status_code == 200:
-				return True
-			retry -= 1
-			time.sleep(5)
-		frappe.throw(_("Failed to connect to the Event Producer site. Retry after some time."))
-
-
-def get_producer_site(producer_url):
-	"""create a FrappeClient object for event producer site"""
-	producer_doc = frappe.get_doc("Event Producer", producer_url)
-	producer_site = FrappeClient(
-		url=producer_url,
-		api_key=producer_doc.api_key,
-		api_secret=producer_doc.get_password("api_secret"),
-	)
-	return producer_site
-
-
-def get_approval_status(config, ref_doctype):
-	"""check the approval status for consumption"""
-	for entry in config:
-		if entry.get("ref_doctype") == ref_doctype:
-			return entry.get("status")
-	return "Pending"
-
-
-@frappe.whitelist()
-def pull_producer_data():
-	"""Fetch data from producer node."""
-	response = requests.get(get_url())
-	if response.status_code == 200:
-		for event_producer in frappe.get_all("Event Producer"):
-			pull_from_node(event_producer.name)
-		return "success"
-	return None
-
-
-@frappe.whitelist()
-def pull_from_node(event_producer):
-	"""pull all updates after the last update timestamp from event producer site"""
-	event_producer = frappe.get_doc("Event Producer", event_producer)
-	producer_site = get_producer_site(event_producer.producer_url)
-	last_update = event_producer.get_last_update()
-
-	(doctypes, mapping_config, naming_config) = get_config(event_producer.producer_doctypes)
-
-	updates = get_updates(producer_site, last_update, doctypes)
-
-	for update in updates:
-		update.use_same_name = naming_config.get(update.ref_doctype)
-		mapping = mapping_config.get(update.ref_doctype)
-		if mapping:
-			update.mapping = mapping
-			update = get_mapped_update(update, producer_site)
-		if not update.update_type == "Delete":
-			update.data = json.loads(update.data)
-
-		sync(update, producer_site, event_producer)
-
-
-def get_config(event_config):
-	"""get the doctype mapping and naming configurations for consumption"""
-	doctypes, mapping_config, naming_config = [], {}, {}
-
-	for entry in event_config:
-		if entry.status == "Approved":
-			if entry.has_mapping:
-				(mapped_doctype, mapping) = frappe.db.get_value(
-					"Document Type Mapping", entry.mapping, ["remote_doctype", "name"]
-				)
-				mapping_config[mapped_doctype] = mapping
-				naming_config[mapped_doctype] = entry.use_same_name
-				doctypes.append(mapped_doctype)
-			else:
-				naming_config[entry.ref_doctype] = entry.use_same_name
-				doctypes.append(entry.ref_doctype)
-	return (doctypes, mapping_config, naming_config)
-
-
-def sync(update, producer_site, event_producer, in_retry=False):
-	"""Sync the individual update"""
-	try:
-		if update.update_type == "Create":
-			set_insert(update, producer_site, event_producer.name)
-		if update.update_type == "Update":
-			set_update(update, producer_site)
-		if update.update_type == "Delete":
-			set_delete(update)
-		if in_retry:
-			return "Synced"
-		log_event_sync(update, event_producer.name, "Synced")
-
-	except Exception:
-		if in_retry:
-			if frappe.flags.in_test:
-				print(frappe.get_traceback())
-			return "Failed"
-		log_event_sync(update, event_producer.name, "Failed", frappe.get_traceback())
-
-	event_producer.set_last_update(update.creation)
-	frappe.db.commit()
-
-
-def set_insert(update, producer_site, event_producer):
-	"""Sync insert type update"""
-	if frappe.db.get_value(update.ref_doctype, update.docname):
-		# doc already created
-		return
-	doc = frappe.get_doc(update.data)
-
-	if update.mapping:
-		if update.get("dependencies"):
-			dependencies_created = sync_mapped_dependencies(update.dependencies, producer_site)
-			for fieldname, value in dependencies_created.items():
-				doc.update({fieldname: value})
-	else:
-		sync_dependencies(doc, producer_site)
-
-	if update.use_same_name:
-		doc.insert(set_name=update.docname, set_child_names=False)
-	else:
-		# if event consumer is not saving documents with the same name as the producer
-		# store the remote docname in a custom field for future updates
-		doc.remote_docname = update.docname
-		doc.remote_site_name = event_producer
-		doc.insert(set_child_names=False)
-
-
-def set_update(update, producer_site):
-	"""Sync update type update"""
-	local_doc = get_local_doc(update)
-	if local_doc:
-		data = frappe._dict(update.data)
-
-		if data.changed:
-			local_doc.update(data.changed)
-		if data.removed:
-			local_doc = update_row_removed(local_doc, data.removed)
-		if data.row_changed:
-			update_row_changed(local_doc, data.row_changed)
-		if data.added:
-			local_doc = update_row_added(local_doc, data.added)
-
-		if update.mapping:
-			if update.get("dependencies"):
-				dependencies_created = sync_mapped_dependencies(update.dependencies, producer_site)
-				for fieldname, value in dependencies_created.items():
-					local_doc.update({fieldname: value})
-		else:
-			sync_dependencies(local_doc, producer_site)
-
-		local_doc.save()
-		local_doc.db_update_all()
-
-
-def update_row_removed(local_doc, removed):
-	"""Sync child table row deletion type update"""
-	for tablename, rownames in removed.items():
-		table = local_doc.get_table_field_doctype(tablename)
-		for row in rownames:
-			table_rows = local_doc.get(tablename)
-			child_table_row = get_child_table_row(table_rows, row)
-			table_rows.remove(child_table_row)
-			local_doc.set(tablename, table_rows)
-	return local_doc
-
-
-def get_child_table_row(table_rows, row):
-	for entry in table_rows:
-		if entry.get("name") == row:
-			return entry
-
-
-def update_row_changed(local_doc, changed):
-	"""Sync child table row updation type update"""
-	for tablename, rows in changed.items():
-		old = local_doc.get(tablename)
-		for doc in old:
-			for row in rows:
-				if row["name"] == doc.get("name"):
-					doc.update(row)
-
-
-def update_row_added(local_doc, added):
-	"""Sync child table row addition type update"""
-	for tablename, rows in added.items():
-		local_doc.extend(tablename, rows)
-		for child in rows:
-			child_doc = frappe.get_doc(child)
-			child_doc.parent = local_doc.name
-			child_doc.parenttype = local_doc.doctype
-			child_doc.insert(set_name=child_doc.name)
-	return local_doc
-
-
-def set_delete(update):
-	"""Sync delete type update"""
-	local_doc = get_local_doc(update)
-	if local_doc:
-		local_doc.delete()
-
-
-def get_updates(producer_site, last_update, doctypes):
-	"""Get all updates generated after the last update timestamp"""
-	docs = producer_site.post_request(
-		{
-			"cmd": "frappe.event_streaming.doctype.event_update_log.event_update_log.get_update_logs_for_consumer",
-			"event_consumer": get_url(),
-			"doctypes": frappe.as_json(doctypes),
-			"last_update": last_update,
-		}
-	)
-	return [frappe._dict(d) for d in (docs or [])]
-
-
-def get_local_doc(update):
-	"""Get the local document if created with a different name"""
-	try:
-		if not update.use_same_name:
-			return frappe.get_doc(update.ref_doctype, {"remote_docname": update.docname})
-		return frappe.get_doc(update.ref_doctype, update.docname)
-	except frappe.DoesNotExistError:
-		return None
-
-
-def sync_dependencies(document, producer_site):
-	"""
-	dependencies is a dictionary to store all the docs
-	having dependencies and their sync status,
-	which is shared among all nested functions.
-	"""
-	dependencies = {document: True}
-
-	def check_doc_has_dependencies(doc, producer_site):
-		"""Sync child table link fields first,
-		then sync link fields,
-		then dynamic links"""
-		meta = frappe.get_meta(doc.doctype)
-		table_fields = meta.get_table_fields()
-		link_fields = meta.get_link_fields()
-		dl_fields = meta.get_dynamic_link_fields()
-		if table_fields:
-			sync_child_table_dependencies(doc, table_fields, producer_site)
-		if link_fields:
-			sync_link_dependencies(doc, link_fields, producer_site)
-		if dl_fields:
-			sync_dynamic_link_dependencies(doc, dl_fields, producer_site)
-
-	def sync_child_table_dependencies(doc, table_fields, producer_site):
-		for df in table_fields:
-			child_table = doc.get(df.fieldname)
-			for entry in child_table:
-				child_doc = producer_site.get_doc(entry.doctype, entry.name)
-				if child_doc:
-					child_doc = frappe._dict(child_doc)
-					set_dependencies(child_doc, frappe.get_meta(entry.doctype).get_link_fields(), producer_site)
-
-	def sync_link_dependencies(doc, link_fields, producer_site):
-		set_dependencies(doc, link_fields, producer_site)
-
-	def sync_dynamic_link_dependencies(doc, dl_fields, producer_site):
-		for df in dl_fields:
-			docname = doc.get(df.fieldname)
-			linked_doctype = doc.get(df.options)
-			if docname and not check_dependency_fulfilled(linked_doctype, docname):
-				master_doc = producer_site.get_doc(linked_doctype, docname)
-				frappe.get_doc(master_doc).insert(set_name=docname)
-
-	def set_dependencies(doc, link_fields, producer_site):
-		for df in link_fields:
-			docname = doc.get(df.fieldname)
-			linked_doctype = df.get_link_doctype()
-			if docname and not check_dependency_fulfilled(linked_doctype, docname):
-				master_doc = producer_site.get_doc(linked_doctype, docname)
-				try:
-					master_doc = frappe.get_doc(master_doc)
-					master_doc.insert(set_name=docname)
-					frappe.db.commit()
-
-				# for dependency inside a dependency
-				except Exception:
-					dependencies[master_doc] = True
-
-	def check_dependency_fulfilled(linked_doctype, docname):
-		return frappe.db.exists(linked_doctype, docname)
-
-	while dependencies[document]:
-		# find the first non synced dependency
-		for item in reversed(list(dependencies.keys())):
-			if dependencies[item]:
-				dependency = item
-				break
-
-		check_doc_has_dependencies(dependency, producer_site)
-
-		# mark synced for nested dependency
-		if dependency != document:
-			dependencies[dependency] = False
-			dependency.insert()
-
-		# no more dependencies left to be synced, the main doc is ready to be synced
-		# end the dependency loop
-		if not any(list(dependencies.values())[1:]):
-			dependencies[document] = False
-
-
-def sync_mapped_dependencies(dependencies, producer_site):
-	dependencies_created = {}
-	for entry in dependencies:
-		doc = frappe._dict(json.loads(entry[1]))
-		docname = frappe.db.exists(doc.doctype, doc.name)
-		if not docname:
-			doc = frappe.get_doc(doc).insert(set_child_names=False)
-			dependencies_created[entry[0]] = doc.name
-		else:
-			dependencies_created[entry[0]] = docname
-
-	return dependencies_created
-
-
-def log_event_sync(update, event_producer, sync_status, error=None):
-	"""Log event update received with the sync_status as Synced or Failed"""
-	doc = frappe.new_doc("Event Sync Log")
-	doc.update_type = update.update_type
-	doc.ref_doctype = update.ref_doctype
-	doc.status = sync_status
-	doc.event_producer = event_producer
-	doc.producer_doc = update.docname
-	doc.data = frappe.as_json(update.data)
-	doc.use_same_name = update.use_same_name
-	doc.mapping = update.mapping if update.mapping else None
-	if update.use_same_name:
-		doc.docname = update.docname
-	else:
-		doc.docname = frappe.db.get_value(update.ref_doctype, {"remote_docname": update.docname}, "name")
-	if error:
-		doc.error = error
-	doc.insert()
-
-
-def get_mapped_update(update, producer_site):
-	"""get the new update document with mapped fields"""
-	mapping = frappe.get_doc("Document Type Mapping", update.mapping)
-	if update.update_type == "Create":
-		doc = frappe._dict(json.loads(update.data))
-		mapped_update = mapping.get_mapping(doc, producer_site, update.update_type)
-		update.data = mapped_update.get("doc")
-		update.dependencies = mapped_update.get("dependencies", None)
-	elif update.update_type == "Update":
-		mapped_update = mapping.get_mapped_update(update, producer_site)
-		update.data = mapped_update.get("doc")
-		update.dependencies = mapped_update.get("dependencies", None)
-
-	update["ref_doctype"] = mapping.local_doctype
-	return update
-
-
-@frappe.whitelist()
-def new_event_notification(producer_url):
-	"""Pull data from producer when notified"""
-	enqueued_method = "frappe.event_streaming.doctype.event_producer.event_producer.pull_from_node"
-	jobs = get_jobs()
-	if not jobs or enqueued_method not in jobs[frappe.local.site]:
-		frappe.enqueue(enqueued_method, queue="default", **{"event_producer": producer_url})
-
-
-@frappe.whitelist()
-def resync(update):
-	"""Retry syncing update if failed"""
-	update = frappe._dict(json.loads(update))
-	producer_site = get_producer_site(update.event_producer)
-	event_producer = frappe.get_doc("Event Producer", update.event_producer)
-	if update.mapping:
-		update = get_mapped_update(update, producer_site)
-		update.data = json.loads(update.data)
-	return sync(update, producer_site, event_producer, in_retry=True)
diff --git a/frappe/event_streaming/doctype/event_producer/test_event_producer.py b/frappe/event_streaming/doctype/event_producer/test_event_producer.py
deleted file mode 100644
index 168c9a61cf..0000000000
--- a/frappe/event_streaming/doctype/event_producer/test_event_producer.py
+++ /dev/null
@@ -1,446 +0,0 @@
-# Copyright (c) 2019, Frappe Technologies and Contributors
-# License: MIT. See LICENSE
-import json
-import unittest
-
-import frappe
-from frappe.core.doctype.user.user import generate_keys
-from frappe.event_streaming.doctype.event_producer.event_producer import pull_from_node
-from frappe.frappeclient import FrappeClient
-from frappe.query_builder.utils import db_type_is
-from frappe.tests.test_query_builder import run_only_if
-
-producer_url = "http://test_site_producer:8000"
-
-
-class TestEventProducer(unittest.TestCase):
-	# @classmethod
-	# def setUpClass(cls):
-	# 	frappe.print_sql(True)
-
-	# @classmethod
-	# def tearDownClass(cls):
-	# 	frappe.print_sql(False)
-
-	def setUp(self):
-		create_event_producer(producer_url)
-
-	def tearDown(self):
-		unsubscribe_doctypes(producer_url)
-
-	def test_insert(self):
-		producer = get_remote_site()
-		producer_doc = insert_into_producer(producer, "test creation 1 sync")
-		self.pull_producer_data()
-		self.assertTrue(frappe.db.exists("ToDo", producer_doc.name))
-
-	def test_update(self):
-		producer = get_remote_site()
-		producer_doc = insert_into_producer(producer, "test update 1")
-		producer_doc["description"] = "test update 2"
-		producer_doc = producer.update(producer_doc)
-		self.pull_producer_data()
-		local_doc = frappe.get_doc(producer_doc.doctype, producer_doc.name)
-		self.assertEqual(local_doc.description, producer_doc.description)
-
-	def test_delete(self):
-		producer = get_remote_site()
-		producer_doc = insert_into_producer(producer, "test delete sync")
-		self.pull_producer_data()
-		self.assertTrue(frappe.db.exists("ToDo", producer_doc.name))
-		producer.delete("ToDo", producer_doc.name)
-		self.pull_producer_data()
-		self.assertFalse(frappe.db.exists("ToDo", producer_doc.name))
-
-	@run_only_if(db_type_is.MARIADB)
-	def test_multiple_doctypes_sync(self):
-		# TODO: This test is extremely flaky with Postgres. Rewrite this!
-		producer = get_remote_site()
-
-		# insert todo and note in producer
-		producer_todo = insert_into_producer(producer, "test multiple doc sync")
-		producer_note1 = frappe._dict(doctype="Note", title="test multiple doc sync 1")
-		delete_on_remote_if_exists(producer, "Note", {"title": producer_note1["title"]})
-		frappe.db.delete("Note", {"title": producer_note1["title"]})
-		producer_note1 = producer.insert(producer_note1)
-		producer_note2 = frappe._dict(doctype="Note", title="test multiple doc sync 2")
-		delete_on_remote_if_exists(producer, "Note", {"title": producer_note2["title"]})
-		frappe.db.delete("Note", {"title": producer_note2["title"]})
-		producer_note2 = producer.insert(producer_note2)
-
-		# update in producer
-		producer_todo["description"] = "test multiple doc update sync"
-		producer_todo = producer.update(producer_todo)
-		producer_note1["content"] = "testing update sync"
-		producer_note1 = producer.update(producer_note1)
-
-		producer.delete("Note", producer_note2.name)
-
-		self.pull_producer_data()
-
-		# check inserted
-		self.assertTrue(frappe.db.exists("ToDo", producer_todo.name))
-
-		# check update
-		local_todo = frappe.get_doc("ToDo", producer_todo.name)
-		self.assertEqual(local_todo.description, producer_todo.description)
-		local_note1 = frappe.get_doc("Note", producer_note1.name)
-		self.assertEqual(local_note1.content, producer_note1.content)
-
-		# check delete
-		self.assertFalse(frappe.db.exists("Note", producer_note2.name))
-
-	def test_child_table_sync_with_dependencies(self):
-		producer = get_remote_site()
-		producer_user = frappe._dict(
-			doctype="User",
-			email="test_user@sync.com",
-			send_welcome_email=0,
-			first_name="Test Sync User",
-			enabled=1,
-			roles=[{"role": "System Manager"}],
-		)
-		delete_on_remote_if_exists(producer, "User", {"email": producer_user.email})
-		frappe.db.delete("User", {"email": producer_user.email})
-		producer_user = producer.insert(producer_user)
-
-		producer_note = frappe._dict(
-			doctype="Note", title="test child table dependency sync", seen_by=[{"user": producer_user.name}]
-		)
-		delete_on_remote_if_exists(producer, "Note", {"title": producer_note.title})
-		frappe.db.delete("Note", {"title": producer_note.title})
-		producer_note = producer.insert(producer_note)
-
-		self.pull_producer_data()
-		self.assertTrue(frappe.db.exists("User", producer_user.name))
-		if self.assertTrue(frappe.db.exists("Note", producer_note.name)):
-			local_note = frappe.get_doc("Note", producer_note.name)
-			self.assertEqual(len(local_note.seen_by), 1)
-
-	def test_dynamic_link_dependencies_synced(self):
-		producer = get_remote_site()
-		# unsubscribe for Note to check whether dependency is fulfilled
-		event_producer = frappe.get_doc("Event Producer", producer_url, for_update=True)
-		event_producer.producer_doctypes = []
-		event_producer.append("producer_doctypes", {"ref_doctype": "ToDo", "use_same_name": 1})
-		event_producer.save()
-
-		producer_link_doc = frappe._dict(doctype="Note", title="Test Dynamic Link 1")
-
-		delete_on_remote_if_exists(producer, "Note", {"title": producer_link_doc.title})
-		frappe.db.delete("Note", {"title": producer_link_doc.title})
-		producer_link_doc = producer.insert(producer_link_doc)
-		producer_doc = frappe._dict(
-			doctype="ToDo",
-			description="Test Dynamic Link 2",
-			assigned_by="Administrator",
-			reference_type="Note",
-			reference_name=producer_link_doc.name,
-		)
-		producer_doc = producer.insert(producer_doc)
-
-		self.pull_producer_data()
-
-		# check dynamic link dependency created
-		self.assertTrue(frappe.db.exists("Note", producer_link_doc.name))
-		self.assertEqual(
-			producer_link_doc.name, frappe.db.get_value("ToDo", producer_doc.name, "reference_name")
-		)
-
-		reset_configuration(producer_url)
-
-	def test_naming_configuration(self):
-		# test with use_same_name = 0
-		producer = get_remote_site()
-		event_producer = frappe.get_doc("Event Producer", producer_url, for_update=True)
-		event_producer.producer_doctypes = []
-		event_producer.append("producer_doctypes", {"ref_doctype": "ToDo", "use_same_name": 0})
-		event_producer.save()
-
-		producer_doc = insert_into_producer(producer, "test different name sync")
-		self.pull_producer_data()
-		self.assertTrue(
-			frappe.db.exists(
-				"ToDo", {"remote_docname": producer_doc.name, "remote_site_name": producer_url}
-			)
-		)
-
-		reset_configuration(producer_url)
-
-	def test_conditional_events(self):
-		producer = get_remote_site()
-
-		# Add Condition
-		event_producer = frappe.get_doc("Event Producer", producer_url)
-		note_producer_entry = [x for x in event_producer.producer_doctypes if x.ref_doctype == "Note"][0]
-		note_producer_entry.condition = "doc.public == 1"
-		event_producer.save()
-
-		# Make test doc
-		producer_note1 = frappe._dict(doctype="Note", public=0, title="test conditional sync")
-		delete_on_remote_if_exists(producer, "Note", {"title": producer_note1["title"]})
-		producer_note1 = producer.insert(producer_note1)
-
-		# Make Update
-		producer_note1["content"] = "Test Conditional Sync Content"
-		producer_note1 = producer.update(producer_note1)
-
-		self.pull_producer_data()
-
-		# Check if synced here
-		self.assertFalse(frappe.db.exists("Note", producer_note1.name))
-
-		# Lets satisfy the condition
-		producer_note1["public"] = 1
-		producer_note1 = producer.update(producer_note1)
-
-		self.pull_producer_data()
-
-		# it should sync now
-		self.assertTrue(frappe.db.exists("Note", producer_note1.name))
-		local_note = frappe.get_doc("Note", producer_note1.name)
-		self.assertEqual(local_note.content, producer_note1.content)
-
-		reset_configuration(producer_url)
-
-	def test_conditional_events_with_cmd(self):
-		producer = get_remote_site()
-
-		# Add Condition
-		event_producer = frappe.get_doc("Event Producer", producer_url)
-		note_producer_entry = [x for x in event_producer.producer_doctypes if x.ref_doctype == "Note"][0]
-		note_producer_entry.condition = (
-			"cmd: frappe.event_streaming.doctype.event_producer.test_event_producer.can_sync_note"
-		)
-		event_producer.save()
-
-		# Make test doc
-		producer_note1 = frappe._dict(doctype="Note", public=0, title="test conditional sync cmd")
-		delete_on_remote_if_exists(producer, "Note", {"title": producer_note1["title"]})
-		producer_note1 = producer.insert(producer_note1)
-
-		# Make Update
-		producer_note1["content"] = "Test Conditional Sync Content"
-		producer_note1 = producer.update(producer_note1)
-
-		self.pull_producer_data()
-
-		# Check if synced here
-		self.assertFalse(frappe.db.exists("Note", producer_note1.name))
-
-		# Lets satisfy the condition
-		producer_note1["public"] = 1
-		producer_note1 = producer.update(producer_note1)
-
-		self.pull_producer_data()
-
-		# it should sync now
-		self.assertTrue(frappe.db.exists("Note", producer_note1.name))
-		local_note = frappe.get_doc("Note", producer_note1.name)
-		self.assertEqual(local_note.content, producer_note1.content)
-
-		reset_configuration(producer_url)
-
-	def test_update_log(self):
-		producer = get_remote_site()
-		producer_doc = insert_into_producer(producer, "test update log")
-		update_log_doc = producer.get_value(
-			"Event Update Log", "docname", {"docname": producer_doc.get("name")}
-		)
-		self.assertEqual(update_log_doc.get("docname"), producer_doc.get("name"))
-
-	def test_event_sync_log(self):
-		producer = get_remote_site()
-		producer_doc = insert_into_producer(producer, "test event sync log")
-		self.pull_producer_data()
-		self.assertTrue(frappe.db.exists("Event Sync Log", {"docname": producer_doc.name}))
-
-	def pull_producer_data(self):
-		pull_from_node(producer_url)
-
-	def test_mapping(self):
-		producer = get_remote_site()
-		event_producer = frappe.get_doc("Event Producer", producer_url, for_update=True)
-		event_producer.producer_doctypes = []
-		mapping = [{"local_fieldname": "description", "remote_fieldname": "content"}]
-		event_producer.append(
-			"producer_doctypes",
-			{
-				"ref_doctype": "ToDo",
-				"use_same_name": 1,
-				"has_mapping": 1,
-				"mapping": get_mapping("ToDo to Note", "ToDo", "Note", mapping),
-			},
-		)
-		event_producer.save()
-
-		producer_note = frappe._dict(doctype="Note", title="Test Mapping", content="Test Mapping")
-		delete_on_remote_if_exists(producer, "Note", {"title": producer_note.title})
-		producer_note = producer.insert(producer_note)
-		self.pull_producer_data()
-		# check inserted
-		self.assertTrue(frappe.db.exists("ToDo", {"description": producer_note.content}))
-
-		# update in producer
-		producer_note["content"] = "test mapped doc update sync"
-		producer_note = producer.update(producer_note)
-		self.pull_producer_data()
-
-		# check updated
-		self.assertTrue(frappe.db.exists("ToDo", {"description": producer_note["content"]}))
-
-		producer.delete("Note", producer_note.name)
-		self.pull_producer_data()
-		# check delete
-		self.assertFalse(frappe.db.exists("ToDo", {"description": producer_note.content}))
-
-		reset_configuration(producer_url)
-
-	def test_inner_mapping(self):
-		producer = get_remote_site()
-
-		setup_event_producer_for_inner_mapping()
-		producer_note = frappe._dict(
-			doctype="Note", title="Inner Mapping Tester", content="Test Inner Mapping"
-		)
-		delete_on_remote_if_exists(producer, "Note", {"title": producer_note.title})
-		producer_note = producer.insert(producer_note)
-		self.pull_producer_data()
-
-		# check dependency inserted
-		self.assertTrue(frappe.db.exists("Role", {"role_name": producer_note.title}))
-		# check doc inserted
-		self.assertTrue(frappe.db.exists("ToDo", {"description": producer_note.content}))
-
-		reset_configuration(producer_url)
-
-
-def can_sync_note(consumer, doc, update_log):
-	return doc.public == 1
-
-
-def setup_event_producer_for_inner_mapping():
-	event_producer = frappe.get_doc("Event Producer", producer_url, for_update=True)
-	event_producer.producer_doctypes = []
-	inner_mapping = [{"local_fieldname": "role_name", "remote_fieldname": "title"}]
-	inner_map = get_mapping("Role to Note Dependency Creation", "Role", "Note", inner_mapping)
-	mapping = [
-		{
-			"local_fieldname": "description",
-			"remote_fieldname": "content",
-		},
-		{
-			"local_fieldname": "role",
-			"remote_fieldname": "title",
-			"mapping_type": "Document",
-			"mapping": inner_map,
-			"remote_value_filters": json.dumps({"title": "title"}),
-		},
-	]
-	event_producer.append(
-		"producer_doctypes",
-		{
-			"ref_doctype": "ToDo",
-			"use_same_name": 1,
-			"has_mapping": 1,
-			"mapping": get_mapping("ToDo to Note Mapping", "ToDo", "Note", mapping),
-		},
-	)
-	event_producer.save()
-	return event_producer
-
-
-def insert_into_producer(producer, description):
-	# create and insert todo on remote site
-	todo = dict(doctype="ToDo", description=description, assigned_by="Administrator")
-	return producer.insert(todo)
-
-
-def delete_on_remote_if_exists(producer, doctype, filters):
-	remote_doc = producer.get_value(doctype, "name", filters)
-	if remote_doc:
-		producer.delete(doctype, remote_doc.get("name"))
-
-
-def get_mapping(mapping_name, local, remote, field_map):
-	name = frappe.db.exists("Document Type Mapping", mapping_name)
-	if name:
-		doc = frappe.get_doc("Document Type Mapping", name)
-	else:
-		doc = frappe.new_doc("Document Type Mapping")
-
-	doc.mapping_name = mapping_name
-	doc.local_doctype = local
-	doc.remote_doctype = remote
-	for entry in field_map:
-		doc.append("field_mapping", entry)
-	doc.save()
-	return doc.name
-
-
-def create_event_producer(producer_url):
-	if frappe.db.exists("Event Producer", producer_url):
-		event_producer = frappe.get_doc("Event Producer", producer_url)
-		for entry in event_producer.producer_doctypes:
-			entry.unsubscribe = 0
-		event_producer.save()
-		return
-
-	generate_keys("Administrator")
-
-	producer_site = connect()
-
-	response = producer_site.post_api(
-		"frappe.core.doctype.user.user.generate_keys", params={"user": "Administrator"}
-	)
-
-	api_secret = response.get("api_secret")
-
-	response = producer_site.get_value("User", "api_key", {"name": "Administrator"})
-	api_key = response.get("api_key")
-
-	event_producer = frappe.new_doc("Event Producer")
-	event_producer.producer_doctypes = []
-	event_producer.producer_url = producer_url
-	event_producer.append("producer_doctypes", {"ref_doctype": "ToDo", "use_same_name": 1})
-	event_producer.append("producer_doctypes", {"ref_doctype": "Note", "use_same_name": 1})
-	event_producer.user = "Administrator"
-	event_producer.api_key = api_key
-	event_producer.api_secret = api_secret
-	event_producer.save()
-
-
-def reset_configuration(producer_url):
-	event_producer = frappe.get_doc("Event Producer", producer_url, for_update=True)
-	event_producer.producer_doctypes = []
-	event_producer.conditions = []
-	event_producer.producer_url = producer_url
-	event_producer.append("producer_doctypes", {"ref_doctype": "ToDo", "use_same_name": 1})
-	event_producer.append("producer_doctypes", {"ref_doctype": "Note", "use_same_name": 1})
-	event_producer.user = "Administrator"
-	event_producer.save()
-
-
-def get_remote_site():
-	producer_doc = frappe.get_doc("Event Producer", producer_url)
-	producer_site = FrappeClient(
-		url=producer_doc.producer_url, username="Administrator", password="admin", verify=False
-	)
-	return producer_site
-
-
-def unsubscribe_doctypes(producer_url):
-	event_producer = frappe.get_doc("Event Producer", producer_url)
-	for entry in event_producer.producer_doctypes:
-		entry.unsubscribe = 1
-	event_producer.save()
-
-
-def connect():
-	def _connect():
-		return FrappeClient(url=producer_url, username="Administrator", password="admin", verify=False)
-
-	try:
-		return _connect()
-	except Exception:
-		return _connect()
diff --git a/frappe/event_streaming/doctype/event_producer_document_type/event_producer_document_type.json b/frappe/event_streaming/doctype/event_producer_document_type/event_producer_document_type.json
deleted file mode 100644
index 17fd51d12d..0000000000
--- a/frappe/event_streaming/doctype/event_producer_document_type/event_producer_document_type.json
+++ /dev/null
@@ -1,86 +0,0 @@
-{
- "actions": [],
- "creation": "2019-10-03 21:08:25.890352",
- "doctype": "DocType",
- "editable_grid": 1,
- "engine": "InnoDB",
- "field_order": [
-  "ref_doctype",
-  "status",
-  "use_same_name",
-  "unsubscribe",
-  "has_mapping",
-  "mapping",
-  "condition"
- ],
- "fields": [
-  {
-   "columns": 3,
-   "fieldname": "ref_doctype",
-   "fieldtype": "Link",
-   "in_list_view": 1,
-   "label": "Document Type",
-   "options": "DocType",
-   "reqd": 1,
-   "set_only_once": 1
-  },
-  {
-   "default": "0",
-   "description": "If the document has different field names on the Producer and Consumer's end check this and set up the Mapping",
-   "fieldname": "has_mapping",
-   "fieldtype": "Check",
-   "label": "Has Mapping"
-  },
-  {
-   "depends_on": "eval: doc.has_mapping",
-   "fieldname": "mapping",
-   "fieldtype": "Link",
-   "label": "Mapping",
-   "options": "Document Type Mapping"
-  },
-  {
-   "columns": 2,
-   "default": "0",
-   "description": "If this is checked the documents will have the same name as they have on the Event Producer's site",
-   "fieldname": "use_same_name",
-   "fieldtype": "Check",
-   "in_list_view": 1,
-   "label": "Use Same Name"
-  },
-  {
-   "columns": 3,
-   "default": "Pending",
-   "fieldname": "status",
-   "fieldtype": "Select",
-   "in_list_view": 1,
-   "label": "Approval Status",
-   "options": "Pending\nApproved\nRejected",
-   "read_only": 1
-  },
-  {
-   "columns": 2,
-   "default": "0",
-   "fieldname": "unsubscribe",
-   "fieldtype": "Check",
-   "in_list_view": 1,
-   "label": "Unsubscribe"
-  },
-  {
-   "fieldname": "condition",
-   "fieldtype": "Code",
-   "label": "Condition"
-  }
- ],
- "istable": 1,
- "links": [],
- "modified": "2020-11-07 09:26:58.463868",
- "modified_by": "Administrator",
- "module": "Event Streaming",
- "name": "Event Producer Document Type",
- "owner": "Administrator",
- "permissions": [],
- "quick_entry": 1,
- "sort_field": "modified",
- "sort_order": "DESC",
- "track_changes": 1
-}
\ No newline at end of file
diff --git a/frappe/event_streaming/doctype/event_producer_document_type/event_producer_document_type.py b/frappe/event_streaming/doctype/event_producer_document_type/event_producer_document_type.py
deleted file mode 100644
index 8f4c936792..0000000000
--- a/frappe/event_streaming/doctype/event_producer_document_type/event_producer_document_type.py
+++ /dev/null
@@ -1,9 +0,0 @@
-# Copyright (c) 2019, Frappe Technologies and contributors
-# License: MIT. See LICENSE
-
-# import frappe
-from frappe.model.document import Document
-
-
-class EventProducerDocumentType(Document):
-	pass
diff --git a/frappe/event_streaming/doctype/event_producer_last_update/event_producer_last_update.js b/frappe/event_streaming/doctype/event_producer_last_update/event_producer_last_update.js
deleted file mode 100644
index 15730e4c5f..0000000000
--- a/frappe/event_streaming/doctype/event_producer_last_update/event_producer_last_update.js
+++ /dev/null
@@ -1,8 +0,0 @@
-// Copyright (c) 2020, Frappe Technologies and contributors
-// For license information, please see license.txt
-
-frappe.ui.form.on('Event Producer Last Update', {
-	// refresh: function(frm) {
-
-	// }
-});
diff --git a/frappe/event_streaming/doctype/event_producer_last_update/event_producer_last_update.json b/frappe/event_streaming/doctype/event_producer_last_update/event_producer_last_update.json
deleted file mode 100644
index 27f8ed2f81..0000000000
--- a/frappe/event_streaming/doctype/event_producer_last_update/event_producer_last_update.json
+++ /dev/null
@@ -1,53 +0,0 @@
-{
- "actions": [],
- "autoname": "field:event_producer",
- "creation": "2020-10-26 12:53:11.940177",
- "doctype": "DocType",
- "editable_grid": 1,
- "engine": "InnoDB",
- "field_order": [
-  "event_producer",
-  "last_update"
- ],
- "fields": [
-  {
-   "fieldname": "event_producer",
-   "fieldtype": "Data",
-   "in_list_view": 1,
-   "label": "Event Producer",
-   "reqd": 1,
-   "unique": 1
-  },
-  {
-   "fieldname": "last_update",
-   "fieldtype": "Data",
-   "label": "Last Update"
-  }
- ],
- "in_create": 1,
- "index_web_pages_for_search": 1,
- "links": [],
- "modified": "2020-10-26 13:22:27.056599",
- "modified_by": "Administrator",
- "module": "Event Streaming",
- "name": "Event Producer Last Update",
- "owner": "Administrator",
- "permissions": [
-  {
-   "create": 1,
-   "delete": 1,
-   "email": 1,
-   "export": 1,
-   "print": 1,
-   "read": 1,
-   "report": 1,
-   "role": "System Manager",
-   "share": 1,
-   "write": 1
-  }
- ],
- "read_only": 1,
- "sort_field": "modified",
- "sort_order": "DESC",
- "track_changes": 1
-}
\ No newline at end of file
diff --git a/frappe/event_streaming/doctype/event_producer_last_update/event_producer_last_update.py b/frappe/event_streaming/doctype/event_producer_last_update/event_producer_last_update.py
deleted file mode 100644
index ec5cee7e78..0000000000
--- a/frappe/event_streaming/doctype/event_producer_last_update/event_producer_last_update.py
+++ /dev/null
@@ -1,9 +0,0 @@
-# Copyright (c) 2020, Frappe Technologies and contributors
-# License: MIT. See LICENSE
-
-# import frappe
-from frappe.model.document import Document
-
-
-class EventProducerLastUpdate(Document):
-	pass
diff --git a/frappe/event_streaming/doctype/event_producer_last_update/test_event_producer_last_update.py b/frappe/event_streaming/doctype/event_producer_last_update/test_event_producer_last_update.py
deleted file mode 100644
index ccdea6c694..0000000000
--- a/frappe/event_streaming/doctype/event_producer_last_update/test_event_producer_last_update.py
+++ /dev/null
@@ -1,8 +0,0 @@
-# Copyright (c) 2020, Frappe Technologies and Contributors
-# License: MIT. See LICENSE
-# import frappe
-import unittest
-
-
-class TestEventProducerLastUpdate(unittest.TestCase):
-	pass
diff --git a/frappe/event_streaming/doctype/event_sync_log/event_sync_log.js b/frappe/event_streaming/doctype/event_sync_log/event_sync_log.js
deleted file mode 100644
index 5199e3f02d..0000000000
--- a/frappe/event_streaming/doctype/event_sync_log/event_sync_log.js
+++ /dev/null
@@ -1,24 +0,0 @@
-// Copyright (c) 2019, Frappe Technologies and contributors
-// For license information, please see license.txt
-
-frappe.ui.form.on('Event Sync Log', {
-	refresh: function(frm) {
-		if (frm.doc.status == 'Failed') {
-			frm.add_custom_button(__('Resync'), function() {
-				frappe.call({
-					method: "frappe.event_streaming.doctype.event_producer.event_producer.resync",
-					args: {
-						update: frm.doc,
-					},
-					callback: function(r) {
-						if (r.message) {
-							frappe.msgprint(r.message);
-							frm.set_value('status', r.message);
-							frm.save();
-						}
-					}
-				});
-			});
-		}
-	}
-});
diff --git a/frappe/event_streaming/doctype/event_sync_log/event_sync_log.json b/frappe/event_streaming/doctype/event_sync_log/event_sync_log.json
deleted file mode 100644
index f82128bd7b..0000000000
--- a/frappe/event_streaming/doctype/event_sync_log/event_sync_log.json
+++ /dev/null
@@ -1,137 +0,0 @@
-{
- "creation": "2019-09-24 22:22:05.845089",
- "doctype": "DocType",
- "editable_grid": 1,
- "engine": "InnoDB",
- "field_order": [
-  "update_type",
-  "ref_doctype",
-  "docname",
-  "column_break_4",
-  "status",
-  "event_producer",
-  "producer_doc",
-  "event_configurations_section",
-  "use_same_name",
-  "column_break_9",
-  "mapping",
-  "section_break_8",
-  "data",
-  "error"
- ],
- "fields": [
-  {
-   "fieldname": "update_type",
-   "fieldtype": "Select",
-   "in_list_view": 1,
-   "label": "Update Type",
-   "options": "Create\nUpdate\nDelete",
-   "read_only": 1
-  },
-  {
-   "fieldname": "ref_doctype",
-   "fieldtype": "Link",
-   "label": "Doctype",
-   "options": "DocType",
-   "read_only": 1
-  },
-  {
-   "fieldname": "docname",
-   "fieldtype": "Data",
-   "in_list_view": 1,
-   "label": "Document Name",
-   "options": "ref_doctype",
-   "read_only": 1
-  },
-  {
-   "fieldname": "column_break_4",
-   "fieldtype": "Column Break"
-  },
-  {
-   "fieldname": "status",
-   "fieldtype": "Select",
-   "in_list_view": 1,
-   "label": "Status",
-   "options": "\nSynced\nFailed",
-   "read_only": 1
-  },
-  {
-   "fieldname": "event_producer",
-   "fieldtype": "Data",
-   "in_list_view": 1,
-   "label": "Event Producer",
-   "options": "Event Producer",
-   "read_only": 1
-  },
-  {
-   "fieldname": "section_break_8",
-   "fieldtype": "Section Break",
-   "label": "Data"
-  },
-  {
-   "fieldname": "data",
-   "fieldtype": "Code",
-   "label": "Data",
-   "read_only": 1
-  },
-  {
-   "fieldname": "producer_doc",
-   "fieldtype": "Data",
-   "label": "Producer Document Name",
-   "read_only": 1
-  },
-  {
-   "depends_on": "eval:doc.status=='Failed'",
-   "fieldname": "error",
-   "fieldtype": "Code",
-   "label": "Error",
-   "read_only": 1
-  },
-  {
-   "fieldname": "event_configurations_section",
-   "fieldtype": "Section Break",
-   "label": "Event Configurations"
-  },
-  {
-   "default": "0",
-   "fieldname": "use_same_name",
-   "fieldtype": "Data",
-   "label": "Use Same Name",
-   "read_only": 1
-  },
-  {
-   "fieldname": "column_break_9",
-   "fieldtype": "Column Break"
-  },
-  {
-   "fieldname": "mapping",
-   "fieldtype": "Data",
-   "label": "Mapping",
-   "read_only": 1
-  }
- ],
- "in_create": 1,
- "modified": "2019-10-07 13:22:10.401479",
- "modified_by": "Administrator",
- "module": "Event Streaming",
- "name": "Event Sync Log",
- "owner": "Administrator",
- "permissions": [
-  {
-   "create": 1,
-   "delete": 1,
-   "email": 1,
-   "export": 1,
-   "print": 1,
-   "read": 1,
-   "report": 1,
-   "role": "System Manager",
-   "share": 1,
-   "write": 1
-  }
- ],
- "quick_entry": 1,
- "sort_field": "modified",
- "sort_order": "DESC",
- "track_changes": 1
-}
\ No newline at end of file
diff --git a/frappe/event_streaming/doctype/event_sync_log/event_sync_log.py b/frappe/event_streaming/doctype/event_sync_log/event_sync_log.py
deleted file mode 100644
index a1d82ad08f..0000000000
--- a/frappe/event_streaming/doctype/event_sync_log/event_sync_log.py
+++ /dev/null
@@ -1,9 +0,0 @@
-# Copyright (c) 2019, Frappe Technologies and contributors
-# License: MIT. See LICENSE
-
-# import frappe
-from frappe.model.document import Document
-
-
-class EventSyncLog(Document):
-	pass
diff --git a/frappe/event_streaming/doctype/event_sync_log/event_sync_log_list.js b/frappe/event_streaming/doctype/event_sync_log/event_sync_log_list.js
deleted file mode 100644
index 75d67003c4..0000000000
--- a/frappe/event_streaming/doctype/event_sync_log/event_sync_log_list.js
+++ /dev/null
@@ -1,9 +0,0 @@
-frappe.listview_settings['Event Sync Log'] = {
-	get_indicator: function(doc) {
-		var colors = {
-			"Failed": "red",
-			"Synced": "green"
-		};
-		return [__(doc.status), colors[doc.status], "status,=," + doc.status];
-	}
-};
diff --git a/frappe/event_streaming/doctype/event_sync_log/test_event_sync_log.py b/frappe/event_streaming/doctype/event_sync_log/test_event_sync_log.py
deleted file mode 100644
index 13028cbac7..0000000000
--- a/frappe/event_streaming/doctype/event_sync_log/test_event_sync_log.py
+++ /dev/null
@@ -1,8 +0,0 @@
-# Copyright (c) 2019, Frappe Technologies and Contributors
-# License: MIT. See LICENSE
-# import frappe
-import unittest
-
-
-class TestEventSyncLog(unittest.TestCase):
-	pass
diff --git a/frappe/event_streaming/doctype/event_update_log/__init__.py b/frappe/event_streaming/doctype/event_update_log/__init__.py
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/frappe/event_streaming/doctype/event_update_log/event_update_log.js b/frappe/event_streaming/doctype/event_update_log/event_update_log.js
deleted file mode 100644
index c5e8ed5915..0000000000
--- a/frappe/event_streaming/doctype/event_update_log/event_update_log.js
+++ /dev/null
@@ -1,8 +0,0 @@
-// Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and contributors
-// For license information, please see license.txt
-
-frappe.ui.form.on('Event Update Log', {
-	// refresh: function(frm) {
-
-	// }
-});
diff --git a/frappe/event_streaming/doctype/event_update_log/event_update_log.json b/frappe/event_streaming/doctype/event_update_log/event_update_log.json
deleted file mode 100644
index a42bc7ec87..0000000000
--- a/frappe/event_streaming/doctype/event_update_log/event_update_log.json
+++ /dev/null
@@ -1,77 +0,0 @@
-{
- "actions": [],
- "creation": "2019-07-30 15:31:26.352527",
- "doctype": "DocType",
- "editable_grid": 1,
- "engine": "InnoDB",
- "field_order": [
-  "update_type",
-  "ref_doctype",
-  "docname",
-  "data",
-  "consumers"
- ],
- "fields": [
-  {
-   "fieldname": "update_type",
-   "fieldtype": "Select",
-   "in_list_view": 1,
-   "label": "Update Type",
-   "options": "Create\nUpdate\nDelete",
-   "read_only": 1
-  },
-  {
-   "fieldname": "ref_doctype",
-   "fieldtype": "Link",
-   "in_list_view": 1,
-   "label": "DocType",
-   "options": "DocType",
-   "read_only": 1
-  },
-  {
-   "fieldname": "docname",
-   "fieldtype": "Data",
-   "in_list_view": 1,
-   "label": "Document Name",
-   "read_only": 1
-  },
-  {
-   "fieldname": "data",
-   "fieldtype": "Code",
-   "label": "Data",
-   "read_only": 1
-  },
-  {
-   "fieldname": "consumers",
-   "fieldtype": "Table MultiSelect",
-   "label": "Consumers",
-   "options": "Event Update Log Consumer",
-   "read_only": 1
-  }
- ],
- "in_create": 1,
- "links": [],
- "modified": "2020-09-04 07:31:52.599804",
- "modified_by": "Administrator",
- "module": "Event Streaming",
- "name": "Event Update Log",
- "owner": "Administrator",
- "permissions": [
-  {
-   "create": 1,
-   "delete": 1,
-   "email": 1,
-   "export": 1,
-   "print": 1,
-   "read": 1,
-   "report": 1,
-   "role": "System Manager",
-   "share": 1,
-   "write": 1
-  }
- ],
- "quick_entry": 1,
- "sort_field": "modified",
- "sort_order": "DESC",
- "track_changes": 1
-}
\ No newline at end of file
diff --git a/frappe/event_streaming/doctype/event_update_log/event_update_log.py b/frappe/event_streaming/doctype/event_update_log/event_update_log.py
deleted file mode 100644
index e40f600484..0000000000
--- a/frappe/event_streaming/doctype/event_update_log/event_update_log.py
+++ /dev/null
@@ -1,296 +0,0 @@
-# Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and contributors
-# License: MIT. See LICENSE
-
-import frappe
-from frappe.model import no_value_fields, table_fields
-from frappe.model.document import Document
-from frappe.utils.background_jobs import get_jobs
-
-
-class EventUpdateLog(Document):
-	def after_insert(self):
-		"""Send update notification updates to event consumers
-		whenever update log is generated"""
-		enqueued_method = (
-			"frappe.event_streaming.doctype.event_consumer.event_consumer.notify_event_consumers"
-		)
-		jobs = get_jobs()
-		if not jobs or enqueued_method not in jobs[frappe.local.site]:
-			frappe.enqueue(
-				enqueued_method, doctype=self.ref_doctype, queue="long", enqueue_after_commit=True
-			)
-
-
-def notify_consumers(doc, event):
-	"""called via hooks"""
-	# make event update log for doctypes having event consumers
-	if frappe.flags.in_install or frappe.flags.in_migrate:
-		return
-
-	consumers = check_doctype_has_consumers(doc.doctype)
-	if consumers:
-		if event == "after_insert":
-			doc.flags.event_update_log = make_event_update_log(doc, update_type="Create")
-		elif event == "on_trash":
-			make_event_update_log(doc, update_type="Delete")
-		else:
-			# on_update
-			# called after saving
-			if not doc.flags.event_update_log:  # if not already inserted
-				diff = get_update(doc.get_doc_before_save(), doc)
-				if diff:
-					doc.diff = diff
-					make_event_update_log(doc, update_type="Update")
-
-
-def check_doctype_has_consumers(doctype):
-	"""Check if doctype has event consumers for event streaming"""
-	return frappe.cache_manager.get_doctype_map(
-		"Event Consumer Document Type",
-		doctype,
-		dict(ref_doctype=doctype, status="Approved", unsubscribed=0),
-	)
-
-
-def get_update(old, new, for_child=False):
-	"""
-	Get document objects with updates only
-	If there is a change, then returns a dict like:
-	{
-	        "changed"		: {fieldname1: new_value1, fieldname2: new_value2, },
-	        "added"			: {table_fieldname1: [{row_dict1}, {row_dict2}], },
-	        "removed"		: {table_fieldname1: [row_name1, row_name2], },
-	        "row_changed"	: {table_fieldname1:
-	                {
-	                        child_fieldname1: new_val,
-	                        child_fieldname2: new_val
-	                },
-	        },
-	}
-	"""
-	if not new:
-		return None
-
-	out = frappe._dict(changed={}, added={}, removed={}, row_changed={})
-	for df in new.meta.fields:
-		if df.fieldtype in no_value_fields and df.fieldtype not in table_fields:
-			continue
-
-		old_value, new_value = old.get(df.fieldname), new.get(df.fieldname)
-
-		if df.fieldtype in table_fields:
-			old_row_by_name, new_row_by_name = make_maps(old_value, new_value)
-			out = check_for_additions(out, df, new_value, old_row_by_name)
-			out = check_for_deletions(out, df, old_value, new_row_by_name)
-
-		elif old_value != new_value:
-			out.changed[df.fieldname] = new_value
-
-	out = check_docstatus(out, old, new, for_child)
-	if any((out.changed, out.added, out.removed, out.row_changed)):
-		return out
-	return None
-
-
-def make_event_update_log(doc, update_type):
-	"""Save update info for doctypes that have event consumers"""
-	if update_type != "Delete":
-		# diff for update type, doc for create type
-		data = frappe.as_json(doc) if not doc.get("diff") else frappe.as_json(doc.diff)
-	else:
-		data = None
-	return frappe.get_doc(
-		{
-			"doctype": "Event Update Log",
-			"update_type": update_type,
-			"ref_doctype": doc.doctype,
-			"docname": doc.name,
-			"data": data,
-		}
-	).insert(ignore_permissions=True)
-
-
-def make_maps(old_value, new_value):
-	"""make maps"""
-	old_row_by_name, new_row_by_name = {}, {}
-	for d in old_value:
-		old_row_by_name[d.name] = d
-	for d in new_value:
-		new_row_by_name[d.name] = d
-	return old_row_by_name, new_row_by_name
-
-
-def check_for_additions(out, df, new_value, old_row_by_name):
-	"""check rows for additions, changes"""
-	for _i, d in enumerate(new_value):
-		if d.name in old_row_by_name:
-			diff = get_update(old_row_by_name[d.name], d, for_child=True)
-			if diff and diff.changed:
-				if not out.row_changed.get(df.fieldname):
-					out.row_changed[df.fieldname] = []
-				diff.changed["name"] = d.name
-				out.row_changed[df.fieldname].append(diff.changed)
-		else:
-			if not out.added.get(df.fieldname):
-				out.added[df.fieldname] = []
-			out.added[df.fieldname].append(d.as_dict())
-	return out
-
-
-def check_for_deletions(out, df, old_value, new_row_by_name):
-	"""check for deletions"""
-	for d in old_value:
-		if d.name not in new_row_by_name:
-			if not out.removed.get(df.fieldname):
-				out.removed[df.fieldname] = []
-			out.removed[df.fieldname].append(d.name)
-	return out
-
-
-def check_docstatus(out, old, new, for_child):
-	"""docstatus changes"""
-	if not for_child and old.docstatus != new.docstatus:
-		out.changed["docstatus"] = new.docstatus
-	return out
-
-
-def is_consumer_uptodate(update_log, consumer):
-	"""
-	Checks if Consumer has read all the UpdateLogs before the specified update_log
-	:param update_log: The UpdateLog Doc in context
-	:param consumer: The EventConsumer doc
-	"""
-	if update_log.update_type == "Create":
-		# consumer is obviously up to date
-		return True
-
-	prev_logs = frappe.get_all(
-		"Event Update Log",
-		filters={
-			"ref_doctype": update_log.ref_doctype,
-			"docname": update_log.docname,
-			"creation": ["<", update_log.creation],
-		},
-		order_by="creation desc",
-		limit_page_length=1,
-	)
-
-	if not len(prev_logs):
-		return False
-
-	prev_log_consumers = frappe.get_all(
-		"Event Update Log Consumer",
-		fields=["consumer"],
-		filters={
-			"parent": prev_logs[0].name,
-			"parenttype": "Event Update Log",
-			"consumer": consumer.name,
-		},
-	)
-
-	return len(prev_log_consumers) > 0
-
-
-def mark_consumer_read(update_log_name, consumer_name):
-	"""
-	This function appends the Consumer to the list of Consumers that has 'read' an Update Log
-	"""
-	update_log = frappe.get_doc("Event Update Log", update_log_name)
-	if len([x for x in update_log.consumers if x.consumer == consumer_name]):
-		return
-
-	frappe.get_doc(
-		frappe._dict(
-			doctype="Event Update Log Consumer",
-			consumer=consumer_name,
-			parent=update_log_name,
-			parenttype="Event Update Log",
-			parentfield="consumers",
-		)
-	).insert(ignore_permissions=True)
-
-
-def get_unread_update_logs(consumer_name, dt, dn):
-	"""
-	Get old logs unread by the consumer on a particular document
-	"""
-	already_consumed = [
-		x[0]
-		for x in frappe.db.sql(
-			"""
-		SELECT
-			update_log.name
-		FROM `tabEvent Update Log` update_log
-		JOIN `tabEvent Update Log Consumer` consumer ON consumer.parent = %(log_name)s
-		WHERE
-			consumer.consumer = %(consumer)s
-			AND update_log.ref_doctype = %(dt)s
-			AND update_log.docname = %(dn)s
-	""",
-			{
-				"consumer": consumer_name,
-				"dt": dt,
-				"dn": dn,
-				"log_name": "update_log.name"
-				if frappe.conf.db_type == "mariadb"
-				else "CAST(update_log.name AS VARCHAR)",
-			},
-			as_dict=0,
-		)
-	]
-
-	logs = frappe.get_all(
-		"Event Update Log",
-		fields=["update_type", "ref_doctype", "docname", "data", "name", "creation"],
-		filters={"ref_doctype": dt, "docname": dn, "name": ["not in", already_consumed]},
-		order_by="creation",
-	)
-
-	return logs
-
-
-@frappe.whitelist()
-def get_update_logs_for_consumer(event_consumer, doctypes, last_update):
-	"""
-	Fetches all the UpdateLogs for the consumer
-	It will inject old un-consumed Update Logs if a doc was just found to be accessible to the Consumer
-	"""
-
-	if isinstance(doctypes, str):
-		doctypes = frappe.parse_json(doctypes)
-
-	from frappe.event_streaming.doctype.event_consumer.event_consumer import has_consumer_access
-
-	consumer = frappe.get_doc("Event Consumer", event_consumer)
-	docs = frappe.get_list(
-		doctype="Event Update Log",
-		filters={"ref_doctype": ("in", doctypes), "creation": (">", last_update)},
-		fields=["update_type", "ref_doctype", "docname", "data", "name", "creation"],
-		order_by="creation desc",
-	)
-
-	result = []
-	to_update_history = []
-	for d in docs:
-		if (d.ref_doctype, d.docname) in to_update_history:
-			# will be notified by background jobs
-			continue
-
-		if not has_consumer_access(consumer=consumer, update_log=d):
-			continue
-
-		if not is_consumer_uptodate(d, consumer):
-			to_update_history.append((d.ref_doctype, d.docname))
-			# get_unread_update_logs will have the current log
-			old_logs = get_unread_update_logs(consumer.name, d.ref_doctype, d.docname)
-			if old_logs:
-				old_logs.reverse()
-				result.extend(old_logs)
-		else:
-			result.append(d)
-
-	for d in result:
-		mark_consumer_read(update_log_name=d.name, consumer_name=consumer.name)
-
-	result.reverse()
-	return result
diff --git a/frappe/event_streaming/doctype/event_update_log/test_event_update_log.py b/frappe/event_streaming/doctype/event_update_log/test_event_update_log.py
deleted file mode 100644
index 0cbff47912..0000000000
--- a/frappe/event_streaming/doctype/event_update_log/test_event_update_log.py
+++ /dev/null
@@ -1,8 +0,0 @@
-# Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and Contributors
-# License: MIT. See LICENSE
-# import frappe
-import unittest
-
-
-class TestEventUpdateLog(unittest.TestCase):
-	pass
diff --git a/frappe/event_streaming/doctype/event_update_log_consumer/__init__.py b/frappe/event_streaming/doctype/event_update_log_consumer/__init__.py
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/frappe/event_streaming/doctype/event_update_log_consumer/event_update_log_consumer.json b/frappe/event_streaming/doctype/event_update_log_consumer/event_update_log_consumer.json
deleted file mode 100644
index b3484c6481..0000000000
--- a/frappe/event_streaming/doctype/event_update_log_consumer/event_update_log_consumer.json
+++ /dev/null
@@ -1,32 +0,0 @@
-{
- "actions": [],
- "creation": "2020-06-30 10:54:53.301787",
- "doctype": "DocType",
- "editable_grid": 1,
- "engine": "InnoDB",
- "field_order": [
-  "consumer"
- ],
- "fields": [
-  {
-   "fieldname": "consumer",
-   "fieldtype": "Link",
-   "in_list_view": 1,
-   "label": "Consumer",
-   "options": "Event Consumer",
-   "reqd": 1
-  }
- ],
- "istable": 1,
- "links": [],
- "modified": "2020-06-30 10:54:53.301787",
- "modified_by": "Administrator",
- "module": "Event Streaming",
- "name": "Event Update Log Consumer",
- "owner": "Administrator",
- "permissions": [],
- "quick_entry": 1,
- "sort_field": "modified",
- "sort_order": "DESC",
- "track_changes": 1
-}
\ No newline at end of file
diff --git a/frappe/event_streaming/doctype/event_update_log_consumer/event_update_log_consumer.py b/frappe/event_streaming/doctype/event_update_log_consumer/event_update_log_consumer.py
deleted file mode 100644
index 69da7db92e..0000000000
--- a/frappe/event_streaming/doctype/event_update_log_consumer/event_update_log_consumer.py
+++ /dev/null
@@ -1,9 +0,0 @@
-# Copyright (c) 2020, Frappe Technologies and contributors
-# License: MIT. See LICENSE
-
-# import frappe
-from frappe.model.document import Document
-
-
-class EventUpdateLogConsumer(Document):
-	pass
diff --git a/frappe/exceptions.py b/frappe/exceptions.py
index c3bb45caea..26c323352d 100644
--- a/frappe/exceptions.py
+++ b/frappe/exceptions.py
@@ -15,6 +15,10 @@ class ValidationError(Exception):
 	http_status_code = 417
 
 
+class FrappeTypeError(TypeError):
+	http_status_code = 417
+
+
 class AuthenticationError(Exception):
 	http_status_code = 401
 
@@ -236,6 +240,14 @@ class QueryDeadlockError(Exception):
 	pass
 
 
+class InReadOnlyMode(ValidationError):
+	http_status_code = 503  # temporarily not available
+
+
+class SessionBootFailed(ValidationError):
+	http_status_code = 500
+
+
 class TooManyWritesError(Exception):
 	pass
 
@@ -261,6 +273,10 @@ class ExecutableNotFound(FileNotFoundError):
 	pass
 
 
+class InvalidRoundingMethod(FileNotFoundError):
+	pass
+
+
 class InvalidRemoteException(Exception):
 	pass
 
diff --git a/frappe/frappeclient.py b/frappe/frappeclient.py
index 474a5b06c4..3f7577fac6 100644
--- a/frappe/frappeclient.py
+++ b/frappe/frappeclient.py
@@ -106,7 +106,9 @@ class FrappeClient:
 			headers=self.headers,
 		)
 
-	def get_list(self, doctype, fields='["name"]', filters=None, limit_start=0, limit_page_length=0):
+	def get_list(
+		self, doctype, fields='["name"]', filters=None, limit_start=0, limit_page_length=None
+	):
 		"""Returns list of records of a particular type"""
 		if not isinstance(fields, str):
 			fields = json.dumps(fields)
@@ -115,7 +117,7 @@ class FrappeClient:
 		}
 		if filters:
 			params["filters"] = json.dumps(filters)
-		if limit_page_length:
+		if limit_page_length is not None:
 			params["limit_start"] = limit_start
 			params["limit_page_length"] = limit_page_length
 		res = self.session.get(
@@ -286,7 +288,11 @@ class FrappeClient:
 
 			if doctype != "User" and not frappe.db.exists("User", doc.get("owner")):
 				frappe.get_doc(
-					{"doctype": "User", "email": doc.get("owner"), "first_name": doc.get("owner").split("@")[0]}
+					{
+						"doctype": "User",
+						"email": doc.get("owner"),
+						"first_name": doc.get("owner").split("@", 1)[0],
+					}
 				).insert()
 
 			if update:
diff --git a/frappe/geo/country_info.json b/frappe/geo/country_info.json
index ed44b1c7f8..f308dc63e3 100644
--- a/frappe/geo/country_info.json
+++ b/frappe/geo/country_info.json
@@ -2723,7 +2723,7 @@
   "currency_fraction": "Kuru\u015f",
   "currency_fraction_units": 100,
   "currency_symbol": "\u20ba",
-  "number_format": "#,###.##",
+  "number_format": "#.###,##",
   "timezones": [
    "Europe/Istanbul"
   ],
diff --git a/frappe/geo/country_info.py b/frappe/geo/country_info.py
index 2aefa27170..3267149d4c 100644
--- a/frappe/geo/country_info.py
+++ b/frappe/geo/country_info.py
@@ -5,6 +5,7 @@ import json
 
 # all country info
 import os
+from functools import lru_cache
 
 import frappe
 from frappe.utils.momentjs import get_all_timezones
@@ -27,8 +28,13 @@ def get_all():
 	return all_data
 
 
-@frappe.whitelist()
+@frappe.whitelist(allow_guest=True)
 def get_country_timezone_info():
+	return _get_country_timezone_info()
+
+
+@lru_cache(maxsize=2)
+def _get_country_timezone_info():
 	return {"country_info": get_all(), "all_timezones": get_all_timezones()}
 
 
diff --git a/frappe/geo/doctype/country/country.js b/frappe/geo/doctype/country/country.js
index 62159a1fe7..75bb3f46d5 100644
--- a/frappe/geo/doctype/country/country.js
+++ b/frappe/geo/doctype/country/country.js
@@ -1,8 +1,6 @@
 // Copyright (c) 2016, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Country', {
-	refresh: function(frm) {
-
-	}
+frappe.ui.form.on("Country", {
+	refresh: function (frm) {},
 });
diff --git a/frappe/geo/doctype/country/country.json b/frappe/geo/doctype/country/country.json
index 8ac3741865..8f62458ad1 100644
--- a/frappe/geo/doctype/country/country.json
+++ b/frappe/geo/doctype/country/country.json
@@ -54,7 +54,7 @@
  "icon": "fa fa-globe",
  "idx": 1,
  "links": [],
- "modified": "2020-02-24 15:44:31.837133",
+ "modified": "2022-08-05 18:33:27.880783",
  "modified_by": "Administrator",
  "module": "Geo",
  "name": "Country",
@@ -69,7 +69,6 @@
    "read": 1,
    "report": 1,
    "role": "System Manager",
-   "set_user_permissions": 1,
    "share": 1,
    "write": 1
   },
@@ -84,5 +83,7 @@
  "quick_entry": 1,
  "sort_field": "country_name",
  "sort_order": "ASC",
- "track_changes": 1
-}
\ No newline at end of file
+ "states": [],
+ "track_changes": 1,
+ "translated_doctype": 1
+}
diff --git a/frappe/geo/doctype/country/country.py b/frappe/geo/doctype/country/country.py
index c6edb38e94..8b1ec1364f 100644
--- a/frappe/geo/doctype/country/country.py
+++ b/frappe/geo/doctype/country/country.py
@@ -1,8 +1,63 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
+# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
 
-from frappe.model.document import Document
+import frappe
+from frappe.model.document import Document, bulk_insert
 
 
 class Country(Document):
+	# NOTE: During installation country docs are bulk inserted.
 	pass
+
+
+def import_country_and_currency():
+	from frappe.geo.doctype.currency.currency import enable_default_currencies
+
+	countries, currencies = get_countries_and_currencies()
+
+	bulk_insert("Country", countries, ignore_duplicates=True)
+	bulk_insert("Currency", currencies, ignore_duplicates=True)
+
+	enable_default_currencies()
+
+
+def get_countries_and_currencies():
+	from frappe.geo.country_info import get_all as get_geo_data
+
+	data = get_geo_data()
+
+	countries = []
+	currencies = []
+
+	added_currencies = set()
+
+	for name, country in data.items():
+		country = frappe._dict(country)
+		countries.append(
+			frappe.get_doc(
+				doctype="Country",
+				name=name,
+				country_name=name,
+				code=country.code,
+				date_format=country.date_format or "dd-mm-yyyy",
+				time_format=country.time_format or "HH:mm:ss",
+				time_zones="\n".join(country.timezones or []),
+			)
+		)
+		if country.currency and country.currency not in added_currencies:
+			added_currencies.add(country.currency)
+
+			currencies.append(
+				frappe.get_doc(
+					doctype="Currency",
+					name=country.currency,
+					currency_name=country.currency,
+					fraction=country.currency_fraction,
+					symbol=country.currency_symbol,
+					fraction_units=country.currency_fraction_units,
+					smallest_currency_fraction_value=country.smallest_currency_fraction_value,
+					number_format=country.number_format,
+				)
+			)
+
+	return countries, currencies
diff --git a/frappe/geo/doctype/country/test_country.py b/frappe/geo/doctype/country/test_country.py
index ecc2fb6863..cf4590aabc 100644
--- a/frappe/geo/doctype/country/test_country.py
+++ b/frappe/geo/doctype/country/test_country.py
@@ -2,5 +2,52 @@
 # License: MIT. See LICENSE
 
 import frappe
+from frappe.geo.doctype.country.country import (
+	get_countries_and_currencies,
+	import_country_and_currency,
+)
+from frappe.geo.doctype.currency.currency import enable_default_currencies
+from frappe.tests.utils import FrappeTestCase
 
 test_records = frappe.get_test_records("Country")
+
+
+def get_table_snapshot(doctype):
+	data = frappe.db.sql(f"select * from `tab{doctype}` order by name", as_dict=True)
+
+	inconsequential_keys = ["modified", "creation"]
+	for row in data:
+		for key in inconsequential_keys:
+			row.pop(key, None)
+	return data
+
+
+class TestCountry(FrappeTestCase):
+	def test_bulk_insert_correctness(self):
+		def clear_tables():
+			frappe.db.delete("Currency")
+			frappe.db.delete("Country")
+
+		# Clear data
+		clear_tables()
+
+		# Reimport and verify same results
+		import_country_and_currency()
+
+		countries_before = get_table_snapshot("Country")
+		currencies_before = get_table_snapshot("Currency")
+
+		clear_tables()
+
+		countries, currencies = get_countries_and_currencies()
+		for country in countries:
+			country.db_insert(ignore_if_duplicate=True)
+		for currency in currencies:
+			currency.db_insert(ignore_if_duplicate=True)
+		enable_default_currencies()
+
+		countries_after = get_table_snapshot("Country")
+		currencies_after = get_table_snapshot("Currency")
+
+		self.assertEqual(countries_before, countries_after)
+		self.assertEqual(currencies_before, currencies_after)
diff --git a/frappe/geo/doctype/currency/currency.js b/frappe/geo/doctype/currency/currency.js
index af2d6ebc4e..08915893a5 100644
--- a/frappe/geo/doctype/currency/currency.js
+++ b/frappe/geo/doctype/currency/currency.js
@@ -1,11 +1,11 @@
 // Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 // License: See license.txt
 
-frappe.ui.form.on('Currency', {
+frappe.ui.form.on("Currency", {
 	refresh(frm) {
 		frm.set_intro("");
-		if(!frm.doc.enabled) {
+		if (!frm.doc.enabled) {
 			frm.set_intro(__("This Currency is disabled. Enable to use in transactions"));
 		}
-	}
+	},
 });
diff --git a/frappe/geo/doctype/currency/currency.py b/frappe/geo/doctype/currency/currency.py
index 93bcc063f8..51317e8771 100644
--- a/frappe/geo/doctype/currency/currency.py
+++ b/frappe/geo/doctype/currency/currency.py
@@ -4,8 +4,14 @@
 import frappe
 from frappe.model.document import Document
 
+DEFAULT_ENABLED_CURRENCIES = ("INR", "USD", "GBP", "EUR", "AED", "AUD", "JPY", "CNY", "CHF")
+
 
 class Currency(Document):
+	# NOTE: During installation country docs are bulk inserted.
 	def validate(self):
-		if not frappe.flags.in_install_app:
-			frappe.clear_cache()
+		frappe.clear_cache()
+
+
+def enable_default_currencies():
+	frappe.db.set_value("Currency", {"name": ("in", DEFAULT_ENABLED_CURRENCIES)}, "enabled", 1)
diff --git a/frappe/geo/utils.py b/frappe/geo/utils.py
index c8e05be357..d89e2a16cd 100644
--- a/frappe/geo/utils.py
+++ b/frappe/geo/utils.py
@@ -1,8 +1,6 @@
 # Copyright (c) 2020, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
-from pymysql import InternalError
-
 import frappe
 
 
@@ -53,7 +51,7 @@ def create_gps_markers(coords):
 	for i in coords:
 		node = {"type": "Feature", "properties": {}, "geometry": {"type": "Point", "coordinates": None}}
 		node["properties"]["name"] = i.name
-		node["geometry"]["coordinates"] = [i.latitude, i.longitude]
+		node["geometry"]["coordinates"] = [i.longitude, i.latitude]  # geojson needs it reverse!
 		geojson_dict.append(node.copy())
 
 	return geojson_dict
@@ -66,7 +64,7 @@ def return_location(doctype, filters_sql):
 			coords = frappe.db.sql(
 				f"""SELECT name, location FROM `tab{doctype}`  WHERE {filters_sql}""", as_dict=True
 			)
-		except InternalError:
+		except frappe.db.InternalError:
 			frappe.msgprint(frappe._("This Doctype does not contain location fields"), raise_exception=True)
 			return
 	else:
@@ -82,7 +80,7 @@ def return_coordinates(doctype, filters_sql):
 				f"""SELECT name, latitude, longitude FROM `tab{doctype}`  WHERE {filters_sql}""",
 				as_dict=True,
 			)
-		except InternalError:
+		except frappe.db.InternalError:
 			frappe.msgprint(
 				frappe._("This Doctype does not contain latitude and longitude fields"), raise_exception=True
 			)
diff --git a/frappe/handler.py b/frappe/handler.py
index cee6d3fbde..58241c1223 100644
--- a/frappe/handler.py
+++ b/frappe/handler.py
@@ -12,6 +12,7 @@ import frappe.sessions
 import frappe.utils
 from frappe import _, is_whitelisted
 from frappe.core.doctype.server_script.server_script_utils import get_server_script_map
+from frappe.monitor import add_data_to_monitor
 from frappe.utils import cint
 from frappe.utils.csvutils import build_csv_response
 from frappe.utils.image import optimize_image
@@ -32,6 +33,8 @@ ALLOWED_MIMETYPES = (
 	"application/vnd.oasis.opendocument.text",
 	"application/vnd.oasis.opendocument.spreadsheet",
 	"text/plain",
+	"video/quicktime",
+	"video/mp4",
 )
 
 
@@ -195,7 +198,7 @@ def upload_file():
 		filename = file.filename
 
 		content_type = guess_type(filename)[0]
-		if optimize and content_type.startswith("image/"):
+		if optimize and content_type and content_type.startswith("image/"):
 			args = {"content": content, "content_type": content_type}
 			if frappe.form_dict.max_width:
 				args["max_width"] = int(frappe.form_dict.max_width)
@@ -269,7 +272,7 @@ def ping():
 
 def run_doc_method(method, docs=None, dt=None, dn=None, arg=None, args=None):
 	"""run a whitelisted controller method"""
-	from inspect import getfullargspec
+	from inspect import signature
 
 	if not args and arg:
 		args = arg
@@ -298,7 +301,7 @@ def run_doc_method(method, docs=None, dt=None, dn=None, arg=None, args=None):
 	is_whitelisted(fn)
 	is_valid_http_method(fn)
 
-	fnargs = getfullargspec(method_obj).args
+	fnargs = list(signature(method_obj).parameters)
 
 	if not fnargs or (len(fnargs) == 1 and fnargs[0] == "self"):
 		response = doc.run_method(method)
@@ -320,6 +323,8 @@ def run_doc_method(method, docs=None, dt=None, dn=None, arg=None, args=None):
 
 	frappe.response["message"] = response
 
+	add_data_to_monitor(methodname=method)
+
 
 # for backwards compatibility
 runserverobj = run_doc_method
diff --git a/frappe/hooks.py b/frappe/hooks.py
index a337d8e0d3..6d8c00d483 100644
--- a/frappe/hooks.py
+++ b/frappe/hooks.py
@@ -8,7 +8,7 @@ source_link = "https://github.com/frappe/frappe"
 app_license = "MIT"
 app_logo_url = "/assets/frappe/images/frappe-framework-logo.svg"
 
-develop_version = "14.x.x-develop"
+develop_version = "15.x.x-develop"
 
 app_email = "developers@frappe.io"
 
@@ -29,6 +29,7 @@ app_include_js = [
 	"form.bundle.js",
 	"controls.bundle.js",
 	"report.bundle.js",
+	"telemetry.bundle.js",
 ]
 app_include_css = [
 	"desk.bundle.css",
@@ -83,6 +84,11 @@ on_logout = (
 	"frappe.core.doctype.session_default_settings.session_default_settings.clear_session_defaults"
 )
 
+# PDF
+pdf_header_html = "frappe.utils.pdf.pdf_header_html"
+pdf_body_html = "frappe.utils.pdf.pdf_body_html"
+pdf_footer_html = "frappe.utils.pdf.pdf_footer_html"
+
 # permissions
 
 permission_query_conditions = {
@@ -108,7 +114,6 @@ has_permission = {
 	"Event": "frappe.desk.doctype.event.event.has_permission",
 	"ToDo": "frappe.desk.doctype.todo.todo.has_permission",
 	"User": "frappe.core.doctype.user.user.has_permission",
-	"Note": "frappe.desk.doctype.note.note.has_permission",
 	"Dashboard Chart": "frappe.desk.doctype.dashboard_chart.dashboard_chart.has_permission",
 	"Number Card": "frappe.desk.doctype.number_card.number_card.has_permission",
 	"Kanban Board": "frappe.desk.doctype.kanban_board.kanban_board.has_permission",
@@ -138,16 +143,11 @@ standard_queries = {"User": "frappe.core.doctype.user.user.user_query"}
 
 doc_events = {
 	"*": {
-		"after_insert": [
-			"frappe.event_streaming.doctype.event_update_log.event_update_log.notify_consumers"
-		],
 		"on_update": [
 			"frappe.desk.notifications.clear_doctype_notifications",
-			"frappe.core.doctype.activity_log.feed.update_feed",
 			"frappe.workflow.doctype.workflow_action.workflow_action.process_workflow_actions",
 			"frappe.automation.doctype.assignment_rule.assignment_rule.apply",
 			"frappe.core.doctype.file.utils.attach_files_to_document",
-			"frappe.event_streaming.doctype.event_update_log.event_update_log.notify_consumers",
 			"frappe.automation.doctype.assignment_rule.assignment_rule.update_due_date",
 			"frappe.core.doctype.user_type.user_type.apply_permissions_for_non_standard_user_type",
 		],
@@ -155,12 +155,10 @@ doc_events = {
 		"on_cancel": [
 			"frappe.desk.notifications.clear_doctype_notifications",
 			"frappe.workflow.doctype.workflow_action.workflow_action.process_workflow_actions",
-			"frappe.event_streaming.doctype.event_update_log.event_update_log.notify_consumers",
 		],
 		"on_trash": [
 			"frappe.desk.notifications.clear_doctype_notifications",
 			"frappe.workflow.doctype.workflow_action.workflow_action.process_workflow_actions",
-			"frappe.event_streaming.doctype.event_update_log.event_update_log.notify_consumers",
 		],
 		"on_update_after_submit": [
 			"frappe.workflow.doctype.workflow_action.workflow_action.process_workflow_actions"
@@ -180,12 +178,10 @@ doc_events = {
 		"on_update": "frappe.integrations.doctype.google_contacts.google_contacts.update_contacts_to_google_contacts",
 	},
 	"DocType": {
-		"after_insert": "frappe.cache_manager.build_domain_restriced_doctype_cache",
-		"after_save": "frappe.cache_manager.build_domain_restriced_doctype_cache",
+		"on_update": "frappe.cache_manager.build_domain_restriced_doctype_cache",
 	},
 	"Page": {
-		"after_insert": "frappe.cache_manager.build_domain_restriced_page_cache",
-		"after_save": "frappe.cache_manager.build_domain_restriced_page_cache",
+		"on_update": "frappe.cache_manager.build_domain_restriced_page_cache",
 	},
 }
 
@@ -195,15 +191,17 @@ scheduler_events = {
 			"frappe.oauth.delete_oauth2_data",
 			"frappe.website.doctype.web_page.web_page.check_publish_status",
 			"frappe.twofactor.delete_all_barcodes_for_users",
-		]
+		],
+		"0/10 * * * *": [
+			"frappe.email.doctype.email_account.email_account.pull",
+		],
 	},
 	"all": [
 		"frappe.email.queue.flush",
-		"frappe.email.doctype.email_account.email_account.pull",
 		"frappe.email.doctype.email_account.email_account.notify_unreplied",
-		"frappe.integrations.doctype.razorpay_settings.razorpay_settings.capture_payment",
 		"frappe.utils.global_search.sync_global_search",
 		"frappe.monitor.flush",
+		"frappe.automation.doctype.reminder.reminder.send_reminders",
 	],
 	"hourly": [
 		"frappe.model.utils.link_count.update_link_count",
@@ -229,7 +227,6 @@ scheduler_events = {
 		"frappe.automation.doctype.auto_repeat.auto_repeat.make_auto_repeat_entry",
 		"frappe.automation.doctype.auto_repeat.auto_repeat.set_auto_repeat_as_completed",
 		"frappe.email.doctype.unhandled_email.unhandled_email.remove_old_unhandled_emails",
-		"frappe.core.doctype.prepared_report.prepared_report.delete_expired_prepared_reports",
 		"frappe.core.doctype.log_settings.log_settings.run_log_clean_up",
 	],
 	"daily_long": [
@@ -276,7 +273,7 @@ setup_wizard_exception = [
 	"frappe.desk.page.setup_wizard.setup_wizard.log_setup_wizard_exception",
 ]
 
-before_migrate = []
+before_migrate = ["frappe.core.doctype.patch_log.patch_log.before_migrate"]
 after_migrate = ["frappe.website.doctype.website_theme.website_theme.after_migrate"]
 
 otp_methods = ["OTP App", "Email", "SMS"]
@@ -366,6 +363,7 @@ global_search_doctypes = {
 }
 
 override_whitelisted_methods = {
+	# Legacy File APIs
 	"frappe.core.doctype.file.file.download_file": "download_file",
 	"frappe.core.doctype.file.file.unzip_file": "frappe.core.api.file.unzip_file",
 	"frappe.core.doctype.file.file.get_attached_images": "frappe.core.api.file.get_attached_images",
@@ -375,6 +373,53 @@ override_whitelisted_methods = {
 	"frappe.core.doctype.file.file.create_new_folder": "frappe.core.api.file.create_new_folder",
 	"frappe.core.doctype.file.file.move_file": "frappe.core.api.file.move_file",
 	"frappe.core.doctype.file.file.zip_files": "frappe.core.api.file.zip_files",
+	# Legacy (& Consistency) OAuth2 APIs
+	"frappe.www.login.login_via_google": "frappe.integrations.oauth2_logins.login_via_google",
+	"frappe.www.login.login_via_github": "frappe.integrations.oauth2_logins.login_via_github",
+	"frappe.www.login.login_via_facebook": "frappe.integrations.oauth2_logins.login_via_facebook",
+	"frappe.www.login.login_via_frappe": "frappe.integrations.oauth2_logins.login_via_frappe",
+	"frappe.www.login.login_via_office365": "frappe.integrations.oauth2_logins.login_via_office365",
+	"frappe.www.login.login_via_salesforce": "frappe.integrations.oauth2_logins.login_via_salesforce",
+	"frappe.www.login.login_via_fairlogin": "frappe.integrations.oauth2_logins.login_via_fairlogin",
 }
 
-translated_search_doctypes = ["DocType", "Role", "Country", "Gender", "Salutation"]
+ignore_links_on_delete = [
+	"Communication",
+	"ToDo",
+	"DocShare",
+	"Email Unsubscribe",
+	"Activity Log",
+	"File",
+	"Version",
+	"Document Follow",
+	"Comment",
+	"View Log",
+	"Tag Link",
+	"Notification Log",
+	"Email Queue",
+	"Document Share Key",
+	"Integration Request",
+	"Unhandled Email",
+	"Webhook Request Log",
+]
+
+# Request Hooks
+before_request = [
+	"frappe.recorder.record",
+	"frappe.monitor.start",
+	"frappe.rate_limiter.apply",
+]
+after_request = ["frappe.rate_limiter.update", "frappe.monitor.stop", "frappe.recorder.dump"]
+
+# Background Job Hooks
+before_job = [
+	"frappe.monitor.start",
+]
+after_job = [
+	"frappe.monitor.stop",
+	"frappe.utils.file_lock.release_document_locks",
+]
+
+extend_bootinfo = [
+	"frappe.utils.telemetry.add_bootinfo",
+]
diff --git a/frappe/installer.py b/frappe/installer.py
index 9d35f04e0d..9c2807d7cd 100644
--- a/frappe/installer.py
+++ b/frappe/installer.py
@@ -1,16 +1,35 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
+# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
 
 import json
 import os
+import re
+import subprocess
 import sys
 from collections import OrderedDict
+from contextlib import suppress
+from shutil import which
 
 import click
 
 import frappe
 from frappe.defaults import _clear_cache
-from frappe.utils import is_git_url
+from frappe.utils import cint, is_git_url
+from frappe.utils.dashboard import sync_dashboards
+from frappe.utils.synchronization import filelock
+
+
+def _is_scheduler_enabled() -> bool:
+	enable_scheduler = False
+	try:
+		frappe.connect()
+		enable_scheduler = cint(frappe.db.get_single_value("System Settings", "enable_scheduler"))
+	except Exception:
+		pass
+	finally:
+		frappe.db.close()
+
+	return bool(enable_scheduler)
 
 
 def _new_site(
@@ -29,11 +48,9 @@ def _new_site(
 	db_type=None,
 	db_host=None,
 	db_port=None,
-	new_site=False,
 ):
 	"""Install a new Frappe site"""
 
-	from frappe.commands.scheduler import _is_scheduler_enabled
 	from frappe.utils import get_site_path, scheduler, touch_file
 
 	if not force and os.path.exists(site):
@@ -59,40 +76,38 @@ def _new_site(
 
 	make_site_dirs()
 
-	installing = touch_file(get_site_path("locks", "installing.lock"))
+	with filelock("bench_new_site", timeout=1):
+		install_db(
+			root_login=db_root_username,
+			root_password=db_root_password,
+			db_name=db_name,
+			admin_password=admin_password,
+			verbose=verbose,
+			source_sql=source_sql,
+			force=force,
+			reinstall=reinstall,
+			db_password=db_password,
+			db_type=db_type,
+			db_host=db_host,
+			db_port=db_port,
+			no_mariadb_socket=no_mariadb_socket,
+		)
 
-	install_db(
-		root_login=db_root_username,
-		root_password=db_root_password,
-		db_name=db_name,
-		admin_password=admin_password,
-		verbose=verbose,
-		source_sql=source_sql,
-		force=force,
-		reinstall=reinstall,
-		db_password=db_password,
-		db_type=db_type,
-		db_host=db_host,
-		db_port=db_port,
-		no_mariadb_socket=no_mariadb_socket,
-	)
-	apps_to_install = (
-		["frappe"] + (frappe.conf.get("install_apps") or []) + (list(install_apps) or [])
-	)
+		apps_to_install = (
+			["frappe"] + (frappe.conf.get("install_apps") or []) + (list(install_apps) or [])
+		)
 
-	for app in apps_to_install:
-		# NOTE: not using force here for 2 reasons:
-		# 	1. It's not really needed here as we've freshly installed a new db
-		# 	2. If someone uses a sql file to do restore and that file already had
-		# 		installed_apps then it might cause problems as that sql file can be of any previous version(s)
-		# 		which might be incompatible with the current version and using force might cause problems.
-		# 		Example: the DocType DocType might not have `migration_hash` column which will cause failure in the restore.
-		install_app(app, verbose=verbose, set_as_patched=not source_sql, force=False)
+		for app in apps_to_install:
+			# NOTE: not using force here for 2 reasons:
+			# 	1. It's not really needed here as we've freshly installed a new db
+			# 	2. If someone uses a sql file to do restore and that file already had
+			# 		installed_apps then it might cause problems as that sql file can be of any previous version(s)
+			# 		which might be incompatible with the current version and using force might cause problems.
+			# 		Example: the DocType DocType might not have `migration_hash` column which will cause failure in the restore.
+			install_app(app, verbose=verbose, set_as_patched=not source_sql, force=False)
 
-	os.remove(installing)
-
-	scheduler.toggle_scheduler(enable_scheduler)
-	frappe.db.commit()
+		scheduler.toggle_scheduler(enable_scheduler)
+		frappe.db.commit()
 
 	scheduler_status = "disabled" if frappe.utils.scheduler.is_scheduler_disabled() else "enabled"
 	print("*** Scheduler is", scheduler_status, "***")
@@ -227,7 +242,7 @@ def parse_app_name(name: str) -> str:
 			_repo = name.split(":")[1].rsplit("/", 1)[1]
 		else:
 			_repo = name.rsplit("/", 2)[2]
-		repo = _repo.split(".")[0]
+		repo = _repo.split(".", 1)[0]
 	else:
 		_, repo, _ = fetch_details_from_tag(name)
 	return repo
@@ -256,7 +271,7 @@ def install_app(name, verbose=False, set_as_patched=True, force=False):
 	frappe.clear_cache()
 
 	if name not in frappe.get_all_apps():
-		raise Exception("App not in apps.txt")
+		raise Exception(f"App {name} not in apps.txt")
 
 	if not force and name in installed_apps:
 		click.secho(f"App {name} already installed", fg="yellow")
@@ -290,6 +305,7 @@ def install_app(name, verbose=False, set_as_patched=True, force=False):
 	sync_jobs()
 	sync_fixtures(name)
 	sync_customizations(name)
+	sync_dashboards(name)
 
 	for after_sync in app_hooks.after_sync or []:
 		frappe.get_attr(after_sync)()  #
@@ -389,7 +405,7 @@ def _delete_modules(modules: list[str], dry_run: bool) -> list[str]:
 
 			if not dry_run:
 				if doctype.issingle:
-					frappe.delete_doc("DocType", doctype.name, ignore_on_trash=True)
+					frappe.delete_doc("DocType", doctype.name, ignore_on_trash=True, force=True)
 				else:
 					drop_doctypes.append(doctype.name)
 
@@ -447,7 +463,7 @@ def _delete_doctypes(doctypes: list[str], dry_run: bool) -> None:
 	for doctype in set(doctypes):
 		print(f"* dropping Table for '{doctype}'...")
 		if not dry_run:
-			frappe.delete_doc("DocType", doctype, ignore_on_trash=True)
+			frappe.delete_doc("DocType", doctype, ignore_on_trash=True, force=True)
 			frappe.db.sql_ddl(f"DROP TABLE IF EXISTS `tab{doctype}`")
 
 
@@ -482,7 +498,7 @@ def init_singles():
 			doc.flags.ignore_mandatory = True
 			doc.flags.ignore_validate = True
 			doc.save()
-		except ImportError:
+		except (ImportError, frappe.DoesNotExistError):
 			# The doctype exists, but controller is deleted,
 			# no need to attempt to init such single, ref: #16917
 			continue
@@ -525,10 +541,21 @@ def make_site_config(
 
 def update_site_config(key, value, validate=True, site_config_path=None):
 	"""Update a value in site_config"""
+	from frappe.utils.synchronization import filelock
+
 	if not site_config_path:
 		site_config_path = get_site_config_path()
 
-	with open(site_config_path) as f:
+	# Sometimes global config file is passed directly to this function
+	_is_global_conf = "common_site_config" in site_config_path
+
+	with filelock("site_config", is_global=_is_global_conf):
+		_update_config_file(key=key, value=value, config_file=site_config_path)
+
+
+def _update_config_file(key: str, value, config_file: str):
+	"""Updates site or common config"""
+	with open(config_file) as f:
 		site_config = json.loads(f.read())
 
 	# In case of non-int value
@@ -548,7 +575,7 @@ def update_site_config(key, value, validate=True, site_config_path=None):
 	else:
 		site_config[key] = value
 
-	with open(site_config_path, "w") as f:
+	with open(config_file, "w") as f:
 		f.write(json.dumps(site_config, indent=1, sort_keys=True))
 
 	if hasattr(frappe.local, "conf"):
@@ -640,10 +667,22 @@ def convert_archive_content(sql_file_path):
 	if frappe.conf.db_type == "mariadb":
 		# ever since mariaDB 10.6, row_format COMPRESSED has been deprecated and removed
 		# this step is added to ease restoring sites depending on older mariaDB servers
+		# This change was reverted by mariadb in 10.6.6
+		# Ref: https://mariadb.com/kb/en/innodb-compressed-row-format/#read-only
 		from pathlib import Path
 
 		from frappe.utils import random_string
 
+		version = _guess_mariadb_version()
+		if not version or (version <= (10, 6, 0) or version >= (10, 6, 6)):
+			return
+
+		click.secho(
+			"MariaDB version being used does not support ROW_FORMAT=COMPRESSED, "
+			"converting into DYNAMIC format.",
+			fg="yellow",
+		)
+
 		old_sql_file_path = Path(f"{sql_file_path}_{random_string(10)}")
 		sql_file_path = Path(sql_file_path)
 
@@ -671,6 +710,20 @@ def extract_sql_gzip(sql_gz_path):
 	return decompressed_file
 
 
+def _guess_mariadb_version() -> tuple[int] | None:
+	# Using command-line because we *might* not have a connection yet and this command is required
+	# in non-interactive mode.
+	# Use db.sql("select version()") instead if connection is available.
+	with suppress(Exception):
+		mysql = which("mysql")
+		version_output = subprocess.getoutput(f"{mysql} --version")
+		version_regex = r"(?P\d+\.\d+\.\d+)-MariaDB"
+
+		version = re.search(version_regex, version_output).group("version")
+
+		return tuple(int(v) for v in version.split("."))
+
+
 def extract_files(site_name, file_path):
 	import shutil
 	import subprocess
@@ -732,7 +785,7 @@ def is_downgrade(sql_file_path, verbose=False):
 
 				for app in all_apps:
 					app_name = app[0]
-					app_version = app[1].split(" ")[0]
+					app_version = app[1].split(" ", 1)[0]
 
 					if app_name == "frappe":
 						try:
diff --git a/frappe/integrations/doctype/braintree_settings/__init__.py b/frappe/integrations/doctype/braintree_settings/__init__.py
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/frappe/integrations/doctype/braintree_settings/braintree_settings.js b/frappe/integrations/doctype/braintree_settings/braintree_settings.js
deleted file mode 100644
index c844022cec..0000000000
--- a/frappe/integrations/doctype/braintree_settings/braintree_settings.js
+++ /dev/null
@@ -1,6 +0,0 @@
-// Copyright (c) 2018, Frappe Technologies and contributors
-// For license information, please see license.txt
-
-frappe.ui.form.on('Braintree Settings', {
-
-});
diff --git a/frappe/integrations/doctype/braintree_settings/braintree_settings.json b/frappe/integrations/doctype/braintree_settings/braintree_settings.json
deleted file mode 100644
index eebf64dfd1..0000000000
--- a/frappe/integrations/doctype/braintree_settings/braintree_settings.json
+++ /dev/null
@@ -1,273 +0,0 @@
-{
- "allow_copy": 0, 
- "allow_guest_to_view": 0, 
- "allow_import": 0, 
- "allow_rename": 0, 
- "autoname": "field:gateway_name", 
- "beta": 0, 
- "creation": "2018-02-05 13:46:12.101852", 
- "custom": 0, 
- "docstatus": 0, 
- "doctype": "DocType", 
- "document_type": "", 
- "editable_grid": 1, 
- "engine": "InnoDB", 
- "fields": [
-  {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "gateway_name", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Payment Gateway Name", 
-   "length": 0, 
-   "no_copy": 0, 
-   "options": "Company", 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "section_break_2", 
-   "fieldtype": "Section Break", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "merchant_id", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Merchant ID", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "public_key", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Public Key", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "private_key", 
-   "fieldtype": "Password", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Private Key", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "use_sandbox", 
-   "fieldtype": "Check", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Use Sandbox", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "header_img", 
-   "fieldtype": "Attach Image", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Header Image", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }
- ], 
- "has_web_view": 0, 
- "hide_heading": 0, 
- "hide_toolbar": 0, 
- "idx": 0, 
- "image_view": 0, 
- "in_create": 0, 
- "is_submittable": 0, 
- "issingle": 0, 
- "istable": 0, 
- "max_attachments": 0, 
- "modified": "2018-02-05 14:33:06.050377", 
- "modified_by": "Administrator", 
- "module": "Integrations", 
- "name": "Braintree Settings", 
- "name_case": "", 
- "owner": "Administrator", 
- "permissions": [
-  {
-   "amend": 0, 
-   "apply_user_permissions": 0, 
-   "cancel": 0, 
-   "create": 1, 
-   "delete": 1, 
-   "email": 1, 
-   "export": 0, 
-   "if_owner": 0, 
-   "import": 0, 
-   "permlevel": 0, 
-   "print": 1, 
-   "read": 1, 
-   "report": 0, 
-   "role": "System Manager", 
-   "set_user_permissions": 0, 
-   "share": 1, 
-   "submit": 0, 
-   "write": 1
-  }
- ], 
- "quick_entry": 1, 
- "read_only": 0, 
- "read_only_onload": 0, 
- "show_name_in_global_search": 0, 
- "sort_field": "modified", 
- "sort_order": "DESC", 
- "track_changes": 0, 
- "track_seen": 0
-}
\ No newline at end of file
diff --git a/frappe/integrations/doctype/braintree_settings/braintree_settings.py b/frappe/integrations/doctype/braintree_settings/braintree_settings.py
deleted file mode 100644
index 35481c67c1..0000000000
--- a/frappe/integrations/doctype/braintree_settings/braintree_settings.py
+++ /dev/null
@@ -1,287 +0,0 @@
-# Copyright (c) 2018, Frappe Technologies and contributors
-# License: MIT. See LICENSE
-
-from urllib.parse import urlencode
-
-import braintree
-
-import frappe
-from frappe import _
-from frappe.integrations.utils import create_payment_gateway, create_request_log
-from frappe.model.document import Document
-from frappe.utils import call_hook_method, get_url
-
-
-class BraintreeSettings(Document):
-	supported_currencies = [
-		"AED",
-		"AMD",
-		"AOA",
-		"ARS",
-		"AUD",
-		"AWG",
-		"AZN",
-		"BAM",
-		"BBD",
-		"BDT",
-		"BGN",
-		"BIF",
-		"BMD",
-		"BND",
-		"BOB",
-		"BRL",
-		"BSD",
-		"BWP",
-		"BYN",
-		"BZD",
-		"CAD",
-		"CHF",
-		"CLP",
-		"CNY",
-		"COP",
-		"CRC",
-		"CVE",
-		"CZK",
-		"DJF",
-		"DKK",
-		"DOP",
-		"DZD",
-		"EGP",
-		"ETB",
-		"EUR",
-		"FJD",
-		"FKP",
-		"GBP",
-		"GEL",
-		"GHS",
-		"GIP",
-		"GMD",
-		"GNF",
-		"GTQ",
-		"GYD",
-		"HKD",
-		"HNL",
-		"HRK",
-		"HTG",
-		"HUF",
-		"IDR",
-		"ILS",
-		"INR",
-		"ISK",
-		"JMD",
-		"JPY",
-		"KES",
-		"KGS",
-		"KHR",
-		"KMF",
-		"KRW",
-		"KYD",
-		"KZT",
-		"LAK",
-		"LBP",
-		"LKR",
-		"LRD",
-		"LSL",
-		"LTL",
-		"MAD",
-		"MDL",
-		"MKD",
-		"MNT",
-		"MOP",
-		"MUR",
-		"MVR",
-		"MWK",
-		"MXN",
-		"MYR",
-		"MZN",
-		"NAD",
-		"NGN",
-		"NIO",
-		"NOK",
-		"NPR",
-		"NZD",
-		"PAB",
-		"PEN",
-		"PGK",
-		"PHP",
-		"PKR",
-		"PLN",
-		"PYG",
-		"QAR",
-		"RON",
-		"RSD",
-		"RUB",
-		"RWF",
-		"SAR",
-		"SBD",
-		"SCR",
-		"SEK",
-		"SGD",
-		"SHP",
-		"SLL",
-		"SOS",
-		"SRD",
-		"STD",
-		"SVC",
-		"SYP",
-		"SZL",
-		"THB",
-		"TJS",
-		"TOP",
-		"TRY",
-		"TTD",
-		"TWD",
-		"TZS",
-		"UAH",
-		"UGX",
-		"USD",
-		"UYU",
-		"UZS",
-		"VEF",
-		"VND",
-		"VUV",
-		"WST",
-		"XAF",
-		"XCD",
-		"XOF",
-		"XPF",
-		"YER",
-		"ZAR",
-		"ZMK",
-		"ZWD",
-	]
-
-	def validate(self):
-		if not self.flags.ignore_mandatory:
-			self.configure_braintree()
-
-	def on_update(self):
-		create_payment_gateway(
-			"Braintree-" + self.gateway_name, settings="Braintree Settings", controller=self.gateway_name
-		)
-		call_hook_method("payment_gateway_enabled", gateway="Braintree-" + self.gateway_name)
-
-	def configure_braintree(self):
-		if self.use_sandbox:
-			environment = "sandbox"
-		else:
-			environment = "production"
-
-		braintree.Configuration.configure(
-			environment=environment,
-			merchant_id=self.merchant_id,
-			public_key=self.public_key,
-			private_key=self.get_password(fieldname="private_key", raise_exception=False),
-		)
-
-	def validate_transaction_currency(self, currency):
-		if currency not in self.supported_currencies:
-			frappe.throw(
-				_(
-					"Please select another payment method. Stripe does not support transactions in currency '{0}'"
-				).format(currency)
-			)
-
-	def get_payment_url(self, **kwargs):
-		return get_url(f"./integrations/braintree_checkout?{urlencode(kwargs)}")
-
-	def create_payment_request(self, data):
-		self.data = frappe._dict(data)
-
-		try:
-			self.integration_request = create_request_log(self.data, service_name="Braintree")
-			return self.create_charge_on_braintree()
-
-		except Exception:
-			frappe.log_error(frappe.get_traceback())
-			return {
-				"redirect_to": frappe.redirect_to_message(
-					_("Server Error"),
-					_(
-						"There seems to be an issue with the server's braintree configuration. Don't worry, in case of failure, the amount will get refunded to your account."
-					),
-				),
-				"status": 401,
-			}
-
-	def create_charge_on_braintree(self):
-		self.configure_braintree()
-
-		redirect_to = self.data.get("redirect_to") or None
-		redirect_message = self.data.get("redirect_message") or None
-
-		result = braintree.Transaction.sale(
-			{
-				"amount": self.data.amount,
-				"payment_method_nonce": self.data.payload_nonce,
-				"options": {"submit_for_settlement": True},
-			}
-		)
-
-		if result.is_success:
-			self.integration_request.db_set("status", "Completed", update_modified=False)
-			self.flags.status_changed_to = "Completed"
-			self.integration_request.db_set("output", result.transaction.status, update_modified=False)
-
-		elif result.transaction:
-			self.integration_request.db_set("status", "Failed", update_modified=False)
-			error_log = frappe.log_error(
-				"code: "
-				+ str(result.transaction.processor_response_code)
-				+ " | text: "
-				+ str(result.transaction.processor_response_text),
-				"Braintree Payment Error",
-			)
-			self.integration_request.db_set("error", error_log.error, update_modified=False)
-		else:
-			self.integration_request.db_set("status", "Failed", update_modified=False)
-			for error in result.errors.deep_errors:
-				error_log = frappe.log_error(
-					"code: " + str(error.code) + " | message: " + str(error.message), "Braintree Payment Error"
-				)
-				self.integration_request.db_set("error", error_log.error, update_modified=False)
-
-		if self.flags.status_changed_to == "Completed":
-			status = "Completed"
-			if self.data.reference_doctype and self.data.reference_docname:
-				custom_redirect_to = None
-				try:
-					custom_redirect_to = frappe.get_doc(
-						self.data.reference_doctype, self.data.reference_docname
-					).run_method("on_payment_authorized", self.flags.status_changed_to)
-					braintree_success_page = frappe.get_hooks("braintree_success_page")
-					if braintree_success_page:
-						custom_redirect_to = frappe.get_attr(braintree_success_page[-1])(self.data)
-				except Exception:
-					frappe.log_error(frappe.get_traceback())
-
-				if custom_redirect_to:
-					redirect_to = custom_redirect_to
-
-			redirect_url = "payment-success"
-		else:
-			status = "Error"
-			redirect_url = "payment-failed"
-
-		if redirect_to:
-			redirect_url += "?" + urlencode({"redirect_to": redirect_to})
-		if redirect_message:
-			redirect_url += "&" + urlencode({"redirect_message": redirect_message})
-
-		return {"redirect_to": redirect_url, "status": status}
-
-
-def get_gateway_controller(doc):
-	payment_request = frappe.get_doc("Payment Request", doc)
-	gateway_controller = frappe.db.get_value(
-		"Payment Gateway", payment_request.payment_gateway, "gateway_controller"
-	)
-	return gateway_controller
-
-
-def get_client_token(doc):
-	gateway_controller = get_gateway_controller(doc)
-	settings = frappe.get_doc("Braintree Settings", gateway_controller)
-	settings.configure_braintree()
-
-	return braintree.ClientToken.generate()
diff --git a/frappe/integrations/doctype/braintree_settings/test_braintree_settings.py b/frappe/integrations/doctype/braintree_settings/test_braintree_settings.py
deleted file mode 100644
index 38d8909dfd..0000000000
--- a/frappe/integrations/doctype/braintree_settings/test_braintree_settings.py
+++ /dev/null
@@ -1,7 +0,0 @@
-# Copyright (c) 2018, Frappe Technologies and Contributors
-# License: MIT. See LICENSE
-import unittest
-
-
-class TestBraintreeSettings(unittest.TestCase):
-	pass
diff --git a/frappe/integrations/doctype/connected_app/connected_app.js b/frappe/integrations/doctype/connected_app/connected_app.js
index 4d20f65559..11dcda235e 100644
--- a/frappe/integrations/doctype/connected_app/connected_app.js
+++ b/frappe/integrations/doctype/connected_app/connected_app.js
@@ -1,38 +1,38 @@
 // Copyright (c) 2019, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Connected App', {
-	refresh: frm => {
-		frm.add_custom_button(__('Get OpenID Configuration'), async () => {
+frappe.ui.form.on("Connected App", {
+	refresh: (frm) => {
+		frm.add_custom_button(__("Get OpenID Configuration"), async () => {
 			if (!frm.doc.openid_configuration) {
-				frappe.msgprint(__('Please enter OpenID Configuration URL'));
+				frappe.msgprint(__("Please enter OpenID Configuration URL"));
 			} else {
 				try {
 					const response = await fetch(frm.doc.openid_configuration);
 					const oidc = await response.json();
-					frm.set_value('authorization_uri', oidc.authorization_endpoint);
-					frm.set_value('token_uri', oidc.token_endpoint);
-					frm.set_value('userinfo_uri', oidc.userinfo_endpoint);
-					frm.set_value('introspection_uri', oidc.introspection_endpoint);
-					frm.set_value('revocation_uri', oidc.revocation_endpoint);
+					frm.set_value("authorization_uri", oidc.authorization_endpoint);
+					frm.set_value("token_uri", oidc.token_endpoint);
+					frm.set_value("userinfo_uri", oidc.userinfo_endpoint);
+					frm.set_value("introspection_uri", oidc.introspection_endpoint);
+					frm.set_value("revocation_uri", oidc.revocation_endpoint);
 				} catch (error) {
-					frappe.msgprint(__('Please check OpenID Configuration URL'));
+					frappe.msgprint(__("Please check OpenID Configuration URL"));
 				}
 			}
 		});
 
 		if (!frm.is_new()) {
-			frm.add_custom_button(__('Connect to {}', [frm.doc.provider_name]), async () => {
+			frm.add_custom_button(__("Connect to {}", [frm.doc.provider_name]), async () => {
 				frappe.call({
-					method: 'initiate_web_application_flow',
+					method: "initiate_web_application_flow",
 					doc: frm.doc,
-					callback: function(r) {
-						window.open(r.message, '_blank');
-					}
+					callback: function (r) {
+						window.open(r.message, "_blank");
+					},
 				});
 			});
 		}
 
-		frm.toggle_display('sb_client_credentials_section', !frm.is_new());
-	}
+		frm.toggle_display("sb_client_credentials_section", !frm.is_new());
+	},
 });
diff --git a/frappe/integrations/doctype/connected_app/connected_app.py b/frappe/integrations/doctype/connected_app/connected_app.py
index 308d1ca84a..536b63fe7b 100644
--- a/frappe/integrations/doctype/connected_app/connected_app.py
+++ b/frappe/integrations/doctype/connected_app/connected_app.py
@@ -14,6 +14,8 @@ if any((os.getenv("CI"), frappe.conf.developer_mode, frappe.conf.allow_tests)):
 	# Disable mandatory TLS in developer mode and tests
 	os.environ["OAUTHLIB_INSECURE_TRANSPORT"] = "1"
 
+os.environ["OAUTHLIB_RELAX_TOKEN_SCOPE"] = "1"
+
 
 class ConnectedApp(Document):
 	"""Connect to a remote oAuth Server. Retrieve and store user's access token
@@ -57,7 +59,7 @@ class ConnectedApp(Document):
 	def initiate_web_application_flow(self, user=None, success_uri=None):
 		"""Return an authorization URL for the user. Save state in Token Cache."""
 		user = user or frappe.session.user
-		oauth = self.get_oauth2_session(init=True)
+		oauth = self.get_oauth2_session(user, init=True)
 		query_params = self.get_query_params()
 		authorization_url, state = oauth.authorization_url(self.authorization_uri, **query_params)
 		token_cache = self.get_token_cache(user)
@@ -102,8 +104,27 @@ class ConnectedApp(Document):
 	def get_query_params(self):
 		return {param.key: param.value for param in self.query_parameters}
 
+	def get_active_token(self, user=None):
+		user = user or frappe.session.user
+		token_cache = self.get_token_cache(user)
+		if token_cache and token_cache.is_expired():
+			oauth_session = self.get_oauth2_session(user)
 
-@frappe.whitelist(allow_guest=True)
+			try:
+				token = oauth_session.refresh_token(
+					body=f"redirect_uri={self.redirect_uri}",
+					token_url=self.token_uri,
+				)
+			except Exception:
+				self.log_error("Token Refresh Error")
+				return None
+
+			token_cache.update_data(token)
+
+		return token_cache
+
+
+@frappe.whitelist(methods=["GET"], allow_guest=True)
 def callback(code=None, state=None):
 	"""Handle client's code.
 
@@ -111,8 +132,6 @@ def callback(code=None, state=None):
 	transmit a code that can be used by the local server to obtain an access
 	token.
 	"""
-	if frappe.request.method != "GET":
-		frappe.throw(_("Invalid request method: {}").format(frappe.request.method))
 
 	if frappe.session.user == "Guest":
 		frappe.local.response["type"] = "redirect"
@@ -136,9 +155,16 @@ def callback(code=None, state=None):
 		code=code,
 		client_secret=connected_app.get_password("client_secret"),
 		include_client_id=True,
-		**query_params
+		**query_params,
 	)
 	token_cache.update_data(token)
 
 	frappe.local.response["type"] = "redirect"
 	frappe.local.response["location"] = token_cache.get("success_uri") or connected_app.get_url()
+
+
+@frappe.whitelist()
+def has_token(connected_app, connected_user=None):
+	app = frappe.get_doc("Connected App", connected_app)
+	token_cache = app.get_token_cache(connected_user or frappe.session.user)
+	return bool(token_cache and token_cache.get_password("access_token", False))
diff --git a/frappe/integrations/doctype/connected_app/test_connected_app.py b/frappe/integrations/doctype/connected_app/test_connected_app.py
index 1acedff160..88441db6b2 100644
--- a/frappe/integrations/doctype/connected_app/test_connected_app.py
+++ b/frappe/integrations/doctype/connected_app/test_connected_app.py
@@ -1,6 +1,5 @@
 # Copyright (c) 2019, Frappe Technologies and contributors
 # License: MIT. See LICENSE
-import unittest
 from urllib.parse import urljoin
 
 import requests
@@ -9,6 +8,7 @@ import frappe
 from frappe.integrations.doctype.social_login_key.test_social_login_key import (
 	create_or_update_social_login_key,
 )
+from frappe.tests.utils import FrappeTestCase
 
 
 def get_user(usr, pwd):
@@ -48,7 +48,7 @@ def get_oauth_client():
 	return oauth_client
 
 
-class TestConnectedApp(unittest.TestCase):
+class TestConnectedApp(FrappeTestCase):
 	def setUp(self):
 		"""Set up a Connected App that connects to our own oAuth provider.
 
diff --git a/frappe/integrations/doctype/dropbox_settings/dropbox_settings.js b/frappe/integrations/doctype/dropbox_settings/dropbox_settings.js
index ea731fafc2..6fee47f1bd 100644
--- a/frappe/integrations/doctype/dropbox_settings/dropbox_settings.js
+++ b/frappe/integrations/doctype/dropbox_settings/dropbox_settings.js
@@ -1,51 +1,47 @@
 // Copyright (c) 2016, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Dropbox Settings', {
-	refresh: function(frm) {
-		frm.toggle_display(["app_access_key", "app_secret_key"], !(frm.doc.__onload && frm.doc.__onload.dropbox_setup_via_site_config));
-		frm.clear_custom_buttons();
+frappe.ui.form.on("Dropbox Settings", {
+	refresh: function (frm) {
+		frm.toggle_display(
+			["app_access_key", "app_secret_key"],
+			!frm.doc.__onload?.dropbox_setup_via_site_config
+		);
 		frm.events.take_backup(frm);
 	},
 
-	allow_dropbox_access: function(frm) {
-		if (frm.doc.app_access_key && frm.doc.app_secret_key) {
-			frappe.call({
-				method: "frappe.integrations.doctype.dropbox_settings.dropbox_settings.get_dropbox_authorize_url",
-				freeze: true,
-				callback: function(r) {
-					if(!r.exc) {
-						window.open(r.message.auth_url);
-					}
-				}
-			})
-		}
-		else if (frm.doc.__onload && frm.doc.__onload.dropbox_setup_via_site_config) {
-			frappe.call({
-				method: "frappe.integrations.doctype.dropbox_settings.dropbox_settings.get_redirect_url",
-				freeze: true,
-				callback: function(r) {
-					if(!r.exc) {
-						window.open(r.message.auth_url);
-					}
-				}
-			})
-		}
-		else {
-			frappe.msgprint(__("Please enter values for App Access Key and App Secret Key"))
-		}
+	are_keys_present: function (frm) {
+		return (
+			(frm.doc.app_access_key && frm.doc.app_secret_key) ||
+			frm.doc.__onload?.dropbox_setup_via_site_config
+		);
 	},
 
-	take_backup: function(frm) {
-		if (frm.doc.enabled && ((frm.doc.app_access_key && frm.doc.app_secret_key)
-			|| (frm.doc.__onload && frm.doc.__onload.dropbox_setup_via_site_config))) {
-			frm.add_custom_button(__("Take Backup Now"), function(frm){
+	allow_dropbox_access: function (frm) {
+		if (!frm.events.are_keys_present(frm)) {
+			frappe.msgprint(__("App Access Key and/or Secret Key are not present."));
+			return;
+		}
+
+		frappe.call({
+			method: "frappe.integrations.doctype.dropbox_settings.dropbox_settings.get_dropbox_authorize_url",
+			freeze: true,
+			callback: function (r) {
+				if (!r.exc) {
+					window.open(r.message.auth_url);
+				}
+			},
+		});
+	},
+
+	take_backup: function (frm) {
+		if (frm.doc.enabled && (frm.doc.dropbox_refresh_token || frm.doc.dropbox_access_token)) {
+			frm.add_custom_button(__("Take Backup Now"), function () {
 				frappe.call({
 					method: "frappe.integrations.doctype.dropbox_settings.dropbox_settings.take_backup",
-					freeze: true
-				})
-			}).addClass("btn-primary")
+					freeze: true,
+				});
+			});
 		}
-	}
+	},
 });
-
diff --git a/frappe/integrations/doctype/dropbox_settings/dropbox_settings.json b/frappe/integrations/doctype/dropbox_settings/dropbox_settings.json
index 858469647a..15535f08c4 100644
--- a/frappe/integrations/doctype/dropbox_settings/dropbox_settings.json
+++ b/frappe/integrations/doctype/dropbox_settings/dropbox_settings.json
@@ -1,8 +1,10 @@
 {
+ "actions": [],
  "creation": "2016-09-21 10:12:57.399174",
  "doctype": "DocType",
  "document_type": "System",
  "editable_grid": 1,
+ "engine": "InnoDB",
  "field_order": [
   "enabled",
   "send_notifications_to",
@@ -14,8 +16,7 @@
   "app_access_key",
   "app_secret_key",
   "allow_dropbox_access",
-  "dropbox_access_key",
-  "dropbox_access_secret",
+  "dropbox_refresh_token",
   "dropbox_access_token"
  ],
  "fields": [
@@ -82,17 +83,11 @@
    "label": "Allow Dropbox Access"
   },
   {
-   "fieldname": "dropbox_access_key",
+   "fieldname": "dropbox_refresh_token",
    "fieldtype": "Password",
    "hidden": 1,
-   "label": "Dropbox Access Key",
-   "read_only": 1
-  },
-  {
-   "fieldname": "dropbox_access_secret",
-   "fieldtype": "Password",
-   "hidden": 1,
-   "label": "Dropbox Access Secret",
+   "label": "Dropbox Refresh Token",
+   "no_copy": 1,
    "read_only": 1
   },
   {
@@ -104,7 +99,8 @@
  ],
  "in_create": 1,
  "issingle": 1,
- "modified": "2019-08-22 16:26:44.468391",
+ "links": [],
+ "modified": "2023-03-20 14:20:19.180611",
  "modified_by": "Administrator",
  "module": "Integrations",
  "name": "Dropbox Settings",
@@ -125,5 +121,6 @@
  "read_only": 1,
  "sort_field": "modified",
  "sort_order": "DESC",
+ "states": [],
  "track_changes": 1
 }
\ No newline at end of file
diff --git a/frappe/integrations/doctype/dropbox_settings/dropbox_settings.py b/frappe/integrations/doctype/dropbox_settings/dropbox_settings.py
index 50c5fa8fe6..c213ba8340 100644
--- a/frappe/integrations/doctype/dropbox_settings/dropbox_settings.py
+++ b/frappe/integrations/doctype/dropbox_settings/dropbox_settings.py
@@ -1,7 +1,6 @@
 # Copyright (c) 2015, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
-import json
 import os
 from urllib.parse import parse_qs, urlparse
 
@@ -16,16 +15,8 @@ from frappe.integrations.offsite_backup_utils import (
 	send_email,
 	validate_file_size,
 )
-from frappe.integrations.utils import make_post_request
 from frappe.model.document import Document
-from frappe.utils import (
-	cint,
-	encode,
-	get_backups_path,
-	get_files_path,
-	get_request_site_address,
-	get_url,
-)
+from frappe.utils import cint, encode, get_backups_path, get_files_path, get_request_site_address
 from frappe.utils.background_jobs import enqueue
 from frappe.utils.backups import new_backup
 
@@ -62,21 +53,21 @@ def take_backups_weekly():
 
 
 def take_backups_if(freq):
-	if frappe.db.get_value("Dropbox Settings", None, "backup_frequency") == freq:
+	if frappe.db.get_single_value("Dropbox Settings", "backup_frequency") == freq:
 		take_backup_to_dropbox()
 
 
 def take_backup_to_dropbox(retry_count=0, upload_db_backup=True):
 	did_not_upload, error_log = [], []
 	try:
-		if cint(frappe.db.get_value("Dropbox Settings", None, "enabled")):
+		if cint(frappe.db.get_single_value("Dropbox Settings", "enabled")):
 			validate_file_size()
 
 			did_not_upload, error_log = backup_to_dropbox(upload_db_backup)
 			if did_not_upload:
 				raise Exception
 
-			if cint(frappe.db.get_value("Dropbox Settings", None, "send_email_for_successful_backup")):
+			if cint(frappe.db.get_single_value("Dropbox Settings", "send_email_for_successful_backup")):
 				send_email(True, "Dropbox", "Dropbox Settings", "send_notifications_to")
 	except JobTimeoutException:
 		if retry_count < 2:
@@ -101,27 +92,9 @@ def take_backup_to_dropbox(retry_count=0, upload_db_backup=True):
 
 
 def backup_to_dropbox(upload_db_backup=True):
-	if not frappe.db:
-		frappe.connect()
-
 	# upload database
 	dropbox_settings = get_dropbox_settings()
-
-	if not dropbox_settings["access_token"]:
-		access_token = generate_oauth2_access_token_from_oauth1_token(dropbox_settings)
-
-		if not access_token.get("oauth2_token"):
-			return (
-				"Failed backup upload",
-				"No Access Token exists! Please generate the access token for Dropbox.",
-			)
-
-		dropbox_settings["access_token"] = access_token["oauth2_token"]
-		set_dropbox_access_token(access_token["oauth2_token"])
-
-	dropbox_client = dropbox.Dropbox(
-		oauth2_access_token=dropbox_settings["access_token"], timeout=None
-	)
+	dropbox_client = get_dropbox_client(dropbox_settings)
 
 	if upload_db_backup:
 		if frappe.flags.create_new_backup:
@@ -267,24 +240,36 @@ def get_uploaded_files_meta(dropbox_folder, dropbox_client):
 		# folder not found
 		if isinstance(e.error, dropbox.files.ListFolderError):
 			return frappe._dict({"entries": []})
-		else:
-			raise
+		raise
+
+
+def get_dropbox_client(dropbox_settings):
+	dropbox_client = dropbox.Dropbox(
+		oauth2_access_token=dropbox_settings["access_token"],
+		oauth2_refresh_token=dropbox_settings["refresh_token"],
+		app_key=dropbox_settings["app_key"],
+		app_secret=dropbox_settings["app_secret"],
+		timeout=None,
+	)
+
+	# checking if the access token has expired
+	dropbox_client.files_list_folder("")
+	if dropbox_settings["access_token"] != dropbox_client._oauth2_access_token:
+		set_dropbox_token(dropbox_client._oauth2_access_token)
+
+	return dropbox_client
 
 
 def get_dropbox_settings(redirect_uri=False):
-	if not frappe.conf.dropbox_broker_site:
-		frappe.conf.dropbox_broker_site = "https://dropbox.erpnext.com"
+	# NOTE: access token is kept for legacy dropbox apps
 	settings = frappe.get_doc("Dropbox Settings")
 	app_details = {
 		"app_key": settings.app_access_key or frappe.conf.dropbox_access_key,
 		"app_secret": settings.get_password(fieldname="app_secret_key", raise_exception=False)
 		if settings.app_secret_key
 		else frappe.conf.dropbox_secret_key,
-		"access_token": settings.get_password("dropbox_access_token", raise_exception=False)
-		if settings.dropbox_access_token
-		else "",
-		"access_key": settings.get_password("dropbox_access_key", raise_exception=False),
-		"access_secret": settings.get_password("dropbox_access_secret", raise_exception=False),
+		"refresh_token": settings.get_password("dropbox_refresh_token", raise_exception=False),
+		"access_token": settings.get_password("dropbox_access_token", raise_exception=False),
 		"file_backup": settings.file_backup,
 		"no_of_backups": settings.no_of_backups if settings.limit_no_of_backups else None,
 	}
@@ -294,14 +279,11 @@ def get_dropbox_settings(redirect_uri=False):
 			{
 				"redirect_uri": get_request_site_address(True)
 				+ "/api/method/frappe.integrations.doctype.dropbox_settings.dropbox_settings.dropbox_auth_finish"
-				if settings.app_secret_key
-				else frappe.conf.dropbox_broker_site
-				+ "/api/method/dropbox_erpnext_broker.www.setup_dropbox.generate_dropbox_access_token",
 			}
 		)
 
-	if not app_details["app_key"] or not app_details["app_secret"]:
-		raise Exception(_("Please set Dropbox access keys in your site config"))
+	if not (app_details["app_key"] and app_details["app_secret"]):
+		raise Exception(_("Please set Dropbox access keys in site config or doctype"))
 
 	return app_details
 
@@ -321,28 +303,6 @@ def delete_older_backups(dropbox_client, folder_path, to_keep):
 		dropbox_client.files_delete(os.path.join(folder_path, f.name))
 
 
-@frappe.whitelist()
-def get_redirect_url():
-	if not frappe.conf.dropbox_broker_site:
-		frappe.conf.dropbox_broker_site = "https://dropbox.erpnext.com"
-	url = "{}/api/method/dropbox_erpnext_broker.www.setup_dropbox.get_authotize_url".format(
-		frappe.conf.dropbox_broker_site
-	)
-
-	try:
-		response = make_post_request(url, data={"site": get_url()})
-		if response.get("message"):
-			return response["message"]
-
-	except Exception:
-		frappe.log_error()
-		frappe.throw(
-			_(
-				"Something went wrong while generating dropbox access token. Please check error log for more details."
-			)
-		)
-
-
 @frappe.whitelist()
 def get_dropbox_authorize_url():
 	app_details = get_dropbox_settings(redirect_uri=True)
@@ -352,6 +312,7 @@ def get_dropbox_authorize_url():
 		session={},
 		csrf_token_session_key="dropbox-auth-csrf-token",
 		consumer_secret=app_details["app_secret"],
+		token_access_type="offline",
 	)
 
 	auth_url = dropbox_oauth_flow.start()
@@ -360,11 +321,20 @@ def get_dropbox_authorize_url():
 
 
 @frappe.whitelist()
-def dropbox_auth_finish(return_access_token=False):
+def dropbox_auth_finish():
 	app_details = get_dropbox_settings(redirect_uri=True)
 	callback = frappe.form_dict
 	close = '
        ' + _("Please close this window") + "
        "
 
+	if not callback.state or not callback.code:
+		frappe.respond_as_web_page(
+			_("Dropbox Setup"),
+			_("Illegal Access Token. Please try again") + close,
+			indicator_color="red",
+			http_status_code=frappe.AuthenticationError.http_status_code,
+		)
+		return
+
 	dropbox_oauth_flow = dropbox.DropboxOAuth2Flow(
 		consumer_key=app_details["app_key"],
 		redirect_uri=app_details["redirect_uri"],
@@ -373,40 +343,20 @@ def dropbox_auth_finish(return_access_token=False):
 		consumer_secret=app_details["app_secret"],
 	)
 
-	if callback.state or callback.code:
-		token = dropbox_oauth_flow.finish({"state": callback.state, "code": callback.code})
-		if return_access_token and token.access_token:
-			return token.access_token, callback.state
+	token = dropbox_oauth_flow.finish({"state": callback.state, "code": callback.code})
+	set_dropbox_token(token.access_token, token.refresh_token)
 
-		set_dropbox_access_token(token.access_token)
-	else:
-		frappe.respond_as_web_page(
-			_("Dropbox Setup"),
-			_("Illegal Access Token. Please try again") + close,
-			indicator_color="red",
-			http_status_code=frappe.AuthenticationError.http_status_code,
-		)
-
-	frappe.respond_as_web_page(
-		_("Dropbox Setup"), _("Dropbox access is approved!") + close, indicator_color="green"
-	)
+	frappe.local.response["type"] = "redirect"
+	frappe.local.response["location"] = "/app/dropbox-settings"
 
 
-def set_dropbox_access_token(access_token):
-	frappe.db.set_value("Dropbox Settings", None, "dropbox_access_token", access_token)
+def set_dropbox_token(access_token, refresh_token=None):
+	# NOTE: used doc object instead of db.set_value so that password field is set properly
+	dropbox_settings = frappe.get_single("Dropbox Settings")
+	dropbox_settings.dropbox_access_token = access_token
+	if refresh_token:
+		dropbox_settings.dropbox_refresh_token = refresh_token
+
+	dropbox_settings.save()
+
 	frappe.db.commit()
-
-
-def generate_oauth2_access_token_from_oauth1_token(dropbox_settings=None):
-	if not dropbox_settings.get("access_key") or not dropbox_settings.get("access_secret"):
-		return {}
-
-	url = "https://api.dropboxapi.com/2/auth/token/from_oauth1"
-	headers = {"Content-Type": "application/json"}
-	auth = (dropbox_settings["app_key"], dropbox_settings["app_secret"])
-	data = {
-		"oauth1_token": dropbox_settings["access_key"],
-		"oauth1_token_secret": dropbox_settings["access_secret"],
-	}
-
-	return make_post_request(url, auth=auth, headers=headers, data=json.dumps(data))
diff --git a/frappe/integrations/doctype/dropbox_settings/test_dropbox_settings.py b/frappe/integrations/doctype/dropbox_settings/test_dropbox_settings.py
index b165e03780..1c95b130e8 100644
--- a/frappe/integrations/doctype/dropbox_settings/test_dropbox_settings.py
+++ b/frappe/integrations/doctype/dropbox_settings/test_dropbox_settings.py
@@ -1,8 +1,8 @@
 # Copyright (c) 2019, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
 # import frappe
-import unittest
+from frappe.tests.utils import FrappeTestCase
 
 
-class TestDropboxSettings(unittest.TestCase):
+class TestDropboxSettings(FrappeTestCase):
 	pass
diff --git a/frappe/integrations/doctype/google_calendar/google_calendar.js b/frappe/integrations/doctype/google_calendar/google_calendar.js
index f30c52b2f2..977dee8dfe 100644
--- a/frappe/integrations/doctype/google_calendar/google_calendar.js
+++ b/frappe/integrations/doctype/google_calendar/google_calendar.js
@@ -2,15 +2,22 @@
 // For license information, please see license.txt
 
 frappe.ui.form.on("Google Calendar", {
-	refresh: function(frm) {
+	refresh: function (frm) {
 		if (frm.is_new()) {
-			frm.dashboard.set_headline(__("To use Google Calendar, enable {0}.", [`${__('Google Settings')}`]));
+			frm.dashboard.set_headline(
+				__("To use Google Calendar, enable {0}.", [
+					`${__("Google Settings")}`,
+				])
+			);
 		}
 
 		frappe.realtime.on("import_google_calendar", (data) => {
 			if (data.progress) {
-				frm.dashboard.show_progress("Syncing Google Calendar", data.progress / data.total * 100,
-					__("Syncing {0} of {1}", [data.progress, data.total]));
+				frm.dashboard.show_progress(
+					"Syncing Google Calendar",
+					(data.progress / data.total) * 100,
+					__("Syncing {0} of {1}", [data.progress, data.total])
+				);
 				if (data.progress === data.total) {
 					frm.dashboard.hide_progress("Syncing Google Calendar");
 				}
@@ -21,38 +28,40 @@ frappe.ui.form.on("Google Calendar", {
 			frm.add_custom_button(__("Sync Calendar"), function () {
 				frappe.show_alert({
 					indicator: "green",
-					message: __("Syncing")
-				});
-				frappe.call({
-					method: "frappe.integrations.doctype.google_calendar.google_calendar.sync",
-					args: {
-						"g_calendar": frm.doc.name
-					},
-				}).then((r) => {
-					frappe.hide_progress();
-					frappe.msgprint(r.message);
+					message: __("Syncing"),
 				});
+				frappe
+					.call({
+						method: "frappe.integrations.doctype.google_calendar.google_calendar.sync",
+						args: {
+							g_calendar: frm.doc.name,
+						},
+					})
+					.then((r) => {
+						frappe.hide_progress();
+						frappe.msgprint(r.message);
+					});
 			});
 		}
 	},
-	authorize_google_calendar_access: function(frm) {
+	authorize_google_calendar_access: function (frm) {
 		let reauthorize = 0;
-		if(frm.doc.authorization_code) {
+		if (frm.doc.authorization_code) {
 			reauthorize = 1;
 		}
 
 		frappe.call({
 			method: "frappe.integrations.doctype.google_calendar.google_calendar.authorize_access",
 			args: {
-				"g_calendar": frm.doc.name,
-				"reauthorize": reauthorize
+				g_calendar: frm.doc.name,
+				reauthorize: reauthorize,
 			},
-			callback: function(r) {
-				if(!r.exc) {
+			callback: function (r) {
+				if (!r.exc) {
 					frm.save();
 					window.open(r.message.url);
 				}
-			}
+			},
 		});
-	}
+	},
 });
diff --git a/frappe/integrations/doctype/google_calendar/google_calendar.py b/frappe/integrations/doctype/google_calendar/google_calendar.py
index 09ed012454..d1cdd0d9e7 100644
--- a/frappe/integrations/doctype/google_calendar/google_calendar.py
+++ b/frappe/integrations/doctype/google_calendar/google_calendar.py
@@ -4,6 +4,7 @@
 
 from datetime import datetime, timedelta
 from urllib.parse import quote
+from zoneinfo import ZoneInfo
 
 import google.oauth2.credentials
 import requests
@@ -20,7 +21,7 @@ from frappe.utils import (
 	add_to_date,
 	get_datetime,
 	get_request_site_address,
-	get_time_zone,
+	get_system_timezone,
 	get_weekdays,
 	now_datetime,
 )
@@ -110,6 +111,7 @@ def authorize_access(g_calendar, reauthorize=None):
 	"""
 	google_settings = frappe.get_doc("Google Settings")
 	google_calendar = frappe.get_doc("Google Calendar", g_calendar)
+	google_calendar.check_permission("write")
 
 	redirect_uri = (
 		get_request_site_address(True)
@@ -274,7 +276,7 @@ def sync_events_from_google_calendar(g_calendar, method=None):
 			if err.resp.status == 410:
 				set_encrypted_password("Google Calendar", account.name, "", "next_sync_token")
 				frappe.db.commit()
-				msg += " " + _("Sync token was invalid and has been resetted, Retry syncing.")
+				msg += " " + _("Sync token was invalid and has been reset, Retry syncing.")
 				frappe.msgprint(msg, title="Invalid Sync Token", indicator="blue")
 			else:
 				frappe.throw(msg)
@@ -356,6 +358,7 @@ def insert_event_to_calendar(account, event, recurrence=None):
 		"google_calendar": account.name,
 		"google_calendar_id": account.google_calendar_id,
 		"google_calendar_event_id": event.get("id"),
+		"google_meet_link": event.get("hangoutLink"),
 		"pulled_from_google_calendar": 1,
 	}
 	calendar_event.update(
@@ -373,6 +376,7 @@ def update_event_in_calendar(account, event, recurrence=None):
 	calendar_event = frappe.get_doc("Event", {"google_calendar_event_id": event.get("id")})
 	calendar_event.subject = event.get("summary")
 	calendar_event.description = event.get("description")
+	calendar_event.google_meet_link = event.get("hangoutLink")
 	calendar_event.update(
 		google_calendar_to_repeat_on(
 			recurrence=recurrence, start=event.get("start"), end=event.get("end")
@@ -400,18 +404,40 @@ def insert_event_in_google_calendar(doc, method=None):
 	event = {"summary": doc.subject, "description": doc.description, "google_calendar_event": 1}
 	event.update(
 		format_date_according_to_google_calendar(
-			doc.all_day, get_datetime(doc.starts_on), get_datetime(doc.ends_on)
+			doc.all_day, get_datetime(doc.starts_on), get_datetime(doc.ends_on) if doc.ends_on else None
 		)
 	)
 
 	if doc.repeat_on:
 		event.update({"recurrence": repeat_on_to_google_calendar_recurrence_rule(doc)})
 
+	event.update({"attendees": get_attendees(doc)})
+
+	conference_data_version = 0
+
+	if doc.add_video_conferencing:
+		event.update({"conferenceData": get_conference_data(doc)})
+		conference_data_version = 1
+
 	try:
-		event = google_calendar.events().insert(calendarId=doc.google_calendar_id, body=event).execute()
-		frappe.db.set_value(
-			"Event", doc.name, "google_calendar_event_id", event.get("id"), update_modified=False
+		event = (
+			google_calendar.events()
+			.insert(
+				calendarId=doc.google_calendar_id,
+				body=event,
+				conferenceDataVersion=conference_data_version,
+				sendUpdates="all",
+			)
+			.execute()
 		)
+
+		frappe.db.set_value(
+			"Event",
+			doc.name,
+			{"google_calendar_event_id": event.get("id"), "google_meet_link": event.get("hangoutLink")},
+			update_modified=False,
+		)
+
 		frappe.msgprint(_("Event Synced with Google Calendar."))
 	except HttpError as err:
 		frappe.throw(
@@ -450,6 +476,7 @@ def update_event_in_google_calendar(doc, method=None):
 			.get(calendarId=doc.google_calendar_id, eventId=doc.google_calendar_event_id)
 			.execute()
 		)
+
 		event["summary"] = doc.subject
 		event["description"] = doc.description
 		event["recurrence"] = repeat_on_to_google_calendar_recurrence_rule(doc)
@@ -458,13 +485,43 @@ def update_event_in_google_calendar(doc, method=None):
 		)
 		event.update(
 			format_date_according_to_google_calendar(
-				doc.all_day, get_datetime(doc.starts_on), get_datetime(doc.ends_on)
+				doc.all_day, get_datetime(doc.starts_on), get_datetime(doc.ends_on) if doc.ends_on else None
 			)
 		)
 
-		google_calendar.events().update(
-			calendarId=doc.google_calendar_id, eventId=doc.google_calendar_event_id, body=event
-		).execute()
+		conference_data_version = 0
+
+		if doc.add_video_conferencing:
+			event.update({"conferenceData": get_conference_data(doc)})
+			conference_data_version = 1
+		elif doc.get_doc_before_save().add_video_conferencing or event.get("hangoutLink"):
+			# remove google meet from google calendar event, if turning off add_video_conferencing
+			event.update({"conferenceData": None})
+			conference_data_version = 1
+
+		event.update({"attendees": get_attendees(doc)})
+
+		event = (
+			google_calendar.events()
+			.update(
+				calendarId=doc.google_calendar_id,
+				eventId=doc.google_calendar_event_id,
+				body=event,
+				conferenceDataVersion=conference_data_version,
+				sendUpdates="all",
+			)
+			.execute()
+		)
+
+		# if add_video_conferencing enabled or disabled during update, overwrite
+		frappe.db.set_value(
+			"Event",
+			doc.name,
+			{"google_meet_link": event.get("hangoutLink")},
+			update_modified=False,
+		)
+		doc.notify_update()
+
 		frappe.msgprint(_("Event Synced with Google Calendar."))
 	except HttpError as err:
 		frappe.throw(
@@ -515,12 +572,20 @@ def google_calendar_to_repeat_on(start, end, recurrence=None):
 	Both have been mapped in a dict for easier mapping.
 	"""
 	repeat_on = {
-		"starts_on": get_datetime(start.get("date"))
-		if start.get("date")
-		else parser.parse(start.get("dateTime")).utcnow(),
-		"ends_on": get_datetime(end.get("date"))
-		if end.get("date")
-		else parser.parse(end.get("dateTime")).utcnow(),
+		"starts_on": (
+			get_datetime(start.get("date"))
+			if start.get("date")
+			else parser.parse(start.get("dateTime"))
+			.astimezone(ZoneInfo(get_system_timezone()))
+			.replace(tzinfo=None)
+		),
+		"ends_on": (
+			get_datetime(end.get("date"))
+			if end.get("date")
+			else parser.parse(end.get("dateTime"))
+			.astimezone(ZoneInfo(get_system_timezone()))
+			.replace(tzinfo=None)
+		),
 		"all_day": 1 if start.get("date") else 0,
 		"repeat_this_event": 1 if recurrence else 0,
 		"repeat_on": None,
@@ -584,11 +649,11 @@ def format_date_according_to_google_calendar(all_day, starts_on, ends_on=None):
 	date_format = {
 		"start": {
 			"dateTime": starts_on.isoformat(),
-			"timeZone": get_time_zone(),
+			"timeZone": get_system_timezone(),
 		},
 		"end": {
 			"dateTime": ends_on.isoformat(),
-			"timeZone": get_time_zone(),
+			"timeZone": get_system_timezone(),
 		},
 	}
 
@@ -682,6 +747,39 @@ def get_recurrence_parameters(recurrence):
 	return frequency, until, byday
 
 
+def get_conference_data(doc):
+	return {
+		"createRequest": {"requestId": doc.name, "conferenceSolutionKey": {"type": "hangoutsMeet"}},
+		"notes": doc.description,
+	}
+
+
+def get_attendees(doc):
+	"""
+	Returns a list of dicts with attendee emails, if available in event_participants table
+	"""
+	attendees, email_not_found = [], []
+
+	for participant in doc.event_participants:
+		if participant.get("email"):
+			attendees.append({"email": participant.email})
+		else:
+			email_not_found.append(
+				{"dt": participant.reference_doctype, "dn": participant.reference_docname}
+			)
+
+	if email_not_found:
+		frappe.msgprint(
+			_("Google Calendar - Contact / email not found. Did not add attendee for -
        {0}").format(
+				"
        ".join(f"{d.get('dt')} {d.get('dn')}" for d in email_not_found)
+			),
+			alert=True,
+			indicator="yellow",
+		)
+
+	return attendees
+
+
 """API Response
 	{
 		'kind': 'calendar#events',
@@ -721,6 +819,32 @@ def get_recurrence_parameters(recurrence):
 				'recurrence': *recurrence,
 				'iCalUID': 'uid',
 				'sequence': 1,
+				'hangoutLink': 'https://meet.google.com/mee-ting-uri',
+				'conferenceData': {
+					'createRequest': {
+						'requestId': 'EV00001',
+						'conferenceSolutionKey': {
+							'type': 'hangoutsMeet'
+						},
+						'status': {
+							'statusCode': 'success'
+						}
+					},
+					'entryPoints': [
+						{
+							'entryPointType': 'video',
+							'uri': 'https://meet.google.com/mee-ting-uri',
+							'label': 'meet.google.com/mee-ting-uri'
+						}
+					],
+					'conferenceSolution': {
+						'key': {
+							'type': 'hangoutsMeet'
+						},
+						'name': 'Google Meet',
+						'iconUri': 'https://fonts.gstatic.com/s/i/productlogos/meet_2020q4/v6/web-512dp/logo_meet_2020q4_color_2x_web_512dp.png'
+					},
+					'conferenceId': 'mee-ting-uri'
 				'reminders': {
 					'useDefault': True
 				}
diff --git a/frappe/integrations/doctype/google_contacts/google_contacts.js b/frappe/integrations/doctype/google_contacts/google_contacts.js
index 6e8035f38d..06289b0ca5 100644
--- a/frappe/integrations/doctype/google_contacts/google_contacts.js
+++ b/frappe/integrations/doctype/google_contacts/google_contacts.js
@@ -1,54 +1,63 @@
 // Copyright (c) 2019, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Google Contacts', {
-	refresh: function(frm) {
+frappe.ui.form.on("Google Contacts", {
+	refresh: function (frm) {
 		if (!frm.doc.enable) {
-			frm.dashboard.set_headline(__("To use Google Contacts, enable {0}.", [`${__('Google Settings')}`]));
+			frm.dashboard.set_headline(
+				__("To use Google Contacts, enable {0}.", [
+					`${__("Google Settings")}`,
+				])
+			);
 		}
 
-		frappe.realtime.on('import_google_contacts', (data) => {
+		frappe.realtime.on("import_google_contacts", (data) => {
 			if (data.progress) {
-				frm.dashboard.show_progress('Import Google Contacts', data.progress / data.total * 100,
-					__('Importing {0} of {1}', [data.progress, data.total]));
+				frm.dashboard.show_progress(
+					"Import Google Contacts",
+					(data.progress / data.total) * 100,
+					__("Importing {0} of {1}", [data.progress, data.total])
+				);
 				if (data.progress === data.total) {
-					frm.dashboard.hide_progress('Import Google Contacts');
+					frm.dashboard.hide_progress("Import Google Contacts");
 				}
 			}
 		});
 
 		if (frm.doc.refresh_token) {
-			let sync_button = frm.add_custom_button(__('Sync Contacts'), function () {
+			let sync_button = frm.add_custom_button(__("Sync Contacts"), function () {
 				frappe.show_alert({
-					indicator: 'green',
-					message: __('Syncing')
-				});
-				frappe.call({
-					method: "frappe.integrations.doctype.google_contacts.google_contacts.sync",
-					args: {
-						"g_contact": frm.doc.name
-					},
-					btn: sync_button
-				}).then((r) => {
-					frappe.hide_progress();
-					frappe.msgprint(r.message);
+					indicator: "green",
+					message: __("Syncing"),
 				});
+				frappe
+					.call({
+						method: "frappe.integrations.doctype.google_contacts.google_contacts.sync",
+						args: {
+							g_contact: frm.doc.name,
+						},
+						btn: sync_button,
+					})
+					.then((r) => {
+						frappe.hide_progress();
+						frappe.msgprint(r.message);
+					});
 			});
 		}
 	},
-	authorize_google_contacts_access: function(frm) {
+	authorize_google_contacts_access: function (frm) {
 		frappe.call({
 			method: "frappe.integrations.doctype.google_contacts.google_contacts.authorize_access",
 			args: {
-				"g_contact": frm.doc.name,
-				"reauthorize": frm.doc.authorization_code ? 1 : 0
+				g_contact: frm.doc.name,
+				reauthorize: frm.doc.authorization_code ? 1 : 0,
 			},
-			callback: function(r) {
-				if(!r.exc) {
+			callback: function (r) {
+				if (!r.exc) {
 					frm.save();
 					window.open(r.message.url);
 				}
-			}
+			},
 		});
-	}
+	},
 });
diff --git a/frappe/integrations/doctype/google_contacts/google_contacts.json b/frappe/integrations/doctype/google_contacts/google_contacts.json
index 76781fe47f..4a72651e67 100644
--- a/frappe/integrations/doctype/google_contacts/google_contacts.json
+++ b/frappe/integrations/doctype/google_contacts/google_contacts.json
@@ -1,4 +1,5 @@
 {
+ "actions": [],
  "autoname": "format:GC-{email_id}",
  "creation": "2019-06-14 00:09:39.441961",
  "doctype": "DocType",
@@ -97,10 +98,12 @@
    "label": "Push to Google Contacts"
   }
  ],
- "modified": "2020-09-18 17:26:09.703215",
+ "links": [],
+ "modified": "2023-03-30 11:25:48.832384",
  "modified_by": "Administrator",
  "module": "Integrations",
  "name": "Google Contacts",
+ "naming_rule": "Expression",
  "owner": "Administrator",
  "permissions": [
   {
@@ -116,17 +119,14 @@
   {
    "create": 1,
    "delete": 1,
-   "email": 1,
-   "export": 1,
-   "print": 1,
+   "if_owner": 1,
    "read": 1,
-   "report": 1,
    "role": "All",
-   "share": 1,
    "write": 1
   }
  ],
  "sort_field": "modified",
  "sort_order": "ASC",
+ "states": [],
  "track_changes": 1
 }
\ No newline at end of file
diff --git a/frappe/integrations/doctype/google_contacts/google_contacts.py b/frappe/integrations/doctype/google_contacts/google_contacts.py
index c1f445b599..7c845da330 100644
--- a/frappe/integrations/doctype/google_contacts/google_contacts.py
+++ b/frappe/integrations/doctype/google_contacts/google_contacts.py
@@ -36,27 +36,24 @@ def authorize_access(g_contact, reauthorize=False, code=None):
 	If no Authorization code get it from Google and then request for Refresh Token.
 	Google Contact Name is set to flags to set_value after Authorization Code is obtained.
 	"""
+	contact = frappe.get_doc("Google Contacts", g_contact)
+	contact.check_permission("write")
 
-	oauth_code = (
-		frappe.db.get_value("Google Contacts", g_contact, "authorization_code") if not code else code
-	)
+	oauth_code = code or contact.get_password("authorization_code")
 	oauth_obj = GoogleOAuth("contacts")
 
 	if not oauth_code or reauthorize:
 		return oauth_obj.get_authentication_url(
 			{
-				"method": "frappe.integrations.doctype.google_contacts.google_contacts.authorize_access",
 				"g_contact": g_contact,
 				"redirect": f"/app/Form/{quote('Google Contacts')}/{quote(g_contact)}",
 			},
 		)
 
 	r = oauth_obj.authorize(oauth_code)
-	frappe.db.set_value(
-		"Google Contacts",
-		g_contact,
-		{"authorization_code": oauth_code, "refresh_token": r.get("refresh_token")},
-	)
+	contact.authorization_code = oauth_code
+	contact.refresh_token = r.get("refresh_token")
+	contact.save()
 
 
 def get_google_contacts_object(g_contact):
diff --git a/frappe/integrations/doctype/google_contacts/test_google_contacts.py b/frappe/integrations/doctype/google_contacts/test_google_contacts.py
new file mode 100644
index 0000000000..d7ca08a082
--- /dev/null
+++ b/frappe/integrations/doctype/google_contacts/test_google_contacts.py
@@ -0,0 +1,9 @@
+# Copyright (c) 2023, Frappe Technologies and Contributors
+# See license.txt
+
+# import frappe
+from frappe.tests.utils import FrappeTestCase
+
+
+class TestGoogleContacts(FrappeTestCase):
+	pass
diff --git a/frappe/integrations/doctype/google_drive/google_drive.js b/frappe/integrations/doctype/google_drive/google_drive.js
index b38c0fb8e6..208c1e5e1a 100644
--- a/frappe/integrations/doctype/google_drive/google_drive.js
+++ b/frappe/integrations/doctype/google_drive/google_drive.js
@@ -1,16 +1,23 @@
 // Copyright (c) 2019, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Google Drive', {
-	refresh: function(frm) {
+frappe.ui.form.on("Google Drive", {
+	refresh: function (frm) {
 		if (!frm.doc.enable) {
-			frm.dashboard.set_headline(__("To use Google Drive, enable {0}.", [`${__('Google Settings')}`]));
+			frm.dashboard.set_headline(
+				__("To use Google Drive, enable {0}.", [
+					`${__("Google Settings")}`,
+				])
+			);
 		}
 
 		frappe.realtime.on("upload_to_google_drive", (data) => {
 			if (data.progress) {
-				frm.dashboard.show_progress("Uploading to Google Drive", data.progress / data.total * 100,
-					__("{0}", [data.message]));
+				frm.dashboard.show_progress(
+					"Uploading to Google Drive",
+					(data.progress / data.total) * 100,
+					__("{0}", [data.message])
+				);
 				if (data.progress === data.total) {
 					frm.dashboard.hide_progress("Uploading to Google Drive");
 				}
@@ -21,37 +28,43 @@ frappe.ui.form.on('Google Drive', {
 			let sync_button = frm.add_custom_button(__("Take Backup"), function () {
 				frappe.show_alert({
 					indicator: "green",
-					message: __("Backing up to Google Drive.")
-				});
-				frappe.call({
-					method: "frappe.integrations.doctype.google_drive.google_drive.take_backup",
-					btn: sync_button
-				}).then((r) => {
-					frappe.msgprint(r.message);
+					message: __("Backing up to Google Drive."),
 				});
+				frappe
+					.call({
+						method: "frappe.integrations.doctype.google_drive.google_drive.take_backup",
+						btn: sync_button,
+					})
+					.then((r) => {
+						frappe.msgprint(r.message);
+					});
 			});
 		}
 
 		if (frm.doc.enable && frm.doc.backup_folder_name && !frm.doc.refresh_token) {
-			frm.dashboard.set_headline(__("Click on Authorize Google Drive Access to authorize Google Drive Access."));
+			frm.dashboard.set_headline(
+				__(
+					"Click on Authorize Google Drive Access to authorize Google Drive Access."
+				)
+			);
 		}
 
 		if (frm.doc.enable && frm.doc.refresh_token && frm.doc.authorization_code) {
 			frm.page.set_indicator("Authorized", "green");
 		}
 	},
-	authorize_google_drive_access: function(frm) {
+	authorize_google_drive_access: function (frm) {
 		frappe.call({
 			method: "frappe.integrations.doctype.google_drive.google_drive.authorize_access",
 			args: {
-				"reauthorize": frm.doc.authorization_code ? 1 : 0
+				reauthorize: frm.doc.authorization_code ? 1 : 0,
 			},
-			callback: function(r) {
+			callback: function (r) {
 				if (!r.exc) {
 					frm.save();
 					window.open(r.message.url);
 				}
-			}
+			},
 		});
-	}
+	},
 });
diff --git a/frappe/integrations/doctype/google_drive/google_drive.json b/frappe/integrations/doctype/google_drive/google_drive.json
index 6742d9ee5d..592281be68 100644
--- a/frappe/integrations/doctype/google_drive/google_drive.json
+++ b/frappe/integrations/doctype/google_drive/google_drive.json
@@ -1,4 +1,5 @@
 {
+ "actions": [],
  "creation": "2019-08-13 17:24:05.470876",
  "doctype": "DocType",
  "engine": "InnoDB",
@@ -100,7 +101,8 @@
   }
  ],
  "issingle": 1,
- "modified": "2020-09-18 17:26:09.703215",
+ "links": [],
+ "modified": "2022-12-04 15:53:58.702389",
  "modified_by": "Administrator",
  "module": "Integrations",
  "name": "Google Drive",
@@ -115,19 +117,10 @@
    "role": "System Manager",
    "share": 1,
    "write": 1
-  },
-  {
-   "create": 1,
-   "delete": 1,
-   "email": 1,
-   "print": 1,
-   "read": 1,
-   "role": "All",
-   "share": 1,
-   "write": 1
   }
  ],
  "sort_field": "modified",
  "sort_order": "ASC",
+ "states": [],
  "track_changes": 1
 }
\ No newline at end of file
diff --git a/frappe/integrations/doctype/google_drive/google_drive.py b/frappe/integrations/doctype/google_drive/google_drive.py
index 62100ae7c5..fbb970de46 100644
--- a/frappe/integrations/doctype/google_drive/google_drive.py
+++ b/frappe/integrations/doctype/google_drive/google_drive.py
@@ -48,23 +48,21 @@ def authorize_access(reauthorize=False, code=None):
 	"""
 
 	oauth_code = (
-		frappe.db.get_value("Google Drive", "Google Drive", "authorization_code") if not code else code
+		frappe.db.get_single_value("Google Drive", "authorization_code") if not code else code
 	)
 	oauth_obj = GoogleOAuth("drive")
 
 	if not oauth_code or reauthorize:
 		if reauthorize:
-			frappe.db.set_value("Google Drive", None, "backup_folder_id", "")
+			frappe.db.set_single_value("Google Drive", "backup_folder_id", "")
 		return oauth_obj.get_authentication_url(
 			{
-				"method": "frappe.integrations.doctype.google_drive.google_drive.authorize_access",
 				"redirect": f"/app/Form/{quote('Google Drive')}",
 			},
 		)
 
 	r = oauth_obj.authorize(oauth_code)
-	frappe.db.set_value(
-		"Google Drive",
+	frappe.db.set_single_value(
 		"Google Drive",
 		{"authorization_code": oauth_code, "refresh_token": r.get("refresh_token")},
 	)
@@ -96,7 +94,7 @@ def check_for_folder_in_google_drive():
 
 		try:
 			folder = google_drive.files().create(body=file_metadata, fields="id").execute()
-			frappe.db.set_value("Google Drive", None, "backup_folder_id", folder.get("id"))
+			frappe.db.set_single_value("Google Drive", "backup_folder_id", folder.get("id"))
 			frappe.db.commit()
 		except HttpError as e:
 			frappe.throw(
@@ -121,7 +119,7 @@ def check_for_folder_in_google_drive():
 
 	for f in google_drive_folders.get("files"):
 		if f.get("name") == account.backup_folder_name:
-			frappe.db.set_value("Google Drive", None, "backup_folder_id", f.get("id"))
+			frappe.db.set_single_value("Google Drive", "backup_folder_id", f.get("id"))
 			frappe.db.commit()
 			backup_folder_exists = True
 			break
@@ -171,7 +169,7 @@ def upload_system_backup_to_google_drive():
 		if not fileurl:
 			continue
 
-		file_metadata = {"name": fileurl, "parents": [account.backup_folder_id]}
+		file_metadata = {"name": os.path.basename(fileurl), "parents": [account.backup_folder_id]}
 
 		try:
 			media = MediaFileUpload(
@@ -187,7 +185,7 @@ def upload_system_backup_to_google_drive():
 			send_email(False, "Google Drive", "Google Drive", "email", error_status=e)
 
 	set_progress(3, "Uploading successful.")
-	frappe.db.set_value("Google Drive", None, "last_backup_on", frappe.utils.now_datetime())
+	frappe.db.set_single_value("Google Drive", "last_backup_on", frappe.utils.now_datetime())
 	send_email(True, "Google Drive", "Google Drive", "email")
 	return _("Google Drive Backup Successful.")
 
diff --git a/frappe/integrations/doctype/google_drive/test_google_drive.py b/frappe/integrations/doctype/google_drive/test_google_drive.py
index 4dcc79afd6..1dd048048c 100644
--- a/frappe/integrations/doctype/google_drive/test_google_drive.py
+++ b/frappe/integrations/doctype/google_drive/test_google_drive.py
@@ -1,8 +1,8 @@
 # Copyright (c) 2019, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
 # import frappe
-import unittest
+from frappe.tests.utils import FrappeTestCase
 
 
-class TestGoogleDrive(unittest.TestCase):
+class TestGoogleDrive(FrappeTestCase):
 	pass
diff --git a/frappe/integrations/doctype/google_settings/google_settings.js b/frappe/integrations/doctype/google_settings/google_settings.js
index 01a127db7f..58093034b5 100644
--- a/frappe/integrations/doctype/google_settings/google_settings.js
+++ b/frappe/integrations/doctype/google_settings/google_settings.js
@@ -1,8 +1,14 @@
 // Copyright (c) 2019, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Google Settings', {
-	refresh: function(frm) {
-		frm.dashboard.set_headline(__("For more information, {0}.", [`${__('Click here')}`]));
-	}
+frappe.ui.form.on("Google Settings", {
+	refresh: function (frm) {
+		frm.dashboard.set_headline(
+			__("For more information, {0}.", [
+				`${__(
+					"Click here"
+				)}`,
+			])
+		);
+	},
 });
diff --git a/frappe/integrations/doctype/google_settings/test_google_settings.py b/frappe/integrations/doctype/google_settings/test_google_settings.py
index 8d07ffa54f..d4bb830779 100644
--- a/frappe/integrations/doctype/google_settings/test_google_settings.py
+++ b/frappe/integrations/doctype/google_settings/test_google_settings.py
@@ -1,14 +1,13 @@
 # Copyright (c) 2021, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
 
-import unittest
-
 import frappe
+from frappe.tests.utils import FrappeTestCase
 
 from .google_settings import get_file_picker_settings
 
 
-class TestGoogleSettings(unittest.TestCase):
+class TestGoogleSettings(FrappeTestCase):
 	def setUp(self):
 		settings = frappe.get_single("Google Settings")
 		settings.client_id = "test_client_id"
@@ -18,24 +17,24 @@ class TestGoogleSettings(unittest.TestCase):
 
 	def test_picker_disabled(self):
 		"""Google Drive Picker should be disabled if it is not enabled in Google Settings."""
-		frappe.db.set_value("Google Settings", None, "enable", 1)
-		frappe.db.set_value("Google Settings", None, "google_drive_picker_enabled", 0)
+		frappe.db.set_single_value("Google Settings", "enable", 1)
+		frappe.db.set_single_value("Google Settings", "google_drive_picker_enabled", 0)
 		settings = get_file_picker_settings()
 
 		self.assertEqual(settings, {})
 
 	def test_google_disabled(self):
 		"""Google Drive Picker should be disabled if Google integration is not enabled."""
-		frappe.db.set_value("Google Settings", None, "enable", 0)
-		frappe.db.set_value("Google Settings", None, "google_drive_picker_enabled", 1)
+		frappe.db.set_single_value("Google Settings", "enable", 0)
+		frappe.db.set_single_value("Google Settings", "google_drive_picker_enabled", 1)
 		settings = get_file_picker_settings()
 
 		self.assertEqual(settings, {})
 
 	def test_picker_enabled(self):
 		"""If picker is enabled, get_file_picker_settings should return the credentials."""
-		frappe.db.set_value("Google Settings", None, "enable", 1)
-		frappe.db.set_value("Google Settings", None, "google_drive_picker_enabled", 1)
+		frappe.db.set_single_value("Google Settings", "enable", 1)
+		frappe.db.set_single_value("Google Settings", "google_drive_picker_enabled", 1)
 		settings = get_file_picker_settings()
 
 		self.assertEqual(True, settings.get("enabled", False))
diff --git a/frappe/integrations/doctype/integration_request/integration_request.js b/frappe/integrations/doctype/integration_request/integration_request.js
index 4b3b9a2de7..ac810f4d73 100644
--- a/frappe/integrations/doctype/integration_request/integration_request.js
+++ b/frappe/integrations/doctype/integration_request/integration_request.js
@@ -1,8 +1,6 @@
 // Copyright (c) 2016, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Integration Request', {
-	refresh: function(frm) {
-
-	}
+frappe.ui.form.on("Integration Request", {
+	refresh: function (frm) {},
 });
diff --git a/frappe/integrations/doctype/integration_request/integration_request.py b/frappe/integrations/doctype/integration_request/integration_request.py
index 334736bc9b..7ca185bd70 100644
--- a/frappe/integrations/doctype/integration_request/integration_request.py
+++ b/frappe/integrations/doctype/integration_request/integration_request.py
@@ -13,6 +13,13 @@ class IntegrationRequest(Document):
 		if self.flags._name:
 			self.name = self.flags._name
 
+	def clear_old_logs(days=30):
+		from frappe.query_builder import Interval
+		from frappe.query_builder.functions import Now
+
+		table = frappe.qb.DocType("Integration Request")
+		frappe.db.delete(table, filters=(table.modified < (Now() - Interval(days=days))))
+
 	def update_status(self, params, status):
 		data = json.loads(self.data)
 		data.update(params)
diff --git a/frappe/integrations/doctype/integration_request/integration_request_list.js b/frappe/integrations/doctype/integration_request/integration_request_list.js
new file mode 100644
index 0000000000..9aede34f29
--- /dev/null
+++ b/frappe/integrations/doctype/integration_request/integration_request_list.js
@@ -0,0 +1,7 @@
+frappe.listview_settings["Integration Request"] = {
+	onload: function (list_view) {
+		frappe.require("logtypes.bundle.js", () => {
+			frappe.utils.logtypes.show_log_retention_message(list_view.doctype);
+		});
+	},
+};
diff --git a/frappe/integrations/doctype/integration_request/test_integration_request.py b/frappe/integrations/doctype/integration_request/test_integration_request.py
index 45963d5096..d80d72fd7a 100644
--- a/frappe/integrations/doctype/integration_request/test_integration_request.py
+++ b/frappe/integrations/doctype/integration_request/test_integration_request.py
@@ -1,11 +1,10 @@
 # Copyright (c) 2015, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
-import unittest
-
 import frappe
+from frappe.tests.utils import FrappeTestCase
 
 # test_records = frappe.get_test_records('Integration Request')
 
 
-class TestIntegrationRequest(unittest.TestCase):
+class TestIntegrationRequest(FrappeTestCase):
 	pass
diff --git a/frappe/integrations/doctype/ldap_group_mapping/ldap_group_mapping.json b/frappe/integrations/doctype/ldap_group_mapping/ldap_group_mapping.json
index 92db68e962..9bfe1eac56 100644
--- a/frappe/integrations/doctype/ldap_group_mapping/ldap_group_mapping.json
+++ b/frappe/integrations/doctype/ldap_group_mapping/ldap_group_mapping.json
@@ -1,4 +1,5 @@
 {
+ "actions": [],
  "creation": "2019-05-29 01:24:29.585060",
  "doctype": "DocType",
  "editable_grid": 1,
@@ -19,13 +20,14 @@
    "fieldname": "erpnext_role",
    "fieldtype": "Link",
    "in_list_view": 1,
-   "label": "ERPNext Role",
+   "label": "User Role",
    "options": "Role",
    "reqd": 1
   }
  ],
  "istable": 1,
- "modified": "2019-07-15 06:46:38.050408",
+ "links": [],
+ "modified": "2022-07-07 16:28:44.828514",
  "modified_by": "Administrator",
  "module": "Integrations",
  "name": "LDAP Group Mapping",
@@ -34,5 +36,6 @@
  "quick_entry": 1,
  "sort_field": "modified",
  "sort_order": "ASC",
+ "states": [],
  "track_changes": 1
 }
\ No newline at end of file
diff --git a/frappe/integrations/doctype/ldap_settings/ldap_settings.js b/frappe/integrations/doctype/ldap_settings/ldap_settings.js
index 9ac95883b7..2ca7370ecf 100644
--- a/frappe/integrations/doctype/ldap_settings/ldap_settings.js
+++ b/frappe/integrations/doctype/ldap_settings/ldap_settings.js
@@ -1,8 +1,6 @@
 // Copyright (c) 2016, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('LDAP Settings', {
-	refresh: function(frm) {
-
-	}
+frappe.ui.form.on("LDAP Settings", {
+	refresh: function (frm) {},
 });
diff --git a/frappe/integrations/doctype/ldap_settings/ldap_settings.json b/frappe/integrations/doctype/ldap_settings/ldap_settings.json
index fd45a71538..0b3bf06239 100644
--- a/frappe/integrations/doctype/ldap_settings/ldap_settings.json
+++ b/frappe/integrations/doctype/ldap_settings/ldap_settings.json
@@ -24,6 +24,7 @@
   "ldap_email_field",
   "ldap_username_field",
   "ldap_first_name_field",
+  "do_not_create_new_user",
   "column_break_19",
   "ldap_middle_name_field",
   "ldap_last_name_field",
@@ -42,7 +43,10 @@
   "column_break_33",
   "ldap_group_member_attribute",
   "ldap_group_mappings_section",
+  "default_user_type",
+  "column_break_38",
   "default_role",
+  "section_break_40",
   "ldap_groups",
   "ldap_group_field"
  ],
@@ -79,11 +83,12 @@
    "reqd": 1
   },
   {
+   "depends_on": "eval: doc.default_user_type == \"System User\"",
    "fieldname": "default_role",
    "fieldtype": "Link",
-   "label": "Default Role on Creation",
-   "options": "Role",
-   "reqd": 1
+   "label": "Default User Role",
+   "mandatory_depends_on": "eval: doc.default_user_type == \"System User\"",
+   "options": "Role"
   },
   {
    "description": "Must be enclosed in '()' and include '{0}', which is a placeholder for the user/login name. i.e. (&(objectclass=user)(uid={0}))",
@@ -249,10 +254,10 @@
    "label": "Group Object Class"
   },
   {
-    "description": "string value, i.e. {0} or uid={0},ou=users,dc=example,dc=com",
-    "fieldname": "ldap_custom_group_search",
-    "fieldtype": "Data",
-    "label": "Custom Group Search"
+   "description": "string value, i.e. {0} or uid={0},ou=users,dc=example,dc=com",
+   "fieldname": "ldap_custom_group_search",
+   "fieldtype": "Data",
+   "label": "Custom Group Search"
   },
   {
    "description": "Requires any valid fdn path. i.e. ou=users,dc=example,dc=com",
@@ -268,12 +273,35 @@
    "fieldtype": "Data",
    "label": "LDAP search path for Groups",
    "reqd": 1
+  },
+  {
+   "fieldname": "default_user_type",
+   "fieldtype": "Link",
+   "label": "Default User Type",
+   "options": "User Type",
+   "reqd": 1
+  },
+  {
+   "fieldname": "column_break_38",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "section_break_40",
+   "fieldtype": "Section Break",
+   "hide_border": 1
+  },
+  {
+   "default": "0",
+   "description": "Do not create new user if user with email does not exist in the system",
+   "fieldname": "do_not_create_new_user",
+   "fieldtype": "Check",
+   "label": "Do Not Create New User "
   }
  ],
  "in_create": 1,
  "issingle": 1,
  "links": [],
- "modified": "2021-07-27 11:51:43.328271",
+ "modified": "2023-01-24 11:20:06.049708",
  "modified_by": "Administrator",
  "module": "Integrations",
  "name": "LDAP Settings",
@@ -294,5 +322,6 @@
  "read_only": 1,
  "sort_field": "modified",
  "sort_order": "DESC",
+ "states": [],
  "track_changes": 1
 }
\ No newline at end of file
diff --git a/frappe/integrations/doctype/ldap_settings/ldap_settings.py b/frappe/integrations/doctype/ldap_settings/ldap_settings.py
index ef6493717f..5b8ad1c901 100644
--- a/frappe/integrations/doctype/ldap_settings/ldap_settings.py
+++ b/frappe/integrations/doctype/ldap_settings/ldap_settings.py
@@ -1,19 +1,37 @@
-# Copyright (c) 2015, Frappe Technologies and contributors
+# Copyright (c) 2022, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
+import ssl
+from typing import TYPE_CHECKING
+
+import ldap3
+from ldap3 import AUTO_BIND_TLS_BEFORE_BIND, HASHED_SALTED_SHA, MODIFY_REPLACE
+from ldap3.abstract.entry import Entry
+from ldap3.core.exceptions import (
+	LDAPAttributeError,
+	LDAPInvalidCredentialsResult,
+	LDAPInvalidFilterError,
+	LDAPNoSuchObjectResult,
+)
+from ldap3.utils.hashed import hashed
+
 import frappe
 from frappe import _, safe_encode
 from frappe.model.document import Document
 from frappe.twofactor import authenticate_for_2factor, confirm_otp_token, should_run_2fa
 
+if TYPE_CHECKING:
+	from frappe.core.doctype.user.user import User
+
 
 class LDAPSettings(Document):
 	def validate(self):
+		self.default_user_type = self.default_user_type or "Website User"
+
 		if not self.enabled:
 			return
 
 		if not self.flags.ignore_mandatory:
-
 			if (
 				self.ldap_search_string.count("(") == self.ldap_search_string.count(")")
 				and self.ldap_search_string.startswith("(")
@@ -28,8 +46,6 @@ class LDAPSettings(Document):
 
 				try:
 					if conn.result["type"] == "bindResponse" and self.base_dn:
-						import ldap3
-
 						conn.search(
 							search_base=self.ldap_search_path_user,
 							search_filter="(objectClass=*)",
@@ -40,13 +56,13 @@ class LDAPSettings(Document):
 							search_base=self.ldap_search_path_group, search_filter="(objectClass=*)", attributes=["cn"]
 						)
 
-				except ldap3.core.exceptions.LDAPAttributeError as ex:
+				except LDAPAttributeError as ex:
 					frappe.throw(
 						_("LDAP settings incorrect. validation response was: {0}").format(ex),
 						title=_("Misconfigured"),
 					)
 
-				except ldap3.core.exceptions.LDAPNoSuchObjectResult:
+				except LDAPNoSuchObjectResult:
 					frappe.throw(
 						_("Ensure the user and group search paths are correct."), title=_("Misconfigured")
 					)
@@ -75,12 +91,8 @@ class LDAPSettings(Document):
 					)
 				)
 
-	def connect_to_ldap(self, base_dn, password, read_only=True):
+	def connect_to_ldap(self, base_dn, password, read_only=True) -> ldap3.Connection:
 		try:
-			import ssl
-
-			import ldap3
-
 			if self.require_trusted_certificate == "Yes":
 				tls_configuration = ldap3.Tls(validate=ssl.CERT_REQUIRED, version=ssl.PROTOCOL_TLS_CLIENT)
 			else:
@@ -94,9 +106,9 @@ class LDAPSettings(Document):
 				tls_configuration.ca_certs_file = self.local_ca_certs_file
 
 			server = ldap3.Server(host=self.ldap_server_url, tls=tls_configuration)
-			bind_type = ldap3.AUTO_BIND_TLS_BEFORE_BIND if self.ssl_tls_mode == "StartTLS" else True
+			bind_type = AUTO_BIND_TLS_BEFORE_BIND if self.ssl_tls_mode == "StartTLS" else True
 
-			conn = ldap3.Connection(
+			return ldap3.Connection(
 				server=server,
 				user=base_dn,
 				password=password,
@@ -105,18 +117,16 @@ class LDAPSettings(Document):
 				raise_exceptions=True,
 			)
 
-			return conn
-
 		except ImportError:
 			msg = _("Please Install the ldap3 library via pip to use ldap functionality.")
 			frappe.throw(msg, title=_("LDAP Not Installed"))
-		except ldap3.core.exceptions.LDAPInvalidCredentialsResult:
+		except LDAPInvalidCredentialsResult:
 			frappe.throw(_("Invalid username or password"))
 		except Exception as ex:
 			frappe.throw(_(str(ex)))
 
 	@staticmethod
-	def get_ldap_client_settings():
+	def get_ldap_client_settings() -> dict:
 		# return the settings to be used on the client side.
 		result = {"enabled": False}
 		ldap = frappe.get_cached_doc("LDAP Settings")
@@ -126,21 +136,19 @@ class LDAPSettings(Document):
 		return result
 
 	@classmethod
-	def update_user_fields(cls, user, user_data):
-
+	def update_user_fields(cls, user: "User", user_data: dict):
 		updatable_data = {key: value for key, value in user_data.items() if key != "email"}
 
 		for key, value in updatable_data.items():
 			setattr(user, key, value)
 		user.save(ignore_permissions=True)
 
-	def sync_roles(self, user, additional_groups=None):
-
+	def sync_roles(self, user: "User", additional_groups: list = None):
 		current_roles = {d.role for d in user.get("roles")}
-
-		needed_roles = set()
-		needed_roles.add(self.default_role)
-
+		if self.default_user_type == "System User":
+			needed_roles = {self.default_role}
+		else:
+			needed_roles = set()
 		lower_groups = [g.lower() for g in additional_groups or []]
 
 		all_mapped_roles = {r.erpnext_role for r in self.ldap_groups}
@@ -157,28 +165,37 @@ class LDAPSettings(Document):
 
 		user.remove_roles(*roles_to_remove)
 
-	def create_or_update_user(self, user_data, groups=None):
-		user = None
+	def create_or_update_user(self, user_data: dict, groups: list = None):
+		user: "User" = None
+		role: str = None
+
 		if frappe.db.exists("User", user_data["email"]):
 			user = frappe.get_doc("User", user_data["email"])
 			LDAPSettings.update_user_fields(user=user, user_data=user_data)
-		else:
-			doc = user_data
-			doc.update(
-				{
-					"doctype": "User",
-					"send_welcome_email": 0,
-					"language": "",
-					"user_type": "System User",
-					# "roles": [{
-					# 	"role": self.default_role
-					# }]
-				}
-			)
+		elif not self.do_not_create_new_user:
+			doc = user_data | {
+				"doctype": "User",
+				"send_welcome_email": 0,
+				"language": "",
+				"user_type": self.default_user_type,
+			}
 			user = frappe.get_doc(doc)
 			user.insert(ignore_permissions=True)
-		# always add default role.
-		user.add_roles(self.default_role)
+		else:
+			frappe.throw(
+				_(
+					"User with email: {0} does not exist in the system. Please ask 'System Administrator' to create the user for you."
+				).format(user_data["email"])
+			)
+
+		if self.default_user_type == "System User":
+			role = self.default_role
+		else:
+			role = frappe.db.get_value("User Type", user.user_type, "role")
+
+		if role:
+			user.add_roles(role)
+
 		self.sync_roles(user, groups)
 
 		return user
@@ -203,38 +220,28 @@ class LDAPSettings(Document):
 
 		return ldap_attributes
 
-	def fetch_ldap_groups(self, user, conn):
-		import ldap3
+	def fetch_ldap_groups(self, user: Entry, conn: ldap3.Connection) -> list:
+		if not isinstance(user, Entry):
+			raise TypeError("Invalid type, attribute 'user' must be of type 'ldap3.abstract.entry.Entry'")
 
-		if type(user) is not ldap3.abstract.entry.Entry:
-			raise TypeError(
-				"Invalid type, attribute {} must be of type '{}'".format("user", "ldap3.abstract.entry.Entry")
-			)
-
-		if type(conn) is not ldap3.core.connection.Connection:
-			raise TypeError(
-				"Invalid type, attribute {} must be of type '{}'".format("conn", "ldap3.Connection")
-			)
+		if not isinstance(conn, ldap3.Connection):
+			raise TypeError("Invalid type, attribute 'conn' must be of type 'ldap3.Connection'")
 
 		fetch_ldap_groups = None
-
 		ldap_object_class = None
 		ldap_group_members_attribute = None
 
 		if self.ldap_directory_server.lower() == "active directory":
-
 			ldap_object_class = "Group"
 			ldap_group_members_attribute = "member"
 			user_search_str = user.entry_dn
 
 		elif self.ldap_directory_server.lower() == "openldap":
-
 			ldap_object_class = "posixgroup"
 			ldap_group_members_attribute = "memberuid"
 			user_search_str = getattr(user, self.ldap_username_field).value
 
 		elif self.ldap_directory_server.lower() == "custom":
-
 			ldap_object_class = self.ldap_group_objectclass
 			ldap_group_members_attribute = self.ldap_group_member_attribute
 			ldap_custom_group_search = self.ldap_custom_group_search or "{0}"
@@ -245,39 +252,31 @@ class LDAPSettings(Document):
 			# this path will be hit for everyone with preconfigured ldap settings. this must be taken into account so as not to break ldap for those users.
 
 			if self.ldap_group_field:
-
 				fetch_ldap_groups = getattr(user, self.ldap_group_field).values
 
 		if ldap_object_class is not None:
 			conn.search(
 				search_base=self.ldap_search_path_group,
-				search_filter="(&(objectClass={})({}={}))".format(
-					ldap_object_class, ldap_group_members_attribute, user_search_str
-				),
+				search_filter=f"(&(objectClass={ldap_object_class})({ldap_group_members_attribute}={user_search_str}))",
 				attributes=["cn"],
 			)  # Build search query
 
 		if len(conn.entries) >= 1:
-
 			fetch_ldap_groups = []
 			for group in conn.entries:
 				fetch_ldap_groups.append(group["cn"].value)
 
 		return fetch_ldap_groups
 
-	def authenticate(self, username, password):
-
+	def authenticate(self, username: str, password: str):
 		if not self.enabled:
 			frappe.throw(_("LDAP is not enabled."))
 
 		user_filter = self.ldap_search_string.format(username)
 		ldap_attributes = self.get_ldap_attributes()
-
 		conn = self.connect_to_ldap(self.base_dn, self.get_password(raise_exception=False))
 
 		try:
-			import ldap3
-
 			conn.search(
 				search_base=self.ldap_search_path_user,
 				search_filter=f"{user_filter}",
@@ -286,26 +285,21 @@ class LDAPSettings(Document):
 
 			if len(conn.entries) == 1 and conn.entries[0]:
 				user = conn.entries[0]
-
 				groups = self.fetch_ldap_groups(user, conn)
 
 				# only try and connect as the user, once we have their fqdn entry.
 				if user.entry_dn and password and conn.rebind(user=user.entry_dn, password=password):
-
 					return self.create_or_update_user(self.convert_ldap_entry_to_dict(user), groups=groups)
 
-			raise ldap3.core.exceptions.LDAPInvalidCredentialsResult  # even though nothing foundor failed authentication raise invalid credentials
+			raise LDAPInvalidCredentialsResult  # even though nothing foundor failed authentication raise invalid credentials
 
-		except ldap3.core.exceptions.LDAPInvalidFilterError:
+		except LDAPInvalidFilterError:
 			frappe.throw(_("Please use a valid LDAP search filter"), title=_("Misconfigured"))
 
-		except ldap3.core.exceptions.LDAPInvalidCredentialsResult:
+		except LDAPInvalidCredentialsResult:
 			frappe.throw(_("Invalid username or password"))
 
 	def reset_password(self, user, password, logout_sessions=False):
-		from ldap3 import HASHED_SALTED_SHA, MODIFY_REPLACE
-		from ldap3.utils.hashed import hashed
-
 		search_filter = f"({self.ldap_email_field}={user})"
 
 		conn = self.connect_to_ldap(
@@ -334,19 +328,27 @@ class LDAPSettings(Document):
 		else:
 			frappe.throw(_("No LDAP User found for email: {0}").format(user))
 
-	def convert_ldap_entry_to_dict(self, user_entry):
-
+	def convert_ldap_entry_to_dict(self, user_entry: Entry):
 		# support multiple email values
-		email = user_entry[self.ldap_email_field]
+		email = user_entry[self.ldap_email_field].value
+
+		if isinstance(email, list):
+			# check if any of the email in the list already exist
+			for e in email:
+				if frappe.db.exists("User", e):
+					email = e
+					break
+			else:
+				# if none of the email exist, use the first email
+				email = email[0]
 
 		data = {
 			"username": user_entry[self.ldap_username_field].value,
-			"email": str(email.value[0] if isinstance(email.value, list) else email.value),
+			"email": email,
 			"first_name": user_entry[self.ldap_first_name_field].value,
 		}
 
 		# optional fields
-
 		if self.ldap_middle_name_field:
 			data["middle_name"] = user_entry[self.ldap_middle_name_field].value
 
@@ -366,7 +368,7 @@ class LDAPSettings(Document):
 def login():
 	# LDAP LOGIN LOGIC
 	args = frappe.form_dict
-	ldap = frappe.get_doc("LDAP Settings")
+	ldap: LDAPSettings = frappe.get_doc("LDAP Settings")
 
 	user = ldap.authenticate(frappe.as_unicode(args.usr), frappe.as_unicode(args.pwd))
 
@@ -375,6 +377,8 @@ def login():
 		authenticate_for_2factor(user.name)
 		if not confirm_otp_token(frappe.local.login_manager):
 			return False
+
+	frappe.form_dict.pop("pwd", None)
 	frappe.local.login_manager.post_login()
 
 	# because of a GET request!
@@ -383,7 +387,7 @@ def login():
 
 @frappe.whitelist()
 def reset_password(user, password, logout):
-	ldap = frappe.get_doc("LDAP Settings")
+	ldap: LDAPSettings = frappe.get_doc("LDAP Settings")
 	if not ldap.enabled:
 		frappe.throw(_("LDAP is not enabled."))
 	ldap.reset_password(user, password, logout_sessions=int(logout))
diff --git a/frappe/integrations/doctype/ldap_settings/test_ldap_settings.py b/frappe/integrations/doctype/ldap_settings/test_ldap_settings.py
index f53b5291b3..0417ea30e4 100644
--- a/frappe/integrations/doctype/ldap_settings/test_ldap_settings.py
+++ b/frappe/integrations/doctype/ldap_settings/test_ldap_settings.py
@@ -1,15 +1,16 @@
-# Copyright (c) 2019, Frappe Technologies and Contributors
+# Copyright (c) 2022, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
+import contextlib
 import functools
 import os
 import ssl
-import unittest
-from unittest import mock
+from unittest import TestCase, mock
 
 import ldap3
 from ldap3 import MOCK_SYNC, OFFLINE_AD_2012_R2, OFFLINE_SLAPD_2_4, Connection, Server
 
 import frappe
+from frappe.exceptions import MandatoryError, ValidationError
 from frappe.integrations.doctype.ldap_settings.ldap_settings import LDAPSettings
 
 
@@ -22,15 +23,19 @@ class LDAP_TestCase:
 	LDAP_LDIF_JSON = None
 	TEST_VALUES_LDAP_COMPLEX_SEARCH_STRING = None
 
+	# for adding type hints during development ^_^
+	assertTrue = TestCase.assertTrue
+	assertEqual = TestCase.assertEqual
+	assertIn = TestCase.assertIn
+
 	def mock_ldap_connection(f):
 		@functools.wraps(f)
 		def wrapped(self, *args, **kwargs):
 
 			with mock.patch(
-				"frappe.integrations.doctype.ldap_settings.ldap_settings.LDAPSettings.connect_to_ldap"
-			) as mock_connection:
-				mock_connection.return_value = self.connection
-
+				"frappe.integrations.doctype.ldap_settings.ldap_settings.LDAPSettings.connect_to_ldap",
+				return_value=self.connection,
+			):
 				self.test_class = LDAPSettings(self.doc)
 
 				# Create a clean doc
@@ -47,80 +52,66 @@ class LDAP_TestCase:
 		return wrapped
 
 	def clean_test_users():
-		try:  # clean up test user 1
+		with contextlib.suppress(Exception):
 			frappe.get_doc("User", "posix.user1@unit.testing").delete()
-		except Exception:
-			pass
-
-		try:  # clean up test user 2
+		with contextlib.suppress(Exception):
 			frappe.get_doc("User", "posix.user2@unit.testing").delete()
-		except Exception:
-			pass
+		with contextlib.suppress(Exception):
+			frappe.get_doc("User", "website_ldap_user@test.com").delete()
 
 	@classmethod
-	def setUpClass(self, ldapServer="OpenLDAP"):
-
-		self.clean_test_users()
+	def setUpClass(cls):
+		cls.clean_test_users()
 		# Save user data for restoration in tearDownClass()
-		self.user_ldap_settings = frappe.get_doc("LDAP Settings")
+		cls.user_ldap_settings = frappe.get_doc("LDAP Settings")
 
 		# Create test user1
-		self.user1doc = {
+		cls.user1doc = {
 			"username": "posix.user",
 			"email": "posix.user1@unit.testing",
 			"first_name": "posix",
+			"doctype": "User",
+			"send_welcome_email": 0,
+			"language": "",
+			"user_type": "System User",
 		}
-		self.user1doc.update(
-			{
-				"doctype": "User",
-				"send_welcome_email": 0,
-				"language": "",
-				"user_type": "System User",
-			}
-		)
 
-		user = frappe.get_doc(self.user1doc)
+		user = frappe.get_doc(cls.user1doc)
 		user.insert(ignore_permissions=True)
 
-		# Create test user1
-		self.user2doc = {
+		cls.user2doc = {
 			"username": "posix.user2",
 			"email": "posix.user2@unit.testing",
 			"first_name": "posix",
+			"doctype": "User",
+			"send_welcome_email": 0,
+			"language": "",
+			"user_type": "System User",
 		}
-		self.user2doc.update(
-			{
-				"doctype": "User",
-				"send_welcome_email": 0,
-				"language": "",
-				"user_type": "System User",
-			}
-		)
-
-		user = frappe.get_doc(self.user2doc)
+		user = frappe.get_doc(cls.user2doc)
 		user.insert(ignore_permissions=True)
 
 		# Setup Mock OpenLDAP Directory
-		self.ldap_dc_path = "dc=unit,dc=testing"
-		self.ldap_user_path = "ou=users," + self.ldap_dc_path
-		self.ldap_group_path = "ou=groups," + self.ldap_dc_path
-		self.base_dn = "cn=base_dn_user," + self.ldap_dc_path
-		self.base_password = "my_password"
-		self.ldap_server = "ldap://my_fake_server:389"
+		cls.ldap_dc_path = "dc=unit,dc=testing"
+		cls.ldap_user_path = f"ou=users,{cls.ldap_dc_path}"
+		cls.ldap_group_path = f"ou=groups,{cls.ldap_dc_path}"
+		cls.base_dn = f"cn=base_dn_user,{cls.ldap_dc_path}"
+		cls.base_password = "my_password"
+		cls.ldap_server = "ldap://my_fake_server:389"
 
-		self.doc = {
+		cls.doc = {
 			"doctype": "LDAP Settings",
 			"enabled": True,
-			"ldap_directory_server": self.TEST_LDAP_SERVER,
-			"ldap_server_url": self.ldap_server,
-			"base_dn": self.base_dn,
-			"password": self.base_password,
-			"ldap_search_path_user": self.ldap_user_path,
-			"ldap_search_string": self.TEST_LDAP_SEARCH_STRING,
-			"ldap_search_path_group": self.ldap_group_path,
+			"ldap_directory_server": cls.TEST_LDAP_SERVER,
+			"ldap_server_url": cls.ldap_server,
+			"base_dn": cls.base_dn,
+			"password": cls.base_password,
+			"ldap_search_path_user": cls.ldap_user_path,
+			"ldap_search_string": cls.TEST_LDAP_SEARCH_STRING,
+			"ldap_search_path_group": cls.ldap_group_path,
 			"ldap_user_creation_and_mapping_section": "",
 			"ldap_email_field": "mail",
-			"ldap_username_field": self.LDAP_USERNAME_FIELD,
+			"ldap_username_field": cls.LDAP_USERNAME_FIELD,
 			"ldap_first_name_field": "givenname",
 			"ldap_middle_name_field": "",
 			"ldap_last_name_field": "sn",
@@ -135,50 +126,41 @@ class LDAP_TestCase:
 			"ldap_group_objectclass": "",
 			"ldap_group_member_attribute": "",
 			"default_role": "Newsletter Manager",
-			"ldap_groups": self.DOCUMENT_GROUP_MAPPINGS,
+			"ldap_groups": cls.DOCUMENT_GROUP_MAPPINGS,
 			"ldap_group_field": "",
+			"default_user_type": "System User",
 		}
 
-		self.server = Server(host=self.ldap_server, port=389, get_info=self.LDAP_SCHEMA)
-
-		self.connection = Connection(
-			self.server,
-			user=self.base_dn,
-			password=self.base_password,
+		cls.server = Server(host=cls.ldap_server, port=389, get_info=cls.LDAP_SCHEMA)
+		cls.connection = Connection(
+			cls.server,
+			user=cls.base_dn,
+			password=cls.base_password,
 			read_only=True,
 			client_strategy=MOCK_SYNC,
 		)
-
-		self.connection.strategy.entries_from_json(
-			os.path.abspath(os.path.dirname(__file__)) + "/" + self.LDAP_LDIF_JSON
+		cls.connection.strategy.entries_from_json(
+			f"{os.path.abspath(os.path.dirname(__file__))}/{cls.LDAP_LDIF_JSON}"
 		)
-
-		self.connection.bind()
+		cls.connection.bind()
 
 	@classmethod
-	def tearDownClass(self):
-		try:
+	def tearDownClass(cls):
+		with contextlib.suppress(Exception):
 			frappe.get_doc("LDAP Settings").delete()
 
-		except Exception:
-			pass
-
-		try:
-			# return doc back to user data
-			self.user_ldap_settings.save()
-
-		except Exception:
-			pass
+		# return doc back to user data
+		with contextlib.suppress(Exception):
+			cls.user_ldap_settings.save()
 
 		# Clean-up test users
-		self.clean_test_users()
+		cls.clean_test_users()
 
 		# Clear OpenLDAP connection
-		self.connection = None
+		cls.connection = None
 
 	@mock_ldap_connection
 	def test_mandatory_fields(self):
-
 		mandatory_fields = [
 			"ldap_server_url",
 			"ldap_directory_server",
@@ -191,30 +173,17 @@ class LDAP_TestCase:
 			"ldap_username_field",
 			"ldap_first_name_field",
 			"require_trusted_certificate",
-			"default_role",
 		]  # fields that are required to have ldap functioning need to be mandatory
 
 		for mandatory_field in mandatory_fields:
-
 			localdoc = self.doc.copy()
 			localdoc[mandatory_field] = ""
 
-			try:
-
+			with contextlib.suppress(MandatoryError, ValidationError):
 				frappe.get_doc(localdoc).save()
-
 				self.fail(f"Document LDAP Settings field [{mandatory_field}] is not mandatory")
 
-			except frappe.exceptions.MandatoryError:
-				pass
-
-			except frappe.exceptions.ValidationError:
-				if mandatory_field == "ldap_search_string":
-					# additional validation is done on this field, pass in this instance
-					pass
-
 		for non_mandatory_field in self.doc:  # Ensure remaining fields have not been made mandatory
-
 			if non_mandatory_field == "doctype" or non_mandatory_field in mandatory_fields:
 				continue
 
@@ -222,15 +191,12 @@ class LDAP_TestCase:
 			localdoc[non_mandatory_field] = ""
 
 			try:
-
 				frappe.get_doc(localdoc).save()
-
-			except frappe.exceptions.MandatoryError:
+			except MandatoryError:
 				self.fail(f"Document LDAP Settings field [{non_mandatory_field}] should not be mandatory")
 
 	@mock_ldap_connection
 	def test_validation_ldap_search_string(self):
-
 		invalid_ldap_search_strings = [
 			"",
 			"uid={0}",
@@ -242,19 +208,26 @@ class LDAP_TestCase:
 		]  # ldap search string must be enclosed in '()' for ldap search to work for finding user and have the same number of opening and closing brackets.
 
 		for invalid_search_string in invalid_ldap_search_strings:
-
 			localdoc = self.doc.copy()
 			localdoc["ldap_search_string"] = invalid_search_string
 
-			try:
+			with contextlib.suppress(ValidationError):
 				frappe.get_doc(localdoc).save()
-
 				self.fail(f"LDAP search string [{invalid_search_string}] should not validate")
 
-			except frappe.exceptions.ValidationError:
-				pass
-
 	def test_connect_to_ldap(self):
+		# prevent these parameters for security or lack of the und user from being able to configure
+		prevent_connection_parameters = {
+			"mode": {
+				"IP_V4_ONLY": "Locks the user to IPv4 without frappe providing a way to configure",
+				"IP_V6_ONLY": "Locks the user to IPv6 without frappe providing a way to configure",
+			},
+			"auto_bind": {
+				"NONE": "ldap3.Connection must autobind with base_dn",
+				"NO_TLS": "ldap3.Connection must have TLS",
+				"TLS_AFTER_BIND": "[Security] ldap3.Connection TLS bind must occur before bind",
+			},
+		}
 
 		# setup a clean doc with ldap disabled so no validation occurs (this is tested seperatly)
 		local_doc = self.doc.copy()
@@ -262,48 +235,25 @@ class LDAP_TestCase:
 		self.test_class = LDAPSettings(self.doc)
 
 		with mock.patch("ldap3.Server") as ldap3_server_method:
-
-			with mock.patch("ldap3.Connection") as ldap3_connection_method:
-				ldap3_connection_method.return_value = self.connection
-
+			with mock.patch("ldap3.Connection", return_value=self.connection) as ldap3_connection_method:
 				with mock.patch("ldap3.Tls") as ldap3_Tls_method:
-
 					function_return = self.test_class.connect_to_ldap(
 						base_dn=self.base_dn, password=self.base_password
 					)
-
 					args, kwargs = ldap3_connection_method.call_args
 
-					prevent_connection_parameters = {
-						# prevent these parameters for security or lack of the und user from being able to configure
-						"mode": {
-							"IP_V4_ONLY": "Locks the user to IPv4 without frappe providing a way to configure",
-							"IP_V6_ONLY": "Locks the user to IPv6 without frappe providing a way to configure",
-						},
-						"auto_bind": {
-							"NONE": "ldap3.Connection must autobind with base_dn",
-							"NO_TLS": "ldap3.Connection must have TLS",
-							"TLS_AFTER_BIND": "[Security] ldap3.Connection TLS bind must occur before bind",
-						},
-					}
-
 					for connection_arg in kwargs:
-
 						if (
 							connection_arg in prevent_connection_parameters
 							and kwargs[connection_arg] in prevent_connection_parameters[connection_arg]
 						):
-
 							self.fail(
-								"ldap3.Connection was called with {}, failed reason: [{}]".format(
-									kwargs[connection_arg],
-									prevent_connection_parameters[connection_arg][kwargs[connection_arg]],
-								)
+								f"ldap3.Connection was called with {kwargs[connection_arg]}, failed reason: [{prevent_connection_parameters[connection_arg][kwargs[connection_arg]]}]"
 							)
 
+					tls_version = ssl.PROTOCOL_TLS_CLIENT
 					if local_doc["require_trusted_certificate"] == "Yes":
 						tls_validate = ssl.CERT_REQUIRED
-						tls_version = ssl.PROTOCOL_TLS_CLIENT
 						tls_configuration = ldap3.Tls(validate=tls_validate, version=tls_version)
 
 						self.assertTrue(
@@ -313,7 +263,6 @@ class LDAP_TestCase:
 
 					else:
 						tls_validate = ssl.CERT_NONE
-						tls_version = ssl.PROTOCOL_TLS_CLIENT
 						tls_configuration = ldap3.Tls(validate=tls_validate, version=tls_version)
 
 						self.assertTrue(kwargs["auto_bind"], "ldap3.Connection must autobind")
@@ -347,7 +296,7 @@ class LDAP_TestCase:
 					)
 
 					self.assertTrue(
-						type(function_return) is ldap3.core.connection.Connection,
+						type(function_return) is Connection,
 						"The return type must be of ldap3.Connection",
 					)
 
@@ -364,24 +313,20 @@ class LDAP_TestCase:
 
 	@mock_ldap_connection
 	def test_get_ldap_client_settings(self):
-
 		result = self.test_class.get_ldap_client_settings()
 
 		self.assertIsInstance(result, dict)
-
 		self.assertTrue(result["enabled"] == self.doc["enabled"])  # settings should match doc
 
 		localdoc = self.doc.copy()
 		localdoc["enabled"] = False
 		frappe.get_doc(localdoc).save()
-
 		result = self.test_class.get_ldap_client_settings()
 
 		self.assertFalse(result["enabled"])  # must match the edited doc
 
 	@mock_ldap_connection
 	def test_update_user_fields(self):
-
 		test_user_data = {
 			"username": "posix.user",
 			"email": "posix.user1@unit.testing",
@@ -391,11 +336,8 @@ class LDAP_TestCase:
 			"phone": "08 1234 5678",
 			"mobile_no": "0421 123 456",
 		}
-
 		test_user = frappe.get_doc("User", test_user_data["email"])
-
 		self.test_class.update_user_fields(test_user, test_user_data)
-
 		updated_user = frappe.get_doc("User", test_user_data["email"])
 
 		self.assertTrue(updated_user.middle_name == test_user_data["middle_name"])
@@ -403,9 +345,23 @@ class LDAP_TestCase:
 		self.assertTrue(updated_user.phone == test_user_data["phone"])
 		self.assertTrue(updated_user.mobile_no == test_user_data["mobile_no"])
 
+		self.assertEqual(updated_user.user_type, self.test_class.default_user_type)
+		self.assertIn(self.test_class.default_role, frappe.get_roles(updated_user.name))
+
+	@mock_ldap_connection
+	def test_create_website_user(self):
+		new_test_user_data = {
+			"username": "website_ldap_user.test",
+			"email": "website_ldap_user@test.com",
+			"first_name": "Website User - LDAP Test",
+		}
+		self.test_class.default_user_type = "Website User"
+		self.test_class.create_or_update_user(user_data=new_test_user_data, groups=[])
+		new_user = frappe.get_doc("User", new_test_user_data["email"])
+		self.assertEqual(new_user.user_type, "Website User")
+
 	@mock_ldap_connection
 	def test_sync_roles(self):
-
 		if self.TEST_LDAP_SERVER.lower() == "openldap":
 			test_user_data = {
 				"posix.user1": [
@@ -457,9 +413,8 @@ class LDAP_TestCase:
 		user.insert(ignore_permissions=True)
 
 		for test_user in test_user_data:
-
-			test_user_doc = frappe.get_doc("User", test_user + "@unit.testing")
-			test_user_roles = frappe.get_roles(test_user + "@unit.testing")
+			test_user_doc = frappe.get_doc("User", f"{test_user}@unit.testing")
+			test_user_roles = frappe.get_roles(f"{test_user}@unit.testing")
 
 			self.assertTrue(
 				len(test_user_roles) == 2, "User should only be a part of the All and Guest roles"
@@ -467,28 +422,22 @@ class LDAP_TestCase:
 
 			self.test_class.sync_roles(test_user_doc, test_user_data[test_user])  # update user roles
 
-			frappe.get_doc("User", test_user + "@unit.testing")
-			updated_user_roles = frappe.get_roles(test_user + "@unit.testing")
+			frappe.get_doc("User", f"{test_user}@unit.testing")
+			updated_user_roles = frappe.get_roles(f"{test_user}@unit.testing")
 
 			self.assertTrue(
 				len(updated_user_roles) == len(test_user_data[test_user]),
-				"syncing of the user roles failed. {} != {} for user {}".format(
-					len(updated_user_roles), len(test_user_data[test_user]), test_user
-				),
+				f"syncing of the user roles failed. {len(updated_user_roles)} != {len(test_user_data[test_user])} for user {test_user}",
 			)
 
 			for user_role in updated_user_roles:  # match each users role mapped to ldap groups
-
 				self.assertTrue(
 					role_to_group_map[user_role] in test_user_data[test_user],
-					"during sync_roles(), the user was given role {} which should not have occured".format(
-						user_role
-					),
+					f"during sync_roles(), the user was given role {user_role} which should not have occured",
 				)
 
 	@mock_ldap_connection
 	def test_create_or_update_user(self):
-
 		test_user_data = {
 			"posix.user1": [
 				"Users",
@@ -498,28 +447,21 @@ class LDAP_TestCase:
 				"frappe_default_guest",
 			],
 		}
-
 		test_user = "posix.user1"
 
-		frappe.get_doc("User", test_user + "@unit.testing").delete()  # remove user 1
+		frappe.get_doc("User", f"{test_user}@unit.testing").delete()
 
 		with self.assertRaises(
 			frappe.exceptions.DoesNotExistError
 		):  # ensure user deleted so function can be tested
-			frappe.get_doc("User", test_user + "@unit.testing")
+			frappe.get_doc("User", f"{test_user}@unit.testing")
 
 		with mock.patch(
 			"frappe.integrations.doctype.ldap_settings.ldap_settings.LDAPSettings.update_user_fields"
 		) as update_user_fields_method:
-
-			update_user_fields_method.return_value = None
-
 			with mock.patch(
 				"frappe.integrations.doctype.ldap_settings.ldap_settings.LDAPSettings.sync_roles"
 			) as sync_roles_method:
-
-				sync_roles_method.return_value = None
-
 				# New user
 				self.test_class.create_or_update_user(self.user1doc, test_user_data[test_user])
 
@@ -539,14 +481,11 @@ class LDAP_TestCase:
 
 	@mock_ldap_connection
 	def test_get_ldap_attributes(self):
-
 		method_return = self.test_class.get_ldap_attributes()
-
 		self.assertTrue(type(method_return) is list)
 
 	@mock_ldap_connection
 	def test_fetch_ldap_groups(self):
-
 		if self.TEST_LDAP_SERVER.lower() == "openldap":
 			test_users = {"posix.user": ["Users", "Administrators"], "posix.user2": ["Users", "Group3"]}
 		elif self.TEST_LDAP_SERVER.lower() == "active directory":
@@ -556,7 +495,6 @@ class LDAP_TestCase:
 			}
 
 		for test_user in test_users:
-
 			self.connection.search(
 				search_base=self.ldap_user_path,
 				search_filter=self.TEST_LDAP_SEARCH_STRING.format(test_user),
@@ -569,18 +507,13 @@ class LDAP_TestCase:
 			self.assertTrue(len(method_return) == len(test_users[test_user]))
 
 			for returned_group in method_return:
-
 				self.assertTrue(returned_group in test_users[test_user])
 
 	@mock_ldap_connection
 	def test_authenticate(self):
-
 		with mock.patch(
 			"frappe.integrations.doctype.ldap_settings.ldap_settings.LDAPSettings.fetch_ldap_groups"
 		) as fetch_ldap_groups_function:
-
-			fetch_ldap_groups_function.return_value = None
-
 			self.assertTrue(self.test_class.authenticate("posix.user", "posix_user_password"))
 
 		self.assertTrue(
@@ -599,25 +532,19 @@ class LDAP_TestCase:
 		]  # All invalid users should return 'invalid username or password'
 
 		for username, password in enumerate(invalid_users):
-
 			with self.assertRaises(frappe.exceptions.ValidationError) as display_massage:
-
 				self.test_class.authenticate(username, password)
 
 			self.assertTrue(
 				str(display_massage.exception).lower() == "invalid username or password",
-				"invalid credentials passed authentication [user: {}, password: {}]".format(
-					username, password
-				),
+				f"invalid credentials passed authentication [user: {username}, password: {password}]",
 			)
 
 	@mock_ldap_connection
 	def test_complex_ldap_search_filter(self):
-
 		ldap_search_filters = self.TEST_VALUES_LDAP_COMPLEX_SEARCH_STRING
 
 		for search_filter in ldap_search_filters:
-
 			self.test_class.ldap_search_string = search_filter
 
 			if (
@@ -634,55 +561,44 @@ class LDAP_TestCase:
 				self.assertTrue(self.test_class.authenticate("posix.user", "posix_user_password"))
 
 	def test_reset_password(self):
-
 		self.test_class = LDAPSettings(self.doc)
 
 		# Create a clean doc
 		localdoc = self.doc.copy()
-
 		localdoc["enabled"] = False
 		frappe.get_doc(localdoc).save()
 
 		with mock.patch(
-			"frappe.integrations.doctype.ldap_settings.ldap_settings.LDAPSettings.connect_to_ldap"
+			"frappe.integrations.doctype.ldap_settings.ldap_settings.LDAPSettings.connect_to_ldap",
+			return_value=self.connection,
 		) as connect_to_ldap:
-			connect_to_ldap.return_value = self.connection
-
 			with self.assertRaises(
 				frappe.exceptions.ValidationError
 			) as validation:  # Fail if username string used
 				self.test_class.reset_password("posix.user", "posix_user_password")
-
 			self.assertTrue(str(validation.exception) == "No LDAP User found for email: posix.user")
 
-			try:
+			with contextlib.suppress(Exception):
 				self.test_class.reset_password(
 					"posix.user1@unit.testing", "posix_user_password"
 				)  # Change Password
-
-			except Exception:  # An exception from the tested class is ok, as long as the connection to LDAP was made writeable
-				pass
-
 			connect_to_ldap.assert_called_with(self.base_dn, self.base_password, read_only=False)
 
 	@mock_ldap_connection
 	def test_convert_ldap_entry_to_dict(self):
-
 		self.connection.search(
 			search_base=self.ldap_user_path,
 			search_filter=self.TEST_LDAP_SEARCH_STRING.format("posix.user"),
 			attributes=self.test_class.get_ldap_attributes(),
 		)
-
 		test_ldap_entry = self.connection.entries[0]
-
 		method_return = self.test_class.convert_ldap_entry_to_dict(test_ldap_entry)
 
 		self.assertTrue(type(method_return) is dict)  # must be dict
 		self.assertTrue(len(method_return) == 6)  # there are 6 fields in mock_ldap for use
 
 
-class Test_OpenLDAP(LDAP_TestCase, unittest.TestCase):
+class Test_OpenLDAP(LDAP_TestCase, TestCase):
 	TEST_LDAP_SERVER = "OpenLDAP"
 	TEST_LDAP_SEARCH_STRING = "(uid={0})"
 	DOCUMENT_GROUP_MAPPINGS = [
@@ -706,7 +622,7 @@ class Test_OpenLDAP(LDAP_TestCase, unittest.TestCase):
 	]
 
 
-class Test_ActiveDirectory(LDAP_TestCase, unittest.TestCase):
+class Test_ActiveDirectory(LDAP_TestCase, TestCase):
 	TEST_LDAP_SERVER = "Active Directory"
 	TEST_LDAP_SEARCH_STRING = "(samaccountname={0})"
 	DOCUMENT_GROUP_MAPPINGS = [
diff --git a/frappe/integrations/doctype/oauth_authorization_code/oauth_authorization_code.js b/frappe/integrations/doctype/oauth_authorization_code/oauth_authorization_code.js
index 32746e6752..83ad1b3ee5 100644
--- a/frappe/integrations/doctype/oauth_authorization_code/oauth_authorization_code.js
+++ b/frappe/integrations/doctype/oauth_authorization_code/oauth_authorization_code.js
@@ -1,8 +1,6 @@
 // Copyright (c) 2016, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('OAuth Authorization Code', {
-	refresh: function(frm) {
-
-	}
+frappe.ui.form.on("OAuth Authorization Code", {
+	refresh: function (frm) {},
 });
diff --git a/frappe/integrations/doctype/oauth_authorization_code/test_oauth_authorization_code.py b/frappe/integrations/doctype/oauth_authorization_code/test_oauth_authorization_code.py
index 2036a42f15..775bc0f968 100644
--- a/frappe/integrations/doctype/oauth_authorization_code/test_oauth_authorization_code.py
+++ b/frappe/integrations/doctype/oauth_authorization_code/test_oauth_authorization_code.py
@@ -1,11 +1,10 @@
 # Copyright (c) 2015, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
-import unittest
-
 import frappe
+from frappe.tests.utils import FrappeTestCase
 
 # test_records = frappe.get_test_records('OAuth Authorization Code')
 
 
-class TestOAuthAuthorizationCode(unittest.TestCase):
+class TestOAuthAuthorizationCode(FrappeTestCase):
 	pass
diff --git a/frappe/integrations/doctype/oauth_bearer_token/oauth_bearer_token.js b/frappe/integrations/doctype/oauth_bearer_token/oauth_bearer_token.js
index da69753903..7794f2fb70 100644
--- a/frappe/integrations/doctype/oauth_bearer_token/oauth_bearer_token.js
+++ b/frappe/integrations/doctype/oauth_bearer_token/oauth_bearer_token.js
@@ -1,8 +1,6 @@
 // Copyright (c) 2016, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('OAuth Bearer Token', {
-	refresh: function(frm) {
-
-	}
+frappe.ui.form.on("OAuth Bearer Token", {
+	refresh: function (frm) {},
 });
diff --git a/frappe/integrations/doctype/oauth_bearer_token/oauth_bearer_token.json b/frappe/integrations/doctype/oauth_bearer_token/oauth_bearer_token.json
index 083f1c9c54..2060c48fb9 100644
--- a/frappe/integrations/doctype/oauth_bearer_token/oauth_bearer_token.json
+++ b/frappe/integrations/doctype/oauth_bearer_token/oauth_bearer_token.json
@@ -67,6 +67,7 @@
   {
    "fieldname": "status",
    "fieldtype": "Select",
+   "in_list_view": 1,
    "in_standard_filter": 1,
    "label": "Status",
    "options": "Active\nRevoked",
@@ -74,10 +75,11 @@
   }
  ],
  "links": [],
- "modified": "2021-04-26 06:40:34.922441",
+ "modified": "2023-04-07 07:08:00.249740",
  "modified_by": "Administrator",
  "module": "Integrations",
  "name": "OAuth Bearer Token",
+ "naming_rule": "By fieldname",
  "owner": "Administrator",
  "permissions": [
   {
@@ -92,5 +94,6 @@
   }
  ],
  "sort_field": "modified",
- "sort_order": "DESC"
+ "sort_order": "DESC",
+ "states": []
 }
\ No newline at end of file
diff --git a/frappe/integrations/doctype/oauth_bearer_token/test_oauth_bearer_token.py b/frappe/integrations/doctype/oauth_bearer_token/test_oauth_bearer_token.py
index 3439096809..84b65893f6 100644
--- a/frappe/integrations/doctype/oauth_bearer_token/test_oauth_bearer_token.py
+++ b/frappe/integrations/doctype/oauth_bearer_token/test_oauth_bearer_token.py
@@ -1,11 +1,10 @@
 # Copyright (c) 2015, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
-import unittest
-
 import frappe
+from frappe.tests.utils import FrappeTestCase
 
 # test_records = frappe.get_test_records('OAuth Bearer Token')
 
 
-class TestOAuthBearerToken(unittest.TestCase):
+class TestOAuthBearerToken(FrappeTestCase):
 	pass
diff --git a/frappe/integrations/doctype/oauth_client/oauth_client.js b/frappe/integrations/doctype/oauth_client/oauth_client.js
index b0caa562b1..3ddd1a046b 100644
--- a/frappe/integrations/doctype/oauth_client/oauth_client.js
+++ b/frappe/integrations/doctype/oauth_client/oauth_client.js
@@ -1,8 +1,6 @@
 // Copyright (c) 2016, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('OAuth Client', {
-	refresh: function(frm) {
-
-	}
+frappe.ui.form.on("OAuth Client", {
+	refresh: function (frm) {},
 });
diff --git a/frappe/integrations/doctype/oauth_client/oauth_client.json b/frappe/integrations/doctype/oauth_client/oauth_client.json
index d0d45c36ab..8b863f62ad 100644
--- a/frappe/integrations/doctype/oauth_client/oauth_client.json
+++ b/frappe/integrations/doctype/oauth_client/oauth_client.json
@@ -1,517 +1,144 @@
 {
- "allow_copy": 0,
- "allow_guest_to_view": 0,
- "allow_import": 0,
- "allow_rename": 0,
- "autoname": "",
- "beta": 0,
+ "actions": [],
  "creation": "2016-08-24 14:07:21.955052",
- "custom": 0,
- "docstatus": 0,
  "doctype": "DocType",
  "document_type": "Document",
  "editable_grid": 1,
  "engine": "InnoDB",
+ "field_order": [
+  "client_id",
+  "app_name",
+  "user",
+  "cb_1",
+  "client_secret",
+  "skip_authorization",
+  "sb_1",
+  "scopes",
+  "cb_3",
+  "redirect_uris",
+  "default_redirect_uri",
+  "sb_advanced",
+  "grant_type",
+  "cb_2",
+  "response_type"
+ ],
  "fields": [
   {
-   "allow_bulk_edit": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
-   "default": "",
    "fieldname": "client_id",
    "fieldtype": "Data",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
    "label": "App Client ID",
-   "length": 0,
-   "no_copy": 0,
-   "permlevel": 0,
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 1,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "unique": 0
+   "read_only": 1
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
    "fieldname": "app_name",
    "fieldtype": "Data",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
    "label": "App Name",
-   "length": 0,
-   "no_copy": 0,
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 1,
-   "search_index": 0,
-   "set_only_once": 0,
-   "unique": 0
+   "reqd": 1
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
    "fieldname": "user",
    "fieldtype": "Link",
    "hidden": 1,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
    "label": "User",
-   "length": 0,
-   "no_copy": 0,
-   "options": "User",
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "unique": 0
+   "options": "User"
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
    "fieldname": "cb_1",
-   "fieldtype": "Column Break",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
-   "length": 0,
-   "no_copy": 0,
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "unique": 0
+   "fieldtype": "Column Break"
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
    "fieldname": "client_secret",
    "fieldtype": "Data",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
    "label": "App Client Secret",
-   "length": 0,
-   "no_copy": 0,
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 1,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "unique": 0
+   "read_only": 1
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
+   "default": "0",
    "description": "If checked, users will not see the Confirm Access dialog.",
    "fieldname": "skip_authorization",
    "fieldtype": "Check",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
-   "label": "Skip Authorization",
-   "length": 0,
-   "no_copy": 0,
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "unique": 0
+   "label": "Skip Authorization"
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
-   "description": "",
    "fieldname": "sb_1",
-   "fieldtype": "Section Break",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
-   "label": "",
-   "length": 0,
-   "no_copy": 0,
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "unique": 0
+   "fieldtype": "Section Break"
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
    "default": "all openid",
    "description": "A list of resources which the Client App will have access to after the user allows it.
         e.g. project",
    "fieldname": "scopes",
    "fieldtype": "Text",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
    "label": "Scopes",
-   "length": 0,
-   "no_copy": 0,
-   "permlevel": 0,
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 1,
-   "search_index": 0,
-   "set_only_once": 0,
-   "unique": 0
+   "reqd": 1
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
    "fieldname": "cb_3",
-   "fieldtype": "Column Break",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
-   "length": 0,
-   "no_copy": 0,
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "unique": 0
+   "fieldtype": "Column Break"
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
-   "description": "URIs for receiving authorization code once the user allows access, as well as failure responses. Typically a REST endpoint exposed by the Client App.\n
        e.g. http://hostname//api/method/frappe.www.login.login_via_facebook",
+   "description": "URIs for receiving authorization code once the user allows access, as well as failure responses. Typically a REST endpoint exposed by the Client App.\n
        e.g. http://hostname/api/method/frappe.www.login.login_via_facebook",
    "fieldname": "redirect_uris",
    "fieldtype": "Text",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
-   "label": "Redirect URIs",
-   "length": 0,
-   "no_copy": 0,
-   "permlevel": 0,
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "unique": 0
+   "label": "Redirect URIs"
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
    "fieldname": "default_redirect_uri",
    "fieldtype": "Data",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
    "label": "Default Redirect URI",
-   "length": 0,
-   "no_copy": 0,
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 1,
-   "search_index": 0,
-   "set_only_once": 0,
-   "unique": 0
+   "reqd": 1
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
    "collapsible": 1,
    "collapsible_depends_on": "1",
-   "columns": 0,
    "fieldname": "sb_advanced",
    "fieldtype": "Section Break",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
-   "label": "Advanced Settings",
-   "length": 0,
-   "no_copy": 0,
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "unique": 0
+   "label": "Advanced Settings"
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
    "fieldname": "grant_type",
    "fieldtype": "Select",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
    "in_list_view": 1,
    "in_standard_filter": 1,
    "label": "Grant Type",
-   "length": 0,
-   "no_copy": 0,
-   "options": "Authorization Code\nImplicit",
-   "permlevel": 0,
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "unique": 0
+   "options": "Authorization Code\nImplicit"
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
    "fieldname": "cb_2",
-   "fieldtype": "Column Break",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
-   "in_list_view": 0,
-   "in_standard_filter": 0,
-   "length": 0,
-   "no_copy": 0,
-   "permlevel": 0,
-   "precision": "",
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "unique": 0
+   "fieldtype": "Column Break"
   },
   {
-   "allow_bulk_edit": 0,
-   "allow_on_submit": 0,
-   "bold": 0,
-   "collapsible": 0,
-   "columns": 0,
    "default": "Code",
    "fieldname": "response_type",
    "fieldtype": "Select",
-   "hidden": 0,
-   "ignore_user_permissions": 0,
-   "ignore_xss_filter": 0,
-   "in_filter": 0,
-   "in_global_search": 0,
    "in_list_view": 1,
    "in_standard_filter": 1,
    "label": "Response Type",
-   "length": 0,
-   "no_copy": 0,
-   "options": "Code\nToken",
-   "permlevel": 0,
-   "print_hide": 0,
-   "print_hide_if_no_value": 0,
-   "read_only": 0,
-   "remember_last_selected_value": 0,
-   "report_hide": 0,
-   "reqd": 0,
-   "search_index": 0,
-   "set_only_once": 0,
-   "unique": 0
+   "options": "Code\nToken"
   }
  ],
- "has_web_view": 0,
- "hide_heading": 0,
- "hide_toolbar": 0,
- "idx": 0,
- "image_view": 0,
- "in_create": 0,
- "is_submittable": 0,
- "issingle": 0,
- "istable": 0,
- "max_attachments": 0,
- "modified": "2020-04-07 21:07:39.476360",
+ "links": [],
+ "modified": "2023-04-07 07:06:35.765981",
  "modified_by": "Administrator",
  "module": "Integrations",
  "name": "OAuth Client",
- "name_case": "",
  "owner": "Administrator",
  "permissions": [
   {
-   "amend": 0,
-   "apply_user_permissions": 0,
-   "cancel": 0,
    "create": 1,
    "delete": 1,
    "email": 1,
    "export": 1,
-   "if_owner": 0,
-   "import": 0,
-   "permlevel": 0,
    "print": 1,
    "read": 1,
    "report": 1,
    "role": "System Manager",
-   "set_user_permissions": 0,
    "share": 1,
-   "submit": 0,
    "write": 1
   }
  ],
- "quick_entry": 0,
- "read_only": 0,
- "read_only_onload": 0,
- "show_name_in_global_search": 0,
  "sort_field": "modified",
  "sort_order": "DESC",
+ "states": [],
  "title_field": "app_name",
- "track_changes": 1,
- "track_seen": 0
+ "track_changes": 1
 }
\ No newline at end of file
diff --git a/frappe/integrations/doctype/oauth_client/test_oauth_client.py b/frappe/integrations/doctype/oauth_client/test_oauth_client.py
index 8fd732673e..53e7bff40e 100644
--- a/frappe/integrations/doctype/oauth_client/test_oauth_client.py
+++ b/frappe/integrations/doctype/oauth_client/test_oauth_client.py
@@ -1,11 +1,10 @@
 # Copyright (c) 2015, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
-import unittest
-
 import frappe
+from frappe.tests.utils import FrappeTestCase
 
 # test_records = frappe.get_test_records('OAuth Client')
 
 
-class TestOAuthClient(unittest.TestCase):
+class TestOAuthClient(FrappeTestCase):
 	pass
diff --git a/frappe/integrations/doctype/oauth_client/test_records.json b/frappe/integrations/doctype/oauth_client/test_records.json
deleted file mode 100644
index 11e6338a87..0000000000
--- a/frappe/integrations/doctype/oauth_client/test_records.json
+++ /dev/null
@@ -1,15 +0,0 @@
-[
- {
-  "app_name": "_Test OAuth Client",
-  "client_secret": "test_client_secret", 
-  "default_redirect_uri": "http://localhost", 
-  "docstatus": 0, 
-  "doctype": "OAuth Client", 
-  "grant_type": "Authorization Code", 
-  "name": "test_client_id", 
-  "redirect_uris": "http://localhost", 
-  "response_type": "Code", 
-  "scopes": "all openid", 
-  "skip_authorization": 1
- }
-]
diff --git a/frappe/integrations/doctype/oauth_provider_settings/oauth_provider_settings.js b/frappe/integrations/doctype/oauth_provider_settings/oauth_provider_settings.js
index 6d7d071934..0071b4e977 100644
--- a/frappe/integrations/doctype/oauth_provider_settings/oauth_provider_settings.js
+++ b/frappe/integrations/doctype/oauth_provider_settings/oauth_provider_settings.js
@@ -1,8 +1,6 @@
 // Copyright (c) 2016, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('OAuth Provider Settings', {
-	refresh: function(frm) {
-
-	}
+frappe.ui.form.on("OAuth Provider Settings", {
+	refresh: function (frm) {},
 });
diff --git a/frappe/integrations/doctype/oauth_provider_settings/oauth_provider_settings.json b/frappe/integrations/doctype/oauth_provider_settings/oauth_provider_settings.json
index bf19eee6b1..219a87f2f4 100644
--- a/frappe/integrations/doctype/oauth_provider_settings/oauth_provider_settings.json
+++ b/frappe/integrations/doctype/oauth_provider_settings/oauth_provider_settings.json
@@ -1,90 +1,43 @@
 {
- "allow_copy": 0, 
- "allow_import": 0, 
- "allow_rename": 0, 
- "beta": 0, 
- "creation": "2016-09-03 11:42:42.575525", 
- "custom": 0, 
- "docstatus": 0, 
- "doctype": "DocType", 
- "document_type": "Setup", 
- "editable_grid": 1, 
- "engine": "InnoDB", 
+ "actions": [],
+ "creation": "2016-09-03 11:42:42.575525",
+ "doctype": "DocType",
+ "document_type": "Setup",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "skip_authorization"
+ ],
  "fields": [
   {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "skip_authorization", 
-   "fieldtype": "Select", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Skip Authorization", 
-   "length": 0, 
-   "no_copy": 0, 
-   "options": "Force\nAuto", 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
+   "fieldname": "skip_authorization",
+   "fieldtype": "Select",
+   "label": "Skip Authorization",
+   "options": "Force\nAuto"
   }
- ], 
- "hide_heading": 0, 
- "hide_toolbar": 0, 
- "idx": 0, 
- "image_view": 0, 
- "in_create": 0, 
-
- "is_submittable": 0, 
- "issingle": 1, 
- "istable": 0, 
- "max_attachments": 0, 
- "modified": "2016-12-29 14:40:30.718685", 
- "modified_by": "Administrator", 
- "module": "Integrations", 
- "name": "OAuth Provider Settings", 
- "name_case": "", 
- "owner": "Administrator", 
+ ],
+ "issingle": 1,
+ "links": [],
+ "modified": "2022-08-03 12:20:52.328415",
+ "modified_by": "Administrator",
+ "module": "Integrations",
+ "name": "OAuth Provider Settings",
+ "owner": "Administrator",
  "permissions": [
   {
-   "amend": 0, 
-   "apply_user_permissions": 0, 
-   "cancel": 0, 
-   "create": 1, 
-   "delete": 1, 
-   "email": 1, 
-   "export": 0, 
-   "if_owner": 0, 
-   "import": 0, 
-   "is_custom": 0, 
-   "permlevel": 0, 
-   "print": 1, 
-   "read": 1, 
-   "report": 0, 
-   "role": "System Manager", 
-   "set_user_permissions": 0, 
-   "share": 1, 
-   "submit": 0, 
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "print": 1,
+   "read": 1,
+   "role": "System Manager",
+   "share": 1,
    "write": 1
   }
- ], 
- "quick_entry": 1, 
- "read_only": 0, 
- "read_only_onload": 0, 
- "sort_field": "modified", 
- "sort_order": "DESC", 
- "track_changes": 1, 
- "track_seen": 0
+ ],
+ "quick_entry": 1,
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "states": [],
+ "track_changes": 1
 }
\ No newline at end of file
diff --git a/frappe/integrations/doctype/oauth_provider_settings/oauth_provider_settings.py b/frappe/integrations/doctype/oauth_provider_settings/oauth_provider_settings.py
index 984382df9d..5a918db587 100644
--- a/frappe/integrations/doctype/oauth_provider_settings/oauth_provider_settings.py
+++ b/frappe/integrations/doctype/oauth_provider_settings/oauth_provider_settings.py
@@ -14,7 +14,9 @@ def get_oauth_settings():
 	"""Returns oauth settings"""
 	out = frappe._dict(
 		{
-			"skip_authorization": frappe.db.get_value("OAuth Provider Settings", None, "skip_authorization")
+			"skip_authorization": frappe.db.get_single_value(
+				"OAuth Provider Settings", "skip_authorization"
+			)
 		}
 	)
 
diff --git a/frappe/integrations/doctype/paypal_settings/__init__.py b/frappe/integrations/doctype/paypal_settings/__init__.py
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/frappe/integrations/doctype/paypal_settings/paypal_settings.js b/frappe/integrations/doctype/paypal_settings/paypal_settings.js
deleted file mode 100644
index 63480bc927..0000000000
--- a/frappe/integrations/doctype/paypal_settings/paypal_settings.js
+++ /dev/null
@@ -1,8 +0,0 @@
-// Copyright (c) 2016, Frappe Technologies and contributors
-// For license information, please see license.txt
-
-frappe.ui.form.on('PayPal Settings', {
-	refresh: function(frm) {
-
-	}
-});
diff --git a/frappe/integrations/doctype/paypal_settings/paypal_settings.json b/frappe/integrations/doctype/paypal_settings/paypal_settings.json
deleted file mode 100644
index 8d48496a4c..0000000000
--- a/frappe/integrations/doctype/paypal_settings/paypal_settings.json
+++ /dev/null
@@ -1,202 +0,0 @@
-{
- "allow_copy": 0, 
- "allow_import": 0, 
- "allow_rename": 0, 
- "beta": 0, 
- "creation": "2016-09-21 08:03:01.009852", 
- "custom": 0, 
- "docstatus": 0, 
- "doctype": "DocType", 
- "document_type": "System", 
- "editable_grid": 1, 
- "fields": [
-  {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "api_username", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "API Username", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "api_password", 
-   "fieldtype": "Password", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "API Password", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "signature", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Signature", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "description": "Check this if you are testing your payment using the Sandbox API", 
-   "fieldname": "paypal_sandbox", 
-   "fieldtype": "Check", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Use Sandbox", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "description": "Mention transaction completion page URL", 
-   "fieldname": "redirect_to", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Redirect To", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }
- ], 
- "hide_heading": 0, 
- "hide_toolbar": 0, 
- "idx": 0, 
- "image_view": 0, 
- "in_create": 1, 
-
- "is_submittable": 0, 
- "issingle": 1, 
- "istable": 0, 
- "max_attachments": 0, 
- "modified": "2016-12-29 14:40:31.574789", 
- "modified_by": "Administrator", 
- "module": "Integrations", 
- "name": "PayPal Settings", 
- "name_case": "", 
- "owner": "Administrator", 
- "permissions": [
-  {
-   "amend": 0, 
-   "apply_user_permissions": 0, 
-   "cancel": 0, 
-   "create": 1, 
-   "delete": 1, 
-   "email": 1, 
-   "export": 1, 
-   "if_owner": 0, 
-   "import": 0, 
-   "is_custom": 0, 
-   "permlevel": 0, 
-   "print": 1, 
-   "read": 1, 
-   "report": 0, 
-   "role": "System Manager", 
-   "set_user_permissions": 0, 
-   "share": 1, 
-   "submit": 0, 
-   "write": 1
-  }
- ], 
- "quick_entry": 0, 
- "read_only": 1, 
- "read_only_onload": 0, 
- "sort_field": "modified", 
- "sort_order": "DESC", 
- "track_changes": 1, 
- "track_seen": 0
-}
\ No newline at end of file
diff --git a/frappe/integrations/doctype/paypal_settings/paypal_settings.py b/frappe/integrations/doctype/paypal_settings/paypal_settings.py
deleted file mode 100644
index 99c499200b..0000000000
--- a/frappe/integrations/doctype/paypal_settings/paypal_settings.py
+++ /dev/null
@@ -1,505 +0,0 @@
-# Copyright (c) 2015, Frappe Technologies and contributors
-# License: MIT. See LICENSE
-
-"""
-# Integrating PayPal
-
-### 1. Validate Currency Support
-
-Example:
-
-	from frappe.integrations.utils import get_payment_gateway_controller
-
-	controller = get_payment_gateway_controller("PayPal")
-	controller().validate_transaction_currency(currency)
-
-### 2. Redirect for payment
-
-Example:
-
-	payment_details = {
-		"amount": 600,
-		"title": "Payment for bill : 111",
-		"description": "payment via cart",
-		"reference_doctype": "Payment Request",
-		"reference_docname": "PR0001",
-		"payer_email": "NuranVerkleij@example.com",
-		"payer_name": "Nuran Verkleij",
-		"order_id": "111",
-		"currency": "USD",
-		"payment_gateway": "Razorpay",
-		"subscription_details": {
-			"plan_id": "plan_12313", # if Required
-			"start_date": "2018-08-30",
-			"billing_period": "Month" #(Day, Week, SemiMonth, Month, Year),
-			"billing_frequency": 1,
-			"customer_notify": 1,
-			"upfront_amount": 1000
-		}
-	}
-
-	# redirect the user to this url
-	url = controller().get_payment_url(**payment_details)
-
-
-### 3. On Completion of Payment
-
-Write a method for `on_payment_authorized` in the reference doctype
-
-Example:
-
-	def on_payment_authorized(payment_status):
-		# your code to handle callback
-
-##### Note:
-
-payment_status - payment gateway will put payment status on callback.
-For paypal payment status parameter is one from: [Completed, Cancelled, Failed]
-
-
-More Details:
-
        For details on how to get your API credentials, follow this link: https://developer.paypal.com/docs/classic/api/apiCredentials/
        
-
-"""
-
-import json
-from urllib.parse import urlencode
-
-import pytz
-
-import frappe
-from frappe import _
-from frappe.integrations.utils import create_payment_gateway, create_request_log, make_post_request
-from frappe.model.document import Document
-from frappe.utils import call_hook_method, cint, get_datetime, get_url
-
-api_path = "/api/method/frappe.integrations.doctype.paypal_settings.paypal_settings"
-
-
-class PayPalSettings(Document):
-	supported_currencies = [
-		"AUD",
-		"BRL",
-		"CAD",
-		"CZK",
-		"DKK",
-		"EUR",
-		"HKD",
-		"HUF",
-		"ILS",
-		"JPY",
-		"MYR",
-		"MXN",
-		"TWD",
-		"NZD",
-		"NOK",
-		"PHP",
-		"PLN",
-		"GBP",
-		"RUB",
-		"SGD",
-		"SEK",
-		"CHF",
-		"THB",
-		"TRY",
-		"USD",
-	]
-
-	def __setup__(self):
-		setattr(self, "use_sandbox", 0)
-
-	def setup_sandbox_env(self, token):
-		data = json.loads(frappe.db.get_value("Integration Request", token, "data"))
-		setattr(self, "use_sandbox", cint(frappe._dict(data).use_sandbox) or 0)
-
-	def validate(self):
-		create_payment_gateway("PayPal")
-		call_hook_method("payment_gateway_enabled", gateway="PayPal")
-		if not self.flags.ignore_mandatory:
-			self.validate_paypal_credentails()
-
-	def on_update(self):
-		pass
-
-	def validate_transaction_currency(self, currency):
-		if currency not in self.supported_currencies:
-			frappe.throw(
-				_(
-					"Please select another payment method. PayPal does not support transactions in currency '{0}'"
-				).format(currency)
-			)
-
-	def get_paypal_params_and_url(self):
-		params = {
-			"USER": self.api_username,
-			"PWD": self.get_password(fieldname="api_password", raise_exception=False),
-			"SIGNATURE": self.signature,
-			"VERSION": "98",
-			"METHOD": "GetPalDetails",
-		}
-
-		if hasattr(self, "use_sandbox") and self.use_sandbox:
-			params.update(
-				{
-					"USER": frappe.conf.sandbox_api_username,
-					"PWD": frappe.conf.sandbox_api_password,
-					"SIGNATURE": frappe.conf.sandbox_signature,
-				}
-			)
-
-		api_url = (
-			"https://api-3t.sandbox.paypal.com/nvp"
-			if (self.paypal_sandbox or self.use_sandbox)
-			else "https://api-3t.paypal.com/nvp"
-		)
-
-		return params, api_url
-
-	def validate_paypal_credentails(self):
-		params, url = self.get_paypal_params_and_url()
-		params = urlencode(params)
-
-		try:
-			res = make_post_request(url=url, data=params.encode("utf-8"))
-
-			if res["ACK"][0] == "Failure":
-				raise Exception
-
-		except Exception:
-			frappe.throw(_("Invalid payment gateway credentials"))
-
-	def get_payment_url(self, **kwargs):
-		setattr(self, "use_sandbox", cint(kwargs.get("use_sandbox", 0)))
-
-		response = self.execute_set_express_checkout(**kwargs)
-
-		if self.paypal_sandbox or self.use_sandbox:
-			return_url = "https://www.sandbox.paypal.com/cgi-bin/webscr?cmd=_express-checkout&token={0}"
-		else:
-			return_url = "https://www.paypal.com/cgi-bin/webscr?cmd=_express-checkout&token={0}"
-
-		kwargs.update(
-			{"token": response.get("TOKEN")[0], "correlation_id": response.get("CORRELATIONID")[0]}
-		)
-
-		create_request_log(kwargs, service_name="PayPal", name=kwargs["token"])
-
-		return return_url.format(kwargs["token"])
-
-	def execute_set_express_checkout(self, **kwargs):
-		params, url = self.get_paypal_params_and_url()
-
-		params.update(
-			{
-				"METHOD": "SetExpressCheckout",
-				"returnUrl": get_url(f"{api_path}.get_express_checkout_details"),
-				"cancelUrl": get_url("/payment-cancel"),
-				"PAYMENTREQUEST_0_PAYMENTACTION": "SALE",
-				"PAYMENTREQUEST_0_AMT": kwargs["amount"],
-				"PAYMENTREQUEST_0_CURRENCYCODE": kwargs["currency"].upper(),
-			}
-		)
-
-		if kwargs.get("subscription_details"):
-			self.configure_recurring_payments(params, kwargs)
-
-		params = urlencode(params)
-		response = make_post_request(url, data=params.encode("utf-8"))
-
-		if response.get("ACK")[0] != "Success":
-			frappe.throw(_("Looks like something is wrong with this site's Paypal configuration."))
-
-		return response
-
-	def configure_recurring_payments(self, params, kwargs):
-		# removing the params as we have to setup rucurring payments
-		for param in (
-			"PAYMENTREQUEST_0_PAYMENTACTION",
-			"PAYMENTREQUEST_0_AMT",
-			"PAYMENTREQUEST_0_CURRENCYCODE",
-		):
-			del params[param]
-
-		params.update(
-			{
-				"L_BILLINGTYPE0": "RecurringPayments",  # The type of billing agreement
-				"L_BILLINGAGREEMENTDESCRIPTION0": kwargs["description"],
-			}
-		)
-
-
-def get_paypal_and_transaction_details(token):
-	doc = frappe.get_doc("PayPal Settings")
-	doc.setup_sandbox_env(token)
-	params, url = doc.get_paypal_params_and_url()
-
-	integration_request = frappe.get_doc("Integration Request", token)
-	data = json.loads(integration_request.data)
-
-	return data, params, url
-
-
-def setup_redirect(data, redirect_url, custom_redirect_to=None, redirect=True):
-	redirect_to = data.get("redirect_to") or None
-	redirect_message = data.get("redirect_message") or None
-
-	if custom_redirect_to:
-		redirect_to = custom_redirect_to
-
-	if redirect_to:
-		redirect_url += "&" + urlencode({"redirect_to": redirect_to})
-	if redirect_message:
-		redirect_url += "&" + urlencode({"redirect_message": redirect_message})
-
-	# this is done so that functions called via hooks can update flags.redirect_to
-	if redirect:
-		frappe.local.response["type"] = "redirect"
-		frappe.local.response["location"] = get_url(redirect_url)
-
-
-@frappe.whitelist(allow_guest=True, xss_safe=True)
-def get_express_checkout_details(token):
-	try:
-		doc = frappe.get_doc("PayPal Settings")
-		doc.setup_sandbox_env(token)
-
-		params, url = doc.get_paypal_params_and_url()
-		params.update({"METHOD": "GetExpressCheckoutDetails", "TOKEN": token})
-
-		response = make_post_request(url, data=params)
-
-		if response.get("ACK")[0] != "Success":
-			frappe.respond_as_web_page(
-				_("Something went wrong"),
-				_(
-					"Looks like something went wrong during the transaction. Since we haven't confirmed the payment, Paypal will automatically refund you this amount. If it doesn't, please send us an email and mention the Correlation ID: {0}."
-				).format(response.get("CORRELATIONID", [None])[0]),
-				indicator_color="red",
-				http_status_code=frappe.ValidationError.http_status_code,
-			)
-
-			return
-
-		doc = frappe.get_doc("Integration Request", token)
-		update_integration_request_status(
-			token,
-			{"payerid": response.get("PAYERID")[0], "payer_email": response.get("EMAIL")[0]},
-			"Authorized",
-			doc=doc,
-		)
-
-		frappe.local.response["type"] = "redirect"
-		frappe.local.response["location"] = get_redirect_uri(doc, token, response.get("PAYERID")[0])
-
-	except Exception:
-		frappe.log_error(frappe.get_traceback())
-
-
-@frappe.whitelist(allow_guest=True, xss_safe=True)
-def confirm_payment(token):
-	try:
-		custom_redirect_to = None
-		data, params, url = get_paypal_and_transaction_details(token)
-
-		params.update(
-			{
-				"METHOD": "DoExpressCheckoutPayment",
-				"PAYERID": data.get("payerid"),
-				"TOKEN": token,
-				"PAYMENTREQUEST_0_PAYMENTACTION": "SALE",
-				"PAYMENTREQUEST_0_AMT": data.get("amount"),
-				"PAYMENTREQUEST_0_CURRENCYCODE": data.get("currency").upper(),
-			}
-		)
-
-		response = make_post_request(url, data=params)
-
-		if response.get("ACK")[0] == "Success":
-			update_integration_request_status(
-				token,
-				{
-					"transaction_id": response.get("PAYMENTINFO_0_TRANSACTIONID")[0],
-					"correlation_id": response.get("CORRELATIONID")[0],
-				},
-				"Completed",
-			)
-
-			if data.get("reference_doctype") and data.get("reference_docname"):
-				custom_redirect_to = frappe.get_doc(
-					data.get("reference_doctype"), data.get("reference_docname")
-				).run_method("on_payment_authorized", "Completed")
-				frappe.db.commit()
-
-			redirect_url = "/integrations/payment-success?doctype={}&docname={}".format(
-				data.get("reference_doctype"), data.get("reference_docname")
-			)
-		else:
-			redirect_url = "/integrations/payment-failed"
-
-		setup_redirect(data, redirect_url, custom_redirect_to)
-
-	except Exception:
-		frappe.log_error(frappe.get_traceback())
-
-
-@frappe.whitelist(allow_guest=True, xss_safe=True)
-def create_recurring_profile(token, payerid):
-	try:
-		custom_redirect_to = None
-		updating = False
-		data, params, url = get_paypal_and_transaction_details(token)
-
-		addons = data.get("addons")
-		subscription_details = data.get("subscription_details")
-
-		if data.get("subscription_id"):
-			if addons:
-				updating = True
-			manage_recurring_payment_profile_status(data["subscription_id"], "Cancel", params, url)
-
-		params.update(
-			{
-				"METHOD": "CreateRecurringPaymentsProfile",
-				"PAYERID": payerid,
-				"TOKEN": token,
-				"DESC": data.get("description"),
-				"BILLINGPERIOD": subscription_details.get("billing_period"),
-				"BILLINGFREQUENCY": subscription_details.get("billing_frequency"),
-				"AMT": data.get("amount")
-				if data.get("subscription_amount") == data.get("amount")
-				else data.get("subscription_amount"),
-				"CURRENCYCODE": data.get("currency").upper(),
-				"INITAMT": data.get("upfront_amount"),
-			}
-		)
-
-		status_changed_to = "Completed" if data.get("starting_immediately") or updating else "Verified"
-
-		starts_at = get_datetime(subscription_details.get("start_date")) or frappe.utils.now_datetime()
-		starts_at = starts_at.replace(tzinfo=pytz.timezone(frappe.utils.get_time_zone())).astimezone(
-			pytz.utc
-		)
-
-		# "PROFILESTARTDATE": datetime.utcfromtimestamp(get_timestamp(starts_at)).isoformat()
-		params.update({"PROFILESTARTDATE": starts_at.isoformat()})
-
-		response = make_post_request(url, data=params)
-
-		if response.get("ACK")[0] == "Success":
-			update_integration_request_status(
-				token,
-				{
-					"profile_id": response.get("PROFILEID")[0],
-				},
-				"Completed",
-			)
-
-			if data.get("reference_doctype") and data.get("reference_docname"):
-				data["subscription_id"] = response.get("PROFILEID")[0]
-
-				frappe.flags.data = data
-				custom_redirect_to = frappe.get_doc(
-					data.get("reference_doctype"), data.get("reference_docname")
-				).run_method("on_payment_authorized", status_changed_to)
-				frappe.db.commit()
-
-			redirect_url = "/integrations/payment-success?doctype={}&docname={}".format(
-				data.get("reference_doctype"), data.get("reference_docname")
-			)
-		else:
-			redirect_url = "/integrations/payment-failed"
-
-		setup_redirect(data, redirect_url, custom_redirect_to)
-
-	except Exception:
-		frappe.log_error(frappe.get_traceback())
-
-
-def update_integration_request_status(token, data, status, error=False, doc=None):
-	if not doc:
-		doc = frappe.get_doc("Integration Request", token)
-
-	doc.update_status(data, status)
-
-
-def get_redirect_uri(doc, token, payerid):
-	data = json.loads(doc.data)
-
-	if data.get("subscription_details") or data.get("subscription_id"):
-		return get_url(f"{api_path}.create_recurring_profile?token={token}&payerid={payerid}")
-	else:
-		return get_url(f"{api_path}.confirm_payment?token={token}")
-
-
-def manage_recurring_payment_profile_status(profile_id, action, args, url):
-	args.update(
-		{"METHOD": "ManageRecurringPaymentsProfileStatus", "PROFILEID": profile_id, "ACTION": action}
-	)
-
-	response = make_post_request(url, data=args)
-
-	# error code 11556 indicates profile is not in active state(or already cancelled)
-	# thus could not cancel the subscription.
-	# thus raise an exception only if the error code is not equal to 11556
-
-	if response.get("ACK")[0] != "Success" and response.get("L_ERRORCODE0", [])[0] != "11556":
-		frappe.throw(_("Failed while amending subscription"))
-
-
-@frappe.whitelist(allow_guest=True)
-def ipn_handler():
-	try:
-		data = frappe.local.form_dict
-
-		validate_ipn_request(data)
-
-		data.update({"payment_gateway": "PayPal"})
-
-		doc = frappe.get_doc(
-			{
-				"data": json.dumps(frappe.local.form_dict),
-				"doctype": "Integration Request",
-				"request_description": "Subscription Notification",
-				"is_remote_request": 1,
-				"status": "Queued",
-			}
-		).insert(ignore_permissions=True)
-		frappe.db.commit()
-
-		frappe.enqueue(
-			method="frappe.integrations.doctype.paypal_settings.paypal_settings.handle_subscription_notification",
-			queue="long",
-			timeout=600,
-			is_async=True,
-			**{"doctype": "Integration Request", "docname": doc.name},
-		)
-
-	except frappe.InvalidStatusError:
-		pass
-	except Exception as e:
-		frappe.log(frappe.log_error(title=e))
-
-
-def validate_ipn_request(data):
-	def _throw():
-		frappe.throw(_("In Valid Request"), exc=frappe.InvalidStatusError)
-
-	if not data.get("recurring_payment_id"):
-		_throw()
-
-	doc = frappe.get_doc("PayPal Settings")
-	params, url = doc.get_paypal_params_and_url()
-
-	params.update(
-		{"METHOD": "GetRecurringPaymentsProfileDetails", "PROFILEID": data.get("recurring_payment_id")}
-	)
-
-	params = urlencode(params)
-	res = make_post_request(url=url, data=params.encode("utf-8"))
-
-	if res["ACK"][0] != "Success":
-		_throw()
-
-
-def handle_subscription_notification(doctype, docname):
-	call_hook_method("handle_subscription_notification", doctype=doctype, docname=docname)
diff --git a/frappe/integrations/doctype/paytm_settings/__init__.py b/frappe/integrations/doctype/paytm_settings/__init__.py
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/frappe/integrations/doctype/paytm_settings/paytm_settings.js b/frappe/integrations/doctype/paytm_settings/paytm_settings.js
deleted file mode 100644
index fe2ee7c952..0000000000
--- a/frappe/integrations/doctype/paytm_settings/paytm_settings.js
+++ /dev/null
@@ -1,8 +0,0 @@
-// Copyright (c) 2020, Frappe Technologies and contributors
-// For license information, please see license.txt
-
-frappe.ui.form.on('Paytm Settings', {
-	refresh: function(frm) {
-		frm.dashboard.set_headline(__("For more information, {0}.", [`${__('Click here')}`]));
-	}
-});
diff --git a/frappe/integrations/doctype/paytm_settings/paytm_settings.json b/frappe/integrations/doctype/paytm_settings/paytm_settings.json
deleted file mode 100644
index 93fbd0df09..0000000000
--- a/frappe/integrations/doctype/paytm_settings/paytm_settings.json
+++ /dev/null
@@ -1,89 +0,0 @@
-{
- "actions": [],
- "creation": "2020-04-02 00:11:22.846697",
- "doctype": "DocType",
- "editable_grid": 1,
- "engine": "InnoDB",
- "field_order": [
-  "merchant_id",
-  "merchant_key",
-  "staging",
-  "column_break_4",
-  "industry_type_id",
-  "website"
- ],
- "fields": [
-  {
-   "fieldname": "merchant_id",
-   "fieldtype": "Data",
-   "in_list_view": 1,
-   "label": "Merchant ID",
-   "reqd": 1,
-   "show_days": 1,
-   "show_seconds": 1
-  },
-  {
-   "fieldname": "merchant_key",
-   "fieldtype": "Password",
-   "in_list_view": 1,
-   "label": "Merchant Key",
-   "reqd": 1,
-   "show_days": 1,
-   "show_seconds": 1
-  },
-  {
-   "default": "0",
-   "fieldname": "staging",
-   "fieldtype": "Check",
-   "label": "Staging",
-   "show_days": 1,
-   "show_seconds": 1
-  },
-  {
-   "depends_on": "eval: !doc.staging",
-   "fieldname": "website",
-   "fieldtype": "Data",
-   "label": "Website",
-   "mandatory_depends_on": "eval: !doc.staging",
-   "show_days": 1,
-   "show_seconds": 1
-  },
-  {
-   "fieldname": "column_break_4",
-   "fieldtype": "Column Break",
-   "show_days": 1,
-   "show_seconds": 1
-  },
-  {
-   "depends_on": "eval: !doc.staging",
-   "fieldname": "industry_type_id",
-   "fieldtype": "Data",
-   "label": "Industry Type ID",
-   "mandatory_depends_on": "eval: !doc.staging",
-   "show_days": 1,
-   "show_seconds": 1
-  }
- ],
- "issingle": 1,
- "links": [],
- "modified": "2020-06-08 13:36:09.703143",
- "modified_by": "Administrator",
- "module": "Integrations",
- "name": "Paytm Settings",
- "owner": "Administrator",
- "permissions": [
-  {
-   "create": 1,
-   "delete": 1,
-   "email": 1,
-   "print": 1,
-   "read": 1,
-   "role": "System Manager",
-   "share": 1,
-   "write": 1
-  }
- ],
- "sort_field": "modified",
- "sort_order": "DESC",
- "track_changes": 1
-}
\ No newline at end of file
diff --git a/frappe/integrations/doctype/paytm_settings/paytm_settings.py b/frappe/integrations/doctype/paytm_settings/paytm_settings.py
deleted file mode 100644
index 81a5f45f47..0000000000
--- a/frappe/integrations/doctype/paytm_settings/paytm_settings.py
+++ /dev/null
@@ -1,181 +0,0 @@
-# Copyright (c) 2020, Frappe Technologies and contributors
-# License: MIT. See LICENSE
-
-import json
-from urllib.parse import urlencode
-
-import requests
-from paytmchecksum import generateSignature, verifySignature
-
-import frappe
-from frappe import _
-from frappe.integrations.utils import create_payment_gateway, create_request_log
-from frappe.model.document import Document
-from frappe.utils import call_hook_method, cint, cstr, flt, get_request_site_address, get_url
-from frappe.utils.password import get_decrypted_password
-
-
-class PaytmSettings(Document):
-	supported_currencies = ["INR"]
-
-	def validate(self):
-		create_payment_gateway("Paytm")
-		call_hook_method("payment_gateway_enabled", gateway="Paytm")
-
-	def validate_transaction_currency(self, currency):
-		if currency not in self.supported_currencies:
-			frappe.throw(
-				_(
-					"Please select another payment method. Paytm does not support transactions in currency '{0}'"
-				).format(currency)
-			)
-
-	def get_payment_url(self, **kwargs):
-		"""Return payment url with several params"""
-		# create unique order id by making it equal to the integration request
-		integration_request = create_request_log(kwargs, service_name="Paytm")
-		kwargs.update(dict(order_id=integration_request.name))
-
-		return get_url(f"./integrations/paytm_checkout?{urlencode(kwargs)}")
-
-
-def get_paytm_config():
-	"""Returns paytm config"""
-
-	paytm_config = frappe.db.get_singles_dict("Paytm Settings")
-	paytm_config.update(
-		dict(merchant_key=get_decrypted_password("Paytm Settings", "Paytm Settings", "merchant_key"))
-	)
-
-	if cint(paytm_config.staging):
-		paytm_config.update(
-			dict(
-				website="WEBSTAGING",
-				url="https://securegw-stage.paytm.in/order/process",
-				transaction_status_url="https://securegw-stage.paytm.in/order/status",
-				industry_type_id="RETAIL",
-			)
-		)
-	else:
-		paytm_config.update(
-			dict(
-				url="https://securegw.paytm.in/order/process",
-				transaction_status_url="https://securegw.paytm.in/order/status",
-			)
-		)
-	return paytm_config
-
-
-def get_paytm_params(payment_details, order_id, paytm_config):
-
-	# initialize a dictionary
-	paytm_params = dict()
-
-	redirect_uri = (
-		get_request_site_address(True)
-		+ "/api/method/frappe.integrations.doctype.paytm_settings.paytm_settings.verify_transaction"
-	)
-
-	paytm_params.update(
-		{
-			"MID": paytm_config.merchant_id,
-			"WEBSITE": paytm_config.website,
-			"INDUSTRY_TYPE_ID": paytm_config.industry_type_id,
-			"CHANNEL_ID": "WEB",
-			"ORDER_ID": order_id,
-			"CUST_ID": payment_details["payer_email"],
-			"EMAIL": payment_details["payer_email"],
-			"TXN_AMOUNT": cstr(flt(payment_details["amount"], 2)),
-			"CALLBACK_URL": redirect_uri,
-		}
-	)
-
-	checksum = generateSignature(paytm_params, paytm_config.merchant_key)
-
-	paytm_params.update({"CHECKSUMHASH": checksum})
-
-	return paytm_params
-
-
-@frappe.whitelist(allow_guest=True)
-def verify_transaction(**paytm_params):
-	"""Verify checksum for received data in the callback and then verify the transaction"""
-	paytm_config = get_paytm_config()
-	is_valid_checksum = False
-
-	paytm_params.pop("cmd", None)
-	paytm_checksum = paytm_params.pop("CHECKSUMHASH", None)
-
-	if paytm_params and paytm_config and paytm_checksum:
-		# Verify checksum
-		is_valid_checksum = verifySignature(paytm_params, paytm_config.merchant_key, paytm_checksum)
-
-	if is_valid_checksum and paytm_params.get("RESPCODE") == "01":
-		verify_transaction_status(paytm_config, paytm_params["ORDERID"])
-	else:
-		frappe.respond_as_web_page(
-			"Payment Failed",
-			"Transaction failed to complete. In case of any deductions, deducted amount will get refunded to your account.",
-			http_status_code=401,
-			indicator_color="red",
-		)
-		frappe.log_error(
-			"Order unsuccessful. Failed Response:" + cstr(paytm_params), "Paytm Payment Failed"
-		)
-
-
-def verify_transaction_status(paytm_config, order_id):
-	"""Verify transaction completion after checksum has been verified"""
-	paytm_params = dict(MID=paytm_config.merchant_id, ORDERID=order_id)
-
-	checksum = generateSignature(paytm_params, paytm_config.merchant_key)
-	paytm_params["CHECKSUMHASH"] = checksum
-
-	post_data = json.dumps(paytm_params)
-	url = paytm_config.transaction_status_url
-
-	response = requests.post(url, data=post_data, headers={"Content-type": "application/json"}).json()
-	finalize_request(order_id, response)
-
-
-def finalize_request(order_id, transaction_response):
-	request = frappe.get_doc("Integration Request", order_id)
-	transaction_data = frappe._dict(json.loads(request.data))
-	redirect_to = transaction_data.get("redirect_to") or None
-	redirect_message = transaction_data.get("redirect_message") or None
-
-	if transaction_response["STATUS"] == "TXN_SUCCESS":
-		if transaction_data.reference_doctype and transaction_data.reference_docname:
-			custom_redirect_to = None
-			try:
-				custom_redirect_to = frappe.get_doc(
-					transaction_data.reference_doctype, transaction_data.reference_docname
-				).run_method("on_payment_authorized", "Completed")
-				request.db_set("status", "Completed")
-			except Exception:
-				request.db_set("status", "Failed")
-				frappe.log_error(frappe.get_traceback())
-
-			if custom_redirect_to:
-				redirect_to = custom_redirect_to
-
-			redirect_url = "/integrations/payment-success"
-	else:
-		request.db_set("status", "Failed")
-		redirect_url = "/integrations/payment-failed"
-
-	if redirect_to:
-		redirect_url += "?" + urlencode({"redirect_to": redirect_to})
-	if redirect_message:
-		redirect_url += "&" + urlencode({"redirect_message": redirect_message})
-
-	frappe.local.response["type"] = "redirect"
-	frappe.local.response["location"] = redirect_url
-
-
-def get_gateway_controller(doctype, docname):
-	reference_doc = frappe.get_doc(doctype, docname)
-	gateway_controller = frappe.db.get_value(
-		"Payment Gateway", reference_doc.payment_gateway, "gateway_controller"
-	)
-	return gateway_controller
diff --git a/frappe/integrations/doctype/paytm_settings/test_paytm_settings.py b/frappe/integrations/doctype/paytm_settings/test_paytm_settings.py
deleted file mode 100644
index 91b69d5aec..0000000000
--- a/frappe/integrations/doctype/paytm_settings/test_paytm_settings.py
+++ /dev/null
@@ -1,8 +0,0 @@
-# Copyright (c) 2020, Frappe Technologies and Contributors
-# License: MIT. See LICENSE
-# import frappe
-import unittest
-
-
-class TestPaytmSettings(unittest.TestCase):
-	pass
diff --git a/frappe/integrations/doctype/razorpay_settings/__init__.py b/frappe/integrations/doctype/razorpay_settings/__init__.py
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/frappe/integrations/doctype/razorpay_settings/razorpay_settings.js b/frappe/integrations/doctype/razorpay_settings/razorpay_settings.js
deleted file mode 100644
index 6915c5c582..0000000000
--- a/frappe/integrations/doctype/razorpay_settings/razorpay_settings.js
+++ /dev/null
@@ -1,8 +0,0 @@
-// Copyright (c) 2016, Frappe Technologies and contributors
-// For license information, please see license.txt
-
-frappe.ui.form.on('Razorpay Settings', {
-	refresh: function(frm) {
-
-	}
-});
\ No newline at end of file
diff --git a/frappe/integrations/doctype/razorpay_settings/razorpay_settings.json b/frappe/integrations/doctype/razorpay_settings/razorpay_settings.json
deleted file mode 100644
index 3fdea79e2b..0000000000
--- a/frappe/integrations/doctype/razorpay_settings/razorpay_settings.json
+++ /dev/null
@@ -1,145 +0,0 @@
-{
- "allow_copy": 0, 
- "allow_import": 0, 
- "allow_rename": 0, 
- "beta": 0, 
- "creation": "2016-09-20 03:44:03.799402", 
- "custom": 0, 
- "docstatus": 0, 
- "doctype": "DocType", 
- "document_type": "System", 
- "editable_grid": 1, 
- "fields": [
-  {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "api_key", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "API Key", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "api_secret", 
-   "fieldtype": "Password", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "API Secret", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "description": "Mention transaction completion page URL", 
-   "fieldname": "redirect_to", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Redirect To", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }
- ], 
- "hide_heading": 0, 
- "hide_toolbar": 0, 
- "idx": 0, 
- "image_view": 0, 
- "in_create": 1, 
-
- "is_submittable": 0, 
- "issingle": 1, 
- "istable": 0, 
- "max_attachments": 0, 
- "modified": "2016-12-29 14:40:31.658270", 
- "modified_by": "Administrator", 
- "module": "Integrations", 
- "name": "Razorpay Settings", 
- "name_case": "", 
- "owner": "Administrator", 
- "permissions": [
-  {
-   "amend": 0, 
-   "apply_user_permissions": 0, 
-   "cancel": 0, 
-   "create": 1, 
-   "delete": 1, 
-   "email": 1, 
-   "export": 1, 
-   "if_owner": 0, 
-   "import": 0, 
-   "is_custom": 0, 
-   "permlevel": 0, 
-   "print": 1, 
-   "read": 1, 
-   "report": 0, 
-   "role": "System Manager", 
-   "set_user_permissions": 0, 
-   "share": 1, 
-   "submit": 0, 
-   "write": 1
-  }
- ], 
- "quick_entry": 0, 
- "read_only": 1, 
- "read_only_onload": 0, 
- "sort_field": "modified", 
- "sort_order": "DESC", 
- "track_changes": 1, 
- "track_seen": 0
-}
\ No newline at end of file
diff --git a/frappe/integrations/doctype/razorpay_settings/razorpay_settings.py b/frappe/integrations/doctype/razorpay_settings/razorpay_settings.py
deleted file mode 100644
index a79e626b49..0000000000
--- a/frappe/integrations/doctype/razorpay_settings/razorpay_settings.py
+++ /dev/null
@@ -1,530 +0,0 @@
-# Copyright (c) 2015, Frappe Technologies and contributors
-# License: MIT. See LICENSE
-
-"""
-# Integrating RazorPay
-
-### Validate Currency
-
-Example:
-
-	from frappe.integrations.utils import get_payment_gateway_controller
-
-	controller = get_payment_gateway_controller("Razorpay")
-	controller().validate_transaction_currency(currency)
-
-### 2. Redirect for payment
-
-Example:
-
-	payment_details = {
-		"amount": 600,
-		"title": "Payment for bill : 111",
-		"description": "payment via cart",
-		"reference_doctype": "Payment Request",
-		"reference_docname": "PR0001",
-		"payer_email": "NuranVerkleij@example.com",
-		"payer_name": "Nuran Verkleij",
-		"order_id": "111",
-		"currency": "INR",
-		"payment_gateway": "Razorpay",
-		"subscription_details": {
-			"plan_id": "plan_12313", # if Required
-			"start_date": "2018-08-30",
-			"billing_period": "Month" #(Day, Week, Month, Year),
-			"billing_frequency": 1,
-			"customer_notify": 1,
-			"upfront_amount": 1000
-		}
-	}
-
-	# Redirect the user to this url
-	url = controller().get_payment_url(**payment_details)
-
-
-### 3. On Completion of Payment
-
-Write a method for `on_payment_authorized` in the reference doctype
-
-Example:
-
-	def on_payment_authorized(payment_status):
-		# this method will be called when payment is complete
-
-
-##### Notes:
-
-payment_status - payment gateway will put payment status on callback.
-For razorpay payment status is Authorized
-
-"""
-
-import hashlib
-import hmac
-import json
-from urllib.parse import urlencode
-
-import razorpay
-
-import frappe
-from frappe import _
-from frappe.integrations.utils import (
-	create_payment_gateway,
-	create_request_log,
-	make_get_request,
-	make_post_request,
-)
-from frappe.model.document import Document
-from frappe.utils import call_hook_method, cint, get_timestamp, get_url
-
-
-class RazorpaySettings(Document):
-	supported_currencies = ["INR"]
-
-	def init_client(self):
-		if self.api_key:
-			secret = self.get_password(fieldname="api_secret", raise_exception=False)
-			self.client = razorpay.Client(auth=(self.api_key, secret))
-
-	def validate(self):
-		create_payment_gateway("Razorpay")
-		call_hook_method("payment_gateway_enabled", gateway="Razorpay")
-		if not self.flags.ignore_mandatory:
-			self.validate_razorpay_credentails()
-
-	def validate_razorpay_credentails(self):
-		if self.api_key and self.api_secret:
-			try:
-				make_get_request(
-					url="https://api.razorpay.com/v1/payments",
-					auth=(self.api_key, self.get_password(fieldname="api_secret", raise_exception=False)),
-				)
-			except Exception:
-				frappe.throw(_("Seems API Key or API Secret is wrong !!!"))
-
-	def validate_transaction_currency(self, currency):
-		if currency not in self.supported_currencies:
-			frappe.throw(
-				_(
-					"Please select another payment method. Razorpay does not support transactions in currency '{0}'"
-				).format(currency)
-			)
-
-	def setup_addon(self, settings, **kwargs):
-		"""
-		Addon template:
-		{
-		        "item": {
-		                "name": row.upgrade_type,
-		                "amount": row.amount,
-		                "currency": currency,
-		                "description": "add-on description"
-		        },
-		        "quantity": 1 (The total amount is calculated as item.amount * quantity)
-		}
-		"""
-		url = "https://api.razorpay.com/v1/subscriptions/{}/addons".format(kwargs.get("subscription_id"))
-
-		try:
-			if not frappe.conf.converted_rupee_to_paisa:
-				convert_rupee_to_paisa(**kwargs)
-
-			for addon in kwargs.get("addons"):
-				resp = make_post_request(
-					url,
-					auth=(settings.api_key, settings.api_secret),
-					data=json.dumps(addon),
-					headers={"content-type": "application/json"},
-				)
-				if not resp.get("id"):
-					frappe.log_error(message=str(resp), title="Razorpay Failed while creating subscription")
-		except Exception:
-			frappe.log_error()
-			# failed
-			pass
-
-	def setup_subscription(self, settings, **kwargs):
-		start_date = (
-			get_timestamp(kwargs.get("subscription_details").get("start_date"))
-			if kwargs.get("subscription_details").get("start_date")
-			else None
-		)
-
-		subscription_details = {
-			"plan_id": kwargs.get("subscription_details").get("plan_id"),
-			"total_count": kwargs.get("subscription_details").get("billing_frequency"),
-			"customer_notify": kwargs.get("subscription_details").get("customer_notify"),
-		}
-
-		if start_date:
-			subscription_details["start_at"] = cint(start_date)
-
-		if kwargs.get("addons"):
-			convert_rupee_to_paisa(**kwargs)
-			subscription_details.update({"addons": kwargs.get("addons")})
-
-		try:
-			resp = make_post_request(
-				"https://api.razorpay.com/v1/subscriptions",
-				auth=(settings.api_key, settings.api_secret),
-				data=json.dumps(subscription_details),
-				headers={"content-type": "application/json"},
-			)
-
-			if resp.get("status") == "created":
-				kwargs["subscription_id"] = resp.get("id")
-				frappe.flags.status = "created"
-				return kwargs
-			else:
-				frappe.log_error(message=str(resp), title="Razorpay Failed while creating subscription")
-
-		except Exception:
-			frappe.log_error()
-
-	def prepare_subscription_details(self, settings, **kwargs):
-		if not kwargs.get("subscription_id"):
-			kwargs = self.setup_subscription(settings, **kwargs)
-
-		if frappe.flags.status != "created":
-			kwargs["subscription_id"] = None
-
-		return kwargs
-
-	def get_payment_url(self, **kwargs):
-		integration_request = create_request_log(kwargs, service_name="Razorpay")
-		return get_url(f"./integrations/razorpay_checkout?token={integration_request.name}")
-
-	def create_order(self, **kwargs):
-		# Creating Orders https://razorpay.com/docs/api/orders/
-
-		# convert rupees to paisa
-		kwargs["amount"] *= 100
-
-		# Create integration log
-		integration_request = create_request_log(kwargs, service_name="Razorpay")
-
-		# Setup payment options
-		payment_options = {
-			"amount": kwargs.get("amount"),
-			"currency": kwargs.get("currency", "INR"),
-			"receipt": kwargs.get("receipt"),
-			"payment_capture": kwargs.get("payment_capture"),
-		}
-		if self.api_key and self.api_secret:
-			try:
-				order = make_post_request(
-					"https://api.razorpay.com/v1/orders",
-					auth=(self.api_key, self.get_password(fieldname="api_secret", raise_exception=False)),
-					data=payment_options,
-				)
-				order["integration_request"] = integration_request.name
-				return order  # Order returned to be consumed by razorpay.js
-			except Exception:
-				frappe.log(frappe.get_traceback())
-				frappe.throw(_("Could not create razorpay order"))
-
-	def create_request(self, data):
-		self.data = frappe._dict(data)
-
-		try:
-			self.integration_request = frappe.get_doc("Integration Request", self.data.token)
-			self.integration_request.update_status(self.data, "Queued")
-			return self.authorize_payment()
-
-		except Exception:
-			frappe.log_error(frappe.get_traceback())
-			return {
-				"redirect_to": frappe.redirect_to_message(
-					_("Server Error"),
-					_(
-						"Seems issue with server's razorpay config. Don't worry, in case of failure amount will get refunded to your account."
-					),
-				),
-				"status": 401,
-			}
-
-	def authorize_payment(self):
-		"""
-		An authorization is performed when user’s payment details are successfully authenticated by the bank.
-		The money is deducted from the customer’s account, but will not be transferred to the merchant’s account
-		until it is explicitly captured by merchant.
-		"""
-		data = json.loads(self.integration_request.data)
-		settings = self.get_settings(data)
-
-		try:
-			resp = make_get_request(
-				f"https://api.razorpay.com/v1/payments/{self.data.razorpay_payment_id}",
-				auth=(settings.api_key, settings.api_secret),
-			)
-
-			if resp.get("status") == "authorized":
-				self.integration_request.update_status(data, "Authorized")
-				self.flags.status_changed_to = "Authorized"
-
-			if resp.get("status") == "captured":
-				self.integration_request.update_status(data, "Completed")
-				self.flags.status_changed_to = "Completed"
-
-			elif data.get("subscription_id"):
-				if resp.get("status") == "refunded":
-					# if subscription start date is in future then
-					# razorpay refunds the amount after authorizing the card details
-					# thus changing status to Verified
-
-					self.integration_request.update_status(data, "Completed")
-					self.flags.status_changed_to = "Verified"
-
-			else:
-				frappe.log_error(message=str(resp), title="Razorpay Payment not authorized")
-
-		except Exception:
-			frappe.log_error()
-
-		status = frappe.flags.integration_request.status_code
-
-		redirect_to = data.get("redirect_to") or None
-		redirect_message = data.get("redirect_message") or None
-		if self.flags.status_changed_to in ("Authorized", "Verified", "Completed"):
-			if self.data.reference_doctype and self.data.reference_docname:
-				custom_redirect_to = None
-				try:
-					frappe.flags.data = data
-					custom_redirect_to = frappe.get_doc(
-						self.data.reference_doctype, self.data.reference_docname
-					).run_method("on_payment_authorized", self.flags.status_changed_to)
-
-				except Exception:
-					frappe.log_error(frappe.get_traceback())
-
-				if custom_redirect_to:
-					redirect_to = custom_redirect_to
-
-			redirect_url = "payment-success?doctype={}&docname={}".format(
-				self.data.reference_doctype, self.data.reference_docname
-			)
-		else:
-			redirect_url = "payment-failed"
-
-		if redirect_to:
-			redirect_url += "&" + urlencode({"redirect_to": redirect_to})
-		if redirect_message:
-			redirect_url += "&" + urlencode({"redirect_message": redirect_message})
-
-		return {"redirect_to": redirect_url, "status": status}
-
-	def get_settings(self, data):
-		settings = frappe._dict(
-			{
-				"api_key": self.api_key,
-				"api_secret": self.get_password(fieldname="api_secret", raise_exception=False),
-			}
-		)
-
-		if cint(data.get("notes", {}).get("use_sandbox")) or data.get("use_sandbox"):
-			settings.update(
-				{
-					"api_key": frappe.conf.sandbox_api_key,
-					"api_secret": frappe.conf.sandbox_api_secret,
-				}
-			)
-
-		return settings
-
-	def cancel_subscription(self, subscription_id):
-		settings = self.get_settings({})
-
-		try:
-			resp = make_post_request(
-				f"https://api.razorpay.com/v1/subscriptions/{subscription_id}/cancel",
-				auth=(settings.api_key, settings.api_secret),
-			)
-		except Exception:
-			frappe.log_error(frappe.get_traceback())
-
-	def verify_signature(self, body, signature, key):
-		key = bytes(key, "utf-8")
-		body = bytes(body, "utf-8")
-
-		dig = hmac.new(key=key, msg=body, digestmod=hashlib.sha256)
-
-		generated_signature = dig.hexdigest()
-		result = hmac.compare_digest(generated_signature, signature)
-
-		if not result:
-			frappe.throw(_("Razorpay Signature Verification Failed"), exc=frappe.PermissionError)
-
-		return result
-
-
-def capture_payment(is_sandbox=False, sanbox_response=None):
-	"""
-	Verifies the purchase as complete by the merchant.
-	After capture, the amount is transferred to the merchant within T+3 days
-	where T is the day on which payment is captured.
-
-	Note: Attempting to capture a payment whose status is not authorized will produce an error.
-	"""
-	controller = frappe.get_doc("Razorpay Settings")
-
-	for doc in frappe.get_all(
-		"Integration Request",
-		filters={"status": "Authorized", "integration_request_service": "Razorpay"},
-		fields=["name", "data"],
-	):
-		try:
-			if is_sandbox:
-				resp = sanbox_response
-			else:
-				data = json.loads(doc.data)
-				settings = controller.get_settings(data)
-
-				resp = make_get_request(
-					"https://api.razorpay.com/v1/payments/{}".format(data.get("razorpay_payment_id")),
-					auth=(settings.api_key, settings.api_secret),
-					data={"amount": data.get("amount")},
-				)
-
-				if resp.get("status") == "authorized":
-					resp = make_post_request(
-						"https://api.razorpay.com/v1/payments/{}/capture".format(data.get("razorpay_payment_id")),
-						auth=(settings.api_key, settings.api_secret),
-						data={"amount": data.get("amount")},
-					)
-
-			if resp.get("status") == "captured":
-				frappe.db.set_value("Integration Request", doc.name, "status", "Completed")
-
-		except Exception:
-			doc = frappe.get_doc("Integration Request", doc.name)
-			doc.status = "Failed"
-			doc.error = frappe.get_traceback()
-			doc.save()
-			frappe.log_error(doc.error, f"{doc.name} Failed")
-
-
-@frappe.whitelist(allow_guest=True)
-def get_api_key():
-	controller = frappe.get_doc("Razorpay Settings")
-	return controller.api_key
-
-
-@frappe.whitelist(allow_guest=True)
-def get_order(doctype, docname):
-	# Order returned to be consumed by razorpay.js
-	doc = frappe.get_doc(doctype, docname)
-	try:
-		# Do not use run_method here as it fails silently
-		return doc.get_razorpay_order()
-	except AttributeError:
-		frappe.log_error(frappe.get_traceback(), _("Controller method get_razorpay_order missing"))
-		frappe.throw(_("Could not create Razorpay order. Please contact Administrator"))
-
-
-@frappe.whitelist(allow_guest=True)
-def order_payment_success(integration_request, params):
-	"""Called by razorpay.js on order payment success, the params
-	contains razorpay_payment_id, razorpay_order_id, razorpay_signature
-	that is updated in the data field of integration request
-
-	Args:
-	        integration_request (string): Name for integration request doc
-	        params (string): Params to be updated for integration request.
-	"""
-	params = json.loads(params)
-	integration = frappe.get_doc("Integration Request", integration_request)
-
-	# Update integration request
-	integration.update_status(params, integration.status)
-	integration.reload()
-
-	data = json.loads(integration.data)
-	controller = frappe.get_doc("Razorpay Settings")
-
-	# Update payment and integration data for payment controller object
-	controller.integration_request = integration
-	controller.data = frappe._dict(data)
-
-	# Authorize payment
-	controller.authorize_payment()
-
-
-@frappe.whitelist(allow_guest=True)
-def order_payment_failure(integration_request, params):
-	"""Called by razorpay.js on failure
-
-	Args:
-	        integration_request (TYPE): Description
-	        params (TYPE): error data to be updated
-	"""
-	frappe.log_error(params, "Razorpay Payment Failure")
-	params = json.loads(params)
-	integration = frappe.get_doc("Integration Request", integration_request)
-	integration.update_status(params, integration.status)
-
-
-def convert_rupee_to_paisa(**kwargs):
-	for addon in kwargs.get("addons"):
-		addon["item"]["amount"] *= 100
-
-	frappe.conf.converted_rupee_to_paisa = True
-
-
-@frappe.whitelist(allow_guest=True)
-def razorpay_subscription_callback():
-	try:
-		data = frappe.local.form_dict
-
-		validate_payment_callback(data)
-
-		data.update({"payment_gateway": "Razorpay"})
-
-		doc = frappe.get_doc(
-			{
-				"data": json.dumps(frappe.local.form_dict),
-				"doctype": "Integration Request",
-				"request_description": "Subscription Notification",
-				"is_remote_request": 1,
-				"status": "Queued",
-			}
-		).insert(ignore_permissions=True)
-		frappe.db.commit()
-
-		frappe.enqueue(
-			method="frappe.integrations.doctype.razorpay_settings.razorpay_settings.handle_subscription_notification",
-			queue="long",
-			timeout=600,
-			is_async=True,
-			**{"doctype": "Integration Request", "docname": doc.name},
-		)
-
-	except frappe.InvalidStatusError:
-		pass
-	except Exception as e:
-		frappe.log(frappe.log_error(title=e))
-
-
-def validate_payment_callback(data):
-	def _throw():
-		frappe.throw(_("Invalid Subscription"), exc=frappe.InvalidStatusError)
-
-	subscription_id = data.get("payload").get("subscription").get("entity").get("id")
-
-	if not (subscription_id):
-		_throw()
-
-	controller = frappe.get_doc("Razorpay Settings")
-
-	settings = controller.get_settings(data)
-
-	resp = make_get_request(
-		f"https://api.razorpay.com/v1/subscriptions/{subscription_id}",
-		auth=(settings.api_key, settings.api_secret),
-	)
-
-	if resp.get("status") != "active":
-		_throw()
-
-
-def handle_subscription_notification(doctype, docname):
-	call_hook_method("handle_subscription_notification", doctype=doctype, docname=docname)
diff --git a/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.js b/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.js
index 1a1b8a7c67..6db4087cf3 100755
--- a/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.js
+++ b/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.js
@@ -1,26 +1,26 @@
 // Copyright (c) 2017, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('S3 Backup Settings', {
-	refresh: function(frm) {
+frappe.ui.form.on("S3 Backup Settings", {
+	refresh: function (frm) {
 		frm.clear_custom_buttons();
 		frm.events.take_backup(frm);
 	},
 
-	take_backup: function(frm) {
+	take_backup: function (frm) {
 		if (frm.doc.access_key_id && frm.doc.secret_access_key) {
-			frm.add_custom_button(__("Take Backup Now"), function(){
+			frm.add_custom_button(__("Take Backup Now"), function () {
 				frm.dashboard.set_headline_alert("S3 Backup Started!");
 				frappe.call({
 					method: "frappe.integrations.doctype.s3_backup_settings.s3_backup_settings.take_backups_s3",
-					callback: function(r) {
-						if(!r.exc) {
+					callback: function (r) {
+						if (!r.exc) {
 							frappe.msgprint(__("S3 Backup complete!"));
 							frm.dashboard.clear_headline();
 						}
-					}
+					},
 				});
 			}).addClass("btn-primary");
 		}
-	}
+	},
 });
diff --git a/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.json b/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.json
index 2ca1723cb2..e2256861cd 100755
--- a/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.json
+++ b/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.json
@@ -71,6 +71,7 @@
   },
   {
    "default": "https://s3.amazonaws.com",
+   "description": "Only change this if you want to use other S3 compatible object storage backends.",
    "fieldname": "endpoint_url",
    "fieldtype": "Data",
    "label": "Endpoint URL"
@@ -129,7 +130,7 @@
  "hide_toolbar": 1,
  "issingle": 1,
  "links": [],
- "modified": "2020-12-07 15:30:55.047689",
+ "modified": "2023-01-11 15:38:20.333833",
  "modified_by": "Administrator",
  "module": "Integrations",
  "name": "S3 Backup Settings",
@@ -149,5 +150,6 @@
  "quick_entry": 1,
  "sort_field": "modified",
  "sort_order": "DESC",
+ "states": [],
  "track_changes": 1
 }
\ No newline at end of file
diff --git a/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.py b/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.py
index 1c2d39be10..568ff71b00 100755
--- a/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.py
+++ b/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.py
@@ -76,8 +76,8 @@ def take_backups_monthly():
 
 
 def take_backups_if(freq):
-	if cint(frappe.db.get_value("S3 Backup Settings", None, "enabled")):
-		if frappe.db.get_value("S3 Backup Settings", None, "frequency") == freq:
+	if cint(frappe.db.get_single_value("S3 Backup Settings", "enabled")):
+		if frappe.db.get_single_value("S3 Backup Settings", "frequency") == freq:
 			take_backups_s3()
 
 
diff --git a/frappe/integrations/doctype/s3_backup_settings/test_s3_backup_settings.py b/frappe/integrations/doctype/s3_backup_settings/test_s3_backup_settings.py
index 48b1ccd113..add4e7a1b2 100755
--- a/frappe/integrations/doctype/s3_backup_settings/test_s3_backup_settings.py
+++ b/frappe/integrations/doctype/s3_backup_settings/test_s3_backup_settings.py
@@ -1,7 +1,7 @@
 # Copyright (c) 2017, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
-import unittest
+from frappe.tests.utils import FrappeTestCase
 
 
-class TestS3BackupSettings(unittest.TestCase):
+class TestS3BackupSettings(FrappeTestCase):
 	pass
diff --git a/frappe/integrations/doctype/slack_webhook_url/slack_webhook_url.js b/frappe/integrations/doctype/slack_webhook_url/slack_webhook_url.js
index b7a972bdc1..49991fcffe 100644
--- a/frappe/integrations/doctype/slack_webhook_url/slack_webhook_url.js
+++ b/frappe/integrations/doctype/slack_webhook_url/slack_webhook_url.js
@@ -1,6 +1,4 @@
 // Copyright (c) 2018, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Slack Webhook URL', {
-
-});
+frappe.ui.form.on("Slack Webhook URL", {});
diff --git a/frappe/integrations/doctype/slack_webhook_url/test_slack_webhook_url.py b/frappe/integrations/doctype/slack_webhook_url/test_slack_webhook_url.py
index 16b1bcd3c2..6697c2a320 100644
--- a/frappe/integrations/doctype/slack_webhook_url/test_slack_webhook_url.py
+++ b/frappe/integrations/doctype/slack_webhook_url/test_slack_webhook_url.py
@@ -1,7 +1,7 @@
 # Copyright (c) 2018, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
-import unittest
+from frappe.tests.utils import FrappeTestCase
 
 
-class TestSlackWebhookURL(unittest.TestCase):
+class TestSlackWebhookURL(FrappeTestCase):
 	pass
diff --git a/frappe/integrations/doctype/social_login_key/social_login_key.js b/frappe/integrations/doctype/social_login_key/social_login_key.js
index e2cbb3459f..033beffff0 100644
--- a/frappe/integrations/doctype/social_login_key/social_login_key.js
+++ b/frappe/integrations/doctype/social_login_key/social_login_key.js
@@ -1,12 +1,19 @@
 // Copyright (c) 2017, Frappe Technologies and contributors
 // For license information, please see license.txt
 const fields = [
-	"provider_name", "base_url", "custom_base_url",
-	"icon", "authorize_url", "access_token_url", "redirect_url",
-	"api_endpoint", "api_endpoint_args", "auth_url_data"
+	"provider_name",
+	"base_url",
+	"custom_base_url",
+	"icon",
+	"authorize_url",
+	"access_token_url",
+	"redirect_url",
+	"api_endpoint",
+	"api_endpoint_args",
+	"auth_url_data",
 ];
 
-frappe.ui.form.on('Social Login Key', {
+frappe.ui.form.on("Social Login Key", {
 	refresh(frm) {
 		frm.trigger("setup_fields");
 	},
@@ -16,23 +23,25 @@ frappe.ui.form.on('Social Login Key', {
 	},
 
 	social_login_provider(frm) {
-		if(frm.doc.social_login_provider != "Custom") {
-			frappe.call({
-				"doc": frm.doc,
-				"method": "get_social_login_provider",
-				"args": {
-					"provider": frm.doc.social_login_provider
-				}
-			}).done((r) => {
-				const provider = r.message;
-				for(var field of fields) {
-					frm.set_value(field, provider[field]);
-					frm.set_df_property(field, "read_only", 1);
-					if (frm.doc.custom_base_url) {
-						frm.toggle_enable("base_url", 1);
+		if (frm.doc.social_login_provider != "Custom") {
+			frappe
+				.call({
+					doc: frm.doc,
+					method: "get_social_login_provider",
+					args: {
+						provider: frm.doc.social_login_provider,
+					},
+				})
+				.done((r) => {
+					const provider = r.message;
+					for (var field of fields) {
+						frm.set_value(field, provider[field]);
+						frm.set_df_property(field, "read_only", 1);
+						if (frm.doc.custom_base_url) {
+							frm.toggle_enable("base_url", 1);
+						}
 					}
-				}
-			});
+				});
 		} else {
 			frm.trigger("clear_fields");
 			frm.trigger("setup_fields");
@@ -41,38 +50,35 @@ frappe.ui.form.on('Social Login Key', {
 
 	setup_fields(frm) {
 		// set custom_base_url to read only for "Custom" provider
-		if(frm.doc.social_login_provider == "Custom") {
+		if (frm.doc.social_login_provider == "Custom") {
 			frm.set_value("custom_base_url", 1);
 			frm.set_df_property("custom_base_url", "read_only", 1);
 		}
 
 		// set fields to read only for providers from template
-		for(var f of fields) {
-			if(frm.doc.social_login_provider != "Custom"){
+		for (var f of fields) {
+			if (frm.doc.social_login_provider != "Custom") {
 				frm.set_df_property(f, "read_only", 1);
 			}
 		}
 
 		// enable base_url for providers with custom_base_url
-		if(frm.doc.custom_base_url) {
+		if (frm.doc.custom_base_url) {
 			frm.set_df_property("base_url", "read_only", 0);
 			frm.fields_dict["sb_identity_details"].collapse(false);
 		}
 
 		// hide social_login_provider and provider_name for non local
-		if(!frm.doc.__islocal &&
-			(frm.doc.social_login_provider ||
-				frm.doc.provider_name)) {
+		if (!frm.doc.__islocal && (frm.doc.social_login_provider || frm.doc.provider_name)) {
 			frm.set_df_property("social_login_provider", "hidden", 1);
 			frm.set_df_property("provider_name", "hidden", 1);
 		}
 	},
 
 	clear_fields(frm) {
-		for(var field of fields){
+		for (var field of fields) {
 			frm.set_value(field, "");
 			frm.set_df_property(field, "read_only", 0);
 		}
-	}
-
+	},
 });
diff --git a/frappe/integrations/doctype/social_login_key/test_social_login_key.py b/frappe/integrations/doctype/social_login_key/test_social_login_key.py
index c51ccb2c0f..c8eedf23da 100644
--- a/frappe/integrations/doctype/social_login_key/test_social_login_key.py
+++ b/frappe/integrations/doctype/social_login_key/test_social_login_key.py
@@ -1,6 +1,5 @@
 # Copyright (c) 2017, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
-import unittest
 from unittest.mock import MagicMock, patch
 
 from rauth import OAuth2Service
@@ -8,11 +7,12 @@ from rauth import OAuth2Service
 import frappe
 from frappe.auth import CookieManager, LoginManager
 from frappe.integrations.doctype.social_login_key.social_login_key import BaseUrlNotSetError
+from frappe.tests.utils import FrappeTestCase
 from frappe.utils import set_request
 from frappe.utils.oauth import login_via_oauth2
 
 
-class TestSocialLoginKey(unittest.TestCase):
+class TestSocialLoginKey(FrappeTestCase):
 	def test_adding_frappe_social_login_provider(self):
 		provider_name = "Frappe"
 		social_login_key = make_social_login_key(social_login_provider=provider_name)
diff --git a/frappe/integrations/doctype/stripe_settings/__init__.py b/frappe/integrations/doctype/stripe_settings/__init__.py
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/frappe/integrations/doctype/stripe_settings/stripe_settings.js b/frappe/integrations/doctype/stripe_settings/stripe_settings.js
deleted file mode 100644
index 578ae94906..0000000000
--- a/frappe/integrations/doctype/stripe_settings/stripe_settings.js
+++ /dev/null
@@ -1,8 +0,0 @@
-// Copyright (c) 2017, Frappe Technologies and contributors
-// For license information, please see license.txt
-
-frappe.ui.form.on('Stripe Settings', {
-	refresh: function(frm) {
-
-	}
-});
diff --git a/frappe/integrations/doctype/stripe_settings/stripe_settings.json b/frappe/integrations/doctype/stripe_settings/stripe_settings.json
deleted file mode 100644
index 306355319b..0000000000
--- a/frappe/integrations/doctype/stripe_settings/stripe_settings.json
+++ /dev/null
@@ -1,315 +0,0 @@
-{
- "allow_copy": 0, 
- "allow_guest_to_view": 0, 
- "allow_import": 0, 
- "allow_rename": 0, 
- "autoname": "field:gateway_name", 
- "beta": 0, 
- "creation": "2017-03-09 17:18:29.458397", 
- "custom": 0, 
- "docstatus": 0, 
- "doctype": "DocType", 
- "document_type": "", 
- "editable_grid": 1, 
- "engine": "InnoDB", 
- "fields": [
-  {
-   "allow_bulk_edit": 0, 
-   "allow_in_quick_entry": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "gateway_name", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Payment Gateway Name", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "translatable": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_in_quick_entry": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "publishable_key", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Publishable Key", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "translatable": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_in_quick_entry": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "column_break_3", 
-   "fieldtype": "Column Break", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "translatable": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_in_quick_entry": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "secret_key", 
-   "fieldtype": "Password", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Secret Key", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "translatable": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_in_quick_entry": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "section_break_5", 
-   "fieldtype": "Section Break", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "translatable": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_in_quick_entry": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "header_img", 
-   "fieldtype": "Attach Image", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Header Image", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "translatable": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_in_quick_entry": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "column_break_7", 
-   "fieldtype": "Column Break", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "translatable": 0, 
-   "unique": 0
-  }, 
-  {
-   "allow_bulk_edit": 0, 
-   "allow_in_quick_entry": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "redirect_url", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Redirect URL", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "translatable": 0, 
-   "unique": 0
-  }
- ], 
- "has_web_view": 0, 
- "hide_heading": 0, 
- "hide_toolbar": 0, 
- "idx": 0, 
- "image_view": 0, 
- "in_create": 0, 
- "is_submittable": 0, 
- "issingle": 0, 
- "istable": 0, 
- "max_attachments": 0, 
- "modified": "2018-05-23 13:32:14.429916", 
- "modified_by": "Administrator", 
- "module": "Integrations", 
- "name": "Stripe Settings", 
- "name_case": "", 
- "owner": "Administrator", 
- "permissions": [
-  {
-   "amend": 0, 
-   "cancel": 0, 
-   "create": 1, 
-   "delete": 1, 
-   "email": 1, 
-   "export": 0, 
-   "if_owner": 0, 
-   "import": 0, 
-   "permlevel": 0, 
-   "print": 1, 
-   "read": 1, 
-   "report": 0, 
-   "role": "System Manager", 
-   "set_user_permissions": 0, 
-   "share": 1, 
-   "submit": 0, 
-   "write": 1
-  }
- ], 
- "quick_entry": 1, 
- "read_only": 0, 
- "read_only_onload": 0, 
- "show_name_in_global_search": 0, 
- "sort_field": "modified", 
- "sort_order": "DESC", 
- "track_changes": 0, 
- "track_seen": 0
-}
\ No newline at end of file
diff --git a/frappe/integrations/doctype/stripe_settings/stripe_settings.py b/frappe/integrations/doctype/stripe_settings/stripe_settings.py
deleted file mode 100644
index 8e1d383790..0000000000
--- a/frappe/integrations/doctype/stripe_settings/stripe_settings.py
+++ /dev/null
@@ -1,280 +0,0 @@
-# Copyright (c) 2017, Frappe Technologies and contributors
-# License: MIT. See LICENSE
-
-from urllib.parse import urlencode
-
-import frappe
-from frappe import _
-from frappe.integrations.utils import (
-	create_payment_gateway,
-	create_request_log,
-	make_get_request,
-	make_post_request,
-)
-from frappe.model.document import Document
-from frappe.utils import call_hook_method, cint, flt, get_url
-
-
-class StripeSettings(Document):
-	supported_currencies = [
-		"AED",
-		"ALL",
-		"ANG",
-		"ARS",
-		"AUD",
-		"AWG",
-		"BBD",
-		"BDT",
-		"BIF",
-		"BMD",
-		"BND",
-		"BOB",
-		"BRL",
-		"BSD",
-		"BWP",
-		"BZD",
-		"CAD",
-		"CHF",
-		"CLP",
-		"CNY",
-		"COP",
-		"CRC",
-		"CVE",
-		"CZK",
-		"DJF",
-		"DKK",
-		"DOP",
-		"DZD",
-		"EGP",
-		"ETB",
-		"EUR",
-		"FJD",
-		"FKP",
-		"GBP",
-		"GIP",
-		"GMD",
-		"GNF",
-		"GTQ",
-		"GYD",
-		"HKD",
-		"HNL",
-		"HRK",
-		"HTG",
-		"HUF",
-		"IDR",
-		"ILS",
-		"INR",
-		"ISK",
-		"JMD",
-		"JPY",
-		"KES",
-		"KHR",
-		"KMF",
-		"KRW",
-		"KYD",
-		"KZT",
-		"LAK",
-		"LBP",
-		"LKR",
-		"LRD",
-		"MAD",
-		"MDL",
-		"MNT",
-		"MOP",
-		"MRO",
-		"MUR",
-		"MVR",
-		"MWK",
-		"MXN",
-		"MYR",
-		"NAD",
-		"NGN",
-		"NIO",
-		"NOK",
-		"NPR",
-		"NZD",
-		"PAB",
-		"PEN",
-		"PGK",
-		"PHP",
-		"PKR",
-		"PLN",
-		"PYG",
-		"QAR",
-		"RUB",
-		"SAR",
-		"SBD",
-		"SCR",
-		"SEK",
-		"SGD",
-		"SHP",
-		"SLL",
-		"SOS",
-		"STD",
-		"SVC",
-		"SZL",
-		"THB",
-		"TOP",
-		"TTD",
-		"TWD",
-		"TZS",
-		"UAH",
-		"UGX",
-		"USD",
-		"UYU",
-		"UZS",
-		"VND",
-		"VUV",
-		"WST",
-		"XAF",
-		"XOF",
-		"XPF",
-		"YER",
-		"ZAR",
-	]
-
-	currency_wise_minimum_charge_amount = {
-		"JPY": 50,
-		"MXN": 10,
-		"DKK": 2.50,
-		"HKD": 4.00,
-		"NOK": 3.00,
-		"SEK": 3.00,
-		"USD": 0.50,
-		"AUD": 0.50,
-		"BRL": 0.50,
-		"CAD": 0.50,
-		"CHF": 0.50,
-		"EUR": 0.50,
-		"GBP": 0.30,
-		"NZD": 0.50,
-		"SGD": 0.50,
-	}
-
-	def on_update(self):
-		create_payment_gateway(
-			"Stripe-" + self.gateway_name, settings="Stripe Settings", controller=self.gateway_name
-		)
-		call_hook_method("payment_gateway_enabled", gateway="Stripe-" + self.gateway_name)
-		if not self.flags.ignore_mandatory:
-			self.validate_stripe_credentails()
-
-	def validate_stripe_credentails(self):
-		if self.publishable_key and self.secret_key:
-			header = {
-				"Authorization": "Bearer {}".format(
-					self.get_password(fieldname="secret_key", raise_exception=False)
-				)
-			}
-			try:
-				make_get_request(url="https://api.stripe.com/v1/charges", headers=header)
-			except Exception:
-				frappe.throw(_("Seems Publishable Key or Secret Key is wrong !!!"))
-
-	def validate_transaction_currency(self, currency):
-		if currency not in self.supported_currencies:
-			frappe.throw(
-				_(
-					"Please select another payment method. Stripe does not support transactions in currency '{0}'"
-				).format(currency)
-			)
-
-	def validate_minimum_transaction_amount(self, currency, amount):
-		if currency in self.currency_wise_minimum_charge_amount:
-			if flt(amount) < self.currency_wise_minimum_charge_amount.get(currency, 0.0):
-				frappe.throw(
-					_("For currency {0}, the minimum transaction amount should be {1}").format(
-						currency, self.currency_wise_minimum_charge_amount.get(currency, 0.0)
-					)
-				)
-
-	def get_payment_url(self, **kwargs):
-		return get_url(f"./integrations/stripe_checkout?{urlencode(kwargs)}")
-
-	def create_request(self, data):
-		import stripe
-
-		self.data = frappe._dict(data)
-		stripe.api_key = self.get_password(fieldname="secret_key", raise_exception=False)
-		stripe.default_http_client = stripe.http_client.RequestsClient()
-
-		try:
-			self.integration_request = create_request_log(self.data, service_name="Stripe")
-			return self.create_charge_on_stripe()
-
-		except Exception:
-			frappe.log_error(frappe.get_traceback())
-			return {
-				"redirect_to": frappe.redirect_to_message(
-					_("Server Error"),
-					_(
-						"It seems that there is an issue with the server's stripe configuration. In case of failure, the amount will get refunded to your account."
-					),
-				),
-				"status": 401,
-			}
-
-	def create_charge_on_stripe(self):
-		import stripe
-
-		try:
-			charge = stripe.Charge.create(
-				amount=cint(flt(self.data.amount) * 100),
-				currency=self.data.currency,
-				source=self.data.stripe_token_id,
-				description=self.data.description,
-				receipt_email=self.data.payer_email,
-			)
-
-			if charge.captured == True:
-				self.integration_request.db_set("status", "Completed", update_modified=False)
-				self.flags.status_changed_to = "Completed"
-
-			else:
-				frappe.log_error(charge.failure_message, "Stripe Payment not completed")
-
-		except Exception:
-			frappe.log_error(frappe.get_traceback())
-
-		return self.finalize_request()
-
-	def finalize_request(self):
-		redirect_to = self.data.get("redirect_to") or None
-		redirect_message = self.data.get("redirect_message") or None
-		status = self.integration_request.status
-
-		if self.flags.status_changed_to == "Completed":
-			if self.data.reference_doctype and self.data.reference_docname:
-				custom_redirect_to = None
-				try:
-					custom_redirect_to = frappe.get_doc(
-						self.data.reference_doctype, self.data.reference_docname
-					).run_method("on_payment_authorized", self.flags.status_changed_to)
-				except Exception:
-					frappe.log_error(frappe.get_traceback())
-
-				if custom_redirect_to:
-					redirect_to = custom_redirect_to
-
-				redirect_url = "payment-success"
-
-			if self.redirect_url:
-				redirect_url = self.redirect_url
-				redirect_to = None
-		else:
-			redirect_url = "payment-failed"
-
-		if redirect_to:
-			redirect_url += "?" + urlencode({"redirect_to": redirect_to})
-		if redirect_message:
-			redirect_url += "&" + urlencode({"redirect_message": redirect_message})
-
-		return {"redirect_to": redirect_url, "status": status}
-
-
-def get_gateway_controller(doctype, docname):
-	reference_doc = frappe.get_doc(doctype, docname)
-	gateway_controller = frappe.db.get_value(
-		"Payment Gateway", reference_doc.payment_gateway, "gateway_controller"
-	)
-	return gateway_controller
diff --git a/frappe/integrations/doctype/stripe_settings/test_stripe_settings.py b/frappe/integrations/doctype/stripe_settings/test_stripe_settings.py
deleted file mode 100644
index eed87bfcaf..0000000000
--- a/frappe/integrations/doctype/stripe_settings/test_stripe_settings.py
+++ /dev/null
@@ -1,7 +0,0 @@
-# Copyright (c) 2018, Frappe Technologies and Contributors
-# License: MIT. See LICENSE
-import unittest
-
-
-class TestStripeSettings(unittest.TestCase):
-	pass
diff --git a/frappe/integrations/doctype/token_cache/test_token_cache.py b/frappe/integrations/doctype/token_cache/test_token_cache.py
index a9366d84d3..44b58fb238 100644
--- a/frappe/integrations/doctype/token_cache/test_token_cache.py
+++ b/frappe/integrations/doctype/token_cache/test_token_cache.py
@@ -1,13 +1,12 @@
 # Copyright (c) 2019, Frappe Technologies and contributors
 # License: MIT. See LICENSE
-import unittest
-
 import frappe
+from frappe.tests.utils import FrappeTestCase
 
 test_dependencies = ["User", "Connected App", "Token Cache"]
 
 
-class TestTokenCache(unittest.TestCase):
+class TestTokenCache(FrappeTestCase):
 	def setUp(self):
 		self.token_cache = frappe.get_last_doc("Token Cache")
 		self.token_cache.update({"connected_app": frappe.get_last_doc("Connected App").name})
diff --git a/frappe/integrations/doctype/token_cache/token_cache.js b/frappe/integrations/doctype/token_cache/token_cache.js
index b7cac9b804..c8074c876b 100644
--- a/frappe/integrations/doctype/token_cache/token_cache.js
+++ b/frappe/integrations/doctype/token_cache/token_cache.js
@@ -1,8 +1,7 @@
 // Copyright (c) 2019, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Token Cache', {
+frappe.ui.form.on("Token Cache", {
 	// refresh: function(frm) {
-
 	// }
 });
diff --git a/frappe/integrations/doctype/token_cache/token_cache.json b/frappe/integrations/doctype/token_cache/token_cache.json
index c016405031..0e6601fd98 100644
--- a/frappe/integrations/doctype/token_cache/token_cache.json
+++ b/frappe/integrations/doctype/token_cache/token_cache.json
@@ -86,10 +86,11 @@
   }
  ],
  "links": [],
- "modified": "2020-11-13 13:35:53.714352",
+ "modified": "2023-01-01 21:01:24.405729",
  "modified_by": "Administrator",
  "module": "Integrations",
  "name": "Token Cache",
+ "naming_rule": "Expression",
  "owner": "Administrator",
  "permissions": [
   {
@@ -106,5 +107,5 @@
  ],
  "sort_field": "modified",
  "sort_order": "DESC",
- "track_changes": 1
+ "states": []
 }
\ No newline at end of file
diff --git a/frappe/integrations/doctype/token_cache/token_cache.py b/frappe/integrations/doctype/token_cache/token_cache.py
index 25f07a16ba..b79dfe0abf 100644
--- a/frappe/integrations/doctype/token_cache/token_cache.py
+++ b/frappe/integrations/doctype/token_cache/token_cache.py
@@ -3,10 +3,12 @@
 
 from datetime import datetime, timedelta
 
+import pytz
+
 import frappe
 from frappe import _
 from frappe.model.document import Document
-from frappe.utils import cint, cstr
+from frappe.utils import cint, cstr, get_system_timezone
 
 
 class TokenCache(Document):
@@ -50,16 +52,20 @@ class TokenCache(Document):
 		return self
 
 	def get_expires_in(self):
-		expiry_time = frappe.utils.get_datetime(self.modified) + timedelta(self.expires_in)
-		return (datetime.now() - expiry_time).total_seconds()
+		system_timezone = pytz.timezone(get_system_timezone())
+		modified = frappe.utils.get_datetime(self.modified)
+		modified = system_timezone.localize(modified)
+		expiry_utc = modified.astimezone(pytz.utc) + timedelta(seconds=self.expires_in)
+		now_utc = datetime.utcnow().replace(tzinfo=pytz.utc)
+		return cint((expiry_utc - now_utc).total_seconds())
 
 	def is_expired(self):
 		return self.get_expires_in() < 0
 
 	def get_json(self):
 		return {
-			"access_token": self.get_password("access_token", ""),
-			"refresh_token": self.get_password("refresh_token", ""),
+			"access_token": self.get_password("access_token", False),
+			"refresh_token": self.get_password("refresh_token", False),
 			"expires_in": self.get_expires_in(),
 			"token_type": self.token_type,
 		}
diff --git a/frappe/integrations/doctype/webhook/__init__.py b/frappe/integrations/doctype/webhook/__init__.py
index 192cd2fa12..b9c96190ca 100644
--- a/frappe/integrations/doctype/webhook/__init__.py
+++ b/frappe/integrations/doctype/webhook/__init__.py
@@ -25,7 +25,7 @@ def run_webhooks(doc, method):
 			# query webhooks
 			webhooks_list = frappe.get_all(
 				"Webhook",
-				fields=["name", "`condition`", "webhook_docevent", "webhook_doctype"],
+				fields=["name", "condition", "webhook_docevent", "webhook_doctype"],
 				filters={"enabled": True},
 			)
 
diff --git a/frappe/integrations/doctype/webhook/test_webhook.py b/frappe/integrations/doctype/webhook/test_webhook.py
index 7d9d05cd9e..8284db7fd3 100644
--- a/frappe/integrations/doctype/webhook/test_webhook.py
+++ b/frappe/integrations/doctype/webhook/test_webhook.py
@@ -1,7 +1,6 @@
 # Copyright (c) 2017, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
 import json
-import unittest
 from contextlib import contextmanager
 
 import frappe
@@ -10,6 +9,7 @@ from frappe.integrations.doctype.webhook.webhook import (
 	get_webhook_data,
 	get_webhook_headers,
 )
+from frappe.tests.utils import FrappeTestCase
 
 
 @contextmanager
@@ -22,13 +22,14 @@ def get_test_webhook(config):
 		wh.delete()
 
 
-class TestWebhook(unittest.TestCase):
+class TestWebhook(FrappeTestCase):
 	@classmethod
 	def setUpClass(cls):
 		# delete any existing webhooks
 		frappe.db.delete("Webhook")
 		# Delete existing logs if any
 		frappe.db.delete("Webhook Request Log")
+		super().setUpClass()
 		# create test webhooks
 		cls.create_sample_webhooks()
 
@@ -176,7 +177,7 @@ class TestWebhook(unittest.TestCase):
 		webhook = frappe.get_doc("Webhook", {"webhook_doctype": "User"})
 		enqueue_webhook(user, webhook)
 
-		self.assertTrue(frappe.db.get_all("Webhook Request Log", pluck="name"))
+		self.assertTrue(frappe.get_all("Webhook Request Log", pluck="name"))
 
 	def test_webhook_with_array_body(self):
 		"""Check if array request body are supported."""
diff --git a/frappe/integrations/doctype/webhook/webhook.js b/frappe/integrations/doctype/webhook/webhook.js
index f4cb4373ea..302cacc194 100644
--- a/frappe/integrations/doctype/webhook/webhook.js
+++ b/frappe/integrations/doctype/webhook/webhook.js
@@ -6,27 +6,41 @@ frappe.webhook = {
 		if (frm.doc.webhook_doctype) {
 			frappe.model.with_doctype(frm.doc.webhook_doctype, () => {
 				// get doctype fields
-				let fields = $.map(frappe.get_doc("DocType", frm.doc.webhook_doctype).fields, (d) => {
-					if (frappe.model.no_value_type.includes(d.fieldtype) && !(frappe.model.table_fields.includes(d.fieldtype))) {
-						return null;
-					} else if (d.fieldtype === 'Currency' || d.fieldtype === 'Float') {
-						return { label: d.label, value: d.fieldname };
-					} else {
-						return {  label: `${__(d.label)} (${d.fieldtype})`, value: d.fieldname };
+				let fields = $.map(
+					frappe.get_doc("DocType", frm.doc.webhook_doctype).fields,
+					(d) => {
+						if (
+							frappe.model.no_value_type.includes(d.fieldtype) &&
+							!frappe.model.table_fields.includes(d.fieldtype)
+						) {
+							return null;
+						} else if (d.fieldtype === "Currency" || d.fieldtype === "Float") {
+							return { label: d.label, value: d.fieldname };
+						} else {
+							return {
+								label: `${__(d.label)} (${d.fieldtype})`,
+								value: d.fieldname,
+							};
+						}
 					}
-				});
+				);
 
 				// add meta fields
 				for (let field of frappe.model.std_fields) {
 					if (field.fieldname == "name") {
 						fields.unshift({ label: "Name (Doc Name)", value: "name" });
 					} else {
-						fields.push({ label: `${__(field.label)} (${field.fieldtype})`, value: field.fieldname });
+						fields.push({
+							label: `${__(field.label)} (${field.fieldtype})`,
+							value: field.fieldname,
+						});
 					}
 				}
 
 				frm.fields_dict.webhook_data.grid.update_docfield_property(
-					'fieldname', 'options', [""].concat(fields)
+					"fieldname",
+					"options",
+					[""].concat(fields)
 				);
 			});
 		}
@@ -42,22 +56,29 @@ frappe.webhook = {
 			}
 
 			if (header_value) {
-				let header_row = (frm.doc.webhook_headers || []).find(row => row.key === 'Content-Type');
+				let header_row = (frm.doc.webhook_headers || []).find(
+					(row) => row.key === "Content-Type"
+				);
 				if (header_row) {
-					frappe.model.set_value(header_row.doctype, header_row.name, "value", header_value);
+					frappe.model.set_value(
+						header_row.doctype,
+						header_row.name,
+						"value",
+						header_value
+					);
 				} else {
 					frm.add_child("webhook_headers", {
-						"key": "Content-Type",
-						"value": header_value
+						key: "Content-Type",
+						value: header_value,
 					});
 				}
 				frm.refresh();
 			}
 		}
-	}
+	},
 };
 
-frappe.ui.form.on('Webhook', {
+frappe.ui.form.on("Webhook", {
 	refresh: (frm) => {
 		frappe.webhook.set_fieldname_select(frm);
 	},
@@ -71,7 +92,7 @@ frappe.ui.form.on('Webhook', {
 	},
 
 	enable_security: (frm) => {
-		frm.toggle_reqd('webhook_secret', frm.doc.enable_security);
+		frm.toggle_reqd("webhook_secret", frm.doc.enable_security);
 	},
 
 	preview_document: (frm) => {
@@ -83,13 +104,15 @@ frappe.ui.form.on('Webhook', {
 				frm.refresh_field("preview_request_body");
 			},
 		});
-	}
+	},
 });
 
 frappe.ui.form.on("Webhook Data", {
 	fieldname: (frm, cdt, cdn) => {
 		let row = locals[cdt][cdn];
-		let df = frappe.get_meta(frm.doc.webhook_doctype).fields.filter((field) => field.fieldname == row.fieldname);
+		let df = frappe
+			.get_meta(frm.doc.webhook_doctype)
+			.fields.filter((field) => field.fieldname == row.fieldname);
 
 		if (!df.length) {
 			// check if field is a meta field
@@ -98,5 +121,5 @@ frappe.ui.form.on("Webhook Data", {
 
 		row.key = df.length ? df[0].fieldname : "name";
 		frm.refresh_field("webhook_data");
-	}
+	},
 });
diff --git a/frappe/integrations/doctype/webhook/webhook.json b/frappe/integrations/doctype/webhook/webhook.json
index a21e460659..c4fc4f675d 100644
--- a/frappe/integrations/doctype/webhook/webhook.json
+++ b/frappe/integrations/doctype/webhook/webhook.json
@@ -129,9 +129,11 @@
   },
   {
    "depends_on": "eval: doc.request_structure == \"JSON\"",
+   "description": "To add dynamic values from the document, use jinja tags like\n\n
        \n
        { \"id\": \"{{ doc.name }}\" }\n
        \n
        ",
    "fieldname": "webhook_json",
    "fieldtype": "Code",
-   "label": "JSON Request Body"
+   "label": "JSON Request Body",
+   "options": "JSON"
   },
   {
    "fieldname": "naming_series",
@@ -202,8 +204,13 @@
    "fieldtype": "Section Break"
   }
  ],
- "links": [],
- "modified": "2022-07-11 08:54:10.740512",
+ "links": [
+  {
+   "link_doctype": "Webhook Request Log",
+   "link_fieldname": "webhook"
+  }
+ ],
+ "modified": "2023-05-21 15:42:58.844826",
  "modified_by": "Administrator",
  "module": "Integrations",
  "name": "Webhook",
diff --git a/frappe/integrations/doctype/webhook/webhook.py b/frappe/integrations/doctype/webhook/webhook.py
index 64a98a61e1..7d168c659f 100644
--- a/frappe/integrations/doctype/webhook/webhook.py
+++ b/frappe/integrations/doctype/webhook/webhook.py
@@ -115,6 +115,7 @@ def enqueue_webhook(doc, webhook) -> None:
 	webhook: Webhook = frappe.get_doc("Webhook", webhook.get("name"))
 	headers = get_webhook_headers(doc, webhook)
 	data = get_webhook_data(doc, webhook)
+	r = None
 
 	for i in range(3):
 		try:
@@ -127,32 +128,40 @@ def enqueue_webhook(doc, webhook) -> None:
 			)
 			r.raise_for_status()
 			frappe.logger().debug({"webhook_success": r.text})
-			log_request(webhook.request_url, headers, data, r)
+			log_request(webhook.name, doc.name, webhook.request_url, headers, data, r)
 			break
 
 		except requests.exceptions.ReadTimeout as e:
 			frappe.logger().debug({"webhook_error": e, "try": i + 1})
-			log_request(webhook.request_url, headers, data)
+			log_request(webhook.name, doc.name, webhook.request_url, headers, data)
 
 		except Exception as e:
 			frappe.logger().debug({"webhook_error": e, "try": i + 1})
-			log_request(webhook.request_url, headers, data, r)
+			log_request(webhook.name, doc.name, webhook.request_url, headers, data, r)
 			sleep(3 * i + 1)
 			if i != 2:
 				continue
-			else:
-				webhook.log_error("Webhook failed")
 
 
-def log_request(url: str, headers: dict, data: dict, res: requests.Response | None = None):
+def log_request(
+	webhook: str,
+	docname: str,
+	url: str,
+	headers: dict,
+	data: dict,
+	res: requests.Response | None = None,
+):
 	request_log = frappe.get_doc(
 		{
 			"doctype": "Webhook Request Log",
+			"webhook": webhook,
+			"reference_document": docname,
 			"user": frappe.session.user if frappe.session.user else None,
 			"url": url,
 			"headers": frappe.as_json(headers) if headers else None,
 			"data": frappe.as_json(data) if data else None,
-			"response": frappe.as_json(res.json()) if res else None,
+			"response": res and res.text,
+			"error": frappe.get_traceback(),
 		}
 	)
 
diff --git a/frappe/integrations/doctype/webhook_data/webhook_data.json b/frappe/integrations/doctype/webhook_data/webhook_data.json
index 96ae7f786a..2ace6a9237 100644
--- a/frappe/integrations/doctype/webhook_data/webhook_data.json
+++ b/frappe/integrations/doctype/webhook_data/webhook_data.json
@@ -1,130 +1,43 @@
 {
- "allow_copy": 0, 
- "allow_guest_to_view": 0, 
- "allow_import": 0, 
- "allow_rename": 0, 
- "beta": 0, 
- "creation": "2017-09-14 12:08:50.302810", 
- "custom": 0, 
- "docstatus": 0, 
- "doctype": "DocType", 
- "document_type": "", 
- "editable_grid": 1, 
- "engine": "InnoDB", 
+ "actions": [],
+ "creation": "2017-09-14 12:08:50.302810",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "fieldname",
+  "cb_doc_data",
+  "key"
+ ],
  "fields": [
   {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "fieldname", 
-   "fieldtype": "Select", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Fieldname", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "fieldname": "fieldname",
+   "fieldtype": "Select",
+   "in_list_view": 1,
+   "label": "Fieldname",
+   "reqd": 1
+  },
   {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "cb_doc_data", 
-   "fieldtype": "Column Break", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "fieldname": "cb_doc_data",
+   "fieldtype": "Column Break"
+  },
   {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "key", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Key", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
+   "fieldname": "key",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Key",
+   "reqd": 1
   }
- ], 
- "has_web_view": 0, 
- "hide_heading": 0, 
- "hide_toolbar": 0, 
- "idx": 0, 
- "image_view": 0, 
- "in_create": 0, 
- "is_submittable": 0, 
- "issingle": 0, 
- "istable": 1, 
- "max_attachments": 0, 
- "modified": "2017-09-14 13:16:58.252176", 
- "modified_by": "Administrator", 
- "module": "Integrations", 
- "name": "Webhook Data", 
- "name_case": "", 
- "owner": "Administrator", 
- "permissions": [], 
- "quick_entry": 0, 
- "read_only": 0, 
- "read_only_onload": 0, 
- "show_name_in_global_search": 0, 
- "sort_field": "modified", 
- "sort_order": "DESC", 
- "track_changes": 0, 
- "track_seen": 0
+ ],
+ "istable": 1,
+ "links": [],
+ "modified": "2022-08-03 12:20:52.208987",
+ "modified_by": "Administrator",
+ "module": "Integrations",
+ "name": "Webhook Data",
+ "owner": "Administrator",
+ "permissions": [],
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "states": []
 }
\ No newline at end of file
diff --git a/frappe/integrations/doctype/webhook_header/webhook_header.json b/frappe/integrations/doctype/webhook_header/webhook_header.json
index 315d28335f..4aea5d02ed 100644
--- a/frappe/integrations/doctype/webhook_header/webhook_header.json
+++ b/frappe/integrations/doctype/webhook_header/webhook_header.json
@@ -1,101 +1,38 @@
 {
- "allow_copy": 0, 
- "allow_guest_to_view": 0, 
- "allow_import": 0, 
- "allow_rename": 0, 
- "beta": 0, 
- "creation": "2017-09-08 16:27:39.195379", 
- "custom": 0, 
- "docstatus": 0, 
- "doctype": "DocType", 
- "document_type": "", 
- "editable_grid": 1, 
- "engine": "InnoDB", 
+ "actions": [],
+ "creation": "2017-09-08 16:27:39.195379",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "key",
+  "value"
+ ],
  "fields": [
   {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "key", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Key", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "fieldname": "key",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Key"
+  },
   {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "value", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Value", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
+   "fieldname": "value",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "Value"
   }
- ], 
- "has_web_view": 0, 
- "hide_heading": 0, 
- "hide_toolbar": 0, 
- "idx": 0, 
- "image_view": 0, 
- "in_create": 0, 
- "is_submittable": 0, 
- "issingle": 0, 
- "istable": 1, 
- "max_attachments": 0, 
- "modified": "2017-09-08 16:28:20.025612", 
- "modified_by": "Administrator", 
- "module": "Integrations", 
- "name": "Webhook Header", 
- "name_case": "", 
- "owner": "Administrator", 
- "permissions": [], 
- "quick_entry": 1, 
- "read_only": 0, 
- "read_only_onload": 0, 
- "show_name_in_global_search": 0, 
- "sort_field": "modified", 
- "sort_order": "DESC", 
- "track_changes": 1, 
- "track_seen": 0
+ ],
+ "istable": 1,
+ "links": [],
+ "modified": "2022-08-03 12:20:51.949422",
+ "modified_by": "Administrator",
+ "module": "Integrations",
+ "name": "Webhook Header",
+ "owner": "Administrator",
+ "permissions": [],
+ "quick_entry": 1,
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "states": [],
+ "track_changes": 1
 }
\ No newline at end of file
diff --git a/frappe/integrations/doctype/webhook_request_log/test_webhook_request_log.py b/frappe/integrations/doctype/webhook_request_log/test_webhook_request_log.py
index e3ad5a88a4..fe15bb8b58 100644
--- a/frappe/integrations/doctype/webhook_request_log/test_webhook_request_log.py
+++ b/frappe/integrations/doctype/webhook_request_log/test_webhook_request_log.py
@@ -2,8 +2,8 @@
 # License: MIT. See LICENSE
 
 # import frappe
-import unittest
+from frappe.tests.utils import FrappeTestCase
 
 
-class TestWebhookRequestLog(unittest.TestCase):
+class TestWebhookRequestLog(FrappeTestCase):
 	pass
diff --git a/frappe/integrations/doctype/webhook_request_log/webhook_request_log.js b/frappe/integrations/doctype/webhook_request_log/webhook_request_log.js
index 9ec4f11536..1cb8c5ec76 100644
--- a/frappe/integrations/doctype/webhook_request_log/webhook_request_log.js
+++ b/frappe/integrations/doctype/webhook_request_log/webhook_request_log.js
@@ -1,8 +1,7 @@
 // Copyright (c) 2021, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Webhook Request Log', {
+frappe.ui.form.on("Webhook Request Log", {
 	// refresh: function(frm) {
-
 	// }
 });
diff --git a/frappe/integrations/doctype/webhook_request_log/webhook_request_log.json b/frappe/integrations/doctype/webhook_request_log/webhook_request_log.json
index d9410a2f82..b07197b12a 100644
--- a/frappe/integrations/doctype/webhook_request_log/webhook_request_log.json
+++ b/frappe/integrations/doctype/webhook_request_log/webhook_request_log.json
@@ -6,12 +6,15 @@
  "editable_grid": 1,
  "engine": "InnoDB",
  "field_order": [
-  "user",
+  "webhook",
+  "reference_document",
   "headers",
   "data",
   "column_break_4",
+  "user",
   "url",
-  "response"
+  "response",
+  "error"
  ],
  "fields": [
   {
@@ -51,12 +54,32 @@
    "label": "User",
    "options": "User",
    "read_only": 1
+  },
+  {
+   "fieldname": "reference_document",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "in_standard_filter": 1,
+   "label": "Reference Document",
+   "read_only": 1
+  },
+  {
+   "fieldname": "error",
+   "fieldtype": "Text",
+   "label": "Error",
+   "read_only": 1
+  },
+  {
+   "fieldname": "webhook",
+   "fieldtype": "Link",
+   "label": "Webhook",
+   "options": "Webhook"
   }
  ],
  "in_create": 1,
  "index_web_pages_for_search": 1,
  "links": [],
- "modified": "2022-05-03 09:33:49.240777",
+ "modified": "2023-05-21 15:50:10.414002",
  "modified_by": "Administrator",
  "module": "Integrations",
  "name": "Webhook Request Log",
@@ -78,6 +101,5 @@
  ],
  "sort_field": "modified",
  "sort_order": "DESC",
- "states": [],
- "track_changes": 1
+ "states": []
 }
\ No newline at end of file
diff --git a/frappe/integrations/doctype/webhook_request_log/webhook_request_log.py b/frappe/integrations/doctype/webhook_request_log/webhook_request_log.py
index 8fbc73f5e5..175215f4d4 100644
--- a/frappe/integrations/doctype/webhook_request_log/webhook_request_log.py
+++ b/frappe/integrations/doctype/webhook_request_log/webhook_request_log.py
@@ -1,9 +1,15 @@
 # Copyright (c) 2021, Frappe Technologies and contributors
 # License: MIT. See LICENSE
 
-# import frappe
+import frappe
 from frappe.model.document import Document
 
 
 class WebhookRequestLog(Document):
-	pass
+	@staticmethod
+	def clear_old_logs(days=30):
+		from frappe.query_builder import Interval
+		from frappe.query_builder.functions import Now
+
+		table = frappe.qb.DocType("Webhook Request Log")
+		frappe.db.delete(table, filters=(table.modified < (Now() - Interval(days=days))))
diff --git a/frappe/integrations/doctype/webhook_request_log/webhook_request_log_list.js b/frappe/integrations/doctype/webhook_request_log/webhook_request_log_list.js
new file mode 100644
index 0000000000..dd4e215157
--- /dev/null
+++ b/frappe/integrations/doctype/webhook_request_log/webhook_request_log_list.js
@@ -0,0 +1,7 @@
+frappe.listview_settings["Webhook Request Log"] = {
+	onload: function (list_view) {
+		frappe.require("logtypes.bundle.js", () => {
+			frappe.utils.logtypes.show_log_retention_message(list_view.doctype);
+		});
+	},
+};
diff --git a/frappe/integrations/google_oauth.py b/frappe/integrations/google_oauth.py
index edce63493e..8bc54e0b1d 100644
--- a/frappe/integrations/google_oauth.py
+++ b/frappe/integrations/google_oauth.py
@@ -19,6 +19,12 @@ _SERVICES = {
 	"drive": ("drive", "v3"),
 	"indexing": ("indexing", "v3"),
 }
+_DOMAIN_CALLBACK_METHODS = {
+	"mail": "frappe.email.oauth.authorize_google_access",
+	"contacts": "frappe.integrations.doctype.google_contacts.google_contacts.authorize_access",
+	"drive": "frappe.integrations.doctype.google_drive.google_drive.authorize_access",
+	"indexing": "frappe.website.doctype.website_settings.google_indexing.authorize_access",
+}
 
 
 class GoogleAuthenticationError(Exception):
@@ -53,7 +59,6 @@ class GoogleOAuth:
 		"""Returns a dict with access and refresh token.
 
 		:param oauth_code: code got back from google upon successful auhtorization
-		:param site_address: side address from which the request is being made
 		"""
 
 		data = {
@@ -96,10 +101,10 @@ class GoogleOAuth:
 	def get_authentication_url(self, state: dict[str, str]) -> dict[str, str]:
 		"""Returns google authentication url.
 
-		:param site_address: side address from which the request is being made (for redirect back to site)
-		:param state: [optional] dict of values which you need on callback (for calling methods, redirection back to the form, doc name, etc)
+		:param state: dict of values which you need on callback (for calling methods, redirection back to the form, doc name, etc)
 		"""
 
+		state.update({"domain": self.domain})
 		state = json.dumps(state)
 		callback_url = get_request_site_address(True) + CALLBACK_METHOD
 
@@ -175,13 +180,22 @@ def callback(state: str, code: str = None, error: str = None) -> None:
 	failure_query_param = state.pop("failure_query_param", "")
 
 	if not error:
-		state.update({"code": code})
-		frappe.get_attr(state.pop("method"))(**state)
+		if (domain := state.pop("domain")) in _DOMAIN_CALLBACK_METHODS:
+			state.update({"code": code})
+			frappe.get_attr(_DOMAIN_CALLBACK_METHODS[domain])(**state)
 
-		# GET request, hence using commit to persist changes
-		frappe.db.commit()  # nosemgrep
-
-	redirect = f"{redirect}?{failure_query_param if error else success_query_param}"
+			# GET request, hence using commit to persist changes
+			frappe.db.commit()  # nosemgrep
+		else:
+			return frappe.respond_as_web_page(
+				"Invalid Google Callback",
+				"The callback domain provided is not valid for Google Authentication",
+				http_status_code=400,
+				indicator_color="red",
+				width=640,
+			)
 
 	frappe.local.response["type"] = "redirect"
-	frappe.local.response["location"] = redirect
+	frappe.local.response[
+		"location"
+	] = f"{redirect}?{failure_query_param if error else success_query_param}"
diff --git a/frappe/integrations/oauth2_logins.py b/frappe/integrations/oauth2_logins.py
index c8252b0f70..942ad2b51b 100644
--- a/frappe/integrations/oauth2_logins.py
+++ b/frappe/integrations/oauth2_logins.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
+# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
 
 import json
@@ -9,46 +9,46 @@ from frappe.utils.oauth import login_via_oauth2, login_via_oauth2_id_token
 
 
 @frappe.whitelist(allow_guest=True)
-def login_via_google(code, state):
+def login_via_google(code: str, state: str):
 	login_via_oauth2("google", code, state, decoder=decoder_compat)
 
 
 @frappe.whitelist(allow_guest=True)
-def login_via_github(code, state):
+def login_via_github(code: str, state: str):
 	login_via_oauth2("github", code, state)
 
 
 @frappe.whitelist(allow_guest=True)
-def login_via_facebook(code, state):
+def login_via_facebook(code: str, state: str):
 	login_via_oauth2("facebook", code, state, decoder=decoder_compat)
 
 
 @frappe.whitelist(allow_guest=True)
-def login_via_frappe(code, state):
+def login_via_frappe(code: str, state: str):
 	login_via_oauth2("frappe", code, state, decoder=decoder_compat)
 
 
 @frappe.whitelist(allow_guest=True)
-def login_via_office365(code, state):
+def login_via_office365(code: str, state: str):
 	login_via_oauth2_id_token("office_365", code, state, decoder=decoder_compat)
 
 
 @frappe.whitelist(allow_guest=True)
-def login_via_salesforce(code, state):
+def login_via_salesforce(code: str, state: str):
 	login_via_oauth2("salesforce", code, state, decoder=decoder_compat)
 
 
 @frappe.whitelist(allow_guest=True)
-def login_via_fairlogin(code, state):
+def login_via_fairlogin(code: str, state: str):
 	login_via_oauth2("fairlogin", code, state, decoder=decoder_compat)
 
 
 @frappe.whitelist(allow_guest=True)
-def custom(code, state):
+def custom(code: str, state: str):
 	"""
 	Callback for processing code and state for user added providers
 
-	process social login from /api/method/frappe.integrations.custom/
+	process social login from /api/method/frappe.integrations.oauth2_logins.custom/
 	"""
 	path = frappe.request.path[1:].split("/")
 	if len(path) == 4 and path[3]:
diff --git a/frappe/integrations/utils.py b/frappe/integrations/utils.py
index f215a73dc6..5ae8965c83 100644
--- a/frappe/integrations/utils.py
+++ b/frappe/integrations/utils.py
@@ -96,54 +96,6 @@ def get_json(obj):
 	return obj if isinstance(obj, str) else frappe.as_json(obj, indent=1)
 
 
-def get_payment_gateway_controller(payment_gateway):
-	"""Return payment gateway controller"""
-	gateway = frappe.get_doc("Payment Gateway", payment_gateway)
-	if gateway.gateway_controller is None:
-		try:
-			return frappe.get_doc(f"{payment_gateway} Settings")
-		except Exception:
-			frappe.throw(_("{0} Settings not found").format(payment_gateway))
-	else:
-		try:
-			return frappe.get_doc(gateway.gateway_settings, gateway.gateway_controller)
-		except Exception:
-			frappe.throw(_("{0} Settings not found").format(payment_gateway))
-
-
-@frappe.whitelist(allow_guest=True, xss_safe=True)
-def get_checkout_url(**kwargs):
-	try:
-		if kwargs.get("payment_gateway"):
-			doc = frappe.get_doc("{} Settings".format(kwargs.get("payment_gateway")))
-			return doc.get_payment_url(**kwargs)
-		else:
-			raise Exception
-	except Exception:
-		frappe.respond_as_web_page(
-			_("Something went wrong"),
-			_(
-				"Looks like something is wrong with this site's payment gateway configuration. No payment has been made."
-			),
-			indicator_color="red",
-			http_status_code=frappe.ValidationError.http_status_code,
-		)
-
-
-def create_payment_gateway(gateway, settings=None, controller=None):
-	# NOTE: we don't translate Payment Gateway name because it is an internal doctype
-	if not frappe.db.exists("Payment Gateway", gateway):
-		payment_gateway = frappe.get_doc(
-			{
-				"doctype": "Payment Gateway",
-				"gateway": gateway,
-				"gateway_settings": settings,
-				"gateway_controller": controller,
-			}
-		)
-		payment_gateway.insert(ignore_permissions=True)
-
-
 def json_handler(obj):
 	if isinstance(obj, (datetime.date, datetime.timedelta, datetime.datetime)):
 		return str(obj)
diff --git a/frappe/integrations/workspace/integrations/integrations.json b/frappe/integrations/workspace/integrations/integrations.json
index bbd2e1199f..8d1dfd64af 100644
--- a/frappe/integrations/workspace/integrations/integrations.json
+++ b/frappe/integrations/workspace/integrations/integrations.json
@@ -1,6 +1,6 @@
 {
  "charts": [],
- "content": "[{\"type\":\"header\",\"data\":{\"text\":\"Reports & Masters\",\"col\":12}},{\"type\":\"card\",\"data\":{\"card_name\":\"Backup\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Google Services\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Authentication\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Payments\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Settings\",\"col\":4}}]",
+ "content": "[{\"type\":\"header\",\"data\":{\"text\":\"Reports & Masters\",\"col\":12}},{\"type\":\"card\",\"data\":{\"card_name\":\"Backup\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Google Services\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Authentication\",\"col\":4}},{\"type\":\"card\",\"data\":{\"card_name\":\"Settings\",\"col\":4}}]",
  "creation": "2020-03-02 15:16:18.714190",
  "docstatus": 0,
  "doctype": "Workspace",
@@ -106,11 +106,52 @@
   {
    "hidden": 0,
    "is_query_report": 0,
-   "label": "Authentication",
+   "label": "Settings",
    "link_count": 0,
    "onboard": 0,
    "type": "Card Break"
   },
+  {
+   "dependencies": "",
+   "hidden": 0,
+   "is_query_report": 0,
+   "label": "Webhook",
+   "link_count": 0,
+   "link_to": "Webhook",
+   "link_type": "DocType",
+   "onboard": 0,
+   "type": "Link"
+  },
+  {
+   "dependencies": "",
+   "hidden": 0,
+   "is_query_report": 0,
+   "label": "Slack Webhook URL",
+   "link_count": 0,
+   "link_to": "Slack Webhook URL",
+   "link_type": "DocType",
+   "onboard": 0,
+   "type": "Link"
+  },
+  {
+   "dependencies": "",
+   "hidden": 0,
+   "is_query_report": 0,
+   "label": "SMS Settings",
+   "link_count": 0,
+   "link_to": "SMS Settings",
+   "link_type": "DocType",
+   "onboard": 0,
+   "type": "Link"
+  },
+  {
+   "hidden": 0,
+   "is_query_report": 0,
+   "label": "Authentication",
+   "link_count": 4,
+   "onboard": 0,
+   "type": "Card Break"
+  },
   {
    "dependencies": "",
    "hidden": 0,
@@ -154,119 +195,16 @@
    "link_type": "DocType",
    "onboard": 0,
    "type": "Link"
-  },
-  {
-   "hidden": 0,
-   "is_query_report": 0,
-   "label": "Payments",
-   "link_count": 0,
-   "onboard": 0,
-   "type": "Card Break"
-  },
-  {
-   "dependencies": "",
-   "hidden": 0,
-   "is_query_report": 0,
-   "label": "Braintree Settings",
-   "link_count": 0,
-   "link_to": "Braintree Settings",
-   "link_type": "DocType",
-   "onboard": 0,
-   "type": "Link"
-  },
-  {
-   "dependencies": "",
-   "hidden": 0,
-   "is_query_report": 0,
-   "label": "PayPal Settings",
-   "link_count": 0,
-   "link_to": "PayPal Settings",
-   "link_type": "DocType",
-   "onboard": 0,
-   "type": "Link"
-  },
-  {
-   "dependencies": "",
-   "hidden": 0,
-   "is_query_report": 0,
-   "label": "Razorpay Settings",
-   "link_count": 0,
-   "link_to": "Razorpay Settings",
-   "link_type": "DocType",
-   "onboard": 0,
-   "type": "Link"
-  },
-  {
-   "dependencies": "",
-   "hidden": 0,
-   "is_query_report": 0,
-   "label": "Stripe Settings",
-   "link_count": 0,
-   "link_to": "Stripe Settings",
-   "link_type": "DocType",
-   "onboard": 0,
-   "type": "Link"
-  },
-  {
-   "dependencies": "",
-   "hidden": 0,
-   "is_query_report": 0,
-   "label": "Paytm Settings",
-   "link_count": 0,
-   "link_to": "Paytm Settings",
-   "link_type": "DocType",
-   "onboard": 0,
-   "type": "Link"
-  },
-  {
-   "hidden": 0,
-   "is_query_report": 0,
-   "label": "Settings",
-   "link_count": 0,
-   "onboard": 0,
-   "type": "Card Break"
-  },
-  {
-   "dependencies": "",
-   "hidden": 0,
-   "is_query_report": 0,
-   "label": "Webhook",
-   "link_count": 0,
-   "link_to": "Webhook",
-   "link_type": "DocType",
-   "onboard": 0,
-   "type": "Link"
-  },
-  {
-   "dependencies": "",
-   "hidden": 0,
-   "is_query_report": 0,
-   "label": "Slack Webhook URL",
-   "link_count": 0,
-   "link_to": "Slack Webhook URL",
-   "link_type": "DocType",
-   "onboard": 0,
-   "type": "Link"
-  },
-  {
-   "dependencies": "",
-   "hidden": 0,
-   "is_query_report": 0,
-   "label": "SMS Settings",
-   "link_count": 0,
-   "link_to": "SMS Settings",
-   "link_type": "DocType",
-   "onboard": 0,
-   "type": "Link"
   }
  ],
- "modified": "2022-01-13 17:39:01.292154",
+ "modified": "2022-07-23 18:00:28.805405",
  "modified_by": "Administrator",
  "module": "Integrations",
  "name": "Integrations",
  "owner": "Administrator",
  "parent_page": "",
  "public": 1,
+ "quick_lists": [],
  "restrict_to_domain": "",
  "roles": [],
  "sequence_id": 15.0,
diff --git a/frappe/migrate.py b/frappe/migrate.py
index 1c249dfdb1..3241b14152 100644
--- a/frappe/migrate.py
+++ b/frappe/migrate.py
@@ -13,6 +13,7 @@ from frappe.cache_manager import clear_global_cache
 from frappe.core.doctype.language.language import sync_languages
 from frappe.core.doctype.scheduled_job_type.scheduled_job_type import sync_jobs
 from frappe.database.schema import add_column
+from frappe.deferred_insert import save_to_db as flush_deferred_inserts
 from frappe.desk.notifications import clear_notifications
 from frappe.modules.patch_handler import PatchType
 from frappe.modules.utils import sync_customizations
@@ -89,8 +90,8 @@ class SiteMigration:
 			json.dump(list(frappe.flags.touched_tables), f, sort_keys=True, indent=4)
 
 		if not self.skip_search_index:
-			print(f"Building search index for {frappe.local.site}")
-			build_index_for_all_routes()
+			print(f"Queued rebuilding of search index for {frappe.local.site}")
+			frappe.enqueue(build_index_for_all_routes, queue="long")
 
 		frappe.publish_realtime("version-update")
 		frappe.flags.touched_tables.clear()
@@ -123,6 +124,7 @@ class SiteMigration:
 		* Sync in-Desk Module Dashboards
 		* Sync customizations: Custom Fields, Property Setters, Custom Permissions
 		* Sync Frappe's internal language master
+		* Flush deferred inserts made during maintenance mode.
 		* Sync Portal Menu Items
 		* Sync Installed Applications Version History
 		* Execute `after_migrate` hooks
@@ -132,6 +134,7 @@ class SiteMigration:
 		sync_dashboards()
 		sync_customizations()
 		sync_languages()
+		flush_deferred_inserts()
 
 		frappe.get_single("Portal Settings").sync_menu()
 		frappe.get_single("Installed Applications").update_versions()
@@ -159,6 +162,8 @@ class SiteMigration:
 		"""Run Migrate operation on site specified. This method initializes
 		and destroys connections to the site database.
 		"""
+		from frappe.utils.synchronization import filelock
+
 		if site:
 			frappe.init(site=site)
 			frappe.connect()
@@ -166,11 +171,12 @@ class SiteMigration:
 		if not self.required_services_running():
 			raise SystemExit(1)
 
-		self.setUp()
-		try:
-			self.pre_schema_updates()
-			self.run_schema_updates()
-		finally:
-			self.post_schema_updates()
-			self.tearDown()
-			frappe.destroy()
+		with filelock("bench_migrate", timeout=1):
+			self.setUp()
+			try:
+				self.pre_schema_updates()
+				self.run_schema_updates()
+				self.post_schema_updates()
+			finally:
+				self.tearDown()
+				frappe.destroy()
diff --git a/frappe/model/__init__.py b/frappe/model/__init__.py
index 29991fa403..ff6ad36c42 100644
--- a/frappe/model/__init__.py
+++ b/frappe/model/__init__.py
@@ -95,6 +95,7 @@ optional_fields = ("_user_tags", "_comments", "_assign", "_liked_by", "_seen")
 table_fields = ("Table", "Table MultiSelect")
 
 core_doctypes_list = (
+	"DefaultValue",
 	"DocType",
 	"DocField",
 	"DocPerm",
@@ -114,6 +115,8 @@ core_doctypes_list = (
 	"Client Script",
 )
 
+# NOTE: this is being used for dynamic autoincrement in new sites,
+# removing any of these will require patches.
 log_types = (
 	"Version",
 	"Error Log",
@@ -186,3 +189,39 @@ def delete_fields(args_dict, delete=0):
 		if frappe.db.db_type == "postgres":
 			# commit the results to db
 			frappe.db.commit()
+
+
+def get_permitted_fields(
+	doctype: str,
+	parenttype: str | None = None,
+	user: str | None = None,
+	permission_type: str | None = None,
+) -> list[str]:
+	meta = frappe.get_meta(doctype)
+	valid_columns = meta.get_valid_columns()
+
+	if doctype in core_doctypes_list:
+		return valid_columns
+
+	# DocType has only fields of type Table (Table, Table MultiSelect)
+	if set(valid_columns).issubset(default_fields):
+		return valid_columns
+
+	if permission_type is None:
+		permission_type = "select" if frappe.only_has_select_perm(doctype, user=user) else "read"
+
+	if permitted_fields := meta.get_permitted_fieldnames(
+		parenttype=parenttype, user=user, permission_type=permission_type
+	):
+		if permission_type == "select":
+			return permitted_fields
+
+		meta_fields = meta.default_fields.copy()
+		optional_meta_fields = [x for x in optional_fields if x in valid_columns]
+
+		if meta.istable:
+			meta_fields.extend(child_table_fields)
+
+		return meta_fields + permitted_fields + optional_meta_fields
+
+	return []
diff --git a/frappe/model/base_document.py b/frappe/model/base_document.py
index 0207571e14..811ba5894c 100644
--- a/frappe/model/base_document.py
+++ b/frappe/model/base_document.py
@@ -11,16 +11,17 @@ from frappe.model import (
 	default_fields,
 	display_fieldtypes,
 	float_like_fields,
+	get_permitted_fields,
 	table_fields,
 )
 from frappe.model.docstatus import DocStatus
 from frappe.model.naming import set_new_name
 from frappe.model.utils.link_count import notify_link_count
 from frappe.modules import load_doctype_module
-from frappe.utils import cast_fieldtype, cint, cstr, flt, now, sanitize_html, strip_html
+from frappe.utils import cast_fieldtype, cint, compare, cstr, flt, now, sanitize_html, strip_html
 from frappe.utils.html_utils import unescape_html
 
-max_positive_value = {"smallint": 2**15, "int": 2**31, "bigint": 2**63}
+max_positive_value = {"smallint": 2**15 - 1, "int": 2**31 - 1, "bigint": 2**63 - 1}
 
 DOCTYPE_TABLE_FIELDS = [
 	_dict(fieldname="fields", options="DocField"),
@@ -35,62 +36,70 @@ DOCTYPES_FOR_DOCTYPE = {"DocType", *TABLE_DOCTYPES_FOR_DOCTYPE.values()}
 
 
 def get_controller(doctype):
-	"""Returns the **class** object of the given DocType.
+	"""
+	Returns the locally cached **class** object of the given DocType.
 	For `custom` type, returns `frappe.model.document.Document`.
 
-	:param doctype: DocType name as string."""
+	:param doctype: DocType name as string.
+	"""
 
-	def _get_controller():
-		from frappe.model.document import Document
-		from frappe.utils.nestedset import NestedSet
-
-		module_name, custom = frappe.db.get_value(
-			"DocType", doctype, ("module", "custom"), cache=True
-		) or ("Core", False)
-
-		if custom:
-			is_tree = frappe.db.get_value("DocType", doctype, "is_tree", ignore=True, cache=True)
-			_class = NestedSet if is_tree else Document
-		else:
-			class_overrides = frappe.get_hooks("override_doctype_class")
-			if class_overrides and class_overrides.get(doctype):
-				import_path = class_overrides[doctype][-1]
-				module_path, classname = import_path.rsplit(".", 1)
-				module = frappe.get_module(module_path)
-				if not hasattr(module, classname):
-					raise ImportError(f"{doctype}: {classname} does not exist in module {module_path}")
-			else:
-				module = load_doctype_module(doctype, module_name)
-				classname = doctype.replace(" ", "").replace("-", "")
-
-			if hasattr(module, classname):
-				_class = getattr(module, classname)
-				if issubclass(_class, BaseDocument):
-					_class = getattr(module, classname)
-				else:
-					raise ImportError(doctype)
-			else:
-				raise ImportError(doctype)
-		return _class
-
-	if frappe.local.dev_server:
-		return _get_controller()
+	if frappe.local.dev_server or frappe.flags.in_migrate:
+		return import_controller(doctype)
 
 	site_controllers = frappe.controllers.setdefault(frappe.local.site, {})
 	if doctype not in site_controllers:
-		site_controllers[doctype] = _get_controller()
+		site_controllers[doctype] = import_controller(doctype)
 
 	return site_controllers[doctype]
 
 
+def import_controller(doctype):
+	from frappe.model.document import Document
+	from frappe.utils.nestedset import NestedSet
+
+	module_name = "Core"
+	if doctype not in DOCTYPES_FOR_DOCTYPE:
+		doctype_info = frappe.db.get_value("DocType", doctype, fieldname="*")
+		if doctype_info:
+			if doctype_info.custom:
+				return NestedSet if doctype_info.is_tree else Document
+			module_name = doctype_info.module
+
+	module_path = None
+	class_overrides = frappe.get_hooks("override_doctype_class")
+	if class_overrides and class_overrides.get(doctype):
+		import_path = class_overrides[doctype][-1]
+		module_path, classname = import_path.rsplit(".", 1)
+		module = frappe.get_module(module_path)
+
+	else:
+		module = load_doctype_module(doctype, module_name)
+		classname = doctype.replace(" ", "").replace("-", "")
+
+	class_ = getattr(module, classname, None)
+	if class_ is None:
+		raise ImportError(
+			doctype
+			if module_path is None
+			else f"{doctype}: {classname} does not exist in module {module_path}"
+		)
+
+	if not issubclass(class_, BaseDocument):
+		raise ImportError(f"{doctype}: {classname} is not a subclass of BaseDocument")
+
+	return class_
+
+
 class BaseDocument:
 	_reserved_keywords = {
 		"doctype",
 		"meta",
 		"_meta",
 		"flags",
+		"parent_doc",
 		"_table_fields",
 		"_valid_columns",
+		"_doc_before_save",
 		"_table_fieldnames",
 		"_reserved_keywords",
 		"dont_update_if_missing",
@@ -100,12 +109,7 @@ class BaseDocument:
 		if d.get("doctype"):
 			self.doctype = d["doctype"]
 
-		self._table_fieldnames = (
-			d["_table_fieldnames"]  # from cache
-			if "_table_fieldnames" in d
-			else {df.fieldname for df in self._get_table_fields()}
-		)
-
+		self._table_fieldnames = {df.fieldname for df in self._get_table_fields()}
 		self.update(d)
 		self.dont_update_if_missing = []
 
@@ -152,8 +156,9 @@ class BaseDocument:
 		if "name" in d:
 			self.name = d["name"]
 
+		ignore_children = hasattr(self, "flags") and self.flags.ignore_children
 		for key, value in d.items():
-			self.set(key, value)
+			self.set(key, value, as_value=ignore_children)
 
 		return self
 
@@ -290,7 +295,7 @@ class BaseDocument:
 			return DOCTYPE_TABLE_FIELDS
 
 		# child tables don't have child tables
-		if self.doctype in DOCTYPES_FOR_DOCTYPE or getattr(self, "parentfield", None):
+		if self.doctype in DOCTYPES_FOR_DOCTYPE:
 			return ()
 
 		return self.meta.get_table_fields()
@@ -299,19 +304,26 @@ class BaseDocument:
 		self, sanitize=True, convert_dates_to_str=False, ignore_nulls=False, ignore_virtual=False
 	) -> dict:
 		d = _dict()
+		permitted_fields = get_permitted_fields(
+			doctype=self.doctype, parenttype=getattr(self, "parenttype", None)
+		)
+
 		for fieldname in self.meta.get_valid_columns():
+			field_value = getattr(self, fieldname, None)
+
 			# column is valid, we can use getattr
-			d[fieldname] = getattr(self, fieldname, None)
+			d[fieldname] = field_value
 
 			# if no need for sanitization and value is None, continue
 			if not sanitize and d[fieldname] is None:
 				continue
 
 			df = self.meta.get_field(fieldname)
+			is_virtual_field = getattr(df, "is_virtual", False)
 
 			if df:
-				if getattr(df, "is_virtual", False):
-					if ignore_virtual:
+				if is_virtual_field:
+					if ignore_virtual or fieldname not in permitted_fields:
 						del d[fieldname]
 						continue
 
@@ -349,7 +361,7 @@ class BaseDocument:
 			):
 				d[fieldname] = str(d[fieldname])
 
-			if ignore_nulls and d[fieldname] is None:
+			if ignore_nulls and not is_virtual_field and d[fieldname] is None:
 				del d[fieldname]
 
 		return d
@@ -671,10 +683,23 @@ class BaseDocument:
 
 			return _("Error: Value missing for {0}: {1}").format(_(df.parent), _(df.label))
 
+		def has_content(df):
+			value = cstr(self.get(df.fieldname))
+			has_text_content = strip_html(value).strip()
+			has_img_tag = " int | None:
 		"""Returns float precision for a particular field (or get global default).
 
 		:param fieldname: Fieldname for which precision is required.
@@ -1086,7 +1117,8 @@ class BaseDocument:
 			df = get_default_df(fieldname)
 
 		if (
-			df.fieldtype == "Currency"
+			df
+			and df.fieldtype == "Currency"
 			and not currency
 			and (currency_field := df.get("options"))
 			and (currency_value := self.get(currency_field))
@@ -1165,7 +1197,10 @@ class BaseDocument:
 				# get values from old doc
 				if self.get("parent_doc"):
 					parent_doc = self.parent_doc.get_latest()
-					ref_doc = [d for d in parent_doc.get(self.parentfield) if d.name == self.name][0]
+					child_docs = [d for d in parent_doc.get(self.parentfield) if d.name == self.name]
+					if not child_docs:
+						return
+					ref_doc = child_docs[0]
 				else:
 					ref_doc = self.get_latest()
 
@@ -1219,7 +1254,7 @@ def _filter(data, filters, limit=None):
 
 	for d in data:
 		for f, fval in _filters.items():
-			if not frappe.compare(getattr(d, f, None), fval[0], fval[1]):
+			if not compare(getattr(d, f, None), fval[0], fval[1]):
 				break
 		else:
 			out.append(d)
diff --git a/frappe/model/create_new.py b/frappe/model/create_new.py
index 51810c3e18..f8b7a73a3b 100644
--- a/frappe/model/create_new.py
+++ b/frappe/model/create_new.py
@@ -115,7 +115,7 @@ def get_static_default_value(df, doctype_user_permissions, allowed_records):
 				return df.default
 
 	elif df.fieldtype == "Select" and df.options and df.options not in ("[Select]", "Loading..."):
-		return df.options.split("\n")[0]
+		return df.options.split("\n", 1)[0]
 
 
 def validate_value_via_user_permissions(
diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py
index 9cf831a8b9..340b3a97f4 100644
--- a/frappe/model/db_query.py
+++ b/frappe/model/db_query.py
@@ -13,8 +13,10 @@ import frappe.permissions
 import frappe.share
 from frappe import _
 from frappe.core.doctype.server_script.server_script_utils import get_server_script_map
-from frappe.model import optional_fields
+from frappe.database.utils import DefaultOrderBy, FallBackDateTimeStr, NestedSetHierarchy
+from frappe.model import get_permitted_fields, optional_fields
 from frappe.model.meta import get_table_columns
+from frappe.model.utils import is_virtual_doctype
 from frappe.model.utils.user_settings import get_user_settings, update_user_settings
 from frappe.query_builder.utils import Column
 from frappe.utils import (
@@ -27,6 +29,7 @@ from frappe.utils import (
 	get_timespan_date_range,
 	make_filter_tuple,
 )
+from frappe.utils.data import sbool
 
 LOCATE_PATTERN = re.compile(r"locate\([^,]+,\s*[`\"]?name[`\"]?\s*\)", flags=re.IGNORECASE)
 LOCATE_CAST_PATTERN = re.compile(
@@ -49,6 +52,8 @@ FIELD_COMMA_PATTERN = re.compile(r"[0-9a-zA-Z]+\s*,")
 STRICT_FIELD_PATTERN = re.compile(r".*/\*.*")
 STRICT_UNION_PATTERN = re.compile(r".*\s(union).*\s")
 ORDER_GROUP_PATTERN = re.compile(r".*[^a-z0-9-_ ,`'\"\.\(\)].*")
+FN_PARAMS_PATTERN = re.compile(r".*?\((.*)\).*")
+SPECIAL_FIELD_CHARS = frozenset(("(", "`", ".", "'", '"', "*"))
 
 
 class DatabaseQuery:
@@ -63,6 +68,17 @@ class DatabaseQuery:
 		self.ignore_ifnull = False
 		self.flags = frappe._dict()
 		self.reference_doctype = None
+		self.permission_map = {}
+
+	@property
+	def doctype_meta(self):
+		if not hasattr(self, "_doctype_meta"):
+			self._doctype_meta = frappe.get_meta(self.doctype)
+		return self._doctype_meta
+
+	@property
+	def query_tables(self):
+		return self.tables + [d.table_name for d in self.link_tables]
 
 	def execute(
 		self,
@@ -71,7 +87,7 @@ class DatabaseQuery:
 		or_filters=None,
 		docstatus=None,
 		group_by=None,
-		order_by="KEEP_DEFAULT_ORDERING",
+		order_by=DefaultOrderBy,
 		limit_start=False,
 		limit_page_length=None,
 		as_list=False,
@@ -96,18 +112,12 @@ class DatabaseQuery:
 		strict=True,
 		pluck=None,
 		ignore_ddl=False,
+		*,
 		parent_doctype=None,
 	) -> list:
 
-		if (
-			not ignore_permissions
-			and not frappe.has_permission(self.doctype, "select", user=user, parent_doctype=parent_doctype)
-			and not frappe.has_permission(self.doctype, "read", user=user, parent_doctype=parent_doctype)
-		):
-			frappe.flags.error_message = _("Insufficient Permission for {0}").format(
-				frappe.bold(self.doctype)
-			)
-			raise frappe.PermissionError(self.doctype)
+		if not ignore_permissions:
+			self.check_read_permission(self.doctype, parent_doctype=parent_doctype)
 
 		# filters and fields swappable
 		# its hard to remember what comes first
@@ -153,6 +163,7 @@ class DatabaseQuery:
 		self.run = run
 		self.strict = strict
 		self.ignore_ddl = ignore_ddl
+		self.parent_doctype = parent_doctype
 
 		# for contextual user permission check
 		# to determine which user permission is applicable on link field of specific doctype
@@ -161,6 +172,21 @@ class DatabaseQuery:
 		if user_settings:
 			self.user_settings = json.loads(user_settings)
 
+		if is_virtual_doctype(self.doctype):
+			from frappe.model.base_document import get_controller
+
+			controller = get_controller(self.doctype)
+			self.parse_args()
+			kwargs = {
+				"as_list": as_list,
+				"with_comment_count": with_comment_count,
+				"save_user_settings": save_user_settings,
+				"save_user_settings_fields": save_user_settings_fields,
+				"pluck": pluck,
+				"parent_doctype": parent_doctype,
+			} | self.__dict__
+			return controller.get_list(kwargs)
+
 		self.columns = self.get_table_columns()
 
 		# no table & ignore_ddl, return
@@ -169,7 +195,7 @@ class DatabaseQuery:
 
 		result = self.build_and_run()
 
-		if with_comment_count and not as_list and self.doctype:
+		if sbool(with_comment_count) and not as_list and self.doctype:
 			self.add_comment_count(result)
 
 		if save_user_settings:
@@ -222,13 +248,14 @@ class DatabaseQuery:
 		self.extract_tables()
 		self.set_optional_columns()
 		self.build_conditions()
+		self.apply_fieldlevel_read_permissions()
 
 		args = frappe._dict()
 
 		if self.with_childnames:
 			for t in self.tables:
-				if t != "`tab" + self.doctype + "`":
-					self.fields.append(t + ".name as '%s:name'" % t[4:-1])
+				if t != f"`tab{self.doctype}`":
+					self.fields.append(f"{t}.name as '{t[4:-1]}:name'")
 
 		# query dict
 		args.tables = self.tables[0]
@@ -236,7 +263,7 @@ class DatabaseQuery:
 		# left join parent, child tables
 		for child in self.tables[1:]:
 			parent_name = cast_name(f"{self.tables[0]}.name")
-			args.tables += f" {self.join} {child} on ({child}.parent = {parent_name})"
+			args.tables += f" {self.join} {child} on ({child}.parenttype = {frappe.db.escape(self.doctype)} and {child}.parent = {parent_name})"
 
 		# left join link tables
 		for link in self.link_tables:
@@ -258,17 +285,19 @@ class DatabaseQuery:
 		# Wrapping fields with grave quotes to allow support for sql keywords
 		# TODO: Add support for wrapping fields with sql functions and distinct keyword
 		for field in self.fields:
+			if field is None:
+				fields.append("NULL")
+				continue
+
 			stripped_field = field.strip().lower()
-			skip_wrapping = any(
-				[
-					stripped_field.startswith(("`", "*", '"', "'")),
-					"(" in stripped_field,
-					"distinct" in stripped_field,
-				]
-			)
-			if skip_wrapping:
+
+			if (
+				stripped_field[0] in {"`", "*", '"', "'"}
+				or "(" in stripped_field
+				or "distinct" in stripped_field
+			):
 				fields.append(field)
-			elif "as" in field.lower().split(" "):
+			elif "as" in stripped_field.split(" "):
 				col, _, new = field.split()
 				fields.append(f"`{col}` as {new}")
 			else:
@@ -315,13 +344,16 @@ class DatabaseQuery:
 
 		# convert child_table.fieldname to `tabChild DocType`.`fieldname`
 		for field in self.fields:
-			if "." in field and "tab" not in field:
+			if "." in field:
 				original_field = field
 				alias = None
 				if " as " in field:
-					field, alias = field.split(" as ")
-				linked_fieldname, fieldname = field.split(".")
+					field, alias = field.split(" as ", 1)
+				linked_fieldname, fieldname = field.split(".", 1)
 				linked_field = frappe.get_meta(self.doctype).get_field(linked_fieldname)
+				# this is not a link field
+				if not linked_field:
+					continue
 				linked_doctype = linked_field.options
 				if linked_field.fieldtype == "Link":
 					self.append_link_table(linked_doctype, linked_fieldname)
@@ -382,14 +414,21 @@ class DatabaseQuery:
 				_raise_exception()
 
 		for field in self.fields:
+			lower_field = field.lower().strip()
+
 			if SUB_QUERY_PATTERN.match(field):
-				if any(f"({keyword}" in field.lower() for keyword in blacklisted_keywords):
-					_raise_exception()
+				if lower_field[0] == "(":
+					subquery_token = lower_field[1:].lstrip().split(" ", 1)[0]
+					if subquery_token in blacklisted_keywords:
+						_raise_exception()
 
-				if any(f"{keyword}(" in field.lower() for keyword in blacklisted_functions):
-					_raise_exception()
+				function = lower_field.split("(", 1)[0].rstrip()
+				if function in blacklisted_functions:
+					frappe.throw(
+						_("Use of function {0} in field is restricted").format(function), exc=frappe.DataError
+					)
 
-				if "@" in field.lower():
+				if "@" in lower_field:
 					# prevent access to global variables
 					_raise_exception()
 
@@ -405,7 +444,7 @@ class DatabaseQuery:
 				if STRICT_FIELD_PATTERN.match(field):
 					frappe.throw(_("Illegal SQL Query"))
 
-				if STRICT_UNION_PATTERN.match(field.lower()):
+				if STRICT_UNION_PATTERN.match(lower_field):
 					frappe.throw(_("Illegal SQL Query"))
 
 	def extract_tables(self):
@@ -426,15 +465,13 @@ class DatabaseQuery:
 				if not ("tab" in field and "." in field) or any(x for x in sql_functions if x in field):
 					continue
 
-				table_name = field.split(".")[0]
+				table_name = field.split(".", 1)[0]
 
 				if table_name.lower().startswith("group_concat("):
 					table_name = table_name[13:]
 				if not table_name[0] == "`":
 					table_name = f"`{table_name}`"
-				if table_name not in self.tables and table_name not in (
-					d.table_name for d in self.link_tables
-				):
+				if table_name not in self.query_tables:
 					self.append_table(table_name)
 
 	def append_table(self, table_name):
@@ -452,14 +489,26 @@ class DatabaseQuery:
 			frappe._dict(doctype=doctype, fieldname=fieldname, table_name=f"`tab{doctype}`")
 		)
 
-	def check_read_permission(self, doctype):
-		ptype = "select" if frappe.only_has_select_perm(doctype) else "read"
+	def check_read_permission(self, doctype: str, parent_doctype: str | None = None):
+		if self.flags.ignore_permissions:
+			return
 
-		if not self.flags.ignore_permissions and not frappe.has_permission(
-			doctype, ptype=ptype, parent_doctype=self.doctype
-		):
+		if doctype not in self.permission_map:
+			self._set_permission_map(doctype, parent_doctype)
+
+		return self.permission_map[doctype]
+
+	def _set_permission_map(self, doctype: str, parent_doctype: str | None = None):
+		ptype = "select" if frappe.only_has_select_perm(doctype) else "read"
+		val = frappe.has_permission(
+			doctype,
+			ptype=ptype,
+			parent_doctype=parent_doctype or self.doctype,
+		)
+		if not val:
 			frappe.flags.error_message = _("Insufficient Permission for {0}").format(frappe.bold(doctype))
 			raise frappe.PermissionError(doctype)
+		self.permission_map[doctype] = ptype
 
 	def set_field_tables(self):
 		"""If there are more than one table, the fieldname must not be ambiguous.
@@ -471,12 +520,13 @@ class DatabaseQuery:
 
 		if len(self.tables) > 1 or len(self.link_tables) > 0:
 			for idx, field in enumerate(self.fields):
-				if "." not in field and not _in_standard_sql_methods(field):
+				if field is not None and "." not in field and not _in_standard_sql_methods(field):
 					self.fields[idx] = f"{self.tables[0]}.{field}"
 
 	def cast_name_fields(self):
 		for i, field in enumerate(self.fields):
-			self.fields[i] = cast_name(field)
+			if field is not None:
+				self.fields[i] = cast_name(field)
 
 	def get_table_columns(self):
 		try:
@@ -527,7 +577,7 @@ class DatabaseQuery:
 			if match_conditions:
 				self.conditions.append(f"({match_conditions})")
 
-	def build_filter_conditions(self, filters, conditions, ignore_permissions=None):
+	def build_filter_conditions(self, filters, conditions: list, ignore_permissions=None):
 		"""build conditions from user filters"""
 		if ignore_permissions is not None:
 			self.flags.ignore_permissions = ignore_permissions
@@ -541,6 +591,107 @@ class DatabaseQuery:
 			else:
 				conditions.append(self.prepare_filter_condition(f))
 
+	def remove_field(self, idx: int):
+		if self.as_list:
+			self.fields[idx] = None
+		else:
+			self.fields.pop(idx)
+
+	def apply_fieldlevel_read_permissions(self):
+		"""Apply fieldlevel read permissions to the query
+
+		Note: Does not apply to `frappe.model.core_doctype_list`
+
+		Remove fields that user is not allowed to read. If `fields=["*"]` is passed, only permitted fields will
+		be returned.
+
+		Example:
+		        - User has read permission only on `title` for DocType `Note`
+		        - Query: fields=["*"]
+		        - Result: fields=["title", ...] // will also include Frappe's meta field like `name`, `owner`, etc.
+		"""
+		if self.flags.ignore_permissions:
+			return
+
+		asterisk_fields = []
+		permitted_fields = get_permitted_fields(
+			doctype=self.doctype,
+			parenttype=self.parent_doctype,
+			permission_type=self.permission_map.get(self.doctype),
+		)
+
+		for i, field in enumerate(self.fields):
+			if "distinct" in field.lower():
+				# field: 'count(distinct `tabPhoto`.name) as total_count'
+				# column: 'tabPhoto.name'
+				if _fn := FN_PARAMS_PATTERN.findall(field):
+					column = _fn[0].replace("distinct ", "").replace("DISTINCT ", "").replace("`", "")
+				# field: 'distinct name'
+				# column: 'name'
+				else:
+					column = field.split(" ", 2)[1].replace("`", "")
+			else:
+				# field: 'count(`tabPhoto`.name) as total_count'
+				# column: 'tabPhoto.name'
+				column = field.split("(")[-1].split(")", 1)[0]
+				column = strip_alias(column).replace("`", "")
+
+			if column == "*" and not in_function("*", field):
+				asterisk_fields.append(i)
+				continue
+
+			# handle pseudo columns
+			elif not column or column.isnumeric():
+				continue
+
+			# labels / pseudo columns or frappe internals
+			elif column[0] in {"'", '"'} or column in optional_fields:
+				continue
+
+			# handle child / joined table fields
+			elif "." in field:
+				table, column = column.split(".", 1)
+				ch_doctype = table.replace("`", "").replace("tab", "", 1)
+
+				if wrap_grave_quotes(table) in self.query_tables:
+					permitted_child_table_fields = get_permitted_fields(
+						doctype=ch_doctype, parenttype=self.doctype
+					)
+					if column in permitted_child_table_fields or column in optional_fields:
+						continue
+					else:
+						self.remove_field(i)
+				else:
+					raise frappe.PermissionError(ch_doctype)
+
+			elif column in permitted_fields:
+				continue
+
+			# field inside function calls / * handles things like count(*)
+			elif "(" in field:
+				if "*" in field:
+					continue
+				elif _params := FN_PARAMS_PATTERN.findall(field):
+					params = (x.strip() for x in _params[0].split(","))
+					for param in params:
+						if not (
+							not param or param in permitted_fields or param.isnumeric() or "'" in param or '"' in param
+						):
+							self.remove_field(i)
+							break
+					continue
+				self.remove_field(i)
+
+			# remove if access not allowed
+			else:
+				self.remove_field(i)
+
+		# handle * fields
+		j = 0
+		for i in asterisk_fields:
+			self.fields[i + j : i + j + 1] = permitted_fields
+			j = j + len(permitted_fields) - 1
+
 	def prepare_filter_condition(self, f):
 		"""Returns a filter condition in the format:
 		ifnull(`tabDocType`.`fieldname`, fallback) operator "value"
@@ -566,21 +717,14 @@ class DatabaseQuery:
 		can_be_null = True
 
 		# prepare in condition
-		if f.operator.lower() in (
-			"ancestors of",
-			"descendants of",
-			"not ancestors of",
-			"not descendants of",
-		):
+		if f.operator.lower() in NestedSetHierarchy:
 			values = f.value or ""
 
 			# TODO: handle list and tuple
 			# if not isinstance(values, (list, tuple)):
 			# 	values = values.split(",")
-
 			field = meta.get_field(f.fieldname)
 			ref_doctype = field.options if field else f.doctype
-
 			lft, rgt = "", ""
 			if f.value:
 				lft, rgt = frappe.db.get_value(ref_doctype, f.value, ["lft", "rgt"])
@@ -610,6 +754,12 @@ class DatabaseQuery:
 			)
 
 		elif f.operator.lower() in ("in", "not in"):
+			# if values contain '' or falsy values then only coalesce column
+			# for `in` query this is only required if values contain '' or values are empty.
+			# for `not in` queries we can't be sure as column values might contain null.
+			if f.operator.lower() == "in":
+				can_be_null = not f.value or any(v is None or v == "" for v in f.value)
+
 			values = f.value or ""
 			if isinstance(values, str):
 				values = values.split(",")
@@ -622,6 +772,7 @@ class DatabaseQuery:
 				value = "('')"
 
 		else:
+			escape = True
 			df = meta.get("fields", {"fieldname": f.fieldname})
 			df = df[0] if df else None
 
@@ -632,19 +783,20 @@ class DatabaseQuery:
 				date_range = get_date_range(f.operator.lower(), f.value)
 				f.operator = "Between"
 				f.value = date_range
-				fallback = "'0001-01-01 00:00:00'"
+				fallback = f"'{FallBackDateTimeStr}'"
 
-			if f.operator in (">", "<") and (f.fieldname in ("creation", "modified")):
+			if f.operator in (">", "<", ">=", "<=") and (f.fieldname in ("creation", "modified")):
 				value = cstr(f.value)
-				fallback = "'0001-01-01 00:00:00'"
+				fallback = f"'{FallBackDateTimeStr}'"
 
 			elif f.operator.lower() in ("between") and (
 				f.fieldname in ("creation", "modified")
 				or (df and (df.fieldtype == "Date" or df.fieldtype == "Datetime"))
 			):
 
+				escape = False
 				value = get_between_date_filter(f.value, df)
-				fallback = "'0001-01-01 00:00:00'"
+				fallback = f"'{FallBackDateTimeStr}'"
 
 			elif f.operator.lower() == "is":
 				if f.value == "set":
@@ -665,7 +817,7 @@ class DatabaseQuery:
 
 			elif (df and df.fieldtype == "Datetime") or isinstance(f.value, datetime):
 				value = frappe.db.format_datetime(f.value)
-				fallback = "'0001-01-01 00:00:00'"
+				fallback = f"'{FallBackDateTimeStr}'"
 
 			elif df and df.fieldtype == "Time":
 				value = get_time(f.value).strftime("%H:%M:%S.%f")
@@ -702,7 +854,7 @@ class DatabaseQuery:
 				value = f"{tname}.{quote}{f.value.name}{quote}"
 
 			# escape value
-			elif isinstance(value, str) and f.operator.lower() != "between":
+			elif escape and isinstance(value, str):
 				value = f"{frappe.db.escape(value, percent=False)}"
 
 		if (
@@ -719,7 +871,7 @@ class DatabaseQuery:
 
 		return condition
 
-	def build_match_conditions(self, as_condition=True):
+	def build_match_conditions(self, as_condition=True) -> str | list:
 		"""add match conditions if applicable"""
 		self.match_filters = []
 		self.match_conditions = []
@@ -730,12 +882,11 @@ class DatabaseQuery:
 		if not self.tables:
 			self.extract_tables()
 
-		meta = frappe.get_meta(self.doctype)
-		role_permissions = frappe.permissions.get_role_permissions(meta, user=self.user)
+		role_permissions = frappe.permissions.get_role_permissions(self.doctype_meta, user=self.user)
 		self.shared = frappe.share.get_shared(self.doctype, self.user)
 
 		if (
-			not meta.istable
+			not self.doctype_meta.istable
 			and not (role_permissions.get("select") or role_permissions.get("read"))
 			and not self.flags.ignore_permissions
 			and not has_any_user_permission_for_doctype(self.doctype, self.user, self.reference_doctype)
@@ -771,7 +922,7 @@ class DatabaseQuery:
 
 			# share is an OR condition, if there is a role permission
 			if not only_if_shared and self.shared and conditions:
-				conditions = f"({conditions}) or ({self.get_share_condition()})"
+				conditions = f"(({conditions}) or ({self.get_share_condition()}))"
 
 			return conditions
 
@@ -785,9 +936,8 @@ class DatabaseQuery:
 		)
 
 	def add_user_permissions(self, user_permissions):
-		meta = frappe.get_meta(self.doctype)
 		doctype_link_fields = []
-		doctype_link_fields = meta.get_link_fields()
+		doctype_link_fields = self.doctype_meta.get_link_fields()
 
 		# append current doctype with fieldname as 'name' as first link field
 		doctype_link_fields.append(
@@ -862,8 +1012,6 @@ class DatabaseQuery:
 		return " and ".join(conditions) if conditions else ""
 
 	def set_order_by(self, args):
-		meta = frappe.get_meta(self.doctype)
-
 		if self.order_by and self.order_by != "KEEP_DEFAULT_ORDERING":
 			args.order_by = self.order_by
 		else:
@@ -882,33 +1030,38 @@ class DatabaseQuery:
 
 			if not group_function_without_group_by:
 				sort_field = sort_order = None
-				if meta.sort_field and "," in meta.sort_field:
+				if self.doctype_meta.sort_field and "," in self.doctype_meta.sort_field:
 					# multiple sort given in doctype definition
 					# Example:
 					# `idx desc, modified desc`
 					# will covert to
 					# `tabItem`.`idx` desc, `tabItem`.`modified` desc
 					args.order_by = ", ".join(
-						f"`tab{self.doctype}`.`{f.split()[0].strip()}` {f.split()[1].strip()}"
-						for f in meta.sort_field.split(",")
+						f"`tab{self.doctype}`.`{f_split[0].strip()}` {f_split[1].strip()}"
+						for f in self.doctype_meta.sort_field.split(",")
+						if (f_split := f.split(maxsplit=2))
 					)
 				else:
-					sort_field = meta.sort_field or "modified"
-					sort_order = (meta.sort_field and meta.sort_order) or "desc"
+					sort_field = self.doctype_meta.sort_field or "modified"
+					sort_order = (self.doctype_meta.sort_field and self.doctype_meta.sort_order) or "desc"
 					if self.order_by:
 						args.order_by = f"`tab{self.doctype}`.`{sort_field or 'modified'}` {sort_order or 'desc'}"
 
 				# draft docs always on top
-				if hasattr(meta, "is_submittable") and meta.is_submittable:
+				if hasattr(self.doctype_meta, "is_submittable") and self.doctype_meta.is_submittable:
 					if self.order_by:
 						args.order_by = f"`tab{self.doctype}`.docstatus asc, {args.order_by}"
 
-	def validate_order_by_and_group_by(self, parameters):
+	def validate_order_by_and_group_by(self, parameters: str):
 		"""Check order by, group by so that atleast one column is selected and does not have subquery"""
 		if not parameters:
 			return
 
+		blacklisted_sql_functions = {
+			"sleep",
+		}
 		_lower = parameters.lower()
+
 		if "select" in _lower and "from" in _lower:
 			frappe.throw(_("Cannot use sub-query in order by"))
 
@@ -916,13 +1069,20 @@ class DatabaseQuery:
 			frappe.throw(_("Illegal SQL Query"))
 
 		for field in parameters.split(","):
-			if "." in field and field.strip().startswith("`tab"):
-				tbl = field.strip().split(".")[0]
+			field = field.strip()
+			function = field.split("(", 1)[0].rstrip().lower()
+			full_field_name = "." in field and field.startswith("`tab")
+
+			if full_field_name:
+				tbl = field.split(".", 1)[0]
 				if tbl not in self.tables:
 					if tbl.startswith("`"):
 						tbl = tbl[4:-1]
 					frappe.throw(_("Please select atleast 1 column from {0} to sort/group").format(tbl))
 
+			if function in blacklisted_sql_functions:
+				frappe.throw(_("Cannot use {0} in order/group by").format(field))
+
 	def add_limit(self):
 		if self.limit_page_length:
 			return f"limit {self.limit_page_length} offset {self.limit_start}"
@@ -1010,8 +1170,9 @@ def get_order_by(doctype, meta):
 		# will covert to
 		# `tabItem`.`idx` desc, `tabItem`.`modified` desc
 		order_by = ", ".join(
-			f"`tab{doctype}`.`{f.split()[0].strip()}` {f.split()[1].strip()}"
+			f"`tab{doctype}`.`{f_split[0].strip()}` {f_split[1].strip()}"
 			for f in meta.sort_field.split(",")
+			if (f_split := f.split(maxsplit=2))
 		)
 
 	else:
@@ -1026,20 +1187,6 @@ def get_order_by(doctype, meta):
 	return order_by
 
 
-def is_parent_only_filter(doctype, filters):
-	# check if filters contains only parent doctype
-	only_parent_doctype = True
-
-	if isinstance(filters, list):
-		for filter in filters:
-			if doctype not in filter:
-				only_parent_doctype = False
-			if "Between" in filter:
-				filter[3] = get_between_date_filter(flt[3])
-
-	return only_parent_doctype
-
-
 def has_any_user_permission_for_doctype(doctype, user, applicable_for):
 	user_permissions = frappe.permissions.get_user_permissions(user=user)
 	doctype_user_permissions = user_permissions.get(doctype, [])
@@ -1129,3 +1276,30 @@ def requires_owner_constraint(role_permissions):
 	# not checking if either select or read if present in if_owner_perms
 	# because either of those is required to perform a query
 	return True
+
+
+def wrap_grave_quotes(table: str) -> str:
+	if table[0] != "`":
+		table = f"`{table}`"
+	return table
+
+
+def is_plain_field(field: str) -> bool:
+	for char in field:
+		if char in SPECIAL_FIELD_CHARS:
+			return False
+	return True
+
+
+def in_function(substr: str, field: str) -> bool:
+	try:
+		return substr in field and field.index("(") < field.index(substr) < field.index(")")
+	except ValueError:
+		return False
+
+
+def strip_alias(field: str) -> str:
+	# Note: Currently only supports aliases that use the " AS " syntax
+	if " as " in field.lower():
+		return field.split(" as ", 1)[0]
+	return field
diff --git a/frappe/model/delete_doc.py b/frappe/model/delete_doc.py
index b555dfc5dc..150be95476 100644
--- a/frappe/model/delete_doc.py
+++ b/frappe/model/delete_doc.py
@@ -9,30 +9,14 @@ import frappe.defaults
 import frappe.model.meta
 from frappe import _, get_module_path
 from frappe.desk.doctype.tag.tag import delete_tags_for_document
+from frappe.model.docstatus import DocStatus
 from frappe.model.dynamic_links import get_dynamic_link_map
 from frappe.model.naming import revert_series_if_last
+from frappe.model.utils import is_virtual_doctype
 from frappe.utils.file_manager import remove_all
 from frappe.utils.global_search import delete_for_document
 from frappe.utils.password import delete_all_passwords_for
 
-doctypes_to_skip = (
-	"Communication",
-	"ToDo",
-	"DocShare",
-	"Email Unsubscribe",
-	"Activity Log",
-	"File",
-	"Version",
-	"Document Follow",
-	"Comment",
-	"View Log",
-	"Tag Link",
-	"Notification Log",
-	"Email Queue",
-	"Document Share Key",
-	"Integration Request",
-)
-
 
 def delete_doc(
 	doctype=None,
@@ -57,11 +41,16 @@ def delete_doc(
 		doctype = frappe.form_dict.get("dt")
 		name = frappe.form_dict.get("dn")
 
+	is_virtual = is_virtual_doctype(doctype)
+
 	names = name
 	if isinstance(name, str) or isinstance(name, int):
 		names = [name]
 
 	for name in names or []:
+		if is_virtual:
+			frappe.get_doc(doctype, name).delete()
+			continue
 
 		# already deleted..?
 		if not frappe.db.exists(doctype, name):
@@ -86,9 +75,15 @@ def delete_doc(
 
 			else:
 				doc = frappe.get_doc(doctype, name)
+				if not (doc.custom or frappe.conf.developer_mode or frappe.flags.in_patch or force):
+					frappe.throw(_("Standard DocType can not be deleted."))
 
 				update_flags(doc, flags, ignore_permissions)
 				check_permission_and_not_submitted(doc)
+				# delete custom table fields using this doctype.
+				frappe.db.delete(
+					"Custom Field", {"options": name, "fieldtype": ("in", frappe.model.table_fields)}
+				)
 				frappe.db.delete("__global_search", {"doctype": name})
 
 			delete_from_table(doctype, name, ignore_doctypes, None)
@@ -161,9 +156,6 @@ def delete_doc(
 				except ImportError:
 					pass
 
-			# delete user_permissions
-			frappe.defaults.clear_default(parenttype="User Permission", key=doctype, value=name)
-
 
 def add_to_deleted_document(doc):
 	"""Add this document to Deleted Document table. Called after delete"""
@@ -184,7 +176,7 @@ def update_naming_series(doc):
 		if doc.meta.autoname.startswith("naming_series:") and getattr(doc, "naming_series", None):
 			revert_series_if_last(doc.naming_series, doc.name, doc)
 
-		elif doc.meta.autoname.split(":")[0] not in ("Prompt", "field", "hash", "autoincrement"):
+		elif doc.meta.autoname.split(":", 1)[0] not in ("Prompt", "field", "hash", "autoincrement"):
 			revert_series_if_last(doc.meta.autoname, doc.name, doc)
 
 
@@ -268,13 +260,13 @@ def check_if_doc_is_linked(doc, method="Delete"):
 				item_parent = getattr(item, "parent", None)
 				linked_doctype = item.parenttype if item_parent else link_dt
 
-				if linked_doctype in doctypes_to_skip or (
+				if linked_doctype in frappe.get_hooks("ignore_links_on_delete") or (
 					linked_doctype in ignore_linked_doctypes and method == "Cancel"
 				):
 					# don't check for communication and todo!
 					continue
 
-				if method != "Delete" and (method != "Cancel" or item.docstatus != 1):
+				if method != "Delete" and (method != "Cancel" or not DocStatus(item.docstatus).is_submitted()):
 					# don't raise exception if not
 					# linked to a non-cancelled doc when deleting or to a submitted doc when cancelling
 					continue
@@ -297,7 +289,9 @@ def check_if_doc_is_dynamically_linked(doc, method="Delete"):
 
 		ignore_linked_doctypes = doc.get("ignore_linked_doctypes") or []
 
-		if df.parent in doctypes_to_skip or (df.parent in ignore_linked_doctypes and method == "Cancel"):
+		if df.parent in frappe.get_hooks("ignore_links_on_delete") or (
+			df.parent in ignore_linked_doctypes and method == "Cancel"
+		):
 			# don't check for communication and todo!
 			continue
 
@@ -309,13 +303,12 @@ def check_if_doc_is_dynamically_linked(doc, method="Delete"):
 				refdoc.get(df.options) == doc.doctype
 				and refdoc.get(df.fieldname) == doc.name
 				and (
-					(method == "Delete" and refdoc.docstatus < 2)
-					or (method == "Cancel" and refdoc.docstatus == 1)
+					# linked to an non-cancelled doc when deleting
+					(method == "Delete" and not DocStatus(refdoc.docstatus).is_cancelled())
+					# linked to a submitted doc when cancelling
+					or (method == "Cancel" and DocStatus(refdoc.docstatus).is_submitted())
 				)
 			):
-				# raise exception only if
-				# linked to an non-cancelled doc when deleting
-				# or linked to a submitted doc when cancelling
 				raise_link_exists_exception(doc, df.parent, df.parent)
 		else:
 			# dynamic link in table
@@ -328,16 +321,20 @@ def check_if_doc_is_dynamically_linked(doc, method="Delete"):
 				(doc.doctype, doc.name),
 				as_dict=True,
 			):
-
-				if (method == "Delete" and refdoc.docstatus < 2) or (
-					method == "Cancel" and refdoc.docstatus == 1
+				# linked to an non-cancelled doc when deleting
+				# or linked to a submitted doc when cancelling
+				if (method == "Delete" and not DocStatus(refdoc.docstatus).is_cancelled()) or (
+					method == "Cancel" and DocStatus(refdoc.docstatus).is_submitted()
 				):
-					# raise exception only if
-					# linked to an non-cancelled doc when deleting
-					# or linked to a submitted doc when cancelling
-
 					reference_doctype = refdoc.parenttype if meta.istable else df.parent
 					reference_docname = refdoc.parent if meta.istable else refdoc.name
+
+					if reference_doctype in frappe.get_hooks("ignore_links_on_delete") or (
+						reference_doctype in ignore_linked_doctypes and method == "Cancel"
+					):
+						# don't check for communication and todo!
+						continue
+
 					at_position = f"at Row: {refdoc.idx}" if meta.istable else ""
 
 					raise_link_exists_exception(doc, reference_doctype, reference_docname, at_position)
@@ -355,7 +352,7 @@ def raise_link_exists_exception(doc, reference_doctype, reference_docname, row="
 
 	frappe.throw(
 		_("Cannot delete or cancel because {0} {1} is linked with {2} {3} {4}").format(
-			doc.doctype, doc_link, reference_doctype, reference_link, row
+			_(doc.doctype), doc_link, _(reference_doctype), reference_link, row
 		),
 		frappe.LinkExistsError,
 	)
diff --git a/frappe/model/docfield.py b/frappe/model/docfield.py
index c54a3855cb..e3740dcd7f 100644
--- a/frappe/model/docfield.py
+++ b/frappe/model/docfield.py
@@ -6,58 +6,5 @@
 import frappe
 
 
-def rename(doctype, fieldname, newname):
-	"""rename docfield"""
-	df = frappe.db.sql(
-		"""select * from tabDocField where parent=%s and fieldname=%s""", (doctype, fieldname), as_dict=1
-	)
-	if not df:
-		return
-
-	df = df[0]
-
-	if frappe.db.get_value("DocType", doctype, "issingle"):
-		update_single(df, newname)
-	else:
-		update_table(df, newname)
-		update_parent_field(df, newname)
-
-
-def update_single(f, new):
-	"""update in tabSingles"""
-	frappe.db.begin()
-	frappe.db.sql(
-		"""update tabSingles set field=%s where doctype=%s and field=%s""",
-		(new, f["parent"], f["fieldname"]),
-	)
-	frappe.db.commit()
-
-
-def update_table(f, new):
-	"""update table"""
-	query = get_change_column_query(f, new)
-	if query:
-		frappe.db.sql(query)
-
-
-def update_parent_field(f, new):
-	"""update 'parentfield' in tables"""
-	if f["fieldtype"] in frappe.model.table_fields:
-		frappe.db.begin()
-		frappe.db.sql(
-			"""update `tab{}` set parentfield={} where parentfield={}""".format(f["options"], "%s", "%s"),
-			(new, f["fieldname"]),
-		)
-		frappe.db.commit()
-
-
-def get_change_column_query(f, new):
-	"""generate change fieldname query"""
-	desc = frappe.db.sql("desc `tab%s`" % f["parent"])
-	for d in desc:
-		if d[0] == f["fieldname"]:
-			return "alter table `tab{}` change `{}` `{}` {}".format(f["parent"], f["fieldname"], new, d[1])
-
-
 def supports_translation(fieldtype):
 	return fieldtype in ["Data", "Select", "Text", "Small Text", "Text Editor"]
diff --git a/frappe/model/document.py b/frappe/model/document.py
index 9b781b1999..8477d35418 100644
--- a/frappe/model/document.py
+++ b/frappe/model/document.py
@@ -3,6 +3,7 @@
 import hashlib
 import json
 import time
+from typing import Any, Generator, Iterable
 
 from werkzeug.exceptions import NotFound
 
@@ -15,8 +16,9 @@ from frappe.model import optional_fields, table_fields
 from frappe.model.base_document import BaseDocument, get_controller
 from frappe.model.docstatus import DocStatus
 from frappe.model.naming import set_new_name, validate_name
+from frappe.model.utils import is_virtual_doctype
 from frappe.model.workflow import set_workflow_state_on_action, validate_workflow
-from frappe.utils import cstr, date_diff, file_lock, flt, get_datetime_str, now
+from frappe.utils import compare, cstr, date_diff, file_lock, flt, get_datetime_str, now
 from frappe.utils.data import get_absolute_url
 from frappe.utils.global_search import update_global_search
 
@@ -119,6 +121,10 @@ class Document(BaseDocument):
 			# incorrect arguments. let's not proceed.
 			raise ValueError("Illegal arguments")
 
+	@property
+	def is_locked(self):
+		return file_lock.lock_exists(self.get_signature())
+
 	@staticmethod
 	def whitelist(fn):
 		"""Decorator: Whitelist method to be called remotely via REST API."""
@@ -128,6 +134,7 @@ class Document(BaseDocument):
 	def load_from_db(self):
 		"""Load document and children from database and create properties
 		from fields"""
+		self.flags.ignore_children = True
 		if not getattr(self, "_metaclass", False) and self.meta.issingle:
 			single_doc = frappe.db.get_singles_dict(self.doctype, for_update=self.flags.for_update)
 			if not single_doc:
@@ -140,25 +147,27 @@ class Document(BaseDocument):
 			self._fix_numeric_types()
 
 		else:
+			get_value_kwargs = {"for_update": self.flags.for_update, "as_dict": True}
+			if not isinstance(self.name, (dict, list)):
+				get_value_kwargs["order_by"] = None
+
 			d = frappe.db.get_value(
-				self.doctype, self.name, "*", as_dict=1, for_update=self.flags.for_update
+				doctype=self.doctype, filters=self.name, fieldname="*", **get_value_kwargs
 			)
+
 			if not d:
 				frappe.throw(
 					_("{0} {1} not found").format(_(self.doctype), self.name), frappe.DoesNotExistError
 				)
 
 			super().__init__(d)
+		self.flags.pop("ignore_children", None)
 
 		for df in self._get_table_fields():
 			# Make sure not to query the DB for a child table, if it is a virtual one.
 			# During frappe is installed, the property "is_virtual" is not available in tabDocType, so
 			# we need to filter those cases for the access to frappe.db.get_value() as it would crash otherwise.
-			if (
-				hasattr(self, "doctype")
-				and not hasattr(self, "module")
-				and frappe.db.get_value("DocType", df.options, "is_virtual", cache=True)
-			):
+			if hasattr(self, "doctype") and not hasattr(self, "module") and is_virtual_doctype(df.options):
 				self.set(df.fieldname, [])
 				continue
 
@@ -169,6 +178,7 @@ class Document(BaseDocument):
 					"*",
 					as_dict=True,
 					order_by="idx asc",
+					for_update=self.flags.for_update,
 				)
 				or []
 			)
@@ -179,12 +189,15 @@ class Document(BaseDocument):
 		if hasattr(self, "__setup__"):
 			self.__setup__()
 
-	reload = load_from_db
+	def reload(self):
+		"""Reload document from database"""
+		self.load_from_db()
 
 	def get_latest(self):
-		if not getattr(self, "latest", None):
-			self.latest = frappe.get_doc(self.doctype, self.name)
-		return self.latest
+		if not getattr(self, "_doc_before_save", None):
+			self.load_doc_before_save()
+
+		return self._doc_before_save
 
 	def check_permission(self, permtype="read", permlevel=None):
 		"""Raise `frappe.PermissionError` if not permitted"""
@@ -192,15 +205,19 @@ class Document(BaseDocument):
 			self.raise_no_permission_to(permlevel or permtype)
 
 	def has_permission(self, permtype="read", verbose=False) -> bool:
-		"""Call `frappe.has_permission` if `self.flags.ignore_permissions`
-		is not set.
+		"""
+		Call `frappe.permissions.has_permission` if `ignore_permissions` flag isn't truthy
 
-		:param permtype: one of `read`, `write`, `submit`, `cancel`, `delete`"""
-		import frappe.permissions
+		:param permtype: `read`, `write`, `submit`, `cancel`, `delete`, etc.
+		:param verbose: DEPRECATED, will be removed in a future release.
+		"""
 
 		if self.flags.ignore_permissions:
 			return True
-		return frappe.permissions.has_permission(self.doctype, permtype, self, verbose=verbose)
+
+		import frappe.permissions
+
+		return frappe.permissions.has_permission(self.doctype, permtype, self)
 
 	def raise_no_permission_to(self, perm_type):
 		"""Raise `frappe.PermissionError`."""
@@ -272,9 +289,6 @@ class Document(BaseDocument):
 		if self.get("amended_from"):
 			self.copy_attachments_from_amended_from()
 
-		# flag to prevent creation of event update log for create and update both
-		# during document creation
-		self.flags.update_log_for_doc_creation = True
 		self.run_post_save_methods()
 		self.flags.in_insert = False
 
@@ -293,6 +307,10 @@ class Document(BaseDocument):
 				follow_document(self.doctype, self.name, frappe.session.user)
 		return self
 
+	def check_if_locked(self):
+		if self.creation and self.is_locked:
+			raise frappe.DocumentLockedError
+
 	def save(self, *args, **kwargs):
 		"""Wrapper for _save"""
 		return self._save(*args, **kwargs)
@@ -319,6 +337,7 @@ class Document(BaseDocument):
 		if self.get("__islocal") or not self.get("name"):
 			return self.insert()
 
+		self.check_if_locked()
 		self.check_permission("write", "save")
 
 		self.set_user_and_timestamp()
@@ -417,7 +436,7 @@ class Document(BaseDocument):
 				df.options, {"parent": self.name, "parenttype": self.doctype, "parentfield": fieldname}
 			)
 
-	def get_doc_before_save(self):
+	def get_doc_before_save(self) -> "Document":
 		return getattr(self, "_doc_before_save", None)
 
 	def has_value_changed(self, fieldname):
@@ -656,14 +675,19 @@ class Document(BaseDocument):
 		has_access_to = self.get_permlevel_access("read")
 
 		for df in self.meta.fields:
-			if df.permlevel and not df.permlevel in has_access_to:
-				self.set(df.fieldname, None)
+			if df.permlevel and hasattr(self, df.fieldname) and df.permlevel not in has_access_to:
+				try:
+					delattr(self, df.fieldname)
+				except AttributeError:
+					# hasattr might return True for class attribute which can't be delattr-ed.
+					continue
 
 		for table_field in self.meta.get_table_fields():
 			for df in frappe.get_meta(table_field.options).fields or []:
-				if df.permlevel and not df.permlevel in has_access_to:
+				if df.permlevel and df.permlevel not in has_access_to:
 					for child in self.get(table_field.fieldname) or []:
-						child.set(df.fieldname, None)
+						if hasattr(child, df.fieldname):
+							delattr(child, df.fieldname)
 
 	def validate_higher_perm_levels(self):
 		"""If the user does not have permissions at permlevel > 0, then reset the values to original / default"""
@@ -691,17 +715,16 @@ class Document(BaseDocument):
 					d.reset_values_if_no_permlevel_access(has_access_to, high_permlevel_fields)
 
 	def get_permlevel_access(self, permission_type="write"):
-		if not hasattr(self, "_has_access_to"):
-			self._has_access_to = {}
-
-		self._has_access_to[permission_type] = []
+		allowed_permlevels = []
 		roles = frappe.get_roles()
-		for perm in self.get_permissions():
-			if perm.role in roles and perm.get(permission_type):
-				if perm.permlevel not in self._has_access_to[permission_type]:
-					self._has_access_to[permission_type].append(perm.permlevel)
 
-		return self._has_access_to[permission_type]
+		for perm in self.get_permissions():
+			if (
+				perm.role in roles and perm.get(permission_type) and perm.permlevel not in allowed_permlevels
+			):
+				allowed_permlevels.append(perm.permlevel)
+
+		return allowed_permlevels
 
 	def has_permlevel_access_to(self, fieldname, df=None, permission_type="read"):
 		if not df:
@@ -741,49 +764,27 @@ class Document(BaseDocument):
 
 		Will also validate document transitions (Save > Submit > Cancel) calling
 		`self.check_docstatus_transition`."""
-		conflict = False
+
+		self.load_doc_before_save(raise_exception=True)
+
 		self._action = "save"
-		if not self.get("__islocal") and not self.meta.get("is_virtual"):
-			if self.meta.issingle:
-				modified = frappe.db.sql(
-					"""select value from tabSingles
-					where doctype=%s and field='modified' for update""",
-					self.doctype,
-				)
-				modified = modified and modified[0][0]
-				if modified and modified != cstr(self._original_modified):
-					conflict = True
-			else:
-				tmp = frappe.db.sql(
-					"""select modified, docstatus from `tab{}`
-					where name = %s for update""".format(
-						self.doctype
-					),
-					self.name,
-					as_dict=True,
-				)
+		previous = self._doc_before_save
 
-				if not tmp:
-					frappe.throw(_("Record does not exist"))
-				else:
-					tmp = tmp[0]
-
-				modified = cstr(tmp.modified)
-
-				if modified and modified != cstr(self._original_modified):
-					conflict = True
-
-				self.check_docstatus_transition(tmp.docstatus)
-
-			if conflict:
-				frappe.msgprint(
-					_("Error: Document has been modified after you have opened it")
-					+ (f" ({modified}, {self.modified}). ")
-					+ _("Please refresh to get the latest document."),
-					raise_exception=frappe.TimestampMismatchError,
-				)
-		else:
+		# previous is None for new document insert
+		if not previous:
 			self.check_docstatus_transition(0)
+			return
+
+		if cstr(previous.modified) != cstr(self._original_modified):
+			frappe.msgprint(
+				_("Error: Document has been modified after you have opened it")
+				+ (f" ({previous.modified}, {self.modified}). ")
+				+ _("Please refresh to get the latest document."),
+				raise_exception=frappe.TimestampMismatchError,
+			)
+
+		if not self.meta.issingle:
+			self.check_docstatus_transition(previous.docstatus)
 
 	def check_docstatus_transition(self, to_docstatus):
 		"""Ensures valid `docstatus` transition.
@@ -948,15 +949,19 @@ class Document(BaseDocument):
 		from frappe.email.doctype.notification.notification import evaluate_alert
 
 		if self.flags.notifications is None:
-			alerts = frappe.cache().hget("notifications", self.doctype)
-			if alerts is None:
-				alerts = frappe.get_all(
+
+			def _get_notifications():
+				"""returns enabled notifications for the current doctype"""
+
+				return frappe.get_all(
 					"Notification",
 					fields=["name", "event", "method"],
 					filters={"enabled": 1, "document_type": self.doctype},
 				)
-				frappe.cache().hset("notifications", self.doctype, alerts)
-			self.flags.notifications = alerts
+
+			self.flags.notifications = frappe.cache().hget(
+				"notifications", self.doctype, _get_notifications
+			)
 
 		if not self.flags.notifications:
 			return
@@ -1023,10 +1028,14 @@ class Document(BaseDocument):
 		"""Rename the document to `name`. This transforms the current object."""
 		return self._rename(name=name, merge=merge, force=force, validate_rename=validate_rename)
 
-	def delete(self, ignore_permissions=False):
+	def delete(self, ignore_permissions=False, force=False):
 		"""Delete document."""
 		return frappe.delete_doc(
-			self.doctype, self.name, ignore_permissions=ignore_permissions, flags=self.flags
+			self.doctype,
+			self.name,
+			ignore_permissions=ignore_permissions,
+			flags=self.flags,
+			force=force,
 		)
 
 	def run_before_save_methods(self):
@@ -1039,7 +1048,6 @@ class Document(BaseDocument):
 
 		Will also update title_field if set"""
 
-		self.load_doc_before_save()
 		self.reset_seen()
 
 		# before_validate method should be executed before ignoring validations
@@ -1062,15 +1070,21 @@ class Document(BaseDocument):
 
 		self.set_title_field()
 
-	def load_doc_before_save(self):
-		"""Save load document from db before saving"""
+	def load_doc_before_save(self, *, raise_exception: bool = False):
+		"""load existing document from db before saving"""
+
 		self._doc_before_save = None
-		if not self.is_new():
-			try:
-				self._doc_before_save = frappe.get_doc(self.doctype, self.name)
-			except frappe.DoesNotExistError:
-				self._doc_before_save = None
-				frappe.clear_last_message()
+
+		if self.is_new():
+			return
+
+		try:
+			self._doc_before_save = frappe.get_doc(self.doctype, self.name, for_update=True)
+		except frappe.DoesNotExistError:
+			if raise_exception:
+				raise
+
+			frappe.clear_last_message()
 
 	def run_post_save_methods(self):
 		"""Run standard methods after `INSERT` or `UPDATE`. Standard Methods are:
@@ -1092,7 +1106,9 @@ class Document(BaseDocument):
 			self.run_method("on_update_after_submit")
 
 		self.clear_cache()
-		self.notify_update()
+
+		if self.flags.get("notify_update", True):
+			self.notify_update()
 
 		update_global_search(self)
 
@@ -1103,8 +1119,6 @@ class Document(BaseDocument):
 		if (self.doctype, self.name) in frappe.flags.currently_saving:
 			frappe.flags.currently_saving.remove((self.doctype, self.name))
 
-		self.latest = None
-
 	def clear_cache(self):
 		frappe.clear_document_cache(self.doctype, self.name)
 
@@ -1145,7 +1159,7 @@ class Document(BaseDocument):
 		:param fieldname: fieldname of the property to be updated, or a {"field":"value"} dictionary
 		:param value: value of the property to be updated
 		:param update_modified: default True. updates the `modified` and `modified_by` properties
-		:param notify: default False. run doc.notify_updated() to send updates via socketio
+		:param notify: default False. run doc.notify_update() to send updates via socketio
 		:param commit: default False. run frappe.db.commit()
 		"""
 		if isinstance(fieldname, dict):
@@ -1166,6 +1180,9 @@ class Document(BaseDocument):
 		# to trigger notification on value change
 		self.run_method("before_change")
 
+		if self.name is None:
+			return
+
 		frappe.db.set_value(
 			self.doctype,
 			self.name,
@@ -1288,7 +1305,7 @@ class Document(BaseDocument):
 		df = doc.meta.get_field(fieldname)
 		val2 = doc.cast(val2, df)
 
-		if not frappe.compare(val1, condition, val2):
+		if not compare(val1, condition, val2):
 			label = doc.meta.get_label(fieldname)
 			condition_str = error_condition_map.get(condition, condition)
 			if doc.get("parentfield"):
@@ -1360,7 +1377,7 @@ class Document(BaseDocument):
 		if not user:
 			user = frappe.session.user
 
-		if self.meta.track_seen:
+		if self.meta.track_seen and not frappe.flags.read_only:
 			_seen = self.get("_seen") or []
 			_seen = frappe.parse_json(_seen)
 
@@ -1369,21 +1386,32 @@ class Document(BaseDocument):
 				frappe.db.set_value(self.doctype, self.name, "_seen", json.dumps(_seen), update_modified=False)
 				frappe.local.flags.commit = True
 
-	def add_viewed(self, user=None):
+	def add_viewed(self, user=None, force=False, unique_views=False):
 		"""add log to communication when a user views a document"""
 		if not user:
 			user = frappe.session.user
 
-		if hasattr(self.meta, "track_views") and self.meta.track_views:
-			frappe.get_doc(
+		if unique_views and frappe.db.exists(
+			"View Log", {"reference_doctype": self.doctype, "reference_name": self.name, "viewed_by": user}
+		):
+			return
+
+		if (hasattr(self.meta, "track_views") and self.meta.track_views) or force:
+			view_log = frappe.get_doc(
 				{
 					"doctype": "View Log",
-					"viewed_by": frappe.session.user,
+					"viewed_by": user,
 					"reference_doctype": self.doctype,
 					"reference_name": self.name,
 				}
-			).insert(ignore_permissions=True)
-			frappe.local.flags.commit = True
+			)
+			if frappe.flags.read_only:
+				view_log.deferred_insert()
+			else:
+				view_log.insert(ignore_permissions=True)
+				frappe.local.flags.commit = True
+
+			return view_log
 
 	def log_error(self, title=None, message=None):
 		"""Helper function to create an Error Log"""
@@ -1489,16 +1517,18 @@ class Document(BaseDocument):
 		if self in frappe.local.locked_documents:
 			frappe.local.locked_documents.remove(self)
 
-	# validation helpers
-	def validate_from_to_dates(self, from_date_field, to_date_field):
-		"""
-		Generic validation to verify date sequence
-		"""
-		if date_diff(self.get(to_date_field), self.get(from_date_field)) < 0:
+	def validate_from_to_dates(self, from_date_field: str, to_date_field: str) -> None:
+		"""Validate that the value of `from_date_field` is not later than the value of `to_date_field`."""
+		from_date = self.get(from_date_field)
+		to_date = self.get(to_date_field)
+		if not (from_date and to_date):
+			return
+
+		if date_diff(to_date, from_date) < 0:
 			frappe.throw(
 				_("{0} must be after {1}").format(
-					frappe.bold(self.meta.get_label(to_date_field)),
-					frappe.bold(self.meta.get_label(from_date_field)),
+					frappe.bold(_(self.meta.get_label(to_date_field))),
+					frappe.bold(_(self.meta.get_label(from_date_field))),
 				),
 				frappe.exceptions.InvalidDates,
 			)
@@ -1530,6 +1560,20 @@ class Document(BaseDocument):
 
 		return DocTags(self.doctype).get_tags(self.name).split(",")[1:]
 
+	def deferred_insert(self) -> None:
+		"""Push the document to redis temporarily and insert later.
+
+		WARN: This doesn't guarantee insertion as redis can be restarted
+		before data is flushed to database.
+		"""
+
+		from frappe.deferred_insert import deferred_insert
+
+		self.set_user_and_timestamp()
+
+		doc = self.get_valid_dict(convert_dates_to_str=True, ignore_virtual=True)
+		deferred_insert(doctype=self.doctype, records=doc)
+
 	def __repr__(self):
 		name = self.name or "unsaved"
 		doctype = self.__class__.__name__
@@ -1563,3 +1607,40 @@ def execute_action(__doctype, __name, __action, **kwargs):
 
 		doc.add_comment("Comment", _("Action Failed") + "

        " + msg)
 	doc.notify_update()
+
+
+def bulk_insert(
+	doctype: str,
+	documents: Iterable["Document"],
+	ignore_duplicates: bool = False,
+	chunk_size=10_000,
+):
+	"""Insert simple Documents objects to database in bulk.
+
+	Warning/Info:
+	        - All documents are inserted without triggering ANY hooks.
+	        - This function assumes you've done the due dilligence and inserts in similar fashion as db_insert
+	        - Documents can be any iterable / generator containing Document objects
+	"""
+
+	columns = frappe.get_meta(doctype).get_valid_columns()
+	values = _document_values_generator(documents, columns)
+
+	frappe.db.bulk_insert(
+		doctype, columns, values, ignore_duplicates=ignore_duplicates, chunk_size=chunk_size
+	)
+
+
+def _document_values_generator(
+	documents: Iterable["Document"],
+	columns: list[str],
+) -> Generator[tuple[Any], None, None]:
+	for doc in documents:
+		doc.creation = doc.modified = now()
+		doc.created_by = doc.modified_by = frappe.session.user
+		doc_values = doc.get_valid_dict(
+			convert_dates_to_str=True,
+			ignore_nulls=True,
+			ignore_virtual=True,
+		)
+		yield tuple(doc_values.get(col) for col in columns)
diff --git a/frappe/model/meta.py b/frappe/model/meta.py
index 014dd5faf1..32c1326170 100644
--- a/frappe/model/meta.py
+++ b/frappe/model/meta.py
@@ -40,21 +40,31 @@ from frappe.model.workflow import get_workflow_name
 from frappe.modules import load_doctype_module
 from frappe.utils import cast, cint, cstr
 
+DEFAULT_FIELD_LABELS = {
+	"name": lambda: _("ID"),
+	"creation": lambda: _("Created On"),
+	"docstatus": lambda: _("Document Status"),
+	"idx": lambda: _("Index"),
+	"modified": lambda: _("Last Updated On"),
+	"modified_by": lambda: _("Last Updated By"),
+	"owner": lambda: _("Created By"),
+	"_user_tags": lambda: _("Tags"),
+	"_liked_by": lambda: _("Liked By"),
+	"_comments": lambda: _("Comments"),
+	"_assign": lambda: _("Assigned To"),
+}
+
 
 def get_meta(doctype, cached=True) -> "Meta":
-	if cached:
-		if not frappe.local.meta_cache.get(doctype):
-			meta = frappe.cache().hget("meta", doctype)
-			if meta:
-				meta = Meta(meta)
-			else:
-				meta = Meta(doctype)
-				frappe.cache().hset("meta", doctype, meta.as_dict())
-			frappe.local.meta_cache[doctype] = meta
+	if not cached:
+		return Meta(doctype)
 
-		return frappe.local.meta_cache[doctype]
-	else:
-		return load_meta(doctype)
+	if meta := frappe.cache().hget("doctype_meta", doctype):
+		return meta
+
+	meta = Meta(doctype)
+	frappe.cache().hset("doctype_meta", doctype, meta)
+	return meta
 
 
 def load_meta(doctype):
@@ -86,7 +96,7 @@ def load_doctype_from_file(doctype):
 class Meta(Document):
 	_metaclass = True
 	default_fields = list(default_fields)[1:]
-	special_doctypes = (
+	special_doctypes = {
 		"DocField",
 		"DocPerm",
 		"DocType",
@@ -94,24 +104,19 @@ class Meta(Document):
 		"DocType Action",
 		"DocType Link",
 		"DocType State",
-	)
+	}
 	standard_set_once_fields = [
 		frappe._dict(fieldname="creation", fieldtype="Datetime"),
 		frappe._dict(fieldname="owner", fieldtype="Data"),
 	]
 
 	def __init__(self, doctype):
-		self._fields = {}
-		if isinstance(doctype, dict):
-			super().__init__(doctype)
-
-		elif isinstance(doctype, Document):
+		if isinstance(doctype, Document):
 			super().__init__(doctype.as_dict())
-			self.process()
-
 		else:
 			super().__init__("DocType", doctype)
-			self.process()
+
+		self.process()
 
 	def load_from_db(self):
 		try:
@@ -124,13 +129,18 @@ class Meta(Document):
 
 	def process(self):
 		# don't process for special doctypes
-		# prevent's circular dependency
+		# prevents circular dependency
 		if self.name in self.special_doctypes:
+			self.init_field_caches()
 			return
 
-		self.add_custom_fields()
+		has_custom_fields = self.add_custom_fields()
 		self.apply_property_setters()
-		self.sort_fields()
+		self.init_field_caches()
+
+		if has_custom_fields:
+			self.sort_fields()
+
 		self.get_valid_columns()
 		self.set_custom_permissions()
 		self.add_custom_links_and_actions()
@@ -198,12 +208,6 @@ class Meta(Document):
 		return self._set_only_once_fields
 
 	def get_table_fields(self):
-		if not hasattr(self, "_table_fields"):
-			if self.name != "DocType":
-				self._table_fields = self.get("fields", {"fieldtype": ["in", table_fields]})
-			else:
-				self._table_fields = DOCTYPE_TABLE_FIELDS
-
 		return self._table_fields
 
 	def get_global_search_fields(self):
@@ -214,7 +218,7 @@ class Meta(Document):
 
 		return fields
 
-	def get_valid_columns(self):
+	def get_valid_columns(self) -> list[str]:
 		if not hasattr(self, "_valid_columns"):
 			table_exists = frappe.db.table_exists(self.name)
 			if self.name in self.special_doctypes and table_exists:
@@ -233,36 +237,23 @@ class Meta(Document):
 
 	def get_field(self, fieldname):
 		"""Return docfield from meta"""
-		if not self._fields:
-			for f in self.get("fields"):
-				self._fields[f.fieldname] = f
 
 		return self._fields.get(fieldname)
 
 	def has_field(self, fieldname):
 		"""Returns True if fieldname exists"""
-		return True if self.get_field(fieldname) else False
+
+		return fieldname in self._fields
 
 	def get_label(self, fieldname):
 		"""Get label of the given fieldname"""
-		df = self.get_field(fieldname)
-		if df:
-			label = df.label
-		else:
-			label = {
-				"name": _("ID"),
-				"creation": _("Created On"),
-				"docstatus": _("Document Status"),
-				"idx": _("Index"),
-				"modified": _("Last Updated On"),
-				"modified_by": _("Last Updated By"),
-				"owner": _("Created By"),
-				"_user_tags": _("Tags"),
-				"_liked_by": _("Liked By"),
-				"_comments": _("Comments"),
-				"_assign": _("Assigned To"),
-			}.get(fieldname) or _("No Label")
-		return label
+		if df := self.get_field(fieldname):
+			return df.label
+
+		if fieldname in DEFAULT_FIELD_LABELS:
+			return DEFAULT_FIELD_LABELS[fieldname]()
+
+		return "No Label"
 
 	def get_options(self, fieldname):
 		return self.get_field(fieldname).options
@@ -273,12 +264,9 @@ class Meta(Document):
 		if df.fieldtype == "Link":
 			return df.options
 
-		elif df.fieldtype == "Dynamic Link":
+		if df.fieldtype == "Dynamic Link":
 			return self.get_options(df.options)
 
-		else:
-			return None
-
 	def get_search_fields(self):
 		search_fields = self.search_fields or "name"
 		search_fields = [d.strip() for d in search_fields.split(",")]
@@ -321,7 +309,7 @@ class Meta(Document):
 		return list_fields
 
 	def get_custom_fields(self):
-		return [d for d in self.fields if d.get("is_custom_field")]
+		return [d for d in self.fields if getattr(d, "is_custom_field", False)]
 
 	def get_title_field(self):
 		"""Return the title field of this doctype,
@@ -340,8 +328,9 @@ class Meta(Document):
 
 	def is_translatable(self, fieldname):
 		"""Return true of false given a field"""
-		field = self.get_field(fieldname)
-		return field and field.translatable
+
+		if field := self.get_field(fieldname):
+			return field.translatable
 
 	def get_workflow(self):
 		return get_workflow_name(self.name)
@@ -349,28 +338,30 @@ class Meta(Document):
 	def get_naming_series_options(self) -> list[str]:
 		"""Get list naming series options."""
 
-		field = self.get_field("naming_series")
-		if field:
+		if field := self.get_field("naming_series"):
 			options = field.options or ""
-
 			return options.split("\n")
+
 		return []
 
 	def add_custom_fields(self):
 		if not frappe.db.table_exists("Custom Field"):
 			return
 
-		custom_fields = frappe.db.sql(
-			"""
-			SELECT * FROM `tabCustom Field`
-			WHERE dt = %s AND docstatus < 2
-		""",
-			(self.name,),
-			as_dict=1,
+		custom_fields = frappe.db.get_values(
+			"Custom Field",
+			filters={"dt": self.name},
+			fieldname="*",
+			as_dict=True,
+			order_by="idx",
 			update={"is_custom_field": 1},
 		)
 
+		if not custom_fields:
+			return
+
 		self.extend("fields", custom_fields)
+		return True
 
 	def apply_property_setters(self):
 		"""
@@ -450,44 +441,44 @@ class Meta(Document):
 
 				self.set(fieldname, new_list)
 
+	def init_field_caches(self):
+		# field map
+		self._fields = {field.fieldname: field for field in self.fields}
+
+		# table fields
+		if self.name == "DocType":
+			self._table_fields = DOCTYPE_TABLE_FIELDS
+		else:
+			self._table_fields = self.get("fields", {"fieldtype": ["in", table_fields]})
+
 	def sort_fields(self):
-		"""sort on basis of insert_after"""
-		custom_fields = sorted(self.get_custom_fields(), key=lambda df: df.idx)
+		"""Sort custom fields on the basis of insert_after"""
 
-		if custom_fields:
-			newlist = []
+		field_order = []
+		insert_after_map = {}
 
-			# if custom field is at top
-			# insert_after is false
-			for c in list(custom_fields):
-				if not c.insert_after:
-					newlist.append(c)
-					custom_fields.pop(custom_fields.index(c))
+		for field in self.fields:
+			if not getattr(field, "is_custom_field", False):
+				field_order.append(field.fieldname)
 
-			# standard fields
-			newlist += [df for df in self.get("fields") if not df.get("is_custom_field")]
+			elif insert_after := getattr(field, "insert_after", None):
+				insert_after_map.setdefault(insert_after, []).append(field.fieldname)
 
-			newlist_fieldnames = [df.fieldname for df in newlist]
-			for i in range(2):
-				for df in list(custom_fields):
-					if df.insert_after in newlist_fieldnames:
-						cf = custom_fields.pop(custom_fields.index(df))
-						idx = newlist_fieldnames.index(df.insert_after)
-						newlist.insert(idx + 1, cf)
-						newlist_fieldnames.insert(idx + 1, cf.fieldname)
+			else:
+				# if custom field is at the top, insert after is None
+				field_order.insert(0, field.fieldname)
 
-				if not custom_fields:
-					break
+		if insert_after_map:
+			_update_field_order_based_on_insert_after(field_order, insert_after_map)
 
-			# worst case, add remaining custom fields to last
-			if custom_fields:
-				newlist += custom_fields
+		sorted_fields = []
 
-			# renum idx
-			for i, f in enumerate(newlist):
-				f.idx = i + 1
+		for idx, fieldname in enumerate(field_order, 1):
+			field = self._fields[fieldname]
+			field.idx = idx
+			sorted_fields.append(field)
 
-			self.fields = newlist
+		self.fields = sorted_fields
 
 	def set_custom_permissions(self):
 		"""Reset `permissions` with Custom DocPerm if exists"""
@@ -504,9 +495,11 @@ class Meta(Document):
 			if custom_perms:
 				self.permissions = [Document(d) for d in custom_perms]
 
-	def get_fieldnames_with_value(self, with_field_meta=False):
+	def get_fieldnames_with_value(self, with_field_meta=False, with_virtual_fields=False):
 		def is_value_field(docfield):
-			return not (docfield.get("is_virtual") or docfield.fieldtype in no_value_fields)
+			return not (
+				not with_virtual_fields and docfield.get("is_virtual") or docfield.fieldtype in no_value_fields
+			)
 
 		if with_field_meta:
 			return [df for df in self.fields if is_value_field(df)]
@@ -539,9 +532,38 @@ class Meta(Document):
 
 		return self.high_permlevel_fields
 
-	def get_permlevel_access(self, permission_type="read", parenttype=None):
+	def get_permitted_fieldnames(self, parenttype=None, *, user=None, permission_type="read"):
+		"""Build list of `fieldname` with read perm level and all the higher perm levels defined.
+
+		Note: If permissions are not defined for DocType, return all the fields with value.
+		"""
+		permitted_fieldnames = []
+
+		if self.istable and not parenttype:
+			return permitted_fieldnames
+
+		if not permission_type:
+			permission_type = "select" if frappe.only_has_select_perm(self.name, user=user) else "read"
+
+		if permission_type == "select":
+			return self.get_search_fields()
+
+		if not self.get_permissions(parenttype=parenttype):
+			return self.get_fieldnames_with_value()
+
+		permlevel_access = set(
+			self.get_permlevel_access(permission_type=permission_type, parenttype=parenttype, user=user)
+		)
+
+		for df in self.get_fieldnames_with_value(with_field_meta=True, with_virtual_fields=True):
+			if df.permlevel in permlevel_access:
+				permitted_fieldnames.append(df.fieldname)
+
+		return permitted_fieldnames
+
+	def get_permlevel_access(self, permission_type="read", parenttype=None, *, user=None):
 		has_access_to = []
-		roles = frappe.get_roles()
+		roles = frappe.get_roles(user)
 		for perm in self.get_permissions(parenttype):
 			if perm.role in roles and perm.get(permission_type):
 				if perm.permlevel not in has_access_to:
@@ -666,10 +688,17 @@ def is_single(doctype):
 
 
 def get_parent_dt(dt):
-	parent_dt = frappe.db.get_all(
-		"DocField", "parent", dict(fieldtype=["in", frappe.model.table_fields], options=dt), limit=1
+	if not frappe.is_table(dt):
+		return ""
+
+	return (
+		frappe.db.get_value(
+			"DocField",
+			{"fieldtype": ("in", frappe.model.table_fields), "options": dt},
+			"parent",
+		)
+		or ""
 	)
-	return parent_dt and parent_dt[0].parent or ""
 
 
 def set_fieldname(field_id, fieldname):
@@ -760,11 +789,11 @@ def trim_tables(doctype=None, dry_run=False, quiet=False):
 	delete the db field.
 	"""
 	UPDATED_TABLES = {}
-	filters = {"issingle": 0}
+	filters = {"issingle": 0, "is_virtual": 0}
 	if doctype:
 		filters["name"] = doctype
 
-	for doctype in frappe.db.get_all("DocType", filters=filters, pluck="name"):
+	for doctype in frappe.get_all("DocType", filters=filters, pluck="name"):
 		try:
 			dropped_columns = trim_table(doctype, dry_run=dry_run)
 			if dropped_columns:
@@ -801,3 +830,28 @@ def trim_table(doctype, dry_run=True):
 		frappe.db.sql_ddl(f"ALTER TABLE `tab{doctype}` {columns_to_remove}")
 
 	return DROPPED_COLUMNS
+
+
+def _update_field_order_based_on_insert_after(field_order, insert_after_map):
+	"""Update the field order based on insert_after_map"""
+
+	retry_field_insertion = True
+
+	while retry_field_insertion:
+		retry_field_insertion = False
+
+		for fieldname in list(insert_after_map):
+			if fieldname not in field_order:
+				continue
+
+			custom_field_index = field_order.index(fieldname)
+			for custom_field_name in insert_after_map.pop(fieldname):
+				custom_field_index += 1
+				field_order.insert(custom_field_index, custom_field_name)
+
+			retry_field_insertion = True
+
+	if insert_after_map:
+		# insert_after is an invalid fieldname, add these fields to the end
+		for fields in insert_after_map.values():
+			field_order.extend(fields)
diff --git a/frappe/model/naming.py b/frappe/model/naming.py
index 0ce6704c39..73b5930563 100644
--- a/frappe/model/naming.py
+++ b/frappe/model/naming.py
@@ -1,6 +1,7 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
 
+import datetime
 import re
 from typing import TYPE_CHECKING, Callable, Optional
 
@@ -23,6 +24,17 @@ NAMING_SERIES_PATTERN = re.compile(r"^[\w\- \/.#{}]+$", re.UNICODE)
 BRACED_PARAMS_PATTERN = re.compile(r"(\{[\w | #]+\})")
 
 
+# Types that can be using in naming series fields
+NAMING_SERIES_PART_TYPES = (
+	int,
+	str,
+	datetime.datetime,
+	datetime.date,
+	datetime.time,
+	datetime.timedelta,
+)
+
+
 class InvalidNamingSeriesError(frappe.ValidationError):
 	pass
 
@@ -47,8 +59,8 @@ class NamingSeries:
 		if not NAMING_SERIES_PATTERN.match(self.series):
 			frappe.throw(
 				_(
-					'Special Characters except "-", "#", ".", "/", "{" and "}" not allowed in naming series',
-				),
+					"Special Characters except '-', '#', '.', '/', '{{' and '}}' not allowed in naming series {0}"
+				).format(frappe.bold(self.series)),
 				exc=InvalidNamingSeriesError,
 			)
 
@@ -153,20 +165,11 @@ def set_new_name(doc):
 	if not doc.name and autoname:
 		set_name_from_naming_options(autoname, doc)
 
-	# if the autoname option is 'field:' and no name was derived, we need to
-	# notify
-	if not doc.name and autoname.startswith("field:"):
-		fieldname = autoname[6:]
-		frappe.throw(_("{0} is required").format(doc.meta.get_label(fieldname)))
-
 	# at this point, we fall back to name generation with the hash option
-	if not doc.name and autoname == "hash":
-		doc.name = make_autoname("hash", doc.doctype)
-
 	if not doc.name:
 		doc.name = make_autoname("hash", doc.doctype)
 
-	doc.name = validate_name(doc.doctype, doc.name, meta.get_field("name_case"))
+	doc.name = validate_name(doc.doctype, doc.name)
 
 
 def is_autoincremented(doctype: str, meta: Optional["Meta"] = None) -> bool:
@@ -208,6 +211,13 @@ def set_name_from_naming_options(autoname, doc):
 
 	if _autoname.startswith("field:"):
 		doc.name = _field_autoname(autoname, doc)
+
+		# if the autoname option is 'field:' and no name was derived, we need to
+		# notify
+		if not doc.name:
+			fieldname = autoname[6:]
+			frappe.throw(_("{0} is required").format(doc.meta.get_label(fieldname)))
+
 	elif _autoname.startswith("naming_series:"):
 		set_name_by_naming_series(doc)
 	elif _autoname.startswith("prompt"):
@@ -222,16 +232,21 @@ def set_naming_from_document_naming_rule(doc):
 	"""
 	Evaluate rules based on "Document Naming Series" doctype
 	"""
-	if doc.doctype in log_types:
+	from frappe.model.base_document import DOCTYPES_FOR_DOCTYPE
+
+	IGNORED_DOCTYPES = {*log_types, *DOCTYPES_FOR_DOCTYPE, "DefaultValue", "Patch Log"}
+
+	if doc.doctype in IGNORED_DOCTYPES:
 		return
 
-	# ignore_ddl if naming is not yet bootstrapped
-	for d in frappe.get_all(
+	document_naming_rules = frappe.cache_manager.get_doctype_map(
 		"Document Naming Rule",
-		dict(document_type=doc.doctype, disabled=0),
+		doc.doctype,
+		filters={"document_type": doc.doctype, "disabled": 0},
 		order_by="priority desc",
-		ignore_ddl=True,
-	):
+	)
+
+	for d in document_naming_rules:
 		frappe.get_cached_doc("Document Naming Rule", d.name).apply(doc)
 		if doc.name:
 			break
@@ -263,11 +278,11 @@ def make_autoname(key="", doctype="", doc=""):
 
 	*Example:*
 
-	              * DE/./.YY./.MM./.##### will create a series like
-	                DE/09/01/0001 where 09 is the year, 01 is the month and 0001 is the series
+	              * DE./.YY./.MM./.##### will create a series like
+	                DE/09/01/00001 where 09 is the year, 01 is the month and 00001 is the series
 	"""
 	if key == "hash":
-		return frappe.generate_hash(doctype, 10)
+		return frappe.generate_hash(length=10)
 
 	series = NamingSeries(key)
 	return series.generate_next_name(doc)
@@ -289,6 +304,7 @@ def parse_naming_series(
 	"""
 
 	name = ""
+	_sentinel = object()
 	if isinstance(parts, str):
 		parts = parts.split(".")
 
@@ -298,6 +314,9 @@ def parse_naming_series(
 	series_set = False
 	today = now_datetime()
 	for e in parts:
+		if not e:
+			continue
+
 		part = ""
 		if e.startswith("#"):
 			if not series_set:
@@ -318,16 +337,16 @@ def parse_naming_series(
 			part = str(today)
 		elif e == "FY":
 			part = frappe.defaults.get_user_default("fiscal_year")
-		elif e.startswith("{") and doc:
+		elif doc and (e.startswith("{") or doc.get(e, _sentinel) is not _sentinel):
 			e = e.replace("{", "").replace("}", "")
-			part = (cstr(doc.get(e)) or "").strip()
-		elif doc and doc.get(e):
-			part = (cstr(doc.get(e)) or "").strip()
+			part = doc.get(e)
 		else:
 			part = e
 
 		if isinstance(part, str):
 			name += part
+		elif isinstance(part, NAMING_SERIES_PART_TYPES):
+			name += cstr(part).strip()
 
 	return name
 
@@ -424,7 +443,7 @@ def get_default_naming_series(doctype: str) -> str | None:
 			return option
 
 
-def validate_name(doctype: str, name: int | str, case: str | None = None):
+def validate_name(doctype: str, name: int | str):
 
 	if not name:
 		frappe.throw(_("No Name Specified for {0}").format(doctype))
@@ -442,10 +461,6 @@ def validate_name(doctype: str, name: int | str, case: str | None = None):
 		frappe.throw(
 			_("There were some errors setting the name, please contact the administrator"), frappe.NameError
 		)
-	if case == "Title Case":
-		name = name.title()
-	if case == "UPPER CASE":
-		name = name.upper()
 	name = name.strip()
 
 	if not frappe.get_meta(doctype).get("issingle") and (doctype == name) and (name != "DocType"):
@@ -534,9 +549,7 @@ def _format_autoname(autoname, doc):
 
 	def get_param_value_for_match(match):
 		param = match.group()
-		# trim braces
-		trimmed_param = param[1:-1]
-		return parse_naming_series([trimmed_param], doc=doc)
+		return parse_naming_series([param[1:-1]], doc=doc)
 
 	# Replace braced params with their parsed value
 	name = BRACED_PARAMS_PATTERN.sub(get_param_value_for_match, autoname_value)
diff --git a/frappe/model/rename_doc.py b/frappe/model/rename_doc.py
index b05df57364..3908365291 100644
--- a/frappe/model/rename_doc.py
+++ b/frappe/model/rename_doc.py
@@ -1,5 +1,6 @@
 # Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
+from types import NoneType
 from typing import TYPE_CHECKING
 
 import frappe
@@ -46,7 +47,7 @@ def update_document_title(
 
 	# TODO: omit this after runtime type checking (ref: https://github.com/frappe/frappe/pull/14927)
 	for obj in [docname, updated_title, updated_name]:
-		if not isinstance(obj, (str, type(None))):
+		if not isinstance(obj, (str, NoneType)):
 			frappe.throw(f"{obj=} must be of type str or None")
 
 	# handle bad API usages
@@ -196,14 +197,6 @@ def rename_doc(
 	if not merge:
 		rename_password(doctype, old, new)
 
-	# update user_permissions
-	DefaultValue = frappe.qb.DocType("DefaultValue")
-	frappe.qb.update(DefaultValue).set(DefaultValue.defvalue, new).where(
-		(DefaultValue.parenttype == "User Permission")
-		& (DefaultValue.defkey == doctype)
-		& (DefaultValue.defvalue == old)
-	).run()
-
 	if merge:
 		new_doc.add_comment("Edit", _("merged {0} into {1}").format(frappe.bold(old), frappe.bold(new)))
 	else:
@@ -236,7 +229,7 @@ def update_assignments(old: str, new: str, doctype: str) -> None:
 
 	for user in common_assignments:
 		# delete todos linked to old doc
-		todos = frappe.db.get_all(
+		todos = frappe.get_all(
 			"ToDo",
 			{
 				"owner": user,
@@ -397,11 +390,6 @@ def rename_doctype(doctype: str, old: str, new: str) -> None:
 	for fieldtype in fields_with_options:
 		update_options_for_fieldtype(fieldtype, old, new)
 
-	# change options where select options are hardcoded i.e. listed
-	select_fields = get_select_fields(old, new)
-	update_link_field_values(select_fields, old, new, doctype)
-	update_select_field_values(old, new)
-
 	# change parenttype for fieldtype Table
 	update_parenttype_values(old, new)
 
@@ -409,7 +397,11 @@ def rename_doctype(doctype: str, old: str, new: str) -> None:
 def update_child_docs(old: str, new: str, meta: "Meta") -> None:
 	# update "parent"
 	for df in meta.get_table_fields():
-		frappe.qb.update(df.options).set("parent", new).where(Field("parent") == old).run()
+		(
+			frappe.qb.update(df.options)
+			.set("parent", new)
+			.where((Field("parent") == old) & (Field("parenttype") == meta.name))
+		).run()
 
 
 def update_link_field_values(link_fields: list[dict], old: str, new: str, doctype: str) -> None:
@@ -496,6 +488,9 @@ def update_options_for_fieldtype(fieldtype: str, old: str, new: str) -> None:
 
 	if frappe.conf.developer_mode:
 		for name in frappe.get_all("DocField", filters={"options": old}, pluck="parent"):
+			if name in (old, new):
+				continue
+
 			doctype = frappe.get_doc("DocType", name)
 			save = False
 			for f in doctype.fields:
@@ -504,11 +499,11 @@ def update_options_for_fieldtype(fieldtype: str, old: str, new: str) -> None:
 					save = True
 			if save:
 				doctype.save()
-	else:
-		DocField = frappe.qb.DocType("DocField")
-		frappe.qb.update(DocField).set(DocField.options, new).where(
-			(DocField.fieldtype == fieldtype) & (DocField.options == old)
-		).run()
+
+	DocField = frappe.qb.DocType("DocField")
+	frappe.qb.update(DocField).set(DocField.options, new).where(
+		(DocField.fieldtype == fieldtype) & (DocField.options == old)
+	).run()
 
 	frappe.qb.update(CustomField).set(CustomField.options, new).where(
 		(CustomField.fieldtype == fieldtype) & (CustomField.options == old)
@@ -534,7 +529,12 @@ def get_select_fields(old: str, new: str) -> list[dict]:
 	standard_fields = (
 		frappe.qb.from_(df)
 		.select(df.parent, df.fieldname, st_issingle)
-		.where((df.parent != new) & (df.fieldtype == "Select") & (df.options.like(f"%{old}%")))
+		.where(
+			(df.parent != new)
+			& (df.fieldname != "fieldtype")
+			& (df.fieldtype == "Select")
+			& (df.options.like(f"%{old}%"))
+		)
 		.run(as_dict=True)
 	)
 
diff --git a/frappe/model/sync.py b/frappe/model/sync.py
index df3999054a..6272c9cb7d 100644
--- a/frappe/model/sync.py
+++ b/frappe/model/sync.py
@@ -8,17 +8,17 @@ import os
 
 import frappe
 from frappe.modules.import_file import import_file_by_path
-from frappe.modules.patch_handler import block_user
+from frappe.modules.patch_handler import _patch_mode
 from frappe.utils import update_progress_bar
 
 
 def sync_all(force=0, reset_permissions=False):
-	block_user(True)
+	_patch_mode(True)
 
 	for app in frappe.get_installed_apps():
 		sync_for(app, force, reset_permissions=reset_permissions)
 
-	block_user(False)
+	_patch_mode(False)
 
 	frappe.clear_cache()
 
@@ -65,6 +65,8 @@ def sync_for(app_name, force=0, reset_permissions=False):
 			"workspace_chart",
 			"workspace_shortcut",
 			"workspace_quick_list",
+			"workspace_number_card",
+			"workspace_custom_block",
 			"workspace",
 		]:
 			files.append(os.path.join(FRAPPE_PATH, "desk", "doctype", desk_module, f"{desk_module}.json"))
diff --git a/frappe/model/utils/__init__.py b/frappe/model/utils/__init__.py
index 351b19c8eb..2935872fc7 100644
--- a/frappe/model/utils/__init__.py
+++ b/frappe/model/utils/__init__.py
@@ -1,12 +1,12 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
-import io
 import re
 
 import frappe
 from frappe import _
 from frappe.build import html_to_js_template
 from frappe.utils import cstr
+from frappe.utils.caching import site_cache
 
 STANDARD_FIELD_CONVERSION_MAP = {
 	"name": "Link",
@@ -90,12 +90,46 @@ def get_fetch_values(doctype, fieldname, value):
 	:param fieldname: Link fieldname selected
 	:param value: Value selected
 	"""
-	out = {}
-	meta = frappe.get_meta(doctype)
-	link_df = meta.get_field(fieldname)
-	for df in meta.get_fields_to_fetch(fieldname):
-		# example shipping_address.gistin
-		link_field, source_fieldname = df.fetch_from.split(".", 1)
-		out[df.fieldname] = frappe.db.get_value(link_df.options, value, source_fieldname)
 
-	return out
+	result = frappe._dict()
+	meta = frappe.get_meta(doctype)
+
+	# fieldname in target doctype: fieldname in source doctype
+	fields_to_fetch = {
+		df.fieldname: df.fetch_from.split(".", 1)[1] for df in meta.get_fields_to_fetch(fieldname)
+	}
+
+	# nothing to fetch
+	if not fields_to_fetch:
+		return result
+
+	# initialise empty values for target fields
+	for target_fieldname in fields_to_fetch:
+		result[target_fieldname] = None
+
+	# fetch only if Link field has a truthy value
+	if not value:
+		return result
+
+	db_values = frappe.db.get_value(
+		meta.get_options(fieldname),  # source doctype
+		value,
+		tuple(set(fields_to_fetch.values())),  # unique source fieldnames
+		as_dict=True,
+	)
+
+	# if value doesn't exist in source doctype, get_value returns None
+	if not db_values:
+		return result
+
+	for target_fieldname, source_fieldname in fields_to_fetch.items():
+		result[target_fieldname] = db_values.get(source_fieldname)
+
+	return result
+
+
+@site_cache()
+def is_virtual_doctype(doctype: str):
+	if frappe.db.has_column("DocType", "is_virtual"):
+		return frappe.db.get_value("DocType", doctype, "is_virtual")
+	return False
diff --git a/frappe/model/utils/rename_field.py b/frappe/model/utils/rename_field.py
index 9e4fc5d84a..c17d01183b 100644
--- a/frappe/model/utils/rename_field.py
+++ b/frappe/model/utils/rename_field.py
@@ -27,7 +27,7 @@ def rename_field(doctype, old_fieldname, new_fieldname):
 		frappe.db.sql(
 			"""update `tab%s` set parentfield=%s
 			where parentfield=%s"""
-			% (new_field.options.split("\n")[0], "%s", "%s"),
+			% (new_field.options.split("\n", 1)[0], "%s", "%s"),
 			(new_fieldname, old_fieldname),
 		)
 
diff --git a/frappe/model/virtual_doctype.py b/frappe/model/virtual_doctype.py
new file mode 100644
index 0000000000..6d8088ed80
--- /dev/null
+++ b/frappe/model/virtual_doctype.py
@@ -0,0 +1,95 @@
+import inspect
+from typing import Protocol, runtime_checkable
+
+import frappe
+from frappe import _
+from frappe.model.base_document import get_controller
+
+
+@runtime_checkable
+class VirtualDoctype(Protocol):
+	"""This class documents requirements that must be met by a doctype controller to function as virtual doctype
+
+
+	Additional requirements:
+	- DocType controller has to inherit from `frappe.model.document.Document` class
+
+	Note:
+	- "Backend" here means any storage service, it can be a database, flat file or network call to API.
+	"""
+
+	# ============ class/static methods ============
+
+	@staticmethod
+	def get_list(args) -> list[frappe._dict]:
+		"""Similar to reportview.get_list"""
+		...
+
+	@staticmethod
+	def get_count(args) -> int:
+		"""Similar to reportview.get_count, return total count of documents on listview."""
+		...
+
+	@staticmethod
+	def get_stats(args):
+		"""Similar to reportview.get_stats, return sidebar stats."""
+		...
+
+	# ============ instance methods ============
+
+	def db_insert(self, *args, **kwargs) -> None:
+		"""Serialize the `Document` object and insert it in backend."""
+		...
+
+	def load_from_db(self) -> None:
+		"""Using self.name initialize current document from backend data.
+
+		This is responsible for updatinng __dict__ of class with all the fields on doctype."""
+		...
+
+	def db_update(self, *args, **kwargs) -> None:
+		"""Serialize the `Document` object and update existing document in backend."""
+		...
+
+	def delete(self, *args, **kwargs) -> None:
+		"""Delete the current document from backend"""
+		...
+
+
+def validate_controller(doctype: str) -> None:
+	try:
+		controller = get_controller(doctype)
+	except ImportError:
+		frappe.msgprint(
+			_("Failed to import virtual doctype {}, is controller file present?").format(doctype)
+		)
+		return
+
+	def _as_str(method):
+		if hasattr(method, "__module__"):
+			return f"{method.__module__}.{method.__qualname__}"
+		return "None"
+
+	expected_static_method = ["get_list", "get_count", "get_stats"]
+	for m in expected_static_method:
+		method = inspect.getattr_static(controller, m, None)
+		if not isinstance(method, staticmethod):
+			frappe.msgprint(
+				_("Virtual DocType {} requires a static method called {} found {}").format(
+					frappe.bold(doctype), frappe.bold(m), frappe.bold(_as_str(method))
+				),
+				title=_("Incomplete Virtual Doctype Implementation"),
+			)
+
+	expected_instance_methods = ["db_insert", "db_update", "load_from_db", "delete"]
+	parent_class = controller.mro()[1]
+	for m in expected_instance_methods:
+		method = getattr(controller, m, None)
+		original_method = getattr(parent_class, m, None)
+		if method == original_method:
+			frappe.msgprint(
+				_("Virtual DocType {} requires overriding an instance method called {} found {}").format(
+					frappe.bold(doctype), frappe.bold(m), frappe.bold(_as_str(method))
+				),
+				title=_("Incomplete Virtual Doctype Implementation"),
+			)
diff --git a/frappe/model/workflow.py b/frappe/model/workflow.py
index 923fbc1b3b..8338157996 100644
--- a/frappe/model/workflow.py
+++ b/frappe/model/workflow.py
@@ -1,12 +1,17 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
 import json
+from typing import TYPE_CHECKING, Union
 
 import frappe
 from frappe import _
 from frappe.model.docstatus import DocStatus
 from frappe.utils import cint
 
+if TYPE_CHECKING:
+	from frappe.model.document import Document
+	from frappe.workflow.doctype.workflow.workflow import Workflow
+
 
 class WorkflowStateError(frappe.ValidationError):
 	pass
@@ -32,20 +37,22 @@ def get_workflow_name(doctype):
 
 
 @frappe.whitelist()
-def get_transitions(doc, workflow=None, raise_exception=False):
+def get_transitions(
+	doc: Union["Document", str, dict], workflow: "Workflow" = None, raise_exception: bool = False
+) -> list[dict]:
 	"""Return list of possible transitions for the given doc"""
-	doc = frappe.get_doc(frappe.parse_json(doc))
+	from frappe.model.document import Document
+
+	if not isinstance(doc, Document):
+		doc = frappe.get_doc(frappe.parse_json(doc))
+		doc.load_from_db()
 
 	if doc.is_new():
 		return []
 
-	doc.load_from_db()
+	doc.check_permission("read")
 
-	frappe.has_permission(doc, "read", throw=True)
-	roles = frappe.get_roles()
-
-	if not workflow:
-		workflow = get_workflow(doc.doctype)
+	workflow = workflow or get_workflow(doc.doctype)
 	current_state = doc.get(workflow.workflow_state_field)
 
 	if not current_state:
@@ -55,11 +62,14 @@ def get_transitions(doc, workflow=None, raise_exception=False):
 			frappe.throw(_("Workflow State not set"), WorkflowStateError)
 
 	transitions = []
+	roles = frappe.get_roles()
+
 	for transition in workflow.transitions:
 		if transition.state == current_state and transition.allowed in roles:
 			if not is_transition_condition_satisfied(transition, doc):
 				continue
 			transitions.append(transition.as_dict())
+
 	return transitions
 
 
@@ -79,7 +89,7 @@ def get_workflow_safe_globals():
 	)
 
 
-def is_transition_condition_satisfied(transition, doc):
+def is_transition_condition_satisfied(transition, doc) -> bool:
 	if not transition.condition:
 		return True
 	else:
@@ -198,7 +208,7 @@ def validate_workflow(doc):
 			)
 
 
-def get_workflow(doctype):
+def get_workflow(doctype) -> "Workflow":
 	return frappe.get_doc("Workflow", get_workflow_name(doctype))
 
 
diff --git a/frappe/modules.txt b/frappe/modules.txt
index fb7817f6ba..863c448594 100644
--- a/frappe/modules.txt
+++ b/frappe/modules.txt
@@ -9,5 +9,4 @@ Integrations
 Printing
 Contacts
 Social
-Automation
-Event Streaming
\ No newline at end of file
+Automation
\ No newline at end of file
diff --git a/frappe/modules/export_file.py b/frappe/modules/export_file.py
index e3a80d6679..edbf5ccaca 100644
--- a/frappe/modules/export_file.py
+++ b/frappe/modules/export_file.py
@@ -2,6 +2,7 @@
 # License: MIT. See LICENSE
 
 import os
+import shutil
 
 import frappe
 import frappe.model
@@ -47,8 +48,10 @@ def write_document_file(doc, record_module=None, create_init=True, folder_name=N
 	write_code_files(folder, fname, doc, doc_export)
 
 	# write the data file
-	with open(os.path.join(folder, fname + ".json"), "w+") as txtfile:
+	path = os.path.join(folder, f"{fname}.json")
+	with open(path, "w+") as txtfile:
 		txtfile.write(frappe.as_json(doc_export))
+	print(f"Wrote document file for {doc.doctype} {doc.name} at {path}")
 
 
 def strip_default_fields(doc, doc_export):
@@ -90,6 +93,21 @@ def get_module_name(doc):
 	return module
 
 
+def delete_folder(module, dt, dn):
+	if frappe.db.get_value("Module Def", module, "custom"):
+		module_path = get_custom_module_path(module)
+	else:
+		module_path = get_module_path(module)
+
+	dt, dn = scrub_dt_dn(dt, dn)
+
+	# delete folder
+	folder = os.path.join(module_path, dt, dn)
+
+	if os.path.exists(folder):
+		shutil.rmtree(folder)
+
+
 def create_folder(module, dt, dn, create_init):
 	if frappe.db.get_value("Module Def", module, "custom"):
 		module_path = get_custom_module_path(module)
diff --git a/frappe/modules/import_file.py b/frappe/modules/import_file.py
index 3690da0657..36e329409a 100644
--- a/frappe/modules/import_file.py
+++ b/frappe/modules/import_file.py
@@ -252,7 +252,7 @@ def load_code_properties(doc, path):
 		if hasattr(doc, "get_code_fields"):
 			dirname, filename = os.path.split(path)
 			for key, extn in doc.get_code_fields().items():
-				codefile = os.path.join(dirname, filename.split(".")[0] + "." + extn)
+				codefile = os.path.join(dirname, filename.split(".", 1)[0] + "." + extn)
 				if os.path.exists(codefile):
 					with open(codefile) as txtfile:
 						doc.set(key, txtfile.read())
diff --git a/frappe/modules/patch_handler.py b/frappe/modules/patch_handler.py
index f389312a4f..8b25ffcb8e 100644
--- a/frappe/modules/patch_handler.py
+++ b/frappe/modules/patch_handler.py
@@ -53,7 +53,7 @@ class PatchType(Enum):
 
 def run_all(skip_failing: bool = False, patch_type: PatchType | None = None) -> None:
 	"""run all pending patches"""
-	executed = set(frappe.get_all("Patch Log", fields="patch", pluck="patch"))
+	executed = set(frappe.get_all("Patch Log", filters={"skipped": 0}, fields="patch", pluck="patch"))
 
 	frappe.flags.final_patches = []
 
@@ -65,8 +65,9 @@ def run_all(skip_failing: bool = False, patch_type: PatchType | None = None) ->
 		except Exception:
 			if not skip_failing:
 				raise
-			else:
-				print("Failed to execute patch")
+
+			print("Failed to execute patch")
+			update_patch_log(patch, skipped=True)
 
 	patches = get_all_patches(patch_type=patch_type)
 
@@ -152,9 +153,9 @@ def run_single(patchmodule=None, method=None, methodargs=None, force=False):
 		return True
 
 
-def execute_patch(patchmodule, method=None, methodargs=None):
+def execute_patch(patchmodule: str, method=None, methodargs=None):
 	"""execute the patch"""
-	block_user(True)
+	_patch_mode(True)
 
 	if patchmodule.startswith("execute:"):
 		has_patch_file = False
@@ -162,7 +163,7 @@ def execute_patch(patchmodule, method=None, methodargs=None):
 		docstring = ""
 	else:
 		has_patch_file = True
-		patch = f"{patchmodule.split()[0]}.execute"
+		patch = f"{patchmodule.split(maxsplit=1)[0]}.execute"
 		_patch = frappe.get_attr(patch)
 		docstring = _patch.__doc__ or ""
 
@@ -173,8 +174,9 @@ def execute_patch(patchmodule, method=None, methodargs=None):
 		f"Executing {patchmodule or methodargs} in {frappe.local.site} ({frappe.db.cur_db_name}){docstring}"
 	)
 
-	start_time = time.time()
+	start_time = time.monotonic()
 	frappe.db.begin()
+	frappe.db.auto_commit_on_many_writes = 0
 	try:
 		if patchmodule:
 			if patchmodule.startswith("finally:"):
@@ -196,16 +198,25 @@ def execute_patch(patchmodule, method=None, methodargs=None):
 
 	else:
 		frappe.db.commit()
-		end_time = time.time()
-		block_user(False)
+		end_time = time.monotonic()
+		_patch_mode(False)
 		print(f"Success: Done in {round(end_time - start_time, 3)}s")
 
 	return True
 
 
-def update_patch_log(patchmodule):
+def update_patch_log(patchmodule, skipped=False):
 	"""update patch_file in patch log"""
-	frappe.get_doc({"doctype": "Patch Log", "patch": patchmodule}).insert(ignore_permissions=True)
+
+	patch = frappe.get_doc({"doctype": "Patch Log", "patch": patchmodule})
+
+	if skipped:
+		traceback = frappe.get_traceback(with_context=True)
+		patch.skipped = 1
+		patch.traceback = traceback
+		print(traceback, end="\n\n")
+
+	patch.insert(ignore_permissions=True)
 
 
 def executed(patchmodule):
@@ -213,21 +224,10 @@ def executed(patchmodule):
 	if patchmodule.startswith("finally:"):
 		# patches are saved without the finally: tag
 		patchmodule = patchmodule.replace("finally:", "")
-	return frappe.db.get_value("Patch Log", {"patch": patchmodule})
+	return frappe.db.get_value("Patch Log", {"patch": patchmodule, "skipped": 0})
 
 
-def block_user(block, msg=None):
+def _patch_mode(enable):
 	"""stop/start execution till patch is run"""
-	frappe.local.flags.in_patch = block
-	frappe.db.begin()
-	if not msg:
-		msg = "Patches are being executed in the system. Please try again in a few moments."
-	frappe.db.set_global("__session_status", block and "stop" or None)
-	frappe.db.set_global("__session_status_message", block and msg or None)
+	frappe.local.flags.in_patch = enable
 	frappe.db.commit()
-
-
-def check_session_stopped():
-	if frappe.db.get_global("__session_status") == "stop":
-		frappe.msgprint(frappe.db.get_global("__session_status_message"))
-		raise frappe.SessionStopped("Session Stopped")
diff --git a/frappe/modules/utils.py b/frappe/modules/utils.py
index f4a386cfc9..57d3e8f7ad 100644
--- a/frappe/modules/utils.py
+++ b/frappe/modules/utils.py
@@ -1,21 +1,35 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
+# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
 """
 	Utilities for using modules
 """
 import json
 import os
+from textwrap import dedent, indent
+from typing import TYPE_CHECKING, Union
 
 import frappe
-import frappe.utils
-from frappe import _
-from frappe.utils import cint
+from frappe import _, get_module_path, scrub
+from frappe.utils import cint, cstr, now_datetime
+
+if TYPE_CHECKING:
+	from types import ModuleType
+
+	from frappe.model.document import Document
 
 
-def export_module_json(doc, is_standard, module):
+doctype_python_modules = {}
+
+
+def export_module_json(doc: "Document", is_standard: bool, module: str) -> str | None:
 	"""Make a folder for the given doc and add its json file (make it a standard
-	object that will be synced)"""
-	if not frappe.flags.in_import and getattr(frappe.get_conf(), "developer_mode", 0) and is_standard:
+	object that will be synced)
+
+	Returns the absolute file_path without the extension.
+	Eg: For exporting a Print Format "_Test Print Format 1", the return value will be
+	`/home/gavin/frappe-bench/apps/frappe/frappe/core/print_format/_test_print_format_1/_test_print_format_1`
+	"""
+	if not frappe.flags.in_import and is_standard and frappe.conf.developer_mode:
 		from frappe.modules.export_file import export_to_files
 
 		# json
@@ -23,14 +37,12 @@ def export_module_json(doc, is_standard, module):
 			record_list=[[doc.doctype, doc.name]], record_module=module, create_init=is_standard
 		)
 
-		path = os.path.join(
+		return os.path.join(
 			frappe.get_module_path(module), scrub(doc.doctype), scrub(doc.name), scrub(doc.name)
 		)
 
-		return path
 
-
-def get_doc_module(module, doctype, name):
+def get_doc_module(module: str, doctype: str, name: str) -> "ModuleType":
 	"""Get custom module for given document"""
 	module_name = "{app}.{module}.{doctype}.{name}.{name}".format(
 		app=frappe.local.module_app[scrub(module)],
@@ -42,34 +54,27 @@ def get_doc_module(module, doctype, name):
 
 
 @frappe.whitelist()
-def export_customizations(module, doctype, sync_on_migrate=0, with_permissions=0):
+def export_customizations(
+	module: str, doctype: str, sync_on_migrate: bool = False, with_permissions: bool = False
+):
 	"""Export Custom Field and Property Setter for the current document to the app folder.
 	This will be synced with bench migrate"""
 
 	sync_on_migrate = cint(sync_on_migrate)
 	with_permissions = cint(with_permissions)
 
-	if not frappe.get_conf().developer_mode:
-		raise Exception("Not developer mode")
+	if not frappe.conf.developer_mode:
+		frappe.throw(_("Only allowed to export customizations in developer mode"))
 
 	custom = {
-		"custom_fields": [],
-		"property_setters": [],
+		"custom_fields": frappe.get_all("Custom Field", fields="*", filters={"dt": doctype}),
+		"property_setters": frappe.get_all("Property Setter", fields="*", filters={"doc_type": doctype}),
 		"custom_perms": [],
-		"links": [],
+		"links": frappe.get_all("DocType Link", fields="*", filters={"parent": doctype}),
 		"doctype": doctype,
 		"sync_on_migrate": sync_on_migrate,
 	}
 
-	def add(_doctype):
-		custom["custom_fields"] += frappe.get_all("Custom Field", fields="*", filters={"dt": _doctype})
-		custom["property_setters"] += frappe.get_all(
-			"Property Setter", fields="*", filters={"doc_type": _doctype}
-		)
-		custom["links"] += frappe.get_all("DocType Link", fields="*", filters={"parent": _doctype})
-
-	add(doctype)
-
 	if with_permissions:
 		custom["custom_perms"] = frappe.get_all(
 			"Custom DocPerm", fields="*", filters={"parent": doctype}
@@ -89,6 +94,7 @@ def export_customizations(module, doctype, sync_on_migrate=0, with_permissions=0
 			f.write(frappe.as_json(custom))
 
 		frappe.msgprint(_("Customizations for {0} exported to:
        {1}").format(doctype, path))
+		return path
 
 
 def sync_customizations(app=None):
@@ -108,10 +114,10 @@ def sync_customizations(app=None):
 						with open(os.path.join(folder, fname)) as f:
 							data = json.loads(f.read())
 						if data.get("sync_on_migrate"):
-							sync_customizations_for_doctype(data, folder)
+							sync_customizations_for_doctype(data, folder, fname)
 
 
-def sync_customizations_for_doctype(data, folder):
+def sync_customizations_for_doctype(data: dict, folder: str, filename: str = ""):
 	"""Sync doctype customzations for a particular data set"""
 	from frappe.core.doctype.doctype.doctype import validate_fields_for_doctype
 
@@ -149,11 +155,14 @@ def sync_customizations_for_doctype(data, folder):
 
 		for doc_type in doctypes:
 			# only sync the parent doctype and child doctype if there isn't any other child table json file
-			if doc_type == doctype or not os.path.exists(
-				os.path.join(folder, frappe.scrub(doc_type) + ".json")
-			):
+			if doc_type == doctype or not os.path.exists(os.path.join(folder, scrub(doc_type) + ".json")):
 				sync_single_doctype(doc_type)
 
+	if not frappe.db.exists("DocType", doctype):
+		print(_("DocType {0} does not exist.").format(doctype))
+		print(_("Skipping fixture syncing for doctype {0} from file {1}").format(doctype, filename))
+		return
+
 	if data["custom_fields"]:
 		sync("custom_fields", "Custom Field", "dt")
 		update_schema = True
@@ -161,36 +170,34 @@ def sync_customizations_for_doctype(data, folder):
 	if data["property_setters"]:
 		sync("property_setters", "Property Setter", "doc_type")
 
+	print(f"Updating customizations for {doctype}")
 	if data.get("custom_perms"):
 		sync("custom_perms", "Custom DocPerm", "parent")
 
-	print(f"Updating customizations for {doctype}")
 	validate_fields_for_doctype(doctype)
 
 	if update_schema and not frappe.db.get_value("DocType", doctype, "issingle"):
 		frappe.db.updatedb(doctype)
 
 
-def scrub(txt):
-	return frappe.scrub(txt)
-
-
-def scrub_dt_dn(dt, dn):
+def scrub_dt_dn(dt: str, dn: str) -> tuple[str, str]:
 	"""Returns in lowercase and code friendly names of doctype and name for certain types"""
 	return scrub(dt), scrub(dn)
 
 
-def get_module_path(module):
-	"""Returns path of the given module"""
-	return frappe.get_module_path(module)
+def get_doc_path(module: str, doctype: str, name: str) -> str:
+	"""Returns path of a doc in a module"""
+	return os.path.join(get_module_path(module), *scrub_dt_dn(doctype, name))
 
 
-def get_doc_path(module, doctype, name):
-	dt, dn = scrub_dt_dn(doctype, name)
-	return os.path.join(get_module_path(module), dt, dn)
-
-
-def reload_doc(module, dt=None, dn=None, force=False, reset_permissions=False):
+def reload_doc(
+	module: str,
+	dt: str = None,
+	dn: str = None,
+	force: bool = False,
+	reset_permissions: bool = False,
+):
+	"""Reload Document from model (`[module]//[name]/[name].json`) files"""
 	from frappe.modules.import_file import import_files
 
 	return import_files(module, dt, dn, force=force, reset_permissions=reset_permissions)
@@ -200,20 +207,17 @@ def export_doc(doctype, name, module=None):
 	"""Write a doc to standard path."""
 	from frappe.modules.export_file import write_document_file
 
-	print(doctype, name)
-
-	if not module:
-		module = frappe.db.get_value("DocType", name, "module")
+	print(f"Exporting Document {doctype} {name}")
+	module = module or frappe.db.get_value("DocType", name, "module")
 	write_document_file(frappe.get_doc(doctype, name), module)
 
 
-def get_doctype_module(doctype) -> str:
+def get_doctype_module(doctype: str) -> str:
 	"""Returns **Module Def** name of given doctype."""
-
-	def make_modules_dict():
-		return dict(frappe.db.sql("select name, module from tabDocType"))
-
-	doctype_module_map = frappe.cache().get_value("doctype_modules", make_modules_dict)
+	doctype_module_map = frappe.cache().get_value(
+		"doctype_modules",
+		generator=lambda: dict(frappe.qb.from_("DocType").select("name", "module").run()),
+	)
 
 	if module_name := doctype_module_map.get(doctype):
 		return module_name
@@ -221,39 +225,35 @@ def get_doctype_module(doctype) -> str:
 		frappe.throw(_("DocType {} not found").format(doctype), exc=frappe.DoesNotExistError)
 
 
-doctype_python_modules = {}
-
-
 def load_doctype_module(doctype, module=None, prefix="", suffix=""):
-	"""Returns the module object for given doctype."""
-	if not module:
-		module = get_doctype_module(doctype)
+	"""Returns the module object for given doctype.
 
+	Note: This will return the standard defined module object for the doctype irrespective
+	of the `override_doctype_class` hook.
+	"""
+	module = module or get_doctype_module(doctype)
 	app = get_module_app(module)
-
 	key = (app, doctype, prefix, suffix)
-
 	module_name = get_module_name(doctype, module, prefix, suffix)
 
-	try:
-		if key not in doctype_python_modules:
+	if key not in doctype_python_modules:
+		try:
 			doctype_python_modules[key] = frappe.get_module(module_name)
-	except ImportError as e:
-		msg = f"Module import failed for {doctype}, the DocType you're trying to open might be deleted."
-		msg += f"
         Error: {e}"
-		raise ImportError(msg) from e
+		except ImportError as e:
+			msg = f"Module import failed for {doctype}, the DocType you're trying to open might be deleted."
+			msg += f"
         Error: {e}"
+			raise ImportError(msg) from e
 
 	return doctype_python_modules[key]
 
 
-def get_module_name(doctype, module, prefix="", suffix="", app=None):
-	return "{app}.{module}.doctype.{doctype}.{prefix}{doctype}{suffix}".format(
-		app=scrub(app or get_module_app(module)),
-		module=scrub(module),
-		doctype=scrub(doctype),
-		prefix=prefix,
-		suffix=suffix,
-	)
+def get_module_name(
+	doctype: str, module: str, prefix: str = "", suffix: str = "", app: str | None = None
+):
+	app = scrub(app or get_module_app(module))
+	module = scrub(module)
+	doctype = scrub(doctype)
+	return f"{app}.{module}.doctype.{doctype}.{prefix}{doctype}{suffix}"
 
 
 def get_module_app(module: str) -> str:
@@ -266,69 +266,76 @@ def get_module_app(module: str) -> str:
 def get_app_publisher(module: str) -> str:
 	app = get_module_app(module)
 	if not app:
-		frappe.throw(_("App not found"))
-	app_publisher = frappe.get_hooks(hook="app_publisher", app_name=app)[0]
-	return app_publisher
+		frappe.throw(_("App not found for module: {0}").format(module))
+	return frappe.get_hooks(hook="app_publisher", app_name=app)[0]
 
 
-def make_boilerplate(template, doc, opts=None):
+def make_boilerplate(
+	template: str, doc: Union["Document", "frappe._dict"], opts: Union[dict, "frappe._dict"] = None
+):
 	target_path = get_doc_path(doc.module, doc.doctype, doc.name)
 	template_name = template.replace("controller", scrub(doc.name))
 	if template_name.endswith("._py"):
 		template_name = template_name[:-4] + ".py"
 	target_file_path = os.path.join(target_path, template_name)
+	template_file_path = os.path.join(
+		get_module_path("core"), "doctype", scrub(doc.doctype), "boilerplate", template
+	)
 
-	if not doc:
-		doc = {}
+	if os.path.exists(target_file_path):
+		print(f"{target_file_path} already exists, skipping...")
+		return
 
+	doc = doc or frappe._dict()
+	opts = opts or frappe._dict()
 	app_publisher = get_app_publisher(doc.module)
+	base_class = "Document"
+	base_class_import = "from frappe.model.document import Document"
+	controller_body = "pass"
 
-	if not os.path.exists(target_file_path):
-		if not opts:
-			opts = {}
+	if doc.get("is_tree"):
+		base_class = "NestedSet"
+		base_class_import = "from frappe.utils.nestedset import NestedSet"
 
-		base_class = "Document"
-		base_class_import = "from frappe.model.document import Document"
-		if doc.get("is_tree"):
-			base_class = "NestedSet"
-			base_class_import = "from frappe.utils.nestedset import NestedSet"
+	if doc.get("is_virtual"):
+		controller_body = indent(
+			dedent(
+				"""
+			def db_insert(self, *args, **kwargs):
+				pass
 
-		custom_controller = "pass"
-		if doc.get("is_virtual"):
-			custom_controller = """
-	def db_insert(self):
-		pass
+			def load_from_db(self):
+				pass
 
-	def load_from_db(self):
-		pass
+			def db_update(self):
+				pass
 
-	def db_update(self):
-		pass
+			@staticmethod
+			def get_list(args):
+				pass
 
-	def get_list(self, args):
-		pass
+			@staticmethod
+			def get_count(args):
+				pass
 
-	def get_count(self, args):
-		pass
+			@staticmethod
+			def get_stats(args):
+				pass
+			"""
+			),
+			"\t",
+		)
 
-	def get_stats(self, args):
-		pass"""
-
-		with open(target_file_path, "w") as target:
-			with open(
-				os.path.join(get_module_path("core"), "doctype", scrub(doc.doctype), "boilerplate", template),
-			) as source:
-				target.write(
-					frappe.as_unicode(
-						frappe.utils.cstr(source.read()).format(
-							app_publisher=app_publisher,
-							year=frappe.utils.nowdate()[:4],
-							classname=doc.name.replace(" ", "").replace("-", ""),
-							base_class_import=base_class_import,
-							base_class=base_class,
-							doctype=doc.name,
-							**opts,
-							custom_controller=custom_controller,
-						)
-					)
-				)
+	with open(target_file_path, "w") as target, open(template_file_path) as source:
+		template = source.read()
+		controller_file_content = cstr(template).format(
+			app_publisher=app_publisher,
+			year=now_datetime().year,
+			classname=doc.name.replace(" ", "").replace("-", ""),
+			base_class_import=base_class_import,
+			base_class=base_class,
+			doctype=doc.name,
+			**opts,
+			custom_controller=controller_body,
+		)
+		target.write(frappe.as_unicode(controller_file_content))
diff --git a/frappe/monitor.py b/frappe/monitor.py
index 8d5391cb77..b93ba1d3bb 100644
--- a/frappe/monitor.py
+++ b/frappe/monitor.py
@@ -25,6 +25,13 @@ def stop(response=None):
 		frappe.local.monitor.dump(response)
 
 
+def add_data_to_monitor(**kwargs) -> None:
+	"""Add additional custom key-value pairs along with monitor log.
+	Note: Key-value pairs should be simple JSON exportable types."""
+	if hasattr(frappe.local, "monitor"):
+		frappe.local.monitor.add_custom_data(**kwargs)
+
+
 def log_file():
 	return os.path.join(frappe.utils.get_bench_path(), "logs", "monitor.json.log")
 
@@ -71,6 +78,10 @@ class Monitor:
 			waitdiff = self.data.timestamp - job.enqueued_at
 			self.data.job.wait = int(waitdiff.total_seconds() * 1000000)
 
+	def add_custom_data(self, **kwargs):
+		if self.data:
+			self.data.update(kwargs)
+
 	def dump(self, response=None):
 		try:
 			timediff = datetime.utcnow() - self.data.timestamp
@@ -78,8 +89,11 @@ class Monitor:
 			self.data.duration = int(timediff.total_seconds() * 1000000)
 
 			if self.data.transaction_type == "request":
-				self.data.request.status_code = response.status_code
-				self.data.request.response_length = int(response.headers.get("Content-Length", 0))
+				if response:
+					self.data.request.status_code = response.status_code
+					self.data.request.response_length = int(response.headers.get("Content-Length", 0))
+				else:
+					self.data.request.status_code = 500
 
 				if hasattr(frappe.local, "rate_limiter"):
 					limiter = frappe.local.rate_limiter
@@ -94,7 +108,7 @@ class Monitor:
 	def store(self):
 		if frappe.cache().llen(MONITOR_REDIS_KEY) > MONITOR_MAX_ENTRIES:
 			frappe.cache().ltrim(MONITOR_REDIS_KEY, 1, -1)
-		serialized = json.dumps(self.data, sort_keys=True, default=str)
+		serialized = json.dumps(self.data, sort_keys=True, default=str, separators=(",", ":"))
 		frappe.cache().rpush(MONITOR_REDIS_KEY, serialized)
 
 
diff --git a/frappe/oauth.py b/frappe/oauth.py
index 68e21ac88b..aa486fe8ba 100644
--- a/frappe/oauth.py
+++ b/frappe/oauth.py
@@ -3,7 +3,7 @@ import datetime
 import hashlib
 import re
 from http import cookies
-from urllib.parse import unquote, urlparse
+from urllib.parse import unquote, urljoin, urlparse
 
 import jwt
 import pytz
@@ -11,6 +11,7 @@ from oauthlib.openid import RequestValidator
 
 import frappe
 from frappe.auth import LoginManager
+from frappe.utils.data import get_system_timezone
 
 
 class OAuthWebRequestValidator(RequestValidator):
@@ -248,7 +249,7 @@ class OAuthWebRequestValidator(RequestValidator):
 		# Remember to check expiration and scope membership
 		otoken = frappe.get_doc("OAuth Bearer Token", token)
 		token_expiration_local = otoken.expiration_time.replace(
-			tzinfo=pytz.timezone(frappe.utils.get_time_zone())
+			tzinfo=pytz.timezone(get_system_timezone())
 		)
 		token_expiration_utc = token_expiration_local.astimezone(pytz.utc)
 		is_token_valid = (
@@ -330,6 +331,8 @@ class OAuthWebRequestValidator(RequestValidator):
 
 		userinfo = get_userinfo(user)
 
+		id_token["exp"] = id_token.get("iat") + token.get("expires_in")
+
 		if userinfo.get("iss"):
 			id_token["iss"] = userinfo.get("iss")
 
@@ -362,6 +365,7 @@ class OAuthWebRequestValidator(RequestValidator):
 
 	def get_jwt_bearer_token(self, token, token_handler, request):
 		now = datetime.datetime.now()
+
 		id_token = dict(
 			aud=token.client_id,
 			iat=round(now.timestamp()),
@@ -574,7 +578,7 @@ def get_userinfo(user):
 		if frappe.utils.validate_url(user.user_image, valid_schemes=valid_url_schemes):
 			picture = user.user_image
 		else:
-			picture = frappe_server_url + "/" + user.user_image
+			picture = urljoin(frappe_server_url, user.user_image)
 
 	userinfo = frappe._dict(
 		{
diff --git a/frappe/parallel_test_runner.py b/frappe/parallel_test_runner.py
index 39a00235cb..b7c3966df1 100644
--- a/frappe/parallel_test_runner.py
+++ b/frappe/parallel_test_runner.py
@@ -18,11 +18,12 @@ if click_ctx:
 
 
 class ParallelTestRunner:
-	def __init__(self, app, site, build_number=1, total_builds=1):
+	def __init__(self, app, site, build_number=1, total_builds=1, dry_run=False):
 		self.app = app
 		self.site = site
 		self.build_number = frappe.utils.cint(build_number) or 1
 		self.total_builds = frappe.utils.cint(total_builds)
+		self.dry_run = dry_run
 		self.setup_test_site()
 		self.run_tests()
 
@@ -31,6 +32,9 @@ class ParallelTestRunner:
 		if not frappe.db:
 			frappe.connect()
 
+		if self.dry_run:
+			return
+
 		frappe.flags.in_test = True
 		frappe.clear_cache()
 		frappe.utils.scheduler.disable_scheduler()
@@ -38,7 +42,7 @@ class ParallelTestRunner:
 		self.before_test_setup()
 
 	def before_test_setup(self):
-		start_time = time.time()
+		start_time = time.monotonic()
 		for fn in frappe.get_hooks("before_tests", app_name=self.app):
 			frappe.get_attr(fn)()
 
@@ -48,7 +52,7 @@ class ParallelTestRunner:
 			for doctype in test_module.global_test_dependencies:
 				make_test_records(doctype, commit=True)
 
-		elapsed = time.time() - start_time
+		elapsed = time.monotonic() - start_time
 		elapsed = click.style(f" ({elapsed:.03}s)", fg="red")
 		click.echo(f"Before Test {elapsed}")
 
@@ -64,6 +68,10 @@ class ParallelTestRunner:
 		if not file_info:
 			return
 
+		if self.dry_run:
+			print("running tests from", "/".join(file_info))
+			return
+
 		frappe.set_user("Administrator")
 		path, filename = file_info
 		module = self.get_module(path, filename)
@@ -108,17 +116,53 @@ class ParallelTestRunner:
 				sys.exit(1)
 
 	def get_test_file_list(self):
+		# Load balance based on total # of tests ~ each runner should get roughly same # of tests.
 		test_list = get_all_tests(self.app)
-		split_size = frappe.utils.ceil(len(test_list) / self.total_builds)
-		# [1,2,3,4,5,6] to [[1,2], [3,4], [4,6]] if split_size is 2
-		test_chunks = [test_list[x : x + split_size] for x in range(0, len(test_list), split_size)]
+
+		test_counts = [self.get_test_count(test) for test in test_list]
+		test_chunks = split_by_weight(test_list, test_counts, chunk_count=self.total_builds)
+
 		return test_chunks[self.build_number - 1]
 
+	@staticmethod
+	def get_test_count(test):
+		"""Get approximate count of tests inside a file"""
+		file_name = "/".join(test)
+
+		with open(file_name) as f:
+			test_count = f.read().count("def test_")
+
+		return test_count
+
+
+def split_by_weight(work, weights, chunk_count):
+	"""Roughly split work by respective weight while keep ordering."""
+	expected_weight = sum(weights) // chunk_count
+
+	chunks = [[] for _ in range(chunk_count)]
+
+	chunk_no = 0
+	chunk_weight = 0
+
+	for task, weight in zip(work, weights):
+		if chunk_weight > expected_weight:
+			chunk_weight = 0
+			chunk_no += 1
+			assert chunk_no < chunk_count
+
+		chunks[chunk_no].append(task)
+		chunk_weight += weight
+
+	assert len(work) == sum(len(chunk) for chunk in chunks)
+	assert len(chunks) == chunk_count
+
+	return chunks
+
 
 class ParallelTestResult(unittest.TextTestResult):
 	def startTest(self, test):
 		self.tb_locals = True
-		self._started_at = time.time()
+		self._started_at = time.monotonic()
 		super(unittest.TextTestResult, self).startTest(test)
 		test_class = unittest.util.strclass(test.__class__)
 		if not hasattr(self, "current_test_class") or self.current_test_class != test_class:
@@ -130,7 +174,7 @@ class ParallelTestResult(unittest.TextTestResult):
 
 	def addSuccess(self, test):
 		super(unittest.TextTestResult, self).addSuccess(test)
-		elapsed = time.time() - self._started_at
+		elapsed = time.monotonic() - self._started_at
 		threshold_passed = elapsed >= SLOW_TEST_THRESHOLD
 		elapsed = click.style(f" ({elapsed:.03}s)", fg="red") if threshold_passed else ""
 		click.echo(f"  {click.style(' ✔ ', fg='green')} {self.getTestMethodName(test)}{elapsed}")
diff --git a/frappe/patches.txt b/frappe/patches.txt
index 425468f06c..fa9d884386 100644
--- a/frappe/patches.txt
+++ b/frappe/patches.txt
@@ -66,14 +66,12 @@ execute:frappe.delete_doc_if_exists('Page', 'user-permissions')
 frappe.patches.v10_0.set_no_copy_to_workflow_state
 frappe.patches.v10_0.increase_single_table_column_length
 frappe.patches.v11_0.create_contact_for_user
-frappe.patches.v11_0.sync_stripe_settings_before_migrate
 frappe.patches.v11_0.update_list_user_settings
 frappe.patches.v11_0.rename_workflow_action_to_workflow_action_master #13-06-2018
 frappe.patches.v11_0.rename_email_alert_to_notification #13-06-2018
 frappe.patches.v11_0.delete_duplicate_user_permissions
 frappe.patches.v11_0.set_dropbox_file_backup
 frappe.patches.v10_0.set_default_locking_time
-frappe.patches.v11_0.rename_google_maps_doctype
 frappe.patches.v10_0.modify_smallest_currency_fraction
 frappe.patches.v10_0.modify_naming_series_table
 frappe.patches.v10_0.enhance_security
@@ -101,7 +99,7 @@ frappe.patches.v12_0.delete_feedback_request_if_exists #1
 frappe.patches.v12_0.rename_events_repeat_on
 frappe.patches.v12_0.fix_public_private_files
 frappe.patches.v12_0.move_email_and_phone_to_child_table
-frappe.patches.v12_0.delete_duplicate_indexes
+frappe.patches.v12_0.delete_duplicate_indexes  # 2022-12-15
 frappe.patches.v12_0.set_default_incoming_email_port
 frappe.patches.v12_0.update_global_search
 frappe.patches.v12_0.setup_tags
@@ -158,21 +156,18 @@ frappe.patches.v13_0.set_route_for_blog_category
 frappe.patches.v13_0.enable_custom_script
 frappe.patches.v13_0.update_newsletter_content_type
 execute:frappe.db.set_value('Website Settings', 'Website Settings', {'navbar_template': 'Standard Navbar', 'footer_template': 'Standard Footer'})
-frappe.patches.v13_0.delete_event_producer_and_consumer_keys
 frappe.patches.v13_0.web_template_set_module #2020-10-05
 frappe.patches.v13_0.remove_custom_link
 execute:frappe.delete_doc("DocType", "Footer Item")
 execute:frappe.reload_doctype('user')
 execute:frappe.reload_doctype('docperm')
 frappe.patches.v13_0.replace_field_target_with_open_in_new_tab
-frappe.core.doctype.role.patches.v13_set_default_desk_properties
 frappe.patches.v13_0.add_switch_theme_to_navbar_settings
 frappe.patches.v13_0.update_icons_in_customized_desk_pages
 execute:frappe.db.set_default('desktop:home_page', 'space')
 execute:frappe.delete_doc_if_exists('Page', 'workspace')
 execute:frappe.delete_doc_if_exists('Page', 'dashboard', force=1)
 frappe.core.doctype.page.patches.drop_unused_pages
-execute:frappe.get_doc('Role', 'Guest').save() # remove desk access
 frappe.patches.v13_0.remove_chat
 frappe.patches.v13_0.rename_desk_page_to_workspace # 02.02.2021
 frappe.patches.v13_0.delete_package_publish_tool
@@ -183,7 +178,11 @@ frappe.patches.v13_0.queryreport_columns
 frappe.patches.v13_0.jinja_hook
 frappe.patches.v13_0.update_notification_channel_if_empty
 frappe.patches.v13_0.set_first_day_of_the_week
+frappe.patches.v13_0.encrypt_2fa_secrets
+frappe.patches.v13_0.reset_corrupt_defaults
+frappe.patches.v13_0.remove_share_for_std_users
 execute:frappe.reload_doc('custom', 'doctype', 'custom_field')
+frappe.email.doctype.email_queue.patches.drop_search_index_on_message_id
 frappe.patches.v14_0.update_workspace2 # 20.09.2021
 frappe.patches.v14_0.save_ratings_in_fraction #23-12-2021
 frappe.patches.v14_0.transform_todo_schema
@@ -192,8 +191,15 @@ frappe.patches.v14_0.reset_creation_datetime
 frappe.patches.v14_0.remove_is_first_startup
 frappe.patches.v14_0.clear_long_pending_stale_logs
 frappe.patches.v14_0.log_settings_migration
+frappe.patches.v14_0.setup_likes_from_feedback
+frappe.patches.v14_0.update_webforms
+frappe.patches.v14_0.delete_payment_gateways
+frappe.patches.v15_0.remove_event_streaming
+frappe.patches.v15_0.copy_disable_prepared_report_to_prepared_report
 
 [post_model_sync]
+execute:frappe.get_doc('Role', 'Guest').save() # remove desk access
+frappe.core.doctype.role.patches.v13_set_default_desk_properties
 frappe.patches.v14_0.drop_data_import_legacy
 frappe.patches.v14_0.copy_mail_data #08.03.21
 frappe.patches.v14_0.update_github_endpoints #08-11-2021
@@ -204,3 +210,17 @@ frappe.patches.v14_0.update_auto_account_deletion_duration
 frappe.patches.v14_0.update_integration_request
 frappe.patches.v14_0.set_document_expiry_default
 frappe.patches.v14_0.delete_data_migration_tool
+frappe.patches.v14_0.set_suspend_email_queue_default
+frappe.patches.v14_0.different_encryption_key
+frappe.patches.v14_0.update_multistep_webforms
+execute:frappe.delete_doc('Page', 'background_jobs', ignore_missing=True, force=True)
+frappe.patches.v14_0.drop_unused_indexes
+frappe.patches.v15_0.drop_modified_index
+frappe.patches.v14_0.update_attachment_comment
+frappe.patches.v15_0.set_contact_full_name
+execute:frappe.delete_doc("Page", "activity", force=1)
+frappe.patches.v14_0.disable_email_accounts_with_oauth
+execute:frappe.delete_doc("Page", "translation-tool", force=1)
+frappe.patches.v15_0.remove_prepared_report_settings_from_system_settings
+frappe.patches.v14_0.remove_manage_subscriptions_from_navbar
+frappe.patches.v15_0.remove_background_jobs_from_dropdown
\ No newline at end of file
diff --git a/frappe/patches/v10_0/set_default_locking_time.py b/frappe/patches/v10_0/set_default_locking_time.py
index 7c9f3ceff1..4eb120873d 100644
--- a/frappe/patches/v10_0/set_default_locking_time.py
+++ b/frappe/patches/v10_0/set_default_locking_time.py
@@ -6,4 +6,4 @@ import frappe
 
 def execute():
 	frappe.reload_doc("core", "doctype", "system_settings")
-	frappe.db.set_value("System Settings", None, "allow_login_after_fail", 60)
+	frappe.db.set_single_value("System Settings", "allow_login_after_fail", 60)
diff --git a/frappe/patches/v11_0/remove_skip_for_doctype.py b/frappe/patches/v11_0/remove_skip_for_doctype.py
index e7c1d71a0a..b3471ca4e8 100644
--- a/frappe/patches/v11_0/remove_skip_for_doctype.py
+++ b/frappe/patches/v11_0/remove_skip_for_doctype.py
@@ -60,7 +60,7 @@ def execute():
 					# Maintain sequence (name, user, allow, for_value, applicable_for, apply_to_all_doctypes, creation, modified)
 					new_user_permissions_list.append(
 						(
-							frappe.generate_hash("", 10),
+							frappe.generate_hash(length=10),
 							user_permission.user,
 							user_permission.allow,
 							user_permission.for_value,
diff --git a/frappe/patches/v11_0/rename_google_maps_doctype.py b/frappe/patches/v11_0/rename_google_maps_doctype.py
deleted file mode 100644
index 354c19a7ed..0000000000
--- a/frappe/patches/v11_0/rename_google_maps_doctype.py
+++ /dev/null
@@ -1,9 +0,0 @@
-import frappe
-from frappe.model.rename_doc import rename_doc
-
-
-def execute():
-	if frappe.db.exists("DocType", "Google Maps") and not frappe.db.exists(
-		"DocType", "Google Maps Settings"
-	):
-		rename_doc("DocType", "Google Maps", "Google Maps Settings")
diff --git a/frappe/patches/v11_0/set_dropbox_file_backup.py b/frappe/patches/v11_0/set_dropbox_file_backup.py
index c9dec31414..5770c55bf4 100644
--- a/frappe/patches/v11_0/set_dropbox_file_backup.py
+++ b/frappe/patches/v11_0/set_dropbox_file_backup.py
@@ -4,6 +4,6 @@ from frappe.utils import cint
 
 def execute():
 	frappe.reload_doctype("Dropbox Settings")
-	check_dropbox_enabled = cint(frappe.db.get_value("Dropbox Settings", None, "enabled"))
+	check_dropbox_enabled = cint(frappe.db.get_single_value("Dropbox Settings", "enabled"))
 	if check_dropbox_enabled == 1:
-		frappe.db.set_value("Dropbox Settings", None, "file_backup", 1)
+		frappe.db.set_single_value("Dropbox Settings", "file_backup", 1)
diff --git a/frappe/patches/v11_0/sync_stripe_settings_before_migrate.py b/frappe/patches/v11_0/sync_stripe_settings_before_migrate.py
deleted file mode 100644
index 019ecef67c..0000000000
--- a/frappe/patches/v11_0/sync_stripe_settings_before_migrate.py
+++ /dev/null
@@ -1,25 +0,0 @@
-import frappe
-from frappe.utils.password import get_decrypted_password
-
-
-def execute():
-	publishable_key = frappe.db.sql(
-		"select value from tabSingles where doctype='Stripe Settings' and field='publishable_key'"
-	)
-	if publishable_key:
-		secret_key = get_decrypted_password(
-			"Stripe Settings", "Stripe Settings", fieldname="secret_key", raise_exception=False
-		)
-		if secret_key:
-			frappe.reload_doc("integrations", "doctype", "stripe_settings")
-			frappe.db.commit()
-
-			settings = frappe.new_doc("Stripe Settings")
-			settings.gateway_name = (
-				frappe.db.get_value("Global Defaults", None, "default_company") or "Stripe Settings"
-			)
-			settings.publishable_key = publishable_key
-			settings.secret_key = secret_key
-			settings.save(ignore_permissions=True)
-
-	frappe.db.delete("Singles", {"doctype": "Stripe Settings"})
diff --git a/frappe/patches/v12_0/change_existing_dashboard_chart_filters.py b/frappe/patches/v12_0/change_existing_dashboard_chart_filters.py
index 19f976084b..b1770ae577 100644
--- a/frappe/patches/v12_0/change_existing_dashboard_chart_filters.py
+++ b/frappe/patches/v12_0/change_existing_dashboard_chart_filters.py
@@ -7,7 +7,7 @@ def execute():
 	if not frappe.db.table_exists("Dashboard Chart"):
 		return
 
-	charts_to_modify = frappe.db.get_all(
+	charts_to_modify = frappe.get_all(
 		"Dashboard Chart",
 		fields=["name", "filters_json", "document_type"],
 		filters={"chart_type": ["not in", ["Report", "Custom"]]},
diff --git a/frappe/patches/v12_0/create_notification_settings_for_user.py b/frappe/patches/v12_0/create_notification_settings_for_user.py
index eb6858967b..50c4b12787 100644
--- a/frappe/patches/v12_0/create_notification_settings_for_user.py
+++ b/frappe/patches/v12_0/create_notification_settings_for_user.py
@@ -8,6 +8,6 @@ def execute():
 	frappe.reload_doc("desk", "doctype", "notification_settings")
 	frappe.reload_doc("desk", "doctype", "notification_subscribed_document")
 
-	users = frappe.db.get_all("User", fields=["name"])
+	users = frappe.get_all("User", fields=["name"])
 	for user in users:
 		create_notification_settings(user.name)
diff --git a/frappe/patches/v12_0/delete_duplicate_indexes.py b/frappe/patches/v12_0/delete_duplicate_indexes.py
index f1def21f7f..45f495fe69 100644
--- a/frappe/patches/v12_0/delete_duplicate_indexes.py
+++ b/frappe/patches/v12_0/delete_duplicate_indexes.py
@@ -1,5 +1,3 @@
-from pymysql import InternalError
-
 import frappe
 
 # This patch deletes all the duplicate indexes created for same column
@@ -17,39 +15,41 @@ def execute():
 		indexes_to_keep_map = frappe._dict()
 		indexes_to_delete = []
 		index_info = frappe.db.sql(
-			"""
-			SELECT
-				column_name,
-				index_name,
-				non_unique
-			FROM information_schema.STATISTICS
-			WHERE table_name=%s
-			AND column_name!='name'
-			AND non_unique=0
-			ORDER BY index_name;
-		""",
-			table,
+			f"""SHOW INDEX FROM `{table}`
+				WHERE Seq_in_index = 1
+				AND Non_unique=0""",
 			as_dict=1,
 		)
 
 		for index in index_info:
-			if not indexes_to_keep_map.get(index.column_name):
-				indexes_to_keep_map[index.column_name] = index
+			if not indexes_to_keep_map.get(index.Column_name):
+				indexes_to_keep_map[index.Column_name] = index
 			else:
-				indexes_to_delete.append(index.index_name)
+				indexes_to_delete.append(index.Key_name)
+
 		if indexes_to_delete:
 			final_deletion_map[table] = indexes_to_delete
 
-	# build drop index query
-	for (table_name, index_list) in final_deletion_map.items():
-		query_list = []
-		alter_query = f"ALTER TABLE `{table_name}`"
-
+	for table_name, index_list in final_deletion_map.items():
 		for index in index_list:
-			query_list.append(f"{alter_query} DROP INDEX `{index}`")
-
-		for query in query_list:
 			try:
-				frappe.db.sql(query)
-			except InternalError:
-				pass
+				if is_clustered_index(table_name, index):
+					continue
+				frappe.db.sql_ddl(f"ALTER TABLE `{table_name}` DROP INDEX `{index}`")
+			except Exception as e:
+				frappe.log_error("Failed to drop index")
+				print(f"x Failed to drop index {index} from {table_name}\n {str(e)}")
+			else:
+				print(f"✓ dropped {index} index from {table}")
+
+
+def is_clustered_index(table, index_name):
+	return bool(
+		frappe.db.sql(
+			f"""SHOW INDEX FROM `{table}`
+			WHERE Key_name = "{index_name}"
+				AND Seq_in_index = 2
+			""",
+			as_dict=True,
+		)
+	)
diff --git a/frappe/patches/v12_0/move_email_and_phone_to_child_table.py b/frappe/patches/v12_0/move_email_and_phone_to_child_table.py
index 1a369b4e12..7283760c23 100644
--- a/frappe/patches/v12_0/move_email_and_phone_to_child_table.py
+++ b/frappe/patches/v12_0/move_email_and_phone_to_child_table.py
@@ -27,7 +27,7 @@ def execute():
 			email_values.append(
 				(
 					1,
-					frappe.generate_hash(contact_detail.email_id, 10),
+					frappe.generate_hash(length=10),
 					contact_detail.email_id,
 					"email_ids",
 					"Contact",
@@ -44,7 +44,7 @@ def execute():
 			phone_values.append(
 				(
 					phone_counter,
-					frappe.generate_hash(contact_detail.email_id, 10),
+					frappe.generate_hash(length=10),
 					contact_detail.phone,
 					"phone_nos",
 					"Contact",
@@ -63,7 +63,7 @@ def execute():
 			phone_values.append(
 				(
 					phone_counter,
-					frappe.generate_hash(contact_detail.email_id, 10),
+					frappe.generate_hash(length=10),
 					contact_detail.mobile_no,
 					"phone_nos",
 					"Contact",
diff --git a/frappe/patches/v12_0/set_default_incoming_email_port.py b/frappe/patches/v12_0/set_default_incoming_email_port.py
index 822ce06f70..766e31fe67 100644
--- a/frappe/patches/v12_0/set_default_incoming_email_port.py
+++ b/frappe/patches/v12_0/set_default_incoming_email_port.py
@@ -15,9 +15,7 @@ def execute():
 
 
 def setup_incoming_email_port_in_email_domains():
-	email_domains = frappe.db.get_all(
-		"Email Domain", ["incoming_port", "use_imap", "use_ssl", "name"]
-	)
+	email_domains = frappe.get_all("Email Domain", ["incoming_port", "use_imap", "use_ssl", "name"])
 	for domain in email_domains:
 		if not domain.incoming_port:
 			incoming_port = get_port(domain)
@@ -33,7 +31,7 @@ def setup_incoming_email_port_in_email_domains():
 
 
 def setup_incoming_email_port_in_email_accounts():
-	email_accounts = frappe.db.get_all(
+	email_accounts = frappe.get_all(
 		"Email Account", ["incoming_port", "use_imap", "use_ssl", "name", "enable_incoming"]
 	)
 
diff --git a/frappe/patches/v12_0/set_default_password_reset_limit.py b/frappe/patches/v12_0/set_default_password_reset_limit.py
index 9e29e75c36..98a397b606 100644
--- a/frappe/patches/v12_0/set_default_password_reset_limit.py
+++ b/frappe/patches/v12_0/set_default_password_reset_limit.py
@@ -6,4 +6,4 @@ import frappe
 
 def execute():
 	frappe.reload_doc("core", "doctype", "system_settings", force=1)
-	frappe.db.set_value("System Settings", None, "password_reset_limit", 3)
+	frappe.db.set_single_value("System Settings", "password_reset_limit", 3)
diff --git a/frappe/patches/v12_0/setup_tags.py b/frappe/patches/v12_0/setup_tags.py
index 6bff8d3dac..cb0d46a45d 100644
--- a/frappe/patches/v12_0/setup_tags.py
+++ b/frappe/patches/v12_0/setup_tags.py
@@ -28,7 +28,7 @@ def execute():
 
 				tag_list.append((tag.strip(), time, time, "Administrator"))
 
-				tag_link_name = frappe.generate_hash(_user_tags.name + tag.strip() + doctype.name, 10)
+				tag_link_name = frappe.generate_hash(length=10)
 				tag_links.append(
 					(tag_link_name, doctype.name, _user_tags.name, tag.strip(), time, time, "Administrator")
 				)
diff --git a/frappe/patches/v13_0/delete_event_producer_and_consumer_keys.py b/frappe/patches/v13_0/delete_event_producer_and_consumer_keys.py
deleted file mode 100644
index 9cb081e15a..0000000000
--- a/frappe/patches/v13_0/delete_event_producer_and_consumer_keys.py
+++ /dev/null
@@ -1,11 +0,0 @@
-# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors
-# License: MIT. See LICENSE
-
-import frappe
-
-
-def execute():
-	if frappe.db.exists("DocType", "Event Producer"):
-		frappe.db.sql("""UPDATE `tabEvent Producer` SET api_key='', api_secret=''""")
-	if frappe.db.exists("DocType", "Event Consumer"):
-		frappe.db.sql("""UPDATE `tabEvent Consumer` SET api_key='', api_secret=''""")
diff --git a/frappe/patches/v13_0/encrypt_2fa_secrets.py b/frappe/patches/v13_0/encrypt_2fa_secrets.py
new file mode 100644
index 0000000000..1814ff50c5
--- /dev/null
+++ b/frappe/patches/v13_0/encrypt_2fa_secrets.py
@@ -0,0 +1,45 @@
+import frappe
+import frappe.defaults
+from frappe.cache_manager import clear_defaults_cache
+from frappe.twofactor import PARENT_FOR_DEFAULTS
+from frappe.utils.password import encrypt
+
+DOCTYPE = "DefaultValue"
+OLD_PARENT = "__default"
+
+
+def execute():
+	table = frappe.qb.DocType(DOCTYPE)
+
+	# set parent for `*_otplogin`
+	(
+		frappe.qb.update(table)
+		.set(table.parent, PARENT_FOR_DEFAULTS)
+		.where(table.parent == OLD_PARENT)
+		.where(table.defkey.like("%_otplogin"))
+	).run()
+
+	# update records for `*_otpsecret`
+	secrets = {
+		key: value
+		for key, value in frappe.defaults.get_defaults_for(parent=OLD_PARENT).items()
+		if key.endswith("_otpsecret")
+	}
+
+	if not secrets:
+		return
+
+	defvalue_cases = frappe.qb.terms.Case()
+
+	for key, value in secrets.items():
+		defvalue_cases.when(table.defkey == key, encrypt(value))
+
+	(
+		frappe.qb.update(table)
+		.set(table.parent, PARENT_FOR_DEFAULTS)
+		.set(table.defvalue, defvalue_cases)
+		.where(table.parent == OLD_PARENT)
+		.where(table.defkey.like("%_otpsecret"))
+	).run()
+
+	clear_defaults_cache()
diff --git a/frappe/patches/v13_0/generate_theme_files_in_public_folder.py b/frappe/patches/v13_0/generate_theme_files_in_public_folder.py
index 29ddca1108..62c7bcdfde 100644
--- a/frappe/patches/v13_0/generate_theme_files_in_public_folder.py
+++ b/frappe/patches/v13_0/generate_theme_files_in_public_folder.py
@@ -6,13 +6,12 @@ import frappe
 
 def execute():
 	frappe.reload_doc("website", "doctype", "website_theme_ignore_app")
-	themes = frappe.db.get_all(
+	themes = frappe.get_all(
 		"Website Theme", filters={"theme_url": ("not like", "/files/website_theme/%")}
 	)
 	for theme in themes:
 		doc = frappe.get_doc("Website Theme", theme.name)
 		try:
-			doc.generate_bootstrap_theme()
 			doc.save()
 		except Exception:
 			print("Ignoring....")
diff --git a/frappe/patches/v13_0/queryreport_columns.py b/frappe/patches/v13_0/queryreport_columns.py
index 3081823db6..e9176952d4 100644
--- a/frappe/patches/v13_0/queryreport_columns.py
+++ b/frappe/patches/v13_0/queryreport_columns.py
@@ -15,4 +15,4 @@ def execute():
 		if isinstance(data, list):
 			# double escape braces
 			jstr = f'{{"columns":{jstr}}}'
-			frappe.db.update("Report", record["name"], "json", jstr)
+			frappe.db.set_value("Report", record["name"], "json", jstr)
diff --git a/frappe/patches/v13_0/remove_share_for_std_users.py b/frappe/patches/v13_0/remove_share_for_std_users.py
new file mode 100644
index 0000000000..39b449995d
--- /dev/null
+++ b/frappe/patches/v13_0/remove_share_for_std_users.py
@@ -0,0 +1,7 @@
+import frappe
+import frappe.share
+
+
+def execute():
+	for user in frappe.STANDARD_USERS:
+		frappe.share.remove("User", user, user)
diff --git a/frappe/patches/v13_0/rename_list_view_setting_to_list_view_settings.py b/frappe/patches/v13_0/rename_list_view_setting_to_list_view_settings.py
index 2147a2da94..c7c8cbc724 100644
--- a/frappe/patches/v13_0/rename_list_view_setting_to_list_view_settings.py
+++ b/frappe/patches/v13_0/rename_list_view_setting_to_list_view_settings.py
@@ -5,22 +5,27 @@ import frappe
 
 
 def execute():
-	if frappe.db.table_exists("List View Setting"):
-		if not frappe.db.table_exists("List View Settings"):
-			frappe.reload_doc("desk", "doctype", "List View Settings")
+	if not frappe.db.table_exists("List View Setting"):
+		return
+	if not frappe.db.exists("DocType", "List View Setting"):
+		return
 
-		existing_list_view_settings = frappe.get_all("List View Settings", as_list=True)
-		for list_view_setting in frappe.get_all(
-			"List View Setting",
-			fields=["disable_count", "disable_sidebar_stats", "disable_auto_refresh", "name"],
-		):
-			name = list_view_setting.pop("name")
-			if name not in [x[0] for x in existing_list_view_settings]:
-				list_view_setting["doctype"] = "List View Settings"
-				list_view_settings = frappe.get_doc(list_view_setting)
-				# setting name here is necessary because autoname is set as prompt
-				list_view_settings.name = name
-				list_view_settings.insert()
+	frappe.reload_doc("desk", "doctype", "List View Settings")
 
-		frappe.delete_doc("DocType", "List View Setting", force=True)
-		frappe.db.commit()
+	existing_list_view_settings = frappe.get_all(
+		"List View Settings", as_list=True, order_by="modified"
+	)
+	for list_view_setting in frappe.get_all(
+		"List View Setting",
+		fields=["disable_count", "disable_sidebar_stats", "disable_auto_refresh", "name"],
+		order_by="modified",
+	):
+		name = list_view_setting.pop("name")
+		if name not in [x[0] for x in existing_list_view_settings]:
+			list_view_setting["doctype"] = "List View Settings"
+			list_view_settings = frappe.get_doc(list_view_setting)
+			# setting name here is necessary because autoname is set as prompt
+			list_view_settings.name = name
+			list_view_settings.insert()
+
+	frappe.delete_doc("DocType", "List View Setting", force=True)
diff --git a/frappe/patches/v13_0/reset_corrupt_defaults.py b/frappe/patches/v13_0/reset_corrupt_defaults.py
new file mode 100644
index 0000000000..10e81c7ff1
--- /dev/null
+++ b/frappe/patches/v13_0/reset_corrupt_defaults.py
@@ -0,0 +1,33 @@
+import frappe
+from frappe.patches.v13_0.encrypt_2fa_secrets import DOCTYPE
+from frappe.patches.v13_0.encrypt_2fa_secrets import PARENT_FOR_DEFAULTS as TWOFACTOR_PARENT
+from frappe.utils import cint
+
+
+def execute():
+	"""
+	This patch is needed to fix parent incorrectly set as `__2fa` because of
+	https://github.com/frappe/frappe/commit/a822092211533ff17ff9b92dd86f6f868ed63e2e
+	"""
+
+	if not frappe.db.get_value(
+		DOCTYPE, {"parent": TWOFACTOR_PARENT, "defkey": ("not like", "%_otp%")}, "defkey"
+	):
+		return
+
+	# system settings
+	system_settings = frappe.get_single("System Settings")
+	system_settings.set_defaults()
+
+	# home page
+	frappe.db.set_default(
+		"desktop:home_page", "workspace" if cint(system_settings.setup_complete) else "setup-wizard"
+	)
+
+	# letter head
+	try:
+		letter_head = frappe.get_doc("Letter Head", {"is_default": 1})
+		letter_head.set_as_default()
+
+	except frappe.DoesNotExistError:
+		pass
diff --git a/frappe/patches/v13_0/set_existing_dashboard_charts_as_public.py b/frappe/patches/v13_0/set_existing_dashboard_charts_as_public.py
index 11015033b9..54140ee7bf 100644
--- a/frappe/patches/v13_0/set_existing_dashboard_charts_as_public.py
+++ b/frappe/patches/v13_0/set_existing_dashboard_charts_as_public.py
@@ -15,7 +15,7 @@ def execute():
 	)
 
 	users = [item.parent for item in users_with_permission]
-	charts = frappe.db.get_all("Dashboard Chart", filters={"owner": ["in", users]})
+	charts = frappe.get_all("Dashboard Chart", filters={"owner": ["in", users]})
 
 	for chart in charts:
 		frappe.db.set_value("Dashboard Chart", chart.name, "is_public", 1)
diff --git a/frappe/patches/v13_0/set_first_day_of_the_week.py b/frappe/patches/v13_0/set_first_day_of_the_week.py
index 165ec3c42b..a48ca09672 100644
--- a/frappe/patches/v13_0/set_first_day_of_the_week.py
+++ b/frappe/patches/v13_0/set_first_day_of_the_week.py
@@ -5,4 +5,4 @@ def execute():
 	frappe.reload_doctype("System Settings")
 	# setting first_day_of_the_week value as "Monday" to avoid breaking change
 	# because before the configuration was introduced, system used to consider "Monday" as start of the week
-	frappe.db.set_value("System Settings", "System Settings", "first_day_of_the_week", "Monday")
+	frappe.db.set_single_value("System Settings", "first_day_of_the_week", "Monday")
diff --git a/frappe/patches/v13_0/website_theme_custom_scss.py b/frappe/patches/v13_0/website_theme_custom_scss.py
index d1a5e11228..d72b763bd9 100644
--- a/frappe/patches/v13_0/website_theme_custom_scss.py
+++ b/frappe/patches/v13_0/website_theme_custom_scss.py
@@ -8,21 +8,31 @@ def execute():
 
 	for theme in frappe.get_all("Website Theme"):
 		doc = frappe.get_doc("Website Theme", theme.name)
+		setup_color_record(doc)
 		if not doc.get("custom_scss") and doc.theme_scss:
 			# move old theme to new theme
 			doc.custom_scss = doc.theme_scss
-
-			if doc.background_color:
-				setup_color_record(doc.background_color)
-
 			doc.save()
 
 
-def setup_color_record(color):
-	frappe.get_doc(
-		{
-			"doctype": "Color",
-			"__newname": color,
-			"color": color,
-		}
-	).save()
+def setup_color_record(doc):
+	color_fields = [
+		"primary_color",
+		"text_color",
+		"light_color",
+		"dark_color",
+		"background_color",
+	]
+
+	for color_field in color_fields:
+		color_code = doc.get(color_field)
+		if not color_code or frappe.db.exists("Color", color_code):
+			continue
+
+		frappe.get_doc(
+			{
+				"doctype": "Color",
+				"__newname": color_code,
+				"color": color_code,
+			}
+		).insert()
diff --git a/frappe/patches/v14_0/delete_data_migration_tool.py b/frappe/patches/v14_0/delete_data_migration_tool.py
index d0416cb1e7..9ecd714c86 100644
--- a/frappe/patches/v14_0/delete_data_migration_tool.py
+++ b/frappe/patches/v14_0/delete_data_migration_tool.py
@@ -5,7 +5,7 @@ import frappe
 
 
 def execute():
-	doctypes = frappe.db.get_all("DocType", {"module": "Data Migration", "custom": 0}, pluck="name")
+	doctypes = frappe.get_all("DocType", {"module": "Data Migration", "custom": 0}, pluck="name")
 	for doctype in doctypes:
 		frappe.delete_doc("DocType", doctype, ignore_missing=True)
 
diff --git a/frappe/patches/v14_0/delete_payment_gateways.py b/frappe/patches/v14_0/delete_payment_gateways.py
new file mode 100644
index 0000000000..c06f63a2d3
--- /dev/null
+++ b/frappe/patches/v14_0/delete_payment_gateways.py
@@ -0,0 +1,16 @@
+import frappe
+
+
+def execute():
+	if "payments" in frappe.get_installed_apps():
+		return
+
+	for doctype in (
+		"Payment Gateway",
+		"Razorpay Settings",
+		"Braintree Settings",
+		"PayPal Settings",
+		"Paytm Settings",
+		"Stripe Settings",
+	):
+		frappe.delete_doc_if_exists("DocType", doctype, force=True)
diff --git a/frappe/patches/v14_0/different_encryption_key.py b/frappe/patches/v14_0/different_encryption_key.py
new file mode 100644
index 0000000000..3b80e15a73
--- /dev/null
+++ b/frappe/patches/v14_0/different_encryption_key.py
@@ -0,0 +1,16 @@
+import pathlib
+
+import frappe
+from frappe.installer import update_site_config
+from frappe.utils.backups import BACKUP_ENCRYPTION_CONFIG_KEY, get_backup_path
+
+
+def execute():
+	if frappe.conf.get(BACKUP_ENCRYPTION_CONFIG_KEY):
+		return
+
+	backup_path = pathlib.Path(get_backup_path())
+	encrypted_backups_present = bool(list(backup_path.glob("*-enc*")))
+
+	if encrypted_backups_present:
+		update_site_config(BACKUP_ENCRYPTION_CONFIG_KEY, frappe.local.conf.encryption_key)
diff --git a/frappe/patches/v14_0/disable_email_accounts_with_oauth.py b/frappe/patches/v14_0/disable_email_accounts_with_oauth.py
new file mode 100644
index 0000000000..2066b5f640
--- /dev/null
+++ b/frappe/patches/v14_0/disable_email_accounts_with_oauth.py
@@ -0,0 +1,38 @@
+import frappe
+from frappe.desk.doctype.notification_log.notification_log import make_notification_logs
+
+
+def execute():
+	if frappe.get_all(
+		"Email Account", {"auth_method": "OAuth", "connected_user": ["is", "set"]}, limit=1
+	):
+		return
+
+	# Setting awaiting password to 1 for email accounts where Oauth is enabled.
+	# This is done so that people can resetup their email accounts with connected app mechanism.
+	frappe.db.set_value("Email Account", {"auth_method": "OAuth"}, "awaiting_password", 1)
+
+	message = "Email Accounts with auth method as OAuth have been disabled.\
+	Please re-setup your OAuth based email accounts with the connected app mechanism to re-enable them."
+
+	if sysmanagers := get_system_managers():
+		make_notification_logs(
+			{
+				"type": "Alert",
+				"subject": frappe._(message),
+			},
+			sysmanagers,
+		)
+
+
+def get_system_managers():
+	user_doctype = frappe.qb.DocType("User").as_("user")
+	user_role_doctype = frappe.qb.DocType("Has Role").as_("user_role")
+	return (
+		frappe.qb.from_(user_doctype)
+		.from_(user_role_doctype)
+		.select(user_doctype.email)
+		.where(user_role_doctype.role == "System Manager")
+		.where(user_doctype.enabled == 1)
+		.where(user_role_doctype.parent == user_doctype.name)
+	).run(pluck=True)
diff --git a/frappe/patches/v14_0/drop_unused_indexes.py b/frappe/patches/v14_0/drop_unused_indexes.py
new file mode 100644
index 0000000000..896ea78fed
--- /dev/null
+++ b/frappe/patches/v14_0/drop_unused_indexes.py
@@ -0,0 +1,56 @@
+"""
+This patch just drops some known indexes which aren't being used anymore or never were used.
+"""
+
+import click
+
+import frappe
+
+UNUSED_INDEXES = [
+	("Comment", ["link_doctype", "link_name"]),
+	("Activity Log", ["link_doctype", "link_name"]),
+]
+
+
+def execute():
+	if frappe.db.db_type == "postgres":
+		return
+
+	db_tables = frappe.db.get_tables(cached=False)
+
+	# All parent indexes
+	parent_doctypes = frappe.get_all(
+		"DocType",
+		{"istable": 0, "is_virtual": 0, "issingle": 0},
+		pluck="name",
+	)
+	db_tables = frappe.db.get_tables(cached=False)
+
+	for doctype in parent_doctypes:
+		table = f"tab{doctype}"
+		if table not in db_tables:
+			continue
+		drop_index_if_exists(table, "parent")
+
+	# Unused composite indexes
+	for doctype, index_fields in UNUSED_INDEXES:
+		table = f"tab{doctype}"
+		index_name = frappe.db.get_index_name(index_fields)
+		if table not in db_tables:
+			continue
+		drop_index_if_exists(table, index_name)
+
+
+def drop_index_if_exists(table: str, index: str):
+	if not frappe.db.has_index(table, index):
+		click.echo(f"- Skipped {index} index for {table} because it doesn't exist")
+		return
+
+	try:
+		frappe.db.sql_ddl(f"ALTER TABLE `{table}` DROP INDEX `{index}`")
+	except Exception as e:
+		frappe.log_error("Failed to drop index")
+		click.secho(f"x Failed to drop index {index} from {table}\n {str(e)}", fg="red")
+		return
+
+	click.echo(f"✓ dropped {index} index from {table}")
diff --git a/frappe/patches/v14_0/remove_db_aggregation.py b/frappe/patches/v14_0/remove_db_aggregation.py
index 4b0a58c2d6..cff2b583ce 100644
--- a/frappe/patches/v14_0/remove_db_aggregation.py
+++ b/frappe/patches/v14_0/remove_db_aggregation.py
@@ -32,4 +32,4 @@ def execute():
 		for agg in ["avg", "max", "min", "sum"]:
 			script = re.sub(f"frappe.db.{agg}\\(", f"frappe.qb.{agg}(", script)
 
-		frappe.db.update("Server Script", name, "script", script)
+		frappe.db.set_value("Server Script", name, "script", script)
diff --git a/frappe/patches/v14_0/remove_manage_subscriptions_from_navbar.py b/frappe/patches/v14_0/remove_manage_subscriptions_from_navbar.py
new file mode 100644
index 0000000000..cd35dd6c9c
--- /dev/null
+++ b/frappe/patches/v14_0/remove_manage_subscriptions_from_navbar.py
@@ -0,0 +1,10 @@
+import frappe
+
+
+def execute():
+	navbar_settings = frappe.get_single("Navbar Settings")
+	for i, l in enumerate(navbar_settings.settings_dropdown):
+		if l.item_label == "Manage Subscriptions":
+			navbar_settings.settings_dropdown.pop(i)
+			navbar_settings.save()
+			break
diff --git a/frappe/patches/v14_0/set_document_expiry_default.py b/frappe/patches/v14_0/set_document_expiry_default.py
index 59a9db6c4d..0b193bcd9b 100644
--- a/frappe/patches/v14_0/set_document_expiry_default.py
+++ b/frappe/patches/v14_0/set_document_expiry_default.py
@@ -2,8 +2,7 @@ import frappe
 
 
 def execute():
-	frappe.db.set_value(
-		"System Settings",
+	frappe.db.set_single_value(
 		"System Settings",
 		{"document_share_key_expiry": 30, "allow_older_web_view_links": 1},
 	)
diff --git a/frappe/patches/v14_0/set_suspend_email_queue_default.py b/frappe/patches/v14_0/set_suspend_email_queue_default.py
new file mode 100644
index 0000000000..8cdb05a177
--- /dev/null
+++ b/frappe/patches/v14_0/set_suspend_email_queue_default.py
@@ -0,0 +1,13 @@
+import frappe
+from frappe.cache_manager import clear_defaults_cache
+
+
+def execute():
+	frappe.db.set_default(
+		"suspend_email_queue",
+		frappe.db.get_default("hold_queue", "Administrator") or 0,
+		parent="__default",
+	)
+
+	frappe.db.delete("DefaultValue", {"defkey": "hold_queue"})
+	clear_defaults_cache()
diff --git a/frappe/patches/v14_0/setup_likes_from_feedback.py b/frappe/patches/v14_0/setup_likes_from_feedback.py
new file mode 100644
index 0000000000..d88f69ce4b
--- /dev/null
+++ b/frappe/patches/v14_0/setup_likes_from_feedback.py
@@ -0,0 +1,30 @@
+import frappe
+
+
+def execute():
+	frappe.reload_doctype("Comment")
+
+	if frappe.db.count("Feedback") > 20000:
+		frappe.db.auto_commit_on_many_writes = True
+
+	for feedback in frappe.get_all("Feedback", fields=["*"]):
+		if feedback.like:
+			new_comment = frappe.new_doc("Comment")
+			new_comment.comment_type = "Like"
+			new_comment.comment_email = feedback.owner
+			new_comment.content = "Liked by: " + feedback.owner
+			new_comment.reference_doctype = feedback.reference_doctype
+			new_comment.reference_name = feedback.reference_name
+			new_comment.creation = feedback.creation
+			new_comment.modified = feedback.modified
+			new_comment.owner = feedback.owner
+			new_comment.modified_by = feedback.modified_by
+			new_comment.ip_address = feedback.ip_address
+			new_comment.db_insert()
+
+	if frappe.db.auto_commit_on_many_writes:
+		frappe.db.auto_commit_on_many_writes = False
+
+	# clean up
+	frappe.db.delete("Feedback")
+	frappe.db.commit()
diff --git a/frappe/patches/v14_0/update_attachment_comment.py b/frappe/patches/v14_0/update_attachment_comment.py
new file mode 100644
index 0000000000..042579d86d
--- /dev/null
+++ b/frappe/patches/v14_0/update_attachment_comment.py
@@ -0,0 +1,33 @@
+import frappe
+
+
+def execute():
+	frappe.db.auto_commit_on_many_writes = 1
+
+	# Strip everything except link to attachment and icon from comments of type "Attached"
+	for name, content in frappe.get_all(
+		"Comment", filters={"comment_type": "Attachment"}, fields=["name", "content"], as_list=True
+	):
+		if not content:
+			continue
+
+		start = content.find("")
+		end = content.find("") if end == -1 else end
+		if end != -1:
+			content = content[: end + 4]
+
+		frappe.db.set_value("Comment", name, "content", content, update_modified=False)
+
+	# Strip "Removed " from comments of type "Attachment Removed"
+	for name, content in frappe.get_all(
+		"Comment",
+		filters={"comment_type": "Attachment Removed"},
+		fields=["name", "content"],
+		as_list=True,
+	):
+		if content and content.startswith("Removed "):
+			frappe.db.set_value("Comment", name, "content", content[8:], update_modified=False)
diff --git a/frappe/patches/v14_0/update_auto_account_deletion_duration.py b/frappe/patches/v14_0/update_auto_account_deletion_duration.py
index 6b01816092..ed7b3f21fa 100644
--- a/frappe/patches/v14_0/update_auto_account_deletion_duration.py
+++ b/frappe/patches/v14_0/update_auto_account_deletion_duration.py
@@ -3,4 +3,4 @@ import frappe
 
 def execute():
 	days = frappe.db.get_single_value("Website Settings", "auto_account_deletion")
-	frappe.db.set_value("Website Settings", None, "auto_account_deletion", days * 24)
+	frappe.db.set_single_value("Website Settings", "auto_account_deletion", days * 24)
diff --git a/frappe/patches/v14_0/update_color_names_in_kanban_board_column.py b/frappe/patches/v14_0/update_color_names_in_kanban_board_column.py
index b568151273..c68015b43d 100644
--- a/frappe/patches/v14_0/update_color_names_in_kanban_board_column.py
+++ b/frappe/patches/v14_0/update_color_names_in_kanban_board_column.py
@@ -17,6 +17,6 @@ def execute():
 		"yellow": "Yellow",
 		"lightblue": "Light Blue",
 	}
-	for d in frappe.db.get_all("Kanban Board Column", fields=["name", "indicator"]):
+	for d in frappe.get_all("Kanban Board Column", fields=["name", "indicator"]):
 		color_name = indicator_map.get(d.indicator, "Gray")
 		frappe.db.set_value("Kanban Board Column", d.name, "indicator", color_name)
diff --git a/frappe/patches/v14_0/update_github_endpoints.py b/frappe/patches/v14_0/update_github_endpoints.py
index 5ea638f0a6..f7bded4e96 100644
--- a/frappe/patches/v14_0/update_github_endpoints.py
+++ b/frappe/patches/v14_0/update_github_endpoints.py
@@ -1,6 +1,7 @@
-import frappe
 import json
 
+import frappe
+
 
 def execute():
 	if frappe.db.exists("Social Login Key", "github"):
diff --git a/frappe/patches/v14_0/update_multistep_webforms.py b/frappe/patches/v14_0/update_multistep_webforms.py
new file mode 100644
index 0000000000..9919ef6e15
--- /dev/null
+++ b/frappe/patches/v14_0/update_multistep_webforms.py
@@ -0,0 +1,12 @@
+import frappe
+
+
+def execute():
+	if not frappe.db.has_column("Web Form", "is_multi_step_form"):
+		return
+
+	for web_form in frappe.get_all("Web Form", filters={"is_multi_step_form": 1}):
+		web_form_fields = frappe.get_doc("Web Form", web_form.name).web_form_fields
+		for web_form_field in web_form_fields:
+			if web_form_field.fieldtype == "Section Break" and web_form_field.idx != 1:
+				frappe.db.set_value("Web Form Field", web_form_field.name, "fieldtype", "Page Break")
diff --git a/frappe/patches/v14_0/update_webforms.py b/frappe/patches/v14_0/update_webforms.py
new file mode 100644
index 0000000000..40d90c426b
--- /dev/null
+++ b/frappe/patches/v14_0/update_webforms.py
@@ -0,0 +1,14 @@
+# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors
+# MIT License. See license.txt
+
+
+import frappe
+
+
+def execute():
+	frappe.reload_doc("website", "doctype", "web_form_list_column")
+	frappe.reload_doctype("Web Form")
+
+	for web_form in frappe.get_all("Web Form", fields=["*"]):
+		if web_form.allow_multiple and not web_form.show_list:
+			frappe.db.set_value("Web Form", web_form.name, "show_list", True)
diff --git a/frappe/patches/v15_0/copy_disable_prepared_report_to_prepared_report.py b/frappe/patches/v15_0/copy_disable_prepared_report_to_prepared_report.py
new file mode 100644
index 0000000000..f1e4afb9a4
--- /dev/null
+++ b/frappe/patches/v15_0/copy_disable_prepared_report_to_prepared_report.py
@@ -0,0 +1,6 @@
+import frappe
+
+
+def execute():
+	table = frappe.qb.DocType("Report")
+	frappe.qb.update(table).set(table.prepared_report, 0).where(table.disable_prepared_report == 1)
diff --git a/frappe/patches/v15_0/drop_modified_index.py b/frappe/patches/v15_0/drop_modified_index.py
new file mode 100644
index 0000000000..8cdcf12ece
--- /dev/null
+++ b/frappe/patches/v15_0/drop_modified_index.py
@@ -0,0 +1,21 @@
+import frappe
+from frappe.patches.v14_0.drop_unused_indexes import drop_index_if_exists
+
+
+def execute():
+	if frappe.db.db_type == "postgres":
+		return
+
+	db_tables = frappe.db.get_tables(cached=False)
+
+	child_tables = frappe.get_all(
+		"DocType",
+		{"istable": 1, "is_virtual": 0},
+		pluck="name",
+	)
+
+	for doctype in child_tables:
+		table = f"tab{doctype}"
+		if table not in db_tables:
+			continue
+		drop_index_if_exists(table, "modified")
diff --git a/frappe/patches/v15_0/remove_background_jobs_from_dropdown.py b/frappe/patches/v15_0/remove_background_jobs_from_dropdown.py
new file mode 100644
index 0000000000..b070e0805c
--- /dev/null
+++ b/frappe/patches/v15_0/remove_background_jobs_from_dropdown.py
@@ -0,0 +1,9 @@
+import frappe
+
+
+def execute():
+	item = frappe.db.exists("Navbar Item", {"item_label": "Background Jobs"})
+	if not item:
+		return
+
+	frappe.delete_doc("Navbar Item", item)
diff --git a/frappe/patches/v15_0/remove_event_streaming.py b/frappe/patches/v15_0/remove_event_streaming.py
new file mode 100644
index 0000000000..4c6a1ce079
--- /dev/null
+++ b/frappe/patches/v15_0/remove_event_streaming.py
@@ -0,0 +1,22 @@
+import frappe
+
+
+def execute():
+	if "event_streaming" in frappe.get_installed_apps():
+		return
+
+	frappe.delete_doc_if_exists("Module Def", "Event Streaming", force=True)
+
+	for doc in [
+		"Event Consumer Document Type",
+		"Document Type Mapping",
+		"Event Producer",
+		"Event Producer Last Update",
+		"Event Producer Document Type",
+		"Event Consumer",
+		"Document Type Field Mapping",
+		"Event Update Log",
+		"Event Update Log Consumer",
+		"Event Sync Log",
+	]:
+		frappe.delete_doc_if_exists("DocType", doc, force=True)
diff --git a/frappe/patches/v15_0/remove_prepared_report_settings_from_system_settings.py b/frappe/patches/v15_0/remove_prepared_report_settings_from_system_settings.py
new file mode 100644
index 0000000000..2c203784df
--- /dev/null
+++ b/frappe/patches/v15_0/remove_prepared_report_settings_from_system_settings.py
@@ -0,0 +1,9 @@
+import frappe
+from frappe.utils import cint
+
+
+def execute():
+	expiry_period = (
+		cint(frappe.db.get_singles_dict("System Settings").get("prepared_report_expiry_period")) or 30
+	)
+	frappe.get_single("Log Settings").register_doctype("Prepared Report", expiry_period)
diff --git a/frappe/patches/v15_0/set_contact_full_name.py b/frappe/patches/v15_0/set_contact_full_name.py
new file mode 100644
index 0000000000..6dc3036f34
--- /dev/null
+++ b/frappe/patches/v15_0/set_contact_full_name.py
@@ -0,0 +1,25 @@
+import frappe
+from frappe.contacts.doctype.contact.contact import get_full_name
+from frappe.utils import update_progress_bar
+
+
+def execute():
+	"""Set full name for all contacts"""
+	frappe.db.auto_commit_on_many_writes = 1
+
+	contacts = frappe.get_all(
+		"Contact",
+		fields=["name", "first_name", "middle_name", "last_name", "company_name"],
+		filters={"full_name": ("is", "not set")},
+		as_list=True,
+	)
+	total = len(contacts)
+	for idx, (name, first, middle, last, company) in enumerate(contacts):
+		update_progress_bar("Setting full name for contacts", idx, total)
+		frappe.db.set_value(
+			"Contact",
+			name,
+			"full_name",
+			get_full_name(first, middle, last, company),
+			update_modified=False,
+		)
diff --git a/frappe/permissions.py b/frappe/permissions.py
index acbdf76989..431132a0ae 100644
--- a/frappe/permissions.py
+++ b/frappe/permissions.py
@@ -22,25 +22,16 @@ rights = (
 	"report",
 	"import",
 	"export",
-	"set_user_permissions",
 	"share",
 )
 
 
-def check_admin_or_system_manager(user=None):
-	if not user:
-		user = frappe.session.user
-
-	if ("System Manager" not in frappe.get_roles(user)) and (user != "Administrator"):
-		frappe.throw(_("Not permitted"), frappe.PermissionError)
-
-
 def print_has_permission_check_logs(func):
 	def inner(*args, **kwargs):
 		frappe.flags["has_permission_check_logs"] = []
 		result = func(*args, **kwargs)
 		self_perm_check = True if not kwargs.get("user") else kwargs.get("user") == frappe.session.user
-		raise_exception = False if kwargs.get("raise_exception") is False else True
+		raise_exception = kwargs.get("raise_exception", True)
 
 		# print only if access denied
 		# and if user is checking his own permission
@@ -60,28 +51,42 @@ def has_permission(
 	verbose=False,
 	user=None,
 	raise_exception=True,
+	*,
 	parent_doctype=None,
 ):
 	"""Returns True if user has permission `ptype` for given `doctype`.
 	If `doc` is passed, it also checks user, share and owner permissions.
 
-	Note: if Table DocType is passed, it always returns True.
+	:param doctype: DocType to check permission for
+	:param ptype: Permission Type to check
+	:param doc: Check User Permissions for specified document.
+	:param verbose: DEPRECATED, will be removed in a future release.
+	:param user: User to check permission for. Defaults to current user.
+	:param raise_exception:
+	        DOES NOT raise an exception.
+	        If not False, will display a message using frappe.msgprint
+	                which explains why the permission check failed.
+
+	:param parent_doctype:
+	        Required when checking permission for a child DocType (unless doc is specified)
 	"""
+
 	if not user:
 		user = frappe.session.user
 
+	if user == "Administrator":
+		return True
+
+	if ptype == "share" and frappe.get_system_settings("disable_document_sharing"):
+		return False
+
 	if not doc and hasattr(doctype, "doctype"):
 		# first argument can be doc or doctype
 		doc = doctype
 		doctype = doc.doctype
 
-	if user == "Administrator":
-		return True
-
 	if frappe.is_table(doctype):
-		return has_child_table_permission(
-			doctype, ptype, doc, verbose, user, raise_exception, parent_doctype
-		)
+		return has_child_permission(doctype, ptype, doc, user, raise_exception, parent_doctype)
 
 	meta = frappe.get_meta(doctype)
 
@@ -201,7 +206,7 @@ def get_role_permissions(doctype_meta, user=None, is_owner=None):
 	if not user:
 		user = frappe.session.user
 
-	cache_key = (doctype_meta.name, user)
+	cache_key = (doctype_meta.name, user, bool(is_owner))
 
 	if user == "Administrator":
 		return allow_everything()
@@ -314,14 +319,26 @@ def has_user_permission(doc, user=None):
 				# restricted for this link field, and no matching values found
 				# make the right message and exit
 				if d.get("parentfield"):
-					# "Not allowed for Company = Restricted Company in Row 3. Restricted field: reference_type"
-					msg = _("Not allowed for {0}: {1} in Row {2}. Restricted field: {3}").format(
-						_(field.options), d.get(field.fieldname), d.idx, field.fieldname
+					# "You are not allowed to access this Employee record because it is linked
+					# to Company 'Restricted Company' in row 3, field Reference Type"
+					msg = _(
+						"You are not allowed to access this {0} record because it is linked to {1} '{2}' in row {3}, field {4}"
+					).format(
+						_(meta.doctype),
+						_(field.options),
+						d.get(field.fieldname) or _("empty"),
+						d.idx,
+						_(field.label) if field.label else field.fieldname,
 					)
 				else:
-					# "Not allowed for Company = Restricted Company. Restricted field: reference_type"
-					msg = _("Not allowed for {0}: {1}. Restricted field: {2}").format(
-						_(field.options), d.get(field.fieldname), field.fieldname
+					# "You are not allowed to access Company 'Restricted Company' in field Reference Type"
+					msg = _(
+						"You are not allowed to access this {0} record because it is linked to {1} '{2}' in field {3}"
+					).format(
+						_(meta.doctype),
+						_(field.options),
+						d.get(field.fieldname) or _("empty"),
+						_(field.label) if field.label else field.fieldname,
 					)
 
 				push_perm_check_log(msg)
@@ -398,7 +415,7 @@ def get_roles(user=None, with_standard=True):
 	if not user:
 		user = frappe.session.user
 
-	if user == "Guest":
+	if user == "Guest" or not user:
 		return ["Guest"]
 
 	def get():
@@ -408,7 +425,9 @@ def get_roles(user=None, with_standard=True):
 			table = DocType("Has Role")
 			roles = (
 				frappe.qb.from_(table)
-				.where((table.parent == user) & (table.role.notin(["All", "Guest"])))
+				.where(
+					(table.parenttype == "User") & (table.parent == user) & (table.role.notin(["All", "Guest"]))
+				)
 				.select(table.role)
 				.run(pluck=True)
 			)
@@ -432,39 +451,16 @@ def get_doctype_roles(doctype, access_type="read"):
 def get_perms_for(roles, perm_doctype="DocPerm"):
 	"""Get perms for given roles"""
 	filters = {"permlevel": 0, "docstatus": 0, "role": ["in", roles]}
-	return frappe.db.get_all(perm_doctype, fields=["*"], filters=filters)
+	return frappe.get_all(perm_doctype, fields=["*"], filters=filters)
 
 
 def get_doctypes_with_custom_docperms():
 	"""Returns all the doctypes with Custom Docperms"""
 
-	doctypes = frappe.db.get_all("Custom DocPerm", fields=["parent"], distinct=1)
+	doctypes = frappe.get_all("Custom DocPerm", fields=["parent"], distinct=1)
 	return [d.parent for d in doctypes]
 
 
-def can_set_user_permissions(doctype, docname=None):
-	# System Manager can always set user permissions
-	if frappe.session.user == "Administrator" or "System Manager" in frappe.get_roles():
-		return True
-
-	meta = frappe.get_meta(doctype)
-
-	# check if current user has read permission for docname
-	if docname and not has_permission(doctype, "read", docname):
-		return False
-
-	# check if current user has a role that can set permission
-	if get_role_permissions(meta).set_user_permissions != 1:
-		return False
-
-	return True
-
-
-def set_user_permission_if_allowed(doctype, name, user, with_message=False):
-	if get_role_permissions(frappe.get_meta(doctype), user).set_user_permissions != 1:
-		add_user_permission(doctype, name, user)
-
-
 def add_user_permission(
 	doctype,
 	name,
@@ -489,6 +485,7 @@ def add_user_permission(
 				for_value=name,
 				is_default=is_default,
 				applicable_for=applicable_for,
+				apply_to_all_doctypes=0 if applicable_for else 1,
 				hide_descendants=hide_descendants,
 			)
 		).insert(ignore_permissions=ignore_permissions)
@@ -505,7 +502,7 @@ def clear_user_permissions_for_doctype(doctype, user=None):
 	filters = {"allow": doctype}
 	if user:
 		filters["user"] = user
-	user_permissions_for_doctype = frappe.db.get_all("User Permission", filters=filters)
+	user_permissions_for_doctype = frappe.get_all("User Permission", filters=filters)
 	for d in user_permissions_for_doctype:
 		frappe.delete_doc("User Permission", d.name)
 
@@ -619,7 +616,7 @@ def get_linked_doctypes(dt: str) -> list:
 def get_doc_name(doc):
 	if not doc:
 		return None
-	return doc if isinstance(doc, str) else doc.name
+	return doc if isinstance(doc, str) else str(doc.name)
 
 
 def allow_everything():
@@ -658,52 +655,74 @@ def push_perm_check_log(log):
 	frappe.flags.get("has_permission_check_logs").append(_(log))
 
 
-def has_child_table_permission(
+def has_child_permission(
 	child_doctype,
 	ptype="read",
 	child_doc=None,
-	verbose=False,
 	user=None,
 	raise_exception=True,
 	parent_doctype=None,
 ):
-	parent_doc = None
+	if isinstance(child_doc, str):
+		child_doc = frappe.db.get_value(
+			child_doctype,
+			child_doc,
+			("parent", "parenttype", "parentfield"),
+			as_dict=True,
+		)
 
 	if child_doc:
-		parent_doctype = child_doc.get("parenttype")
-		parent_doc = frappe.get_cached_doc(
-			{"doctype": parent_doctype, "docname": child_doc.get("parent")}
-		)
+		parent_doctype = child_doc.parenttype
 
-	if parent_doctype:
-		if not is_parent_valid(child_doctype, parent_doctype):
-			frappe.throw(
-				_("{0} is not a valid parent DocType for {1}").format(
-					frappe.bold(parent_doctype), frappe.bold(child_doctype)
-				),
-				title=_("Invalid Parent DocType"),
-			)
-	else:
-		frappe.throw(
-			_("Please specify a valid parent DocType for {0}").format(frappe.bold(child_doctype)),
-			title=_("Parent DocType Required"),
+	if not parent_doctype:
+		push_perm_check_log(
+			_("Please specify a valid parent DocType for {0}").format(frappe.bold(child_doctype))
 		)
+		return False
+
+	parent_meta = frappe.get_meta(parent_doctype)
+
+	if parent_meta.istable or not (
+		valid_parentfields := [
+			df.fieldname for df in parent_meta.get_table_fields() if df.options == child_doctype
+		]
+	):
+		push_perm_check_log(
+			_("{0} is not a valid parent DocType for {1}").format(
+				frappe.bold(parent_doctype), frappe.bold(child_doctype)
+			)
+		)
+		return False
+
+	if child_doc:
+		parentfield = child_doc.parentfield
+		if not parentfield:
+			push_perm_check_log(
+				_("Parentfield not specified in {0}: {1}").format(
+					frappe.bold(child_doctype), frappe.bold(child_doc.name)
+				)
+			)
+			return False
+
+		if parentfield not in valid_parentfields:
+			push_perm_check_log(
+				_("{0} is not a valid parentfield for {1}").format(
+					frappe.bold(parentfield), frappe.bold(child_doctype)
+				)
+			)
+			return False
+
+		permlevel = parent_meta.get_field(parentfield).permlevel
+		if permlevel > 0 and permlevel not in parent_meta.get_permlevel_access(ptype, user=user):
+			push_perm_check_log(
+				_("Insufficient Permission Level for {0}").format(frappe.bold(parent_doctype))
+			)
+			return False
 
 	return has_permission(
 		parent_doctype,
 		ptype=ptype,
-		doc=parent_doc,
-		verbose=verbose,
+		doc=child_doc and getattr(child_doc, "parent_doc", child_doc.parent),
 		user=user,
 		raise_exception=raise_exception,
 	)
-
-
-def is_parent_valid(child_doctype, parent_doctype):
-	from frappe.core.utils import find
-
-	parent_meta = frappe.get_meta(parent_doctype)
-	child_table_field_exists = find(
-		parent_meta.get_table_fields(), lambda d: d.options == child_doctype
-	)
-	return not parent_meta.istable and child_table_field_exists
diff --git a/frappe/printing/doctype/letter_head/letter_head.js b/frappe/printing/doctype/letter_head/letter_head.js
index ca4dad2d07..55d97cf37f 100644
--- a/frappe/printing/doctype/letter_head/letter_head.js
+++ b/frappe/printing/doctype/letter_head/letter_head.js
@@ -1,8 +1,8 @@
 // Copyright (c) 2017, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Letter Head', {
-	refresh: function(frm) {
+frappe.ui.form.on("Letter Head", {
+	refresh: function (frm) {
 		frm.flag_public_attachments = true;
-	}
+	},
 });
diff --git a/frappe/printing/doctype/letter_head/letter_head.py b/frappe/printing/doctype/letter_head/letter_head.py
index 9edd84a425..c48fd1fe25 100644
--- a/frappe/printing/doctype/letter_head/letter_head.py
+++ b/frappe/printing/doctype/letter_head/letter_head.py
@@ -61,9 +61,12 @@ class LetterHead(Document):
 
 		# To preserve the aspect ratio of the image, apply constraints only on
 		# the greater dimension and allow the other to scale accordingly
-		dimension = "width" if width > height else "height"
+		dimension = "width" if self.get(width) > self.get(height) else "height"
 		dimension_value = self.get(f"{dimension_prefix}{dimension}")
 
+		if not dimension_value:
+			dimension_value = ""
+
 		self.set(
 			html_field,
 			f"""
        
diff --git a/frappe/printing/doctype/letter_head/test_letter_head.py b/frappe/printing/doctype/letter_head/test_letter_head.py
index 75019ce275..b36daeae03 100644
--- a/frappe/printing/doctype/letter_head/test_letter_head.py
+++ b/frappe/printing/doctype/letter_head/test_letter_head.py
@@ -1,11 +1,10 @@
 # Copyright (c) 2017, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
-import unittest
-
 import frappe
+from frappe.tests.utils import FrappeTestCase
 
 
-class TestLetterHead(unittest.TestCase):
+class TestLetterHead(FrappeTestCase):
 	def test_auto_image(self):
 		letter_head = frappe.get_doc(
 			dict(doctype="Letter Head", letter_head_name="Test", source="Image", image="/public/test.png")
diff --git a/frappe/printing/doctype/network_printer_settings/network_printer_settings.js b/frappe/printing/doctype/network_printer_settings/network_printer_settings.js
index 043afd388f..2d094d8038 100644
--- a/frappe/printing/doctype/network_printer_settings/network_printer_settings.js
+++ b/frappe/printing/doctype/network_printer_settings/network_printer_settings.js
@@ -1,29 +1,29 @@
 // Copyright (c) 2021, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Network Printer Settings', {
-	onload (frm) {
+frappe.ui.form.on("Network Printer Settings", {
+	onload(frm) {
 		frm.trigger("connect_print_server");
 	},
-	server_ip (frm) {
+	server_ip(frm) {
 		frm.trigger("connect_print_server");
 	},
-	port (frm) {
+	port(frm) {
 		frm.trigger("connect_print_server");
 	},
-	connect_print_server (frm) {
+	connect_print_server(frm) {
 		if (frm.doc.server_ip && frm.doc.port) {
 			frappe.call({
-				"doc": frm.doc,
-				"method": "get_printers_list",
-				"args": {
+				doc: frm.doc,
+				method: "get_printers_list",
+				args: {
 					ip: frm.doc.server_ip,
-					port: frm.doc.port
+					port: frm.doc.port,
+				},
+				callback: function (data) {
+					frm.set_df_property("printer_name", "options", [""].concat(data.message));
 				},
-				callback: function(data) {
-					frm.set_df_property('printer_name', 'options', [""].concat(data.message));
-				}
 			});
 		}
-	}
+	},
 });
diff --git a/frappe/printing/doctype/network_printer_settings/test_network_printer_settings.py b/frappe/printing/doctype/network_printer_settings/test_network_printer_settings.py
index 17ce16cb82..de04b5277b 100644
--- a/frappe/printing/doctype/network_printer_settings/test_network_printer_settings.py
+++ b/frappe/printing/doctype/network_printer_settings/test_network_printer_settings.py
@@ -2,8 +2,8 @@
 # See license.txt
 
 # import frappe
-import unittest
+from frappe.tests.utils import FrappeTestCase
 
 
-class TestNetworkPrinterSettings(unittest.TestCase):
+class TestNetworkPrinterSettings(FrappeTestCase):
 	pass
diff --git a/frappe/printing/doctype/print_format/print_format.js b/frappe/printing/doctype/print_format/print_format.js
index 3fd1d9d148..64cd36a727 100644
--- a/frappe/printing/doctype/print_format/print_format.js
+++ b/frappe/printing/doctype/print_format/print_format.js
@@ -17,9 +17,9 @@ frappe.ui.form.on("Print Format", {
 		if (frm.doc.standard === "Yes" && frappe.session.user !== "Administrator") {
 			frm.set_intro(__("Please duplicate this to make changes"));
 		}
-		frm.trigger('render_buttons');
-		frm.toggle_display('standard', frappe.boot.developer_mode);
-		frm.trigger('hide_absolute_value_field');
+		frm.trigger("render_buttons");
+		frm.toggle_display("standard", frappe.boot.developer_mode);
+		frm.trigger("hide_absolute_value_field");
 	},
 	render_buttons: function (frm) {
 		frm.page.clear_inner_toolbar();
@@ -36,38 +36,40 @@ frappe.ui.form.on("Print Format", {
 						frappe.set_route("print-format-builder", frm.doc.name);
 					}
 				});
-			}
-			else if (frm.doc.custom_format && !frm.doc.raw_printing) {
+			} else if (frm.doc.custom_format && !frm.doc.raw_printing) {
 				frm.set_df_property("html", "reqd", 1);
 			}
-			if (frappe.model.can_read(frm.doc.doc_type)) {
-				frappe.db.get_value('DocType', frm.doc.doc_type, 'default_print_format', (r) => {
-					if (r.default_print_format != frm.doc.name) {
-						frm.add_custom_button(__("Set as Default"), function () {
-							frappe.call({
-								method: "frappe.printing.doctype.print_format.print_format.make_default",
-								args: {
-									name: frm.doc.name
-								},
-								callback: function() {
-									frm.refresh();
-								}
-							});
-						});
+			if (frappe.model.can_write("Customize Form")) {
+				frappe.model.with_doctype(frm.doc.doc_type, function () {
+					let current_format = frappe.get_meta(frm.doc.doc_type).default_print_format;
+					if (current_format == frm.doc.name) {
+						return;
 					}
+
+					frm.add_custom_button(__("Set as Default"), function () {
+						frappe.call({
+							method: "frappe.printing.doctype.print_format.print_format.make_default",
+							args: {
+								name: frm.doc.name,
+							},
+							callback: function () {
+								frm.refresh();
+							},
+						});
+					});
 				});
 			}
 		}
 	},
 	custom_format: function (frm) {
 		var value = frm.doc.custom_format ? 0 : 1;
-		frm.set_value('align_labels_right', value);
-		frm.set_value('show_section_headings', value);
-		frm.set_value('line_breaks', value);
-		frm.trigger('render_buttons');
+		frm.set_value("align_labels_right", value);
+		frm.set_value("show_section_headings", value);
+		frm.set_value("line_breaks", value);
+		frm.trigger("render_buttons");
 	},
 	doc_type: function (frm) {
-		frm.trigger('hide_absolute_value_field');
+		frm.trigger("hide_absolute_value_field");
 	},
 	hide_absolute_value_field: function (frm) {
 		// TODO: make it work with frm.doc.doc_type
@@ -76,9 +78,11 @@ frappe.ui.form.on("Print Format", {
 		if (doctype) {
 			frappe.model.with_doctype(doctype, () => {
 				const meta = frappe.get_meta(doctype);
-				const has_int_float_currency_field = meta.fields.filter(df => in_list(['Int', 'Float', 'Currency'], df.fieldtype));
-				frm.toggle_display('absolute_value', has_int_float_currency_field.length);
+				const has_int_float_currency_field = meta.fields.filter((df) =>
+					in_list(["Int", "Float", "Currency"], df.fieldtype)
+				);
+				frm.toggle_display("absolute_value", has_int_float_currency_field.length);
 			});
 		}
-	}
+	},
 });
diff --git a/frappe/printing/doctype/print_format/print_format.json b/frappe/printing/doctype/print_format/print_format.json
index 75ec0fa7fd..7f4408c950 100644
--- a/frappe/printing/doctype/print_format/print_format.json
+++ b/frappe/printing/doctype/print_format/print_format.json
@@ -257,9 +257,8 @@
  ],
  "icon": "fa fa-print",
  "idx": 1,
- "index_web_pages_for_search": 1,
  "links": [],
- "modified": "2021-10-12 17:52:41.167107",
+ "modified": "2022-11-09 15:29:46.709305",
  "modified_by": "Administrator",
  "module": "Printing",
  "name": "Print Format",
@@ -276,9 +275,14 @@
    "role": "System Manager",
    "share": 1,
    "write": 1
+  },
+  {
+   "role": "All",
+   "select": 1
   }
  ],
  "sort_field": "modified",
  "sort_order": "DESC",
+ "states": [],
  "track_changes": 1
 }
\ No newline at end of file
diff --git a/frappe/printing/doctype/print_format/print_format.py b/frappe/printing/doctype/print_format/print_format.py
index e49db67512..599a16bee3 100644
--- a/frappe/printing/doctype/print_format/print_format.py
+++ b/frappe/printing/doctype/print_format/print_format.py
@@ -13,7 +13,7 @@ from frappe.utils.weasyprint import download_pdf, get_html
 
 class PrintFormat(Document):
 	def onload(self):
-		templates = frappe.db.get_all(
+		templates = frappe.get_all(
 			"Print Format Field Template",
 			fields=["template", "field", "name"],
 			filters={"document_type": self.doc_type},
@@ -95,7 +95,7 @@ class PrintFormat(Document):
 	def export_doc(self):
 		from frappe.modules.utils import export_module_json
 
-		export_module_json(self, self.standard == "Yes", self.module)
+		return export_module_json(self, self.standard == "Yes", self.module)
 
 	def on_trash(self):
 		if self.doc_type:
@@ -109,13 +109,12 @@ def make_default(name):
 
 	print_format = frappe.get_doc("Print Format", name)
 
-	if (frappe.conf.get("developer_mode") or 0) == 1:
-		# developer mode, set it default in doctype
-		doctype = frappe.get_doc("DocType", print_format.doc_type)
+	doctype = frappe.get_doc("DocType", print_format.doc_type)
+	if doctype.custom:
 		doctype.default_print_format = name
 		doctype.save()
 	else:
-		# customization
+		# "Customize form"
 		frappe.make_property_setter(
 			{
 				"doctype_or_field": "DocType",
diff --git a/frappe/printing/doctype/print_format/test_print_format.py b/frappe/printing/doctype/print_format/test_print_format.py
index fbfeecb3ab..7caa5c6102 100644
--- a/frappe/printing/doctype/print_format/test_print_format.py
+++ b/frappe/printing/doctype/print_format/test_print_format.py
@@ -1,14 +1,19 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
+import os
 import re
-import unittest
+from typing import TYPE_CHECKING
 
 import frappe
+from frappe.tests.utils import FrappeTestCase
+
+if TYPE_CHECKING:
+	from frappe.printing.doctype.print_format.print_format import PrintFormat
 
 test_records = frappe.get_test_records("Print Format")
 
 
-class TestPrintFormat(unittest.TestCase):
+class TestPrintFormat(FrappeTestCase):
 	def test_print_user(self, style=None):
 		print_html = frappe.get_print("User", "Administrator", style=style)
 		self.assertTrue("" in print_html)
@@ -30,3 +35,29 @@ class TestPrintFormat(unittest.TestCase):
 	def test_print_user_classic(self):
 		print_html = self.test_print_user("Classic")
 		self.assertTrue("/* classic format: for-test */" in print_html)
+
+	def test_export_doc(self):
+		doc: "PrintFormat" = frappe.get_doc("Print Format", test_records[0]["name"])
+
+		# this is only to make export_doc happy
+		doc.standard = "Yes"
+		_before = frappe.conf.developer_mode
+		frappe.conf.developer_mode = True
+		export_path = doc.export_doc()
+		frappe.conf.developer_mode = _before
+
+		exported_doc_path = f"{export_path}.json"
+		doc.reload()
+		doc_dict = doc.as_dict(no_nulls=True, convert_dates_to_str=True)
+
+		self.assertTrue(os.path.exists(exported_doc_path))
+
+		with open(exported_doc_path) as f:
+			exported_doc = frappe.parse_json(f.read())
+
+		for key, value in exported_doc.items():
+			if key in doc_dict:
+				with self.subTest(key=key):
+					self.assertEqual(value, doc_dict[key])
+
+		self.addCleanup(os.remove, exported_doc_path)
diff --git a/frappe/printing/doctype/print_format_field_template/print_format_field_template.js b/frappe/printing/doctype/print_format_field_template/print_format_field_template.js
index 7fbb0d7359..4aa00ae5e7 100644
--- a/frappe/printing/doctype/print_format_field_template/print_format_field_template.js
+++ b/frappe/printing/doctype/print_format_field_template/print_format_field_template.js
@@ -1,8 +1,7 @@
 // Copyright (c) 2021, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Print Format Field Template', {
+frappe.ui.form.on("Print Format Field Template", {
 	// refresh: function(frm) {
-
 	// }
 });
diff --git a/frappe/printing/doctype/print_format_field_template/print_format_field_template.py b/frappe/printing/doctype/print_format_field_template/print_format_field_template.py
index 428979b3ad..cc938da34f 100644
--- a/frappe/printing/doctype/print_format_field_template/print_format_field_template.py
+++ b/frappe/printing/doctype/print_format_field_template/print_format_field_template.py
@@ -27,7 +27,7 @@ class PrintFormatFieldTemplate(Document):
 		filters = {"document_type": self.document_type, "field": self.field}
 		if not self.is_new():
 			filters.update({"name": ("!=", self.name)})
-		result = frappe.db.get_all("Print Format Field Template", filters=filters, limit=1)
+		result = frappe.get_all("Print Format Field Template", filters=filters, limit=1)
 		if result:
 			frappe.throw(
 				_("A template already exists for field {0} of {1}").format(
diff --git a/frappe/printing/doctype/print_format_field_template/test_print_format_field_template.py b/frappe/printing/doctype/print_format_field_template/test_print_format_field_template.py
index b55a492635..74f495bd3a 100644
--- a/frappe/printing/doctype/print_format_field_template/test_print_format_field_template.py
+++ b/frappe/printing/doctype/print_format_field_template/test_print_format_field_template.py
@@ -2,8 +2,8 @@
 # See license.txt
 
 # import frappe
-import unittest
+from frappe.tests.utils import FrappeTestCase
 
 
-class TestPrintFormatFieldTemplate(unittest.TestCase):
+class TestPrintFormatFieldTemplate(FrappeTestCase):
 	pass
diff --git a/frappe/printing/doctype/print_heading/print_heading.js b/frappe/printing/doctype/print_heading/print_heading.js
index 39f26a2e0f..3a2c615363 100644
--- a/frappe/printing/doctype/print_heading/print_heading.js
+++ b/frappe/printing/doctype/print_heading/print_heading.js
@@ -1,8 +1,6 @@
 // Copyright (c) 2017, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Print Heading', {
-	refresh: function(frm) {
-
-	}
+frappe.ui.form.on("Print Heading", {
+	refresh: function (frm) {},
 });
diff --git a/frappe/printing/doctype/print_heading/print_heading.json b/frappe/printing/doctype/print_heading/print_heading.json
index 418429f29e..9951b533cb 100644
--- a/frappe/printing/doctype/print_heading/print_heading.json
+++ b/frappe/printing/doctype/print_heading/print_heading.json
@@ -1,145 +1,143 @@
 {
- "allow_copy": 0, 
- "allow_guest_to_view": 0, 
- "allow_import": 1, 
- "allow_rename": 1, 
- "autoname": "field:print_heading", 
- "beta": 0, 
- "creation": "2013-01-10 16:34:24", 
- "custom": 0, 
- "docstatus": 0, 
- "doctype": "DocType", 
- "document_type": "Setup", 
- "editable_grid": 0, 
+ "allow_copy": 0,
+ "allow_guest_to_view": 0,
+ "allow_import": 1,
+ "allow_rename": 1,
+ "autoname": "field:print_heading",
+ "beta": 0,
+ "creation": "2013-01-10 16:34:24",
+ "custom": 0,
+ "docstatus": 0,
+ "doctype": "DocType",
+ "document_type": "Setup",
+ "editable_grid": 0,
  "fields": [
   {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 1, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "print_heading", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 1, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Print Heading", 
-   "length": 0, 
-   "no_copy": 0, 
-   "oldfieldname": "print_heading", 
-   "oldfieldtype": "Data", 
-   "permlevel": 0, 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
+   "allow_bulk_edit": 0,
+   "allow_on_submit": 1,
+   "bold": 0,
+   "collapsible": 0,
+   "columns": 0,
+   "fieldname": "print_heading",
+   "fieldtype": "Data",
+   "hidden": 0,
+   "ignore_user_permissions": 0,
+   "ignore_xss_filter": 0,
+   "in_filter": 1,
+   "in_global_search": 0,
+   "in_list_view": 1,
+   "in_standard_filter": 0,
+   "label": "Print Heading",
+   "length": 0,
+   "no_copy": 0,
+   "oldfieldname": "print_heading",
+   "oldfieldtype": "Data",
+   "permlevel": 0,
+   "print_hide": 0,
+   "print_hide_if_no_value": 0,
+   "read_only": 0,
+   "remember_last_selected_value": 0,
+   "report_hide": 0,
+   "reqd": 1,
+   "search_index": 0,
+   "set_only_once": 0,
    "unique": 0
-  }, 
+  },
   {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "description", 
-   "fieldtype": "Small Text", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Description", 
-   "length": 0, 
-   "no_copy": 0, 
-   "oldfieldname": "description", 
-   "oldfieldtype": "Small Text", 
-   "permlevel": 0, 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0, 
+   "allow_bulk_edit": 0,
+   "allow_on_submit": 0,
+   "bold": 0,
+   "collapsible": 0,
+   "columns": 0,
+   "fieldname": "description",
+   "fieldtype": "Small Text",
+   "hidden": 0,
+   "ignore_user_permissions": 0,
+   "ignore_xss_filter": 0,
+   "in_filter": 0,
+   "in_global_search": 0,
+   "in_list_view": 1,
+   "in_standard_filter": 0,
+   "label": "Description",
+   "length": 0,
+   "no_copy": 0,
+   "oldfieldname": "description",
+   "oldfieldtype": "Small Text",
+   "permlevel": 0,
+   "print_hide": 0,
+   "print_hide_if_no_value": 0,
+   "read_only": 0,
+   "remember_last_selected_value": 0,
+   "report_hide": 0,
+   "reqd": 0,
+   "search_index": 0,
+   "set_only_once": 0,
+   "unique": 0,
    "width": "300px"
   }
- ], 
- "has_web_view": 0, 
- "hide_heading": 0, 
- "hide_toolbar": 0, 
- "icon": "fa fa-font", 
- "idx": 1, 
- "image_view": 0, 
- "in_create": 0, 
- "is_submittable": 0, 
- "issingle": 0, 
- "istable": 0, 
- "max_attachments": 0, 
- "modified": "2017-05-03 05:59:09.131569", 
- "modified_by": "Administrator", 
- "module": "Printing", 
- "name": "Print Heading", 
- "owner": "Administrator", 
+ ],
+ "has_web_view": 0,
+ "hide_heading": 0,
+ "hide_toolbar": 0,
+ "icon": "fa fa-font",
+ "idx": 1,
+ "image_view": 0,
+ "in_create": 0,
+ "is_submittable": 0,
+ "issingle": 0,
+ "istable": 0,
+ "max_attachments": 0,
+ "modified": "2017-05-03 05:59:09.131569",
+ "modified_by": "Administrator",
+ "module": "Printing",
+ "name": "Print Heading",
+ "owner": "Administrator",
  "permissions": [
   {
-   "amend": 0, 
-   "apply_user_permissions": 0, 
-   "cancel": 0, 
-   "create": 1, 
-   "delete": 1, 
-   "email": 1, 
-   "export": 0, 
-   "if_owner": 0, 
-   "import": 0, 
-   "permlevel": 0, 
-   "print": 1, 
-   "read": 1, 
-   "report": 1, 
-   "role": "System Manager", 
-   "set_user_permissions": 0, 
-   "share": 1, 
-   "submit": 0, 
+   "amend": 0,
+   "apply_user_permissions": 0,
+   "cancel": 0,
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "export": 0,
+   "if_owner": 0,
+   "import": 0,
+   "permlevel": 0,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "System Manager",
+   "share": 1,
+   "submit": 0,
    "write": 1
-  }, 
+  },
   {
-   "amend": 0, 
-   "apply_user_permissions": 0, 
-   "cancel": 0, 
-   "create": 0, 
-   "delete": 0, 
-   "email": 0, 
-   "export": 0, 
-   "if_owner": 0, 
-   "import": 0, 
-   "permlevel": 0, 
-   "print": 0, 
-   "read": 1, 
-   "report": 0, 
-   "role": "All", 
-   "set_user_permissions": 0, 
-   "share": 0, 
-   "submit": 0, 
+   "amend": 0,
+   "apply_user_permissions": 0,
+   "cancel": 0,
+   "create": 0,
+   "delete": 0,
+   "email": 0,
+   "export": 0,
+   "if_owner": 0,
+   "import": 0,
+   "permlevel": 0,
+   "print": 0,
+   "read": 1,
+   "report": 0,
+   "role": "All",
+   "share": 0,
+   "submit": 0,
    "write": 0
   }
- ], 
- "quick_entry": 1, 
- "read_only": 0, 
- "read_only_onload": 0, 
- "search_fields": "print_heading", 
- "show_name_in_global_search": 0, 
- "sort_order": "DESC", 
- "track_changes": 0, 
+ ],
+ "quick_entry": 1,
+ "read_only": 0,
+ "read_only_onload": 0,
+ "search_fields": "print_heading",
+ "show_name_in_global_search": 0,
+ "sort_order": "DESC",
+ "track_changes": 0,
  "track_seen": 0
-}
\ No newline at end of file
+}
diff --git a/frappe/printing/doctype/print_heading/test_print_heading.py b/frappe/printing/doctype/print_heading/test_print_heading.py
index 74ff7ce74f..8fab30e79f 100644
--- a/frappe/printing/doctype/print_heading/test_print_heading.py
+++ b/frappe/printing/doctype/print_heading/test_print_heading.py
@@ -1,9 +1,8 @@
 # Copyright (c) 2017, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
-import unittest
-
 import frappe
+from frappe.tests.utils import FrappeTestCase
 
 
-class TestPrintHeading(unittest.TestCase):
+class TestPrintHeading(FrappeTestCase):
 	pass
diff --git a/frappe/printing/doctype/print_settings/print_settings.js b/frappe/printing/doctype/print_settings/print_settings.js
index b1311166ee..dc939c298d 100644
--- a/frappe/printing/doctype/print_settings/print_settings.js
+++ b/frappe/printing/doctype/print_settings/print_settings.js
@@ -1,19 +1,23 @@
 // Copyright (c) 2018, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Print Settings', {
-	print_style: function(frm) {
-		frappe.db.get_value('Print Style', frm.doc.print_style, 'preview').then((r) => {
-			if(r.message.preview) {
+frappe.ui.form.on("Print Settings", {
+	print_style: function (frm) {
+		frappe.db.get_value("Print Style", frm.doc.print_style, "preview").then((r) => {
+			if (r.message.preview) {
 				frm.get_field("print_style_preview").$wrapper.html(
-					``);
+					``
+				);
 			} else {
 				frm.get_field("print_style_preview").$wrapper.html(
-					`
        ${__("No Preview")}
        `);
+					`
        ${__(
+						"No Preview"
+					)}
        `
+				);
 			}
 		});
 	},
-	onload: function(frm) {
+	onload: function (frm) {
 		frm.script_manager.trigger("print_style");
-	}
+	},
 });
diff --git a/frappe/printing/doctype/print_settings/test_print_settings.py b/frappe/printing/doctype/print_settings/test_print_settings.py
index 6a6437bf97..80069824ab 100644
--- a/frappe/printing/doctype/print_settings/test_print_settings.py
+++ b/frappe/printing/doctype/print_settings/test_print_settings.py
@@ -1,7 +1,7 @@
 # Copyright (c) 2018, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
-import unittest
+from frappe.tests.utils import FrappeTestCase
 
 
-class TestPrintSettings(unittest.TestCase):
+class TestPrintSettings(FrappeTestCase):
 	pass
diff --git a/frappe/printing/doctype/print_style/print_style.js b/frappe/printing/doctype/print_style/print_style.js
index 44c4a528f4..3177e1aa09 100644
--- a/frappe/printing/doctype/print_style/print_style.js
+++ b/frappe/printing/doctype/print_style/print_style.js
@@ -1,10 +1,10 @@
 // Copyright (c) 2017, Frappe Technologies and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Print Style', {
-	refresh: function(frm) {
-		frm.add_custom_button(__('Print Settings'), () => {
-			frappe.set_route('Form', 'Print Settings');
-		})
-	}
+frappe.ui.form.on("Print Style", {
+	refresh: function (frm) {
+		frm.add_custom_button(__("Print Settings"), () => {
+			frappe.set_route("Form", "Print Settings");
+		});
+	},
 });
diff --git a/frappe/printing/doctype/print_style/print_style.json b/frappe/printing/doctype/print_style/print_style.json
index 29e88a460a..1d3c9a6189 100644
--- a/frappe/printing/doctype/print_style/print_style.json
+++ b/frappe/printing/doctype/print_style/print_style.json
@@ -1,214 +1,75 @@
 {
- "allow_copy": 0, 
- "allow_guest_to_view": 0, 
- "allow_import": 0, 
- "allow_rename": 1, 
- "autoname": "field:print_style_name", 
- "beta": 0, 
- "creation": "2017-08-17 01:25:56.910716", 
- "custom": 0, 
- "docstatus": 0, 
- "doctype": "DocType", 
- "document_type": "", 
- "editable_grid": 1, 
- "engine": "InnoDB", 
+ "actions": [],
+ "allow_rename": 1,
+ "autoname": "field:print_style_name",
+ "creation": "2017-08-17 01:25:56.910716",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "print_style_name",
+  "disabled",
+  "standard",
+  "css",
+  "preview"
+ ],
  "fields": [
   {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "print_style_name", 
-   "fieldtype": "Data", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Print Style Name", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "fieldname": "print_style_name",
+   "fieldtype": "Data",
+   "label": "Print Style Name",
+   "reqd": 1,
+   "unique": 1
+  },
   {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "disabled", 
-   "fieldtype": "Check", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 1, 
-   "in_standard_filter": 0, 
-   "label": "Disabled", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "default": "0",
+   "fieldname": "disabled",
+   "fieldtype": "Check",
+   "in_list_view": 1,
+   "label": "Disabled"
+  },
   {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "standard", 
-   "fieldtype": "Check", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Standard", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "default": "0",
+   "fieldname": "standard",
+   "fieldtype": "Check",
+   "label": "Standard"
+  },
   {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "css", 
-   "fieldtype": "Code", 
-   "hidden": 0, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "CSS", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 1, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
-  }, 
+   "fieldname": "css",
+   "fieldtype": "Code",
+   "label": "CSS",
+   "reqd": 1
+  },
   {
-   "allow_bulk_edit": 0, 
-   "allow_on_submit": 0, 
-   "bold": 0, 
-   "collapsible": 0, 
-   "columns": 0, 
-   "fieldname": "preview", 
-   "fieldtype": "Attach Image", 
-   "hidden": 1, 
-   "ignore_user_permissions": 0, 
-   "ignore_xss_filter": 0, 
-   "in_filter": 0, 
-   "in_global_search": 0, 
-   "in_list_view": 0, 
-   "in_standard_filter": 0, 
-   "label": "Preview", 
-   "length": 0, 
-   "no_copy": 0, 
-   "permlevel": 0, 
-   "precision": "", 
-   "print_hide": 0, 
-   "print_hide_if_no_value": 0, 
-   "read_only": 0, 
-   "remember_last_selected_value": 0, 
-   "report_hide": 0, 
-   "reqd": 0, 
-   "search_index": 0, 
-   "set_only_once": 0, 
-   "unique": 0
+   "fieldname": "preview",
+   "fieldtype": "Attach Image",
+   "hidden": 1,
+   "label": "Preview"
   }
- ], 
- "has_web_view": 0, 
- "hide_heading": 0, 
- "hide_toolbar": 0, 
- "idx": 0, 
- "image_field": "preview", 
- "image_view": 0, 
- "in_create": 0, 
- "is_submittable": 0, 
- "issingle": 0, 
- "istable": 0, 
- "max_attachments": 0, 
- "modified": "2017-08-17 02:18:08.132853", 
- "modified_by": "Administrator", 
- "module": "Printing", 
- "name": "Print Style", 
- "name_case": "", 
- "owner": "Administrator", 
+ ],
+ "image_field": "preview",
+ "links": [],
+ "modified": "2022-08-03 12:20:51.295775",
+ "modified_by": "Administrator",
+ "module": "Printing",
+ "name": "Print Style",
+ "owner": "Administrator",
  "permissions": [
   {
-   "amend": 0, 
-   "apply_user_permissions": 0, 
-   "cancel": 0, 
-   "create": 1, 
-   "delete": 1, 
-   "email": 1, 
-   "export": 1, 
-   "if_owner": 0, 
-   "import": 0, 
-   "permlevel": 0, 
-   "print": 1, 
-   "read": 1, 
-   "report": 1, 
-   "role": "System Manager", 
-   "set_user_permissions": 0, 
-   "share": 1, 
-   "submit": 0, 
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "export": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "System Manager",
+   "share": 1,
    "write": 1
   }
- ], 
- "quick_entry": 0, 
- "read_only": 0, 
- "read_only_onload": 0, 
- "show_name_in_global_search": 0, 
- "sort_field": "modified", 
- "sort_order": "DESC", 
- "track_changes": 1, 
- "track_seen": 0
+ ],
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "states": [],
+ "track_changes": 1
 }
\ No newline at end of file
diff --git a/frappe/printing/doctype/print_style/test_print_style.py b/frappe/printing/doctype/print_style/test_print_style.py
index f8ce54b9bb..744181ad3e 100644
--- a/frappe/printing/doctype/print_style/test_print_style.py
+++ b/frappe/printing/doctype/print_style/test_print_style.py
@@ -1,9 +1,8 @@
 # Copyright (c) 2017, Frappe Technologies and Contributors
 # License: MIT. See LICENSE
-import unittest
-
 import frappe
+from frappe.tests.utils import FrappeTestCase
 
 
-class TestPrintStyle(unittest.TestCase):
+class TestPrintStyle(FrappeTestCase):
 	pass
diff --git a/frappe/printing/page/print/print.js b/frappe/printing/page/print/print.js
index 7db6930a60..8e5e165c78 100644
--- a/frappe/printing/page/print/print.js
+++ b/frappe/printing/page/print/print.js
@@ -1,11 +1,11 @@
-frappe.pages['print'].on_page_load = function(wrapper) {
+frappe.pages["print"].on_page_load = function (wrapper) {
 	frappe.ui.make_app_page({
 		parent: wrapper,
 	});
 
 	let print_view = new frappe.ui.form.PrintView(wrapper);
 
-	$(wrapper).bind('show', () => {
+	$(wrapper).bind("show", () => {
 		const route = frappe.get_route();
 		const doctype = route[1];
 		const docname = route.slice(2).join("/");
@@ -19,8 +19,9 @@ frappe.pages['print'].on_page_load = function(wrapper) {
 				});
 			});
 		} else {
-			print_view.frm = frappe.route_options.frm.doctype ?
-				frappe.route_options.frm : frappe.route_options.frm.frm;
+			print_view.frm = frappe.route_options.frm.doctype
+				? frappe.route_options.frm
+				: frappe.route_options.frm.frm;
 			frappe.route_options.frm = null;
 			print_view.show(print_view.frm);
 		}
@@ -37,7 +38,7 @@ frappe.ui.form.PrintView = class {
 	make() {
 		this.print_wrapper = this.page.main.empty().html(
 			`
        
-				${frappe.render_template('print_skeleton_loading')}
+				${frappe.render_template("print_skeleton_loading")}
 				
 			
        
@@ -49,10 +50,7 @@ frappe.ui.form.PrintView = class {
 		`
 		);
 
-		this.print_settings = frappe.model.get_doc(
-			':Print Settings',
-			'Print Settings'
-		);
+		this.print_settings = frappe.model.get_doc(":Print Settings", "Print Settings");
 		this.setup_menu();
 		this.setup_toolbar();
 		this.setup_sidebar();
@@ -64,87 +62,68 @@ frappe.ui.form.PrintView = class {
 	}
 
 	setup_toolbar() {
-		this.page.set_primary_action(
-			__('Print'),
-			() => this.printit(), 'printer'
-		);
+		this.page.set_primary_action(__("Print"), () => this.printit(), "printer");
 
-		this.page.add_button(
-			__('Full Page'),
-			() => this.render_page('/printview?'),
-			{ icon: 'full-page' }
-		);
+		this.page.add_button(__("Full Page"), () => this.render_page("/printview?"), {
+			icon: "full-page",
+		});
 
-		this.page.add_button(
-			__('PDF'),
-			() => this.render_pdf(),
-			{ icon: 'small-file' }
-		);
+		this.page.add_button(__("PDF"), () => this.render_pdf(), { icon: "small-file" });
 
-		this.page.add_button(
-			__('Refresh'),
-			() => this.refresh_print_format(),
-			{ icon: 'refresh' }
-		);
+		this.page.add_button(__("Refresh"), () => this.refresh_print_format(), {
+			icon: "refresh",
+		});
 
-		this.page.add_action_icon("file", () => {
-			this.go_to_form_view();
-		}, '', __("Form"));
+		this.page.add_action_icon(
+			"file",
+			() => {
+				this.go_to_form_view();
+			},
+			"",
+			__("Form")
+		);
 	}
 
 	setup_sidebar() {
-		this.sidebar = this.page.sidebar.addClass('print-preview-sidebar');
+		this.sidebar = this.page.sidebar.addClass("print-preview-sidebar");
 
-		this.print_sel = this.add_sidebar_item(
-			{
-				fieldtype: 'Select',
-				fieldname: 'print_format',
-				label: 'Print Format',
-				options: [this.get_default_option_for_select(__('Select Print Format'))],
-				change: () => this.refresh_print_format(),
-				default: __('Select Print Format')
+		this.print_format_selector = this.add_sidebar_item({
+			fieldtype: "Link",
+			fieldname: "print_format",
+			options: "Print Format",
+			placeholder: __("Print Format"),
+			get_query: () => {
+				return { filters: { doc_type: this.frm.doctype } };
 			},
-		).$input;
+			change: () => this.refresh_print_format(),
+		}).$input;
 
-		this.language_sel = this.add_sidebar_item(
-			{
-				fieldtype: 'Select',
-				fieldname: 'language',
-				placeholder: 'Language',
-				options: [
-					this.get_default_option_for_select(__('Select Language')),
-					...this.get_language_options()
-				],
-				default: __('Select Language'),
-				change: () => {
-					this.set_user_lang();
-					this.preview();
-				},
+		this.language_selector = this.add_sidebar_item({
+			fieldtype: "Link",
+			fieldname: "language",
+			placeholder: __("Language"),
+			options: "Language",
+			change: () => {
+				this.set_user_lang();
+				this.preview();
 			},
-		).$input;
+		}).$input;
 
-		this.letterhead_selector_df = this.add_sidebar_item(
-			{
-				fieldtype: 'Autocomplete',
-				fieldname: 'letterhead',
-				label: __('Select Letterhead'),
-				placeholder: __('Select Letterhead'),
-				options: [__('No Letterhead')],
-				change: () => this.preview(),
-				default: this.print_settings.with_letterhead
-					? __('No Letterhead')
-					: __('Select Letterhead')
-			},
+		this.letterhead_selector = this.add_sidebar_item({
+			fieldtype: "Link",
+			fieldname: "letterhead",
+			options: "Letter Head",
+			placeholder: __("Letter Head"),
+			change: () => this.preview(),
+		}).$input;
+		this.sidebar_dynamic_section = $(`
        `).appendTo(
+			this.sidebar
 		);
-		this.letterhead_selector = this.letterhead_selector_df.$input;
-		this.sidebar_dynamic_section = $(
-			`
        `
-		).appendTo(this.sidebar);
 	}
 
 	add_sidebar_item(df, is_dynamic) {
-		if (df.fieldtype == 'Select') {
-			df.input_class = 'btn btn-default btn-sm text-left';
+		if (df.fieldtype == "Select") {
+			df.input_class = "btn btn-default btn-sm text-left";
 		}
 
 		let field = frappe.ui.form.make_control({
@@ -160,35 +139,25 @@ frappe.ui.form.PrintView = class {
 		return field;
 	}
 
-	get_default_option_for_select(value) {
-		return {
-			label: value,
-			value: value,
-			disabled: true
-		};
-	}
-
 	setup_menu() {
 		this.page.clear_menu();
 
-		this.page.add_menu_item(__('Print Settings'), () => {
-			frappe.set_route('Form', 'Print Settings');
+		this.page.add_menu_item(__("Print Settings"), () => {
+			frappe.set_route("Form", "Print Settings");
 		});
 
-		if (this.print_settings.enable_raw_printing == '1') {
-			this.page.add_menu_item(__('Raw Printing Setting'), () => {
+		if (this.print_settings.enable_raw_printing == "1") {
+			this.page.add_menu_item(__("Raw Printing Setting"), () => {
 				this.printer_setting_dialog();
 			});
 		}
 
-		if (frappe.model.can_create('Print Format')) {
-			this.page.add_menu_item(__('Customize'), () =>
-				this.edit_print_format()
-			);
+		if (frappe.model.can_create("Print Format")) {
+			this.page.add_menu_item(__("Customize"), () => this.edit_print_format());
 		}
 
 		if (cint(this.print_settings.enable_print_server)) {
-			this.page.add_menu_item(__('Select Network Printer'), () =>
+			this.page.add_menu_item(__("Select Network Printer"), () =>
 				this.network_printer_setting_dialog()
 			);
 		}
@@ -203,14 +172,14 @@ frappe.ui.form.PrintView = class {
 		// print format builder beta
 		this.page.add_inner_message(`
 			
-				${__('Try the new Print Format Builder')}
+				${__("Try the new Print Format Builder")}
 			
 		`);
 
 		let tasks = [
-			this.refresh_print_options,
+			this.set_default_print_format,
 			this.set_default_print_language,
-			this.set_letterhead_options,
+			this.set_default_letterhead,
 			this.preview,
 		].map((fn) => fn.bind(this));
 
@@ -226,23 +195,26 @@ frappe.ui.form.PrintView = class {
 		this.additional_settings = {};
 		this.sidebar_dynamic_section.empty();
 		frappe
-			.xcall('frappe.printing.page.print.print.get_print_settings_to_show', {
+			.xcall("frappe.printing.page.print.print.get_print_settings_to_show", {
 				doctype: this.frm.doc.doctype,
-				docname: this.frm.doc.name
+				docname: this.frm.doc.name,
 			})
 			.then((settings) => this.add_settings_to_sidebar(settings));
 	}
 
 	add_settings_to_sidebar(settings) {
 		for (let df of settings) {
-			let field = this.add_sidebar_item({
-				...df,
-				change: () => {
-					const val = field.get_value();
-					this.additional_settings[field.df.fieldname] = val;
-					this.preview();
+			let field = this.add_sidebar_item(
+				{
+					...df,
+					change: () => {
+						const val = field.get_value();
+						this.additional_settings[field.df.fieldname] = val;
+						this.preview();
+					},
 				},
-			}, true);
+				true
+			);
 		}
 	}
 
@@ -251,19 +223,18 @@ frappe.ui.form.PrintView = class {
 		let is_custom_format =
 			print_format.name &&
 			(print_format.print_format_builder || print_format.print_format_builder_beta) &&
-			print_format.standard === 'No';
-		let is_standard_but_editable =
-			print_format.name && print_format.custom_format;
+			print_format.standard === "No";
+		let is_standard_but_editable = print_format.name && print_format.custom_format;
 
 		if (is_standard_but_editable) {
-			frappe.set_route('Form', 'Print Format', print_format.name);
+			frappe.set_route("Form", "Print Format", print_format.name);
 			return;
 		}
 		if (is_custom_format) {
 			if (print_format.print_format_builder_beta) {
-				frappe.set_route('print-format-builder-beta', print_format.name);
+				frappe.set_route("print-format-builder-beta", print_format.name);
 			} else {
-				frappe.set_route('print-format-builder', print_format.name);
+				frappe.set_route("print-format-builder", print_format.name);
 			}
 			return;
 		}
@@ -271,21 +242,21 @@ frappe.ui.form.PrintView = class {
 		frappe.prompt(
 			[
 				{
-					label: __('New Print Format Name'),
-					fieldname: 'print_format_name',
-					fieldtype: 'Data',
+					label: __("New Print Format Name"),
+					fieldname: "print_format_name",
+					fieldtype: "Data",
 					reqd: 1,
 				},
 				{
-					label: __('Based On'),
-					fieldname: 'based_on',
-					fieldtype: 'Read Only',
-					default: print_format.name || 'Standard',
+					label: __("Based On"),
+					fieldname: "based_on",
+					fieldtype: "Read Only",
+					default: print_format.name || "Standard",
 				},
 				{
-					label: __('Use the new Print Format Builder'),
-					fieldname: 'beta',
-					fieldtype: 'Check'
+					label: __("Use the new Print Format Builder"),
+					fieldname: "beta",
+					fieldtype: "Check",
 				},
 			],
 			(data) => {
@@ -294,13 +265,13 @@ frappe.ui.form.PrintView = class {
 					doctype: this.frm.doctype,
 					name: data.print_format_name,
 					based_on: data.based_on,
-					beta: data.beta
+					beta: data.beta,
 				};
-				frappe.set_route('print-format-builder');
-				this.print_sel.val(data.print_format_name);
+				frappe.set_route("print-format-builder");
+				this.print_format_selector.val(data.print_format_name);
 			},
-			__('New Custom Print Format'),
-			__('Start')
+			__("New Custom Print Format"),
+			__("Start")
 		);
 	}
 
@@ -321,25 +292,20 @@ frappe.ui.form.PrintView = class {
 
 	setup_customize_dialog() {
 		let print_format = this.get_print_format();
-		$(document).on('new-print-format', (e) => {
-			this.refresh_print_options();
-			if (e.print_format) {
-				this.print_sel.val(e.print_format);
-			}
-			// start a new print format
+		$(document).on("new-print-format", (e) => {
 			frappe.prompt(
 				[
 					{
-						label: __('New Print Format Name'),
-						fieldname: 'print_format_name',
-						fieldtype: 'Data',
+						label: __("New Print Format Name"),
+						fieldname: "print_format_name",
+						fieldtype: "Data",
 						reqd: 1,
 					},
 					{
-						label: __('Based On'),
-						fieldname: 'based_on',
-						fieldtype: 'Read Only',
-						default: print_format.name || 'Standard',
+						label: __("Based On"),
+						fieldname: "based_on",
+						fieldtype: "Read Only",
+						default: print_format.name || "Standard",
 					},
 				],
 				(data) => {
@@ -349,80 +315,61 @@ frappe.ui.form.PrintView = class {
 						name: data.print_format_name,
 						based_on: data.based_on,
 					};
-					frappe.set_route('print-format-builder');
+					frappe.set_route("print-format-builder");
 				},
-				__('New Custom Print Format'),
-				__('Start')
+				__("New Custom Print Format"),
+				__("Start")
 			);
 		});
 	}
 
 	setup_keyboard_shortcuts() {
-		this.wrapper.find('.print-toolbar a.btn-default').each((i, el) => {
+		this.wrapper.find(".print-toolbar a.btn-default").each((i, el) => {
 			frappe.ui.keys.get_shortcut_group(this.frm.page).add($(el));
 		});
 	}
 
-	set_letterhead_options() {
-		let letterhead_options = [__('No Letterhead')];
-		let default_letterhead;
-		let doc_letterhead = this.frm.doc.letter_head;
+	set_default_letterhead() {
+		if (this.frm.doc.letter_head) {
+			this.letterhead_selector.val(this.frm.doc.letter_head);
+			return;
+		}
 
 		return frappe.db
-			.get_list('Letter Head', {
-				filters: {'disabled': 0},
-				fields: ['name', 'is_default'],
-				limit: 0
-			})
-			.then((letterheads) => {
-				letterheads.map((letterhead) => {
-					if (letterhead.is_default) default_letterhead = letterhead.name;
-					return letterhead_options.push(letterhead.name);
-				});
-
-				this.letterhead_selector_df.set_data(letterhead_options);
-				let selected_letterhead = doc_letterhead || default_letterhead;
-				if (selected_letterhead)
-					this.letterhead_selector.val(selected_letterhead);
-			});
+			.get_value("Letter Head", { disabled: 0, is_default: 1 }, "name")
+			.then(({ message }) => this.letterhead_selector.val(message.name));
 	}
 
 	set_user_lang() {
-		this.lang_code = this.language_sel.val();
-	}
-
-	get_language_options() {
-		return frappe.get_languages();
+		this.lang_code = this.language_selector.val();
 	}
 
 	set_default_print_language() {
 		let print_format = this.get_print_format();
 		this.lang_code =
-			print_format.default_print_language ||
-			this.frm.doc.language ||
-			frappe.boot.lang;
-		this.language_sel.val(this.lang_code);
+			this.frm.doc.language || print_format.default_print_language || frappe.boot.lang;
+		this.language_selector.val(this.lang_code);
 	}
 
 	toggle_raw_printing() {
 		const is_raw_printing = this.is_raw_printing();
-		this.wrapper.find('.btn-print-preview').toggle(!is_raw_printing);
-		this.wrapper.find('.btn-download-pdf').toggle(!is_raw_printing);
+		this.wrapper.find(".btn-print-preview").toggle(!is_raw_printing);
+		this.wrapper.find(".btn-download-pdf").toggle(!is_raw_printing);
 	}
 
 	preview() {
 		let print_format = this.get_print_format();
 		if (print_format.print_format_builder_beta) {
-			this.print_wrapper.find('.print-preview-wrapper').hide();
-			this.print_wrapper.find('.preview-beta-wrapper').show();
+			this.print_wrapper.find(".print-preview-wrapper").hide();
+			this.print_wrapper.find(".preview-beta-wrapper").show();
 			this.preview_beta();
 			return;
 		}
 
-		this.print_wrapper.find('.preview-beta-wrapper').hide();
-		this.print_wrapper.find('.print-preview-wrapper').show();
+		this.print_wrapper.find(".preview-beta-wrapper").hide();
+		this.print_wrapper.find(".print-preview-wrapper").show();
 
-		const $print_format = this.print_wrapper.find('iframe');
+		const $print_format = this.print_wrapper.find("iframe");
 		this.$print_format_body = $print_format.contents();
 		this.get_print_html((out) => {
 			if (!out.html) {
@@ -432,73 +379,78 @@ frappe.ui.form.PrintView = class {
 			this.setup_print_format_dom(out, $print_format);
 
 			const print_height = $print_format.get(0).offsetHeight;
-			const $message = this.wrapper.find('.page-break-message');
+			const $message = this.wrapper.find(".page-break-message");
 
 			const print_height_inches = frappe.dom.pixel_to_inches(print_height);
 			// if contents are large enough, indicate that it will get printed on multiple pages
 			// Maximum height for an A4 document is 11.69 inches
 			if (print_height_inches > 11.69) {
-				$message.text(__('This may get printed on multiple pages'));
+				$message.text(__("This may get printed on multiple pages"));
 			} else {
-				$message.text('');
+				$message.text("");
 			}
 		});
 	}
 
 	preview_beta() {
 		let print_format = this.get_print_format();
-		const iframe = this.print_wrapper.find('.preview-beta-wrapper iframe');
+		const iframe = this.print_wrapper.find(".preview-beta-wrapper iframe");
 		let params = new URLSearchParams({
 			doctype: this.frm.doc.doctype,
 			name: this.frm.doc.name,
-			print_format: print_format.name
+			print_format: print_format.name,
 		});
 		let letterhead = this.get_letterhead();
 		if (letterhead) {
 			params.append("letterhead", letterhead);
 		}
-		iframe.prop('src', `/printpreview?${params.toString()}`);
+		iframe.prop("src", `/printpreview?${params.toString()}`);
 	}
 
 	setup_print_format_dom(out, $print_format) {
-		this.print_wrapper.find('.print-format-skeleton').remove();
+		this.print_wrapper.find(".print-format-skeleton").remove();
 		let base_url = frappe.urllib.get_base_url();
-		let print_css = frappe.assets.bundled_asset('print.bundle.css', frappe.utils.is_rtl(this.lang_code));
-		this.$print_format_body.find('html').attr('dir', frappe.utils.is_rtl(this.lang_code) ? 'rtl': 'ltr');
-		this.$print_format_body.find('html').attr('lang', this.lang_code);
-		this.$print_format_body.find('head').html(
+		let print_css = frappe.assets.bundled_asset(
+			"print.bundle.css",
+			frappe.utils.is_rtl(this.lang_code)
+		);
+		this.$print_format_body
+			.find("html")
+			.attr("dir", frappe.utils.is_rtl(this.lang_code) ? "rtl" : "ltr");
+		this.$print_format_body.find("html").attr("lang", this.lang_code);
+		this.$print_format_body.find("head").html(
 			`
 			`
 		);
 
-		this.$print_format_body.find('body').html(
-			`
        ${out.html}
        `
-		);
+		this.$print_format_body
+			.find("body")
+			.html(`
        ${out.html}
        `);
 
 		this.show_footer();
 
-		this.$print_format_body.find('.print-format').css({
-			display: 'flex',
-			flexDirection: 'column',
+		this.$print_format_body.find(".print-format").css({
+			display: "flex",
+			flexDirection: "column",
 		});
 
-		this.$print_format_body.find('.page-break').css({
-			display: 'flex',
-			'flex-direction': 'column',
-			flex: '1',
+		this.$print_format_body.find(".page-break").css({
+			display: "flex",
+			"flex-direction": "column",
+			flex: "1",
 		});
 
 		setTimeout(() => {
-			$print_format.height(this.$print_format_body.find('.print-format').outerHeight());
+			$print_format.height(this.$print_format_body.find(".print-format").outerHeight());
 		}, 500);
 	}
 
 	hide() {
-		if (this.frm.setup_done && this.frm.page.current_view_name === 'print') {
+		if (this.frm.setup_done && this.frm.page.current_view_name === "print") {
 			this.frm.page.set_view(
-				this.frm.page.previous_view_name === 'print'
-					? 'main'
-					: this.frm.page.previous_view_name || 'main'
+				this.frm.page.previous_view_name === "print"
+					? "main"
+					: this.frm.page.previous_view_name || "main"
 			);
 		}
 	}
@@ -507,15 +459,15 @@ frappe.ui.form.PrintView = class {
 		frappe.route_options = {
 			frm: this,
 		};
-		frappe.set_route('Form', this.frm.doctype, this.frm.docname);
+		frappe.set_route("Form", this.frm.doctype, this.frm.docname);
 	}
 
 	show_footer() {
 		// footer is hidden by default as reqd by pdf generation
 		// simple hack to show it in print preview
 
-		this.$print_format_body.find('#footer-html').attr(
-			'style',
+		this.$print_format_body.find("#footer-html").attr(
+			"style",
 			`
 			display: block !important;
 			order: 1;
@@ -529,7 +481,7 @@ frappe.ui.form.PrintView = class {
 		let me = this;
 
 		if (cint(me.print_settings.enable_print_server)) {
-			if (localStorage.getItem('network_printer')) {
+			if (localStorage.getItem("network_printer")) {
 				me.print_by_server();
 			} else {
 				me.network_printer_setting_dialog(() => me.print_by_server());
@@ -537,10 +489,10 @@ frappe.ui.form.PrintView = class {
 		} else if (me.get_mapped_printer().length === 1) {
 			// printer is already mapped in localstorage (applies for both raw and pdf )
 			if (me.is_raw_printing()) {
-				me.get_raw_commands(function(out) {
+				me.get_raw_commands(function (out) {
 					frappe.ui.form
 						.qz_connect()
-						.then(function() {
+						.then(function () {
 							let printer_map = me.get_mapped_printer()[0];
 							let data = [out.raw_commands];
 							let config = qz.configs.create(printer_map.printer);
@@ -556,9 +508,9 @@ frappe.ui.form.PrintView = class {
 					{
 						message: __('PDF printing via "Raw Print" is not supported.'),
 						subtitle: __(
-							'Please remove the printer mapping in Printer Settings and try again.'
+							"Please remove the printer mapping in Printer Settings and try again."
 						),
-						indicator: 'info',
+						indicator: "info",
 					},
 					14
 				);
@@ -568,61 +520,61 @@ frappe.ui.form.PrintView = class {
 			// printer not mapped in localstorage and the current print format is raw printing
 			frappe.show_alert(
 				{
-					message: __('Printer mapping not set.'),
+					message: __("Printer mapping not set."),
 					subtitle: __(
-						'Please set a printer mapping for this print format in the Printer Settings'
+						"Please set a printer mapping for this print format in the Printer Settings"
 					),
-					indicator: 'warning',
+					indicator: "warning",
 				},
 				14
 			);
 			me.printer_setting_dialog();
 		} else {
-			me.render_page('/printview?', true);
+			me.render_page("/printview?", true);
 		}
 	}
 
 	print_by_server() {
 		let me = this;
-		if (localStorage.getItem('network_printer')) {
+		if (localStorage.getItem("network_printer")) {
 			frappe.call({
-				method: 'frappe.utils.print_format.print_by_server',
+				method: "frappe.utils.print_format.print_by_server",
 				args: {
 					doctype: me.frm.doc.doctype,
 					name: me.frm.doc.name,
-					printer_setting: localStorage.getItem('network_printer'),
+					printer_setting: localStorage.getItem("network_printer"),
 					print_format: me.selected_format(),
 					no_letterhead: me.with_letterhead(),
 					letterhead: me.get_letterhead(),
 				},
-				callback: function() {},
+				callback: function () {},
 			});
 		}
 	}
 	network_printer_setting_dialog(callback) {
 		frappe.call({
-			method: 'frappe.printing.doctype.network_printer_settings.network_printer_settings.get_network_printer_settings',
-			callback: function(r) {
+			method: "frappe.printing.doctype.network_printer_settings.network_printer_settings.get_network_printer_settings",
+			callback: function (r) {
 				if (r.message) {
 					let d = new frappe.ui.Dialog({
-						title: __('Select Network Printer'),
+						title: __("Select Network Printer"),
 						fields: [
 							{
-								"label": "Printer",
-								"fieldname": "printer",
-								"fieldtype": "Select",
-								"reqd": 1,
-								"options": r.message
-							}
+								label: "Printer",
+								fieldname: "printer",
+								fieldtype: "Select",
+								reqd: 1,
+								options: r.message,
+							},
 						],
-						primary_action: function() {
-							localStorage.setItem('network_printer', d.get_values().printer);
+						primary_action: function () {
+							localStorage.setItem("network_printer", d.get_values().printer);
 							if (typeof callback == "function") {
 								callback();
 							}
 							d.hide();
 						},
-						primary_action_label: __('Select')
+						primary_action_label: __("Select"),
 					});
 					d.show();
 				}
@@ -637,15 +589,15 @@ frappe.ui.form.PrintView = class {
 				doctype: this.frm.doc.doctype,
 				name: this.frm.doc.name,
 				print_format: print_format.name,
-				letterhead: this.get_letterhead()
+				letterhead: this.get_letterhead(),
 			});
 			let w = window.open(`/api/method/frappe.utils.weasyprint.download_pdf?${params}`);
 			if (!w) {
-				frappe.msgprint(__('Please enable pop-ups'));
+				frappe.msgprint(__("Please enable pop-ups"));
 				return;
 			}
 		} else {
-			this.render_page('/api/method/frappe.utils.print_format.download_pdf?');
+			this.render_page("/api/method/frappe.utils.print_format.download_pdf?");
 		}
 	}
 
@@ -653,24 +605,24 @@ frappe.ui.form.PrintView = class {
 		let w = window.open(
 			frappe.urllib.get_full_url(
 				method +
-					'doctype=' +
+					"doctype=" +
 					encodeURIComponent(this.frm.doc.doctype) +
-					'&name=' +
+					"&name=" +
 					encodeURIComponent(this.frm.doc.name) +
-					(printit ? '&trigger_print=1' : '') +
-					'&format=' +
+					(printit ? "&trigger_print=1" : "") +
+					"&format=" +
 					encodeURIComponent(this.selected_format()) +
-					'&no_letterhead=' +
-					(this.with_letterhead() ? '0' : '1') +
-					'&letterhead=' +
+					"&no_letterhead=" +
+					(this.with_letterhead() ? "0" : "1") +
+					"&letterhead=" +
 					encodeURIComponent(this.get_letterhead()) +
-					'&settings=' +
+					"&settings=" +
 					encodeURIComponent(JSON.stringify(this.additional_settings)) +
-					(this.lang_code ? '&_lang=' + this.lang_code : '')
+					(this.lang_code ? "&_lang=" + this.lang_code : "")
 			)
 		);
 		if (!w) {
-			frappe.msgprint(__('Please enable pop-ups'));
+			frappe.msgprint(__("Please enable pop-ups"));
 			return;
 		}
 	}
@@ -687,7 +639,7 @@ frappe.ui.form.PrintView = class {
 			this._req.abort();
 		}
 		this._req = frappe.call({
-			method: 'frappe.www.printview.get_html_and_style',
+			method: "frappe.www.printview.get_html_and_style",
 			args: {
 				doc: this.frm.doc,
 				print_format: this.selected_format(),
@@ -696,7 +648,7 @@ frappe.ui.form.PrintView = class {
 				settings: this.additional_settings,
 				_lang: this.lang_code,
 			},
-			callback: function(r) {
+			callback: function (r) {
 				if (!r.exc) {
 					callback(r.message);
 				}
@@ -705,25 +657,25 @@ frappe.ui.form.PrintView = class {
 	}
 
 	get_letterhead() {
-		return this.letterhead_selector.val();
+		return this.letterhead_selector.val() || __("No Letterhead");
 	}
 
 	get_no_preview_html() {
 		return `
        
-			${__('No Preview Available')}
+			${__("No Preview Available")}
 		
        `;
 	}
 
 	get_raw_commands(callback) {
 		// fetches rendered raw commands from the server for the current print format.
 		frappe.call({
-			method: 'frappe.www.printview.get_rendered_raw_commands',
+			method: "frappe.www.printview.get_rendered_raw_commands",
 			args: {
 				doc: this.frm.doc,
 				print_format: this.selected_format(),
 				_lang: this.lang_code,
 			},
-			callback: function(r) {
+			callback: function (r) {
 				if (!r.exc) {
 					callback(r.message);
 				}
@@ -746,30 +698,28 @@ frappe.ui.form.PrintView = class {
 	get_print_format_printer_map() {
 		// returns the whole object "print_format_printer_map" stored in the localStorage.
 		try {
-			let print_format_printer_map = JSON.parse(
-				localStorage.print_format_printer_map
-			);
+			let print_format_printer_map = JSON.parse(localStorage.print_format_printer_map);
 			return print_format_printer_map;
 		} catch (e) {
 			return {};
 		}
 	}
 
-	refresh_print_options() {
-		this.print_formats = frappe.meta.get_print_formats(this.frm.doctype);
-		const print_format_select_val = this.print_sel.val();
-		this.print_sel.empty().add_options([
-			this.get_default_option_for_select(__('Select Print Format')),
-			...this.print_formats
-		]);
-		return this.print_formats.includes(print_format_select_val)
-			&& this.print_sel.val(print_format_select_val);
+	set_default_print_format() {
+		if (
+			frappe.meta
+				.get_print_formats(this.frm.doctype)
+				.includes(this.print_format_selector.val()) ||
+			!this.frm.meta.default_print_format
+		)
+			return;
+
+		this.print_format_selector.empty();
+		this.print_format_selector.val(this.frm.meta.default_print_format);
 	}
 
 	selected_format() {
-		return (
-			this.print_sel.val() || this.frm.meta.default_print_format || 'Standard'
-		);
+		return this.print_format_selector.val() || "Standard";
 	}
 
 	is_raw_printing(format) {
@@ -782,19 +732,19 @@ frappe.ui.form.PrintView = class {
 			format = this.selected_format();
 		}
 
-		if (locals['Print Format'] && locals['Print Format'][format]) {
-			print_format = locals['Print Format'][format];
+		if (locals["Print Format"] && locals["Print Format"][format]) {
+			print_format = locals["Print Format"][format];
 		}
 
 		return print_format;
 	}
 
 	with_letterhead() {
-		return cint(this.get_letterhead() !== __('No Letterhead'));
+		return cint(this.get_letterhead() !== __("No Letterhead"));
 	}
 
 	set_style(style) {
-		frappe.dom.set_style(style || frappe.boot.print_css, 'print-style');
+		frappe.dom.set_style(style || frappe.boot.print_css, "print-style");
 	}
 
 	printer_setting_dialog() {
@@ -805,15 +755,15 @@ frappe.ui.form.PrintView = class {
 		frappe.ui.form.qz_get_printer_list().then((data) => {
 			this.printer_list = data;
 			const dialog = new frappe.ui.Dialog({
-				title: __('Printer Settings'),
+				title: __("Printer Settings"),
 				fields: [
 					{
-						fieldtype: 'Section Break',
+						fieldtype: "Section Break",
 					},
 					{
-						fieldname: 'printer_mapping',
-						fieldtype: 'Table',
-						label: __('Printer Mapping'),
+						fieldname: "printer_mapping",
+						fieldtype: "Table",
+						label: __("Printer Mapping"),
 						in_place_edit: true,
 						data: this.data,
 						get_data: () => {
@@ -821,28 +771,28 @@ frappe.ui.form.PrintView = class {
 						},
 						fields: [
 							{
-								fieldtype: 'Select',
-								fieldname: 'print_format',
+								fieldtype: "Select",
+								fieldname: "print_format",
 								default: 0,
-								options: this.print_formats,
+								options: frappe.meta.get_print_formats(this.frm.doctype),
 								read_only: 0,
 								in_list_view: 1,
-								label: __('Print Format'),
+								label: __("Print Format"),
 							},
 							{
-								fieldtype: 'Select',
-								fieldname: 'printer',
+								fieldtype: "Select",
+								fieldname: "printer",
 								default: 0,
 								options: this.printer_list,
 								read_only: 0,
 								in_list_view: 1,
-								label: __('Printer'),
+								label: __("Printer"),
 							},
 						],
 					},
 				],
 				primary_action: () => {
-					let printer_mapping = dialog.get_values()['printer_mapping'];
+					let printer_mapping = dialog.get_values()["printer_mapping"];
 					if (printer_mapping && printer_mapping.length) {
 						let print_format_list = printer_mapping.map((a) => a.print_format);
 						let has_duplicate = print_format_list.some(
@@ -851,7 +801,7 @@ frappe.ui.form.PrintView = class {
 						if (has_duplicate)
 							frappe.throw(
 								__(
-									'Cannot have multiple printers mapped to a single print format.'
+									"Cannot have multiple printers mapped to a single print format."
 								)
 							);
 					} else {
@@ -864,11 +814,11 @@ frappe.ui.form.PrintView = class {
 					);
 					dialog.hide();
 				},
-				primary_action_label: __('Save'),
+				primary_action_label: __("Save"),
 			});
 			dialog.show();
 			if (!(this.printer_list && this.printer_list.length)) {
-				frappe.throw(__('No Printer is Available.'));
+				frappe.throw(__("No Printer is Available."));
 			}
 		});
 	}
diff --git a/frappe/printing/page/print_format_builder/print_format_builder.js b/frappe/printing/page/print_format_builder/print_format_builder.js
index d12a7cc96c..657904f32e 100644
--- a/frappe/printing/page/print_format_builder/print_format_builder.js
+++ b/frappe/printing/page/print_format_builder/print_format_builder.js
@@ -1,17 +1,17 @@
-frappe.pages['print-format-builder'].on_page_load = function(wrapper) {
+frappe.pages["print-format-builder"].on_page_load = function (wrapper) {
 	frappe.print_format_builder = new frappe.PrintFormatBuilder(wrapper);
 	frappe.breadcrumbs.add("Setup", "Print Format");
-}
+};
 
-frappe.pages['print-format-builder'].on_page_show = function(wrapper) {
+frappe.pages["print-format-builder"].on_page_show = function (wrapper) {
 	var route = frappe.get_route();
-	if(route.length>1) {
-		frappe.model.with_doc('Print Format', route[1], function() {
-			frappe.print_format_builder.print_format = frappe.get_doc('Print Format', route[1]);
+	if (route.length > 1) {
+		frappe.model.with_doc("Print Format", route[1], function () {
+			frappe.print_format_builder.print_format = frappe.get_doc("Print Format", route[1]);
 			frappe.print_format_builder.refresh();
 		});
-	} else if(frappe.route_options) {
-		if(frappe.route_options.make_new) {
+	} else if (frappe.route_options) {
+		if (frappe.route_options.make_new) {
 			let { doctype, name, based_on, beta } = frappe.route_options;
 			frappe.route_options = null;
 			frappe.print_format_builder.setup_new_print_format(doctype, name, based_on, beta);
@@ -21,7 +21,7 @@ frappe.pages['print-format-builder'].on_page_show = function(wrapper) {
 			frappe.print_format_builder.refresh();
 		}
 	}
-}
+};
 
 frappe.PrintFormatBuilder = class PrintFormatBuilder {
 	constructor(parent) {
@@ -31,7 +31,7 @@ frappe.PrintFormatBuilder = class PrintFormatBuilder {
 	}
 	refresh() {
 		this.custom_html_count = 0;
-		if(!this.print_format) {
+		if (!this.print_format) {
 			this.show_start();
 		} else {
 			this.page.set_title(this.print_format.name);
@@ -44,10 +44,14 @@ frappe.PrintFormatBuilder = class PrintFormatBuilder {
 			title: __("Print Format Builder"),
 		});
 
-		this.page.main.css({"border-color": "transparent"});
+		this.page.main.css({ "border-color": "transparent" });
 
-		this.page.sidebar = $('
        ').appendTo(this.page.sidebar);
-		this.page.main = $('
        ').appendTo(this.page.main);
+		this.page.sidebar = $('
        ').appendTo(
+			this.page.sidebar
+		);
+		this.page.main = $(
+			'
        '
+		).appendTo(this.page.main);
 
 		// future-bindings for buttons on sections / fields
 		// bind only once
@@ -73,20 +77,20 @@ frappe.PrintFormatBuilder = class PrintFormatBuilder {
 				fieldtype: "Link",
 				options: "Print Format",
 				filters: {
-					print_format_builder: 1
+					print_format_builder: 1,
 				},
 				label: __("Select Print Format to Edit"),
-				only_select: true
+				only_select: true,
 			},
-			render_input: true
+			render_input: true,
 		});
 
 		// create a new print format.
-		this.page.main.find(".btn-edit-print-format").on("click", function() {
+		this.page.main.find(".btn-edit-print-format").on("click", function () {
 			var name = me.print_format_input.get_value();
-			if(!name) return;
-			frappe.model.with_doc("Print Format", name, function(doc) {
-				frappe.set_route('print-format-builder', name);
+			if (!name) return;
+			frappe.model.with_doc("Print Format", name, function (doc) {
+				frappe.set_route("print-format-builder", name);
 			});
 		});
 	}
@@ -98,12 +102,12 @@ frappe.PrintFormatBuilder = class PrintFormatBuilder {
 				fieldtype: "Link",
 				options: "DocType",
 				filters: {
-					"istable": 0,
-					"issingle": 0
+					istable: 0,
+					issingle: 0,
 				},
-				label: __("Select a DocType to make a new format")
+				label: __("Select a DocType to make a new format"),
 			},
-			render_input: true
+			render_input: true,
 		});
 
 		this.name_input = frappe.ui.form.make_control({
@@ -112,34 +116,33 @@ frappe.PrintFormatBuilder = class PrintFormatBuilder {
 				fieldtype: "Data",
 				label: __("Name of the new Print Format"),
 			},
-			render_input: true
+			render_input: true,
 		});
 
-		this.page.main.find(".btn-new-print-format").on("click", function() {
+		this.page.main.find(".btn-new-print-format").on("click", function () {
 			var doctype = me.doctype_input.get_value(),
 				name = me.name_input.get_value();
-			if(!(doctype && name)) {
+			if (!(doctype && name)) {
 				frappe.msgprint(__("Both DocType and Name required"));
 				return;
 			}
 			me.setup_new_print_format(doctype, name);
-
 		});
 	}
 	setup_new_print_format(doctype, name, based_on, beta) {
 		frappe.call({
-			method: 'frappe.printing.page.print_format_builder.print_format_builder.create_custom_format',
+			method: "frappe.printing.page.print_format_builder.print_format_builder.create_custom_format",
 			args: {
 				doctype: doctype,
 				name: name,
 				based_on: based_on,
-				beta: Boolean(beta)
+				beta: Boolean(beta),
 			},
 			callback: (r) => {
 				if (r.message) {
 					let print_format = r.message;
 					if (print_format.print_format_builder_beta) {
-						frappe.set_route('print-format-builder-beta', print_format.name);
+						frappe.set_route("print-format-builder-beta", print_format.name);
 					} else {
 						this.print_format = print_format;
 						this.refresh();
@@ -150,20 +153,24 @@ frappe.PrintFormatBuilder = class PrintFormatBuilder {
 	}
 	setup_print_format() {
 		var me = this;
-		frappe.model.with_doctype(this.print_format.doc_type, function(doctype) {
+		frappe.model.with_doctype(this.print_format.doc_type, function (doctype) {
 			me.meta = frappe.get_meta(me.print_format.doc_type);
 			me.setup_sidebar();
 			me.render_layout();
-			me.page.set_primary_action(__("Save"), function() {
+			me.page.set_primary_action(__("Save"), function () {
 				me.save_print_format();
 			});
 			me.page.clear_menu();
-			me.page.add_menu_item(__("Start new Format"), function() {
-				me.print_format = null;
-				me.refresh();
-			}, true);
+			me.page.add_menu_item(
+				__("Start new Format"),
+				function () {
+					me.print_format = null;
+					me.refresh();
+				},
+				true
+			);
 			me.page.clear_inner_toolbar();
-			me.page.add_inner_button(__("Edit Properties"), function() {
+			me.page.add_inner_button(__("Edit Properties"), function () {
 				frappe.set_route("Form", "Print Format", me.print_format.name);
 			});
 		});
@@ -172,7 +179,7 @@ frappe.PrintFormatBuilder = class PrintFormatBuilder {
 		// prepend custom HTML field
 		var fields = [this.get_custom_html_field()].concat(this.meta.fields);
 		this.page.sidebar.html(
-			$(frappe.render_template("print_format_builder_sidebar", {fields: fields}))
+			$(frappe.render_template("print_format_builder_sidebar", { fields: fields }))
 		);
 		this.setup_field_filter();
 	}
@@ -180,15 +187,18 @@ frappe.PrintFormatBuilder = class PrintFormatBuilder {
 		return {
 			fieldtype: "Custom HTML",
 			fieldname: "_custom_html",
-			label: __("Custom HTML")
+			label: __("Custom HTML"),
 		};
 	}
 	render_layout() {
 		this.page.main.empty();
 		this.prepare_data();
-		$(frappe.render_template("print_format_builder_layout", {
-			data: this.layout_data, me: this}))
-			.appendTo(this.page.main);
+		$(
+			frappe.render_template("print_format_builder_layout", {
+				data: this.layout_data,
+				me: this,
+			})
+		).appendTo(this.page.main);
 		this.setup_sortable();
 		this.setup_add_section();
 		this.setup_edit_heading();
@@ -198,82 +208,86 @@ frappe.PrintFormatBuilder = class PrintFormatBuilder {
 	prepare_data() {
 		this.print_heading_template = null;
 		this.data = JSON.parse(this.print_format.format_data || "[]");
-		if(!this.data.length) {
+		if (!this.data.length) {
 			// new layout
 			this.data = this.meta.fields;
 		} else {
 			// extract print_heading_template if found
-			if(this.data[0].fieldname==="print_heading_template") {
+			if (this.data[0].fieldname === "print_heading_template") {
 				this.print_heading_template = this.data[0].options;
 				this.data = this.data.splice(1);
 			}
 		}
 
-		if(!this.print_heading_template) {
+		if (!this.print_heading_template) {
 			// default print heading template
-			this.print_heading_template = '
        \
-				
        '+__(this.print_format.doc_type)
-					+'

        {{ doc.name }}\
+			this.print_heading_template =
+				'
        \
+				
        ' +
+				__(this.print_format.doc_type) +
+				'

        {{ doc.name }}\
 				
        ';
 		}
 
 		this.layout_data = [];
 		this.fields_dict = {};
 		this.custom_html_dict = {};
-		var section = null, column = null, me = this, custom_html_count = 0;
+		var section = null,
+			column = null,
+			me = this,
+			custom_html_count = 0;
 
 		// create a new placeholder for column and set
 		// it as "column"
-		var set_column = function() {
-			if(!section) set_section();
+		var set_column = function () {
+			if (!section) set_section();
 			column = me.get_new_column();
 			section.columns.push(column);
 			section.no_of_columns += 1;
-		}
+		};
 
-		var set_section = function(label) {
+		var set_section = function (label) {
 			section = me.get_new_section();
-			if(label) section.label = label;
+			if (label) section.label = label;
 			column = null;
 			me.layout_data.push(section);
-		}
+		};
 
 		// break the layout into sections and columns
 		// so that it is easier to render in a template
-		$.each(this.data, function(i, f) {
+		$.each(this.data, function (i, f) {
 			me.fields_dict[f.fieldname] = f;
-			if(!f.name && f.fieldname) {
+			if (!f.name && f.fieldname) {
 				// from format_data (designed format)
 				// print_hide should always be false
-				if(f.fieldname==="_custom_html") {
+				if (f.fieldname === "_custom_html") {
 					f.label = "Custom HTML";
 					f.fieldtype = "Custom HTML";
 
 					// set custom html id to map data properties later
 					custom_html_count++;
 					f.custom_html_id = custom_html_count;
-					me.custom_html_dict[f.custom_html_id] = f
+					me.custom_html_dict[f.custom_html_id] = f;
 				} else {
-					f = $.extend(frappe.meta.get_docfield(me.print_format.doc_type,
-						f.fieldname) || {}, f);
+					f = $.extend(
+						frappe.meta.get_docfield(me.print_format.doc_type, f.fieldname) || {},
+						f
+					);
 				}
 			}
 
-			if(f.fieldtype==="Section Break") {
+			if (f.fieldtype === "Section Break") {
 				set_section(f.label);
-
-			} else if(f.fieldtype==="Column Break") {
+			} else if (f.fieldtype === "Column Break") {
 				set_column();
+			} else if (!in_list(frappe.model.layout_fields, f.fieldtype)) {
+				if (!column) set_column();
 
-			} else if (!in_list(["Section Break", "Column Break", "Tab Break", "Fold"], f.fieldtype)
-				&& f.label) {
-				if(!column) set_column();
-
-				if(f.fieldtype==="Table") {
+				if (f.fieldtype === "Table") {
 					me.add_table_properties(f);
 				}
 
-				if(!f.print_hide) {
+				if (!f.print_hide) {
 					column.fields.push(f);
 					section.has_fields = true;
 				}
@@ -281,33 +295,38 @@ frappe.PrintFormatBuilder = class PrintFormatBuilder {
 		});
 
 		// strip out empty sections
-		this.layout_data = $.map(this.layout_data, function(s) {
-			return s.has_fields ? s : null
+		this.layout_data = $.map(this.layout_data, function (s) {
+			return s.has_fields ? s : null;
 		});
 	}
 	get_new_section() {
-		return {columns: [], no_of_columns: 0, label:''};
+		return { columns: [], no_of_columns: 0, label: "" };
 	}
 	get_new_column() {
-		return {fields: []}
+		return { fields: [] };
 	}
 	add_table_properties(f) {
 		// build table columns and widths in a dict
 		// visible_columns
 		var me = this;
-		if(!f.visible_columns) {
+		if (!f.visible_columns) {
 			me.init_visible_columns(f);
 		}
 	}
 	init_visible_columns(f) {
-		f.visible_columns = []
-		$.each(frappe.get_meta(f.options).fields, function(i, _f) {
-			if (!in_list(["Section Break", "Column Break", "Tab Break"], _f.fieldtype) &&
-				!_f.print_hide && f.label) {
-
+		f.visible_columns = [];
+		$.each(frappe.get_meta(f.options).fields, function (i, _f) {
+			if (
+				!in_list(["Section Break", "Column Break", "Tab Break"], _f.fieldtype) &&
+				!_f.print_hide &&
+				f.label
+			) {
 				// column names set as fieldname|width
-				f.visible_columns.push({fieldname: _f.fieldname,
-					print_width: (_f.width || ""), print_hide:0});
+				f.visible_columns.push({
+					fieldname: _f.fieldname,
+					print_width: _f.width || "",
+					print_hide: 0,
+				});
 			}
 		});
 	}
@@ -315,118 +334,125 @@ frappe.PrintFormatBuilder = class PrintFormatBuilder {
 		var me = this;
 
 		// drag from fields library
-		Sortable.create(this.page.sidebar.find(".print-format-builder-sidebar-fields").get(0),
-			{
-				group: {
-					name:'field', put: true, pull:"clone"
-				},
-				sort: false,
-				onAdd: function(evt) {
-					// on drop, trash!
-					$(evt.item).fadeOut();
-				}
-			});
+		Sortable.create(this.page.sidebar.find(".print-format-builder-sidebar-fields").get(0), {
+			group: {
+				name: "field",
+				put: true,
+				pull: "clone",
+			},
+			sort: false,
+			onAdd: function (evt) {
+				// on drop, trash!
+				$(evt.item).fadeOut();
+			},
+		});
 
 		// sort, drag and drop between columns
-		this.page.main.find(".print-format-builder-column").each(function() {
+		this.page.main.find(".print-format-builder-column").each(function () {
 			me.setup_sortable_for_column(this);
 		});
 
 		// section sorting
-		Sortable.create(this.page.main.find(".print-format-builder-layout").get(0),
-			{ handle: ".print-format-builder-section-head" }
-		);
+		Sortable.create(this.page.main.find(".print-format-builder-layout").get(0), {
+			handle: ".print-format-builder-section-head",
+		});
 	}
 	setup_sortable_for_column(col) {
 		var me = this;
 		Sortable.create(col, {
 			group: {
-				name: 'field',
+				name: "field",
 				put: true,
-				pull: true
+				pull: true,
 			},
-			onAdd: function(evt) {
+			onAdd: function (evt) {
 				// on drop, change the HTML
 
 				var $item = $(evt.item);
-				if(!$item.hasClass("print-format-builder-field")) {
+				if (!$item.hasClass("print-format-builder-field")) {
 					var fieldname = $item.attr("data-fieldname");
 
-					if(fieldname==="_custom_html") {
+					if (fieldname === "_custom_html") {
 						var field = me.get_custom_html_field();
 					} else {
-						var field = frappe.meta.get_docfield(me.print_format.doc_type,
-							fieldname);
+						var field = frappe.meta.get_docfield(me.print_format.doc_type, fieldname);
 					}
 
-					var html = frappe.render_template("print_format_builder_field",
-						{field: field, me:me});
+					var html = frappe.render_template("print_format_builder_field", {
+						field: field,
+						me: me,
+					});
 
 					$item.replaceWith(html);
 				}
-			}
+			},
 		});
-
 	}
 	setup_field_filter() {
 		var me = this;
-		this.page.sidebar.find(".filter-fields").on("keyup", function() {
+		this.page.sidebar.find(".filter-fields").on("keyup", function () {
 			var text = $(this).val();
-			me.page.sidebar.find(".field-label").each(function() {
-				var show = !text || $(this).text().toLowerCase().indexOf(text.toLowerCase())!==-1;
+			me.page.sidebar.find(".field-label").each(function () {
+				var show =
+					!text || $(this).text().toLowerCase().indexOf(text.toLowerCase()) !== -1;
 				$(this).parent().toggle(show);
-			})
+			});
 		});
 	}
 	setup_section_settings() {
 		var me = this;
-		this.page.main.on("click", ".section-settings", function() {
+		this.page.main.on("click", ".section-settings", function () {
 			var section = $(this).parent().parent();
 			var no_of_columns = section.find(".section-column").length;
-			var label = section.attr('data-label');
+			var label = section.attr("data-label");
 
 			// new dialog
 			var d = new frappe.ui.Dialog({
 				title: "Edit Section",
 				fields: [
 					{
-						label:__("No of Columns"),
-						fieldname:"no_of_columns",
-						fieldtype:"Select",
+						label: __("No of Columns"),
+						fieldname: "no_of_columns",
+						fieldtype: "Select",
 						options: ["1", "2", "3", "4"],
 					},
 					{
-						label:__("Section Heading"),
-						fieldname:"label",
-						fieldtype:"Data",
-						description: __('Will only be shown if section headings are enabled')
+						label: __("Section Heading"),
+						fieldname: "label",
+						fieldtype: "Data",
+						description: __("Will only be shown if section headings are enabled"),
 					},
 					{
 						label: __("Remove Section"),
 						fieldname: "remove_section",
 						fieldtype: "Button",
-						click: function() {
+						click: function () {
 							d.hide();
-							section.fadeOut(function() {section.remove()});
+							section.fadeOut(function () {
+								section.remove();
+							});
 						},
 						input_class: "btn-danger",
 						input_css: {
-							"margin-top": "20px"
-						}
-					}
+							"margin-top": "20px",
+						},
+					},
 				],
 			});
 
 			d.set_input("no_of_columns", no_of_columns + "");
 			d.set_input("label", label || "");
 
-			d.set_primary_action(__("Update"), function() {
+			d.set_primary_action(__("Update"), function () {
 				// resize number of columns
-				me.update_columns_in_section(section, no_of_columns,
-					cint(d.get_value("no_of_columns")));
+				me.update_columns_in_section(
+					section,
+					no_of_columns,
+					cint(d.get_value("no_of_columns"))
+				);
 
-				section.attr('data-label', d.get_value('label') || '');
-				section.find('.section-label').html(d.get_value('label') || '');
+				section.attr("data-label", d.get_value("label") || "");
+				section.find(".section-label").html(d.get_value("label") || "");
 
 				d.hide();
 			});
@@ -437,7 +463,7 @@ frappe.PrintFormatBuilder = class PrintFormatBuilder {
 		});
 	}
 	setup_field_settings() {
-		this.page.main.find(".field-settings").on("click", e => {
+		this.page.main.find(".field-settings").on("click", (e) => {
 			const field = $(e.currentTarget).parent();
 			// new dialog
 			var d = new frappe.ui.Dialog({
@@ -446,41 +472,43 @@ frappe.PrintFormatBuilder = class PrintFormatBuilder {
 					{
 						label: __("Label"),
 						fieldname: "label",
-						fieldtype: "Data"
+						fieldtype: "Data",
 					},
 					{
 						label: __("Align Value"),
 						fieldname: "align",
 						fieldtype: "Select",
-						options: [{'label': __('Left', null, 'alignment'), 'value': 'left'},
-							{'label': __('Right', null, 'alignment'), 'value': 'right'}]
+						options: [
+							{ label: __("Left", null, "alignment"), value: "left" },
+							{ label: __("Right", null, "alignment"), value: "right" },
+						],
 					},
 					{
 						label: __("Remove Field"),
 						fieldtype: "Button",
-						click: function() {
+						click: function () {
 							d.hide();
 							field.remove();
 						},
 						input_class: "btn-danger",
-					}
+					},
 				],
 			});
 
-			d.set_value('label', field.attr("data-label"));
+			d.set_value("label", field.attr("data-label"));
 
-			d.set_primary_action(__("Update"), function() {
-				field.attr('data-align', d.get_value('align'));
-				field.attr('data-label', d.get_value('label'));
-				field.find('.field-label').html(d.get_value('label'));
+			d.set_primary_action(__("Update"), function () {
+				field.attr("data-align", d.get_value("align"));
+				field.attr("data-label", d.get_value("label"));
+				field.find(".field-label").html(d.get_value("label"));
 				d.hide();
 			});
 
 			// set current value
-			if (field.attr('data-align')) {
-				d.set_value('align', field.attr('data-align'));
+			if (field.attr("data-align")) {
+				d.set_value("align", field.attr("data-align"));
 			} else {
-				d.set_value('align', 'left');
+				d.set_value("align", "left");
 			}
 
 			d.show();
@@ -495,32 +523,33 @@ frappe.PrintFormatBuilder = class PrintFormatBuilder {
 		// this is based on a dummy attribute custom_html_id, since all custom html
 		// fields have the same fieldname `_custom_html`
 		var me = this;
-		this.page.main.find('[data-fieldtype="Custom HTML"]').each(function() {
-			var fieldname = $(this).attr('data-fieldname');
-			var content = $($(this).find('.html-content')[0]);
-			var html = me.custom_html_dict[parseInt(content.attr('data-custom-html-id'))].options;
-			content.data('content', html);
-		})
+		this.page.main.find('[data-fieldtype="Custom HTML"]').each(function () {
+			var fieldname = $(this).attr("data-fieldname");
+			var content = $($(this).find(".html-content")[0]);
+			var html = me.custom_html_dict[parseInt(content.attr("data-custom-html-id"))].options;
+			content.data("content", html);
+		});
 	}
 	update_columns_in_section(section, no_of_columns, new_no_of_columns) {
 		var col_size = 12 / new_no_of_columns,
 			me = this,
-			resize = function() {
-				section.find(".section-column")
+			resize = function () {
+				section
+					.find(".section-column")
 					.removeClass()
 					.addClass("section-column")
-					.addClass("col-md-" + col_size)
+					.addClass("col-md-" + col_size);
 			};
 
-		if(new_no_of_columns < no_of_columns) {
+		if (new_no_of_columns < no_of_columns) {
 			// move contents of last n columns to previous column
-			for(var i=no_of_columns; i > new_no_of_columns; i--) {
-				var $col = $(section.find(".print-format-builder-column").get(i-1));
-				var prev = section.find(".print-format-builder-column").get(i-2);
+			for (var i = no_of_columns; i > new_no_of_columns; i--) {
+				var $col = $(section.find(".print-format-builder-column").get(i - 1));
+				var prev = section.find(".print-format-builder-column").get(i - 2);
 
 				// append each field to prev
 				$col.parent().addClass("to-drop");
-				$col.find(".print-format-builder-field").each(function() {
+				$col.find(".print-format-builder-field").each(function () {
 					$(this).appendTo(prev);
 				});
 			}
@@ -530,32 +559,33 @@ frappe.PrintFormatBuilder = class PrintFormatBuilder {
 
 			// resize
 			resize();
-
-		} else if(new_no_of_columns > no_of_columns) {
+		} else if (new_no_of_columns > no_of_columns) {
 			// add empty column and resize old columns
-			for(var i=no_of_columns; i < new_no_of_columns; i++) {
-				var col = $('
        \
-					
        ')
-				.appendTo(section);
-				me.setup_sortable_for_column(col
-					.find(".print-format-builder-column").get(0));
+			for (var i = no_of_columns; i < new_no_of_columns; i++) {
+				var col = $(
+					'
        \
+					
        '
+				).appendTo(section);
+				me.setup_sortable_for_column(col.find(".print-format-builder-column").get(0));
 			}
 			// resize
 			resize();
 		}
-
 	}
 	setup_add_section() {
 		var me = this;
-		this.page.main.find(".print-format-builder-add-section").on("click", function() {
+		this.page.main.find(".print-format-builder-add-section").on("click", function () {
 			// boostrap new section info
 			var section = me.get_new_section();
 			section.columns.push(me.get_new_column());
 			section.no_of_columns = 1;
 
-			var $section = $(frappe.render_template("print_format_builder_section",
-				{section: section, me: me}))
-				.appendTo(me.page.main.find(".print-format-builder-layout"))
+			var $section = $(
+				frappe.render_template("print_format_builder_section", {
+					section: section,
+					me: me,
+				})
+			).appendTo(me.page.main.find(".print-format-builder-layout"));
 
 			me.setup_sortable_for_column($section.find(".print-format-builder-column").get(0));
 		});
@@ -565,23 +595,25 @@ frappe.PrintFormatBuilder = class PrintFormatBuilder {
 		var $heading = this.page.main.find(".print-format-builder-print-heading");
 
 		// set content property
-		$heading.data('content', this.print_heading_template);
+		$heading.data("content", this.print_heading_template);
 
-		this.page.main.find(".edit-heading").on("click", function() {
+		this.page.main.find(".edit-heading").on("click", function () {
 			var d = me.get_edit_html_dialog(__("Edit Heading"), __("Heading"), $heading);
-		})
+		});
 	}
 	setup_column_selector() {
 		var me = this;
-		this.page.main.on("click", ".select-columns", function() {
+		this.page.main.on("click", ".select-columns", function () {
 			var parent = $(this).parents(".print-format-builder-field:first"),
 				doctype = parent.attr("data-doctype"),
 				label = parent.attr("data-label"),
 				columns = parent.attr("data-columns").split(","),
-				column_names = $.map(columns, function(v) { return v.split("|")[0]; }),
+				column_names = $.map(columns, function (v) {
+					return v.split("|")[0];
+				}),
 				widths = {};
 
-			$.each(columns, function(i, v) {
+			$.each(columns, function (i, v) {
 				var parts = v.split("|");
 				widths[parts[0]] = parts[1] || "";
 			});
@@ -592,46 +624,51 @@ frappe.PrintFormatBuilder = class PrintFormatBuilder {
 
 			var $body = $(d.body);
 
-
 			var doc_fields = frappe.get_meta(doctype).fields;
 			var docfields_by_name = {};
 
 			// docfields by fieldname
-			$.each(doc_fields, function(j, f) {
-				if(f) docfields_by_name[f.fieldname] = f;
-			})
+			$.each(doc_fields, function (j, f) {
+				if (f) docfields_by_name[f.fieldname] = f;
+			});
 
 			// add field which are in column_names first to preserve order
 			var fields = [];
-			$.each(column_names, function(i, v) {
-				if(in_list(Object.keys(docfields_by_name), v)) {
+			$.each(column_names, function (i, v) {
+				if (in_list(Object.keys(docfields_by_name), v)) {
 					fields.push(docfields_by_name[v]);
 				}
-			})
+			});
 			// add remaining fields
-			$.each(doc_fields, function(j, f) {
-				if (f && !in_list(column_names, f.fieldname)
-					&& !in_list(["Section Break", "Column Break", "Tab Break"], f.fieldtype) && f.label) {
+			$.each(doc_fields, function (j, f) {
+				if (
+					f &&
+					!in_list(column_names, f.fieldname) &&
+					!in_list(["Section Break", "Column Break", "Tab Break"], f.fieldtype) &&
+					f.label
+				) {
 					fields.push(f);
 				}
-			})
+			});
 			// render checkboxes
-			$(frappe.render_template("print_format_builder_column_selector", {
-				fields: fields,
-				column_names: column_names,
-				widths: widths
-			})).appendTo(d.body);
+			$(
+				frappe.render_template("print_format_builder_column_selector", {
+					fields: fields,
+					column_names: column_names,
+					widths: widths,
+				})
+			).appendTo(d.body);
 
 			Sortable.create($body.find(".column-selector-list").get(0));
 
-			var get_width_input = function(fieldname) {
-				return $body.find(".column-width[data-fieldname='"+ fieldname +"']")
-			}
+			var get_width_input = function (fieldname) {
+				return $body.find(".column-width[data-fieldname='" + fieldname + "']");
+			};
 
 			// update data-columns property on update
-			d.set_primary_action(__("Update"), function() {
+			d.set_primary_action(__("Update"), function () {
 				var visible_columns = [];
-				$body.find("input:checked").each(function() {
+				$body.find("input:checked").each(function () {
 					var fieldname = $(this).attr("data-fieldname"),
 						width = get_width_input(fieldname).val() || "";
 					visible_columns.push(fieldname + "|" + width);
@@ -643,17 +680,17 @@ frappe.PrintFormatBuilder = class PrintFormatBuilder {
 			let update_column_count_message = () => {
 				// show a warning if user selects more than 10 columns for a table
 				let columns_count = $body.find("input:checked").length;
-				$body.find('.help-message').toggle(columns_count > 10);
-			}
+				$body.find(".help-message").toggle(columns_count > 10);
+			};
 			update_column_count_message();
 
 			// enable / disable input based on selection
-			$body.on("click", "input[type='checkbox']", function() {
+			$body.on("click", "input[type='checkbox']", function () {
 				var disabled = !$(this).prop("checked"),
 					input = get_width_input($(this).attr("data-fieldname"));
 
 				input.prop("disabled", disabled);
-				if(disabled) input.val("");
+				if (disabled) input.val("");
 
 				update_column_count_message();
 			});
@@ -664,19 +701,24 @@ frappe.PrintFormatBuilder = class PrintFormatBuilder {
 		});
 	}
 	get_visible_columns_string(f) {
-		if(!f.visible_columns) {
+		if (!f.visible_columns) {
 			this.init_visible_columns(f);
 		}
-		return $.map(f.visible_columns, function(v) { return v.fieldname + "|" + (v.print_width || "") }).join(",");
+		return $.map(f.visible_columns, function (v) {
+			return v.fieldname + "|" + (v.print_width || "");
+		}).join(",");
 	}
 	get_no_content() {
-		return __("Edit to add content")
+		return __("Edit to add content");
 	}
 	setup_edit_custom_html() {
 		var me = this;
-		this.page.main.on("click", ".edit-html", function() {
-			me.get_edit_html_dialog(__("Edit Custom HTML"), __("Custom HTML"),
-				$(this).parents(".print-format-builder-field:first").find(".html-content"));
+		this.page.main.on("click", ".edit-html", function () {
+			me.get_edit_html_dialog(
+				__("Edit Custom HTML"),
+				__("Custom HTML"),
+				$(this).parents(".print-format-builder-field:first").find(".html-content")
+			);
 		});
 	}
 	get_edit_html_dialog(title, label, $content) {
@@ -688,26 +730,31 @@ frappe.PrintFormatBuilder = class PrintFormatBuilder {
 					fieldname: "content",
 					fieldtype: "Code",
 					label: label,
-					options: "HTML"
+					options: "HTML",
 				},
 				{
 					fieldname: "help",
 					fieldtype: "HTML",
-					options: '
        '
-						+ __("You can add dynamic properties from the document by using Jinja templating.")
-						+ __("For example: If you want to include the document ID, use {0}", ["{{ doc.name }}"])
-					+ '
        '
-				}
-			]
+					options:
+						"
        " +
+						__(
+							"You can add dynamic properties from the document by using Jinja templating."
+						) +
+						__("For example: If you want to include the document ID, use {0}", [
+							"{{ doc.name }}",
+						]) +
+						"
        ",
+				},
+			],
 		});
 
 		// set existing content in input
-		var content = $content.data('content') || "";
-		if(content.indexOf(me.get_no_content())!==-1) content = "";
+		var content = $content.data("content") || "";
+		if (content.indexOf(me.get_no_content()) !== -1) content = "";
 		d.set_input("content", content);
 
-		d.set_primary_action(__("Update"), function() {
-			$($content[0]).data('content', d.get_value("content"));
+		d.set_primary_action(__("Update"), function () {
+			$($content[0]).data("content", d.get_value("content"));
 			$content.html(d.get_value("content"));
 			d.hide();
 		});
@@ -725,55 +772,59 @@ frappe.PrintFormatBuilder = class PrintFormatBuilder {
 		// before rendering
 		data.push({
 			fieldname: "print_heading_template",
-			fieldtype:"Custom HTML",
-			options: this.page.main.find(".print-format-builder-print-heading").data('content')
+			fieldtype: "Custom HTML",
+			options: this.page.main.find(".print-format-builder-print-heading").data("content"),
 		});
 
 		// add pages
-		this.page.main.find(".print-format-builder-section").each(function() {
-			var section = {"fieldtype": "Section Break", 'label': $(this).attr('data-label') || '' };
+		this.page.main.find(".print-format-builder-section").each(function () {
+			var section = { fieldtype: "Section Break", label: $(this).attr("data-label") || "" };
 			data.push(section);
-			$(this).find(".print-format-builder-column").each(function() {
-				data.push({"fieldtype": "Column Break"});
-				$(this).find(".print-format-builder-field").each(function() {
-					var $this = $(this),
-						fieldtype = $this.attr("data-fieldtype"),
-						align = $this.attr('data-align'),
-						label = $this.attr('data-label'),
-						df = {
-							fieldname: $this.attr("data-fieldname"),
-							print_hide: 0
-						};
+			$(this)
+				.find(".print-format-builder-column")
+				.each(function () {
+					data.push({ fieldtype: "Column Break" });
+					$(this)
+						.find(".print-format-builder-field")
+						.each(function () {
+							var $this = $(this),
+								fieldtype = $this.attr("data-fieldtype"),
+								align = $this.attr("data-align"),
+								label = $this.attr("data-label"),
+								df = {
+									fieldname: $this.attr("data-fieldname"),
+									print_hide: 0,
+								};
 
-					if(align) {
-						df.align = align;
-					}
+							if (align) {
+								df.align = align;
+							}
 
-					if(label) {
-						df.label = label;
-					}
+							if (label) {
+								df.label = label;
+							}
 
-					if(fieldtype==="Table") {
-						// append the user selected columns to visible_columns
-						var columns = $this.attr("data-columns").split(",");
-						df.visible_columns = [];
-						$.each(columns, function(i, c) {
-							var parts = c.split("|");
-							df.visible_columns.push({
-								fieldname:parts[0],
-								print_width:parts[1],
-								print_hide:0
-							});
+							if (fieldtype === "Table") {
+								// append the user selected columns to visible_columns
+								var columns = $this.attr("data-columns").split(",");
+								df.visible_columns = [];
+								$.each(columns, function (i, c) {
+									var parts = c.split("|");
+									df.visible_columns.push({
+										fieldname: parts[0],
+										print_width: parts[1],
+										print_hide: 0,
+									});
+								});
+							}
+							if (fieldtype === "Custom HTML") {
+								// custom html as HTML field
+								df.fieldtype = "HTML";
+								df.options = $($this.find(".html-content")[0]).data("content");
+							}
+							data.push(df);
 						});
-					}
-					if(fieldtype==="Custom HTML") {
-						// custom html as HTML field
-						df.fieldtype = "HTML";
-						df.options = $($this.find(".html-content")[0]).data('content');
-					}
-					data.push(df);
 				});
-			});
 		});
 
 		// save format_data
@@ -787,11 +838,11 @@ frappe.PrintFormatBuilder = class PrintFormatBuilder {
 			},
 			freeze: true,
 			btn: this.page.btn_primary,
-			callback: function(r) {
+			callback: function (r) {
 				me.print_format = r.message;
-				locals['Print Format'][me.print_format.name] = r.message;
-				frappe.show_alert({message: __("Saved"), indicator: 'green'});
-			}
+				locals["Print Format"][me.print_format.name] = r.message;
+				frappe.show_alert({ message: __("Saved"), indicator: "green" });
+			},
 		});
 	}
 };
diff --git a/frappe/printing/page/print_format_builder/print_format_builder_column_selector.html b/frappe/printing/page/print_format_builder/print_format_builder_column_selector.html
index 495b837f48..adc87fff22 100644
--- a/frappe/printing/page/print_format_builder/print_format_builder_column_selector.html
+++ b/frappe/printing/page/print_format_builder/print_format_builder_column_selector.html
@@ -17,10 +17,10 @@
 				
        
 				
        
 					
 				
        
 			
        
diff --git a/frappe/printing/page/print_format_builder/print_format_builder_field.html b/frappe/printing/page/print_format_builder/print_format_builder_field.html
index beb9de2f5a..90eb4f1f0a 100644
--- a/frappe/printing/page/print_format_builder/print_format_builder_field.html
+++ b/frappe/printing/page/print_format_builder/print_format_builder_field.html
@@ -34,7 +34,7 @@
 			
 				
 			
-			{{ __(field.label) }}
+			{{ __(field.label) || __(field.fieldname) }}
 				 ({%= __("Table") %})
 			
 			
diff --git a/frappe/printing/page/print_format_builder_beta/print_format_builder_beta.js b/frappe/printing/page/print_format_builder_beta/print_format_builder_beta.js
index e923bbcb00..9c3eee1273 100644
--- a/frappe/printing/page/print_format_builder_beta/print_format_builder_beta.js
+++ b/frappe/printing/page/print_format_builder_beta/print_format_builder_beta.js
@@ -1,8 +1,8 @@
-frappe.pages["print-format-builder-beta"].on_page_load = function(wrapper) {
+frappe.pages["print-format-builder-beta"].on_page_load = function (wrapper) {
 	frappe.ui.make_app_page({
 		parent: wrapper,
 		title: __("Print Format Builder"),
-		single_column: true
+		single_column: true,
 	});
 
 	// hot reload in development
@@ -12,7 +12,7 @@ frappe.pages["print-format-builder-beta"].on_page_load = function(wrapper) {
 	}
 };
 
-frappe.pages["print-format-builder-beta"].on_page_show = function(wrapper) {
+frappe.pages["print-format-builder-beta"].on_page_show = function (wrapper) {
 	load_print_format_builder_beta(wrapper);
 };
 
@@ -26,7 +26,7 @@ function load_print_format_builder_beta(wrapper) {
 			frappe.print_format_builder = new frappe.ui.PrintFormatBuilder({
 				wrapper: $parent,
 				page: wrapper.page,
-				print_format: route[1]
+				print_format: route[1],
 			});
 		});
 	} else {
@@ -39,14 +39,12 @@ function load_print_format_builder_beta(wrapper) {
 					fieldtype: "Select",
 					options: [
 						{ label: __("Create New"), value: "Create" },
-						{ label: __("Edit Existing"), value: "Edit" }
+						{ label: __("Edit Existing"), value: "Edit" },
 					],
 					change() {
 						let action = d.get_value("action");
-						d.get_primary_btn().text(
-							action === "Create" ? __("Create") : __("Edit")
-						);
-					}
+						d.get_primary_btn().text(action === "Create" ? __("Create") : __("Edit"));
+					},
 				},
 				{
 					label: __("Select Document Type"),
@@ -54,19 +52,17 @@ function load_print_format_builder_beta(wrapper) {
 					fieldtype: "Link",
 					options: "DocType",
 					filters: {
-						istable: 0
+						istable: 0,
 					},
 					reqd: 1,
-					default: frappe.route_options
-						? frappe.route_options.doctype
-						: null
+					default: frappe.route_options ? frappe.route_options.doctype : null,
 				},
 				{
-					label: __("Print Format Name"),
+					label: __("New Print Format Name"),
 					fieldname: "print_format_name",
 					fieldtype: "Data",
-					depends_on: doc => doc.action === "Create",
-					mandatory_depends_on: doc => doc.action === "Create"
+					depends_on: (doc) => doc.action === "Create",
+					mandatory_depends_on: (doc) => doc.action === "Create",
 				},
 				{
 					label: __("Select Print Format"),
@@ -74,25 +70,20 @@ function load_print_format_builder_beta(wrapper) {
 					fieldtype: "Link",
 					options: "Print Format",
 					only_select: 1,
-					depends_on: doc => doc.action === "Edit",
+					depends_on: (doc) => doc.action === "Edit",
 					get_query() {
 						return {
 							filters: {
 								doc_type: d.get_value("doctype"),
-								print_format_builder_beta: 1
-							}
+								print_format_builder_beta: 1,
+							},
 						};
 					},
-					mandatory_depends_on: doc => doc.action === "Edit"
-				}
+					mandatory_depends_on: (doc) => doc.action === "Edit",
+				},
 			],
 			primary_action_label: __("Edit"),
-			primary_action({
-				action,
-				doctype,
-				print_format,
-				print_format_name
-			}) {
+			primary_action({ action, doctype, print_format, print_format_name }) {
 				if (action === "Edit") {
 					frappe.set_route("print-format-builder-beta", print_format);
 				} else if (action === "Create") {
@@ -102,19 +93,16 @@ function load_print_format_builder_beta(wrapper) {
 							doctype: "Print Format",
 							name: print_format_name,
 							doc_type: doctype,
-							print_format_builder_beta: 1
+							print_format_builder_beta: 1,
 						})
-						.then(doc => {
-							frappe.set_route(
-								"print-format-builder-beta",
-								doc.name
-							);
+						.then((doc) => {
+							frappe.set_route("print-format-builder-beta", doc.name);
 						})
 						.finally(() => {
 							d.get_primary_btn().prop("disabled", false);
 						});
 				}
-			}
+			},
 		});
 		d.set_value("action", "Create");
 		d.show();
diff --git a/frappe/public/html/print_template.html b/frappe/public/html/print_template.html
index e2ff9c9c76..80069f4bf3 100644
--- a/frappe/public/html/print_template.html
+++ b/frappe/public/html/print_template.html
@@ -27,9 +27,11 @@
 			
        
 		{% endif %}
 
-		
         20 %}
-					style="font-size: 4.0pt"
+					{% if can_use_smaller_font %}
+						style="font-size: 4.0pt"
+					{% endif %}
 				{% endif %}
 			>
 			{% if print_settings.letter_head %}
diff --git a/frappe/public/icons/timeless/icons.svg b/frappe/public/icons/timeless/icons.svg
index af63d0c8ff..f5a34a14c9 100644
--- a/frappe/public/icons/timeless/icons.svg
+++ b/frappe/public/icons/timeless/icons.svg
@@ -43,6 +43,21 @@
 		
 	
 
+	
+		
+		
+	
+
+	
+		
+		
+	
+
+	
+		
+		
+	
+
 	
 		
 		
@@ -116,8 +131,14 @@
 		
 	
 
-	
-		
+	
+		
+		
+	
+
+	
+		
+		
 	
 
 	
@@ -216,8 +237,14 @@
 		
 	
 
-	
-		
+	
+		
+	
+
+	
+		
+		
+		
 	
 
 	
@@ -814,6 +841,13 @@
 		
 	
 
+	
+		
+		
+		
+		
+	
+
 	
 		
 		
@@ -821,6 +855,13 @@
 		
 	
 
+	
+		
+		
+		
+		
+	
+
 	
 		
 	
@@ -859,6 +900,12 @@
 		
 	
 
+	
+		
+		
+		
+	
+
 	
 		
 		
 						
        ${title}
        
 						
 					
        
 					
        
 						${content}
 					
        
 					
 				
        
@@ -49,16 +46,23 @@ frappe.ui.Dialog = class Dialog extends frappe.ui.Dialog {
 		return this.$wrapper.find(".modal-footer .btn-primary");
 	}
 
+	get_secondary_btn() {
+		return this.$wrapper.find(".modal-footer .btn-secondary");
+	}
+
 	set_primary_action(label, click) {
-		this.$wrapper.find('.modal-footer').removeClass('hidden');
-		return super.set_primary_action(label, click)
-			.removeClass('hidden');
+		this.$wrapper.find(".modal-footer").removeClass("hidden");
+		return super.set_primary_action(label, click).removeClass("hidden");
+	}
+
+	set_secondary_action(click) {
+		return super.set_secondary_action(click).removeClass("hidden");
 	}
 
 	make() {
 		super.make();
 		if (this.fields) {
-			this.$wrapper.find('.section-body').addClass('w-100');
+			this.$wrapper.find(".section-body").addClass("w-100");
 		}
 	}
 };
diff --git a/frappe/public/js/checkout.bundle.js b/frappe/public/js/checkout.bundle.js
deleted file mode 100644
index 954e838fa8..0000000000
--- a/frappe/public/js/checkout.bundle.js
+++ /dev/null
@@ -1 +0,0 @@
-import "./integrations/razorpay";
diff --git a/frappe/public/js/form.bundle.js b/frappe/public/js/form.bundle.js
index 38d76f1f26..f2d19f5f3b 100644
--- a/frappe/public/js/form.bundle.js
+++ b/frappe/public/js/form.bundle.js
@@ -13,4 +13,4 @@ import "./frappe/form/templates/users_in_sidebar.html";
 import "./frappe/views/formview.js";
 import "./frappe/form/form.js";
 import "./frappe/meta_tag.js";
-import "./frappe/doctype/"
+import "./frappe/doctype/";
diff --git a/frappe/public/js/form_builder/FormBuilder.vue b/frappe/public/js/form_builder/FormBuilder.vue
new file mode 100644
index 0000000000..fe7eaaa682
--- /dev/null
+++ b/frappe/public/js/form_builder/FormBuilder.vue
@@ -0,0 +1,289 @@
+
+
+
+
+
diff --git a/frappe/public/js/form_builder/components/Column.vue b/frappe/public/js/form_builder/components/Column.vue
new file mode 100644
index 0000000000..acb1ff735e
--- /dev/null
+++ b/frappe/public/js/form_builder/components/Column.vue
@@ -0,0 +1,257 @@
+
+
+
+
+
diff --git a/frappe/public/js/form_builder/components/EditableInput.vue b/frappe/public/js/form_builder/components/EditableInput.vue
new file mode 100644
index 0000000000..8964838f4a
--- /dev/null
+++ b/frappe/public/js/form_builder/components/EditableInput.vue
@@ -0,0 +1,88 @@
+
+
+
+
+
diff --git a/frappe/public/js/form_builder/components/Field.vue b/frappe/public/js/form_builder/components/Field.vue
new file mode 100644
index 0000000000..e0230765b5
--- /dev/null
+++ b/frappe/public/js/form_builder/components/Field.vue
@@ -0,0 +1,180 @@
+
+
+
+
+
diff --git a/frappe/public/js/form_builder/components/FieldProperties.vue b/frappe/public/js/form_builder/components/FieldProperties.vue
new file mode 100644
index 0000000000..b8e687ac06
--- /dev/null
+++ b/frappe/public/js/form_builder/components/FieldProperties.vue
@@ -0,0 +1,100 @@
+
+
+
+
+
diff --git a/frappe/public/js/form_builder/components/FieldTypes.vue b/frappe/public/js/form_builder/components/FieldTypes.vue
new file mode 100644
index 0000000000..2907614a51
--- /dev/null
+++ b/frappe/public/js/form_builder/components/FieldTypes.vue
@@ -0,0 +1,93 @@
+
+
+
+
+
diff --git a/frappe/public/js/form_builder/components/SearchBox.vue b/frappe/public/js/form_builder/components/SearchBox.vue
new file mode 100644
index 0000000000..80f7ed97e7
--- /dev/null
+++ b/frappe/public/js/form_builder/components/SearchBox.vue
@@ -0,0 +1,35 @@
+
+
+
+
+
diff --git a/frappe/public/js/form_builder/components/Section.vue b/frappe/public/js/form_builder/components/Section.vue
new file mode 100644
index 0000000000..c97fe1e4d8
--- /dev/null
+++ b/frappe/public/js/form_builder/components/Section.vue
@@ -0,0 +1,270 @@
+
+
+
+
+
diff --git a/frappe/public/js/form_builder/components/Sidebar.vue b/frappe/public/js/form_builder/components/Sidebar.vue
new file mode 100644
index 0000000000..ed50510cc3
--- /dev/null
+++ b/frappe/public/js/form_builder/components/Sidebar.vue
@@ -0,0 +1,130 @@
+
+
+
+
+
diff --git a/frappe/public/js/form_builder/components/Tabs.vue b/frappe/public/js/form_builder/components/Tabs.vue
new file mode 100644
index 0000000000..b587d9d37e
--- /dev/null
+++ b/frappe/public/js/form_builder/components/Tabs.vue
@@ -0,0 +1,331 @@
+
+
+
+
+
diff --git a/frappe/public/js/form_builder/components/controls/AttachControl.vue b/frappe/public/js/form_builder/components/controls/AttachControl.vue
new file mode 100644
index 0000000000..86cdf7c5ac
--- /dev/null
+++ b/frappe/public/js/form_builder/components/controls/AttachControl.vue
@@ -0,0 +1,20 @@
+
+
+
+
diff --git a/frappe/public/js/form_builder/components/controls/ButtonControl.vue b/frappe/public/js/form_builder/components/controls/ButtonControl.vue
new file mode 100644
index 0000000000..bb6d4b8388
--- /dev/null
+++ b/frappe/public/js/form_builder/components/controls/ButtonControl.vue
@@ -0,0 +1,32 @@
+
+
+
+
+
+
diff --git a/frappe/public/js/form_builder/components/controls/CheckControl.vue b/frappe/public/js/form_builder/components/controls/CheckControl.vue
new file mode 100644
index 0000000000..0d786de0b4
--- /dev/null
+++ b/frappe/public/js/form_builder/components/controls/CheckControl.vue
@@ -0,0 +1,52 @@
+
+
+
+
+
diff --git a/frappe/public/js/form_builder/components/controls/CodeControl.vue b/frappe/public/js/form_builder/components/controls/CodeControl.vue
new file mode 100644
index 0000000000..d42bc43f81
--- /dev/null
+++ b/frappe/public/js/form_builder/components/controls/CodeControl.vue
@@ -0,0 +1,72 @@
+
+
+
+
diff --git a/frappe/public/js/form_builder/components/controls/DataControl.vue b/frappe/public/js/form_builder/components/controls/DataControl.vue
new file mode 100644
index 0000000000..2052912130
--- /dev/null
+++ b/frappe/public/js/form_builder/components/controls/DataControl.vue
@@ -0,0 +1,101 @@
+
+
+
+
+
+
diff --git a/frappe/public/js/form_builder/components/controls/GeolocationControl.vue b/frappe/public/js/form_builder/components/controls/GeolocationControl.vue
new file mode 100644
index 0000000000..b46f6aed33
--- /dev/null
+++ b/frappe/public/js/form_builder/components/controls/GeolocationControl.vue
@@ -0,0 +1,37 @@
+
+
+
+
+
diff --git a/frappe/public/js/form_builder/components/controls/ImageControl.vue b/frappe/public/js/form_builder/components/controls/ImageControl.vue
new file mode 100644
index 0000000000..f76cf0300d
--- /dev/null
+++ b/frappe/public/js/form_builder/components/controls/ImageControl.vue
@@ -0,0 +1,14 @@
+
+
+
diff --git a/frappe/public/js/form_builder/components/controls/LinkControl.vue b/frappe/public/js/form_builder/components/controls/LinkControl.vue
new file mode 100644
index 0000000000..ce58c9a81e
--- /dev/null
+++ b/frappe/public/js/form_builder/components/controls/LinkControl.vue
@@ -0,0 +1,86 @@
+
+
+
+
\ No newline at end of file
diff --git a/frappe/public/js/form_builder/components/controls/RatingControl.vue b/frappe/public/js/form_builder/components/controls/RatingControl.vue
new file mode 100644
index 0000000000..9247f3e925
--- /dev/null
+++ b/frappe/public/js/form_builder/components/controls/RatingControl.vue
@@ -0,0 +1,48 @@
+
+
+
+
+
diff --git a/frappe/public/js/form_builder/components/controls/SelectControl.vue b/frappe/public/js/form_builder/components/controls/SelectControl.vue
new file mode 100644
index 0000000000..091430a6fe
--- /dev/null
+++ b/frappe/public/js/form_builder/components/controls/SelectControl.vue
@@ -0,0 +1,124 @@
+
+
+
+
+
diff --git a/frappe/public/js/form_builder/components/controls/SignatureControl.vue b/frappe/public/js/form_builder/components/controls/SignatureControl.vue
new file mode 100644
index 0000000000..4a07b6af83
--- /dev/null
+++ b/frappe/public/js/form_builder/components/controls/SignatureControl.vue
@@ -0,0 +1,31 @@
+
+
+
+
+
diff --git a/frappe/public/js/form_builder/components/controls/TableControl.vue b/frappe/public/js/form_builder/components/controls/TableControl.vue
new file mode 100644
index 0000000000..425997e7ea
--- /dev/null
+++ b/frappe/public/js/form_builder/components/controls/TableControl.vue
@@ -0,0 +1,86 @@
+
+
+
+
+
diff --git a/frappe/public/js/form_builder/components/controls/TextControl.vue b/frappe/public/js/form_builder/components/controls/TextControl.vue
new file mode 100644
index 0000000000..8a5dbdf947
--- /dev/null
+++ b/frappe/public/js/form_builder/components/controls/TextControl.vue
@@ -0,0 +1,131 @@
+
+
+
+