Merge branch 'develop' of https://github.com/frappe/frappe into put-req-integrations

.github/helper/semgrep_rules/frappe_correctness.yml

@@ -98,8 +98,6 @@ rules:
   languages: [python]
   severity: WARNING
   paths:
-      exclude:
-        - test_*.py
       include:
         - "*/**/doctype/*"
 

.github/helper/semgrep_rules/security.yml

@@ -8,10 +8,6 @@ rules:
     dynamic content. Avoid it or use safe_eval().
   languages: [python]
   severity: ERROR
-  paths:
-    exclude:
-      - frappe/__init__.py
-      - frappe/commands/utils.py
 
 - id: frappe-sqli-format-strings
   patterns:

.github/semantic.yml

@@ -11,3 +11,20 @@ allowRevertCommits: true
 
 # For allowed PR types: https://github.com/commitizen/conventional-commit-types/blob/v3.0.0/index.json
 # Tool Reference: https://github.com/zeke/semantic-pull-requests
+
+# By default types specified in commitizen/conventional-commit-types is used.
+# See: https://github.com/commitizen/conventional-commit-types/blob/v3.0.0/index.json
+# You can override the valid types
+types:
+  - BREAKING CHANGE
+  - feat
+  - fix
+  - docs
+  - style
+  - refactor
+  - perf
+  - test
+  - build
+  - ci
+  - chore
+  - revert

.github/workflows/semgrep.yml

@@ -1,34 +1,18 @@
 name: Semgrep
 
 on:
-  pull_request:
-    branches:
-      - develop
-      - version-13-hotfix
-      - version-13-pre-release
+  pull_request: { }
+
 jobs:
   semgrep:
     name: Frappe Linter
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
-    - name: Setup python3
-      uses: actions/setup-python@v2
-      with:
-        python-version: 3.8
-
-    - name: Setup semgrep
-      run: |
-        python -m pip install -q semgrep
-        git fetch origin $GITHUB_BASE_REF:$GITHUB_BASE_REF -q
-
-    - name: Semgrep errors
-      run: |
-        files=$(git diff --name-only --diff-filter=d $GITHUB_BASE_REF)
-        [[ -d .github/helper/semgrep_rules ]] && semgrep --severity ERROR --config=.github/helper/semgrep_rules --quiet --error $files
-        semgrep --config="r/python.lang.correctness" --quiet --error $files
-
-    - name: Semgrep warnings
-      run: |
-        files=$(git diff --name-only --diff-filter=d $GITHUB_BASE_REF)
-        [[ -d .github/helper/semgrep_rules ]] && semgrep --severity WARNING --severity INFO --config=.github/helper/semgrep_rules --quiet $files
+      - uses: actions/checkout@v2
+      - uses: returntocorp/semgrep-action@v1
+        env:
+            SEMGREP_TIMEOUT: 120
+        with:
+            config: >-
+              r/python.lang.correctness
+              .github/helper/semgrep_rules

cypress/integration/navigation.js

@@ -0,0 +1,14 @@
+context('Navigation', () => {
+	before(() => {
+		cy.login();
+		cy.visit('/app/website');
+	});
+	it('Navigate to route with hash in document name', () => {
+		cy.insert_doc('ToDo', {'__newname': 'ABC#123', 'description': 'Test this', 'ignore_duplicate': true});
+		cy.visit('/app/todo/ABC#123');
+		cy.title().should('eq', 'Test this - ABC#123');
+		cy.get_field('description', 'Text Editor').contains('Test this');
+		cy.go('back');
+		cy.title().should('eq', 'Website');
+	});
+});

esbuild/esbuild.js

@@ -8,6 +8,7 @@ let yargs = require("yargs");
 let cliui = require("cliui")();
 let chalk = require("chalk");
 let html_plugin = require("./frappe-html");
+let rtlcss = require('rtlcss');
 let postCssPlugin = require("esbuild-plugin-postcss2").default;
 let ignore_assets = require("./ignore-assets");
 let sass_options = require("./sass_options");
@@ -96,9 +97,9 @@ async function execute() {
 		await clean_dist_folders(APPS);
 	}
 
-	let result;
+	let results;
 	try {
-		result = await build_assets_for_apps(APPS, FILES_TO_BUILD);
+		results = await build_assets_for_apps(APPS, FILES_TO_BUILD);
 	} catch (e) {
 		log_error("There were some problems during build");
 		log();
@@ -107,13 +108,15 @@ async function execute() {
 	}
 
 	if (!WATCH_MODE) {
-		log_built_assets(result.metafile);
+		log_built_assets(results);
 		console.timeEnd(TOTAL_BUILD_TIME);
 		log();
 	} else {
 		log("Watching for changes...");
 	}
-	return await write_assets_json(result.metafile);
+	for (const result of results) {
+		await write_assets_json(result.metafile);
+	}
 }
 
 function build_assets_for_apps(apps, files) {
@@ -125,6 +128,8 @@ function build_assets_for_apps(apps, files) {
 		let output_path = assets_path;
 
 		let file_map = {};
+		let style_file_map = {};
+		let rtl_style_file_map = {};
 		for (let file of files) {
 			let relative_app_path = path.relative(apps_path, file);
 			let app = relative_app_path.split(path.sep)[0];
@@ -140,19 +145,32 @@ function build_assets_for_apps(apps, files) {
 			}
 			output_name = path.join(app, "dist", output_name);
 
-			if (Object.keys(file_map).includes(output_name)) {
+			if (Object.keys(file_map).includes(output_name) || Object.keys(style_file_map).includes(output_name)) {
 				log_warn(
 					`Duplicate output file ${output_name} generated from ${file}`
 				);
 			}
-
-			file_map[output_name] = file;
+			if ([".css", ".scss", ".less", ".sass", ".styl"].includes(extension)) {
+				style_file_map[output_name] = file;
+				rtl_style_file_map[output_name.replace('/css/', '/css-rtl/')] = file;
+			} else {
+				file_map[output_name] = file;
+			}
 		}
-
-		return build_files({
+		let build = build_files({
 			files: file_map,
 			outdir: output_path
 		});
+		let style_build = build_style_files({
+			files: style_file_map,
+			outdir: output_path
+		});
+		let rtl_style_build = build_style_files({
+			files: rtl_style_file_map,
+			outdir: output_path,
+			rtl_style: true
+		});
+		return Promise.all([build, style_build, rtl_style_build]);
 	});
 }
 
@@ -203,7 +221,33 @@ function get_files_to_build(files) {
 }
 
 function build_files({ files, outdir }) {
-	return esbuild.build({
+	let build_plugins = [
+		html_plugin,
+		vue(),
+	];
+	return esbuild.build(get_build_options(files, outdir, build_plugins));
+}
+
+function build_style_files({ files, outdir, rtl_style=false }) {
+	let plugins = [];
+	if (rtl_style) {
+		plugins.push(rtlcss);
+	}
+
+	let build_plugins = [
+		ignore_assets,
+		postCssPlugin({
+			plugins: plugins,
+			sassOptions: sass_options
+		})
+	];
+
+	plugins.push(require("autoprefixer"));
+	return esbuild.build(get_build_options(files, outdir, build_plugins));
+}
+
+function get_build_options(files, outdir, plugins) {
+	return {
 		entryPoints: files,
 		entryNames: "[dir]/[name].[hash]",
 		outdir,
@@ -217,17 +261,9 @@ function build_files({ files, outdir }) {
 				PRODUCTION ? "production" : "development"
 			)
 		},
-		plugins: [
-			html_plugin,
-			ignore_assets,
-			vue(),
-			postCssPlugin({
-				plugins: [require("autoprefixer")],
-				sassOptions: sass_options
-			})
-		],
+		plugins: plugins,
 		watch: get_watch_config()
-	});
+	};
 }
 
 function get_watch_config() {
@@ -260,7 +296,8 @@ async function clean_dist_folders(apps) {
 		let public_path = get_public_path(app);
 		let paths = [
 			path.resolve(public_path, "dist", "js"),
-			path.resolve(public_path, "dist", "css")
+			path.resolve(public_path, "dist", "css"),
+			path.resolve(public_path, "dist", "css-rtl")
 		];
 		for (let target of paths) {
 			if (fs.existsSync(target)) {
@@ -272,7 +309,11 @@ async function clean_dist_folders(apps) {
 	}
 }
 
-function log_built_assets(metafile) {
+function log_built_assets(results) {
+	let outputs = {};
+	for (const result of results) {
+		outputs = Object.assign(outputs, result.metafile.outputs);
+	}
 	let column_widths = [60, 20];
 	cliui.div(
 		{
@@ -287,9 +328,9 @@ function log_built_assets(metafile) {
 	cliui.div("");
 
 	let output_by_dist_path = {};
-	for (let outfile in metafile.outputs) {
+	for (let outfile in outputs) {
 		if (outfile.endsWith(".map")) continue;
-		let data = metafile.outputs[outfile];
+		let data = outputs[outfile];
 		outfile = path.resolve(outfile);
 		outfile = path.relative(assets_path, outfile);
 		let filename = path.basename(outfile);
@@ -344,7 +385,11 @@ async function write_assets_json(metafile) {
 		let info = metafile.outputs[output];
 		let asset_path = "/" + path.relative(sites_path, output);
 		if (info.entryPoint) {
-			out[path.basename(info.entryPoint)] = asset_path;
+			let key = path.basename(info.entryPoint);
+			if (key.endsWith('.css') && asset_path.includes('/css-rtl/')) {
+				key = `rtl_${key}`;
+			}
+			out[key] = asset_path;
 		}
 	}
 
@@ -483,4 +528,4 @@ function log_rebuilt_assets(prev_assets, new_assets) {
 		log("    " + filename);
 	}
 	log();
-}
+}

frappe/__init__.py

@@ -28,6 +28,8 @@ from .exceptions import *
 from .utils.jinja import (get_jenv, get_template, render_template, get_email_from_template, get_jloader)
 from .utils.lazy_loader import lazy_import
 
+from frappe.query_builder import get_query_builder
+
 # Lazy imports
 faker = lazy_import('faker')
 
@@ -118,6 +120,7 @@ def set_user_lang(user, user_language=None):
 
 # local-globals
 db = local("db")
+qb = local("qb")
 conf = local("conf")
 form = form_dict = local("form_dict")
 request = local("request")
@@ -202,6 +205,7 @@ def init(site, sites_path=None, new_site=False):
 	local.form_dict = _dict()
 	local.session = _dict()
 	local.dev_server = _dev_server
+	local.qb = get_query_builder(local.conf.db_type or "mariadb")
 
 	setup_module_map()
 

frappe/auth.py

@@ -1,31 +1,58 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
-import datetime
-from frappe import _
+# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors
+# MIT License. See LICENSE
+from urllib.parse import quote
+
 import frappe
 import frappe.database
 import frappe.utils
-from frappe.utils import cint, flt, get_datetime, datetime, date_diff, today
 import frappe.utils.user
-from frappe import conf
-from frappe.sessions import Session, clear_sessions, delete_session
-from frappe.modules.patch_handler import check_session_stopped
-from frappe.translate import get_lang_code
-from frappe.utils.password import check_password, delete_login_failed_cache
+from frappe import _, conf
 from frappe.core.doctype.activity_log.activity_log import add_authentication_log
-from frappe.twofactor import (should_run_2fa, authenticate_for_2factor,
-	confirm_otp_token, get_cached_user_pass)
+from frappe.modules.patch_handler import check_session_stopped
+from frappe.sessions import Session, clear_sessions, delete_session
+from frappe.translate import get_language
+from frappe.twofactor import authenticate_for_2factor, confirm_otp_token, get_cached_user_pass, should_run_2fa
+from frappe.utils import cint, date_diff, datetime, get_datetime, today
+from frappe.utils.password import check_password
 from frappe.website.utils import get_home_page
-from urllib.parse import quote
 
 
 class HTTPRequest:
 	def __init__(self):
-		# Get Environment variables
-		self.domain = frappe.request.host
-		if self.domain and self.domain.startswith('www.'):
-			self.domain = self.domain[4:]
+		# set frappe.local.request_ip
+		self.set_request_ip()
 
+		# load cookies
+		self.set_cookies()
+
+		# set frappe.local.db
+		self.connect()
+
+		# login and start/resume user session
+		self.set_session()
+
+		# set request language
+		self.set_lang()
+
+		# match csrf token from current session
+		self.validate_csrf_token()
+
+		# write out latest cookies
+		frappe.local.cookie_manager.init_cookies()
+
+		# check session status
+		check_session_stopped()
+
+	@property
+	def domain(self):
+		if not getattr(self, "_domain", None):
+			self._domain = frappe.request.host
+			if self._domain and self._domain.startswith('www.'):
+				self._domain = self._domain[4:]
+
+		return self._domain
+
+	def set_request_ip(self):
 		if frappe.get_request_header('X-Forwarded-For'):
 			frappe.local.request_ip = (frappe.get_request_header('X-Forwarded-For').split(",")[0]).strip()
 
@@ -35,37 +62,21 @@ class HTTPRequest:
 		else:
 			frappe.local.request_ip = '127.0.0.1'
 
-		# language
-		self.set_lang()
-
-		# load cookies
+	def set_cookies(self):
 		frappe.local.cookie_manager = CookieManager()
 
-		# set db
-		self.connect()
-
-		# login
+	def set_session(self):
 		frappe.local.login_manager = LoginManager()
 
-		if frappe.form_dict._lang:
-			lang = get_lang_code(frappe.form_dict._lang)
-			if lang:
-				frappe.local.lang = lang
-
-		self.validate_csrf_token()
-
-		# write out latest cookies
-		frappe.local.cookie_manager.init_cookies()
-
-		# check status
-		check_session_stopped()
-
 	def validate_csrf_token(self):
 		if frappe.local.request and frappe.local.request.method in ("POST", "PUT", "DELETE"):
-			if not frappe.local.session: return
-			if not frappe.local.session.data.csrf_token \
-				or frappe.local.session.data.device=="mobile" \
-				or frappe.conf.get('ignore_csrf', None):
+			if not frappe.local.session:
+				return
+			if (
+				not frappe.local.session.data.csrf_token
+				or frappe.local.session.data.device == "mobile"
+				or frappe.conf.get('ignore_csrf', None)
+			):
 				# not via boot
 				return
 
@@ -79,17 +90,18 @@ class HTTPRequest:
 				frappe.throw(_("Invalid Request"), frappe.CSRFTokenError)
 
 	def set_lang(self):
-		from frappe.translate import guess_language
-		frappe.local.lang = guess_language()
+		frappe.local.lang = get_language()
 
 	def get_db_name(self):
 		"""get database name from conf"""
 		return conf.db_name
 
-	def connect(self, ac_name = None):
+	def connect(self):
 		"""connect to db, from ac_name or db_name"""
-		frappe.local.db = frappe.database.get_db(user = self.get_db_name(), \
-			password = getattr(conf, 'db_password', ''))
+		frappe.local.db = frappe.database.get_db(
+			user=self.get_db_name(),
+			password=getattr(conf, 'db_password', '')
+		)
 
 class LoginManager:
 	def __init__(self):
@@ -142,8 +154,9 @@ class LoginManager:
 		self.make_session()
 		self.setup_boot_cache()
 		self.set_user_info()
+		self.clear_preferred_language()
 
-	def get_user_info(self, resume=False):
+	def get_user_info(self):
 		self.info = frappe.db.get_value("User", self.user,
 			["user_type", "first_name", "last_name", "user_image"], as_dict=1)
 
@@ -181,11 +194,13 @@ class LoginManager:
 			frappe.local.response["redirect_to"] = redirect_to
 			frappe.cache().hdel('redirect_after_login', self.user)
 
-
 		frappe.local.cookie_manager.set_cookie("full_name", self.full_name)
 		frappe.local.cookie_manager.set_cookie("user_id", self.user)
 		frappe.local.cookie_manager.set_cookie("user_image", self.info.user_image or "")
 
+	def clear_preferred_language(self):
+		frappe.local.cookie_manager.delete_cookie("preferred_language")
+
 	def make_session(self, resume=False):
 		# start session
 		frappe.local.session_obj = Session(user=self.user, resume=resume,

frappe/cache_manager.py

@@ -141,17 +141,13 @@ def build_table_count_cache():
 		return
 
 	_cache = frappe.cache()
-	data = frappe.db.multisql({
-		"mariadb": """
-			SELECT 	table_name AS name,
-					table_rows AS count
-			FROM information_schema.tables""",
-		"postgres": """
-			SELECT 	"relname" AS name,
-					"n_tup_ins" AS count
-			FROM "pg_stat_all_tables"
-		"""
-	}, as_dict=1)
+	table_name = frappe.qb.Field("table_name").as_("name")
+	table_rows = frappe.qb.Field("table_rows").as_("count")
+	information_schema = frappe.qb.Schema("information_schema")
+
+	query = frappe.qb.from_(information_schema.tables).select(table_name, table_rows)
+
+	data = frappe.db.sql(query, as_dict=1)
 
 	counts = {d.get('name').lstrip('tab'): d.get('count', None) for d in data}
 	_cache.set_value("information_schema:counts", counts)

frappe/commands/__init__.py

@@ -102,7 +102,9 @@ def get_commands():
 	from .site import commands as site_commands
 	from .translate import commands as translate_commands
 	from .utils import commands as utils_commands
+	from .redis import commands as redis_commands
 
-	return list(set(scheduler_commands + site_commands + translate_commands + utils_commands))
+	all_commands = scheduler_commands + site_commands + translate_commands + utils_commands + redis_commands
+	return list(set(all_commands))
 
 commands = get_commands()

frappe/commands/redis.py

@@ -0,0 +1,53 @@
+import os
+
+import click
+
+import frappe
+from frappe.utils.rq import RedisQueue
+from frappe.installer import update_site_config
+
+@click.command('create-rq-users')
+@click.option('--set-admin-password', is_flag=True, default=False, help='Set new Redis admin(default user) password')
+@click.option('--use-rq-auth', is_flag=True, default=False, help='Enable Redis authentication for sites')
+def create_rq_users(set_admin_password=False, use_rq_auth=False):
+	"""Create Redis Queue users and add to acl and app configs.
+
+	acl config file will be used by redis server while starting the server
+	and app config is used by app while connecting to redis server.
+	"""
+	acl_file_path = os.path.abspath('../config/redis_queue.acl')
+
+	with frappe.init_site():
+		acl_list, user_credentials = RedisQueue.gen_acl_list(
+			set_admin_password=set_admin_password)
+
+	with open(acl_file_path, 'w') as f:
+		f.writelines([acl+'\n' for acl in acl_list])
+
+	sites_path = os.getcwd()
+	common_site_config_path = os.path.join(sites_path, 'common_site_config.json')
+	update_site_config("rq_username", user_credentials['bench'][0], validate=False,
+		site_config_path=common_site_config_path)
+	update_site_config("rq_password", user_credentials['bench'][1], validate=False,
+		site_config_path=common_site_config_path)
+	update_site_config("use_rq_auth", use_rq_auth, validate=False,
+		site_config_path=common_site_config_path)
+
+	click.secho('* ACL and site configs are updated with new user credentials. '
+		'Please restart Redis Queue server to enable namespaces.',
+		fg='green')
+
+	if set_admin_password:
+		env_key = 'RQ_ADMIN_PASWORD'
+		click.secho('* Redis admin password is successfully set up. '
+			'Include below line in .bashrc file for system to use',
+			fg='green')
+		click.secho(f"`export {env_key}={user_credentials['default'][1]}`")
+		click.secho('NOTE: Please save the admin password as you '
+			'can not access redis server without the password',
+			fg='yellow')
+
+
+commands = [
+	create_rq_users
+]

frappe/commands/scheduler.py

@@ -172,9 +172,13 @@ def start_scheduler():
 @click.command('worker')
 @click.option('--queue', type=str)
 @click.option('--quiet', is_flag = True, default = False, help = 'Hide Log Outputs')
-def start_worker(queue, quiet = False):
+@click.option('-u', '--rq-username', default=None, help='Redis ACL user')
+@click.option('-p', '--rq-password', default=None, help='Redis ACL user password')
+def start_worker(queue, quiet = False, rq_username=None, rq_password=None):
+	"""Site is used to find redis credentals.
+	"""
 	from frappe.utils.background_jobs import start_worker
-	start_worker(queue, quiet = quiet)
+	start_worker(queue, quiet = quiet, rq_username=rq_username, rq_password=rq_password)
 
 @click.command('ready-for-migration')
 @click.option('--site', help='site name')

frappe/commands/utils.py

@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-
 import json
 import os
 import subprocess
@@ -14,6 +12,13 @@ from frappe.exceptions import SiteNotSpecifiedError
 from frappe.utils import get_bench_path, update_progress_bar, cint
 
 
+DATA_IMPORT_DEPRECATION = click.style(
+	"[DEPRECATED] The `import-csv` command used 'Data Import Legacy' which has been deprecated.\n"
+	"Use `data-import` command instead to import data via 'Data Import'.",
+	fg="yellow"
+)
+
+
 @click.command('build')
 @click.option('--app', help='Build assets for app')
 @click.option('--apps', help='Build assets for specific apps')
@@ -350,7 +355,8 @@ def import_doc(context, path, force=False):
 	if not context.sites:
 		raise SiteNotSpecifiedError
 
-@click.command('import-csv')
+
+@click.command('import-csv', help=DATA_IMPORT_DEPRECATION)
 @click.argument('path')
 @click.option('--only-insert', default=False, is_flag=True, help='Do not overwrite existing records')
 @click.option('--submit-after-import', default=False, is_flag=True, help='Submit document after importing it')
@@ -358,32 +364,8 @@ def import_doc(context, path, force=False):
 @click.option('--no-email', default=True, is_flag=True, help='Send email if applicable')
 @pass_context
 def import_csv(context, path, only_insert=False, submit_after_import=False, ignore_encoding_errors=False, no_email=True):
-	"Import CSV using data import"
-	from frappe.core.doctype.data_import_legacy import importer
-	from frappe.utils.csvutils import read_csv_content
-	site = get_site(context)
-
-	if not os.path.exists(path):
-		path = os.path.join('..', path)
-	if not os.path.exists(path):
-		print('Invalid path {0}'.format(path))
-		sys.exit(1)
-
-	with open(path, 'r') as csvfile:
-		content = read_csv_content(csvfile.read())
-
-	frappe.init(site=site)
-	frappe.connect()
-
-	try:
-		importer.upload(content, submit_after_import=submit_after_import, no_email=no_email,
-			ignore_encoding_errors=ignore_encoding_errors, overwrite=not only_insert,
-			via_console=True)
-		frappe.db.commit()
-	except Exception:
-		print(frappe.get_traceback())
-
-	frappe.destroy()
+	click.secho(DATA_IMPORT_DEPRECATION)
+	sys.exit(1)
 
 
 @click.command('data-import')
@@ -569,32 +551,16 @@ def run_tests(context, app=None, module=None, doctype=None, test=(), profile=Fal
 
 	if coverage:
 		from coverage import Coverage
+		from frappe.coverage import STANDARD_INCLUSIONS, STANDARD_EXCLUSIONS, FRAPPE_EXCLUSIONS
 
 		# Generate coverage report only for app that is being tested
 		source_path = os.path.join(get_bench_path(), 'apps', app or 'frappe')
-		incl = [
-			'*.py',
-		]
-		omit = [
-			'*.js',
-			'*.xml',
-			'*.pyc',
-			'*.css',
-			'*.less',
-			'*.scss',
-			'*.vue',
-			'*.html',
-			'*/test_*',
-			'*/node_modules/*',
-			'*/doctype/*/*_dashboard.py',
-			'*/patches/*',
-		]
+		omit = STANDARD_EXCLUSIONS[:]
 
 		if not app or app == 'frappe':
-			omit.append('*/tests/*')
-			omit.append('*/commands/*')
+			omit.extend(FRAPPE_EXCLUSIONS)
 
-		cov = Coverage(source=[source_path], omit=omit, include=incl)
+		cov = Coverage(source=[source_path], omit=omit, include=STANDARD_INCLUSIONS)
 		cov.start()
 
 	ret = frappe.test_runner.main(app, module, doctype, context.verbose, tests=tests,
@@ -767,26 +733,49 @@ def set_config(context, key, value, global_=False, parse=False, as_dict=False):
 			frappe.destroy()
 
 
-@click.command('version')
-def get_version():
-	"Show the versions of all the installed apps"
+@click.command("version")
+@click.option("-f", "--format", "output",
+	type=click.Choice(["plain", "table", "json", "legacy"]), help="Output format", default="legacy")
+def get_version(output):
+	"""Show the versions of all the installed apps."""
 	from git import Repo
+	from frappe.utils.commands import render_table
 	from frappe.utils.change_log import get_app_branch
-	frappe.init('')
+
+	frappe.init("")
+	data = []
 
 	for app in sorted(frappe.get_all_apps()):
-		branch_name = get_app_branch(app)
 		module = frappe.get_module(app)
 		app_hooks = frappe.get_module(app + ".hooks")
 		repo = Repo(frappe.get_app_path(app, ".."))
-		branch = repo.head.ref.name
-		commit = repo.head.ref.commit.hexsha[:7]
 
-		if hasattr(app_hooks, '{0}_version'.format(branch_name)):
-			click.echo("{0} {1} {2} ({3})".format(app, getattr(app_hooks, '{0}_version'.format(branch_name)), branch, commit))
+		app_info = frappe._dict()
+		app_info.app = app
+		app_info.branch = get_app_branch(app)
+		app_info.commit = repo.head.object.hexsha[:7]
+		app_info.version = getattr(app_hooks, f"{app_info.branch}_version", None) or module.__version__
 
-		elif hasattr(module, "__version__"):
-			click.echo("{0} {1} {2} ({3})".format(app, module.__version__, branch, commit))
+		data.append(app_info)
+
+	{
+		"legacy": lambda: [
+			click.echo(f"{app_info.app} {app_info.version}")
+			for app_info in data
+		],
+		"plain": lambda: [
+			click.echo(f"{app_info.app} {app_info.version} {app_info.branch} ({app_info.commit})")
+			for app_info in data
+		],
+		"table": lambda: render_table(
+			[["App", "Version", "Branch", "Commit"]] +
+			[
+				[app_info.app, app_info.version, app_info.branch, app_info.commit]
+				for app_info in data
+			]
+		),
+		"json": lambda: click.echo(json.dumps(data, indent=4)),
+	}[output]()
 
 
 @click.command('rebuild-global-search')

frappe/config/__init__.py

@@ -39,18 +39,13 @@ def get_modules_from_app(app):
 	)
 
 def get_all_empty_tables_by_module():
-	empty_tables = set(r[0] for r in frappe.db.multisql({
-		"mariadb": """
-			SELECT table_name
-			FROM information_schema.tables
-			WHERE table_rows = 0 and table_schema = "{}"
-			""".format(frappe.conf.db_name),
-		"postgres": """
-			SELECT "relname" as "table_name"
-			FROM "pg_stat_all_tables"
-			WHERE n_tup_ins = 0
-		"""
-	}))
+	table_rows = frappe.qb.Field("table_rows")
+	table_name = frappe.qb.Field("table_name")
+	information_schema = frappe.qb.Schema("information_schema")
+
+	query = frappe.qb.from_(information_schema.tables).select(table_name).where(table_rows == 0)
+
+	empty_tables = {r[0] for r in frappe.db.sql(query)}
 
 	results = frappe.get_all("DocType", fields=["name", "module"])
 	empty_tables_by_module = {}

frappe/core/doctype/access_log/access_log.py

@@ -1,9 +1,5 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2019, Frappe Technologies and contributors
 # For license information, please see license.txt
-
-# imports - standard imports
-# imports - module imports
 import frappe
 from frappe.model.document import Document
 

frappe/core/doctype/activity_log/feed.py

@@ -29,10 +29,12 @@ def update_feed(doc, method=None):
 			name = feed.name or doc.name
 
 			# delete earlier feed
-			frappe.db.sql("""delete from `tabActivity Log`
-				where
-					reference_doctype=%s and reference_name=%s
-					and link_doctype=%s""", (doctype, name,feed.link_doctype))
+			frappe.db.delete("Activity Log", {
+				"reference_doctype": doctype,
+				"reference_name": name,
+				"link_doctype": feed.link_doctype
+			})
+
 			frappe.get_doc({
 				"doctype": "Activity Log",
 				"reference_doctype": doctype,

frappe/core/doctype/communication/mixins.py

@@ -3,6 +3,7 @@ from frappe import _
 from frappe.core.utils import get_parent_doc
 from frappe.utils import parse_addr, get_formatted_email, get_url
 from frappe.email.doctype.email_account.email_account import EmailAccount
+from frappe.desk.doctype.todo.todo import ToDo
 
 class CommunicationEmailMixin:
 	"""Mixin class to handle communication mails.
@@ -76,6 +77,7 @@ class CommunicationEmailMixin:
 		if is_inbound_mail_communcation:
 			cc.append(self.get_owner())
 			cc = set(cc) - {self.sender_mailid}
+			cc.update(self.get_assignees())
 
 		cc = set(cc) - set(self.filter_thread_notification_disbled_users(cc))
 		cc = cc - set(self.mail_recipients(is_inbound_mail_communcation=is_inbound_mail_communcation))
@@ -176,8 +178,8 @@ class CommunicationEmailMixin:
 	def mail_attachments(self, print_format=None, print_html=None):
 		final_attachments = []
 
-		if print_format and print_html:
-			d = {'print_format': print_format, 'print_html': print_html, 'print_format_attachment': 1,
+		if print_format or print_html:
+			d = {'print_format': print_format, 'html': print_html, 'print_format_attachment': 1,
 					'doctype': self.reference_doctype, 'name': self.reference_name}
 			final_attachments.append(d)
 
@@ -201,6 +203,13 @@ class CommunicationEmailMixin:
 			self.mail_cc(is_inbound_mail_communcation = is_inbound_mail_communcation, include_sender=include_sender)
 		return set(all_ids) - set(final_ids)
 
+	def get_assignees(self):
+		"""Get owners of the reference document.
+		"""
+		filters = {'status': 'Open', 'reference_name': self.reference_name,
+			'reference_type': self.reference_doctype}
+		return ToDo.get_owners(filters)
+
 	@staticmethod
 	def filter_thread_notification_disbled_users(emails):
 		"""Filter users based on notifications for email threads setting is disabled.

frappe/core/doctype/data_export/exporter.py

@@ -7,7 +7,6 @@ import frappe.permissions
 import re, csv, os
 from frappe.utils.csvutils import UnicodeWriter
 from frappe.utils import cstr, formatdate, format_datetime, parse_json, cint, format_duration
-from frappe.core.doctype.data_import_legacy.importer import get_data_keys
 from frappe.core.doctype.access_log.access_log import make_access_log
 
 reflags = {
@@ -20,6 +19,15 @@ reflags = {
 	"D": re.DEBUG
 }
 
+def get_data_keys():
+	return frappe._dict({
+		"data_separator": _('Start entering data below this line'),
+		"main_table": _("Table") + ":",
+		"parent_table": _("Parent Table") + ":",
+		"columns": _("Column Name") + ":",
+		"doctype": _("DocType") + ":"
+	})
+
 @frappe.whitelist()
 def export_data(doctype=None, parent_doctype=None, all_doctypes=True, with_data=False,
 		select_columns=None, file_type='CSV', template=False, filters=None):

frappe/core/doctype/data_import/data_import.py

@@ -171,9 +171,6 @@ def import_file(
 	i.import_data()
 
 
-##############
-
-
 def import_doc(path, pre_process=None):
 	if os.path.isdir(path):
 		files = [os.path.join(path, f) for f in os.listdir(path)]
@@ -192,19 +189,8 @@ def import_doc(path, pre_process=None):
 			)
 			frappe.flags.mute_emails = False
 			frappe.db.commit()
-		elif f.endswith(".csv"):
-			validate_csv_import_file(f)
-			frappe.db.commit()
-
-
-def validate_csv_import_file(path):
-	if path.endswith(".csv"):
-		print()
-		print("This method is deprecated.")
-		print('Import CSV files using the command "bench --site sitename data-import"')
-		print("Or use the method frappe.core.doctype.data_import.data_import.import_file")
-		print()
-		raise Exception("Method deprecated")
+		else:
+			raise NotImplementedError("Only .json files can be imported")
 
 
 def export_json(

frappe/core/doctype/data_import_legacy/data_import_legacy.js

@@ -1,324 +0,0 @@
-// Copyright (c) 2017, Frappe Technologies and contributors
-// For license information, please see license.txt
-
-frappe.ui.form.on('Data Import Legacy', {
-	onload: function(frm) {
-		if (frm.doc.__islocal) {
-			frm.set_value("action", "");
-		}
-
-		frappe.call({
-			method: "frappe.core.doctype.data_import_legacy.data_import_legacy.get_importable_doctypes",
-			callback: function (r) {
-				let importable_doctypes = r.message;
-				frm.set_query("reference_doctype", function () {
-					return {
-						"filters": {
-							"issingle": 0,
-							"istable": 0,
-							"name": ['in', importable_doctypes]
-						}
-					};
-				});
-			}
-		}),
-
-		// should never check public
-		frm.fields_dict["import_file"].df.is_private = 1;
-
-		frappe.realtime.on("data_import_progress", function(data) {
-			if (data.data_import === frm.doc.name) {
-				if (data.reload && data.reload === true) {
-					frm.reload_doc();
-				}
-				if (data.progress) {
-					let progress_bar = $(frm.dashboard.progress_area.body).find(".progress-bar");
-					if (progress_bar) {
-						$(progress_bar).removeClass("progress-bar-danger").addClass("progress-bar-success progress-bar-striped");
-						$(progress_bar).css("width", data.progress + "%");
-					}
-				}
-			}
-		});
-	},
-
-	reference_doctype: function(frm){
-		if (frm.doc.reference_doctype) {
-			frappe.model.with_doctype(frm.doc.reference_doctype);
-		}
-	},
-
-	refresh: function(frm) {
-		frm.disable_save();
-		frm.dashboard.clear_headline();
-		if (frm.doc.reference_doctype && !frm.doc.import_file) {
-			frm.page.set_indicator(__('Attach file'), 'orange');
-		} else {
-			if (frm.doc.import_status) {
-				const listview_settings = frappe.listview_settings['Data Import Legacy'];
-				const indicator = listview_settings.get_indicator(frm.doc);
-
-				frm.page.set_indicator(indicator[0], indicator[1]);
-
-				if (frm.doc.import_status === "In Progress") {
-					frm.dashboard.add_progress("Data Import Progress", "0");
-					frm.set_read_only();
-					frm.refresh_fields();
-				}
-			}
-		}
-
-		if (frm.doc.reference_doctype) {
-			frappe.model.with_doctype(frm.doc.reference_doctype);
-		}
-
-		if(frm.doc.action == "Insert new records" || frm.doc.action == "Update records") {
-			frm.set_df_property("action", "read_only", 1);
-		}
-
-		frm.add_custom_button(__("Help"), function() {
-			frappe.help.show_video("6wiriRKPhmg");
-		});
-
-		if (frm.doc.reference_doctype && frm.doc.docstatus === 0) {
-			frm.add_custom_button(__("Download template"), function() {
-				frappe.data_import.download_dialog(frm).show();
-			});
-		}
-
-		if (frm.doc.reference_doctype && frm.doc.import_file && frm.doc.total_rows &&
-			frm.doc.docstatus === 0 && (!frm.doc.import_status || frm.doc.import_status == "Failed")) {
-			frm.page.set_primary_action(__("Start Import"), function() {
-				frappe.call({
-					btn: frm.page.btn_primary,
-					method: "frappe.core.doctype.data_import_legacy.data_import_legacy.import_data",
-					args: {
-						data_import: frm.doc.name
-					}
-				});
-			}).addClass('btn btn-primary');
-		}
-
-		if (frm.doc.log_details) {
-			frm.events.create_log_table(frm);
-		} else {
-			$(frm.fields_dict.import_log.wrapper).empty();
-		}
-	},
-
-	action: function(frm) {
-		if(!frm.doc.action) return;
-		if(!frm.doc.reference_doctype) {
-			frappe.msgprint(__("Please select document type first."));
-			frm.set_value("action", "");
-			return;
-		}
-
-		if(frm.doc.action == "Insert new records") {
-			frm.doc.insert_new = 1;
-		} else if (frm.doc.action == "Update records"){
-			frm.doc.overwrite = 1;
-		}
-		frm.save();
-	},
-
-	only_update: function(frm) {
-		frm.save();
-	},
-
-	submit_after_import: function(frm) {
-		frm.save();
-	},
-
-	skip_errors: function(frm) {
-		frm.save();
-	},
-
-	ignore_encoding_errors: function(frm) {
-		frm.save();
-	},
-
-	no_email: function(frm) {
-		frm.save();
-	},
-
-	show_only_errors: function(frm) {
-		frm.events.create_log_table(frm);
-	},
-
-	create_log_table: function(frm) {
-		let msg = JSON.parse(frm.doc.log_details);
-		var $log_wrapper = $(frm.fields_dict.import_log.wrapper).empty();
-		$(frappe.render_template("log_details", {
-			data: msg.messages,
-			import_status: frm.doc.import_status,
-			show_only_errors: frm.doc.show_only_errors,
-		})).appendTo($log_wrapper);
-	}
-});
-
-frappe.provide('frappe.data_import');
-frappe.data_import.download_dialog = function(frm) {
-	var dialog;
-	const filter_fields = df => frappe.model.is_value_type(df) && !df.hidden;
-	const get_fields = dt => frappe.meta.get_docfields(dt).filter(filter_fields);
-
-	const get_doctype_checkbox_fields = () => {
-		return dialog.fields.filter(df => df.fieldname.endsWith('_fields'))
-			.map(df => dialog.fields_dict[df.fieldname]);
-	};
-
-	const doctype_fields = get_fields(frm.doc.reference_doctype)
-		.map(df => {
-			let reqd = (df.reqd || df.fieldname == 'naming_series') ? 1 : 0;
-			return {
-				label: df.label,
-				reqd: reqd,
-				danger: reqd,
-				value: df.fieldname,
-				checked: 1
-			};
-		});
-
-	let fields = [
-		{
-			"label": __("Select Columns"),
-			"fieldname": "select_columns",
-			"fieldtype": "Select",
-			"options": "All\nMandatory\nManually",
-			"reqd": 1,
-			"onchange": function() {
-				const fields = get_doctype_checkbox_fields();
-				fields.map(f => f.toggle(true));
-				if(this.value == 'Mandatory' || this.value == 'Manually') {
-					checkbox_toggle(true);
-					fields.map(multicheck_field => {
-						multicheck_field.options.map(option => {
-							if(!option.reqd) return;
-							$(multicheck_field.$wrapper).find(`:checkbox[data-unit="${option.value}"]`)
-								.prop('checked', false)
-								.trigger('click');
-						});
-					});
-				} else if(this.value == 'All'){
-					$(dialog.body).find(`[data-fieldtype="MultiCheck"] :checkbox`)
-						.prop('disabled', true);
-				}
-			}
-		},
-		{
-			"label": __("File Type"),
-			"fieldname": "file_type",
-			"fieldtype": "Select",
-			"options": "Excel\nCSV",
-			"default": "Excel"
-		},
-		{
-			"label": __("Download with Data"),
-			"fieldname": "with_data",
-			"fieldtype": "Check",
-			"hidden": !frm.doc.overwrite,
-			"default": 1
-		},
-		{
-			"label": __("Select All"),
-			"fieldname": "select_all",
-			"fieldtype": "Button",
-			"depends_on": "eval:doc.select_columns=='Manually'",
-			click: function() {
-				checkbox_toggle();
-			}
-		},
-		{
-			"label": __("Unselect All"),
-			"fieldname": "unselect_all",
-			"fieldtype": "Button",
-			"depends_on": "eval:doc.select_columns=='Manually'",
-			click: function() {
-				checkbox_toggle(true);
-			}
-		},
-		{
-			"label": frm.doc.reference_doctype,
-			"fieldname": "doctype_fields",
-			"fieldtype": "MultiCheck",
-			"options": doctype_fields,
-			"columns": 2,
-			"hidden": 1
-		}
-	];
-
-	const child_table_fields = frappe.meta.get_table_fields(frm.doc.reference_doctype)
-		.map(df => {
-			return {
-				"label": df.options,
-				"fieldname": df.fieldname + '_fields',
-				"fieldtype": "MultiCheck",
-				"options": frappe.meta.get_docfields(df.options)
-					.filter(filter_fields)
-					.map(df => ({
-						label: df.label,
-						reqd: df.reqd ? 1 : 0,
-						value: df.fieldname,
-						checked: 1,
-						danger: df.reqd
-					})),
-				"columns": 2,
-				"hidden": 1
-			};
-		});
-
-	fields = fields.concat(child_table_fields);
-
-	dialog = new frappe.ui.Dialog({
-		title: __('Download Template'),
-		fields: fields,
-		primary_action: function(values) {
-			var data = values;
-			if (frm.doc.reference_doctype) {
-				var export_params = () => {
-					let columns = {};
-					if(values.select_columns) {
-						columns = get_doctype_checkbox_fields().reduce((columns, field) => {
-							const options = field.get_checked_options();
-							columns[field.df.label] = options;
-							return columns;
-						}, {});
-					}
-
-					return {
-						doctype: frm.doc.reference_doctype,
-						parent_doctype: frm.doc.reference_doctype,
-						select_columns: JSON.stringify(columns),
-						with_data: frm.doc.overwrite && data.with_data,
-						all_doctypes: true,
-						file_type: data.file_type,
-						template: true
-					};
-				};
-				let get_template_url = '/api/method/frappe.core.doctype.data_export.exporter.export_data';
-				open_url_post(get_template_url, export_params());
-			} else {
-				frappe.msgprint(__("Please select the Document Type."));
-			}
-			dialog.hide();
-		},
-		primary_action_label: __('Download')
-	});
-
-	$(dialog.body).find('div[data-fieldname="select_all"], div[data-fieldname="unselect_all"]')
-		.wrapAll('<div class="inline-buttons" />');
-	const button_container = $(dialog.body).find('.inline-buttons');
-	button_container.addClass('flex');
-	$(button_container).find('.frappe-control').map((index, button) => {
-		$(button).css({"margin-right": "1em"});
-	});
-
-	function checkbox_toggle(checked=false) {
-		$(dialog.body).find('[data-fieldtype="MultiCheck"]').map((index, element) => {
-			$(element).find(`:checkbox`).prop("checked", checked).trigger('click');
-		});
-	}
-
-	return dialog;
-};

frappe/core/doctype/data_import_legacy/data_import_legacy.json

@@ -1,218 +0,0 @@
-{
- "actions": [],
- "allow_copy": 1,
- "creation": "2020-06-11 16:13:23.813709",
- "doctype": "DocType",
- "document_type": "Document",
- "editable_grid": 1,
- "engine": "InnoDB",
- "field_order": [
-  "reference_doctype",
-  "action",
-  "insert_new",
-  "overwrite",
-  "only_update",
-  "section_break_4",
-  "import_file",
-  "column_break_4",
-  "error_file",
-  "section_break_6",
-  "skip_errors",
-  "submit_after_import",
-  "ignore_encoding_errors",
-  "no_email",
-  "import_detail",
-  "import_status",
-  "show_only_errors",
-  "import_log",
-  "log_details",
-  "amended_from",
-  "total_rows",
-  "amended_from"
- ],
- "fields": [
-  {
-   "fieldname": "reference_doctype",
-   "fieldtype": "Link",
-   "ignore_user_permissions": 1,
-   "in_list_view": 1,
-   "label": "Document Type",
-   "options": "DocType",
-   "reqd": 1
-  },
-  {
-   "fieldname": "action",
-   "fieldtype": "Select",
-   "label": "Action",
-   "options": "Insert new records\nUpdate records",
-   "reqd": 1
-  },
-  {
-   "default": "0",
-   "depends_on": "eval:!doc.overwrite",
-   "description": "New data will be inserted.",
-   "fieldname": "insert_new",
-   "fieldtype": "Check",
-   "hidden": 1,
-   "label": "Insert new records",
-   "set_only_once": 1
-  },
-  {
-   "default": "0",
-   "depends_on": "eval:!doc.insert_new",
-   "description": "If you are updating/overwriting already created records.",
-   "fieldname": "overwrite",
-   "fieldtype": "Check",
-   "hidden": 1,
-   "label": "Update records",
-   "set_only_once": 1
-  },
-  {
-   "default": "0",
-   "depends_on": "overwrite",
-   "description": "If you don't want to create any new records while updating the older records.",
-   "fieldname": "only_update",
-   "fieldtype": "Check",
-   "label": "Don't create new records"
-  },
-  {
-   "depends_on": "eval:(!doc.__islocal)",
-   "fieldname": "section_break_4",
-   "fieldtype": "Section Break"
-  },
-  {
-   "fieldname": "import_file",
-   "fieldtype": "Attach",
-   "label": "Attach file for Import"
-  },
-  {
-   "fieldname": "column_break_4",
-   "fieldtype": "Column Break"
-  },
-  {
-   "depends_on": "eval: doc.import_status == \"Partially Successful\"",
-   "description": "This is the template file generated with only the rows having some error. You should use this file for correction and import.",
-   "fieldname": "error_file",
-   "fieldtype": "Attach",
-   "label": "Generated File"
-  },
-  {
-   "depends_on": "eval:(!doc.__islocal)",
-   "fieldname": "section_break_6",
-   "fieldtype": "Section Break"
-  },
-  {
-   "default": "0",
-   "description": "If this is checked, rows with valid data will be imported and invalid rows will be dumped into a new file for you to import later.",
-   "fieldname": "skip_errors",
-   "fieldtype": "Check",
-   "label": "Skip rows with errors"
-  },
-  {
-   "default": "0",
-   "fieldname": "submit_after_import",
-   "fieldtype": "Check",
-   "label": "Submit after importing"
-  },
-  {
-   "default": "0",
-   "fieldname": "ignore_encoding_errors",
-   "fieldtype": "Check",
-   "label": "Ignore encoding errors"
-  },
-  {
-   "default": "1",
-   "fieldname": "no_email",
-   "fieldtype": "Check",
-   "label": "Do not send Emails"
-  },
-  {
-   "collapsible": 1,
-   "collapsible_depends_on": "eval: doc.import_status == \"Failed\"",
-   "depends_on": "import_status",
-   "fieldname": "import_detail",
-   "fieldtype": "Section Break",
-   "label": "Import Log"
-  },
-  {
-   "fieldname": "import_status",
-   "fieldtype": "Select",
-   "label": "Import Status",
-   "options": "\nSuccessful\nFailed\nIn Progress\nPartially Successful",
-   "read_only": 1
-  },
-  {
-   "allow_on_submit": 1,
-   "default": "1",
-   "fieldname": "show_only_errors",
-   "fieldtype": "Check",
-   "label": "Show only errors",
-   "no_copy": 1,
-   "print_hide": 1
-  },
-  {
-   "allow_on_submit": 1,
-   "depends_on": "import_status",
-   "fieldname": "import_log",
-   "fieldtype": "HTML",
-   "label": "Import Log"
-  },
-  {
-   "allow_on_submit": 1,
-   "fieldname": "log_details",
-   "fieldtype": "Code",
-   "hidden": 1,
-   "label": "Log Details",
-   "read_only": 1
-  },
-  {
-   "fieldname": "amended_from",
-   "fieldtype": "Link",
-   "label": "Amended From",
-   "no_copy": 1,
-   "options": "Data Import",
-   "print_hide": 1,
-   "read_only": 1
-  },
-  {
-   "fieldname": "total_rows",
-   "fieldtype": "Int",
-   "hidden": 1,
-   "label": "Total Rows",
-   "read_only": 1
-  },
-  {
-   "fieldname": "amended_from",
-   "fieldtype": "Link",
-   "label": "Amended From",
-   "no_copy": 1,
-   "options": "Data Import Legacy",
-   "print_hide": 1,
-   "read_only": 1
-  }
- ],
- "is_submittable": 1,
- "links": [],
- "max_attachments": 1,
- "modified": "2020-06-11 16:13:23.813709",
- "modified_by": "Administrator",
- "module": "Core",
- "name": "Data Import Legacy",
- "owner": "Administrator",
- "permissions": [
-  {
-   "create": 1,
-   "delete": 1,
-   "email": 1,
-   "read": 1,
-   "role": "System Manager",
-   "share": 1,
-   "submit": 1,
-   "write": 1
-  }
- ],
- "sort_field": "modified",
- "sort_order": "DESC",
- "track_changes": 1,
- "track_seen": 1
-}

frappe/core/doctype/data_import_legacy/data_import_legacy.py

@@ -1,126 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright (c) 2017, Frappe Technologies and contributors
-# For license information, please see license.txt
-
-import os
-
-import frappe
-import frappe.modules.import_file
-from frappe import _
-from frappe.core.doctype.data_import_legacy.importer import upload
-from frappe.model.document import Document
-from frappe.modules.import_file import import_file_by_path as _import_file_by_path
-from frappe.utils.background_jobs import enqueue
-from frappe.utils.data import format_datetime
-
-
-class DataImportLegacy(Document):
-	def autoname(self):
-		if not self.name:
-			self.name = "Import on " + format_datetime(self.creation)
-
-	def validate(self):
-		if not self.import_file:
-			self.db_set("total_rows", 0)
-		if self.import_status == "In Progress":
-			frappe.throw(_("Can't save the form as data import is in progress."))
-
-		# validate the template just after the upload
-		# if there is total_rows in the doc, it means that the template is already validated and error free
-		if self.import_file and not self.total_rows:
-			upload(data_import_doc=self, from_data_import="Yes", validate_template=True)
-
-
-@frappe.whitelist()
-def get_importable_doctypes():
-	return frappe.cache().hget("can_import", frappe.session.user)
-
-
-@frappe.whitelist()
-def import_data(data_import):
-	frappe.db.set_value("Data Import Legacy", data_import, "import_status", "In Progress", update_modified=False)
-	frappe.publish_realtime("data_import_progress", {"progress": "0",
-		"data_import": data_import, "reload": True}, user=frappe.session.user)
-
-	from frappe.core.page.background_jobs.background_jobs import get_info
-	enqueued_jobs = [d.get("job_name") for d in get_info()]
-
-	if data_import not in enqueued_jobs:
-		enqueue(upload, queue='default', timeout=6000, event='data_import', job_name=data_import,
-			data_import_doc=data_import, from_data_import="Yes", user=frappe.session.user)
-
-
-def import_doc(path, overwrite=False, ignore_links=False, ignore_insert=False,
-	insert=False, submit=False, pre_process=None):
-	if os.path.isdir(path):
-		files = [os.path.join(path, f) for f in os.listdir(path)]
-	else:
-		files = [path]
-
-	for f in files:
-		if f.endswith(".json"):
-			frappe.flags.mute_emails = True
-			_import_file_by_path(f, data_import=True, force=True, pre_process=pre_process, reset_permissions=True)
-			frappe.flags.mute_emails = False
-			frappe.db.commit()
-		elif f.endswith(".csv"):
-			import_file_by_path(f, ignore_links=ignore_links, overwrite=overwrite, submit=submit, pre_process=pre_process)
-			frappe.db.commit()
-
-
-def import_file_by_path(path, ignore_links=False, overwrite=False, submit=False, pre_process=None, no_email=True):
-	from frappe.utils.csvutils import read_csv_content
-	print("Importing " + path)
-	with open(path, "r") as infile:
-		upload(rows=read_csv_content(infile.read()), ignore_links=ignore_links, no_email=no_email, overwrite=overwrite,
-			submit_after_import=submit, pre_process=pre_process)
-
-
-def export_json(doctype, path, filters=None, or_filters=None, name=None, order_by="creation asc"):
-	def post_process(out):
-		del_keys = ('modified_by', 'creation', 'owner', 'idx')
-		for doc in out:
-			for key in del_keys:
-				if key in doc:
-					del doc[key]
-			for k, v in doc.items():
-				if isinstance(v, list):
-					for child in v:
-						for key in del_keys + ('docstatus', 'doctype', 'modified', 'name'):
-							if key in child:
-								del child[key]
-
-	out = []
-	if name:
-		out.append(frappe.get_doc(doctype, name).as_dict())
-	elif frappe.db.get_value("DocType", doctype, "issingle"):
-		out.append(frappe.get_doc(doctype).as_dict())
-	else:
-		for doc in frappe.get_all(doctype, fields=["name"], filters=filters, or_filters=or_filters, limit_page_length=0, order_by=order_by):
-			out.append(frappe.get_doc(doctype, doc.name).as_dict())
-	post_process(out)
-
-	dirname = os.path.dirname(path)
-	if not os.path.exists(dirname):
-		path = os.path.join('..', path)
-
-	with open(path, "w") as outfile:
-		outfile.write(frappe.as_json(out))
-
-
-def export_csv(doctype, path):
-	from frappe.core.doctype.data_export.exporter import export_data
-	with open(path, "wb") as csvfile:
-		export_data(doctype=doctype, all_doctypes=True, template=True, with_data=True)
-		csvfile.write(frappe.response.result.encode("utf-8"))
-
-
-@frappe.whitelist()
-def export_fixture(doctype, app):
-	if frappe.session.user != "Administrator":
-		raise frappe.PermissionError
-
-	if not os.path.exists(frappe.get_app_path(app, "fixtures")):
-		os.mkdir(frappe.get_app_path(app, "fixtures"))
-
-	export_json(doctype, frappe.get_app_path(app, "fixtures", frappe.scrub(doctype) + ".json"), order_by="name asc")

frappe/core/doctype/data_import_legacy/data_import_legacy_list.js

@@ -1,24 +0,0 @@
-frappe.listview_settings['Data Import Legacy'] = {
-	add_fields: ["import_status"],
-	has_indicator_for_draft: 1,
-	get_indicator: function(doc) {
-
-		let status = {
-			'Successful': [__("Success"), "green", "import_status,=,Successful"],
-			'Partially Successful': [__("Partial Success"), "blue", "import_status,=,Partially Successful"],
-			'In Progress': [__("In Progress"), "orange", "import_status,=,In Progress"],
-			'Failed': [__("Failed"), "red", "import_status,=,Failed"],
-			'Pending': [__("Pending"), "orange", "import_status,=,"]
-		}
-
-		if (doc.import_status) {
-			return status[doc.import_status];
-		}
-
-		if (doc.docstatus == 0) {
-			return status['Pending'];
-		}
-
-		return status['Pending'];
-	}
-};

frappe/core/doctype/data_import_legacy/importer.py

@@ -1,538 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
-
-import requests
-import frappe, json
-import frappe.permissions
-
-from frappe import _
-
-from frappe.utils.csvutils import getlink
-from frappe.utils.dateutils import parse_date
-
-from frappe.utils import cint, cstr, flt, getdate, get_datetime, get_url, get_absolute_url, duration_to_seconds
-
-
-@frappe.whitelist()
-def get_data_keys():
-	return frappe._dict({
-		"data_separator": _('Start entering data below this line'),
-		"main_table": _("Table") + ":",
-		"parent_table": _("Parent Table") + ":",
-		"columns": _("Column Name") + ":",
-		"doctype": _("DocType") + ":"
-	})
-
-
-
-@frappe.whitelist()
-def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False, no_email=True, overwrite=None,
-	update_only = None, ignore_links=False, pre_process=None, via_console=False, from_data_import="No",
-	skip_errors = True, data_import_doc=None, validate_template=False, user=None):
-	"""upload data"""
-
-	# for translations
-	if user:
-		frappe.cache().hdel("lang", user)
-		frappe.set_user_lang(user)
-
-	if data_import_doc and isinstance(data_import_doc, str):
-		data_import_doc = frappe.get_doc("Data Import Legacy", data_import_doc)
-	if data_import_doc and from_data_import == "Yes":
-		no_email = data_import_doc.no_email
-		ignore_encoding_errors = data_import_doc.ignore_encoding_errors
-		update_only = data_import_doc.only_update
-		submit_after_import = data_import_doc.submit_after_import
-		overwrite = data_import_doc.overwrite
-		skip_errors = data_import_doc.skip_errors
-	else:
-		# extra input params
-		params = json.loads(frappe.form_dict.get("params") or '{}')
-		if params.get("submit_after_import"):
-			submit_after_import = True
-		if params.get("ignore_encoding_errors"):
-			ignore_encoding_errors = True
-		if not params.get("no_email"):
-			no_email = False
-		if params.get('update_only'):
-			update_only = True
-		if params.get('from_data_import'):
-			from_data_import = params.get('from_data_import')
-		if not params.get('skip_errors'):
-			skip_errors = params.get('skip_errors')
-
-	frappe.flags.in_import = True
-	frappe.flags.mute_emails = no_email
-
-	def get_data_keys_definition():
-		return get_data_keys()
-
-	def bad_template():
-		frappe.throw(_("Please do not change the rows above {0}").format(get_data_keys_definition().data_separator))
-
-	def check_data_length():
-		if not data:
-			frappe.throw(_("No data found in the file. Please reattach the new file with data."))
-
-	def get_start_row():
-		for i, row in enumerate(rows):
-			if row and row[0]==get_data_keys_definition().data_separator:
-				return i+1
-		bad_template()
-
-	def get_header_row(key):
-		return get_header_row_and_idx(key)[0]
-
-	def get_header_row_and_idx(key):
-		for i, row in enumerate(header):
-			if row and row[0]==key:
-				return row, i
-		return [], -1
-
-	def filter_empty_columns(columns):
-		empty_cols = list(filter(lambda x: x in ("", None), columns))
-
-		if empty_cols:
-			if columns[-1*len(empty_cols):] == empty_cols:
-				# filter empty columns if they exist at the end
-				columns = columns[:-1*len(empty_cols)]
-			else:
-				frappe.msgprint(_("Please make sure that there are no empty columns in the file."),
-					raise_exception=1)
-
-		return columns
-
-	def make_column_map():
-		doctype_row, row_idx = get_header_row_and_idx(get_data_keys_definition().doctype)
-		if row_idx == -1: # old style
-			return
-
-		dt = None
-		for i, d in enumerate(doctype_row[1:]):
-			if d not in ("~", "-"):
-				if d and doctype_row[i] in (None, '' ,'~', '-', _("DocType") + ":"):
-					dt, parentfield = d, None
-					# xls format truncates the row, so it may not have more columns
-					if len(doctype_row) > i+2:
-						parentfield = doctype_row[i+2]
-					doctypes.append((dt, parentfield))
-					column_idx_to_fieldname[(dt, parentfield)] = {}
-					column_idx_to_fieldtype[(dt, parentfield)] = {}
-				if dt:
-					column_idx_to_fieldname[(dt, parentfield)][i+1] = rows[row_idx + 2][i+1]
-					column_idx_to_fieldtype[(dt, parentfield)][i+1] = rows[row_idx + 4][i+1]
-
-	def get_doc(start_idx):
-		if doctypes:
-			doc = {}
-			attachments = []
-			last_error_row_idx = None
-			for idx in range(start_idx, len(rows)):
-				last_error_row_idx = idx	# pylint: disable=W0612
-				if (not doc) or main_doc_empty(rows[idx]):
-					for dt, parentfield in doctypes:
-						d = {}
-						for column_idx in column_idx_to_fieldname[(dt, parentfield)]:
-							try:
-								fieldname = column_idx_to_fieldname[(dt, parentfield)][column_idx]
-								fieldtype = column_idx_to_fieldtype[(dt, parentfield)][column_idx]
-
-								if not fieldname or not rows[idx][column_idx]:
-									continue
-
-								d[fieldname] = rows[idx][column_idx]
-								if fieldtype in ("Int", "Check"):
-									d[fieldname] = cint(d[fieldname])
-								elif fieldtype in ("Float", "Currency", "Percent"):
-									d[fieldname] = flt(d[fieldname])
-								elif fieldtype == "Date":
-									if d[fieldname] and isinstance(d[fieldname], str):
-										d[fieldname] = getdate(parse_date(d[fieldname]))
-								elif fieldtype == "Datetime":
-									if d[fieldname]:
-										if " " in d[fieldname]:
-											_date, _time = d[fieldname].split()
-										else:
-											_date, _time = d[fieldname], '00:00:00'
-										_date = parse_date(d[fieldname])
-										d[fieldname] = get_datetime(_date + " " + _time)
-									else:
-										d[fieldname] = None
-								elif fieldtype == "Duration":
-									d[fieldname] = duration_to_seconds(cstr(d[fieldname]))
-								elif fieldtype in ("Image", "Attach Image", "Attach"):
-									# added file to attachments list
-									attachments.append(d[fieldname])
-
-								elif fieldtype in ("Link", "Dynamic Link", "Data") and d[fieldname]:
-									# as fields can be saved in the number format(long type) in data import template
-									d[fieldname] = cstr(d[fieldname])
-
-							except IndexError:
-								pass
-
-						# scrub quotes from name and modified
-						if d.get("name") and d["name"].startswith('"'):
-							d["name"] = d["name"][1:-1]
-
-						if sum(0 if not val else 1 for val in d.values()):
-							d['doctype'] = dt
-							if dt == doctype:
-								doc.update(d)
-							else:
-								if not overwrite and doc.get("name"):
-									d['parent'] = doc["name"]
-								d['parenttype'] = doctype
-								d['parentfield'] = parentfield
-								doc.setdefault(d['parentfield'], []).append(d)
-				else:
-					break
-
-			return doc, attachments, last_error_row_idx
-		else:
-			doc = frappe._dict(zip(columns, rows[start_idx][1:]))
-			doc['doctype'] = doctype
-			return doc, [], None
-
-	# used in testing whether a row is empty or parent row or child row
-	# checked only 3 first columns since first two columns can be blank for example the case of
-	# importing the item variant where item code and item name will be blank.
-	def main_doc_empty(row):
-		if row:
-			for i in range(3,0,-1):
-				if len(row) > i and row[i]:
-					return False
-		return True
-
-	def validate_naming(doc):
-		autoname = frappe.get_meta(doctype).autoname
-		if autoname:
-			if autoname[0:5] == 'field':
-				autoname = autoname[6:]
-			elif autoname == 'naming_series:':
-				autoname = 'naming_series'
-			else:
-				return True
-
-			if (autoname not in doc) or (not doc[autoname]):
-				from frappe.model.base_document import get_controller
-				if not hasattr(get_controller(doctype), "autoname"):
-					frappe.throw(_("{0} is a mandatory field").format(autoname))
-		return True
-
-	users = frappe.db.sql_list("select name from tabUser")
-	def prepare_for_insert(doc):
-		# don't block data import if user is not set
-		# migrating from another system
-		if not doc.owner in users:
-			doc.owner = frappe.session.user
-		if not doc.modified_by in users:
-			doc.modified_by = frappe.session.user
-
-	def is_valid_url(url):
-		is_valid = False
-		if url.startswith("/files") or url.startswith("/private/files"):
-			url = get_url(url)
-
-		try:
-			r = requests.get(url)
-			is_valid = True if r.status_code == 200 else False
-		except Exception:
-			pass
-
-		return is_valid
-
-	def attach_file_to_doc(doctype, docname, file_url):
-		# check if attachment is already available
-		# check if the attachement link is relative or not
-		if not file_url:
-			return
-		if not is_valid_url(file_url):
-			return
-
-		files = frappe.db.sql("""Select name from `tabFile` where attached_to_doctype='{doctype}' and
-			attached_to_name='{docname}' and (file_url='{file_url}' or thumbnail_url='{file_url}')""".format(
-				doctype=doctype,
-				docname=docname,
-				file_url=file_url
-			))
-
-		if files:
-			# file is already attached
-			return
-
-		_file = frappe.get_doc({
-			"doctype": "File",
-			"file_url": file_url,
-			"attached_to_name": docname,
-			"attached_to_doctype": doctype,
-			"attached_to_field": 0,
-			"folder": "Home/Attachments"})
-		_file.save()
-
-
-	# header
-	filename, file_extension = ['','']
-	if not rows:
-		_file = frappe.get_doc("File", {"file_url": data_import_doc.import_file})
-		fcontent = _file.get_content()
-		filename, file_extension = _file.get_extension()
-
-		if file_extension == '.xlsx' and from_data_import == 'Yes':
-			from frappe.utils.xlsxutils import read_xlsx_file_from_attached_file
-			rows = read_xlsx_file_from_attached_file(file_url=data_import_doc.import_file)
-
-		elif file_extension == '.csv':
-			from frappe.utils.csvutils import read_csv_content
-			rows = read_csv_content(fcontent, ignore_encoding_errors)
-
-		else:
-			frappe.throw(_("Unsupported File Format"))
-
-	start_row = get_start_row()
-	header = rows[:start_row]
-	data = rows[start_row:]
-	try:
-		doctype = get_header_row(get_data_keys_definition().main_table)[1]
-		columns = filter_empty_columns(get_header_row(get_data_keys_definition().columns)[1:])
-	except:
-		frappe.throw(_("Cannot change header content"))
-	doctypes = []
-	column_idx_to_fieldname = {}
-	column_idx_to_fieldtype = {}
-
-	if skip_errors:
-		data_rows_with_error = header
-
-	if submit_after_import and not cint(frappe.db.get_value("DocType",
-			doctype, "is_submittable")):
-		submit_after_import = False
-
-	parenttype = get_header_row(get_data_keys_definition().parent_table)
-
-	if len(parenttype) > 1:
-		parenttype = parenttype[1]
-
-	# check permissions
-	if not frappe.permissions.can_import(parenttype or doctype):
-		frappe.flags.mute_emails = False
-		return {"messages": [_("Not allowed to Import") + ": " + _(doctype)], "error": True}
-
-	# Throw expception in case of the empty data file
-	check_data_length()
-	make_column_map()
-	total = len(data)
-
-	if validate_template:
-		if total:
-			data_import_doc.total_rows = total
-		return True
-
-	if overwrite==None:
-		overwrite = params.get('overwrite')
-
-	# delete child rows (if parenttype)
-	parentfield = None
-	if parenttype:
-		parentfield = get_parent_field(doctype, parenttype)
-
-		if overwrite:
-			delete_child_rows(data, doctype)
-
-	import_log = []
-	def log(**kwargs):
-		if via_console:
-			print((kwargs.get("title") + kwargs.get("message")).encode('utf-8'))
-		else:
-			import_log.append(kwargs)
-
-	def as_link(doctype, name):
-		if via_console:
-			return "{0}: {1}".format(doctype, name)
-		else:
-			return getlink(doctype, name)
-
-	# publish realtime task update
-	def publish_progress(achieved, reload=False):
-		if data_import_doc:
-			frappe.publish_realtime("data_import_progress", {"progress": str(int(100.0*achieved/total)),
-				"data_import": data_import_doc.name, "reload": reload}, user=frappe.session.user)
-
-
-	error_flag = rollback_flag = False
-
-	batch_size = frappe.conf.data_import_batch_size or 1000
-
-	for batch_start in range(0, total, batch_size):
-		batch = data[batch_start:batch_start + batch_size]
-
-		for i, row in enumerate(batch):
-			# bypass empty rows
-			if main_doc_empty(row):
-				continue
-
-			row_idx = i + start_row
-			doc = None
-
-			publish_progress(i)
-
-			try:
-				doc, attachments, last_error_row_idx = get_doc(row_idx)
-				validate_naming(doc)
-				if pre_process:
-					pre_process(doc)
-
-				original = None
-				if parentfield:
-					parent = frappe.get_doc(parenttype, doc["parent"])
-					doc = parent.append(parentfield, doc)
-					parent.save()
-				else:
-					if overwrite and doc.get("name") and frappe.db.exists(doctype, doc["name"]):
-						original = frappe.get_doc(doctype, doc["name"])
-						original_name = original.name
-						original.update(doc)
-						# preserve original name for case sensitivity
-						original.name = original_name
-						original.flags.ignore_links = ignore_links
-						original.save()
-						doc = original
-					else:
-						if not update_only:
-							doc = frappe.get_doc(doc)
-							prepare_for_insert(doc)
-							doc.flags.ignore_links = ignore_links
-							doc.insert()
-					if attachments:
-						# check file url and create a File document
-						for file_url in attachments:
-							attach_file_to_doc(doc.doctype, doc.name, file_url)
-					if submit_after_import:
-						doc.submit()
-
-				# log errors
-				if parentfield:
-					log(**{"row": doc.idx, "title": 'Inserted row for "%s"' % (as_link(parenttype, doc.parent)),
-						"link": get_absolute_url(parenttype, doc.parent), "message": 'Document successfully saved', "indicator": "green"})
-				elif submit_after_import:
-					log(**{"row": row_idx + 1, "title":'Submitted row for "%s"' % (as_link(doc.doctype, doc.name)),
-						"message": "Document successfully submitted", "link": get_absolute_url(doc.doctype, doc.name), "indicator": "blue"})
-				elif original:
-					log(**{"row": row_idx + 1,"title":'Updated row for "%s"' % (as_link(doc.doctype, doc.name)),
-						"message": "Document successfully updated", "link": get_absolute_url(doc.doctype, doc.name), "indicator": "green"})
-				elif not update_only:
-					log(**{"row": row_idx + 1, "title":'Inserted row for "%s"' % (as_link(doc.doctype, doc.name)),
-						"message": "Document successfully saved", "link": get_absolute_url(doc.doctype, doc.name), "indicator": "green"})
-				else:
-					log(**{"row": row_idx + 1, "title":'Ignored row for %s' % (row[1]), "link": None,
-						"message": "Document updation ignored", "indicator": "orange"})
-
-			except Exception as e:
-				error_flag = True
-
-				# build error message
-				if frappe.local.message_log:
-					err_msg = "\n".join(['<p class="border-bottom small">{}</p>'.format(json.loads(msg).get('message')) for msg in frappe.local.message_log])
-				else:
-					err_msg = '<p class="border-bottom small">{}</p>'.format(cstr(e))
-
-				error_trace = frappe.get_traceback()
-				if error_trace:
-					error_log_doc = frappe.log_error(error_trace)
-					error_link = get_absolute_url("Error Log", error_log_doc.name)
-				else:
-					error_link = None
-
-				log(**{
-					"row": row_idx + 1,
-					"title": 'Error for row %s' % (len(row)>1 and frappe.safe_decode(row[1]) or ""),
-					"message": err_msg,
-					"indicator": "red",
-					"link":error_link
-				})
-
-				# data with error to create a new file
-				# include the errored data in the last row as last_error_row_idx will not be updated for the last row
-				if skip_errors:
-					if last_error_row_idx == len(rows)-1:
-						last_error_row_idx = len(rows)
-					data_rows_with_error += rows[row_idx:last_error_row_idx]
-				else:
-					rollback_flag = True
-			finally:
-				frappe.local.message_log = []
-
-		start_row += batch_size
-		if rollback_flag:
-			frappe.db.rollback()
-		else:
-			frappe.db.commit()
-
-	frappe.flags.mute_emails = False
-	frappe.flags.in_import = False
-
-	log_message = {"messages": import_log, "error": error_flag}
-	if data_import_doc:
-		data_import_doc.log_details = json.dumps(log_message)
-
-		import_status = None
-		if error_flag and data_import_doc.skip_errors and len(data) != len(data_rows_with_error):
-			import_status = "Partially Successful"
-			# write the file with the faulty row
-			file_name = 'error_' + filename + file_extension
-			if file_extension == '.xlsx':
-				from frappe.utils.xlsxutils import make_xlsx
-				xlsx_file = make_xlsx(data_rows_with_error, "Data Import Template")
-				file_data = xlsx_file.getvalue()
-			else:
-				from frappe.utils.csvutils import to_csv
-				file_data = to_csv(data_rows_with_error)
-			_file = frappe.get_doc({
-				"doctype": "File",
-				"file_name": file_name,
-				"attached_to_doctype": "Data Import Legacy",
-				"attached_to_name": data_import_doc.name,
-				"folder": "Home/Attachments",
-				"content": file_data})
-			_file.save()
-			data_import_doc.error_file = _file.file_url
-
-		elif error_flag:
-			import_status = "Failed"
-		else:
-			import_status = "Successful"
-
-		data_import_doc.import_status = import_status
-		data_import_doc.save()
-		if data_import_doc.import_status in ["Successful", "Partially Successful"]:
-			data_import_doc.submit()
-			publish_progress(100, True)
-		else:
-			publish_progress(0, True)
-		frappe.db.commit()
-	else:
-		return log_message
-
-def get_parent_field(doctype, parenttype):
-	parentfield = None
-
-	# get parentfield
-	if parenttype:
-		for d in frappe.get_meta(parenttype).get_table_fields():
-			if d.options==doctype:
-				parentfield = d.fieldname
-				break
-
-		if not parentfield:
-			frappe.msgprint(_("Did not find {0} for {0} ({1})").format("parentfield", parenttype, doctype))
-			raise Exception
-
-	return parentfield
-
-def delete_child_rows(rows, doctype):
-	"""delete child rows for all parents"""
-	for p in list(set(r[1] for r in rows)):
-		if p:
-			frappe.db.sql("""delete from `tab{0}` where parent=%s""".format(doctype), p)

frappe/core/doctype/data_import_legacy/log_details.html

@@ -1,38 +0,0 @@
-<div>
-	<div class="table-responsive">
-		<table class="table table-bordered table-hover log-details-table">
-				<tr>
-					<th style="width:10%"> {{ __("Row No") }} </th>
-					<th style="width:40%"> {{ __("Row Status") }} </th>
-					<th style="width:50%"> {{ __("Message") }} </th>
-				</tr>
-
-			{% for row in data %}
-				{% if (!show_only_errors) || (show_only_errors && row.indicator == "red") %}
-				<tr>
-					<td>
-						<span>{{ row.row }} </span>
-					</td>
-					<td>
-						<span class="indicator {{ row.indicator }}"> {{ row.title }} </span>
-					</td>
-					<td>
-						{% if (import_status != "Failed" || (row.indicator == "red")) { %}
-							<div>{{ row.message }}</div>
-							{% if row.link %}
-								<span style="width: 10%; float:right;">
-									<a class="btn-open no-decoration" title="Open Link" href="{{ row.link }}">
-										<i class="octicon octicon-arrow-right"></i>
-									</a>
-								</span>
-							{% endif %}
-						{% } else { %}
-							<span> {{ __("Document can't saved.") }} </span>
-						{% } %}
-					</td>
-				</tr>
-				{% endif %}
-			{% endfor %}
-		</table>
-	</div>
-</div>

frappe/core/doctype/data_import_legacy/test_data_import_legacy.py

@@ -1,8 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright (c) 2020, Frappe Technologies and Contributors
-# See license.txt
-# import frappe
-import unittest
-
-class TestDataImportLegacy(unittest.TestCase):
-	pass

frappe/core/doctype/doctype/doctype.json

@@ -76,6 +76,7 @@
   "index_web_pages_for_search",
   "route",
   "is_published_field",
+  "website_search_field",
   "advanced",
   "engine"
  ],
@@ -547,6 +548,12 @@
   {
    "fieldname": "column_break_51",
    "fieldtype": "Column Break"
+  },
+  {
+   "depends_on": "has_web_view",
+   "fieldname": "website_search_field",
+   "fieldtype": "Data",
+   "label": "Website Search Field"
   }
  ],
  "icon": "fa fa-bolt",
@@ -628,7 +635,7 @@
    "link_fieldname": "reference_doctype"
   }
  ],
- "modified": "2021-04-16 12:26:41.031135",
+ "modified": "2021-06-17 23:31:44.974199",
  "modified_by": "Administrator",
  "module": "Core",
  "name": "DocType",
@@ -662,4 +669,4 @@
  "sort_field": "modified",
  "sort_order": "DESC",
  "track_changes": 1
-}
+}

frappe/core/doctype/doctype/doctype.py

@@ -396,10 +396,7 @@ class DocType(Document):
 			frappe.db.sql("""update tabSingles set value=%s
 				where doctype=%s and field='name' and value = %s""", (new, new, old))
 		else:
-			frappe.db.multisql({
-				"mariadb": f"RENAME TABLE `tab{old}` TO `tab{new}`",
-				"postgres": f"ALTER TABLE `tab{old}` RENAME TO `tab{new}`"
-			})
+			frappe.db.rename_table(old, new)
 			frappe.db.commit()
 
 		# Do not rename and move files and folders for custom doctype
@@ -927,6 +924,13 @@ def validate_fields(meta):
 		if meta.is_published_field not in fieldname_list:
 			frappe.throw(_("Is Published Field must be a valid fieldname"), InvalidFieldNameError)
 
+	def check_website_search_field(meta):
+		if not meta.website_search_field:
+			return
+
+		if meta.website_search_field not in fieldname_list:
+			frappe.throw(_("Website Search Field must be a valid fieldname"), InvalidFieldNameError)
+
 	def check_timeline_field(meta):
 		if not meta.timeline_field:
 			return
@@ -1046,6 +1050,7 @@ def validate_fields(meta):
 	check_title_field(meta)
 	check_timeline_field(meta)
 	check_is_published_field(meta)
+	check_website_search_field(meta)
 	check_sort_field(meta)
 	check_image_field(meta)
 

frappe/core/doctype/domain_settings/domain_settings.py

@@ -34,7 +34,7 @@ class DomainSettings(Document):
 		all_domains = list((frappe.get_hooks('domains') or {}))
 
 		def remove_role(role):
-			frappe.db.sql('delete from `tabHas Role` where role=%s', role)
+			frappe.db.delete("Has Role", {"role": role})
 			frappe.set_value('Role', role, 'disabled', 1)
 
 		for domain in all_domains:

frappe/core/doctype/error_log/error_log.py

@@ -20,4 +20,4 @@ def set_old_logs_as_seen():
 def clear_error_logs():
 	'''Flush all Error Logs'''
 	frappe.only_for('System Manager')
-	frappe.db.sql('''DELETE FROM `tabError Log`''')
+	frappe.db.truncate("Error Log")

frappe/core/doctype/report/test_report.py

@@ -82,9 +82,11 @@ class TestReport(unittest.TestCase):
 
 	def test_report_permissions(self):
 		frappe.set_user('test@example.com')
-		frappe.db.sql("""delete from `tabHas Role` where parent = %s
-			and role = 'Test Has Role'""", frappe.session.user, auto_commit=1)
-
+		frappe.db.delete("Has Role", {
+			"parent": frappe.session.user,
+			"role": "Test Has Role"
+		})
+		frappe.db.commit()
 		if not frappe.db.exists('Role', 'Test Has Role'):
 			role = frappe.get_doc({
 				'doctype': 'Role',

frappe/core/doctype/role/role.py

@@ -38,7 +38,7 @@ class Role(Document):
 				self.set(key, 0)
 
 	def remove_roles(self):
-		frappe.db.sql("delete from `tabHas Role` where role = %s", self.name)
+		frappe.db.delete("Has Role", {"role": self.name})
 		frappe.clear_cache()
 
 	def on_update(self):

frappe/core/doctype/scheduled_job_type/scheduled_job_type.py

@@ -1,5 +1,4 @@
-# -*- coding: utf-8 -*-
-# Copyright (c) 2019, Frappe Technologies and contributors
+# Copyright (c) 2021, Frappe Technologies and contributors
 # For license information, please see license.txt
 
 import json
@@ -110,7 +109,7 @@ class ScheduledJobType(Document):
 		return 'long' if ('Long' in self.frequency) else 'default'
 
 	def on_trash(self):
-		frappe.db.sql('delete from `tabScheduled Job Log` where scheduled_job_type=%s', self.name)
+		frappe.db.delete("Scheduled Job Log", {"scheduled_job_type": self.name})
 
 
 @frappe.whitelist()

frappe/core/doctype/user/user.py

@@ -53,6 +53,7 @@ class User(Document):
 	def after_insert(self):
 		create_notification_settings(self.name)
 		frappe.cache().delete_key('users_for_mentions')
+		frappe.cache().delete_key('enabled_users')
 
 	def validate(self):
 		self.check_demo()
@@ -129,6 +130,9 @@ class User(Document):
 		if self.has_value_changed('allow_in_mentions') or self.has_value_changed('user_type'):
 			frappe.cache().delete_key('users_for_mentions')
 
+		if self.has_value_changed('enabled'):
+			frappe.cache().delete_key('enabled_users')
+
 	def has_website_permission(self, ptype, user, verbose=False):
 		"""Returns true if current user is the session user"""
 		return self.name == frappe.session.user
@@ -364,17 +368,15 @@ class User(Document):
 			frappe.local.login_manager.logout(user=self.name)
 
 		# delete todos
-		frappe.db.sql("""DELETE FROM `tabToDo` WHERE `owner`=%s""", (self.name,))
+		frappe.db.delete("ToDo", {"owner": self.name})
 		frappe.db.sql("""UPDATE `tabToDo` SET `assigned_by`=NULL WHERE `assigned_by`=%s""",
 			(self.name,))
 
 		# delete events
-		frappe.db.sql("""delete from `tabEvent` where owner=%s
-			and event_type='Private'""", (self.name,))
+		frappe.db.delete("Event", {"owner": self.name, "event_type": "Private"})
 
 		# delete shares
-		frappe.db.sql("""delete from `tabDocShare` where user=%s""", self.name)
-
+		frappe.db.delete("DocShare", {"user": self.name})
 		# delete messages
 		frappe.db.sql("""delete from `tabCommunication`
 			where communication_type in ('Chat', 'Notification')
@@ -392,6 +394,8 @@ class User(Document):
 		if self.get('allow_in_mentions'):
 			frappe.cache().delete_key('users_for_mentions')
 
+		frappe.cache().delete_key('enabled_users')
+
 
 	def before_rename(self, old_name, new_name, merge=False):
 		self.check_demo()
@@ -1230,3 +1234,10 @@ def generate_keys(user):
 def switch_theme(theme):
 	if theme in ["Dark", "Light"]:
 		frappe.db.set_value("User", frappe.session.user, "desk_theme", theme)
+
+def get_enabled_users():
+	def _get_enabled_users():
+		enabled_users = frappe.get_all("User", filters={"enabled": "1"}, pluck="name")
+		return enabled_users
+
+	return frappe.cache().get_value("enabled_users", _get_enabled_users)

frappe/core/doctype/user_permission/test_user_permission.py

@@ -1,6 +1,5 @@
-# -*- coding: utf-8 -*-
-# Copyright (c) 2017, Frappe Technologies and Contributors
-# See license.txt
+# Copyright (c) 2021, Frappe Technologies and Contributors
+# See LICENSE
 from frappe.core.doctype.user_permission.user_permission import add_user_permissions, remove_applicable
 from frappe.permissions import has_user_permission
 from frappe.core.doctype.doctype.test_doctype import new_doctype
@@ -10,11 +9,14 @@ import unittest
 
 class TestUserPermission(unittest.TestCase):
 	def setUp(self):
-		frappe.db.sql("""DELETE FROM `tabUser Permission`
-			WHERE `user` in (
-				'test_bulk_creation_update@example.com',
-				'test_user_perm1@example.com',
-				'nested_doc_user@example.com')""")
+		test_users = (
+			"test_bulk_creation_update@example.com",
+			"test_user_perm1@example.com",
+			"nested_doc_user@example.com",
+		)
+		frappe.db.delete("User Permission", {
+			"user": ("in", test_users)
+		})
 		frappe.delete_doc_if_exists("DocType", "Person")
 		frappe.db.sql_ddl("DROP TABLE IF EXISTS `tabPerson`")
 		frappe.delete_doc_if_exists("DocType", "Doc A")

frappe/core/doctype/user_permission/user_permission.py

@@ -1,5 +1,4 @@
-# -*- coding: utf-8 -*-
-# Copyright (c) 2017, Frappe Technologies and contributors
+# Copyright (c) 2021, Frappe Technologies and contributors
 # For license information, please see license.txt
 
 import frappe, json
@@ -179,11 +178,16 @@ def check_applicable_doc_perm(user, doctype, docname):
 
 @frappe.whitelist()
 def clear_user_permissions(user, for_doctype):
-	frappe.only_for('System Manager')
-	total = frappe.db.count('User Permission', filters = dict(user=user, allow=for_doctype))
+	frappe.only_for("System Manager")
+	total = frappe.db.count("User Permission", {"user": user, "allow": for_doctype})
+
 	if total:
-		frappe.db.sql('DELETE FROM `tabUser Permission` WHERE `user`=%s AND `allow`=%s', (user, for_doctype))
+		frappe.db.delete("User Permission", {
+			"allow": for_doctype,
+			"user": user,
+		})
 		frappe.clear_cache()
+
 	return total
 
 @frappe.whitelist()
@@ -225,7 +229,7 @@ def insert_user_perm(user, doctype, docname, is_default=0, hide_descendants=0, a
 	user_perm.is_default = is_default
 	user_perm.hide_descendants = hide_descendants
 	if applicable:
-		user_perm.applicable_for  = applicable
+		user_perm.applicable_for = applicable
 		user_perm.apply_to_all_doctypes = 0
 	else:
 		user_perm.apply_to_all_doctypes = 1
@@ -233,27 +237,27 @@ def insert_user_perm(user, doctype, docname, is_default=0, hide_descendants=0, a
 
 def remove_applicable(perm_applied_docs, user, doctype, docname):
 	for applicable_for in perm_applied_docs:
-		frappe.db.sql("""DELETE FROM `tabUser Permission`
-			WHERE `user`=%s
-			AND `applicable_for`=%s
-			AND `allow`=%s
-			AND `for_value`=%s
-		""", (user, applicable_for, doctype, docname))
+		frappe.db.delete("User Permission", {
+			"applicable_for": applicable_for,
+			"for_value": docname,
+			"allow": doctype,
+			"user": user,
+		})
 
 def remove_apply_to_all(user, doctype, docname):
-	frappe.db.sql("""DELETE from `tabUser Permission`
-		WHERE `user`=%s
-		AND `apply_to_all_doctypes`=1
-		AND `allow`=%s
-		AND `for_value`=%s
-	""",(user, doctype, docname))
+	frappe.db.delete("User Permission", {
+		"apply_to_all_doctypes": 1,
+		"for_value": docname,
+		"allow": doctype,
+		"user": user,
+	})
 
 def update_applicable(already_applied, to_apply, user, doctype, docname):
 	for applied in already_applied:
 		if applied not in to_apply:
-			frappe.db.sql("""DELETE FROM `tabUser Permission`
-				WHERE `user`=%s
-				AND `applicable_for`=%s
-				AND `allow`=%s
-				AND `for_value`=%s
-			""",(user, applied, doctype, docname))
+			frappe.db.delete("User Permission", {
+				"applicable_for": applied,
+				"for_value": docname,
+				"allow": doctype,
+				"user": user,
+			})

frappe/core/page/background_jobs/background_jobs.py

@@ -4,12 +4,12 @@
 import json
 from typing import TYPE_CHECKING, Dict, List
 
-from rq import Queue, Worker
+from rq import Worker
 
 import frappe
 from frappe import _
 from frappe.utils import convert_utc_to_user_timezone, format_datetime
-from frappe.utils.background_jobs import get_redis_conn
+from frappe.utils.background_jobs import get_redis_conn, get_queues
 from frappe.utils.scheduler import is_scheduler_inactive
 
 if TYPE_CHECKING:
@@ -29,7 +29,7 @@ def get_info(show_failed=False) -> List[Dict]:
 		show_failed = json.loads(show_failed)
 
 	conn = get_redis_conn()
-	queues = Queue.all(conn)
+	queues = get_queues()
 	workers = Worker.all(conn)
 	jobs = []
 
@@ -75,7 +75,7 @@ def get_info(show_failed=False) -> List[Dict]:
 @frappe.whitelist()
 def remove_failed_jobs():
 	conn = get_redis_conn()
-	queues = Queue.all(conn)
+	queues = get_queues()
 	for queue in queues:
 		fail_registry = queue.failed_job_registry
 		for job_id in fail_registry.get_job_ids():

frappe/core/page/permission_manager/permission_manager.py

@@ -92,14 +92,14 @@ def update(doctype, role, permlevel, ptype, value=None):
 	"""Update role permission params
 
 	Args:
-	    doctype (str): Name of the DocType to update params for
-	    role (str): Role to be updated for, eg "Website Manager".
-	    permlevel (int): perm level the provided rule applies to
-	    ptype (str): permission type, example "read", "delete", etc.
-	    value (None, optional): value for ptype, None indicates False
+		doctype (str): Name of the DocType to update params for
+		role (str): Role to be updated for, eg "Website Manager".
+		permlevel (int): perm level the provided rule applies to
+		ptype (str): permission type, example "read", "delete", etc.
+		value (None, optional): value for ptype, None indicates False
 
 	Returns:
-	    str: Refresh flag is permission is updated successfully
+		str: Refresh flag is permission is updated successfully
 	"""
 	frappe.only_for("System Manager")
 	out = update_permission_property(doctype, role, permlevel, ptype, value)
@@ -110,10 +110,9 @@ def remove(doctype, role, permlevel):
 	frappe.only_for("System Manager")
 	setup_custom_perms(doctype)
 
-	name = frappe.get_value('Custom DocPerm', dict(parent=doctype, role=role, permlevel=permlevel))
+	frappe.db.delete("Custom DocPerm", {"parent": doctype, "role": role, "permlevel": permlevel})
 
-	frappe.db.sql('delete from `tabCustom DocPerm` where name=%s', name)
-	if not frappe.get_all('Custom DocPerm', dict(parent=doctype)):
+	if not frappe.get_all('Custom DocPerm', {"parent": doctype}):
 		frappe.throw(_('There must be atleast one permission rule.'), title=_('Cannot Remove'))
 
 	validate_permissions_for_doctype(doctype, for_remove=True, alert=True)

frappe/coverage.py

@@ -0,0 +1,35 @@
+# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors
+# MIT License. See LICENSE
+"""
+	frappe.coverage
+	~~~~~~~~~~~~~~~~
+
+	Coverage settings for frappe
+"""
+
+STANDARD_INCLUSIONS = ["*.py"]
+
+STANDARD_EXCLUSIONS = [
+	'*.js',
+	'*.xml',
+	'*.pyc',
+	'*.css',
+	'*.less',
+	'*.scss',
+	'*.vue',
+	'*.html',
+	'*/test_*',
+	'*/node_modules/*',
+	'*/doctype/*/*_dashboard.py',
+	'*/patches/*',
+]
+
+FRAPPE_EXCLUSIONS = [
+	"*/tests/*",
+	"*/commands/*",
+	"*/frappe/change_log/*",
+	"*/frappe/exceptions*",
+	"*frappe/setup.py",
+	"*/doctype/*/*_dashboard.py",
+	"*/patches/*",
+]

frappe/custom/doctype/custom_field/custom_field.json

@@ -120,7 +120,7 @@
    "label": "Field Type",
    "oldfieldname": "fieldtype",
    "oldfieldtype": "Select",
-   "options": "Attach\nAttach Image\nBarcode\nButton\nCheck\nCode\nColor\nColumn Break\nCurrency\nData\nDate\nDatetime\nDuration\nDynamic Link\nFloat\nGeolocation\nHTML\nImage\nInt\nLink\nLong Text\nMarkdown Editor\nPassword\nPercent\nRating\nRead Only\nSection Break\nSelect\nSmall Text\nTable\nTable MultiSelect\nText\nText Editor\nTime\nSignature",
+   "options": "Attach\nAttach Image\nBarcode\nButton\nCheck\nCode\nColor\nColumn Break\nCurrency\nData\nDate\nDatetime\nDuration\nDynamic Link\nFloat\nFold\nGeolocation\nHeading\nHTML\nHTML Editor\nImage\nInt\nLink\nLong Text\nMarkdown Editor\nPassword\nPercent\nRead Only\nRating\nSection Break\nSelect\nSmall Text\nTable\nTable MultiSelect\nText\nText Editor\nTime\nSignature",
    "reqd": 1
   },
   {
@@ -417,7 +417,7 @@
  "idx": 1,
  "index_web_pages_for_search": 1,
  "links": [],
- "modified": "2020-10-29 06:14:43.073329",
+ "modified": "2021-07-12 04:54:12.042319",
  "modified_by": "Administrator",
  "module": "Custom",
  "name": "Custom Field",

frappe/custom/doctype/custom_field/custom_field.py

@@ -85,12 +85,10 @@ class CustomField(Document):
 					frappe.bold(self.label)))
 
 		# delete property setter entries
-		frappe.db.sql("""\
-			DELETE FROM `tabProperty Setter`
-			WHERE doc_type = %s
-			AND field_name = %s""",
-				(self.dt, self.fieldname))
-
+		frappe.db.delete("Property Setter", {
+			"doc_type": self.dt,
+			"field_name": self.fieldname
+		})
 		frappe.clear_cache(doctype=self.dt)
 
 	def validate_insert_after(self, meta):

frappe/custom/doctype/customize_form/customize_form.py

@@ -1,5 +1,5 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
-# MIT License. See license.txt
+# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors
+# MIT License. See LICENSE
 
 """
 	Customize Form is a Single DocType used to mask the Property Setter
@@ -18,10 +18,11 @@ from frappe.custom.doctype.property_setter.property_setter import delete_propert
 from frappe.model.docfield import supports_translation
 from frappe.core.doctype.doctype.doctype import validate_series
 
+
 class CustomizeForm(Document):
 	def on_update(self):
-		frappe.db.sql("delete from tabSingles where doctype='Customize Form'")
-		frappe.db.sql("delete from `tabCustomize Form Field`")
+		frappe.db.delete("Singles", {"doctype": "Customize Form"})
+		frappe.db.delete("Customize Form Field")
 
 	@frappe.whitelist()
 	def fetch_to_customize(self):

frappe/database/database.py

@@ -6,6 +6,7 @@
 
 import re
 import time
+from typing import Dict, List, Union
 import frappe
 import datetime
 import frappe.defaults
@@ -13,7 +14,7 @@ import frappe.model.meta
 
 from frappe import _
 from time import time
-from frappe.utils import now, getdate, cast_fieldtype, get_datetime
+from frappe.utils import now, getdate, cast_fieldtype, get_datetime, get_table_name
 from frappe.model.utils.link_count import flush_local_link_count
 
 
@@ -103,6 +104,7 @@ class Database(object):
 				{"name": "a%", "owner":"test@example.com"})
 
 		"""
+		query = str(query)
 		if re.search(r'ifnull\(', query, flags=re.IGNORECASE):
 			# replaces ifnull in query with coalesce
 			query = re.sub(r'ifnull\(', 'coalesce(', query, flags=re.IGNORECASE)
@@ -951,15 +953,37 @@ class Database(object):
 		query = sql_dict.get(current_dialect)
 		return self.sql(query, values, **kwargs)
 
-	def delete(self, doctype, conditions, debug=False):
-		if conditions:
-			conditions, values = self.build_conditions(conditions)
-			return self.sql("DELETE FROM `tab{doctype}` where {conditions}".format(
-				doctype=doctype,
-				conditions=conditions
-			), values, debug=debug)
-		else:
-			frappe.throw(_('No conditions provided'))
+	def delete(self, doctype: str, filters: Union[Dict, List] = None, debug=False, **kwargs):
+		"""Delete rows from a table in site which match the passed filters. This
+		does trigger DocType hooks. Simply runs a DELETE query in the database.
+
+		Doctype name can be passed directly, it will be pre-pended with `tab`.
+		"""
+		values = ()
+		filters = filters or kwargs.get("conditions")
+		table = get_table_name(doctype)
+		query = f"DELETE FROM `{table}`"
+
+		if "debug" not in kwargs:
+			kwargs["debug"] = debug
+
+		if filters:
+			conditions, values = self.build_conditions(filters)
+			query = f"{query} WHERE {conditions}"
+
+		return self.sql(query, values, **kwargs)
+
+	def truncate(self, doctype: str):
+		"""Truncate a table in the database. This runs a DDL command `TRUNCATE TABLE`.
+		This cannot be rolled back.
+
+		Doctype name can be passed directly, it will be pre-pended with `tab`.
+		"""
+		table = doctype if doctype.startswith("__") else f"tab{doctype}"
+		return self.sql_ddl(f"truncate `{table}`")
+
+	def clear_table(self, doctype):
+		return self.truncate(doctype)
 
 	def get_last_created(self, doctype):
 		last_record = self.get_all(doctype, ('creation'), limit=1, order_by='creation desc')
@@ -968,9 +992,6 @@ class Database(object):
 		else:
 			return None
 
-	def clear_table(self, doctype):
-		self.sql('truncate `tab{}`'.format(doctype))
-
 	def log_touched_tables(self, query, values=None):
 		if values:
 			query = frappe.safe_decode(self._cursor.mogrify(query, values))
@@ -1021,6 +1042,7 @@ class Database(object):
 					), tuple(insert_list))
 				insert_list = []
 
+
 def enqueue_jobs_after_commit():
 	from frappe.utils.background_jobs import execute_job, get_queue
 

frappe/database/mariadb/database.py

@@ -1,3 +1,5 @@
+from typing import List, Tuple, Union
+
 import pymysql
 from pymysql.constants import ER, FIELD_TYPE
 from pymysql.converters import conversions, escape_string
@@ -5,7 +7,7 @@ from pymysql.converters import conversions, escape_string
 import frappe
 from frappe.database.database import Database
 from frappe.database.mariadb.schema import MariaDBTable
-from frappe.utils import UnicodeWithAttrs, cstr, get_datetime
+from frappe.utils import UnicodeWithAttrs, cstr, get_datetime, get_table_name
 
 
 class MariaDBDatabase(Database):
@@ -123,6 +125,19 @@ class MariaDBDatabase(Database):
 	def is_type_datetime(code):
 		return code in (pymysql.DATE, pymysql.DATETIME)
 
+	def rename_table(self, old_name: str, new_name: str) -> Union[List, Tuple]:
+		old_name = get_table_name(old_name)
+		new_name = get_table_name(new_name)
+		return self.sql(f"RENAME TABLE `{old_name}` TO `{new_name}`")
+
+	def describe(self, doctype: str) -> Union[List, Tuple]:
+		table_name = get_table_name(doctype)
+		return self.sql(f"DESC `{table_name}`")
+
+	def change_column_type(self, table: str, column: str, type: str) -> Union[List, Tuple]:
+		table_name = get_table_name(table)
+		return self.sql(f"ALTER TABLE `{table_name}` MODIFY `{column}` {type} NOT NULL")
+
 	# exception types
 	@staticmethod
 	def is_deadlocked(e):

frappe/database/mariadb/framework_mariadb.sql

@@ -220,6 +220,7 @@ CREATE TABLE `tabDocType` (
   `allow_guest_to_view` int(1) NOT NULL DEFAULT 0,
   `route` varchar(255) DEFAULT NULL,
   `is_published_field` varchar(255) DEFAULT NULL,
+  `website_search_field` varchar(255) DEFAULT NULL,
   `email_append_to` int(1) NOT NULL DEFAULT 0,
   `subject_field` varchar(255) DEFAULT NULL,
   `sender_field` varchar(255) DEFAULT NULL,

frappe/database/postgres/database.py

@@ -1,12 +1,14 @@
 import re
-import frappe
+from typing import List, Tuple, Union
+
 import psycopg2
 import psycopg2.extensions
-from frappe.utils import cstr
 from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
 
+import frappe
 from frappe.database.database import Database
 from frappe.database.postgres.schema import PostgresTable
+from frappe.utils import cstr, get_table_name
 
 # cast decimals as floats
 DEC2FLOAT = psycopg2.extensions.new_type(
@@ -170,6 +172,19 @@ class PostgresDatabase(Database):
 	def is_data_too_long(e):
 		return e.pgcode == '22001'
 
+	def rename_table(self, old_name: str, new_name: str) -> Union[List, Tuple]:
+		old_name = get_table_name(old_name)
+		new_name = get_table_name(new_name)
+		return self.sql(f"ALTER TABLE `{old_name}` RENAME TO `{new_name}`")
+
+	def describe(self, doctype: str)-> Union[List, Tuple]:
+		table_name = get_table_name(doctype)
+		return self.sql(f"SELECT COLUMN_NAME FROM information_schema.COLUMNS WHERE TABLE_NAME = '{table_name}'")
+
+	def change_column_type(self, table: str, column: str, type: str) -> Union[List, Tuple]:
+		table_name = get_table_name(table)
+		return self.sql(f'ALTER TABLE "{table_name}" ALTER COLUMN "{column}" TYPE {type}')
+
 	def create_auth_table(self):
 		self.sql_ddl("""create table if not exists "__Auth" (
 				"doctype" VARCHAR(140) NOT NULL,
@@ -297,6 +312,7 @@ class PostgresDatabase(Database):
 def modify_query(query):
 	""""Modifies query according to the requirements of postgres"""
 	# replace ` with " for definitions
+	query = str(query)
 	query = query.replace('`', '"')
 	query = replace_locate_with_strpos(query)
 	# select from requires ""

frappe/database/postgres/framework_postgres.sql

@@ -225,6 +225,7 @@ CREATE TABLE "tabDocType" (
   "allow_guest_to_view" smallint NOT NULL DEFAULT 0,
   "route" varchar(255) DEFAULT NULL,
   "is_published_field" varchar(255) DEFAULT NULL,
+  "website_search_field" varchar(255) DEFAULT NULL,
   "email_append_to" smallint NOT NULL DEFAULT 0,
   "subject_field" varchar(255) DEFAULT NULL,
   "sender_field" varchar(255) DEFAULT NULL,

frappe/defaults.py

@@ -124,11 +124,10 @@ def set_default(key, value, parent, parenttype="__default"):
 		where
 			defkey=%s and parent=%s
 		for update''', (key, parent)):
-		frappe.db.sql("""
-			delete from
-				`tabDefaultValue`
-			where
-				defkey=%s and parent=%s""", (key, parent))
+		frappe.db.delete("DefaultValue", {
+			"defkey": key,
+			"parent": parent
+		})
 	if value != None:
 		add_default(key, value, parent)
 	else:
@@ -155,29 +154,23 @@ def clear_default(key=None, value=None, parent=None, name=None, parenttype=None)
 	:param name: Default ID.
 	:param parenttype: Clear defaults table for a particular type e.g. **User**.
 	"""
-	conditions = []
-	values = []
+	filters = {}
 
 	if name:
-		conditions.append("name=%s")
-		values.append(name)
+		filters.update({"name": name})
 
 	else:
 		if key:
-			conditions.append("defkey=%s")
-			values.append(key)
+			filters.update({"defkey": key})
 
 		if value:
-			conditions.append("defvalue=%s")
-			values.append(value)
+			filters.update({"defvalue": value})
 
 		if parent:
-			conditions.append("parent=%s")
-			values.append(parent)
+			filters.update({"parent": parent})
 
 		if parenttype:
-			conditions.append("parenttype=%s")
-			values.append(parenttype)
+			filters.update({"parenttype": parenttype})
 
 	if parent:
 		clear_defaults_cache(parent)
@@ -185,11 +178,10 @@ def clear_default(key=None, value=None, parent=None, name=None, parenttype=None)
 		clear_defaults_cache("__default")
 		clear_defaults_cache("__global")
 
-	if not conditions:
+	if not filters:
 		raise Exception("[clear_default] No key specified.")
 
-	frappe.db.sql("""delete from tabDefaultValue where {0}""".format(" and ".join(conditions)),
-		tuple(values))
+	frappe.db.delete("DefaultValue", filters)
 
 	_clear_cache(parent)
 

frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py

@@ -64,7 +64,7 @@ class TestDashboardChart(unittest.TestCase):
 		if frappe.db.exists('Dashboard Chart', 'Test Empty Dashboard Chart'):
 			frappe.delete_doc('Dashboard Chart', 'Test Empty Dashboard Chart')
 
-		frappe.db.sql('delete from `tabError Log`')
+		frappe.db.delete("Error Log")
 
 		frappe.get_doc(dict(
 			doctype = 'Dashboard Chart',
@@ -94,7 +94,7 @@ class TestDashboardChart(unittest.TestCase):
 		if frappe.db.exists('Dashboard Chart', 'Test Empty Dashboard Chart 2'):
 			frappe.delete_doc('Dashboard Chart', 'Test Empty Dashboard Chart 2')
 
-		frappe.db.sql('delete from `tabError Log`')
+		frappe.db.delete("Error Log")
 
 		# create one data point
 		frappe.get_doc(dict(doctype = 'Error Log', creation = '2018-06-01 00:00:00')).insert()

frappe/desk/doctype/desktop_icon/desktop_icon.py

@@ -197,7 +197,7 @@ def set_desktop_icons(visible_list, ignore_duplicate=True):
 
 	# clear all custom only if setup is not complete
 	if not int(frappe.defaults.get_defaults().setup_complete or 0):
-		frappe.db.sql('delete from `tabDesktop Icon` where standard=0')
+		frappe.db.delete("Desktop Icon", {"standard": 0})
 
 	# set standard as blocked and hidden if setting first active domain
 	if not frappe.flags.keep_desktop_icons:

frappe/desk/doctype/event/event.py

@@ -338,9 +338,8 @@ def delete_events(ref_type, ref_name, delete_event=False):
 				total_participants = frappe.get_all("Event Participants", filters={"parenttype": "Event", "parent": participation.parent})
 
 				if len(total_participants) <= 1:
-					frappe.db.sql("DELETE FROM `tabEvent` WHERE `name` = %(name)s", {'name': participation.parent})
-
-				frappe.db.sql("DELETE FROM `tabEvent Participants ` WHERE `name` = %(name)s", {'name': participation.name})
+					frappe.db.delete("Event", {"name": participation.parent})
+					frappe.db.delete("Event Participants", {"name": participation.name})
 
 # Close events if ends_on or repeat_till is less than now_datetime
 def set_status_of_events():

frappe/desk/doctype/form_tour/form_tour.js

@@ -15,14 +15,14 @@ frappe.ui.form.on('Form Tour', {
 
 		frm.add_custom_button(__('Show Tour'), async () => {
 			const issingle = await check_if_single(frm.doc.reference_doctype);
+			let route_changed = null;
 
 			if (issingle) {
-				frappe.set_route('Form', frm.doc.reference_doctype);
+				route_changed = frappe.set_route('Form', frm.doc.reference_doctype);
 			} else {
-				const new_name =  'new-' + frappe.scrub(frm.doc.reference_doctype) + '-1';
-				frappe.set_route('Form', frm.doc.reference_doctype, new_name);
+				route_changed = frappe.set_route('Form', frm.doc.reference_doctype, 'new');
 			}
-			frappe.utils.sleep(500).then(() => {
+			route_changed.then(() => {
 				const tour_name = frm.doc.name;
 				cur_frm.tour
 					.init({ tour_name })

frappe/desk/doctype/notification_log/notification_log.py

@@ -12,7 +12,10 @@ class NotificationLog(Document):
 		frappe.publish_realtime('notification', after_commit=True, user=self.for_user)
 		set_notifications_as_unseen(self.for_user)
 		if is_email_notifications_enabled_for_type(self.for_user, self.type):
-			send_notification_email(self)
+			try:
+				send_notification_email(self)
+			except frappe.OutgoingEmailError:
+				frappe.log_error(message=frappe.get_traceback(), title=_("Failed to send notification email"))
 
 
 def get_permission_query_conditions(for_user):

frappe/desk/doctype/route_history/route_history.py

@@ -1,5 +1,4 @@
-# -*- coding: utf-8 -*-
-# Copyright (c) 2018, Frappe Technologies and contributors
+# Copyright (c) 2021, Frappe Technologies and contributors
 # For license information, please see license.txt
 
 import frappe
@@ -8,6 +7,7 @@ from frappe.model.document import Document
 class RouteHistory(Document):
 	pass
 
+
 def flush_old_route_records():
 	"""Deletes all route records except last 500 records per user"""
 
@@ -24,19 +24,14 @@ def flush_old_route_records():
 	for user in users:
 		user = user[0]
 		last_record_to_keep = frappe.db.get_all('Route History',
-			filters={
-				'user': user,
-			},
+			filters={'user': user},
 			limit=1,
 			limit_start=500,
 			fields=['modified'],
-			order_by='modified desc')
+			order_by='modified desc'
+		)
 
-		frappe.db.sql('''
-			DELETE
-			FROM `tabRoute History`
-			WHERE `modified` <= %(modified)s and `user`=%(modified)s
-		''', {
-			"modified": last_record_to_keep[0].modified,
+		frappe.db.delete("Route History", {
+			"modified": ("<=", last_record_to_keep[0].modified),
 			"user": user
-		})
+		})

frappe/desk/doctype/tag/tag.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright (c) 2019, Frappe Technologies and contributors
 # For license information, please see license.txt
 
@@ -123,7 +122,10 @@ def delete_tags_for_document(doc):
 	if not frappe.db.table_exists("Tag Link"):
 		return
 
-	frappe.db.sql("""DELETE FROM `tabTag Link` WHERE `document_type`=%s AND `document_name`=%s""", (doc.doctype, doc.name))
+	frappe.db.delete("Tag Link", {
+		"document_type": doc.doctype,
+		"document_name": doc.name
+	})
 
 def update_tags(doc, tags):
 	"""
@@ -161,7 +163,11 @@ def get_deleted_tags(new_tags, existing_tags):
 	return list(set(existing_tags) - set(new_tags))
 
 def delete_tag_for_document(dt, dn, tag):
-	frappe.db.sql("""DELETE FROM `tabTag Link` WHERE `document_type`=%s AND `document_name`=%s AND tag=%s""", (dt, dn, tag))
+	frappe.db.delete("Tag Link", {
+		"document_type": dt,
+		"document_name": dn,
+		"tag": tag
+	})
 
 @frappe.whitelist()
 def get_documents_for_tag(tag):

frappe/desk/doctype/tag/test_tag.py

@@ -1,8 +1,26 @@
-# -*- coding: utf-8 -*-
-# Copyright (c) 2019, Frappe Technologies and Contributors
-# See license.txt
-# import frappe
 import unittest
+import frappe
+
+from frappe.desk.reportview import get_stats
+from frappe.desk.doctype.tag.tag import add_tag
 
 class TestTag(unittest.TestCase):
-	pass
+	def setUp(self) -> None:
+		frappe.db.sql("DELETE from `tabTag`")
+		frappe.db.sql("UPDATE `tabDocType` set _user_tags=''")
+
+	def test_tag_count_query(self):
+		self.assertDictEqual(get_stats('["_user_tags"]', 'DocType'),
+			{'_user_tags': [['No Tags', frappe.db.count('DocType')]]})
+		add_tag('Standard', 'DocType', 'User')
+		add_tag('Standard', 'DocType', 'ToDo')
+
+		# count with no filter
+		self.assertDictEqual(get_stats('["_user_tags"]', 'DocType'),
+			{'_user_tags': [['Standard', 2], ['No Tags', frappe.db.count('DocType') - 2]]})
+
+		# count with child table field filter
+		self.assertDictEqual(get_stats('["_user_tags"]',
+			'DocType',
+			filters='[["DocField", "fieldname", "like", "%last_name%"], ["DocType", "name", "like", "%use%"]]'),
+			{'_user_tags': [['Standard', 1], ['No Tags', 0]]})

frappe/desk/doctype/todo/todo.py

@@ -5,11 +5,13 @@ import frappe
 import json
 
 from frappe.model.document import Document
-from frappe.utils import get_fullname
+from frappe.utils import get_fullname, parse_addr
 
 exclude_from_linked_with = True
 
 class ToDo(Document):
+	DocType = 'ToDo'
+
 	def validate(self):
 		self._assignment = None
 		if self.is_new():
@@ -39,13 +41,7 @@ class ToDo(Document):
 		self.update_in_reference()
 
 	def on_trash(self):
-		# unlink todo from linked comments
-		frappe.db.sql("""
-			delete from `tabCommunication Link`
-			where link_doctype=%(doctype)s and link_name=%(name)s""", {
-				"doctype": self.doctype, "name": self.name
-		})
-
+		self.delete_communication_links()
 		self.update_in_reference()
 
 	def add_assign_comment(self, text, comment_type):
@@ -54,6 +50,13 @@ class ToDo(Document):
 
 		frappe.get_doc(self.reference_type, self.reference_name).add_comment(comment_type, text)
 
+	def delete_communication_links(self):
+		# unlink todo from linked comments
+		return frappe.db.delete("Communication Link", {
+			"link_doctype": self.doctype,
+			"link_name": self.name
+		})
+
 	def update_in_reference(self):
 		if not (self.reference_type and self.reference_name):
 			return
@@ -84,6 +87,13 @@ class ToDo(Document):
 			else:
 				raise
 
+	@classmethod
+	def get_owners(cls, filters=None):
+		"""Returns list of owners after applying filters on todo's.
+		"""
+		rows = frappe.get_all(cls.DocType, filters=filters or {}, fields=['owner'])
+		return [parse_addr(row.owner)[1] for row in rows if row.owner]
+
 # NOTE: todo is viewable if a user is an owner, or set as assigned_to value, or has any role that is allowed to access ToDo doctype.
 def on_doctype_update():
 	frappe.db.add_index("ToDo", ["reference_type", "reference_name"])

frappe/desk/reportview.py

@@ -445,24 +445,36 @@ def get_stats(stats, doctype, filters=[]):
 	for tag in tags:
 		if not tag in columns: continue
 		try:
-			tagcount = frappe.get_list(doctype, fields=[tag, "count(*)"],
-				#filters=["ifnull(`%s`,'')!=''" % tag], group_by=tag, as_list=True)
-				filters = filters + ["ifnull(`%s`,'')!=''" % tag], group_by = tag, as_list = True)
+			tag_count = frappe.get_list(doctype,
+				fields=[tag, "count(*)"],
+				filters=filters + [[tag, '!=', '']],
+				group_by=tag,
+				as_list=True,
+				distinct=1,
+			)
 
-			if tag=='_user_tags':
-				stats[tag] = scrub_user_tags(tagcount)
-				stats[tag].append([_("No Tags"), frappe.get_list(doctype,
+			if tag == '_user_tags':
+				stats[tag] = scrub_user_tags(tag_count)
+				no_tag_count = frappe.get_list(doctype,
 					fields=[tag, "count(*)"],
-					filters=filters +["({0} = ',' or {0} = '' or {0} is null)".format(tag)], as_list=True)[0][1]])
+					filters=filters + [[tag, "in", ('', ',')]],
+					as_list=True,
+					group_by=tag,
+					order_by=tag,
+				)
+
+				no_tag_count = no_tag_count[0][1] if no_tag_count else 0
+
+				stats[tag].append([_("No Tags"), no_tag_count])
 			else:
-				stats[tag] = tagcount
+				stats[tag] = tag_count
 
 		except frappe.db.SQLError:
-			# does not work for child tables
 			pass
-		except frappe.db.InternalError:
+		except frappe.db.InternalError as e:
 			# raised when _user_tags column is added on the fly
 			pass
+
 	return stats
 
 @frappe.whitelist()

frappe/desk/search.py

@@ -168,7 +168,18 @@ def search_widget(doctype, txt, query=None, searchfield=None, start=0,
 				strict=False)
 
 			if doctype in UNTRANSLATED_DOCTYPES:
-				values = tuple([v for v in list(values) if re.search(re.escape(txt)+".*", (_(v.name) if as_dict else _(v[0])), re.IGNORECASE)])
+				# Filtering the values array so that query is included in very element
+				values = (
+					v for v in values
+					if re.search(
+						f"{re.escape(txt)}.*", _(v.name if as_dict else v[0]), re.IGNORECASE
+					)
+				)
+
+			# Sorting the values array so that relevant results always come first
+			# This will first bring elements on top in which query is a prefix of element
+			# Then it will bring the rest of the elements and sort them in lexicographical order
+			values = sorted(values, key=lambda x: relevance_sorter(x, txt, as_dict))
 
 			# remove _relevance from results
 			if as_dict:
@@ -208,6 +219,13 @@ def scrub_custom_query(query, key, txt):
 		query = query.replace('%s', ((txt or '') + '%'))
 	return query
 
+def relevance_sorter(key, query, as_dict):
+	value = _(key.name if as_dict else key[0])
+	return (
+		value.lower().startswith(query.lower()) is not True,
+		value
+	)
+
 @wrapt.decorator
 def validate_and_sanitize_search_inputs(fn, instance, args, kwargs):
 	kwargs.update(dict(zip(fn.__code__.co_varnames, args)))

frappe/email/doctype/unhandled_email/unhandled_email.py

@@ -10,5 +10,6 @@ class UnhandledEmail(Document):
 
 
 def remove_old_unhandled_emails():
-	frappe.db.sql("""DELETE FROM `tabUnhandled Email`
-	WHERE creation < %s""", frappe.utils.add_days(frappe.utils.nowdate(), -30))
+	frappe.db.delete("Unhandled Email", {
+		"creation": ("<", frappe.utils.add_days(frappe.utils.nowdate(), -30))
+	})

frappe/email/queue.py

@@ -173,13 +173,8 @@ def clear_outbox(days=None):
 		WHERE `priority`=0 AND `modified` < (NOW() - INTERVAL '{0}' DAY)""".format(days))
 
 	if email_queues:
-		frappe.db.sql("""DELETE FROM `tabEmail Queue` WHERE `name` IN ({0})""".format(
-			','.join(['%s']*len(email_queues)
-		)), tuple(email_queues))
-
-		frappe.db.sql("""DELETE FROM `tabEmail Queue Recipient` WHERE `parent` IN ({0})""".format(
-			','.join(['%s']*len(email_queues)
-		)), tuple(email_queues))
+		frappe.db.delete("Email Queue", {"name": ("in", email_queues)})
+		frappe.db.delete("Email Queue Recipient", {"parent": ("in", email_queues)})
 
 def set_expiry_for_email_queue():
 	''' Mark emails as expire that has not sent for 7 days.

frappe/email/test_email_body.py

@@ -127,7 +127,7 @@ w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
 '''
 		transformed_html = '''
 <h3>Hi John</h3>
-<p style="margin:5px 0 !important">This is a test email</p>
+<p style="margin:1em 0 !important">This is a test email</p>
 '''
 		self.assertTrue(transformed_html in inline_style_in_html(html))
 

frappe/hooks.py

@@ -171,6 +171,9 @@ doc_events = {
 			"frappe.workflow.doctype.workflow_action.workflow_action.process_workflow_actions",
 			"frappe.event_streaming.doctype.event_update_log.event_update_log.notify_consumers"
 		],
+		"on_update_after_submit": [
+			"frappe.workflow.doctype.workflow_action.workflow_action.process_workflow_actions"
+		],
 		"on_change": [
 			"frappe.social.doctype.energy_point_rule.energy_point_rule.process_energy_points",
 			"frappe.automation.doctype.milestone_tracker.milestone_tracker.evaluate_milestone"

frappe/model/__init__.py

@@ -152,32 +152,22 @@ def delete_fields(args_dict, delete=0):
 		if not fields:
 			continue
 
-		frappe.db.sql("""
-			DELETE FROM `tabDocField`
-			WHERE parent='%s' AND fieldname IN (%s)
-		""" % (dt, ", ".join(["'{}'".format(f) for f in fields])))
+		frappe.db.delete("DocField", {
+			"parent": dt,
+			"fieldname": ("in", fields),
+		})
 
 		# Delete the data/column only if delete is specified
 		if not delete:
 			continue
 
 		if frappe.db.get_value("DocType", dt, "issingle"):
-			frappe.db.sql("""
-				DELETE FROM `tabSingles`
-				WHERE doctype='%s' AND field IN (%s)
-			""" % (dt, ", ".join("'{}'".format(f) for f in fields)))
+			frappe.db.delete("Singles", {
+				"doctype": dt,
+				"field": ("in", fields),
+			})
 		else:
-			existing_fields = frappe.db.multisql({
-					"mariadb": "DESC `tab%s`" % dt,
-					"postgres": """
-						SELECT
- 							COLUMN_NAME
-						FROM
- 							information_schema.COLUMNS
-						WHERE
- 							TABLE_NAME = 'tab%s';
-					""" % dt,
-				})
+			existing_fields = frappe.db.describe(dt)
 			existing_fields = existing_fields and [e[0] for e in existing_fields] or []
 			fields_need_to_delete = set(fields) & set(existing_fields)
 			if not fields_need_to_delete:

frappe/model/delete_doc.py

@@ -65,12 +65,12 @@ def delete_doc(doctype=None, name=None, force=0, ignore_doctypes=None, for_reloa
 				update_flags(doc, flags, ignore_permissions)
 				check_permission_and_not_submitted(doc)
 
-				frappe.db.sql("delete from `tabCustom Field` where dt = %s", name)
-				frappe.db.sql("delete from `tabClient Script` where dt = %s", name)
-				frappe.db.sql("delete from `tabProperty Setter` where doc_type = %s", name)
-				frappe.db.sql("delete from `tabReport` where ref_doctype=%s", name)
-				frappe.db.sql("delete from `tabCustom DocPerm` where parent=%s", name)
-				frappe.db.sql("delete from `__global_search` where doctype=%s", name)
+				frappe.db.delete("Custom Field", {"dt": name})
+				frappe.db.delete("Client Script", {"dt": name})
+				frappe.db.delete("Property Setter", {"doc_type": name})
+				frappe.db.delete("Report", {"ref_doctype": name})
+				frappe.db.delete("Custom DocPerm", {"parent": name})
+				frappe.db.delete("__global_search", {"doctype": name})
 
 			delete_from_table(doctype, name, ignore_doctypes, None)
 
@@ -162,10 +162,9 @@ def update_naming_series(doc):
 
 def delete_from_table(doctype, name, ignore_doctypes, doc):
 	if doctype!="DocType" and doctype==name:
-		frappe.db.sql("delete from `tabSingles` where `doctype`=%s", name)
+		frappe.db.delete("Singles", {"doctype": name})
 	else:
-		frappe.db.sql("delete from `tab{0}` where `name`=%s".format(doctype), name)
-
+		frappe.db.delete(doctype, {"name": name})
 	# get child tables
 	if doc:
 		tables = [d.options for d in doc.meta.get_table_fields()]
@@ -339,8 +338,10 @@ def clear_references(doctype, reference_doctype, reference_name,
 		(reference_doctype, reference_name))
 
 def clear_timeline_references(link_doctype, link_name):
-	frappe.db.sql("""DELETE FROM `tabCommunication Link`
-		WHERE `tabCommunication Link`.link_doctype=%s AND `tabCommunication Link`.link_name=%s""", (link_doctype, link_name))
+	frappe.db.delete("Communication Link", {
+		"link_doctype": link_doctype,
+		"link_name": link_name
+	})
 
 def insert_feed(doc):
 	if (

frappe/model/document.py

@@ -390,10 +390,11 @@ class Document(BaseDocument):
 
 		else:
 			# no rows found, delete all rows
-			frappe.db.sql("""delete from `tab{0}` where parent=%s
-				and parenttype=%s and parentfield=%s""".format(df.options),
-				(self.name, self.doctype, fieldname))
-
+			frappe.db.delete(df.options, {
+				"parent": self.name,
+				"parenttype": self.doctype,
+				"parentfield": fieldname
+			})
 	def get_doc_before_save(self):
 		return getattr(self, '_doc_before_save', None)
 
@@ -451,7 +452,9 @@ class Document(BaseDocument):
 
 	def update_single(self, d):
 		"""Updates values for Single type Document in `tabSingles`."""
-		frappe.db.sql("""delete from `tabSingles` where doctype=%s""", self.doctype)
+		frappe.db.delete("Singles", {
+			"doctype": self.doctype
+		})
 		for field, value in d.items():
 			if field != "doctype":
 				frappe.db.sql("""insert into `tabSingles` (doctype, field, value)

frappe/parallel_test_runner.py

@@ -111,33 +111,16 @@ class ParallelTestRunner():
 		if self.with_coverage:
 			from coverage import Coverage
 			from frappe.utils import get_bench_path
+			from frappe.coverage import STANDARD_INCLUSIONS, STANDARD_EXCLUSIONS, FRAPPE_EXCLUSIONS
 
 			# Generate coverage report only for app that is being tested
 			source_path = os.path.join(get_bench_path(), 'apps', self.app)
-			incl = [
-				'*.py',
-			]
-			omit = [
-				'*.js',
-				'*.xml',
-				'*.pyc',
-				'*.css',
-				'*.less',
-				'*.scss',
-				'*.vue',
-				'*.pyc',
-				'*.html',
-				'*/test_*',
-				'*/node_modules/*',
-				'*/doctype/*/*_dashboard.py',
-				'*/patches/*',
-			]
+			omit = STANDARD_EXCLUSIONS[:]
 
 			if self.app == 'frappe':
-				omit.append('*/tests/*')
-				omit.append('*/commands/*')
+				omit.extend(FRAPPE_EXCLUSIONS)
 
-			self.coverage = Coverage(source=[source_path], omit=omit, include=incl)
+			self.coverage = Coverage(source=[source_path], omit=omit, include=STANDARD_INCLUSIONS)
 			self.coverage.start()
 
 	def save_coverage(self):

frappe/patches.txt

@@ -180,3 +180,4 @@ frappe.patches.v12_0.rename_uploaded_files_with_proper_name
 frappe.patches.v13_0.queryreport_columns
 frappe.patches.v13_0.jinja_hook
 frappe.patches.v13_0.update_notification_channel_if_empty
+frappe.patches.v14_0.drop_data_import_legacy

frappe/patches/v11_0/apply_customization_to_custom_doctype.py

@@ -28,7 +28,7 @@ def execute():
 
 		for prop in property_setters:
 			property_setter_map[prop.field_name] = prop
-			frappe.db.sql('DELETE FROM `tabProperty Setter` WHERE `name`=%s', prop.name)
+			frappe.db.delete("Property Setter", {"name": prop.name})
 
 		meta = frappe.get_meta(doctype.name)
 
@@ -50,6 +50,6 @@ def execute():
 				df = frappe.new_doc('DocField', meta, 'fields')
 				df.update(cf)
 				meta.fields.append(df)
-			frappe.db.sql('DELETE FROM `tabCustom Field` WHERE name=%s', cf.name)
+			frappe.db.delete("Custom Field", {"name": cf.name})
 
 		meta.save()

frappe/patches/v11_0/sync_stripe_settings_before_migrate.py

@@ -17,4 +17,4 @@ def execute():
 			settings.secret_key = secret_key
 			settings.save(ignore_permissions=True)
 
-	frappe.db.sql("""DELETE FROM tabSingles WHERE doctype='Stripe Settings'""")
+	frappe.db.delete("Singles", {"doctype": "Stripe Settings"})

frappe/patches/v12_0/delete_feedback_request_if_exists.py

@@ -2,7 +2,4 @@
 import frappe
 
 def execute():
-    frappe.db.sql('''
-        DELETE from `tabDocType`
-        WHERE name = 'Feedback Request'
-    ''')
+	frappe.db.delete("DocType", {"name": "Feedback Request"})

frappe/patches/v12_0/remove_deprecated_fields_from_doctype.py

@@ -8,7 +8,6 @@ def execute():
 		'DocType': ['hide_heading', 'image_view', 'read_only_onload']
 	}, delete=1)
 
-	frappe.db.sql('''
-		DELETE from `tabProperty Setter`
-		WHERE property = 'read_only_onload'
-	''')
+	frappe.db.delete("Property Setter", {
+		"property": "read_only_onload"
+	})

frappe/patches/v12_0/set_correct_assign_value_in_docs.py

@@ -1,32 +1,29 @@
 import frappe
+from frappe.query_builder.functions import GroupConcat, Coalesce
 
 def execute():
-	frappe.reload_doc('desk', 'doctype', 'todo')
+	frappe.reload_doc("desk", "doctype", "todo")
 
-	query = '''
-		SELECT
-			name, reference_type, reference_name, {} as assignees
-		FROM
-			`tabToDo`
-		WHERE
-			COALESCE(reference_type, '') != '' AND
-			COALESCE(reference_name, '') != '' AND
-			status != 'Cancelled'
-		GROUP BY
-			reference_type, reference_name
-	'''
+	ToDo = frappe.qb.Table("ToDo")
+	assignees = GroupConcat("owner").distinct().as_("assignees")
 
-	assignments = frappe.db.multisql({
-		'mariadb': query.format('GROUP_CONCAT(DISTINCT `owner`)'),
-		'postgres': query.format('STRING_AGG(DISTINCT "owner", ",")')
-	}, as_dict=True)
+	query = (
+		frappe.qb.from_(ToDo)
+		.select(ToDo.name, ToDo.reference_type, assignees)
+		.where(Coalesce(ToDo.reference_type, "") != "")
+		.where(Coalesce(ToDo.reference_name, "") != "")
+		.where(ToDo.status != "Cancelled")
+		.groupby(ToDo.reference_type, ToDo.reference_name)
+	)
+
+	assignments = frappe.db.sql(query, as_dict=True)
 
 	for doc in assignments:
-		assignments = doc.assignees.split(',')
+		assignments = doc.assignees.split(",")
 		frappe.db.set_value(
 			doc.reference_type,
 			doc.reference_name,
-			'_assign',
+			"_assign",
 			frappe.as_json(assignments),
 			update_modified=False
-		)
+		)

frappe/patches/v12_0/set_primary_key_in_series.py

@@ -1,21 +1,24 @@
 import frappe
 
 def execute():
-    #if current = 0, simply delete the key as it'll be recreated on first entry
-    frappe.db.sql('delete from `tabSeries` where current = 0')
-    duplicate_keys = frappe.db.sql('''
-        SELECT name, max(current) as current
-        from
-            `tabSeries`
-        group by
-            name
-        having count(name) > 1
-    ''', as_dict=True)
-    for row in duplicate_keys:
-        frappe.db.sql('delete from `tabSeries` where name = %(key)s', {
-            'key': row.name
-        })
-        if row.current:
-            frappe.db.sql('insert into `tabSeries`(`name`, `current`) values (%(name)s, %(current)s)', row)
-    frappe.db.commit()
-    frappe.db.sql('ALTER table `tabSeries` ADD PRIMARY KEY IF NOT EXISTS (name)')
+	#if current = 0, simply delete the key as it'll be recreated on first entry
+	frappe.db.delete("Series", {"current": 0})
+
+	duplicate_keys = frappe.db.sql('''
+		SELECT name, max(current) as current
+		from
+			`tabSeries`
+		group by
+			name
+		having count(name) > 1
+	''', as_dict=True)
+
+	for row in duplicate_keys:
+		frappe.db.delete("Series", {
+			"name": row.name
+		})
+		if row.current:
+			frappe.db.sql('insert into `tabSeries`(`name`, `current`) values (%(name)s, %(current)s)', row)
+	frappe.db.commit()
+
+	frappe.db.sql('ALTER table `tabSeries` ADD PRIMARY KEY IF NOT EXISTS (name)')

frappe/patches/v12_0/setup_comments_from_communications.py

@@ -29,4 +29,6 @@ def execute():
 		frappe.db.auto_commit_on_many_writes = False
 
 	# clean up
-	frappe.db.sql("delete from `tabCommunication` where communication_type = 'Comment'")
+	frappe.db.delete("Communication", {
+		"communication_type": "Comment"
+	})

frappe/patches/v13_0/increase_password_length.py

@@ -1,7 +1,4 @@
 import frappe
 
 def execute():
-	frappe.db.multisql({
-		"mariadb": "ALTER  TABLE `__Auth` MODIFY `password` TEXT NOT NULL",
-		"postgres": 'ALTER TABLE "__Auth" ALTER COLUMN "password" TYPE TEXT'
-	})
+	frappe.db.change_column_type(table="__Auth", column="password", type="TEXT")

frappe/patches/v13_0/remove_twilio_settings.py

@@ -12,7 +12,9 @@ def execute():
 	frappe.delete_doc_if_exists('DocType', 'Twilio Number Group')
 	if twilio_settings_doctype_in_integrations():
 		frappe.delete_doc_if_exists('DocType', 'Twilio Settings')
-		frappe.db.sql("delete from `tabSingles` where `doctype`=%s", 'Twilio Settings')
+		frappe.db.delete("Singles", {
+			"doctype": "Twilio Settings"
+		})
 
 def twilio_settings_doctype_in_integrations() -> bool:
 	"""Check Twilio Settings doctype exists in integrations module or not.

frappe/patches/v14_0/drop_data_import_legacy.py

@@ -0,0 +1,22 @@
+import frappe
+import click
+
+
+def execute():
+	doctype = "Data Import Legacy"
+	table = frappe.utils.get_table_name(doctype)
+
+	# delete the doctype record to avoid broken links
+	frappe.db.delete("DocType", {"name": doctype})
+
+	# leaving table in database for manual cleanup
+	click.secho(
+		f"`{doctype}` has been deprecated. The DocType is deleted, but the data still"
+		" exists on the database. If this data is worth recovering, you may export it"
+		f" using\n\n\tbench --site {frappe.local.site} backup -i '{doctype}'\n\nAfter"
+		" this, the table will continue to persist in the database, until you choose"
+		" to remove it yourself. If you want to drop the table, you may run\n\n\tbench"
+		f" --site {frappe.local.site} execute frappe.db.sql --args \"('DROP TABLE IF"
+		f" EXISTS `{table}`', )\"\n",
+		fg="yellow",
+	)

frappe/permissions.py

@@ -7,9 +7,11 @@ import frappe.share
 from frappe import _, msgprint
 from frappe.utils import cint
 
+
 rights = ("select", "read", "write", "create", "delete", "submit", "cancel", "amend",
 	"print", "email", "report", "import", "export", "set_user_permissions", "share")
 
+
 def check_admin_or_system_manager(user=None):
 	if not user: user = frappe.session.user
 
@@ -516,8 +518,7 @@ def reset_perms(doctype):
 	"""Reset permissions for given doctype."""
 	from frappe.desk.notifications import delete_notification_count_for
 	delete_notification_count_for(doctype)
-
-	frappe.db.sql("""delete from `tabCustom DocPerm` where parent=%s""", doctype)
+	frappe.db.delete("Custom DocPerm", {"parent": doctype})
 
 def get_linked_doctypes(dt):
 	return list(set([dt] + [d.options for d in

frappe/printing/page/print/print.js

@@ -403,19 +403,14 @@ frappe.ui.form.PrintView = class {
 	setup_print_format_dom(out, $print_format) {
 		this.print_wrapper.find('.print-format-skeleton').remove();
 		let base_url = frappe.urllib.get_base_url();
-		let print_css = frappe.assets.bundled_asset('print.bundle.css');
+		let print_css = frappe.assets.bundled_asset('print.bundle.css', frappe.utils.is_rtl(this.lang_code));
+		this.$print_format_body.find('html').attr('dir', frappe.utils.is_rtl(this.lang_code) ? 'rtl': 'ltr');
+		this.$print_format_body.find('html').attr('lang', this.lang_code);
 		this.$print_format_body.find('head').html(
 			`<style type="text/css">${out.style}</style>
 			<link href="${base_url}${print_css}" rel="stylesheet">`
 		);
 
-		if (frappe.utils.is_rtl(this.lang_code)) {
-			let rtl_css = frappe.assets.bundled_asset('frappe-rtl.bundle.css');
-			this.$print_format_body.find('head').append(
-				`<link type="text/css" rel="stylesheet" href="${base_url}${rtl_css}"></link>`
-			);
-		}
-
 		this.$print_format_body.find('body').html(
 			`<div class="print-format print-format-preview">${out.html}</div>`
 		);

frappe/public/build.json

@@ -1,299 +0,0 @@
-{
-	"css/frappe-web-b4.css": "public/scss/website.scss",
-	"css/frappe-chat-web.css": [
-		"public/css/font-awesome.css",
-		"public/css/octicons/octicons.css",
-		"public/less/chat.less"
-	],
-	"concat:js/moment-bundle.min.js": [
-		"node_modules/moment/min/moment-with-locales.min.js",
-		"node_modules/moment-timezone/builds/moment-timezone-with-data.min.js"
-	],
-	"js/chat.js": "public/js/frappe/chat.js",
-	"js/frappe-recorder.min.js": "public/js/frappe/recorder/recorder.js",
-	"js/checkout.min.js": "public/js/integrations/razorpay.js",
-	"js/frappe-web.min.js": [
-		"public/js/frappe/class.js",
-		"public/js/frappe/polyfill.js",
-		"public/js/lib/md5.min.js",
-		"public/js/frappe/provide.js",
-		"public/js/frappe/format.js",
-		"public/js/frappe/utils/number_format.js",
-		"public/js/frappe/utils/utils.js",
-		"public/js/frappe/utils/common.js",
-		"public/js/frappe/ui/messages.js",
-		"public/js/frappe/translate.js",
-		"public/js/frappe/utils/pretty_date.js",
-		"public/js/frappe/microtemplate.js",
-		"public/js/frappe/query_string.js",
-
-		"public/js/frappe/upload.js",
-
-		"public/js/frappe/model/meta.js",
-		"public/js/frappe/model/model.js",
-		"public/js/frappe/model/perm.js",
-
-		"website/js/website.js",
-		"public/js/frappe/socketio_client.js"
-	],
-	"js/bootstrap-4-web.min.js": "website/js/bootstrap-4.js",
-	"js/control.min.js": [
-		"node_modules/air-datepicker/dist/js/datepicker.min.js",
-		"node_modules/air-datepicker/dist/js/i18n/datepicker.cs.js",
-		"node_modules/air-datepicker/dist/js/i18n/datepicker.da.js",
-		"node_modules/air-datepicker/dist/js/i18n/datepicker.de.js",
-		"node_modules/air-datepicker/dist/js/i18n/datepicker.en.js",
-		"node_modules/air-datepicker/dist/js/i18n/datepicker.es.js",
-		"node_modules/air-datepicker/dist/js/i18n/datepicker.fi.js",
-		"node_modules/air-datepicker/dist/js/i18n/datepicker.fr.js",
-		"node_modules/air-datepicker/dist/js/i18n/datepicker.hu.js",
-		"node_modules/air-datepicker/dist/js/i18n/datepicker.nl.js",
-		"node_modules/air-datepicker/dist/js/i18n/datepicker.pl.js",
-		"node_modules/air-datepicker/dist/js/i18n/datepicker.pt-BR.js",
-		"node_modules/air-datepicker/dist/js/i18n/datepicker.pt.js",
-		"node_modules/air-datepicker/dist/js/i18n/datepicker.ro.js",
-		"node_modules/air-datepicker/dist/js/i18n/datepicker.sk.js",
-		"node_modules/air-datepicker/dist/js/i18n/datepicker.zh.js",
-		"public/js/frappe/ui/capture.js",
-		"public/js/frappe/form/controls/control.js"
-	],
-	"js/dialog.min.js": [
-		"public/js/frappe/dom.js",
-		"public/js/frappe/form/formatters.js",
-		"public/js/frappe/form/layout.js",
-		"public/js/frappe/ui/field_group.js",
-		"public/js/frappe/form/link_selector.js",
-		"public/js/frappe/form/multi_select_dialog.js",
-		"public/js/frappe/ui/dialog.js"
-	],
-	"css/desk.min.css": [
-		"public/js/lib/leaflet/leaflet.css",
-		"public/js/lib/leaflet/leaflet.draw.css",
-		"public/js/lib/leaflet/L.Control.Locate.css",
-		"public/js/lib/leaflet/easy-button.css",
-		"public/css/font-awesome.css",
-		"public/css/octicons/octicons.css",
-		"public/less/desk.less",
-		"public/less/module.less",
-		"public/less/mobile.less",
-		"public/less/controls.less",
-		"public/less/chat.less",
-		"public/css/fonts/inter/inter.css",
-		"node_modules/frappe-charts/dist/frappe-charts.min.css",
-		"node_modules/plyr/dist/plyr.css",
-		"public/scss/desk.scss"
-	],
-	"css/frappe-rtl.css": [
-		"public/css/bootstrap-rtl.css",
-		"public/css/desk-rtl.css",
-		"public/css/report-rtl.css"
-	],
-	"css/printview.css": [
-		"public/css/bootstrap.css",
-		"public/scss/print.scss"
-	],
-	"concat:js/libs.min.js": [
-		"public/js/lib/Sortable.min.js",
-		"public/js/lib/jquery/jquery.hotkeys.js",
-		"node_modules/bootstrap/dist/js/bootstrap.bundle.min.js",
-		"node_modules/vue/dist/vue.min.js",
-		"node_modules/moment/min/moment-with-locales.min.js",
-		"node_modules/moment-timezone/builds/moment-timezone-with-data.min.js",
-		"node_modules/socket.io-client/dist/socket.io.slim.js",
-		"node_modules/localforage/dist/localforage.min.js",
-		"public/js/lib/jSignature.min.js",
-		"public/js/lib/leaflet/leaflet.js",
-		"public/js/lib/leaflet/leaflet.draw.js",
-		"public/js/lib/leaflet/L.Control.Locate.js",
-		"public/js/lib/leaflet/easy-button.js"
-	],
-	"js/desk.min.js": [
-		"public/js/frappe/translate.js",
-		"public/js/frappe/class.js",
-		"public/js/frappe/polyfill.js",
-		"public/js/frappe/provide.js",
-		"public/js/frappe/assets.js",
-		"public/js/frappe/format.js",
-		"public/js/frappe/form/formatters.js",
-		"public/js/frappe/dom.js",
-		"public/js/frappe/ui/messages.js",
-		"public/js/frappe/ui/keyboard.js",
-		"public/js/frappe/ui/colors.js",
-		"public/js/frappe/ui/sidebar.js",
-		"public/js/frappe/ui/link_preview.js",
-
-		"public/js/frappe/request.js",
-		"public/js/frappe/socketio_client.js",
-		"public/js/frappe/utils/utils.js",
-		"public/js/frappe/event_emitter.js",
-		"public/js/frappe/router.js",
-		"public/js/frappe/router_history.js",
-		"public/js/frappe/defaults.js",
-		"public/js/frappe/roles_editor.js",
-		"public/js/frappe/module_editor.js",
-		"public/js/frappe/microtemplate.js",
-
-		"public/js/frappe/ui/page.html",
-		"public/js/frappe/ui/page.js",
-		"public/js/frappe/ui/slides.js",
-		"public/js/frappe/ui/onboarding_dialog.js",
-		"public/js/frappe/ui/find.js",
-		"public/js/frappe/ui/iconbar.js",
-		"public/js/frappe/form/layout.js",
-		"public/js/frappe/ui/field_group.js",
-		"public/js/frappe/form/link_selector.js",
-		"public/js/frappe/form/multi_select_dialog.js",
-		"public/js/frappe/ui/dialog.js",
-		"public/js/frappe/ui/capture.js",
-		"public/js/frappe/ui/app_icon.js",
-		"public/js/frappe/ui/theme_switcher.js",
-
-		"public/js/frappe/model/model.js",
-		"public/js/frappe/db.js",
-		"public/js/frappe/model/meta.js",
-		"public/js/frappe/model/sync.js",
-		"public/js/frappe/model/create_new.js",
-		"public/js/frappe/model/perm.js",
-		"public/js/frappe/model/workflow.js",
-		"public/js/frappe/model/user_settings.js",
-
-		"public/js/lib/md5.min.js",
-		"public/js/frappe/utils/user.js",
-		"public/js/frappe/utils/common.js",
-		"public/js/frappe/utils/urllib.js",
-		"public/js/frappe/utils/pretty_date.js",
-		"public/js/frappe/utils/tools.js",
-		"public/js/frappe/utils/datetime.js",
-		"public/js/frappe/utils/number_format.js",
-		"public/js/frappe/utils/help.js",
-		"public/js/frappe/utils/help_links.js",
-		"public/js/frappe/utils/address_and_contact.js",
-		"public/js/frappe/utils/preview_email.js",
-		"public/js/frappe/utils/file_manager.js",
-
-		"public/js/frappe/upload.js",
-		"public/js/frappe/ui/tree.js",
-
-		"public/js/frappe/views/container.js",
-		"public/js/frappe/views/breadcrumbs.js",
-		"public/js/frappe/views/factory.js",
-		"public/js/frappe/views/pageview.js",
-
-		"public/js/frappe/ui/toolbar/awesome_bar.js",
-		"public/js/frappe/ui/toolbar/energy_points_notifications.js",
-		"public/js/frappe/ui/notifications/notifications.js",
-		"public/js/frappe/ui/toolbar/search.js",
-		"public/js/frappe/ui/toolbar/tag_utils.js",
-		"public/js/frappe/ui/toolbar/search.html",
-		"public/js/frappe/ui/toolbar/search_utils.js",
-		"public/js/frappe/ui/toolbar/about.js",
-		"public/js/frappe/ui/toolbar/navbar.html",
-		"public/js/frappe/ui/toolbar/toolbar.js",
-		"public/js/frappe/ui/toolbar/notifications.js",
-		"public/js/frappe/views/communication.js",
-		"public/js/frappe/views/translation_manager.js",
-		"public/js/frappe/views/workspace/workspace.js",
-
-		"public/js/frappe/widgets/widget_group.js",
-
-		"public/js/frappe/ui/sort_selector.html",
-		"public/js/frappe/ui/sort_selector.js",
-
-		"public/js/frappe/change_log.html",
-		"public/js/frappe/ui/workspace_loading_skeleton.html",
-		"public/js/frappe/desk.js",
-		"public/js/frappe/query_string.js",
-
-		"public/js/frappe/ui/comment.js",
-
-		"public/js/frappe/chat.js",
-		"public/js/frappe/utils/energy_point_utils.js",
-		"public/js/frappe/utils/dashboard_utils.js",
-		"public/js/frappe/ui/chart.js",
-		"public/js/frappe/ui/datatable.js",
-		"public/js/frappe/ui/driver.js",
-		"public/js/frappe/ui/plyr.js",
-		"public/js/frappe/barcode_scanner/index.js"
-	],
-	"js/form.min.js": [
-		"public/js/frappe/form/templates/**.html",
-		"public/js/frappe/form/controls/control.js",
-		"public/js/frappe/views/formview.js",
-		"public/js/frappe/form/form.js",
-		"public/js/frappe/meta_tag.js"
-	],
-	"js/list.min.js": [
-		"public/js/frappe/ui/listing.html",
-
-		"public/js/frappe/model/indicator.js",
-		"public/js/frappe/ui/filters/filter.js",
-		"public/js/frappe/ui/filters/filter_list.js",
-		"public/js/frappe/ui/filters/field_select.js",
-		"public/js/frappe/ui/filters/edit_filter.html",
-		"public/js/frappe/ui/tags.js",
-		"public/js/frappe/ui/tag_editor.js",
-		"public/js/frappe/ui/like.js",
-		"public/js/frappe/ui/liked_by.html",
-		"public/html/print_template.html",
-
-		"public/js/frappe/list/base_list.js",
-		"public/js/frappe/list/list_view.js",
-		"public/js/frappe/list/list_factory.js",
-
-		"public/js/frappe/list/list_view_select.js",
-		"public/js/frappe/list/list_sidebar.js",
-		"public/js/frappe/list/list_sidebar.html",
-		"public/js/frappe/list/list_sidebar_stat.html",
-		"public/js/frappe/list/list_sidebar_group_by.js",
-		"public/js/frappe/list/list_view_permission_restrictions.html",
-
-		"public/js/frappe/views/gantt/gantt_view.js",
-		"public/js/frappe/views/calendar/calendar.js",
-		"public/js/frappe/views/dashboard/dashboard_view.js",
-		"public/js/frappe/views/image/image_view.js",
-		"public/js/frappe/views/map/map_view.js",
-		"public/js/frappe/views/kanban/kanban_view.js",
-		"public/js/frappe/views/inbox/inbox_view.js",
-		"public/js/frappe/views/file/file_view.js",
-
-		"public/js/frappe/views/treeview.js",
-		"public/js/frappe/views/interaction.js",
-
-		"public/js/frappe/views/image/image_view_item_row.html",
-		"public/js/frappe/views/image/photoswipe_dom.html",
-
-		"public/js/frappe/views/kanban/kanban_board.html",
-		"public/js/frappe/views/kanban/kanban_column.html",
-		"public/js/frappe/views/kanban/kanban_card.html"
-	],
-	"css/report.min.css": [
-		"node_modules/frappe-datatable/dist/frappe-datatable.css",
-		"public/css/tree_grid.css"
-	],
-	"js/report.min.js": [
-		"public/js/lib/clusterize.min.js",
-		"public/js/frappe/views/reports/report_factory.js",
-		"public/js/frappe/views/reports/report_view.js",
-		"public/js/frappe/views/reports/query_report.js",
-		"public/js/frappe/views/reports/print_grid.html",
-		"public/js/frappe/views/reports/print_tree.html",
-		"public/js/frappe/ui/group_by/group_by.html",
-		"public/js/frappe/ui/group_by/group_by.js",
-		"public/js/frappe/views/reports/report_utils.js"
-	],
-	"js/web_form.min.js": [
-		"public/js/frappe/utils/datetime.js",
-		"public/js/frappe/web_form/webform_script.js"
-	],
-	"css/web_form.css": [
-		"website/css/web_form.css",
-		"public/css/octicons/octicons.css",
-		"public/scss/controls.scss",
-		"node_modules/frappe-datatable/dist/frappe-datatable.css"
-	],
-	"css/email.css": "public/scss/email.scss",
-	"js/barcode_scanner.min.js": "public/js/frappe/barcode_scanner/quagga.js",
-	"js/user_profile_controller.min.js": "desk/page/user_profile/user_profile_controller.js",
-	"css/login.css": "public/scss/login.scss",
-	"js/data_import_tools.min.js": "public/js/frappe/data_import/index.js"
-}

frappe/public/css/desk-rtl.css

@@ -1,118 +0,0 @@
-.navbar .navbar-search-icon{
-  right: auto;
-  left: 24px;
-}
-.navbar > .container > .navbar-header{
-  float: right !important;
-}
-body[data-sidebar="0"] .navbar-home {
-  margin-left: auto !important;
-  margin-right: 15px !important;
-}
-.navbar-desk ~ ul > li {
-  float: right !important;
-}
-body.no-breadcrumbs .navbar .navbar-home:before {
-  margin-right: auto;
-  margin-left: 10px !important;
-  -ms-transform:rotate(180deg);     /* Internet Explorer 9 */
-  -webkit-transform:rotate(180deg); /* Chrome, Safari, Opera */
-  transform:rotate(180deg);         /* Standard syntax */
-}
-.layout-side-section .overlay-sidebar {
-  left: auto !important;
-  right: 0 !important;
-}
-.layout-side-section .overlay-sidebar.opened {
-  transform:translateX(0) !important;
-}
-.navbar-right {
-  float: left !important;
-}
-#navbar-breadcrumbs > li > a:before {
-  margin-right: auto;
-  margin-left: 10px;
-  top: 6px;
-  -ms-transform:rotate(180deg);     /* Internet Explorer 9 */
-  -webkit-transform:rotate(180deg); /* Chrome, Safari, Opera */
-  transform:rotate(180deg);         /* Standard syntax */
-}
-.case-wrapper {
-  float: right;
-}
-.link-btn {
-  right: auto;
-  left:  4px;
-  transform:rotate(180deg);         /* Rotate icon*/
-}
-.sidebar-menu .badge {
-  right: auto;
-  left: 0px;
-}
-.indicator::before {
-  margin: 0 0 0 4px;
-}
-.pull-left {
-  float: right !important;
-}
-.grid-row > .row .col:last-child {
-  margin-right: auto;
-  margin-left: -10px;
-}
-.text-right {
-  text-align: left;
-}
-.list-row-head .octicon-heart {
-  margin-right: auto;
-  margin-left: 13px;
-}
-.list-id {
-  margin-left: 7px !important;
-}
-.avatar-small .avatar-sm {
-  margin-left: 5px;
-  margin-right: auto;
-}
-.list-row-right .list-row-modified {
-  margin-right: auto;
-  margin-left: 9px;
-}
-.list-comment-count {
-  text-align: right;
-}
-ul.tree-children {
-  padding-right: 20px;
-  padding-left: inherit !important;
-}
-.balance-area {
-  float: left !important;
-}
-.tree.opened::before, .tree-node.opened::before, .tree:last-child::after, .tree-node:last-child::after {
-  left: inherit !important;
-  right: 8px;
-}
-.tree.opened > .tree-children > .tree-node > .tree-link::before, .tree-node.opened > .tree-children > .tree-node > .tree-link::before {
-  left: inherit !important;
-  right: -11px;
-}
-.tree:last-child::after, .tree-node:last-child::after {
-  right: -13px !important;
-}
-.tree.opened::before {
-  left: auto !important;
-  right: 23px;
-}
-.results {
-  direction: ltr;
-}
-.data-table {
-  direction: ltr;
-}
-.section-header {
-  direction: ltr;
-}
-
-.ql-editor {
-  direction: rtl;
-  text-align: right;
-}

frappe/public/css/report-rtl.css

@@ -1,15 +0,0 @@
-.grid-report {
-  direction: ltr;
-}
-
-.page-form .awesomplete > ul {
-    left: auto;
-}
-
-.chart_area{
-	direction: ltr;
-}
-
-.grid-report .show-zero{
-	direction: rtl;
-}

frappe/public/html/print_template.html

@@ -1,5 +1,5 @@
 <!DOCTYPE html>
-<html lang="en">
+<html lang="{{ lang }}" dir="{{ layout_direction }}">
 <head>
 	<meta charset="utf-8">
 	<meta http-equiv="X-UA-Compatible" content="IE=edge">
@@ -7,7 +7,7 @@
 	<meta name="description" content="">
 	<meta name="author" content="">
 	<title>{{ title }}</title>
-	<link href="{{ base_url }}{{ frappe.assets.bundled_asset('print.bundle.css') }}" rel="stylesheet">
+	<link href="{{ base_url }}{{ frappe.assets.bundled_asset('print.bundle.css', frappe.utils.is_rtl(lang)) }}" rel="stylesheet">
 	<style>
 		{{ print_css }}
 	</style>

frappe/public/js/frappe/assets.js

@@ -168,9 +168,13 @@ frappe.assets = {
 		}
 	},
 
-	bundled_asset(path) {
+	bundled_asset(path, is_rtl=null) {
 		if (!path.startsWith('/assets') && path.includes('.bundle.')) {
-			return frappe.boot.assets_json[path] || path;
+			if (path.endsWith('.css') && is_rtl) {
+				path = `rtl_${path}`;
+			}
+			path = frappe.boot.assets_json[path] || path;
+			return path;
 		}
 		return path;
 	}

frappe/public/js/frappe/desk.js

@@ -64,8 +64,6 @@ frappe.Application = class Application {
 			}
 		});
 
-		this.set_rtl();
-
 		// page container
 		this.make_page_container();
 		this.set_route();
@@ -489,16 +487,6 @@ frappe.Application = class Application {
 		}, 100);
 	}
 
-	set_rtl() {
-		if (frappe.utils.is_rtl()) {
-			var ls = document.createElement('link');
-			ls.rel="stylesheet";
-			ls.type = "text/css";
-			ls.href= frappe.assets.bundled_asset("frappe-rtl.bundle.css");
-			document.getElementsByTagName('head')[0].appendChild(ls);
-			$('body').addClass('frappe-rtl');
-		}
-	}
 
 	show_change_log() {
 		var me = this;

frappe/public/js/frappe/form/controls/signature.js

@@ -53,13 +53,15 @@ frappe.ui.form.ControlSignature = class ControlSignature extends frappe.ui.form.
 		this.img = $("<img class='img-responsive attach-image-display'>")
 			.appendTo(this.img_wrapper).toggle(false);
 	}
-	refresh_input(e) {
+	refresh_input() {
+		// signature dom is not ready
+		if (!this.body) return;
 		// prevent to load the second time
 		this.make_pad();
 		this.$wrapper.find(".control-input").toggle(false);
 		this.set_editable(this.get_status()=="Write");
 		this.load_pad();
-		if(this.get_status()=="Read") {
+		if (this.get_status() == "Read") {
 			$(this.disp_area).toggle(false);
 		}
 	}

frappe/public/js/frappe/form/dashboard.js

@@ -204,7 +204,7 @@ frappe.ui.form.Dashboard = class FormDashboard {
 				$(this).removeClass('hidden');
 			}
 		});
-		this.set_open_count();
+		!this.frm.is_new() && this.set_open_count();
 	}
 
 	init_data() {

frappe/public/js/frappe/form/form.js

@@ -1265,7 +1265,9 @@ frappe.ui.form.Form = class FrappeForm {
 		if (df && df[property] != value) {
 			df[property] = value;
 			if (table_field && table_row_name) {
-				this.fields_dict[fieldname].grid.grid_rows_by_docname[table_row_name].refresh_field(fieldname);
+				if (this.fields_dict[fieldname].grid.grid_rows_by_docname[table_row_name]) {
+					this.fields_dict[fieldname].grid.grid_rows_by_docname[table_row_name].refresh_field(fieldname);
+				}
 			} else {
 				this.refresh_field(fieldname);
 			}

frappe/public/js/frappe/form/form_tour.js

@@ -2,8 +2,6 @@ frappe.ui.form.FormTour = class FormTour {
 	constructor({ frm }) {
 		this.frm = frm;
 		this.driver_steps = [];
-
-		this.init_driver();
 	}
 
 	init_driver() {
@@ -37,11 +35,17 @@ frappe.ui.form.FormTour = class FormTour {
 		if (tour_name) {
 			this.tour = await frappe.db.get_doc('Form Tour', tour_name);
 		} else {
-			this.tour = { steps: frappe.tour[this.frm.doctype] };
+			const doctype_tour_exists = await frappe.db.exists('Form Tour', this.frm.doctype);
+			if (doctype_tour_exists) {
+				this.tour = await frappe.db.get_doc('Form Tour', this.frm.doctype);
+			} else {
+				this.tour = { steps: frappe.tour[this.frm.doctype] };
+			}
 		}
 		
 		if (on_finish) this.on_finish = on_finish;
 
+		this.init_driver();
 		this.build_steps();
 		this.update_driver_steps();
 	}
@@ -232,7 +236,7 @@ frappe.ui.form.FormTour = class FormTour {
 	}
 
 	add_step_to_save() {
-		const page_id = `#page-${this.frm.doctype}`;
+		const page_id = `[id="page-${this.frm.doctype}"]`;
 		const $save_btn = `${page_id} .standard-actions .primary-action`;
 		const save_step = {
 			element: $save_btn,
@@ -244,6 +248,9 @@ frappe.ui.form.FormTour = class FormTour {
 				description: "",
 				position: "left",
 				doneBtnText: __("Save")
+			},
+			onNext: () => {
+				this.frm.save();
 			}
 		};
 		this.driver_steps.push(save_step);

frappe/public/js/frappe/form/layout.js

@@ -263,9 +263,6 @@ frappe.ui.form.Layout = class Layout {
 				section.addClass("empty-section");
 			}
 		});
-
-		this.frm && this.frm.dashboard.refresh();
-
 	}
 
 	refresh_fields (fields) {

frappe/public/js/frappe/microtemplate.js

@@ -119,6 +119,10 @@ frappe.render_grid = function(opts) {
 	// render HTML wrapper page
 	opts.base_url = frappe.urllib.get_base_url();
 	opts.print_css = frappe.boot.print_css;
+
+	opts.lang = opts.lang || frappe.boot.lang,
+	opts.layout_direction = opts.layout_direction || frappe.utils.is_rtl() ? "rtl" : "ltr";
+
 	var html = frappe.render_template("print_template", opts);
 
 	var w = window.open();

frappe/public/js/frappe/router.js

@@ -14,11 +14,10 @@ frappe.view_factories = [];
 frappe.route_options = null;
 frappe.route_hooks = {};
 
-$(window).on('hashchange', function() {
+$(window).on('hashchange', function(e) {
 	// v1 style routing, route is in hash
-	if (window.location.hash) {
+	if (window.location.hash && !frappe.router.is_app_route(e.currentTarget.pathname)) {
 		let sub_path = frappe.router.get_sub_path(window.location.hash);
-		window.location.hash = '';
 		frappe.router.push_state(sub_path);
 		return false;
 	}
@@ -48,18 +47,18 @@ $('body').on('click', 'a', function(e) {
 		return;
 	}
 
-	if (href==='') {
+	if (href === '') {
 		return override('/app');
 	}
 
-	// target has "#" ,this is a v1 style route, so remake it.
-	if (e.currentTarget.hash) {
+	if (href.startsWith('#')) {
+		// target startswith "#", this is a v1 style route, so remake it.
 		return override(e.currentTarget.hash);
 	}
 
-	// target has "/app, this is a v2 style route.
-	if (e.currentTarget.pathname && frappe.router.is_app_route(e.currentTarget.pathname)) {
-		return override(e.currentTarget.pathname);
+	if (frappe.router.is_app_route(e.currentTarget.pathname)) {
+		// target has "/app, this is a v2 style route.
+		return override(e.currentTarget.pathname + e.currentTarget.hash);
 	}
 });
 
@@ -172,7 +171,7 @@ frappe.router = {
 			standard_route = ['List', doctype_route.doctype, frappe.utils.to_title_case(route[2])];
 			if (route[3]) {
 				// calendar / kanban / dashboard / folder name
-				standard_route.push(...route.splice(3, route.length));
+				standard_route.push([...route].splice(3, route.length));
 			}
 		}
 		return standard_route;
@@ -298,7 +297,7 @@ frappe.router = {
 				new_route = [this.slug(route[1]), 'view', route[2].toLowerCase()];
 
 				// calendar / inbox / file folder
-				if (route[3]) new_route.push(...route.slice(3, route.length));
+				if (route[3]) new_route.push([...route].slice(3, route.length));
 			} else {
 				if ($.isPlainObject(route[2])) {
 					frappe.route_options = route[2];
@@ -349,8 +348,6 @@ frappe.router = {
 	push_state(url) {
 		// change the URL and call the router
 		if (window.location.pathname !== url) {
-			// cleanup any remenants of v1 routing
-			window.location.hash = '';
 
 			// push state so the browser looks fine
 			history.pushState(null, null, url);
@@ -364,7 +361,11 @@ frappe.router = {
 		// return clean sub_path from hash or url
 		// supports both v1 and v2 routing
 		if (!route) {
-			route = window.location.hash || (window.location.pathname + window.location.search);
+			route = window.location.pathname + window.location.hash + window.location.search;
+			if (route.includes('app#')) {
+				// to support v1
+				route = window.location.hash;
+			}
 		}
 
 		return this.strip_prefix(route);

frappe/public/js/frappe/ui/field_group.js

@@ -40,14 +40,16 @@ frappe.ui.FieldGroup = class FieldGroup extends frappe.ui.form.Layout {
 				this.catch_enter_as_submit();
 			}
 
-			$(this.wrapper).find('input, select').on('change', () => {
-				this.dirty = true;
-
-				frappe.run_serially([
-					() => frappe.timeout(0.1),
-					() => me.refresh_dependency()
-				]);
-			});
+			$(this.wrapper).find('input, select').on(
+				'change awesomplete-selectcomplete',
+				() => {
+					this.dirty = true;
+					frappe.run_serially([
+						() => frappe.timeout(0.1),
+						() => me.refresh_dependency()
+					]);
+				}
+			);
 
 		}
 	}

frappe/public/js/frappe/views/reports/print_grid.html

@@ -11,6 +11,7 @@
 	<!-- heading -->
 	<thead>
 		<tr>
+		<th> # </th>
 		{% for col in columns %}
 			{% if col.name && col._id !== "_check" %}
 			<th
@@ -30,6 +31,9 @@
 	<tbody>
 		{% for row in data %}
 			<tr style="height: 30px">
+			<td {% if row.bold == 1 %} style="font-weight: bold" {% endif %}>
+				<span> {{ row._index + 1 }} </span>
+			</td>
 			{% for col in columns %}
 				{% if col.name && col._id !== "_check" %}
 

frappe/public/js/frappe/views/reports/query_report.js

@@ -1264,7 +1264,9 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList {
 			print_css: print_css,
 			print_settings: print_settings,
 			landscape: landscape,
-			columns: columns
+			columns: columns,
+			lang: frappe.boot.lang,
+			layout_direction: frappe.utils.is_rtl() ? "rtl" : "ltr"
 		});
 
 		frappe.render_pdf(html, print_settings);

frappe/public/scss/common/controls.scss

@@ -166,6 +166,9 @@ select.form-control {
 	.ace_print-margin {
 		background-color: var(--dark-border-color);
 	}
+	.ace_scrollbar {
+		z-index: 3;
+	}
 }
 
 .frappe-control[data-fieldtype="Attach"],

frappe/public/scss/common/css_variables.scss

@@ -179,9 +179,20 @@
 	--text-on-pink: var(--pink-500);
 	--text-on-cyan: var(--cyan-600);
 
+	// Layout Colors
+	--bg-color: var(--gray-50);
+	--fg-color: white;
+	--navbar-bg: white;
+	--fg-hover-color: var(--gray-100);
+	--card-bg: var(--fg-color);
+	--disabled-text-color: var(--gray-700);
 	--disabled-control-bg: var(--gray-50);
 	--control-bg: var(--gray-100);
 	--control-bg-on-gray: var(--gray-200);
+	--awesomebar-focus-bg: var(--fg-color);
+	--modal-bg: white;
+	--toast-bg: var(--modal-bg);
+	--popover-bg: white;
 
 	--awesomplete-hover-bg: var(--control-bg);