From e369919d6517de225576da09b65fa9841c0c2b34 Mon Sep 17 00:00:00 2001 From: hasnain2808 Date: Thu, 17 Jun 2021 13:51:42 +0530 Subject: [PATCH 001/164] fix: add route building as a background job --- frappe/migrate.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/frappe/migrate.py b/frappe/migrate.py index d19e255639..cdaf75af9a 100644 --- a/frappe/migrate.py +++ b/frappe/migrate.py @@ -11,6 +11,7 @@ import frappe.model.sync from frappe.utils.fixtures import sync_fixtures from frappe.utils.connections import check_connection from frappe.utils.dashboard import sync_dashboards +from frappe.utils.background_jobs import enqueue from frappe.cache_manager import clear_global_cache from frappe.desk.notifications import clear_notifications from frappe.website import render @@ -86,18 +87,19 @@ Otherwise, check the server logs and ensure that all the required services are r for fn in frappe.get_hooks('after_migrate', app_name=app): frappe.get_attr(fn)() - # build web_routes index - if not skip_search_index: - # Run this last as it updates the current session - print('Building search index for {}'.format(frappe.local.site)) - build_index_for_all_routes() - frappe.db.commit() clear_notifications() frappe.publish_realtime("version-update") frappe.flags.in_migrate = False + + # build web_routes index + if not skip_search_index: + # Run this last as it updates the current session + print('Queuing search index build for {}'.format(frappe.local.site)) + enqueue(method=build_index_for_all_routes, job_name='Search index build for {}'.format(frappe.local.site), now=0) + finally: with open(touched_tables_file, 'w') as f: json.dump(list(frappe.flags.touched_tables), f, sort_keys=True, indent=4) From 292d7f914cd4de1f306ac6299a0e029a672c6a2f Mon Sep 17 00:00:00 2001 From: hasnain2808 Date: Thu, 17 Jun 2021 15:52:39 +0530 Subject: [PATCH 002/164] fix: be explicit with timeout and queue --- frappe/migrate.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/frappe/migrate.py b/frappe/migrate.py index cdaf75af9a..5885c05039 100644 --- a/frappe/migrate.py +++ b/frappe/migrate.py @@ -4,6 +4,8 @@ import json import os import sys + +from rq import queue import frappe import frappe.translate import frappe.modules.patch_handler @@ -98,7 +100,13 @@ Otherwise, check the server logs and ensure that all the required services are r if not skip_search_index: # Run this last as it updates the current session print('Queuing search index build for {}'.format(frappe.local.site)) - enqueue(method=build_index_for_all_routes, job_name='Search index build for {}'.format(frappe.local.site), now=0) + enqueue( + method=build_index_for_all_routes, + job_name='Search index build for {}'.format(frappe.local.site), + now=0, + queue='background', + timeout=10000 + ) finally: with open(touched_tables_file, 'w') as f: From 9bea81e30cdc59a645993fc538fdf1321de88cf2 Mon Sep 17 00:00:00 2001 From: hasnain2808 Date: Fri, 18 Jun 2021 00:12:07 +0530 Subject: [PATCH 003/164] feat: website_search_field in doctype doctype website_search_field to declare the content field remove the background job execution of search indexing --- frappe/core/doctype/doctype/doctype.json | 11 +++++++++-- frappe/migrate.py | 18 ++++++------------ frappe/search/website_search.py | 18 ++++++++++++------ 3 files changed, 27 insertions(+), 20 deletions(-) diff --git a/frappe/core/doctype/doctype/doctype.json b/frappe/core/doctype/doctype/doctype.json index 7f93d3130a..6a427f71e1 100644 --- a/frappe/core/doctype/doctype/doctype.json +++ b/frappe/core/doctype/doctype/doctype.json @@ -76,6 +76,7 @@ "index_web_pages_for_search", "route", "is_published_field", + "website_search_field", "advanced", "engine" ], @@ -547,6 +548,12 @@ { "fieldname": "column_break_51", "fieldtype": "Column Break" + }, + { + "depends_on": "has_web_view", + "fieldname": "website_search_field", + "fieldtype": "Data", + "label": "Website Search Field" } ], "icon": "fa fa-bolt", @@ -628,7 +635,7 @@ "link_fieldname": "reference_doctype" } ], - "modified": "2021-04-16 12:26:41.031135", + "modified": "2021-06-17 23:31:44.974199", "modified_by": "Administrator", "module": "Core", "name": "DocType", @@ -662,4 +669,4 @@ "sort_field": "modified", "sort_order": "DESC", "track_changes": 1 -} +} \ No newline at end of file diff --git a/frappe/migrate.py b/frappe/migrate.py index 5885c05039..c984371927 100644 --- a/frappe/migrate.py +++ b/frappe/migrate.py @@ -89,6 +89,12 @@ Otherwise, check the server logs and ensure that all the required services are r for fn in frappe.get_hooks('after_migrate', app_name=app): frappe.get_attr(fn)() + # build web_routes index + if not skip_search_index: + # Run this last as it updates the current session + print('Building search index for {}'.format(frappe.local.site)) + build_index_for_all_routes() + frappe.db.commit() clear_notifications() @@ -96,18 +102,6 @@ Otherwise, check the server logs and ensure that all the required services are r frappe.publish_realtime("version-update") frappe.flags.in_migrate = False - # build web_routes index - if not skip_search_index: - # Run this last as it updates the current session - print('Queuing search index build for {}'.format(frappe.local.site)) - enqueue( - method=build_index_for_all_routes, - job_name='Search index build for {}'.format(frappe.local.site), - now=0, - queue='background', - timeout=10000 - ) - finally: with open(touched_tables_file, 'w') as f: json.dump(list(frappe.flags.touched_tables), f, sort_keys=True, indent=4) diff --git a/frappe/search/website_search.py b/frappe/search/website_search.py index 452ea2a427..10764d49d4 100644 --- a/frappe/search/website_search.py +++ b/frappe/search/website_search.py @@ -35,10 +35,12 @@ class WebsiteSearch(FullTextSearch): if getattr(self, "_items_to_index", False): return self._items_to_index - routes = get_static_pages_from_all_apps() + slugs_with_web_view() - self._items_to_index = [] + + routes = get_static_pages_from_all_apps() + slugs_with_web_view(self._items_to_index ) + + for i, route in enumerate(routes): update_progress_bar("Retrieving Routes", i, len(routes)) self._items_to_index += [self.get_document_to_index(route)] @@ -85,16 +87,20 @@ class WebsiteSearch(FullTextSearch): ) -def slugs_with_web_view(): +def slugs_with_web_view(_items_to_index): all_routes = [] filters = { "has_web_view": 1, "allow_guest_to_view": 1, "index_web_pages_for_search": 1} - fields = ["name", "is_published_field"] + fields = ["name", "is_published_field", 'website_search_field'] doctype_with_web_views = frappe.get_all("DocType", filters=filters, fields=fields) for doctype in doctype_with_web_views: if doctype.is_published_field: - routes = frappe.get_all(doctype.name, filters={doctype.is_published_field: 1}, fields="route") - all_routes += [route.route for route in routes] + docs = frappe.get_all(doctype.name, filters={doctype.is_published_field: 1}, fields=["route", doctype.website_search_field]) + if doctype.website_search_field: + for doc in docs: + _items_to_index += [frappe._dict(title=doc.title, content=getattr(doc, doctype.website_search_field), path=doc.route)] + else: + all_routes += [route.route for route in docs] return all_routes From e71ae4b3feda96a591593c612b769bcd78b02985 Mon Sep 17 00:00:00 2001 From: hasnain2808 Date: Fri, 18 Jun 2021 00:25:31 +0530 Subject: [PATCH 004/164] fix: title needed too --- frappe/search/website_search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/search/website_search.py b/frappe/search/website_search.py index 10764d49d4..41d51f6b69 100644 --- a/frappe/search/website_search.py +++ b/frappe/search/website_search.py @@ -95,7 +95,7 @@ def slugs_with_web_view(_items_to_index): for doctype in doctype_with_web_views: if doctype.is_published_field: - docs = frappe.get_all(doctype.name, filters={doctype.is_published_field: 1}, fields=["route", doctype.website_search_field]) + docs = frappe.get_all(doctype.name, filters={doctype.is_published_field: 1}, fields=["route", doctype.website_search_field, 'title']) if doctype.website_search_field: for doc in docs: _items_to_index += [frappe._dict(title=doc.title, content=getattr(doc, doctype.website_search_field), path=doc.route)] From c7d7c54e2c66385511457ad4d48a145e7f3a0902 Mon Sep 17 00:00:00 2001 From: hasnain2808 Date: Fri, 18 Jun 2021 11:06:52 +0530 Subject: [PATCH 005/164] feat: better indexing --- frappe/search/website_search.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/frappe/search/website_search.py b/frappe/search/website_search.py index 41d51f6b69..566c0f6a6d 100644 --- a/frappe/search/website_search.py +++ b/frappe/search/website_search.py @@ -98,7 +98,10 @@ def slugs_with_web_view(_items_to_index): docs = frappe.get_all(doctype.name, filters={doctype.is_published_field: 1}, fields=["route", doctype.website_search_field, 'title']) if doctype.website_search_field: for doc in docs: - _items_to_index += [frappe._dict(title=doc.title, content=getattr(doc, doctype.website_search_field), path=doc.route)] + content = frappe.utils.md_to_html(getattr(doc, doctype.website_search_field)) + soup = BeautifulSoup(content, "html.parser") + text_content = soup.text if soup else "" + _items_to_index += [frappe._dict(title=doc.title, content=text_content, path=doc.route)] else: all_routes += [route.route for route in docs] From aa6520b5b010ac777b0eb054b2fc5752649fcd0f Mon Sep 17 00:00:00 2001 From: codescientist703 Date: Fri, 18 Jun 2021 13:53:57 +0530 Subject: [PATCH 006/164] fix: add sorting for paren tree doctype to appear above child --- frappe/desk/search.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/frappe/desk/search.py b/frappe/desk/search.py index 040a8c2118..be62b9699d 100644 --- a/frappe/desk/search.py +++ b/frappe/desk/search.py @@ -167,6 +167,9 @@ def search_widget(doctype, txt, query=None, searchfield=None, start=0, as_list=not as_dict, strict=False) + # Bring Parent doctype to top and child doctype below it using sorting + values = sorted(values, key=lambda x: ((x[1] is not None), (x[0]))) + if doctype in UNTRANSLATED_DOCTYPES: values = tuple([v for v in list(values) if re.search(re.escape(txt)+".*", (_(v.name) if as_dict else _(v[0])), re.IGNORECASE)]) From 85bbcaaac41bbbf08b0475596e17479fb60b770d Mon Sep 17 00:00:00 2001 From: codescientist703 Date: Wed, 23 Jun 2021 13:13:51 +0530 Subject: [PATCH 007/164] fix: added sorting and filtering logic for relevant input results --- frappe/desk/search.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/frappe/desk/search.py b/frappe/desk/search.py index be62b9699d..64ae7f7b7a 100644 --- a/frappe/desk/search.py +++ b/frappe/desk/search.py @@ -167,11 +167,20 @@ def search_widget(doctype, txt, query=None, searchfield=None, start=0, as_list=not as_dict, strict=False) - # Bring Parent doctype to top and child doctype below it using sorting - values = sorted(values, key=lambda x: ((x[1] is not None), (x[0]))) + # Filtering the values array so that query is included in very element + values = tuple( + [ + v for v in list(values) + if re.search( + re.escape(txt) + ".*", (_(v.name) if as_dict else _(v[0])), re.IGNORECASE + ) + ] + ) - if doctype in UNTRANSLATED_DOCTYPES: - values = tuple([v for v in list(values) if re.search(re.escape(txt)+".*", (_(v.name) if as_dict else _(v[0])), re.IGNORECASE)]) + # Sorting the values array so that relevant results always come first + # This will first bring elements on top in which query is a prefix of element + # Then it will bring the rest of the elements and sort them in lexicographical order + values = sorted(values, key=lambda x: sorting_comparator(x, txt, as_dict)) # remove _relevance from results if as_dict: @@ -211,6 +220,13 @@ def scrub_custom_query(query, key, txt): query = query.replace('%s', ((txt or '') + '%')) return query +def sorting_comparator(key, query, as_dict): + value = (_(key.name) if as_dict else _(key[0])) + return ( + value.lower().startswith(query.lower()) is False, + value + ) + @wrapt.decorator def validate_and_sanitize_search_inputs(fn, instance, args, kwargs): kwargs.update(dict(zip(fn.__code__.co_varnames, args))) From 0c8e72d0f0d9fc31e0f98b6c9d597865cc0fa810 Mon Sep 17 00:00:00 2001 From: hasnain2808 Date: Thu, 24 Jun 2021 19:07:48 +0530 Subject: [PATCH 008/164] fix: cache document page --- frappe/website/page_renderers/document_page.py | 12 +++++++++--- frappe/website/page_renderers/template_page.py | 2 +- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/frappe/website/page_renderers/document_page.py b/frappe/website/page_renderers/document_page.py index f1741c681f..0108a84f79 100644 --- a/frappe/website/page_renderers/document_page.py +++ b/frappe/website/page_renderers/document_page.py @@ -1,7 +1,7 @@ import frappe from frappe.model.document import get_controller from frappe.website.page_renderers.base_template_page import BaseTemplatePage -from frappe.website.utils import build_response +from frappe.website.utils import build_response, cache_html from frappe.website.router import (get_doctypes_with_web_view, get_page_info_from_web_page_with_dynamic_routes) @@ -47,14 +47,20 @@ class DocumentPage(BaseTemplatePage): return False def render(self): + html = self.get_html() + html = self.add_csrf_token(html) + + return build_response(self.path, html, self.http_status_code or 200, self.headers) + + @cache_html + def get_html(self): self.doc = frappe.get_doc(self.doctype, self.docname) self.init_context() self.update_context() self.post_process_context() html = frappe.get_template(self.template_path).render(self.context) - html = self.add_csrf_token(html) - return build_response(self.path, html, self.http_status_code or 200, self.headers) + return html def update_context(self): self.context.doc = self.doc diff --git a/frappe/website/page_renderers/template_page.py b/frappe/website/page_renderers/template_page.py index 5e6e57e33a..2ec3f51f41 100644 --- a/frappe/website/page_renderers/template_page.py +++ b/frappe/website/page_renderers/template_page.py @@ -83,7 +83,7 @@ class TemplatePage(BaseTemplatePage): super(TemplatePage, self).post_process_context() def add_sidebar_and_breadcrumbs(self): - if self.basepath: + if self.basepath and not self.context.sidebar_items: self.context.sidebar_items = get_sidebar_items(self.context.website_sidebar, self.basepath) if self.context.add_breadcrumbs and not self.context.parents: From 40dd5227964b4d138e6731cb2cabc2cb5a7450f0 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Fri, 25 Jun 2021 13:06:09 +0530 Subject: [PATCH 009/164] feat: Auto generate RTL styles using rtlcss https://github.com/MohammadYounes/rtlcss --- esbuild/esbuild.js | 25 ++++++++++- frappe/public/js/frappe/desk.js | 12 ------ frappe/utils/jinja_globals.py | 11 ++++- frappe/www/app.html | 2 +- frappe/www/app.py | 3 ++ frappe/www/printview.html | 3 -- package.json | 1 + yarn.lock | 74 +++++++++++++++++++++++++++++++++ 8 files changed, 113 insertions(+), 18 deletions(-) diff --git a/esbuild/esbuild.js b/esbuild/esbuild.js index efa1959969..a0b07abe43 100644 --- a/esbuild/esbuild.js +++ b/esbuild/esbuild.js @@ -8,6 +8,7 @@ let yargs = require("yargs"); let cliui = require("cliui")(); let chalk = require("chalk"); let html_plugin = require("./frappe-html"); +let rtlcss = require('rtlcss'); let postCssPlugin = require("esbuild-plugin-postcss2").default; let ignore_assets = require("./ignore-assets"); let sass_options = require("./sass_options"); @@ -99,6 +100,7 @@ async function execute() { let result; try { result = await build_assets_for_apps(APPS, FILES_TO_BUILD); + result = await create_rtl_assets(result); } catch (e) { log_error("There were some problems during build"); log(); @@ -260,7 +262,8 @@ async function clean_dist_folders(apps) { let public_path = get_public_path(app); let paths = [ path.resolve(public_path, "dist", "js"), - path.resolve(public_path, "dist", "css") + path.resolve(public_path, "dist", "css"), + path.resolve(public_path, "dist", "css-rtl") ]; for (let target of paths) { if (fs.existsSync(target)) { @@ -484,3 +487,23 @@ function log_rebuilt_assets(prev_assets, new_assets) { } log(); } + +async function create_rtl_assets(result) { + for (let file_path in result.metafile.outputs) { + if (file_path.endsWith('.css')) { + console.log(file_path); + let content = fs.readFileSync(file_path, {'encoding': 'utf-8'}); + let rtl_content = rtlcss.process(content); + let rtl_file_path = file_path.replace('/css/', '/css-rtl/'); + let rtl_folder_path = path.dirname(rtl_file_path); + if (fs.existsSync(rtl_file_path)) { + continue; + } + if (!fs.existsSync(rtl_folder_path)) { + fs.mkdirSync(rtl_folder_path); + } + fs.writeFileSync(rtl_file_path, rtl_content); + } + } + return result; +} \ No newline at end of file diff --git a/frappe/public/js/frappe/desk.js b/frappe/public/js/frappe/desk.js index 65c0139b65..9d106f46f4 100644 --- a/frappe/public/js/frappe/desk.js +++ b/frappe/public/js/frappe/desk.js @@ -64,8 +64,6 @@ frappe.Application = class Application { } }); - this.set_rtl(); - // page container this.make_page_container(); this.set_route(); @@ -489,16 +487,6 @@ frappe.Application = class Application { }, 100); } - set_rtl() { - if (frappe.utils.is_rtl()) { - var ls = document.createElement('link'); - ls.rel="stylesheet"; - ls.type = "text/css"; - ls.href= frappe.assets.bundled_asset("frappe-rtl.bundle.css"); - document.getElementsByTagName('head')[0].appendChild(ls); - $('body').addClass('frappe-rtl'); - } - } show_change_log() { var me = this; diff --git a/frappe/utils/jinja_globals.py b/frappe/utils/jinja_globals.py index 2c14249672..30a0e46232 100644 --- a/frappe/utils/jinja_globals.py +++ b/frappe/utils/jinja_globals.py @@ -73,8 +73,13 @@ def include_script(path): return f'' -def include_style(path): +def include_style(path, rtl=None): path = bundled_asset(path) + if rtl is None: + rtl = is_rtl() + + if rtl: + path = path.replace('/css/', '/css-rtl/') return f'' @@ -87,3 +92,7 @@ def bundled_asset(path): path = bundled_assets.get(path) or path return abs_url(path) + +def is_rtl(): + from frappe import local + return local.lang in ["ar", "he", "fa", "ps"] \ No newline at end of file diff --git a/frappe/www/app.html b/frappe/www/app.html index c8172693b9..8704ed2671 100644 --- a/frappe/www/app.html +++ b/frappe/www/app.html @@ -1,5 +1,5 @@ - + diff --git a/frappe/www/app.py b/frappe/www/app.py index 27505c8131..55992b5a55 100644 --- a/frappe/www/app.py +++ b/frappe/www/app.py @@ -6,6 +6,7 @@ import os, re import frappe from frappe import _ import frappe.sessions +from frappe.utils.jinja_globals import is_rtl def get_context(context): if frappe.session.user == "Guest": @@ -40,6 +41,8 @@ def get_context(context): "build_version": frappe.utils.get_build_version(), "include_js": hooks["app_include_js"], "include_css": hooks["app_include_css"], + "dir": "rtl" if is_rtl() else "ltr", + "lang": frappe.local.lang, "sounds": hooks["sounds"], "boot": boot if context.get("for_mobile") else boot_json, "desk_theme": desk_theme or "Light", diff --git a/frappe/www/printview.html b/frappe/www/printview.html index 8bc6e0cb80..05e85730dd 100644 --- a/frappe/www/printview.html +++ b/frappe/www/printview.html @@ -6,9 +6,6 @@ {{ title }} {{ include_style('print.bundle.css') }} - {%- if has_rtl -%} - {{ include_style('frappe-rtl.bundle.css') }} - {%- endif -%} diff --git a/package.json b/package.json index 8b22a6e92c..e84fa1b581 100644 --- a/package.json +++ b/package.json @@ -67,6 +67,7 @@ "fast-glob": "^3.2.5", "launch-editor": "^2.2.1", "md5": "^2.3.0", + "rtlcss": "^3.2.1", "yargs": "^16.2.0" }, "snyk": true diff --git a/yarn.lock b/yarn.lock index 7b1fb981dd..d031f58685 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2420,6 +2420,14 @@ find-up@^3.0.0: dependencies: locate-path "^3.0.0" +find-up@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/find-up/-/find-up-5.0.0.tgz#4c92819ecb7083561e4f4a240a86be5198f536fc" + integrity sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng== + dependencies: + locate-path "^6.0.0" + path-exists "^4.0.0" + follow-redirects@^1.10.0: version "1.13.3" resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.13.3.tgz#e5598ad50174c1bc4e872301e82ac2cd97f90267" @@ -3697,6 +3705,13 @@ locate-path@^3.0.0: p-locate "^3.0.0" path-exists "^3.0.0" +locate-path@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-6.0.0.tgz#55321eb309febbc59c4801d931a72452a681d286" + integrity sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw== + dependencies: + p-locate "^5.0.0" + lodash.assign@^4.2.0: version "4.2.0" resolved "https://registry.yarnpkg.com/lodash.assign/-/lodash.assign-4.2.0.tgz#0d99f3ccd7a6d261d19bdaeb9245005d285808e7" @@ -4234,6 +4249,11 @@ nanoid@^3.1.22: resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.1.22.tgz#b35f8fb7d151990a8aebd5aa5015c03cf726f844" integrity sha512-/2ZUaJX2ANuLtTvqTlgqBQNJoQO398KyJgZloL0PZkC0dpysjncRUPsFe3DUPzz/y3h+u7C46np8RMuvF3jsSQ== +nanoid@^3.1.23: + version "3.1.23" + resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.1.23.tgz#f744086ce7c2bc47ee0a8472574d5c78e4183a81" + integrity sha512-FiB0kzdP0FFVGDKlRLEQ1BgDzU87dy5NnzjeW9YZNt+/c3+q82EQDUwniSAUxp/F0gFNI1ZhKU1FqYsMuqZVnw== + native-request@^1.0.5: version "1.0.8" resolved "https://registry.yarnpkg.com/native-request/-/native-request-1.0.8.tgz#8f66bf606e0f7ea27c0e5995eb2f5d03e33ae6fb" @@ -4583,6 +4603,13 @@ p-limit@^2.0.0: dependencies: p-try "^2.0.0" +p-limit@^3.0.2: + version "3.1.0" + resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-3.1.0.tgz#e1daccbe78d0d1388ca18c64fea38e3e57e3706b" + integrity sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ== + dependencies: + yocto-queue "^0.1.0" + p-locate@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-3.0.0.tgz#322d69a05c0264b25997d9f40cd8a891ab0064a4" @@ -4590,6 +4617,13 @@ p-locate@^3.0.0: dependencies: p-limit "^2.0.0" +p-locate@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-5.0.0.tgz#83c8315c6785005e3bd021839411c9e110e6d834" + integrity sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw== + dependencies: + p-limit "^3.0.2" + p-map@2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/p-map/-/p-map-2.1.0.tgz#310928feef9c9ecc65b68b17693018a665cea175" @@ -4712,6 +4746,11 @@ path-exists@^3.0.0: resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-3.0.0.tgz#ce0ebeaa5f78cb18925ea7d810d7b59b010fd515" integrity sha1-zg6+ql94yxiSXqfYENe1mwEP1RU= +path-exists@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-4.0.0.tgz#513bdbe2d3b95d7762e8c1137efa195c6c61b5b3" + integrity sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w== + path-is-absolute@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f" @@ -5170,6 +5209,15 @@ postcss@^7.0.32: source-map "^0.6.1" supports-color "^6.1.0" +postcss@^8.2.4: + version "8.3.5" + resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.3.5.tgz#982216b113412bc20a86289e91eb994952a5b709" + integrity sha512-NxTuJocUhYGsMiMFHDUkmjSKT3EdH4/WbGF6GCi1NDGk+vbcUTun4fpbOqaPtD8IIsztA2ilZm2DhYCuyN58gA== + dependencies: + colorette "^1.2.2" + nanoid "^3.1.23" + source-map-js "^0.6.2" + prepend-http@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/prepend-http/-/prepend-http-2.0.0.tgz#e92434bfa5ea8c19f41cdfd401d741a3c819d897" @@ -5852,6 +5900,17 @@ roarr@^2.15.3: semver-compare "^1.0.0" sprintf-js "^1.1.2" +rtlcss@^3.2.1: + version "3.2.1" + resolved "https://registry.yarnpkg.com/rtlcss/-/rtlcss-3.2.1.tgz#654e55ea2f46991f9738d952ba77ba0aa94a670d" + integrity sha512-S9bh35JXwPIhfun7nFu/HjlNrwELL5nvTJqA1suLfbnqY/mauIL5sBkrJNHziVppX9PA2rJ7NV82+RtzB71mJA== + dependencies: + chalk "^4.1.0" + find-up "^5.0.0" + mkdirp "^1.0.4" + postcss "^8.2.4" + strip-json-comments "^3.1.1" + run-async@^2.4.0: version "2.4.1" resolved "https://registry.yarnpkg.com/run-async/-/run-async-2.4.1.tgz#8440eccf99ea3e70bd409d49aab88e10c189a455" @@ -6459,6 +6518,11 @@ sortablejs@^1.7.0: resolved "https://registry.yarnpkg.com/sortablejs/-/sortablejs-1.8.3.tgz#5ae908ef96300966e95440a143340f5dd565a0df" integrity sha512-AftvD4hdKcR5QlGi7L/JST506zGNGrysE8/QohDpwKXJarHWqCt+TUlrtoMk/wkECB607Q019/OZlJViyWiD6A== +source-map-js@^0.6.2: + version "0.6.2" + resolved "https://registry.yarnpkg.com/source-map-js/-/source-map-js-0.6.2.tgz#0bb5de631b41cfbda6cfba8bd05a80efdfd2385e" + integrity sha512-/3GptzWzu0+0MBQFrDKzw/DvvMTUORvgY6k6jd/VS6iCR4RDTKWH6v6WPwQoUO8667uQEf9Oe38DxAYWY5F/Ug== + source-map-resolve@^0.5.2: version "0.5.3" resolved "https://registry.yarnpkg.com/source-map-resolve/-/source-map-resolve-0.5.3.tgz#190866bece7553e1f8f267a2ee82c606b5509a1a" @@ -6756,6 +6820,11 @@ strip-indent@^1.0.1: dependencies: get-stdin "^4.0.1" +strip-json-comments@^3.1.1: + version "3.1.1" + resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-3.1.1.tgz#31f1281b3832630434831c310c01cccda8cbe006" + integrity sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig== + strip-json-comments@~2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-2.0.1.tgz#3c531942e908c2697c0ec344858c286c7ca0a60a" @@ -7568,3 +7637,8 @@ yeast@0.1.2: version "0.1.2" resolved "https://registry.yarnpkg.com/yeast/-/yeast-0.1.2.tgz#008e06d8094320c372dbc2f8ed76a0ca6c8ac419" integrity sha1-AI4G2AlDIMNy28L47XagymyKxBk= + +yocto-queue@^0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-0.1.0.tgz#0294eb3dee05028d31ee1a5fa2c556a6aaf10a1b" + integrity sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q== From 63df7f850e48bdc1d7e08dc4e2ef38a22b7b7d0b Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Sun, 27 Jun 2021 08:54:32 +0530 Subject: [PATCH 010/164] refactor: Remove RTL style files --- frappe/public/css/bootstrap-rtl.css | 1476 --------------------- frappe/public/css/desk-rtl.css | 118 -- frappe/public/css/report-rtl.css | 15 - frappe/public/scss/frappe-rtl.bundle.scss | 3 - 4 files changed, 1612 deletions(-) delete mode 100644 frappe/public/css/bootstrap-rtl.css delete mode 100644 frappe/public/css/desk-rtl.css delete mode 100644 frappe/public/css/report-rtl.css delete mode 100644 frappe/public/scss/frappe-rtl.bundle.scss diff --git a/frappe/public/css/bootstrap-rtl.css b/frappe/public/css/bootstrap-rtl.css deleted file mode 100644 index 5dfa46c055..0000000000 --- a/frappe/public/css/bootstrap-rtl.css +++ /dev/null @@ -1,1476 +0,0 @@ -/******************************************************************************* - * bootstrap-rtl (version 3.3.4) - * Author: Morteza Ansarinia (http://github.com/morteza) - * Created on: August 13,2015 - * Project: bootstrap-rtl - * Copyright: Unlicensed Public Domain - *******************************************************************************/ - -html { - direction: rtl; -} -body { - direction: rtl; -} -.flip.text-left { - text-align: right; -} -.flip.text-right { - text-align: left; -} -.list-unstyled { - padding-right: 0; - padding-left: initial; -} -.list-inline { - padding-right: 0; - padding-left: initial; - margin-right: -5px; - margin-left: 0; -} -dd { - margin-right: 0; - margin-left: initial; -} -@media (min-width: 768px) { - .dl-horizontal dt { - float: right; - clear: right; - text-align: left; - } - .dl-horizontal dd { - margin-right: 180px; - margin-left: 0; - } -} -blockquote { - border-right: 5px solid #eeeeee; - border-left: 0; -} -.blockquote-reverse, -blockquote.pull-left { - padding-left: 15px; - padding-right: 0; - border-left: 5px solid #eeeeee; - border-right: 0; - text-align: left; -} -.col-xs-1, .col-sm-1, .col-md-1, .col-lg-1, .col-xs-2, .col-sm-2, .col-md-2, .col-lg-2, .col-xs-3, .col-sm-3, .col-md-3, .col-lg-3, .col-xs-4, .col-sm-4, .col-md-4, .col-lg-4, .col-xs-5, .col-sm-5, .col-md-5, .col-lg-5, .col-xs-6, .col-sm-6, .col-md-6, .col-lg-6, .col-xs-7, .col-sm-7, .col-md-7, .col-lg-7, .col-xs-8, .col-sm-8, .col-md-8, .col-lg-8, .col-xs-9, .col-sm-9, .col-md-9, .col-lg-9, .col-xs-10, .col-sm-10, .col-md-10, .col-lg-10, .col-xs-11, .col-sm-11, .col-md-11, .col-lg-11, .col-xs-12, .col-sm-12, .col-md-12, .col-lg-12 { - position: relative; - min-height: 1px; - padding-left: 15px; - padding-right: 15px; -} -.col-xs-1, .col-xs-2, .col-xs-3, .col-xs-4, .col-xs-5, .col-xs-6, .col-xs-7, .col-xs-8, .col-xs-9, .col-xs-10, .col-xs-11, .col-xs-12 { - float: right; -} -.col-xs-12 { - width: 100%; -} -.col-xs-11 { - width: 91.66666667%; -} -.col-xs-10 { - width: 83.33333333%; -} -.col-xs-9 { - width: 75%; -} -.col-xs-8 { - width: 66.66666667%; -} -.col-xs-7 { - width: 58.33333333%; -} -.col-xs-6 { - width: 50%; -} -.col-xs-5 { - width: 41.66666667%; -} -.col-xs-4 { - width: 33.33333333%; -} -.col-xs-3 { - width: 25%; -} -.col-xs-2 { - width: 16.66666667%; -} -.col-xs-1 { - width: 8.33333333%; -} -.col-xs-pull-12 { - left: 100%; - right: auto; -} -.col-xs-pull-11 { - left: 91.66666667%; - right: auto; -} -.col-xs-pull-10 { - left: 83.33333333%; - right: auto; -} -.col-xs-pull-9 { - left: 75%; - right: auto; -} -.col-xs-pull-8 { - left: 66.66666667%; - right: auto; -} -.col-xs-pull-7 { - left: 58.33333333%; - right: auto; -} -.col-xs-pull-6 { - left: 50%; - right: auto; -} -.col-xs-pull-5 { - left: 41.66666667%; - right: auto; -} -.col-xs-pull-4 { - left: 33.33333333%; - right: auto; -} -.col-xs-pull-3 { - left: 25%; - right: auto; -} -.col-xs-pull-2 { - left: 16.66666667%; - right: auto; -} -.col-xs-pull-1 { - left: 8.33333333%; - right: auto; -} -.col-xs-pull-0 { - left: auto; - right: auto; -} -.col-xs-push-12 { - right: 100%; - left: 0; -} -.col-xs-push-11 { - right: 91.66666667%; - left: 0; -} -.col-xs-push-10 { - right: 83.33333333%; - left: 0; -} -.col-xs-push-9 { - right: 75%; - left: 0; -} -.col-xs-push-8 { - right: 66.66666667%; - left: 0; -} -.col-xs-push-7 { - right: 58.33333333%; - left: 0; -} -.col-xs-push-6 { - right: 50%; - left: 0; -} -.col-xs-push-5 { - right: 41.66666667%; - left: 0; -} -.col-xs-push-4 { - right: 33.33333333%; - left: 0; -} -.col-xs-push-3 { - right: 25%; - left: 0; -} -.col-xs-push-2 { - right: 16.66666667%; - left: 0; -} -.col-xs-push-1 { - right: 8.33333333%; - left: 0; -} -.col-xs-push-0 { - right: auto; - left: 0; -} -.col-xs-offset-12 { - margin-right: 100%; - margin-left: 0; -} -.col-xs-offset-11 { - margin-right: 91.66666667%; - margin-left: 0; -} -.col-xs-offset-10 { - margin-right: 83.33333333%; - margin-left: 0; -} -.col-xs-offset-9 { - margin-right: 75%; - margin-left: 0; -} -.col-xs-offset-8 { - margin-right: 66.66666667%; - margin-left: 0; -} -.col-xs-offset-7 { - margin-right: 58.33333333%; - margin-left: 0; -} -.col-xs-offset-6 { - margin-right: 50%; - margin-left: 0; -} -.col-xs-offset-5 { - margin-right: 41.66666667%; - margin-left: 0; -} -.col-xs-offset-4 { - margin-right: 33.33333333%; - margin-left: 0; -} -.col-xs-offset-3 { - margin-right: 25%; - margin-left: 0; -} -.col-xs-offset-2 { - margin-right: 16.66666667%; - margin-left: 0; -} -.col-xs-offset-1 { - margin-right: 8.33333333%; - margin-left: 0; -} -.col-xs-offset-0 { - margin-right: 0%; - margin-left: 0; -} -@media (min-width: 768px) { - .col-sm-1, .col-sm-2, .col-sm-3, .col-sm-4, .col-sm-5, .col-sm-6, .col-sm-7, .col-sm-8, .col-sm-9, .col-sm-10, .col-sm-11, .col-sm-12 { - float: right; - } - .col-sm-12 { - width: 100%; - } - .col-sm-11 { - width: 91.66666667%; - } - .col-sm-10 { - width: 83.33333333%; - } - .col-sm-9 { - width: 75%; - } - .col-sm-8 { - width: 66.66666667%; - } - .col-sm-7 { - width: 58.33333333%; - } - .col-sm-6 { - width: 50%; - } - .col-sm-5 { - width: 41.66666667%; - } - .col-sm-4 { - width: 33.33333333%; - } - .col-sm-3 { - width: 25%; - } - .col-sm-2 { - width: 16.66666667%; - } - .col-sm-1 { - width: 8.33333333%; - } - .col-sm-pull-12 { - left: 100%; - right: auto; - } - .col-sm-pull-11 { - left: 91.66666667%; - right: auto; - } - .col-sm-pull-10 { - left: 83.33333333%; - right: auto; - } - .col-sm-pull-9 { - left: 75%; - right: auto; - } - .col-sm-pull-8 { - left: 66.66666667%; - right: auto; - } - .col-sm-pull-7 { - left: 58.33333333%; - right: auto; - } - .col-sm-pull-6 { - left: 50%; - right: auto; - } - .col-sm-pull-5 { - left: 41.66666667%; - right: auto; - } - .col-sm-pull-4 { - left: 33.33333333%; - right: auto; - } - .col-sm-pull-3 { - left: 25%; - right: auto; - } - .col-sm-pull-2 { - left: 16.66666667%; - right: auto; - } - .col-sm-pull-1 { - left: 8.33333333%; - right: auto; - } - .col-sm-pull-0 { - left: auto; - right: auto; - } - .col-sm-push-12 { - right: 100%; - left: 0; - } - .col-sm-push-11 { - right: 91.66666667%; - left: 0; - } - .col-sm-push-10 { - right: 83.33333333%; - left: 0; - } - .col-sm-push-9 { - right: 75%; - left: 0; - } - .col-sm-push-8 { - right: 66.66666667%; - left: 0; - } - .col-sm-push-7 { - right: 58.33333333%; - left: 0; - } - .col-sm-push-6 { - right: 50%; - left: 0; - } - .col-sm-push-5 { - right: 41.66666667%; - left: 0; - } - .col-sm-push-4 { - right: 33.33333333%; - left: 0; - } - .col-sm-push-3 { - right: 25%; - left: 0; - } - .col-sm-push-2 { - right: 16.66666667%; - left: 0; - } - .col-sm-push-1 { - right: 8.33333333%; - left: 0; - } - .col-sm-push-0 { - right: auto; - left: 0; - } - .col-sm-offset-12 { - margin-right: 100%; - margin-left: 0; - } - .col-sm-offset-11 { - margin-right: 91.66666667%; - margin-left: 0; - } - .col-sm-offset-10 { - margin-right: 83.33333333%; - margin-left: 0; - } - .col-sm-offset-9 { - margin-right: 75%; - margin-left: 0; - } - .col-sm-offset-8 { - margin-right: 66.66666667%; - margin-left: 0; - } - .col-sm-offset-7 { - margin-right: 58.33333333%; - margin-left: 0; - } - .col-sm-offset-6 { - margin-right: 50%; - margin-left: 0; - } - .col-sm-offset-5 { - margin-right: 41.66666667%; - margin-left: 0; - } - .col-sm-offset-4 { - margin-right: 33.33333333%; - margin-left: 0; - } - .col-sm-offset-3 { - margin-right: 25%; - margin-left: 0; - } - .col-sm-offset-2 { - margin-right: 16.66666667%; - margin-left: 0; - } - .col-sm-offset-1 { - margin-right: 8.33333333%; - margin-left: 0; - } - .col-sm-offset-0 { - margin-right: 0%; - margin-left: 0; - } -} -@media (min-width: 992px) { - .col-md-1, .col-md-2, .col-md-3, .col-md-4, .col-md-5, .col-md-6, .col-md-7, .col-md-8, .col-md-9, .col-md-10, .col-md-11, .col-md-12 { - float: right; - } - .col-md-12 { - width: 100%; - } - .col-md-11 { - width: 91.66666667%; - } - .col-md-10 { - width: 83.33333333%; - } - .col-md-9 { - width: 75%; - } - .col-md-8 { - width: 66.66666667%; - } - .col-md-7 { - width: 58.33333333%; - } - .col-md-6 { - width: 50%; - } - .col-md-5 { - width: 41.66666667%; - } - .col-md-4 { - width: 33.33333333%; - } - .col-md-3 { - width: 25%; - } - .col-md-2 { - width: 16.66666667%; - } - .col-md-1 { - width: 8.33333333%; - } - .col-md-pull-12 { - left: 100%; - right: auto; - } - .col-md-pull-11 { - left: 91.66666667%; - right: auto; - } - .col-md-pull-10 { - left: 83.33333333%; - right: auto; - } - .col-md-pull-9 { - left: 75%; - right: auto; - } - .col-md-pull-8 { - left: 66.66666667%; - right: auto; - } - .col-md-pull-7 { - left: 58.33333333%; - right: auto; - } - .col-md-pull-6 { - left: 50%; - right: auto; - } - .col-md-pull-5 { - left: 41.66666667%; - right: auto; - } - .col-md-pull-4 { - left: 33.33333333%; - right: auto; - } - .col-md-pull-3 { - left: 25%; - right: auto; - } - .col-md-pull-2 { - left: 16.66666667%; - right: auto; - } - .col-md-pull-1 { - left: 8.33333333%; - right: auto; - } - .col-md-pull-0 { - left: auto; - right: auto; - } - .col-md-push-12 { - right: 100%; - left: 0; - } - .col-md-push-11 { - right: 91.66666667%; - left: 0; - } - .col-md-push-10 { - right: 83.33333333%; - left: 0; - } - .col-md-push-9 { - right: 75%; - left: 0; - } - .col-md-push-8 { - right: 66.66666667%; - left: 0; - } - .col-md-push-7 { - right: 58.33333333%; - left: 0; - } - .col-md-push-6 { - right: 50%; - left: 0; - } - .col-md-push-5 { - right: 41.66666667%; - left: 0; - } - .col-md-push-4 { - right: 33.33333333%; - left: 0; - } - .col-md-push-3 { - right: 25%; - left: 0; - } - .col-md-push-2 { - right: 16.66666667%; - left: 0; - } - .col-md-push-1 { - right: 8.33333333%; - left: 0; - } - .col-md-push-0 { - right: auto; - left: 0; - } - .col-md-offset-12 { - margin-right: 100%; - margin-left: 0; - } - .col-md-offset-11 { - margin-right: 91.66666667%; - margin-left: 0; - } - .col-md-offset-10 { - margin-right: 83.33333333%; - margin-left: 0; - } - .col-md-offset-9 { - margin-right: 75%; - margin-left: 0; - } - .col-md-offset-8 { - margin-right: 66.66666667%; - margin-left: 0; - } - .col-md-offset-7 { - margin-right: 58.33333333%; - margin-left: 0; - } - .col-md-offset-6 { - margin-right: 50%; - margin-left: 0; - } - .col-md-offset-5 { - margin-right: 41.66666667%; - margin-left: 0; - } - .col-md-offset-4 { - margin-right: 33.33333333%; - margin-left: 0; - } - .col-md-offset-3 { - margin-right: 25%; - margin-left: 0; - } - .col-md-offset-2 { - margin-right: 16.66666667%; - margin-left: 0; - } - .col-md-offset-1 { - margin-right: 8.33333333%; - margin-left: 0; - } - .col-md-offset-0 { - margin-right: 0%; - margin-left: 0; - } -} -@media (min-width: 1200px) { - .col-lg-1, .col-lg-2, .col-lg-3, .col-lg-4, .col-lg-5, .col-lg-6, .col-lg-7, .col-lg-8, .col-lg-9, .col-lg-10, .col-lg-11, .col-lg-12 { - float: right; - } - .col-lg-12 { - width: 100%; - } - .col-lg-11 { - width: 91.66666667%; - } - .col-lg-10 { - width: 83.33333333%; - } - .col-lg-9 { - width: 75%; - } - .col-lg-8 { - width: 66.66666667%; - } - .col-lg-7 { - width: 58.33333333%; - } - .col-lg-6 { - width: 50%; - } - .col-lg-5 { - width: 41.66666667%; - } - .col-lg-4 { - width: 33.33333333%; - } - .col-lg-3 { - width: 25%; - } - .col-lg-2 { - width: 16.66666667%; - } - .col-lg-1 { - width: 8.33333333%; - } - .col-lg-pull-12 { - left: 100%; - right: auto; - } - .col-lg-pull-11 { - left: 91.66666667%; - right: auto; - } - .col-lg-pull-10 { - left: 83.33333333%; - right: auto; - } - .col-lg-pull-9 { - left: 75%; - right: auto; - } - .col-lg-pull-8 { - left: 66.66666667%; - right: auto; - } - .col-lg-pull-7 { - left: 58.33333333%; - right: auto; - } - .col-lg-pull-6 { - left: 50%; - right: auto; - } - .col-lg-pull-5 { - left: 41.66666667%; - right: auto; - } - .col-lg-pull-4 { - left: 33.33333333%; - right: auto; - } - .col-lg-pull-3 { - left: 25%; - right: auto; - } - .col-lg-pull-2 { - left: 16.66666667%; - right: auto; - } - .col-lg-pull-1 { - left: 8.33333333%; - right: auto; - } - .col-lg-pull-0 { - left: auto; - right: auto; - } - .col-lg-push-12 { - right: 100%; - left: 0; - } - .col-lg-push-11 { - right: 91.66666667%; - left: 0; - } - .col-lg-push-10 { - right: 83.33333333%; - left: 0; - } - .col-lg-push-9 { - right: 75%; - left: 0; - } - .col-lg-push-8 { - right: 66.66666667%; - left: 0; - } - .col-lg-push-7 { - right: 58.33333333%; - left: 0; - } - .col-lg-push-6 { - right: 50%; - left: 0; - } - .col-lg-push-5 { - right: 41.66666667%; - left: 0; - } - .col-lg-push-4 { - right: 33.33333333%; - left: 0; - } - .col-lg-push-3 { - right: 25%; - left: 0; - } - .col-lg-push-2 { - right: 16.66666667%; - left: 0; - } - .col-lg-push-1 { - right: 8.33333333%; - left: 0; - } - .col-lg-push-0 { - right: auto; - left: 0; - } - .col-lg-offset-12 { - margin-right: 100%; - margin-left: 0; - } - .col-lg-offset-11 { - margin-right: 91.66666667%; - margin-left: 0; - } - .col-lg-offset-10 { - margin-right: 83.33333333%; - margin-left: 0; - } - .col-lg-offset-9 { - margin-right: 75%; - margin-left: 0; - } - .col-lg-offset-8 { - margin-right: 66.66666667%; - margin-left: 0; - } - .col-lg-offset-7 { - margin-right: 58.33333333%; - margin-left: 0; - } - .col-lg-offset-6 { - margin-right: 50%; - margin-left: 0; - } - .col-lg-offset-5 { - margin-right: 41.66666667%; - margin-left: 0; - } - .col-lg-offset-4 { - margin-right: 33.33333333%; - margin-left: 0; - } - .col-lg-offset-3 { - margin-right: 25%; - margin-left: 0; - } - .col-lg-offset-2 { - margin-right: 16.66666667%; - margin-left: 0; - } - .col-lg-offset-1 { - margin-right: 8.33333333%; - margin-left: 0; - } - .col-lg-offset-0 { - margin-right: 0%; - margin-left: 0; - } -} -caption { - text-align: right; -} -th { - text-align: right; -} -@media screen and (max-width: 767px) { - .table-responsive > .table-bordered { - border: 0; - } - .table-responsive > .table-bordered > thead > tr > th:first-child, - .table-responsive > .table-bordered > tbody > tr > th:first-child, - .table-responsive > .table-bordered > tfoot > tr > th:first-child, - .table-responsive > .table-bordered > thead > tr > td:first-child, - .table-responsive > .table-bordered > tbody > tr > td:first-child, - .table-responsive > .table-bordered > tfoot > tr > td:first-child { - border-right: 0; - border-left: initial; - } - .table-responsive > .table-bordered > thead > tr > th:last-child, - .table-responsive > .table-bordered > tbody > tr > th:last-child, - .table-responsive > .table-bordered > tfoot > tr > th:last-child, - .table-responsive > .table-bordered > thead > tr > td:last-child, - .table-responsive > .table-bordered > tbody > tr > td:last-child, - .table-responsive > .table-bordered > tfoot > tr > td:last-child { - border-left: 0; - border-right: initial; - } -} -.radio label, -.checkbox label { - padding-right: 20px; - padding-left: initial; -} -.radio input[type="radio"], -.radio-inline input[type="radio"], -.checkbox input[type="checkbox"], -.checkbox-inline input[type="checkbox"] { - margin-right: -20px; - margin-left: auto; -} -.radio-inline, -.checkbox-inline { - padding-right: 20px; - padding-left: 0; -} -.radio-inline + .radio-inline, -.checkbox-inline + .checkbox-inline { - margin-right: 10px; - margin-left: 0; -} -.has-feedback .form-control { - padding-left: 42.5px; - padding-right: 12px; -} -.form-control-feedback { - left: 0; - right: auto; -} -@media (min-width: 768px) { - .form-inline label { - padding-right: 0; - padding-left: initial; - } - .form-inline .radio input[type="radio"], - .form-inline .checkbox input[type="checkbox"] { - margin-right: 0; - margin-left: auto; - } -} -@media (min-width: 768px) { - .form-horizontal .control-label { - text-align: left; - } -} -.form-horizontal .has-feedback .form-control-feedback { - left: 15px; - right: auto; -} -.caret { - margin-right: 2px; - margin-left: 0; -} -.dropdown-menu { - right: 0; - left: auto; - float: left; - text-align: right; -} -.dropdown-menu.pull-right { - left: 0; - right: auto; - float: right; -} -.dropdown-menu-right { - left: auto; - right: 0; -} -.dropdown-menu-left { - left: 0; - right: auto; -} -@media (min-width: 768px) { - .navbar-right .dropdown-menu { - left: auto; - right: 0; - } - .navbar-right .dropdown-menu-left { - left: 0; - right: auto; - } -} -.btn-group > .btn, -.btn-group-vertical > .btn { - float: right; -} -.btn-group .btn + .btn, -.btn-group .btn + .btn-group, -.btn-group .btn-group + .btn, -.btn-group .btn-group + .btn-group { - margin-right: -1px; - margin-left: 0px; -} -.btn-toolbar { - margin-right: -5px; - margin-left: 0px; -} -.btn-toolbar .btn-group, -.btn-toolbar .input-group { - float: right; -} -.btn-toolbar > .btn, -.btn-toolbar > .btn-group, -.btn-toolbar > .input-group { - margin-right: 5px; - margin-left: 0px; -} -.btn-group > .btn:first-child { - margin-right: 0; -} -.btn-group > .btn:first-child:not(:last-child):not(.dropdown-toggle) { - border-top-right-radius: 4px; - border-bottom-right-radius: 4px; - border-bottom-left-radius: 0; - border-top-left-radius: 0; -} -.btn-group > .btn:last-child:not(:first-child), -.btn-group > .dropdown-toggle:not(:first-child) { - border-top-left-radius: 4px; - border-bottom-left-radius: 4px; - border-bottom-right-radius: 0; - border-top-right-radius: 0; -} -.btn-group > .btn-group { - float: right; -} -.btn-group.btn-group-justified > .btn, -.btn-group.btn-group-justified > .btn-group { - float: none; -} -.btn-group > .btn-group:not(:first-child):not(:last-child) > .btn { - border-radius: 0; -} -.btn-group > .btn-group:first-child > .btn:last-child, -.btn-group > .btn-group:first-child > .dropdown-toggle { - border-top-right-radius: 4px; - border-bottom-right-radius: 4px; - border-bottom-left-radius: 0; - border-top-left-radius: 0; -} -.btn-group > .btn-group:last-child > .btn:first-child { - border-top-left-radius: 4px; - border-bottom-left-radius: 4px; - border-bottom-right-radius: 0; - border-top-right-radius: 0; -} -.btn .caret { - margin-right: 0; -} -.btn-group-vertical > .btn + .btn, -.btn-group-vertical > .btn + .btn-group, -.btn-group-vertical > .btn-group + .btn, -.btn-group-vertical > .btn-group + .btn-group { - margin-top: -1px; - margin-right: 0; -} -.input-group .form-control { - float: right; -} -.input-group .form-control:first-child, -.input-group-addon:first-child, -.input-group-btn:first-child > .btn, -.input-group-btn:first-child > .btn-group > .btn, -.input-group-btn:first-child > .dropdown-toggle, -.input-group-btn:last-child > .btn:not(:last-child):not(.dropdown-toggle), -.input-group-btn:last-child > .btn-group:not(:last-child) > .btn { - border-bottom-right-radius: 4px; - border-top-right-radius: 4px; - border-bottom-left-radius: 0; - border-top-left-radius: 0; -} -.input-group-addon:first-child { - border-left: 0px; - border-right: 1px solid; -} -.input-group .form-control:last-child, -.input-group-addon:last-child, -.input-group-btn:last-child > .btn, -.input-group-btn:last-child > .btn-group > .btn, -.input-group-btn:last-child > .dropdown-toggle, -.input-group-btn:first-child > .btn:not(:first-child), -.input-group-btn:first-child > .btn-group:not(:first-child) > .btn { - border-bottom-left-radius: 4px; - border-top-left-radius: 4px; - border-bottom-right-radius: 0; - border-top-right-radius: 0; -} -.input-group-addon:last-child { - border-left-width: 1px; - border-left-style: solid; - border-right: 0px; -} -.input-group-btn > .btn + .btn { - margin-right: -1px; - margin-left: auto; -} -.input-group-btn:first-child > .btn, -.input-group-btn:first-child > .btn-group { - margin-left: -1px; - margin-right: auto; -} -.input-group-btn:last-child > .btn, -.input-group-btn:last-child > .btn-group { - margin-right: -1px; - margin-left: auto; -} -.nav { - padding-right: 0; - padding-left: initial; -} -.nav-tabs > li { - float: right; -} -.nav-tabs > li > a { - margin-left: auto; - margin-right: -2px; - border-radius: 4px 4px 0 0; -} -.nav-pills > li { - float: right; -} -.nav-pills > li > a { - border-radius: 4px; -} -.nav-pills > li + li { - margin-right: 2px; - margin-left: auto; -} -.nav-stacked > li { - float: none; -} -.nav-stacked > li + li { - margin-right: 0; - margin-left: auto; -} -.nav-justified > .dropdown .dropdown-menu { - right: auto; -} -.nav-tabs-justified > li > a { - margin-left: 0; - margin-right: auto; -} -@media (min-width: 768px) { - .nav-tabs-justified > li > a { - border-radius: 4px 4px 0 0; - } -} -@media (min-width: 768px) { - .navbar-header { - float: right; - } -} -.navbar-collapse { - padding-right: 15px; - padding-left: 15px; -} -.navbar-brand { - float: right; -} -@media (min-width: 768px) { - .navbar > .container .navbar-brand, - .navbar > .container-fluid .navbar-brand { - margin-right: -15px; - margin-left: auto; - } -} -.navbar-toggle { - float: left; - margin-left: 15px; - margin-right: auto; -} -@media (max-width: 767px) { - .navbar-nav .open .dropdown-menu > li > a, - .navbar-nav .open .dropdown-menu .dropdown-header { - padding: 5px 25px 5px 15px; - } -} -@media (min-width: 768px) { - .navbar-nav { - float: right; - } - .navbar-nav > li { - float: right; - } -} -@media (min-width: 768px) { - .navbar-left.flip { - float: right !important; - } - .navbar-right:last-child { - margin-left: -15px; - margin-right: auto; - } - .navbar-right.flip { - float: left !important; - margin-left: -15px; - margin-right: auto; - } - .navbar-right .dropdown-menu { - left: 0; - right: auto; - } -} -@media (min-width: 768px) { - .navbar-text { - float: right; - } - .navbar-text.navbar-right:last-child { - margin-left: 0; - margin-right: auto; - } -} -.pagination { - padding-right: 0; -} -.pagination > li > a, -.pagination > li > span { - float: right; - margin-right: -1px; - margin-left: 0px; -} -.pagination > li:first-child > a, -.pagination > li:first-child > span { - margin-left: 0; - border-bottom-right-radius: 4px; - border-top-right-radius: 4px; - border-bottom-left-radius: 0; - border-top-left-radius: 0; -} -.pagination > li:last-child > a, -.pagination > li:last-child > span { - margin-right: -1px; - border-bottom-left-radius: 4px; - border-top-left-radius: 4px; - border-bottom-right-radius: 0; - border-top-right-radius: 0; -} -.pager { - padding-right: 0; - padding-left: initial; -} -.pager .next > a, -.pager .next > span { - float: left; -} -.pager .previous > a, -.pager .previous > span { - float: right; -} -.nav-pills > li > a > .badge { - margin-left: 0px; - margin-right: 3px; -} -.list-group-item > .badge { - float: left; -} -.list-group-item > .badge + .badge { - margin-left: 5px; - margin-right: auto; -} -.alert-dismissable, -.alert-dismissible { - padding-left: 35px; - padding-right: 15px; -} -.alert-dismissable .close, -.alert-dismissible .close { - right: auto; - left: -21px; -} -.progress-bar { - float: right; -} -.media > .pull-left { - margin-right: 10px; -} -.media > .pull-left.flip { - margin-right: 0; - margin-left: 10px; -} -.media > .pull-right { - margin-left: 10px; -} -.media > .pull-right.flip { - margin-left: 0; - margin-right: 10px; -} -.media-right, -.media > .pull-right { - padding-right: 10px; - padding-left: initial; -} -.media-left, -.media > .pull-left { - padding-left: 10px; - padding-right: initial; -} -.media-list { - padding-right: 0; - padding-left: initial; - list-style: none; -} -.list-group { - padding-right: 0; - padding-left: initial; -} -.panel > .table:first-child > thead:first-child > tr:first-child td:first-child, -.panel > .table-responsive:first-child > .table:first-child > thead:first-child > tr:first-child td:first-child, -.panel > .table:first-child > tbody:first-child > tr:first-child td:first-child, -.panel > .table-responsive:first-child > .table:first-child > tbody:first-child > tr:first-child td:first-child, -.panel > .table:first-child > thead:first-child > tr:first-child th:first-child, -.panel > .table-responsive:first-child > .table:first-child > thead:first-child > tr:first-child th:first-child, -.panel > .table:first-child > tbody:first-child > tr:first-child th:first-child, -.panel > .table-responsive:first-child > .table:first-child > tbody:first-child > tr:first-child th:first-child { - border-top-right-radius: 3px; - border-top-left-radius: 0; -} -.panel > .table:first-child > thead:first-child > tr:first-child td:last-child, -.panel > .table-responsive:first-child > .table:first-child > thead:first-child > tr:first-child td:last-child, -.panel > .table:first-child > tbody:first-child > tr:first-child td:last-child, -.panel > .table-responsive:first-child > .table:first-child > tbody:first-child > tr:first-child td:last-child, -.panel > .table:first-child > thead:first-child > tr:first-child th:last-child, -.panel > .table-responsive:first-child > .table:first-child > thead:first-child > tr:first-child th:last-child, -.panel > .table:first-child > tbody:first-child > tr:first-child th:last-child, -.panel > .table-responsive:first-child > .table:first-child > tbody:first-child > tr:first-child th:last-child { - border-top-left-radius: 3px; - border-top-right-radius: 0; -} -.panel > .table:last-child > tbody:last-child > tr:last-child td:first-child, -.panel > .table-responsive:last-child > .table:last-child > tbody:last-child > tr:last-child td:first-child, -.panel > .table:last-child > tfoot:last-child > tr:last-child td:first-child, -.panel > .table-responsive:last-child > .table:last-child > tfoot:last-child > tr:last-child td:first-child, -.panel > .table:last-child > tbody:last-child > tr:last-child th:first-child, -.panel > .table-responsive:last-child > .table:last-child > tbody:last-child > tr:last-child th:first-child, -.panel > .table:last-child > tfoot:last-child > tr:last-child th:first-child, -.panel > .table-responsive:last-child > .table:last-child > tfoot:last-child > tr:last-child th:first-child { - border-bottom-left-radius: 3px; - border-top-right-radius: 0; -} -.panel > .table:last-child > tbody:last-child > tr:last-child td:last-child, -.panel > .table-responsive:last-child > .table:last-child > tbody:last-child > tr:last-child td:last-child, -.panel > .table:last-child > tfoot:last-child > tr:last-child td:last-child, -.panel > .table-responsive:last-child > .table:last-child > tfoot:last-child > tr:last-child td:last-child, -.panel > .table:last-child > tbody:last-child > tr:last-child th:last-child, -.panel > .table-responsive:last-child > .table:last-child > tbody:last-child > tr:last-child th:last-child, -.panel > .table:last-child > tfoot:last-child > tr:last-child th:last-child, -.panel > .table-responsive:last-child > .table:last-child > tfoot:last-child > tr:last-child th:last-child { - border-bottom-right-radius: 3px; - border-top-left-radius: 0; -} -.panel > .table-bordered > thead > tr > th:first-child, -.panel > .table-responsive > .table-bordered > thead > tr > th:first-child, -.panel > .table-bordered > tbody > tr > th:first-child, -.panel > .table-responsive > .table-bordered > tbody > tr > th:first-child, -.panel > .table-bordered > tfoot > tr > th:first-child, -.panel > .table-responsive > .table-bordered > tfoot > tr > th:first-child, -.panel > .table-bordered > thead > tr > td:first-child, -.panel > .table-responsive > .table-bordered > thead > tr > td:first-child, -.panel > .table-bordered > tbody > tr > td:first-child, -.panel > .table-responsive > .table-bordered > tbody > tr > td:first-child, -.panel > .table-bordered > tfoot > tr > td:first-child, -.panel > .table-responsive > .table-bordered > tfoot > tr > td:first-child { - border-right: 0; - border-left: none; -} -.panel > .table-bordered > thead > tr > th:last-child, -.panel > .table-responsive > .table-bordered > thead > tr > th:last-child, -.panel > .table-bordered > tbody > tr > th:last-child, -.panel > .table-responsive > .table-bordered > tbody > tr > th:last-child, -.panel > .table-bordered > tfoot > tr > th:last-child, -.panel > .table-responsive > .table-bordered > tfoot > tr > th:last-child, -.panel > .table-bordered > thead > tr > td:last-child, -.panel > .table-responsive > .table-bordered > thead > tr > td:last-child, -.panel > .table-bordered > tbody > tr > td:last-child, -.panel > .table-responsive > .table-bordered > tbody > tr > td:last-child, -.panel > .table-bordered > tfoot > tr > td:last-child, -.panel > .table-responsive > .table-bordered > tfoot > tr > td:last-child { - border-right: none; - border-left: 0; -} -.embed-responsive .embed-responsive-item, -.embed-responsive iframe, -.embed-responsive embed, -.embed-responsive object { - right: 0; - left: auto; -} -.close { - float: left; -} -.modal-footer { - text-align: left; -} -.modal-footer.flip { - text-align: right; -} -.modal-footer .btn + .btn { - margin-left: auto; - margin-right: 5px; -} -.modal-footer .btn-group .btn + .btn { - margin-right: -1px; - margin-left: auto; -} -.modal-footer .btn-block + .btn-block { - margin-right: 0; - margin-left: auto; -} -.popover { - left: auto; - text-align: right; -} -.popover.top > .arrow { - right: 50%; - left: auto; - margin-right: -11px; - margin-left: auto; -} -.popover.top > .arrow:after { - margin-right: -10px; - margin-left: auto; -} -.popover.bottom > .arrow { - right: 50%; - left: auto; - margin-right: -11px; - margin-left: auto; -} -.popover.bottom > .arrow:after { - margin-right: -10px; - margin-left: auto; -} -.carousel-control { - right: 0; - bottom: 0; -} -.carousel-control.left { - right: auto; - left: 0; - background-image: -webkit-linear-gradient(left, color-stop(rgba(0, 0, 0, 0.5) 0%), color-stop(rgba(0, 0, 0, 0.0001) 100%)); - background-image: -o-linear-gradient(left, rgba(0, 0, 0, 0.5) 0%, rgba(0, 0, 0, 0.0001) 100%); - background-image: linear-gradient(to right, rgba(0, 0, 0, 0.5) 0%, rgba(0, 0, 0, 0.0001) 100%); - background-repeat: repeat-x; - filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#80000000', endColorstr='#00000000', GradientType=1); -} -.carousel-control.right { - left: auto; - right: 0; - background-image: -webkit-linear-gradient(left, color-stop(rgba(0, 0, 0, 0.0001) 0%), color-stop(rgba(0, 0, 0, 0.5) 100%)); - background-image: -o-linear-gradient(left, rgba(0, 0, 0, 0.0001) 0%, rgba(0, 0, 0, 0.5) 100%); - background-image: linear-gradient(to right, rgba(0, 0, 0, 0.0001) 0%, rgba(0, 0, 0, 0.5) 100%); - background-repeat: repeat-x; - filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#00000000', endColorstr='#80000000', GradientType=1); -} -.carousel-control .icon-prev, -.carousel-control .glyphicon-chevron-left { - left: 50%; - right: auto; - margin-right: -10px; -} -.carousel-control .icon-next, -.carousel-control .glyphicon-chevron-right { - right: 50%; - left: auto; - margin-left: -10px; -} -.carousel-indicators { - right: 50%; - left: 0; - margin-right: -30%; - margin-left: 0; - padding-left: 0; -} -@media screen and (min-width: 768px) { - .carousel-control .glyphicon-chevron-left, - .carousel-control .icon-prev { - margin-left: 0; - margin-right: -15px; - } - .carousel-control .glyphicon-chevron-right, - .carousel-control .icon-next { - margin-left: 0; - margin-right: -15px; - } - .carousel-caption { - left: 20%; - right: 20%; - padding-bottom: 30px; - } -} -.pull-right.flip { - float: left !important; -} -.pull-left.flip { - float: right !important; -} -/*# sourceMappingURL=bootstrap-rtl.css.map */ \ No newline at end of file diff --git a/frappe/public/css/desk-rtl.css b/frappe/public/css/desk-rtl.css deleted file mode 100644 index a38f6864ff..0000000000 --- a/frappe/public/css/desk-rtl.css +++ /dev/null @@ -1,118 +0,0 @@ -.navbar .navbar-search-icon{ - right: auto; - left: 24px; -} -.navbar > .container > .navbar-header{ - float: right !important; -} -body[data-sidebar="0"] .navbar-home { - margin-left: auto !important; - margin-right: 15px !important; -} -.navbar-desk ~ ul > li { - float: right !important; -} -body.no-breadcrumbs .navbar .navbar-home:before { - margin-right: auto; - margin-left: 10px !important; - -ms-transform:rotate(180deg); /* Internet Explorer 9 */ - -webkit-transform:rotate(180deg); /* Chrome, Safari, Opera */ - transform:rotate(180deg); /* Standard syntax */ -} -.layout-side-section .overlay-sidebar { - left: auto !important; - right: 0 !important; -} -.layout-side-section .overlay-sidebar.opened { - transform:translateX(0) !important; -} -.navbar-right { - float: left !important; -} -#navbar-breadcrumbs > li > a:before { - margin-right: auto; - margin-left: 10px; - top: 6px; - -ms-transform:rotate(180deg); /* Internet Explorer 9 */ - -webkit-transform:rotate(180deg); /* Chrome, Safari, Opera */ - transform:rotate(180deg); /* Standard syntax */ -} -.case-wrapper { - float: right; -} -.link-btn { - right: auto; - left: 4px; - transform:rotate(180deg); /* Rotate icon*/ -} -.sidebar-menu .badge { - right: auto; - left: 0px; -} -.indicator::before { - margin: 0 0 0 4px; -} -.pull-left { - float: right !important; -} -.grid-row > .row .col:last-child { - margin-right: auto; - margin-left: -10px; -} -.text-right { - text-align: left; -} -.list-row-head .octicon-heart { - margin-right: auto; - margin-left: 13px; -} -.list-id { - margin-left: 7px !important; -} -.avatar-small .avatar-sm { - margin-left: 5px; - margin-right: auto; -} -.list-row-right .list-row-modified { - margin-right: auto; - margin-left: 9px; -} -.list-comment-count { - text-align: right; -} -ul.tree-children { - padding-right: 20px; - padding-left: inherit !important; -} -.balance-area { - float: left !important; -} -.tree.opened::before, .tree-node.opened::before, .tree:last-child::after, .tree-node:last-child::after { - left: inherit !important; - right: 8px; -} -.tree.opened > .tree-children > .tree-node > .tree-link::before, .tree-node.opened > .tree-children > .tree-node > .tree-link::before { - left: inherit !important; - right: -11px; -} -.tree:last-child::after, .tree-node:last-child::after { - right: -13px !important; -} -.tree.opened::before { - left: auto !important; - right: 23px; -} -.results { - direction: ltr; -} -.data-table { - direction: ltr; -} -.section-header { - direction: ltr; -} - -.ql-editor { - direction: rtl; - text-align: right; -} \ No newline at end of file diff --git a/frappe/public/css/report-rtl.css b/frappe/public/css/report-rtl.css deleted file mode 100644 index 03e986c56b..0000000000 --- a/frappe/public/css/report-rtl.css +++ /dev/null @@ -1,15 +0,0 @@ -.grid-report { - direction: ltr; -} - -.page-form .awesomplete > ul { - left: auto; -} - -.chart_area{ - direction: ltr; -} - -.grid-report .show-zero{ - direction: rtl; -} diff --git a/frappe/public/scss/frappe-rtl.bundle.scss b/frappe/public/scss/frappe-rtl.bundle.scss deleted file mode 100644 index bc618270a8..0000000000 --- a/frappe/public/scss/frappe-rtl.bundle.scss +++ /dev/null @@ -1,3 +0,0 @@ -@import "frappe/public/css/bootstrap-rtl.css"; -@import "frappe/public/css/desk-rtl.css"; -@import "frappe/public/css/report-rtl.css"; From 7e6b00fd3c2aa76c1c3c98664ec1e68b95fb6bd9 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Sun, 27 Jun 2021 08:54:57 +0530 Subject: [PATCH 011/164] refactor: Remove unused build.json --- frappe/public/build.json | 299 --------------------------------------- 1 file changed, 299 deletions(-) delete mode 100755 frappe/public/build.json diff --git a/frappe/public/build.json b/frappe/public/build.json deleted file mode 100755 index 942871ee9b..0000000000 --- a/frappe/public/build.json +++ /dev/null @@ -1,299 +0,0 @@ -{ - "css/frappe-web-b4.css": "public/scss/website.scss", - "css/frappe-chat-web.css": [ - "public/css/font-awesome.css", - "public/css/octicons/octicons.css", - "public/less/chat.less" - ], - "concat:js/moment-bundle.min.js": [ - "node_modules/moment/min/moment-with-locales.min.js", - "node_modules/moment-timezone/builds/moment-timezone-with-data.min.js" - ], - "js/chat.js": "public/js/frappe/chat.js", - "js/frappe-recorder.min.js": "public/js/frappe/recorder/recorder.js", - "js/checkout.min.js": "public/js/integrations/razorpay.js", - "js/frappe-web.min.js": [ - "public/js/frappe/class.js", - "public/js/frappe/polyfill.js", - "public/js/lib/md5.min.js", - "public/js/frappe/provide.js", - "public/js/frappe/format.js", - "public/js/frappe/utils/number_format.js", - "public/js/frappe/utils/utils.js", - "public/js/frappe/utils/common.js", - "public/js/frappe/ui/messages.js", - "public/js/frappe/translate.js", - "public/js/frappe/utils/pretty_date.js", - "public/js/frappe/microtemplate.js", - "public/js/frappe/query_string.js", - - "public/js/frappe/upload.js", - - "public/js/frappe/model/meta.js", - "public/js/frappe/model/model.js", - "public/js/frappe/model/perm.js", - - "website/js/website.js", - "public/js/frappe/socketio_client.js" - ], - "js/bootstrap-4-web.min.js": "website/js/bootstrap-4.js", - "js/control.min.js": [ - "node_modules/air-datepicker/dist/js/datepicker.min.js", - "node_modules/air-datepicker/dist/js/i18n/datepicker.cs.js", - "node_modules/air-datepicker/dist/js/i18n/datepicker.da.js", - "node_modules/air-datepicker/dist/js/i18n/datepicker.de.js", - "node_modules/air-datepicker/dist/js/i18n/datepicker.en.js", - "node_modules/air-datepicker/dist/js/i18n/datepicker.es.js", - "node_modules/air-datepicker/dist/js/i18n/datepicker.fi.js", - "node_modules/air-datepicker/dist/js/i18n/datepicker.fr.js", - "node_modules/air-datepicker/dist/js/i18n/datepicker.hu.js", - "node_modules/air-datepicker/dist/js/i18n/datepicker.nl.js", - "node_modules/air-datepicker/dist/js/i18n/datepicker.pl.js", - "node_modules/air-datepicker/dist/js/i18n/datepicker.pt-BR.js", - "node_modules/air-datepicker/dist/js/i18n/datepicker.pt.js", - "node_modules/air-datepicker/dist/js/i18n/datepicker.ro.js", - "node_modules/air-datepicker/dist/js/i18n/datepicker.sk.js", - "node_modules/air-datepicker/dist/js/i18n/datepicker.zh.js", - "public/js/frappe/ui/capture.js", - "public/js/frappe/form/controls/control.js" - ], - "js/dialog.min.js": [ - "public/js/frappe/dom.js", - "public/js/frappe/form/formatters.js", - "public/js/frappe/form/layout.js", - "public/js/frappe/ui/field_group.js", - "public/js/frappe/form/link_selector.js", - "public/js/frappe/form/multi_select_dialog.js", - "public/js/frappe/ui/dialog.js" - ], - "css/desk.min.css": [ - "public/js/lib/leaflet/leaflet.css", - "public/js/lib/leaflet/leaflet.draw.css", - "public/js/lib/leaflet/L.Control.Locate.css", - "public/js/lib/leaflet/easy-button.css", - "public/css/font-awesome.css", - "public/css/octicons/octicons.css", - "public/less/desk.less", - "public/less/module.less", - "public/less/mobile.less", - "public/less/controls.less", - "public/less/chat.less", - "public/css/fonts/inter/inter.css", - "node_modules/frappe-charts/dist/frappe-charts.min.css", - "node_modules/plyr/dist/plyr.css", - "public/scss/desk.scss" - ], - "css/frappe-rtl.css": [ - "public/css/bootstrap-rtl.css", - "public/css/desk-rtl.css", - "public/css/report-rtl.css" - ], - "css/printview.css": [ - "public/css/bootstrap.css", - "public/scss/print.scss" - ], - "concat:js/libs.min.js": [ - "public/js/lib/Sortable.min.js", - "public/js/lib/jquery/jquery.hotkeys.js", - "node_modules/bootstrap/dist/js/bootstrap.bundle.min.js", - "node_modules/vue/dist/vue.min.js", - "node_modules/moment/min/moment-with-locales.min.js", - "node_modules/moment-timezone/builds/moment-timezone-with-data.min.js", - "node_modules/socket.io-client/dist/socket.io.slim.js", - "node_modules/localforage/dist/localforage.min.js", - "public/js/lib/jSignature.min.js", - "public/js/lib/leaflet/leaflet.js", - "public/js/lib/leaflet/leaflet.draw.js", - "public/js/lib/leaflet/L.Control.Locate.js", - "public/js/lib/leaflet/easy-button.js" - ], - "js/desk.min.js": [ - "public/js/frappe/translate.js", - "public/js/frappe/class.js", - "public/js/frappe/polyfill.js", - "public/js/frappe/provide.js", - "public/js/frappe/assets.js", - "public/js/frappe/format.js", - "public/js/frappe/form/formatters.js", - "public/js/frappe/dom.js", - "public/js/frappe/ui/messages.js", - "public/js/frappe/ui/keyboard.js", - "public/js/frappe/ui/colors.js", - "public/js/frappe/ui/sidebar.js", - "public/js/frappe/ui/link_preview.js", - - "public/js/frappe/request.js", - "public/js/frappe/socketio_client.js", - "public/js/frappe/utils/utils.js", - "public/js/frappe/event_emitter.js", - "public/js/frappe/router.js", - "public/js/frappe/router_history.js", - "public/js/frappe/defaults.js", - "public/js/frappe/roles_editor.js", - "public/js/frappe/module_editor.js", - "public/js/frappe/microtemplate.js", - - "public/js/frappe/ui/page.html", - "public/js/frappe/ui/page.js", - "public/js/frappe/ui/slides.js", - "public/js/frappe/ui/onboarding_dialog.js", - "public/js/frappe/ui/find.js", - "public/js/frappe/ui/iconbar.js", - "public/js/frappe/form/layout.js", - "public/js/frappe/ui/field_group.js", - "public/js/frappe/form/link_selector.js", - "public/js/frappe/form/multi_select_dialog.js", - "public/js/frappe/ui/dialog.js", - "public/js/frappe/ui/capture.js", - "public/js/frappe/ui/app_icon.js", - "public/js/frappe/ui/theme_switcher.js", - - "public/js/frappe/model/model.js", - "public/js/frappe/db.js", - "public/js/frappe/model/meta.js", - "public/js/frappe/model/sync.js", - "public/js/frappe/model/create_new.js", - "public/js/frappe/model/perm.js", - "public/js/frappe/model/workflow.js", - "public/js/frappe/model/user_settings.js", - - "public/js/lib/md5.min.js", - "public/js/frappe/utils/user.js", - "public/js/frappe/utils/common.js", - "public/js/frappe/utils/urllib.js", - "public/js/frappe/utils/pretty_date.js", - "public/js/frappe/utils/tools.js", - "public/js/frappe/utils/datetime.js", - "public/js/frappe/utils/number_format.js", - "public/js/frappe/utils/help.js", - "public/js/frappe/utils/help_links.js", - "public/js/frappe/utils/address_and_contact.js", - "public/js/frappe/utils/preview_email.js", - "public/js/frappe/utils/file_manager.js", - - "public/js/frappe/upload.js", - "public/js/frappe/ui/tree.js", - - "public/js/frappe/views/container.js", - "public/js/frappe/views/breadcrumbs.js", - "public/js/frappe/views/factory.js", - "public/js/frappe/views/pageview.js", - - "public/js/frappe/ui/toolbar/awesome_bar.js", - "public/js/frappe/ui/toolbar/energy_points_notifications.js", - "public/js/frappe/ui/notifications/notifications.js", - "public/js/frappe/ui/toolbar/search.js", - "public/js/frappe/ui/toolbar/tag_utils.js", - "public/js/frappe/ui/toolbar/search.html", - "public/js/frappe/ui/toolbar/search_utils.js", - "public/js/frappe/ui/toolbar/about.js", - "public/js/frappe/ui/toolbar/navbar.html", - "public/js/frappe/ui/toolbar/toolbar.js", - "public/js/frappe/ui/toolbar/notifications.js", - "public/js/frappe/views/communication.js", - "public/js/frappe/views/translation_manager.js", - "public/js/frappe/views/workspace/workspace.js", - - "public/js/frappe/widgets/widget_group.js", - - "public/js/frappe/ui/sort_selector.html", - "public/js/frappe/ui/sort_selector.js", - - "public/js/frappe/change_log.html", - "public/js/frappe/ui/workspace_loading_skeleton.html", - "public/js/frappe/desk.js", - "public/js/frappe/query_string.js", - - "public/js/frappe/ui/comment.js", - - "public/js/frappe/chat.js", - "public/js/frappe/utils/energy_point_utils.js", - "public/js/frappe/utils/dashboard_utils.js", - "public/js/frappe/ui/chart.js", - "public/js/frappe/ui/datatable.js", - "public/js/frappe/ui/driver.js", - "public/js/frappe/ui/plyr.js", - "public/js/frappe/barcode_scanner/index.js" - ], - "js/form.min.js": [ - "public/js/frappe/form/templates/**.html", - "public/js/frappe/form/controls/control.js", - "public/js/frappe/views/formview.js", - "public/js/frappe/form/form.js", - "public/js/frappe/meta_tag.js" - ], - "js/list.min.js": [ - "public/js/frappe/ui/listing.html", - - "public/js/frappe/model/indicator.js", - "public/js/frappe/ui/filters/filter.js", - "public/js/frappe/ui/filters/filter_list.js", - "public/js/frappe/ui/filters/field_select.js", - "public/js/frappe/ui/filters/edit_filter.html", - "public/js/frappe/ui/tags.js", - "public/js/frappe/ui/tag_editor.js", - "public/js/frappe/ui/like.js", - "public/js/frappe/ui/liked_by.html", - "public/html/print_template.html", - - "public/js/frappe/list/base_list.js", - "public/js/frappe/list/list_view.js", - "public/js/frappe/list/list_factory.js", - - "public/js/frappe/list/list_view_select.js", - "public/js/frappe/list/list_sidebar.js", - "public/js/frappe/list/list_sidebar.html", - "public/js/frappe/list/list_sidebar_stat.html", - "public/js/frappe/list/list_sidebar_group_by.js", - "public/js/frappe/list/list_view_permission_restrictions.html", - - "public/js/frappe/views/gantt/gantt_view.js", - "public/js/frappe/views/calendar/calendar.js", - "public/js/frappe/views/dashboard/dashboard_view.js", - "public/js/frappe/views/image/image_view.js", - "public/js/frappe/views/map/map_view.js", - "public/js/frappe/views/kanban/kanban_view.js", - "public/js/frappe/views/inbox/inbox_view.js", - "public/js/frappe/views/file/file_view.js", - - "public/js/frappe/views/treeview.js", - "public/js/frappe/views/interaction.js", - - "public/js/frappe/views/image/image_view_item_row.html", - "public/js/frappe/views/image/photoswipe_dom.html", - - "public/js/frappe/views/kanban/kanban_board.html", - "public/js/frappe/views/kanban/kanban_column.html", - "public/js/frappe/views/kanban/kanban_card.html" - ], - "css/report.min.css": [ - "node_modules/frappe-datatable/dist/frappe-datatable.css", - "public/css/tree_grid.css" - ], - "js/report.min.js": [ - "public/js/lib/clusterize.min.js", - "public/js/frappe/views/reports/report_factory.js", - "public/js/frappe/views/reports/report_view.js", - "public/js/frappe/views/reports/query_report.js", - "public/js/frappe/views/reports/print_grid.html", - "public/js/frappe/views/reports/print_tree.html", - "public/js/frappe/ui/group_by/group_by.html", - "public/js/frappe/ui/group_by/group_by.js", - "public/js/frappe/views/reports/report_utils.js" - ], - "js/web_form.min.js": [ - "public/js/frappe/utils/datetime.js", - "public/js/frappe/web_form/webform_script.js" - ], - "css/web_form.css": [ - "website/css/web_form.css", - "public/css/octicons/octicons.css", - "public/scss/controls.scss", - "node_modules/frappe-datatable/dist/frappe-datatable.css" - ], - "css/email.css": "public/scss/email.scss", - "js/barcode_scanner.min.js": "public/js/frappe/barcode_scanner/quagga.js", - "js/user_profile_controller.min.js": "desk/page/user_profile/user_profile_controller.js", - "css/login.css": "public/scss/login.scss", - "js/data_import_tools.min.js": "public/js/frappe/data_import/index.js" -} From 30f92070910e983238a7661c133f1891d4413cf4 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Mon, 28 Jun 2021 12:03:00 +0530 Subject: [PATCH 012/164] feat: Select css-rtl folder if language is rtl --- frappe/public/js/frappe/assets.js | 8 ++++++-- frappe/utils/jinja_globals.py | 14 ++++++++------ 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/frappe/public/js/frappe/assets.js b/frappe/public/js/frappe/assets.js index 3fca8640f3..1f2659992f 100644 --- a/frappe/public/js/frappe/assets.js +++ b/frappe/public/js/frappe/assets.js @@ -168,9 +168,13 @@ frappe.assets = { } }, - bundled_asset(path) { + bundled_asset(path, is_rtl=null) { if (!path.startsWith('/assets') && path.includes('.bundle.')) { - return frappe.boot.assets_json[path] || path; + path = frappe.boot.assets_json[path] || path; + if (path.endsWith('.css') && is_rtl) { + path = path.replace('/css/', '/css-rtl/'); + } + return path; } return path; } diff --git a/frappe/utils/jinja_globals.py b/frappe/utils/jinja_globals.py index 30a0e46232..dd5ade727b 100644 --- a/frappe/utils/jinja_globals.py +++ b/frappe/utils/jinja_globals.py @@ -75,24 +75,26 @@ def include_script(path): def include_style(path, rtl=None): path = bundled_asset(path) - if rtl is None: - rtl = is_rtl() - if rtl: + if is_rtl(rtl): path = path.replace('/css/', '/css-rtl/') return f'' -def bundled_asset(path): +def bundled_asset(path, rtl=None): from frappe.utils import get_assets_json from frappe.website.utils import abs_url if ".bundle." in path and not path.startswith("/assets"): bundled_assets = get_assets_json() path = bundled_assets.get(path) or path + if path.endswith('.css') and is_rtl(rtl): + path = path.replace('/css/', '/css-rtl/') return abs_url(path) -def is_rtl(): +def is_rtl(rtl=None): from frappe import local - return local.lang in ["ar", "he", "fa", "ps"] \ No newline at end of file + if rtl is None: + return local.lang in ["ar", "he", "fa", "ps"] + return rtl \ No newline at end of file From 6960a192082ecdaff0d8b33ec8240c5844c8f89b Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Mon, 28 Jun 2021 12:03:31 +0530 Subject: [PATCH 013/164] fix: RTL for print views --- frappe/printing/page/print/print.js | 11 +++-------- frappe/public/html/print_template.html | 2 +- frappe/public/js/frappe/views/reports/query_report.js | 4 +++- frappe/www/printview.html | 2 +- frappe/www/printview.py | 4 +++- 5 files changed, 11 insertions(+), 12 deletions(-) diff --git a/frappe/printing/page/print/print.js b/frappe/printing/page/print/print.js index 233bbe0ce7..fc59065c64 100644 --- a/frappe/printing/page/print/print.js +++ b/frappe/printing/page/print/print.js @@ -409,19 +409,14 @@ frappe.ui.form.PrintView = class { setup_print_format_dom(out, $print_format) { this.print_wrapper.find('.print-format-skeleton').remove(); let base_url = frappe.urllib.get_base_url(); - let print_css = frappe.assets.bundled_asset('print.bundle.css'); + let print_css = frappe.assets.bundled_asset('print.bundle.css', frappe.utils.is_rtl(this.lang_code)); + this.$print_format_body.find('html').attr('dir', frappe.utils.is_rtl(this.lang_code) ? 'rtl': 'ltr'); + this.$print_format_body.find('html').attr('lang', this.lang_code); this.$print_format_body.find('head').html( ` ` ); - if (frappe.utils.is_rtl(this.lang_code)) { - let rtl_css = frappe.assets.bundled_asset('frappe-rtl.bundle.css'); - this.$print_format_body.find('head').append( - `` - ); - } - this.$print_format_body.find('body').html( `
${out.html}
` ); diff --git a/frappe/public/html/print_template.html b/frappe/public/html/print_template.html index 721bec7fa7..39cc6b4274 100644 --- a/frappe/public/html/print_template.html +++ b/frappe/public/html/print_template.html @@ -1,5 +1,5 @@ - + diff --git a/frappe/public/js/frappe/views/reports/query_report.js b/frappe/public/js/frappe/views/reports/query_report.js index 208d5b2f67..a4b3564e37 100644 --- a/frappe/public/js/frappe/views/reports/query_report.js +++ b/frappe/public/js/frappe/views/reports/query_report.js @@ -1264,7 +1264,9 @@ frappe.views.QueryReport = class QueryReport extends frappe.views.BaseList { print_css: print_css, print_settings: print_settings, landscape: landscape, - columns: columns + columns: columns, + lang: frappe.boot.lang, + layout_direction: frappe.utils.is_rtl() ? "rtl" : "ltr" }); frappe.render_pdf(html, print_settings); diff --git a/frappe/www/printview.html b/frappe/www/printview.html index 05e85730dd..3f8d4201fb 100644 --- a/frappe/www/printview.html +++ b/frappe/www/printview.html @@ -1,5 +1,5 @@ - + diff --git a/frappe/www/printview.py b/frappe/www/printview.py index 226d5b048a..cdf47790eb 100644 --- a/frappe/www/printview.py +++ b/frappe/www/printview.py @@ -7,6 +7,7 @@ from frappe import _ from frappe.modules import get_doc_path from frappe.core.doctype.access_log.access_log import make_access_log from frappe.utils import cint, sanitize_html, strip_html +from frappe.utils.jinja_globals import is_rtl no_cache = 1 @@ -44,7 +45,8 @@ def get_context(context): "css": get_print_style(frappe.form_dict.style, print_format), "comment": frappe.session.user, "title": doc.get(meta.title_field) if meta.title_field else doc.name, - "has_rtl": True if frappe.local.lang in ["ar", "he", "fa", "ps"] else False + "lang": frappe.local.lang, + "layout_direction": "rtl" if is_rtl() else "ltr" } def get_print_format_doc(print_format_name, meta): From 31d8436979e49119011fa7fe891f4689a586e267 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Mon, 28 Jun 2021 12:04:57 +0530 Subject: [PATCH 014/164] refactor: Rename dir to layout_direction - For readability --- frappe/www/app.html | 2 +- frappe/www/app.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/www/app.html b/frappe/www/app.html index 8704ed2671..68a6dc8e86 100644 --- a/frappe/www/app.html +++ b/frappe/www/app.html @@ -1,5 +1,5 @@ - + diff --git a/frappe/www/app.py b/frappe/www/app.py index 55992b5a55..acf6dde000 100644 --- a/frappe/www/app.py +++ b/frappe/www/app.py @@ -41,7 +41,7 @@ def get_context(context): "build_version": frappe.utils.get_build_version(), "include_js": hooks["app_include_js"], "include_css": hooks["app_include_css"], - "dir": "rtl" if is_rtl() else "ltr", + "layout_direction": "rtl" if is_rtl() else "ltr", "lang": frappe.local.lang, "sounds": hooks["sounds"], "boot": boot if context.get("for_mobile") else boot_json, From d9962370dc6cd6da78b9a0e630ade67939dcd73e Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Mon, 28 Jun 2021 12:06:26 +0530 Subject: [PATCH 015/164] fix: Add language to print_template --- frappe/public/html/print_template.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/html/print_template.html b/frappe/public/html/print_template.html index 39cc6b4274..f63a20377f 100644 --- a/frappe/public/html/print_template.html +++ b/frappe/public/html/print_template.html @@ -1,5 +1,5 @@ - + From ae9aad4d760289549dcd47fb0faa7a5f4cc134c6 Mon Sep 17 00:00:00 2001 From: Saqib Ansari Date: Tue, 29 Jun 2021 13:30:20 +0530 Subject: [PATCH 016/164] fix: form tour loading after route change --- frappe/desk/doctype/form_tour/form_tour.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/frappe/desk/doctype/form_tour/form_tour.js b/frappe/desk/doctype/form_tour/form_tour.js index efb853cfa5..8d70dcd3dc 100644 --- a/frappe/desk/doctype/form_tour/form_tour.js +++ b/frappe/desk/doctype/form_tour/form_tour.js @@ -15,14 +15,14 @@ frappe.ui.form.on('Form Tour', { frm.add_custom_button(__('Show Tour'), async () => { const issingle = await check_if_single(frm.doc.reference_doctype); + let route_changed = null; if (issingle) { - frappe.set_route('Form', frm.doc.reference_doctype); + route_changed = frappe.set_route('Form', frm.doc.reference_doctype); } else { - const new_name = 'new-' + frappe.scrub(frm.doc.reference_doctype) + '-1'; - frappe.set_route('Form', frm.doc.reference_doctype, new_name); + route_changed = frappe.set_route('Form', frm.doc.reference_doctype, 'new'); } - frappe.utils.sleep(500).then(() => { + route_changed.then(() => { const tour_name = frm.doc.name; cur_frm.tour .init({ tour_name }) From 3acc0a24465cd86be58e7718a0da307e8581fb59 Mon Sep 17 00:00:00 2001 From: Saqib Ansari Date: Tue, 29 Jun 2021 13:30:51 +0530 Subject: [PATCH 017/164] feat: get standard form tour for onboarding step --- frappe/public/js/frappe/form/form_tour.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/frappe/public/js/frappe/form/form_tour.js b/frappe/public/js/frappe/form/form_tour.js index 7f7ec9ce4f..5318973a66 100644 --- a/frappe/public/js/frappe/form/form_tour.js +++ b/frappe/public/js/frappe/form/form_tour.js @@ -37,7 +37,12 @@ frappe.ui.form.FormTour = class FormTour { if (tour_name) { this.tour = await frappe.db.get_doc('Form Tour', tour_name); } else { - this.tour = { steps: frappe.tour[this.frm.doctype] }; + const doctype_tour_exists = await frappe.db.exists('Form Tour', this.frm.doctype); + if (doctype_tour_exists) { + this.tour = await frappe.db.get_doc('Form Tour', this.frm.doctype) + } else { + this.tour = { steps: frappe.tour[this.frm.doctype] }; + } } if (on_finish) this.on_finish = on_finish; @@ -232,7 +237,7 @@ frappe.ui.form.FormTour = class FormTour { } add_step_to_save() { - const page_id = `#page-${this.frm.doctype}`; + const page_id = `[id="page-${this.frm.doctype}"]`; const $save_btn = `${page_id} .standard-actions .primary-action`; const save_step = { element: $save_btn, From debc8013057ca0e51a031dbc80aa3d65b21d8fb7 Mon Sep 17 00:00:00 2001 From: Saqib Ansari Date: Wed, 30 Jun 2021 14:21:22 +0530 Subject: [PATCH 018/164] feat: submit on completion --- frappe/desk/doctype/form_tour/form_tour.json | 10 ++++- frappe/public/js/frappe/form/form_tour.js | 45 ++++++++++++++++++-- 2 files changed, 51 insertions(+), 4 deletions(-) diff --git a/frappe/desk/doctype/form_tour/form_tour.json b/frappe/desk/doctype/form_tour/form_tour.json index e4ea528fcc..44f39766b3 100644 --- a/frappe/desk/doctype/form_tour/form_tour.json +++ b/frappe/desk/doctype/form_tour/form_tour.json @@ -11,6 +11,7 @@ "module", "is_standard", "save_on_complete", + "submit_on_complete", "section_break_3", "steps" ], @@ -62,11 +63,18 @@ "label": "Module", "options": "Module Def", "read_only": 1 + }, + { + "default": "0", + "depends_on": "save_on_complete", + "fieldname": "submit_on_complete", + "fieldtype": "Check", + "label": "Submit on Completion" } ], "index_web_pages_for_search": 1, "links": [], - "modified": "2021-06-06 20:32:54.068774", + "modified": "2021-06-29 15:23:26.893068", "modified_by": "Administrator", "module": "Desk", "name": "Form Tour", diff --git a/frappe/public/js/frappe/form/form_tour.js b/frappe/public/js/frappe/form/form_tour.js index 5318973a66..5f4920f415 100644 --- a/frappe/public/js/frappe/form/form_tour.js +++ b/frappe/public/js/frappe/form/form_tour.js @@ -17,8 +17,9 @@ frappe.ui.form.FormTour = class FormTour { prevBtnText: 'Previous', opacity: 0.25, onHighlighted: (step) => { - // if last step is to save, then attach a listener to save button - if (step.options.is_save_step) { + // if last step is to save, or if is submit step + // then attach a listener on highlighted elem to reset the driver + if ((step.options.is_save_step && !this.driver.hasNextStep())|| step.options.is_submit_step) { $(step.options.element).one('click', () => this.driver.reset()); } @@ -74,6 +75,10 @@ frappe.ui.form.FormTour = class FormTour { if (this.tour.save_on_complete) { this.add_step_to_save(); } + + if (this.tour.submit_on_complete) { + this.add_step_to_submit(); + } } is_next_condition_satisfied(step) { @@ -249,9 +254,43 @@ frappe.ui.form.FormTour = class FormTour { description: "", position: "left", doneBtnText: __("Save") + }, + onNext: () => { + this.frm.save(); } }; this.driver_steps.push(save_step); - frappe.ui.form.on(this.frm.doctype, 'after_save', () => this.on_finish && this.on_finish()); + + let after_save = () => this.on_finish && this.on_finish(); + + if (this.tour.submit_on_complete) { + after_save = () => { + this.update_driver_steps(); + this.driver.start(this.driver.steps.length - 1); + } + } + frappe.ui.form.on(this.frm.doctype, 'after_save', after_save); + } + + add_step_to_submit() { + const page_id = `[id="page-${this.frm.doctype}"]`; + const $submit_btn = `${page_id} .standard-actions .primary-action`; + const submit_step = { + element: $submit_btn, + is_submit_step: true, + allowClose: false, + overlayClickNext: false, + popover: { + title: __("Submit"), + description: "", + position: "left", + doneBtnText: __("Submit") + }, + onNext: () => { + this.frm.savesubmit(); + } + }; + this.driver_steps.push(submit_step); + frappe.ui.form.on(this.frm.doctype, 'after_submit', () => this.on_finish && this.on_finish()); } }; \ No newline at end of file From ef0a5e904bc7bef582b29629956c99deb466de49 Mon Sep 17 00:00:00 2001 From: barredterra <14891507+barredterra@users.noreply.github.com> Date: Fri, 2 Jul 2021 17:24:34 +0200 Subject: [PATCH 019/164] feat: different output formats for `bench version` --- frappe/commands/utils.py | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py index b944f02af7..7d3aecca1f 100644 --- a/frappe/commands/utils.py +++ b/frappe/commands/utils.py @@ -768,25 +768,45 @@ def set_config(context, key, value, global_=False, parse=False, as_dict=False): @click.command('version') -def get_version(): +@click.option('--output', help='Output format. One of: plain, table, json, legacy.', default='legacy') +def get_version(output): "Show the versions of all the installed apps" from git import Repo - from frappe.utils.change_log import get_app_branch + from frappe.utils.commands import render_table + frappe.init('') + data = [] for app in sorted(frappe.get_all_apps()): - branch_name = get_app_branch(app) module = frappe.get_module(app) app_hooks = frappe.get_module(app + ".hooks") repo = Repo(frappe.get_app_path(app, "..")) - branch = repo.head.ref.name - commit = repo.head.ref.commit.hexsha[:7] - if hasattr(app_hooks, '{0}_version'.format(branch_name)): - click.echo("{0} {1} {2} ({3})".format(app, getattr(app_hooks, '{0}_version'.format(branch_name)), branch, commit)) + app_info = frappe._dict() + app_info.app = app + app_info.branch = repo.head.ref.name + app_info.commit = repo.head.ref.commit.hexsha[:7] + if hasattr(app_hooks, '{0}_version'.format(app_info.branch)): + app_info.version = getattr(app_hooks, '{0}_version'.format(app_info.branch)) elif hasattr(module, "__version__"): - click.echo("{0} {1} {2} ({3})".format(app, module.__version__, branch, commit)) + app_info.version = module.__version__ + + data.append(app_info) + + if output == 'table': + table = [['App', 'Version', 'Branch', 'Commit']] + for app_info in data: + table.append([app_info.app, app_info.version, app_info.branch, app_info.commit]) + render_table(table) + elif output == 'json': + click.echo(json.dumps(data, indent=4)) + elif output == 'plain': + for app_info in data: + click.echo(f'{app_info.app} {app_info.version} {app_info.branch} ({app_info.commit})') + else: # legacy + for app_info in data: + click.echo(f'{app_info.app} {app_info.version}') @click.command('rebuild-global-search') From 51207fe582f28eeaf5a0692c143e243339d6cce6 Mon Sep 17 00:00:00 2001 From: hasnain2808 Date: Sat, 3 Jul 2021 13:00:43 +0530 Subject: [PATCH 020/164] feat: option to not preload assets --- frappe/website/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frappe/website/utils.py b/frappe/website/utils.py index 0f5f182ea2..171c90e1cf 100644 --- a/frappe/website/utils.py +++ b/frappe/website/utils.py @@ -449,15 +449,15 @@ def cache_html(func): return cache_html_decorator -def build_response(path, data, http_status_code, headers=None): +def build_response(path, data, http_status_code, headers=None, preload_assets=True): # build response response = Response() response.data = set_content_type(response, data, path) response.status_code = http_status_code response.headers["X-Page-Name"] = path.encode("ascii", errors="xmlcharrefreplace") response.headers["X-From-Cache"] = frappe.local.response.from_cache or False - - add_preload_headers(response) + if preload_assets: + add_preload_headers(response) if headers: for key, val in iteritems(headers): response.headers[key] = val.encode("ascii", errors="xmlcharrefreplace") From 1c5ad30bf1069eb8469f2a47d9fe25dd4199ce40 Mon Sep 17 00:00:00 2001 From: hasnain2808 Date: Sat, 3 Jul 2021 18:44:40 +0530 Subject: [PATCH 021/164] perf: let's preload assets --- frappe/website/utils.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/frappe/website/utils.py b/frappe/website/utils.py index 171c90e1cf..419a5199df 100644 --- a/frappe/website/utils.py +++ b/frappe/website/utils.py @@ -449,15 +449,14 @@ def cache_html(func): return cache_html_decorator -def build_response(path, data, http_status_code, headers=None, preload_assets=True): +def build_response(path, data, http_status_code, headers=None): # build response response = Response() response.data = set_content_type(response, data, path) response.status_code = http_status_code response.headers["X-Page-Name"] = path.encode("ascii", errors="xmlcharrefreplace") response.headers["X-From-Cache"] = frappe.local.response.from_cache or False - if preload_assets: - add_preload_headers(response) + add_preload_headers(response) if headers: for key, val in iteritems(headers): response.headers[key] = val.encode("ascii", errors="xmlcharrefreplace") @@ -486,11 +485,12 @@ def set_content_type(response, data, path): return data def add_preload_headers(response): - from bs4 import BeautifulSoup + from bs4 import BeautifulSoup, SoupStrainer try: preload = [] - soup = BeautifulSoup(response.data, "lxml") + strainer = SoupStrainer(re.compile("script|link")) + soup = BeautifulSoup(response.data, "lxml", parse_only=strainer) for elem in soup.find_all('script', src=re.compile(".*")): preload.append(("script", elem.get("src"))) From 4ddf8c18d8d8e156dcdfa67c191a02015dda66f3 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Mon, 5 Jul 2021 10:38:55 +0530 Subject: [PATCH 022/164] feat: Execute separate build command for style files - Execute style build twice (one for LTR and one for RTL) because rtlcss is dependant on comments and esbuild strips all comments so we cannot run rtlcss on built css files. --- esbuild/esbuild.js | 106 ++++++++++++++++++++++++++------------------- 1 file changed, 62 insertions(+), 44 deletions(-) diff --git a/esbuild/esbuild.js b/esbuild/esbuild.js index a0b07abe43..aa273ab85c 100644 --- a/esbuild/esbuild.js +++ b/esbuild/esbuild.js @@ -97,10 +97,9 @@ async function execute() { await clean_dist_folders(APPS); } - let result; + let results; try { - result = await build_assets_for_apps(APPS, FILES_TO_BUILD); - result = await create_rtl_assets(result); + results = await build_assets_for_apps(APPS, FILES_TO_BUILD); } catch (e) { log_error("There were some problems during build"); log(); @@ -109,13 +108,15 @@ async function execute() { } if (!WATCH_MODE) { - log_built_assets(result.metafile); + log_built_assets(results); console.timeEnd(TOTAL_BUILD_TIME); log(); } else { log("Watching for changes..."); } - return await write_assets_json(result.metafile); + for (const result of results) { + write_assets_json(result.metafile); + } } function build_assets_for_apps(apps, files) { @@ -127,6 +128,8 @@ function build_assets_for_apps(apps, files) { let output_path = assets_path; let file_map = {}; + let style_file_map = {}; + let rtl_style_file_map = {}; for (let file of files) { let relative_app_path = path.relative(apps_path, file); let app = relative_app_path.split(path.sep)[0]; @@ -142,19 +145,32 @@ function build_assets_for_apps(apps, files) { } output_name = path.join(app, "dist", output_name); - if (Object.keys(file_map).includes(output_name)) { + if (Object.keys(file_map).includes(output_name) || Object.keys(style_file_map).includes(output_name)) { log_warn( `Duplicate output file ${output_name} generated from ${file}` ); } - - file_map[output_name] = file; + if ([".css", ".scss", ".less", ".sass", ".styl"].includes(extension)) { + style_file_map[output_name] = file; + rtl_style_file_map[output_name.replace('/css/', '/css-rtl/')] = file; + } else { + file_map[output_name] = file; + } } - - return build_files({ + let build = build_files({ files: file_map, outdir: output_path }); + let style_build = build_style_files({ + files: style_file_map, + outdir: output_path + }); + let rtl_style_build = build_style_files({ + files: rtl_style_file_map, + outdir: output_path, + rtl_style: true + }); + return Promise.all([build, style_build, rtl_style_build]); }); } @@ -205,7 +221,33 @@ function get_files_to_build(files) { } function build_files({ files, outdir }) { - return esbuild.build({ + let build_plugins = [ + html_plugin, + vue(), + ]; + return esbuild.build(get_build_options(files, outdir, build_plugins)); +} + +function build_style_files({ files, outdir, rtl_style=false }) { + let plugins = []; + if (rtl_style) { + plugins.push(rtlcss); + } + + let build_plugins = [ + ignore_assets, + postCssPlugin({ + plugins: plugins, + sassOptions: sass_options + }) + ]; + + plugins.push(require("autoprefixer")); + return esbuild.build(get_build_options(files, outdir, build_plugins)); +} + +function get_build_options(files, outdir, plugins) { + return { entryPoints: files, entryNames: "[dir]/[name].[hash]", outdir, @@ -219,17 +261,9 @@ function build_files({ files, outdir }) { PRODUCTION ? "production" : "development" ) }, - plugins: [ - html_plugin, - ignore_assets, - vue(), - postCssPlugin({ - plugins: [require("autoprefixer")], - sassOptions: sass_options - }) - ], + plugins: plugins, watch: get_watch_config() - }); + }; } function get_watch_config() { @@ -275,7 +309,11 @@ async function clean_dist_folders(apps) { } } -function log_built_assets(metafile) { +function log_built_assets(results) { + let outputs = {}; + for (const result of results) { + outputs = Object.assign(outputs, result.metafile.outputs); + } let column_widths = [60, 20]; cliui.div( { @@ -290,9 +328,9 @@ function log_built_assets(metafile) { cliui.div(""); let output_by_dist_path = {}; - for (let outfile in metafile.outputs) { + for (let outfile in outputs) { if (outfile.endsWith(".map")) continue; - let data = metafile.outputs[outfile]; + let data = outputs[outfile]; outfile = path.resolve(outfile); outfile = path.relative(assets_path, outfile); let filename = path.basename(outfile); @@ -486,24 +524,4 @@ function log_rebuilt_assets(prev_assets, new_assets) { log(" " + filename); } log(); -} - -async function create_rtl_assets(result) { - for (let file_path in result.metafile.outputs) { - if (file_path.endsWith('.css')) { - console.log(file_path); - let content = fs.readFileSync(file_path, {'encoding': 'utf-8'}); - let rtl_content = rtlcss.process(content); - let rtl_file_path = file_path.replace('/css/', '/css-rtl/'); - let rtl_folder_path = path.dirname(rtl_file_path); - if (fs.existsSync(rtl_file_path)) { - continue; - } - if (!fs.existsSync(rtl_folder_path)) { - fs.mkdirSync(rtl_folder_path); - } - fs.writeFileSync(rtl_file_path, rtl_content); - } - } - return result; } \ No newline at end of file From 6df9fd2bf89b3506a958db9dfd0049c89c2e006d Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Mon, 5 Jul 2021 14:10:51 +0530 Subject: [PATCH 023/164] fix: Build asset file from results sequentially --- esbuild/esbuild.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esbuild/esbuild.js b/esbuild/esbuild.js index aa273ab85c..19ab559ccd 100644 --- a/esbuild/esbuild.js +++ b/esbuild/esbuild.js @@ -115,7 +115,7 @@ async function execute() { log("Watching for changes..."); } for (const result of results) { - write_assets_json(result.metafile); + await write_assets_json(result.metafile); } } From 70e4b8a459d53dec867bf294336335b9d02a9f2c Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Mon, 5 Jul 2021 14:39:46 +0530 Subject: [PATCH 024/164] fix: Use different key for RTL assets --- esbuild/esbuild.js | 6 +++++- frappe/public/js/frappe/assets.js | 4 ++-- frappe/utils/jinja_globals.py | 7 ++----- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/esbuild/esbuild.js b/esbuild/esbuild.js index 19ab559ccd..9074beae06 100644 --- a/esbuild/esbuild.js +++ b/esbuild/esbuild.js @@ -385,7 +385,11 @@ async function write_assets_json(metafile) { let info = metafile.outputs[output]; let asset_path = "/" + path.relative(sites_path, output); if (info.entryPoint) { - out[path.basename(info.entryPoint)] = asset_path; + let key = path.basename(info.entryPoint); + if (key.endsWith('.css') && asset_path.includes('/css-rtl/')) { + key = `rtl_${key}`; + } + out[key] = asset_path; } } diff --git a/frappe/public/js/frappe/assets.js b/frappe/public/js/frappe/assets.js index 1f2659992f..0a3fb33cb8 100644 --- a/frappe/public/js/frappe/assets.js +++ b/frappe/public/js/frappe/assets.js @@ -170,10 +170,10 @@ frappe.assets = { bundled_asset(path, is_rtl=null) { if (!path.startsWith('/assets') && path.includes('.bundle.')) { - path = frappe.boot.assets_json[path] || path; if (path.endsWith('.css') && is_rtl) { - path = path.replace('/css/', '/css-rtl/'); + path = `rtl_${path}`; } + path = frappe.boot.assets_json[path] || path; return path; } return path; diff --git a/frappe/utils/jinja_globals.py b/frappe/utils/jinja_globals.py index dd5ade727b..67ca9d108a 100644 --- a/frappe/utils/jinja_globals.py +++ b/frappe/utils/jinja_globals.py @@ -75,9 +75,6 @@ def include_script(path): def include_style(path, rtl=None): path = bundled_asset(path) - - if is_rtl(rtl): - path = path.replace('/css/', '/css-rtl/') return f'' @@ -87,9 +84,9 @@ def bundled_asset(path, rtl=None): if ".bundle." in path and not path.startswith("/assets"): bundled_assets = get_assets_json() - path = bundled_assets.get(path) or path if path.endswith('.css') and is_rtl(rtl): - path = path.replace('/css/', '/css-rtl/') + path = f"rtl_{path}" + path = bundled_assets.get(path) or path return abs_url(path) From 6e92cda34aa91790429e24ab7c25203a6facea7f Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Mon, 5 Jul 2021 14:43:34 +0530 Subject: [PATCH 025/164] fix: Breadcrumb direction for RTL --- frappe/public/scss/desk/breadcrumb.scss | 4 ++-- frappe/public/scss/desk/css_variables.scss | 2 ++ frappe/public/scss/desk/dark.scss | 2 ++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/frappe/public/scss/desk/breadcrumb.scss b/frappe/public/scss/desk/breadcrumb.scss index 6324e6f012..b466bab7ae 100644 --- a/frappe/public/scss/desk/breadcrumb.scss +++ b/frappe/public/scss/desk/breadcrumb.scss @@ -4,7 +4,7 @@ background-color: white; font-size: var(--text-sm); } - +/*! This comment will be included even in compressed mode. */ #navbar-breadcrumbs { margin-left: var(--margin-md); font-size: var(--text-sm); @@ -12,7 +12,7 @@ font-size: var(--text-md); margin-right: 10px; &:before { - content: var(--right-arrow-svg); + content: #{"/*!rtl:var(--left-arrow-svg);*/"}var(--right-arrow-svg); display: inline-block; margin-right: 10px; } diff --git a/frappe/public/scss/desk/css_variables.scss b/frappe/public/scss/desk/css_variables.scss index 5bb2614dcc..c099a8bdeb 100644 --- a/frappe/public/scss/desk/css_variables.scss +++ b/frappe/public/scss/desk/css_variables.scss @@ -77,4 +77,6 @@ $input-height: 28px !default; --skeleton-bg: var(--gray-100); --right-arrow-svg: url("data: image/svg+xml;utf8, "); + + --left-arrow-svg: url("data: image/svg+xml;utf8, "); } diff --git a/frappe/public/scss/desk/dark.scss b/frappe/public/scss/desk/dark.scss index 76dcf90bc3..7f0dfe73b8 100644 --- a/frappe/public/scss/desk/dark.scss +++ b/frappe/public/scss/desk/dark.scss @@ -157,4 +157,6 @@ --skeleton-bg: var(--gray-800); --right-arrow-svg: url("data: image/svg+xml;utf8, "); + + --left-arrow-svg: url("data: image/svg+xml;utf8, "); } From 0b3402236cfd15259cf2bae29ae87f0f306ac951 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Mon, 5 Jul 2021 14:44:04 +0530 Subject: [PATCH 026/164] fix: Add layout direction to pdf template --- frappe/templates/print_formats/pdf_header_footer.html | 2 +- frappe/utils/pdf.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/frappe/templates/print_formats/pdf_header_footer.html b/frappe/templates/print_formats/pdf_header_footer.html index a9c621f505..189cf43cd9 100644 --- a/frappe/templates/print_formats/pdf_header_footer.html +++ b/frappe/templates/print_formats/pdf_header_footer.html @@ -1,5 +1,5 @@ - + diff --git a/frappe/utils/pdf.py b/frappe/utils/pdf.py index fcf483bea6..9cbece5dbe 100644 --- a/frappe/utils/pdf.py +++ b/frappe/utils/pdf.py @@ -13,7 +13,7 @@ from PyPDF2 import PdfFileReader, PdfFileWriter import frappe from frappe import _ from frappe.utils import scrub_urls -from frappe.utils.jinja_globals import bundled_asset +from frappe.utils.jinja_globals import bundled_asset, is_rtl PDF_CONTENT_ERRORS = ["ContentNotFoundError", "ContentOperationNotPermittedError", "UnknownContentError", "RemoteHostClosedError"] @@ -177,7 +177,9 @@ def prepare_header_footer(soup): "content": content, "styles": styles, "html_id": html_id, - "css": css + "css": css, + "lang": frappe.local.lang, + "layout_direction": "rtl" if is_rtl else "ltr" }) # create temp file From 0228b9c8ebdd20b7b76522b697cc6c4bfe46c949 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Mon, 5 Jul 2021 18:02:35 +0530 Subject: [PATCH 027/164] fix: Add RTL control directive to override dropdown style in RTL mode --- frappe/public/scss/desk/global.scss | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/frappe/public/scss/desk/global.scss b/frappe/public/scss/desk/global.scss index afcf2957cc..1168c8ce8c 100644 --- a/frappe/public/scss/desk/global.scss +++ b/frappe/public/scss/desk/global.scss @@ -302,6 +302,15 @@ select.input-xs { } } +/*!rtl:raw: +.dropdown-menu { + right: auto; +} +.popover { + right: auto; +} +*/ + .custom-control.custom-switch { font-size: var(--text-md); line-height: 1.6; From d1555263e1823f9cd6df626e0c5caa4203c508cf Mon Sep 17 00:00:00 2001 From: barredterra <14891507+barredterra@users.noreply.github.com> Date: Mon, 5 Jul 2021 19:52:12 +0200 Subject: [PATCH 028/164] refactor: suggestions from review - use -f / -- format instead of output - get rid of ugly if / else statements --- frappe/commands/utils.py | 46 +++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py index 7d3aecca1f..1cecdffe1b 100644 --- a/frappe/commands/utils.py +++ b/frappe/commands/utils.py @@ -767,14 +767,15 @@ def set_config(context, key, value, global_=False, parse=False, as_dict=False): frappe.destroy() -@click.command('version') -@click.option('--output', help='Output format. One of: plain, table, json, legacy.', default='legacy') +@click.command("version") +@click.option("-f", "--format", "output", + type=click.Choice(["plain", "table", "json", "legacy"]), help="Output format", default="legacy") def get_version(output): - "Show the versions of all the installed apps" + """Show the versions of all the installed apps.""" from git import Repo from frappe.utils.commands import render_table - frappe.init('') + frappe.init("") data = [] for app in sorted(frappe.get_all_apps()): @@ -786,27 +787,28 @@ def get_version(output): app_info.app = app app_info.branch = repo.head.ref.name app_info.commit = repo.head.ref.commit.hexsha[:7] - - if hasattr(app_hooks, '{0}_version'.format(app_info.branch)): - app_info.version = getattr(app_hooks, '{0}_version'.format(app_info.branch)) - elif hasattr(module, "__version__"): - app_info.version = module.__version__ + app_info.version = getattr(app_hooks, f"{app_info.branch}_version", None) or module.__version__ data.append(app_info) - if output == 'table': - table = [['App', 'Version', 'Branch', 'Commit']] - for app_info in data: - table.append([app_info.app, app_info.version, app_info.branch, app_info.commit]) - render_table(table) - elif output == 'json': - click.echo(json.dumps(data, indent=4)) - elif output == 'plain': - for app_info in data: - click.echo(f'{app_info.app} {app_info.version} {app_info.branch} ({app_info.commit})') - else: # legacy - for app_info in data: - click.echo(f'{app_info.app} {app_info.version}') + { + "legacy": lambda: [ + click.echo(f"{app_info.app} {app_info.version}") + for app_info in data + ], + "plain": lambda: [ + click.echo(f"{app_info.app} {app_info.version} {app_info.branch} ({app_info.commit})") + for app_info in data + ], + "table": lambda: render_table( + [["App", "Version", "Branch", "Commit"]] + + [ + [app_info.app, app_info.version, app_info.branch, app_info.commit] + for app_info in data + ] + ), + "json": lambda: click.echo(json.dumps(data, indent=4)), + }[output]() @click.command('rebuild-global-search') From d7e0479ee742073fc1093ef9f2177153e93ae4a2 Mon Sep 17 00:00:00 2001 From: barredterra <14891507+barredterra@users.noreply.github.com> Date: Mon, 5 Jul 2021 19:53:02 +0200 Subject: [PATCH 029/164] test: add test for `bench version` --- frappe/tests/test_commands.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/frappe/tests/test_commands.py b/frappe/tests/test_commands.py index 07bdf8791e..8f9bbb1afb 100644 --- a/frappe/tests/test_commands.py +++ b/frappe/tests/test_commands.py @@ -426,3 +426,13 @@ class TestCommands(BaseTestCommands): self.assertEqual(self.returncode, 0) self.assertIn("pong", self.stdout) + def test_version(self): + self.execute("bench version") + self.assertEqual(self.returncode, 0) + + for output in ["legacy", "plain", "table", "json"]: + self.execute(f"bench version -f {output}") + self.assertEqual(self.returncode, 0) + + self.execute("bench version -f invalid") + self.assertEqual(self.returncode, 1) From 1fa7011b886c7de1ae77b817963994e5dbdf8c3d Mon Sep 17 00:00:00 2001 From: hasnain2808 Date: Tue, 6 Jul 2021 11:59:23 +0530 Subject: [PATCH 030/164] fix: system notifications without email --- frappe/desk/doctype/notification_log/notification_log.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/frappe/desk/doctype/notification_log/notification_log.py b/frappe/desk/doctype/notification_log/notification_log.py index 414f272f59..f5f1b3b124 100644 --- a/frappe/desk/doctype/notification_log/notification_log.py +++ b/frappe/desk/doctype/notification_log/notification_log.py @@ -12,7 +12,10 @@ class NotificationLog(Document): frappe.publish_realtime('notification', after_commit=True, user=self.for_user) set_notifications_as_unseen(self.for_user) if is_email_notifications_enabled_for_type(self.for_user, self.type): - send_notification_email(self) + try: + send_notification_email(self) + except frappe.OutgoingEmailError: + frappe.log_error(message = frappe.get_traceback(), title=_("Error Sending Notification Email(System Notification sent).")) def get_permission_query_conditions(for_user): From 60322c58e1e371ce14ac47c6b579625eb7865a53 Mon Sep 17 00:00:00 2001 From: Saqib Ansari Date: Wed, 7 Jul 2021 12:29:45 +0530 Subject: [PATCH 031/164] fix: init driver only if necessary --- frappe/public/js/frappe/form/form_tour.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/frappe/public/js/frappe/form/form_tour.js b/frappe/public/js/frappe/form/form_tour.js index 5f4920f415..0cdefa1968 100644 --- a/frappe/public/js/frappe/form/form_tour.js +++ b/frappe/public/js/frappe/form/form_tour.js @@ -2,8 +2,6 @@ frappe.ui.form.FormTour = class FormTour { constructor({ frm }) { this.frm = frm; this.driver_steps = []; - - this.init_driver(); } init_driver() { @@ -48,6 +46,7 @@ frappe.ui.form.FormTour = class FormTour { if (on_finish) this.on_finish = on_finish; + this.init_driver(); this.build_steps(); this.update_driver_steps(); } From acf46c4ddd604da766d50cb5c44a16b847a0d59f Mon Sep 17 00:00:00 2001 From: Saqib Ansari Date: Wed, 7 Jul 2021 12:31:04 +0530 Subject: [PATCH 032/164] fix: add missing semicolon --- frappe/public/js/frappe/form/form_tour.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/form/form_tour.js b/frappe/public/js/frappe/form/form_tour.js index 0cdefa1968..ee18db055e 100644 --- a/frappe/public/js/frappe/form/form_tour.js +++ b/frappe/public/js/frappe/form/form_tour.js @@ -38,7 +38,7 @@ frappe.ui.form.FormTour = class FormTour { } else { const doctype_tour_exists = await frappe.db.exists('Form Tour', this.frm.doctype); if (doctype_tour_exists) { - this.tour = await frappe.db.get_doc('Form Tour', this.frm.doctype) + this.tour = await frappe.db.get_doc('Form Tour', this.frm.doctype); } else { this.tour = { steps: frappe.tour[this.frm.doctype] }; } From e9ec018b23c4af7ecd314051e3aa58401c6fe177 Mon Sep 17 00:00:00 2001 From: Saqib Ansari Date: Wed, 7 Jul 2021 12:40:40 +0530 Subject: [PATCH 033/164] revert: "feat: submit on completion" --- frappe/desk/doctype/form_tour/form_tour.json | 10 +---- frappe/public/js/frappe/form/form_tour.js | 42 ++------------------ 2 files changed, 4 insertions(+), 48 deletions(-) diff --git a/frappe/desk/doctype/form_tour/form_tour.json b/frappe/desk/doctype/form_tour/form_tour.json index 44f39766b3..e4ea528fcc 100644 --- a/frappe/desk/doctype/form_tour/form_tour.json +++ b/frappe/desk/doctype/form_tour/form_tour.json @@ -11,7 +11,6 @@ "module", "is_standard", "save_on_complete", - "submit_on_complete", "section_break_3", "steps" ], @@ -63,18 +62,11 @@ "label": "Module", "options": "Module Def", "read_only": 1 - }, - { - "default": "0", - "depends_on": "save_on_complete", - "fieldname": "submit_on_complete", - "fieldtype": "Check", - "label": "Submit on Completion" } ], "index_web_pages_for_search": 1, "links": [], - "modified": "2021-06-29 15:23:26.893068", + "modified": "2021-06-06 20:32:54.068774", "modified_by": "Administrator", "module": "Desk", "name": "Form Tour", diff --git a/frappe/public/js/frappe/form/form_tour.js b/frappe/public/js/frappe/form/form_tour.js index ee18db055e..6bef2a0cb8 100644 --- a/frappe/public/js/frappe/form/form_tour.js +++ b/frappe/public/js/frappe/form/form_tour.js @@ -15,9 +15,8 @@ frappe.ui.form.FormTour = class FormTour { prevBtnText: 'Previous', opacity: 0.25, onHighlighted: (step) => { - // if last step is to save, or if is submit step - // then attach a listener on highlighted elem to reset the driver - if ((step.options.is_save_step && !this.driver.hasNextStep())|| step.options.is_submit_step) { + // if last step is to save, then attach a listener to save button + if (step.options.is_save_step) { $(step.options.element).one('click', () => this.driver.reset()); } @@ -74,10 +73,6 @@ frappe.ui.form.FormTour = class FormTour { if (this.tour.save_on_complete) { this.add_step_to_save(); } - - if (this.tour.submit_on_complete) { - this.add_step_to_submit(); - } } is_next_condition_satisfied(step) { @@ -259,37 +254,6 @@ frappe.ui.form.FormTour = class FormTour { } }; this.driver_steps.push(save_step); - - let after_save = () => this.on_finish && this.on_finish(); - - if (this.tour.submit_on_complete) { - after_save = () => { - this.update_driver_steps(); - this.driver.start(this.driver.steps.length - 1); - } - } - frappe.ui.form.on(this.frm.doctype, 'after_save', after_save); - } - - add_step_to_submit() { - const page_id = `[id="page-${this.frm.doctype}"]`; - const $submit_btn = `${page_id} .standard-actions .primary-action`; - const submit_step = { - element: $submit_btn, - is_submit_step: true, - allowClose: false, - overlayClickNext: false, - popover: { - title: __("Submit"), - description: "", - position: "left", - doneBtnText: __("Submit") - }, - onNext: () => { - this.frm.savesubmit(); - } - }; - this.driver_steps.push(submit_step); - frappe.ui.form.on(this.frm.doctype, 'after_submit', () => this.on_finish && this.on_finish()); + frappe.ui.form.on(this.frm.doctype, 'after_save', () => this.on_finish && this.on_finish()); } }; \ No newline at end of file From ecfa8c843f13e6ab7abb629a51a8ce7c44df386e Mon Sep 17 00:00:00 2001 From: codescientist703 Date: Wed, 7 Jul 2021 17:33:59 +0530 Subject: [PATCH 034/164] fix: child results should appear for parent search query --- frappe/desk/search.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/frappe/desk/search.py b/frappe/desk/search.py index 64ae7f7b7a..b0e9592c72 100644 --- a/frappe/desk/search.py +++ b/frappe/desk/search.py @@ -167,15 +167,16 @@ def search_widget(doctype, txt, query=None, searchfield=None, start=0, as_list=not as_dict, strict=False) - # Filtering the values array so that query is included in very element - values = tuple( - [ - v for v in list(values) - if re.search( - re.escape(txt) + ".*", (_(v.name) if as_dict else _(v[0])), re.IGNORECASE - ) - ] - ) + if doctype in UNTRANSLATED_DOCTYPES: + # Filtering the values array so that query is included in very element + values = tuple( + [ + v for v in list(values) + if re.search( + re.escape(txt) + ".*", (_(v.name) if as_dict else _(v[0])), re.IGNORECASE + ) + ] + ) # Sorting the values array so that relevant results always come first # This will first bring elements on top in which query is a prefix of element From ba062adca84dfecb0e129c292b30303156213db4 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 8 Jul 2021 13:36:25 +0530 Subject: [PATCH 035/164] refactor(search): Improvements in search_widget, search_link APIs * Minor perf enhancements * Renamed sorting_comparator to relevance_sorter --- frappe/desk/search.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/frappe/desk/search.py b/frappe/desk/search.py index b0e9592c72..3cce80a1a0 100644 --- a/frappe/desk/search.py +++ b/frappe/desk/search.py @@ -169,19 +169,17 @@ def search_widget(doctype, txt, query=None, searchfield=None, start=0, if doctype in UNTRANSLATED_DOCTYPES: # Filtering the values array so that query is included in very element - values = tuple( - [ - v for v in list(values) - if re.search( - re.escape(txt) + ".*", (_(v.name) if as_dict else _(v[0])), re.IGNORECASE - ) - ] + values = ( + v for v in values + if re.search( + f"{re.escape(txt)}.*", _(v.name if as_dict else v[0]), re.IGNORECASE ) + ) # Sorting the values array so that relevant results always come first # This will first bring elements on top in which query is a prefix of element # Then it will bring the rest of the elements and sort them in lexicographical order - values = sorted(values, key=lambda x: sorting_comparator(x, txt, as_dict)) + values = sorted(values, key=lambda x: relevance_sorter(x, txt, as_dict)) # remove _relevance from results if as_dict: @@ -221,10 +219,10 @@ def scrub_custom_query(query, key, txt): query = query.replace('%s', ((txt or '') + '%')) return query -def sorting_comparator(key, query, as_dict): - value = (_(key.name) if as_dict else _(key[0])) +def relevance_sorter(key, query, as_dict): + value = _(key.name if as_dict else key[0]) return ( - value.lower().startswith(query.lower()) is False, + value.lower().startswith(query.lower()) == False, value ) From fe21efb76f82554602aa150b04d7a484cff9500a Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Mon, 12 Jul 2021 10:06:06 +0530 Subject: [PATCH 036/164] fix: Keep charts as it is - Frappe Charts do not support RTL and rtlcss does not work on SVG... Setting direction as ltr to keep charts as it is --- frappe/public/scss/desk/global.scss | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/frappe/public/scss/desk/global.scss b/frappe/public/scss/desk/global.scss index 1168c8ce8c..d1205e0e38 100644 --- a/frappe/public/scss/desk/global.scss +++ b/frappe/public/scss/desk/global.scss @@ -302,15 +302,6 @@ select.input-xs { } } -/*!rtl:raw: -.dropdown-menu { - right: auto; -} -.popover { - right: auto; -} -*/ - .custom-control.custom-switch { font-size: var(--text-md); line-height: 1.6; @@ -583,3 +574,15 @@ details > summary:focus { // font-family: 'Octicons'; // content: "\f00b"; // } + +/*rtl:raw: +.dropdown-menu { + right: auto; +} +.popover { + right: auto; +} +.chart-container { + direction: ltr; +} +*/ \ No newline at end of file From 10fde05950d1284a2aeafa15367260f4231fffa5 Mon Sep 17 00:00:00 2001 From: rohitwaghchaure Date: Mon, 12 Jul 2021 17:43:15 +0530 Subject: [PATCH 037/164] fix: custom script to duplicate row not working in v13 (#13667) (cherry picked from commit 1615baf69a476a4c8361cd8af19e2edada6b4784) --- frappe/public/js/frappe/form/form.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js index 8064f90a98..faaa3dfbd9 100644 --- a/frappe/public/js/frappe/form/form.js +++ b/frappe/public/js/frappe/form/form.js @@ -1265,7 +1265,9 @@ frappe.ui.form.Form = class FrappeForm { if (df && df[property] != value) { df[property] = value; if (table_field && table_row_name) { - this.fields_dict[fieldname].grid.grid_rows_by_docname[table_row_name].refresh_field(fieldname); + if (this.fields_dict[fieldname].grid.grid_rows_by_docname[table_row_name]) { + this.fields_dict[fieldname].grid.grid_rows_by_docname[table_row_name].refresh_field(fieldname); + } } else { this.refresh_field(fieldname); } From 45cdf7a2695a4e123184982167e736e04a1c140e Mon Sep 17 00:00:00 2001 From: Mohammad Hasnain Mohsin Rajan Date: Tue, 13 Jul 2021 11:15:22 +0530 Subject: [PATCH 038/164] chore: correct error message --- frappe/desk/doctype/notification_log/notification_log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/desk/doctype/notification_log/notification_log.py b/frappe/desk/doctype/notification_log/notification_log.py index f5f1b3b124..af09d65015 100644 --- a/frappe/desk/doctype/notification_log/notification_log.py +++ b/frappe/desk/doctype/notification_log/notification_log.py @@ -15,7 +15,7 @@ class NotificationLog(Document): try: send_notification_email(self) except frappe.OutgoingEmailError: - frappe.log_error(message = frappe.get_traceback(), title=_("Error Sending Notification Email(System Notification sent).")) + frappe.log_error(message = frappe.get_traceback(), title=_("Error Sending Notification Email.")) def get_permission_query_conditions(for_user): From dc50b77179326bd8f06fa524b6e4182ac1737ef5 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 23 Jun 2021 13:08:25 +0530 Subject: [PATCH 039/164] chore: Drop Data Import Legacy --- frappe/commands/utils.py | 32 +- .../doctype/data_import_legacy/__init__.py | 0 .../data_import_legacy/data_import_legacy.js | 324 ----------- .../data_import_legacy.json | 218 ------- .../data_import_legacy/data_import_legacy.py | 126 ---- .../data_import_legacy_list.js | 24 - .../doctype/data_import_legacy/importer.py | 538 ------------------ .../data_import_legacy/log_details.html | 38 -- .../test_data_import_legacy.py | 8 - 9 files changed, 6 insertions(+), 1302 deletions(-) delete mode 100644 frappe/core/doctype/data_import_legacy/__init__.py delete mode 100644 frappe/core/doctype/data_import_legacy/data_import_legacy.js delete mode 100644 frappe/core/doctype/data_import_legacy/data_import_legacy.json delete mode 100644 frappe/core/doctype/data_import_legacy/data_import_legacy.py delete mode 100644 frappe/core/doctype/data_import_legacy/data_import_legacy_list.js delete mode 100644 frappe/core/doctype/data_import_legacy/importer.py delete mode 100644 frappe/core/doctype/data_import_legacy/log_details.html delete mode 100644 frappe/core/doctype/data_import_legacy/test_data_import_legacy.py diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py index ca58e78870..506ec7816b 100644 --- a/frappe/commands/utils.py +++ b/frappe/commands/utils.py @@ -358,32 +358,12 @@ def import_doc(context, path, force=False): @click.option('--no-email', default=True, is_flag=True, help='Send email if applicable') @pass_context def import_csv(context, path, only_insert=False, submit_after_import=False, ignore_encoding_errors=False, no_email=True): - "Import CSV using data import" - from frappe.core.doctype.data_import_legacy import importer - from frappe.utils.csvutils import read_csv_content - site = get_site(context) - - if not os.path.exists(path): - path = os.path.join('..', path) - if not os.path.exists(path): - print('Invalid path {0}'.format(path)) - sys.exit(1) - - with open(path, 'r') as csvfile: - content = read_csv_content(csvfile.read()) - - frappe.init(site=site) - frappe.connect() - - try: - importer.upload(content, submit_after_import=submit_after_import, no_email=no_email, - ignore_encoding_errors=ignore_encoding_errors, overwrite=not only_insert, - via_console=True) - frappe.db.commit() - except Exception: - print(frappe.get_traceback()) - - frappe.destroy() + click.secho( + "The `import-csv` command used 'Data Import Legacy' which has been deprecated.\n" + "Use `data-import` command instead to import data via 'Data Import'.", + fg="yellow", + ) + sys.exit(1) @click.command('data-import') diff --git a/frappe/core/doctype/data_import_legacy/__init__.py b/frappe/core/doctype/data_import_legacy/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/core/doctype/data_import_legacy/data_import_legacy.js b/frappe/core/doctype/data_import_legacy/data_import_legacy.js deleted file mode 100644 index 8e4f397171..0000000000 --- a/frappe/core/doctype/data_import_legacy/data_import_legacy.js +++ /dev/null @@ -1,324 +0,0 @@ -// Copyright (c) 2017, Frappe Technologies and contributors -// For license information, please see license.txt - -frappe.ui.form.on('Data Import Legacy', { - onload: function(frm) { - if (frm.doc.__islocal) { - frm.set_value("action", ""); - } - - frappe.call({ - method: "frappe.core.doctype.data_import_legacy.data_import_legacy.get_importable_doctypes", - callback: function (r) { - let importable_doctypes = r.message; - frm.set_query("reference_doctype", function () { - return { - "filters": { - "issingle": 0, - "istable": 0, - "name": ['in', importable_doctypes] - } - }; - }); - } - }), - - // should never check public - frm.fields_dict["import_file"].df.is_private = 1; - - frappe.realtime.on("data_import_progress", function(data) { - if (data.data_import === frm.doc.name) { - if (data.reload && data.reload === true) { - frm.reload_doc(); - } - if (data.progress) { - let progress_bar = $(frm.dashboard.progress_area.body).find(".progress-bar"); - if (progress_bar) { - $(progress_bar).removeClass("progress-bar-danger").addClass("progress-bar-success progress-bar-striped"); - $(progress_bar).css("width", data.progress + "%"); - } - } - } - }); - }, - - reference_doctype: function(frm){ - if (frm.doc.reference_doctype) { - frappe.model.with_doctype(frm.doc.reference_doctype); - } - }, - - refresh: function(frm) { - frm.disable_save(); - frm.dashboard.clear_headline(); - if (frm.doc.reference_doctype && !frm.doc.import_file) { - frm.page.set_indicator(__('Attach file'), 'orange'); - } else { - if (frm.doc.import_status) { - const listview_settings = frappe.listview_settings['Data Import Legacy']; - const indicator = listview_settings.get_indicator(frm.doc); - - frm.page.set_indicator(indicator[0], indicator[1]); - - if (frm.doc.import_status === "In Progress") { - frm.dashboard.add_progress("Data Import Progress", "0"); - frm.set_read_only(); - frm.refresh_fields(); - } - } - } - - if (frm.doc.reference_doctype) { - frappe.model.with_doctype(frm.doc.reference_doctype); - } - - if(frm.doc.action == "Insert new records" || frm.doc.action == "Update records") { - frm.set_df_property("action", "read_only", 1); - } - - frm.add_custom_button(__("Help"), function() { - frappe.help.show_video("6wiriRKPhmg"); - }); - - if (frm.doc.reference_doctype && frm.doc.docstatus === 0) { - frm.add_custom_button(__("Download template"), function() { - frappe.data_import.download_dialog(frm).show(); - }); - } - - if (frm.doc.reference_doctype && frm.doc.import_file && frm.doc.total_rows && - frm.doc.docstatus === 0 && (!frm.doc.import_status || frm.doc.import_status == "Failed")) { - frm.page.set_primary_action(__("Start Import"), function() { - frappe.call({ - btn: frm.page.btn_primary, - method: "frappe.core.doctype.data_import_legacy.data_import_legacy.import_data", - args: { - data_import: frm.doc.name - } - }); - }).addClass('btn btn-primary'); - } - - if (frm.doc.log_details) { - frm.events.create_log_table(frm); - } else { - $(frm.fields_dict.import_log.wrapper).empty(); - } - }, - - action: function(frm) { - if(!frm.doc.action) return; - if(!frm.doc.reference_doctype) { - frappe.msgprint(__("Please select document type first.")); - frm.set_value("action", ""); - return; - } - - if(frm.doc.action == "Insert new records") { - frm.doc.insert_new = 1; - } else if (frm.doc.action == "Update records"){ - frm.doc.overwrite = 1; - } - frm.save(); - }, - - only_update: function(frm) { - frm.save(); - }, - - submit_after_import: function(frm) { - frm.save(); - }, - - skip_errors: function(frm) { - frm.save(); - }, - - ignore_encoding_errors: function(frm) { - frm.save(); - }, - - no_email: function(frm) { - frm.save(); - }, - - show_only_errors: function(frm) { - frm.events.create_log_table(frm); - }, - - create_log_table: function(frm) { - let msg = JSON.parse(frm.doc.log_details); - var $log_wrapper = $(frm.fields_dict.import_log.wrapper).empty(); - $(frappe.render_template("log_details", { - data: msg.messages, - import_status: frm.doc.import_status, - show_only_errors: frm.doc.show_only_errors, - })).appendTo($log_wrapper); - } -}); - -frappe.provide('frappe.data_import'); -frappe.data_import.download_dialog = function(frm) { - var dialog; - const filter_fields = df => frappe.model.is_value_type(df) && !df.hidden; - const get_fields = dt => frappe.meta.get_docfields(dt).filter(filter_fields); - - const get_doctype_checkbox_fields = () => { - return dialog.fields.filter(df => df.fieldname.endsWith('_fields')) - .map(df => dialog.fields_dict[df.fieldname]); - }; - - const doctype_fields = get_fields(frm.doc.reference_doctype) - .map(df => { - let reqd = (df.reqd || df.fieldname == 'naming_series') ? 1 : 0; - return { - label: df.label, - reqd: reqd, - danger: reqd, - value: df.fieldname, - checked: 1 - }; - }); - - let fields = [ - { - "label": __("Select Columns"), - "fieldname": "select_columns", - "fieldtype": "Select", - "options": "All\nMandatory\nManually", - "reqd": 1, - "onchange": function() { - const fields = get_doctype_checkbox_fields(); - fields.map(f => f.toggle(true)); - if(this.value == 'Mandatory' || this.value == 'Manually') { - checkbox_toggle(true); - fields.map(multicheck_field => { - multicheck_field.options.map(option => { - if(!option.reqd) return; - $(multicheck_field.$wrapper).find(`:checkbox[data-unit="${option.value}"]`) - .prop('checked', false) - .trigger('click'); - }); - }); - } else if(this.value == 'All'){ - $(dialog.body).find(`[data-fieldtype="MultiCheck"] :checkbox`) - .prop('disabled', true); - } - } - }, - { - "label": __("File Type"), - "fieldname": "file_type", - "fieldtype": "Select", - "options": "Excel\nCSV", - "default": "Excel" - }, - { - "label": __("Download with Data"), - "fieldname": "with_data", - "fieldtype": "Check", - "hidden": !frm.doc.overwrite, - "default": 1 - }, - { - "label": __("Select All"), - "fieldname": "select_all", - "fieldtype": "Button", - "depends_on": "eval:doc.select_columns=='Manually'", - click: function() { - checkbox_toggle(); - } - }, - { - "label": __("Unselect All"), - "fieldname": "unselect_all", - "fieldtype": "Button", - "depends_on": "eval:doc.select_columns=='Manually'", - click: function() { - checkbox_toggle(true); - } - }, - { - "label": frm.doc.reference_doctype, - "fieldname": "doctype_fields", - "fieldtype": "MultiCheck", - "options": doctype_fields, - "columns": 2, - "hidden": 1 - } - ]; - - const child_table_fields = frappe.meta.get_table_fields(frm.doc.reference_doctype) - .map(df => { - return { - "label": df.options, - "fieldname": df.fieldname + '_fields', - "fieldtype": "MultiCheck", - "options": frappe.meta.get_docfields(df.options) - .filter(filter_fields) - .map(df => ({ - label: df.label, - reqd: df.reqd ? 1 : 0, - value: df.fieldname, - checked: 1, - danger: df.reqd - })), - "columns": 2, - "hidden": 1 - }; - }); - - fields = fields.concat(child_table_fields); - - dialog = new frappe.ui.Dialog({ - title: __('Download Template'), - fields: fields, - primary_action: function(values) { - var data = values; - if (frm.doc.reference_doctype) { - var export_params = () => { - let columns = {}; - if(values.select_columns) { - columns = get_doctype_checkbox_fields().reduce((columns, field) => { - const options = field.get_checked_options(); - columns[field.df.label] = options; - return columns; - }, {}); - } - - return { - doctype: frm.doc.reference_doctype, - parent_doctype: frm.doc.reference_doctype, - select_columns: JSON.stringify(columns), - with_data: frm.doc.overwrite && data.with_data, - all_doctypes: true, - file_type: data.file_type, - template: true - }; - }; - let get_template_url = '/api/method/frappe.core.doctype.data_export.exporter.export_data'; - open_url_post(get_template_url, export_params()); - } else { - frappe.msgprint(__("Please select the Document Type.")); - } - dialog.hide(); - }, - primary_action_label: __('Download') - }); - - $(dialog.body).find('div[data-fieldname="select_all"], div[data-fieldname="unselect_all"]') - .wrapAll('
'); - const button_container = $(dialog.body).find('.inline-buttons'); - button_container.addClass('flex'); - $(button_container).find('.frappe-control').map((index, button) => { - $(button).css({"margin-right": "1em"}); - }); - - function checkbox_toggle(checked=false) { - $(dialog.body).find('[data-fieldtype="MultiCheck"]').map((index, element) => { - $(element).find(`:checkbox`).prop("checked", checked).trigger('click'); - }); - } - - return dialog; -}; diff --git a/frappe/core/doctype/data_import_legacy/data_import_legacy.json b/frappe/core/doctype/data_import_legacy/data_import_legacy.json deleted file mode 100644 index 852ccba156..0000000000 --- a/frappe/core/doctype/data_import_legacy/data_import_legacy.json +++ /dev/null @@ -1,218 +0,0 @@ -{ - "actions": [], - "allow_copy": 1, - "creation": "2020-06-11 16:13:23.813709", - "doctype": "DocType", - "document_type": "Document", - "editable_grid": 1, - "engine": "InnoDB", - "field_order": [ - "reference_doctype", - "action", - "insert_new", - "overwrite", - "only_update", - "section_break_4", - "import_file", - "column_break_4", - "error_file", - "section_break_6", - "skip_errors", - "submit_after_import", - "ignore_encoding_errors", - "no_email", - "import_detail", - "import_status", - "show_only_errors", - "import_log", - "log_details", - "amended_from", - "total_rows", - "amended_from" - ], - "fields": [ - { - "fieldname": "reference_doctype", - "fieldtype": "Link", - "ignore_user_permissions": 1, - "in_list_view": 1, - "label": "Document Type", - "options": "DocType", - "reqd": 1 - }, - { - "fieldname": "action", - "fieldtype": "Select", - "label": "Action", - "options": "Insert new records\nUpdate records", - "reqd": 1 - }, - { - "default": "0", - "depends_on": "eval:!doc.overwrite", - "description": "New data will be inserted.", - "fieldname": "insert_new", - "fieldtype": "Check", - "hidden": 1, - "label": "Insert new records", - "set_only_once": 1 - }, - { - "default": "0", - "depends_on": "eval:!doc.insert_new", - "description": "If you are updating/overwriting already created records.", - "fieldname": "overwrite", - "fieldtype": "Check", - "hidden": 1, - "label": "Update records", - "set_only_once": 1 - }, - { - "default": "0", - "depends_on": "overwrite", - "description": "If you don't want to create any new records while updating the older records.", - "fieldname": "only_update", - "fieldtype": "Check", - "label": "Don't create new records" - }, - { - "depends_on": "eval:(!doc.__islocal)", - "fieldname": "section_break_4", - "fieldtype": "Section Break" - }, - { - "fieldname": "import_file", - "fieldtype": "Attach", - "label": "Attach file for Import" - }, - { - "fieldname": "column_break_4", - "fieldtype": "Column Break" - }, - { - "depends_on": "eval: doc.import_status == \"Partially Successful\"", - "description": "This is the template file generated with only the rows having some error. You should use this file for correction and import.", - "fieldname": "error_file", - "fieldtype": "Attach", - "label": "Generated File" - }, - { - "depends_on": "eval:(!doc.__islocal)", - "fieldname": "section_break_6", - "fieldtype": "Section Break" - }, - { - "default": "0", - "description": "If this is checked, rows with valid data will be imported and invalid rows will be dumped into a new file for you to import later.", - "fieldname": "skip_errors", - "fieldtype": "Check", - "label": "Skip rows with errors" - }, - { - "default": "0", - "fieldname": "submit_after_import", - "fieldtype": "Check", - "label": "Submit after importing" - }, - { - "default": "0", - "fieldname": "ignore_encoding_errors", - "fieldtype": "Check", - "label": "Ignore encoding errors" - }, - { - "default": "1", - "fieldname": "no_email", - "fieldtype": "Check", - "label": "Do not send Emails" - }, - { - "collapsible": 1, - "collapsible_depends_on": "eval: doc.import_status == \"Failed\"", - "depends_on": "import_status", - "fieldname": "import_detail", - "fieldtype": "Section Break", - "label": "Import Log" - }, - { - "fieldname": "import_status", - "fieldtype": "Select", - "label": "Import Status", - "options": "\nSuccessful\nFailed\nIn Progress\nPartially Successful", - "read_only": 1 - }, - { - "allow_on_submit": 1, - "default": "1", - "fieldname": "show_only_errors", - "fieldtype": "Check", - "label": "Show only errors", - "no_copy": 1, - "print_hide": 1 - }, - { - "allow_on_submit": 1, - "depends_on": "import_status", - "fieldname": "import_log", - "fieldtype": "HTML", - "label": "Import Log" - }, - { - "allow_on_submit": 1, - "fieldname": "log_details", - "fieldtype": "Code", - "hidden": 1, - "label": "Log Details", - "read_only": 1 - }, - { - "fieldname": "amended_from", - "fieldtype": "Link", - "label": "Amended From", - "no_copy": 1, - "options": "Data Import", - "print_hide": 1, - "read_only": 1 - }, - { - "fieldname": "total_rows", - "fieldtype": "Int", - "hidden": 1, - "label": "Total Rows", - "read_only": 1 - }, - { - "fieldname": "amended_from", - "fieldtype": "Link", - "label": "Amended From", - "no_copy": 1, - "options": "Data Import Legacy", - "print_hide": 1, - "read_only": 1 - } - ], - "is_submittable": 1, - "links": [], - "max_attachments": 1, - "modified": "2020-06-11 16:13:23.813709", - "modified_by": "Administrator", - "module": "Core", - "name": "Data Import Legacy", - "owner": "Administrator", - "permissions": [ - { - "create": 1, - "delete": 1, - "email": 1, - "read": 1, - "role": "System Manager", - "share": 1, - "submit": 1, - "write": 1 - } - ], - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1, - "track_seen": 1 -} \ No newline at end of file diff --git a/frappe/core/doctype/data_import_legacy/data_import_legacy.py b/frappe/core/doctype/data_import_legacy/data_import_legacy.py deleted file mode 100644 index 63f806d75b..0000000000 --- a/frappe/core/doctype/data_import_legacy/data_import_legacy.py +++ /dev/null @@ -1,126 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Frappe Technologies and contributors -# For license information, please see license.txt - -import os - -import frappe -import frappe.modules.import_file -from frappe import _ -from frappe.core.doctype.data_import_legacy.importer import upload -from frappe.model.document import Document -from frappe.modules.import_file import import_file_by_path as _import_file_by_path -from frappe.utils.background_jobs import enqueue -from frappe.utils.data import format_datetime - - -class DataImportLegacy(Document): - def autoname(self): - if not self.name: - self.name = "Import on " + format_datetime(self.creation) - - def validate(self): - if not self.import_file: - self.db_set("total_rows", 0) - if self.import_status == "In Progress": - frappe.throw(_("Can't save the form as data import is in progress.")) - - # validate the template just after the upload - # if there is total_rows in the doc, it means that the template is already validated and error free - if self.import_file and not self.total_rows: - upload(data_import_doc=self, from_data_import="Yes", validate_template=True) - - -@frappe.whitelist() -def get_importable_doctypes(): - return frappe.cache().hget("can_import", frappe.session.user) - - -@frappe.whitelist() -def import_data(data_import): - frappe.db.set_value("Data Import Legacy", data_import, "import_status", "In Progress", update_modified=False) - frappe.publish_realtime("data_import_progress", {"progress": "0", - "data_import": data_import, "reload": True}, user=frappe.session.user) - - from frappe.core.page.background_jobs.background_jobs import get_info - enqueued_jobs = [d.get("job_name") for d in get_info()] - - if data_import not in enqueued_jobs: - enqueue(upload, queue='default', timeout=6000, event='data_import', job_name=data_import, - data_import_doc=data_import, from_data_import="Yes", user=frappe.session.user) - - -def import_doc(path, overwrite=False, ignore_links=False, ignore_insert=False, - insert=False, submit=False, pre_process=None): - if os.path.isdir(path): - files = [os.path.join(path, f) for f in os.listdir(path)] - else: - files = [path] - - for f in files: - if f.endswith(".json"): - frappe.flags.mute_emails = True - _import_file_by_path(f, data_import=True, force=True, pre_process=pre_process, reset_permissions=True) - frappe.flags.mute_emails = False - frappe.db.commit() - elif f.endswith(".csv"): - import_file_by_path(f, ignore_links=ignore_links, overwrite=overwrite, submit=submit, pre_process=pre_process) - frappe.db.commit() - - -def import_file_by_path(path, ignore_links=False, overwrite=False, submit=False, pre_process=None, no_email=True): - from frappe.utils.csvutils import read_csv_content - print("Importing " + path) - with open(path, "r") as infile: - upload(rows=read_csv_content(infile.read()), ignore_links=ignore_links, no_email=no_email, overwrite=overwrite, - submit_after_import=submit, pre_process=pre_process) - - -def export_json(doctype, path, filters=None, or_filters=None, name=None, order_by="creation asc"): - def post_process(out): - del_keys = ('modified_by', 'creation', 'owner', 'idx') - for doc in out: - for key in del_keys: - if key in doc: - del doc[key] - for k, v in doc.items(): - if isinstance(v, list): - for child in v: - for key in del_keys + ('docstatus', 'doctype', 'modified', 'name'): - if key in child: - del child[key] - - out = [] - if name: - out.append(frappe.get_doc(doctype, name).as_dict()) - elif frappe.db.get_value("DocType", doctype, "issingle"): - out.append(frappe.get_doc(doctype).as_dict()) - else: - for doc in frappe.get_all(doctype, fields=["name"], filters=filters, or_filters=or_filters, limit_page_length=0, order_by=order_by): - out.append(frappe.get_doc(doctype, doc.name).as_dict()) - post_process(out) - - dirname = os.path.dirname(path) - if not os.path.exists(dirname): - path = os.path.join('..', path) - - with open(path, "w") as outfile: - outfile.write(frappe.as_json(out)) - - -def export_csv(doctype, path): - from frappe.core.doctype.data_export.exporter import export_data - with open(path, "wb") as csvfile: - export_data(doctype=doctype, all_doctypes=True, template=True, with_data=True) - csvfile.write(frappe.response.result.encode("utf-8")) - - -@frappe.whitelist() -def export_fixture(doctype, app): - if frappe.session.user != "Administrator": - raise frappe.PermissionError - - if not os.path.exists(frappe.get_app_path(app, "fixtures")): - os.mkdir(frappe.get_app_path(app, "fixtures")) - - export_json(doctype, frappe.get_app_path(app, "fixtures", frappe.scrub(doctype) + ".json"), order_by="name asc") diff --git a/frappe/core/doctype/data_import_legacy/data_import_legacy_list.js b/frappe/core/doctype/data_import_legacy/data_import_legacy_list.js deleted file mode 100644 index fcf2391313..0000000000 --- a/frappe/core/doctype/data_import_legacy/data_import_legacy_list.js +++ /dev/null @@ -1,24 +0,0 @@ -frappe.listview_settings['Data Import Legacy'] = { - add_fields: ["import_status"], - has_indicator_for_draft: 1, - get_indicator: function(doc) { - - let status = { - 'Successful': [__("Success"), "green", "import_status,=,Successful"], - 'Partially Successful': [__("Partial Success"), "blue", "import_status,=,Partially Successful"], - 'In Progress': [__("In Progress"), "orange", "import_status,=,In Progress"], - 'Failed': [__("Failed"), "red", "import_status,=,Failed"], - 'Pending': [__("Pending"), "orange", "import_status,=,"] - } - - if (doc.import_status) { - return status[doc.import_status]; - } - - if (doc.docstatus == 0) { - return status['Pending']; - } - - return status['Pending']; - } -}; diff --git a/frappe/core/doctype/data_import_legacy/importer.py b/frappe/core/doctype/data_import_legacy/importer.py deleted file mode 100644 index ceefff4410..0000000000 --- a/frappe/core/doctype/data_import_legacy/importer.py +++ /dev/null @@ -1,538 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors -# MIT License. See license.txt - -import requests -import frappe, json -import frappe.permissions - -from frappe import _ - -from frappe.utils.csvutils import getlink -from frappe.utils.dateutils import parse_date - -from frappe.utils import cint, cstr, flt, getdate, get_datetime, get_url, get_absolute_url, duration_to_seconds - - -@frappe.whitelist() -def get_data_keys(): - return frappe._dict({ - "data_separator": _('Start entering data below this line'), - "main_table": _("Table") + ":", - "parent_table": _("Parent Table") + ":", - "columns": _("Column Name") + ":", - "doctype": _("DocType") + ":" - }) - - - -@frappe.whitelist() -def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False, no_email=True, overwrite=None, - update_only = None, ignore_links=False, pre_process=None, via_console=False, from_data_import="No", - skip_errors = True, data_import_doc=None, validate_template=False, user=None): - """upload data""" - - # for translations - if user: - frappe.cache().hdel("lang", user) - frappe.set_user_lang(user) - - if data_import_doc and isinstance(data_import_doc, str): - data_import_doc = frappe.get_doc("Data Import Legacy", data_import_doc) - if data_import_doc and from_data_import == "Yes": - no_email = data_import_doc.no_email - ignore_encoding_errors = data_import_doc.ignore_encoding_errors - update_only = data_import_doc.only_update - submit_after_import = data_import_doc.submit_after_import - overwrite = data_import_doc.overwrite - skip_errors = data_import_doc.skip_errors - else: - # extra input params - params = json.loads(frappe.form_dict.get("params") or '{}') - if params.get("submit_after_import"): - submit_after_import = True - if params.get("ignore_encoding_errors"): - ignore_encoding_errors = True - if not params.get("no_email"): - no_email = False - if params.get('update_only'): - update_only = True - if params.get('from_data_import'): - from_data_import = params.get('from_data_import') - if not params.get('skip_errors'): - skip_errors = params.get('skip_errors') - - frappe.flags.in_import = True - frappe.flags.mute_emails = no_email - - def get_data_keys_definition(): - return get_data_keys() - - def bad_template(): - frappe.throw(_("Please do not change the rows above {0}").format(get_data_keys_definition().data_separator)) - - def check_data_length(): - if not data: - frappe.throw(_("No data found in the file. Please reattach the new file with data.")) - - def get_start_row(): - for i, row in enumerate(rows): - if row and row[0]==get_data_keys_definition().data_separator: - return i+1 - bad_template() - - def get_header_row(key): - return get_header_row_and_idx(key)[0] - - def get_header_row_and_idx(key): - for i, row in enumerate(header): - if row and row[0]==key: - return row, i - return [], -1 - - def filter_empty_columns(columns): - empty_cols = list(filter(lambda x: x in ("", None), columns)) - - if empty_cols: - if columns[-1*len(empty_cols):] == empty_cols: - # filter empty columns if they exist at the end - columns = columns[:-1*len(empty_cols)] - else: - frappe.msgprint(_("Please make sure that there are no empty columns in the file."), - raise_exception=1) - - return columns - - def make_column_map(): - doctype_row, row_idx = get_header_row_and_idx(get_data_keys_definition().doctype) - if row_idx == -1: # old style - return - - dt = None - for i, d in enumerate(doctype_row[1:]): - if d not in ("~", "-"): - if d and doctype_row[i] in (None, '' ,'~', '-', _("DocType") + ":"): - dt, parentfield = d, None - # xls format truncates the row, so it may not have more columns - if len(doctype_row) > i+2: - parentfield = doctype_row[i+2] - doctypes.append((dt, parentfield)) - column_idx_to_fieldname[(dt, parentfield)] = {} - column_idx_to_fieldtype[(dt, parentfield)] = {} - if dt: - column_idx_to_fieldname[(dt, parentfield)][i+1] = rows[row_idx + 2][i+1] - column_idx_to_fieldtype[(dt, parentfield)][i+1] = rows[row_idx + 4][i+1] - - def get_doc(start_idx): - if doctypes: - doc = {} - attachments = [] - last_error_row_idx = None - for idx in range(start_idx, len(rows)): - last_error_row_idx = idx # pylint: disable=W0612 - if (not doc) or main_doc_empty(rows[idx]): - for dt, parentfield in doctypes: - d = {} - for column_idx in column_idx_to_fieldname[(dt, parentfield)]: - try: - fieldname = column_idx_to_fieldname[(dt, parentfield)][column_idx] - fieldtype = column_idx_to_fieldtype[(dt, parentfield)][column_idx] - - if not fieldname or not rows[idx][column_idx]: - continue - - d[fieldname] = rows[idx][column_idx] - if fieldtype in ("Int", "Check"): - d[fieldname] = cint(d[fieldname]) - elif fieldtype in ("Float", "Currency", "Percent"): - d[fieldname] = flt(d[fieldname]) - elif fieldtype == "Date": - if d[fieldname] and isinstance(d[fieldname], str): - d[fieldname] = getdate(parse_date(d[fieldname])) - elif fieldtype == "Datetime": - if d[fieldname]: - if " " in d[fieldname]: - _date, _time = d[fieldname].split() - else: - _date, _time = d[fieldname], '00:00:00' - _date = parse_date(d[fieldname]) - d[fieldname] = get_datetime(_date + " " + _time) - else: - d[fieldname] = None - elif fieldtype == "Duration": - d[fieldname] = duration_to_seconds(cstr(d[fieldname])) - elif fieldtype in ("Image", "Attach Image", "Attach"): - # added file to attachments list - attachments.append(d[fieldname]) - - elif fieldtype in ("Link", "Dynamic Link", "Data") and d[fieldname]: - # as fields can be saved in the number format(long type) in data import template - d[fieldname] = cstr(d[fieldname]) - - except IndexError: - pass - - # scrub quotes from name and modified - if d.get("name") and d["name"].startswith('"'): - d["name"] = d["name"][1:-1] - - if sum(0 if not val else 1 for val in d.values()): - d['doctype'] = dt - if dt == doctype: - doc.update(d) - else: - if not overwrite and doc.get("name"): - d['parent'] = doc["name"] - d['parenttype'] = doctype - d['parentfield'] = parentfield - doc.setdefault(d['parentfield'], []).append(d) - else: - break - - return doc, attachments, last_error_row_idx - else: - doc = frappe._dict(zip(columns, rows[start_idx][1:])) - doc['doctype'] = doctype - return doc, [], None - - # used in testing whether a row is empty or parent row or child row - # checked only 3 first columns since first two columns can be blank for example the case of - # importing the item variant where item code and item name will be blank. - def main_doc_empty(row): - if row: - for i in range(3,0,-1): - if len(row) > i and row[i]: - return False - return True - - def validate_naming(doc): - autoname = frappe.get_meta(doctype).autoname - if autoname: - if autoname[0:5] == 'field': - autoname = autoname[6:] - elif autoname == 'naming_series:': - autoname = 'naming_series' - else: - return True - - if (autoname not in doc) or (not doc[autoname]): - from frappe.model.base_document import get_controller - if not hasattr(get_controller(doctype), "autoname"): - frappe.throw(_("{0} is a mandatory field").format(autoname)) - return True - - users = frappe.db.sql_list("select name from tabUser") - def prepare_for_insert(doc): - # don't block data import if user is not set - # migrating from another system - if not doc.owner in users: - doc.owner = frappe.session.user - if not doc.modified_by in users: - doc.modified_by = frappe.session.user - - def is_valid_url(url): - is_valid = False - if url.startswith("/files") or url.startswith("/private/files"): - url = get_url(url) - - try: - r = requests.get(url) - is_valid = True if r.status_code == 200 else False - except Exception: - pass - - return is_valid - - def attach_file_to_doc(doctype, docname, file_url): - # check if attachment is already available - # check if the attachement link is relative or not - if not file_url: - return - if not is_valid_url(file_url): - return - - files = frappe.db.sql("""Select name from `tabFile` where attached_to_doctype='{doctype}' and - attached_to_name='{docname}' and (file_url='{file_url}' or thumbnail_url='{file_url}')""".format( - doctype=doctype, - docname=docname, - file_url=file_url - )) - - if files: - # file is already attached - return - - _file = frappe.get_doc({ - "doctype": "File", - "file_url": file_url, - "attached_to_name": docname, - "attached_to_doctype": doctype, - "attached_to_field": 0, - "folder": "Home/Attachments"}) - _file.save() - - - # header - filename, file_extension = ['',''] - if not rows: - _file = frappe.get_doc("File", {"file_url": data_import_doc.import_file}) - fcontent = _file.get_content() - filename, file_extension = _file.get_extension() - - if file_extension == '.xlsx' and from_data_import == 'Yes': - from frappe.utils.xlsxutils import read_xlsx_file_from_attached_file - rows = read_xlsx_file_from_attached_file(file_url=data_import_doc.import_file) - - elif file_extension == '.csv': - from frappe.utils.csvutils import read_csv_content - rows = read_csv_content(fcontent, ignore_encoding_errors) - - else: - frappe.throw(_("Unsupported File Format")) - - start_row = get_start_row() - header = rows[:start_row] - data = rows[start_row:] - try: - doctype = get_header_row(get_data_keys_definition().main_table)[1] - columns = filter_empty_columns(get_header_row(get_data_keys_definition().columns)[1:]) - except: - frappe.throw(_("Cannot change header content")) - doctypes = [] - column_idx_to_fieldname = {} - column_idx_to_fieldtype = {} - - if skip_errors: - data_rows_with_error = header - - if submit_after_import and not cint(frappe.db.get_value("DocType", - doctype, "is_submittable")): - submit_after_import = False - - parenttype = get_header_row(get_data_keys_definition().parent_table) - - if len(parenttype) > 1: - parenttype = parenttype[1] - - # check permissions - if not frappe.permissions.can_import(parenttype or doctype): - frappe.flags.mute_emails = False - return {"messages": [_("Not allowed to Import") + ": " + _(doctype)], "error": True} - - # Throw expception in case of the empty data file - check_data_length() - make_column_map() - total = len(data) - - if validate_template: - if total: - data_import_doc.total_rows = total - return True - - if overwrite==None: - overwrite = params.get('overwrite') - - # delete child rows (if parenttype) - parentfield = None - if parenttype: - parentfield = get_parent_field(doctype, parenttype) - - if overwrite: - delete_child_rows(data, doctype) - - import_log = [] - def log(**kwargs): - if via_console: - print((kwargs.get("title") + kwargs.get("message")).encode('utf-8')) - else: - import_log.append(kwargs) - - def as_link(doctype, name): - if via_console: - return "{0}: {1}".format(doctype, name) - else: - return getlink(doctype, name) - - # publish realtime task update - def publish_progress(achieved, reload=False): - if data_import_doc: - frappe.publish_realtime("data_import_progress", {"progress": str(int(100.0*achieved/total)), - "data_import": data_import_doc.name, "reload": reload}, user=frappe.session.user) - - - error_flag = rollback_flag = False - - batch_size = frappe.conf.data_import_batch_size or 1000 - - for batch_start in range(0, total, batch_size): - batch = data[batch_start:batch_start + batch_size] - - for i, row in enumerate(batch): - # bypass empty rows - if main_doc_empty(row): - continue - - row_idx = i + start_row - doc = None - - publish_progress(i) - - try: - doc, attachments, last_error_row_idx = get_doc(row_idx) - validate_naming(doc) - if pre_process: - pre_process(doc) - - original = None - if parentfield: - parent = frappe.get_doc(parenttype, doc["parent"]) - doc = parent.append(parentfield, doc) - parent.save() - else: - if overwrite and doc.get("name") and frappe.db.exists(doctype, doc["name"]): - original = frappe.get_doc(doctype, doc["name"]) - original_name = original.name - original.update(doc) - # preserve original name for case sensitivity - original.name = original_name - original.flags.ignore_links = ignore_links - original.save() - doc = original - else: - if not update_only: - doc = frappe.get_doc(doc) - prepare_for_insert(doc) - doc.flags.ignore_links = ignore_links - doc.insert() - if attachments: - # check file url and create a File document - for file_url in attachments: - attach_file_to_doc(doc.doctype, doc.name, file_url) - if submit_after_import: - doc.submit() - - # log errors - if parentfield: - log(**{"row": doc.idx, "title": 'Inserted row for "%s"' % (as_link(parenttype, doc.parent)), - "link": get_absolute_url(parenttype, doc.parent), "message": 'Document successfully saved', "indicator": "green"}) - elif submit_after_import: - log(**{"row": row_idx + 1, "title":'Submitted row for "%s"' % (as_link(doc.doctype, doc.name)), - "message": "Document successfully submitted", "link": get_absolute_url(doc.doctype, doc.name), "indicator": "blue"}) - elif original: - log(**{"row": row_idx + 1,"title":'Updated row for "%s"' % (as_link(doc.doctype, doc.name)), - "message": "Document successfully updated", "link": get_absolute_url(doc.doctype, doc.name), "indicator": "green"}) - elif not update_only: - log(**{"row": row_idx + 1, "title":'Inserted row for "%s"' % (as_link(doc.doctype, doc.name)), - "message": "Document successfully saved", "link": get_absolute_url(doc.doctype, doc.name), "indicator": "green"}) - else: - log(**{"row": row_idx + 1, "title":'Ignored row for %s' % (row[1]), "link": None, - "message": "Document updation ignored", "indicator": "orange"}) - - except Exception as e: - error_flag = True - - # build error message - if frappe.local.message_log: - err_msg = "\n".join(['

{}

'.format(json.loads(msg).get('message')) for msg in frappe.local.message_log]) - else: - err_msg = '

{}

'.format(cstr(e)) - - error_trace = frappe.get_traceback() - if error_trace: - error_log_doc = frappe.log_error(error_trace) - error_link = get_absolute_url("Error Log", error_log_doc.name) - else: - error_link = None - - log(**{ - "row": row_idx + 1, - "title": 'Error for row %s' % (len(row)>1 and frappe.safe_decode(row[1]) or ""), - "message": err_msg, - "indicator": "red", - "link":error_link - }) - - # data with error to create a new file - # include the errored data in the last row as last_error_row_idx will not be updated for the last row - if skip_errors: - if last_error_row_idx == len(rows)-1: - last_error_row_idx = len(rows) - data_rows_with_error += rows[row_idx:last_error_row_idx] - else: - rollback_flag = True - finally: - frappe.local.message_log = [] - - start_row += batch_size - if rollback_flag: - frappe.db.rollback() - else: - frappe.db.commit() - - frappe.flags.mute_emails = False - frappe.flags.in_import = False - - log_message = {"messages": import_log, "error": error_flag} - if data_import_doc: - data_import_doc.log_details = json.dumps(log_message) - - import_status = None - if error_flag and data_import_doc.skip_errors and len(data) != len(data_rows_with_error): - import_status = "Partially Successful" - # write the file with the faulty row - file_name = 'error_' + filename + file_extension - if file_extension == '.xlsx': - from frappe.utils.xlsxutils import make_xlsx - xlsx_file = make_xlsx(data_rows_with_error, "Data Import Template") - file_data = xlsx_file.getvalue() - else: - from frappe.utils.csvutils import to_csv - file_data = to_csv(data_rows_with_error) - _file = frappe.get_doc({ - "doctype": "File", - "file_name": file_name, - "attached_to_doctype": "Data Import Legacy", - "attached_to_name": data_import_doc.name, - "folder": "Home/Attachments", - "content": file_data}) - _file.save() - data_import_doc.error_file = _file.file_url - - elif error_flag: - import_status = "Failed" - else: - import_status = "Successful" - - data_import_doc.import_status = import_status - data_import_doc.save() - if data_import_doc.import_status in ["Successful", "Partially Successful"]: - data_import_doc.submit() - publish_progress(100, True) - else: - publish_progress(0, True) - frappe.db.commit() - else: - return log_message - -def get_parent_field(doctype, parenttype): - parentfield = None - - # get parentfield - if parenttype: - for d in frappe.get_meta(parenttype).get_table_fields(): - if d.options==doctype: - parentfield = d.fieldname - break - - if not parentfield: - frappe.msgprint(_("Did not find {0} for {0} ({1})").format("parentfield", parenttype, doctype)) - raise Exception - - return parentfield - -def delete_child_rows(rows, doctype): - """delete child rows for all parents""" - for p in list(set(r[1] for r in rows)): - if p: - frappe.db.sql("""delete from `tab{0}` where parent=%s""".format(doctype), p) diff --git a/frappe/core/doctype/data_import_legacy/log_details.html b/frappe/core/doctype/data_import_legacy/log_details.html deleted file mode 100644 index aa160a742b..0000000000 --- a/frappe/core/doctype/data_import_legacy/log_details.html +++ /dev/null @@ -1,38 +0,0 @@ -
-
- - - - - - - - {% for row in data %} - {% if (!show_only_errors) || (show_only_errors && row.indicator == "red") %} - - - - - - {% endif %} - {% endfor %} -
{{ __("Row No") }} {{ __("Row Status") }} {{ __("Message") }}
- {{ row.row }} - - {{ row.title }} - - {% if (import_status != "Failed" || (row.indicator == "red")) { %} -
{{ row.message }}
- {% if row.link %} - - - - - - {% endif %} - {% } else { %} - {{ __("Document can't saved.") }} - {% } %} -
-
-
\ No newline at end of file diff --git a/frappe/core/doctype/data_import_legacy/test_data_import_legacy.py b/frappe/core/doctype/data_import_legacy/test_data_import_legacy.py deleted file mode 100644 index 6f9964e8f5..0000000000 --- a/frappe/core/doctype/data_import_legacy/test_data_import_legacy.py +++ /dev/null @@ -1,8 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2020, Frappe Technologies and Contributors -# See license.txt -# import frappe -import unittest - -class TestDataImportLegacy(unittest.TestCase): - pass From 0d0e527416c268b5a74284dbd849d79be010ebfe Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 23 Jun 2021 13:22:11 +0530 Subject: [PATCH 040/164] chore: Show deprecation warning in command help --- frappe/commands/utils.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py index 506ec7816b..2fe5ca5929 100644 --- a/frappe/commands/utils.py +++ b/frappe/commands/utils.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - import json import os import subprocess @@ -14,6 +12,13 @@ from frappe.exceptions import SiteNotSpecifiedError from frappe.utils import get_bench_path, update_progress_bar, cint +DATA_IMPORT_DEPRECATION = click.style( + "[DEPRECATED] The `import-csv` command used 'Data Import Legacy' which has been deprecated.\n" + "Use `data-import` command instead to import data via 'Data Import'.", + fg="yellow" +) + + @click.command('build') @click.option('--app', help='Build assets for app') @click.option('--apps', help='Build assets for specific apps') @@ -350,7 +355,8 @@ def import_doc(context, path, force=False): if not context.sites: raise SiteNotSpecifiedError -@click.command('import-csv') + +@click.command('import-csv', help=DATA_IMPORT_DEPRECATION) @click.argument('path') @click.option('--only-insert', default=False, is_flag=True, help='Do not overwrite existing records') @click.option('--submit-after-import', default=False, is_flag=True, help='Submit document after importing it') @@ -358,11 +364,7 @@ def import_doc(context, path, force=False): @click.option('--no-email', default=True, is_flag=True, help='Send email if applicable') @pass_context def import_csv(context, path, only_insert=False, submit_after_import=False, ignore_encoding_errors=False, no_email=True): - click.secho( - "The `import-csv` command used 'Data Import Legacy' which has been deprecated.\n" - "Use `data-import` command instead to import data via 'Data Import'.", - fg="yellow", - ) + click.secho(DATA_IMPORT_DEPRECATION) sys.exit(1) From baea532c3d19fd9d55dd2b10cd676e7547b378bb Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 13 Jul 2021 11:27:01 +0530 Subject: [PATCH 041/164] fix: Move depr module module utils in exporter --- frappe/core/doctype/data_export/exporter.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/frappe/core/doctype/data_export/exporter.py b/frappe/core/doctype/data_export/exporter.py index 389948449e..ffd828bfdb 100644 --- a/frappe/core/doctype/data_export/exporter.py +++ b/frappe/core/doctype/data_export/exporter.py @@ -7,7 +7,6 @@ import frappe.permissions import re, csv, os from frappe.utils.csvutils import UnicodeWriter from frappe.utils import cstr, formatdate, format_datetime, parse_json, cint, format_duration -from frappe.core.doctype.data_import_legacy.importer import get_data_keys from frappe.core.doctype.access_log.access_log import make_access_log reflags = { @@ -20,6 +19,15 @@ reflags = { "D": re.DEBUG } +def get_data_keys(): + return frappe._dict({ + "data_separator": _('Start entering data below this line'), + "main_table": _("Table") + ":", + "parent_table": _("Parent Table") + ":", + "columns": _("Column Name") + ":", + "doctype": _("DocType") + ":" + }) + @frappe.whitelist() def export_data(doctype=None, parent_doctype=None, all_doctypes=True, with_data=False, select_columns=None, file_type='CSV', template=False, filters=None): From 2aa2586a0cb1391fe07e40811b7d2c7cf09abc8d Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 13 Jul 2021 12:04:18 +0530 Subject: [PATCH 042/164] fix: Drop deprecated validate_csv_import_file def --- frappe/core/doctype/data_import/data_import.py | 18 ++---------------- frappe/utils/fixtures.py | 2 +- 2 files changed, 3 insertions(+), 17 deletions(-) diff --git a/frappe/core/doctype/data_import/data_import.py b/frappe/core/doctype/data_import/data_import.py index 7e8374a0a2..50469eeb4d 100644 --- a/frappe/core/doctype/data_import/data_import.py +++ b/frappe/core/doctype/data_import/data_import.py @@ -171,9 +171,6 @@ def import_file( i.import_data() -############## - - def import_doc(path, pre_process=None): if os.path.isdir(path): files = [os.path.join(path, f) for f in os.listdir(path)] @@ -192,19 +189,8 @@ def import_doc(path, pre_process=None): ) frappe.flags.mute_emails = False frappe.db.commit() - elif f.endswith(".csv"): - validate_csv_import_file(f) - frappe.db.commit() - - -def validate_csv_import_file(path): - if path.endswith(".csv"): - print() - print("This method is deprecated.") - print('Import CSV files using the command "bench --site sitename data-import"') - print("Or use the method frappe.core.doctype.data_import.data_import.import_file") - print() - raise Exception("Method deprecated") + else: + raise NotImplementedError("Only .json files can be imported") def export_json( diff --git a/frappe/utils/fixtures.py b/frappe/utils/fixtures.py index 1f33c36b13..895a3c8373 100644 --- a/frappe/utils/fixtures.py +++ b/frappe/utils/fixtures.py @@ -20,7 +20,7 @@ def sync_fixtures(app=None): if os.path.exists(frappe.get_app_path(app, "fixtures")): fixture_files = sorted(os.listdir(frappe.get_app_path(app, "fixtures"))) for fname in fixture_files: - if fname.endswith(".json") or fname.endswith(".csv"): + if fname.endswith(".json"): import_doc(frappe.get_app_path(app, "fixtures", fname)) import_custom_scripts(app) From 8dcb7a1eaac14f4e68e4644b4481ceb06cae8ca5 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 13 Jul 2021 12:04:50 +0530 Subject: [PATCH 043/164] style: Remove redundant comments --- frappe/core/doctype/access_log/access_log.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/frappe/core/doctype/access_log/access_log.py b/frappe/core/doctype/access_log/access_log.py index d2fbee108b..2ea014f981 100644 --- a/frappe/core/doctype/access_log/access_log.py +++ b/frappe/core/doctype/access_log/access_log.py @@ -1,9 +1,5 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and contributors # For license information, please see license.txt - -# imports - standard imports -# imports - module imports import frappe from frappe.model.document import Document From 58cb8e5dd1516432ef319802e856472f419d3135 Mon Sep 17 00:00:00 2001 From: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> Date: Tue, 13 Jul 2021 12:46:01 +0530 Subject: [PATCH 044/164] style: Fix minor formatting issues - Also, updated error title --- frappe/desk/doctype/notification_log/notification_log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/desk/doctype/notification_log/notification_log.py b/frappe/desk/doctype/notification_log/notification_log.py index af09d65015..d7d7f68b74 100644 --- a/frappe/desk/doctype/notification_log/notification_log.py +++ b/frappe/desk/doctype/notification_log/notification_log.py @@ -15,7 +15,7 @@ class NotificationLog(Document): try: send_notification_email(self) except frappe.OutgoingEmailError: - frappe.log_error(message = frappe.get_traceback(), title=_("Error Sending Notification Email.")) + frappe.log_error(message=frappe.get_traceback(), title=_("Failed to send notification email")) def get_permission_query_conditions(for_user): From e826866503287ed67696113b5b789ced053744ae Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Tue, 13 Jul 2021 14:24:12 +0530 Subject: [PATCH 045/164] fix: Tag count query --- frappe/desk/reportview.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/frappe/desk/reportview.py b/frappe/desk/reportview.py index 55515856f1..b9382bc830 100644 --- a/frappe/desk/reportview.py +++ b/frappe/desk/reportview.py @@ -445,17 +445,24 @@ def get_stats(stats, doctype, filters=[]): for tag in tags: if not tag in columns: continue try: - tagcount = frappe.get_list(doctype, fields=[tag, "count(*)"], - #filters=["ifnull(`%s`,'')!=''" % tag], group_by=tag, as_list=True) - filters = filters + ["ifnull(`%s`,'')!=''" % tag], group_by = tag, as_list = True) + tag_count = frappe.get_list(doctype, + fields=[tag, "count(*)"], + filters=filters + [[tag, '!=', '']], + group_by=tag, + as_list=True, + ) - if tag=='_user_tags': - stats[tag] = scrub_user_tags(tagcount) - stats[tag].append([_("No Tags"), frappe.get_list(doctype, + if tag == '_user_tags': + stats[tag] = scrub_user_tags(tag_count) + no_tag_count = frappe.get_list(doctype, fields=[tag, "count(*)"], - filters=filters +["({0} = ',' or {0} = '' or {0} is null)".format(tag)], as_list=True)[0][1]]) + filters=filters + [[tag, "in", ('', ',')]], + as_list=True, + )[0][1] + + stats[tag].append([_("No Tags"), no_tag_count]) else: - stats[tag] = tagcount + stats[tag] = tag_count except frappe.db.SQLError: # does not work for child tables From ae70502f910c85f6a4528b487eea3b535cec6c39 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Tue, 13 Jul 2021 15:34:36 +0530 Subject: [PATCH 046/164] test: Add test case to validate tag count query --- frappe/desk/doctype/tag/test_tag.py | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/frappe/desk/doctype/tag/test_tag.py b/frappe/desk/doctype/tag/test_tag.py index 442a891fd8..6eb7219c26 100644 --- a/frappe/desk/doctype/tag/test_tag.py +++ b/frappe/desk/doctype/tag/test_tag.py @@ -1,8 +1,26 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2019, Frappe Technologies and Contributors -# See license.txt -# import frappe import unittest +import frappe + +from frappe.desk.reportview import get_stats +from frappe.desk.doctype.tag.tag import add_tag class TestTag(unittest.TestCase): - pass + def setUp(self) -> None: + frappe.db.sql("DELETE from `tabTag`") + frappe.db.sql("UPDATE `tabDocType` set _user_tags=''") + + def test_tag_count_query(self): + self.assertDictEqual(get_stats('["_user_tags"]', 'DocType'), + {'_user_tags': [['No Tags', frappe.db.count('DocType')]]}) + add_tag('Standard', 'DocType', 'User') + add_tag('Standard', 'DocType', 'ToDo') + + # count with no filter + self.assertDictEqual(get_stats('["_user_tags"]', 'DocType'), + {'_user_tags': [['Standard', 2], ['No Tags', frappe.db.count('DocType') - 2]]}) + + # count with child table field filter + self.assertDictEqual(get_stats('["_user_tags"]', + 'DocType', + filters='[["DocField", "fieldname", "like", "%last_name%"], ["DocType", "name", "like", "%use%"]]'), + {'_user_tags': [['Standard', 1], ['No Tags', 0]]}) \ No newline at end of file From 68930a022aafd97cf95d7fdce312ff82bf963ee8 Mon Sep 17 00:00:00 2001 From: Bhavesh Maheshwari <34086262+bhavesh95863@users.noreply.github.com> Date: Tue, 13 Jul 2021 15:55:24 +0530 Subject: [PATCH 047/164] fix: Duplicate if condition remove --- frappe/utils/global_search.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/frappe/utils/global_search.py b/frappe/utils/global_search.py index efe92232d9..8fa2ea474f 100644 --- a/frappe/utils/global_search.py +++ b/frappe/utils/global_search.py @@ -231,9 +231,6 @@ def update_global_search(doc): if frappe.local.conf.get('disable_global_search'): return - if frappe.local.conf.get('disable_global_search'): - return - if doc.docstatus > 1 or (doc.meta.has_field("enabled") and not doc.get("enabled")) \ or doc.get("disabled"): return From 8bc498c18cc4b6932768bb618ee8d27fce93fa21 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Tue, 13 Jul 2021 10:42:00 +0000 Subject: [PATCH 048/164] chore(deps): [security] bump minimist from 1.2.0 to 1.2.5 Bumps [minimist](https://github.com/substack/minimist) from 1.2.0 to 1.2.5. **This update includes security fixes.** - [Release notes](https://github.com/substack/minimist/releases) - [Commits](https://github.com/substack/minimist/compare/1.2.0...1.2.5) Signed-off-by: dependabot-preview[bot] --- yarn.lock | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/yarn.lock b/yarn.lock index ddb5623e5e..da7f89872f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4148,16 +4148,11 @@ minimatch@^3.0.4, minimatch@~3.0.2: dependencies: brace-expansion "^1.1.7" -minimist@^1.1.3, minimist@^1.2.5: +minimist@^1.1.3, minimist@^1.2.0, minimist@^1.2.5: version "1.2.5" resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.5.tgz#67d66014b66a6a8aaa0c083c5fd58df4e4e97602" integrity sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw== -minimist@^1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.0.tgz#a35008b20f41383eec1fb914f4cd5df79a264284" - integrity sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ= - minipass@^3.0.0: version "3.1.3" resolved "https://registry.yarnpkg.com/minipass/-/minipass-3.1.3.tgz#7d42ff1f39635482e15f9cdb53184deebd5815fd" From 04094110f5ddb5b2d4c514be43328d9f0e893654 Mon Sep 17 00:00:00 2001 From: barredterra <14891507+barredterra@users.noreply.github.com> Date: Tue, 13 Jul 2021 13:11:15 +0200 Subject: [PATCH 049/164] fix: handle detached head Workaround for https://github.com/gitpython-developers/GitPython/issues/633 --- frappe/commands/utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py index 5c6274d348..542f41725b 100644 --- a/frappe/commands/utils.py +++ b/frappe/commands/utils.py @@ -774,6 +774,7 @@ def get_version(output): """Show the versions of all the installed apps.""" from git import Repo from frappe.utils.commands import render_table + from frappe.utils.change_log import get_app_branch frappe.init("") data = [] @@ -785,8 +786,8 @@ def get_version(output): app_info = frappe._dict() app_info.app = app - app_info.branch = repo.head.ref.name - app_info.commit = repo.head.ref.commit.hexsha[:7] + app_info.branch = get_app_branch(app) + app_info.commit = repo.head.object.hexsha[:7] app_info.version = getattr(app_hooks, f"{app_info.branch}_version", None) or module.__version__ data.append(app_info) From fc2c887635168cd18bf18b6eae74fedc58cb9a22 Mon Sep 17 00:00:00 2001 From: barredterra <14891507+barredterra@users.noreply.github.com> Date: Tue, 13 Jul 2021 13:23:25 +0200 Subject: [PATCH 050/164] test: fix return code --- frappe/tests/test_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/tests/test_commands.py b/frappe/tests/test_commands.py index 8f9bbb1afb..54103f0151 100644 --- a/frappe/tests/test_commands.py +++ b/frappe/tests/test_commands.py @@ -435,4 +435,4 @@ class TestCommands(BaseTestCommands): self.assertEqual(self.returncode, 0) self.execute("bench version -f invalid") - self.assertEqual(self.returncode, 1) + self.assertEqual(self.returncode, 2) From cf67dbce5abd96687ab3c8a05a74cecf3bf7792b Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Tue, 13 Jul 2021 16:56:49 +0530 Subject: [PATCH 051/164] fix: Add missing fieldtypes in for custom fields --- frappe/custom/doctype/custom_field/custom_field.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/custom/doctype/custom_field/custom_field.json b/frappe/custom/doctype/custom_field/custom_field.json index 2f0819ab68..19462e79de 100644 --- a/frappe/custom/doctype/custom_field/custom_field.json +++ b/frappe/custom/doctype/custom_field/custom_field.json @@ -120,7 +120,7 @@ "label": "Field Type", "oldfieldname": "fieldtype", "oldfieldtype": "Select", - "options": "Attach\nAttach Image\nBarcode\nButton\nCheck\nCode\nColor\nColumn Break\nCurrency\nData\nDate\nDatetime\nDuration\nDynamic Link\nFloat\nGeolocation\nHTML\nImage\nInt\nLink\nLong Text\nMarkdown Editor\nPassword\nPercent\nRating\nRead Only\nSection Break\nSelect\nSmall Text\nTable\nTable MultiSelect\nText\nText Editor\nTime\nSignature", + "options": "Attach\nAttach Image\nBarcode\nButton\nCheck\nCode\nColor\nColumn Break\nCurrency\nData\nDate\nDatetime\nDuration\nDynamic Link\nFloat\nFold\nGeolocation\nHeading\nHTML\nHTML Editor\nImage\nInt\nLink\nLong Text\nMarkdown Editor\nPassword\nPercent\nRead Only\nRating\nSection Break\nSelect\nSmall Text\nTable\nTable MultiSelect\nText\nText Editor\nTime\nSignature", "reqd": 1 }, { @@ -417,7 +417,7 @@ "idx": 1, "index_web_pages_for_search": 1, "links": [], - "modified": "2020-10-29 06:14:43.073329", + "modified": "2021-07-12 04:54:12.042319", "modified_by": "Administrator", "module": "Custom", "name": "Custom Field", From b23dd711162ff03922c7469fd478623bad45dff1 Mon Sep 17 00:00:00 2001 From: Sagar Vora Date: Tue, 13 Jul 2021 17:56:47 +0530 Subject: [PATCH 052/164] fix: escape quotes before declaring variables when making a new app --- frappe/utils/boilerplate.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/frappe/utils/boilerplate.py b/frappe/utils/boilerplate.py index 8dab9b748f..d88eaa5745 100755 --- a/frappe/utils/boilerplate.py +++ b/frappe/utils/boilerplate.py @@ -66,9 +66,6 @@ def make_boilerplate(dest, app_name): with open(os.path.join(dest, hooks.app_name, ".gitignore"), "w") as f: f.write(frappe.as_unicode(gitignore_template.format(app_name = hooks.app_name))) - with open(os.path.join(dest, hooks.app_name, "setup.py"), "w") as f: - f.write(frappe.as_unicode(setup_template.format(**hooks))) - with open(os.path.join(dest, hooks.app_name, "requirements.txt"), "w") as f: f.write("# frappe -- https://github.com/frappe/frappe is installed via 'bench init'") @@ -82,6 +79,14 @@ def make_boilerplate(dest, app_name): with open(os.path.join(dest, hooks.app_name, hooks.app_name, "modules.txt"), "w") as f: f.write(frappe.as_unicode(hooks.app_title)) + # These values could contain quotes and can break string declarations + # So escaping them before setting variables in setup.py and hooks.py + for key in ("app_publisher", "app_description", "app_license"): + hooks[key] = hooks[key].replace("\\", "\\\\").replace("'", "\\'").replace("\"", "\\\"") + + with open(os.path.join(dest, hooks.app_name, "setup.py"), "w") as f: + f.write(frappe.as_unicode(setup_template.format(**hooks))) + with open(os.path.join(dest, hooks.app_name, hooks.app_name, "hooks.py"), "w") as f: f.write(frappe.as_unicode(hooks_template.format(**hooks))) @@ -328,18 +333,18 @@ def get_data(): setup_template = """from setuptools import setup, find_packages -with open('requirements.txt') as f: - install_requires = f.read().strip().split('\\n') +with open("requirements.txt") as f: + install_requires = f.read().strip().split("\\n") # get version from __version__ variable in {app_name}/__init__.py from {app_name} import __version__ as version setup( - name='{app_name}', + name="{app_name}", version=version, - description='{app_description}', - author='{app_publisher}', - author_email='{app_email}', + description="{app_description}", + author="{app_publisher}", + author_email="{app_email}", packages=find_packages(), zip_safe=False, include_package_data=True, From 79124873c3357086580edd53c56b0756b4c68e1b Mon Sep 17 00:00:00 2001 From: codescientist703 Date: Tue, 13 Jul 2021 18:43:34 +0530 Subject: [PATCH 053/164] fix: fixed sider recommendation --- frappe/desk/search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/desk/search.py b/frappe/desk/search.py index 3cce80a1a0..f9b65fc98e 100644 --- a/frappe/desk/search.py +++ b/frappe/desk/search.py @@ -222,7 +222,7 @@ def scrub_custom_query(query, key, txt): def relevance_sorter(key, query, as_dict): value = _(key.name if as_dict else key[0]) return ( - value.lower().startswith(query.lower()) == False, + value.lower().startswith(query.lower()) is not True, value ) From bed64ef0adcecacd98958aae940d43a11e264177 Mon Sep 17 00:00:00 2001 From: Pruthvi Patel Date: Tue, 13 Jul 2021 18:49:26 +0530 Subject: [PATCH 054/164] test: change app description to test if quotes are being escaped --- frappe/tests/test_boilerplate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/tests/test_boilerplate.py b/frappe/tests/test_boilerplate.py index 4ae78c94de..259d5a9194 100644 --- a/frappe/tests/test_boilerplate.py +++ b/frappe/tests/test_boilerplate.py @@ -20,7 +20,7 @@ class TestBoilerPlate(unittest.TestCase): def test_create_app(self): title = "Test App" - description = "Test app for unit testing" + description = "This app's description contains 'single quotes' and \"double quotes\"." publisher = "Test Publisher" email = "example@example.org" icon = "" # empty -> default From b619c3d33044a0f1b4951a98924ace3fdeb7069f Mon Sep 17 00:00:00 2001 From: codescientist703 Date: Tue, 13 Jul 2021 20:24:24 +0530 Subject: [PATCH 055/164] test: added test for order of link --- frappe/tests/test_search.py | 56 ++++++++++++++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/frappe/tests/test_search.py b/frappe/tests/test_search.py index 9ad02f49a6..fb99db2dc7 100644 --- a/frappe/tests/test_search.py +++ b/frappe/tests/test_search.py @@ -7,6 +7,48 @@ from frappe.desk.search import search_link from frappe.desk.search import search_widget class TestSearch(unittest.TestCase): + def setUp(self): + self.tree_doctype_name = 'Test Tree Order' + + # Create Tree doctype + self.tree_doc = frappe.get_doc({ + 'doctype': 'DocType', + 'name': self.tree_doctype_name, + 'module': 'Custom', + 'custom': 1, + 'is_tree': 1, + 'autoname': 'field:random', + 'fields': [{ + 'fieldname': 'random', + 'label': 'Random', + 'fieldtype': 'Data' + }] + }).insert() + self.tree_doc.search_fields = 'parent_test_tree_order' + self.tree_doc.save() + + # Create root for the tree doctype + self.parent_doctype_name = 'All Territories' + frappe.get_doc(doctype=self.tree_doctype_name, random=self.parent_doctype_name, + is_group=1).insert() + + # Create children for the root + self.child_doctypes_names = ['USA', 'India', 'Russia', 'China'] + self.child_doctype_list = [] + for child_name in self.child_doctypes_names: + temp = frappe.get_doc(doctype=self.tree_doctype_name, random=child_name, + parent_test_tree_order=self.parent_doctype_name) + temp.insert() + self.child_doctype_list.append(temp) + + def tearDown(self): + # Deleting all the created doctype + for child_doctype in self.child_doctype_list: + child_doctype.delete() + frappe.delete_doc(self.tree_doctype_name, self.parent_doctype_name, + force=1, ignore_permissions=True, for_reload=True) + self.tree_doc.delete() + def test_search_field_sanitizer(self): # pass search_link('DocType', 'User', query=None, filters=None, page_length=20, searchfield='name') @@ -38,6 +80,18 @@ class TestSearch(unittest.TestCase): search_link, 'DocType', 'Customer', query=None, filters=None, page_length=20, searchfield=';') + def test_link_field_order(self): + # Making a request to the search_link with the tree doctype + search_link(doctype=self.tree_doctype_name, txt='all', query=None, + filters=None, page_length=20, searchfield=None) + result = frappe.response['results'] + + # Check whether the result is sorted or not + self.assertEquals(self.parent_doctype_name, result[0]['value']) + + # Check whether searching for parent also list out children + self.assertEquals(len(result), len(self.child_doctypes_names) + 1) + #Search for the word "pay", part of the word "pays" (country) in french. def test_link_search_in_foreign_language(self): try: @@ -80,4 +134,4 @@ class TestSearch(unittest.TestCase): @frappe.validate_and_sanitize_search_inputs def get_data(doctype, txt, searchfield, start, page_len, filters): - return [doctype, txt, searchfield, start, page_len, filters] \ No newline at end of file + return [doctype, txt, searchfield, start, page_len, filters] From f2267bec25e9edb1b947b9ac21c652a6daaec78c Mon Sep 17 00:00:00 2001 From: leela Date: Wed, 14 Jul 2021 10:28:10 +0530 Subject: [PATCH 056/164] fix: sending document as an attachment through mail Currently sending document as an attachment is failing and the fix is to make it work by fixing the bug introduced in latest refactoring. --- frappe/core/doctype/communication/mixins.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/core/doctype/communication/mixins.py b/frappe/core/doctype/communication/mixins.py index 160018c4a2..09a8a0ac22 100644 --- a/frappe/core/doctype/communication/mixins.py +++ b/frappe/core/doctype/communication/mixins.py @@ -176,8 +176,8 @@ class CommunicationEmailMixin: def mail_attachments(self, print_format=None, print_html=None): final_attachments = [] - if print_format and print_html: - d = {'print_format': print_format, 'print_html': print_html, 'print_format_attachment': 1, + if print_format or print_html: + d = {'print_format': print_format, 'html': print_html, 'print_format_attachment': 1, 'doctype': self.reference_doctype, 'name': self.reference_name} final_attachments.append(d) From 6f17e3d45fa90b9aec60b6b0c5403c57e707f9df Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 14 Jul 2021 12:02:33 +0530 Subject: [PATCH 057/164] perf(test_search): Setup and teardown tests only if needed --- frappe/tests/test_search.py | 99 ++++++++++++++++++++++--------------- 1 file changed, 60 insertions(+), 39 deletions(-) diff --git a/frappe/tests/test_search.py b/frappe/tests/test_search.py index fb99db2dc7..afb584ea15 100644 --- a/frappe/tests/test_search.py +++ b/frappe/tests/test_search.py @@ -1,4 +1,4 @@ -# Copyright (c) 2018, Frappe Technologies Pvt. Ltd. and Contributors +# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt import unittest @@ -6,48 +6,15 @@ import frappe from frappe.desk.search import search_link from frappe.desk.search import search_widget + class TestSearch(unittest.TestCase): def setUp(self): - self.tree_doctype_name = 'Test Tree Order' - - # Create Tree doctype - self.tree_doc = frappe.get_doc({ - 'doctype': 'DocType', - 'name': self.tree_doctype_name, - 'module': 'Custom', - 'custom': 1, - 'is_tree': 1, - 'autoname': 'field:random', - 'fields': [{ - 'fieldname': 'random', - 'label': 'Random', - 'fieldtype': 'Data' - }] - }).insert() - self.tree_doc.search_fields = 'parent_test_tree_order' - self.tree_doc.save() - - # Create root for the tree doctype - self.parent_doctype_name = 'All Territories' - frappe.get_doc(doctype=self.tree_doctype_name, random=self.parent_doctype_name, - is_group=1).insert() - - # Create children for the root - self.child_doctypes_names = ['USA', 'India', 'Russia', 'China'] - self.child_doctype_list = [] - for child_name in self.child_doctypes_names: - temp = frappe.get_doc(doctype=self.tree_doctype_name, random=child_name, - parent_test_tree_order=self.parent_doctype_name) - temp.insert() - self.child_doctype_list.append(temp) + if self._testMethodName == "test_link_field_order": + setup_test_link_field_order(self) def tearDown(self): - # Deleting all the created doctype - for child_doctype in self.child_doctype_list: - child_doctype.delete() - frappe.delete_doc(self.tree_doctype_name, self.parent_doctype_name, - force=1, ignore_permissions=True, for_reload=True) - self.tree_doc.delete() + if self._testMethodName == "test_link_field_order": + teardown_test_link_field_order(self) def test_search_field_sanitizer(self): # pass @@ -135,3 +102,57 @@ class TestSearch(unittest.TestCase): @frappe.validate_and_sanitize_search_inputs def get_data(doctype, txt, searchfield, start, page_len, filters): return [doctype, txt, searchfield, start, page_len, filters] + +def setup_test_link_field_order(TestCase): + TestCase.tree_doctype_name = 'Test Tree Order' + TestCase.child_doctype_list = [] + TestCase.child_doctypes_names = ['USA', 'India', 'Russia', 'China'] + TestCase.parent_doctype_name = 'All Territories' + + # Create Tree doctype + TestCase.tree_doc = frappe.get_doc({ + 'doctype': 'DocType', + 'name': TestCase.tree_doctype_name, + 'module': 'Custom', + 'custom': 1, + 'is_tree': 1, + 'autoname': 'field:random', + 'fields': [{ + 'fieldname': 'random', + 'label': 'Random', + 'fieldtype': 'Data' + }] + }).insert() + TestCase.tree_doc.search_fields = 'parent_test_tree_order' + TestCase.tree_doc.save() + + # Create root for the tree doctype + frappe.get_doc({ + "doctype": TestCase.tree_doctype_name, + "random": TestCase.parent_doctype_name, + "is_group": 1 + }).insert() + + # Create children for the root + for child_name in TestCase.child_doctypes_names: + temp = frappe.get_doc({ + "doctype": TestCase.tree_doctype_name, + "random": child_name, + "parent_test_tree_order": TestCase.parent_doctype_name + }).insert() + TestCase.child_doctype_list.append(temp) + +def teardown_test_link_field_order(TestCase): + # Deleting all the created doctype + for child_doctype in TestCase.child_doctype_list: + child_doctype.delete() + + frappe.delete_doc( + TestCase.tree_doctype_name, + TestCase.parent_doctype_name, + ignore_permissions=True, + force=True, + for_reload=True, + ) + + TestCase.tree_doc.delete() From 0eafadf002e45c7a851149783829dc1060a55011 Mon Sep 17 00:00:00 2001 From: shariquerik Date: Wed, 14 Jul 2021 12:10:11 +0530 Subject: [PATCH 058/164] fix: Code Editor scrollbar glitch --- frappe/public/scss/common/controls.scss | 3 +++ 1 file changed, 3 insertions(+) diff --git a/frappe/public/scss/common/controls.scss b/frappe/public/scss/common/controls.scss index c939c6de39..a7a8d49510 100644 --- a/frappe/public/scss/common/controls.scss +++ b/frappe/public/scss/common/controls.scss @@ -166,6 +166,9 @@ select.form-control { .ace_print-margin { background-color: var(--dark-border-color); } + .ace_scrollbar { + z-index: 3; + } } .frappe-control[data-fieldtype="Attach"], From 9d9d76efe626f63a06150ec9519bd8c7becd1d01 Mon Sep 17 00:00:00 2001 From: Fahim Ali Zain Date: Fri, 4 Jun 2021 11:23:02 +0530 Subject: [PATCH 059/164] fix: frappe.local.lang resolution --- frappe/auth.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frappe/auth.py b/frappe/auth.py index ef79d96ddb..7c513aed63 100644 --- a/frappe/auth.py +++ b/frappe/auth.py @@ -35,9 +35,6 @@ class HTTPRequest: else: frappe.local.request_ip = '127.0.0.1' - # language - self.set_lang() - # load cookies frappe.local.cookie_manager = CookieManager() @@ -47,6 +44,9 @@ class HTTPRequest: # login frappe.local.login_manager = LoginManager() + # language + self.set_lang() + if frappe.form_dict._lang: lang = get_lang_code(frappe.form_dict._lang) if lang: From f96ebb1f664e009ed2bf61505067ddacf4b21e78 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 10 Jun 2021 12:47:23 +0530 Subject: [PATCH 060/164] perf: Skip guess_language if _lang is provided --- frappe/auth.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/frappe/auth.py b/frappe/auth.py index 7c513aed63..479ec6bda4 100644 --- a/frappe/auth.py +++ b/frappe/auth.py @@ -10,7 +10,7 @@ import frappe.utils.user from frappe import conf from frappe.sessions import Session, clear_sessions, delete_session from frappe.modules.patch_handler import check_session_stopped -from frappe.translate import get_lang_code +from frappe.translate import get_lang_code, guess_language from frappe.utils.password import check_password, delete_login_failed_cache from frappe.core.doctype.activity_log.activity_log import add_authentication_log from frappe.twofactor import (should_run_2fa, authenticate_for_2factor, @@ -47,11 +47,6 @@ class HTTPRequest: # language self.set_lang() - if frappe.form_dict._lang: - lang = get_lang_code(frappe.form_dict._lang) - if lang: - frappe.local.lang = lang - self.validate_csrf_token() # write out latest cookies @@ -79,8 +74,12 @@ class HTTPRequest: frappe.throw(_("Invalid Request"), frappe.CSRFTokenError) def set_lang(self): - from frappe.translate import guess_language - frappe.local.lang = guess_language() + if frappe.form_dict._lang: + lang = get_lang_code(frappe.form_dict._lang) + if lang: + frappe.local.lang = lang + else: + frappe.local.lang = guess_language() def get_db_name(self): """get database name from conf""" From f5bd21cf46357d5e8bb98f30db7d68610a667b98 Mon Sep 17 00:00:00 2001 From: Fahim Ali Zain Date: Sat, 10 Jul 2021 10:52:43 +0530 Subject: [PATCH 061/164] fix: preferred_language cookie support for all users --- frappe/translate.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/frappe/translate.py b/frappe/translate.py index 4ff50d3fd0..8344436d94 100644 --- a/frappe/translate.py +++ b/frappe/translate.py @@ -22,12 +22,12 @@ from frappe.utils import is_html, strip, strip_html_tags def guess_language(lang_list=None): """Set `frappe.local.lang` from HTTP headers at beginning of request""" - user_preferred_language = frappe.request.cookies.get('preferred_language') - is_guest_user = not frappe.session.user or frappe.session.user == 'Guest' - if is_guest_user and user_preferred_language: - return user_preferred_language + preferred_language_cookie = frappe.request.cookies.get('preferred_language') + lang_codes = list(frappe.request.accept_languages.values()) + + if preferred_language_cookie: + lang_codes.append(preferred_language_cookie) - lang_codes = frappe.request.accept_languages.values() if not lang_codes: return frappe.local.lang From caafd9e2b519204b94b2203444dd1ea73086ac4d Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 13 Jul 2021 13:29:22 +0530 Subject: [PATCH 062/164] refactor: Simplify HTTPRequest class * For the sake of Readability and ease of understanding * Style updates --- frappe/auth.py | 74 ++++++++++++++++++++++++++++++++------------------ 1 file changed, 47 insertions(+), 27 deletions(-) diff --git a/frappe/auth.py b/frappe/auth.py index 479ec6bda4..a95c41906d 100644 --- a/frappe/auth.py +++ b/frappe/auth.py @@ -21,11 +21,40 @@ from urllib.parse import quote class HTTPRequest: def __init__(self): - # Get Environment variables - self.domain = frappe.request.host - if self.domain and self.domain.startswith('www.'): - self.domain = self.domain[4:] + # set frappe.local.request_ip + self.set_request_ip() + # load cookies + self.set_cookies() + + # set frappe.local.db + self.connect() + + # login and start/resume user session + self.set_session() + + # set request language + self.set_lang() + + # match csrf token from current session + self.validate_csrf_token() + + # write out latest cookies + frappe.local.cookie_manager.init_cookies() + + # check session status + check_session_stopped() + + @property + def domain(self): + if not getattr(self, "_domain", None): + self._domain = frappe.request.host + if self._domain and self._domain.startswith('www.'): + self._domain = self._domain[4:] + + return self._domain + + def set_request_ip(self): if frappe.get_request_header('X-Forwarded-For'): frappe.local.request_ip = (frappe.get_request_header('X-Forwarded-For').split(",")[0]).strip() @@ -35,32 +64,21 @@ class HTTPRequest: else: frappe.local.request_ip = '127.0.0.1' - # load cookies + def set_cookies(self): frappe.local.cookie_manager = CookieManager() - # set db - self.connect() - - # login + def set_session(self): frappe.local.login_manager = LoginManager() - # language - self.set_lang() - - self.validate_csrf_token() - - # write out latest cookies - frappe.local.cookie_manager.init_cookies() - - # check status - check_session_stopped() - def validate_csrf_token(self): if frappe.local.request and frappe.local.request.method in ("POST", "PUT", "DELETE"): - if not frappe.local.session: return - if not frappe.local.session.data.csrf_token \ - or frappe.local.session.data.device=="mobile" \ - or frappe.conf.get('ignore_csrf', None): + if not frappe.local.session: + return + if ( + not frappe.local.session.data.csrf_token + or frappe.local.session.data.device == "mobile" + or frappe.conf.get('ignore_csrf', None) + ): # not via boot return @@ -85,10 +103,12 @@ class HTTPRequest: """get database name from conf""" return conf.db_name - def connect(self, ac_name = None): + def connect(self): """connect to db, from ac_name or db_name""" - frappe.local.db = frappe.database.get_db(user = self.get_db_name(), \ - password = getattr(conf, 'db_password', '')) + frappe.local.db = frappe.database.get_db( + user=self.get_db_name(), + password=getattr(conf, 'db_password', '') + ) class LoginManager: def __init__(self): From c47cbfd2efb2f72133494b7b68940fe14410ccf8 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 13 Jul 2021 17:03:30 +0530 Subject: [PATCH 063/164] refactor: Set Language in HTTPHeader Order of priority for setting language: 1. Form Dict => _lang 2. Cookie => preferred_language 3. Request Header => Accept-Language 4. User document => language 5. System Settings => language Cookie is placed at #2 since the language picker in the navbar depends on it. And the Accept-Language header sends values based on the client's locales. --- Form Dict _lang now accepts language codes too. Previously, language names were used...for whatever reason. --- frappe/auth.py | 13 +++----- frappe/translate.py | 78 ++++++++++++++++++++++++++++++++------------- 2 files changed, 59 insertions(+), 32 deletions(-) diff --git a/frappe/auth.py b/frappe/auth.py index a95c41906d..9135b139ae 100644 --- a/frappe/auth.py +++ b/frappe/auth.py @@ -5,13 +5,13 @@ from frappe import _ import frappe import frappe.database import frappe.utils -from frappe.utils import cint, flt, get_datetime, datetime, date_diff, today +from frappe.utils import cint, get_datetime, datetime, date_diff, today import frappe.utils.user from frappe import conf from frappe.sessions import Session, clear_sessions, delete_session from frappe.modules.patch_handler import check_session_stopped -from frappe.translate import get_lang_code, guess_language -from frappe.utils.password import check_password, delete_login_failed_cache +from frappe.translate import guess_language +from frappe.utils.password import check_password from frappe.core.doctype.activity_log.activity_log import add_authentication_log from frappe.twofactor import (should_run_2fa, authenticate_for_2factor, confirm_otp_token, get_cached_user_pass) @@ -92,12 +92,7 @@ class HTTPRequest: frappe.throw(_("Invalid Request"), frappe.CSRFTokenError) def set_lang(self): - if frappe.form_dict._lang: - lang = get_lang_code(frappe.form_dict._lang) - if lang: - frappe.local.lang = lang - else: - frappe.local.lang = guess_language() + frappe.local.lang = guess_language() def get_db_name(self): """get database name from conf""" diff --git a/frappe/translate.py b/frappe/translate.py index 8344436d94..c567d0615c 100644 --- a/frappe/translate.py +++ b/frappe/translate.py @@ -11,6 +11,7 @@ import io import itertools import json import operator +import functools import os import re from csv import reader @@ -21,36 +22,64 @@ from frappe.utils import is_html, strip, strip_html_tags def guess_language(lang_list=None): - """Set `frappe.local.lang` from HTTP headers at beginning of request""" + """Set `frappe.local.lang` from HTTP headers at beginning of request + + Order of priority for setting language: + 1. Form Dict => _lang + 2. Cookie => preferred_language + 3. Request Header => Accept-Language + 4. User document => language + 5. System Settings => language + """ + + # fetch language from form_dict + if frappe.form_dict._lang: + language = get_lang_code( + frappe.form_dict._lang or get_parent_language(frappe.form_dict._lang) + ) + if language: + return language + + lang_set = set(lang_list or get_all_languages() or []) + + # fetch language from cookie preferred_language_cookie = frappe.request.cookies.get('preferred_language') - lang_codes = list(frappe.request.accept_languages.values()) if preferred_language_cookie: - lang_codes.append(preferred_language_cookie) + if preferred_language_cookie in lang_set: + return preferred_language_cookie - if not lang_codes: - return frappe.local.lang + parent_language = get_parent_language(language) + if parent_language in lang_set: + return parent_language - guess = None - if not lang_list: - lang_list = get_all_languages() or [] + # fetch language from request headers + accept_language = list(frappe.request.accept_languages.values()) - for l in lang_codes: - code = l.strip() - if not isinstance(code, str): - code = str(code, 'utf-8') - if code in lang_list or code == "en": - guess = code - break + for language in accept_language: + if language in lang_set: + return language - # check if parent language (pt) is setup, if variant (pt-BR) - if "-" in code: - code = code.split("-")[0] - if code in lang_list: - guess = code - break + parent_language = get_parent_language(language) + if parent_language in lang_set: + return parent_language + + # fallback to language set in User or System Settings + return frappe.local.lang + + +@functools.lru_cache(maxsize=None) +def get_parent_language(lang: str) -> str: + """If the passed language is a variant, return its parent + + Eg: + 1. zh-TW -> zh + 2. sr-BA -> sr + """ + is_language_variant = "-" in lang + if is_language_variant: + return lang[:lang.index("-")] - return guess or frappe.local.lang def get_user_lang(user=None): """Set frappe.local.lang from user preferences on session beginning or resumption""" @@ -75,7 +104,10 @@ def get_user_lang(user=None): return lang def get_lang_code(lang): - return frappe.db.get_value('Language', {'language_name': lang}) or lang + return ( + frappe.db.get_value("Language", {"name": lang}) + or frappe.db.get_value("Language", {"language_name": lang}) + ) def set_default_language(lang): """Set Global default language""" From 76ec9e44e4e9a7ade9509bc1f080c057aa5dbb3c Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 13 Jul 2021 17:09:03 +0530 Subject: [PATCH 064/164] refactor: Rename guess_language as get_language Guess suggests there's some AI involvement. The get_language function has a defined priority. It is deterministic, hence teh name change. --- frappe/auth.py | 4 ++-- frappe/translate.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/frappe/auth.py b/frappe/auth.py index 9135b139ae..7c8c119414 100644 --- a/frappe/auth.py +++ b/frappe/auth.py @@ -10,7 +10,7 @@ import frappe.utils.user from frappe import conf from frappe.sessions import Session, clear_sessions, delete_session from frappe.modules.patch_handler import check_session_stopped -from frappe.translate import guess_language +from frappe.translate import get_language from frappe.utils.password import check_password from frappe.core.doctype.activity_log.activity_log import add_authentication_log from frappe.twofactor import (should_run_2fa, authenticate_for_2factor, @@ -92,7 +92,7 @@ class HTTPRequest: frappe.throw(_("Invalid Request"), frappe.CSRFTokenError) def set_lang(self): - frappe.local.lang = guess_language() + frappe.local.lang = get_language() def get_db_name(self): """get database name from conf""" diff --git a/frappe/translate.py b/frappe/translate.py index c567d0615c..50da6e7297 100644 --- a/frappe/translate.py +++ b/frappe/translate.py @@ -21,7 +21,7 @@ from frappe.model.utils import InvalidIncludePath, render_include from frappe.utils import is_html, strip, strip_html_tags -def guess_language(lang_list=None): +def get_language(lang_list=None): """Set `frappe.local.lang` from HTTP headers at beginning of request Order of priority for setting language: From 736c6c9b8a388c5c9f87509432dd3d53a65de4d9 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 13 Jul 2021 18:18:40 +0530 Subject: [PATCH 065/164] fix: Don't redefine datetime * Sort imports * Update file header --- frappe/auth.py | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/frappe/auth.py b/frappe/auth.py index 7c8c119414..2e92e9fbbb 100644 --- a/frappe/auth.py +++ b/frappe/auth.py @@ -1,22 +1,20 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors -# MIT License. See license.txt -import datetime -from frappe import _ +# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors +# MIT License. See LICENSE +from urllib.parse import quote + import frappe import frappe.database import frappe.utils -from frappe.utils import cint, get_datetime, datetime, date_diff, today import frappe.utils.user -from frappe import conf -from frappe.sessions import Session, clear_sessions, delete_session -from frappe.modules.patch_handler import check_session_stopped -from frappe.translate import get_language -from frappe.utils.password import check_password +from frappe import _, conf from frappe.core.doctype.activity_log.activity_log import add_authentication_log -from frappe.twofactor import (should_run_2fa, authenticate_for_2factor, - confirm_otp_token, get_cached_user_pass) +from frappe.modules.patch_handler import check_session_stopped +from frappe.sessions import Session, clear_sessions, delete_session +from frappe.translate import get_language +from frappe.twofactor import authenticate_for_2factor, confirm_otp_token, get_cached_user_pass, should_run_2fa +from frappe.utils import cint, date_diff, datetime, get_datetime, today +from frappe.utils.password import check_password from frappe.website.utils import get_home_page -from urllib.parse import quote class HTTPRequest: From 0598ddf70ef83cf1bada858ffd7c7b6d8017a098 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 13 Jul 2021 18:19:38 +0530 Subject: [PATCH 066/164] fix: Clear preferred_language cookie post login If preferred_language was set in cookie pre login, clear it after a successful login so that User or Site specific settings can be applied --- frappe/auth.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/frappe/auth.py b/frappe/auth.py index 2e92e9fbbb..fc1cb09e1a 100644 --- a/frappe/auth.py +++ b/frappe/auth.py @@ -154,8 +154,9 @@ class LoginManager: self.make_session() self.setup_boot_cache() self.set_user_info() + self.clear_preferred_language() - def get_user_info(self, resume=False): + def get_user_info(self): self.info = frappe.db.get_value("User", self.user, ["user_type", "first_name", "last_name", "user_image"], as_dict=1) @@ -193,11 +194,13 @@ class LoginManager: frappe.local.response["redirect_to"] = redirect_to frappe.cache().hdel('redirect_after_login', self.user) - frappe.local.cookie_manager.set_cookie("full_name", self.full_name) frappe.local.cookie_manager.set_cookie("user_id", self.user) frappe.local.cookie_manager.set_cookie("user_image", self.info.user_image or "") + def clear_preferred_language(self): + frappe.local.cookie_manager.delete_cookie("preferred_language") + def make_session(self, resume=False): # start session frappe.local.session_obj = Session(user=self.user, resume=resume, From b4ea6c8890c50364a62d2151618f982597138761 Mon Sep 17 00:00:00 2001 From: Sagar Vora Date: Wed, 14 Jul 2021 12:21:28 +0530 Subject: [PATCH 067/164] fix: dont refresh form dashboard in `layout.refresh_sections` (#13650) --- frappe/public/js/frappe/form/layout.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/frappe/public/js/frappe/form/layout.js b/frappe/public/js/frappe/form/layout.js index 045e5dc1b3..528f485354 100644 --- a/frappe/public/js/frappe/form/layout.js +++ b/frappe/public/js/frappe/form/layout.js @@ -263,9 +263,6 @@ frappe.ui.form.Layout = class Layout { section.addClass("empty-section"); } }); - - this.frm && this.frm.dashboard.refresh(); - } refresh_fields (fields) { From bd854bb368023c43a90d1cb044a5282a082e9371 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Wed, 14 Jul 2021 18:16:50 +0530 Subject: [PATCH 068/164] fix: Do not create energy points for disabled users --- frappe/core/doctype/user/user.py | 13 +++++++++++++ .../doctype/energy_point_rule/energy_point_rule.py | 8 +++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py index 5b605504e8..53d1c9ffe5 100644 --- a/frappe/core/doctype/user/user.py +++ b/frappe/core/doctype/user/user.py @@ -53,6 +53,7 @@ class User(Document): def after_insert(self): create_notification_settings(self.name) frappe.cache().delete_key('users_for_mentions') + frappe.cache().delete_key('enabled_users') def validate(self): self.check_demo() @@ -129,6 +130,9 @@ class User(Document): if self.has_value_changed('allow_in_mentions') or self.has_value_changed('user_type'): frappe.cache().delete_key('users_for_mentions') + if self.has_value_changed('enabled'): + frappe.cache().delete_key('enabled_users') + def has_website_permission(self, ptype, user, verbose=False): """Returns true if current user is the session user""" return self.name == frappe.session.user @@ -392,6 +396,8 @@ class User(Document): if self.get('allow_in_mentions'): frappe.cache().delete_key('users_for_mentions') + frappe.cache().delete_key('enabled_users') + def before_rename(self, old_name, new_name, merge=False): self.check_demo() @@ -1230,3 +1236,10 @@ def generate_keys(user): def switch_theme(theme): if theme in ["Dark", "Light"]: frappe.db.set_value("User", frappe.session.user, "desk_theme", theme) + +def get_enabled_users(): + def _get_enabled_users(): + enabled_users = frappe.get_all("User", filters={"enabled": "1"}, pluck="name") + return enabled_users + + return frappe.cache().get_value("enabled_users", _get_enabled_users) \ No newline at end of file diff --git a/frappe/social/doctype/energy_point_rule/energy_point_rule.py b/frappe/social/doctype/energy_point_rule/energy_point_rule.py index 1c736528de..b673e8e199 100644 --- a/frappe/social/doctype/energy_point_rule/energy_point_rule.py +++ b/frappe/social/doctype/energy_point_rule/energy_point_rule.py @@ -5,6 +5,7 @@ import frappe from frappe import _ import frappe.cache_manager +from frappe.core.doctype.user.user import get_enabled_users from frappe.model import log_types from frappe.model.document import Document from frappe.social.doctype.energy_point_settings.energy_point_settings import is_energy_point_enabled @@ -44,7 +45,7 @@ class EnergyPointRule(Document): try: for user in users: - if not user or user == 'Administrator': continue + if not is_eligible_user(user): continue create_energy_points_log(reference_doctype, reference_name, { 'points': points, 'user': user, @@ -119,3 +120,8 @@ def get_energy_point_doctypes(): d.reference_doctype for d in frappe.get_all('Energy Point Rule', ['reference_doctype'], {'enabled': 1}) ] + +def is_eligible_user(user): + '''Checks if user is eligible to get energy points''' + enabled_users = get_enabled_users() + return user and user in enabled_users and user != 'Administrator' From e00aaf8cc492332ba18fc05c67013eee51f572a7 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 14 Jul 2021 19:30:01 +0530 Subject: [PATCH 069/164] BREAKING CHANGE: Drop frappe.lang in favour of frappe.local.lang --- frappe/__init__.py | 2 -- frappe/boot.py | 2 +- frappe/core/doctype/page/page.py | 2 +- frappe/desk/desk_page.py | 2 +- frappe/desk/query_report.py | 2 +- frappe/sessions.py | 2 +- 6 files changed, 5 insertions(+), 7 deletions(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index 1c978945c7..4015ea8090 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -130,8 +130,6 @@ error_log = local("error_log") debug_log = local("debug_log") message_log = local("message_log") -lang = local("lang") - # This if block is never executed when running the code. It is only used for # telling static code analyzer where to find dynamically defined attributes. if typing.TYPE_CHECKING: diff --git a/frappe/boot.py b/frappe/boot.py index 0589e32ac8..6636cb4329 100644 --- a/frappe/boot.py +++ b/frappe/boot.py @@ -208,7 +208,7 @@ def get_column(doctype): def load_translations(bootinfo): messages = frappe.get_lang_dict("boot") - bootinfo["lang"] = frappe.lang + bootinfo["lang"] = frappe.local.lang # load translated report names for name in bootinfo.user.all_reports: diff --git a/frappe/core/doctype/page/page.py b/frappe/core/doctype/page/page.py index 0ba0e309dd..3a6baef9c4 100644 --- a/frappe/core/doctype/page/page.py +++ b/frappe/core/doctype/page/page.py @@ -141,7 +141,7 @@ class Page(Document): # flag for not caching this page self._dynamic_page = True - if frappe.lang != 'en': + if frappe.local.lang != 'en': from frappe.translate import get_lang_js self.script += get_lang_js("page", self.name) diff --git a/frappe/desk/desk_page.py b/frappe/desk/desk_page.py index d373dbda0e..45e266b957 100644 --- a/frappe/desk/desk_page.py +++ b/frappe/desk/desk_page.py @@ -31,7 +31,7 @@ def getpage(): doc = get(page) # load translations - if frappe.lang != "en": + if frappe.local.lang != "en": send_translations(frappe.get_lang_dict("page", page)) frappe.response.docs.append(doc) diff --git a/frappe/desk/query_report.py b/frappe/desk/query_report.py index 3c0ebf11c1..a127594af1 100644 --- a/frappe/desk/query_report.py +++ b/frappe/desk/query_report.py @@ -187,7 +187,7 @@ def get_script(report_name): script = "frappe.query_reports['%s']={}" % report_name # load translations - if frappe.lang != "en": + if frappe.local.lang != "en": send_translations(frappe.get_lang_dict("report", report_name)) return { diff --git a/frappe/sessions.py b/frappe/sessions.py index 4d922d6769..85a13523ff 100644 --- a/frappe/sessions.py +++ b/frappe/sessions.py @@ -335,7 +335,7 @@ class Session: now = frappe.utils.now() self.data['data']['last_updated'] = now - self.data['data']['lang'] = str(frappe.lang) + self.data['data']['lang'] = str(frappe.local.lang) # update session in db last_updated = frappe.cache().hget("last_db_session_update", self.sid) From e8d50b9d3c6c0a39ed3602ffc7627214f0c7ed72 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 14 Jul 2021 19:32:30 +0530 Subject: [PATCH 070/164] chore: Add types for frappe.translate module *only for recently modified functions --- frappe/translate.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/frappe/translate.py b/frappe/translate.py index 50da6e7297..c3c88f9b0a 100644 --- a/frappe/translate.py +++ b/frappe/translate.py @@ -15,13 +15,14 @@ import functools import os import re from csv import reader +from typing import List, Union import frappe from frappe.model.utils import InvalidIncludePath, render_include from frappe.utils import is_html, strip, strip_html_tags -def get_language(lang_list=None): +def get_language(lang_list: List = None) -> str: """Set `frappe.local.lang` from HTTP headers at beginning of request Order of priority for setting language: @@ -81,7 +82,7 @@ def get_parent_language(lang: str) -> str: return lang[:lang.index("-")] -def get_user_lang(user=None): +def get_user_lang(user: str = None) -> str: """Set frappe.local.lang from user preferences on session beginning or resumption""" if not user: user = frappe.session.user @@ -103,7 +104,7 @@ def get_user_lang(user=None): return lang -def get_lang_code(lang): +def get_lang_code(lang: str) -> Union[str, None]: return ( frappe.db.get_value("Language", {"name": lang}) or frappe.db.get_value("Language", {"language_name": lang}) From 8faf2fd759095b3a54dd3ed0ff16e9ab949e1473 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 14 Jul 2021 19:33:48 +0530 Subject: [PATCH 071/164] refactor(minor): frappe.translate module * Remove unset limit for lru cache in get_parent_language * Simplify get_user_lang and add relevant comment --- frappe/translate.py | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/frappe/translate.py b/frappe/translate.py index c3c88f9b0a..15d771ecdd 100644 --- a/frappe/translate.py +++ b/frappe/translate.py @@ -1,4 +1,4 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors +# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt """ frappe.translate @@ -69,7 +69,7 @@ def get_language(lang_list: List = None) -> str: return frappe.local.lang -@functools.lru_cache(maxsize=None) +@functools.lru_cache() def get_parent_language(lang: str) -> str: """If the passed language is a variant, return its parent @@ -84,21 +84,17 @@ def get_parent_language(lang: str) -> str: def get_user_lang(user: str = None) -> str: """Set frappe.local.lang from user preferences on session beginning or resumption""" - if not user: - user = frappe.session.user - - # via cache + user = user or frappe.session.user lang = frappe.cache().hget("lang", user) if not lang: - - # if defined in user profile - lang = frappe.db.get_value("User", user, "language") - if not lang: - lang = frappe.db.get_default("lang") - - if not lang: - lang = frappe.local.lang or 'en' + # User.language => Session Defaults => frappe.local.lang => 'en' + lang = ( + frappe.db.get_value("User", user, "language") + or frappe.db.get_default("lang") + or frappe.local.lang + or "en" + ) frappe.cache().hset("lang", user, lang) From 2ac1c45c664af135bdaefc934ea258a01d76661f Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 14 Jul 2021 20:12:10 +0530 Subject: [PATCH 072/164] refactor: Maintain common list for Frappe Coverage settings * Maintain common settings for coverage settings of parallel and base test suites * Expand FRAPPE_EXCLUSIONS list based on coveralls.io report --- frappe/commands/utils.py | 24 ++++------------------- frappe/coverage.py | 35 ++++++++++++++++++++++++++++++++++ frappe/parallel_test_runner.py | 25 ++++-------------------- 3 files changed, 43 insertions(+), 41 deletions(-) create mode 100644 frappe/coverage.py diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py index 212642e01b..874ec90d47 100644 --- a/frappe/commands/utils.py +++ b/frappe/commands/utils.py @@ -551,32 +551,16 @@ def run_tests(context, app=None, module=None, doctype=None, test=(), profile=Fal if coverage: from coverage import Coverage + from frappe.coverage import STANDARD_INCLUSIONS, STANDARD_EXCLUSIONS, FRAPPE_EXCLUSIONS # Generate coverage report only for app that is being tested source_path = os.path.join(get_bench_path(), 'apps', app or 'frappe') - incl = [ - '*.py', - ] - omit = [ - '*.js', - '*.xml', - '*.pyc', - '*.css', - '*.less', - '*.scss', - '*.vue', - '*.html', - '*/test_*', - '*/node_modules/*', - '*/doctype/*/*_dashboard.py', - '*/patches/*', - ] + omit = STANDARD_EXCLUSIONS[:] if not app or app == 'frappe': - omit.append('*/tests/*') - omit.append('*/commands/*') + omit.extend(FRAPPE_EXCLUSIONS) - cov = Coverage(source=[source_path], omit=omit, include=incl) + cov = Coverage(source=[source_path], omit=omit, include=STANDARD_INCLUSIONS) cov.start() ret = frappe.test_runner.main(app, module, doctype, context.verbose, tests=tests, diff --git a/frappe/coverage.py b/frappe/coverage.py new file mode 100644 index 0000000000..a59c24a714 --- /dev/null +++ b/frappe/coverage.py @@ -0,0 +1,35 @@ +# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors +# MIT License. See LICENSE +""" + frappe.coverage + ~~~~~~~~~~~~~~~~ + + Coverage settings for frappe +""" + +STANDARD_INCLUSIONS = ["*.py"] + +STANDARD_EXCLUSIONS = [ + '*.js', + '*.xml', + '*.pyc', + '*.css', + '*.less', + '*.scss', + '*.vue', + '*.html', + '*/test_*', + '*/node_modules/*', + '*/doctype/*/*_dashboard.py', + '*/patches/*', +] + +FRAPPE_EXCLUSIONS = [ + "*/tests/*", + "*/commands/*", + "*/frappe/change_log/*", + "*/frappe/exceptions*", + "*frappe/setup.py", + "*/doctype/*/*_dashboard.py", + "*/patches/*", +] diff --git a/frappe/parallel_test_runner.py b/frappe/parallel_test_runner.py index 2f83b88572..6265498c96 100644 --- a/frappe/parallel_test_runner.py +++ b/frappe/parallel_test_runner.py @@ -111,33 +111,16 @@ class ParallelTestRunner(): if self.with_coverage: from coverage import Coverage from frappe.utils import get_bench_path + from frappe.coverage import STANDARD_INCLUSIONS, STANDARD_EXCLUSIONS, FRAPPE_EXCLUSIONS # Generate coverage report only for app that is being tested source_path = os.path.join(get_bench_path(), 'apps', self.app) - incl = [ - '*.py', - ] - omit = [ - '*.js', - '*.xml', - '*.pyc', - '*.css', - '*.less', - '*.scss', - '*.vue', - '*.pyc', - '*.html', - '*/test_*', - '*/node_modules/*', - '*/doctype/*/*_dashboard.py', - '*/patches/*', - ] + omit = STANDARD_EXCLUSIONS[:] if self.app == 'frappe': - omit.append('*/tests/*') - omit.append('*/commands/*') + omit.extend(FRAPPE_EXCLUSIONS) - self.coverage = Coverage(source=[source_path], omit=omit, include=incl) + self.coverage = Coverage(source=[source_path], omit=omit, include=STANDARD_INCLUSIONS) self.coverage.start() def save_coverage(self): From e71eef0ecc1b7601d5df83e007488db0fde236e9 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 14 Jul 2021 20:14:03 +0530 Subject: [PATCH 073/164] chore: Drop dead code (pythonrc file) --- frappe/pythonrc.py | 8 -------- 1 file changed, 8 deletions(-) delete mode 100755 frappe/pythonrc.py diff --git a/frappe/pythonrc.py b/frappe/pythonrc.py deleted file mode 100755 index 6761ead05b..0000000000 --- a/frappe/pythonrc.py +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env python2.7 - -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors -# MIT License. See license.txt - -import os -import frappe -frappe.connect(site=os.environ.get("site")) \ No newline at end of file From 4959dd02f1c8d7cac0ed677eed43ed0bc454a2a5 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 14 Jul 2021 20:14:37 +0530 Subject: [PATCH 074/164] refactor(minor): frappe.translate.get_messages_from_file * Don't re-define frappe util - get_bench_path * Add Python types * Style changes --- frappe/commands/utils.py | 2 +- frappe/translate.py | 20 +++++++++----------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py index 874ec90d47..8fc6877d4f 100644 --- a/frappe/commands/utils.py +++ b/frappe/commands/utils.py @@ -770,7 +770,7 @@ def get_version(output): "table": lambda: render_table( [["App", "Version", "Branch", "Commit"]] + [ - [app_info.app, app_info.version, app_info.branch, app_info.commit] + [app_info.app, app_info.version, app_info.branch, app_info.commit] for app_info in data ] ), diff --git a/frappe/translate.py b/frappe/translate.py index 15d771ecdd..ce4c3abf3d 100644 --- a/frappe/translate.py +++ b/frappe/translate.py @@ -15,11 +15,11 @@ import functools import os import re from csv import reader -from typing import List, Union +from typing import List, Union, Tuple import frappe from frappe.model.utils import InvalidIncludePath, render_include -from frappe.utils import is_html, strip, strip_html_tags +from frappe.utils import get_bench_path, is_html, strip, strip_html_tags def get_language(lang_list: List = None) -> str: @@ -558,7 +558,7 @@ def get_all_messages_from_js_files(app_name=None): return messages -def get_messages_from_file(path): +def get_messages_from_file(path: str) -> List[Tuple[str, str, str, str]]: """Returns a list of transatable strings from a code file :param path: path of the code file @@ -571,7 +571,7 @@ def get_messages_from_file(path): frappe.flags.scanned_files.append(path) - apps_path = get_bench_dir() + bench_path = get_bench_path() if os.path.exists(path): with open(path, 'r') as sourcefile: try: @@ -579,11 +579,12 @@ def get_messages_from_file(path): except Exception: print("Could not scan file for translation: {0}".format(path)) return [] - data = [(os.path.relpath(path, apps_path), message, context, line) \ - for line, message, context in extract_messages_from_code(file_contents)] - return data + + return [ + (os.path.relpath(path, bench_path), message, context, line) + for (line, message, context) in extract_messages_from_code(file_contents) + ] else: - # print "Translate: {0} missing".format(os.path.abspath(path)) return [] def extract_messages_from_code(code): @@ -797,9 +798,6 @@ def deduplicate_messages(messages): ret.append(next(g)) return ret -def get_bench_dir(): - return os.path.join(frappe.__file__, '..', '..', '..', '..') - def rename_language(old_name, new_name): if not frappe.db.exists('Language', new_name): return From bd91a33bb91ee57a3b6b73ea7cda1c285e81a0e4 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Wed, 14 Jul 2021 21:35:04 +0530 Subject: [PATCH 075/164] test: Add test case to validate EP for disabled user --- .../energy_point_log/test_energy_point_log.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/frappe/social/doctype/energy_point_log/test_energy_point_log.py b/frappe/social/doctype/energy_point_log/test_energy_point_log.py index 8c4dba5d6b..4a6e86463e 100644 --- a/frappe/social/doctype/energy_point_log/test_energy_point_log.py +++ b/frappe/social/doctype/energy_point_log/test_energy_point_log.py @@ -273,6 +273,31 @@ class TestEnergyPointLog(unittest.TestCase): self.assertEqual(points_after_reverting_todo, points_after_closing_todo - rule_points) self.assertEqual(points_after_saving_todo_again, points_after_reverting_todo + rule_points) + def test_energy_points_disabled_user(self): + frappe.set_user('test@example.com') + user = frappe.get_doc('User', 'test@example.com') + user.enabled = 0 + user.save() + todo_point_rule = create_energy_point_rule_for_todo() + energy_point_of_user = get_points('test@example.com') + + created_todo = create_a_todo() + + created_todo.status = 'Closed' + created_todo.save() + points_after_closing_todo = get_points('test@example.com') + + # do not update energy points for disabled user + self.assertEqual(points_after_closing_todo, energy_point_of_user) + + user.enabled = 1 + user.save() + + created_todo.save() + points_after_re_saving_todo = get_points('test@example.com') + self.assertEqual(points_after_re_saving_todo, energy_point_of_user + todo_point_rule.points) + + def create_energy_point_rule_for_todo(multiplier_field=None, for_doc_event='Custom', max_points=None, for_assigned_users=0, field_to_check=None, apply_once=False, user_field='owner'): name = 'ToDo Closed' From 421220a872e07da153b33392005c74a87f08b1ea Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 15 Jul 2021 01:04:04 +0530 Subject: [PATCH 076/164] test: Added tests for frappe.translate.get_language --- frappe/tests/test_translate.py | 54 ++++++++++++++++++++++++++++++++-- frappe/translate.py | 5 +++- 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/frappe/tests/test_translate.py b/frappe/tests/test_translate.py index f51f31d509..717b18ced8 100644 --- a/frappe/tests/test_translate.py +++ b/frappe/tests/test_translate.py @@ -1,13 +1,27 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors +# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -import frappe, unittest, os +import os +import unittest +from random import choices +from unittest.mock import patch + +import frappe import frappe.translate from frappe import _ +from frappe.auth import HTTPRequest +from frappe.translate import get_parent_language +from frappe.utils import set_request dirname = os.path.dirname(__file__) translation_string_file = os.path.join(dirname, 'translation_test_file.txt') +first_lang, second_lang, third_lang, fourth_lang, fifth_lang = choices( + frappe.get_all("Language", pluck="name"), k=5 +) class TestTranslate(unittest.TestCase): + def tearDown(self): + frappe.form_dict.pop("_lang", None) + def test_extract_message_from_file(self): data = frappe.translate.get_messages_from_file(translation_string_file) self.assertListEqual(data, expected_output) @@ -20,6 +34,42 @@ class TestTranslate(unittest.TestCase): finally: frappe.local.lang = 'en' + def test_request_language_resolution_with_form_dict(self): + """Test for frappe.translate.get_language + + Case 1: frappe.form_dict._lang is set + """ + + frappe.form_dict._lang = first_lang + + with patch.object(frappe.translate, "get_preferred_language_cookie", return_value=second_lang): + set_request(method="POST", path="/", headers=[("Accept-Language", third_lang)]) + HTTPRequest() + + self.assertIn(frappe.local.lang, [first_lang, get_parent_language(first_lang)]) + + def test_request_language_resolution_with_cookie(self): + """Test for frappe.translate.get_language + + Case 2: frappe.form_dict._lang is not set, but preferred_language cookie is + """ + + with patch.object(frappe.translate, "get_preferred_language_cookie", return_value=second_lang): + set_request(method="POST", path="/", headers=[("Accept-Language", third_lang)]) + HTTPRequest() + + self.assertIn(frappe.local.lang, [second_lang, get_parent_language(second_lang)]) + + def test_request_language_resolution_with_request_header(self): + """Test for frappe.translate.get_language + + Case 3: frappe.form_dict._lang & preferred_language cookie is not set, but Accept-Language header is + """ + set_request(method="POST", path="/", headers=[("Accept-Language", third_lang)]) + HTTPRequest() + self.assertIn(frappe.local.lang, [third_lang, get_parent_language(third_lang)]) + + expected_output = [ ('apps/frappe/frappe/tests/translation_test_file.txt', 'Warning: Unable to find {0} in any table related to {1}', 'This is some context', 2), ('apps/frappe/frappe/tests/translation_test_file.txt', 'Warning: Unable to find {0} in any table related to {1}', None, 4), diff --git a/frappe/translate.py b/frappe/translate.py index ce4c3abf3d..d5916f1761 100644 --- a/frappe/translate.py +++ b/frappe/translate.py @@ -44,7 +44,7 @@ def get_language(lang_list: List = None) -> str: lang_set = set(lang_list or get_all_languages() or []) # fetch language from cookie - preferred_language_cookie = frappe.request.cookies.get('preferred_language') + preferred_language_cookie = get_preferred_language_cookie() if preferred_language_cookie: if preferred_language_cookie in lang_set: @@ -906,3 +906,6 @@ def get_all_languages(with_language_name=False): @frappe.whitelist(allow_guest=True) def set_preferred_language_cookie(preferred_language): frappe.local.cookie_manager.set_cookie("preferred_language", preferred_language) + +def get_preferred_language_cookie(): + return frappe.request.cookies.get("preferred_language") From 2ccfb547b50a43e39ce4cbe8d53b81d525689fe6 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 15 Jul 2021 01:11:10 +0530 Subject: [PATCH 077/164] test: Added test to check sanity of HTTPRequest.set_lang --- frappe/tests/test_auth.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/frappe/tests/test_auth.py b/frappe/tests/test_auth.py index bc23cb591c..8447150006 100644 --- a/frappe/tests/test_auth.py +++ b/frappe/tests/test_auth.py @@ -4,8 +4,9 @@ import time import unittest import frappe -from frappe.auth import LoginAttemptTracker +from frappe.auth import HTTPRequest, LoginAttemptTracker from frappe.frappeclient import FrappeClient, AuthError +from frappe.utils import set_request class TestAuth(unittest.TestCase): def __init__(self, *args, **kwargs): @@ -124,3 +125,20 @@ class TestLoginAttemptTracker(unittest.TestCase): tracker.add_failure_attempt() self.assertTrue(tracker.is_user_allowed()) + +class TestFrappeHTTPRequest(unittest.TestCase): + # test frappe.auth.HTTPRequest + def test_set_language(self): + """Check if language is set on object initialization + + This is a test to ensure that language has changed. To test correctness + of frappe.local.lang, check out the tests of frappe.translate.get_language + """ + lang_before_request = frappe.local.lang + random_lang = frappe.get_all("Language", limit=1, pluck="name")[0] + set_request(method='POST', path='/') + frappe.form_dict._lang = random_lang + HTTPRequest() + self.assertTrue(hasattr(frappe.local, "lang")) + self.assertIsInstance(frappe.local.lang, str) + self.assertNotEqual(lang_before_request, frappe.local.lang) From f54894b1e75f30a60e5f611c097f123967901078 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 15 Jul 2021 01:12:13 +0530 Subject: [PATCH 078/164] chore: Update copyright year info in file header --- frappe/sessions.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/frappe/sessions.py b/frappe/sessions.py index 85a13523ff..8e30d0660c 100644 --- a/frappe/sessions.py +++ b/frappe/sessions.py @@ -1,4 +1,4 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors +# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt """ Boot session from cache or build @@ -249,7 +249,6 @@ class Session: data = self.get_session_record() if data: - # set language self.data.update({'data': data, 'user':data.user, 'sid': self.sid}) self.user = data.user validate_ip_address(self.user) From ffb020ce0ea9a0fb2c2ec7c0f8d03d986bc1cd1e Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Thu, 15 Jul 2021 08:37:47 +0530 Subject: [PATCH 079/164] fix: Make query postgres compatible --- frappe/desk/reportview.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/frappe/desk/reportview.py b/frappe/desk/reportview.py index b9382bc830..1dbc52eb5b 100644 --- a/frappe/desk/reportview.py +++ b/frappe/desk/reportview.py @@ -450,6 +450,7 @@ def get_stats(stats, doctype, filters=[]): filters=filters + [[tag, '!=', '']], group_by=tag, as_list=True, + distinct=1, ) if tag == '_user_tags': @@ -458,18 +459,22 @@ def get_stats(stats, doctype, filters=[]): fields=[tag, "count(*)"], filters=filters + [[tag, "in", ('', ',')]], as_list=True, - )[0][1] + group_by=tag, + order_by=tag, + ) + + no_tag_count = no_tag_count[0][1] if no_tag_count else 0 stats[tag].append([_("No Tags"), no_tag_count]) else: stats[tag] = tag_count except frappe.db.SQLError: - # does not work for child tables pass - except frappe.db.InternalError: + except frappe.db.InternalError as e: # raised when _user_tags column is added on the fly pass + return stats @frappe.whitelist() From 187c777539353b9b9ebfb1cd6089e776a114e5d7 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 15 Jul 2021 14:18:31 +0530 Subject: [PATCH 080/164] chore: Trigger GHA From a6537ce987b208eacc7f1e3b19a2b25de3dae127 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 15 Jul 2021 15:38:36 +0530 Subject: [PATCH 081/164] test: Invoke get_language directly instead of via HTTPRequest --- frappe/tests/test_translate.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/frappe/tests/test_translate.py b/frappe/tests/test_translate.py index 717b18ced8..edab3a82c3 100644 --- a/frappe/tests/test_translate.py +++ b/frappe/tests/test_translate.py @@ -8,8 +8,7 @@ from unittest.mock import patch import frappe import frappe.translate from frappe import _ -from frappe.auth import HTTPRequest -from frappe.translate import get_parent_language +from frappe.translate import get_language, get_parent_language from frappe.utils import set_request dirname = os.path.dirname(__file__) @@ -43,10 +42,9 @@ class TestTranslate(unittest.TestCase): frappe.form_dict._lang = first_lang with patch.object(frappe.translate, "get_preferred_language_cookie", return_value=second_lang): - set_request(method="POST", path="/", headers=[("Accept-Language", third_lang)]) - HTTPRequest() + return_val = get_language() - self.assertIn(frappe.local.lang, [first_lang, get_parent_language(first_lang)]) + self.assertIn(return_val, [first_lang, get_parent_language(first_lang)]) def test_request_language_resolution_with_cookie(self): """Test for frappe.translate.get_language @@ -56,9 +54,9 @@ class TestTranslate(unittest.TestCase): with patch.object(frappe.translate, "get_preferred_language_cookie", return_value=second_lang): set_request(method="POST", path="/", headers=[("Accept-Language", third_lang)]) - HTTPRequest() + return_val = get_language() - self.assertIn(frappe.local.lang, [second_lang, get_parent_language(second_lang)]) + self.assertIn(return_val, [second_lang, get_parent_language(second_lang)]) def test_request_language_resolution_with_request_header(self): """Test for frappe.translate.get_language @@ -66,8 +64,8 @@ class TestTranslate(unittest.TestCase): Case 3: frappe.form_dict._lang & preferred_language cookie is not set, but Accept-Language header is """ set_request(method="POST", path="/", headers=[("Accept-Language", third_lang)]) - HTTPRequest() - self.assertIn(frappe.local.lang, [third_lang, get_parent_language(third_lang)]) + return_val = get_language() + self.assertIn(return_val, [third_lang, get_parent_language(third_lang)]) expected_output = [ From 275a4335c291a56784000139e9e5f32f7afb2eb1 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 15 Jul 2021 15:53:21 +0530 Subject: [PATCH 082/164] ci: Override acceptable semantic commit name types * Add "BREAKING CHANGE" --- .github/semantic.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/semantic.yml b/.github/semantic.yml index e1e53bc1a4..fa15046b4a 100644 --- a/.github/semantic.yml +++ b/.github/semantic.yml @@ -11,3 +11,20 @@ allowRevertCommits: true # For allowed PR types: https://github.com/commitizen/conventional-commit-types/blob/v3.0.0/index.json # Tool Reference: https://github.com/zeke/semantic-pull-requests + +# By default types specified in commitizen/conventional-commit-types is used. +# See: https://github.com/commitizen/conventional-commit-types/blob/v3.0.0/index.json +# You can override the valid types +types: + - BREAKING CHANGE + - feat + - fix + - docs + - style + - refactor + - perf + - test + - build + - ci + - chore + - revert From 42b3c178002e65c7b025561ce3f86e73274b50f3 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 15 Jul 2021 16:02:44 +0530 Subject: [PATCH 083/164] Revert "BREAKING CHANGE: Drop frappe.lang in favour of frappe.local.lang" This reverts commit e00aaf8cc492332ba18fc05c67013eee51f572a7. --- This was pretty much a pointless change since frappe.lang just proxies to frappe.local.lang --- frappe/__init__.py | 2 ++ frappe/boot.py | 2 +- frappe/core/doctype/page/page.py | 2 +- frappe/desk/desk_page.py | 2 +- frappe/desk/query_report.py | 2 +- frappe/sessions.py | 2 +- 6 files changed, 7 insertions(+), 5 deletions(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index 4015ea8090..1c978945c7 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -130,6 +130,8 @@ error_log = local("error_log") debug_log = local("debug_log") message_log = local("message_log") +lang = local("lang") + # This if block is never executed when running the code. It is only used for # telling static code analyzer where to find dynamically defined attributes. if typing.TYPE_CHECKING: diff --git a/frappe/boot.py b/frappe/boot.py index 6636cb4329..0589e32ac8 100644 --- a/frappe/boot.py +++ b/frappe/boot.py @@ -208,7 +208,7 @@ def get_column(doctype): def load_translations(bootinfo): messages = frappe.get_lang_dict("boot") - bootinfo["lang"] = frappe.local.lang + bootinfo["lang"] = frappe.lang # load translated report names for name in bootinfo.user.all_reports: diff --git a/frappe/core/doctype/page/page.py b/frappe/core/doctype/page/page.py index 3a6baef9c4..0ba0e309dd 100644 --- a/frappe/core/doctype/page/page.py +++ b/frappe/core/doctype/page/page.py @@ -141,7 +141,7 @@ class Page(Document): # flag for not caching this page self._dynamic_page = True - if frappe.local.lang != 'en': + if frappe.lang != 'en': from frappe.translate import get_lang_js self.script += get_lang_js("page", self.name) diff --git a/frappe/desk/desk_page.py b/frappe/desk/desk_page.py index 45e266b957..d373dbda0e 100644 --- a/frappe/desk/desk_page.py +++ b/frappe/desk/desk_page.py @@ -31,7 +31,7 @@ def getpage(): doc = get(page) # load translations - if frappe.local.lang != "en": + if frappe.lang != "en": send_translations(frappe.get_lang_dict("page", page)) frappe.response.docs.append(doc) diff --git a/frappe/desk/query_report.py b/frappe/desk/query_report.py index a127594af1..3c0ebf11c1 100644 --- a/frappe/desk/query_report.py +++ b/frappe/desk/query_report.py @@ -187,7 +187,7 @@ def get_script(report_name): script = "frappe.query_reports['%s']={}" % report_name # load translations - if frappe.local.lang != "en": + if frappe.lang != "en": send_translations(frappe.get_lang_dict("report", report_name)) return { diff --git a/frappe/sessions.py b/frappe/sessions.py index 8e30d0660c..0b469616b8 100644 --- a/frappe/sessions.py +++ b/frappe/sessions.py @@ -334,7 +334,7 @@ class Session: now = frappe.utils.now() self.data['data']['last_updated'] = now - self.data['data']['lang'] = str(frappe.local.lang) + self.data['data']['lang'] = str(frappe.lang) # update session in db last_updated = frappe.cache().hget("last_db_session_update", self.sid) From 144880bfb10e2f11c23ff3e83b8e0a50d05631a3 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Thu, 15 Jul 2021 19:34:49 +0530 Subject: [PATCH 084/164] ci: make semgrep check diff-aware --- .../semgrep_rules/frappe_correctness.yml | 2 -- .github/helper/semgrep_rules/security.yml | 4 --- .github/workflows/semgrep.yml | 36 ++++++------------- 3 files changed, 10 insertions(+), 32 deletions(-) diff --git a/.github/helper/semgrep_rules/frappe_correctness.yml b/.github/helper/semgrep_rules/frappe_correctness.yml index faab3344a6..d9603e89aa 100644 --- a/.github/helper/semgrep_rules/frappe_correctness.yml +++ b/.github/helper/semgrep_rules/frappe_correctness.yml @@ -98,8 +98,6 @@ rules: languages: [python] severity: WARNING paths: - exclude: - - test_*.py include: - "*/**/doctype/*" diff --git a/.github/helper/semgrep_rules/security.yml b/.github/helper/semgrep_rules/security.yml index b2cc4b16fc..5a5098bf50 100644 --- a/.github/helper/semgrep_rules/security.yml +++ b/.github/helper/semgrep_rules/security.yml @@ -8,10 +8,6 @@ rules: dynamic content. Avoid it or use safe_eval(). languages: [python] severity: ERROR - paths: - exclude: - - frappe/__init__.py - - frappe/commands/utils.py - id: frappe-sqli-format-strings patterns: diff --git a/.github/workflows/semgrep.yml b/.github/workflows/semgrep.yml index 389524e968..e27b406df0 100644 --- a/.github/workflows/semgrep.yml +++ b/.github/workflows/semgrep.yml @@ -1,34 +1,18 @@ name: Semgrep on: - pull_request: - branches: - - develop - - version-13-hotfix - - version-13-pre-release + pull_request: { } + jobs: semgrep: name: Frappe Linter runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: Setup python3 - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - - name: Setup semgrep - run: | - python -m pip install -q semgrep - git fetch origin $GITHUB_BASE_REF:$GITHUB_BASE_REF -q - - - name: Semgrep errors - run: | - files=$(git diff --name-only --diff-filter=d $GITHUB_BASE_REF) - [[ -d .github/helper/semgrep_rules ]] && semgrep --severity ERROR --config=.github/helper/semgrep_rules --quiet --error $files - semgrep --config="r/python.lang.correctness" --quiet --error $files - - - name: Semgrep warnings - run: | - files=$(git diff --name-only --diff-filter=d $GITHUB_BASE_REF) - [[ -d .github/helper/semgrep_rules ]] && semgrep --severity WARNING --severity INFO --config=.github/helper/semgrep_rules --quiet $files + - uses: actions/checkout@v2 + - uses: returntocorp/semgrep-action@v1 + env: + SEMGREP_TIMEOUT: 120 + with: + config: >- + r/python.lang.correctness + .github/helper/semgrep_rules From c85ff679ef7a4c88dc4c625f69faed76ef195111 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 16 Jul 2021 12:02:13 +0530 Subject: [PATCH 085/164] fix(NotFoundPage): Set default value for http_status_code --- frappe/website/page_renderers/not_found_page.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/website/page_renderers/not_found_page.py b/frappe/website/page_renderers/not_found_page.py index af510fecfc..de631f5dfe 100644 --- a/frappe/website/page_renderers/not_found_page.py +++ b/frappe/website/page_renderers/not_found_page.py @@ -8,11 +8,11 @@ from frappe.website.utils import can_cache HOMEPAGE_PATHS = ('/', '/index', 'index') class NotFoundPage(TemplatePage): - def __init__(self, path, http_status_code): + def __init__(self, path, http_status_code=None): self.request_path = path self.request_url = frappe.local.request.url if hasattr(frappe.local, 'request') else '' path = '404' - http_status_code = 404 + http_status_code = http_status_code or 404 super().__init__(path=path, http_status_code=http_status_code) def can_render(self): From 39d5820e05fe7f7a1f4212174e273324352ad3b6 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Fri, 16 Jul 2021 13:15:20 +0530 Subject: [PATCH 086/164] fix: Allow navigation to the document with # in the docname --- frappe/public/js/frappe/router.js | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/frappe/public/js/frappe/router.js b/frappe/public/js/frappe/router.js index 2bfa7c7be6..f5967c0826 100644 --- a/frappe/public/js/frappe/router.js +++ b/frappe/public/js/frappe/router.js @@ -14,11 +14,10 @@ frappe.view_factories = []; frappe.route_options = null; frappe.route_hooks = {}; -$(window).on('hashchange', function() { +$(window).on('hashchange', function(e) { // v1 style routing, route is in hash - if (window.location.hash) { + if (window.location.hash && !frappe.router.is_app_route(e.currentTarget.pathname)) { let sub_path = frappe.router.get_sub_path(window.location.hash); - window.location.hash = ''; frappe.router.push_state(sub_path); return false; } @@ -52,14 +51,16 @@ $('body').on('click', 'a', function(e) { return override('/app'); } - // target has "#" ,this is a v1 style route, so remake it. - if (e.currentTarget.hash) { - return override(e.currentTarget.hash); - } - - // target has "/app, this is a v2 style route. - if (e.currentTarget.pathname && frappe.router.is_app_route(e.currentTarget.pathname)) { - return override(e.currentTarget.pathname); + if (frappe.router.is_app_route(e.currentTarget.pathname)) { + // target has "/app, this is a v2 style route. + if (e.currentTarget.pathname) { + return override(e.currentTarget.pathname + e.currentTarget.hash); + } + } else { + // target has "#" ,this is a v1 style route, so remake it. + if (e.currentTarget.hash) { + return override(e.currentTarget.hash); + } } }); @@ -349,8 +350,6 @@ frappe.router = { push_state(url) { // change the URL and call the router if (window.location.pathname !== url) { - // cleanup any remenants of v1 routing - window.location.hash = ''; // push state so the browser looks fine history.pushState(null, null, url); @@ -364,7 +363,11 @@ frappe.router = { // return clean sub_path from hash or url // supports both v1 and v2 routing if (!route) { - route = window.location.hash || (window.location.pathname + window.location.search); + route = window.location.pathname + window.location.hash + window.location.search; + if (route.includes('app#')) { + // to supports v1 + route = window.location.hash; + } } return this.strip_prefix(route); From d974fd8d4dac5b6ff397d9dc0f7a3f99f9b63cc3 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Fri, 16 Jul 2021 13:19:41 +0530 Subject: [PATCH 087/164] fix: Typo --- frappe/public/js/frappe/router.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/router.js b/frappe/public/js/frappe/router.js index f5967c0826..8f8704713e 100644 --- a/frappe/public/js/frappe/router.js +++ b/frappe/public/js/frappe/router.js @@ -365,7 +365,7 @@ frappe.router = { if (!route) { route = window.location.pathname + window.location.hash + window.location.search; if (route.includes('app#')) { - // to supports v1 + // to support v1 route = window.location.hash; } } From b690374afeb17fda8230151370766b5488de054a Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Fri, 16 Jul 2021 21:58:20 +0530 Subject: [PATCH 088/164] fix: Trigger worklow action on update_after_submit event --- frappe/hooks.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/frappe/hooks.py b/frappe/hooks.py index ac42a03461..f3d25d6bf4 100644 --- a/frappe/hooks.py +++ b/frappe/hooks.py @@ -171,6 +171,9 @@ doc_events = { "frappe.workflow.doctype.workflow_action.workflow_action.process_workflow_actions", "frappe.event_streaming.doctype.event_update_log.event_update_log.notify_consumers" ], + "on_update_after_submit": [ + "frappe.workflow.doctype.workflow_action.workflow_action.process_workflow_actions" + ], "on_change": [ "frappe.social.doctype.energy_point_rule.energy_point_rule.process_energy_points", "frappe.automation.doctype.milestone_tracker.milestone_tracker.evaluate_milestone" From 9b437f8c90b3a7ca955f5f0c168bf8001d8e2975 Mon Sep 17 00:00:00 2001 From: leela Date: Mon, 19 Jul 2021 12:18:25 +0530 Subject: [PATCH 089/164] feat: Add rq users --- frappe/commands/__init__.py | 4 +- frappe/commands/redis.py | 48 ++++++++++++++++++++ frappe/utils/__init__.py | 6 +++ frappe/utils/rq.py | 87 +++++++++++++++++++++++++++++++++++++ 4 files changed, 144 insertions(+), 1 deletion(-) create mode 100644 frappe/commands/redis.py create mode 100644 frappe/utils/rq.py diff --git a/frappe/commands/__init__.py b/frappe/commands/__init__.py index be9d107025..9ed333d034 100644 --- a/frappe/commands/__init__.py +++ b/frappe/commands/__init__.py @@ -102,7 +102,9 @@ def get_commands(): from .site import commands as site_commands from .translate import commands as translate_commands from .utils import commands as utils_commands + from .redis import commands as redis_commands - return list(set(scheduler_commands + site_commands + translate_commands + utils_commands)) + all_commands = scheduler_commands + site_commands + translate_commands + utils_commands + redis_commands + return list(set(all_commands)) commands = get_commands() diff --git a/frappe/commands/redis.py b/frappe/commands/redis.py new file mode 100644 index 0000000000..c608296cac --- /dev/null +++ b/frappe/commands/redis.py @@ -0,0 +1,48 @@ +import os + +import click +import redis + +import frappe +from frappe.utils.rq import RedisQueue +from frappe.installer import update_site_config + +@click.command('create-rq-users') +@click.option('--set-admin-password', is_flag=True, default=False, help='Set new Redis admin(default user) password') +@click.option('--reset-passwords', is_flag=True, default=False, help='Remove all existing passwords') +def create_rq_users(set_admin_password=False, reset_passwords=False): + """Create Redis Queue users and add to acl and app configs. + + acl config file will be used by redis server while starting the server + and app config is used by app while connecting to redis server. + """ + acl_file_path = os.path.abspath('../config/redis_queue.acl') + acl_list, user_credentials = RedisQueue.gen_acl_list( + reset_passwords=reset_passwords, set_admin_password=set_admin_password) + + with open(acl_file_path, 'w') as f: + f.writelines([acl+'\n' for acl in acl_list]) + + sites_path = os.getcwd() + common_site_config_path = os.path.join(sites_path, 'common_site_config.json') + update_site_config("rq_username", user_credentials['bench'][0], validate=False, + site_config_path=common_site_config_path) + update_site_config("rq_password", user_credentials['bench'][1], validate=False, + site_config_path=common_site_config_path) + + if set_admin_password: + env_key = 'RQ_ADMIN_PASWORD' + click.secho('Redis admin password is successfully set up. ' + 'Include below line in .bashrc file for system to use', + fg='green' + ) + click.secho(f"`export {env_key}={user_credentials['default'][1]}`") + click.secho('NOTE: Please save the admin password as you ' + 'can not access redis server without the password', + fg='yellow' + ) + + +commands = [ + create_rq_users +] diff --git a/frappe/utils/__init__.py b/frappe/utils/__init__.py index af9d5de1ee..bf139173d6 100644 --- a/frappe/utils/__init__.py +++ b/frappe/utils/__init__.py @@ -383,6 +383,12 @@ def get_files_path(*path, **kwargs): def get_bench_path(): return os.path.realpath(os.path.join(os.path.dirname(frappe.__file__), '..', '..', '..')) +def get_bench_id(): + return frappe.local.conf.get('bench_id', 'DefaultBench') + +def get_site_id(site=None): + return f"{site or frappe.local.site}@{get_bench_id()}" + def get_backups_path(): return get_site_path("private", "backups") diff --git a/frappe/utils/rq.py b/frappe/utils/rq.py new file mode 100644 index 0000000000..5e9b9dcd5d --- /dev/null +++ b/frappe/utils/rq.py @@ -0,0 +1,87 @@ +import redis + +import frappe +from frappe.utils import get_site_id, get_bench_id, random_string + + +class RedisQueue: + def __init__(self, conn): + self.conn = conn + + def add_user(self, username, password=None): + """Create or update the user. + """ + password = password or self.conn.acl_genpass() + user_settings = self.get_new_user_settings(username, password) + is_created = self.conn.acl_setuser(**user_settings) + return frappe._dict(user_settings) if is_created else {} + + @classmethod + def get_connection(cls, username='default', password=None): + domain = frappe.local.conf.redis_queue.split("redis://", 1)[-1] + url = f"redis://{username}:{password or ''}@{domain}" + conn = redis.from_url(url) + conn.ping() + return conn + + @classmethod + def new(cls, username='default', password=None): + return cls(cls.get_connection(username, password)) + + @classmethod + def set_admin_password(cls, cur_password=None, new_password=None, reset_passwords=False): + username = 'default' + conn = cls.get_connection(username, cur_password) + password = '+'+(new_password or conn.acl_genpass()) + conn.acl_setuser( + username=username, enabled=True, reset_passwords=reset_passwords, passwords=password + ) + return password[1:] + + @classmethod + def get_new_user_settings(cls, username, password): + d = {} + d['username'] = username + d['passwords'] = '+'+password + d['reset_keys'] = True + d['enabled'] = True + d['keys'] = cls.get_acl_key_rules() + d['commands'] = cls.get_acl_command_rules() + return d + + @classmethod + def get_acl_key_rules(cls, include_key_prefix=False): + """FIXME: Find better way + """ + rules = ['rq:[^q]*', 'rq:queues', f'rq:queue:{get_bench_id()}:*'] + if include_key_prefix: + return ['~'+pattern for pattern in rules] + return rules + + @classmethod + def get_acl_command_rules(cls): + return ['+@all', '-@admin'] + + @classmethod + def gen_acl_list(cls, reset_passwords=False, set_admin_password=False): + """Generate list of ACL users needed for this branch. + + This list contains default ACL user and the bench ACL user(used by all sites incase of ACL is enabled). + """ + with frappe.init_site(): + bench_username = get_bench_id() + bench_user_rules = cls.get_acl_key_rules(include_key_prefix=True) + cls.get_acl_command_rules() + + bench_user_rule_str = ' '.join(bench_user_rules).strip() + bench_user_password = random_string(20) + bench_user_resetpass = (reset_passwords and 'resetpass') or '' + + default_username = 'default' + _default_user_password = random_string(20) if set_admin_password else '' + default_user_password = '>'+_default_user_password if _default_user_password else 'nopass' + default_user_resetpass = (reset_passwords and set_admin_password and 'resetpass') or '' + + return [ + f'user {default_username} on {default_user_password} {default_user_resetpass} ~* &* +@all', + f'user {bench_username} on >{bench_user_password} {bench_user_resetpass} {bench_user_rule_str}' + ], {'bench': (bench_username, bench_user_password), 'default': (default_username, _default_user_password)} From cd499af6b28ef18dbe664ba757240ddfcf46d244 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Tue, 20 Jul 2021 08:43:42 +0530 Subject: [PATCH 090/164] fix: Move layout colors to common css_variables.scss - Layout colors are used by website as well --- frappe/public/scss/common/css_variables.scss | 11 +++++++++++ frappe/public/scss/desk/css_variables.scss | 15 +-------------- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/frappe/public/scss/common/css_variables.scss b/frappe/public/scss/common/css_variables.scss index 48a8a48f5f..112238bfe5 100644 --- a/frappe/public/scss/common/css_variables.scss +++ b/frappe/public/scss/common/css_variables.scss @@ -179,9 +179,20 @@ --text-on-pink: var(--pink-500); --text-on-cyan: var(--cyan-600); + // Layout Colors + --bg-color: var(--gray-50); + --fg-color: white; + --navbar-bg: white; + --fg-hover-color: var(--gray-100); + --card-bg: var(--fg-color); + --disabled-text-color: var(--gray-700); --disabled-control-bg: var(--gray-50); --control-bg: var(--gray-100); --control-bg-on-gray: var(--gray-200); + --awesomebar-focus-bg: var(--fg-color); + --modal-bg: white; + --toast-bg: var(--modal-bg); + --popover-bg: white; --awesomplete-hover-bg: var(--control-bg); diff --git a/frappe/public/scss/desk/css_variables.scss b/frappe/public/scss/desk/css_variables.scss index 5bb2614dcc..282e21433b 100644 --- a/frappe/public/scss/desk/css_variables.scss +++ b/frappe/public/scss/desk/css_variables.scss @@ -25,20 +25,7 @@ $input-height: 28px !default; --navbar-height: 60px; - // Layout Colors - --bg-color: var(--gray-50); - --fg-color: white; - --navbar-bg: white; - --fg-hover-color: var(--gray-100); - --card-bg: var(--fg-color); - --disabled-text-color: var(--gray-700); - --disabled-control-bg: var(--gray-50); - --control-bg: var(--gray-100); - --control-bg-on-gray: var(--gray-200); - --awesomebar-focus-bg: var(--fg-color); - --modal-bg: white; - --toast-bg: var(--modal-bg); - --popover-bg: white; + --appreciation-color: var(--dark-green-600); --appreciation-bg: var(--dark-green-100); From 2a9b9875f904804df6663e78416d4eee7890bd03 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Tue, 20 Jul 2021 10:11:45 +0530 Subject: [PATCH 091/164] fix: Make pad only after dom is ready - Signature control used to break on first form load --- frappe/public/js/frappe/form/controls/signature.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/frappe/public/js/frappe/form/controls/signature.js b/frappe/public/js/frappe/form/controls/signature.js index bcbdfa0d27..445f5b7b3b 100644 --- a/frappe/public/js/frappe/form/controls/signature.js +++ b/frappe/public/js/frappe/form/controls/signature.js @@ -53,13 +53,15 @@ frappe.ui.form.ControlSignature = class ControlSignature extends frappe.ui.form. this.img = $("") .appendTo(this.img_wrapper).toggle(false); } - refresh_input(e) { + refresh_input() { + // signature dom is not ready + if (!this.body) return; // prevent to load the second time this.make_pad(); this.$wrapper.find(".control-input").toggle(false); this.set_editable(this.get_status()=="Write"); this.load_pad(); - if(this.get_status()=="Read") { + if (this.get_status() == "Read") { $(this.disp_area).toggle(false); } } From 1e1f9115402f99e7ec25b7f07f867b7aebf6327b Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Tue, 20 Jul 2021 15:00:07 +0530 Subject: [PATCH 092/164] fix: Use 1em margin for p outside container Revert https://github.com/frappe/frappe/commit/397848f9f7f692c3b8ac43efbc85b231ac32cb08#diff-cad6d40df66d9378e252342fe4148d3f8114a3c361dffd048fc3b09d5d9024b3R39 --- frappe/public/scss/email.bundle.scss | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/frappe/public/scss/email.bundle.scss b/frappe/public/scss/email.bundle.scss index b1a8d6c485..9b58c130a2 100644 --- a/frappe/public/scss/email.bundle.scss +++ b/frappe/public/scss/email.bundle.scss @@ -36,7 +36,13 @@ a { } p { - margin: 5px 0 !important; + margin: 1em 0 !important; +} + +.with-container { + p { + margin: 5px 0 !important; + } } .ql-editor { From e1c78bb9b357ba6b3c601dc595a8e6836e5571c6 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Tue, 20 Jul 2021 15:21:22 +0530 Subject: [PATCH 093/164] test: Update test --- frappe/email/test_email_body.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/email/test_email_body.py b/frappe/email/test_email_body.py index 8e637273ed..2c7d119fce 100644 --- a/frappe/email/test_email_body.py +++ b/frappe/email/test_email_body.py @@ -127,7 +127,7 @@ w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> ''' transformed_html = '''

Hi John

-

This is a test email

+

This is a test email

''' self.assertTrue(transformed_html in inline_style_in_html(html)) From e1ade10a54df10d66fea326b350e628ae3a4568a Mon Sep 17 00:00:00 2001 From: leela Date: Tue, 20 Jul 2021 18:59:40 +0530 Subject: [PATCH 094/164] fix: `show Report` button in report view (cherry picked from commit 0917d59a40f77f4514b29599ae96cf10ff0573ac) --- frappe/public/js/frappe/router.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/public/js/frappe/router.js b/frappe/public/js/frappe/router.js index 2bfa7c7be6..1f5a250872 100644 --- a/frappe/public/js/frappe/router.js +++ b/frappe/public/js/frappe/router.js @@ -172,7 +172,7 @@ frappe.router = { standard_route = ['List', doctype_route.doctype, frappe.utils.to_title_case(route[2])]; if (route[3]) { // calendar / kanban / dashboard / folder name - standard_route.push(...route.splice(3, route.length)); + standard_route.push([...route].splice(3, route.length)); } } return standard_route; @@ -298,7 +298,7 @@ frappe.router = { new_route = [this.slug(route[1]), 'view', route[2].toLowerCase()]; // calendar / inbox / file folder - if (route[3]) new_route.push(...route.slice(3, route.length)); + if (route[3]) new_route.push([...route].slice(3, route.length)); } else { if ($.isPlainObject(route[2])) { frappe.route_options = route[2]; From d459847ae39f8e9917686c3db6340ee95103085e Mon Sep 17 00:00:00 2001 From: leela Date: Tue, 13 Jul 2021 20:25:32 +0530 Subject: [PATCH 095/164] refactor: set amended docname to original docname Currently, whenever a document is amended it's name is set to name-X(X is a counter) when amended again and so on. In this PR, we have changed all cancelled doc patterns to name-CAN-X, so that amended docs can use the original name instead of name-X. --- frappe/core/doctype/doctype/test_doctype.py | 8 +- frappe/model/document.py | 9 +- frappe/model/naming.py | 83 ++++++++++++++++--- frappe/patches.txt | 1 + frappe/patches/v13_0/rename_cancelled_docs.py | 27 ++++++ frappe/public/js/frappe/form/form.js | 24 +++--- frappe/public/js/frappe/router.js | 6 ++ frappe/tests/test_naming.py | 34 ++++++++ 8 files changed, 164 insertions(+), 28 deletions(-) create mode 100644 frappe/patches/v13_0/rename_cancelled_docs.py diff --git a/frappe/core/doctype/doctype/test_doctype.py b/frappe/core/doctype/doctype/test_doctype.py index 1e1a01a685..4b6d0e4794 100644 --- a/frappe/core/doctype/doctype/test_doctype.py +++ b/frappe/core/doctype/doctype/test_doctype.py @@ -348,6 +348,7 @@ class TestDocType(unittest.TestCase): dump_docs = json.dumps(docs.get('docs')) cancel_all_linked_docs(dump_docs) data_link_doc.cancel() + data_doc.name = '{}-CAN-0'.format(data_doc.name) data_doc.load_from_db() self.assertEqual(data_link_doc.docstatus, 2) self.assertEqual(data_doc.docstatus, 2) @@ -371,7 +372,7 @@ class TestDocType(unittest.TestCase): for data in link_doc.get('permissions'): data.submit = 1 data.cancel = 1 - link_doc.insert() + link_doc.insert(ignore_if_duplicate=True) #create first parent doctype test_doc_1 = new_doctype('Test Doctype 1') @@ -386,7 +387,7 @@ class TestDocType(unittest.TestCase): for data in test_doc_1.get('permissions'): data.submit = 1 data.cancel = 1 - test_doc_1.insert() + test_doc_1.insert(ignore_if_duplicate=True) #crete second parent doctype doc = new_doctype('Test Doctype 2') @@ -401,7 +402,7 @@ class TestDocType(unittest.TestCase): for data in link_doc.get('permissions'): data.submit = 1 data.cancel = 1 - doc.insert() + doc.insert(ignore_if_duplicate=True) # create doctype data data_link_doc_1 = frappe.new_doc('Test Linked Doctype 1') @@ -432,6 +433,7 @@ class TestDocType(unittest.TestCase): # checking that doc for Test Doctype 2 is not canceled self.assertRaises(frappe.LinkExistsError, data_link_doc_1.cancel) + data_doc_2.name = '{}-CAN-0'.format(data_doc_2.name) data_doc.load_from_db() data_doc_2.load_from_db() self.assertEqual(data_link_doc_1.docstatus, 2) diff --git a/frappe/model/document.py b/frappe/model/document.py index 61160e1f01..e974ae2a3e 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -5,7 +5,7 @@ import time from frappe import _, msgprint, is_whitelisted from frappe.utils import flt, cstr, now, get_datetime_str, file_lock, date_diff from frappe.model.base_document import BaseDocument, get_controller -from frappe.model.naming import set_new_name +from frappe.model.naming import set_new_name, gen_new_name_for_cancelled_doc from werkzeug.exceptions import NotFound, Forbidden import hashlib, json from frappe.model import optional_fields, table_fields @@ -705,7 +705,6 @@ class Document(BaseDocument): else: tmp = frappe.db.sql("""select modified, docstatus from `tab{0}` where name = %s for update""".format(self.doctype), self.name, as_dict=True) - if not tmp: frappe.throw(_("Record does not exist")) else: @@ -916,8 +915,12 @@ class Document(BaseDocument): @whitelist.__func__ def _cancel(self): - """Cancel the document. Sets `docstatus` = 2, then saves.""" + """Cancel the document. Sets `docstatus` = 2, then saves. + """ self.docstatus = 2 + new_name = gen_new_name_for_cancelled_doc(self) + frappe.rename_doc(self.doctype, self.name, new_name, force=True, show_alert=False) + self.name = new_name self.save() @whitelist.__func__ diff --git a/frappe/model/naming.py b/frappe/model/naming.py index fe136adce8..6dff0aaff6 100644 --- a/frappe/model/naming.py +++ b/frappe/model/naming.py @@ -28,7 +28,7 @@ def set_new_name(doc): doc.name = None if getattr(doc, "amended_from", None): - _set_amended_name(doc) + doc.name = _get_amended_name(doc) return elif getattr(doc.meta, "issingle", False): @@ -221,6 +221,15 @@ def revert_series_if_last(key, name, doc=None): * prefix = #### and hashes = 2021 (hash doesn't exist) * will search hash in key then accordingly get prefix = "" """ + if hasattr(doc, 'amended_from'): + # do not revert if doc is amended, since cancelled docs still exist + if doc.docstatus != 2 and doc.amended_from: + return + + # for first cancelled doc + if doc.docstatus == 2 and not doc.amended_from: + name, _ = NameParser.parse_docname(doc.name, sep='-CAN-') + if ".#" in key: prefix, hashes = key.rsplit(".", 1) if "#" not in hashes: @@ -303,16 +312,9 @@ def append_number_if_name_exists(doctype, value, fieldname="name", separator="-" return value -def _set_amended_name(doc): - am_id = 1 - am_prefix = doc.amended_from - if frappe.db.get_value(doc.doctype, doc.amended_from, "amended_from"): - am_id = cint(doc.amended_from.split("-")[-1]) + 1 - am_prefix = "-".join(doc.amended_from.split("-")[:-1]) # except the last hyphen - - doc.name = am_prefix + "-" + str(am_id) - return doc.name - +def _get_amended_name(doc): + name, _ = NameParser(doc).parse_amended_from() + return name def _field_autoname(autoname, doc, skip_slicing=None): """ @@ -323,7 +325,6 @@ def _field_autoname(autoname, doc, skip_slicing=None): name = (cstr(doc.get(fieldname)) or "").strip() return name - def _prompt_autoname(autoname, doc): """ Generate a name using Prompt option. This simply means the user will have to set the name manually. @@ -354,3 +355,61 @@ def _format_autoname(autoname, doc): name = re.sub(r"(\{[\w | #]+\})", get_param_value_for_match, autoname_value) return name + +class NameParser: + """Parse document name and return all the parts of it. + + NOTE: It handles cancellend and amended doc parsing for now. It can be expanded. + """ + def __init__(self, doc): + self.doc = doc + + def parse_name(self): + if not hasattr(self.doc, "amended_from"): + return (self.doc.name, None, None) + + #If document is cancelled document + if hasattr(self.doc, "amended_from") and self.doc.docstatus == 2: + return self.parse_docname(self.doc.name, sep='-CAN-') + return self.parse_docname(self.doc.name) + + def parse_amended_from(self): + if not getattr(self.doc, 'amended_from', None): + return (None, None) + return self.parse_docname(self.doc.amended_from, '-CAN-') + + @classmethod + def parse_docname(cls, name, sep='-'): + split_list = name.rsplit(sep, 1) + + if len(split_list) == 1: + return (name, None) + return (split_list[0], split_list[1]) + +def get_cancelled_doc_latest_counter(tname, docname): + """Get the latest counter used for cancelled docs of given docname. + """ + name_prefix = f'{docname}-CAN-' + + rows = frappe.db.sql(""" + select + name + from `tab{tname}` + where + name like %(name_prefix)s and docstatus=2 + """.format(tname=tname), {'name_prefix': name_prefix+'%'}, as_dict=1) + + if not rows: + return -1 + return max([int(row.name.replace(name_prefix, '') or -1) for row in rows]) + +def gen_new_name_for_cancelled_doc(doc): + """Generate a new name for cancelled document. + """ + if getattr(doc, "amended_from", None): + name, _ = NameParser(doc).parse_amended_from() + else: + name = doc.name + + counter = get_cancelled_doc_latest_counter(doc.doctype, name) + return f'{name}-CAN-{counter+1}' diff --git a/frappe/patches.txt b/frappe/patches.txt index 7605d8ea2b..a9c5807df0 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -180,3 +180,4 @@ frappe.patches.v12_0.rename_uploaded_files_with_proper_name frappe.patches.v13_0.queryreport_columns frappe.patches.v13_0.jinja_hook frappe.patches.v13_0.update_notification_channel_if_empty +frappe.patches.v13_0.rename_cancelled_docs diff --git a/frappe/patches/v13_0/rename_cancelled_docs.py b/frappe/patches/v13_0/rename_cancelled_docs.py new file mode 100644 index 0000000000..2e99a6f3cd --- /dev/null +++ b/frappe/patches/v13_0/rename_cancelled_docs.py @@ -0,0 +1,27 @@ +import frappe +from frappe.model.naming import NameParser +from frappe.model.rename_doc import rename_doc + +def execute(): + """Rename already cancelled documents by adding `CAN-X` postfix instead of `-X`. + """ + for doctype in frappe.db.get_all('DocType'): + doctype = frappe.get_doc('DocType', doctype.name) + if doctype.is_submittable and frappe.db.table_exists(doctype.name): + cancelled_docs = frappe.db.get_all(doctype.name, ['amended_from', 'name'], {'docstatus':2}) + + for doc in cancelled_docs: + if '-CAN-' in doc.name: + continue + + current_name = doc.name + + if getattr(doc, "amended_from", None): + orig_name, counter = NameParser.parse_docname(doc.name) + else: + orig_name, counter = doc.name, 0 + new_name = f'{orig_name}-CAN-{counter or 0}' + + print(f"Renaming {doctype.name} record from {current_name} to {new_name}") + rename_doc(doctype.name, current_name, new_name, ignore_permissions=True, show_alert=False) + frappe.db.commit() diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js index 8064f90a98..30f023c987 100644 --- a/frappe/public/js/frappe/form/form.js +++ b/frappe/public/js/frappe/form/form.js @@ -770,32 +770,36 @@ frappe.ui.form.Form = class FrappeForm { } _cancel(btn, callback, on_error, skip_confirm) { - const me = this; const cancel_doc = () => { frappe.validated = true; - me.script_manager.trigger("before_cancel").then(() => { + this.script_manager.trigger("before_cancel").then(() => { if (!frappe.validated) { - return me.handle_save_fail(btn, on_error); + return this.handle_save_fail(btn, on_error); } - var after_cancel = function(r) { + const original_name = this.docname; + const after_cancel = (r) => { if (r.exc) { - me.handle_save_fail(btn, on_error); + this.handle_save_fail(btn, on_error); } else { frappe.utils.play_sound("cancel"); - me.refresh(); callback && callback(); - me.script_manager.trigger("after_cancel"); + this.script_manager.trigger("after_cancel"); + frappe.run_serially([ + () => this.rename_notify(this.doctype, original_name, r.docs[0].name), + () => frappe.router.clear_re_route(this.doctype, original_name), + () => this.refresh(), + ]); } }; - frappe.ui.form.save(me, "cancel", after_cancel, btn); + frappe.ui.form.save(this, "cancel", after_cancel, btn); }); } if (skip_confirm) { cancel_doc(); } else { - frappe.confirm(__("Permanently Cancel {0}?", [this.docname]), cancel_doc, me.handle_save_fail(btn, on_error)); + frappe.confirm(__("Permanently Cancel {0}?", [this.docname]), cancel_doc, this.handle_save_fail(btn, on_error)); } }; @@ -817,7 +821,7 @@ frappe.ui.form.Form = class FrappeForm { 'docname': this.doc.name }).then(is_amended => { if (is_amended) { - frappe.throw(__('This document is already amended, you cannot ammend it again')); + frappe.throw(__('This document is already amended, you cannot amend it again')); } this.validate_form_action("Amend"); var me = this; diff --git a/frappe/public/js/frappe/router.js b/frappe/public/js/frappe/router.js index 2bfa7c7be6..17e0e689d3 100644 --- a/frappe/public/js/frappe/router.js +++ b/frappe/public/js/frappe/router.js @@ -235,6 +235,12 @@ frappe.router = { } }, + clear_re_route(doctype, docname) { + delete frappe.re_route[ + `${encodeURIComponent(frappe.router.slug(doctype))}/${encodeURIComponent(docname)}` + ]; + }, + set_title(sub_path) { if (frappe.route_titles[sub_path]) { frappe.utils.set_title(frappe.route_titles[sub_path]); diff --git a/frappe/tests/test_naming.py b/frappe/tests/test_naming.py index 557993882f..78071a4120 100644 --- a/frappe/tests/test_naming.py +++ b/frappe/tests/test_naming.py @@ -116,3 +116,37 @@ class TestNaming(unittest.TestCase): self.assertEqual(current_index.get('current'), 2) frappe.db.sql("""delete from `tabSeries` where name = %s""", series) + + def test_naming_for_cancelled_and_amended_doc(self): + submittable_doctype = frappe.get_doc({ + "doctype": "DocType", + "module": "Core", + "custom": 1, + "is_submittable": 1, + "permissions": [{ + "role": "System Manager", + "read": 1 + }], + "name": 'Submittable Doctype' + }).insert(ignore_if_duplicate=True) + + doc = frappe.new_doc('Submittable Doctype') + doc.save() + original_name = doc.name + + doc.submit() + doc.cancel() + cancelled_name = doc.name + self.assertEqual(cancelled_name, "{}-CAN-0".format(original_name)) + + amended_doc = frappe.copy_doc(doc) + amended_doc.docstatus = 0 + amended_doc.amended_from = doc.name + amended_doc.save() + self.assertEqual(amended_doc.name, original_name) + + amended_doc.submit() + amended_doc.cancel() + self.assertEqual(amended_doc.name, "{}-CAN-1".format(original_name)) + + submittable_doctype.delete() From 09ca4d5af3c8aeda0496d8d2a16b025df8ca275f Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Wed, 21 Jul 2021 14:21:44 +0530 Subject: [PATCH 096/164] fix: V1 support and external links --- frappe/public/js/frappe/router.js | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/frappe/public/js/frappe/router.js b/frappe/public/js/frappe/router.js index 8f8704713e..0d8315bbc8 100644 --- a/frappe/public/js/frappe/router.js +++ b/frappe/public/js/frappe/router.js @@ -47,20 +47,18 @@ $('body').on('click', 'a', function(e) { return; } - if (href==='') { + if (href === '') { return override('/app'); } + if (href.startsWith('#')) { + // target startswith "#", this is a v1 style route, so remake it. + return override(e.currentTarget.hash); + } + if (frappe.router.is_app_route(e.currentTarget.pathname)) { // target has "/app, this is a v2 style route. - if (e.currentTarget.pathname) { - return override(e.currentTarget.pathname + e.currentTarget.hash); - } - } else { - // target has "#" ,this is a v1 style route, so remake it. - if (e.currentTarget.hash) { - return override(e.currentTarget.hash); - } + return override(e.currentTarget.pathname + e.currentTarget.hash); } }); From fa6cb14ee5221ec4d9db82bc731be8910eefb7b0 Mon Sep 17 00:00:00 2001 From: Aradhya-Tripathi Date: Wed, 21 Jul 2021 19:15:38 +0530 Subject: [PATCH 097/164] refactor: moved from raw queries to frappe orm in core --- frappe/core/doctype/activity_log/feed.py | 15 +++-- .../domain_settings/domain_settings.py | 3 +- .../core/doctype/log_settings/log_settings.py | 1 + frappe/core/doctype/report/test_report.py | 9 ++- frappe/core/doctype/role/role.py | 3 +- .../scheduled_job_type/scheduled_job_type.py | 7 +- frappe/core/doctype/user/user.py | 27 +++++--- .../user_permission/test_user_permission.py | 19 ++++-- .../user_permission/user_permission.py | 66 +++++++++++++------ .../permission_manager/permission_manager.py | 6 +- 10 files changed, 106 insertions(+), 50 deletions(-) diff --git a/frappe/core/doctype/activity_log/feed.py b/frappe/core/doctype/activity_log/feed.py index caa3cae613..cf8f7416e8 100644 --- a/frappe/core/doctype/activity_log/feed.py +++ b/frappe/core/doctype/activity_log/feed.py @@ -27,12 +27,15 @@ def update_feed(doc, method=None): feed = frappe._dict(feed) doctype = feed.doctype or doc.doctype name = feed.name or doc.name - - # delete earlier feed - frappe.db.sql("""delete from `tabActivity Log` - where - reference_doctype=%s and reference_name=%s - and link_doctype=%s""", (doctype, name,feed.link_doctype)) + + + frappe.db.delete(doctype="Activity Log", conditions={"reference_doctype": doctype, + "reference_name": name, + "link_doctype": name.feed.link_doctype}) + # frappe.db.sql("""delete from `tabActivity Log` + # where + # reference_doctype=%s and reference_name=%s + # and link_doctype=%s""", (doctype, name,feed.link_doctype)) frappe.get_doc({ "doctype": "Activity Log", "reference_doctype": doctype, diff --git a/frappe/core/doctype/domain_settings/domain_settings.py b/frappe/core/doctype/domain_settings/domain_settings.py index 7ad0aeff21..6ca180def1 100644 --- a/frappe/core/doctype/domain_settings/domain_settings.py +++ b/frappe/core/doctype/domain_settings/domain_settings.py @@ -34,7 +34,8 @@ class DomainSettings(Document): all_domains = list((frappe.get_hooks('domains') or {})) def remove_role(role): - frappe.db.sql('delete from `tabHas Role` where role=%s', role) + frappe.db.delete(doctype="Has Role", conditions={"role": role}) + # frappe.db.sql('delete from `tabHas Role` where role=%s', role) frappe.set_value('Role', role, 'disabled', 1) for domain in all_domains: diff --git a/frappe/core/doctype/log_settings/log_settings.py b/frappe/core/doctype/log_settings/log_settings.py index e73aa8dac1..776fcc92e9 100644 --- a/frappe/core/doctype/log_settings/log_settings.py +++ b/frappe/core/doctype/log_settings/log_settings.py @@ -13,6 +13,7 @@ class LogSettings(Document): self.clear_email_queue() def clear_error_logs(self): + # frappe.db.delete(doctype="Error Log", conditions="") frappe.db.sql(""" DELETE FROM `tabError Log` WHERE `creation` < (NOW() - INTERVAL '{0}' DAY) """.format(self.clear_error_log_after)) diff --git a/frappe/core/doctype/report/test_report.py b/frappe/core/doctype/report/test_report.py index 9d0c0b9af0..2bab6adf19 100644 --- a/frappe/core/doctype/report/test_report.py +++ b/frappe/core/doctype/report/test_report.py @@ -81,9 +81,14 @@ class TestReport(unittest.TestCase): self.assertDictEqual({'name': 'Administrator', 'user_type': 'System User', 'email': 'admin@example.com'}, admin_dict) def test_report_permissions(self): + frappe.set_user('test@example.com') - frappe.db.sql("""delete from `tabHas Role` where parent = %s - and role = 'Test Has Role'""", frappe.session.user, auto_commit=1) + frappe.db.delete(doctype="Has Role", conditions={ + "parent": frappe.session.user, + "role": "Test Has Role" + }) + # frappe.db.sql("""delete from `tabHas Role` where parent = %s + # and role = 'Test Has Role'""", frappe.session.user, auto_commit=1) if not frappe.db.exists('Role', 'Test Has Role'): role = frappe.get_doc({ diff --git a/frappe/core/doctype/role/role.py b/frappe/core/doctype/role/role.py index 02482c75ca..fae8ae2d28 100644 --- a/frappe/core/doctype/role/role.py +++ b/frappe/core/doctype/role/role.py @@ -38,7 +38,8 @@ class Role(Document): self.set(key, 0) def remove_roles(self): - frappe.db.sql("delete from `tabHas Role` where role = %s", self.name) + frappe.db.delete(doctype="Has Role", conditions={"role": self.name}) + # frappe.db.sql("delete from `tabHas Role` where role = %s", self.name) frappe.clear_cache() def on_update(self): diff --git a/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py b/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py index 59089d12ad..d6c94aded1 100644 --- a/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py +++ b/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py @@ -13,7 +13,6 @@ from frappe.model.document import Document from frappe.utils import get_datetime, now_datetime from frappe.utils.background_jobs import enqueue, get_jobs - class ScheduledJobType(Document): def autoname(self): self.name = ".".join(self.method.split(".")[-2:]) @@ -110,7 +109,11 @@ class ScheduledJobType(Document): return 'long' if ('Long' in self.frequency) else 'default' def on_trash(self): - frappe.db.sql('delete from `tabScheduled Job Log` where scheduled_job_type=%s', self.name) + + frappe.db.delete(doctype="Scheduled Job Log", conditions={ + "scheduled_job_type": self.name + }) + # frappe.db.sql('delete from `tabScheduled Job Log` where scheduled_job_type=%s', self.name) @frappe.whitelist() diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py index 53d1c9ffe5..f69af4e4a8 100644 --- a/frappe/core/doctype/user/user.py +++ b/frappe/core/doctype/user/user.py @@ -17,7 +17,7 @@ from frappe.website.utils import is_signup_disabled from frappe.rate_limiter import rate_limit from frappe.utils.background_jobs import enqueue from frappe.core.doctype.user_type.user_type import user_linked_with_permission_on_doctype - +from frappe.database.database import Database STANDARD_USERS = ("Guest", "Administrator") @@ -367,23 +367,30 @@ class User(Document): if getattr(frappe.local, "login_manager", None): frappe.local.login_manager.logout(user=self.name) - # delete todos - frappe.db.sql("""DELETE FROM `tabToDo` WHERE `owner`=%s""", (self.name,)) + frappe.db.delete("Todo", {"owner": self.name}) + # frappe.db.sql("""DELETE FROM `tabToDo` WHERE `owner`=%s""", (self.name,)) frappe.db.sql("""UPDATE `tabToDo` SET `assigned_by`=NULL WHERE `assigned_by`=%s""", (self.name,)) # delete events - frappe.db.sql("""delete from `tabEvent` where owner=%s - and event_type='Private'""", (self.name,)) + frappe.db.delete("Event", {"owner": self.name, "event_type": "Private"}) + # frappe.db.sql("""delete from `tabEvent` where owner=%s + # and event_type='Private'""", (self.name,)) # delete shares - frappe.db.sql("""delete from `tabDocShare` where user=%s""", self.name) + frappe.db.delete("DocShare", {"user": self.name}) + # frappe.db.sql("""delete from `tabDocShare` where user=%s""", self.name) # delete messages - frappe.db.sql("""delete from `tabCommunication` - where communication_type in ('Chat', 'Notification') - and reference_doctype='User' - and (reference_name=%s or owner=%s)""", (self.name, self.name)) + # TODO: CHANGE THIS FROM ABHISHEK KA PYPIKA + frappe.db.delete("Communication", { + "reference_doctype": "User", + "communication_type": ("in", ("Chat", "Notification")), + }) + # frappe.db.sql("""delete from `tabCommunication` + # where communication_type in ('Chat', 'Notification') + # and reference_doctype='User' + # and (reference_name=%s or owner=%s)""", (self.name, self.name)) # unlink contact frappe.db.sql("""update `tabContact` diff --git a/frappe/core/doctype/user_permission/test_user_permission.py b/frappe/core/doctype/user_permission/test_user_permission.py index 1a442b53e7..9393e47ee9 100644 --- a/frappe/core/doctype/user_permission/test_user_permission.py +++ b/frappe/core/doctype/user_permission/test_user_permission.py @@ -10,12 +10,19 @@ import unittest class TestUserPermission(unittest.TestCase): def setUp(self): - frappe.db.sql("""DELETE FROM `tabUser Permission` - WHERE `user` in ( - 'test_bulk_creation_update@example.com', - 'test_user_perm1@example.com', - 'nested_doc_user@example.com')""") - frappe.delete_doc_if_exists("DocType", "Person") + + frappe.db.delete("User Permission", { + "user": ("in", ("test_bulk_creation_update@example.com", + "test_user_perm1@example.com", + "nested_doc_user@example.com")) + }) + + # frappe.db.sql("""DELETE FROM `tabUser Permission` + # WHERE `user` in ( + # 'test_bulk_creation_update@example.com', + # 'test_user_perm1@example.com', + # 'nested_doc_user@example.com')""") + # frappe.delete_doc_if_exists("DocType", "Person") frappe.db.sql_ddl("DROP TABLE IF EXISTS `tabPerson`") frappe.delete_doc_if_exists("DocType", "Doc A") frappe.db.sql_ddl("DROP TABLE IF EXISTS `tabDoc A`") diff --git a/frappe/core/doctype/user_permission/user_permission.py b/frappe/core/doctype/user_permission/user_permission.py index 4aa5797c7f..5b3822d82f 100644 --- a/frappe/core/doctype/user_permission/user_permission.py +++ b/frappe/core/doctype/user_permission/user_permission.py @@ -182,7 +182,11 @@ def clear_user_permissions(user, for_doctype): frappe.only_for('System Manager') total = frappe.db.count('User Permission', filters = dict(user=user, allow=for_doctype)) if total: - frappe.db.sql('DELETE FROM `tabUser Permission` WHERE `user`=%s AND `allow`=%s', (user, for_doctype)) + frappe.db.delete("User Permission", { + "user": user, + "allow": for_doctype + }) + # frappe.db.sql('DELETE FROM `tabUser Permission` WHERE `user`=%s AND `allow`=%s', (user, for_doctype)) frappe.clear_cache() return total @@ -232,28 +236,50 @@ def insert_user_perm(user, doctype, docname, is_default=0, hide_descendants=0, a user_perm.insert() def remove_applicable(perm_applied_docs, user, doctype, docname): + for applicable_for in perm_applied_docs: - frappe.db.sql("""DELETE FROM `tabUser Permission` - WHERE `user`=%s - AND `applicable_for`=%s - AND `allow`=%s - AND `for_value`=%s - """, (user, applicable_for, doctype, docname)) - + frappe.db.delete("User Permission", { + "user": user, + "applicable_for": applicable_for, + "allow": doctype, + "for_value": docname + }) + # + # frappe.db.sql("""DELETE FROM `tabUser Permission` + # WHERE `user`=%s + # AND `applicable_for`=%s + # AND `allow`=%s + # AND `for_value`=%s + # """, (user, applicable_for, doctype, docname)) def remove_apply_to_all(user, doctype, docname): - frappe.db.sql("""DELETE from `tabUser Permission` - WHERE `user`=%s - AND `apply_to_all_doctypes`=1 - AND `allow`=%s - AND `for_value`=%s - """,(user, doctype, docname)) + + frappe.db.delete("User Permission", { + "user": user, + "apply_to_all_doctypes": 1, + "allow": doctype, + "for_value": docname + }) + # frappe.db.sql("""DELETE from `tabUser Permission` + # WHERE `user`=%s + # AND `apply_to_all_doctypes`=1 + # AND `allow`=%s + # AND `for_value`=%s + # """,(user, doctype, docname)) def update_applicable(already_applied, to_apply, user, doctype, docname): for applied in already_applied: if applied not in to_apply: - frappe.db.sql("""DELETE FROM `tabUser Permission` - WHERE `user`=%s - AND `applicable_for`=%s - AND `allow`=%s - AND `for_value`=%s - """,(user, applied, doctype, docname)) + + frappe.db.delete("User Permission", { + "user": user, + "applicable_for": applied, + "allow": doctype, + "for_value": docname + }) + + # frappe.db.sql("""DELETE FROM `tabUser Permission` + # WHERE `user`=%s + # AND `applicable_for`=%s + # AND `allow`=%s + # AND `for_value`=%s + # """,(user, applied, doctype, docname)) diff --git a/frappe/core/page/permission_manager/permission_manager.py b/frappe/core/page/permission_manager/permission_manager.py index 15c7cb55ae..0a76283813 100644 --- a/frappe/core/page/permission_manager/permission_manager.py +++ b/frappe/core/page/permission_manager/permission_manager.py @@ -111,8 +111,10 @@ def remove(doctype, role, permlevel): setup_custom_perms(doctype) name = frappe.get_value('Custom DocPerm', dict(parent=doctype, role=role, permlevel=permlevel)) - - frappe.db.sql('delete from `tabCustom DocPerm` where name=%s', name) + frappe.db.delete("Custom DocPerm", { + "name": name + }) + # frappe.db.sql('delete from `tabCustom DocPerm` where name=%s', name) if not frappe.get_all('Custom DocPerm', dict(parent=doctype)): frappe.throw(_('There must be atleast one permission rule.'), title=_('Cannot Remove')) From 9aa17db392d54f52cc7eda223eb3ece276ee74a1 Mon Sep 17 00:00:00 2001 From: Aradhya-Tripathi Date: Wed, 21 Jul 2021 19:16:17 +0530 Subject: [PATCH 098/164] refactor: moved from raw queries to frappe orm in desk --- .../dashboard_chart/test_dashboard_chart.py | 6 ++++-- frappe/desk/doctype/event/event.py | 11 ++++++++--- .../desk/doctype/route_history/route_history.py | 17 +++++++++++------ frappe/desk/doctype/tag/tag.py | 13 +++++++++++-- 4 files changed, 34 insertions(+), 13 deletions(-) diff --git a/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py b/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py index 78d133b2d5..7133640c21 100644 --- a/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py +++ b/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py @@ -64,7 +64,8 @@ class TestDashboardChart(unittest.TestCase): if frappe.db.exists('Dashboard Chart', 'Test Empty Dashboard Chart'): frappe.delete_doc('Dashboard Chart', 'Test Empty Dashboard Chart') - frappe.db.sql('delete from `tabError Log`') + frappe.db.delete("Error Log") + # frappe.db.sql('delete from `tabError Log`') frappe.get_doc(dict( doctype = 'Dashboard Chart', @@ -94,7 +95,8 @@ class TestDashboardChart(unittest.TestCase): if frappe.db.exists('Dashboard Chart', 'Test Empty Dashboard Chart 2'): frappe.delete_doc('Dashboard Chart', 'Test Empty Dashboard Chart 2') - frappe.db.sql('delete from `tabError Log`') + frappe.db.delete("Error Log") + # frappe.db.sql('delete from `tabError Log`') # create one data point frappe.get_doc(dict(doctype = 'Error Log', creation = '2018-06-01 00:00:00')).insert() diff --git a/frappe/desk/doctype/event/event.py b/frappe/desk/doctype/event/event.py index 57c89eaf2e..f9a0dd74fd 100644 --- a/frappe/desk/doctype/event/event.py +++ b/frappe/desk/doctype/event/event.py @@ -338,9 +338,14 @@ def delete_events(ref_type, ref_name, delete_event=False): total_participants = frappe.get_all("Event Participants", filters={"parenttype": "Event", "parent": participation.parent}) if len(total_participants) <= 1: - frappe.db.sql("DELETE FROM `tabEvent` WHERE `name` = %(name)s", {'name': participation.parent}) - - frappe.db.sql("DELETE FROM `tabEvent Participants ` WHERE `name` = %(name)s", {'name': participation.name}) + frappe.db.delete("Event", { + "name": participation.parent + }) + # frappe.db.sql("DELETE FROM `tabEvent` WHERE `name` = %(name)s", {'name': participation.parent}) + frappe.db.delete("Event Participants", { + "name": participation.name + }) + # frappe.db.sql("DELETE FROM `tabEvent Participants ` WHERE `name` = %(name)s", {'name': participation.name}) # Close events if ends_on or repeat_till is less than now_datetime def set_status_of_events(): diff --git a/frappe/desk/doctype/route_history/route_history.py b/frappe/desk/doctype/route_history/route_history.py index b82077f485..bfa2711b8f 100644 --- a/frappe/desk/doctype/route_history/route_history.py +++ b/frappe/desk/doctype/route_history/route_history.py @@ -32,11 +32,16 @@ def flush_old_route_records(): fields=['modified'], order_by='modified desc') - frappe.db.sql(''' - DELETE - FROM `tabRoute History` - WHERE `modified` <= %(modified)s and `user`=%(modified)s - ''', { + frappe.db.delete("Route History", { "modified": last_record_to_keep[0].modified, "user": user - }) \ No newline at end of file + }) + + # frappe.db.sql(''' + # DELETE + # FROM `tabRoute History` + # WHERE `modified` <= %(modified)s and `user`=%(modified)s + # ''', { + # "modified": last_record_to_keep[0].modified, + # "user": user + # }) \ No newline at end of file diff --git a/frappe/desk/doctype/tag/tag.py b/frappe/desk/doctype/tag/tag.py index 4ea5c9cd7e..8b78405d06 100644 --- a/frappe/desk/doctype/tag/tag.py +++ b/frappe/desk/doctype/tag/tag.py @@ -123,7 +123,11 @@ def delete_tags_for_document(doc): if not frappe.db.table_exists("Tag Link"): return - frappe.db.sql("""DELETE FROM `tabTag Link` WHERE `document_type`=%s AND `document_name`=%s""", (doc.doctype, doc.name)) + frappe.db.delete("Tag Link", { + "document_type": doc.doctype, + "document_name": doc.name + }) + # frappe.db.sql("""DELETE FROM `tabTag Link` WHERE `document_type`=%s AND `document_name`=%s""", (doc.doctype, doc.name)) def update_tags(doc, tags): """ @@ -161,7 +165,12 @@ def get_deleted_tags(new_tags, existing_tags): return list(set(existing_tags) - set(new_tags)) def delete_tag_for_document(dt, dn, tag): - frappe.db.sql("""DELETE FROM `tabTag Link` WHERE `document_type`=%s AND `document_name`=%s AND tag=%s""", (dt, dn, tag)) + frappe.db.delete("Tag Link", { + "document_type": dt, + "document_name": dn, + "tag": tag + }) + # frappe.db.sql("""DELETE FROM `tabTag Link` WHERE `document_type`=%s AND `document_name`=%s AND tag=%s""", (dt, dn, tag)) @frappe.whitelist() def get_documents_for_tag(tag): From a256219dd27b5251b90e0d04b1a5b7797117ea6c Mon Sep 17 00:00:00 2001 From: Aradhya-Tripathi Date: Wed, 21 Jul 2021 19:16:59 +0530 Subject: [PATCH 099/164] refactor: moved from raw queries in frappe email to frappe orm --- frappe/email/doctype/unhandled_email/unhandled_email.py | 7 +++++-- frappe/email/queue.py | 1 + 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/frappe/email/doctype/unhandled_email/unhandled_email.py b/frappe/email/doctype/unhandled_email/unhandled_email.py index 6414dbece3..850007d85f 100644 --- a/frappe/email/doctype/unhandled_email/unhandled_email.py +++ b/frappe/email/doctype/unhandled_email/unhandled_email.py @@ -10,5 +10,8 @@ class UnhandledEmail(Document): def remove_old_unhandled_emails(): - frappe.db.sql("""DELETE FROM `tabUnhandled Email` - WHERE creation < %s""", frappe.utils.add_days(frappe.utils.nowdate(), -30)) + frappe.db.delete("Unhandled Email", { + "creation": ("<", frappe.utils.add_days(frappe.utils.nowdate(), -30)) + }) + # frappe.db.sql("""DELETE FROM `tabUnhandled Email` + # WHERE creation < %s""", frappe.utils.add_days(frappe.utils.nowdate(), -30)) \ No newline at end of file diff --git a/frappe/email/queue.py b/frappe/email/queue.py index 885a306cfb..40f1c7be3a 100755 --- a/frappe/email/queue.py +++ b/frappe/email/queue.py @@ -173,6 +173,7 @@ def clear_outbox(days=None): WHERE `priority`=0 AND `modified` < (NOW() - INTERVAL '{0}' DAY)""".format(days)) if email_queues: + # TODO: email_queues IN frappe.db.sql frappe.db.sql("""DELETE FROM `tabEmail Queue` WHERE `name` IN ({0})""".format( ','.join(['%s']*len(email_queues) )), tuple(email_queues)) From eb7e95ae26a542e82066b302e6bc85edffe7df25 Mon Sep 17 00:00:00 2001 From: Aradhya-Tripathi Date: Wed, 21 Jul 2021 19:19:06 +0530 Subject: [PATCH 100/164] refactor: using frappe orm --- .../doctype/custom_field/custom_field.py | 14 ++++-- .../doctype/customize_form/customize_form.py | 8 +++- frappe/model/__init__.py | 26 +++++++---- frappe/model/delete_doc.py | 46 +++++++++++++++---- frappe/permissions.py | 7 ++- frappe/sessions.py | 5 +- 6 files changed, 78 insertions(+), 28 deletions(-) diff --git a/frappe/custom/doctype/custom_field/custom_field.py b/frappe/custom/doctype/custom_field/custom_field.py index 7e6ea1875a..1d8f4ae67a 100644 --- a/frappe/custom/doctype/custom_field/custom_field.py +++ b/frappe/custom/doctype/custom_field/custom_field.py @@ -85,11 +85,15 @@ class CustomField(Document): frappe.bold(self.label))) # delete property setter entries - frappe.db.sql("""\ - DELETE FROM `tabProperty Setter` - WHERE doc_type = %s - AND field_name = %s""", - (self.dt, self.fieldname)) + frappe.db.delete("Property Setter", { + "doc_type": self.dt, + "field_name": self.fieldname + }) + # frappe.db.sql("""\ + # DELETE FROM `tabProperty Setter` + # WHERE doc_type = %s + # AND field_name = %s""", + # (self.dt, self.fieldname)) frappe.clear_cache(doctype=self.dt) diff --git a/frappe/custom/doctype/customize_form/customize_form.py b/frappe/custom/doctype/customize_form/customize_form.py index 1b8977acc4..3b7de448d3 100644 --- a/frappe/custom/doctype/customize_form/customize_form.py +++ b/frappe/custom/doctype/customize_form/customize_form.py @@ -20,8 +20,12 @@ from frappe.core.doctype.doctype.doctype import validate_series class CustomizeForm(Document): def on_update(self): - frappe.db.sql("delete from tabSingles where doctype='Customize Form'") - frappe.db.sql("delete from `tabCustomize Form Field`") + frappe.db.delete("Singles", { + "doctype": "Customize Form" + }) + frappe.db.delete("Customize Form Field") + # frappe.db.sql("delete from tabSingles where doctype='Customize Form'") + # frappe.db.sql("delete from `tabCustomize Form Field`") @frappe.whitelist() def fetch_to_customize(self): diff --git a/frappe/model/__init__.py b/frappe/model/__init__.py index 75122f5aba..763fd36cc7 100644 --- a/frappe/model/__init__.py +++ b/frappe/model/__init__.py @@ -151,21 +151,29 @@ def delete_fields(args_dict, delete=0): fields = args_dict[dt] if not fields: continue - - frappe.db.sql(""" - DELETE FROM `tabDocField` - WHERE parent='%s' AND fieldname IN (%s) - """ % (dt, ", ".join(["'{}'".format(f) for f in fields]))) + + frappe.db.delete("DocField", { + "parent": dt, + "fieldname": ("in", ", ".join(["'{}'".format(f) for f in fields])) + }) + # frappe.db.sql(""" + # DELETE FROM `tabDocField` + # WHERE parent='%s' AND fieldname IN (%s) + # """ % (dt, ", ".join(["'{}'".format(f) for f in fields]))) # Delete the data/column only if delete is specified if not delete: continue if frappe.db.get_value("DocType", dt, "issingle"): - frappe.db.sql(""" - DELETE FROM `tabSingles` - WHERE doctype='%s' AND field IN (%s) - """ % (dt, ", ".join("'{}'".format(f) for f in fields))) + frappe.db.delete("Singles", { + "doctype": dt, + "field": ("in", ", ".join("'{}'".format(f) for f in fields)) + }) + # frappe.db.sql(""" + # DELETE FROM `tabSingles` + # WHERE doctype='%s' AND field IN (%s) + # """ % (dt, ", ".join("'{}'".format(f) for f in fields))) else: existing_fields = frappe.db.multisql({ "mariadb": "DESC `tab%s`" % dt, diff --git a/frappe/model/delete_doc.py b/frappe/model/delete_doc.py index cc88cfa106..ec53d060fd 100644 --- a/frappe/model/delete_doc.py +++ b/frappe/model/delete_doc.py @@ -65,12 +65,36 @@ def delete_doc(doctype=None, name=None, force=0, ignore_doctypes=None, for_reloa update_flags(doc, flags, ignore_permissions) check_permission_and_not_submitted(doc) - frappe.db.sql("delete from `tabCustom Field` where dt = %s", name) - frappe.db.sql("delete from `tabClient Script` where dt = %s", name) - frappe.db.sql("delete from `tabProperty Setter` where doc_type = %s", name) - frappe.db.sql("delete from `tabReport` where ref_doctype=%s", name) - frappe.db.sql("delete from `tabCustom DocPerm` where parent=%s", name) - frappe.db.sql("delete from `__global_search` where doctype=%s", name) + frappe.db.delete("Custom Field", { + "dt": name + }) + + frappe.db.delete("Client Script", { + "dt": name + }) + + frappe.db.delete("Property Setter", { + "doc_type": name + }) + + frappe.db.delete("Report", { + "ref_doctype": name + }) + + frappe.db.delete("Custom DocPerm", { + "parent": name + }) + + frappe.db.delete("__global_search", { + "doctype": name + }) + + # frappe.db.sql("delete from `tabCustom Field` where dt = %s", name) + # frappe.db.sql("delete from `tabClient Script` where dt = %s", name) + # frappe.db.sql("delete from `tabProperty Setter` where doc_type = %s", name) + # frappe.db.sql("delete from `tabReport` where ref_doctype=%s", name) + # frappe.db.sql("delete from `tabCustom DocPerm` where parent=%s", name) + # frappe.db.sql("delete from `__global_search` where doctype=%s", name) delete_from_table(doctype, name, ignore_doctypes, None) @@ -162,9 +186,15 @@ def update_naming_series(doc): def delete_from_table(doctype, name, ignore_doctypes, doc): if doctype!="DocType" and doctype==name: - frappe.db.sql("delete from `tabSingles` where `doctype`=%s", name) + frappe.db.delete("Singles", { + "doctype": name + }) + # frappe.db.sql("delete from `tabSingles` where `doctype`=%s", name) else: - frappe.db.sql("delete from `tab{0}` where `name`=%s".format(doctype), name) + frappe.db.delete(f"{doctype}", { + "name": name + }) + # frappe.db.sql("delete from `tab{0}` where `name`=%s".format(doctype), name) # get child tables if doc: diff --git a/frappe/permissions.py b/frappe/permissions.py index 07b4a2e68f..4cd846e52c 100644 --- a/frappe/permissions.py +++ b/frappe/permissions.py @@ -6,10 +6,12 @@ import frappe import frappe.share from frappe import _, msgprint from frappe.utils import cint - +from frappe.database.database import Database rights = ("select", "read", "write", "create", "delete", "submit", "cancel", "amend", "print", "email", "report", "import", "export", "set_user_permissions", "share") + + def check_admin_or_system_manager(user=None): if not user: user = frappe.session.user @@ -516,8 +518,9 @@ def reset_perms(doctype): """Reset permissions for given doctype.""" from frappe.desk.notifications import delete_notification_count_for delete_notification_count_for(doctype) + frappe.db.delete(doctype="Custom DocPerm", conditions={"parent": doctype}) - frappe.db.sql("""delete from `tabCustom DocPerm` where parent=%s""", doctype) + # frappe.db.sql("""delete from `tabCustom DocPerm` where parent=%s""", doctype) def get_linked_doctypes(dt): return list(set([dt] + [d.options for d in diff --git a/frappe/sessions.py b/frappe/sessions.py index 4d922d6769..ae2bd02d98 100644 --- a/frappe/sessions.py +++ b/frappe/sessions.py @@ -16,6 +16,7 @@ import frappe.translate import redis from urllib.parse import unquote from frappe.cache_manager import clear_user_cache +from frappe.database.database import Database @frappe.whitelist(allow_guest=True) def clear(user=None): @@ -76,7 +77,6 @@ def get_sessions_to_clear(user=None, keep_current=False, device=None): def delete_session(sid=None, user=None, reason="Session Expired"): from frappe.core.doctype.activity_log.feed import logout_feed - frappe.cache().hdel("session", sid) frappe.cache().hdel("last_db_session_update", sid) if sid and not user: @@ -84,7 +84,8 @@ def delete_session(sid=None, user=None, reason="Session Expired"): if user_details: user = user_details[0].get("user") logout_feed(user, reason) - frappe.db.sql("""delete from tabSessions where sid=%s""", sid) + frappe.db.delete(doctype="Sessions", conditions={"sid": sid}) + # frappe.db.sql("""delete from tabSessions where sid=%s""", sid) frappe.db.commit() def clear_all_sessions(reason=None): From 931549ee054e8193591e4ff5e6796144a74d94ee Mon Sep 17 00:00:00 2001 From: Aradhya-Tripathi Date: Wed, 21 Jul 2021 19:20:35 +0530 Subject: [PATCH 101/164] refactor: changed delete to accept no conditions --- frappe/database/database.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index 6012e47445..5b4d813011 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -951,15 +951,25 @@ class Database(object): query = sql_dict.get(current_dialect) return self.sql(query, values, **kwargs) - def delete(self, doctype, conditions, debug=False): + def delete(self, doctype, conditions=None, debug=False): if conditions: conditions, values = self.build_conditions(conditions) - return self.sql("DELETE FROM `tab{doctype}` where {conditions}".format( + if doctype.startswith("__"): + return self.sql("DELETE FROM `{doctype}` where {conditions}".format( doctype=doctype, conditions=conditions ), values, debug=debug) + else: + return self.sql("DELETE FROM `tab{doctype}` where {conditions}".format( + doctype=doctype, + conditions=conditions + ), values, debug=debug) + else: - frappe.throw(_('No conditions provided')) + return self.sql("DELETE FROM `tab{doctype}`".format( + doctype=doctype + ), debug=debug) + def get_last_created(self, doctype): last_record = self.get_all(doctype, ('creation'), limit=1, order_by='creation desc') From dd2648b1414e47b46071109ab3bba9c5ed6a37f9 Mon Sep 17 00:00:00 2001 From: Aradhya-Tripathi Date: Wed, 21 Jul 2021 23:28:17 +0530 Subject: [PATCH 102/164] refactor: Moving from core raw queries to frappe orm --- frappe/core/doctype/activity_log/feed.py | 6 +++--- .../domain_settings/domain_settings.py | 2 +- frappe/core/doctype/error_log/error_log.py | 3 ++- frappe/core/doctype/report/test_report.py | 2 +- frappe/core/doctype/role/role.py | 2 +- .../scheduled_job_type/scheduled_job_type.py | 2 +- frappe/core/doctype/user/user.py | 19 +++++++++---------- 7 files changed, 18 insertions(+), 18 deletions(-) diff --git a/frappe/core/doctype/activity_log/feed.py b/frappe/core/doctype/activity_log/feed.py index cf8f7416e8..92e18d4a5a 100644 --- a/frappe/core/doctype/activity_log/feed.py +++ b/frappe/core/doctype/activity_log/feed.py @@ -29,9 +29,9 @@ def update_feed(doc, method=None): name = feed.name or doc.name - frappe.db.delete(doctype="Activity Log", conditions={"reference_doctype": doctype, - "reference_name": name, - "link_doctype": name.feed.link_doctype}) + frappe.db.delete("Activity Log", {"reference_doctype": doctype, + "reference_name": name, + "link_doctype": name.feed.link_doctype}) # frappe.db.sql("""delete from `tabActivity Log` # where # reference_doctype=%s and reference_name=%s diff --git a/frappe/core/doctype/domain_settings/domain_settings.py b/frappe/core/doctype/domain_settings/domain_settings.py index 6ca180def1..4ed9d1e15a 100644 --- a/frappe/core/doctype/domain_settings/domain_settings.py +++ b/frappe/core/doctype/domain_settings/domain_settings.py @@ -34,7 +34,7 @@ class DomainSettings(Document): all_domains = list((frappe.get_hooks('domains') or {})) def remove_role(role): - frappe.db.delete(doctype="Has Role", conditions={"role": role}) + frappe.db.delete("Has Role", {"role": role}) # frappe.db.sql('delete from `tabHas Role` where role=%s', role) frappe.set_value('Role', role, 'disabled', 1) diff --git a/frappe/core/doctype/error_log/error_log.py b/frappe/core/doctype/error_log/error_log.py index 8223238c57..9a7333995c 100644 --- a/frappe/core/doctype/error_log/error_log.py +++ b/frappe/core/doctype/error_log/error_log.py @@ -20,4 +20,5 @@ def set_old_logs_as_seen(): def clear_error_logs(): '''Flush all Error Logs''' frappe.only_for('System Manager') - frappe.db.sql('''DELETE FROM `tabError Log`''') \ No newline at end of file + frappe.db.delete("Error Log") + # frappe.db.sql('''DELETE FROM `tabError Log`''') \ No newline at end of file diff --git a/frappe/core/doctype/report/test_report.py b/frappe/core/doctype/report/test_report.py index 2bab6adf19..2ac41cca2b 100644 --- a/frappe/core/doctype/report/test_report.py +++ b/frappe/core/doctype/report/test_report.py @@ -83,7 +83,7 @@ class TestReport(unittest.TestCase): def test_report_permissions(self): frappe.set_user('test@example.com') - frappe.db.delete(doctype="Has Role", conditions={ + frappe.db.delete("Has Role", { "parent": frappe.session.user, "role": "Test Has Role" }) diff --git a/frappe/core/doctype/role/role.py b/frappe/core/doctype/role/role.py index fae8ae2d28..dee25eb476 100644 --- a/frappe/core/doctype/role/role.py +++ b/frappe/core/doctype/role/role.py @@ -38,7 +38,7 @@ class Role(Document): self.set(key, 0) def remove_roles(self): - frappe.db.delete(doctype="Has Role", conditions={"role": self.name}) + frappe.db.delete("Has Role", {"role": self.name}) # frappe.db.sql("delete from `tabHas Role` where role = %s", self.name) frappe.clear_cache() diff --git a/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py b/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py index d6c94aded1..9882f0edfe 100644 --- a/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py +++ b/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py @@ -110,7 +110,7 @@ class ScheduledJobType(Document): def on_trash(self): - frappe.db.delete(doctype="Scheduled Job Log", conditions={ + frappe.db.delete("Scheduled Job Log", { "scheduled_job_type": self.name }) # frappe.db.sql('delete from `tabScheduled Job Log` where scheduled_job_type=%s', self.name) diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py index f69af4e4a8..166f61de8d 100644 --- a/frappe/core/doctype/user/user.py +++ b/frappe/core/doctype/user/user.py @@ -17,7 +17,6 @@ from frappe.website.utils import is_signup_disabled from frappe.rate_limiter import rate_limit from frappe.utils.background_jobs import enqueue from frappe.core.doctype.user_type.user_type import user_linked_with_permission_on_doctype -from frappe.database.database import Database STANDARD_USERS = ("Guest", "Administrator") @@ -382,15 +381,15 @@ class User(Document): # frappe.db.sql("""delete from `tabDocShare` where user=%s""", self.name) # delete messages - # TODO: CHANGE THIS FROM ABHISHEK KA PYPIKA - frappe.db.delete("Communication", { - "reference_doctype": "User", - "communication_type": ("in", ("Chat", "Notification")), - }) - # frappe.db.sql("""delete from `tabCommunication` - # where communication_type in ('Chat', 'Notification') - # and reference_doctype='User' - # and (reference_name=%s or owner=%s)""", (self.name, self.name)) + + # frappe.db.delete("Communication", { + # "reference_doctype": "User", + # "communication_type": ("in", ("Chat", "Notification")), + # }) + frappe.db.sql("""delete from `tabCommunication` + where communication_type in ('Chat', 'Notification') + and reference_doctype='User' + and (reference_name=%s or owner=%s)""", (self.name, self.name)) # unlink contact frappe.db.sql("""update `tabContact` From 79ccaa95f3aff078a3ebdc68291b270b7cbeed26 Mon Sep 17 00:00:00 2001 From: Aradhya-Tripathi Date: Wed, 21 Jul 2021 23:29:50 +0530 Subject: [PATCH 103/164] refactor: Moved raw queries to frappe orm --- .../personal_data_deletion_request.py | 18 ++++++++++++------ .../doctype/workflow_action/workflow_action.py | 12 +++++++++--- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py b/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py index 63ba96d138..b23c7fa11d 100644 --- a/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py +++ b/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py @@ -323,12 +323,18 @@ class PersonalDataDeletionRequest(Document): def remove_unverified_record(): - frappe.db.sql( - """ - DELETE FROM `tabPersonal Data Deletion Request` - WHERE `status` = 'Pending Verification' - AND `creation` < (NOW() - INTERVAL '7' DAY)""" - ) + + frappe.db.delete("Personal Data Deletion Request", { + "status": "Pending Verification", + "creation": ("<", ["Now()", "-", "INTERVAL 7 DAY"]) + }) + + # frappe.db.sql( + # """ + # DELETE FROM `tabPersonal Data Deletion Request` + # WHERE `status` = 'Pending Verification' + # AND `creation` < (NOW() - INTERVAL '7' DAY)""" + # ) @frappe.whitelist(allow_guest=True) diff --git a/frappe/workflow/doctype/workflow_action/workflow_action.py b/frappe/workflow/doctype/workflow_action/workflow_action.py index b70ffb2406..af168d3477 100644 --- a/frappe/workflow/doctype/workflow_action/workflow_action.py +++ b/frappe/workflow/doctype/workflow_action/workflow_action.py @@ -133,9 +133,15 @@ def return_link_expired_page(doc, doc_workflow_state): def clear_old_workflow_actions(doc, user=None): user = user if user else frappe.session.user - frappe.db.sql("""DELETE FROM `tabWorkflow Action` - WHERE `reference_doctype`=%s AND `reference_name`=%s AND `user`!=%s AND `status`='Open'""", - (doc.get('doctype'), doc.get('name'), user)) + frappe.db.delete("Workflow Action", { + "reference_doctype": doc.get("doctype"), + "reference_name": doc.get("name"), + "user": ("!=", user), + "status": "Open" + }) + # frappe.db.sql("""DELETE FROM `tabWorkflow Action` + # WHERE `reference_doctype`=%s AND `reference_name`=%s AND `user`!=%s AND `status`='Open'""", + # (doc.get('doctype'), doc.get('name'), user)) def update_completed_workflow_actions(doc, user=None): user = user if user else frappe.session.user From 888783682072330987f713bd700b8bb1f7db916d Mon Sep 17 00:00:00 2001 From: Aradhya-Tripathi Date: Wed, 21 Jul 2021 23:31:15 +0530 Subject: [PATCH 104/164] refactor: Moved util queries to frappe orm --- frappe/utils/global_search.py | 25 +++++++++++++++++-------- frappe/utils/password.py | 15 +++++++++++---- 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/frappe/utils/global_search.py b/frappe/utils/global_search.py index 8fa2ea474f..a018761370 100644 --- a/frappe/utils/global_search.py +++ b/frappe/utils/global_search.py @@ -23,7 +23,8 @@ def reset(): Deletes all data in __global_search :return: """ - frappe.db.sql('DELETE FROM `__global_search`') + frappe.db.delete("__global_search") + # frappe.db.sql('DELETE FROM `__global_search`') def get_doctypes_with_global_search(with_child_tables=True): @@ -146,9 +147,12 @@ def rebuild_for_doctype(doctype): def delete_global_search_records_for_doctype(doctype): - frappe.db.sql('''DELETE - FROM `__global_search` - WHERE doctype = %s''', doctype, as_dict=True) + frappe.db.delete("__global_search", { + "doctype": doctype + }) + # frappe.db.sql('''DELETE + # FROM `__global_search` + # WHERE doctype = %s''', doctype, as_dict=True) def get_selected_fields(meta, global_search_fields): @@ -400,10 +404,15 @@ def delete_for_document(doc): :param doc: Deleted document """ - frappe.db.sql('''DELETE - FROM `__global_search` - WHERE doctype = %s - AND name = %s''', (doc.doctype, doc.name), as_dict=True) + frappe.db.delete("__global_search", { + "doctype": doc.doctype, + "name": doc.name + }) + + # frappe.db.sql('''DELETE + # FROM `__global_search` + # WHERE doctype = %s + # AND name = %s''', (doc.doctype, doc.name), as_dict=True) @frappe.whitelist() diff --git a/frappe/utils/password.py b/frappe/utils/password.py index 428f2e9577..caa89af64e 100644 --- a/frappe/utils/password.py +++ b/frappe/utils/password.py @@ -65,10 +65,17 @@ def set_encrypted_password(doctype, name, pwd, fieldname='password'): def remove_encrypted_password(doctype, name, fieldname='password'): - frappe.db.sql( - 'DELETE FROM `__Auth` WHERE doctype = %s and name = %s and fieldname = %s', - values=[doctype, name, fieldname] - ) + + frappe.db.delete("__Auth", { + "doctype": doctype, + "name": name, + "fieldname": fieldname + }) + + # frappe.db.sql( + # 'DELETE FROM `__Auth` WHERE doctype = %s and name = %s and fieldname = %s', + # values=[doctype, name, fieldname] + # ) def check_password(user, pwd, doctype='User', fieldname='password', delete_tracker_cache=True): From 8feb430371d38b34ca90f6ae2f08bd651e812a64 Mon Sep 17 00:00:00 2001 From: Aradhya-Tripathi Date: Wed, 21 Jul 2021 23:32:18 +0530 Subject: [PATCH 105/164] refactor: Moved raw queries to frappe orm --- frappe/model/__init__.py | 2 +- frappe/model/delete_doc.py | 8 ++++++-- .../v11_0/apply_customization_to_custom_doctype.py | 10 ++++++++-- .../v11_0/sync_stripe_settings_before_migrate.py | 5 ++++- .../v12_0/delete_feedback_request_if_exists.py | 11 +++++++---- .../v12_0/remove_deprecated_fields_from_doctype.py | 12 ++++++++---- 6 files changed, 34 insertions(+), 14 deletions(-) diff --git a/frappe/model/__init__.py b/frappe/model/__init__.py index 763fd36cc7..99eaa7ff5d 100644 --- a/frappe/model/__init__.py +++ b/frappe/model/__init__.py @@ -154,7 +154,7 @@ def delete_fields(args_dict, delete=0): frappe.db.delete("DocField", { "parent": dt, - "fieldname": ("in", ", ".join(["'{}'".format(f) for f in fields])) + "fieldname": ("in", fields) }) # frappe.db.sql(""" # DELETE FROM `tabDocField` diff --git a/frappe/model/delete_doc.py b/frappe/model/delete_doc.py index ec53d060fd..9b5ebb1031 100644 --- a/frappe/model/delete_doc.py +++ b/frappe/model/delete_doc.py @@ -369,8 +369,12 @@ def clear_references(doctype, reference_doctype, reference_name, (reference_doctype, reference_name)) def clear_timeline_references(link_doctype, link_name): - frappe.db.sql("""DELETE FROM `tabCommunication Link` - WHERE `tabCommunication Link`.link_doctype=%s AND `tabCommunication Link`.link_name=%s""", (link_doctype, link_name)) + frappe.db.delete("Communication Link", { + "link_doctype": link_doctype, + "link_name": link_name + }) + # frappe.db.sql("""DELETE FROM `tabCommunication Link` + # WHERE `tabCommunication Link`.link_doctype=%s AND `tabCommunication Link`.link_name=%s""", (link_doctype, link_name)) def insert_feed(doc): if ( diff --git a/frappe/patches/v11_0/apply_customization_to_custom_doctype.py b/frappe/patches/v11_0/apply_customization_to_custom_doctype.py index 49b68ed240..c13ff83a1f 100644 --- a/frappe/patches/v11_0/apply_customization_to_custom_doctype.py +++ b/frappe/patches/v11_0/apply_customization_to_custom_doctype.py @@ -28,7 +28,10 @@ def execute(): for prop in property_setters: property_setter_map[prop.field_name] = prop - frappe.db.sql('DELETE FROM `tabProperty Setter` WHERE `name`=%s', prop.name) + frappe.db.delete("Property Setter", { + "name": prop.name + }) + # frappe.db.sql('DELETE FROM `tabProperty Setter` WHERE `name`=%s', prop.name) meta = frappe.get_meta(doctype.name) @@ -50,6 +53,9 @@ def execute(): df = frappe.new_doc('DocField', meta, 'fields') df.update(cf) meta.fields.append(df) - frappe.db.sql('DELETE FROM `tabCustom Field` WHERE name=%s', cf.name) + frappe.db.delete("Custom Field", { + "name": cf.name + }) + # frappe.db.sql('DELETE FROM `tabCustom Field` WHERE name=%s', cf.name) meta.save() diff --git a/frappe/patches/v11_0/sync_stripe_settings_before_migrate.py b/frappe/patches/v11_0/sync_stripe_settings_before_migrate.py index a8e9bd4de1..1a12d4dc1d 100644 --- a/frappe/patches/v11_0/sync_stripe_settings_before_migrate.py +++ b/frappe/patches/v11_0/sync_stripe_settings_before_migrate.py @@ -17,4 +17,7 @@ def execute(): settings.secret_key = secret_key settings.save(ignore_permissions=True) - frappe.db.sql("""DELETE FROM tabSingles WHERE doctype='Stripe Settings'""") \ No newline at end of file + frappe.db.delete("Singles", { + "doctype": "Stripe Settings" + }) + # frappe.db.sql("""DELETE FROM tabSingles WHERE doctype='Stripe Settings'""") \ No newline at end of file diff --git a/frappe/patches/v12_0/delete_feedback_request_if_exists.py b/frappe/patches/v12_0/delete_feedback_request_if_exists.py index fdbcecfc5a..537d509424 100644 --- a/frappe/patches/v12_0/delete_feedback_request_if_exists.py +++ b/frappe/patches/v12_0/delete_feedback_request_if_exists.py @@ -2,7 +2,10 @@ import frappe def execute(): - frappe.db.sql(''' - DELETE from `tabDocType` - WHERE name = 'Feedback Request' - ''') \ No newline at end of file + frappe.db.delete("DocType", { + "name": "Feedback Request" + }) + # frappe.db.sql(''' + # DELETE from `tabDocType` + # WHERE name = 'Feedback Request' + # ''') \ No newline at end of file diff --git a/frappe/patches/v12_0/remove_deprecated_fields_from_doctype.py b/frappe/patches/v12_0/remove_deprecated_fields_from_doctype.py index 60599066e6..17dfbc5ca1 100644 --- a/frappe/patches/v12_0/remove_deprecated_fields_from_doctype.py +++ b/frappe/patches/v12_0/remove_deprecated_fields_from_doctype.py @@ -8,7 +8,11 @@ def execute(): 'DocType': ['hide_heading', 'image_view', 'read_only_onload'] }, delete=1) - frappe.db.sql(''' - DELETE from `tabProperty Setter` - WHERE property = 'read_only_onload' - ''') + frappe.db.delete("Property Setter", { + "property": "read_only_onload" + }) + + # frappe.db.sql(''' + # DELETE from `tabProperty Setter` + # WHERE property = 'read_only_onload' + # ''') From 6f70dcf52dd66cc2d200d41c07ad7900a10aa58e Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Thu, 22 Jul 2021 12:02:47 +0530 Subject: [PATCH 106/164] test: Add navingation test case --- cypress/integration/navigation.js | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 cypress/integration/navigation.js diff --git a/cypress/integration/navigation.js b/cypress/integration/navigation.js new file mode 100644 index 0000000000..3fe12f3547 --- /dev/null +++ b/cypress/integration/navigation.js @@ -0,0 +1,15 @@ +context('Navigation', () => { + before(() => { + cy.login(); + cy.visit('/app/website'); + }); + it('Navigate to route with hash in document name', () => { + cy.insert_doc('ToDo', {'__newname': 'ABC#123', 'description': 'Test this', 'ignore_duplicate': true}); + cy.visit('/app/todo/ABC#123'); + cy.title().should('eq', 'Test this - ABC#123'); + cy.get_field('description', 'Text Editor').contains('Test this'); + cy.go('back'); + cy.title().should('eq', 'Website'); + cy.remove_doc('ToDo', 'ABC#123'); + }); +}); \ No newline at end of file From a9cab2cf786560689678be4a686d113c83e414aa Mon Sep 17 00:00:00 2001 From: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> Date: Thu, 22 Jul 2021 13:07:31 +0530 Subject: [PATCH 107/164] test: Remove `remove_doc`. to avoid flaky test --- cypress/integration/navigation.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cypress/integration/navigation.js b/cypress/integration/navigation.js index 3fe12f3547..7e1426aa46 100644 --- a/cypress/integration/navigation.js +++ b/cypress/integration/navigation.js @@ -10,6 +10,5 @@ context('Navigation', () => { cy.get_field('description', 'Text Editor').contains('Test this'); cy.go('back'); cy.title().should('eq', 'Website'); - cy.remove_doc('ToDo', 'ABC#123'); }); -}); \ No newline at end of file +}); From 4dc476da6acad27c20afba1bf66074a774b899ec Mon Sep 17 00:00:00 2001 From: Aradhya-Tripathi Date: Thu, 22 Jul 2021 17:00:24 +0530 Subject: [PATCH 108/164] fix(update_feed): Malformed query in activity log --- frappe/core/doctype/activity_log/feed.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/core/doctype/activity_log/feed.py b/frappe/core/doctype/activity_log/feed.py index 92e18d4a5a..9b48d0550a 100644 --- a/frappe/core/doctype/activity_log/feed.py +++ b/frappe/core/doctype/activity_log/feed.py @@ -31,7 +31,7 @@ def update_feed(doc, method=None): frappe.db.delete("Activity Log", {"reference_doctype": doctype, "reference_name": name, - "link_doctype": name.feed.link_doctype}) + "link_doctype": feed.link_doctype}) # frappe.db.sql("""delete from `tabActivity Log` # where # reference_doctype=%s and reference_name=%s From 1204ae8ce40c2ebb34f54dfdf970199aa48511e0 Mon Sep 17 00:00:00 2001 From: Aradhya-Tripathi Date: Thu, 22 Jul 2021 17:26:24 +0530 Subject: [PATCH 109/164] fix: fixed spacing and ui tests --- frappe/core/doctype/activity_log/feed.py | 12 ++++++------ frappe/defaults.py | 14 +++++++++----- frappe/desk/doctype/desktop_icon/desktop_icon.py | 5 ++++- frappe/model/delete_doc.py | 6 ------ frappe/permissions.py | 2 +- frappe/sessions.py | 2 +- frappe/utils/password.py | 2 -- .../personal_data_deletion_request.py | 2 -- 8 files changed, 21 insertions(+), 24 deletions(-) diff --git a/frappe/core/doctype/activity_log/feed.py b/frappe/core/doctype/activity_log/feed.py index 9b48d0550a..d91a21ec44 100644 --- a/frappe/core/doctype/activity_log/feed.py +++ b/frappe/core/doctype/activity_log/feed.py @@ -26,12 +26,12 @@ def update_feed(doc, method=None): feed = frappe._dict(feed) doctype = feed.doctype or doc.doctype - name = feed.name or doc.name - - - frappe.db.delete("Activity Log", {"reference_doctype": doctype, - "reference_name": name, - "link_doctype": feed.link_doctype}) + name = feed.name or doc.name + frappe.db.delete("Activity Log", { + "reference_doctype": doctype, + "reference_name": name, + "link_doctype": feed.link_doctype + }) # frappe.db.sql("""delete from `tabActivity Log` # where # reference_doctype=%s and reference_name=%s diff --git a/frappe/defaults.py b/frappe/defaults.py index fde48d71ff..ee73e86a91 100644 --- a/frappe/defaults.py +++ b/frappe/defaults.py @@ -124,11 +124,15 @@ def set_default(key, value, parent, parenttype="__default"): where defkey=%s and parent=%s for update''', (key, parent)): - frappe.db.sql(""" - delete from - `tabDefaultValue` - where - defkey=%s and parent=%s""", (key, parent)) + frappe.db.delete("DefaultValue", { + "defkey": key, + "parent": parent + }) + # frappe.db.sql(""" + # delete from + # `tabDefaultValue` + # where + # defkey=%s and parent=%s""", (key, parent)) if value != None: add_default(key, value, parent) else: diff --git a/frappe/desk/doctype/desktop_icon/desktop_icon.py b/frappe/desk/doctype/desktop_icon/desktop_icon.py index 81a79cdb09..bc5677e9b9 100644 --- a/frappe/desk/doctype/desktop_icon/desktop_icon.py +++ b/frappe/desk/doctype/desktop_icon/desktop_icon.py @@ -197,7 +197,10 @@ def set_desktop_icons(visible_list, ignore_duplicate=True): # clear all custom only if setup is not complete if not int(frappe.defaults.get_defaults().setup_complete or 0): - frappe.db.sql('delete from `tabDesktop Icon` where standard=0') + frappe.db.delete("Desktop Icon", { + "standard": 0 + }) + # frappe.db.sql('delete from `tabDesktop Icon` where standard=0') # set standard as blocked and hidden if setting first active domain if not frappe.flags.keep_desktop_icons: diff --git a/frappe/model/delete_doc.py b/frappe/model/delete_doc.py index 9b5ebb1031..03510d489d 100644 --- a/frappe/model/delete_doc.py +++ b/frappe/model/delete_doc.py @@ -64,27 +64,21 @@ def delete_doc(doctype=None, name=None, force=0, ignore_doctypes=None, for_reloa update_flags(doc, flags, ignore_permissions) check_permission_and_not_submitted(doc) - frappe.db.delete("Custom Field", { "dt": name }) - frappe.db.delete("Client Script", { "dt": name }) - frappe.db.delete("Property Setter", { "doc_type": name }) - frappe.db.delete("Report", { "ref_doctype": name }) - frappe.db.delete("Custom DocPerm", { "parent": name }) - frappe.db.delete("__global_search", { "doctype": name }) diff --git a/frappe/permissions.py b/frappe/permissions.py index 4cd846e52c..293ac14f27 100644 --- a/frappe/permissions.py +++ b/frappe/permissions.py @@ -518,7 +518,7 @@ def reset_perms(doctype): """Reset permissions for given doctype.""" from frappe.desk.notifications import delete_notification_count_for delete_notification_count_for(doctype) - frappe.db.delete(doctype="Custom DocPerm", conditions={"parent": doctype}) + frappe.db.delete("Custom DocPerm", {"parent": doctype}) # frappe.db.sql("""delete from `tabCustom DocPerm` where parent=%s""", doctype) diff --git a/frappe/sessions.py b/frappe/sessions.py index ae2bd02d98..3a584dfb74 100644 --- a/frappe/sessions.py +++ b/frappe/sessions.py @@ -84,7 +84,7 @@ def delete_session(sid=None, user=None, reason="Session Expired"): if user_details: user = user_details[0].get("user") logout_feed(user, reason) - frappe.db.delete(doctype="Sessions", conditions={"sid": sid}) + frappe.db.delete("Sessions", {"sid": sid}) # frappe.db.sql("""delete from tabSessions where sid=%s""", sid) frappe.db.commit() diff --git a/frappe/utils/password.py b/frappe/utils/password.py index caa89af64e..c144d866f3 100644 --- a/frappe/utils/password.py +++ b/frappe/utils/password.py @@ -65,7 +65,6 @@ def set_encrypted_password(doctype, name, pwd, fieldname='password'): def remove_encrypted_password(doctype, name, fieldname='password'): - frappe.db.delete("__Auth", { "doctype": doctype, "name": name, @@ -77,7 +76,6 @@ def remove_encrypted_password(doctype, name, fieldname='password'): # values=[doctype, name, fieldname] # ) - def check_password(user, pwd, doctype='User', fieldname='password', delete_tracker_cache=True): '''Checks if user and password are correct, else raises frappe.AuthenticationError''' diff --git a/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py b/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py index b23c7fa11d..bde6f2a4ea 100644 --- a/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py +++ b/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py @@ -323,7 +323,6 @@ class PersonalDataDeletionRequest(Document): def remove_unverified_record(): - frappe.db.delete("Personal Data Deletion Request", { "status": "Pending Verification", "creation": ("<", ["Now()", "-", "INTERVAL 7 DAY"]) @@ -336,7 +335,6 @@ def remove_unverified_record(): # AND `creation` < (NOW() - INTERVAL '7' DAY)""" # ) - @frappe.whitelist(allow_guest=True) def confirm_deletion(email, name, host_name): if not verify_request(): From bc39fd869fac10ee10b4cafd471e85227eace2b9 Mon Sep 17 00:00:00 2001 From: Aradhya-Tripathi Date: Thu, 22 Jul 2021 19:09:39 +0530 Subject: [PATCH 110/164] fix: Fixed typos --- frappe/core/doctype/user/user.py | 2 +- frappe/database/database.py | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py index 166f61de8d..69d3b67220 100644 --- a/frappe/core/doctype/user/user.py +++ b/frappe/core/doctype/user/user.py @@ -366,7 +366,7 @@ class User(Document): if getattr(frappe.local, "login_manager", None): frappe.local.login_manager.logout(user=self.name) - frappe.db.delete("Todo", {"owner": self.name}) + frappe.db.delete("ToDo", {"owner": self.name}) # frappe.db.sql("""DELETE FROM `tabToDo` WHERE `owner`=%s""", (self.name,)) frappe.db.sql("""UPDATE `tabToDo` SET `assigned_by`=NULL WHERE `assigned_by`=%s""", (self.name,)) diff --git a/frappe/database/database.py b/frappe/database/database.py index 5b4d813011..60185106e1 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -966,9 +966,15 @@ class Database(object): ), values, debug=debug) else: - return self.sql("DELETE FROM `tab{doctype}`".format( - doctype=doctype - ), debug=debug) + if doctype.startwith("__"): + return self.sql("DELETE FROM `{doctype}`".format( + doctype=doctype + ), debug=debug) + + else: + return self.sql("DELETE FROM `tab{doctype}`".format( + doctype=doctype + ), debug=debug) def get_last_created(self, doctype): From 1c82b83be07f016ca86149f40a5fac1f21f705e9 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 23 Jul 2021 10:52:22 +0530 Subject: [PATCH 111/164] refactor: Simplify frappe.db.delete logic * Rename conditions kwarg to filters. Handle conditions kwarg if passed * Re-added exc raise for no filters...to risky. Thinking of having an extra kwarg like `force` or something...undecided * Added Python type hints * Accept and pass kwargs to frappe.db.sql...Now pass any kwarg db.sql accepts * Pass debug from fn def if someone is still using it as a positional arg :thonk: --- frappe/database/database.py | 38 ++++++++++++++++--------------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index 60185106e1..cdf15f9c3d 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -6,6 +6,7 @@ import re import time +from typing import Dict, List, Union import frappe import datetime import frappe.defaults @@ -951,30 +952,23 @@ class Database(object): query = sql_dict.get(current_dialect) return self.sql(query, values, **kwargs) - def delete(self, doctype, conditions=None, debug=False): - if conditions: - conditions, values = self.build_conditions(conditions) - if doctype.startswith("__"): - return self.sql("DELETE FROM `{doctype}` where {conditions}".format( - doctype=doctype, - conditions=conditions - ), values, debug=debug) - else: - return self.sql("DELETE FROM `tab{doctype}` where {conditions}".format( - doctype=doctype, - conditions=conditions - ), values, debug=debug) + def delete(self, doctype: str, filters: Union[Dict, List], debug=False, **kwargs): + """Delete rows from a table in site which match the passed filters. This + does trigger DocType hooks. Simply runs a DELETE query in the database. + """ + if kwargs: + filters = filters or kwargs.get("conditions") + if not filters: + raise TypeError("No filters passed for `frappe.db.delete`") + if "debug" not in kwargs: + kwargs["debug"] = debug - else: - if doctype.startwith("__"): - return self.sql("DELETE FROM `{doctype}`".format( - doctype=doctype - ), debug=debug) + table = doctype if doctype.startswith("__") else f"tab{doctype}" + query = f"DELETE FROM `{table}`" + conditions, values = self.build_conditions(filters) + query += f"WHERE {conditions}" - else: - return self.sql("DELETE FROM `tab{doctype}`".format( - doctype=doctype - ), debug=debug) + return self.sql(query, values, **kwargs) def get_last_created(self, doctype): From 36c4be9a8d5c456316d11f7b56e427a01d22f167 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 23 Jul 2021 11:50:17 +0530 Subject: [PATCH 112/164] feat: frappe.db.truncate ORM wrapper for SQL DDL statement * Alias frappe.db.clear_table to use truncate * Support for __ tables is added here --- frappe/database/database.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index cdf15f9c3d..0c18ae91f7 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -955,21 +955,36 @@ class Database(object): def delete(self, doctype: str, filters: Union[Dict, List], debug=False, **kwargs): """Delete rows from a table in site which match the passed filters. This does trigger DocType hooks. Simply runs a DELETE query in the database. + + Doctype name can be passed directly, it will be pre-pended with `tab`. """ if kwargs: filters = filters or kwargs.get("conditions") if not filters: - raise TypeError("No filters passed for `frappe.db.delete`") + raise TypeError( + "No filters passed for `frappe.db.delete`. If you wish to clear the whole " + "table, consider using `frappe.db.truncate` instead?" + ) if "debug" not in kwargs: kwargs["debug"] = debug table = doctype if doctype.startswith("__") else f"tab{doctype}" - query = f"DELETE FROM `{table}`" conditions, values = self.build_conditions(filters) - query += f"WHERE {conditions}" + query = f"DELETE FROM `{table}` WHERE {conditions}" return self.sql(query, values, **kwargs) + def truncate(self, doctype: str): + """Truncate a table in the database. This runs a DDL command `TRUNCATE TABLE`. + This cannot be rolled back. + + Doctype name can be passed directly, it will be pre-pended with `tab`. + """ + table = doctype if doctype.startswith("__") else f"tab{doctype}" + return self.sql_ddl(f"truncate `{table}`") + + def clear_table(self, doctype): + return self.truncate(doctype) def get_last_created(self, doctype): last_record = self.get_all(doctype, ('creation'), limit=1, order_by='creation desc') @@ -978,9 +993,6 @@ class Database(object): else: return None - def clear_table(self, doctype): - self.sql('truncate `tab{}`'.format(doctype)) - def log_touched_tables(self, query, values=None): if values: query = frappe.safe_decode(self._cursor.mogrify(query, values)) From e3ea5cd05af53c95d96341c4ae52c0ce59c75c09 Mon Sep 17 00:00:00 2001 From: Sagar Vora Date: Fri, 23 Jul 2021 14:08:16 +0530 Subject: [PATCH 113/164] fix: refresh dependencies on `awesomplete-selectcomplete` event (#13756) --- frappe/public/js/frappe/ui/field_group.js | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/frappe/public/js/frappe/ui/field_group.js b/frappe/public/js/frappe/ui/field_group.js index b8b908eb95..db06d99615 100644 --- a/frappe/public/js/frappe/ui/field_group.js +++ b/frappe/public/js/frappe/ui/field_group.js @@ -40,14 +40,16 @@ frappe.ui.FieldGroup = class FieldGroup extends frappe.ui.form.Layout { this.catch_enter_as_submit(); } - $(this.wrapper).find('input, select').on('change', () => { - this.dirty = true; - - frappe.run_serially([ - () => frappe.timeout(0.1), - () => me.refresh_dependency() - ]); - }); + $(this.wrapper).find('input, select').on( + 'change awesomplete-selectcomplete', + () => { + this.dirty = true; + frappe.run_serially([ + () => frappe.timeout(0.1), + () => me.refresh_dependency() + ]); + } + ); } } From 2f3371d873b976afe08a4003533263a9a636ce28 Mon Sep 17 00:00:00 2001 From: Aradhya-Tripathi Date: Fri, 23 Jul 2021 23:14:24 +0530 Subject: [PATCH 114/164] refactor: Moved raw queries to frappe orm --- .../core/doctype/activity_log/activity_log.py | 6 +-- .../core/doctype/log_settings/log_settings.py | 8 ++-- frappe/database/database.py | 13 ++++--- frappe/desk/doctype/todo/todo.py | 12 ++++-- frappe/model/document.py | 17 ++++++-- .../v12_0/set_primary_key_in_series.py | 39 ++++++++++--------- .../setup_comments_from_communications.py | 5 ++- .../patches/v13_0/remove_twilio_settings.py | 5 ++- frappe/utils/error.py | 7 +++- frappe/utils/install.py | 6 ++- frappe/utils/password.py | 8 +++- frappe/utils/testutils.py | 5 ++- .../workflow_action/workflow_action.py | 11 ++++-- 13 files changed, 90 insertions(+), 52 deletions(-) diff --git a/frappe/core/doctype/activity_log/activity_log.py b/frappe/core/doctype/activity_log/activity_log.py index efec0dc217..3f9f4f3cb0 100644 --- a/frappe/core/doctype/activity_log/activity_log.py +++ b/frappe/core/doctype/activity_log/activity_log.py @@ -44,6 +44,6 @@ def clear_activity_logs(days=None): if not days: days = 90 - - frappe.db.sql("""delete from `tabActivity Log` where \ - creation< (NOW() - INTERVAL '{0}' DAY)""".format(days)) \ No newline at end of file + frappe.db.delete("Activity Log", filters={"creation": ("<", ["NOW()", "-", f"INTERVAL {days} DAY"])}) + # frappe.db.sql("""delete from `tabActivity Log` where \ + # creation< (NOW() - INTERVAL '{0}' DAY)""".format(days)) \ No newline at end of file diff --git a/frappe/core/doctype/log_settings/log_settings.py b/frappe/core/doctype/log_settings/log_settings.py index 776fcc92e9..eea73ee748 100644 --- a/frappe/core/doctype/log_settings/log_settings.py +++ b/frappe/core/doctype/log_settings/log_settings.py @@ -13,10 +13,10 @@ class LogSettings(Document): self.clear_email_queue() def clear_error_logs(self): - # frappe.db.delete(doctype="Error Log", conditions="") - frappe.db.sql(""" DELETE FROM `tabError Log` - WHERE `creation` < (NOW() - INTERVAL '{0}' DAY) - """.format(self.clear_error_log_after)) + frappe.db.delete("Error Log", {"creation": ("<", ["NOW()", "-", f"INTERVAL {self.clear_error_log_after} DAY"])}) + # frappe.db.sql(""" DELETE FROM `tabError Log` + # WHERE `creation` < (NOW() - INTERVAL '{0}' DAY) + # """.format(self.clear_error_log_after)) def clear_activity_logs(self): from frappe.core.doctype.activity_log.activity_log import clear_activity_logs diff --git a/frappe/database/database.py b/frappe/database/database.py index 0c18ae91f7..8b4fd3ad9e 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -952,7 +952,7 @@ class Database(object): query = sql_dict.get(current_dialect) return self.sql(query, values, **kwargs) - def delete(self, doctype: str, filters: Union[Dict, List], debug=False, **kwargs): + def delete(self, doctype: str, filters: Union[Dict, List] = None, debug=False, **kwargs): """Delete rows from a table in site which match the passed filters. This does trigger DocType hooks. Simply runs a DELETE query in the database. @@ -960,14 +960,15 @@ class Database(object): """ if kwargs: filters = filters or kwargs.get("conditions") - if not filters: - raise TypeError( - "No filters passed for `frappe.db.delete`. If you wish to clear the whole " - "table, consider using `frappe.db.truncate` instead?" - ) + if "debug" not in kwargs: kwargs["debug"] = debug + if not filters: + table = doctype if doctype.startswith("__") else f"tab{doctype}" + query = f"DELETE FROM `{table}`" + return self.sql(query, **kwargs) + table = doctype if doctype.startswith("__") else f"tab{doctype}" conditions, values = self.build_conditions(filters) query = f"DELETE FROM `{table}` WHERE {conditions}" diff --git a/frappe/desk/doctype/todo/todo.py b/frappe/desk/doctype/todo/todo.py index 4696563445..388ba37e1f 100644 --- a/frappe/desk/doctype/todo/todo.py +++ b/frappe/desk/doctype/todo/todo.py @@ -40,11 +40,15 @@ class ToDo(Document): def on_trash(self): # unlink todo from linked comments - frappe.db.sql(""" - delete from `tabCommunication Link` - where link_doctype=%(doctype)s and link_name=%(name)s""", { - "doctype": self.doctype, "name": self.name + frappe.db.delete("Communication Link", { + "link_doctype": self.doctype, + "link_name": self.name }) + # frappe.db.sql(""" + # delete from `tabCommunication Link` + # where link_doctype=%(doctype)s and link_name=%(name)s""", { + # "doctype": self.doctype, "name": self.name + # }) self.update_in_reference() diff --git a/frappe/model/document.py b/frappe/model/document.py index 61160e1f01..f8adfdb3cf 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -390,9 +390,15 @@ class Document(BaseDocument): else: # no rows found, delete all rows - frappe.db.sql("""delete from `tab{0}` where parent=%s - and parenttype=%s and parentfield=%s""".format(df.options), - (self.name, self.doctype, fieldname)) + frappe.db.delete(df.options, { + "parent": self.name, + "parenttype": self.doctype, + "parentfield": fieldname + }) + + # frappe.db.sql("""delete from `tab{0}` where parent=%s + # and parenttype=%s and parentfield=%s""".format(df.options), + # (self.name, self.doctype, fieldname)) def get_doc_before_save(self): return getattr(self, '_doc_before_save', None) @@ -451,7 +457,10 @@ class Document(BaseDocument): def update_single(self, d): """Updates values for Single type Document in `tabSingles`.""" - frappe.db.sql("""delete from `tabSingles` where doctype=%s""", self.doctype) + frappe.db.delete("Singles", { + "doctype": self.doctype + }) + # frappe.db.sql("""delete from `tabSingles` where doctype=%s""", self.doctype) for field, value in d.items(): if field != "doctype": frappe.db.sql("""insert into `tabSingles` (doctype, field, value) diff --git a/frappe/patches/v12_0/set_primary_key_in_series.py b/frappe/patches/v12_0/set_primary_key_in_series.py index e5ed2204ba..a8409cbbba 100644 --- a/frappe/patches/v12_0/set_primary_key_in_series.py +++ b/frappe/patches/v12_0/set_primary_key_in_series.py @@ -1,21 +1,24 @@ import frappe def execute(): - #if current = 0, simply delete the key as it'll be recreated on first entry - frappe.db.sql('delete from `tabSeries` where current = 0') - duplicate_keys = frappe.db.sql(''' - SELECT name, max(current) as current - from - `tabSeries` - group by - name - having count(name) > 1 - ''', as_dict=True) - for row in duplicate_keys: - frappe.db.sql('delete from `tabSeries` where name = %(key)s', { - 'key': row.name - }) - if row.current: - frappe.db.sql('insert into `tabSeries`(`name`, `current`) values (%(name)s, %(current)s)', row) - frappe.db.commit() - frappe.db.sql('ALTER table `tabSeries` ADD PRIMARY KEY IF NOT EXISTS (name)') + #if current = 0, simply delete the key as it'll be recreated on first entry + frappe.db.sql('delete from `tabSeries` where current = 0') + duplicate_keys = frappe.db.sql(''' + SELECT name, max(current) as current + from + `tabSeries` + group by + name + having count(name) > 1 + ''', as_dict=True) + for row in duplicate_keys: + frappe.db.delete("Series", { + "name": row.name + }) + # frappe.db.sql('delete from `tabSeries` where name = %(key)s', { + # 'key': row.name + # }) + if row.current: + frappe.db.sql('insert into `tabSeries`(`name`, `current`) values (%(name)s, %(current)s)', row) + frappe.db.commit() + frappe.db.sql('ALTER table `tabSeries` ADD PRIMARY KEY IF NOT EXISTS (name)') diff --git a/frappe/patches/v12_0/setup_comments_from_communications.py b/frappe/patches/v12_0/setup_comments_from_communications.py index 039ceeff35..a4e3fb4261 100644 --- a/frappe/patches/v12_0/setup_comments_from_communications.py +++ b/frappe/patches/v12_0/setup_comments_from_communications.py @@ -29,4 +29,7 @@ def execute(): frappe.db.auto_commit_on_many_writes = False # clean up - frappe.db.sql("delete from `tabCommunication` where communication_type = 'Comment'") + frappe.db.delete("Communication", { + "communication_type": "Comment" + }) + # frappe.db.sql("delete from `tabCommunication` where communication_type = 'Comment'") diff --git a/frappe/patches/v13_0/remove_twilio_settings.py b/frappe/patches/v13_0/remove_twilio_settings.py index 363cbdd4b6..5d17364847 100644 --- a/frappe/patches/v13_0/remove_twilio_settings.py +++ b/frappe/patches/v13_0/remove_twilio_settings.py @@ -12,7 +12,10 @@ def execute(): frappe.delete_doc_if_exists('DocType', 'Twilio Number Group') if twilio_settings_doctype_in_integrations(): frappe.delete_doc_if_exists('DocType', 'Twilio Settings') - frappe.db.sql("delete from `tabSingles` where `doctype`=%s", 'Twilio Settings') + frappe.db.delete("Singles", { + "doctype": "Twilio Settings" + }) + # frappe.db.sql("delete from `tabSingles` where `doctype`=%s", 'Twilio Settings') def twilio_settings_doctype_in_integrations() -> bool: """Check Twilio Settings doctype exists in integrations module or not. diff --git a/frappe/utils/error.py b/frappe/utils/error.py index 07e34674fe..75b5c71da5 100644 --- a/frappe/utils/error.py +++ b/frappe/utils/error.py @@ -176,8 +176,11 @@ def collect_error_snapshots(): def clear_old_snapshots(): """Clear snapshots that are older than a month""" - frappe.db.sql("""delete from `tabError Snapshot` - where creation < (NOW() - INTERVAL '1' MONTH)""") + + frappe.db.delete("Error Snapshot", filters={"creation": ("<", ["NOW()", "-", f"INTERVAL '1' MONTH"])}) + + # frappe.db.sql("""delete from `tabError Snapshot` + # where creation < (NOW() - INTERVAL '1' MONTH)""") path = get_error_snapshot_path() today = datetime.datetime.now() diff --git a/frappe/utils/install.py b/frappe/utils/install.py index 91d8f04eb4..9d21a0d945 100644 --- a/frappe/utils/install.py +++ b/frappe/utils/install.py @@ -111,8 +111,10 @@ def before_tests(): # don't run before tests if any other app is installed return - frappe.db.sql("delete from `tabCustom Field`") - frappe.db.sql("delete from `tabEvent`") + frappe.db.delete("Custom Field") + frappe.db.delete("Event") + # frappe.db.sql("delete from `tabCustom Field`") + # frappe.db.sql("delete from `tabEvent`") frappe.db.commit() frappe.clear_cache() diff --git a/frappe/utils/password.py b/frappe/utils/password.py index c144d866f3..4bac6537ce 100644 --- a/frappe/utils/password.py +++ b/frappe/utils/password.py @@ -136,8 +136,12 @@ def update_password(user, pwd, doctype='User', fieldname='password', logout_all_ def delete_all_passwords_for(doctype, name): try: - frappe.db.sql("""delete from `__Auth` where `doctype`=%(doctype)s and `name`=%(name)s""", - { 'doctype': doctype, 'name': name }) + frappe.db.delete("__Auth", { + "doctype": doctype, + "name": name + }) + # frappe.db.sql("""delete from `__Auth` where `doctype`=%(doctype)s and `name`=%(name)s""", + # { 'doctype': doctype, 'name': name }) except Exception as e: if not frappe.db.is_missing_column(e): raise diff --git a/frappe/utils/testutils.py b/frappe/utils/testutils.py index c451d090f1..74b800bdcf 100644 --- a/frappe/utils/testutils.py +++ b/frappe/utils/testutils.py @@ -12,5 +12,8 @@ def add_custom_field(doctype, fieldname, fieldtype='Data', options=None): }).insert() def clear_custom_fields(doctype): - frappe.db.sql('delete from `tabCustom Field` where dt=%s', doctype) + frappe.db.delete("Custom Field", { + "dt": doctype + }) + # frappe.db.sql('delete from `tabCustom Field` where dt=%s', doctype) frappe.clear_cache(doctype=doctype) diff --git a/frappe/workflow/doctype/workflow_action/workflow_action.py b/frappe/workflow/doctype/workflow_action/workflow_action.py index af168d3477..6c1c31f0e5 100644 --- a/frappe/workflow/doctype/workflow_action/workflow_action.py +++ b/frappe/workflow/doctype/workflow_action/workflow_action.py @@ -259,10 +259,13 @@ def is_workflow_action_already_created(doc): def clear_workflow_actions(doctype, name): if not (doctype and name): return - - frappe.db.sql('''delete from `tabWorkflow Action` - where reference_doctype=%s and reference_name=%s''', - (doctype, name)) + frappe.db.delete("Workflow Action", { + "reference_doctype": doctype, + "reference_name": name + }) + # frappe.db.sql('''delete from `tabWorkflow Action` + # where reference_doctype=%s and reference_name=%s''', + # (doctype, name)) def get_doc_workflow_state(doc): workflow_name = get_workflow_name(doc.get('doctype')) From a86d717f17058203ced2e4a4b7b4cfffa42080ea Mon Sep 17 00:00:00 2001 From: Aradhya-Tripathi Date: Fri, 23 Jul 2021 23:31:44 +0530 Subject: [PATCH 115/164] fix: Convert indentation to tabs --- frappe/core/doctype/activity_log/feed.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/frappe/core/doctype/activity_log/feed.py b/frappe/core/doctype/activity_log/feed.py index d91a21ec44..d064022b25 100644 --- a/frappe/core/doctype/activity_log/feed.py +++ b/frappe/core/doctype/activity_log/feed.py @@ -28,10 +28,10 @@ def update_feed(doc, method=None): doctype = feed.doctype or doc.doctype name = feed.name or doc.name frappe.db.delete("Activity Log", { - "reference_doctype": doctype, - "reference_name": name, - "link_doctype": feed.link_doctype - }) + "reference_doctype": doctype, + "reference_name": name, + "link_doctype": feed.link_doctype + }) # frappe.db.sql("""delete from `tabActivity Log` # where # reference_doctype=%s and reference_name=%s From b8f2c979b579aae1288abc692448fa6947d0d288 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sat, 24 Jul 2021 01:56:29 +0530 Subject: [PATCH 116/164] ci: Remove test suite TestFrappeHTTPRequest Remove tests for HTTPRequest under test_auth because Frappe develop CI started breaking after PR merge --- frappe/tests/test_auth.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/frappe/tests/test_auth.py b/frappe/tests/test_auth.py index 8447150006..b02b53338d 100644 --- a/frappe/tests/test_auth.py +++ b/frappe/tests/test_auth.py @@ -125,20 +125,3 @@ class TestLoginAttemptTracker(unittest.TestCase): tracker.add_failure_attempt() self.assertTrue(tracker.is_user_allowed()) - -class TestFrappeHTTPRequest(unittest.TestCase): - # test frappe.auth.HTTPRequest - def test_set_language(self): - """Check if language is set on object initialization - - This is a test to ensure that language has changed. To test correctness - of frappe.local.lang, check out the tests of frappe.translate.get_language - """ - lang_before_request = frappe.local.lang - random_lang = frappe.get_all("Language", limit=1, pluck="name")[0] - set_request(method='POST', path='/') - frappe.form_dict._lang = random_lang - HTTPRequest() - self.assertTrue(hasattr(frappe.local, "lang")) - self.assertIsInstance(frappe.local.lang, str) - self.assertNotEqual(lang_before_request, frappe.local.lang) From 538ef2168bd5c77e19726d7ffe25816d765f2c90 Mon Sep 17 00:00:00 2001 From: Aradhya-Tripathi Date: Sat, 24 Jul 2021 22:58:12 +0530 Subject: [PATCH 117/164] fix: removed wrong queries --- .../core/doctype/activity_log/activity_log.py | 6 +++--- .../core/doctype/log_settings/log_settings.py | 8 ++++---- frappe/core/doctype/report/test_report.py | 2 +- frappe/core/doctype/user/user.py | 6 +++--- .../user_permission/test_user_permission.py | 2 +- .../user_permission/user_permission.py | 8 ++++---- frappe/model/__init__.py | 6 +++--- frappe/model/delete_doc.py | 4 ++-- .../delete_feedback_request_if_exists.py | 8 ++++---- frappe/permissions.py | 2 +- frappe/sessions.py | 1 - frappe/utils/error.py | 6 +++--- frappe/utils/password.py | 4 ++-- .../personal_data_deletion_request.py | 20 +++++++++---------- .../workflow_action/workflow_action.py | 4 ++-- 15 files changed, 43 insertions(+), 44 deletions(-) diff --git a/frappe/core/doctype/activity_log/activity_log.py b/frappe/core/doctype/activity_log/activity_log.py index 3f9f4f3cb0..2571330f9e 100644 --- a/frappe/core/doctype/activity_log/activity_log.py +++ b/frappe/core/doctype/activity_log/activity_log.py @@ -44,6 +44,6 @@ def clear_activity_logs(days=None): if not days: days = 90 - frappe.db.delete("Activity Log", filters={"creation": ("<", ["NOW()", "-", f"INTERVAL {days} DAY"])}) - # frappe.db.sql("""delete from `tabActivity Log` where \ - # creation< (NOW() - INTERVAL '{0}' DAY)""".format(days)) \ No newline at end of file + # frappe.db.delete("Activity Log", filters={"creation": ("<", ["NOW()", "-", f"INTERVAL {days} DAY"])}) + frappe.db.sql("""delete from `tabActivity Log` where \ + creation< (NOW() - INTERVAL '{0}' DAY)""".format(days)) \ No newline at end of file diff --git a/frappe/core/doctype/log_settings/log_settings.py b/frappe/core/doctype/log_settings/log_settings.py index eea73ee748..7f25998dc5 100644 --- a/frappe/core/doctype/log_settings/log_settings.py +++ b/frappe/core/doctype/log_settings/log_settings.py @@ -13,10 +13,10 @@ class LogSettings(Document): self.clear_email_queue() def clear_error_logs(self): - frappe.db.delete("Error Log", {"creation": ("<", ["NOW()", "-", f"INTERVAL {self.clear_error_log_after} DAY"])}) - # frappe.db.sql(""" DELETE FROM `tabError Log` - # WHERE `creation` < (NOW() - INTERVAL '{0}' DAY) - # """.format(self.clear_error_log_after)) + # frappe.db.delete("Error Log", {"creation": ("<", ["NOW()", "-", f"INTERVAL {self.clear_error_log_after} DAY"])}) + frappe.db.sql(""" DELETE FROM `tabError Log` + WHERE `creation` < (NOW() - INTERVAL '{0}' DAY) + """.format(self.clear_error_log_after)) def clear_activity_logs(self): from frappe.core.doctype.activity_log.activity_log import clear_activity_logs diff --git a/frappe/core/doctype/report/test_report.py b/frappe/core/doctype/report/test_report.py index 2ac41cca2b..efc33b888c 100644 --- a/frappe/core/doctype/report/test_report.py +++ b/frappe/core/doctype/report/test_report.py @@ -88,7 +88,7 @@ class TestReport(unittest.TestCase): "role": "Test Has Role" }) # frappe.db.sql("""delete from `tabHas Role` where parent = %s - # and role = 'Test Has Role'""", frappe.session.user, auto_commit=1) + # and role = 'Test Has Role'""", frappe.session.user, auto_commit=1) if not frappe.db.exists('Role', 'Test Has Role'): role = frappe.get_doc({ diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py index 69d3b67220..7ff4b8bea9 100644 --- a/frappe/core/doctype/user/user.py +++ b/frappe/core/doctype/user/user.py @@ -383,9 +383,9 @@ class User(Document): # delete messages # frappe.db.delete("Communication", { - # "reference_doctype": "User", - # "communication_type": ("in", ("Chat", "Notification")), - # }) + # "reference_doctype": "User", + # "communication_type": ("in", ("Chat", "Notification")), + # }) frappe.db.sql("""delete from `tabCommunication` where communication_type in ('Chat', 'Notification') and reference_doctype='User' diff --git a/frappe/core/doctype/user_permission/test_user_permission.py b/frappe/core/doctype/user_permission/test_user_permission.py index 9393e47ee9..60a91e6a9b 100644 --- a/frappe/core/doctype/user_permission/test_user_permission.py +++ b/frappe/core/doctype/user_permission/test_user_permission.py @@ -14,7 +14,7 @@ class TestUserPermission(unittest.TestCase): frappe.db.delete("User Permission", { "user": ("in", ("test_bulk_creation_update@example.com", "test_user_perm1@example.com", - "nested_doc_user@example.com")) + "nested_doc_user@example.com")) }) # frappe.db.sql("""DELETE FROM `tabUser Permission` diff --git a/frappe/core/doctype/user_permission/user_permission.py b/frappe/core/doctype/user_permission/user_permission.py index 5b3822d82f..338b063a63 100644 --- a/frappe/core/doctype/user_permission/user_permission.py +++ b/frappe/core/doctype/user_permission/user_permission.py @@ -278,8 +278,8 @@ def update_applicable(already_applied, to_apply, user, doctype, docname): }) # frappe.db.sql("""DELETE FROM `tabUser Permission` - # WHERE `user`=%s - # AND `applicable_for`=%s - # AND `allow`=%s - # AND `for_value`=%s + # WHERE `user`=%s + # AND `applicable_for`=%s + # AND `allow`=%s + # AND `for_value`=%s # """,(user, applied, doctype, docname)) diff --git a/frappe/model/__init__.py b/frappe/model/__init__.py index 99eaa7ff5d..7459493fcc 100644 --- a/frappe/model/__init__.py +++ b/frappe/model/__init__.py @@ -168,11 +168,11 @@ def delete_fields(args_dict, delete=0): if frappe.db.get_value("DocType", dt, "issingle"): frappe.db.delete("Singles", { "doctype": dt, - "field": ("in", ", ".join("'{}'".format(f) for f in fields)) + "field": ("in", fields) }) # frappe.db.sql(""" - # DELETE FROM `tabSingles` - # WHERE doctype='%s' AND field IN (%s) + # DELETE FROM `tabSingles` + # WHERE doctype='%s' AND field IN (%s) # """ % (dt, ", ".join("'{}'".format(f) for f in fields))) else: existing_fields = frappe.db.multisql({ diff --git a/frappe/model/delete_doc.py b/frappe/model/delete_doc.py index 03510d489d..9187310a84 100644 --- a/frappe/model/delete_doc.py +++ b/frappe/model/delete_doc.py @@ -185,7 +185,7 @@ def delete_from_table(doctype, name, ignore_doctypes, doc): }) # frappe.db.sql("delete from `tabSingles` where `doctype`=%s", name) else: - frappe.db.delete(f"{doctype}", { + frappe.db.delete(doctype, { "name": name }) # frappe.db.sql("delete from `tab{0}` where `name`=%s".format(doctype), name) @@ -368,7 +368,7 @@ def clear_timeline_references(link_doctype, link_name): "link_name": link_name }) # frappe.db.sql("""DELETE FROM `tabCommunication Link` - # WHERE `tabCommunication Link`.link_doctype=%s AND `tabCommunication Link`.link_name=%s""", (link_doctype, link_name)) + # WHERE `tabCommunication Link`.link_doctype=%s AND `tabCommunication Link`.link_name=%s""", (link_doctype, link_name)) def insert_feed(doc): if ( diff --git a/frappe/patches/v12_0/delete_feedback_request_if_exists.py b/frappe/patches/v12_0/delete_feedback_request_if_exists.py index 537d509424..de799b1e21 100644 --- a/frappe/patches/v12_0/delete_feedback_request_if_exists.py +++ b/frappe/patches/v12_0/delete_feedback_request_if_exists.py @@ -5,7 +5,7 @@ def execute(): frappe.db.delete("DocType", { "name": "Feedback Request" }) - # frappe.db.sql(''' - # DELETE from `tabDocType` - # WHERE name = 'Feedback Request' - # ''') \ No newline at end of file + # frappe.db.sql(''' + #DELETE from `tabDocType` + #WHERE name = 'Feedback Request' + # ''') \ No newline at end of file diff --git a/frappe/permissions.py b/frappe/permissions.py index 293ac14f27..7724f09e78 100644 --- a/frappe/permissions.py +++ b/frappe/permissions.py @@ -6,7 +6,7 @@ import frappe import frappe.share from frappe import _, msgprint from frappe.utils import cint -from frappe.database.database import Database + rights = ("select", "read", "write", "create", "delete", "submit", "cancel", "amend", "print", "email", "report", "import", "export", "set_user_permissions", "share") diff --git a/frappe/sessions.py b/frappe/sessions.py index 3a584dfb74..f4d05538bc 100644 --- a/frappe/sessions.py +++ b/frappe/sessions.py @@ -16,7 +16,6 @@ import frappe.translate import redis from urllib.parse import unquote from frappe.cache_manager import clear_user_cache -from frappe.database.database import Database @frappe.whitelist(allow_guest=True) def clear(user=None): diff --git a/frappe/utils/error.py b/frappe/utils/error.py index 75b5c71da5..f2cbc5ec10 100644 --- a/frappe/utils/error.py +++ b/frappe/utils/error.py @@ -177,10 +177,10 @@ def collect_error_snapshots(): def clear_old_snapshots(): """Clear snapshots that are older than a month""" - frappe.db.delete("Error Snapshot", filters={"creation": ("<", ["NOW()", "-", f"INTERVAL '1' MONTH"])}) + # frappe.db.delete("Error Snapshot", filters={"creation": ("<", ["NOW()", "-", f"INTERVAL '1' MONTH"])}) - # frappe.db.sql("""delete from `tabError Snapshot` - # where creation < (NOW() - INTERVAL '1' MONTH)""") + frappe.db.sql("""delete from `tabError Snapshot` + where creation < (NOW() - INTERVAL '1' MONTH)""") path = get_error_snapshot_path() today = datetime.datetime.now() diff --git a/frappe/utils/password.py b/frappe/utils/password.py index 4bac6537ce..21e1bc9586 100644 --- a/frappe/utils/password.py +++ b/frappe/utils/password.py @@ -72,8 +72,8 @@ def remove_encrypted_password(doctype, name, fieldname='password'): }) # frappe.db.sql( - # 'DELETE FROM `__Auth` WHERE doctype = %s and name = %s and fieldname = %s', - # values=[doctype, name, fieldname] + # 'DELETE FROM `__Auth` WHERE doctype = %s and name = %s and fieldname = %s', + # values=[doctype, name, fieldname] # ) def check_password(user, pwd, doctype='User', fieldname='password', delete_tracker_cache=True): diff --git a/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py b/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py index bde6f2a4ea..96898cc159 100644 --- a/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py +++ b/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py @@ -323,17 +323,17 @@ class PersonalDataDeletionRequest(Document): def remove_unverified_record(): - frappe.db.delete("Personal Data Deletion Request", { - "status": "Pending Verification", - "creation": ("<", ["Now()", "-", "INTERVAL 7 DAY"]) - }) + # frappe.db.delete("Personal Data Deletion Request", { + # "status": "Pending Verification", + # "creation": ("<", ["Now()", "-", "INTERVAL 7 DAY"]) + # }) - # frappe.db.sql( - # """ - # DELETE FROM `tabPersonal Data Deletion Request` - # WHERE `status` = 'Pending Verification' - # AND `creation` < (NOW() - INTERVAL '7' DAY)""" - # ) + frappe.db.sql( + """ + DELETE FROM `tabPersonal Data Deletion Request` + WHERE `status` = 'Pending Verification' + AND `creation` < (NOW() - INTERVAL '7' DAY)""" + ) @frappe.whitelist(allow_guest=True) def confirm_deletion(email, name, host_name): diff --git a/frappe/workflow/doctype/workflow_action/workflow_action.py b/frappe/workflow/doctype/workflow_action/workflow_action.py index 6c1c31f0e5..5957bf8671 100644 --- a/frappe/workflow/doctype/workflow_action/workflow_action.py +++ b/frappe/workflow/doctype/workflow_action/workflow_action.py @@ -264,8 +264,8 @@ def clear_workflow_actions(doctype, name): "reference_name": name }) # frappe.db.sql('''delete from `tabWorkflow Action` - # where reference_doctype=%s and reference_name=%s''', - # (doctype, name)) + # where reference_doctype=%s and reference_name=%s''', + # (doctype, name)) def get_doc_workflow_state(doc): workflow_name = get_workflow_name(doc.get('doctype')) From 326bd463fc36994d11cf9470101ab43e7b575dd8 Mon Sep 17 00:00:00 2001 From: pateljannat Date: Mon, 26 Jul 2021 17:09:57 +0530 Subject: [PATCH 118/164] fix: web form child table issue --- frappe/website/doctype/web_form/web_form.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/website/doctype/web_form/web_form.py b/frappe/website/doctype/web_form/web_form.py index 6e4200b54b..32f7e030a6 100644 --- a/frappe/website/doctype/web_form/web_form.py +++ b/frappe/website/doctype/web_form/web_form.py @@ -542,7 +542,7 @@ def get_form_data(doctype, docname=None, web_form_name=None): # For Table fields, server-side processing for meta for field in out.web_form.web_form_fields: if field.fieldtype == "Table": - field.fields = get_in_list_view_fields(field.options) + field.fields = frappe.get_meta(field.options).fields out.update({field.fieldname: field.fields}) if field.fieldtype == "Link": From db09a85183a5068bf9792f8d2e4101ab4382eec6 Mon Sep 17 00:00:00 2001 From: leela Date: Tue, 20 Jul 2021 06:11:01 +0530 Subject: [PATCH 119/164] refactor: Add authentication for Redis Queue --- frappe/commands/redis.py | 25 +++--- frappe/commands/scheduler.py | 8 +- .../page/background_jobs/background_jobs.py | 8 +- frappe/test_runner.py | 1 + frappe/tests/test_background_jobs.py | 6 +- frappe/tests/test_redis.py | 70 ++++++++++++++++ frappe/utils/__init__.py | 2 +- frappe/utils/background_jobs.py | 80 +++++++++++++++---- frappe/utils/rq.py | 24 +++--- 9 files changed, 173 insertions(+), 51 deletions(-) create mode 100644 frappe/tests/test_redis.py diff --git a/frappe/commands/redis.py b/frappe/commands/redis.py index c608296cac..38a46c2142 100644 --- a/frappe/commands/redis.py +++ b/frappe/commands/redis.py @@ -1,7 +1,6 @@ import os import click -import redis import frappe from frappe.utils.rq import RedisQueue @@ -9,16 +8,18 @@ from frappe.installer import update_site_config @click.command('create-rq-users') @click.option('--set-admin-password', is_flag=True, default=False, help='Set new Redis admin(default user) password') -@click.option('--reset-passwords', is_flag=True, default=False, help='Remove all existing passwords') -def create_rq_users(set_admin_password=False, reset_passwords=False): +@click.option('--use-rq-auth', is_flag=True, default=False, help='Enable Redis authentication for sites') +def create_rq_users(set_admin_password=False, use_rq_auth=False): """Create Redis Queue users and add to acl and app configs. acl config file will be used by redis server while starting the server and app config is used by app while connecting to redis server. """ acl_file_path = os.path.abspath('../config/redis_queue.acl') - acl_list, user_credentials = RedisQueue.gen_acl_list( - reset_passwords=reset_passwords, set_admin_password=set_admin_password) + + with frappe.init_site(): + acl_list, user_credentials = RedisQueue.gen_acl_list( + set_admin_password=set_admin_password) with open(acl_file_path, 'w') as f: f.writelines([acl+'\n' for acl in acl_list]) @@ -29,18 +30,22 @@ def create_rq_users(set_admin_password=False, reset_passwords=False): site_config_path=common_site_config_path) update_site_config("rq_password", user_credentials['bench'][1], validate=False, site_config_path=common_site_config_path) + update_site_config("use_rq_auth", use_rq_auth, validate=False, + site_config_path=common_site_config_path) + + click.secho('* ACL and site configs are updated with new user credentials. ' + 'Please restart Redis Queue server to enable namespaces.', + fg='green') if set_admin_password: env_key = 'RQ_ADMIN_PASWORD' - click.secho('Redis admin password is successfully set up. ' + click.secho('* Redis admin password is successfully set up. ' 'Include below line in .bashrc file for system to use', - fg='green' - ) + fg='green') click.secho(f"`export {env_key}={user_credentials['default'][1]}`") click.secho('NOTE: Please save the admin password as you ' 'can not access redis server without the password', - fg='yellow' - ) + fg='yellow') commands = [ diff --git a/frappe/commands/scheduler.py b/frappe/commands/scheduler.py index d69ebb3024..f82473fd55 100755 --- a/frappe/commands/scheduler.py +++ b/frappe/commands/scheduler.py @@ -172,9 +172,13 @@ def start_scheduler(): @click.command('worker') @click.option('--queue', type=str) @click.option('--quiet', is_flag = True, default = False, help = 'Hide Log Outputs') -def start_worker(queue, quiet = False): +@click.option('-u', '--rq-username', default=None, help='Redis ACL user') +@click.option('-p', '--rq-password', default=None, help='Redis ACL user password') +def start_worker(queue, quiet = False, rq_username=None, rq_password=None): + """Site is used to find redis credentals. + """ from frappe.utils.background_jobs import start_worker - start_worker(queue, quiet = quiet) + start_worker(queue, quiet = quiet, rq_username=rq_username, rq_password=rq_password) @click.command('ready-for-migration') @click.option('--site', help='site name') diff --git a/frappe/core/page/background_jobs/background_jobs.py b/frappe/core/page/background_jobs/background_jobs.py index 847b23bd3e..1f3555e351 100644 --- a/frappe/core/page/background_jobs/background_jobs.py +++ b/frappe/core/page/background_jobs/background_jobs.py @@ -4,12 +4,12 @@ import json from typing import TYPE_CHECKING, Dict, List -from rq import Queue, Worker +from rq import Worker import frappe from frappe import _ from frappe.utils import convert_utc_to_user_timezone, format_datetime -from frappe.utils.background_jobs import get_redis_conn +from frappe.utils.background_jobs import get_redis_conn, get_queues from frappe.utils.scheduler import is_scheduler_inactive if TYPE_CHECKING: @@ -29,7 +29,7 @@ def get_info(show_failed=False) -> List[Dict]: show_failed = json.loads(show_failed) conn = get_redis_conn() - queues = Queue.all(conn) + queues = get_queues() workers = Worker.all(conn) jobs = [] @@ -75,7 +75,7 @@ def get_info(show_failed=False) -> List[Dict]: @frappe.whitelist() def remove_failed_jobs(): conn = get_redis_conn() - queues = Queue.all(conn) + queues = get_queues() for queue in queues: fail_registry = queue.failed_job_registry for job_id in fail_registry.get_job_ids(): diff --git a/frappe/test_runner.py b/frappe/test_runner.py index 0c30fbbd00..8112362f34 100644 --- a/frappe/test_runner.py +++ b/frappe/test_runner.py @@ -56,6 +56,7 @@ def main(app=None, module=None, doctype=None, verbose=False, tests=(), frappe.clear_cache() frappe.utils.scheduler.disable_scheduler() set_test_email_config() + frappe.conf.update({'bench_id': 'test_bench', 'use_rq_auth': False}) if not frappe.flags.skip_before_tests: if verbose: diff --git a/frappe/tests/test_background_jobs.py b/frappe/tests/test_background_jobs.py index 88783f14f1..48e0dd2ee9 100644 --- a/frappe/tests/test_background_jobs.py +++ b/frappe/tests/test_background_jobs.py @@ -4,7 +4,7 @@ from rq import Queue import frappe from frappe.core.page.background_jobs.background_jobs import remove_failed_jobs -from frappe.utils.background_jobs import get_redis_conn +from frappe.utils.background_jobs import get_redis_conn, rename_queue import time @@ -17,14 +17,14 @@ class TestBackgroundJobs(unittest.TestCase): queues = Queue.all(conn) for queue in queues: - if queue.name == "short": + if queue.name == rename_queue("short"): fail_registry = queue.failed_job_registry self.assertGreater(fail_registry.count, 0) remove_failed_jobs() for queue in queues: - if queue.name == "short": + if queue.name == rename_queue("short"): fail_registry = queue.failed_job_registry self.assertEqual(fail_registry.count, 0) diff --git a/frappe/tests/test_redis.py b/frappe/tests/test_redis.py new file mode 100644 index 0000000000..72af1ac699 --- /dev/null +++ b/frappe/tests/test_redis.py @@ -0,0 +1,70 @@ +import unittest +import functools + +import redis + +import frappe +from frappe.utils import get_bench_id +from frappe.utils.rq import RedisQueue +from frappe.utils.background_jobs import get_redis_conn + +def version_tuple(version): + return tuple(map(int, (version.split(".")))) + +def skip_if_redis_version_lt(version): + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + conn = get_redis_conn() + redis_version = conn.execute_command('info')['redis_version'] + if version_tuple(redis_version) < version_tuple(version): + return + return func(*args, **kwargs) + return wrapper + return decorator + +class TestRedisAuth(unittest.TestCase): + @skip_if_redis_version_lt('6.0') + def test_rq_gen_acllist(self): + """Make sure that ACL list is genrated + """ + acl_list = RedisQueue.gen_acl_list() + self.assertEqual(acl_list[1]['bench'][0], get_bench_id()) + + @skip_if_redis_version_lt('6.0') + def test_adding_redis_user(self): + acl_list = RedisQueue.gen_acl_list() + username, password = acl_list[1]['bench'] + conn = get_redis_conn() + + conn.acl_deluser(username) + _ = RedisQueue(conn).add_user(username, password) + self.assertTrue(conn.acl_getuser(username)) + conn.acl_deluser(username) + + @skip_if_redis_version_lt('6.0') + def test_rq_namespace(self): + """Make sure that user can access only their respective namespace. + """ + # Current bench ID + bench_id = frappe.conf.get('bench_id') + conn = get_redis_conn() + conn.set('rq:queue:test_bench1:abc', 'value') + conn.set(f'rq:queue:{bench_id}:abc', 'value') + + # Create new Redis Queue user + tmp_bench_id = 'test_bench1' + username, password = tmp_bench_id, 'password1' + conn.acl_deluser(username) + frappe.conf.update({'bench_id': tmp_bench_id}) + _ = RedisQueue(conn).add_user(username, password) + test_bench1_conn = RedisQueue.get_connection(username, password) + + self.assertEqual(test_bench1_conn.get('rq:queue:test_bench1:abc'), b'value') + + # User should not be able to access queues apart from their bench queues + with self.assertRaises(redis.exceptions.NoPermissionError): + test_bench1_conn.get(f'rq:queue:{bench_id}:abc') + + frappe.conf.update({'bench_id': bench_id}) + conn.acl_deluser(username) diff --git a/frappe/utils/__init__.py b/frappe/utils/__init__.py index bf139173d6..80c6cda98c 100644 --- a/frappe/utils/__init__.py +++ b/frappe/utils/__init__.py @@ -384,7 +384,7 @@ def get_bench_path(): return os.path.realpath(os.path.join(os.path.dirname(frappe.__file__), '..', '..', '..')) def get_bench_id(): - return frappe.local.conf.get('bench_id', 'DefaultBench') + return frappe.get_conf().get('bench_id', 'DefaultBench') def get_site_id(site=None): return f"{site or frappe.local.site}@{get_bench_id()}" diff --git a/frappe/utils/background_jobs.py b/frappe/utils/background_jobs.py index 8456835ca7..4241c95c5d 100755 --- a/frappe/utils/background_jobs.py +++ b/frappe/utils/background_jobs.py @@ -1,13 +1,21 @@ +import os +import socket +import time +from uuid import uuid4 +from collections import defaultdict + + import redis +from typing import List from rq import Connection, Queue, Worker from rq.logutils import setup_loghandlers -from frappe.utils import cstr -from collections import defaultdict + import frappe -import os, socket, time from frappe import _ -from uuid import uuid4 import frappe.monitor +from frappe.utils import cstr, get_bench_id +from frappe.utils.rq import RedisQueue +from frappe.utils.commands import log default_timeout = 300 @@ -131,21 +139,22 @@ def execute_job(site, method, event, job_name, kwargs, user=None, is_async=True, if is_async: frappe.destroy() -def start_worker(queue=None, quiet = False): +def start_worker(queue=None, quiet = False, rq_username=None, rq_password=None): '''Wrapper to start rq worker. Connects to redis and monitors these queues.''' with frappe.init_site(): # empty init is required to get redis_queue from common_site_config.json - redis_connection = get_redis_conn() + redis_connection = get_redis_conn(username=rq_username, password=rq_password) + queues = get_queue_list(queue, build_queue_name=True) + queue_name = queue and rename_queue(queue) if os.environ.get('CI'): setup_loghandlers('ERROR') with Connection(redis_connection): - queues = get_queue_list(queue) logging_level = "INFO" if quiet: logging_level = "WARNING" - Worker(queues, name=get_worker_name(queue)).work(logging_level = logging_level) + Worker(queues, name=get_worker_name(queue_name)).work(logging_level = logging_level) def get_worker_name(queue): '''When limiting worker to a specific queue, also append queue name to default worker name''' @@ -186,7 +195,7 @@ def get_jobs(site=None, queue=None, key='method'): return jobs_per_site -def get_queue_list(queue_list=None): +def get_queue_list(queue_list=None, build_queue_name=False): '''Defines possible queues. Also wraps a given queue in a list after validating.''' default_queue_list = list(queue_timeout) if queue_list: @@ -195,11 +204,9 @@ def get_queue_list(queue_list=None): for queue in queue_list: validate_queue(queue, default_queue_list) - - return queue_list - else: - return default_queue_list + queue_list = default_queue_list + return [rename_queue(q) for q in queue_list] if build_queue_name else queue_list def get_workers(queue): '''Returns a list of Worker objects tied to a queue object''' @@ -218,7 +225,7 @@ def get_running_jobs_in_queue(queue): def get_queue(queue, is_async=True): '''Returns a Queue object tied to a redis connection''' validate_queue(queue) - return Queue(queue, connection=get_redis_conn(), is_async=is_async) + return Queue(rename_queue(queue), connection=get_redis_conn(), is_async=is_async) def validate_queue(queue, default_queue_list=None): if not default_queue_list: @@ -227,7 +234,7 @@ def validate_queue(queue, default_queue_list=None): if queue not in default_queue_list: frappe.throw(_("Queue should be one of {0}").format(', '.join(default_queue_list))) -def get_redis_conn(): +def get_redis_conn(username=None, password=None): if not hasattr(frappe.local, 'conf'): raise Exception('You need to call frappe.init') @@ -236,11 +243,50 @@ def get_redis_conn(): global redis_connection - if not redis_connection: - redis_connection = redis.from_url(frappe.local.conf.redis_queue) + cred = frappe._dict() + if frappe.conf.get('use_rq_auth'): + if username: + cred['username'] = username + cred['password'] = password + else: + cred['username'] = frappe.get_site_config().rq_username or get_bench_id() + cred['password'] = frappe.get_site_config().rq_password + + elif os.environ.get('RQ_ADMIN_PASWORD'): + cred['username'] = 'default' + cred['password'] = os.environ.get('RQ_ADMIN_PASWORD') + try: + redis_connection = RedisQueue.get_connection(**cred) + except (redis.exceptions.AuthenticationError, redis.exceptions.ResponseError): + log(f'Wrong credentials used for {cred.username or "default user"}. ' + 'You can reset credentials using `bench create-rq-users` CLI and restart the server', + colour='red') + raise + except Exception: + log(f'Please make sure that Redis Queue runs @ {frappe.get_conf().redis_queue}', colour='red') + raise return redis_connection +def get_queues() -> List[Queue]: + """Get all the queues linked to the current bench. + """ + queues = Queue.all(connection=get_redis_conn()) + return [q for q in queues if is_queue_accessible(q)] + +def rename_queue(qname: str) -> str: + """Rename qname by adding bench name as prefix. + + Renamed queues are useful to define namespaces of customers. + """ + return f"{get_bench_id()}:{qname}" + +def is_queue_accessible(qobj: Queue) -> bool: + """Checks whether queue is relate to current bench or not. + """ + accessible_queues = [rename_queue(q) for q in list(queue_timeout)] + return qobj.name in accessible_queues + def enqueue_test_job(): enqueue('frappe.utils.background_jobs.test_job', s=100) diff --git a/frappe/utils/rq.py b/frappe/utils/rq.py index 5e9b9dcd5d..b344b0caa5 100644 --- a/frappe/utils/rq.py +++ b/frappe/utils/rq.py @@ -1,8 +1,7 @@ import redis import frappe -from frappe.utils import get_site_id, get_bench_id, random_string - +from frappe.utils import get_bench_id, random_string class RedisQueue: def __init__(self, conn): @@ -17,9 +16,10 @@ class RedisQueue: return frappe._dict(user_settings) if is_created else {} @classmethod - def get_connection(cls, username='default', password=None): - domain = frappe.local.conf.redis_queue.split("redis://", 1)[-1] - url = f"redis://{username}:{password or ''}@{domain}" + def get_connection(cls, username=None, password=None): + rq_url = frappe.local.conf.redis_queue + domain = rq_url.split("redis://", 1)[-1] + url = (username and f"redis://{username}:{password or ''}@{domain}") or rq_url conn = redis.from_url(url) conn.ping() return conn @@ -63,25 +63,21 @@ class RedisQueue: return ['+@all', '-@admin'] @classmethod - def gen_acl_list(cls, reset_passwords=False, set_admin_password=False): + def gen_acl_list(cls, set_admin_password=False): """Generate list of ACL users needed for this branch. This list contains default ACL user and the bench ACL user(used by all sites incase of ACL is enabled). """ - with frappe.init_site(): - bench_username = get_bench_id() - bench_user_rules = cls.get_acl_key_rules(include_key_prefix=True) + cls.get_acl_command_rules() - + bench_username = get_bench_id() + bench_user_rules = cls.get_acl_key_rules(include_key_prefix=True) + cls.get_acl_command_rules() bench_user_rule_str = ' '.join(bench_user_rules).strip() bench_user_password = random_string(20) - bench_user_resetpass = (reset_passwords and 'resetpass') or '' default_username = 'default' _default_user_password = random_string(20) if set_admin_password else '' default_user_password = '>'+_default_user_password if _default_user_password else 'nopass' - default_user_resetpass = (reset_passwords and set_admin_password and 'resetpass') or '' return [ - f'user {default_username} on {default_user_password} {default_user_resetpass} ~* &* +@all', - f'user {bench_username} on >{bench_user_password} {bench_user_resetpass} {bench_user_rule_str}' + f'user {default_username} on {default_user_password} ~* &* +@all', + f'user {bench_username} on >{bench_user_password} {bench_user_rule_str}' ], {'bench': (bench_username, bench_user_password), 'default': (default_username, _default_user_password)} From 36feb72c89cb34c07bbb2fea94d45d339bf1d2bd Mon Sep 17 00:00:00 2001 From: hasnain2808 Date: Tue, 27 Jul 2021 11:29:00 +0530 Subject: [PATCH 120/164] fix: add column entries in doctypes --- frappe/database/mariadb/framework_mariadb.sql | 1 + frappe/database/postgres/framework_postgres.sql | 1 + 2 files changed, 2 insertions(+) diff --git a/frappe/database/mariadb/framework_mariadb.sql b/frappe/database/mariadb/framework_mariadb.sql index a52efd01e3..f8841e9417 100644 --- a/frappe/database/mariadb/framework_mariadb.sql +++ b/frappe/database/mariadb/framework_mariadb.sql @@ -220,6 +220,7 @@ CREATE TABLE `tabDocType` ( `allow_guest_to_view` int(1) NOT NULL DEFAULT 0, `route` varchar(255) DEFAULT NULL, `is_published_field` varchar(255) DEFAULT NULL, + `website_search_field` varchar(255) DEFAULT NULL, `email_append_to` int(1) NOT NULL DEFAULT 0, `subject_field` varchar(255) DEFAULT NULL, `sender_field` varchar(255) DEFAULT NULL, diff --git a/frappe/database/postgres/framework_postgres.sql b/frappe/database/postgres/framework_postgres.sql index eeb0eecd3f..a4e94aa326 100644 --- a/frappe/database/postgres/framework_postgres.sql +++ b/frappe/database/postgres/framework_postgres.sql @@ -225,6 +225,7 @@ CREATE TABLE "tabDocType" ( "allow_guest_to_view" smallint NOT NULL DEFAULT 0, "route" varchar(255) DEFAULT NULL, "is_published_field" varchar(255) DEFAULT NULL, + "website_search_field" varchar(255) DEFAULT NULL, "email_append_to" smallint NOT NULL DEFAULT 0, "subject_field" varchar(255) DEFAULT NULL, "sender_field" varchar(255) DEFAULT NULL, From 780778d7e3ef442db9e82002123994974ace2df9 Mon Sep 17 00:00:00 2001 From: Mohammad Hasnain Mohsin Rajan Date: Tue, 27 Jul 2021 11:36:14 +0530 Subject: [PATCH 121/164] chore: remove unrelated code --- frappe/migrate.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/frappe/migrate.py b/frappe/migrate.py index d4060e6067..061e4c98d7 100644 --- a/frappe/migrate.py +++ b/frappe/migrate.py @@ -4,8 +4,6 @@ import json import os import sys - -from rq import queue import frappe import frappe.translate import frappe.modules.patch_handler @@ -13,7 +11,6 @@ import frappe.model.sync from frappe.utils.fixtures import sync_fixtures from frappe.utils.connections import check_connection from frappe.utils.dashboard import sync_dashboards -from frappe.utils.background_jobs import enqueue from frappe.cache_manager import clear_global_cache from frappe.desk.notifications import clear_notifications from frappe.website.utils import clear_website_cache @@ -101,7 +98,6 @@ Otherwise, check the server logs and ensure that all the required services are r frappe.publish_realtime("version-update") frappe.flags.in_migrate = False - finally: with open(touched_tables_file, 'w') as f: json.dump(list(frappe.flags.touched_tables), f, sort_keys=True, indent=4) From 574bc57abe56b621053ee7644e5eefe753d4c6a3 Mon Sep 17 00:00:00 2001 From: Mohammad Hasnain Mohsin Rajan Date: Tue, 27 Jul 2021 11:38:51 +0530 Subject: [PATCH 122/164] chore: remove unrelated changes --- frappe/website/page_renderers/document_page.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/frappe/website/page_renderers/document_page.py b/frappe/website/page_renderers/document_page.py index 0108a84f79..01d8ec54c5 100644 --- a/frappe/website/page_renderers/document_page.py +++ b/frappe/website/page_renderers/document_page.py @@ -1,7 +1,7 @@ import frappe from frappe.model.document import get_controller from frappe.website.page_renderers.base_template_page import BaseTemplatePage -from frappe.website.utils import build_response, cache_html +from frappe.website.utils import build_response from frappe.website.router import (get_doctypes_with_web_view, get_page_info_from_web_page_with_dynamic_routes) @@ -47,7 +47,11 @@ class DocumentPage(BaseTemplatePage): return False def render(self): - html = self.get_html() + self.doc = frappe.get_doc(self.doctype, self.docname) + self.init_context() + self.update_context() + self.post_process_context() + html = frappe.get_template(self.template_path).render(self.context) html = self.add_csrf_token(html) return build_response(self.path, html, self.http_status_code or 200, self.headers) From 2a116a59b91e8cd8766e275ea8ab2d7e50f3d4b8 Mon Sep 17 00:00:00 2001 From: Mohammad Hasnain Mohsin Rajan Date: Tue, 27 Jul 2021 11:39:26 +0530 Subject: [PATCH 123/164] chore: remove unrelated changes --- frappe/website/page_renderers/template_page.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/website/page_renderers/template_page.py b/frappe/website/page_renderers/template_page.py index 2ec3f51f41..5e6e57e33a 100644 --- a/frappe/website/page_renderers/template_page.py +++ b/frappe/website/page_renderers/template_page.py @@ -83,7 +83,7 @@ class TemplatePage(BaseTemplatePage): super(TemplatePage, self).post_process_context() def add_sidebar_and_breadcrumbs(self): - if self.basepath and not self.context.sidebar_items: + if self.basepath: self.context.sidebar_items = get_sidebar_items(self.context.website_sidebar, self.basepath) if self.context.add_breadcrumbs and not self.context.parents: From 27608d85900b96e843d7cc05c6c1b3e1e54c6883 Mon Sep 17 00:00:00 2001 From: Mohammad Hasnain Mohsin Rajan Date: Tue, 27 Jul 2021 11:41:16 +0530 Subject: [PATCH 124/164] chore: remove unrelated changes --- frappe/website/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frappe/website/utils.py b/frappe/website/utils.py index 419a5199df..0f5f182ea2 100644 --- a/frappe/website/utils.py +++ b/frappe/website/utils.py @@ -456,6 +456,7 @@ def build_response(path, data, http_status_code, headers=None): response.status_code = http_status_code response.headers["X-Page-Name"] = path.encode("ascii", errors="xmlcharrefreplace") response.headers["X-From-Cache"] = frappe.local.response.from_cache or False + add_preload_headers(response) if headers: for key, val in iteritems(headers): @@ -485,12 +486,11 @@ def set_content_type(response, data, path): return data def add_preload_headers(response): - from bs4 import BeautifulSoup, SoupStrainer + from bs4 import BeautifulSoup try: preload = [] - strainer = SoupStrainer(re.compile("script|link")) - soup = BeautifulSoup(response.data, "lxml", parse_only=strainer) + soup = BeautifulSoup(response.data, "lxml") for elem in soup.find_all('script', src=re.compile(".*")): preload.append(("script", elem.get("src"))) From ea7dd9e8df0b18fe968dad0714a07681ee796972 Mon Sep 17 00:00:00 2001 From: Mohammad Hasnain Mohsin Rajan Date: Tue, 27 Jul 2021 11:42:13 +0530 Subject: [PATCH 125/164] chore: remove unrelated changes --- frappe/website/page_renderers/document_page.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/frappe/website/page_renderers/document_page.py b/frappe/website/page_renderers/document_page.py index 01d8ec54c5..f1741c681f 100644 --- a/frappe/website/page_renderers/document_page.py +++ b/frappe/website/page_renderers/document_page.py @@ -56,16 +56,6 @@ class DocumentPage(BaseTemplatePage): return build_response(self.path, html, self.http_status_code or 200, self.headers) - @cache_html - def get_html(self): - self.doc = frappe.get_doc(self.doctype, self.docname) - self.init_context() - self.update_context() - self.post_process_context() - html = frappe.get_template(self.template_path).render(self.context) - - return html - def update_context(self): self.context.doc = self.doc self.context.update(self.context.doc.as_dict()) From 266a5511c91a7ae223d29a3773643de58c68681f Mon Sep 17 00:00:00 2001 From: Mohammad Hasnain Mohsin Rajan Date: Tue, 27 Jul 2021 12:31:34 +0530 Subject: [PATCH 126/164] chore: fix sider --- frappe/search/website_search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/search/website_search.py b/frappe/search/website_search.py index 1fb84531b0..70962e8172 100644 --- a/frappe/search/website_search.py +++ b/frappe/search/website_search.py @@ -38,7 +38,7 @@ class WebsiteSearch(FullTextSearch): self._items_to_index = [] - routes = get_static_pages_from_all_apps() + slugs_with_web_view(self._items_to_index ) + routes = get_static_pages_from_all_apps() + slugs_with_web_view(self._items_to_index) for i, route in enumerate(routes): From ee6d4f50b4a27e9cc8c3b5f8a821a6d9c0cf4f21 Mon Sep 17 00:00:00 2001 From: Mohammad Hasnain Mohsin Rajan Date: Tue, 27 Jul 2021 13:40:00 +0530 Subject: [PATCH 127/164] chore: remove unwanted changes --- frappe/website/page_renderers/document_page.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/frappe/website/page_renderers/document_page.py b/frappe/website/page_renderers/document_page.py index f1741c681f..6b8d973ead 100644 --- a/frappe/website/page_renderers/document_page.py +++ b/frappe/website/page_renderers/document_page.py @@ -1,7 +1,7 @@ import frappe from frappe.model.document import get_controller from frappe.website.page_renderers.base_template_page import BaseTemplatePage -from frappe.website.utils import build_response +from frappe.website.utils import cache_html from frappe.website.router import (get_doctypes_with_web_view, get_page_info_from_web_page_with_dynamic_routes) @@ -47,14 +47,19 @@ class DocumentPage(BaseTemplatePage): return False def render(self): + html = self.get_html() + html = self.add_csrf_token(html) + + return self.build_response(html) + + @cache_html + def get_html(self): self.doc = frappe.get_doc(self.doctype, self.docname) self.init_context() self.update_context() self.post_process_context() html = frappe.get_template(self.template_path).render(self.context) - html = self.add_csrf_token(html) - - return build_response(self.path, html, self.http_status_code or 200, self.headers) + return html def update_context(self): self.context.doc = self.doc From 9a282428685f8d0305853392887272560a6b7830 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Wed, 28 Jul 2021 11:32:20 +0530 Subject: [PATCH 128/164] fix: Report print option --- frappe/public/html/print_template.html | 2 +- frappe/public/js/frappe/microtemplate.js | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/frappe/public/html/print_template.html b/frappe/public/html/print_template.html index f63a20377f..e2ff9c9c76 100644 --- a/frappe/public/html/print_template.html +++ b/frappe/public/html/print_template.html @@ -7,7 +7,7 @@ {{ title }} - + diff --git a/frappe/public/js/frappe/microtemplate.js b/frappe/public/js/frappe/microtemplate.js index 7b45db952e..151d008d3e 100644 --- a/frappe/public/js/frappe/microtemplate.js +++ b/frappe/public/js/frappe/microtemplate.js @@ -119,6 +119,10 @@ frappe.render_grid = function(opts) { // render HTML wrapper page opts.base_url = frappe.urllib.get_base_url(); opts.print_css = frappe.boot.print_css; + + opts.lang = opts.lang || frappe.boot.lang, + opts.layout_direction = opts.layout_direction || frappe.utils.is_rtl() ? "rtl" : "ltr"; + var html = frappe.render_template("print_template", opts); var w = window.open(); From 84ff1d0af991fc6be5bd755c716f656baaeef312 Mon Sep 17 00:00:00 2001 From: Aradhya-Tripathi Date: Wed, 28 Jul 2021 13:48:01 +0530 Subject: [PATCH 129/164] style: removed frappe.db.sql comments --- .../core/doctype/activity_log/activity_log.py | 1 - frappe/core/doctype/activity_log/feed.py | 4 ---- .../domain_settings/domain_settings.py | 1 - frappe/core/doctype/error_log/error_log.py | 3 +-- .../core/doctype/log_settings/log_settings.py | 1 - frappe/core/doctype/report/test_report.py | 3 --- frappe/core/doctype/role/role.py | 1 - .../scheduled_job_type/scheduled_job_type.py | 5 +---- frappe/core/doctype/user/user.py | 10 --------- .../user_permission/test_user_permission.py | 6 ------ .../user_permission/user_permission.py | 21 ------------------- .../permission_manager/permission_manager.py | 13 ++++++------ .../doctype/custom_field/custom_field.py | 6 ------ .../doctype/customize_form/customize_form.py | 4 +--- frappe/defaults.py | 5 ----- .../dashboard_chart/test_dashboard_chart.py | 2 -- .../desk/doctype/desktop_icon/desktop_icon.py | 1 - frappe/desk/doctype/event/event.py | 2 -- .../doctype/route_history/route_history.py | 11 +--------- frappe/desk/doctype/tag/tag.py | 2 -- frappe/desk/doctype/todo/todo.py | 6 ------ .../unhandled_email/unhandled_email.py | 4 +--- frappe/model/__init__.py | 9 -------- frappe/model/delete_doc.py | 15 +------------ frappe/model/document.py | 6 ------ .../apply_customization_to_custom_doctype.py | 4 ---- .../sync_stripe_settings_before_migrate.py | 3 +-- .../delete_feedback_request_if_exists.py | 6 +----- .../remove_deprecated_fields_from_doctype.py | 7 +------ .../v12_0/set_primary_key_in_series.py | 3 --- .../setup_comments_from_communications.py | 3 +-- .../patches/v13_0/remove_twilio_settings.py | 3 +-- frappe/permissions.py | 2 -- frappe/sessions.py | 1 - frappe/utils/error.py | 2 -- frappe/utils/global_search.py | 13 ------------ frappe/utils/install.py | 2 -- frappe/utils/password.py | 7 ------- frappe/utils/testutils.py | 1 - .../personal_data_deletion_request.py | 5 ----- .../workflow_action/workflow_action.py | 9 +------- 41 files changed, 18 insertions(+), 195 deletions(-) diff --git a/frappe/core/doctype/activity_log/activity_log.py b/frappe/core/doctype/activity_log/activity_log.py index 2571330f9e..ce5e504f7f 100644 --- a/frappe/core/doctype/activity_log/activity_log.py +++ b/frappe/core/doctype/activity_log/activity_log.py @@ -44,6 +44,5 @@ def clear_activity_logs(days=None): if not days: days = 90 - # frappe.db.delete("Activity Log", filters={"creation": ("<", ["NOW()", "-", f"INTERVAL {days} DAY"])}) frappe.db.sql("""delete from `tabActivity Log` where \ creation< (NOW() - INTERVAL '{0}' DAY)""".format(days)) \ No newline at end of file diff --git a/frappe/core/doctype/activity_log/feed.py b/frappe/core/doctype/activity_log/feed.py index d064022b25..efac825178 100644 --- a/frappe/core/doctype/activity_log/feed.py +++ b/frappe/core/doctype/activity_log/feed.py @@ -32,10 +32,6 @@ def update_feed(doc, method=None): "reference_name": name, "link_doctype": feed.link_doctype }) - # frappe.db.sql("""delete from `tabActivity Log` - # where - # reference_doctype=%s and reference_name=%s - # and link_doctype=%s""", (doctype, name,feed.link_doctype)) frappe.get_doc({ "doctype": "Activity Log", "reference_doctype": doctype, diff --git a/frappe/core/doctype/domain_settings/domain_settings.py b/frappe/core/doctype/domain_settings/domain_settings.py index 4ed9d1e15a..a8c7c6a747 100644 --- a/frappe/core/doctype/domain_settings/domain_settings.py +++ b/frappe/core/doctype/domain_settings/domain_settings.py @@ -35,7 +35,6 @@ class DomainSettings(Document): def remove_role(role): frappe.db.delete("Has Role", {"role": role}) - # frappe.db.sql('delete from `tabHas Role` where role=%s', role) frappe.set_value('Role', role, 'disabled', 1) for domain in all_domains: diff --git a/frappe/core/doctype/error_log/error_log.py b/frappe/core/doctype/error_log/error_log.py index 9a7333995c..081b4e879d 100644 --- a/frappe/core/doctype/error_log/error_log.py +++ b/frappe/core/doctype/error_log/error_log.py @@ -20,5 +20,4 @@ def set_old_logs_as_seen(): def clear_error_logs(): '''Flush all Error Logs''' frappe.only_for('System Manager') - frappe.db.delete("Error Log") - # frappe.db.sql('''DELETE FROM `tabError Log`''') \ No newline at end of file + frappe.db.delete("Error Log") \ No newline at end of file diff --git a/frappe/core/doctype/log_settings/log_settings.py b/frappe/core/doctype/log_settings/log_settings.py index 7f25998dc5..e73aa8dac1 100644 --- a/frappe/core/doctype/log_settings/log_settings.py +++ b/frappe/core/doctype/log_settings/log_settings.py @@ -13,7 +13,6 @@ class LogSettings(Document): self.clear_email_queue() def clear_error_logs(self): - # frappe.db.delete("Error Log", {"creation": ("<", ["NOW()", "-", f"INTERVAL {self.clear_error_log_after} DAY"])}) frappe.db.sql(""" DELETE FROM `tabError Log` WHERE `creation` < (NOW() - INTERVAL '{0}' DAY) """.format(self.clear_error_log_after)) diff --git a/frappe/core/doctype/report/test_report.py b/frappe/core/doctype/report/test_report.py index efc33b888c..1bf9893bd7 100644 --- a/frappe/core/doctype/report/test_report.py +++ b/frappe/core/doctype/report/test_report.py @@ -87,9 +87,6 @@ class TestReport(unittest.TestCase): "parent": frappe.session.user, "role": "Test Has Role" }) - # frappe.db.sql("""delete from `tabHas Role` where parent = %s - # and role = 'Test Has Role'""", frappe.session.user, auto_commit=1) - if not frappe.db.exists('Role', 'Test Has Role'): role = frappe.get_doc({ 'doctype': 'Role', diff --git a/frappe/core/doctype/role/role.py b/frappe/core/doctype/role/role.py index dee25eb476..28b444e1e7 100644 --- a/frappe/core/doctype/role/role.py +++ b/frappe/core/doctype/role/role.py @@ -39,7 +39,6 @@ class Role(Document): def remove_roles(self): frappe.db.delete("Has Role", {"role": self.name}) - # frappe.db.sql("delete from `tabHas Role` where role = %s", self.name) frappe.clear_cache() def on_update(self): diff --git a/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py b/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py index 9882f0edfe..d3afecf378 100644 --- a/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py +++ b/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py @@ -109,13 +109,10 @@ class ScheduledJobType(Document): return 'long' if ('Long' in self.frequency) else 'default' def on_trash(self): - frappe.db.delete("Scheduled Job Log", { "scheduled_job_type": self.name }) - # frappe.db.sql('delete from `tabScheduled Job Log` where scheduled_job_type=%s', self.name) - - + @frappe.whitelist() def execute_event(doc: str): frappe.only_for("System Manager") diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py index 7ff4b8bea9..65a48fb39e 100644 --- a/frappe/core/doctype/user/user.py +++ b/frappe/core/doctype/user/user.py @@ -367,25 +367,15 @@ class User(Document): frappe.local.login_manager.logout(user=self.name) frappe.db.delete("ToDo", {"owner": self.name}) - # frappe.db.sql("""DELETE FROM `tabToDo` WHERE `owner`=%s""", (self.name,)) frappe.db.sql("""UPDATE `tabToDo` SET `assigned_by`=NULL WHERE `assigned_by`=%s""", (self.name,)) # delete events frappe.db.delete("Event", {"owner": self.name, "event_type": "Private"}) - # frappe.db.sql("""delete from `tabEvent` where owner=%s - # and event_type='Private'""", (self.name,)) # delete shares frappe.db.delete("DocShare", {"user": self.name}) - # frappe.db.sql("""delete from `tabDocShare` where user=%s""", self.name) - # delete messages - - # frappe.db.delete("Communication", { - # "reference_doctype": "User", - # "communication_type": ("in", ("Chat", "Notification")), - # }) frappe.db.sql("""delete from `tabCommunication` where communication_type in ('Chat', 'Notification') and reference_doctype='User' diff --git a/frappe/core/doctype/user_permission/test_user_permission.py b/frappe/core/doctype/user_permission/test_user_permission.py index 60a91e6a9b..c3d593ee3b 100644 --- a/frappe/core/doctype/user_permission/test_user_permission.py +++ b/frappe/core/doctype/user_permission/test_user_permission.py @@ -17,12 +17,6 @@ class TestUserPermission(unittest.TestCase): "nested_doc_user@example.com")) }) - # frappe.db.sql("""DELETE FROM `tabUser Permission` - # WHERE `user` in ( - # 'test_bulk_creation_update@example.com', - # 'test_user_perm1@example.com', - # 'nested_doc_user@example.com')""") - # frappe.delete_doc_if_exists("DocType", "Person") frappe.db.sql_ddl("DROP TABLE IF EXISTS `tabPerson`") frappe.delete_doc_if_exists("DocType", "Doc A") frappe.db.sql_ddl("DROP TABLE IF EXISTS `tabDoc A`") diff --git a/frappe/core/doctype/user_permission/user_permission.py b/frappe/core/doctype/user_permission/user_permission.py index 338b063a63..de242efe10 100644 --- a/frappe/core/doctype/user_permission/user_permission.py +++ b/frappe/core/doctype/user_permission/user_permission.py @@ -186,7 +186,6 @@ def clear_user_permissions(user, for_doctype): "user": user, "allow": for_doctype }) - # frappe.db.sql('DELETE FROM `tabUser Permission` WHERE `user`=%s AND `allow`=%s', (user, for_doctype)) frappe.clear_cache() return total @@ -244,13 +243,6 @@ def remove_applicable(perm_applied_docs, user, doctype, docname): "allow": doctype, "for_value": docname }) - # - # frappe.db.sql("""DELETE FROM `tabUser Permission` - # WHERE `user`=%s - # AND `applicable_for`=%s - # AND `allow`=%s - # AND `for_value`=%s - # """, (user, applicable_for, doctype, docname)) def remove_apply_to_all(user, doctype, docname): frappe.db.delete("User Permission", { @@ -259,12 +251,6 @@ def remove_apply_to_all(user, doctype, docname): "allow": doctype, "for_value": docname }) - # frappe.db.sql("""DELETE from `tabUser Permission` - # WHERE `user`=%s - # AND `apply_to_all_doctypes`=1 - # AND `allow`=%s - # AND `for_value`=%s - # """,(user, doctype, docname)) def update_applicable(already_applied, to_apply, user, doctype, docname): for applied in already_applied: @@ -276,10 +262,3 @@ def update_applicable(already_applied, to_apply, user, doctype, docname): "allow": doctype, "for_value": docname }) - - # frappe.db.sql("""DELETE FROM `tabUser Permission` - # WHERE `user`=%s - # AND `applicable_for`=%s - # AND `allow`=%s - # AND `for_value`=%s - # """,(user, applied, doctype, docname)) diff --git a/frappe/core/page/permission_manager/permission_manager.py b/frappe/core/page/permission_manager/permission_manager.py index 0a76283813..af8973811a 100644 --- a/frappe/core/page/permission_manager/permission_manager.py +++ b/frappe/core/page/permission_manager/permission_manager.py @@ -92,14 +92,14 @@ def update(doctype, role, permlevel, ptype, value=None): """Update role permission params Args: - doctype (str): Name of the DocType to update params for - role (str): Role to be updated for, eg "Website Manager". - permlevel (int): perm level the provided rule applies to - ptype (str): permission type, example "read", "delete", etc. - value (None, optional): value for ptype, None indicates False + doctype (str): Name of the DocType to update params for + role (str): Role to be updated for, eg "Website Manager". + permlevel (int): perm level the provided rule applies to + ptype (str): permission type, example "read", "delete", etc. + value (None, optional): value for ptype, None indicates False Returns: - str: Refresh flag is permission is updated successfully + str: Refresh flag is permission is updated successfully """ frappe.only_for("System Manager") out = update_permission_property(doctype, role, permlevel, ptype, value) @@ -114,7 +114,6 @@ def remove(doctype, role, permlevel): frappe.db.delete("Custom DocPerm", { "name": name }) - # frappe.db.sql('delete from `tabCustom DocPerm` where name=%s', name) if not frappe.get_all('Custom DocPerm', dict(parent=doctype)): frappe.throw(_('There must be atleast one permission rule.'), title=_('Cannot Remove')) diff --git a/frappe/custom/doctype/custom_field/custom_field.py b/frappe/custom/doctype/custom_field/custom_field.py index 1d8f4ae67a..e266455f7a 100644 --- a/frappe/custom/doctype/custom_field/custom_field.py +++ b/frappe/custom/doctype/custom_field/custom_field.py @@ -89,12 +89,6 @@ class CustomField(Document): "doc_type": self.dt, "field_name": self.fieldname }) - # frappe.db.sql("""\ - # DELETE FROM `tabProperty Setter` - # WHERE doc_type = %s - # AND field_name = %s""", - # (self.dt, self.fieldname)) - frappe.clear_cache(doctype=self.dt) def validate_insert_after(self, meta): diff --git a/frappe/custom/doctype/customize_form/customize_form.py b/frappe/custom/doctype/customize_form/customize_form.py index 3b7de448d3..5431511d42 100644 --- a/frappe/custom/doctype/customize_form/customize_form.py +++ b/frappe/custom/doctype/customize_form/customize_form.py @@ -24,9 +24,7 @@ class CustomizeForm(Document): "doctype": "Customize Form" }) frappe.db.delete("Customize Form Field") - # frappe.db.sql("delete from tabSingles where doctype='Customize Form'") - # frappe.db.sql("delete from `tabCustomize Form Field`") - + @frappe.whitelist() def fetch_to_customize(self): self.clear_existing_doc() diff --git a/frappe/defaults.py b/frappe/defaults.py index ee73e86a91..794d30a0c8 100644 --- a/frappe/defaults.py +++ b/frappe/defaults.py @@ -128,11 +128,6 @@ def set_default(key, value, parent, parenttype="__default"): "defkey": key, "parent": parent }) - # frappe.db.sql(""" - # delete from - # `tabDefaultValue` - # where - # defkey=%s and parent=%s""", (key, parent)) if value != None: add_default(key, value, parent) else: diff --git a/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py b/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py index 7133640c21..9f10522b12 100644 --- a/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py +++ b/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py @@ -65,7 +65,6 @@ class TestDashboardChart(unittest.TestCase): frappe.delete_doc('Dashboard Chart', 'Test Empty Dashboard Chart') frappe.db.delete("Error Log") - # frappe.db.sql('delete from `tabError Log`') frappe.get_doc(dict( doctype = 'Dashboard Chart', @@ -96,7 +95,6 @@ class TestDashboardChart(unittest.TestCase): frappe.delete_doc('Dashboard Chart', 'Test Empty Dashboard Chart 2') frappe.db.delete("Error Log") - # frappe.db.sql('delete from `tabError Log`') # create one data point frappe.get_doc(dict(doctype = 'Error Log', creation = '2018-06-01 00:00:00')).insert() diff --git a/frappe/desk/doctype/desktop_icon/desktop_icon.py b/frappe/desk/doctype/desktop_icon/desktop_icon.py index bc5677e9b9..283ffad967 100644 --- a/frappe/desk/doctype/desktop_icon/desktop_icon.py +++ b/frappe/desk/doctype/desktop_icon/desktop_icon.py @@ -200,7 +200,6 @@ def set_desktop_icons(visible_list, ignore_duplicate=True): frappe.db.delete("Desktop Icon", { "standard": 0 }) - # frappe.db.sql('delete from `tabDesktop Icon` where standard=0') # set standard as blocked and hidden if setting first active domain if not frappe.flags.keep_desktop_icons: diff --git a/frappe/desk/doctype/event/event.py b/frappe/desk/doctype/event/event.py index f9a0dd74fd..2182ec4c9f 100644 --- a/frappe/desk/doctype/event/event.py +++ b/frappe/desk/doctype/event/event.py @@ -341,11 +341,9 @@ def delete_events(ref_type, ref_name, delete_event=False): frappe.db.delete("Event", { "name": participation.parent }) - # frappe.db.sql("DELETE FROM `tabEvent` WHERE `name` = %(name)s", {'name': participation.parent}) frappe.db.delete("Event Participants", { "name": participation.name }) - # frappe.db.sql("DELETE FROM `tabEvent Participants ` WHERE `name` = %(name)s", {'name': participation.name}) # Close events if ends_on or repeat_till is less than now_datetime def set_status_of_events(): diff --git a/frappe/desk/doctype/route_history/route_history.py b/frappe/desk/doctype/route_history/route_history.py index bfa2711b8f..4303b7554d 100644 --- a/frappe/desk/doctype/route_history/route_history.py +++ b/frappe/desk/doctype/route_history/route_history.py @@ -35,13 +35,4 @@ def flush_old_route_records(): frappe.db.delete("Route History", { "modified": last_record_to_keep[0].modified, "user": user - }) - - # frappe.db.sql(''' - # DELETE - # FROM `tabRoute History` - # WHERE `modified` <= %(modified)s and `user`=%(modified)s - # ''', { - # "modified": last_record_to_keep[0].modified, - # "user": user - # }) \ No newline at end of file + }) \ No newline at end of file diff --git a/frappe/desk/doctype/tag/tag.py b/frappe/desk/doctype/tag/tag.py index 8b78405d06..626a2db085 100644 --- a/frappe/desk/doctype/tag/tag.py +++ b/frappe/desk/doctype/tag/tag.py @@ -127,7 +127,6 @@ def delete_tags_for_document(doc): "document_type": doc.doctype, "document_name": doc.name }) - # frappe.db.sql("""DELETE FROM `tabTag Link` WHERE `document_type`=%s AND `document_name`=%s""", (doc.doctype, doc.name)) def update_tags(doc, tags): """ @@ -170,7 +169,6 @@ def delete_tag_for_document(dt, dn, tag): "document_name": dn, "tag": tag }) - # frappe.db.sql("""DELETE FROM `tabTag Link` WHERE `document_type`=%s AND `document_name`=%s AND tag=%s""", (dt, dn, tag)) @frappe.whitelist() def get_documents_for_tag(tag): diff --git a/frappe/desk/doctype/todo/todo.py b/frappe/desk/doctype/todo/todo.py index 388ba37e1f..34fb3bb905 100644 --- a/frappe/desk/doctype/todo/todo.py +++ b/frappe/desk/doctype/todo/todo.py @@ -44,12 +44,6 @@ class ToDo(Document): "link_doctype": self.doctype, "link_name": self.name }) - # frappe.db.sql(""" - # delete from `tabCommunication Link` - # where link_doctype=%(doctype)s and link_name=%(name)s""", { - # "doctype": self.doctype, "name": self.name - # }) - self.update_in_reference() def add_assign_comment(self, text, comment_type): diff --git a/frappe/email/doctype/unhandled_email/unhandled_email.py b/frappe/email/doctype/unhandled_email/unhandled_email.py index 850007d85f..b445c98aa6 100644 --- a/frappe/email/doctype/unhandled_email/unhandled_email.py +++ b/frappe/email/doctype/unhandled_email/unhandled_email.py @@ -12,6 +12,4 @@ class UnhandledEmail(Document): def remove_old_unhandled_emails(): frappe.db.delete("Unhandled Email", { "creation": ("<", frappe.utils.add_days(frappe.utils.nowdate(), -30)) - }) - # frappe.db.sql("""DELETE FROM `tabUnhandled Email` - # WHERE creation < %s""", frappe.utils.add_days(frappe.utils.nowdate(), -30)) \ No newline at end of file + }) \ No newline at end of file diff --git a/frappe/model/__init__.py b/frappe/model/__init__.py index 7459493fcc..6b38b383bf 100644 --- a/frappe/model/__init__.py +++ b/frappe/model/__init__.py @@ -156,11 +156,6 @@ def delete_fields(args_dict, delete=0): "parent": dt, "fieldname": ("in", fields) }) - # frappe.db.sql(""" - # DELETE FROM `tabDocField` - # WHERE parent='%s' AND fieldname IN (%s) - # """ % (dt, ", ".join(["'{}'".format(f) for f in fields]))) - # Delete the data/column only if delete is specified if not delete: continue @@ -170,10 +165,6 @@ def delete_fields(args_dict, delete=0): "doctype": dt, "field": ("in", fields) }) - # frappe.db.sql(""" - # DELETE FROM `tabSingles` - # WHERE doctype='%s' AND field IN (%s) - # """ % (dt, ", ".join("'{}'".format(f) for f in fields))) else: existing_fields = frappe.db.multisql({ "mariadb": "DESC `tab%s`" % dt, diff --git a/frappe/model/delete_doc.py b/frappe/model/delete_doc.py index 9187310a84..44e675f3c5 100644 --- a/frappe/model/delete_doc.py +++ b/frappe/model/delete_doc.py @@ -82,14 +82,6 @@ def delete_doc(doctype=None, name=None, force=0, ignore_doctypes=None, for_reloa frappe.db.delete("__global_search", { "doctype": name }) - - # frappe.db.sql("delete from `tabCustom Field` where dt = %s", name) - # frappe.db.sql("delete from `tabClient Script` where dt = %s", name) - # frappe.db.sql("delete from `tabProperty Setter` where doc_type = %s", name) - # frappe.db.sql("delete from `tabReport` where ref_doctype=%s", name) - # frappe.db.sql("delete from `tabCustom DocPerm` where parent=%s", name) - # frappe.db.sql("delete from `__global_search` where doctype=%s", name) - delete_from_table(doctype, name, ignore_doctypes, None) if frappe.conf.developer_mode and not doc.custom and not ( @@ -183,13 +175,10 @@ def delete_from_table(doctype, name, ignore_doctypes, doc): frappe.db.delete("Singles", { "doctype": name }) - # frappe.db.sql("delete from `tabSingles` where `doctype`=%s", name) else: frappe.db.delete(doctype, { "name": name }) - # frappe.db.sql("delete from `tab{0}` where `name`=%s".format(doctype), name) - # get child tables if doc: tables = [d.options for d in doc.meta.get_table_fields()] @@ -367,9 +356,7 @@ def clear_timeline_references(link_doctype, link_name): "link_doctype": link_doctype, "link_name": link_name }) - # frappe.db.sql("""DELETE FROM `tabCommunication Link` - # WHERE `tabCommunication Link`.link_doctype=%s AND `tabCommunication Link`.link_name=%s""", (link_doctype, link_name)) - + def insert_feed(doc): if ( frappe.flags.in_install diff --git a/frappe/model/document.py b/frappe/model/document.py index f8adfdb3cf..b44d95716e 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -395,11 +395,6 @@ class Document(BaseDocument): "parenttype": self.doctype, "parentfield": fieldname }) - - # frappe.db.sql("""delete from `tab{0}` where parent=%s - # and parenttype=%s and parentfield=%s""".format(df.options), - # (self.name, self.doctype, fieldname)) - def get_doc_before_save(self): return getattr(self, '_doc_before_save', None) @@ -460,7 +455,6 @@ class Document(BaseDocument): frappe.db.delete("Singles", { "doctype": self.doctype }) - # frappe.db.sql("""delete from `tabSingles` where doctype=%s""", self.doctype) for field, value in d.items(): if field != "doctype": frappe.db.sql("""insert into `tabSingles` (doctype, field, value) diff --git a/frappe/patches/v11_0/apply_customization_to_custom_doctype.py b/frappe/patches/v11_0/apply_customization_to_custom_doctype.py index c13ff83a1f..c01db50d5e 100644 --- a/frappe/patches/v11_0/apply_customization_to_custom_doctype.py +++ b/frappe/patches/v11_0/apply_customization_to_custom_doctype.py @@ -31,8 +31,6 @@ def execute(): frappe.db.delete("Property Setter", { "name": prop.name }) - # frappe.db.sql('DELETE FROM `tabProperty Setter` WHERE `name`=%s', prop.name) - meta = frappe.get_meta(doctype.name) for df in meta.fields: @@ -56,6 +54,4 @@ def execute(): frappe.db.delete("Custom Field", { "name": cf.name }) - # frappe.db.sql('DELETE FROM `tabCustom Field` WHERE name=%s', cf.name) - meta.save() diff --git a/frappe/patches/v11_0/sync_stripe_settings_before_migrate.py b/frappe/patches/v11_0/sync_stripe_settings_before_migrate.py index 1a12d4dc1d..5c949fbb1d 100644 --- a/frappe/patches/v11_0/sync_stripe_settings_before_migrate.py +++ b/frappe/patches/v11_0/sync_stripe_settings_before_migrate.py @@ -19,5 +19,4 @@ def execute(): frappe.db.delete("Singles", { "doctype": "Stripe Settings" - }) - # frappe.db.sql("""DELETE FROM tabSingles WHERE doctype='Stripe Settings'""") \ No newline at end of file + }) \ No newline at end of file diff --git a/frappe/patches/v12_0/delete_feedback_request_if_exists.py b/frappe/patches/v12_0/delete_feedback_request_if_exists.py index de799b1e21..bc3c7b8f97 100644 --- a/frappe/patches/v12_0/delete_feedback_request_if_exists.py +++ b/frappe/patches/v12_0/delete_feedback_request_if_exists.py @@ -4,8 +4,4 @@ import frappe def execute(): frappe.db.delete("DocType", { "name": "Feedback Request" - }) - # frappe.db.sql(''' - #DELETE from `tabDocType` - #WHERE name = 'Feedback Request' - # ''') \ No newline at end of file + }) \ No newline at end of file diff --git a/frappe/patches/v12_0/remove_deprecated_fields_from_doctype.py b/frappe/patches/v12_0/remove_deprecated_fields_from_doctype.py index 17dfbc5ca1..9c9a79ccbf 100644 --- a/frappe/patches/v12_0/remove_deprecated_fields_from_doctype.py +++ b/frappe/patches/v12_0/remove_deprecated_fields_from_doctype.py @@ -10,9 +10,4 @@ def execute(): frappe.db.delete("Property Setter", { "property": "read_only_onload" - }) - - # frappe.db.sql(''' - # DELETE from `tabProperty Setter` - # WHERE property = 'read_only_onload' - # ''') + }) \ No newline at end of file diff --git a/frappe/patches/v12_0/set_primary_key_in_series.py b/frappe/patches/v12_0/set_primary_key_in_series.py index a8409cbbba..e8d3abdde1 100644 --- a/frappe/patches/v12_0/set_primary_key_in_series.py +++ b/frappe/patches/v12_0/set_primary_key_in_series.py @@ -15,9 +15,6 @@ def execute(): frappe.db.delete("Series", { "name": row.name }) - # frappe.db.sql('delete from `tabSeries` where name = %(key)s', { - # 'key': row.name - # }) if row.current: frappe.db.sql('insert into `tabSeries`(`name`, `current`) values (%(name)s, %(current)s)', row) frappe.db.commit() diff --git a/frappe/patches/v12_0/setup_comments_from_communications.py b/frappe/patches/v12_0/setup_comments_from_communications.py index a4e3fb4261..b72d02720e 100644 --- a/frappe/patches/v12_0/setup_comments_from_communications.py +++ b/frappe/patches/v12_0/setup_comments_from_communications.py @@ -31,5 +31,4 @@ def execute(): # clean up frappe.db.delete("Communication", { "communication_type": "Comment" - }) - # frappe.db.sql("delete from `tabCommunication` where communication_type = 'Comment'") + }) \ No newline at end of file diff --git a/frappe/patches/v13_0/remove_twilio_settings.py b/frappe/patches/v13_0/remove_twilio_settings.py index 5d17364847..8b7e5e02c2 100644 --- a/frappe/patches/v13_0/remove_twilio_settings.py +++ b/frappe/patches/v13_0/remove_twilio_settings.py @@ -15,9 +15,8 @@ def execute(): frappe.db.delete("Singles", { "doctype": "Twilio Settings" }) - # frappe.db.sql("delete from `tabSingles` where `doctype`=%s", 'Twilio Settings') def twilio_settings_doctype_in_integrations() -> bool: """Check Twilio Settings doctype exists in integrations module or not. """ - return frappe.db.exists("DocType", {'name': 'Twilio Settings', 'module': 'Integrations'}) + return frappe.db.exists("DocType", {'name': 'Twilio Settings', 'module': 'Integrations'}) \ No newline at end of file diff --git a/frappe/permissions.py b/frappe/permissions.py index 7724f09e78..82f27452fb 100644 --- a/frappe/permissions.py +++ b/frappe/permissions.py @@ -520,8 +520,6 @@ def reset_perms(doctype): delete_notification_count_for(doctype) frappe.db.delete("Custom DocPerm", {"parent": doctype}) - # frappe.db.sql("""delete from `tabCustom DocPerm` where parent=%s""", doctype) - def get_linked_doctypes(dt): return list(set([dt] + [d.options for d in frappe.get_meta(dt).get("fields", { diff --git a/frappe/sessions.py b/frappe/sessions.py index f4d05538bc..58158a2e4e 100644 --- a/frappe/sessions.py +++ b/frappe/sessions.py @@ -84,7 +84,6 @@ def delete_session(sid=None, user=None, reason="Session Expired"): logout_feed(user, reason) frappe.db.delete("Sessions", {"sid": sid}) - # frappe.db.sql("""delete from tabSessions where sid=%s""", sid) frappe.db.commit() def clear_all_sessions(reason=None): diff --git a/frappe/utils/error.py b/frappe/utils/error.py index f2cbc5ec10..d83cbb49ea 100644 --- a/frappe/utils/error.py +++ b/frappe/utils/error.py @@ -177,8 +177,6 @@ def collect_error_snapshots(): def clear_old_snapshots(): """Clear snapshots that are older than a month""" - # frappe.db.delete("Error Snapshot", filters={"creation": ("<", ["NOW()", "-", f"INTERVAL '1' MONTH"])}) - frappe.db.sql("""delete from `tabError Snapshot` where creation < (NOW() - INTERVAL '1' MONTH)""") diff --git a/frappe/utils/global_search.py b/frappe/utils/global_search.py index a018761370..6391fcf7ef 100644 --- a/frappe/utils/global_search.py +++ b/frappe/utils/global_search.py @@ -24,8 +24,6 @@ def reset(): :return: """ frappe.db.delete("__global_search") - # frappe.db.sql('DELETE FROM `__global_search`') - def get_doctypes_with_global_search(with_child_tables=True): """ @@ -150,10 +148,6 @@ def delete_global_search_records_for_doctype(doctype): frappe.db.delete("__global_search", { "doctype": doctype }) - # frappe.db.sql('''DELETE - # FROM `__global_search` - # WHERE doctype = %s''', doctype, as_dict=True) - def get_selected_fields(meta, global_search_fields): fieldnames = [df.fieldname for df in global_search_fields] @@ -403,18 +397,11 @@ def delete_for_document(doc): been deleted :param doc: Deleted document """ - frappe.db.delete("__global_search", { "doctype": doc.doctype, "name": doc.name }) - # frappe.db.sql('''DELETE - # FROM `__global_search` - # WHERE doctype = %s - # AND name = %s''', (doc.doctype, doc.name), as_dict=True) - - @frappe.whitelist() def search(text, start=0, limit=20, doctype=""): """ diff --git a/frappe/utils/install.py b/frappe/utils/install.py index 9d21a0d945..e6d4386ebe 100644 --- a/frappe/utils/install.py +++ b/frappe/utils/install.py @@ -113,8 +113,6 @@ def before_tests(): frappe.db.delete("Custom Field") frappe.db.delete("Event") - # frappe.db.sql("delete from `tabCustom Field`") - # frappe.db.sql("delete from `tabEvent`") frappe.db.commit() frappe.clear_cache() diff --git a/frappe/utils/password.py b/frappe/utils/password.py index 21e1bc9586..a097c58b31 100644 --- a/frappe/utils/password.py +++ b/frappe/utils/password.py @@ -71,11 +71,6 @@ def remove_encrypted_password(doctype, name, fieldname='password'): "fieldname": fieldname }) - # frappe.db.sql( - # 'DELETE FROM `__Auth` WHERE doctype = %s and name = %s and fieldname = %s', - # values=[doctype, name, fieldname] - # ) - def check_password(user, pwd, doctype='User', fieldname='password', delete_tracker_cache=True): '''Checks if user and password are correct, else raises frappe.AuthenticationError''' @@ -140,8 +135,6 @@ def delete_all_passwords_for(doctype, name): "doctype": doctype, "name": name }) - # frappe.db.sql("""delete from `__Auth` where `doctype`=%(doctype)s and `name`=%(name)s""", - # { 'doctype': doctype, 'name': name }) except Exception as e: if not frappe.db.is_missing_column(e): raise diff --git a/frappe/utils/testutils.py b/frappe/utils/testutils.py index 74b800bdcf..5c5bfa7976 100644 --- a/frappe/utils/testutils.py +++ b/frappe/utils/testutils.py @@ -15,5 +15,4 @@ def clear_custom_fields(doctype): frappe.db.delete("Custom Field", { "dt": doctype }) - # frappe.db.sql('delete from `tabCustom Field` where dt=%s', doctype) frappe.clear_cache(doctype=doctype) diff --git a/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py b/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py index 96898cc159..54d2c2e446 100644 --- a/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py +++ b/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py @@ -323,11 +323,6 @@ class PersonalDataDeletionRequest(Document): def remove_unverified_record(): - # frappe.db.delete("Personal Data Deletion Request", { - # "status": "Pending Verification", - # "creation": ("<", ["Now()", "-", "INTERVAL 7 DAY"]) - # }) - frappe.db.sql( """ DELETE FROM `tabPersonal Data Deletion Request` diff --git a/frappe/workflow/doctype/workflow_action/workflow_action.py b/frappe/workflow/doctype/workflow_action/workflow_action.py index 5957bf8671..e3b89ba0e5 100644 --- a/frappe/workflow/doctype/workflow_action/workflow_action.py +++ b/frappe/workflow/doctype/workflow_action/workflow_action.py @@ -139,10 +139,7 @@ def clear_old_workflow_actions(doc, user=None): "user": ("!=", user), "status": "Open" }) - # frappe.db.sql("""DELETE FROM `tabWorkflow Action` - # WHERE `reference_doctype`=%s AND `reference_name`=%s AND `user`!=%s AND `status`='Open'""", - # (doc.get('doctype'), doc.get('name'), user)) - + def update_completed_workflow_actions(doc, user=None): user = user if user else frappe.session.user frappe.db.sql("""UPDATE `tabWorkflow Action` SET `status`='Completed', `completed_by`=%s @@ -263,10 +260,6 @@ def clear_workflow_actions(doctype, name): "reference_doctype": doctype, "reference_name": name }) - # frappe.db.sql('''delete from `tabWorkflow Action` - # where reference_doctype=%s and reference_name=%s''', - # (doctype, name)) - def get_doc_workflow_state(doc): workflow_name = get_workflow_name(doc.get('doctype')) workflow_state_field = get_workflow_state_field(workflow_name) From 9a74255a2e8334b42a5b8f66799f207b0a92309a Mon Sep 17 00:00:00 2001 From: leela Date: Wed, 28 Jul 2021 13:35:08 +0530 Subject: [PATCH 130/164] style: Add row numbering for report view print format --- frappe/public/js/frappe/views/reports/print_grid.html | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frappe/public/js/frappe/views/reports/print_grid.html b/frappe/public/js/frappe/views/reports/print_grid.html index e607f12b52..6f693bc932 100644 --- a/frappe/public/js/frappe/views/reports/print_grid.html +++ b/frappe/public/js/frappe/views/reports/print_grid.html @@ -11,6 +11,7 @@ + # {% for col in columns %} {% if col.name && col._id !== "_check" %} {% for row in data %} + + {{ row._index + 1 }} + {% for col in columns %} {% if col.name && col._id !== "_check" %} From b719e1481de9a629eee42559f5b0599a4c049d6f Mon Sep 17 00:00:00 2001 From: leela Date: Wed, 28 Jul 2021 17:00:12 +0530 Subject: [PATCH 131/164] Revert "refactor: set amended docname to original docname" This reverts commit d459847ae39f8e9917686c3db6340ee95103085e. --- frappe/core/doctype/doctype/test_doctype.py | 8 +- frappe/model/document.py | 9 +- frappe/model/naming.py | 83 +++---------------- frappe/patches.txt | 1 - frappe/patches/v13_0/rename_cancelled_docs.py | 27 ------ frappe/public/js/frappe/form/form.js | 24 +++--- frappe/public/js/frappe/router.js | 6 -- frappe/tests/test_naming.py | 34 -------- 8 files changed, 28 insertions(+), 164 deletions(-) delete mode 100644 frappe/patches/v13_0/rename_cancelled_docs.py diff --git a/frappe/core/doctype/doctype/test_doctype.py b/frappe/core/doctype/doctype/test_doctype.py index 4b6d0e4794..1e1a01a685 100644 --- a/frappe/core/doctype/doctype/test_doctype.py +++ b/frappe/core/doctype/doctype/test_doctype.py @@ -348,7 +348,6 @@ class TestDocType(unittest.TestCase): dump_docs = json.dumps(docs.get('docs')) cancel_all_linked_docs(dump_docs) data_link_doc.cancel() - data_doc.name = '{}-CAN-0'.format(data_doc.name) data_doc.load_from_db() self.assertEqual(data_link_doc.docstatus, 2) self.assertEqual(data_doc.docstatus, 2) @@ -372,7 +371,7 @@ class TestDocType(unittest.TestCase): for data in link_doc.get('permissions'): data.submit = 1 data.cancel = 1 - link_doc.insert(ignore_if_duplicate=True) + link_doc.insert() #create first parent doctype test_doc_1 = new_doctype('Test Doctype 1') @@ -387,7 +386,7 @@ class TestDocType(unittest.TestCase): for data in test_doc_1.get('permissions'): data.submit = 1 data.cancel = 1 - test_doc_1.insert(ignore_if_duplicate=True) + test_doc_1.insert() #crete second parent doctype doc = new_doctype('Test Doctype 2') @@ -402,7 +401,7 @@ class TestDocType(unittest.TestCase): for data in link_doc.get('permissions'): data.submit = 1 data.cancel = 1 - doc.insert(ignore_if_duplicate=True) + doc.insert() # create doctype data data_link_doc_1 = frappe.new_doc('Test Linked Doctype 1') @@ -433,7 +432,6 @@ class TestDocType(unittest.TestCase): # checking that doc for Test Doctype 2 is not canceled self.assertRaises(frappe.LinkExistsError, data_link_doc_1.cancel) - data_doc_2.name = '{}-CAN-0'.format(data_doc_2.name) data_doc.load_from_db() data_doc_2.load_from_db() self.assertEqual(data_link_doc_1.docstatus, 2) diff --git a/frappe/model/document.py b/frappe/model/document.py index e974ae2a3e..61160e1f01 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -5,7 +5,7 @@ import time from frappe import _, msgprint, is_whitelisted from frappe.utils import flt, cstr, now, get_datetime_str, file_lock, date_diff from frappe.model.base_document import BaseDocument, get_controller -from frappe.model.naming import set_new_name, gen_new_name_for_cancelled_doc +from frappe.model.naming import set_new_name from werkzeug.exceptions import NotFound, Forbidden import hashlib, json from frappe.model import optional_fields, table_fields @@ -705,6 +705,7 @@ class Document(BaseDocument): else: tmp = frappe.db.sql("""select modified, docstatus from `tab{0}` where name = %s for update""".format(self.doctype), self.name, as_dict=True) + if not tmp: frappe.throw(_("Record does not exist")) else: @@ -915,12 +916,8 @@ class Document(BaseDocument): @whitelist.__func__ def _cancel(self): - """Cancel the document. Sets `docstatus` = 2, then saves. - """ + """Cancel the document. Sets `docstatus` = 2, then saves.""" self.docstatus = 2 - new_name = gen_new_name_for_cancelled_doc(self) - frappe.rename_doc(self.doctype, self.name, new_name, force=True, show_alert=False) - self.name = new_name self.save() @whitelist.__func__ diff --git a/frappe/model/naming.py b/frappe/model/naming.py index 6dff0aaff6..fe136adce8 100644 --- a/frappe/model/naming.py +++ b/frappe/model/naming.py @@ -28,7 +28,7 @@ def set_new_name(doc): doc.name = None if getattr(doc, "amended_from", None): - doc.name = _get_amended_name(doc) + _set_amended_name(doc) return elif getattr(doc.meta, "issingle", False): @@ -221,15 +221,6 @@ def revert_series_if_last(key, name, doc=None): * prefix = #### and hashes = 2021 (hash doesn't exist) * will search hash in key then accordingly get prefix = "" """ - if hasattr(doc, 'amended_from'): - # do not revert if doc is amended, since cancelled docs still exist - if doc.docstatus != 2 and doc.amended_from: - return - - # for first cancelled doc - if doc.docstatus == 2 and not doc.amended_from: - name, _ = NameParser.parse_docname(doc.name, sep='-CAN-') - if ".#" in key: prefix, hashes = key.rsplit(".", 1) if "#" not in hashes: @@ -312,9 +303,16 @@ def append_number_if_name_exists(doctype, value, fieldname="name", separator="-" return value -def _get_amended_name(doc): - name, _ = NameParser(doc).parse_amended_from() - return name +def _set_amended_name(doc): + am_id = 1 + am_prefix = doc.amended_from + if frappe.db.get_value(doc.doctype, doc.amended_from, "amended_from"): + am_id = cint(doc.amended_from.split("-")[-1]) + 1 + am_prefix = "-".join(doc.amended_from.split("-")[:-1]) # except the last hyphen + + doc.name = am_prefix + "-" + str(am_id) + return doc.name + def _field_autoname(autoname, doc, skip_slicing=None): """ @@ -325,6 +323,7 @@ def _field_autoname(autoname, doc, skip_slicing=None): name = (cstr(doc.get(fieldname)) or "").strip() return name + def _prompt_autoname(autoname, doc): """ Generate a name using Prompt option. This simply means the user will have to set the name manually. @@ -355,61 +354,3 @@ def _format_autoname(autoname, doc): name = re.sub(r"(\{[\w | #]+\})", get_param_value_for_match, autoname_value) return name - -class NameParser: - """Parse document name and return all the parts of it. - - NOTE: It handles cancellend and amended doc parsing for now. It can be expanded. - """ - def __init__(self, doc): - self.doc = doc - - def parse_name(self): - if not hasattr(self.doc, "amended_from"): - return (self.doc.name, None, None) - - #If document is cancelled document - if hasattr(self.doc, "amended_from") and self.doc.docstatus == 2: - return self.parse_docname(self.doc.name, sep='-CAN-') - return self.parse_docname(self.doc.name) - - def parse_amended_from(self): - if not getattr(self.doc, 'amended_from', None): - return (None, None) - return self.parse_docname(self.doc.amended_from, '-CAN-') - - @classmethod - def parse_docname(cls, name, sep='-'): - split_list = name.rsplit(sep, 1) - - if len(split_list) == 1: - return (name, None) - return (split_list[0], split_list[1]) - -def get_cancelled_doc_latest_counter(tname, docname): - """Get the latest counter used for cancelled docs of given docname. - """ - name_prefix = f'{docname}-CAN-' - - rows = frappe.db.sql(""" - select - name - from `tab{tname}` - where - name like %(name_prefix)s and docstatus=2 - """.format(tname=tname), {'name_prefix': name_prefix+'%'}, as_dict=1) - - if not rows: - return -1 - return max([int(row.name.replace(name_prefix, '') or -1) for row in rows]) - -def gen_new_name_for_cancelled_doc(doc): - """Generate a new name for cancelled document. - """ - if getattr(doc, "amended_from", None): - name, _ = NameParser(doc).parse_amended_from() - else: - name = doc.name - - counter = get_cancelled_doc_latest_counter(doc.doctype, name) - return f'{name}-CAN-{counter+1}' diff --git a/frappe/patches.txt b/frappe/patches.txt index a9c5807df0..7605d8ea2b 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -180,4 +180,3 @@ frappe.patches.v12_0.rename_uploaded_files_with_proper_name frappe.patches.v13_0.queryreport_columns frappe.patches.v13_0.jinja_hook frappe.patches.v13_0.update_notification_channel_if_empty -frappe.patches.v13_0.rename_cancelled_docs diff --git a/frappe/patches/v13_0/rename_cancelled_docs.py b/frappe/patches/v13_0/rename_cancelled_docs.py deleted file mode 100644 index 2e99a6f3cd..0000000000 --- a/frappe/patches/v13_0/rename_cancelled_docs.py +++ /dev/null @@ -1,27 +0,0 @@ -import frappe -from frappe.model.naming import NameParser -from frappe.model.rename_doc import rename_doc - -def execute(): - """Rename already cancelled documents by adding `CAN-X` postfix instead of `-X`. - """ - for doctype in frappe.db.get_all('DocType'): - doctype = frappe.get_doc('DocType', doctype.name) - if doctype.is_submittable and frappe.db.table_exists(doctype.name): - cancelled_docs = frappe.db.get_all(doctype.name, ['amended_from', 'name'], {'docstatus':2}) - - for doc in cancelled_docs: - if '-CAN-' in doc.name: - continue - - current_name = doc.name - - if getattr(doc, "amended_from", None): - orig_name, counter = NameParser.parse_docname(doc.name) - else: - orig_name, counter = doc.name, 0 - new_name = f'{orig_name}-CAN-{counter or 0}' - - print(f"Renaming {doctype.name} record from {current_name} to {new_name}") - rename_doc(doctype.name, current_name, new_name, ignore_permissions=True, show_alert=False) - frappe.db.commit() diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js index 3bbc883b0c..faaa3dfbd9 100644 --- a/frappe/public/js/frappe/form/form.js +++ b/frappe/public/js/frappe/form/form.js @@ -770,36 +770,32 @@ frappe.ui.form.Form = class FrappeForm { } _cancel(btn, callback, on_error, skip_confirm) { + const me = this; const cancel_doc = () => { frappe.validated = true; - this.script_manager.trigger("before_cancel").then(() => { + me.script_manager.trigger("before_cancel").then(() => { if (!frappe.validated) { - return this.handle_save_fail(btn, on_error); + return me.handle_save_fail(btn, on_error); } - const original_name = this.docname; - const after_cancel = (r) => { + var after_cancel = function(r) { if (r.exc) { - this.handle_save_fail(btn, on_error); + me.handle_save_fail(btn, on_error); } else { frappe.utils.play_sound("cancel"); + me.refresh(); callback && callback(); - this.script_manager.trigger("after_cancel"); - frappe.run_serially([ - () => this.rename_notify(this.doctype, original_name, r.docs[0].name), - () => frappe.router.clear_re_route(this.doctype, original_name), - () => this.refresh(), - ]); + me.script_manager.trigger("after_cancel"); } }; - frappe.ui.form.save(this, "cancel", after_cancel, btn); + frappe.ui.form.save(me, "cancel", after_cancel, btn); }); } if (skip_confirm) { cancel_doc(); } else { - frappe.confirm(__("Permanently Cancel {0}?", [this.docname]), cancel_doc, this.handle_save_fail(btn, on_error)); + frappe.confirm(__("Permanently Cancel {0}?", [this.docname]), cancel_doc, me.handle_save_fail(btn, on_error)); } }; @@ -821,7 +817,7 @@ frappe.ui.form.Form = class FrappeForm { 'docname': this.doc.name }).then(is_amended => { if (is_amended) { - frappe.throw(__('This document is already amended, you cannot amend it again')); + frappe.throw(__('This document is already amended, you cannot ammend it again')); } this.validate_form_action("Amend"); var me = this; diff --git a/frappe/public/js/frappe/router.js b/frappe/public/js/frappe/router.js index 3f8ce7fee2..8a29dc1040 100644 --- a/frappe/public/js/frappe/router.js +++ b/frappe/public/js/frappe/router.js @@ -234,12 +234,6 @@ frappe.router = { } }, - clear_re_route(doctype, docname) { - delete frappe.re_route[ - `${encodeURIComponent(frappe.router.slug(doctype))}/${encodeURIComponent(docname)}` - ]; - }, - set_title(sub_path) { if (frappe.route_titles[sub_path]) { frappe.utils.set_title(frappe.route_titles[sub_path]); diff --git a/frappe/tests/test_naming.py b/frappe/tests/test_naming.py index 78071a4120..557993882f 100644 --- a/frappe/tests/test_naming.py +++ b/frappe/tests/test_naming.py @@ -116,37 +116,3 @@ class TestNaming(unittest.TestCase): self.assertEqual(current_index.get('current'), 2) frappe.db.sql("""delete from `tabSeries` where name = %s""", series) - - def test_naming_for_cancelled_and_amended_doc(self): - submittable_doctype = frappe.get_doc({ - "doctype": "DocType", - "module": "Core", - "custom": 1, - "is_submittable": 1, - "permissions": [{ - "role": "System Manager", - "read": 1 - }], - "name": 'Submittable Doctype' - }).insert(ignore_if_duplicate=True) - - doc = frappe.new_doc('Submittable Doctype') - doc.save() - original_name = doc.name - - doc.submit() - doc.cancel() - cancelled_name = doc.name - self.assertEqual(cancelled_name, "{}-CAN-0".format(original_name)) - - amended_doc = frappe.copy_doc(doc) - amended_doc.docstatus = 0 - amended_doc.amended_from = doc.name - amended_doc.save() - self.assertEqual(amended_doc.name, original_name) - - amended_doc.submit() - amended_doc.cancel() - self.assertEqual(amended_doc.name, "{}-CAN-1".format(original_name)) - - submittable_doctype.delete() From 01d275f6677f4cd18b72b12d32ba4104a5b0ce30 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 28 Jul 2021 18:55:27 +0530 Subject: [PATCH 132/164] style: Format code for better readability * Remove trailing whitespaces * Format code to fit module conventions * Add appropriate new lines between imports, classes, fn defs, etc * Added comments, docstrings & module headers --- .../core/doctype/activity_log/activity_log.py | 1 + frappe/core/doctype/activity_log/feed.py | 5 ++- .../scheduled_job_type/scheduled_job_type.py | 11 +++--- frappe/core/doctype/user/user.py | 2 ++ .../user_permission/test_user_permission.py | 5 ++- .../user_permission/user_permission.py | 29 ++++++++------- .../doctype/customize_form/customize_form.py | 11 +++--- .../desk/doctype/desktop_icon/desktop_icon.py | 4 +-- frappe/desk/doctype/event/event.py | 8 ++--- .../doctype/route_history/route_history.py | 4 +-- frappe/desk/doctype/tag/tag.py | 3 +- frappe/model/__init__.py | 7 ++-- frappe/model/delete_doc.py | 36 ++++++------------- .../apply_customization_to_custom_doctype.py | 10 +++--- .../sync_stripe_settings_before_migrate.py | 4 +-- .../delete_feedback_request_if_exists.py | 4 +-- .../v12_0/set_primary_key_in_series.py | 2 ++ .../setup_comments_from_communications.py | 2 +- .../patches/v13_0/remove_twilio_settings.py | 2 +- frappe/permissions.py | 2 +- frappe/sessions.py | 1 + frappe/tests/test_commands.py | 2 +- frappe/utils/error.py | 4 +-- frappe/utils/testutils.py | 4 +-- .../personal_data_deletion_request.py | 1 + .../workflow_action/workflow_action.py | 2 +- 26 files changed, 72 insertions(+), 94 deletions(-) diff --git a/frappe/core/doctype/activity_log/activity_log.py b/frappe/core/doctype/activity_log/activity_log.py index ce5e504f7f..efec0dc217 100644 --- a/frappe/core/doctype/activity_log/activity_log.py +++ b/frappe/core/doctype/activity_log/activity_log.py @@ -44,5 +44,6 @@ def clear_activity_logs(days=None): if not days: days = 90 + frappe.db.sql("""delete from `tabActivity Log` where \ creation< (NOW() - INTERVAL '{0}' DAY)""".format(days)) \ No newline at end of file diff --git a/frappe/core/doctype/activity_log/feed.py b/frappe/core/doctype/activity_log/feed.py index efac825178..19d7b77184 100644 --- a/frappe/core/doctype/activity_log/feed.py +++ b/frappe/core/doctype/activity_log/feed.py @@ -26,12 +26,15 @@ def update_feed(doc, method=None): feed = frappe._dict(feed) doctype = feed.doctype or doc.doctype - name = feed.name or doc.name + name = feed.name or doc.name + + # delete earlier feed frappe.db.delete("Activity Log", { "reference_doctype": doctype, "reference_name": name, "link_doctype": feed.link_doctype }) + frappe.get_doc({ "doctype": "Activity Log", "reference_doctype": doctype, diff --git a/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py b/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py index d3afecf378..b6515b1e79 100644 --- a/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py +++ b/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py @@ -1,5 +1,4 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2019, Frappe Technologies and contributors +# Copyright (c) 2021, Frappe Technologies and contributors # For license information, please see license.txt import json @@ -13,6 +12,7 @@ from frappe.model.document import Document from frappe.utils import get_datetime, now_datetime from frappe.utils.background_jobs import enqueue, get_jobs + class ScheduledJobType(Document): def autoname(self): self.name = ".".join(self.method.split(".")[-2:]) @@ -109,10 +109,9 @@ class ScheduledJobType(Document): return 'long' if ('Long' in self.frequency) else 'default' def on_trash(self): - frappe.db.delete("Scheduled Job Log", { - "scheduled_job_type": self.name - }) - + frappe.db.delete("Scheduled Job Log", {"scheduled_job_type": self.name}) + + @frappe.whitelist() def execute_event(doc: str): frappe.only_for("System Manager") diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py index 65a48fb39e..5d799f8ee9 100644 --- a/frappe/core/doctype/user/user.py +++ b/frappe/core/doctype/user/user.py @@ -18,6 +18,7 @@ from frappe.rate_limiter import rate_limit from frappe.utils.background_jobs import enqueue from frappe.core.doctype.user_type.user_type import user_linked_with_permission_on_doctype + STANDARD_USERS = ("Guest", "Administrator") @@ -366,6 +367,7 @@ class User(Document): if getattr(frappe.local, "login_manager", None): frappe.local.login_manager.logout(user=self.name) + # delete todos frappe.db.delete("ToDo", {"owner": self.name}) frappe.db.sql("""UPDATE `tabToDo` SET `assigned_by`=NULL WHERE `assigned_by`=%s""", (self.name,)) diff --git a/frappe/core/doctype/user_permission/test_user_permission.py b/frappe/core/doctype/user_permission/test_user_permission.py index c3d593ee3b..1e48de334b 100644 --- a/frappe/core/doctype/user_permission/test_user_permission.py +++ b/frappe/core/doctype/user_permission/test_user_permission.py @@ -1,6 +1,5 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Frappe Technologies and Contributors -# See license.txt +# Copyright (c) 2021, Frappe Technologies and Contributors +# See LICENSE from frappe.core.doctype.user_permission.user_permission import add_user_permissions, remove_applicable from frappe.permissions import has_user_permission from frappe.core.doctype.doctype.test_doctype import new_doctype diff --git a/frappe/core/doctype/user_permission/user_permission.py b/frappe/core/doctype/user_permission/user_permission.py index de242efe10..5201ffef8d 100644 --- a/frappe/core/doctype/user_permission/user_permission.py +++ b/frappe/core/doctype/user_permission/user_permission.py @@ -1,5 +1,4 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Frappe Technologies and contributors +# Copyright (c) 2021, Frappe Technologies and contributors # For license information, please see license.txt import frappe, json @@ -179,14 +178,16 @@ def check_applicable_doc_perm(user, doctype, docname): @frappe.whitelist() def clear_user_permissions(user, for_doctype): - frappe.only_for('System Manager') - total = frappe.db.count('User Permission', filters = dict(user=user, allow=for_doctype)) + frappe.only_for("System Manager") + total = frappe.db.count("User Permission", {"user": user, "allow": for_doctype}) + if total: frappe.db.delete("User Permission", { + "allow": for_doctype, "user": user, - "allow": for_doctype }) frappe.clear_cache() + return total @frappe.whitelist() @@ -228,37 +229,35 @@ def insert_user_perm(user, doctype, docname, is_default=0, hide_descendants=0, a user_perm.is_default = is_default user_perm.hide_descendants = hide_descendants if applicable: - user_perm.applicable_for = applicable + user_perm.applicable_for = applicable user_perm.apply_to_all_doctypes = 0 else: user_perm.apply_to_all_doctypes = 1 user_perm.insert() def remove_applicable(perm_applied_docs, user, doctype, docname): - for applicable_for in perm_applied_docs: frappe.db.delete("User Permission", { - "user": user, "applicable_for": applicable_for, + "for_value": docname, "allow": doctype, - "for_value": docname + "user": user, }) -def remove_apply_to_all(user, doctype, docname): +def remove_apply_to_all(user, doctype, docname): frappe.db.delete("User Permission", { - "user": user, "apply_to_all_doctypes": 1, + "for_value": docname, "allow": doctype, - "for_value": docname + "user": user, }) def update_applicable(already_applied, to_apply, user, doctype, docname): for applied in already_applied: if applied not in to_apply: - frappe.db.delete("User Permission", { - "user": user, "applicable_for": applied, + "for_value": docname, "allow": doctype, - "for_value": docname + "user": user, }) diff --git a/frappe/custom/doctype/customize_form/customize_form.py b/frappe/custom/doctype/customize_form/customize_form.py index 5431511d42..8de194fb00 100644 --- a/frappe/custom/doctype/customize_form/customize_form.py +++ b/frappe/custom/doctype/customize_form/customize_form.py @@ -1,5 +1,5 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors -# MIT License. See license.txt +# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors +# MIT License. See LICENSE """ Customize Form is a Single DocType used to mask the Property Setter @@ -18,13 +18,12 @@ from frappe.custom.doctype.property_setter.property_setter import delete_propert from frappe.model.docfield import supports_translation from frappe.core.doctype.doctype.doctype import validate_series + class CustomizeForm(Document): def on_update(self): - frappe.db.delete("Singles", { - "doctype": "Customize Form" - }) + frappe.db.delete("Singles", {"doctype": "Customize Form"}) frappe.db.delete("Customize Form Field") - + @frappe.whitelist() def fetch_to_customize(self): self.clear_existing_doc() diff --git a/frappe/desk/doctype/desktop_icon/desktop_icon.py b/frappe/desk/doctype/desktop_icon/desktop_icon.py index 283ffad967..28c5a670cb 100644 --- a/frappe/desk/doctype/desktop_icon/desktop_icon.py +++ b/frappe/desk/doctype/desktop_icon/desktop_icon.py @@ -197,9 +197,7 @@ def set_desktop_icons(visible_list, ignore_duplicate=True): # clear all custom only if setup is not complete if not int(frappe.defaults.get_defaults().setup_complete or 0): - frappe.db.delete("Desktop Icon", { - "standard": 0 - }) + frappe.db.delete("Desktop Icon", {"standard": 0}) # set standard as blocked and hidden if setting first active domain if not frappe.flags.keep_desktop_icons: diff --git a/frappe/desk/doctype/event/event.py b/frappe/desk/doctype/event/event.py index 2182ec4c9f..e7e7be530b 100644 --- a/frappe/desk/doctype/event/event.py +++ b/frappe/desk/doctype/event/event.py @@ -338,12 +338,8 @@ def delete_events(ref_type, ref_name, delete_event=False): total_participants = frappe.get_all("Event Participants", filters={"parenttype": "Event", "parent": participation.parent}) if len(total_participants) <= 1: - frappe.db.delete("Event", { - "name": participation.parent - }) - frappe.db.delete("Event Participants", { - "name": participation.name - }) + frappe.db.delete("Event", {"name": participation.parent}) + frappe.db.delete("Event Participants", {"name": participation.name}) # Close events if ends_on or repeat_till is less than now_datetime def set_status_of_events(): diff --git a/frappe/desk/doctype/route_history/route_history.py b/frappe/desk/doctype/route_history/route_history.py index 4303b7554d..a179119861 100644 --- a/frappe/desk/doctype/route_history/route_history.py +++ b/frappe/desk/doctype/route_history/route_history.py @@ -1,5 +1,4 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2018, Frappe Technologies and contributors +# Copyright (c) 2021, Frappe Technologies and contributors # For license information, please see license.txt import frappe @@ -8,6 +7,7 @@ from frappe.model.document import Document class RouteHistory(Document): pass + def flush_old_route_records(): """Deletes all route records except last 500 records per user""" diff --git a/frappe/desk/doctype/tag/tag.py b/frappe/desk/doctype/tag/tag.py index 626a2db085..2341d721e2 100644 --- a/frappe/desk/doctype/tag/tag.py +++ b/frappe/desk/doctype/tag/tag.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and contributors # For license information, please see license.txt @@ -124,7 +123,7 @@ def delete_tags_for_document(doc): return frappe.db.delete("Tag Link", { - "document_type": doc.doctype, + "document_type": doc.doctype, "document_name": doc.name }) diff --git a/frappe/model/__init__.py b/frappe/model/__init__.py index 6b38b383bf..79b41936c3 100644 --- a/frappe/model/__init__.py +++ b/frappe/model/__init__.py @@ -151,11 +151,12 @@ def delete_fields(args_dict, delete=0): fields = args_dict[dt] if not fields: continue - + frappe.db.delete("DocField", { "parent": dt, - "fieldname": ("in", fields) + "fieldname": ("in", fields), }) + # Delete the data/column only if delete is specified if not delete: continue @@ -163,7 +164,7 @@ def delete_fields(args_dict, delete=0): if frappe.db.get_value("DocType", dt, "issingle"): frappe.db.delete("Singles", { "doctype": dt, - "field": ("in", fields) + "field": ("in", fields), }) else: existing_fields = frappe.db.multisql({ diff --git a/frappe/model/delete_doc.py b/frappe/model/delete_doc.py index 44e675f3c5..fbbf1a4852 100644 --- a/frappe/model/delete_doc.py +++ b/frappe/model/delete_doc.py @@ -64,24 +64,14 @@ def delete_doc(doctype=None, name=None, force=0, ignore_doctypes=None, for_reloa update_flags(doc, flags, ignore_permissions) check_permission_and_not_submitted(doc) - frappe.db.delete("Custom Field", { - "dt": name - }) - frappe.db.delete("Client Script", { - "dt": name - }) - frappe.db.delete("Property Setter", { - "doc_type": name - }) - frappe.db.delete("Report", { - "ref_doctype": name - }) - frappe.db.delete("Custom DocPerm", { - "parent": name - }) - frappe.db.delete("__global_search", { - "doctype": name - }) + + frappe.db.delete("Custom Field", {"dt": name}) + frappe.db.delete("Client Script", {"dt": name}) + frappe.db.delete("Property Setter", {"doc_type": name}) + frappe.db.delete("Report", {"ref_doctype": name}) + frappe.db.delete("Custom DocPerm", {"parent": name}) + frappe.db.delete("__global_search", {"doctype": name}) + delete_from_table(doctype, name, ignore_doctypes, None) if frappe.conf.developer_mode and not doc.custom and not ( @@ -172,13 +162,9 @@ def update_naming_series(doc): def delete_from_table(doctype, name, ignore_doctypes, doc): if doctype!="DocType" and doctype==name: - frappe.db.delete("Singles", { - "doctype": name - }) + frappe.db.delete("Singles", {"doctype": name}) else: - frappe.db.delete(doctype, { - "name": name - }) + frappe.db.delete(doctype, {"name": name}) # get child tables if doc: tables = [d.options for d in doc.meta.get_table_fields()] @@ -356,7 +342,7 @@ def clear_timeline_references(link_doctype, link_name): "link_doctype": link_doctype, "link_name": link_name }) - + def insert_feed(doc): if ( frappe.flags.in_install diff --git a/frappe/patches/v11_0/apply_customization_to_custom_doctype.py b/frappe/patches/v11_0/apply_customization_to_custom_doctype.py index c01db50d5e..7e84c5ae24 100644 --- a/frappe/patches/v11_0/apply_customization_to_custom_doctype.py +++ b/frappe/patches/v11_0/apply_customization_to_custom_doctype.py @@ -28,9 +28,8 @@ def execute(): for prop in property_setters: property_setter_map[prop.field_name] = prop - frappe.db.delete("Property Setter", { - "name": prop.name - }) + frappe.db.delete("Property Setter", {"name": prop.name}) + meta = frappe.get_meta(doctype.name) for df in meta.fields: @@ -51,7 +50,6 @@ def execute(): df = frappe.new_doc('DocField', meta, 'fields') df.update(cf) meta.fields.append(df) - frappe.db.delete("Custom Field", { - "name": cf.name - }) + frappe.db.delete("Custom Field", {"name": cf.name}) + meta.save() diff --git a/frappe/patches/v11_0/sync_stripe_settings_before_migrate.py b/frappe/patches/v11_0/sync_stripe_settings_before_migrate.py index 5c949fbb1d..901ab66bfd 100644 --- a/frappe/patches/v11_0/sync_stripe_settings_before_migrate.py +++ b/frappe/patches/v11_0/sync_stripe_settings_before_migrate.py @@ -17,6 +17,4 @@ def execute(): settings.secret_key = secret_key settings.save(ignore_permissions=True) - frappe.db.delete("Singles", { - "doctype": "Stripe Settings" - }) \ No newline at end of file + frappe.db.delete("Singles", {"doctype": "Stripe Settings"}) diff --git a/frappe/patches/v12_0/delete_feedback_request_if_exists.py b/frappe/patches/v12_0/delete_feedback_request_if_exists.py index bc3c7b8f97..c1bf46b14a 100644 --- a/frappe/patches/v12_0/delete_feedback_request_if_exists.py +++ b/frappe/patches/v12_0/delete_feedback_request_if_exists.py @@ -2,6 +2,4 @@ import frappe def execute(): - frappe.db.delete("DocType", { - "name": "Feedback Request" - }) \ No newline at end of file + frappe.db.delete("DocType", {"name": "Feedback Request"}) diff --git a/frappe/patches/v12_0/set_primary_key_in_series.py b/frappe/patches/v12_0/set_primary_key_in_series.py index e8d3abdde1..b2139f63b6 100644 --- a/frappe/patches/v12_0/set_primary_key_in_series.py +++ b/frappe/patches/v12_0/set_primary_key_in_series.py @@ -11,6 +11,7 @@ def execute(): name having count(name) > 1 ''', as_dict=True) + for row in duplicate_keys: frappe.db.delete("Series", { "name": row.name @@ -18,4 +19,5 @@ def execute(): if row.current: frappe.db.sql('insert into `tabSeries`(`name`, `current`) values (%(name)s, %(current)s)', row) frappe.db.commit() + frappe.db.sql('ALTER table `tabSeries` ADD PRIMARY KEY IF NOT EXISTS (name)') diff --git a/frappe/patches/v12_0/setup_comments_from_communications.py b/frappe/patches/v12_0/setup_comments_from_communications.py index b72d02720e..11e02965f1 100644 --- a/frappe/patches/v12_0/setup_comments_from_communications.py +++ b/frappe/patches/v12_0/setup_comments_from_communications.py @@ -31,4 +31,4 @@ def execute(): # clean up frappe.db.delete("Communication", { "communication_type": "Comment" - }) \ No newline at end of file + }) diff --git a/frappe/patches/v13_0/remove_twilio_settings.py b/frappe/patches/v13_0/remove_twilio_settings.py index 8b7e5e02c2..7efaf876e2 100644 --- a/frappe/patches/v13_0/remove_twilio_settings.py +++ b/frappe/patches/v13_0/remove_twilio_settings.py @@ -19,4 +19,4 @@ def execute(): def twilio_settings_doctype_in_integrations() -> bool: """Check Twilio Settings doctype exists in integrations module or not. """ - return frappe.db.exists("DocType", {'name': 'Twilio Settings', 'module': 'Integrations'}) \ No newline at end of file + return frappe.db.exists("DocType", {'name': 'Twilio Settings', 'module': 'Integrations'}) diff --git a/frappe/permissions.py b/frappe/permissions.py index 82f27452fb..33aef4ab41 100644 --- a/frappe/permissions.py +++ b/frappe/permissions.py @@ -7,11 +7,11 @@ import frappe.share from frappe import _, msgprint from frappe.utils import cint + rights = ("select", "read", "write", "create", "delete", "submit", "cancel", "amend", "print", "email", "report", "import", "export", "set_user_permissions", "share") - def check_admin_or_system_manager(user=None): if not user: user = frappe.session.user diff --git a/frappe/sessions.py b/frappe/sessions.py index 0202931e70..4f769ea88f 100644 --- a/frappe/sessions.py +++ b/frappe/sessions.py @@ -76,6 +76,7 @@ def get_sessions_to_clear(user=None, keep_current=False, device=None): def delete_session(sid=None, user=None, reason="Session Expired"): from frappe.core.doctype.activity_log.feed import logout_feed + frappe.cache().hdel("session", sid) frappe.cache().hdel("last_db_session_update", sid) if sid and not user: diff --git a/frappe/tests/test_commands.py b/frappe/tests/test_commands.py index 54103f0151..f687f70228 100644 --- a/frappe/tests/test_commands.py +++ b/frappe/tests/test_commands.py @@ -433,6 +433,6 @@ class TestCommands(BaseTestCommands): for output in ["legacy", "plain", "table", "json"]: self.execute(f"bench version -f {output}") self.assertEqual(self.returncode, 0) - + self.execute("bench version -f invalid") self.assertEqual(self.returncode, 2) diff --git a/frappe/utils/error.py b/frappe/utils/error.py index d83cbb49ea..05b578d7e8 100644 --- a/frappe/utils/error.py +++ b/frappe/utils/error.py @@ -176,9 +176,9 @@ def collect_error_snapshots(): def clear_old_snapshots(): """Clear snapshots that are older than a month""" - + frappe.db.sql("""delete from `tabError Snapshot` - where creation < (NOW() - INTERVAL '1' MONTH)""") + where creation < (NOW() - INTERVAL '1' MONTH)""") path = get_error_snapshot_path() today = datetime.datetime.now() diff --git a/frappe/utils/testutils.py b/frappe/utils/testutils.py index 5c5bfa7976..9a2b2da791 100644 --- a/frappe/utils/testutils.py +++ b/frappe/utils/testutils.py @@ -12,7 +12,5 @@ def add_custom_field(doctype, fieldname, fieldtype='Data', options=None): }).insert() def clear_custom_fields(doctype): - frappe.db.delete("Custom Field", { - "dt": doctype - }) + frappe.db.delete("Custom Field", {"dt": doctype}) frappe.clear_cache(doctype=doctype) diff --git a/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py b/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py index 54d2c2e446..63ba96d138 100644 --- a/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py +++ b/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py @@ -330,6 +330,7 @@ def remove_unverified_record(): AND `creation` < (NOW() - INTERVAL '7' DAY)""" ) + @frappe.whitelist(allow_guest=True) def confirm_deletion(email, name, host_name): if not verify_request(): diff --git a/frappe/workflow/doctype/workflow_action/workflow_action.py b/frappe/workflow/doctype/workflow_action/workflow_action.py index e3b89ba0e5..5eedc27d9c 100644 --- a/frappe/workflow/doctype/workflow_action/workflow_action.py +++ b/frappe/workflow/doctype/workflow_action/workflow_action.py @@ -139,7 +139,7 @@ def clear_old_workflow_actions(doc, user=None): "user": ("!=", user), "status": "Open" }) - + def update_completed_workflow_actions(doc, user=None): user = user if user else frappe.session.user frappe.db.sql("""UPDATE `tabWorkflow Action` SET `status`='Completed', `completed_by`=%s From a836b2de0c041fe17ca2530a5bf856719ad4f51c Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 28 Jul 2021 18:57:51 +0530 Subject: [PATCH 133/164] perf: Use truncate instead of drop Changed DML DROP statements to use DDL TRUNCATE for better performance wherever possible --- frappe/core/doctype/error_log/error_log.py | 2 +- frappe/utils/install.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/frappe/core/doctype/error_log/error_log.py b/frappe/core/doctype/error_log/error_log.py index 081b4e879d..3d66253b08 100644 --- a/frappe/core/doctype/error_log/error_log.py +++ b/frappe/core/doctype/error_log/error_log.py @@ -20,4 +20,4 @@ def set_old_logs_as_seen(): def clear_error_logs(): '''Flush all Error Logs''' frappe.only_for('System Manager') - frappe.db.delete("Error Log") \ No newline at end of file + frappe.db.truncate("Error Log") diff --git a/frappe/utils/install.py b/frappe/utils/install.py index e6d4386ebe..3d6a2fed97 100644 --- a/frappe/utils/install.py +++ b/frappe/utils/install.py @@ -111,9 +111,9 @@ def before_tests(): # don't run before tests if any other app is installed return - frappe.db.delete("Custom Field") - frappe.db.delete("Event") - frappe.db.commit() + frappe.db.truncate("Custom Field") + frappe.db.truncate("Event") + frappe.clear_cache() # complete setup if missing From 7dac03cea01f534ccaf3df01d34ffc71ecca4d05 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 28 Jul 2021 18:59:18 +0530 Subject: [PATCH 134/164] fix: Re-introduce code erreneously taken out in previous commits Due to bulk updations, some statements were missed out/got deleted. This commit re-introduces them. --- frappe/core/doctype/report/test_report.py | 2 +- .../doctype/user_permission/test_user_permission.py | 12 +++++++----- frappe/desk/doctype/route_history/route_history.py | 11 +++++------ 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/frappe/core/doctype/report/test_report.py b/frappe/core/doctype/report/test_report.py index 1bf9893bd7..9c953db1f0 100644 --- a/frappe/core/doctype/report/test_report.py +++ b/frappe/core/doctype/report/test_report.py @@ -81,12 +81,12 @@ class TestReport(unittest.TestCase): self.assertDictEqual({'name': 'Administrator', 'user_type': 'System User', 'email': 'admin@example.com'}, admin_dict) def test_report_permissions(self): - frappe.set_user('test@example.com') frappe.db.delete("Has Role", { "parent": frappe.session.user, "role": "Test Has Role" }) + frappe.db.commit() if not frappe.db.exists('Role', 'Test Has Role'): role = frappe.get_doc({ 'doctype': 'Role', diff --git a/frappe/core/doctype/user_permission/test_user_permission.py b/frappe/core/doctype/user_permission/test_user_permission.py index 1e48de334b..85db846982 100644 --- a/frappe/core/doctype/user_permission/test_user_permission.py +++ b/frappe/core/doctype/user_permission/test_user_permission.py @@ -9,13 +9,15 @@ import unittest class TestUserPermission(unittest.TestCase): def setUp(self): - + test_users = ( + "test_bulk_creation_update@example.com", + "test_user_perm1@example.com", + "nested_doc_user@example.com", + ) frappe.db.delete("User Permission", { - "user": ("in", ("test_bulk_creation_update@example.com", - "test_user_perm1@example.com", - "nested_doc_user@example.com")) + "user": ("in", test_users) }) - + frappe.delete_doc_if_exists("DocType", "Person") frappe.db.sql_ddl("DROP TABLE IF EXISTS `tabPerson`") frappe.delete_doc_if_exists("DocType", "Doc A") frappe.db.sql_ddl("DROP TABLE IF EXISTS `tabDoc A`") diff --git a/frappe/desk/doctype/route_history/route_history.py b/frappe/desk/doctype/route_history/route_history.py index a179119861..95872440c7 100644 --- a/frappe/desk/doctype/route_history/route_history.py +++ b/frappe/desk/doctype/route_history/route_history.py @@ -24,15 +24,14 @@ def flush_old_route_records(): for user in users: user = user[0] last_record_to_keep = frappe.db.get_all('Route History', - filters={ - 'user': user, - }, + filters={'user': user}, limit=1, limit_start=500, fields=['modified'], - order_by='modified desc') + order_by='modified desc' + ) frappe.db.delete("Route History", { - "modified": last_record_to_keep[0].modified, + "modified": ("<=", last_record_to_keep[0].modified), "user": user - }) \ No newline at end of file + }) From 3ca7fa77967a49b1d391cecd01e33a45d4e42012 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 27 Jul 2021 01:12:19 +0530 Subject: [PATCH 135/164] refactor(minor): Make frappe.db.delete DRY-er --- frappe/database/database.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index 8b4fd3ad9e..ee2d062a81 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -958,20 +958,17 @@ class Database(object): Doctype name can be passed directly, it will be pre-pended with `tab`. """ - if kwargs: - filters = filters or kwargs.get("conditions") + values = () + filters = filters or kwargs.get("conditions") + table = doctype if doctype.startswith("__") else f"tab{doctype}" + query = f"DELETE FROM `{table}`" if "debug" not in kwargs: kwargs["debug"] = debug - if not filters: - table = doctype if doctype.startswith("__") else f"tab{doctype}" - query = f"DELETE FROM `{table}`" - return self.sql(query, **kwargs) - - table = doctype if doctype.startswith("__") else f"tab{doctype}" - conditions, values = self.build_conditions(filters) - query = f"DELETE FROM `{table}` WHERE {conditions}" + if filters: + conditions, values = self.build_conditions(filters) + query = f"{query} WHERE {conditions}" return self.sql(query, values, **kwargs) From d63affc73251cd565bdc5e232eafd4a77f20a6a0 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 27 Jul 2021 02:12:19 +0530 Subject: [PATCH 136/164] refactor(minor): Use ORM instead of raw delete queries Modified query building for "IN" statements, as well as older condition builders to use frappe.db.delete --- frappe/defaults.py | 23 +++++++------------ frappe/desk/doctype/todo/todo.py | 13 +++++++---- frappe/email/queue.py | 10 ++------ .../v12_0/set_primary_key_in_series.py | 3 ++- 4 files changed, 20 insertions(+), 29 deletions(-) diff --git a/frappe/defaults.py b/frappe/defaults.py index 794d30a0c8..d4c338388d 100644 --- a/frappe/defaults.py +++ b/frappe/defaults.py @@ -154,29 +154,23 @@ def clear_default(key=None, value=None, parent=None, name=None, parenttype=None) :param name: Default ID. :param parenttype: Clear defaults table for a particular type e.g. **User**. """ - conditions = [] - values = [] + filters = {} if name: - conditions.append("name=%s") - values.append(name) + filters.update({"name": name}) else: if key: - conditions.append("defkey=%s") - values.append(key) + filters.update({"defkey": key}) if value: - conditions.append("defvalue=%s") - values.append(value) + filters.update({"defvalue": value}) if parent: - conditions.append("parent=%s") - values.append(parent) + filters.update({"parent": parent}) if parenttype: - conditions.append("parenttype=%s") - values.append(parenttype) + filters.update({"parenttype": parenttype}) if parent: clear_defaults_cache(parent) @@ -184,11 +178,10 @@ def clear_default(key=None, value=None, parent=None, name=None, parenttype=None) clear_defaults_cache("__default") clear_defaults_cache("__global") - if not conditions: + if not filters: raise Exception("[clear_default] No key specified.") - frappe.db.sql("""delete from tabDefaultValue where {0}""".format(" and ".join(conditions)), - tuple(values)) + frappe.db.delete("DefaultValue", filters) _clear_cache(parent) diff --git a/frappe/desk/doctype/todo/todo.py b/frappe/desk/doctype/todo/todo.py index 34fb3bb905..01eed41b4f 100644 --- a/frappe/desk/doctype/todo/todo.py +++ b/frappe/desk/doctype/todo/todo.py @@ -39,11 +39,7 @@ class ToDo(Document): self.update_in_reference() def on_trash(self): - # unlink todo from linked comments - frappe.db.delete("Communication Link", { - "link_doctype": self.doctype, - "link_name": self.name - }) + self.delete_communication_links() self.update_in_reference() def add_assign_comment(self, text, comment_type): @@ -52,6 +48,13 @@ class ToDo(Document): frappe.get_doc(self.reference_type, self.reference_name).add_comment(comment_type, text) + def delete_communication_links(self): + # unlink todo from linked comments + return frappe.db.delete("Communication Link", { + "link_doctype": self.doctype, + "link_name": self.name + }) + def update_in_reference(self): if not (self.reference_type and self.reference_name): return diff --git a/frappe/email/queue.py b/frappe/email/queue.py index 40f1c7be3a..ef59302bab 100755 --- a/frappe/email/queue.py +++ b/frappe/email/queue.py @@ -173,14 +173,8 @@ def clear_outbox(days=None): WHERE `priority`=0 AND `modified` < (NOW() - INTERVAL '{0}' DAY)""".format(days)) if email_queues: - # TODO: email_queues IN frappe.db.sql - frappe.db.sql("""DELETE FROM `tabEmail Queue` WHERE `name` IN ({0})""".format( - ','.join(['%s']*len(email_queues) - )), tuple(email_queues)) - - frappe.db.sql("""DELETE FROM `tabEmail Queue Recipient` WHERE `parent` IN ({0})""".format( - ','.join(['%s']*len(email_queues) - )), tuple(email_queues)) + frappe.db.delete("Email Queue", {"name": ("in", email_queues)}) + frappe.db.delete("Email Queue Recipient", {"parent": ("in", email_queues)}) def set_expiry_for_email_queue(): ''' Mark emails as expire that has not sent for 7 days. diff --git a/frappe/patches/v12_0/set_primary_key_in_series.py b/frappe/patches/v12_0/set_primary_key_in_series.py index b2139f63b6..83a903fc2d 100644 --- a/frappe/patches/v12_0/set_primary_key_in_series.py +++ b/frappe/patches/v12_0/set_primary_key_in_series.py @@ -2,7 +2,8 @@ import frappe def execute(): #if current = 0, simply delete the key as it'll be recreated on first entry - frappe.db.sql('delete from `tabSeries` where current = 0') + frappe.db.delete("Series", {"current": 0}) + duplicate_keys = frappe.db.sql(''' SELECT name, max(current) as current from From dfccae524618cd56b91e6da9f1091ea933ef6816 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 29 Jul 2021 02:12:19 +0530 Subject: [PATCH 137/164] perf(minor): Permission Manager remove API Delete using the filters directly instead of selecting rows in one query and deleting those rows in another. DBMS would have to scan the table twice prior --- frappe/core/page/permission_manager/permission_manager.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/frappe/core/page/permission_manager/permission_manager.py b/frappe/core/page/permission_manager/permission_manager.py index af8973811a..2a99283dda 100644 --- a/frappe/core/page/permission_manager/permission_manager.py +++ b/frappe/core/page/permission_manager/permission_manager.py @@ -110,11 +110,9 @@ def remove(doctype, role, permlevel): frappe.only_for("System Manager") setup_custom_perms(doctype) - name = frappe.get_value('Custom DocPerm', dict(parent=doctype, role=role, permlevel=permlevel)) - frappe.db.delete("Custom DocPerm", { - "name": name - }) - if not frappe.get_all('Custom DocPerm', dict(parent=doctype)): + frappe.db.delete("Custom DocPerm", {"parent": doctype, "role": role, "permlevel": permlevel}) + + if not frappe.get_all('Custom DocPerm', {"parent": doctype}): frappe.throw(_('There must be atleast one permission rule.'), title=_('Cannot Remove')) validate_permissions_for_doctype(doctype, for_remove=True, alert=True) From b40721de4b488025d484fa8b9abe5217778dd60e Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 29 Jul 2021 12:53:43 +0530 Subject: [PATCH 138/164] style: Remove extra whitespace --- frappe/database/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index ee2d062a81..2070ba676b 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -968,7 +968,7 @@ class Database(object): if filters: conditions, values = self.build_conditions(filters) - query = f"{query} WHERE {conditions}" + query = f"{query} WHERE {conditions}" return self.sql(query, values, **kwargs) From 5bb01511aa3e30aa7994ca2dfac0cf0d266d13e6 Mon Sep 17 00:00:00 2001 From: shariquerik Date: Thu, 29 Jul 2021 17:50:43 +0530 Subject: [PATCH 139/164] fix: new-doc-1 not found error --- frappe/public/js/frappe/form/dashboard.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/form/dashboard.js b/frappe/public/js/frappe/form/dashboard.js index b2b0c11d54..420704149b 100644 --- a/frappe/public/js/frappe/form/dashboard.js +++ b/frappe/public/js/frappe/form/dashboard.js @@ -204,7 +204,7 @@ frappe.ui.form.Dashboard = class FormDashboard { $(this).removeClass('hidden'); } }); - this.set_open_count(); + !this.frm.is_new() && this.set_open_count(); } init_data() { From ff973477a3e429ac82d8e5cb95ea0377b48b80be Mon Sep 17 00:00:00 2001 From: leela Date: Thu, 29 Jul 2021 20:32:03 +0530 Subject: [PATCH 140/164] fix: Use bench path as default bench_id --- frappe/tests/test_background_jobs.py | 6 +++--- frappe/utils/__init__.py | 2 +- frappe/utils/background_jobs.py | 20 ++++++++++---------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/frappe/tests/test_background_jobs.py b/frappe/tests/test_background_jobs.py index 48e0dd2ee9..188f3e166f 100644 --- a/frappe/tests/test_background_jobs.py +++ b/frappe/tests/test_background_jobs.py @@ -4,7 +4,7 @@ from rq import Queue import frappe from frappe.core.page.background_jobs.background_jobs import remove_failed_jobs -from frappe.utils.background_jobs import get_redis_conn, rename_queue +from frappe.utils.background_jobs import get_redis_conn, generate_qname import time @@ -17,14 +17,14 @@ class TestBackgroundJobs(unittest.TestCase): queues = Queue.all(conn) for queue in queues: - if queue.name == rename_queue("short"): + if queue.name == generate_qname("short"): fail_registry = queue.failed_job_registry self.assertGreater(fail_registry.count, 0) remove_failed_jobs() for queue in queues: - if queue.name == rename_queue("short"): + if queue.name == generate_qname("short"): fail_registry = queue.failed_job_registry self.assertEqual(fail_registry.count, 0) diff --git a/frappe/utils/__init__.py b/frappe/utils/__init__.py index 80c6cda98c..68366eb234 100644 --- a/frappe/utils/__init__.py +++ b/frappe/utils/__init__.py @@ -384,7 +384,7 @@ def get_bench_path(): return os.path.realpath(os.path.join(os.path.dirname(frappe.__file__), '..', '..', '..')) def get_bench_id(): - return frappe.get_conf().get('bench_id', 'DefaultBench') + return frappe.get_conf().get('bench_id', get_bench_path().strip('/').replace('/', '-')) def get_site_id(site=None): return f"{site or frappe.local.site}@{get_bench_id()}" diff --git a/frappe/utils/background_jobs.py b/frappe/utils/background_jobs.py index 4241c95c5d..f0bd06aff4 100755 --- a/frappe/utils/background_jobs.py +++ b/frappe/utils/background_jobs.py @@ -145,7 +145,7 @@ def start_worker(queue=None, quiet = False, rq_username=None, rq_password=None): # empty init is required to get redis_queue from common_site_config.json redis_connection = get_redis_conn(username=rq_username, password=rq_password) queues = get_queue_list(queue, build_queue_name=True) - queue_name = queue and rename_queue(queue) + queue_name = queue and generate_qname(queue) if os.environ.get('CI'): setup_loghandlers('ERROR') @@ -206,7 +206,7 @@ def get_queue_list(queue_list=None, build_queue_name=False): validate_queue(queue, default_queue_list) else: queue_list = default_queue_list - return [rename_queue(q) for q in queue_list] if build_queue_name else queue_list + return [generate_qname(qtype) for qtype in queue_list] if build_queue_name else queue_list def get_workers(queue): '''Returns a list of Worker objects tied to a queue object''' @@ -222,10 +222,10 @@ def get_running_jobs_in_queue(queue): jobs.append(current_job) return jobs -def get_queue(queue, is_async=True): +def get_queue(qtype, is_async=True): '''Returns a Queue object tied to a redis connection''' - validate_queue(queue) - return Queue(rename_queue(queue), connection=get_redis_conn(), is_async=is_async) + validate_queue(qtype) + return Queue(generate_qname(qtype), connection=get_redis_conn(), is_async=is_async) def validate_queue(queue, default_queue_list=None): if not default_queue_list: @@ -274,17 +274,17 @@ def get_queues() -> List[Queue]: queues = Queue.all(connection=get_redis_conn()) return [q for q in queues if is_queue_accessible(q)] -def rename_queue(qname: str) -> str: - """Rename qname by adding bench name as prefix. +def generate_qname(qtype: str) -> str: + """Generate qname by combining bench ID and queue type. - Renamed queues are useful to define namespaces of customers. + qnames are useful to define namespaces of customers. """ - return f"{get_bench_id()}:{qname}" + return f"{get_bench_id()}:{qtype}" def is_queue_accessible(qobj: Queue) -> bool: """Checks whether queue is relate to current bench or not. """ - accessible_queues = [rename_queue(q) for q in list(queue_timeout)] + accessible_queues = [generate_qname(q) for q in list(queue_timeout)] return qobj.name in accessible_queues def enqueue_test_job(): From eb9d2bcd649b6dba7e764ae5f34a61c8b0c51d9c Mon Sep 17 00:00:00 2001 From: saxenabhishek Date: Thu, 15 Jul 2021 01:21:31 +0530 Subject: [PATCH 141/164] feat: Query builder --- frappe/__init__.py | 4 +++ frappe/query_builder/__init__.py | 1 + frappe/query_builder/qb.py | 53 ++++++++++++++++++++++++++++++++ requirements.txt | 1 + 4 files changed, 59 insertions(+) create mode 100644 frappe/query_builder/__init__.py create mode 100644 frappe/query_builder/qb.py diff --git a/frappe/__init__.py b/frappe/__init__.py index 1c978945c7..39e21efb9e 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -28,6 +28,8 @@ from .exceptions import * from .utils.jinja import (get_jenv, get_template, render_template, get_email_from_template, get_jloader) from .utils.lazy_loader import lazy_import +from frappe.query_builder import query_builder + # Lazy imports faker = lazy_import('faker') @@ -118,6 +120,7 @@ def set_user_lang(user, user_language=None): # local-globals db = local("db") +qb = local("qb") conf = local("conf") form = form_dict = local("form_dict") request = local("request") @@ -202,6 +205,7 @@ def init(site, sites_path=None, new_site=False): local.form_dict = _dict() local.session = _dict() local.dev_server = _dev_server + local.qb = query_builder(local.conf.db_type) setup_module_map() diff --git a/frappe/query_builder/__init__.py b/frappe/query_builder/__init__.py new file mode 100644 index 0000000000..da1748beec --- /dev/null +++ b/frappe/query_builder/__init__.py @@ -0,0 +1 @@ +from frappe.query_builder.qb import qb as query_builder diff --git a/frappe/query_builder/qb.py b/frappe/query_builder/qb.py new file mode 100644 index 0000000000..6fa54661e8 --- /dev/null +++ b/frappe/query_builder/qb.py @@ -0,0 +1,53 @@ +from pypika import MySQLQuery, Order, PostgreSQLQuery +from pypika import functions as fn +from pypika import terms +from pypika.queries import Schema, Table + +def qb(db_type): + if not db_type: + db_type = "mariadb" + selecter = {"mariadb": MariaDB, "postgres": Postgres} + return selecter[db_type] + +class common: + fn = fn + terms = terms + desc = Order.desc + Schema = Schema + +class MariaDB(MySQLQuery,common): + Field = terms.Field + + def __init__(self) -> None: + super().__init__() + + @classmethod + def from_(cls, class_name, *args, **kwargs): + if isinstance(class_name,str): + class_name = "tab"+class_name + return super().from_(class_name, *args, **kwargs) + +class Postgres(PostgreSQLQuery,common): + postgres_field = {"table_name": "relname", "table_rows": "n_tup_ins"} + information_schema_translation = {"tables": "pg_stat_all_tables"} + + def __init__(self) -> None: + super().__init__() + + @classmethod + def Field(cls, fieldName, *args, **kwargs): + if fieldName in cls.postgres_field: + fieldName = cls.postgres_field[fieldName] + return terms.Field(fieldName, *args, **kwargs) + + @classmethod + def from_(cls, class_name, *args, **kwargs): + if isinstance(class_name, Table): + if class_name._schema: + if class_name._schema._name == "information_schema": + class_name = cls.information_schema_translation[class_name._table_name] + + elif isinstance(class_name, str): + class_name = "tab" + class_name + + return super().from_(class_name, *args, **kwargs) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 0791f01b27..51327953d5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -49,6 +49,7 @@ pyngrok~=5.0.5 pyOpenSSL~=20.0.1 pyotp~=2.6.0 PyPDF2~=1.26.0 +PyPika~=0.48.6 pypng~=0.0.20 PyQRCode~=1.2.1 python-dateutil~=2.8.1 From 49105ad08a59e60c3b7fdde9705adcce42ff75f0 Mon Sep 17 00:00:00 2001 From: saxenabhishek Date: Thu, 15 Jul 2021 01:22:06 +0530 Subject: [PATCH 142/164] refactor: qb in build_table_count_cache --- frappe/cache_manager.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/frappe/cache_manager.py b/frappe/cache_manager.py index 9f09f26be8..9de05907f5 100644 --- a/frappe/cache_manager.py +++ b/frappe/cache_manager.py @@ -141,17 +141,13 @@ def build_table_count_cache(): return _cache = frappe.cache() - data = frappe.db.multisql({ - "mariadb": """ - SELECT table_name AS name, - table_rows AS count - FROM information_schema.tables""", - "postgres": """ - SELECT "relname" AS name, - "n_tup_ins" AS count - FROM "pg_stat_all_tables" - """ - }, as_dict=1) + name = frappe.qb.Field("table_name").as_("name") + count = frappe.qb.Field("table_rows").as_("count") + information_schema = frappe.qb.Schema("information_schema") + + q = frappe.qb.from_(information_schema.tables).select(name, count).get_sql() + + data = frappe.db.sql(q, as_dict=1) counts = {d.get('name').lstrip('tab'): d.get('count', None) for d in data} _cache.set_value("information_schema:counts", counts) From 8b82815ba65c6baff0b064e1ab2d3ec54c794804 Mon Sep 17 00:00:00 2001 From: saxenabhishek Date: Thu, 15 Jul 2021 01:22:27 +0530 Subject: [PATCH 143/164] refactor: qb in get_all_empty_tables_by_module --- frappe/config/__init__.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/frappe/config/__init__.py b/frappe/config/__init__.py index 62a877be24..8fe110d393 100644 --- a/frappe/config/__init__.py +++ b/frappe/config/__init__.py @@ -39,18 +39,13 @@ def get_modules_from_app(app): ) def get_all_empty_tables_by_module(): - empty_tables = set(r[0] for r in frappe.db.multisql({ - "mariadb": """ - SELECT table_name - FROM information_schema.tables - WHERE table_rows = 0 and table_schema = "{}" - """.format(frappe.conf.db_name), - "postgres": """ - SELECT "relname" as "table_name" - FROM "pg_stat_all_tables" - WHERE n_tup_ins = 0 - """ - })) + table_rows = frappe.qb.Field("table_rows") + table_name = frappe.qb.Field("table_name") + information_schema = frappe.qb.Schema("information_schema") + + q = frappe.qb.from_(information_schema.tables).select(table_name).where(table_rows == 0).get_sql() + + empty_tables = set(r[0] for r in frappe.db.sql(q)) results = frappe.get_all("DocType", fields=["name", "module"]) empty_tables_by_module = {} From a8222126e815142dd073b4437008ebcdcb8e70f1 Mon Sep 17 00:00:00 2001 From: saxenabhishek Date: Thu, 15 Jul 2021 01:24:16 +0530 Subject: [PATCH 144/164] refactor: qb in after_rename --- frappe/core/doctype/doctype/doctype.py | 5 +---- frappe/query_builder/qb.py | 10 +++++++++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/frappe/core/doctype/doctype/doctype.py b/frappe/core/doctype/doctype/doctype.py index 3cdc45ea08..22be912abc 100644 --- a/frappe/core/doctype/doctype/doctype.py +++ b/frappe/core/doctype/doctype/doctype.py @@ -396,10 +396,7 @@ class DocType(Document): frappe.db.sql("""update tabSingles set value=%s where doctype=%s and field='name' and value = %s""", (new, new, old)) else: - frappe.db.multisql({ - "mariadb": f"RENAME TABLE `tab{old}` TO `tab{new}`", - "postgres": f"ALTER TABLE `tab{old}` RENAME TO `tab{new}`" - }) + frappe.db.sql(frappe.qb.rename_table(old,new)) frappe.db.commit() # Do not rename and move files and folders for custom doctype diff --git a/frappe/query_builder/qb.py b/frappe/query_builder/qb.py index 6fa54661e8..2dce79eca3 100644 --- a/frappe/query_builder/qb.py +++ b/frappe/query_builder/qb.py @@ -26,6 +26,10 @@ class MariaDB(MySQLQuery,common): if isinstance(class_name,str): class_name = "tab"+class_name return super().from_(class_name, *args, **kwargs) + + @staticmethod + def rename_table(old_name, new_name): + return f"RENAME TABLE `tab{old_name}` TO `tab{new_name}`" class Postgres(PostgreSQLQuery,common): postgres_field = {"table_name": "relname", "table_rows": "n_tup_ins"} @@ -50,4 +54,8 @@ class Postgres(PostgreSQLQuery,common): elif isinstance(class_name, str): class_name = "tab" + class_name - return super().from_(class_name, *args, **kwargs) \ No newline at end of file + return super().from_(class_name, *args, **kwargs) + + @staticmethod + def rename_table(old_name, new_name): + return f"ALTER TABLE `tab{old_name}` RENAME TO `tab{new_name}`" From 43b0d31cf822b4607b5686e19ac9c45a55208204 Mon Sep 17 00:00:00 2001 From: saxenabhishek Date: Thu, 15 Jul 2021 01:26:01 +0530 Subject: [PATCH 145/164] refactor: qb in delete_fields --- frappe/model/__init__.py | 12 +----------- frappe/query_builder/qb.py | 8 ++++++++ 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/frappe/model/__init__.py b/frappe/model/__init__.py index 79b41936c3..1f72272df1 100644 --- a/frappe/model/__init__.py +++ b/frappe/model/__init__.py @@ -167,17 +167,7 @@ def delete_fields(args_dict, delete=0): "field": ("in", fields), }) else: - existing_fields = frappe.db.multisql({ - "mariadb": "DESC `tab%s`" % dt, - "postgres": """ - SELECT - COLUMN_NAME - FROM - information_schema.COLUMNS - WHERE - TABLE_NAME = 'tab%s'; - """ % dt, - }) + existing_fields = frappe.db.sql(frappe.qb.DESC(dt)) existing_fields = existing_fields and [e[0] for e in existing_fields] or [] fields_need_to_delete = set(fields) & set(existing_fields) if not fields_need_to_delete: diff --git a/frappe/query_builder/qb.py b/frappe/query_builder/qb.py index 2dce79eca3..d085c01027 100644 --- a/frappe/query_builder/qb.py +++ b/frappe/query_builder/qb.py @@ -31,6 +31,10 @@ class MariaDB(MySQLQuery,common): def rename_table(old_name, new_name): return f"RENAME TABLE `tab{old_name}` TO `tab{new_name}`" + @staticmethod + def DESC(dt): + return f"DESC `tab{dt}`" + class Postgres(PostgreSQLQuery,common): postgres_field = {"table_name": "relname", "table_rows": "n_tup_ins"} information_schema_translation = {"tables": "pg_stat_all_tables"} @@ -59,3 +63,7 @@ class Postgres(PostgreSQLQuery,common): @staticmethod def rename_table(old_name, new_name): return f"ALTER TABLE `tab{old_name}` RENAME TO `tab{new_name}`" + + @staticmethod + def DESC(dt): + return f"SELECT COLUMN_NAME FROM information_schema.COLUMNS WHERE TABLE_NAME = 'tab{dt}'" From 90cd4f708ca6b7fb353510bb70c466dce988a28f Mon Sep 17 00:00:00 2001 From: saxenabhishek Date: Thu, 15 Jul 2021 01:29:17 +0530 Subject: [PATCH 146/164] refactor: qb in patch v13 increase_password_length --- frappe/patches/v13_0/increase_password_length.py | 5 +---- frappe/query_builder/qb.py | 8 ++++++++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/frappe/patches/v13_0/increase_password_length.py b/frappe/patches/v13_0/increase_password_length.py index 1bb1979051..3f0e93e72c 100644 --- a/frappe/patches/v13_0/increase_password_length.py +++ b/frappe/patches/v13_0/increase_password_length.py @@ -1,7 +1,4 @@ import frappe def execute(): - frappe.db.multisql({ - "mariadb": "ALTER TABLE `__Auth` MODIFY `password` TEXT NOT NULL", - "postgres": 'ALTER TABLE "__Auth" ALTER COLUMN "password" TYPE TEXT' - }) + frappe.db.sql(frappe.qb.change_table_type(tb = "__Auth",col = "password",type = "TEXT")) diff --git a/frappe/query_builder/qb.py b/frappe/query_builder/qb.py index d085c01027..ddf514ab2c 100644 --- a/frappe/query_builder/qb.py +++ b/frappe/query_builder/qb.py @@ -35,6 +35,10 @@ class MariaDB(MySQLQuery,common): def DESC(dt): return f"DESC `tab{dt}`" + @staticmethod + def change_table_type(tb, col, type): + return f"ALTER TABLE `{tb}` MODIFY `{col}` {type} NOT NULL" + class Postgres(PostgreSQLQuery,common): postgres_field = {"table_name": "relname", "table_rows": "n_tup_ins"} information_schema_translation = {"tables": "pg_stat_all_tables"} @@ -67,3 +71,7 @@ class Postgres(PostgreSQLQuery,common): @staticmethod def DESC(dt): return f"SELECT COLUMN_NAME FROM information_schema.COLUMNS WHERE TABLE_NAME = 'tab{dt}'" + + @staticmethod + def change_table_type(tb, col, type): + return f'ALTER TABLE "{tb}" ALTER COLUMN "{col}" TYPE {type}' From 2f7c78b266a09e27beaf418b42f47249a7e46170 Mon Sep 17 00:00:00 2001 From: saxenabhishek Date: Thu, 15 Jul 2021 02:00:50 +0530 Subject: [PATCH 147/164] refactor: qb in patch v12 set_correct_assign_value_in_docs --- .../v12_0/set_correct_assign_value_in_docs.py | 32 +++++++++---------- frappe/query_builder/custom_functions.py | 12 +++++++ frappe/query_builder/qb.py | 21 ++++++++---- 3 files changed, 42 insertions(+), 23 deletions(-) create mode 100644 frappe/query_builder/custom_functions.py diff --git a/frappe/patches/v12_0/set_correct_assign_value_in_docs.py b/frappe/patches/v12_0/set_correct_assign_value_in_docs.py index 65a635c170..72e71c8314 100644 --- a/frappe/patches/v12_0/set_correct_assign_value_in_docs.py +++ b/frappe/patches/v12_0/set_correct_assign_value_in_docs.py @@ -1,25 +1,23 @@ import frappe + def execute(): - frappe.reload_doc('desk', 'doctype', 'todo') + frappe.reload_doc("desk", "doctype", "todo") - query = ''' - SELECT - name, reference_type, reference_name, {} as assignees - FROM - `tabToDo` - WHERE - COALESCE(reference_type, '') != '' AND - COALESCE(reference_name, '') != '' AND - status != 'Cancelled' - GROUP BY - reference_type, reference_name - ''' + ToDo = frappe.qb.Table("ToDo") + assignees = frappe.qb.GROUP_CONCAT("owner").distinct().as_("assignees") - assignments = frappe.db.multisql({ - 'mariadb': query.format('GROUP_CONCAT(DISTINCT `owner`)'), - 'postgres': query.format('STRING_AGG(DISTINCT "owner", ",")') - }, as_dict=True) + q = ( + frappe.qb.from_(ToDo) + .select(ToDo.name, ToDo.reference_type, assignees) + .where(frappe.qb.fn.Coalesce(ToDo.reference_type, "") != "") + .where(frappe.qb.fn.Coalesce(ToDo.reference_name, "") != "") + .where(ToDo.status != "Cancelled") + .groupby(ToDo.reference_type, ToDo.reference_name) + .get_sql() + ) + + assignments = frappe.db.sql(q, as_dict=True) for doc in assignments: assignments = doc.assignees.split(',') diff --git a/frappe/query_builder/custom_functions.py b/frappe/query_builder/custom_functions.py new file mode 100644 index 0000000000..446a4df2d2 --- /dev/null +++ b/frappe/query_builder/custom_functions.py @@ -0,0 +1,12 @@ +from pypika import functions as fn + + + +class GROUP_CONCAT(fn.DistinctOptionFunction): + def __init__(self, col, alias=None): + super(GROUP_CONCAT, self).__init__("GROUP_CONCAT", col, alias=alias) + + +class STRING_AGG(fn.DistinctOptionFunction): + def __init__(self, col, val=",", alias=None): + super(STRING_AGG, self).__init__("STRING_AGG", col, val, alias=alias) diff --git a/frappe/query_builder/qb.py b/frappe/query_builder/qb.py index ddf514ab2c..fb92eed56b 100644 --- a/frappe/query_builder/qb.py +++ b/frappe/query_builder/qb.py @@ -2,6 +2,7 @@ from pypika import MySQLQuery, Order, PostgreSQLQuery from pypika import functions as fn from pypika import terms from pypika.queries import Schema, Table +from .custom_functions import GROUP_CONCAT, STRING_AGG def qb(db_type): if not db_type: @@ -14,19 +15,25 @@ class common: terms = terms desc = Order.desc Schema = Schema + @staticmethod + def Table(classname:str, *args, **kwargs): + if not classname.startswith("__"): + classname = "tab" + classname + return Table(classname, *args, **kwargs) -class MariaDB(MySQLQuery,common): +class MariaDB(common, MySQLQuery,): Field = terms.Field + GROUP_CONCAT = GROUP_CONCAT def __init__(self) -> None: super().__init__() - + @classmethod def from_(cls, class_name, *args, **kwargs): if isinstance(class_name,str): class_name = "tab"+class_name return super().from_(class_name, *args, **kwargs) - + @staticmethod def rename_table(old_name, new_name): return f"RENAME TABLE `tab{old_name}` TO `tab{new_name}`" @@ -39,9 +46,11 @@ class MariaDB(MySQLQuery,common): def change_table_type(tb, col, type): return f"ALTER TABLE `{tb}` MODIFY `{col}` {type} NOT NULL" -class Postgres(PostgreSQLQuery,common): + +class Postgres(common, PostgreSQLQuery,): postgres_field = {"table_name": "relname", "table_rows": "n_tup_ins"} information_schema_translation = {"tables": "pg_stat_all_tables"} + GROUP_CONCAT = STRING_AGG def __init__(self) -> None: super().__init__() @@ -63,11 +72,11 @@ class Postgres(PostgreSQLQuery,common): class_name = "tab" + class_name return super().from_(class_name, *args, **kwargs) - + @staticmethod def rename_table(old_name, new_name): return f"ALTER TABLE `tab{old_name}` RENAME TO `tab{new_name}`" - + @staticmethod def DESC(dt): return f"SELECT COLUMN_NAME FROM information_schema.COLUMNS WHERE TABLE_NAME = 'tab{dt}'" From 3f081a55e5272bb743599d0b3aff3535c7717c6c Mon Sep 17 00:00:00 2001 From: saxenabhishek Date: Thu, 15 Jul 2021 02:12:12 +0530 Subject: [PATCH 148/164] refactor: qb in Website_analytics.get_data --- .../website_analytics/website_analytics.py | 38 ++++++------------- 1 file changed, 12 insertions(+), 26 deletions(-) diff --git a/frappe/website/report/website_analytics/website_analytics.py b/frappe/website/report/website_analytics/website_analytics.py index d141972679..683eeba487 100644 --- a/frappe/website/report/website_analytics/website_analytics.py +++ b/frappe/website/report/website_analytics/website_analytics.py @@ -56,33 +56,19 @@ class WebsiteAnalytics(object): ] def get_data(self): - pg_query = """ - SELECT - path, - COUNT(*) as count, - COUNT(CASE WHEN CAST(is_unique as Integer) = 1 THEN 1 END) as unique_count - FROM `tabWeb Page View` - WHERE coalesce("tabWeb Page View".creation, '0001-01-01') BETWEEN %s AND %s - GROUP BY path - ORDER BY count desc - """ + Web_Page_View = frappe.qb.Table("Web Page View") + count_all = frappe.qb.fn.Count("*").as_("count") + case = frappe.qb.terms.Case().when(Web_Page_View.is_unique == "1", "1") + count_is_unique = frappe.qb.fn.Count(case).as_("unique_count") - mariadb_query = """ - SELECT - path, - COUNT(*) as count, - COUNT(CASE WHEN is_unique = 1 THEN 1 END) as unique_count - FROM `tabWeb Page View` - WHERE creation BETWEEN %s AND %s - GROUP BY path - ORDER BY count desc - """ - - data = frappe.db.multisql({ - "mariadb": mariadb_query, - "postgres": pg_query - }, (self.filters.from_date, self.filters.to_date)) - return data + curr = ( + frappe.qb.from_(Web_Page_View) + .select("path", count_all, count_is_unique) + .where(frappe.qb.fn.Coalesce(Web_Page_View.creation, "0001-01-01")[self.filters.from_date:self.filters.to_date]) + .groupby(Web_Page_View.path) + .orderby("count", Order=frappe.qb.desc).get_sql() + ) + return frappe.db.sql(curr) def _get_query_for_mariadb(self): filters_range = self.filters.range From c05a0328115049050a3a6aaf16ce9aaad5561326 Mon Sep 17 00:00:00 2001 From: saxenabhishek Date: Fri, 23 Jul 2021 10:38:33 +0530 Subject: [PATCH 149/164] refactor: qb in global_search.search --- frappe/query_builder/custom_functions.py | 46 +++++++++++++++++++++--- frappe/query_builder/qb.py | 10 ++++-- frappe/utils/global_search.py | 34 ++++++------------ 3 files changed, 59 insertions(+), 31 deletions(-) diff --git a/frappe/query_builder/custom_functions.py b/frappe/query_builder/custom_functions.py index 446a4df2d2..6b6dd137ab 100644 --- a/frappe/query_builder/custom_functions.py +++ b/frappe/query_builder/custom_functions.py @@ -1,12 +1,50 @@ from pypika import functions as fn +from pypika.utils import builder class GROUP_CONCAT(fn.DistinctOptionFunction): - def __init__(self, col, alias=None): - super(GROUP_CONCAT, self).__init__("GROUP_CONCAT", col, alias=alias) + def __init__(self, col, alias=None): + super(GROUP_CONCAT, self).__init__("GROUP_CONCAT", col, alias=alias) class STRING_AGG(fn.DistinctOptionFunction): - def __init__(self, col, val=",", alias=None): - super(STRING_AGG, self).__init__("STRING_AGG", col, val, alias=alias) + def __init__(self, col, val=",", alias=None): + super(STRING_AGG, self).__init__("STRING_AGG", col, val, alias=alias) + +class Match(fn.DistinctOptionFunction): + def __init__(self, col, *args, **kwargs): + alias = kwargs.get("alias") + super(Match, self).__init__(" MATCH", col, *args, alias=alias) + self._Against = False + + def get_function_sql(self, **kwargs): + s = super(fn.DistinctOptionFunction, self).get_function_sql(**kwargs) + + # n = len(self.name) + 1 + if self._Against: + return s + f" AGAINST ('+{self._Against}*' IN BOOLEAN MODE)" + return s + + @builder + def Against(self, b): + self._Against = b + + +class TO_TSVECTOR(fn.DistinctOptionFunction): + def __init__(self, col, *args, **kwargs): + alias = kwargs.get("alias") + super(TO_TSVECTOR, self).__init__("TO_TSVECTOR", col, *args, alias=alias) + self._PLAINTO_TSQUERY = False + + def get_function_sql(self, **kwargs): + s = super(fn.DistinctOptionFunction, self).get_function_sql(**kwargs) + + # n = len(self.name) + 1 + if self._PLAINTO_TSQUERY: + return s + f" @@ PLAINTO_TSQUERY('{self._PLAINTO_TSQUERY}')" + return s + + @builder + def Against(self, b): + self._PLAINTO_TSQUERY = b diff --git a/frappe/query_builder/qb.py b/frappe/query_builder/qb.py index fb92eed56b..2c1e3e103a 100644 --- a/frappe/query_builder/qb.py +++ b/frappe/query_builder/qb.py @@ -2,7 +2,7 @@ from pypika import MySQLQuery, Order, PostgreSQLQuery from pypika import functions as fn from pypika import terms from pypika.queries import Schema, Table -from .custom_functions import GROUP_CONCAT, STRING_AGG +import frappe.query_builder.custom_functions as SpecialFuncs def qb(db_type): if not db_type: @@ -15,6 +15,7 @@ class common: terms = terms desc = Order.desc Schema = Schema + @staticmethod def Table(classname:str, *args, **kwargs): if not classname.startswith("__"): @@ -23,7 +24,9 @@ class common: class MariaDB(common, MySQLQuery,): Field = terms.Field - GROUP_CONCAT = GROUP_CONCAT + GROUP_CONCAT = SpecialFuncs.GROUP_CONCAT + Match = SpecialFuncs.Match + def __init__(self) -> None: super().__init__() @@ -50,7 +53,8 @@ class MariaDB(common, MySQLQuery,): class Postgres(common, PostgreSQLQuery,): postgres_field = {"table_name": "relname", "table_rows": "n_tup_ins"} information_schema_translation = {"tables": "pg_stat_all_tables"} - GROUP_CONCAT = STRING_AGG + GROUP_CONCAT = SpecialFuncs.STRING_AGG + Match = SpecialFuncs.TO_TSVECTOR def __init__(self) -> None: super().__init__() diff --git a/frappe/utils/global_search.py b/frappe/utils/global_search.py index 6391fcf7ef..62db9d8e1b 100644 --- a/frappe/utils/global_search.py +++ b/frappe/utils/global_search.py @@ -423,39 +423,25 @@ def search(text, start=0, limit=20, doctype=""): if not text: continue - conditions = '1=1' - offset = '' + global_search = frappe.qb.Table("__global_search") - mariadb_text = frappe.db.escape('+' + text + '*') + rank = frappe.qb.Match(global_search.content).Against(text).as_("rank") - mariadb_fields = '`doctype`, `name`, `content`, MATCH (`content`) AGAINST ({} IN BOOLEAN MODE) AS rank'.format(mariadb_text) - postgres_fields = '`doctype`, `name`, `content`, TO_TSVECTOR("content") @@ PLAINTO_TSQUERY({}) AS rank'.format(frappe.db.escape(text)) - - values = {} + q = (frappe.qb.from_(global_search) + .select(global_search.doctype,global_search.name,global_search.content,rank) + .orderby("rank", order=frappe.qb.desc) + .limit(limit)) if doctype: - conditions = '`doctype` = %(doctype)s' - values['doctype'] = doctype + q = q.where(global_search.doctype == doctype) elif allowed_doctypes: - conditions = '`doctype` IN %(allowed_doctypes)s' - values['allowed_doctypes'] = tuple(allowed_doctypes) + q = q.where(global_search.doctype.isin(allowed_doctypes)) if int(start) > 0: - offset = 'OFFSET {}'.format(start) + q = q.offset(int(start)) - common_query = """ - SELECT {fields} - FROM `__global_search` - WHERE {conditions} - ORDER BY rank DESC - LIMIT {limit} - {offset} - """ - result = frappe.db.multisql({ - 'mariadb': common_query.format(fields=mariadb_fields, conditions=conditions, limit=limit, offset=offset), - 'postgres': common_query.format(fields=postgres_fields, conditions=conditions, limit=limit, offset=offset) - }, values=values, as_dict=True) + result = frappe.db.sql(q.get_sql(),as_dict=True) results.extend(result) From dd36b2a528097be2139da64df82842e435ce5dfc Mon Sep 17 00:00:00 2001 From: saxenabhishek Date: Fri, 23 Jul 2021 18:39:27 +0530 Subject: [PATCH 150/164] refactor: move to get_query_builder --- frappe/__init__.py | 4 +-- frappe/query_builder/__init__.py | 2 +- frappe/query_builder/{qb.py => builder.py} | 30 ++++++++++++------- .../{custom_functions.py => functions.py} | 2 +- 4 files changed, 24 insertions(+), 14 deletions(-) rename frappe/query_builder/{qb.py => builder.py} (80%) rename frappe/query_builder/{custom_functions.py => functions.py} (100%) diff --git a/frappe/__init__.py b/frappe/__init__.py index 39e21efb9e..24faf840bd 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -28,7 +28,7 @@ from .exceptions import * from .utils.jinja import (get_jenv, get_template, render_template, get_email_from_template, get_jloader) from .utils.lazy_loader import lazy_import -from frappe.query_builder import query_builder +from frappe.query_builder import get_query_builder # Lazy imports faker = lazy_import('faker') @@ -205,7 +205,7 @@ def init(site, sites_path=None, new_site=False): local.form_dict = _dict() local.session = _dict() local.dev_server = _dev_server - local.qb = query_builder(local.conf.db_type) + local.qb = get_query_builder(local.conf.db_type) setup_module_map() diff --git a/frappe/query_builder/__init__.py b/frappe/query_builder/__init__.py index da1748beec..00356ab679 100644 --- a/frappe/query_builder/__init__.py +++ b/frappe/query_builder/__init__.py @@ -1 +1 @@ -from frappe.query_builder.qb import qb as query_builder +from frappe.query_builder.builder import get_query_builder diff --git a/frappe/query_builder/qb.py b/frappe/query_builder/builder.py similarity index 80% rename from frappe/query_builder/qb.py rename to frappe/query_builder/builder.py index 2c1e3e103a..db16047ce9 100644 --- a/frappe/query_builder/qb.py +++ b/frappe/query_builder/builder.py @@ -1,15 +1,25 @@ -from pypika import MySQLQuery, Order, PostgreSQLQuery +from pypika import MySQLQuery, Order, PostgreSQLQuery, Query from pypika import functions as fn from pypika import terms from pypika.queries import Schema, Table -import frappe.query_builder.custom_functions as SpecialFuncs +import frappe.query_builder.functions as SpecialFuncs -def qb(db_type): + +def get_query_builder(db_type: str) -> Query: + """[return the query builder object] + + Args: + db_type (str): [string value of the db used] + + Returns: + Query: [Query object] + """ if not db_type: db_type = "mariadb" selecter = {"mariadb": MariaDB, "postgres": Postgres} return selecter[db_type] + class common: fn = fn terms = terms @@ -17,24 +27,24 @@ class common: Schema = Schema @staticmethod - def Table(classname:str, *args, **kwargs): + def Table(classname: str, *args, **kwargs) -> Table: if not classname.startswith("__"): - classname = "tab" + classname + classname = f"tab{classname}" return Table(classname, *args, **kwargs) -class MariaDB(common, MySQLQuery,): + +class MariaDB(common, MySQLQuery): Field = terms.Field GROUP_CONCAT = SpecialFuncs.GROUP_CONCAT Match = SpecialFuncs.Match - def __init__(self) -> None: super().__init__() @classmethod def from_(cls, class_name, *args, **kwargs): - if isinstance(class_name,str): - class_name = "tab"+class_name + if isinstance(class_name, str): + class_name = f"tab{class_name}" return super().from_(class_name, *args, **kwargs) @staticmethod @@ -50,7 +60,7 @@ class MariaDB(common, MySQLQuery,): return f"ALTER TABLE `{tb}` MODIFY `{col}` {type} NOT NULL" -class Postgres(common, PostgreSQLQuery,): +class Postgres(common, PostgreSQLQuery): postgres_field = {"table_name": "relname", "table_rows": "n_tup_ins"} information_schema_translation = {"tables": "pg_stat_all_tables"} GROUP_CONCAT = SpecialFuncs.STRING_AGG diff --git a/frappe/query_builder/custom_functions.py b/frappe/query_builder/functions.py similarity index 100% rename from frappe/query_builder/custom_functions.py rename to frappe/query_builder/functions.py index 6b6dd137ab..3b45d78726 100644 --- a/frappe/query_builder/custom_functions.py +++ b/frappe/query_builder/functions.py @@ -2,7 +2,6 @@ from pypika import functions as fn from pypika.utils import builder - class GROUP_CONCAT(fn.DistinctOptionFunction): def __init__(self, col, alias=None): super(GROUP_CONCAT, self).__init__("GROUP_CONCAT", col, alias=alias) @@ -12,6 +11,7 @@ class STRING_AGG(fn.DistinctOptionFunction): def __init__(self, col, val=",", alias=None): super(STRING_AGG, self).__init__("STRING_AGG", col, val, alias=alias) + class Match(fn.DistinctOptionFunction): def __init__(self, col, *args, **kwargs): alias = kwargs.get("alias") From ee3c84beef82ae37be04414866827510c8d0ebff Mon Sep 17 00:00:00 2001 From: saxenabhishek Date: Sat, 24 Jul 2021 00:28:43 +0530 Subject: [PATCH 151/164] style: typecast to string inside db.sql --- frappe/cache_manager.py | 8 ++--- frappe/config/__init__.py | 4 +-- frappe/database/database.py | 1 + frappe/database/postgres/database.py | 1 + .../v12_0/set_correct_assign_value_in_docs.py | 11 +++---- frappe/utils/global_search.py | 31 ++++++++++++------- .../website_analytics/website_analytics.py | 16 ++++++---- 7 files changed, 42 insertions(+), 30 deletions(-) diff --git a/frappe/cache_manager.py b/frappe/cache_manager.py index 9de05907f5..c17ae583ed 100644 --- a/frappe/cache_manager.py +++ b/frappe/cache_manager.py @@ -141,13 +141,13 @@ def build_table_count_cache(): return _cache = frappe.cache() - name = frappe.qb.Field("table_name").as_("name") - count = frappe.qb.Field("table_rows").as_("count") + table_name = frappe.qb.Field("table_name").as_("name") + table_rows = frappe.qb.Field("table_rows").as_("count") information_schema = frappe.qb.Schema("information_schema") - q = frappe.qb.from_(information_schema.tables).select(name, count).get_sql() + query = frappe.qb.from_(information_schema.tables).select(table_name, table_rows) - data = frappe.db.sql(q, as_dict=1) + data = frappe.db.sql(query, as_dict=1) counts = {d.get('name').lstrip('tab'): d.get('count', None) for d in data} _cache.set_value("information_schema:counts", counts) diff --git a/frappe/config/__init__.py b/frappe/config/__init__.py index 8fe110d393..47f1849a43 100644 --- a/frappe/config/__init__.py +++ b/frappe/config/__init__.py @@ -43,9 +43,9 @@ def get_all_empty_tables_by_module(): table_name = frappe.qb.Field("table_name") information_schema = frappe.qb.Schema("information_schema") - q = frappe.qb.from_(information_schema.tables).select(table_name).where(table_rows == 0).get_sql() + query = frappe.qb.from_(information_schema.tables).select(table_name).where(table_rows == 0) - empty_tables = set(r[0] for r in frappe.db.sql(q)) + empty_tables = set(r[0] for r in frappe.db.sql(query)) results = frappe.get_all("DocType", fields=["name", "module"]) empty_tables_by_module = {} diff --git a/frappe/database/database.py b/frappe/database/database.py index 2070ba676b..bf97317f99 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -104,6 +104,7 @@ class Database(object): {"name": "a%", "owner":"test@example.com"}) """ + query = str(query) if re.search(r'ifnull\(', query, flags=re.IGNORECASE): # replaces ifnull in query with coalesce query = re.sub(r'ifnull\(', 'coalesce(', query, flags=re.IGNORECASE) diff --git a/frappe/database/postgres/database.py b/frappe/database/postgres/database.py index 8235277e30..32cd42eb94 100644 --- a/frappe/database/postgres/database.py +++ b/frappe/database/postgres/database.py @@ -297,6 +297,7 @@ class PostgresDatabase(Database): def modify_query(query): """"Modifies query according to the requirements of postgres""" # replace ` with " for definitions + query = str(query) query = query.replace('`', '"') query = replace_locate_with_strpos(query) # select from requires "" diff --git a/frappe/patches/v12_0/set_correct_assign_value_in_docs.py b/frappe/patches/v12_0/set_correct_assign_value_in_docs.py index 72e71c8314..dd493bc52c 100644 --- a/frappe/patches/v12_0/set_correct_assign_value_in_docs.py +++ b/frappe/patches/v12_0/set_correct_assign_value_in_docs.py @@ -7,24 +7,23 @@ def execute(): ToDo = frappe.qb.Table("ToDo") assignees = frappe.qb.GROUP_CONCAT("owner").distinct().as_("assignees") - q = ( + query = ( frappe.qb.from_(ToDo) .select(ToDo.name, ToDo.reference_type, assignees) .where(frappe.qb.fn.Coalesce(ToDo.reference_type, "") != "") .where(frappe.qb.fn.Coalesce(ToDo.reference_name, "") != "") .where(ToDo.status != "Cancelled") .groupby(ToDo.reference_type, ToDo.reference_name) - .get_sql() ) - assignments = frappe.db.sql(q, as_dict=True) + assignments = frappe.db.sql(query, as_dict=True) for doc in assignments: - assignments = doc.assignees.split(',') + assignments = doc.assignees.split(",") frappe.db.set_value( doc.reference_type, doc.reference_name, - '_assign', + "_assign", frappe.as_json(assignments), update_modified=False - ) + ) \ No newline at end of file diff --git a/frappe/utils/global_search.py b/frappe/utils/global_search.py index 62db9d8e1b..7754c87823 100644 --- a/frappe/utils/global_search.py +++ b/frappe/utils/global_search.py @@ -411,14 +411,16 @@ def search(text, start=0, limit=20, doctype=""): :param limit: number of results to return, default 20 :return: Array of result objects """ - from frappe.desk.doctype.global_search_settings.global_search_settings import get_doctypes_for_global_search + from frappe.desk.doctype.global_search_settings.global_search_settings import ( + get_doctypes_for_global_search, + ) results = [] sorted_results = [] allowed_doctypes = get_doctypes_for_global_search() - for text in set(text.split('&')): + for text in set(text.split("&")): text = text.strip() if not text: continue @@ -427,21 +429,24 @@ def search(text, start=0, limit=20, doctype=""): rank = frappe.qb.Match(global_search.content).Against(text).as_("rank") - q = (frappe.qb.from_(global_search) - .select(global_search.doctype,global_search.name,global_search.content,rank) + query = ( + frappe.qb.from_(global_search) + .select( + global_search.doctype, global_search.name, global_search.content, rank + ) .orderby("rank", order=frappe.qb.desc) - .limit(limit)) + .limit(limit) + ) if doctype: - q = q.where(global_search.doctype == doctype) + query = query.where(global_search.doctype == doctype) elif allowed_doctypes: - q = q.where(global_search.doctype.isin(allowed_doctypes)) + query = query.where(global_search.doctype.isin(allowed_doctypes)) - if int(start) > 0: - q = q.offset(int(start)) + if start > 0: + query = query.offset(start) - - result = frappe.db.sql(q.get_sql(),as_dict=True) + result = frappe.db.sql(query, as_dict=True) results.extend(result) @@ -452,7 +457,9 @@ def search(text, start=0, limit=20, doctype=""): try: meta = frappe.get_meta(r.doctype) if meta.image_field: - r.image = frappe.db.get_value(r.doctype, r.name, meta.image_field) + r.image = frappe.db.get_value( + r.doctype, r.name, meta.image_field + ) except Exception: frappe.clear_messages() diff --git a/frappe/website/report/website_analytics/website_analytics.py b/frappe/website/report/website_analytics/website_analytics.py index 683eeba487..2992108e75 100644 --- a/frappe/website/report/website_analytics/website_analytics.py +++ b/frappe/website/report/website_analytics/website_analytics.py @@ -56,17 +56,21 @@ class WebsiteAnalytics(object): ] def get_data(self): - Web_Page_View = frappe.qb.Table("Web Page View") + WebPageView = frappe.qb.Table("Web Page View") count_all = frappe.qb.fn.Count("*").as_("count") - case = frappe.qb.terms.Case().when(Web_Page_View.is_unique == "1", "1") + case = frappe.qb.terms.Case().when(WebPageView.is_unique == "1", "1") count_is_unique = frappe.qb.fn.Count(case).as_("unique_count") curr = ( - frappe.qb.from_(Web_Page_View) + frappe.qb.from_(WebPageView) .select("path", count_all, count_is_unique) - .where(frappe.qb.fn.Coalesce(Web_Page_View.creation, "0001-01-01")[self.filters.from_date:self.filters.to_date]) - .groupby(Web_Page_View.path) - .orderby("count", Order=frappe.qb.desc).get_sql() + .where( + frappe.qb.fn.Coalesce(WebPageView.creation, "0001-01-01")[ + self.filters.from_date : self.filters.to_date + ] + ) + .groupby(WebPageView.path) + .orderby("count", Order=frappe.qb.desc) ) return frappe.db.sql(curr) From 45d1afe739eabead499a989f5474d9ffc6a53d94 Mon Sep 17 00:00:00 2001 From: saxenabhishek Date: Sat, 24 Jul 2021 01:43:18 +0530 Subject: [PATCH 152/164] refactor: move rename, change_column_type, DESC to db --- frappe/core/doctype/doctype/doctype.py | 2 +- frappe/database/database.py | 4 ++ frappe/database/mariadb/database.py | 15 ++++++++ frappe/database/postgres/database.py | 19 +++++++++- frappe/model/__init__.py | 2 +- .../patches/v13_0/increase_password_length.py | 2 +- frappe/query_builder/builder.py | 38 +++---------------- 7 files changed, 44 insertions(+), 38 deletions(-) diff --git a/frappe/core/doctype/doctype/doctype.py b/frappe/core/doctype/doctype/doctype.py index 22be912abc..f868b68cfa 100644 --- a/frappe/core/doctype/doctype/doctype.py +++ b/frappe/core/doctype/doctype/doctype.py @@ -396,7 +396,7 @@ class DocType(Document): frappe.db.sql("""update tabSingles set value=%s where doctype=%s and field='name' and value = %s""", (new, new, old)) else: - frappe.db.sql(frappe.qb.rename_table(old,new)) + frappe.db.rename_table(old,new) frappe.db.commit() # Do not rename and move files and folders for custom doctype diff --git a/frappe/database/database.py b/frappe/database/database.py index bf97317f99..9c84ec3492 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -1042,6 +1042,10 @@ class Database(object): ), tuple(insert_list)) insert_list = [] + @staticmethod + def add_tab(table_name: str) -> str: + return f"tab{table_name}" if not table_name.startswith("__") else table_name + def enqueue_jobs_after_commit(): from frappe.utils.background_jobs import execute_job, get_queue diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index 879c8394d7..f2b3ad718d 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -1,3 +1,5 @@ +from typing import List, Tuple, Union + import pymysql from pymysql.constants import ER, FIELD_TYPE from pymysql.converters import conversions, escape_string @@ -123,6 +125,19 @@ class MariaDBDatabase(Database): def is_type_datetime(code): return code in (pymysql.DATE, pymysql.DATETIME) + def rename_table(self, old_name: str, new_name: str) -> Union[List,Tuple]: + old_name = self.add_tab(old_name) + new_name = self.add_tab(new_name) + return self.sql(f"RENAME TABLE `{old_name}` TO `{new_name}`") + + def DESC(self, doctype: str) -> Union[List,Tuple]: + doctype = self.add_tab(doctype) + return self.sql(f"DESC `{doctype}`") + + def change_column_type(self, table: str, column: str, type: str) -> Union[List,Tuple]: + table = self.add_tab(table) + return self.sql(f"ALTER TABLE `{table}` MODIFY `{column}` {type} NOT NULL") + # exception types @staticmethod def is_deadlocked(e): diff --git a/frappe/database/postgres/database.py b/frappe/database/postgres/database.py index 32cd42eb94..dd967e6ca7 100644 --- a/frappe/database/postgres/database.py +++ b/frappe/database/postgres/database.py @@ -1,12 +1,14 @@ import re -import frappe +from typing import List, Tuple, Union + import psycopg2 import psycopg2.extensions -from frappe.utils import cstr from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT +import frappe from frappe.database.database import Database from frappe.database.postgres.schema import PostgresTable +from frappe.utils import cstr # cast decimals as floats DEC2FLOAT = psycopg2.extensions.new_type( @@ -170,6 +172,19 @@ class PostgresDatabase(Database): def is_data_too_long(e): return e.pgcode == '22001' + def rename_table(self, old_name: str, new_name: str) -> Union[List,Tuple]: + old_name = self.add_tab(old_name) + new_name = self.add_tab(new_name) + return self.sql(f"ALTER TABLE `{old_name}` RENAME TO `{new_name}`") + + def DESC(self, doctype: str)-> Union[List,Tuple]: + doctype = self.add_tab(doctype) + return self.sql(f"SELECT COLUMN_NAME FROM information_schema.COLUMNS WHERE TABLE_NAME = '{doctype}'") + + def change_column_type(self, table: str, column: str, type: str) -> Union[List,Tuple]: + table = self.add_tab(table) + return self.sql(f'ALTER TABLE "{table}" ALTER COLUMN "{column}" TYPE {type}') + def create_auth_table(self): self.sql_ddl("""create table if not exists "__Auth" ( "doctype" VARCHAR(140) NOT NULL, diff --git a/frappe/model/__init__.py b/frappe/model/__init__.py index 1f72272df1..22a735e3f0 100644 --- a/frappe/model/__init__.py +++ b/frappe/model/__init__.py @@ -167,7 +167,7 @@ def delete_fields(args_dict, delete=0): "field": ("in", fields), }) else: - existing_fields = frappe.db.sql(frappe.qb.DESC(dt)) + existing_fields = frappe.db.DESC(dt) existing_fields = existing_fields and [e[0] for e in existing_fields] or [] fields_need_to_delete = set(fields) & set(existing_fields) if not fields_need_to_delete: diff --git a/frappe/patches/v13_0/increase_password_length.py b/frappe/patches/v13_0/increase_password_length.py index 3f0e93e72c..e2a39d6724 100644 --- a/frappe/patches/v13_0/increase_password_length.py +++ b/frappe/patches/v13_0/increase_password_length.py @@ -1,4 +1,4 @@ import frappe def execute(): - frappe.db.sql(frappe.qb.change_table_type(tb = "__Auth",col = "password",type = "TEXT")) + frappe.db.change_column_type(table = "__Auth",column = "password",type = "TEXT") diff --git a/frappe/query_builder/builder.py b/frappe/query_builder/builder.py index db16047ce9..0531f33fff 100644 --- a/frappe/query_builder/builder.py +++ b/frappe/query_builder/builder.py @@ -9,10 +9,10 @@ def get_query_builder(db_type: str) -> Query: """[return the query builder object] Args: - db_type (str): [string value of the db used] + db_type (str): [string value of the db used] Returns: - Query: [Query object] + Query: [Query object] """ if not db_type: db_type = "mariadb" @@ -38,27 +38,12 @@ class MariaDB(common, MySQLQuery): GROUP_CONCAT = SpecialFuncs.GROUP_CONCAT Match = SpecialFuncs.Match - def __init__(self) -> None: - super().__init__() - @classmethod def from_(cls, class_name, *args, **kwargs): if isinstance(class_name, str): class_name = f"tab{class_name}" return super().from_(class_name, *args, **kwargs) - @staticmethod - def rename_table(old_name, new_name): - return f"RENAME TABLE `tab{old_name}` TO `tab{new_name}`" - - @staticmethod - def DESC(dt): - return f"DESC `tab{dt}`" - - @staticmethod - def change_table_type(tb, col, type): - return f"ALTER TABLE `{tb}` MODIFY `{col}` {type} NOT NULL" - class Postgres(common, PostgreSQLQuery): postgres_field = {"table_name": "relname", "table_rows": "n_tup_ins"} @@ -66,9 +51,6 @@ class Postgres(common, PostgreSQLQuery): GROUP_CONCAT = SpecialFuncs.STRING_AGG Match = SpecialFuncs.TO_TSVECTOR - def __init__(self) -> None: - super().__init__() - @classmethod def Field(cls, fieldName, *args, **kwargs): if fieldName in cls.postgres_field: @@ -80,21 +62,11 @@ class Postgres(common, PostgreSQLQuery): if isinstance(class_name, Table): if class_name._schema: if class_name._schema._name == "information_schema": - class_name = cls.information_schema_translation[class_name._table_name] + class_name = cls.information_schema_translation[ + class_name._table_name + ] elif isinstance(class_name, str): class_name = "tab" + class_name return super().from_(class_name, *args, **kwargs) - - @staticmethod - def rename_table(old_name, new_name): - return f"ALTER TABLE `tab{old_name}` RENAME TO `tab{new_name}`" - - @staticmethod - def DESC(dt): - return f"SELECT COLUMN_NAME FROM information_schema.COLUMNS WHERE TABLE_NAME = 'tab{dt}'" - - @staticmethod - def change_table_type(tb, col, type): - return f'ALTER TABLE "{tb}" ALTER COLUMN "{col}" TYPE {type}' From ce10e367251217382ab6a21d0caf047510ef901b Mon Sep 17 00:00:00 2001 From: saxenabhishek Date: Wed, 28 Jul 2021 16:31:41 +0530 Subject: [PATCH 153/164] refactor: move all pypika functions to a module --- .../v12_0/set_correct_assign_value_in_docs.py | 8 +- frappe/query_builder/__init__.py | 3 +- frappe/query_builder/builder.py | 38 ++------- frappe/query_builder/custom.py | 83 +++++++++++++++++++ frappe/query_builder/functions.py | 63 ++++---------- frappe/query_builder/utils.py | 39 +++++++++ frappe/utils/global_search.py | 3 +- .../website_analytics/website_analytics.py | 11 ++- 8 files changed, 161 insertions(+), 87 deletions(-) create mode 100644 frappe/query_builder/custom.py create mode 100644 frappe/query_builder/utils.py diff --git a/frappe/patches/v12_0/set_correct_assign_value_in_docs.py b/frappe/patches/v12_0/set_correct_assign_value_in_docs.py index dd493bc52c..d06d87726b 100644 --- a/frappe/patches/v12_0/set_correct_assign_value_in_docs.py +++ b/frappe/patches/v12_0/set_correct_assign_value_in_docs.py @@ -5,13 +5,15 @@ def execute(): frappe.reload_doc("desk", "doctype", "todo") ToDo = frappe.qb.Table("ToDo") - assignees = frappe.qb.GROUP_CONCAT("owner").distinct().as_("assignees") + from frappe.query_builder.functions import GroupConcat + assignees = GroupConcat("owner").distinct().as_("assignees") + from frappe.query_builder.functions import Coalesce query = ( frappe.qb.from_(ToDo) .select(ToDo.name, ToDo.reference_type, assignees) - .where(frappe.qb.fn.Coalesce(ToDo.reference_type, "") != "") - .where(frappe.qb.fn.Coalesce(ToDo.reference_name, "") != "") + .where(Coalesce(ToDo.reference_type, "") != "") + .where(Coalesce(ToDo.reference_name, "") != "") .where(ToDo.status != "Cancelled") .groupby(ToDo.reference_type, ToDo.reference_name) ) diff --git a/frappe/query_builder/__init__.py b/frappe/query_builder/__init__.py index 00356ab679..34f7d43386 100644 --- a/frappe/query_builder/__init__.py +++ b/frappe/query_builder/__init__.py @@ -1 +1,2 @@ -from frappe.query_builder.builder import get_query_builder +from frappe.query_builder.utils import get_query_builder +import frappe.query_builder.functions diff --git a/frappe/query_builder/builder.py b/frappe/query_builder/builder.py index 0531f33fff..f8cc73ff4b 100644 --- a/frappe/query_builder/builder.py +++ b/frappe/query_builder/builder.py @@ -1,27 +1,7 @@ -from pypika import MySQLQuery, Order, PostgreSQLQuery, Query -from pypika import functions as fn -from pypika import terms +from pypika import MySQLQuery, Order, PostgreSQLQuery, terms from pypika.queries import Schema, Table -import frappe.query_builder.functions as SpecialFuncs - - -def get_query_builder(db_type: str) -> Query: - """[return the query builder object] - - Args: - db_type (str): [string value of the db used] - - Returns: - Query: [Query object] - """ - if not db_type: - db_type = "mariadb" - selecter = {"mariadb": MariaDB, "postgres": Postgres} - return selecter[db_type] - class common: - fn = fn terms = terms desc = Order.desc Schema = Schema @@ -35,8 +15,6 @@ class common: class MariaDB(common, MySQLQuery): Field = terms.Field - GROUP_CONCAT = SpecialFuncs.GROUP_CONCAT - Match = SpecialFuncs.Match @classmethod def from_(cls, class_name, *args, **kwargs): @@ -46,15 +24,13 @@ class MariaDB(common, MySQLQuery): class Postgres(common, PostgreSQLQuery): - postgres_field = {"table_name": "relname", "table_rows": "n_tup_ins"} - information_schema_translation = {"tables": "pg_stat_all_tables"} - GROUP_CONCAT = SpecialFuncs.STRING_AGG - Match = SpecialFuncs.TO_TSVECTOR + field_translation = {"table_name": "relname", "table_rows": "n_tup_ins"} + schema_translation = {"tables": "pg_stat_all_tables"} @classmethod def Field(cls, fieldName, *args, **kwargs): - if fieldName in cls.postgres_field: - fieldName = cls.postgres_field[fieldName] + if fieldName in cls.field_translation: + fieldName = cls.field_translation[fieldName] return terms.Field(fieldName, *args, **kwargs) @classmethod @@ -62,11 +38,11 @@ class Postgres(common, PostgreSQLQuery): if isinstance(class_name, Table): if class_name._schema: if class_name._schema._name == "information_schema": - class_name = cls.information_schema_translation[ + class_name = cls.schema_translation[ class_name._table_name ] elif isinstance(class_name, str): - class_name = "tab" + class_name + class_name = f"tab{class_name}" return super().from_(class_name, *args, **kwargs) diff --git a/frappe/query_builder/custom.py b/frappe/query_builder/custom.py new file mode 100644 index 0000000000..5aaed463d9 --- /dev/null +++ b/frappe/query_builder/custom.py @@ -0,0 +1,83 @@ +from typing import Optional + +from pypika.functions import DistinctOptionFunction +from pypika.utils import builder + +import frappe + + +class GROUP_CONCAT(DistinctOptionFunction): + def __init__(self, column: str, alias: Optional[str] = None): + """[ Implements the group concat function read more about it at https://www.geeksforgeeks.org/mysql-group_concat-function ] + Args: + column (str): [ name of the column you want to concat] + alias (Optional[str], optional): [ is this an alias? ]. Defaults to None. + """ + super(GROUP_CONCAT, self).__init__("GROUP_CONCAT", column, alias=alias) + + +class STRING_AGG(DistinctOptionFunction): + def __init__(self, column: str, separator: str = ",", alias: Optional[str] = None): + """[ Implements the group concat function read more about it at https://docs.microsoft.com/en-us/sql/t-sql/functions/string-agg-transact-sql?view=sql-server-ver15 ] + + Args: + column (str): [ name of the column you want to concat ] + separator (str, optional): [separator to be used]. Defaults to ",". + alias (Optional[str], optional): [description]. Defaults to None. + """ + super(STRING_AGG, self).__init__("STRING_AGG", column, separator, alias=alias) + + +class MATCH(DistinctOptionFunction): + def __init__(self, column: str, *args, **kwargs): + """[ Implementation of Match Against read more about it https://dev.mysql.com/doc/refman/8.0/en/fulltext-search.html#function_match ] + + Args: + column (str):[ column to search in ] + """ + alias = kwargs.get("alias") + super(MATCH, self).__init__(" MATCH", column, *args, alias=alias) + self._Against = False + + def get_function_sql(self, **kwargs): + s = super(DistinctOptionFunction, self).get_function_sql(**kwargs) + + if self._Against: + return f"{s} AGAINST ({frappe.db.escape(f'+{self._Against}*')} IN BOOLEAN MODE)" + return s + + @builder + def Against(self, text: str): + """[ Text that has to be searched against ] + + Args: + text (str): [ the text string that we match it against ] + """ + self._Against = text + + +class TO_TSVECTOR(DistinctOptionFunction): + def __init__(self, column: str, *args, **kwargs): + """[ Implementation of TO_TSVECTOR read more about it https://www.postgresql.org/docs/9.1/textsearch-controls.html] + + Args: + column (str): [ column to search in ] + """ + alias = kwargs.get("alias") + super(TO_TSVECTOR, self).__init__("TO_TSVECTOR", column, *args, alias=alias) + self._PLAINTO_TSQUERY = False + + def get_function_sql(self, **kwargs): + s = super(DistinctOptionFunction, self).get_function_sql(**kwargs) + if self._PLAINTO_TSQUERY: + return f"{s} @@ PLAINTO_TSQUERY({frappe.db.escape(self._PLAINTO_TSQUERY)})" + return s + + @builder + def Against(self, text: str): + """[ Text that has to be searched against ] + + Args: + text (str): [ the text string that we match it against ] + """ + self._PLAINTO_TSQUERY = text diff --git a/frappe/query_builder/functions.py b/frappe/query_builder/functions.py index 3b45d78726..119c4e6c4f 100644 --- a/frappe/query_builder/functions.py +++ b/frappe/query_builder/functions.py @@ -1,50 +1,17 @@ -from pypika import functions as fn -from pypika.utils import builder +from pypika.functions import * +from frappe.query_builder.utils import ImportMapper, db_type +from frappe.query_builder.custom import GROUP_CONCAT, STRING_AGG, MATCH, TO_TSVECTOR +GroupConcat = ImportMapper( + { + db_type.MARIADB: GROUP_CONCAT, + db_type.POSTGRES: STRING_AGG + } +) -class GROUP_CONCAT(fn.DistinctOptionFunction): - def __init__(self, col, alias=None): - super(GROUP_CONCAT, self).__init__("GROUP_CONCAT", col, alias=alias) - - -class STRING_AGG(fn.DistinctOptionFunction): - def __init__(self, col, val=",", alias=None): - super(STRING_AGG, self).__init__("STRING_AGG", col, val, alias=alias) - - -class Match(fn.DistinctOptionFunction): - def __init__(self, col, *args, **kwargs): - alias = kwargs.get("alias") - super(Match, self).__init__(" MATCH", col, *args, alias=alias) - self._Against = False - - def get_function_sql(self, **kwargs): - s = super(fn.DistinctOptionFunction, self).get_function_sql(**kwargs) - - # n = len(self.name) + 1 - if self._Against: - return s + f" AGAINST ('+{self._Against}*' IN BOOLEAN MODE)" - return s - - @builder - def Against(self, b): - self._Against = b - - -class TO_TSVECTOR(fn.DistinctOptionFunction): - def __init__(self, col, *args, **kwargs): - alias = kwargs.get("alias") - super(TO_TSVECTOR, self).__init__("TO_TSVECTOR", col, *args, alias=alias) - self._PLAINTO_TSQUERY = False - - def get_function_sql(self, **kwargs): - s = super(fn.DistinctOptionFunction, self).get_function_sql(**kwargs) - - # n = len(self.name) + 1 - if self._PLAINTO_TSQUERY: - return s + f" @@ PLAINTO_TSQUERY('{self._PLAINTO_TSQUERY}')" - return s - - @builder - def Against(self, b): - self._PLAINTO_TSQUERY = b +Match = ImportMapper( + { + db_type.MARIADB: MATCH, + db_type.POSTGRES: TO_TSVECTOR + } +) diff --git a/frappe/query_builder/utils.py b/frappe/query_builder/utils.py new file mode 100644 index 0000000000..d8e5698e66 --- /dev/null +++ b/frappe/query_builder/utils.py @@ -0,0 +1,39 @@ +from enum import Enum +from typing import Any, Callable, Dict, Optional + +from pypika import Query + +import frappe +from .builder import MariaDB, Postgres + + +class db_type(Enum): + MARIADB = "mariadb" + POSTGRES = "postgres" + + +class ImportMapper: + def __init__(self, func_map: Dict[db_type, Callable]) -> None: + self.func_map = func_map + + def __call__(self, *args: Any, **kwds: Any) -> Callable: + db = db_type.MARIADB + if frappe.conf.db_type: + db = db_type(frappe.conf.db_type) + return self.func_map[db](*args, **kwds) + + +def get_query_builder(type_of_db: Optional[str]) -> Query: + """[return the query builder object] + + Args: + type_of_db (str): [string value of the db used] + + Returns: + Query: [Query object] + """ + db = db_type["MARIADB"] + if type_of_db: + db = db_type(type_of_db) + selecter = {db_type.MARIADB: MariaDB, db_type.POSTGRES: Postgres} + return selecter[db] diff --git a/frappe/utils/global_search.py b/frappe/utils/global_search.py index 7754c87823..e6e0bcb141 100644 --- a/frappe/utils/global_search.py +++ b/frappe/utils/global_search.py @@ -427,7 +427,8 @@ def search(text, start=0, limit=20, doctype=""): global_search = frappe.qb.Table("__global_search") - rank = frappe.qb.Match(global_search.content).Against(text).as_("rank") + from frappe.query_builder.functions import Match + rank = Match(global_search.content).Against(text).as_("rank") query = ( frappe.qb.from_(global_search) diff --git a/frappe/website/report/website_analytics/website_analytics.py b/frappe/website/report/website_analytics/website_analytics.py index 2992108e75..74e22d3dbb 100644 --- a/frappe/website/report/website_analytics/website_analytics.py +++ b/frappe/website/report/website_analytics/website_analytics.py @@ -57,15 +57,20 @@ class WebsiteAnalytics(object): def get_data(self): WebPageView = frappe.qb.Table("Web Page View") - count_all = frappe.qb.fn.Count("*").as_("count") + + from frappe.query_builder.functions import Count + + count_all = Count("*").as_("count") case = frappe.qb.terms.Case().when(WebPageView.is_unique == "1", "1") - count_is_unique = frappe.qb.fn.Count(case).as_("unique_count") + count_is_unique = Count(case).as_("unique_count") + + from frappe.query_builder.functions import Coalesce curr = ( frappe.qb.from_(WebPageView) .select("path", count_all, count_is_unique) .where( - frappe.qb.fn.Coalesce(WebPageView.creation, "0001-01-01")[ + Coalesce(WebPageView.creation, "0001-01-01")[ self.filters.from_date : self.filters.to_date ] ) From 681d0ab995ff9ea599b6e50c05dbabcee68614b4 Mon Sep 17 00:00:00 2001 From: saxenabhishek Date: Thu, 29 Jul 2021 02:33:44 +0530 Subject: [PATCH 154/164] test: for qb Match and GroupConcat --- frappe/query_builder/utils.py | 3 +-- frappe/tests/test_query_builder.py | 35 ++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 frappe/tests/test_query_builder.py diff --git a/frappe/query_builder/utils.py b/frappe/query_builder/utils.py index d8e5698e66..e2276e15c0 100644 --- a/frappe/query_builder/utils.py +++ b/frappe/query_builder/utils.py @@ -11,7 +11,6 @@ class db_type(Enum): MARIADB = "mariadb" POSTGRES = "postgres" - class ImportMapper: def __init__(self, func_map: Dict[db_type, Callable]) -> None: self.func_map = func_map @@ -32,7 +31,7 @@ def get_query_builder(type_of_db: Optional[str]) -> Query: Returns: Query: [Query object] """ - db = db_type["MARIADB"] + db = db_type.MARIADB if type_of_db: db = db_type(type_of_db) selecter = {db_type.MARIADB: MariaDB, db_type.POSTGRES: Postgres} diff --git a/frappe/tests/test_query_builder.py b/frappe/tests/test_query_builder.py new file mode 100644 index 0000000000..6cbd3b2950 --- /dev/null +++ b/frappe/tests/test_query_builder.py @@ -0,0 +1,35 @@ +import unittest +from typing import Callable + +import frappe +from frappe.query_builder.functions import GroupConcat, Match +from frappe.query_builder.utils import db_type + + +def CheckDB(dbtype: db_type) -> Callable: + return unittest.skipIf( + db_type(frappe.conf.db_type) != dbtype, f"Only runs for{db_type}" + ) + +@CheckDB(dbtype=db_type.MARIADB) +class TestCustomFunctionsMariaDB(unittest.TestCase): + def test_concat(self): + self.assertEqual("GROUP_CONCAT('Notes')", GroupConcat("Notes").get_sql()) + + def test_match(self): + query = Match("Notes").Against("text") + self.assertEqual( + " MATCH('Notes') AGAINST ('+text*' IN BOOLEAN MODE)", query.get_sql() + ) + + +@CheckDB(dbtype=db_type.POSTGRES) +class TestCustomFunctionsPostgres(unittest.TestCase): + def test_concat(self): + self.assertEqual("STRING_AGG('Notes',',')", GroupConcat("Notes").get_sql()) + + def test_match(self): + query = Match("Notes").Against("text") + self.assertEqual( + "TO_TSVECTOR('Notes') @@ PLAINTO_TSQUERY('text')", query.get_sql() + ) From 639b666a0de55c79cfd65192446fd792d9f0c65e Mon Sep 17 00:00:00 2001 From: saxenabhishek Date: Thu, 29 Jul 2021 03:57:03 +0530 Subject: [PATCH 155/164] test: for qb builder classes --- frappe/query_builder/builder.py | 42 ++++++++++++++---------------- frappe/tests/test_query_builder.py | 41 ++++++++++++++++++++++++++++- 2 files changed, 60 insertions(+), 23 deletions(-) diff --git a/frappe/query_builder/builder.py b/frappe/query_builder/builder.py index f8cc73ff4b..4878212aa2 100644 --- a/frappe/query_builder/builder.py +++ b/frappe/query_builder/builder.py @@ -7,20 +7,20 @@ class common: Schema = Schema @staticmethod - def Table(classname: str, *args, **kwargs) -> Table: - if not classname.startswith("__"): - classname = f"tab{classname}" - return Table(classname, *args, **kwargs) + def Table(class_name: str, *args, **kwargs) -> Table: + if not class_name.startswith("__"): + class_name = f"tab{class_name}" + return Table(class_name, *args, **kwargs) class MariaDB(common, MySQLQuery): Field = terms.Field @classmethod - def from_(cls, class_name, *args, **kwargs): - if isinstance(class_name, str): - class_name = f"tab{class_name}" - return super().from_(class_name, *args, **kwargs) + def from_(cls, table, *args, **kwargs): + if isinstance(table, str): + table = cls.Table(table) + return super().from_(table, *args, **kwargs) class Postgres(common, PostgreSQLQuery): @@ -28,21 +28,19 @@ class Postgres(common, PostgreSQLQuery): schema_translation = {"tables": "pg_stat_all_tables"} @classmethod - def Field(cls, fieldName, *args, **kwargs): - if fieldName in cls.field_translation: - fieldName = cls.field_translation[fieldName] - return terms.Field(fieldName, *args, **kwargs) + def Field(cls, field_name, *args, **kwargs): + if field_name in cls.field_translation: + field_name = cls.field_translation[field_name] + return terms.Field(field_name, *args, **kwargs) @classmethod - def from_(cls, class_name, *args, **kwargs): - if isinstance(class_name, Table): - if class_name._schema: - if class_name._schema._name == "information_schema": - class_name = cls.schema_translation[ - class_name._table_name - ] + def from_(cls, table, *args, **kwargs): + if isinstance(table, Table): + if table._schema: + if table._schema._name == "information_schema": + table = cls.schema_translation[table._table_name] - elif isinstance(class_name, str): - class_name = f"tab{class_name}" + elif isinstance(table, str): + table = cls.Table(table) - return super().from_(class_name, *args, **kwargs) + return super().from_(table, *args, **kwargs) diff --git a/frappe/tests/test_query_builder.py b/frappe/tests/test_query_builder.py index 6cbd3b2950..b5e520ce30 100644 --- a/frappe/tests/test_query_builder.py +++ b/frappe/tests/test_query_builder.py @@ -8,9 +8,10 @@ from frappe.query_builder.utils import db_type def CheckDB(dbtype: db_type) -> Callable: return unittest.skipIf( - db_type(frappe.conf.db_type) != dbtype, f"Only runs for{db_type}" + db_type(frappe.conf.db_type) != dbtype, f"Only runs for {dbtype.value}" ) + @CheckDB(dbtype=db_type.MARIADB) class TestCustomFunctionsMariaDB(unittest.TestCase): def test_concat(self): @@ -33,3 +34,41 @@ class TestCustomFunctionsPostgres(unittest.TestCase): self.assertEqual( "TO_TSVECTOR('Notes') @@ PLAINTO_TSQUERY('text')", query.get_sql() ) + + +class TestBuilderBase(object): + def test_adding_tabs(self): + self.assertEqual("tabNotes", frappe.qb.Table("Notes").get_sql()) + self.assertEqual("__Auth", frappe.qb.Table("__Auth").get_sql()) + + +@CheckDB(dbtype=db_type.MARIADB) +class TestBuilderMaria(unittest.TestCase, TestBuilderBase): + def test_adding_tabs_in_from(self): + self.assertEqual( + "SELECT * FROM `tabNotes`", frappe.qb.from_("Notes").select("*").get_sql() + ) + self.assertEqual( + "SELECT * FROM `__Auth`", frappe.qb.from_("__Auth").select("*").get_sql() + ) + + +@CheckDB(dbtype=db_type.POSTGRES) +class TestBuilderPostgres(unittest.TestCase, TestBuilderBase): + def test_adding_tabs_in_from(self): + self.assertEqual( + 'SELECT * FROM "tabNotes"', frappe.qb.from_("Notes").select("*").get_sql() + ) + self.assertEqual( + 'SELECT * FROM "__Auth"', frappe.qb.from_("__Auth").select("*").get_sql() + ) + + def test_replace_tables(self): + info_schema = frappe.qb.Schema("information_schema") + self.assertEqual( + 'SELECT * FROM "pg_stat_all_tables"', + frappe.qb.from_(info_schema.tables).select("*").get_sql(), + ) + + def test_replace_fields_post(self): + self.assertEqual("relname", frappe.qb.Field("table_name").get_sql()) From 9423861c340d996964df3dcdda70fb1266e2d8d3 Mon Sep 17 00:00:00 2001 From: saxenabhishek Date: Thu, 29 Jul 2021 16:30:23 +0530 Subject: [PATCH 156/164] test: for db rename, change_column_type, describe --- frappe/tests/test_db.py | 99 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 98 insertions(+), 1 deletion(-) diff --git a/frappe/tests/test_db.py b/frappe/tests/test_db.py index 04c9a525b1..efb782bd3e 100644 --- a/frappe/tests/test_db.py +++ b/frappe/tests/test_db.py @@ -12,6 +12,8 @@ from frappe.custom.doctype.custom_field.custom_field import create_custom_field from frappe.utils import random_string from frappe.utils.testutils import clear_custom_fields +from .test_query_builder import CheckDB, db_type + class TestDB(unittest.TestCase): def test_get_value(self): @@ -146,7 +148,7 @@ class TestDB(unittest.TestCase): # Create documents under that doctype and query them via ORM for _ in range(10): - docfields = { key.lower(): random_string(10) for key in fields } + docfields = {key.lower(): random_string(10) for key in fields} doc = frappe.get_doc({"doctype": test_doctype, "description": random_string(20), **docfields}) doc.insert() created_docs.append(doc.name) @@ -189,3 +191,98 @@ class TestDB(unittest.TestCase): for doc in created_docs: frappe.delete_doc(test_doctype, doc) clear_custom_fields(test_doctype) + +@CheckDB(db_type.MARIADB) +class TestDDLCommandsMaria(unittest.TestCase): + test_table_name = "TestNotes" + + def setUp(self) -> None: + frappe.db.commit() + frappe.db.sql( + f""" + CREATE TABLE `tab{self.test_table_name}` (`id` INT NULL,PRIMARY KEY (`id`)); + """ + ) + + def tearDown(self) -> None: + frappe.db.sql(f"DROP TABLE tab{self.test_table_name};") + self.test_table_name = "TestNotes" + + def test_rename(self) -> None: + new_table_name = f"{self.test_table_name}_new" + frappe.db.rename_table(self.test_table_name, new_table_name) + check_exists = frappe.db.sql( + f""" + SELECT * FROM INFORMATION_SCHEMA.TABLES + WHERE TABLE_NAME = N'tab{new_table_name}'; + """ + ) + self.assertGreater(len(check_exists), 0) + self.assertIn(f"tab{new_table_name}", check_exists[0]) + + # * so this table is deleted after the rename + self.test_table_name = new_table_name + + def test_describe(self) -> None: + self.assertEqual( + (("id", "int(11)", "NO", "PRI", None, ""),), + frappe.db.DESC(self.test_table_name), + ) + + def test_change_type(self) -> None: + frappe.db.change_column_type("TestNotes", "id", "varchar(255)") + test_table_description = frappe.db.sql(f"DESC tab{self.test_table_name};") + self.assertGreater(len(test_table_description), 0) + self.assertIn("varchar(255)", test_table_description[0]) + + +@CheckDB(dbtype=db_type.POSTGRES) +class TestDDLCommandsPost(unittest.TestCase): + test_table_name = "TestNotes" + + def setUp(self) -> None: + frappe.db.sql( + f""" + CREATE TABLE "tab{self.test_table_name}" ("id" INT NULL,PRIMARY KEY ("id")) + """ + ) + + def tearDown(self) -> None: + frappe.db.sql(f'DROP TABLE "tab{self.test_table_name}"') + self.test_table_name = "TestNotes" + + def test_rename(self) -> None: + new_table_name = f"{self.test_table_name}_new" + frappe.db.rename_table(self.test_table_name, new_table_name) + check_exists = frappe.db.sql( + f""" + SELECT EXISTS ( + SELECT FROM information_schema.tables + WHERE table_name = 'tab{new_table_name}' + ); + """ + ) + self.assertTrue(check_exists[0][0]) + + # * so this table is deleted after the rename + self.test_table_name = new_table_name + + def test_describe(self) -> None: + self.assertEqual([("id",)], frappe.db.DESC(self.test_table_name)) + + def test_change_type(self) -> None: + frappe.db.change_column_type(self.test_table_name, "id", "varchar(255)") + check_change = frappe.db.sql( + f""" + SELECT + table_name, + column_name, + data_type + FROM + information_schema.columns + WHERE + table_name = 'tab{self.test_table_name}' + """ + ) + self.assertGreater(len(check_change), 0) + self.assertIn("character varying", check_change[0]) From 7671e784872ca64437738f7b6d794aab68556592 Mon Sep 17 00:00:00 2001 From: saxenabhishek Date: Fri, 30 Jul 2021 10:48:17 +0530 Subject: [PATCH 157/164] style: minor fixes and improvements --- frappe/config/__init__.py | 2 +- frappe/core/doctype/doctype/doctype.py | 2 +- frappe/database/mariadb/database.py | 6 +++--- frappe/database/postgres/database.py | 6 +++--- frappe/model/__init__.py | 2 +- .../v12_0/set_correct_assign_value_in_docs.py | 4 +--- .../patches/v13_0/increase_password_length.py | 2 +- frappe/query_builder/__init__.py | 1 - frappe/query_builder/builder.py | 15 ++++++++++++--- frappe/tests/test_db.py | 4 ++-- frappe/tests/test_query_builder.py | 1 + frappe/utils/global_search.py | 4 +--- .../website_analytics/website_analytics.py | 18 +++++++----------- 13 files changed, 34 insertions(+), 33 deletions(-) diff --git a/frappe/config/__init__.py b/frappe/config/__init__.py index 47f1849a43..caf169404d 100644 --- a/frappe/config/__init__.py +++ b/frappe/config/__init__.py @@ -45,7 +45,7 @@ def get_all_empty_tables_by_module(): query = frappe.qb.from_(information_schema.tables).select(table_name).where(table_rows == 0) - empty_tables = set(r[0] for r in frappe.db.sql(query)) + empty_tables = {(r[0] for r in frappe.db.sql(query))} results = frappe.get_all("DocType", fields=["name", "module"]) empty_tables_by_module = {} diff --git a/frappe/core/doctype/doctype/doctype.py b/frappe/core/doctype/doctype/doctype.py index f868b68cfa..c8efda6a55 100644 --- a/frappe/core/doctype/doctype/doctype.py +++ b/frappe/core/doctype/doctype/doctype.py @@ -396,7 +396,7 @@ class DocType(Document): frappe.db.sql("""update tabSingles set value=%s where doctype=%s and field='name' and value = %s""", (new, new, old)) else: - frappe.db.rename_table(old,new) + frappe.db.rename_table(old, new) frappe.db.commit() # Do not rename and move files and folders for custom doctype diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index f2b3ad718d..d218ea0338 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -125,16 +125,16 @@ class MariaDBDatabase(Database): def is_type_datetime(code): return code in (pymysql.DATE, pymysql.DATETIME) - def rename_table(self, old_name: str, new_name: str) -> Union[List,Tuple]: + def rename_table(self, old_name: str, new_name: str) -> Union[List, Tuple]: old_name = self.add_tab(old_name) new_name = self.add_tab(new_name) return self.sql(f"RENAME TABLE `{old_name}` TO `{new_name}`") - def DESC(self, doctype: str) -> Union[List,Tuple]: + def describe(self, doctype: str) -> Union[List, Tuple]: doctype = self.add_tab(doctype) return self.sql(f"DESC `{doctype}`") - def change_column_type(self, table: str, column: str, type: str) -> Union[List,Tuple]: + def change_column_type(self, table: str, column: str, type: str) -> Union[List, Tuple]: table = self.add_tab(table) return self.sql(f"ALTER TABLE `{table}` MODIFY `{column}` {type} NOT NULL") diff --git a/frappe/database/postgres/database.py b/frappe/database/postgres/database.py index dd967e6ca7..9c48b2960d 100644 --- a/frappe/database/postgres/database.py +++ b/frappe/database/postgres/database.py @@ -172,16 +172,16 @@ class PostgresDatabase(Database): def is_data_too_long(e): return e.pgcode == '22001' - def rename_table(self, old_name: str, new_name: str) -> Union[List,Tuple]: + def rename_table(self, old_name: str, new_name: str) -> Union[List, Tuple]: old_name = self.add_tab(old_name) new_name = self.add_tab(new_name) return self.sql(f"ALTER TABLE `{old_name}` RENAME TO `{new_name}`") - def DESC(self, doctype: str)-> Union[List,Tuple]: + def describe(self, doctype: str)-> Union[List, Tuple]: doctype = self.add_tab(doctype) return self.sql(f"SELECT COLUMN_NAME FROM information_schema.COLUMNS WHERE TABLE_NAME = '{doctype}'") - def change_column_type(self, table: str, column: str, type: str) -> Union[List,Tuple]: + def change_column_type(self, table: str, column: str, type: str) -> Union[List, Tuple]: table = self.add_tab(table) return self.sql(f'ALTER TABLE "{table}" ALTER COLUMN "{column}" TYPE {type}') diff --git a/frappe/model/__init__.py b/frappe/model/__init__.py index 22a735e3f0..4aa8fb9000 100644 --- a/frappe/model/__init__.py +++ b/frappe/model/__init__.py @@ -167,7 +167,7 @@ def delete_fields(args_dict, delete=0): "field": ("in", fields), }) else: - existing_fields = frappe.db.DESC(dt) + existing_fields = frappe.db.describe(dt) existing_fields = existing_fields and [e[0] for e in existing_fields] or [] fields_need_to_delete = set(fields) & set(existing_fields) if not fields_need_to_delete: diff --git a/frappe/patches/v12_0/set_correct_assign_value_in_docs.py b/frappe/patches/v12_0/set_correct_assign_value_in_docs.py index d06d87726b..90766b5f64 100644 --- a/frappe/patches/v12_0/set_correct_assign_value_in_docs.py +++ b/frappe/patches/v12_0/set_correct_assign_value_in_docs.py @@ -1,14 +1,12 @@ import frappe - +from frappe.query_builder.functions import GroupConcat, Coalesce def execute(): frappe.reload_doc("desk", "doctype", "todo") ToDo = frappe.qb.Table("ToDo") - from frappe.query_builder.functions import GroupConcat assignees = GroupConcat("owner").distinct().as_("assignees") - from frappe.query_builder.functions import Coalesce query = ( frappe.qb.from_(ToDo) .select(ToDo.name, ToDo.reference_type, assignees) diff --git a/frappe/patches/v13_0/increase_password_length.py b/frappe/patches/v13_0/increase_password_length.py index e2a39d6724..62ca2ed779 100644 --- a/frappe/patches/v13_0/increase_password_length.py +++ b/frappe/patches/v13_0/increase_password_length.py @@ -1,4 +1,4 @@ import frappe def execute(): - frappe.db.change_column_type(table = "__Auth",column = "password",type = "TEXT") + frappe.db.change_column_type(table="__Auth", column="password", type="TEXT") diff --git a/frappe/query_builder/__init__.py b/frappe/query_builder/__init__.py index 34f7d43386..798c34b6cc 100644 --- a/frappe/query_builder/__init__.py +++ b/frappe/query_builder/__init__.py @@ -1,2 +1 @@ from frappe.query_builder.utils import get_query_builder -import frappe.query_builder.functions diff --git a/frappe/query_builder/builder.py b/frappe/query_builder/builder.py index 4878212aa2..d0f452e781 100644 --- a/frappe/query_builder/builder.py +++ b/frappe/query_builder/builder.py @@ -1,7 +1,8 @@ from pypika import MySQLQuery, Order, PostgreSQLQuery, terms from pypika.queries import Schema, Table -class common: + +class Base: terms = terms desc = Order.desc Schema = Schema @@ -13,7 +14,7 @@ class common: return Table(class_name, *args, **kwargs) -class MariaDB(common, MySQLQuery): +class MariaDB(Base, MySQLQuery): Field = terms.Field @classmethod @@ -23,9 +24,17 @@ class MariaDB(common, MySQLQuery): return super().from_(table, *args, **kwargs) -class Postgres(common, PostgreSQLQuery): +class Postgres(Base, PostgreSQLQuery): field_translation = {"table_name": "relname", "table_rows": "n_tup_ins"} schema_translation = {"tables": "pg_stat_all_tables"} + # TODO: Find a better way to do this + # These are interdependent query changes that need fixing. These + # translations happen in the same query. But there is no check to see if + # the Fields are changed only when a particular `information_schema` schema + # is used. Replacing them is not straightforward because the "from_" + # function can not see the arguments passed to the "select" function as + # they are two different objects. The quick fix used here is to replace the + # Field names in the "Field" function. @classmethod def Field(cls, field_name, *args, **kwargs): diff --git a/frappe/tests/test_db.py b/frappe/tests/test_db.py index efb782bd3e..9839df1197 100644 --- a/frappe/tests/test_db.py +++ b/frappe/tests/test_db.py @@ -226,7 +226,7 @@ class TestDDLCommandsMaria(unittest.TestCase): def test_describe(self) -> None: self.assertEqual( (("id", "int(11)", "NO", "PRI", None, ""),), - frappe.db.DESC(self.test_table_name), + frappe.db.describe(self.test_table_name), ) def test_change_type(self) -> None: @@ -268,7 +268,7 @@ class TestDDLCommandsPost(unittest.TestCase): self.test_table_name = new_table_name def test_describe(self) -> None: - self.assertEqual([("id",)], frappe.db.DESC(self.test_table_name)) + self.assertEqual([("id",)], frappe.db.describe(self.test_table_name)) def test_change_type(self) -> None: frappe.db.change_column_type(self.test_table_name, "id", "varchar(255)") diff --git a/frappe/tests/test_query_builder.py b/frappe/tests/test_query_builder.py index b5e520ce30..d5a74a33bd 100644 --- a/frappe/tests/test_query_builder.py +++ b/frappe/tests/test_query_builder.py @@ -4,6 +4,7 @@ from typing import Callable import frappe from frappe.query_builder.functions import GroupConcat, Match from frappe.query_builder.utils import db_type +from frappe.query_builder.functions import GroupConcat, Match def CheckDB(dbtype: db_type) -> Callable: diff --git a/frappe/utils/global_search.py b/frappe/utils/global_search.py index e6e0bcb141..072e3a7c62 100644 --- a/frappe/utils/global_search.py +++ b/frappe/utils/global_search.py @@ -414,6 +414,7 @@ def search(text, start=0, limit=20, doctype=""): from frappe.desk.doctype.global_search_settings.global_search_settings import ( get_doctypes_for_global_search, ) + from frappe.query_builder.functions import Match results = [] sorted_results = [] @@ -426,10 +427,7 @@ def search(text, start=0, limit=20, doctype=""): continue global_search = frappe.qb.Table("__global_search") - - from frappe.query_builder.functions import Match rank = Match(global_search.content).Against(text).as_("rank") - query = ( frappe.qb.from_(global_search) .select( diff --git a/frappe/website/report/website_analytics/website_analytics.py b/frappe/website/report/website_analytics/website_analytics.py index 74e22d3dbb..6cc0fb7f97 100644 --- a/frappe/website/report/website_analytics/website_analytics.py +++ b/frappe/website/report/website_analytics/website_analytics.py @@ -1,11 +1,14 @@ # Copyright (c) 2013, Frappe Technologies and contributors # For license information, please see license.txt -import frappe from datetime import datetime + +import frappe +from frappe.query_builder.functions import Coalesce, Count from frappe.utils import getdate from frappe.utils.dateutils import get_dates_from_timegrain + def execute(filters=None): return WebsiteAnalytics(filters).run() @@ -57,27 +60,20 @@ class WebsiteAnalytics(object): def get_data(self): WebPageView = frappe.qb.Table("Web Page View") - - from frappe.query_builder.functions import Count - count_all = Count("*").as_("count") case = frappe.qb.terms.Case().when(WebPageView.is_unique == "1", "1") count_is_unique = Count(case).as_("unique_count") - from frappe.query_builder.functions import Coalesce - - curr = ( + query = ( frappe.qb.from_(WebPageView) .select("path", count_all, count_is_unique) .where( - Coalesce(WebPageView.creation, "0001-01-01")[ - self.filters.from_date : self.filters.to_date - ] + Coalesce(WebPageView.creation, "0001-01-01")[self.filters.from_date:self.filters.to_date] ) .groupby(WebPageView.path) .orderby("count", Order=frappe.qb.desc) ) - return frappe.db.sql(curr) + return frappe.db.sql(query) def _get_query_for_mariadb(self): filters_range = self.filters.range From a017c89afc4253e2d9ff04514fd9a05f7b5beede Mon Sep 17 00:00:00 2001 From: hasnain2808 Date: Fri, 30 Jul 2021 11:31:55 +0530 Subject: [PATCH 158/164] fix: validation for website search field --- frappe/core/doctype/doctype/doctype.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/frappe/core/doctype/doctype/doctype.py b/frappe/core/doctype/doctype/doctype.py index 3cdc45ea08..fac0fc7e2d 100644 --- a/frappe/core/doctype/doctype/doctype.py +++ b/frappe/core/doctype/doctype/doctype.py @@ -927,6 +927,13 @@ def validate_fields(meta): if meta.is_published_field not in fieldname_list: frappe.throw(_("Is Published Field must be a valid fieldname"), InvalidFieldNameError) + def check_website_search_field(meta): + if not meta.website_search_field: + return + + if meta.website_search_field not in fieldname_list: + frappe.throw(_("Website Search Field must be a valid fieldname"), InvalidFieldNameError) + def check_timeline_field(meta): if not meta.timeline_field: return @@ -1046,6 +1053,7 @@ def validate_fields(meta): check_title_field(meta) check_timeline_field(meta) check_is_published_field(meta) + check_website_search_field(meta) check_sort_field(meta) check_image_field(meta) From 42dc8a180f3dacd3a6024c90bd55740248a32372 Mon Sep 17 00:00:00 2001 From: saxenabhishek Date: Fri, 30 Jul 2021 11:48:12 +0530 Subject: [PATCH 159/164] refactor: get_table_name, run_only_if --- frappe/__init__.py | 2 +- frappe/database/database.py | 7 ++----- frappe/database/mariadb/database.py | 14 +++++++------- frappe/database/postgres/database.py | 14 +++++++------- frappe/query_builder/builder.py | 8 ++++---- frappe/query_builder/functions.py | 10 +++++----- frappe/query_builder/utils.py | 20 ++++++++------------ frappe/tests/test_db.py | 6 +++--- frappe/tests/test_query_builder.py | 15 +++++++-------- frappe/utils/__init__.py | 3 +++ 10 files changed, 47 insertions(+), 52 deletions(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index 24faf840bd..b4728f9ac3 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -205,7 +205,7 @@ def init(site, sites_path=None, new_site=False): local.form_dict = _dict() local.session = _dict() local.dev_server = _dev_server - local.qb = get_query_builder(local.conf.db_type) + local.qb = get_query_builder(local.conf.db_type or "mariadb") setup_module_map() diff --git a/frappe/database/database.py b/frappe/database/database.py index 9c84ec3492..b1dec95139 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -14,7 +14,7 @@ import frappe.model.meta from frappe import _ from time import time -from frappe.utils import now, getdate, cast_fieldtype, get_datetime +from frappe.utils import now, getdate, cast_fieldtype, get_datetime, get_table_name from frappe.model.utils.link_count import flush_local_link_count @@ -961,7 +961,7 @@ class Database(object): """ values = () filters = filters or kwargs.get("conditions") - table = doctype if doctype.startswith("__") else f"tab{doctype}" + table = get_table_name(doctype) query = f"DELETE FROM `{table}`" if "debug" not in kwargs: @@ -1042,9 +1042,6 @@ class Database(object): ), tuple(insert_list)) insert_list = [] - @staticmethod - def add_tab(table_name: str) -> str: - return f"tab{table_name}" if not table_name.startswith("__") else table_name def enqueue_jobs_after_commit(): from frappe.utils.background_jobs import execute_job, get_queue diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index d218ea0338..5dd6d9e58a 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -7,7 +7,7 @@ from pymysql.converters import conversions, escape_string import frappe from frappe.database.database import Database from frappe.database.mariadb.schema import MariaDBTable -from frappe.utils import UnicodeWithAttrs, cstr, get_datetime +from frappe.utils import UnicodeWithAttrs, cstr, get_datetime, get_table_name class MariaDBDatabase(Database): @@ -126,17 +126,17 @@ class MariaDBDatabase(Database): return code in (pymysql.DATE, pymysql.DATETIME) def rename_table(self, old_name: str, new_name: str) -> Union[List, Tuple]: - old_name = self.add_tab(old_name) - new_name = self.add_tab(new_name) + old_name = get_table_name(old_name) + new_name = get_table_name(new_name) return self.sql(f"RENAME TABLE `{old_name}` TO `{new_name}`") def describe(self, doctype: str) -> Union[List, Tuple]: - doctype = self.add_tab(doctype) - return self.sql(f"DESC `{doctype}`") + table_name = get_table_name(doctype) + return self.sql(f"DESC `{table_name}`") def change_column_type(self, table: str, column: str, type: str) -> Union[List, Tuple]: - table = self.add_tab(table) - return self.sql(f"ALTER TABLE `{table}` MODIFY `{column}` {type} NOT NULL") + table_name = get_table_name(table) + return self.sql(f"ALTER TABLE `{table_name}` MODIFY `{column}` {type} NOT NULL") # exception types @staticmethod diff --git a/frappe/database/postgres/database.py b/frappe/database/postgres/database.py index 9c48b2960d..0b73c8b44b 100644 --- a/frappe/database/postgres/database.py +++ b/frappe/database/postgres/database.py @@ -8,7 +8,7 @@ from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT import frappe from frappe.database.database import Database from frappe.database.postgres.schema import PostgresTable -from frappe.utils import cstr +from frappe.utils import cstr, get_table_name # cast decimals as floats DEC2FLOAT = psycopg2.extensions.new_type( @@ -173,17 +173,17 @@ class PostgresDatabase(Database): return e.pgcode == '22001' def rename_table(self, old_name: str, new_name: str) -> Union[List, Tuple]: - old_name = self.add_tab(old_name) - new_name = self.add_tab(new_name) + old_name = get_table_name(old_name) + new_name = get_table_name(new_name) return self.sql(f"ALTER TABLE `{old_name}` RENAME TO `{new_name}`") def describe(self, doctype: str)-> Union[List, Tuple]: - doctype = self.add_tab(doctype) - return self.sql(f"SELECT COLUMN_NAME FROM information_schema.COLUMNS WHERE TABLE_NAME = '{doctype}'") + table_name = get_table_name(doctype) + return self.sql(f"SELECT COLUMN_NAME FROM information_schema.COLUMNS WHERE TABLE_NAME = '{table_name}'") def change_column_type(self, table: str, column: str, type: str) -> Union[List, Tuple]: - table = self.add_tab(table) - return self.sql(f'ALTER TABLE "{table}" ALTER COLUMN "{column}" TYPE {type}') + table_name = get_table_name(table) + return self.sql(f'ALTER TABLE "{table_name}" ALTER COLUMN "{column}" TYPE {type}') def create_auth_table(self): self.sql_ddl("""create table if not exists "__Auth" ( diff --git a/frappe/query_builder/builder.py b/frappe/query_builder/builder.py index d0f452e781..da1533fb1a 100644 --- a/frappe/query_builder/builder.py +++ b/frappe/query_builder/builder.py @@ -1,5 +1,6 @@ from pypika import MySQLQuery, Order, PostgreSQLQuery, terms from pypika.queries import Schema, Table +from frappe.utils import get_table_name class Base: @@ -8,10 +9,9 @@ class Base: Schema = Schema @staticmethod - def Table(class_name: str, *args, **kwargs) -> Table: - if not class_name.startswith("__"): - class_name = f"tab{class_name}" - return Table(class_name, *args, **kwargs) + def Table(table_name: str, *args, **kwargs) -> Table: + table_name = get_table_name(table_name) + return Table(table_name, *args, **kwargs) class MariaDB(Base, MySQLQuery): diff --git a/frappe/query_builder/functions.py b/frappe/query_builder/functions.py index 119c4e6c4f..5ccb266945 100644 --- a/frappe/query_builder/functions.py +++ b/frappe/query_builder/functions.py @@ -1,17 +1,17 @@ from pypika.functions import * -from frappe.query_builder.utils import ImportMapper, db_type +from frappe.query_builder.utils import ImportMapper, db_type_is from frappe.query_builder.custom import GROUP_CONCAT, STRING_AGG, MATCH, TO_TSVECTOR GroupConcat = ImportMapper( { - db_type.MARIADB: GROUP_CONCAT, - db_type.POSTGRES: STRING_AGG + db_type_is.MARIADB: GROUP_CONCAT, + db_type_is.POSTGRES: STRING_AGG } ) Match = ImportMapper( { - db_type.MARIADB: MATCH, - db_type.POSTGRES: TO_TSVECTOR + db_type_is.MARIADB: MATCH, + db_type_is.POSTGRES: TO_TSVECTOR } ) diff --git a/frappe/query_builder/utils.py b/frappe/query_builder/utils.py index e2276e15c0..b52a3606e8 100644 --- a/frappe/query_builder/utils.py +++ b/frappe/query_builder/utils.py @@ -1,5 +1,5 @@ from enum import Enum -from typing import Any, Callable, Dict, Optional +from typing import Any, Callable, Dict from pypika import Query @@ -7,22 +7,20 @@ import frappe from .builder import MariaDB, Postgres -class db_type(Enum): +class db_type_is(Enum): MARIADB = "mariadb" POSTGRES = "postgres" class ImportMapper: - def __init__(self, func_map: Dict[db_type, Callable]) -> None: + def __init__(self, func_map: Dict[db_type_is, Callable]) -> None: self.func_map = func_map def __call__(self, *args: Any, **kwds: Any) -> Callable: - db = db_type.MARIADB - if frappe.conf.db_type: - db = db_type(frappe.conf.db_type) + db = db_type_is(frappe.conf.db_type or "mariadb") return self.func_map[db](*args, **kwds) -def get_query_builder(type_of_db: Optional[str]) -> Query: +def get_query_builder(type_of_db: str) -> Query: """[return the query builder object] Args: @@ -31,8 +29,6 @@ def get_query_builder(type_of_db: Optional[str]) -> Query: Returns: Query: [Query object] """ - db = db_type.MARIADB - if type_of_db: - db = db_type(type_of_db) - selecter = {db_type.MARIADB: MariaDB, db_type.POSTGRES: Postgres} - return selecter[db] + db = db_type_is(type_of_db) + picks = {db_type_is.MARIADB: MariaDB, db_type_is.POSTGRES: Postgres} + return picks[db] diff --git a/frappe/tests/test_db.py b/frappe/tests/test_db.py index 9839df1197..044ce455d9 100644 --- a/frappe/tests/test_db.py +++ b/frappe/tests/test_db.py @@ -12,7 +12,7 @@ from frappe.custom.doctype.custom_field.custom_field import create_custom_field from frappe.utils import random_string from frappe.utils.testutils import clear_custom_fields -from .test_query_builder import CheckDB, db_type +from .test_query_builder import run_only_if, db_type_is class TestDB(unittest.TestCase): @@ -192,7 +192,7 @@ class TestDB(unittest.TestCase): frappe.delete_doc(test_doctype, doc) clear_custom_fields(test_doctype) -@CheckDB(db_type.MARIADB) +@run_only_if(db_type_is.MARIADB) class TestDDLCommandsMaria(unittest.TestCase): test_table_name = "TestNotes" @@ -236,7 +236,7 @@ class TestDDLCommandsMaria(unittest.TestCase): self.assertIn("varchar(255)", test_table_description[0]) -@CheckDB(dbtype=db_type.POSTGRES) +@run_only_if(db_type_is.POSTGRES) class TestDDLCommandsPost(unittest.TestCase): test_table_name = "TestNotes" diff --git a/frappe/tests/test_query_builder.py b/frappe/tests/test_query_builder.py index d5a74a33bd..d155dd95db 100644 --- a/frappe/tests/test_query_builder.py +++ b/frappe/tests/test_query_builder.py @@ -3,17 +3,16 @@ from typing import Callable import frappe from frappe.query_builder.functions import GroupConcat, Match -from frappe.query_builder.utils import db_type -from frappe.query_builder.functions import GroupConcat, Match +from frappe.query_builder.utils import db_type_is -def CheckDB(dbtype: db_type) -> Callable: +def run_only_if(dbtype: db_type_is) -> Callable: return unittest.skipIf( - db_type(frappe.conf.db_type) != dbtype, f"Only runs for {dbtype.value}" + db_type_is(frappe.conf.db_type) != dbtype, f"Only runs for {dbtype.value}" ) -@CheckDB(dbtype=db_type.MARIADB) +@run_only_if(db_type_is.MARIADB) class TestCustomFunctionsMariaDB(unittest.TestCase): def test_concat(self): self.assertEqual("GROUP_CONCAT('Notes')", GroupConcat("Notes").get_sql()) @@ -25,7 +24,7 @@ class TestCustomFunctionsMariaDB(unittest.TestCase): ) -@CheckDB(dbtype=db_type.POSTGRES) +@run_only_if(db_type_is.POSTGRES) class TestCustomFunctionsPostgres(unittest.TestCase): def test_concat(self): self.assertEqual("STRING_AGG('Notes',',')", GroupConcat("Notes").get_sql()) @@ -43,7 +42,7 @@ class TestBuilderBase(object): self.assertEqual("__Auth", frappe.qb.Table("__Auth").get_sql()) -@CheckDB(dbtype=db_type.MARIADB) +@run_only_if(db_type_is.MARIADB) class TestBuilderMaria(unittest.TestCase, TestBuilderBase): def test_adding_tabs_in_from(self): self.assertEqual( @@ -54,7 +53,7 @@ class TestBuilderMaria(unittest.TestCase, TestBuilderBase): ) -@CheckDB(dbtype=db_type.POSTGRES) +@run_only_if(db_type_is.POSTGRES) class TestBuilderPostgres(unittest.TestCase, TestBuilderBase): def test_adding_tabs_in_from(self): self.assertEqual( diff --git a/frappe/utils/__init__.py b/frappe/utils/__init__.py index 68366eb234..29883398e4 100644 --- a/frappe/utils/__init__.py +++ b/frappe/utils/__init__.py @@ -849,3 +849,6 @@ def groupby_metric(iterable: typing.Dict[str, list], key: str): for item in items: records.setdefault(item[key], {}).setdefault(category, []).append(item) return records + +def get_table_name(table_name: str) -> str: + return f"tab{table_name}" if not table_name.startswith("__") else table_name \ No newline at end of file From 94701fdb7d0e042e9d10fcf1b1921df095af7882 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 30 Jul 2021 12:27:22 +0530 Subject: [PATCH 160/164] feat(utils): frappe.utils.get_table_name Fetch table name from doctype/__ table name passed --- frappe/utils/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/frappe/utils/__init__.py b/frappe/utils/__init__.py index 68366eb234..b97585aa04 100644 --- a/frappe/utils/__init__.py +++ b/frappe/utils/__init__.py @@ -849,3 +849,6 @@ def groupby_metric(iterable: typing.Dict[str, list], key: str): for item in items: records.setdefault(item[key], {}).setdefault(category, []).append(item) return records + +def get_table_name(table_name: str) -> str: + return f"tab{table_name}" if not table_name.startswith("__") else table_name From 43cde9010922c7c1ac9c6e535843d835e78fbe0a Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 30 Jul 2021 12:28:27 +0530 Subject: [PATCH 161/164] fix(backups): Allow individual backups of all tables Due to previous logic, only tables under DocType table were allowed to take partial backups. This change allows backup to be taken for deprecated doctypes too. --- frappe/utils/backups.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/frappe/utils/backups.py b/frappe/utils/backups.py index 908be52452..f13710dcfe 100644 --- a/frappe/utils/backups.py +++ b/frappe/utils/backups.py @@ -116,16 +116,16 @@ class BackupGenerator: def setup_backup_tables(self): """Sets self.backup_includes, self.backup_excludes based on passed args""" - existing_doctypes = set([x.name for x in frappe.get_all("DocType")]) + existing_tables = frappe.db.get_tables() def get_tables(doctypes): tables = [] for doctype in doctypes: - if doctype and doctype in existing_doctypes: - if doctype.startswith("tab"): - tables.append(doctype) - else: - tables.append("tab" + doctype) + if not doctype: + continue + table = frappe.utils.get_table_name(doctype) + if table in existing_tables: + tables.append(table) return tables passed_tables = { From 5350210ad56d66e21c1f54b633667e0b54337992 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 30 Jul 2021 12:31:07 +0530 Subject: [PATCH 162/164] fix: Add patch to remove Data Import Legacy This patch removes the document record from DocType list/table and adds a message directing how existing data should be handled. --- frappe/patches.txt | 1 + frappe/patches/v14_0/__init__.py | 0 .../patches/v14_0/drop_data_import_legacy.py | 22 +++++++++++++++++++ 3 files changed, 23 insertions(+) create mode 100644 frappe/patches/v14_0/__init__.py create mode 100644 frappe/patches/v14_0/drop_data_import_legacy.py diff --git a/frappe/patches.txt b/frappe/patches.txt index 7605d8ea2b..493c4dc9f6 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -180,3 +180,4 @@ frappe.patches.v12_0.rename_uploaded_files_with_proper_name frappe.patches.v13_0.queryreport_columns frappe.patches.v13_0.jinja_hook frappe.patches.v13_0.update_notification_channel_if_empty +frappe.patches.v14_0.drop_data_import_legacy diff --git a/frappe/patches/v14_0/__init__.py b/frappe/patches/v14_0/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/frappe/patches/v14_0/drop_data_import_legacy.py b/frappe/patches/v14_0/drop_data_import_legacy.py new file mode 100644 index 0000000000..2037930c9f --- /dev/null +++ b/frappe/patches/v14_0/drop_data_import_legacy.py @@ -0,0 +1,22 @@ +import frappe +import click + + +def execute(): + doctype = "Data Import Legacy" + table = frappe.utils.get_table_name(doctype) + + # delete the doctype record to avoid broken links + frappe.db.delete("DocType", {"name": doctype}) + + # leaving table in database for manual cleanup + click.secho( + f"`{doctype}` has been deprecated. The DocType is deleted, but the data still" + " exists on the database. If this data is worth recovering, you may export it" + f" using\n\n\tbench --site {frappe.local.site} backup -i '{doctype}'\n\nAfter" + " this, the table will continue to persist in the database, until you choose" + " to remove it yourself. If you want to drop the table, you may run\n\n\tbench" + f" --site {frappe.local.site} execute frappe.db.sql --args \"('DROP TABLE IF" + f" EXISTS `{table}`', )\"\n", + fg="yellow", + ) From 07d192f95a231ee286b8ce3eca77313dd78e5385 Mon Sep 17 00:00:00 2001 From: leela Date: Fri, 30 Jul 2021 13:02:45 +0530 Subject: [PATCH 163/164] fix: incoming emails are not sending to the task owners --- frappe/core/doctype/communication/mixins.py | 9 +++++++++ frappe/desk/doctype/todo/todo.py | 11 ++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/frappe/core/doctype/communication/mixins.py b/frappe/core/doctype/communication/mixins.py index 09a8a0ac22..52cd370890 100644 --- a/frappe/core/doctype/communication/mixins.py +++ b/frappe/core/doctype/communication/mixins.py @@ -3,6 +3,7 @@ from frappe import _ from frappe.core.utils import get_parent_doc from frappe.utils import parse_addr, get_formatted_email, get_url from frappe.email.doctype.email_account.email_account import EmailAccount +from frappe.desk.doctype.todo.todo import ToDo class CommunicationEmailMixin: """Mixin class to handle communication mails. @@ -76,6 +77,7 @@ class CommunicationEmailMixin: if is_inbound_mail_communcation: cc.append(self.get_owner()) cc = set(cc) - {self.sender_mailid} + cc.update(self.get_assignees()) cc = set(cc) - set(self.filter_thread_notification_disbled_users(cc)) cc = cc - set(self.mail_recipients(is_inbound_mail_communcation=is_inbound_mail_communcation)) @@ -201,6 +203,13 @@ class CommunicationEmailMixin: self.mail_cc(is_inbound_mail_communcation = is_inbound_mail_communcation, include_sender=include_sender) return set(all_ids) - set(final_ids) + def get_assignees(self): + """Get owners of the reference document. + """ + filters = {'status': 'Open', 'reference_name': self.reference_name, + 'reference_type': self.reference_doctype} + return ToDo.get_owners(filters) + @staticmethod def filter_thread_notification_disbled_users(emails): """Filter users based on notifications for email threads setting is disabled. diff --git a/frappe/desk/doctype/todo/todo.py b/frappe/desk/doctype/todo/todo.py index 4696563445..4151cddac0 100644 --- a/frappe/desk/doctype/todo/todo.py +++ b/frappe/desk/doctype/todo/todo.py @@ -5,11 +5,13 @@ import frappe import json from frappe.model.document import Document -from frappe.utils import get_fullname +from frappe.utils import get_fullname, parse_addr exclude_from_linked_with = True class ToDo(Document): + DocType = 'ToDo' + def validate(self): self._assignment = None if self.is_new(): @@ -84,6 +86,13 @@ class ToDo(Document): else: raise + @classmethod + def get_owners(cls, filters=None): + """Returns list of owners after applying filters on todo's. + """ + rows = frappe.get_all(cls.DocType, filters=filters or {}, fields=['owner']) + return [parse_addr(row.owner)[1] for row in rows if row.owner] + # NOTE: todo is viewable if a user is an owner, or set as assigned_to value, or has any role that is allowed to access ToDo doctype. def on_doctype_update(): frappe.db.add_index("ToDo", ["reference_type", "reference_name"]) From b1c6ccffe155cc813d9dd89da004668d98dddde6 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 30 Jul 2021 13:09:12 +0530 Subject: [PATCH 164/164] style: Remove redundant brackets --- frappe/config/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/config/__init__.py b/frappe/config/__init__.py index caf169404d..e7f0f1a763 100644 --- a/frappe/config/__init__.py +++ b/frappe/config/__init__.py @@ -45,7 +45,7 @@ def get_all_empty_tables_by_module(): query = frappe.qb.from_(information_schema.tables).select(table_name).where(table_rows == 0) - empty_tables = {(r[0] for r in frappe.db.sql(query))} + empty_tables = {r[0] for r in frappe.db.sql(query)} results = frappe.get_all("DocType", fields=["name", "module"]) empty_tables_by_module = {}