From e369919d6517de225576da09b65fa9841c0c2b34 Mon Sep 17 00:00:00 2001 From: hasnain2808 Date: Thu, 17 Jun 2021 13:51:42 +0530 Subject: [PATCH 01/86] fix: add route building as a background job --- frappe/migrate.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/frappe/migrate.py b/frappe/migrate.py index d19e255639..cdaf75af9a 100644 --- a/frappe/migrate.py +++ b/frappe/migrate.py @@ -11,6 +11,7 @@ import frappe.model.sync from frappe.utils.fixtures import sync_fixtures from frappe.utils.connections import check_connection from frappe.utils.dashboard import sync_dashboards +from frappe.utils.background_jobs import enqueue from frappe.cache_manager import clear_global_cache from frappe.desk.notifications import clear_notifications from frappe.website import render @@ -86,18 +87,19 @@ Otherwise, check the server logs and ensure that all the required services are r for fn in frappe.get_hooks('after_migrate', app_name=app): frappe.get_attr(fn)() - # build web_routes index - if not skip_search_index: - # Run this last as it updates the current session - print('Building search index for {}'.format(frappe.local.site)) - build_index_for_all_routes() - frappe.db.commit() clear_notifications() frappe.publish_realtime("version-update") frappe.flags.in_migrate = False + + # build web_routes index + if not skip_search_index: + # Run this last as it updates the current session + print('Queuing search index build for {}'.format(frappe.local.site)) + enqueue(method=build_index_for_all_routes, job_name='Search index build for {}'.format(frappe.local.site), now=0) + finally: with open(touched_tables_file, 'w') as f: json.dump(list(frappe.flags.touched_tables), f, sort_keys=True, indent=4) From 292d7f914cd4de1f306ac6299a0e029a672c6a2f Mon Sep 17 00:00:00 2001 From: hasnain2808 Date: Thu, 17 Jun 2021 15:52:39 +0530 Subject: [PATCH 02/86] fix: be explicit with timeout and queue --- frappe/migrate.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/frappe/migrate.py b/frappe/migrate.py index cdaf75af9a..5885c05039 100644 --- a/frappe/migrate.py +++ b/frappe/migrate.py @@ -4,6 +4,8 @@ import json import os import sys + +from rq import queue import frappe import frappe.translate import frappe.modules.patch_handler @@ -98,7 +100,13 @@ Otherwise, check the server logs and ensure that all the required services are r if not skip_search_index: # Run this last as it updates the current session print('Queuing search index build for {}'.format(frappe.local.site)) - enqueue(method=build_index_for_all_routes, job_name='Search index build for {}'.format(frappe.local.site), now=0) + enqueue( + method=build_index_for_all_routes, + job_name='Search index build for {}'.format(frappe.local.site), + now=0, + queue='background', + timeout=10000 + ) finally: with open(touched_tables_file, 'w') as f: From 9bea81e30cdc59a645993fc538fdf1321de88cf2 Mon Sep 17 00:00:00 2001 From: hasnain2808 Date: Fri, 18 Jun 2021 00:12:07 +0530 Subject: [PATCH 03/86] feat: website_search_field in doctype doctype website_search_field to declare the content field remove the background job execution of search indexing --- frappe/core/doctype/doctype/doctype.json | 11 +++++++++-- frappe/migrate.py | 18 ++++++------------ frappe/search/website_search.py | 18 ++++++++++++------ 3 files changed, 27 insertions(+), 20 deletions(-) diff --git a/frappe/core/doctype/doctype/doctype.json b/frappe/core/doctype/doctype/doctype.json index 7f93d3130a..6a427f71e1 100644 --- a/frappe/core/doctype/doctype/doctype.json +++ b/frappe/core/doctype/doctype/doctype.json @@ -76,6 +76,7 @@ "index_web_pages_for_search", "route", "is_published_field", + "website_search_field", "advanced", "engine" ], @@ -547,6 +548,12 @@ { "fieldname": "column_break_51", "fieldtype": "Column Break" + }, + { + "depends_on": "has_web_view", + "fieldname": "website_search_field", + "fieldtype": "Data", + "label": "Website Search Field" } ], "icon": "fa fa-bolt", @@ -628,7 +635,7 @@ "link_fieldname": "reference_doctype" } ], - "modified": "2021-04-16 12:26:41.031135", + "modified": "2021-06-17 23:31:44.974199", "modified_by": "Administrator", "module": "Core", "name": "DocType", @@ -662,4 +669,4 @@ "sort_field": "modified", "sort_order": "DESC", "track_changes": 1 -} +} \ No newline at end of file diff --git a/frappe/migrate.py b/frappe/migrate.py index 5885c05039..c984371927 100644 --- a/frappe/migrate.py +++ b/frappe/migrate.py @@ -89,6 +89,12 @@ Otherwise, check the server logs and ensure that all the required services are r for fn in frappe.get_hooks('after_migrate', app_name=app): frappe.get_attr(fn)() + # build web_routes index + if not skip_search_index: + # Run this last as it updates the current session + print('Building search index for {}'.format(frappe.local.site)) + build_index_for_all_routes() + frappe.db.commit() clear_notifications() @@ -96,18 +102,6 @@ Otherwise, check the server logs and ensure that all the required services are r frappe.publish_realtime("version-update") frappe.flags.in_migrate = False - # build web_routes index - if not skip_search_index: - # Run this last as it updates the current session - print('Queuing search index build for {}'.format(frappe.local.site)) - enqueue( - method=build_index_for_all_routes, - job_name='Search index build for {}'.format(frappe.local.site), - now=0, - queue='background', - timeout=10000 - ) - finally: with open(touched_tables_file, 'w') as f: json.dump(list(frappe.flags.touched_tables), f, sort_keys=True, indent=4) diff --git a/frappe/search/website_search.py b/frappe/search/website_search.py index 452ea2a427..10764d49d4 100644 --- a/frappe/search/website_search.py +++ b/frappe/search/website_search.py @@ -35,10 +35,12 @@ class WebsiteSearch(FullTextSearch): if getattr(self, "_items_to_index", False): return self._items_to_index - routes = get_static_pages_from_all_apps() + slugs_with_web_view() - self._items_to_index = [] + + routes = get_static_pages_from_all_apps() + slugs_with_web_view(self._items_to_index ) + + for i, route in enumerate(routes): update_progress_bar("Retrieving Routes", i, len(routes)) self._items_to_index += [self.get_document_to_index(route)] @@ -85,16 +87,20 @@ class WebsiteSearch(FullTextSearch): ) -def slugs_with_web_view(): +def slugs_with_web_view(_items_to_index): all_routes = [] filters = { "has_web_view": 1, "allow_guest_to_view": 1, "index_web_pages_for_search": 1} - fields = ["name", "is_published_field"] + fields = ["name", "is_published_field", 'website_search_field'] doctype_with_web_views = frappe.get_all("DocType", filters=filters, fields=fields) for doctype in doctype_with_web_views: if doctype.is_published_field: - routes = frappe.get_all(doctype.name, filters={doctype.is_published_field: 1}, fields="route") - all_routes += [route.route for route in routes] + docs = frappe.get_all(doctype.name, filters={doctype.is_published_field: 1}, fields=["route", doctype.website_search_field]) + if doctype.website_search_field: + for doc in docs: + _items_to_index += [frappe._dict(title=doc.title, content=getattr(doc, doctype.website_search_field), path=doc.route)] + else: + all_routes += [route.route for route in docs] return all_routes From e71ae4b3feda96a591593c612b769bcd78b02985 Mon Sep 17 00:00:00 2001 From: hasnain2808 Date: Fri, 18 Jun 2021 00:25:31 +0530 Subject: [PATCH 04/86] fix: title needed too --- frappe/search/website_search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/search/website_search.py b/frappe/search/website_search.py index 10764d49d4..41d51f6b69 100644 --- a/frappe/search/website_search.py +++ b/frappe/search/website_search.py @@ -95,7 +95,7 @@ def slugs_with_web_view(_items_to_index): for doctype in doctype_with_web_views: if doctype.is_published_field: - docs = frappe.get_all(doctype.name, filters={doctype.is_published_field: 1}, fields=["route", doctype.website_search_field]) + docs = frappe.get_all(doctype.name, filters={doctype.is_published_field: 1}, fields=["route", doctype.website_search_field, 'title']) if doctype.website_search_field: for doc in docs: _items_to_index += [frappe._dict(title=doc.title, content=getattr(doc, doctype.website_search_field), path=doc.route)] From c7d7c54e2c66385511457ad4d48a145e7f3a0902 Mon Sep 17 00:00:00 2001 From: hasnain2808 Date: Fri, 18 Jun 2021 11:06:52 +0530 Subject: [PATCH 05/86] feat: better indexing --- frappe/search/website_search.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/frappe/search/website_search.py b/frappe/search/website_search.py index 41d51f6b69..566c0f6a6d 100644 --- a/frappe/search/website_search.py +++ b/frappe/search/website_search.py @@ -98,7 +98,10 @@ def slugs_with_web_view(_items_to_index): docs = frappe.get_all(doctype.name, filters={doctype.is_published_field: 1}, fields=["route", doctype.website_search_field, 'title']) if doctype.website_search_field: for doc in docs: - _items_to_index += [frappe._dict(title=doc.title, content=getattr(doc, doctype.website_search_field), path=doc.route)] + content = frappe.utils.md_to_html(getattr(doc, doctype.website_search_field)) + soup = BeautifulSoup(content, "html.parser") + text_content = soup.text if soup else "" + _items_to_index += [frappe._dict(title=doc.title, content=text_content, path=doc.route)] else: all_routes += [route.route for route in docs] From 0c8e72d0f0d9fc31e0f98b6c9d597865cc0fa810 Mon Sep 17 00:00:00 2001 From: hasnain2808 Date: Thu, 24 Jun 2021 19:07:48 +0530 Subject: [PATCH 06/86] fix: cache document page --- frappe/website/page_renderers/document_page.py | 12 +++++++++--- frappe/website/page_renderers/template_page.py | 2 +- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/frappe/website/page_renderers/document_page.py b/frappe/website/page_renderers/document_page.py index f1741c681f..0108a84f79 100644 --- a/frappe/website/page_renderers/document_page.py +++ b/frappe/website/page_renderers/document_page.py @@ -1,7 +1,7 @@ import frappe from frappe.model.document import get_controller from frappe.website.page_renderers.base_template_page import BaseTemplatePage -from frappe.website.utils import build_response +from frappe.website.utils import build_response, cache_html from frappe.website.router import (get_doctypes_with_web_view, get_page_info_from_web_page_with_dynamic_routes) @@ -47,14 +47,20 @@ class DocumentPage(BaseTemplatePage): return False def render(self): + html = self.get_html() + html = self.add_csrf_token(html) + + return build_response(self.path, html, self.http_status_code or 200, self.headers) + + @cache_html + def get_html(self): self.doc = frappe.get_doc(self.doctype, self.docname) self.init_context() self.update_context() self.post_process_context() html = frappe.get_template(self.template_path).render(self.context) - html = self.add_csrf_token(html) - return build_response(self.path, html, self.http_status_code or 200, self.headers) + return html def update_context(self): self.context.doc = self.doc diff --git a/frappe/website/page_renderers/template_page.py b/frappe/website/page_renderers/template_page.py index 5e6e57e33a..2ec3f51f41 100644 --- a/frappe/website/page_renderers/template_page.py +++ b/frappe/website/page_renderers/template_page.py @@ -83,7 +83,7 @@ class TemplatePage(BaseTemplatePage): super(TemplatePage, self).post_process_context() def add_sidebar_and_breadcrumbs(self): - if self.basepath: + if self.basepath and not self.context.sidebar_items: self.context.sidebar_items = get_sidebar_items(self.context.website_sidebar, self.basepath) if self.context.add_breadcrumbs and not self.context.parents: From 51207fe582f28eeaf5a0692c143e243339d6cce6 Mon Sep 17 00:00:00 2001 From: hasnain2808 Date: Sat, 3 Jul 2021 13:00:43 +0530 Subject: [PATCH 07/86] feat: option to not preload assets --- frappe/website/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frappe/website/utils.py b/frappe/website/utils.py index 0f5f182ea2..171c90e1cf 100644 --- a/frappe/website/utils.py +++ b/frappe/website/utils.py @@ -449,15 +449,15 @@ def cache_html(func): return cache_html_decorator -def build_response(path, data, http_status_code, headers=None): +def build_response(path, data, http_status_code, headers=None, preload_assets=True): # build response response = Response() response.data = set_content_type(response, data, path) response.status_code = http_status_code response.headers["X-Page-Name"] = path.encode("ascii", errors="xmlcharrefreplace") response.headers["X-From-Cache"] = frappe.local.response.from_cache or False - - add_preload_headers(response) + if preload_assets: + add_preload_headers(response) if headers: for key, val in iteritems(headers): response.headers[key] = val.encode("ascii", errors="xmlcharrefreplace") From 1c5ad30bf1069eb8469f2a47d9fe25dd4199ce40 Mon Sep 17 00:00:00 2001 From: hasnain2808 Date: Sat, 3 Jul 2021 18:44:40 +0530 Subject: [PATCH 08/86] perf: let's preload assets --- frappe/website/utils.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/frappe/website/utils.py b/frappe/website/utils.py index 171c90e1cf..419a5199df 100644 --- a/frappe/website/utils.py +++ b/frappe/website/utils.py @@ -449,15 +449,14 @@ def cache_html(func): return cache_html_decorator -def build_response(path, data, http_status_code, headers=None, preload_assets=True): +def build_response(path, data, http_status_code, headers=None): # build response response = Response() response.data = set_content_type(response, data, path) response.status_code = http_status_code response.headers["X-Page-Name"] = path.encode("ascii", errors="xmlcharrefreplace") response.headers["X-From-Cache"] = frappe.local.response.from_cache or False - if preload_assets: - add_preload_headers(response) + add_preload_headers(response) if headers: for key, val in iteritems(headers): response.headers[key] = val.encode("ascii", errors="xmlcharrefreplace") @@ -486,11 +485,12 @@ def set_content_type(response, data, path): return data def add_preload_headers(response): - from bs4 import BeautifulSoup + from bs4 import BeautifulSoup, SoupStrainer try: preload = [] - soup = BeautifulSoup(response.data, "lxml") + strainer = SoupStrainer(re.compile("script|link")) + soup = BeautifulSoup(response.data, "lxml", parse_only=strainer) for elem in soup.find_all('script', src=re.compile(".*")): preload.append(("script", elem.get("src"))) From cf164fffba6742886d67b71c443b48e15ca8506c Mon Sep 17 00:00:00 2001 From: Deepesh Garg Date: Tue, 13 Jul 2021 19:11:07 +0530 Subject: [PATCH 09/86] fix: Error on updating docs via API --- frappe/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/api.py b/frappe/api.py index 36d51e894c..636c6b2888 100644 --- a/frappe/api.py +++ b/frappe/api.py @@ -82,7 +82,7 @@ def handle(): if frappe.local.request.method=="PUT": data = get_request_form_data() - doc = frappe.get_doc(doctype, name) + doc = frappe.get_doc(doctype, name, for_update=True) if "flags" in data: del data["flags"] From 33f8bfcaf29fc2d920ff5110a7b616e902afbcea Mon Sep 17 00:00:00 2001 From: Deepesh Garg Date: Sun, 18 Jul 2021 17:46:16 +0530 Subject: [PATCH 10/86] fix: Add for update in nest set model updates --- frappe/utils/nestedset.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/frappe/utils/nestedset.py b/frappe/utils/nestedset.py index 3c024c40e4..b0657b1108 100644 --- a/frappe/utils/nestedset.py +++ b/frappe/utils/nestedset.py @@ -57,13 +57,14 @@ def update_add_node(doc, parent, parent_field): # get the last sibling of the parent if parent: - left, right = frappe.db.sql("select lft, rgt from `tab{0}` where name=%s" + left, right = frappe.db.sql("select lft, rgt from `tab{0}` where name=%s for update" .format(doctype), parent)[0] validate_loop(doc.doctype, doc.name, left, right) else: # root right = frappe.db.sql(""" SELECT COALESCE(MAX(rgt), 0) + 1 FROM `tab{0}` WHERE COALESCE(`{1}`, '') = '' + FOR UPDATE """.format(doctype, parent_field))[0][0] right = right or 1 @@ -89,7 +90,7 @@ def update_move_node(doc, parent_field): if parent: new_parent = frappe.db.sql("""select lft, rgt from `tab{0}` - where name = %s""".format(doc.doctype), parent, as_dict=1)[0] + where name = %s for update""".format(doc.doctype), parent, as_dict=1)[0] validate_loop(doc.doctype, doc.name, new_parent.lft, new_parent.rgt) @@ -108,7 +109,7 @@ def update_move_node(doc, parent_field): if parent: new_parent = frappe.db.sql("""select lft, rgt from `tab%s` - where name = %s""" % (doc.doctype, '%s'), parent, as_dict=1)[0] + where name = %s for update""" % (doc.doctype, '%s'), parent, as_dict=1)[0] # set parent lft, rgt @@ -128,7 +129,7 @@ def update_move_node(doc, parent_field): new_diff = new_parent.rgt - doc.lft else: # new root - max_rgt = frappe.db.sql("""select max(rgt) from `tab{0}`""".format(doc.doctype))[0][0] + max_rgt = frappe.db.sql("""select max(rgt) from `tab{0}` for update""".format(doc.doctype))[0][0] new_diff = max_rgt + 1 - doc.lft # bring back from dark side From 0c3e923e0fd1dbcab3a6493cea8fb8083bfc46b0 Mon Sep 17 00:00:00 2001 From: Deepesh Garg Date: Mon, 19 Jul 2021 10:49:44 +0530 Subject: [PATCH 11/86] fix: Remove for update flag from aggregate functions --- frappe/utils/nestedset.py | 1 - 1 file changed, 1 deletion(-) diff --git a/frappe/utils/nestedset.py b/frappe/utils/nestedset.py index b0657b1108..75c1b0c2ae 100644 --- a/frappe/utils/nestedset.py +++ b/frappe/utils/nestedset.py @@ -64,7 +64,6 @@ def update_add_node(doc, parent, parent_field): right = frappe.db.sql(""" SELECT COALESCE(MAX(rgt), 0) + 1 FROM `tab{0}` WHERE COALESCE(`{1}`, '') = '' - FOR UPDATE """.format(doctype, parent_field))[0][0] right = right or 1 From f1f8678ee0c631b791e479de87909fdf2853c714 Mon Sep 17 00:00:00 2001 From: Deepesh Garg Date: Mon, 19 Jul 2021 10:54:22 +0530 Subject: [PATCH 12/86] fix: Remove for update flag from aggregate functions --- frappe/utils/nestedset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/utils/nestedset.py b/frappe/utils/nestedset.py index 75c1b0c2ae..4a65140449 100644 --- a/frappe/utils/nestedset.py +++ b/frappe/utils/nestedset.py @@ -128,7 +128,7 @@ def update_move_node(doc, parent_field): new_diff = new_parent.rgt - doc.lft else: # new root - max_rgt = frappe.db.sql("""select max(rgt) from `tab{0}` for update""".format(doc.doctype))[0][0] + max_rgt = frappe.db.sql("""select max(rgt) from `tab{0}`""".format(doc.doctype))[0][0] new_diff = max_rgt + 1 - doc.lft # bring back from dark side From 9b437f8c90b3a7ca955f5f0c168bf8001d8e2975 Mon Sep 17 00:00:00 2001 From: leela Date: Mon, 19 Jul 2021 12:18:25 +0530 Subject: [PATCH 13/86] feat: Add rq users --- frappe/commands/__init__.py | 4 +- frappe/commands/redis.py | 48 ++++++++++++++++++++ frappe/utils/__init__.py | 6 +++ frappe/utils/rq.py | 87 +++++++++++++++++++++++++++++++++++++ 4 files changed, 144 insertions(+), 1 deletion(-) create mode 100644 frappe/commands/redis.py create mode 100644 frappe/utils/rq.py diff --git a/frappe/commands/__init__.py b/frappe/commands/__init__.py index be9d107025..9ed333d034 100644 --- a/frappe/commands/__init__.py +++ b/frappe/commands/__init__.py @@ -102,7 +102,9 @@ def get_commands(): from .site import commands as site_commands from .translate import commands as translate_commands from .utils import commands as utils_commands + from .redis import commands as redis_commands - return list(set(scheduler_commands + site_commands + translate_commands + utils_commands)) + all_commands = scheduler_commands + site_commands + translate_commands + utils_commands + redis_commands + return list(set(all_commands)) commands = get_commands() diff --git a/frappe/commands/redis.py b/frappe/commands/redis.py new file mode 100644 index 0000000000..c608296cac --- /dev/null +++ b/frappe/commands/redis.py @@ -0,0 +1,48 @@ +import os + +import click +import redis + +import frappe +from frappe.utils.rq import RedisQueue +from frappe.installer import update_site_config + +@click.command('create-rq-users') +@click.option('--set-admin-password', is_flag=True, default=False, help='Set new Redis admin(default user) password') +@click.option('--reset-passwords', is_flag=True, default=False, help='Remove all existing passwords') +def create_rq_users(set_admin_password=False, reset_passwords=False): + """Create Redis Queue users and add to acl and app configs. + + acl config file will be used by redis server while starting the server + and app config is used by app while connecting to redis server. + """ + acl_file_path = os.path.abspath('../config/redis_queue.acl') + acl_list, user_credentials = RedisQueue.gen_acl_list( + reset_passwords=reset_passwords, set_admin_password=set_admin_password) + + with open(acl_file_path, 'w') as f: + f.writelines([acl+'\n' for acl in acl_list]) + + sites_path = os.getcwd() + common_site_config_path = os.path.join(sites_path, 'common_site_config.json') + update_site_config("rq_username", user_credentials['bench'][0], validate=False, + site_config_path=common_site_config_path) + update_site_config("rq_password", user_credentials['bench'][1], validate=False, + site_config_path=common_site_config_path) + + if set_admin_password: + env_key = 'RQ_ADMIN_PASWORD' + click.secho('Redis admin password is successfully set up. ' + 'Include below line in .bashrc file for system to use', + fg='green' + ) + click.secho(f"`export {env_key}={user_credentials['default'][1]}`") + click.secho('NOTE: Please save the admin password as you ' + 'can not access redis server without the password', + fg='yellow' + ) + + +commands = [ + create_rq_users +] diff --git a/frappe/utils/__init__.py b/frappe/utils/__init__.py index af9d5de1ee..bf139173d6 100644 --- a/frappe/utils/__init__.py +++ b/frappe/utils/__init__.py @@ -383,6 +383,12 @@ def get_files_path(*path, **kwargs): def get_bench_path(): return os.path.realpath(os.path.join(os.path.dirname(frappe.__file__), '..', '..', '..')) +def get_bench_id(): + return frappe.local.conf.get('bench_id', 'DefaultBench') + +def get_site_id(site=None): + return f"{site or frappe.local.site}@{get_bench_id()}" + def get_backups_path(): return get_site_path("private", "backups") diff --git a/frappe/utils/rq.py b/frappe/utils/rq.py new file mode 100644 index 0000000000..5e9b9dcd5d --- /dev/null +++ b/frappe/utils/rq.py @@ -0,0 +1,87 @@ +import redis + +import frappe +from frappe.utils import get_site_id, get_bench_id, random_string + + +class RedisQueue: + def __init__(self, conn): + self.conn = conn + + def add_user(self, username, password=None): + """Create or update the user. + """ + password = password or self.conn.acl_genpass() + user_settings = self.get_new_user_settings(username, password) + is_created = self.conn.acl_setuser(**user_settings) + return frappe._dict(user_settings) if is_created else {} + + @classmethod + def get_connection(cls, username='default', password=None): + domain = frappe.local.conf.redis_queue.split("redis://", 1)[-1] + url = f"redis://{username}:{password or ''}@{domain}" + conn = redis.from_url(url) + conn.ping() + return conn + + @classmethod + def new(cls, username='default', password=None): + return cls(cls.get_connection(username, password)) + + @classmethod + def set_admin_password(cls, cur_password=None, new_password=None, reset_passwords=False): + username = 'default' + conn = cls.get_connection(username, cur_password) + password = '+'+(new_password or conn.acl_genpass()) + conn.acl_setuser( + username=username, enabled=True, reset_passwords=reset_passwords, passwords=password + ) + return password[1:] + + @classmethod + def get_new_user_settings(cls, username, password): + d = {} + d['username'] = username + d['passwords'] = '+'+password + d['reset_keys'] = True + d['enabled'] = True + d['keys'] = cls.get_acl_key_rules() + d['commands'] = cls.get_acl_command_rules() + return d + + @classmethod + def get_acl_key_rules(cls, include_key_prefix=False): + """FIXME: Find better way + """ + rules = ['rq:[^q]*', 'rq:queues', f'rq:queue:{get_bench_id()}:*'] + if include_key_prefix: + return ['~'+pattern for pattern in rules] + return rules + + @classmethod + def get_acl_command_rules(cls): + return ['+@all', '-@admin'] + + @classmethod + def gen_acl_list(cls, reset_passwords=False, set_admin_password=False): + """Generate list of ACL users needed for this branch. + + This list contains default ACL user and the bench ACL user(used by all sites incase of ACL is enabled). + """ + with frappe.init_site(): + bench_username = get_bench_id() + bench_user_rules = cls.get_acl_key_rules(include_key_prefix=True) + cls.get_acl_command_rules() + + bench_user_rule_str = ' '.join(bench_user_rules).strip() + bench_user_password = random_string(20) + bench_user_resetpass = (reset_passwords and 'resetpass') or '' + + default_username = 'default' + _default_user_password = random_string(20) if set_admin_password else '' + default_user_password = '>'+_default_user_password if _default_user_password else 'nopass' + default_user_resetpass = (reset_passwords and set_admin_password and 'resetpass') or '' + + return [ + f'user {default_username} on {default_user_password} {default_user_resetpass} ~* &* +@all', + f'user {bench_username} on >{bench_user_password} {bench_user_resetpass} {bench_user_rule_str}' + ], {'bench': (bench_username, bench_user_password), 'default': (default_username, _default_user_password)} From fa6cb14ee5221ec4d9db82bc731be8910eefb7b0 Mon Sep 17 00:00:00 2001 From: Aradhya-Tripathi Date: Wed, 21 Jul 2021 19:15:38 +0530 Subject: [PATCH 14/86] refactor: moved from raw queries to frappe orm in core --- frappe/core/doctype/activity_log/feed.py | 15 +++-- .../domain_settings/domain_settings.py | 3 +- .../core/doctype/log_settings/log_settings.py | 1 + frappe/core/doctype/report/test_report.py | 9 ++- frappe/core/doctype/role/role.py | 3 +- .../scheduled_job_type/scheduled_job_type.py | 7 +- frappe/core/doctype/user/user.py | 27 +++++--- .../user_permission/test_user_permission.py | 19 ++++-- .../user_permission/user_permission.py | 66 +++++++++++++------ .../permission_manager/permission_manager.py | 6 +- 10 files changed, 106 insertions(+), 50 deletions(-) diff --git a/frappe/core/doctype/activity_log/feed.py b/frappe/core/doctype/activity_log/feed.py index caa3cae613..cf8f7416e8 100644 --- a/frappe/core/doctype/activity_log/feed.py +++ b/frappe/core/doctype/activity_log/feed.py @@ -27,12 +27,15 @@ def update_feed(doc, method=None): feed = frappe._dict(feed) doctype = feed.doctype or doc.doctype name = feed.name or doc.name - - # delete earlier feed - frappe.db.sql("""delete from `tabActivity Log` - where - reference_doctype=%s and reference_name=%s - and link_doctype=%s""", (doctype, name,feed.link_doctype)) + + + frappe.db.delete(doctype="Activity Log", conditions={"reference_doctype": doctype, + "reference_name": name, + "link_doctype": name.feed.link_doctype}) + # frappe.db.sql("""delete from `tabActivity Log` + # where + # reference_doctype=%s and reference_name=%s + # and link_doctype=%s""", (doctype, name,feed.link_doctype)) frappe.get_doc({ "doctype": "Activity Log", "reference_doctype": doctype, diff --git a/frappe/core/doctype/domain_settings/domain_settings.py b/frappe/core/doctype/domain_settings/domain_settings.py index 7ad0aeff21..6ca180def1 100644 --- a/frappe/core/doctype/domain_settings/domain_settings.py +++ b/frappe/core/doctype/domain_settings/domain_settings.py @@ -34,7 +34,8 @@ class DomainSettings(Document): all_domains = list((frappe.get_hooks('domains') or {})) def remove_role(role): - frappe.db.sql('delete from `tabHas Role` where role=%s', role) + frappe.db.delete(doctype="Has Role", conditions={"role": role}) + # frappe.db.sql('delete from `tabHas Role` where role=%s', role) frappe.set_value('Role', role, 'disabled', 1) for domain in all_domains: diff --git a/frappe/core/doctype/log_settings/log_settings.py b/frappe/core/doctype/log_settings/log_settings.py index e73aa8dac1..776fcc92e9 100644 --- a/frappe/core/doctype/log_settings/log_settings.py +++ b/frappe/core/doctype/log_settings/log_settings.py @@ -13,6 +13,7 @@ class LogSettings(Document): self.clear_email_queue() def clear_error_logs(self): + # frappe.db.delete(doctype="Error Log", conditions="") frappe.db.sql(""" DELETE FROM `tabError Log` WHERE `creation` < (NOW() - INTERVAL '{0}' DAY) """.format(self.clear_error_log_after)) diff --git a/frappe/core/doctype/report/test_report.py b/frappe/core/doctype/report/test_report.py index 9d0c0b9af0..2bab6adf19 100644 --- a/frappe/core/doctype/report/test_report.py +++ b/frappe/core/doctype/report/test_report.py @@ -81,9 +81,14 @@ class TestReport(unittest.TestCase): self.assertDictEqual({'name': 'Administrator', 'user_type': 'System User', 'email': 'admin@example.com'}, admin_dict) def test_report_permissions(self): + frappe.set_user('test@example.com') - frappe.db.sql("""delete from `tabHas Role` where parent = %s - and role = 'Test Has Role'""", frappe.session.user, auto_commit=1) + frappe.db.delete(doctype="Has Role", conditions={ + "parent": frappe.session.user, + "role": "Test Has Role" + }) + # frappe.db.sql("""delete from `tabHas Role` where parent = %s + # and role = 'Test Has Role'""", frappe.session.user, auto_commit=1) if not frappe.db.exists('Role', 'Test Has Role'): role = frappe.get_doc({ diff --git a/frappe/core/doctype/role/role.py b/frappe/core/doctype/role/role.py index 02482c75ca..fae8ae2d28 100644 --- a/frappe/core/doctype/role/role.py +++ b/frappe/core/doctype/role/role.py @@ -38,7 +38,8 @@ class Role(Document): self.set(key, 0) def remove_roles(self): - frappe.db.sql("delete from `tabHas Role` where role = %s", self.name) + frappe.db.delete(doctype="Has Role", conditions={"role": self.name}) + # frappe.db.sql("delete from `tabHas Role` where role = %s", self.name) frappe.clear_cache() def on_update(self): diff --git a/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py b/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py index 59089d12ad..d6c94aded1 100644 --- a/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py +++ b/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py @@ -13,7 +13,6 @@ from frappe.model.document import Document from frappe.utils import get_datetime, now_datetime from frappe.utils.background_jobs import enqueue, get_jobs - class ScheduledJobType(Document): def autoname(self): self.name = ".".join(self.method.split(".")[-2:]) @@ -110,7 +109,11 @@ class ScheduledJobType(Document): return 'long' if ('Long' in self.frequency) else 'default' def on_trash(self): - frappe.db.sql('delete from `tabScheduled Job Log` where scheduled_job_type=%s', self.name) + + frappe.db.delete(doctype="Scheduled Job Log", conditions={ + "scheduled_job_type": self.name + }) + # frappe.db.sql('delete from `tabScheduled Job Log` where scheduled_job_type=%s', self.name) @frappe.whitelist() diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py index 53d1c9ffe5..f69af4e4a8 100644 --- a/frappe/core/doctype/user/user.py +++ b/frappe/core/doctype/user/user.py @@ -17,7 +17,7 @@ from frappe.website.utils import is_signup_disabled from frappe.rate_limiter import rate_limit from frappe.utils.background_jobs import enqueue from frappe.core.doctype.user_type.user_type import user_linked_with_permission_on_doctype - +from frappe.database.database import Database STANDARD_USERS = ("Guest", "Administrator") @@ -367,23 +367,30 @@ class User(Document): if getattr(frappe.local, "login_manager", None): frappe.local.login_manager.logout(user=self.name) - # delete todos - frappe.db.sql("""DELETE FROM `tabToDo` WHERE `owner`=%s""", (self.name,)) + frappe.db.delete("Todo", {"owner": self.name}) + # frappe.db.sql("""DELETE FROM `tabToDo` WHERE `owner`=%s""", (self.name,)) frappe.db.sql("""UPDATE `tabToDo` SET `assigned_by`=NULL WHERE `assigned_by`=%s""", (self.name,)) # delete events - frappe.db.sql("""delete from `tabEvent` where owner=%s - and event_type='Private'""", (self.name,)) + frappe.db.delete("Event", {"owner": self.name, "event_type": "Private"}) + # frappe.db.sql("""delete from `tabEvent` where owner=%s + # and event_type='Private'""", (self.name,)) # delete shares - frappe.db.sql("""delete from `tabDocShare` where user=%s""", self.name) + frappe.db.delete("DocShare", {"user": self.name}) + # frappe.db.sql("""delete from `tabDocShare` where user=%s""", self.name) # delete messages - frappe.db.sql("""delete from `tabCommunication` - where communication_type in ('Chat', 'Notification') - and reference_doctype='User' - and (reference_name=%s or owner=%s)""", (self.name, self.name)) + # TODO: CHANGE THIS FROM ABHISHEK KA PYPIKA + frappe.db.delete("Communication", { + "reference_doctype": "User", + "communication_type": ("in", ("Chat", "Notification")), + }) + # frappe.db.sql("""delete from `tabCommunication` + # where communication_type in ('Chat', 'Notification') + # and reference_doctype='User' + # and (reference_name=%s or owner=%s)""", (self.name, self.name)) # unlink contact frappe.db.sql("""update `tabContact` diff --git a/frappe/core/doctype/user_permission/test_user_permission.py b/frappe/core/doctype/user_permission/test_user_permission.py index 1a442b53e7..9393e47ee9 100644 --- a/frappe/core/doctype/user_permission/test_user_permission.py +++ b/frappe/core/doctype/user_permission/test_user_permission.py @@ -10,12 +10,19 @@ import unittest class TestUserPermission(unittest.TestCase): def setUp(self): - frappe.db.sql("""DELETE FROM `tabUser Permission` - WHERE `user` in ( - 'test_bulk_creation_update@example.com', - 'test_user_perm1@example.com', - 'nested_doc_user@example.com')""") - frappe.delete_doc_if_exists("DocType", "Person") + + frappe.db.delete("User Permission", { + "user": ("in", ("test_bulk_creation_update@example.com", + "test_user_perm1@example.com", + "nested_doc_user@example.com")) + }) + + # frappe.db.sql("""DELETE FROM `tabUser Permission` + # WHERE `user` in ( + # 'test_bulk_creation_update@example.com', + # 'test_user_perm1@example.com', + # 'nested_doc_user@example.com')""") + # frappe.delete_doc_if_exists("DocType", "Person") frappe.db.sql_ddl("DROP TABLE IF EXISTS `tabPerson`") frappe.delete_doc_if_exists("DocType", "Doc A") frappe.db.sql_ddl("DROP TABLE IF EXISTS `tabDoc A`") diff --git a/frappe/core/doctype/user_permission/user_permission.py b/frappe/core/doctype/user_permission/user_permission.py index 4aa5797c7f..5b3822d82f 100644 --- a/frappe/core/doctype/user_permission/user_permission.py +++ b/frappe/core/doctype/user_permission/user_permission.py @@ -182,7 +182,11 @@ def clear_user_permissions(user, for_doctype): frappe.only_for('System Manager') total = frappe.db.count('User Permission', filters = dict(user=user, allow=for_doctype)) if total: - frappe.db.sql('DELETE FROM `tabUser Permission` WHERE `user`=%s AND `allow`=%s', (user, for_doctype)) + frappe.db.delete("User Permission", { + "user": user, + "allow": for_doctype + }) + # frappe.db.sql('DELETE FROM `tabUser Permission` WHERE `user`=%s AND `allow`=%s', (user, for_doctype)) frappe.clear_cache() return total @@ -232,28 +236,50 @@ def insert_user_perm(user, doctype, docname, is_default=0, hide_descendants=0, a user_perm.insert() def remove_applicable(perm_applied_docs, user, doctype, docname): + for applicable_for in perm_applied_docs: - frappe.db.sql("""DELETE FROM `tabUser Permission` - WHERE `user`=%s - AND `applicable_for`=%s - AND `allow`=%s - AND `for_value`=%s - """, (user, applicable_for, doctype, docname)) - + frappe.db.delete("User Permission", { + "user": user, + "applicable_for": applicable_for, + "allow": doctype, + "for_value": docname + }) + # + # frappe.db.sql("""DELETE FROM `tabUser Permission` + # WHERE `user`=%s + # AND `applicable_for`=%s + # AND `allow`=%s + # AND `for_value`=%s + # """, (user, applicable_for, doctype, docname)) def remove_apply_to_all(user, doctype, docname): - frappe.db.sql("""DELETE from `tabUser Permission` - WHERE `user`=%s - AND `apply_to_all_doctypes`=1 - AND `allow`=%s - AND `for_value`=%s - """,(user, doctype, docname)) + + frappe.db.delete("User Permission", { + "user": user, + "apply_to_all_doctypes": 1, + "allow": doctype, + "for_value": docname + }) + # frappe.db.sql("""DELETE from `tabUser Permission` + # WHERE `user`=%s + # AND `apply_to_all_doctypes`=1 + # AND `allow`=%s + # AND `for_value`=%s + # """,(user, doctype, docname)) def update_applicable(already_applied, to_apply, user, doctype, docname): for applied in already_applied: if applied not in to_apply: - frappe.db.sql("""DELETE FROM `tabUser Permission` - WHERE `user`=%s - AND `applicable_for`=%s - AND `allow`=%s - AND `for_value`=%s - """,(user, applied, doctype, docname)) + + frappe.db.delete("User Permission", { + "user": user, + "applicable_for": applied, + "allow": doctype, + "for_value": docname + }) + + # frappe.db.sql("""DELETE FROM `tabUser Permission` + # WHERE `user`=%s + # AND `applicable_for`=%s + # AND `allow`=%s + # AND `for_value`=%s + # """,(user, applied, doctype, docname)) diff --git a/frappe/core/page/permission_manager/permission_manager.py b/frappe/core/page/permission_manager/permission_manager.py index 15c7cb55ae..0a76283813 100644 --- a/frappe/core/page/permission_manager/permission_manager.py +++ b/frappe/core/page/permission_manager/permission_manager.py @@ -111,8 +111,10 @@ def remove(doctype, role, permlevel): setup_custom_perms(doctype) name = frappe.get_value('Custom DocPerm', dict(parent=doctype, role=role, permlevel=permlevel)) - - frappe.db.sql('delete from `tabCustom DocPerm` where name=%s', name) + frappe.db.delete("Custom DocPerm", { + "name": name + }) + # frappe.db.sql('delete from `tabCustom DocPerm` where name=%s', name) if not frappe.get_all('Custom DocPerm', dict(parent=doctype)): frappe.throw(_('There must be atleast one permission rule.'), title=_('Cannot Remove')) From 9aa17db392d54f52cc7eda223eb3ece276ee74a1 Mon Sep 17 00:00:00 2001 From: Aradhya-Tripathi Date: Wed, 21 Jul 2021 19:16:17 +0530 Subject: [PATCH 15/86] refactor: moved from raw queries to frappe orm in desk --- .../dashboard_chart/test_dashboard_chart.py | 6 ++++-- frappe/desk/doctype/event/event.py | 11 ++++++++--- .../desk/doctype/route_history/route_history.py | 17 +++++++++++------ frappe/desk/doctype/tag/tag.py | 13 +++++++++++-- 4 files changed, 34 insertions(+), 13 deletions(-) diff --git a/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py b/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py index 78d133b2d5..7133640c21 100644 --- a/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py +++ b/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py @@ -64,7 +64,8 @@ class TestDashboardChart(unittest.TestCase): if frappe.db.exists('Dashboard Chart', 'Test Empty Dashboard Chart'): frappe.delete_doc('Dashboard Chart', 'Test Empty Dashboard Chart') - frappe.db.sql('delete from `tabError Log`') + frappe.db.delete("Error Log") + # frappe.db.sql('delete from `tabError Log`') frappe.get_doc(dict( doctype = 'Dashboard Chart', @@ -94,7 +95,8 @@ class TestDashboardChart(unittest.TestCase): if frappe.db.exists('Dashboard Chart', 'Test Empty Dashboard Chart 2'): frappe.delete_doc('Dashboard Chart', 'Test Empty Dashboard Chart 2') - frappe.db.sql('delete from `tabError Log`') + frappe.db.delete("Error Log") + # frappe.db.sql('delete from `tabError Log`') # create one data point frappe.get_doc(dict(doctype = 'Error Log', creation = '2018-06-01 00:00:00')).insert() diff --git a/frappe/desk/doctype/event/event.py b/frappe/desk/doctype/event/event.py index 57c89eaf2e..f9a0dd74fd 100644 --- a/frappe/desk/doctype/event/event.py +++ b/frappe/desk/doctype/event/event.py @@ -338,9 +338,14 @@ def delete_events(ref_type, ref_name, delete_event=False): total_participants = frappe.get_all("Event Participants", filters={"parenttype": "Event", "parent": participation.parent}) if len(total_participants) <= 1: - frappe.db.sql("DELETE FROM `tabEvent` WHERE `name` = %(name)s", {'name': participation.parent}) - - frappe.db.sql("DELETE FROM `tabEvent Participants ` WHERE `name` = %(name)s", {'name': participation.name}) + frappe.db.delete("Event", { + "name": participation.parent + }) + # frappe.db.sql("DELETE FROM `tabEvent` WHERE `name` = %(name)s", {'name': participation.parent}) + frappe.db.delete("Event Participants", { + "name": participation.name + }) + # frappe.db.sql("DELETE FROM `tabEvent Participants ` WHERE `name` = %(name)s", {'name': participation.name}) # Close events if ends_on or repeat_till is less than now_datetime def set_status_of_events(): diff --git a/frappe/desk/doctype/route_history/route_history.py b/frappe/desk/doctype/route_history/route_history.py index b82077f485..bfa2711b8f 100644 --- a/frappe/desk/doctype/route_history/route_history.py +++ b/frappe/desk/doctype/route_history/route_history.py @@ -32,11 +32,16 @@ def flush_old_route_records(): fields=['modified'], order_by='modified desc') - frappe.db.sql(''' - DELETE - FROM `tabRoute History` - WHERE `modified` <= %(modified)s and `user`=%(modified)s - ''', { + frappe.db.delete("Route History", { "modified": last_record_to_keep[0].modified, "user": user - }) \ No newline at end of file + }) + + # frappe.db.sql(''' + # DELETE + # FROM `tabRoute History` + # WHERE `modified` <= %(modified)s and `user`=%(modified)s + # ''', { + # "modified": last_record_to_keep[0].modified, + # "user": user + # }) \ No newline at end of file diff --git a/frappe/desk/doctype/tag/tag.py b/frappe/desk/doctype/tag/tag.py index 4ea5c9cd7e..8b78405d06 100644 --- a/frappe/desk/doctype/tag/tag.py +++ b/frappe/desk/doctype/tag/tag.py @@ -123,7 +123,11 @@ def delete_tags_for_document(doc): if not frappe.db.table_exists("Tag Link"): return - frappe.db.sql("""DELETE FROM `tabTag Link` WHERE `document_type`=%s AND `document_name`=%s""", (doc.doctype, doc.name)) + frappe.db.delete("Tag Link", { + "document_type": doc.doctype, + "document_name": doc.name + }) + # frappe.db.sql("""DELETE FROM `tabTag Link` WHERE `document_type`=%s AND `document_name`=%s""", (doc.doctype, doc.name)) def update_tags(doc, tags): """ @@ -161,7 +165,12 @@ def get_deleted_tags(new_tags, existing_tags): return list(set(existing_tags) - set(new_tags)) def delete_tag_for_document(dt, dn, tag): - frappe.db.sql("""DELETE FROM `tabTag Link` WHERE `document_type`=%s AND `document_name`=%s AND tag=%s""", (dt, dn, tag)) + frappe.db.delete("Tag Link", { + "document_type": dt, + "document_name": dn, + "tag": tag + }) + # frappe.db.sql("""DELETE FROM `tabTag Link` WHERE `document_type`=%s AND `document_name`=%s AND tag=%s""", (dt, dn, tag)) @frappe.whitelist() def get_documents_for_tag(tag): From a256219dd27b5251b90e0d04b1a5b7797117ea6c Mon Sep 17 00:00:00 2001 From: Aradhya-Tripathi Date: Wed, 21 Jul 2021 19:16:59 +0530 Subject: [PATCH 16/86] refactor: moved from raw queries in frappe email to frappe orm --- frappe/email/doctype/unhandled_email/unhandled_email.py | 7 +++++-- frappe/email/queue.py | 1 + 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/frappe/email/doctype/unhandled_email/unhandled_email.py b/frappe/email/doctype/unhandled_email/unhandled_email.py index 6414dbece3..850007d85f 100644 --- a/frappe/email/doctype/unhandled_email/unhandled_email.py +++ b/frappe/email/doctype/unhandled_email/unhandled_email.py @@ -10,5 +10,8 @@ class UnhandledEmail(Document): def remove_old_unhandled_emails(): - frappe.db.sql("""DELETE FROM `tabUnhandled Email` - WHERE creation < %s""", frappe.utils.add_days(frappe.utils.nowdate(), -30)) + frappe.db.delete("Unhandled Email", { + "creation": ("<", frappe.utils.add_days(frappe.utils.nowdate(), -30)) + }) + # frappe.db.sql("""DELETE FROM `tabUnhandled Email` + # WHERE creation < %s""", frappe.utils.add_days(frappe.utils.nowdate(), -30)) \ No newline at end of file diff --git a/frappe/email/queue.py b/frappe/email/queue.py index 885a306cfb..40f1c7be3a 100755 --- a/frappe/email/queue.py +++ b/frappe/email/queue.py @@ -173,6 +173,7 @@ def clear_outbox(days=None): WHERE `priority`=0 AND `modified` < (NOW() - INTERVAL '{0}' DAY)""".format(days)) if email_queues: + # TODO: email_queues IN frappe.db.sql frappe.db.sql("""DELETE FROM `tabEmail Queue` WHERE `name` IN ({0})""".format( ','.join(['%s']*len(email_queues) )), tuple(email_queues)) From eb7e95ae26a542e82066b302e6bc85edffe7df25 Mon Sep 17 00:00:00 2001 From: Aradhya-Tripathi Date: Wed, 21 Jul 2021 19:19:06 +0530 Subject: [PATCH 17/86] refactor: using frappe orm --- .../doctype/custom_field/custom_field.py | 14 ++++-- .../doctype/customize_form/customize_form.py | 8 +++- frappe/model/__init__.py | 26 +++++++---- frappe/model/delete_doc.py | 46 +++++++++++++++---- frappe/permissions.py | 7 ++- frappe/sessions.py | 5 +- 6 files changed, 78 insertions(+), 28 deletions(-) diff --git a/frappe/custom/doctype/custom_field/custom_field.py b/frappe/custom/doctype/custom_field/custom_field.py index 7e6ea1875a..1d8f4ae67a 100644 --- a/frappe/custom/doctype/custom_field/custom_field.py +++ b/frappe/custom/doctype/custom_field/custom_field.py @@ -85,11 +85,15 @@ class CustomField(Document): frappe.bold(self.label))) # delete property setter entries - frappe.db.sql("""\ - DELETE FROM `tabProperty Setter` - WHERE doc_type = %s - AND field_name = %s""", - (self.dt, self.fieldname)) + frappe.db.delete("Property Setter", { + "doc_type": self.dt, + "field_name": self.fieldname + }) + # frappe.db.sql("""\ + # DELETE FROM `tabProperty Setter` + # WHERE doc_type = %s + # AND field_name = %s""", + # (self.dt, self.fieldname)) frappe.clear_cache(doctype=self.dt) diff --git a/frappe/custom/doctype/customize_form/customize_form.py b/frappe/custom/doctype/customize_form/customize_form.py index 1b8977acc4..3b7de448d3 100644 --- a/frappe/custom/doctype/customize_form/customize_form.py +++ b/frappe/custom/doctype/customize_form/customize_form.py @@ -20,8 +20,12 @@ from frappe.core.doctype.doctype.doctype import validate_series class CustomizeForm(Document): def on_update(self): - frappe.db.sql("delete from tabSingles where doctype='Customize Form'") - frappe.db.sql("delete from `tabCustomize Form Field`") + frappe.db.delete("Singles", { + "doctype": "Customize Form" + }) + frappe.db.delete("Customize Form Field") + # frappe.db.sql("delete from tabSingles where doctype='Customize Form'") + # frappe.db.sql("delete from `tabCustomize Form Field`") @frappe.whitelist() def fetch_to_customize(self): diff --git a/frappe/model/__init__.py b/frappe/model/__init__.py index 75122f5aba..763fd36cc7 100644 --- a/frappe/model/__init__.py +++ b/frappe/model/__init__.py @@ -151,21 +151,29 @@ def delete_fields(args_dict, delete=0): fields = args_dict[dt] if not fields: continue - - frappe.db.sql(""" - DELETE FROM `tabDocField` - WHERE parent='%s' AND fieldname IN (%s) - """ % (dt, ", ".join(["'{}'".format(f) for f in fields]))) + + frappe.db.delete("DocField", { + "parent": dt, + "fieldname": ("in", ", ".join(["'{}'".format(f) for f in fields])) + }) + # frappe.db.sql(""" + # DELETE FROM `tabDocField` + # WHERE parent='%s' AND fieldname IN (%s) + # """ % (dt, ", ".join(["'{}'".format(f) for f in fields]))) # Delete the data/column only if delete is specified if not delete: continue if frappe.db.get_value("DocType", dt, "issingle"): - frappe.db.sql(""" - DELETE FROM `tabSingles` - WHERE doctype='%s' AND field IN (%s) - """ % (dt, ", ".join("'{}'".format(f) for f in fields))) + frappe.db.delete("Singles", { + "doctype": dt, + "field": ("in", ", ".join("'{}'".format(f) for f in fields)) + }) + # frappe.db.sql(""" + # DELETE FROM `tabSingles` + # WHERE doctype='%s' AND field IN (%s) + # """ % (dt, ", ".join("'{}'".format(f) for f in fields))) else: existing_fields = frappe.db.multisql({ "mariadb": "DESC `tab%s`" % dt, diff --git a/frappe/model/delete_doc.py b/frappe/model/delete_doc.py index cc88cfa106..ec53d060fd 100644 --- a/frappe/model/delete_doc.py +++ b/frappe/model/delete_doc.py @@ -65,12 +65,36 @@ def delete_doc(doctype=None, name=None, force=0, ignore_doctypes=None, for_reloa update_flags(doc, flags, ignore_permissions) check_permission_and_not_submitted(doc) - frappe.db.sql("delete from `tabCustom Field` where dt = %s", name) - frappe.db.sql("delete from `tabClient Script` where dt = %s", name) - frappe.db.sql("delete from `tabProperty Setter` where doc_type = %s", name) - frappe.db.sql("delete from `tabReport` where ref_doctype=%s", name) - frappe.db.sql("delete from `tabCustom DocPerm` where parent=%s", name) - frappe.db.sql("delete from `__global_search` where doctype=%s", name) + frappe.db.delete("Custom Field", { + "dt": name + }) + + frappe.db.delete("Client Script", { + "dt": name + }) + + frappe.db.delete("Property Setter", { + "doc_type": name + }) + + frappe.db.delete("Report", { + "ref_doctype": name + }) + + frappe.db.delete("Custom DocPerm", { + "parent": name + }) + + frappe.db.delete("__global_search", { + "doctype": name + }) + + # frappe.db.sql("delete from `tabCustom Field` where dt = %s", name) + # frappe.db.sql("delete from `tabClient Script` where dt = %s", name) + # frappe.db.sql("delete from `tabProperty Setter` where doc_type = %s", name) + # frappe.db.sql("delete from `tabReport` where ref_doctype=%s", name) + # frappe.db.sql("delete from `tabCustom DocPerm` where parent=%s", name) + # frappe.db.sql("delete from `__global_search` where doctype=%s", name) delete_from_table(doctype, name, ignore_doctypes, None) @@ -162,9 +186,15 @@ def update_naming_series(doc): def delete_from_table(doctype, name, ignore_doctypes, doc): if doctype!="DocType" and doctype==name: - frappe.db.sql("delete from `tabSingles` where `doctype`=%s", name) + frappe.db.delete("Singles", { + "doctype": name + }) + # frappe.db.sql("delete from `tabSingles` where `doctype`=%s", name) else: - frappe.db.sql("delete from `tab{0}` where `name`=%s".format(doctype), name) + frappe.db.delete(f"{doctype}", { + "name": name + }) + # frappe.db.sql("delete from `tab{0}` where `name`=%s".format(doctype), name) # get child tables if doc: diff --git a/frappe/permissions.py b/frappe/permissions.py index 07b4a2e68f..4cd846e52c 100644 --- a/frappe/permissions.py +++ b/frappe/permissions.py @@ -6,10 +6,12 @@ import frappe import frappe.share from frappe import _, msgprint from frappe.utils import cint - +from frappe.database.database import Database rights = ("select", "read", "write", "create", "delete", "submit", "cancel", "amend", "print", "email", "report", "import", "export", "set_user_permissions", "share") + + def check_admin_or_system_manager(user=None): if not user: user = frappe.session.user @@ -516,8 +518,9 @@ def reset_perms(doctype): """Reset permissions for given doctype.""" from frappe.desk.notifications import delete_notification_count_for delete_notification_count_for(doctype) + frappe.db.delete(doctype="Custom DocPerm", conditions={"parent": doctype}) - frappe.db.sql("""delete from `tabCustom DocPerm` where parent=%s""", doctype) + # frappe.db.sql("""delete from `tabCustom DocPerm` where parent=%s""", doctype) def get_linked_doctypes(dt): return list(set([dt] + [d.options for d in diff --git a/frappe/sessions.py b/frappe/sessions.py index 4d922d6769..ae2bd02d98 100644 --- a/frappe/sessions.py +++ b/frappe/sessions.py @@ -16,6 +16,7 @@ import frappe.translate import redis from urllib.parse import unquote from frappe.cache_manager import clear_user_cache +from frappe.database.database import Database @frappe.whitelist(allow_guest=True) def clear(user=None): @@ -76,7 +77,6 @@ def get_sessions_to_clear(user=None, keep_current=False, device=None): def delete_session(sid=None, user=None, reason="Session Expired"): from frappe.core.doctype.activity_log.feed import logout_feed - frappe.cache().hdel("session", sid) frappe.cache().hdel("last_db_session_update", sid) if sid and not user: @@ -84,7 +84,8 @@ def delete_session(sid=None, user=None, reason="Session Expired"): if user_details: user = user_details[0].get("user") logout_feed(user, reason) - frappe.db.sql("""delete from tabSessions where sid=%s""", sid) + frappe.db.delete(doctype="Sessions", conditions={"sid": sid}) + # frappe.db.sql("""delete from tabSessions where sid=%s""", sid) frappe.db.commit() def clear_all_sessions(reason=None): From 931549ee054e8193591e4ff5e6796144a74d94ee Mon Sep 17 00:00:00 2001 From: Aradhya-Tripathi Date: Wed, 21 Jul 2021 19:20:35 +0530 Subject: [PATCH 18/86] refactor: changed delete to accept no conditions --- frappe/database/database.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index 6012e47445..5b4d813011 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -951,15 +951,25 @@ class Database(object): query = sql_dict.get(current_dialect) return self.sql(query, values, **kwargs) - def delete(self, doctype, conditions, debug=False): + def delete(self, doctype, conditions=None, debug=False): if conditions: conditions, values = self.build_conditions(conditions) - return self.sql("DELETE FROM `tab{doctype}` where {conditions}".format( + if doctype.startswith("__"): + return self.sql("DELETE FROM `{doctype}` where {conditions}".format( doctype=doctype, conditions=conditions ), values, debug=debug) + else: + return self.sql("DELETE FROM `tab{doctype}` where {conditions}".format( + doctype=doctype, + conditions=conditions + ), values, debug=debug) + else: - frappe.throw(_('No conditions provided')) + return self.sql("DELETE FROM `tab{doctype}`".format( + doctype=doctype + ), debug=debug) + def get_last_created(self, doctype): last_record = self.get_all(doctype, ('creation'), limit=1, order_by='creation desc') From dd2648b1414e47b46071109ab3bba9c5ed6a37f9 Mon Sep 17 00:00:00 2001 From: Aradhya-Tripathi Date: Wed, 21 Jul 2021 23:28:17 +0530 Subject: [PATCH 19/86] refactor: Moving from core raw queries to frappe orm --- frappe/core/doctype/activity_log/feed.py | 6 +++--- .../domain_settings/domain_settings.py | 2 +- frappe/core/doctype/error_log/error_log.py | 3 ++- frappe/core/doctype/report/test_report.py | 2 +- frappe/core/doctype/role/role.py | 2 +- .../scheduled_job_type/scheduled_job_type.py | 2 +- frappe/core/doctype/user/user.py | 19 +++++++++---------- 7 files changed, 18 insertions(+), 18 deletions(-) diff --git a/frappe/core/doctype/activity_log/feed.py b/frappe/core/doctype/activity_log/feed.py index cf8f7416e8..92e18d4a5a 100644 --- a/frappe/core/doctype/activity_log/feed.py +++ b/frappe/core/doctype/activity_log/feed.py @@ -29,9 +29,9 @@ def update_feed(doc, method=None): name = feed.name or doc.name - frappe.db.delete(doctype="Activity Log", conditions={"reference_doctype": doctype, - "reference_name": name, - "link_doctype": name.feed.link_doctype}) + frappe.db.delete("Activity Log", {"reference_doctype": doctype, + "reference_name": name, + "link_doctype": name.feed.link_doctype}) # frappe.db.sql("""delete from `tabActivity Log` # where # reference_doctype=%s and reference_name=%s diff --git a/frappe/core/doctype/domain_settings/domain_settings.py b/frappe/core/doctype/domain_settings/domain_settings.py index 6ca180def1..4ed9d1e15a 100644 --- a/frappe/core/doctype/domain_settings/domain_settings.py +++ b/frappe/core/doctype/domain_settings/domain_settings.py @@ -34,7 +34,7 @@ class DomainSettings(Document): all_domains = list((frappe.get_hooks('domains') or {})) def remove_role(role): - frappe.db.delete(doctype="Has Role", conditions={"role": role}) + frappe.db.delete("Has Role", {"role": role}) # frappe.db.sql('delete from `tabHas Role` where role=%s', role) frappe.set_value('Role', role, 'disabled', 1) diff --git a/frappe/core/doctype/error_log/error_log.py b/frappe/core/doctype/error_log/error_log.py index 8223238c57..9a7333995c 100644 --- a/frappe/core/doctype/error_log/error_log.py +++ b/frappe/core/doctype/error_log/error_log.py @@ -20,4 +20,5 @@ def set_old_logs_as_seen(): def clear_error_logs(): '''Flush all Error Logs''' frappe.only_for('System Manager') - frappe.db.sql('''DELETE FROM `tabError Log`''') \ No newline at end of file + frappe.db.delete("Error Log") + # frappe.db.sql('''DELETE FROM `tabError Log`''') \ No newline at end of file diff --git a/frappe/core/doctype/report/test_report.py b/frappe/core/doctype/report/test_report.py index 2bab6adf19..2ac41cca2b 100644 --- a/frappe/core/doctype/report/test_report.py +++ b/frappe/core/doctype/report/test_report.py @@ -83,7 +83,7 @@ class TestReport(unittest.TestCase): def test_report_permissions(self): frappe.set_user('test@example.com') - frappe.db.delete(doctype="Has Role", conditions={ + frappe.db.delete("Has Role", { "parent": frappe.session.user, "role": "Test Has Role" }) diff --git a/frappe/core/doctype/role/role.py b/frappe/core/doctype/role/role.py index fae8ae2d28..dee25eb476 100644 --- a/frappe/core/doctype/role/role.py +++ b/frappe/core/doctype/role/role.py @@ -38,7 +38,7 @@ class Role(Document): self.set(key, 0) def remove_roles(self): - frappe.db.delete(doctype="Has Role", conditions={"role": self.name}) + frappe.db.delete("Has Role", {"role": self.name}) # frappe.db.sql("delete from `tabHas Role` where role = %s", self.name) frappe.clear_cache() diff --git a/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py b/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py index d6c94aded1..9882f0edfe 100644 --- a/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py +++ b/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py @@ -110,7 +110,7 @@ class ScheduledJobType(Document): def on_trash(self): - frappe.db.delete(doctype="Scheduled Job Log", conditions={ + frappe.db.delete("Scheduled Job Log", { "scheduled_job_type": self.name }) # frappe.db.sql('delete from `tabScheduled Job Log` where scheduled_job_type=%s', self.name) diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py index f69af4e4a8..166f61de8d 100644 --- a/frappe/core/doctype/user/user.py +++ b/frappe/core/doctype/user/user.py @@ -17,7 +17,6 @@ from frappe.website.utils import is_signup_disabled from frappe.rate_limiter import rate_limit from frappe.utils.background_jobs import enqueue from frappe.core.doctype.user_type.user_type import user_linked_with_permission_on_doctype -from frappe.database.database import Database STANDARD_USERS = ("Guest", "Administrator") @@ -382,15 +381,15 @@ class User(Document): # frappe.db.sql("""delete from `tabDocShare` where user=%s""", self.name) # delete messages - # TODO: CHANGE THIS FROM ABHISHEK KA PYPIKA - frappe.db.delete("Communication", { - "reference_doctype": "User", - "communication_type": ("in", ("Chat", "Notification")), - }) - # frappe.db.sql("""delete from `tabCommunication` - # where communication_type in ('Chat', 'Notification') - # and reference_doctype='User' - # and (reference_name=%s or owner=%s)""", (self.name, self.name)) + + # frappe.db.delete("Communication", { + # "reference_doctype": "User", + # "communication_type": ("in", ("Chat", "Notification")), + # }) + frappe.db.sql("""delete from `tabCommunication` + where communication_type in ('Chat', 'Notification') + and reference_doctype='User' + and (reference_name=%s or owner=%s)""", (self.name, self.name)) # unlink contact frappe.db.sql("""update `tabContact` From 79ccaa95f3aff078a3ebdc68291b270b7cbeed26 Mon Sep 17 00:00:00 2001 From: Aradhya-Tripathi Date: Wed, 21 Jul 2021 23:29:50 +0530 Subject: [PATCH 20/86] refactor: Moved raw queries to frappe orm --- .../personal_data_deletion_request.py | 18 ++++++++++++------ .../doctype/workflow_action/workflow_action.py | 12 +++++++++--- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py b/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py index 63ba96d138..b23c7fa11d 100644 --- a/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py +++ b/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py @@ -323,12 +323,18 @@ class PersonalDataDeletionRequest(Document): def remove_unverified_record(): - frappe.db.sql( - """ - DELETE FROM `tabPersonal Data Deletion Request` - WHERE `status` = 'Pending Verification' - AND `creation` < (NOW() - INTERVAL '7' DAY)""" - ) + + frappe.db.delete("Personal Data Deletion Request", { + "status": "Pending Verification", + "creation": ("<", ["Now()", "-", "INTERVAL 7 DAY"]) + }) + + # frappe.db.sql( + # """ + # DELETE FROM `tabPersonal Data Deletion Request` + # WHERE `status` = 'Pending Verification' + # AND `creation` < (NOW() - INTERVAL '7' DAY)""" + # ) @frappe.whitelist(allow_guest=True) diff --git a/frappe/workflow/doctype/workflow_action/workflow_action.py b/frappe/workflow/doctype/workflow_action/workflow_action.py index b70ffb2406..af168d3477 100644 --- a/frappe/workflow/doctype/workflow_action/workflow_action.py +++ b/frappe/workflow/doctype/workflow_action/workflow_action.py @@ -133,9 +133,15 @@ def return_link_expired_page(doc, doc_workflow_state): def clear_old_workflow_actions(doc, user=None): user = user if user else frappe.session.user - frappe.db.sql("""DELETE FROM `tabWorkflow Action` - WHERE `reference_doctype`=%s AND `reference_name`=%s AND `user`!=%s AND `status`='Open'""", - (doc.get('doctype'), doc.get('name'), user)) + frappe.db.delete("Workflow Action", { + "reference_doctype": doc.get("doctype"), + "reference_name": doc.get("name"), + "user": ("!=", user), + "status": "Open" + }) + # frappe.db.sql("""DELETE FROM `tabWorkflow Action` + # WHERE `reference_doctype`=%s AND `reference_name`=%s AND `user`!=%s AND `status`='Open'""", + # (doc.get('doctype'), doc.get('name'), user)) def update_completed_workflow_actions(doc, user=None): user = user if user else frappe.session.user From 888783682072330987f713bd700b8bb1f7db916d Mon Sep 17 00:00:00 2001 From: Aradhya-Tripathi Date: Wed, 21 Jul 2021 23:31:15 +0530 Subject: [PATCH 21/86] refactor: Moved util queries to frappe orm --- frappe/utils/global_search.py | 25 +++++++++++++++++-------- frappe/utils/password.py | 15 +++++++++++---- 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/frappe/utils/global_search.py b/frappe/utils/global_search.py index 8fa2ea474f..a018761370 100644 --- a/frappe/utils/global_search.py +++ b/frappe/utils/global_search.py @@ -23,7 +23,8 @@ def reset(): Deletes all data in __global_search :return: """ - frappe.db.sql('DELETE FROM `__global_search`') + frappe.db.delete("__global_search") + # frappe.db.sql('DELETE FROM `__global_search`') def get_doctypes_with_global_search(with_child_tables=True): @@ -146,9 +147,12 @@ def rebuild_for_doctype(doctype): def delete_global_search_records_for_doctype(doctype): - frappe.db.sql('''DELETE - FROM `__global_search` - WHERE doctype = %s''', doctype, as_dict=True) + frappe.db.delete("__global_search", { + "doctype": doctype + }) + # frappe.db.sql('''DELETE + # FROM `__global_search` + # WHERE doctype = %s''', doctype, as_dict=True) def get_selected_fields(meta, global_search_fields): @@ -400,10 +404,15 @@ def delete_for_document(doc): :param doc: Deleted document """ - frappe.db.sql('''DELETE - FROM `__global_search` - WHERE doctype = %s - AND name = %s''', (doc.doctype, doc.name), as_dict=True) + frappe.db.delete("__global_search", { + "doctype": doc.doctype, + "name": doc.name + }) + + # frappe.db.sql('''DELETE + # FROM `__global_search` + # WHERE doctype = %s + # AND name = %s''', (doc.doctype, doc.name), as_dict=True) @frappe.whitelist() diff --git a/frappe/utils/password.py b/frappe/utils/password.py index 428f2e9577..caa89af64e 100644 --- a/frappe/utils/password.py +++ b/frappe/utils/password.py @@ -65,10 +65,17 @@ def set_encrypted_password(doctype, name, pwd, fieldname='password'): def remove_encrypted_password(doctype, name, fieldname='password'): - frappe.db.sql( - 'DELETE FROM `__Auth` WHERE doctype = %s and name = %s and fieldname = %s', - values=[doctype, name, fieldname] - ) + + frappe.db.delete("__Auth", { + "doctype": doctype, + "name": name, + "fieldname": fieldname + }) + + # frappe.db.sql( + # 'DELETE FROM `__Auth` WHERE doctype = %s and name = %s and fieldname = %s', + # values=[doctype, name, fieldname] + # ) def check_password(user, pwd, doctype='User', fieldname='password', delete_tracker_cache=True): From 8feb430371d38b34ca90f6ae2f08bd651e812a64 Mon Sep 17 00:00:00 2001 From: Aradhya-Tripathi Date: Wed, 21 Jul 2021 23:32:18 +0530 Subject: [PATCH 22/86] refactor: Moved raw queries to frappe orm --- frappe/model/__init__.py | 2 +- frappe/model/delete_doc.py | 8 ++++++-- .../v11_0/apply_customization_to_custom_doctype.py | 10 ++++++++-- .../v11_0/sync_stripe_settings_before_migrate.py | 5 ++++- .../v12_0/delete_feedback_request_if_exists.py | 11 +++++++---- .../v12_0/remove_deprecated_fields_from_doctype.py | 12 ++++++++---- 6 files changed, 34 insertions(+), 14 deletions(-) diff --git a/frappe/model/__init__.py b/frappe/model/__init__.py index 763fd36cc7..99eaa7ff5d 100644 --- a/frappe/model/__init__.py +++ b/frappe/model/__init__.py @@ -154,7 +154,7 @@ def delete_fields(args_dict, delete=0): frappe.db.delete("DocField", { "parent": dt, - "fieldname": ("in", ", ".join(["'{}'".format(f) for f in fields])) + "fieldname": ("in", fields) }) # frappe.db.sql(""" # DELETE FROM `tabDocField` diff --git a/frappe/model/delete_doc.py b/frappe/model/delete_doc.py index ec53d060fd..9b5ebb1031 100644 --- a/frappe/model/delete_doc.py +++ b/frappe/model/delete_doc.py @@ -369,8 +369,12 @@ def clear_references(doctype, reference_doctype, reference_name, (reference_doctype, reference_name)) def clear_timeline_references(link_doctype, link_name): - frappe.db.sql("""DELETE FROM `tabCommunication Link` - WHERE `tabCommunication Link`.link_doctype=%s AND `tabCommunication Link`.link_name=%s""", (link_doctype, link_name)) + frappe.db.delete("Communication Link", { + "link_doctype": link_doctype, + "link_name": link_name + }) + # frappe.db.sql("""DELETE FROM `tabCommunication Link` + # WHERE `tabCommunication Link`.link_doctype=%s AND `tabCommunication Link`.link_name=%s""", (link_doctype, link_name)) def insert_feed(doc): if ( diff --git a/frappe/patches/v11_0/apply_customization_to_custom_doctype.py b/frappe/patches/v11_0/apply_customization_to_custom_doctype.py index 49b68ed240..c13ff83a1f 100644 --- a/frappe/patches/v11_0/apply_customization_to_custom_doctype.py +++ b/frappe/patches/v11_0/apply_customization_to_custom_doctype.py @@ -28,7 +28,10 @@ def execute(): for prop in property_setters: property_setter_map[prop.field_name] = prop - frappe.db.sql('DELETE FROM `tabProperty Setter` WHERE `name`=%s', prop.name) + frappe.db.delete("Property Setter", { + "name": prop.name + }) + # frappe.db.sql('DELETE FROM `tabProperty Setter` WHERE `name`=%s', prop.name) meta = frappe.get_meta(doctype.name) @@ -50,6 +53,9 @@ def execute(): df = frappe.new_doc('DocField', meta, 'fields') df.update(cf) meta.fields.append(df) - frappe.db.sql('DELETE FROM `tabCustom Field` WHERE name=%s', cf.name) + frappe.db.delete("Custom Field", { + "name": cf.name + }) + # frappe.db.sql('DELETE FROM `tabCustom Field` WHERE name=%s', cf.name) meta.save() diff --git a/frappe/patches/v11_0/sync_stripe_settings_before_migrate.py b/frappe/patches/v11_0/sync_stripe_settings_before_migrate.py index a8e9bd4de1..1a12d4dc1d 100644 --- a/frappe/patches/v11_0/sync_stripe_settings_before_migrate.py +++ b/frappe/patches/v11_0/sync_stripe_settings_before_migrate.py @@ -17,4 +17,7 @@ def execute(): settings.secret_key = secret_key settings.save(ignore_permissions=True) - frappe.db.sql("""DELETE FROM tabSingles WHERE doctype='Stripe Settings'""") \ No newline at end of file + frappe.db.delete("Singles", { + "doctype": "Stripe Settings" + }) + # frappe.db.sql("""DELETE FROM tabSingles WHERE doctype='Stripe Settings'""") \ No newline at end of file diff --git a/frappe/patches/v12_0/delete_feedback_request_if_exists.py b/frappe/patches/v12_0/delete_feedback_request_if_exists.py index fdbcecfc5a..537d509424 100644 --- a/frappe/patches/v12_0/delete_feedback_request_if_exists.py +++ b/frappe/patches/v12_0/delete_feedback_request_if_exists.py @@ -2,7 +2,10 @@ import frappe def execute(): - frappe.db.sql(''' - DELETE from `tabDocType` - WHERE name = 'Feedback Request' - ''') \ No newline at end of file + frappe.db.delete("DocType", { + "name": "Feedback Request" + }) + # frappe.db.sql(''' + # DELETE from `tabDocType` + # WHERE name = 'Feedback Request' + # ''') \ No newline at end of file diff --git a/frappe/patches/v12_0/remove_deprecated_fields_from_doctype.py b/frappe/patches/v12_0/remove_deprecated_fields_from_doctype.py index 60599066e6..17dfbc5ca1 100644 --- a/frappe/patches/v12_0/remove_deprecated_fields_from_doctype.py +++ b/frappe/patches/v12_0/remove_deprecated_fields_from_doctype.py @@ -8,7 +8,11 @@ def execute(): 'DocType': ['hide_heading', 'image_view', 'read_only_onload'] }, delete=1) - frappe.db.sql(''' - DELETE from `tabProperty Setter` - WHERE property = 'read_only_onload' - ''') + frappe.db.delete("Property Setter", { + "property": "read_only_onload" + }) + + # frappe.db.sql(''' + # DELETE from `tabProperty Setter` + # WHERE property = 'read_only_onload' + # ''') From 4dc476da6acad27c20afba1bf66074a774b899ec Mon Sep 17 00:00:00 2001 From: Aradhya-Tripathi Date: Thu, 22 Jul 2021 17:00:24 +0530 Subject: [PATCH 23/86] fix(update_feed): Malformed query in activity log --- frappe/core/doctype/activity_log/feed.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/core/doctype/activity_log/feed.py b/frappe/core/doctype/activity_log/feed.py index 92e18d4a5a..9b48d0550a 100644 --- a/frappe/core/doctype/activity_log/feed.py +++ b/frappe/core/doctype/activity_log/feed.py @@ -31,7 +31,7 @@ def update_feed(doc, method=None): frappe.db.delete("Activity Log", {"reference_doctype": doctype, "reference_name": name, - "link_doctype": name.feed.link_doctype}) + "link_doctype": feed.link_doctype}) # frappe.db.sql("""delete from `tabActivity Log` # where # reference_doctype=%s and reference_name=%s From 1204ae8ce40c2ebb34f54dfdf970199aa48511e0 Mon Sep 17 00:00:00 2001 From: Aradhya-Tripathi Date: Thu, 22 Jul 2021 17:26:24 +0530 Subject: [PATCH 24/86] fix: fixed spacing and ui tests --- frappe/core/doctype/activity_log/feed.py | 12 ++++++------ frappe/defaults.py | 14 +++++++++----- frappe/desk/doctype/desktop_icon/desktop_icon.py | 5 ++++- frappe/model/delete_doc.py | 6 ------ frappe/permissions.py | 2 +- frappe/sessions.py | 2 +- frappe/utils/password.py | 2 -- .../personal_data_deletion_request.py | 2 -- 8 files changed, 21 insertions(+), 24 deletions(-) diff --git a/frappe/core/doctype/activity_log/feed.py b/frappe/core/doctype/activity_log/feed.py index 9b48d0550a..d91a21ec44 100644 --- a/frappe/core/doctype/activity_log/feed.py +++ b/frappe/core/doctype/activity_log/feed.py @@ -26,12 +26,12 @@ def update_feed(doc, method=None): feed = frappe._dict(feed) doctype = feed.doctype or doc.doctype - name = feed.name or doc.name - - - frappe.db.delete("Activity Log", {"reference_doctype": doctype, - "reference_name": name, - "link_doctype": feed.link_doctype}) + name = feed.name or doc.name + frappe.db.delete("Activity Log", { + "reference_doctype": doctype, + "reference_name": name, + "link_doctype": feed.link_doctype + }) # frappe.db.sql("""delete from `tabActivity Log` # where # reference_doctype=%s and reference_name=%s diff --git a/frappe/defaults.py b/frappe/defaults.py index fde48d71ff..ee73e86a91 100644 --- a/frappe/defaults.py +++ b/frappe/defaults.py @@ -124,11 +124,15 @@ def set_default(key, value, parent, parenttype="__default"): where defkey=%s and parent=%s for update''', (key, parent)): - frappe.db.sql(""" - delete from - `tabDefaultValue` - where - defkey=%s and parent=%s""", (key, parent)) + frappe.db.delete("DefaultValue", { + "defkey": key, + "parent": parent + }) + # frappe.db.sql(""" + # delete from + # `tabDefaultValue` + # where + # defkey=%s and parent=%s""", (key, parent)) if value != None: add_default(key, value, parent) else: diff --git a/frappe/desk/doctype/desktop_icon/desktop_icon.py b/frappe/desk/doctype/desktop_icon/desktop_icon.py index 81a79cdb09..bc5677e9b9 100644 --- a/frappe/desk/doctype/desktop_icon/desktop_icon.py +++ b/frappe/desk/doctype/desktop_icon/desktop_icon.py @@ -197,7 +197,10 @@ def set_desktop_icons(visible_list, ignore_duplicate=True): # clear all custom only if setup is not complete if not int(frappe.defaults.get_defaults().setup_complete or 0): - frappe.db.sql('delete from `tabDesktop Icon` where standard=0') + frappe.db.delete("Desktop Icon", { + "standard": 0 + }) + # frappe.db.sql('delete from `tabDesktop Icon` where standard=0') # set standard as blocked and hidden if setting first active domain if not frappe.flags.keep_desktop_icons: diff --git a/frappe/model/delete_doc.py b/frappe/model/delete_doc.py index 9b5ebb1031..03510d489d 100644 --- a/frappe/model/delete_doc.py +++ b/frappe/model/delete_doc.py @@ -64,27 +64,21 @@ def delete_doc(doctype=None, name=None, force=0, ignore_doctypes=None, for_reloa update_flags(doc, flags, ignore_permissions) check_permission_and_not_submitted(doc) - frappe.db.delete("Custom Field", { "dt": name }) - frappe.db.delete("Client Script", { "dt": name }) - frappe.db.delete("Property Setter", { "doc_type": name }) - frappe.db.delete("Report", { "ref_doctype": name }) - frappe.db.delete("Custom DocPerm", { "parent": name }) - frappe.db.delete("__global_search", { "doctype": name }) diff --git a/frappe/permissions.py b/frappe/permissions.py index 4cd846e52c..293ac14f27 100644 --- a/frappe/permissions.py +++ b/frappe/permissions.py @@ -518,7 +518,7 @@ def reset_perms(doctype): """Reset permissions for given doctype.""" from frappe.desk.notifications import delete_notification_count_for delete_notification_count_for(doctype) - frappe.db.delete(doctype="Custom DocPerm", conditions={"parent": doctype}) + frappe.db.delete("Custom DocPerm", {"parent": doctype}) # frappe.db.sql("""delete from `tabCustom DocPerm` where parent=%s""", doctype) diff --git a/frappe/sessions.py b/frappe/sessions.py index ae2bd02d98..3a584dfb74 100644 --- a/frappe/sessions.py +++ b/frappe/sessions.py @@ -84,7 +84,7 @@ def delete_session(sid=None, user=None, reason="Session Expired"): if user_details: user = user_details[0].get("user") logout_feed(user, reason) - frappe.db.delete(doctype="Sessions", conditions={"sid": sid}) + frappe.db.delete("Sessions", {"sid": sid}) # frappe.db.sql("""delete from tabSessions where sid=%s""", sid) frappe.db.commit() diff --git a/frappe/utils/password.py b/frappe/utils/password.py index caa89af64e..c144d866f3 100644 --- a/frappe/utils/password.py +++ b/frappe/utils/password.py @@ -65,7 +65,6 @@ def set_encrypted_password(doctype, name, pwd, fieldname='password'): def remove_encrypted_password(doctype, name, fieldname='password'): - frappe.db.delete("__Auth", { "doctype": doctype, "name": name, @@ -77,7 +76,6 @@ def remove_encrypted_password(doctype, name, fieldname='password'): # values=[doctype, name, fieldname] # ) - def check_password(user, pwd, doctype='User', fieldname='password', delete_tracker_cache=True): '''Checks if user and password are correct, else raises frappe.AuthenticationError''' diff --git a/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py b/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py index b23c7fa11d..bde6f2a4ea 100644 --- a/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py +++ b/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py @@ -323,7 +323,6 @@ class PersonalDataDeletionRequest(Document): def remove_unverified_record(): - frappe.db.delete("Personal Data Deletion Request", { "status": "Pending Verification", "creation": ("<", ["Now()", "-", "INTERVAL 7 DAY"]) @@ -336,7 +335,6 @@ def remove_unverified_record(): # AND `creation` < (NOW() - INTERVAL '7' DAY)""" # ) - @frappe.whitelist(allow_guest=True) def confirm_deletion(email, name, host_name): if not verify_request(): From bc39fd869fac10ee10b4cafd471e85227eace2b9 Mon Sep 17 00:00:00 2001 From: Aradhya-Tripathi Date: Thu, 22 Jul 2021 19:09:39 +0530 Subject: [PATCH 25/86] fix: Fixed typos --- frappe/core/doctype/user/user.py | 2 +- frappe/database/database.py | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py index 166f61de8d..69d3b67220 100644 --- a/frappe/core/doctype/user/user.py +++ b/frappe/core/doctype/user/user.py @@ -366,7 +366,7 @@ class User(Document): if getattr(frappe.local, "login_manager", None): frappe.local.login_manager.logout(user=self.name) - frappe.db.delete("Todo", {"owner": self.name}) + frappe.db.delete("ToDo", {"owner": self.name}) # frappe.db.sql("""DELETE FROM `tabToDo` WHERE `owner`=%s""", (self.name,)) frappe.db.sql("""UPDATE `tabToDo` SET `assigned_by`=NULL WHERE `assigned_by`=%s""", (self.name,)) diff --git a/frappe/database/database.py b/frappe/database/database.py index 5b4d813011..60185106e1 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -966,9 +966,15 @@ class Database(object): ), values, debug=debug) else: - return self.sql("DELETE FROM `tab{doctype}`".format( - doctype=doctype - ), debug=debug) + if doctype.startwith("__"): + return self.sql("DELETE FROM `{doctype}`".format( + doctype=doctype + ), debug=debug) + + else: + return self.sql("DELETE FROM `tab{doctype}`".format( + doctype=doctype + ), debug=debug) def get_last_created(self, doctype): From 1c82b83be07f016ca86149f40a5fac1f21f705e9 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 23 Jul 2021 10:52:22 +0530 Subject: [PATCH 26/86] refactor: Simplify frappe.db.delete logic * Rename conditions kwarg to filters. Handle conditions kwarg if passed * Re-added exc raise for no filters...to risky. Thinking of having an extra kwarg like `force` or something...undecided * Added Python type hints * Accept and pass kwargs to frappe.db.sql...Now pass any kwarg db.sql accepts * Pass debug from fn def if someone is still using it as a positional arg :thonk: --- frappe/database/database.py | 38 ++++++++++++++++--------------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index 60185106e1..cdf15f9c3d 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -6,6 +6,7 @@ import re import time +from typing import Dict, List, Union import frappe import datetime import frappe.defaults @@ -951,30 +952,23 @@ class Database(object): query = sql_dict.get(current_dialect) return self.sql(query, values, **kwargs) - def delete(self, doctype, conditions=None, debug=False): - if conditions: - conditions, values = self.build_conditions(conditions) - if doctype.startswith("__"): - return self.sql("DELETE FROM `{doctype}` where {conditions}".format( - doctype=doctype, - conditions=conditions - ), values, debug=debug) - else: - return self.sql("DELETE FROM `tab{doctype}` where {conditions}".format( - doctype=doctype, - conditions=conditions - ), values, debug=debug) + def delete(self, doctype: str, filters: Union[Dict, List], debug=False, **kwargs): + """Delete rows from a table in site which match the passed filters. This + does trigger DocType hooks. Simply runs a DELETE query in the database. + """ + if kwargs: + filters = filters or kwargs.get("conditions") + if not filters: + raise TypeError("No filters passed for `frappe.db.delete`") + if "debug" not in kwargs: + kwargs["debug"] = debug - else: - if doctype.startwith("__"): - return self.sql("DELETE FROM `{doctype}`".format( - doctype=doctype - ), debug=debug) + table = doctype if doctype.startswith("__") else f"tab{doctype}" + query = f"DELETE FROM `{table}`" + conditions, values = self.build_conditions(filters) + query += f"WHERE {conditions}" - else: - return self.sql("DELETE FROM `tab{doctype}`".format( - doctype=doctype - ), debug=debug) + return self.sql(query, values, **kwargs) def get_last_created(self, doctype): From 36c4be9a8d5c456316d11f7b56e427a01d22f167 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 23 Jul 2021 11:50:17 +0530 Subject: [PATCH 27/86] feat: frappe.db.truncate ORM wrapper for SQL DDL statement * Alias frappe.db.clear_table to use truncate * Support for __ tables is added here --- frappe/database/database.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index cdf15f9c3d..0c18ae91f7 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -955,21 +955,36 @@ class Database(object): def delete(self, doctype: str, filters: Union[Dict, List], debug=False, **kwargs): """Delete rows from a table in site which match the passed filters. This does trigger DocType hooks. Simply runs a DELETE query in the database. + + Doctype name can be passed directly, it will be pre-pended with `tab`. """ if kwargs: filters = filters or kwargs.get("conditions") if not filters: - raise TypeError("No filters passed for `frappe.db.delete`") + raise TypeError( + "No filters passed for `frappe.db.delete`. If you wish to clear the whole " + "table, consider using `frappe.db.truncate` instead?" + ) if "debug" not in kwargs: kwargs["debug"] = debug table = doctype if doctype.startswith("__") else f"tab{doctype}" - query = f"DELETE FROM `{table}`" conditions, values = self.build_conditions(filters) - query += f"WHERE {conditions}" + query = f"DELETE FROM `{table}` WHERE {conditions}" return self.sql(query, values, **kwargs) + def truncate(self, doctype: str): + """Truncate a table in the database. This runs a DDL command `TRUNCATE TABLE`. + This cannot be rolled back. + + Doctype name can be passed directly, it will be pre-pended with `tab`. + """ + table = doctype if doctype.startswith("__") else f"tab{doctype}" + return self.sql_ddl(f"truncate `{table}`") + + def clear_table(self, doctype): + return self.truncate(doctype) def get_last_created(self, doctype): last_record = self.get_all(doctype, ('creation'), limit=1, order_by='creation desc') @@ -978,9 +993,6 @@ class Database(object): else: return None - def clear_table(self, doctype): - self.sql('truncate `tab{}`'.format(doctype)) - def log_touched_tables(self, query, values=None): if values: query = frappe.safe_decode(self._cursor.mogrify(query, values)) From 2f3371d873b976afe08a4003533263a9a636ce28 Mon Sep 17 00:00:00 2001 From: Aradhya-Tripathi Date: Fri, 23 Jul 2021 23:14:24 +0530 Subject: [PATCH 28/86] refactor: Moved raw queries to frappe orm --- .../core/doctype/activity_log/activity_log.py | 6 +-- .../core/doctype/log_settings/log_settings.py | 8 ++-- frappe/database/database.py | 13 ++++--- frappe/desk/doctype/todo/todo.py | 12 ++++-- frappe/model/document.py | 17 ++++++-- .../v12_0/set_primary_key_in_series.py | 39 ++++++++++--------- .../setup_comments_from_communications.py | 5 ++- .../patches/v13_0/remove_twilio_settings.py | 5 ++- frappe/utils/error.py | 7 +++- frappe/utils/install.py | 6 ++- frappe/utils/password.py | 8 +++- frappe/utils/testutils.py | 5 ++- .../workflow_action/workflow_action.py | 11 ++++-- 13 files changed, 90 insertions(+), 52 deletions(-) diff --git a/frappe/core/doctype/activity_log/activity_log.py b/frappe/core/doctype/activity_log/activity_log.py index efec0dc217..3f9f4f3cb0 100644 --- a/frappe/core/doctype/activity_log/activity_log.py +++ b/frappe/core/doctype/activity_log/activity_log.py @@ -44,6 +44,6 @@ def clear_activity_logs(days=None): if not days: days = 90 - - frappe.db.sql("""delete from `tabActivity Log` where \ - creation< (NOW() - INTERVAL '{0}' DAY)""".format(days)) \ No newline at end of file + frappe.db.delete("Activity Log", filters={"creation": ("<", ["NOW()", "-", f"INTERVAL {days} DAY"])}) + # frappe.db.sql("""delete from `tabActivity Log` where \ + # creation< (NOW() - INTERVAL '{0}' DAY)""".format(days)) \ No newline at end of file diff --git a/frappe/core/doctype/log_settings/log_settings.py b/frappe/core/doctype/log_settings/log_settings.py index 776fcc92e9..eea73ee748 100644 --- a/frappe/core/doctype/log_settings/log_settings.py +++ b/frappe/core/doctype/log_settings/log_settings.py @@ -13,10 +13,10 @@ class LogSettings(Document): self.clear_email_queue() def clear_error_logs(self): - # frappe.db.delete(doctype="Error Log", conditions="") - frappe.db.sql(""" DELETE FROM `tabError Log` - WHERE `creation` < (NOW() - INTERVAL '{0}' DAY) - """.format(self.clear_error_log_after)) + frappe.db.delete("Error Log", {"creation": ("<", ["NOW()", "-", f"INTERVAL {self.clear_error_log_after} DAY"])}) + # frappe.db.sql(""" DELETE FROM `tabError Log` + # WHERE `creation` < (NOW() - INTERVAL '{0}' DAY) + # """.format(self.clear_error_log_after)) def clear_activity_logs(self): from frappe.core.doctype.activity_log.activity_log import clear_activity_logs diff --git a/frappe/database/database.py b/frappe/database/database.py index 0c18ae91f7..8b4fd3ad9e 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -952,7 +952,7 @@ class Database(object): query = sql_dict.get(current_dialect) return self.sql(query, values, **kwargs) - def delete(self, doctype: str, filters: Union[Dict, List], debug=False, **kwargs): + def delete(self, doctype: str, filters: Union[Dict, List] = None, debug=False, **kwargs): """Delete rows from a table in site which match the passed filters. This does trigger DocType hooks. Simply runs a DELETE query in the database. @@ -960,14 +960,15 @@ class Database(object): """ if kwargs: filters = filters or kwargs.get("conditions") - if not filters: - raise TypeError( - "No filters passed for `frappe.db.delete`. If you wish to clear the whole " - "table, consider using `frappe.db.truncate` instead?" - ) + if "debug" not in kwargs: kwargs["debug"] = debug + if not filters: + table = doctype if doctype.startswith("__") else f"tab{doctype}" + query = f"DELETE FROM `{table}`" + return self.sql(query, **kwargs) + table = doctype if doctype.startswith("__") else f"tab{doctype}" conditions, values = self.build_conditions(filters) query = f"DELETE FROM `{table}` WHERE {conditions}" diff --git a/frappe/desk/doctype/todo/todo.py b/frappe/desk/doctype/todo/todo.py index 4696563445..388ba37e1f 100644 --- a/frappe/desk/doctype/todo/todo.py +++ b/frappe/desk/doctype/todo/todo.py @@ -40,11 +40,15 @@ class ToDo(Document): def on_trash(self): # unlink todo from linked comments - frappe.db.sql(""" - delete from `tabCommunication Link` - where link_doctype=%(doctype)s and link_name=%(name)s""", { - "doctype": self.doctype, "name": self.name + frappe.db.delete("Communication Link", { + "link_doctype": self.doctype, + "link_name": self.name }) + # frappe.db.sql(""" + # delete from `tabCommunication Link` + # where link_doctype=%(doctype)s and link_name=%(name)s""", { + # "doctype": self.doctype, "name": self.name + # }) self.update_in_reference() diff --git a/frappe/model/document.py b/frappe/model/document.py index 61160e1f01..f8adfdb3cf 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -390,9 +390,15 @@ class Document(BaseDocument): else: # no rows found, delete all rows - frappe.db.sql("""delete from `tab{0}` where parent=%s - and parenttype=%s and parentfield=%s""".format(df.options), - (self.name, self.doctype, fieldname)) + frappe.db.delete(df.options, { + "parent": self.name, + "parenttype": self.doctype, + "parentfield": fieldname + }) + + # frappe.db.sql("""delete from `tab{0}` where parent=%s + # and parenttype=%s and parentfield=%s""".format(df.options), + # (self.name, self.doctype, fieldname)) def get_doc_before_save(self): return getattr(self, '_doc_before_save', None) @@ -451,7 +457,10 @@ class Document(BaseDocument): def update_single(self, d): """Updates values for Single type Document in `tabSingles`.""" - frappe.db.sql("""delete from `tabSingles` where doctype=%s""", self.doctype) + frappe.db.delete("Singles", { + "doctype": self.doctype + }) + # frappe.db.sql("""delete from `tabSingles` where doctype=%s""", self.doctype) for field, value in d.items(): if field != "doctype": frappe.db.sql("""insert into `tabSingles` (doctype, field, value) diff --git a/frappe/patches/v12_0/set_primary_key_in_series.py b/frappe/patches/v12_0/set_primary_key_in_series.py index e5ed2204ba..a8409cbbba 100644 --- a/frappe/patches/v12_0/set_primary_key_in_series.py +++ b/frappe/patches/v12_0/set_primary_key_in_series.py @@ -1,21 +1,24 @@ import frappe def execute(): - #if current = 0, simply delete the key as it'll be recreated on first entry - frappe.db.sql('delete from `tabSeries` where current = 0') - duplicate_keys = frappe.db.sql(''' - SELECT name, max(current) as current - from - `tabSeries` - group by - name - having count(name) > 1 - ''', as_dict=True) - for row in duplicate_keys: - frappe.db.sql('delete from `tabSeries` where name = %(key)s', { - 'key': row.name - }) - if row.current: - frappe.db.sql('insert into `tabSeries`(`name`, `current`) values (%(name)s, %(current)s)', row) - frappe.db.commit() - frappe.db.sql('ALTER table `tabSeries` ADD PRIMARY KEY IF NOT EXISTS (name)') + #if current = 0, simply delete the key as it'll be recreated on first entry + frappe.db.sql('delete from `tabSeries` where current = 0') + duplicate_keys = frappe.db.sql(''' + SELECT name, max(current) as current + from + `tabSeries` + group by + name + having count(name) > 1 + ''', as_dict=True) + for row in duplicate_keys: + frappe.db.delete("Series", { + "name": row.name + }) + # frappe.db.sql('delete from `tabSeries` where name = %(key)s', { + # 'key': row.name + # }) + if row.current: + frappe.db.sql('insert into `tabSeries`(`name`, `current`) values (%(name)s, %(current)s)', row) + frappe.db.commit() + frappe.db.sql('ALTER table `tabSeries` ADD PRIMARY KEY IF NOT EXISTS (name)') diff --git a/frappe/patches/v12_0/setup_comments_from_communications.py b/frappe/patches/v12_0/setup_comments_from_communications.py index 039ceeff35..a4e3fb4261 100644 --- a/frappe/patches/v12_0/setup_comments_from_communications.py +++ b/frappe/patches/v12_0/setup_comments_from_communications.py @@ -29,4 +29,7 @@ def execute(): frappe.db.auto_commit_on_many_writes = False # clean up - frappe.db.sql("delete from `tabCommunication` where communication_type = 'Comment'") + frappe.db.delete("Communication", { + "communication_type": "Comment" + }) + # frappe.db.sql("delete from `tabCommunication` where communication_type = 'Comment'") diff --git a/frappe/patches/v13_0/remove_twilio_settings.py b/frappe/patches/v13_0/remove_twilio_settings.py index 363cbdd4b6..5d17364847 100644 --- a/frappe/patches/v13_0/remove_twilio_settings.py +++ b/frappe/patches/v13_0/remove_twilio_settings.py @@ -12,7 +12,10 @@ def execute(): frappe.delete_doc_if_exists('DocType', 'Twilio Number Group') if twilio_settings_doctype_in_integrations(): frappe.delete_doc_if_exists('DocType', 'Twilio Settings') - frappe.db.sql("delete from `tabSingles` where `doctype`=%s", 'Twilio Settings') + frappe.db.delete("Singles", { + "doctype": "Twilio Settings" + }) + # frappe.db.sql("delete from `tabSingles` where `doctype`=%s", 'Twilio Settings') def twilio_settings_doctype_in_integrations() -> bool: """Check Twilio Settings doctype exists in integrations module or not. diff --git a/frappe/utils/error.py b/frappe/utils/error.py index 07e34674fe..75b5c71da5 100644 --- a/frappe/utils/error.py +++ b/frappe/utils/error.py @@ -176,8 +176,11 @@ def collect_error_snapshots(): def clear_old_snapshots(): """Clear snapshots that are older than a month""" - frappe.db.sql("""delete from `tabError Snapshot` - where creation < (NOW() - INTERVAL '1' MONTH)""") + + frappe.db.delete("Error Snapshot", filters={"creation": ("<", ["NOW()", "-", f"INTERVAL '1' MONTH"])}) + + # frappe.db.sql("""delete from `tabError Snapshot` + # where creation < (NOW() - INTERVAL '1' MONTH)""") path = get_error_snapshot_path() today = datetime.datetime.now() diff --git a/frappe/utils/install.py b/frappe/utils/install.py index 91d8f04eb4..9d21a0d945 100644 --- a/frappe/utils/install.py +++ b/frappe/utils/install.py @@ -111,8 +111,10 @@ def before_tests(): # don't run before tests if any other app is installed return - frappe.db.sql("delete from `tabCustom Field`") - frappe.db.sql("delete from `tabEvent`") + frappe.db.delete("Custom Field") + frappe.db.delete("Event") + # frappe.db.sql("delete from `tabCustom Field`") + # frappe.db.sql("delete from `tabEvent`") frappe.db.commit() frappe.clear_cache() diff --git a/frappe/utils/password.py b/frappe/utils/password.py index c144d866f3..4bac6537ce 100644 --- a/frappe/utils/password.py +++ b/frappe/utils/password.py @@ -136,8 +136,12 @@ def update_password(user, pwd, doctype='User', fieldname='password', logout_all_ def delete_all_passwords_for(doctype, name): try: - frappe.db.sql("""delete from `__Auth` where `doctype`=%(doctype)s and `name`=%(name)s""", - { 'doctype': doctype, 'name': name }) + frappe.db.delete("__Auth", { + "doctype": doctype, + "name": name + }) + # frappe.db.sql("""delete from `__Auth` where `doctype`=%(doctype)s and `name`=%(name)s""", + # { 'doctype': doctype, 'name': name }) except Exception as e: if not frappe.db.is_missing_column(e): raise diff --git a/frappe/utils/testutils.py b/frappe/utils/testutils.py index c451d090f1..74b800bdcf 100644 --- a/frappe/utils/testutils.py +++ b/frappe/utils/testutils.py @@ -12,5 +12,8 @@ def add_custom_field(doctype, fieldname, fieldtype='Data', options=None): }).insert() def clear_custom_fields(doctype): - frappe.db.sql('delete from `tabCustom Field` where dt=%s', doctype) + frappe.db.delete("Custom Field", { + "dt": doctype + }) + # frappe.db.sql('delete from `tabCustom Field` where dt=%s', doctype) frappe.clear_cache(doctype=doctype) diff --git a/frappe/workflow/doctype/workflow_action/workflow_action.py b/frappe/workflow/doctype/workflow_action/workflow_action.py index af168d3477..6c1c31f0e5 100644 --- a/frappe/workflow/doctype/workflow_action/workflow_action.py +++ b/frappe/workflow/doctype/workflow_action/workflow_action.py @@ -259,10 +259,13 @@ def is_workflow_action_already_created(doc): def clear_workflow_actions(doctype, name): if not (doctype and name): return - - frappe.db.sql('''delete from `tabWorkflow Action` - where reference_doctype=%s and reference_name=%s''', - (doctype, name)) + frappe.db.delete("Workflow Action", { + "reference_doctype": doctype, + "reference_name": name + }) + # frappe.db.sql('''delete from `tabWorkflow Action` + # where reference_doctype=%s and reference_name=%s''', + # (doctype, name)) def get_doc_workflow_state(doc): workflow_name = get_workflow_name(doc.get('doctype')) From a86d717f17058203ced2e4a4b7b4cfffa42080ea Mon Sep 17 00:00:00 2001 From: Aradhya-Tripathi Date: Fri, 23 Jul 2021 23:31:44 +0530 Subject: [PATCH 29/86] fix: Convert indentation to tabs --- frappe/core/doctype/activity_log/feed.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/frappe/core/doctype/activity_log/feed.py b/frappe/core/doctype/activity_log/feed.py index d91a21ec44..d064022b25 100644 --- a/frappe/core/doctype/activity_log/feed.py +++ b/frappe/core/doctype/activity_log/feed.py @@ -28,10 +28,10 @@ def update_feed(doc, method=None): doctype = feed.doctype or doc.doctype name = feed.name or doc.name frappe.db.delete("Activity Log", { - "reference_doctype": doctype, - "reference_name": name, - "link_doctype": feed.link_doctype - }) + "reference_doctype": doctype, + "reference_name": name, + "link_doctype": feed.link_doctype + }) # frappe.db.sql("""delete from `tabActivity Log` # where # reference_doctype=%s and reference_name=%s From 538ef2168bd5c77e19726d7ffe25816d765f2c90 Mon Sep 17 00:00:00 2001 From: Aradhya-Tripathi Date: Sat, 24 Jul 2021 22:58:12 +0530 Subject: [PATCH 30/86] fix: removed wrong queries --- .../core/doctype/activity_log/activity_log.py | 6 +++--- .../core/doctype/log_settings/log_settings.py | 8 ++++---- frappe/core/doctype/report/test_report.py | 2 +- frappe/core/doctype/user/user.py | 6 +++--- .../user_permission/test_user_permission.py | 2 +- .../user_permission/user_permission.py | 8 ++++---- frappe/model/__init__.py | 6 +++--- frappe/model/delete_doc.py | 4 ++-- .../delete_feedback_request_if_exists.py | 8 ++++---- frappe/permissions.py | 2 +- frappe/sessions.py | 1 - frappe/utils/error.py | 6 +++--- frappe/utils/password.py | 4 ++-- .../personal_data_deletion_request.py | 20 +++++++++---------- .../workflow_action/workflow_action.py | 4 ++-- 15 files changed, 43 insertions(+), 44 deletions(-) diff --git a/frappe/core/doctype/activity_log/activity_log.py b/frappe/core/doctype/activity_log/activity_log.py index 3f9f4f3cb0..2571330f9e 100644 --- a/frappe/core/doctype/activity_log/activity_log.py +++ b/frappe/core/doctype/activity_log/activity_log.py @@ -44,6 +44,6 @@ def clear_activity_logs(days=None): if not days: days = 90 - frappe.db.delete("Activity Log", filters={"creation": ("<", ["NOW()", "-", f"INTERVAL {days} DAY"])}) - # frappe.db.sql("""delete from `tabActivity Log` where \ - # creation< (NOW() - INTERVAL '{0}' DAY)""".format(days)) \ No newline at end of file + # frappe.db.delete("Activity Log", filters={"creation": ("<", ["NOW()", "-", f"INTERVAL {days} DAY"])}) + frappe.db.sql("""delete from `tabActivity Log` where \ + creation< (NOW() - INTERVAL '{0}' DAY)""".format(days)) \ No newline at end of file diff --git a/frappe/core/doctype/log_settings/log_settings.py b/frappe/core/doctype/log_settings/log_settings.py index eea73ee748..7f25998dc5 100644 --- a/frappe/core/doctype/log_settings/log_settings.py +++ b/frappe/core/doctype/log_settings/log_settings.py @@ -13,10 +13,10 @@ class LogSettings(Document): self.clear_email_queue() def clear_error_logs(self): - frappe.db.delete("Error Log", {"creation": ("<", ["NOW()", "-", f"INTERVAL {self.clear_error_log_after} DAY"])}) - # frappe.db.sql(""" DELETE FROM `tabError Log` - # WHERE `creation` < (NOW() - INTERVAL '{0}' DAY) - # """.format(self.clear_error_log_after)) + # frappe.db.delete("Error Log", {"creation": ("<", ["NOW()", "-", f"INTERVAL {self.clear_error_log_after} DAY"])}) + frappe.db.sql(""" DELETE FROM `tabError Log` + WHERE `creation` < (NOW() - INTERVAL '{0}' DAY) + """.format(self.clear_error_log_after)) def clear_activity_logs(self): from frappe.core.doctype.activity_log.activity_log import clear_activity_logs diff --git a/frappe/core/doctype/report/test_report.py b/frappe/core/doctype/report/test_report.py index 2ac41cca2b..efc33b888c 100644 --- a/frappe/core/doctype/report/test_report.py +++ b/frappe/core/doctype/report/test_report.py @@ -88,7 +88,7 @@ class TestReport(unittest.TestCase): "role": "Test Has Role" }) # frappe.db.sql("""delete from `tabHas Role` where parent = %s - # and role = 'Test Has Role'""", frappe.session.user, auto_commit=1) + # and role = 'Test Has Role'""", frappe.session.user, auto_commit=1) if not frappe.db.exists('Role', 'Test Has Role'): role = frappe.get_doc({ diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py index 69d3b67220..7ff4b8bea9 100644 --- a/frappe/core/doctype/user/user.py +++ b/frappe/core/doctype/user/user.py @@ -383,9 +383,9 @@ class User(Document): # delete messages # frappe.db.delete("Communication", { - # "reference_doctype": "User", - # "communication_type": ("in", ("Chat", "Notification")), - # }) + # "reference_doctype": "User", + # "communication_type": ("in", ("Chat", "Notification")), + # }) frappe.db.sql("""delete from `tabCommunication` where communication_type in ('Chat', 'Notification') and reference_doctype='User' diff --git a/frappe/core/doctype/user_permission/test_user_permission.py b/frappe/core/doctype/user_permission/test_user_permission.py index 9393e47ee9..60a91e6a9b 100644 --- a/frappe/core/doctype/user_permission/test_user_permission.py +++ b/frappe/core/doctype/user_permission/test_user_permission.py @@ -14,7 +14,7 @@ class TestUserPermission(unittest.TestCase): frappe.db.delete("User Permission", { "user": ("in", ("test_bulk_creation_update@example.com", "test_user_perm1@example.com", - "nested_doc_user@example.com")) + "nested_doc_user@example.com")) }) # frappe.db.sql("""DELETE FROM `tabUser Permission` diff --git a/frappe/core/doctype/user_permission/user_permission.py b/frappe/core/doctype/user_permission/user_permission.py index 5b3822d82f..338b063a63 100644 --- a/frappe/core/doctype/user_permission/user_permission.py +++ b/frappe/core/doctype/user_permission/user_permission.py @@ -278,8 +278,8 @@ def update_applicable(already_applied, to_apply, user, doctype, docname): }) # frappe.db.sql("""DELETE FROM `tabUser Permission` - # WHERE `user`=%s - # AND `applicable_for`=%s - # AND `allow`=%s - # AND `for_value`=%s + # WHERE `user`=%s + # AND `applicable_for`=%s + # AND `allow`=%s + # AND `for_value`=%s # """,(user, applied, doctype, docname)) diff --git a/frappe/model/__init__.py b/frappe/model/__init__.py index 99eaa7ff5d..7459493fcc 100644 --- a/frappe/model/__init__.py +++ b/frappe/model/__init__.py @@ -168,11 +168,11 @@ def delete_fields(args_dict, delete=0): if frappe.db.get_value("DocType", dt, "issingle"): frappe.db.delete("Singles", { "doctype": dt, - "field": ("in", ", ".join("'{}'".format(f) for f in fields)) + "field": ("in", fields) }) # frappe.db.sql(""" - # DELETE FROM `tabSingles` - # WHERE doctype='%s' AND field IN (%s) + # DELETE FROM `tabSingles` + # WHERE doctype='%s' AND field IN (%s) # """ % (dt, ", ".join("'{}'".format(f) for f in fields))) else: existing_fields = frappe.db.multisql({ diff --git a/frappe/model/delete_doc.py b/frappe/model/delete_doc.py index 03510d489d..9187310a84 100644 --- a/frappe/model/delete_doc.py +++ b/frappe/model/delete_doc.py @@ -185,7 +185,7 @@ def delete_from_table(doctype, name, ignore_doctypes, doc): }) # frappe.db.sql("delete from `tabSingles` where `doctype`=%s", name) else: - frappe.db.delete(f"{doctype}", { + frappe.db.delete(doctype, { "name": name }) # frappe.db.sql("delete from `tab{0}` where `name`=%s".format(doctype), name) @@ -368,7 +368,7 @@ def clear_timeline_references(link_doctype, link_name): "link_name": link_name }) # frappe.db.sql("""DELETE FROM `tabCommunication Link` - # WHERE `tabCommunication Link`.link_doctype=%s AND `tabCommunication Link`.link_name=%s""", (link_doctype, link_name)) + # WHERE `tabCommunication Link`.link_doctype=%s AND `tabCommunication Link`.link_name=%s""", (link_doctype, link_name)) def insert_feed(doc): if ( diff --git a/frappe/patches/v12_0/delete_feedback_request_if_exists.py b/frappe/patches/v12_0/delete_feedback_request_if_exists.py index 537d509424..de799b1e21 100644 --- a/frappe/patches/v12_0/delete_feedback_request_if_exists.py +++ b/frappe/patches/v12_0/delete_feedback_request_if_exists.py @@ -5,7 +5,7 @@ def execute(): frappe.db.delete("DocType", { "name": "Feedback Request" }) - # frappe.db.sql(''' - # DELETE from `tabDocType` - # WHERE name = 'Feedback Request' - # ''') \ No newline at end of file + # frappe.db.sql(''' + #DELETE from `tabDocType` + #WHERE name = 'Feedback Request' + # ''') \ No newline at end of file diff --git a/frappe/permissions.py b/frappe/permissions.py index 293ac14f27..7724f09e78 100644 --- a/frappe/permissions.py +++ b/frappe/permissions.py @@ -6,7 +6,7 @@ import frappe import frappe.share from frappe import _, msgprint from frappe.utils import cint -from frappe.database.database import Database + rights = ("select", "read", "write", "create", "delete", "submit", "cancel", "amend", "print", "email", "report", "import", "export", "set_user_permissions", "share") diff --git a/frappe/sessions.py b/frappe/sessions.py index 3a584dfb74..f4d05538bc 100644 --- a/frappe/sessions.py +++ b/frappe/sessions.py @@ -16,7 +16,6 @@ import frappe.translate import redis from urllib.parse import unquote from frappe.cache_manager import clear_user_cache -from frappe.database.database import Database @frappe.whitelist(allow_guest=True) def clear(user=None): diff --git a/frappe/utils/error.py b/frappe/utils/error.py index 75b5c71da5..f2cbc5ec10 100644 --- a/frappe/utils/error.py +++ b/frappe/utils/error.py @@ -177,10 +177,10 @@ def collect_error_snapshots(): def clear_old_snapshots(): """Clear snapshots that are older than a month""" - frappe.db.delete("Error Snapshot", filters={"creation": ("<", ["NOW()", "-", f"INTERVAL '1' MONTH"])}) + # frappe.db.delete("Error Snapshot", filters={"creation": ("<", ["NOW()", "-", f"INTERVAL '1' MONTH"])}) - # frappe.db.sql("""delete from `tabError Snapshot` - # where creation < (NOW() - INTERVAL '1' MONTH)""") + frappe.db.sql("""delete from `tabError Snapshot` + where creation < (NOW() - INTERVAL '1' MONTH)""") path = get_error_snapshot_path() today = datetime.datetime.now() diff --git a/frappe/utils/password.py b/frappe/utils/password.py index 4bac6537ce..21e1bc9586 100644 --- a/frappe/utils/password.py +++ b/frappe/utils/password.py @@ -72,8 +72,8 @@ def remove_encrypted_password(doctype, name, fieldname='password'): }) # frappe.db.sql( - # 'DELETE FROM `__Auth` WHERE doctype = %s and name = %s and fieldname = %s', - # values=[doctype, name, fieldname] + # 'DELETE FROM `__Auth` WHERE doctype = %s and name = %s and fieldname = %s', + # values=[doctype, name, fieldname] # ) def check_password(user, pwd, doctype='User', fieldname='password', delete_tracker_cache=True): diff --git a/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py b/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py index bde6f2a4ea..96898cc159 100644 --- a/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py +++ b/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py @@ -323,17 +323,17 @@ class PersonalDataDeletionRequest(Document): def remove_unverified_record(): - frappe.db.delete("Personal Data Deletion Request", { - "status": "Pending Verification", - "creation": ("<", ["Now()", "-", "INTERVAL 7 DAY"]) - }) + # frappe.db.delete("Personal Data Deletion Request", { + # "status": "Pending Verification", + # "creation": ("<", ["Now()", "-", "INTERVAL 7 DAY"]) + # }) - # frappe.db.sql( - # """ - # DELETE FROM `tabPersonal Data Deletion Request` - # WHERE `status` = 'Pending Verification' - # AND `creation` < (NOW() - INTERVAL '7' DAY)""" - # ) + frappe.db.sql( + """ + DELETE FROM `tabPersonal Data Deletion Request` + WHERE `status` = 'Pending Verification' + AND `creation` < (NOW() - INTERVAL '7' DAY)""" + ) @frappe.whitelist(allow_guest=True) def confirm_deletion(email, name, host_name): diff --git a/frappe/workflow/doctype/workflow_action/workflow_action.py b/frappe/workflow/doctype/workflow_action/workflow_action.py index 6c1c31f0e5..5957bf8671 100644 --- a/frappe/workflow/doctype/workflow_action/workflow_action.py +++ b/frappe/workflow/doctype/workflow_action/workflow_action.py @@ -264,8 +264,8 @@ def clear_workflow_actions(doctype, name): "reference_name": name }) # frappe.db.sql('''delete from `tabWorkflow Action` - # where reference_doctype=%s and reference_name=%s''', - # (doctype, name)) + # where reference_doctype=%s and reference_name=%s''', + # (doctype, name)) def get_doc_workflow_state(doc): workflow_name = get_workflow_name(doc.get('doctype')) From 326bd463fc36994d11cf9470101ab43e7b575dd8 Mon Sep 17 00:00:00 2001 From: pateljannat Date: Mon, 26 Jul 2021 17:09:57 +0530 Subject: [PATCH 31/86] fix: web form child table issue --- frappe/website/doctype/web_form/web_form.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/website/doctype/web_form/web_form.py b/frappe/website/doctype/web_form/web_form.py index 6e4200b54b..32f7e030a6 100644 --- a/frappe/website/doctype/web_form/web_form.py +++ b/frappe/website/doctype/web_form/web_form.py @@ -542,7 +542,7 @@ def get_form_data(doctype, docname=None, web_form_name=None): # For Table fields, server-side processing for meta for field in out.web_form.web_form_fields: if field.fieldtype == "Table": - field.fields = get_in_list_view_fields(field.options) + field.fields = frappe.get_meta(field.options).fields out.update({field.fieldname: field.fields}) if field.fieldtype == "Link": From db09a85183a5068bf9792f8d2e4101ab4382eec6 Mon Sep 17 00:00:00 2001 From: leela Date: Tue, 20 Jul 2021 06:11:01 +0530 Subject: [PATCH 32/86] refactor: Add authentication for Redis Queue --- frappe/commands/redis.py | 25 +++--- frappe/commands/scheduler.py | 8 +- .../page/background_jobs/background_jobs.py | 8 +- frappe/test_runner.py | 1 + frappe/tests/test_background_jobs.py | 6 +- frappe/tests/test_redis.py | 70 ++++++++++++++++ frappe/utils/__init__.py | 2 +- frappe/utils/background_jobs.py | 80 +++++++++++++++---- frappe/utils/rq.py | 24 +++--- 9 files changed, 173 insertions(+), 51 deletions(-) create mode 100644 frappe/tests/test_redis.py diff --git a/frappe/commands/redis.py b/frappe/commands/redis.py index c608296cac..38a46c2142 100644 --- a/frappe/commands/redis.py +++ b/frappe/commands/redis.py @@ -1,7 +1,6 @@ import os import click -import redis import frappe from frappe.utils.rq import RedisQueue @@ -9,16 +8,18 @@ from frappe.installer import update_site_config @click.command('create-rq-users') @click.option('--set-admin-password', is_flag=True, default=False, help='Set new Redis admin(default user) password') -@click.option('--reset-passwords', is_flag=True, default=False, help='Remove all existing passwords') -def create_rq_users(set_admin_password=False, reset_passwords=False): +@click.option('--use-rq-auth', is_flag=True, default=False, help='Enable Redis authentication for sites') +def create_rq_users(set_admin_password=False, use_rq_auth=False): """Create Redis Queue users and add to acl and app configs. acl config file will be used by redis server while starting the server and app config is used by app while connecting to redis server. """ acl_file_path = os.path.abspath('../config/redis_queue.acl') - acl_list, user_credentials = RedisQueue.gen_acl_list( - reset_passwords=reset_passwords, set_admin_password=set_admin_password) + + with frappe.init_site(): + acl_list, user_credentials = RedisQueue.gen_acl_list( + set_admin_password=set_admin_password) with open(acl_file_path, 'w') as f: f.writelines([acl+'\n' for acl in acl_list]) @@ -29,18 +30,22 @@ def create_rq_users(set_admin_password=False, reset_passwords=False): site_config_path=common_site_config_path) update_site_config("rq_password", user_credentials['bench'][1], validate=False, site_config_path=common_site_config_path) + update_site_config("use_rq_auth", use_rq_auth, validate=False, + site_config_path=common_site_config_path) + + click.secho('* ACL and site configs are updated with new user credentials. ' + 'Please restart Redis Queue server to enable namespaces.', + fg='green') if set_admin_password: env_key = 'RQ_ADMIN_PASWORD' - click.secho('Redis admin password is successfully set up. ' + click.secho('* Redis admin password is successfully set up. ' 'Include below line in .bashrc file for system to use', - fg='green' - ) + fg='green') click.secho(f"`export {env_key}={user_credentials['default'][1]}`") click.secho('NOTE: Please save the admin password as you ' 'can not access redis server without the password', - fg='yellow' - ) + fg='yellow') commands = [ diff --git a/frappe/commands/scheduler.py b/frappe/commands/scheduler.py index d69ebb3024..f82473fd55 100755 --- a/frappe/commands/scheduler.py +++ b/frappe/commands/scheduler.py @@ -172,9 +172,13 @@ def start_scheduler(): @click.command('worker') @click.option('--queue', type=str) @click.option('--quiet', is_flag = True, default = False, help = 'Hide Log Outputs') -def start_worker(queue, quiet = False): +@click.option('-u', '--rq-username', default=None, help='Redis ACL user') +@click.option('-p', '--rq-password', default=None, help='Redis ACL user password') +def start_worker(queue, quiet = False, rq_username=None, rq_password=None): + """Site is used to find redis credentals. + """ from frappe.utils.background_jobs import start_worker - start_worker(queue, quiet = quiet) + start_worker(queue, quiet = quiet, rq_username=rq_username, rq_password=rq_password) @click.command('ready-for-migration') @click.option('--site', help='site name') diff --git a/frappe/core/page/background_jobs/background_jobs.py b/frappe/core/page/background_jobs/background_jobs.py index 847b23bd3e..1f3555e351 100644 --- a/frappe/core/page/background_jobs/background_jobs.py +++ b/frappe/core/page/background_jobs/background_jobs.py @@ -4,12 +4,12 @@ import json from typing import TYPE_CHECKING, Dict, List -from rq import Queue, Worker +from rq import Worker import frappe from frappe import _ from frappe.utils import convert_utc_to_user_timezone, format_datetime -from frappe.utils.background_jobs import get_redis_conn +from frappe.utils.background_jobs import get_redis_conn, get_queues from frappe.utils.scheduler import is_scheduler_inactive if TYPE_CHECKING: @@ -29,7 +29,7 @@ def get_info(show_failed=False) -> List[Dict]: show_failed = json.loads(show_failed) conn = get_redis_conn() - queues = Queue.all(conn) + queues = get_queues() workers = Worker.all(conn) jobs = [] @@ -75,7 +75,7 @@ def get_info(show_failed=False) -> List[Dict]: @frappe.whitelist() def remove_failed_jobs(): conn = get_redis_conn() - queues = Queue.all(conn) + queues = get_queues() for queue in queues: fail_registry = queue.failed_job_registry for job_id in fail_registry.get_job_ids(): diff --git a/frappe/test_runner.py b/frappe/test_runner.py index 0c30fbbd00..8112362f34 100644 --- a/frappe/test_runner.py +++ b/frappe/test_runner.py @@ -56,6 +56,7 @@ def main(app=None, module=None, doctype=None, verbose=False, tests=(), frappe.clear_cache() frappe.utils.scheduler.disable_scheduler() set_test_email_config() + frappe.conf.update({'bench_id': 'test_bench', 'use_rq_auth': False}) if not frappe.flags.skip_before_tests: if verbose: diff --git a/frappe/tests/test_background_jobs.py b/frappe/tests/test_background_jobs.py index 88783f14f1..48e0dd2ee9 100644 --- a/frappe/tests/test_background_jobs.py +++ b/frappe/tests/test_background_jobs.py @@ -4,7 +4,7 @@ from rq import Queue import frappe from frappe.core.page.background_jobs.background_jobs import remove_failed_jobs -from frappe.utils.background_jobs import get_redis_conn +from frappe.utils.background_jobs import get_redis_conn, rename_queue import time @@ -17,14 +17,14 @@ class TestBackgroundJobs(unittest.TestCase): queues = Queue.all(conn) for queue in queues: - if queue.name == "short": + if queue.name == rename_queue("short"): fail_registry = queue.failed_job_registry self.assertGreater(fail_registry.count, 0) remove_failed_jobs() for queue in queues: - if queue.name == "short": + if queue.name == rename_queue("short"): fail_registry = queue.failed_job_registry self.assertEqual(fail_registry.count, 0) diff --git a/frappe/tests/test_redis.py b/frappe/tests/test_redis.py new file mode 100644 index 0000000000..72af1ac699 --- /dev/null +++ b/frappe/tests/test_redis.py @@ -0,0 +1,70 @@ +import unittest +import functools + +import redis + +import frappe +from frappe.utils import get_bench_id +from frappe.utils.rq import RedisQueue +from frappe.utils.background_jobs import get_redis_conn + +def version_tuple(version): + return tuple(map(int, (version.split(".")))) + +def skip_if_redis_version_lt(version): + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + conn = get_redis_conn() + redis_version = conn.execute_command('info')['redis_version'] + if version_tuple(redis_version) < version_tuple(version): + return + return func(*args, **kwargs) + return wrapper + return decorator + +class TestRedisAuth(unittest.TestCase): + @skip_if_redis_version_lt('6.0') + def test_rq_gen_acllist(self): + """Make sure that ACL list is genrated + """ + acl_list = RedisQueue.gen_acl_list() + self.assertEqual(acl_list[1]['bench'][0], get_bench_id()) + + @skip_if_redis_version_lt('6.0') + def test_adding_redis_user(self): + acl_list = RedisQueue.gen_acl_list() + username, password = acl_list[1]['bench'] + conn = get_redis_conn() + + conn.acl_deluser(username) + _ = RedisQueue(conn).add_user(username, password) + self.assertTrue(conn.acl_getuser(username)) + conn.acl_deluser(username) + + @skip_if_redis_version_lt('6.0') + def test_rq_namespace(self): + """Make sure that user can access only their respective namespace. + """ + # Current bench ID + bench_id = frappe.conf.get('bench_id') + conn = get_redis_conn() + conn.set('rq:queue:test_bench1:abc', 'value') + conn.set(f'rq:queue:{bench_id}:abc', 'value') + + # Create new Redis Queue user + tmp_bench_id = 'test_bench1' + username, password = tmp_bench_id, 'password1' + conn.acl_deluser(username) + frappe.conf.update({'bench_id': tmp_bench_id}) + _ = RedisQueue(conn).add_user(username, password) + test_bench1_conn = RedisQueue.get_connection(username, password) + + self.assertEqual(test_bench1_conn.get('rq:queue:test_bench1:abc'), b'value') + + # User should not be able to access queues apart from their bench queues + with self.assertRaises(redis.exceptions.NoPermissionError): + test_bench1_conn.get(f'rq:queue:{bench_id}:abc') + + frappe.conf.update({'bench_id': bench_id}) + conn.acl_deluser(username) diff --git a/frappe/utils/__init__.py b/frappe/utils/__init__.py index bf139173d6..80c6cda98c 100644 --- a/frappe/utils/__init__.py +++ b/frappe/utils/__init__.py @@ -384,7 +384,7 @@ def get_bench_path(): return os.path.realpath(os.path.join(os.path.dirname(frappe.__file__), '..', '..', '..')) def get_bench_id(): - return frappe.local.conf.get('bench_id', 'DefaultBench') + return frappe.get_conf().get('bench_id', 'DefaultBench') def get_site_id(site=None): return f"{site or frappe.local.site}@{get_bench_id()}" diff --git a/frappe/utils/background_jobs.py b/frappe/utils/background_jobs.py index 8456835ca7..4241c95c5d 100755 --- a/frappe/utils/background_jobs.py +++ b/frappe/utils/background_jobs.py @@ -1,13 +1,21 @@ +import os +import socket +import time +from uuid import uuid4 +from collections import defaultdict + + import redis +from typing import List from rq import Connection, Queue, Worker from rq.logutils import setup_loghandlers -from frappe.utils import cstr -from collections import defaultdict + import frappe -import os, socket, time from frappe import _ -from uuid import uuid4 import frappe.monitor +from frappe.utils import cstr, get_bench_id +from frappe.utils.rq import RedisQueue +from frappe.utils.commands import log default_timeout = 300 @@ -131,21 +139,22 @@ def execute_job(site, method, event, job_name, kwargs, user=None, is_async=True, if is_async: frappe.destroy() -def start_worker(queue=None, quiet = False): +def start_worker(queue=None, quiet = False, rq_username=None, rq_password=None): '''Wrapper to start rq worker. Connects to redis and monitors these queues.''' with frappe.init_site(): # empty init is required to get redis_queue from common_site_config.json - redis_connection = get_redis_conn() + redis_connection = get_redis_conn(username=rq_username, password=rq_password) + queues = get_queue_list(queue, build_queue_name=True) + queue_name = queue and rename_queue(queue) if os.environ.get('CI'): setup_loghandlers('ERROR') with Connection(redis_connection): - queues = get_queue_list(queue) logging_level = "INFO" if quiet: logging_level = "WARNING" - Worker(queues, name=get_worker_name(queue)).work(logging_level = logging_level) + Worker(queues, name=get_worker_name(queue_name)).work(logging_level = logging_level) def get_worker_name(queue): '''When limiting worker to a specific queue, also append queue name to default worker name''' @@ -186,7 +195,7 @@ def get_jobs(site=None, queue=None, key='method'): return jobs_per_site -def get_queue_list(queue_list=None): +def get_queue_list(queue_list=None, build_queue_name=False): '''Defines possible queues. Also wraps a given queue in a list after validating.''' default_queue_list = list(queue_timeout) if queue_list: @@ -195,11 +204,9 @@ def get_queue_list(queue_list=None): for queue in queue_list: validate_queue(queue, default_queue_list) - - return queue_list - else: - return default_queue_list + queue_list = default_queue_list + return [rename_queue(q) for q in queue_list] if build_queue_name else queue_list def get_workers(queue): '''Returns a list of Worker objects tied to a queue object''' @@ -218,7 +225,7 @@ def get_running_jobs_in_queue(queue): def get_queue(queue, is_async=True): '''Returns a Queue object tied to a redis connection''' validate_queue(queue) - return Queue(queue, connection=get_redis_conn(), is_async=is_async) + return Queue(rename_queue(queue), connection=get_redis_conn(), is_async=is_async) def validate_queue(queue, default_queue_list=None): if not default_queue_list: @@ -227,7 +234,7 @@ def validate_queue(queue, default_queue_list=None): if queue not in default_queue_list: frappe.throw(_("Queue should be one of {0}").format(', '.join(default_queue_list))) -def get_redis_conn(): +def get_redis_conn(username=None, password=None): if not hasattr(frappe.local, 'conf'): raise Exception('You need to call frappe.init') @@ -236,11 +243,50 @@ def get_redis_conn(): global redis_connection - if not redis_connection: - redis_connection = redis.from_url(frappe.local.conf.redis_queue) + cred = frappe._dict() + if frappe.conf.get('use_rq_auth'): + if username: + cred['username'] = username + cred['password'] = password + else: + cred['username'] = frappe.get_site_config().rq_username or get_bench_id() + cred['password'] = frappe.get_site_config().rq_password + + elif os.environ.get('RQ_ADMIN_PASWORD'): + cred['username'] = 'default' + cred['password'] = os.environ.get('RQ_ADMIN_PASWORD') + try: + redis_connection = RedisQueue.get_connection(**cred) + except (redis.exceptions.AuthenticationError, redis.exceptions.ResponseError): + log(f'Wrong credentials used for {cred.username or "default user"}. ' + 'You can reset credentials using `bench create-rq-users` CLI and restart the server', + colour='red') + raise + except Exception: + log(f'Please make sure that Redis Queue runs @ {frappe.get_conf().redis_queue}', colour='red') + raise return redis_connection +def get_queues() -> List[Queue]: + """Get all the queues linked to the current bench. + """ + queues = Queue.all(connection=get_redis_conn()) + return [q for q in queues if is_queue_accessible(q)] + +def rename_queue(qname: str) -> str: + """Rename qname by adding bench name as prefix. + + Renamed queues are useful to define namespaces of customers. + """ + return f"{get_bench_id()}:{qname}" + +def is_queue_accessible(qobj: Queue) -> bool: + """Checks whether queue is relate to current bench or not. + """ + accessible_queues = [rename_queue(q) for q in list(queue_timeout)] + return qobj.name in accessible_queues + def enqueue_test_job(): enqueue('frappe.utils.background_jobs.test_job', s=100) diff --git a/frappe/utils/rq.py b/frappe/utils/rq.py index 5e9b9dcd5d..b344b0caa5 100644 --- a/frappe/utils/rq.py +++ b/frappe/utils/rq.py @@ -1,8 +1,7 @@ import redis import frappe -from frappe.utils import get_site_id, get_bench_id, random_string - +from frappe.utils import get_bench_id, random_string class RedisQueue: def __init__(self, conn): @@ -17,9 +16,10 @@ class RedisQueue: return frappe._dict(user_settings) if is_created else {} @classmethod - def get_connection(cls, username='default', password=None): - domain = frappe.local.conf.redis_queue.split("redis://", 1)[-1] - url = f"redis://{username}:{password or ''}@{domain}" + def get_connection(cls, username=None, password=None): + rq_url = frappe.local.conf.redis_queue + domain = rq_url.split("redis://", 1)[-1] + url = (username and f"redis://{username}:{password or ''}@{domain}") or rq_url conn = redis.from_url(url) conn.ping() return conn @@ -63,25 +63,21 @@ class RedisQueue: return ['+@all', '-@admin'] @classmethod - def gen_acl_list(cls, reset_passwords=False, set_admin_password=False): + def gen_acl_list(cls, set_admin_password=False): """Generate list of ACL users needed for this branch. This list contains default ACL user and the bench ACL user(used by all sites incase of ACL is enabled). """ - with frappe.init_site(): - bench_username = get_bench_id() - bench_user_rules = cls.get_acl_key_rules(include_key_prefix=True) + cls.get_acl_command_rules() - + bench_username = get_bench_id() + bench_user_rules = cls.get_acl_key_rules(include_key_prefix=True) + cls.get_acl_command_rules() bench_user_rule_str = ' '.join(bench_user_rules).strip() bench_user_password = random_string(20) - bench_user_resetpass = (reset_passwords and 'resetpass') or '' default_username = 'default' _default_user_password = random_string(20) if set_admin_password else '' default_user_password = '>'+_default_user_password if _default_user_password else 'nopass' - default_user_resetpass = (reset_passwords and set_admin_password and 'resetpass') or '' return [ - f'user {default_username} on {default_user_password} {default_user_resetpass} ~* &* +@all', - f'user {bench_username} on >{bench_user_password} {bench_user_resetpass} {bench_user_rule_str}' + f'user {default_username} on {default_user_password} ~* &* +@all', + f'user {bench_username} on >{bench_user_password} {bench_user_rule_str}' ], {'bench': (bench_username, bench_user_password), 'default': (default_username, _default_user_password)} From 36feb72c89cb34c07bbb2fea94d45d339bf1d2bd Mon Sep 17 00:00:00 2001 From: hasnain2808 Date: Tue, 27 Jul 2021 11:29:00 +0530 Subject: [PATCH 33/86] fix: add column entries in doctypes --- frappe/database/mariadb/framework_mariadb.sql | 1 + frappe/database/postgres/framework_postgres.sql | 1 + 2 files changed, 2 insertions(+) diff --git a/frappe/database/mariadb/framework_mariadb.sql b/frappe/database/mariadb/framework_mariadb.sql index a52efd01e3..f8841e9417 100644 --- a/frappe/database/mariadb/framework_mariadb.sql +++ b/frappe/database/mariadb/framework_mariadb.sql @@ -220,6 +220,7 @@ CREATE TABLE `tabDocType` ( `allow_guest_to_view` int(1) NOT NULL DEFAULT 0, `route` varchar(255) DEFAULT NULL, `is_published_field` varchar(255) DEFAULT NULL, + `website_search_field` varchar(255) DEFAULT NULL, `email_append_to` int(1) NOT NULL DEFAULT 0, `subject_field` varchar(255) DEFAULT NULL, `sender_field` varchar(255) DEFAULT NULL, diff --git a/frappe/database/postgres/framework_postgres.sql b/frappe/database/postgres/framework_postgres.sql index eeb0eecd3f..a4e94aa326 100644 --- a/frappe/database/postgres/framework_postgres.sql +++ b/frappe/database/postgres/framework_postgres.sql @@ -225,6 +225,7 @@ CREATE TABLE "tabDocType" ( "allow_guest_to_view" smallint NOT NULL DEFAULT 0, "route" varchar(255) DEFAULT NULL, "is_published_field" varchar(255) DEFAULT NULL, + "website_search_field" varchar(255) DEFAULT NULL, "email_append_to" smallint NOT NULL DEFAULT 0, "subject_field" varchar(255) DEFAULT NULL, "sender_field" varchar(255) DEFAULT NULL, From 780778d7e3ef442db9e82002123994974ace2df9 Mon Sep 17 00:00:00 2001 From: Mohammad Hasnain Mohsin Rajan Date: Tue, 27 Jul 2021 11:36:14 +0530 Subject: [PATCH 34/86] chore: remove unrelated code --- frappe/migrate.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/frappe/migrate.py b/frappe/migrate.py index d4060e6067..061e4c98d7 100644 --- a/frappe/migrate.py +++ b/frappe/migrate.py @@ -4,8 +4,6 @@ import json import os import sys - -from rq import queue import frappe import frappe.translate import frappe.modules.patch_handler @@ -13,7 +11,6 @@ import frappe.model.sync from frappe.utils.fixtures import sync_fixtures from frappe.utils.connections import check_connection from frappe.utils.dashboard import sync_dashboards -from frappe.utils.background_jobs import enqueue from frappe.cache_manager import clear_global_cache from frappe.desk.notifications import clear_notifications from frappe.website.utils import clear_website_cache @@ -101,7 +98,6 @@ Otherwise, check the server logs and ensure that all the required services are r frappe.publish_realtime("version-update") frappe.flags.in_migrate = False - finally: with open(touched_tables_file, 'w') as f: json.dump(list(frappe.flags.touched_tables), f, sort_keys=True, indent=4) From 574bc57abe56b621053ee7644e5eefe753d4c6a3 Mon Sep 17 00:00:00 2001 From: Mohammad Hasnain Mohsin Rajan Date: Tue, 27 Jul 2021 11:38:51 +0530 Subject: [PATCH 35/86] chore: remove unrelated changes --- frappe/website/page_renderers/document_page.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/frappe/website/page_renderers/document_page.py b/frappe/website/page_renderers/document_page.py index 0108a84f79..01d8ec54c5 100644 --- a/frappe/website/page_renderers/document_page.py +++ b/frappe/website/page_renderers/document_page.py @@ -1,7 +1,7 @@ import frappe from frappe.model.document import get_controller from frappe.website.page_renderers.base_template_page import BaseTemplatePage -from frappe.website.utils import build_response, cache_html +from frappe.website.utils import build_response from frappe.website.router import (get_doctypes_with_web_view, get_page_info_from_web_page_with_dynamic_routes) @@ -47,7 +47,11 @@ class DocumentPage(BaseTemplatePage): return False def render(self): - html = self.get_html() + self.doc = frappe.get_doc(self.doctype, self.docname) + self.init_context() + self.update_context() + self.post_process_context() + html = frappe.get_template(self.template_path).render(self.context) html = self.add_csrf_token(html) return build_response(self.path, html, self.http_status_code or 200, self.headers) From 2a116a59b91e8cd8766e275ea8ab2d7e50f3d4b8 Mon Sep 17 00:00:00 2001 From: Mohammad Hasnain Mohsin Rajan Date: Tue, 27 Jul 2021 11:39:26 +0530 Subject: [PATCH 36/86] chore: remove unrelated changes --- frappe/website/page_renderers/template_page.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/website/page_renderers/template_page.py b/frappe/website/page_renderers/template_page.py index 2ec3f51f41..5e6e57e33a 100644 --- a/frappe/website/page_renderers/template_page.py +++ b/frappe/website/page_renderers/template_page.py @@ -83,7 +83,7 @@ class TemplatePage(BaseTemplatePage): super(TemplatePage, self).post_process_context() def add_sidebar_and_breadcrumbs(self): - if self.basepath and not self.context.sidebar_items: + if self.basepath: self.context.sidebar_items = get_sidebar_items(self.context.website_sidebar, self.basepath) if self.context.add_breadcrumbs and not self.context.parents: From 27608d85900b96e843d7cc05c6c1b3e1e54c6883 Mon Sep 17 00:00:00 2001 From: Mohammad Hasnain Mohsin Rajan Date: Tue, 27 Jul 2021 11:41:16 +0530 Subject: [PATCH 37/86] chore: remove unrelated changes --- frappe/website/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frappe/website/utils.py b/frappe/website/utils.py index 419a5199df..0f5f182ea2 100644 --- a/frappe/website/utils.py +++ b/frappe/website/utils.py @@ -456,6 +456,7 @@ def build_response(path, data, http_status_code, headers=None): response.status_code = http_status_code response.headers["X-Page-Name"] = path.encode("ascii", errors="xmlcharrefreplace") response.headers["X-From-Cache"] = frappe.local.response.from_cache or False + add_preload_headers(response) if headers: for key, val in iteritems(headers): @@ -485,12 +486,11 @@ def set_content_type(response, data, path): return data def add_preload_headers(response): - from bs4 import BeautifulSoup, SoupStrainer + from bs4 import BeautifulSoup try: preload = [] - strainer = SoupStrainer(re.compile("script|link")) - soup = BeautifulSoup(response.data, "lxml", parse_only=strainer) + soup = BeautifulSoup(response.data, "lxml") for elem in soup.find_all('script', src=re.compile(".*")): preload.append(("script", elem.get("src"))) From ea7dd9e8df0b18fe968dad0714a07681ee796972 Mon Sep 17 00:00:00 2001 From: Mohammad Hasnain Mohsin Rajan Date: Tue, 27 Jul 2021 11:42:13 +0530 Subject: [PATCH 38/86] chore: remove unrelated changes --- frappe/website/page_renderers/document_page.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/frappe/website/page_renderers/document_page.py b/frappe/website/page_renderers/document_page.py index 01d8ec54c5..f1741c681f 100644 --- a/frappe/website/page_renderers/document_page.py +++ b/frappe/website/page_renderers/document_page.py @@ -56,16 +56,6 @@ class DocumentPage(BaseTemplatePage): return build_response(self.path, html, self.http_status_code or 200, self.headers) - @cache_html - def get_html(self): - self.doc = frappe.get_doc(self.doctype, self.docname) - self.init_context() - self.update_context() - self.post_process_context() - html = frappe.get_template(self.template_path).render(self.context) - - return html - def update_context(self): self.context.doc = self.doc self.context.update(self.context.doc.as_dict()) From 266a5511c91a7ae223d29a3773643de58c68681f Mon Sep 17 00:00:00 2001 From: Mohammad Hasnain Mohsin Rajan Date: Tue, 27 Jul 2021 12:31:34 +0530 Subject: [PATCH 39/86] chore: fix sider --- frappe/search/website_search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/search/website_search.py b/frappe/search/website_search.py index 1fb84531b0..70962e8172 100644 --- a/frappe/search/website_search.py +++ b/frappe/search/website_search.py @@ -38,7 +38,7 @@ class WebsiteSearch(FullTextSearch): self._items_to_index = [] - routes = get_static_pages_from_all_apps() + slugs_with_web_view(self._items_to_index ) + routes = get_static_pages_from_all_apps() + slugs_with_web_view(self._items_to_index) for i, route in enumerate(routes): From ee6d4f50b4a27e9cc8c3b5f8a821a6d9c0cf4f21 Mon Sep 17 00:00:00 2001 From: Mohammad Hasnain Mohsin Rajan Date: Tue, 27 Jul 2021 13:40:00 +0530 Subject: [PATCH 40/86] chore: remove unwanted changes --- frappe/website/page_renderers/document_page.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/frappe/website/page_renderers/document_page.py b/frappe/website/page_renderers/document_page.py index f1741c681f..6b8d973ead 100644 --- a/frappe/website/page_renderers/document_page.py +++ b/frappe/website/page_renderers/document_page.py @@ -1,7 +1,7 @@ import frappe from frappe.model.document import get_controller from frappe.website.page_renderers.base_template_page import BaseTemplatePage -from frappe.website.utils import build_response +from frappe.website.utils import cache_html from frappe.website.router import (get_doctypes_with_web_view, get_page_info_from_web_page_with_dynamic_routes) @@ -47,14 +47,19 @@ class DocumentPage(BaseTemplatePage): return False def render(self): + html = self.get_html() + html = self.add_csrf_token(html) + + return self.build_response(html) + + @cache_html + def get_html(self): self.doc = frappe.get_doc(self.doctype, self.docname) self.init_context() self.update_context() self.post_process_context() html = frappe.get_template(self.template_path).render(self.context) - html = self.add_csrf_token(html) - - return build_response(self.path, html, self.http_status_code or 200, self.headers) + return html def update_context(self): self.context.doc = self.doc From 61328c29ae367485574d8f5a983c76a1ec55c578 Mon Sep 17 00:00:00 2001 From: shariquerik Date: Tue, 27 Jul 2021 14:12:57 +0530 Subject: [PATCH 41/86] fix: Inbox View not loading --- frappe/public/js/frappe/views/inbox/inbox_view.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/views/inbox/inbox_view.js b/frappe/public/js/frappe/views/inbox/inbox_view.js index 8b53bd49a9..e928327a43 100644 --- a/frappe/public/js/frappe/views/inbox/inbox_view.js +++ b/frappe/public/js/frappe/views/inbox/inbox_view.js @@ -16,7 +16,7 @@ frappe.views.InboxView = class InboxView extends frappe.views.ListView { } frappe.set_route("List", "Communication", "Inbox", email_account); return true; - } else if (!route[3] || (route[3] !== "All Accounts" && !is_valid(route[3]))) { + } else if (!route[3] || (route[3][0] !== "All Accounts" && !is_valid(route[3][0]))) { frappe.throw(__('No email account associated with the User. Please add an account under User > Email Inbox.')); } return false; From 9a282428685f8d0305853392887272560a6b7830 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Wed, 28 Jul 2021 11:32:20 +0530 Subject: [PATCH 42/86] fix: Report print option --- frappe/public/html/print_template.html | 2 +- frappe/public/js/frappe/microtemplate.js | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/frappe/public/html/print_template.html b/frappe/public/html/print_template.html index f63a20377f..e2ff9c9c76 100644 --- a/frappe/public/html/print_template.html +++ b/frappe/public/html/print_template.html @@ -7,7 +7,7 @@ {{ title }} - + diff --git a/frappe/public/js/frappe/microtemplate.js b/frappe/public/js/frappe/microtemplate.js index 7b45db952e..151d008d3e 100644 --- a/frappe/public/js/frappe/microtemplate.js +++ b/frappe/public/js/frappe/microtemplate.js @@ -119,6 +119,10 @@ frappe.render_grid = function(opts) { // render HTML wrapper page opts.base_url = frappe.urllib.get_base_url(); opts.print_css = frappe.boot.print_css; + + opts.lang = opts.lang || frappe.boot.lang, + opts.layout_direction = opts.layout_direction || frappe.utils.is_rtl() ? "rtl" : "ltr"; + var html = frappe.render_template("print_template", opts); var w = window.open(); From 84ff1d0af991fc6be5bd755c716f656baaeef312 Mon Sep 17 00:00:00 2001 From: Aradhya-Tripathi Date: Wed, 28 Jul 2021 13:48:01 +0530 Subject: [PATCH 43/86] style: removed frappe.db.sql comments --- .../core/doctype/activity_log/activity_log.py | 1 - frappe/core/doctype/activity_log/feed.py | 4 ---- .../domain_settings/domain_settings.py | 1 - frappe/core/doctype/error_log/error_log.py | 3 +-- .../core/doctype/log_settings/log_settings.py | 1 - frappe/core/doctype/report/test_report.py | 3 --- frappe/core/doctype/role/role.py | 1 - .../scheduled_job_type/scheduled_job_type.py | 5 +---- frappe/core/doctype/user/user.py | 10 --------- .../user_permission/test_user_permission.py | 6 ------ .../user_permission/user_permission.py | 21 ------------------- .../permission_manager/permission_manager.py | 13 ++++++------ .../doctype/custom_field/custom_field.py | 6 ------ .../doctype/customize_form/customize_form.py | 4 +--- frappe/defaults.py | 5 ----- .../dashboard_chart/test_dashboard_chart.py | 2 -- .../desk/doctype/desktop_icon/desktop_icon.py | 1 - frappe/desk/doctype/event/event.py | 2 -- .../doctype/route_history/route_history.py | 11 +--------- frappe/desk/doctype/tag/tag.py | 2 -- frappe/desk/doctype/todo/todo.py | 6 ------ .../unhandled_email/unhandled_email.py | 4 +--- frappe/model/__init__.py | 9 -------- frappe/model/delete_doc.py | 15 +------------ frappe/model/document.py | 6 ------ .../apply_customization_to_custom_doctype.py | 4 ---- .../sync_stripe_settings_before_migrate.py | 3 +-- .../delete_feedback_request_if_exists.py | 6 +----- .../remove_deprecated_fields_from_doctype.py | 7 +------ .../v12_0/set_primary_key_in_series.py | 3 --- .../setup_comments_from_communications.py | 3 +-- .../patches/v13_0/remove_twilio_settings.py | 3 +-- frappe/permissions.py | 2 -- frappe/sessions.py | 1 - frappe/utils/error.py | 2 -- frappe/utils/global_search.py | 13 ------------ frappe/utils/install.py | 2 -- frappe/utils/password.py | 7 ------- frappe/utils/testutils.py | 1 - .../personal_data_deletion_request.py | 5 ----- .../workflow_action/workflow_action.py | 9 +------- 41 files changed, 18 insertions(+), 195 deletions(-) diff --git a/frappe/core/doctype/activity_log/activity_log.py b/frappe/core/doctype/activity_log/activity_log.py index 2571330f9e..ce5e504f7f 100644 --- a/frappe/core/doctype/activity_log/activity_log.py +++ b/frappe/core/doctype/activity_log/activity_log.py @@ -44,6 +44,5 @@ def clear_activity_logs(days=None): if not days: days = 90 - # frappe.db.delete("Activity Log", filters={"creation": ("<", ["NOW()", "-", f"INTERVAL {days} DAY"])}) frappe.db.sql("""delete from `tabActivity Log` where \ creation< (NOW() - INTERVAL '{0}' DAY)""".format(days)) \ No newline at end of file diff --git a/frappe/core/doctype/activity_log/feed.py b/frappe/core/doctype/activity_log/feed.py index d064022b25..efac825178 100644 --- a/frappe/core/doctype/activity_log/feed.py +++ b/frappe/core/doctype/activity_log/feed.py @@ -32,10 +32,6 @@ def update_feed(doc, method=None): "reference_name": name, "link_doctype": feed.link_doctype }) - # frappe.db.sql("""delete from `tabActivity Log` - # where - # reference_doctype=%s and reference_name=%s - # and link_doctype=%s""", (doctype, name,feed.link_doctype)) frappe.get_doc({ "doctype": "Activity Log", "reference_doctype": doctype, diff --git a/frappe/core/doctype/domain_settings/domain_settings.py b/frappe/core/doctype/domain_settings/domain_settings.py index 4ed9d1e15a..a8c7c6a747 100644 --- a/frappe/core/doctype/domain_settings/domain_settings.py +++ b/frappe/core/doctype/domain_settings/domain_settings.py @@ -35,7 +35,6 @@ class DomainSettings(Document): def remove_role(role): frappe.db.delete("Has Role", {"role": role}) - # frappe.db.sql('delete from `tabHas Role` where role=%s', role) frappe.set_value('Role', role, 'disabled', 1) for domain in all_domains: diff --git a/frappe/core/doctype/error_log/error_log.py b/frappe/core/doctype/error_log/error_log.py index 9a7333995c..081b4e879d 100644 --- a/frappe/core/doctype/error_log/error_log.py +++ b/frappe/core/doctype/error_log/error_log.py @@ -20,5 +20,4 @@ def set_old_logs_as_seen(): def clear_error_logs(): '''Flush all Error Logs''' frappe.only_for('System Manager') - frappe.db.delete("Error Log") - # frappe.db.sql('''DELETE FROM `tabError Log`''') \ No newline at end of file + frappe.db.delete("Error Log") \ No newline at end of file diff --git a/frappe/core/doctype/log_settings/log_settings.py b/frappe/core/doctype/log_settings/log_settings.py index 7f25998dc5..e73aa8dac1 100644 --- a/frappe/core/doctype/log_settings/log_settings.py +++ b/frappe/core/doctype/log_settings/log_settings.py @@ -13,7 +13,6 @@ class LogSettings(Document): self.clear_email_queue() def clear_error_logs(self): - # frappe.db.delete("Error Log", {"creation": ("<", ["NOW()", "-", f"INTERVAL {self.clear_error_log_after} DAY"])}) frappe.db.sql(""" DELETE FROM `tabError Log` WHERE `creation` < (NOW() - INTERVAL '{0}' DAY) """.format(self.clear_error_log_after)) diff --git a/frappe/core/doctype/report/test_report.py b/frappe/core/doctype/report/test_report.py index efc33b888c..1bf9893bd7 100644 --- a/frappe/core/doctype/report/test_report.py +++ b/frappe/core/doctype/report/test_report.py @@ -87,9 +87,6 @@ class TestReport(unittest.TestCase): "parent": frappe.session.user, "role": "Test Has Role" }) - # frappe.db.sql("""delete from `tabHas Role` where parent = %s - # and role = 'Test Has Role'""", frappe.session.user, auto_commit=1) - if not frappe.db.exists('Role', 'Test Has Role'): role = frappe.get_doc({ 'doctype': 'Role', diff --git a/frappe/core/doctype/role/role.py b/frappe/core/doctype/role/role.py index dee25eb476..28b444e1e7 100644 --- a/frappe/core/doctype/role/role.py +++ b/frappe/core/doctype/role/role.py @@ -39,7 +39,6 @@ class Role(Document): def remove_roles(self): frappe.db.delete("Has Role", {"role": self.name}) - # frappe.db.sql("delete from `tabHas Role` where role = %s", self.name) frappe.clear_cache() def on_update(self): diff --git a/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py b/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py index 9882f0edfe..d3afecf378 100644 --- a/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py +++ b/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py @@ -109,13 +109,10 @@ class ScheduledJobType(Document): return 'long' if ('Long' in self.frequency) else 'default' def on_trash(self): - frappe.db.delete("Scheduled Job Log", { "scheduled_job_type": self.name }) - # frappe.db.sql('delete from `tabScheduled Job Log` where scheduled_job_type=%s', self.name) - - + @frappe.whitelist() def execute_event(doc: str): frappe.only_for("System Manager") diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py index 7ff4b8bea9..65a48fb39e 100644 --- a/frappe/core/doctype/user/user.py +++ b/frappe/core/doctype/user/user.py @@ -367,25 +367,15 @@ class User(Document): frappe.local.login_manager.logout(user=self.name) frappe.db.delete("ToDo", {"owner": self.name}) - # frappe.db.sql("""DELETE FROM `tabToDo` WHERE `owner`=%s""", (self.name,)) frappe.db.sql("""UPDATE `tabToDo` SET `assigned_by`=NULL WHERE `assigned_by`=%s""", (self.name,)) # delete events frappe.db.delete("Event", {"owner": self.name, "event_type": "Private"}) - # frappe.db.sql("""delete from `tabEvent` where owner=%s - # and event_type='Private'""", (self.name,)) # delete shares frappe.db.delete("DocShare", {"user": self.name}) - # frappe.db.sql("""delete from `tabDocShare` where user=%s""", self.name) - # delete messages - - # frappe.db.delete("Communication", { - # "reference_doctype": "User", - # "communication_type": ("in", ("Chat", "Notification")), - # }) frappe.db.sql("""delete from `tabCommunication` where communication_type in ('Chat', 'Notification') and reference_doctype='User' diff --git a/frappe/core/doctype/user_permission/test_user_permission.py b/frappe/core/doctype/user_permission/test_user_permission.py index 60a91e6a9b..c3d593ee3b 100644 --- a/frappe/core/doctype/user_permission/test_user_permission.py +++ b/frappe/core/doctype/user_permission/test_user_permission.py @@ -17,12 +17,6 @@ class TestUserPermission(unittest.TestCase): "nested_doc_user@example.com")) }) - # frappe.db.sql("""DELETE FROM `tabUser Permission` - # WHERE `user` in ( - # 'test_bulk_creation_update@example.com', - # 'test_user_perm1@example.com', - # 'nested_doc_user@example.com')""") - # frappe.delete_doc_if_exists("DocType", "Person") frappe.db.sql_ddl("DROP TABLE IF EXISTS `tabPerson`") frappe.delete_doc_if_exists("DocType", "Doc A") frappe.db.sql_ddl("DROP TABLE IF EXISTS `tabDoc A`") diff --git a/frappe/core/doctype/user_permission/user_permission.py b/frappe/core/doctype/user_permission/user_permission.py index 338b063a63..de242efe10 100644 --- a/frappe/core/doctype/user_permission/user_permission.py +++ b/frappe/core/doctype/user_permission/user_permission.py @@ -186,7 +186,6 @@ def clear_user_permissions(user, for_doctype): "user": user, "allow": for_doctype }) - # frappe.db.sql('DELETE FROM `tabUser Permission` WHERE `user`=%s AND `allow`=%s', (user, for_doctype)) frappe.clear_cache() return total @@ -244,13 +243,6 @@ def remove_applicable(perm_applied_docs, user, doctype, docname): "allow": doctype, "for_value": docname }) - # - # frappe.db.sql("""DELETE FROM `tabUser Permission` - # WHERE `user`=%s - # AND `applicable_for`=%s - # AND `allow`=%s - # AND `for_value`=%s - # """, (user, applicable_for, doctype, docname)) def remove_apply_to_all(user, doctype, docname): frappe.db.delete("User Permission", { @@ -259,12 +251,6 @@ def remove_apply_to_all(user, doctype, docname): "allow": doctype, "for_value": docname }) - # frappe.db.sql("""DELETE from `tabUser Permission` - # WHERE `user`=%s - # AND `apply_to_all_doctypes`=1 - # AND `allow`=%s - # AND `for_value`=%s - # """,(user, doctype, docname)) def update_applicable(already_applied, to_apply, user, doctype, docname): for applied in already_applied: @@ -276,10 +262,3 @@ def update_applicable(already_applied, to_apply, user, doctype, docname): "allow": doctype, "for_value": docname }) - - # frappe.db.sql("""DELETE FROM `tabUser Permission` - # WHERE `user`=%s - # AND `applicable_for`=%s - # AND `allow`=%s - # AND `for_value`=%s - # """,(user, applied, doctype, docname)) diff --git a/frappe/core/page/permission_manager/permission_manager.py b/frappe/core/page/permission_manager/permission_manager.py index 0a76283813..af8973811a 100644 --- a/frappe/core/page/permission_manager/permission_manager.py +++ b/frappe/core/page/permission_manager/permission_manager.py @@ -92,14 +92,14 @@ def update(doctype, role, permlevel, ptype, value=None): """Update role permission params Args: - doctype (str): Name of the DocType to update params for - role (str): Role to be updated for, eg "Website Manager". - permlevel (int): perm level the provided rule applies to - ptype (str): permission type, example "read", "delete", etc. - value (None, optional): value for ptype, None indicates False + doctype (str): Name of the DocType to update params for + role (str): Role to be updated for, eg "Website Manager". + permlevel (int): perm level the provided rule applies to + ptype (str): permission type, example "read", "delete", etc. + value (None, optional): value for ptype, None indicates False Returns: - str: Refresh flag is permission is updated successfully + str: Refresh flag is permission is updated successfully """ frappe.only_for("System Manager") out = update_permission_property(doctype, role, permlevel, ptype, value) @@ -114,7 +114,6 @@ def remove(doctype, role, permlevel): frappe.db.delete("Custom DocPerm", { "name": name }) - # frappe.db.sql('delete from `tabCustom DocPerm` where name=%s', name) if not frappe.get_all('Custom DocPerm', dict(parent=doctype)): frappe.throw(_('There must be atleast one permission rule.'), title=_('Cannot Remove')) diff --git a/frappe/custom/doctype/custom_field/custom_field.py b/frappe/custom/doctype/custom_field/custom_field.py index 1d8f4ae67a..e266455f7a 100644 --- a/frappe/custom/doctype/custom_field/custom_field.py +++ b/frappe/custom/doctype/custom_field/custom_field.py @@ -89,12 +89,6 @@ class CustomField(Document): "doc_type": self.dt, "field_name": self.fieldname }) - # frappe.db.sql("""\ - # DELETE FROM `tabProperty Setter` - # WHERE doc_type = %s - # AND field_name = %s""", - # (self.dt, self.fieldname)) - frappe.clear_cache(doctype=self.dt) def validate_insert_after(self, meta): diff --git a/frappe/custom/doctype/customize_form/customize_form.py b/frappe/custom/doctype/customize_form/customize_form.py index 3b7de448d3..5431511d42 100644 --- a/frappe/custom/doctype/customize_form/customize_form.py +++ b/frappe/custom/doctype/customize_form/customize_form.py @@ -24,9 +24,7 @@ class CustomizeForm(Document): "doctype": "Customize Form" }) frappe.db.delete("Customize Form Field") - # frappe.db.sql("delete from tabSingles where doctype='Customize Form'") - # frappe.db.sql("delete from `tabCustomize Form Field`") - + @frappe.whitelist() def fetch_to_customize(self): self.clear_existing_doc() diff --git a/frappe/defaults.py b/frappe/defaults.py index ee73e86a91..794d30a0c8 100644 --- a/frappe/defaults.py +++ b/frappe/defaults.py @@ -128,11 +128,6 @@ def set_default(key, value, parent, parenttype="__default"): "defkey": key, "parent": parent }) - # frappe.db.sql(""" - # delete from - # `tabDefaultValue` - # where - # defkey=%s and parent=%s""", (key, parent)) if value != None: add_default(key, value, parent) else: diff --git a/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py b/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py index 7133640c21..9f10522b12 100644 --- a/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py +++ b/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py @@ -65,7 +65,6 @@ class TestDashboardChart(unittest.TestCase): frappe.delete_doc('Dashboard Chart', 'Test Empty Dashboard Chart') frappe.db.delete("Error Log") - # frappe.db.sql('delete from `tabError Log`') frappe.get_doc(dict( doctype = 'Dashboard Chart', @@ -96,7 +95,6 @@ class TestDashboardChart(unittest.TestCase): frappe.delete_doc('Dashboard Chart', 'Test Empty Dashboard Chart 2') frappe.db.delete("Error Log") - # frappe.db.sql('delete from `tabError Log`') # create one data point frappe.get_doc(dict(doctype = 'Error Log', creation = '2018-06-01 00:00:00')).insert() diff --git a/frappe/desk/doctype/desktop_icon/desktop_icon.py b/frappe/desk/doctype/desktop_icon/desktop_icon.py index bc5677e9b9..283ffad967 100644 --- a/frappe/desk/doctype/desktop_icon/desktop_icon.py +++ b/frappe/desk/doctype/desktop_icon/desktop_icon.py @@ -200,7 +200,6 @@ def set_desktop_icons(visible_list, ignore_duplicate=True): frappe.db.delete("Desktop Icon", { "standard": 0 }) - # frappe.db.sql('delete from `tabDesktop Icon` where standard=0') # set standard as blocked and hidden if setting first active domain if not frappe.flags.keep_desktop_icons: diff --git a/frappe/desk/doctype/event/event.py b/frappe/desk/doctype/event/event.py index f9a0dd74fd..2182ec4c9f 100644 --- a/frappe/desk/doctype/event/event.py +++ b/frappe/desk/doctype/event/event.py @@ -341,11 +341,9 @@ def delete_events(ref_type, ref_name, delete_event=False): frappe.db.delete("Event", { "name": participation.parent }) - # frappe.db.sql("DELETE FROM `tabEvent` WHERE `name` = %(name)s", {'name': participation.parent}) frappe.db.delete("Event Participants", { "name": participation.name }) - # frappe.db.sql("DELETE FROM `tabEvent Participants ` WHERE `name` = %(name)s", {'name': participation.name}) # Close events if ends_on or repeat_till is less than now_datetime def set_status_of_events(): diff --git a/frappe/desk/doctype/route_history/route_history.py b/frappe/desk/doctype/route_history/route_history.py index bfa2711b8f..4303b7554d 100644 --- a/frappe/desk/doctype/route_history/route_history.py +++ b/frappe/desk/doctype/route_history/route_history.py @@ -35,13 +35,4 @@ def flush_old_route_records(): frappe.db.delete("Route History", { "modified": last_record_to_keep[0].modified, "user": user - }) - - # frappe.db.sql(''' - # DELETE - # FROM `tabRoute History` - # WHERE `modified` <= %(modified)s and `user`=%(modified)s - # ''', { - # "modified": last_record_to_keep[0].modified, - # "user": user - # }) \ No newline at end of file + }) \ No newline at end of file diff --git a/frappe/desk/doctype/tag/tag.py b/frappe/desk/doctype/tag/tag.py index 8b78405d06..626a2db085 100644 --- a/frappe/desk/doctype/tag/tag.py +++ b/frappe/desk/doctype/tag/tag.py @@ -127,7 +127,6 @@ def delete_tags_for_document(doc): "document_type": doc.doctype, "document_name": doc.name }) - # frappe.db.sql("""DELETE FROM `tabTag Link` WHERE `document_type`=%s AND `document_name`=%s""", (doc.doctype, doc.name)) def update_tags(doc, tags): """ @@ -170,7 +169,6 @@ def delete_tag_for_document(dt, dn, tag): "document_name": dn, "tag": tag }) - # frappe.db.sql("""DELETE FROM `tabTag Link` WHERE `document_type`=%s AND `document_name`=%s AND tag=%s""", (dt, dn, tag)) @frappe.whitelist() def get_documents_for_tag(tag): diff --git a/frappe/desk/doctype/todo/todo.py b/frappe/desk/doctype/todo/todo.py index 388ba37e1f..34fb3bb905 100644 --- a/frappe/desk/doctype/todo/todo.py +++ b/frappe/desk/doctype/todo/todo.py @@ -44,12 +44,6 @@ class ToDo(Document): "link_doctype": self.doctype, "link_name": self.name }) - # frappe.db.sql(""" - # delete from `tabCommunication Link` - # where link_doctype=%(doctype)s and link_name=%(name)s""", { - # "doctype": self.doctype, "name": self.name - # }) - self.update_in_reference() def add_assign_comment(self, text, comment_type): diff --git a/frappe/email/doctype/unhandled_email/unhandled_email.py b/frappe/email/doctype/unhandled_email/unhandled_email.py index 850007d85f..b445c98aa6 100644 --- a/frappe/email/doctype/unhandled_email/unhandled_email.py +++ b/frappe/email/doctype/unhandled_email/unhandled_email.py @@ -12,6 +12,4 @@ class UnhandledEmail(Document): def remove_old_unhandled_emails(): frappe.db.delete("Unhandled Email", { "creation": ("<", frappe.utils.add_days(frappe.utils.nowdate(), -30)) - }) - # frappe.db.sql("""DELETE FROM `tabUnhandled Email` - # WHERE creation < %s""", frappe.utils.add_days(frappe.utils.nowdate(), -30)) \ No newline at end of file + }) \ No newline at end of file diff --git a/frappe/model/__init__.py b/frappe/model/__init__.py index 7459493fcc..6b38b383bf 100644 --- a/frappe/model/__init__.py +++ b/frappe/model/__init__.py @@ -156,11 +156,6 @@ def delete_fields(args_dict, delete=0): "parent": dt, "fieldname": ("in", fields) }) - # frappe.db.sql(""" - # DELETE FROM `tabDocField` - # WHERE parent='%s' AND fieldname IN (%s) - # """ % (dt, ", ".join(["'{}'".format(f) for f in fields]))) - # Delete the data/column only if delete is specified if not delete: continue @@ -170,10 +165,6 @@ def delete_fields(args_dict, delete=0): "doctype": dt, "field": ("in", fields) }) - # frappe.db.sql(""" - # DELETE FROM `tabSingles` - # WHERE doctype='%s' AND field IN (%s) - # """ % (dt, ", ".join("'{}'".format(f) for f in fields))) else: existing_fields = frappe.db.multisql({ "mariadb": "DESC `tab%s`" % dt, diff --git a/frappe/model/delete_doc.py b/frappe/model/delete_doc.py index 9187310a84..44e675f3c5 100644 --- a/frappe/model/delete_doc.py +++ b/frappe/model/delete_doc.py @@ -82,14 +82,6 @@ def delete_doc(doctype=None, name=None, force=0, ignore_doctypes=None, for_reloa frappe.db.delete("__global_search", { "doctype": name }) - - # frappe.db.sql("delete from `tabCustom Field` where dt = %s", name) - # frappe.db.sql("delete from `tabClient Script` where dt = %s", name) - # frappe.db.sql("delete from `tabProperty Setter` where doc_type = %s", name) - # frappe.db.sql("delete from `tabReport` where ref_doctype=%s", name) - # frappe.db.sql("delete from `tabCustom DocPerm` where parent=%s", name) - # frappe.db.sql("delete from `__global_search` where doctype=%s", name) - delete_from_table(doctype, name, ignore_doctypes, None) if frappe.conf.developer_mode and not doc.custom and not ( @@ -183,13 +175,10 @@ def delete_from_table(doctype, name, ignore_doctypes, doc): frappe.db.delete("Singles", { "doctype": name }) - # frappe.db.sql("delete from `tabSingles` where `doctype`=%s", name) else: frappe.db.delete(doctype, { "name": name }) - # frappe.db.sql("delete from `tab{0}` where `name`=%s".format(doctype), name) - # get child tables if doc: tables = [d.options for d in doc.meta.get_table_fields()] @@ -367,9 +356,7 @@ def clear_timeline_references(link_doctype, link_name): "link_doctype": link_doctype, "link_name": link_name }) - # frappe.db.sql("""DELETE FROM `tabCommunication Link` - # WHERE `tabCommunication Link`.link_doctype=%s AND `tabCommunication Link`.link_name=%s""", (link_doctype, link_name)) - + def insert_feed(doc): if ( frappe.flags.in_install diff --git a/frappe/model/document.py b/frappe/model/document.py index f8adfdb3cf..b44d95716e 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -395,11 +395,6 @@ class Document(BaseDocument): "parenttype": self.doctype, "parentfield": fieldname }) - - # frappe.db.sql("""delete from `tab{0}` where parent=%s - # and parenttype=%s and parentfield=%s""".format(df.options), - # (self.name, self.doctype, fieldname)) - def get_doc_before_save(self): return getattr(self, '_doc_before_save', None) @@ -460,7 +455,6 @@ class Document(BaseDocument): frappe.db.delete("Singles", { "doctype": self.doctype }) - # frappe.db.sql("""delete from `tabSingles` where doctype=%s""", self.doctype) for field, value in d.items(): if field != "doctype": frappe.db.sql("""insert into `tabSingles` (doctype, field, value) diff --git a/frappe/patches/v11_0/apply_customization_to_custom_doctype.py b/frappe/patches/v11_0/apply_customization_to_custom_doctype.py index c13ff83a1f..c01db50d5e 100644 --- a/frappe/patches/v11_0/apply_customization_to_custom_doctype.py +++ b/frappe/patches/v11_0/apply_customization_to_custom_doctype.py @@ -31,8 +31,6 @@ def execute(): frappe.db.delete("Property Setter", { "name": prop.name }) - # frappe.db.sql('DELETE FROM `tabProperty Setter` WHERE `name`=%s', prop.name) - meta = frappe.get_meta(doctype.name) for df in meta.fields: @@ -56,6 +54,4 @@ def execute(): frappe.db.delete("Custom Field", { "name": cf.name }) - # frappe.db.sql('DELETE FROM `tabCustom Field` WHERE name=%s', cf.name) - meta.save() diff --git a/frappe/patches/v11_0/sync_stripe_settings_before_migrate.py b/frappe/patches/v11_0/sync_stripe_settings_before_migrate.py index 1a12d4dc1d..5c949fbb1d 100644 --- a/frappe/patches/v11_0/sync_stripe_settings_before_migrate.py +++ b/frappe/patches/v11_0/sync_stripe_settings_before_migrate.py @@ -19,5 +19,4 @@ def execute(): frappe.db.delete("Singles", { "doctype": "Stripe Settings" - }) - # frappe.db.sql("""DELETE FROM tabSingles WHERE doctype='Stripe Settings'""") \ No newline at end of file + }) \ No newline at end of file diff --git a/frappe/patches/v12_0/delete_feedback_request_if_exists.py b/frappe/patches/v12_0/delete_feedback_request_if_exists.py index de799b1e21..bc3c7b8f97 100644 --- a/frappe/patches/v12_0/delete_feedback_request_if_exists.py +++ b/frappe/patches/v12_0/delete_feedback_request_if_exists.py @@ -4,8 +4,4 @@ import frappe def execute(): frappe.db.delete("DocType", { "name": "Feedback Request" - }) - # frappe.db.sql(''' - #DELETE from `tabDocType` - #WHERE name = 'Feedback Request' - # ''') \ No newline at end of file + }) \ No newline at end of file diff --git a/frappe/patches/v12_0/remove_deprecated_fields_from_doctype.py b/frappe/patches/v12_0/remove_deprecated_fields_from_doctype.py index 17dfbc5ca1..9c9a79ccbf 100644 --- a/frappe/patches/v12_0/remove_deprecated_fields_from_doctype.py +++ b/frappe/patches/v12_0/remove_deprecated_fields_from_doctype.py @@ -10,9 +10,4 @@ def execute(): frappe.db.delete("Property Setter", { "property": "read_only_onload" - }) - - # frappe.db.sql(''' - # DELETE from `tabProperty Setter` - # WHERE property = 'read_only_onload' - # ''') + }) \ No newline at end of file diff --git a/frappe/patches/v12_0/set_primary_key_in_series.py b/frappe/patches/v12_0/set_primary_key_in_series.py index a8409cbbba..e8d3abdde1 100644 --- a/frappe/patches/v12_0/set_primary_key_in_series.py +++ b/frappe/patches/v12_0/set_primary_key_in_series.py @@ -15,9 +15,6 @@ def execute(): frappe.db.delete("Series", { "name": row.name }) - # frappe.db.sql('delete from `tabSeries` where name = %(key)s', { - # 'key': row.name - # }) if row.current: frappe.db.sql('insert into `tabSeries`(`name`, `current`) values (%(name)s, %(current)s)', row) frappe.db.commit() diff --git a/frappe/patches/v12_0/setup_comments_from_communications.py b/frappe/patches/v12_0/setup_comments_from_communications.py index a4e3fb4261..b72d02720e 100644 --- a/frappe/patches/v12_0/setup_comments_from_communications.py +++ b/frappe/patches/v12_0/setup_comments_from_communications.py @@ -31,5 +31,4 @@ def execute(): # clean up frappe.db.delete("Communication", { "communication_type": "Comment" - }) - # frappe.db.sql("delete from `tabCommunication` where communication_type = 'Comment'") + }) \ No newline at end of file diff --git a/frappe/patches/v13_0/remove_twilio_settings.py b/frappe/patches/v13_0/remove_twilio_settings.py index 5d17364847..8b7e5e02c2 100644 --- a/frappe/patches/v13_0/remove_twilio_settings.py +++ b/frappe/patches/v13_0/remove_twilio_settings.py @@ -15,9 +15,8 @@ def execute(): frappe.db.delete("Singles", { "doctype": "Twilio Settings" }) - # frappe.db.sql("delete from `tabSingles` where `doctype`=%s", 'Twilio Settings') def twilio_settings_doctype_in_integrations() -> bool: """Check Twilio Settings doctype exists in integrations module or not. """ - return frappe.db.exists("DocType", {'name': 'Twilio Settings', 'module': 'Integrations'}) + return frappe.db.exists("DocType", {'name': 'Twilio Settings', 'module': 'Integrations'}) \ No newline at end of file diff --git a/frappe/permissions.py b/frappe/permissions.py index 7724f09e78..82f27452fb 100644 --- a/frappe/permissions.py +++ b/frappe/permissions.py @@ -520,8 +520,6 @@ def reset_perms(doctype): delete_notification_count_for(doctype) frappe.db.delete("Custom DocPerm", {"parent": doctype}) - # frappe.db.sql("""delete from `tabCustom DocPerm` where parent=%s""", doctype) - def get_linked_doctypes(dt): return list(set([dt] + [d.options for d in frappe.get_meta(dt).get("fields", { diff --git a/frappe/sessions.py b/frappe/sessions.py index f4d05538bc..58158a2e4e 100644 --- a/frappe/sessions.py +++ b/frappe/sessions.py @@ -84,7 +84,6 @@ def delete_session(sid=None, user=None, reason="Session Expired"): logout_feed(user, reason) frappe.db.delete("Sessions", {"sid": sid}) - # frappe.db.sql("""delete from tabSessions where sid=%s""", sid) frappe.db.commit() def clear_all_sessions(reason=None): diff --git a/frappe/utils/error.py b/frappe/utils/error.py index f2cbc5ec10..d83cbb49ea 100644 --- a/frappe/utils/error.py +++ b/frappe/utils/error.py @@ -177,8 +177,6 @@ def collect_error_snapshots(): def clear_old_snapshots(): """Clear snapshots that are older than a month""" - # frappe.db.delete("Error Snapshot", filters={"creation": ("<", ["NOW()", "-", f"INTERVAL '1' MONTH"])}) - frappe.db.sql("""delete from `tabError Snapshot` where creation < (NOW() - INTERVAL '1' MONTH)""") diff --git a/frappe/utils/global_search.py b/frappe/utils/global_search.py index a018761370..6391fcf7ef 100644 --- a/frappe/utils/global_search.py +++ b/frappe/utils/global_search.py @@ -24,8 +24,6 @@ def reset(): :return: """ frappe.db.delete("__global_search") - # frappe.db.sql('DELETE FROM `__global_search`') - def get_doctypes_with_global_search(with_child_tables=True): """ @@ -150,10 +148,6 @@ def delete_global_search_records_for_doctype(doctype): frappe.db.delete("__global_search", { "doctype": doctype }) - # frappe.db.sql('''DELETE - # FROM `__global_search` - # WHERE doctype = %s''', doctype, as_dict=True) - def get_selected_fields(meta, global_search_fields): fieldnames = [df.fieldname for df in global_search_fields] @@ -403,18 +397,11 @@ def delete_for_document(doc): been deleted :param doc: Deleted document """ - frappe.db.delete("__global_search", { "doctype": doc.doctype, "name": doc.name }) - # frappe.db.sql('''DELETE - # FROM `__global_search` - # WHERE doctype = %s - # AND name = %s''', (doc.doctype, doc.name), as_dict=True) - - @frappe.whitelist() def search(text, start=0, limit=20, doctype=""): """ diff --git a/frappe/utils/install.py b/frappe/utils/install.py index 9d21a0d945..e6d4386ebe 100644 --- a/frappe/utils/install.py +++ b/frappe/utils/install.py @@ -113,8 +113,6 @@ def before_tests(): frappe.db.delete("Custom Field") frappe.db.delete("Event") - # frappe.db.sql("delete from `tabCustom Field`") - # frappe.db.sql("delete from `tabEvent`") frappe.db.commit() frappe.clear_cache() diff --git a/frappe/utils/password.py b/frappe/utils/password.py index 21e1bc9586..a097c58b31 100644 --- a/frappe/utils/password.py +++ b/frappe/utils/password.py @@ -71,11 +71,6 @@ def remove_encrypted_password(doctype, name, fieldname='password'): "fieldname": fieldname }) - # frappe.db.sql( - # 'DELETE FROM `__Auth` WHERE doctype = %s and name = %s and fieldname = %s', - # values=[doctype, name, fieldname] - # ) - def check_password(user, pwd, doctype='User', fieldname='password', delete_tracker_cache=True): '''Checks if user and password are correct, else raises frappe.AuthenticationError''' @@ -140,8 +135,6 @@ def delete_all_passwords_for(doctype, name): "doctype": doctype, "name": name }) - # frappe.db.sql("""delete from `__Auth` where `doctype`=%(doctype)s and `name`=%(name)s""", - # { 'doctype': doctype, 'name': name }) except Exception as e: if not frappe.db.is_missing_column(e): raise diff --git a/frappe/utils/testutils.py b/frappe/utils/testutils.py index 74b800bdcf..5c5bfa7976 100644 --- a/frappe/utils/testutils.py +++ b/frappe/utils/testutils.py @@ -15,5 +15,4 @@ def clear_custom_fields(doctype): frappe.db.delete("Custom Field", { "dt": doctype }) - # frappe.db.sql('delete from `tabCustom Field` where dt=%s', doctype) frappe.clear_cache(doctype=doctype) diff --git a/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py b/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py index 96898cc159..54d2c2e446 100644 --- a/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py +++ b/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py @@ -323,11 +323,6 @@ class PersonalDataDeletionRequest(Document): def remove_unverified_record(): - # frappe.db.delete("Personal Data Deletion Request", { - # "status": "Pending Verification", - # "creation": ("<", ["Now()", "-", "INTERVAL 7 DAY"]) - # }) - frappe.db.sql( """ DELETE FROM `tabPersonal Data Deletion Request` diff --git a/frappe/workflow/doctype/workflow_action/workflow_action.py b/frappe/workflow/doctype/workflow_action/workflow_action.py index 5957bf8671..e3b89ba0e5 100644 --- a/frappe/workflow/doctype/workflow_action/workflow_action.py +++ b/frappe/workflow/doctype/workflow_action/workflow_action.py @@ -139,10 +139,7 @@ def clear_old_workflow_actions(doc, user=None): "user": ("!=", user), "status": "Open" }) - # frappe.db.sql("""DELETE FROM `tabWorkflow Action` - # WHERE `reference_doctype`=%s AND `reference_name`=%s AND `user`!=%s AND `status`='Open'""", - # (doc.get('doctype'), doc.get('name'), user)) - + def update_completed_workflow_actions(doc, user=None): user = user if user else frappe.session.user frappe.db.sql("""UPDATE `tabWorkflow Action` SET `status`='Completed', `completed_by`=%s @@ -263,10 +260,6 @@ def clear_workflow_actions(doctype, name): "reference_doctype": doctype, "reference_name": name }) - # frappe.db.sql('''delete from `tabWorkflow Action` - # where reference_doctype=%s and reference_name=%s''', - # (doctype, name)) - def get_doc_workflow_state(doc): workflow_name = get_workflow_name(doc.get('doctype')) workflow_state_field = get_workflow_state_field(workflow_name) From 9a74255a2e8334b42a5b8f66799f207b0a92309a Mon Sep 17 00:00:00 2001 From: leela Date: Wed, 28 Jul 2021 13:35:08 +0530 Subject: [PATCH 44/86] style: Add row numbering for report view print format --- frappe/public/js/frappe/views/reports/print_grid.html | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frappe/public/js/frappe/views/reports/print_grid.html b/frappe/public/js/frappe/views/reports/print_grid.html index e607f12b52..6f693bc932 100644 --- a/frappe/public/js/frappe/views/reports/print_grid.html +++ b/frappe/public/js/frappe/views/reports/print_grid.html @@ -11,6 +11,7 @@ + # {% for col in columns %} {% if col.name && col._id !== "_check" %} {% for row in data %} + + {{ row._index + 1 }} + {% for col in columns %} {% if col.name && col._id !== "_check" %} From b719e1481de9a629eee42559f5b0599a4c049d6f Mon Sep 17 00:00:00 2001 From: leela Date: Wed, 28 Jul 2021 17:00:12 +0530 Subject: [PATCH 45/86] Revert "refactor: set amended docname to original docname" This reverts commit d459847ae39f8e9917686c3db6340ee95103085e. --- frappe/core/doctype/doctype/test_doctype.py | 8 +- frappe/model/document.py | 9 +- frappe/model/naming.py | 83 +++---------------- frappe/patches.txt | 1 - frappe/patches/v13_0/rename_cancelled_docs.py | 27 ------ frappe/public/js/frappe/form/form.js | 24 +++--- frappe/public/js/frappe/router.js | 6 -- frappe/tests/test_naming.py | 34 -------- 8 files changed, 28 insertions(+), 164 deletions(-) delete mode 100644 frappe/patches/v13_0/rename_cancelled_docs.py diff --git a/frappe/core/doctype/doctype/test_doctype.py b/frappe/core/doctype/doctype/test_doctype.py index 4b6d0e4794..1e1a01a685 100644 --- a/frappe/core/doctype/doctype/test_doctype.py +++ b/frappe/core/doctype/doctype/test_doctype.py @@ -348,7 +348,6 @@ class TestDocType(unittest.TestCase): dump_docs = json.dumps(docs.get('docs')) cancel_all_linked_docs(dump_docs) data_link_doc.cancel() - data_doc.name = '{}-CAN-0'.format(data_doc.name) data_doc.load_from_db() self.assertEqual(data_link_doc.docstatus, 2) self.assertEqual(data_doc.docstatus, 2) @@ -372,7 +371,7 @@ class TestDocType(unittest.TestCase): for data in link_doc.get('permissions'): data.submit = 1 data.cancel = 1 - link_doc.insert(ignore_if_duplicate=True) + link_doc.insert() #create first parent doctype test_doc_1 = new_doctype('Test Doctype 1') @@ -387,7 +386,7 @@ class TestDocType(unittest.TestCase): for data in test_doc_1.get('permissions'): data.submit = 1 data.cancel = 1 - test_doc_1.insert(ignore_if_duplicate=True) + test_doc_1.insert() #crete second parent doctype doc = new_doctype('Test Doctype 2') @@ -402,7 +401,7 @@ class TestDocType(unittest.TestCase): for data in link_doc.get('permissions'): data.submit = 1 data.cancel = 1 - doc.insert(ignore_if_duplicate=True) + doc.insert() # create doctype data data_link_doc_1 = frappe.new_doc('Test Linked Doctype 1') @@ -433,7 +432,6 @@ class TestDocType(unittest.TestCase): # checking that doc for Test Doctype 2 is not canceled self.assertRaises(frappe.LinkExistsError, data_link_doc_1.cancel) - data_doc_2.name = '{}-CAN-0'.format(data_doc_2.name) data_doc.load_from_db() data_doc_2.load_from_db() self.assertEqual(data_link_doc_1.docstatus, 2) diff --git a/frappe/model/document.py b/frappe/model/document.py index e974ae2a3e..61160e1f01 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -5,7 +5,7 @@ import time from frappe import _, msgprint, is_whitelisted from frappe.utils import flt, cstr, now, get_datetime_str, file_lock, date_diff from frappe.model.base_document import BaseDocument, get_controller -from frappe.model.naming import set_new_name, gen_new_name_for_cancelled_doc +from frappe.model.naming import set_new_name from werkzeug.exceptions import NotFound, Forbidden import hashlib, json from frappe.model import optional_fields, table_fields @@ -705,6 +705,7 @@ class Document(BaseDocument): else: tmp = frappe.db.sql("""select modified, docstatus from `tab{0}` where name = %s for update""".format(self.doctype), self.name, as_dict=True) + if not tmp: frappe.throw(_("Record does not exist")) else: @@ -915,12 +916,8 @@ class Document(BaseDocument): @whitelist.__func__ def _cancel(self): - """Cancel the document. Sets `docstatus` = 2, then saves. - """ + """Cancel the document. Sets `docstatus` = 2, then saves.""" self.docstatus = 2 - new_name = gen_new_name_for_cancelled_doc(self) - frappe.rename_doc(self.doctype, self.name, new_name, force=True, show_alert=False) - self.name = new_name self.save() @whitelist.__func__ diff --git a/frappe/model/naming.py b/frappe/model/naming.py index 6dff0aaff6..fe136adce8 100644 --- a/frappe/model/naming.py +++ b/frappe/model/naming.py @@ -28,7 +28,7 @@ def set_new_name(doc): doc.name = None if getattr(doc, "amended_from", None): - doc.name = _get_amended_name(doc) + _set_amended_name(doc) return elif getattr(doc.meta, "issingle", False): @@ -221,15 +221,6 @@ def revert_series_if_last(key, name, doc=None): * prefix = #### and hashes = 2021 (hash doesn't exist) * will search hash in key then accordingly get prefix = "" """ - if hasattr(doc, 'amended_from'): - # do not revert if doc is amended, since cancelled docs still exist - if doc.docstatus != 2 and doc.amended_from: - return - - # for first cancelled doc - if doc.docstatus == 2 and not doc.amended_from: - name, _ = NameParser.parse_docname(doc.name, sep='-CAN-') - if ".#" in key: prefix, hashes = key.rsplit(".", 1) if "#" not in hashes: @@ -312,9 +303,16 @@ def append_number_if_name_exists(doctype, value, fieldname="name", separator="-" return value -def _get_amended_name(doc): - name, _ = NameParser(doc).parse_amended_from() - return name +def _set_amended_name(doc): + am_id = 1 + am_prefix = doc.amended_from + if frappe.db.get_value(doc.doctype, doc.amended_from, "amended_from"): + am_id = cint(doc.amended_from.split("-")[-1]) + 1 + am_prefix = "-".join(doc.amended_from.split("-")[:-1]) # except the last hyphen + + doc.name = am_prefix + "-" + str(am_id) + return doc.name + def _field_autoname(autoname, doc, skip_slicing=None): """ @@ -325,6 +323,7 @@ def _field_autoname(autoname, doc, skip_slicing=None): name = (cstr(doc.get(fieldname)) or "").strip() return name + def _prompt_autoname(autoname, doc): """ Generate a name using Prompt option. This simply means the user will have to set the name manually. @@ -355,61 +354,3 @@ def _format_autoname(autoname, doc): name = re.sub(r"(\{[\w | #]+\})", get_param_value_for_match, autoname_value) return name - -class NameParser: - """Parse document name and return all the parts of it. - - NOTE: It handles cancellend and amended doc parsing for now. It can be expanded. - """ - def __init__(self, doc): - self.doc = doc - - def parse_name(self): - if not hasattr(self.doc, "amended_from"): - return (self.doc.name, None, None) - - #If document is cancelled document - if hasattr(self.doc, "amended_from") and self.doc.docstatus == 2: - return self.parse_docname(self.doc.name, sep='-CAN-') - return self.parse_docname(self.doc.name) - - def parse_amended_from(self): - if not getattr(self.doc, 'amended_from', None): - return (None, None) - return self.parse_docname(self.doc.amended_from, '-CAN-') - - @classmethod - def parse_docname(cls, name, sep='-'): - split_list = name.rsplit(sep, 1) - - if len(split_list) == 1: - return (name, None) - return (split_list[0], split_list[1]) - -def get_cancelled_doc_latest_counter(tname, docname): - """Get the latest counter used for cancelled docs of given docname. - """ - name_prefix = f'{docname}-CAN-' - - rows = frappe.db.sql(""" - select - name - from `tab{tname}` - where - name like %(name_prefix)s and docstatus=2 - """.format(tname=tname), {'name_prefix': name_prefix+'%'}, as_dict=1) - - if not rows: - return -1 - return max([int(row.name.replace(name_prefix, '') or -1) for row in rows]) - -def gen_new_name_for_cancelled_doc(doc): - """Generate a new name for cancelled document. - """ - if getattr(doc, "amended_from", None): - name, _ = NameParser(doc).parse_amended_from() - else: - name = doc.name - - counter = get_cancelled_doc_latest_counter(doc.doctype, name) - return f'{name}-CAN-{counter+1}' diff --git a/frappe/patches.txt b/frappe/patches.txt index a9c5807df0..7605d8ea2b 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -180,4 +180,3 @@ frappe.patches.v12_0.rename_uploaded_files_with_proper_name frappe.patches.v13_0.queryreport_columns frappe.patches.v13_0.jinja_hook frappe.patches.v13_0.update_notification_channel_if_empty -frappe.patches.v13_0.rename_cancelled_docs diff --git a/frappe/patches/v13_0/rename_cancelled_docs.py b/frappe/patches/v13_0/rename_cancelled_docs.py deleted file mode 100644 index 2e99a6f3cd..0000000000 --- a/frappe/patches/v13_0/rename_cancelled_docs.py +++ /dev/null @@ -1,27 +0,0 @@ -import frappe -from frappe.model.naming import NameParser -from frappe.model.rename_doc import rename_doc - -def execute(): - """Rename already cancelled documents by adding `CAN-X` postfix instead of `-X`. - """ - for doctype in frappe.db.get_all('DocType'): - doctype = frappe.get_doc('DocType', doctype.name) - if doctype.is_submittable and frappe.db.table_exists(doctype.name): - cancelled_docs = frappe.db.get_all(doctype.name, ['amended_from', 'name'], {'docstatus':2}) - - for doc in cancelled_docs: - if '-CAN-' in doc.name: - continue - - current_name = doc.name - - if getattr(doc, "amended_from", None): - orig_name, counter = NameParser.parse_docname(doc.name) - else: - orig_name, counter = doc.name, 0 - new_name = f'{orig_name}-CAN-{counter or 0}' - - print(f"Renaming {doctype.name} record from {current_name} to {new_name}") - rename_doc(doctype.name, current_name, new_name, ignore_permissions=True, show_alert=False) - frappe.db.commit() diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js index 3bbc883b0c..faaa3dfbd9 100644 --- a/frappe/public/js/frappe/form/form.js +++ b/frappe/public/js/frappe/form/form.js @@ -770,36 +770,32 @@ frappe.ui.form.Form = class FrappeForm { } _cancel(btn, callback, on_error, skip_confirm) { + const me = this; const cancel_doc = () => { frappe.validated = true; - this.script_manager.trigger("before_cancel").then(() => { + me.script_manager.trigger("before_cancel").then(() => { if (!frappe.validated) { - return this.handle_save_fail(btn, on_error); + return me.handle_save_fail(btn, on_error); } - const original_name = this.docname; - const after_cancel = (r) => { + var after_cancel = function(r) { if (r.exc) { - this.handle_save_fail(btn, on_error); + me.handle_save_fail(btn, on_error); } else { frappe.utils.play_sound("cancel"); + me.refresh(); callback && callback(); - this.script_manager.trigger("after_cancel"); - frappe.run_serially([ - () => this.rename_notify(this.doctype, original_name, r.docs[0].name), - () => frappe.router.clear_re_route(this.doctype, original_name), - () => this.refresh(), - ]); + me.script_manager.trigger("after_cancel"); } }; - frappe.ui.form.save(this, "cancel", after_cancel, btn); + frappe.ui.form.save(me, "cancel", after_cancel, btn); }); } if (skip_confirm) { cancel_doc(); } else { - frappe.confirm(__("Permanently Cancel {0}?", [this.docname]), cancel_doc, this.handle_save_fail(btn, on_error)); + frappe.confirm(__("Permanently Cancel {0}?", [this.docname]), cancel_doc, me.handle_save_fail(btn, on_error)); } }; @@ -821,7 +817,7 @@ frappe.ui.form.Form = class FrappeForm { 'docname': this.doc.name }).then(is_amended => { if (is_amended) { - frappe.throw(__('This document is already amended, you cannot amend it again')); + frappe.throw(__('This document is already amended, you cannot ammend it again')); } this.validate_form_action("Amend"); var me = this; diff --git a/frappe/public/js/frappe/router.js b/frappe/public/js/frappe/router.js index 3f8ce7fee2..8a29dc1040 100644 --- a/frappe/public/js/frappe/router.js +++ b/frappe/public/js/frappe/router.js @@ -234,12 +234,6 @@ frappe.router = { } }, - clear_re_route(doctype, docname) { - delete frappe.re_route[ - `${encodeURIComponent(frappe.router.slug(doctype))}/${encodeURIComponent(docname)}` - ]; - }, - set_title(sub_path) { if (frappe.route_titles[sub_path]) { frappe.utils.set_title(frappe.route_titles[sub_path]); diff --git a/frappe/tests/test_naming.py b/frappe/tests/test_naming.py index 78071a4120..557993882f 100644 --- a/frappe/tests/test_naming.py +++ b/frappe/tests/test_naming.py @@ -116,37 +116,3 @@ class TestNaming(unittest.TestCase): self.assertEqual(current_index.get('current'), 2) frappe.db.sql("""delete from `tabSeries` where name = %s""", series) - - def test_naming_for_cancelled_and_amended_doc(self): - submittable_doctype = frappe.get_doc({ - "doctype": "DocType", - "module": "Core", - "custom": 1, - "is_submittable": 1, - "permissions": [{ - "role": "System Manager", - "read": 1 - }], - "name": 'Submittable Doctype' - }).insert(ignore_if_duplicate=True) - - doc = frappe.new_doc('Submittable Doctype') - doc.save() - original_name = doc.name - - doc.submit() - doc.cancel() - cancelled_name = doc.name - self.assertEqual(cancelled_name, "{}-CAN-0".format(original_name)) - - amended_doc = frappe.copy_doc(doc) - amended_doc.docstatus = 0 - amended_doc.amended_from = doc.name - amended_doc.save() - self.assertEqual(amended_doc.name, original_name) - - amended_doc.submit() - amended_doc.cancel() - self.assertEqual(amended_doc.name, "{}-CAN-1".format(original_name)) - - submittable_doctype.delete() From 01d275f6677f4cd18b72b12d32ba4104a5b0ce30 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 28 Jul 2021 18:55:27 +0530 Subject: [PATCH 46/86] style: Format code for better readability * Remove trailing whitespaces * Format code to fit module conventions * Add appropriate new lines between imports, classes, fn defs, etc * Added comments, docstrings & module headers --- .../core/doctype/activity_log/activity_log.py | 1 + frappe/core/doctype/activity_log/feed.py | 5 ++- .../scheduled_job_type/scheduled_job_type.py | 11 +++--- frappe/core/doctype/user/user.py | 2 ++ .../user_permission/test_user_permission.py | 5 ++- .../user_permission/user_permission.py | 29 ++++++++------- .../doctype/customize_form/customize_form.py | 11 +++--- .../desk/doctype/desktop_icon/desktop_icon.py | 4 +-- frappe/desk/doctype/event/event.py | 8 ++--- .../doctype/route_history/route_history.py | 4 +-- frappe/desk/doctype/tag/tag.py | 3 +- frappe/model/__init__.py | 7 ++-- frappe/model/delete_doc.py | 36 ++++++------------- .../apply_customization_to_custom_doctype.py | 10 +++--- .../sync_stripe_settings_before_migrate.py | 4 +-- .../delete_feedback_request_if_exists.py | 4 +-- .../v12_0/set_primary_key_in_series.py | 2 ++ .../setup_comments_from_communications.py | 2 +- .../patches/v13_0/remove_twilio_settings.py | 2 +- frappe/permissions.py | 2 +- frappe/sessions.py | 1 + frappe/tests/test_commands.py | 2 +- frappe/utils/error.py | 4 +-- frappe/utils/testutils.py | 4 +-- .../personal_data_deletion_request.py | 1 + .../workflow_action/workflow_action.py | 2 +- 26 files changed, 72 insertions(+), 94 deletions(-) diff --git a/frappe/core/doctype/activity_log/activity_log.py b/frappe/core/doctype/activity_log/activity_log.py index ce5e504f7f..efec0dc217 100644 --- a/frappe/core/doctype/activity_log/activity_log.py +++ b/frappe/core/doctype/activity_log/activity_log.py @@ -44,5 +44,6 @@ def clear_activity_logs(days=None): if not days: days = 90 + frappe.db.sql("""delete from `tabActivity Log` where \ creation< (NOW() - INTERVAL '{0}' DAY)""".format(days)) \ No newline at end of file diff --git a/frappe/core/doctype/activity_log/feed.py b/frappe/core/doctype/activity_log/feed.py index efac825178..19d7b77184 100644 --- a/frappe/core/doctype/activity_log/feed.py +++ b/frappe/core/doctype/activity_log/feed.py @@ -26,12 +26,15 @@ def update_feed(doc, method=None): feed = frappe._dict(feed) doctype = feed.doctype or doc.doctype - name = feed.name or doc.name + name = feed.name or doc.name + + # delete earlier feed frappe.db.delete("Activity Log", { "reference_doctype": doctype, "reference_name": name, "link_doctype": feed.link_doctype }) + frappe.get_doc({ "doctype": "Activity Log", "reference_doctype": doctype, diff --git a/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py b/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py index d3afecf378..b6515b1e79 100644 --- a/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py +++ b/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py @@ -1,5 +1,4 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2019, Frappe Technologies and contributors +# Copyright (c) 2021, Frappe Technologies and contributors # For license information, please see license.txt import json @@ -13,6 +12,7 @@ from frappe.model.document import Document from frappe.utils import get_datetime, now_datetime from frappe.utils.background_jobs import enqueue, get_jobs + class ScheduledJobType(Document): def autoname(self): self.name = ".".join(self.method.split(".")[-2:]) @@ -109,10 +109,9 @@ class ScheduledJobType(Document): return 'long' if ('Long' in self.frequency) else 'default' def on_trash(self): - frappe.db.delete("Scheduled Job Log", { - "scheduled_job_type": self.name - }) - + frappe.db.delete("Scheduled Job Log", {"scheduled_job_type": self.name}) + + @frappe.whitelist() def execute_event(doc: str): frappe.only_for("System Manager") diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py index 65a48fb39e..5d799f8ee9 100644 --- a/frappe/core/doctype/user/user.py +++ b/frappe/core/doctype/user/user.py @@ -18,6 +18,7 @@ from frappe.rate_limiter import rate_limit from frappe.utils.background_jobs import enqueue from frappe.core.doctype.user_type.user_type import user_linked_with_permission_on_doctype + STANDARD_USERS = ("Guest", "Administrator") @@ -366,6 +367,7 @@ class User(Document): if getattr(frappe.local, "login_manager", None): frappe.local.login_manager.logout(user=self.name) + # delete todos frappe.db.delete("ToDo", {"owner": self.name}) frappe.db.sql("""UPDATE `tabToDo` SET `assigned_by`=NULL WHERE `assigned_by`=%s""", (self.name,)) diff --git a/frappe/core/doctype/user_permission/test_user_permission.py b/frappe/core/doctype/user_permission/test_user_permission.py index c3d593ee3b..1e48de334b 100644 --- a/frappe/core/doctype/user_permission/test_user_permission.py +++ b/frappe/core/doctype/user_permission/test_user_permission.py @@ -1,6 +1,5 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Frappe Technologies and Contributors -# See license.txt +# Copyright (c) 2021, Frappe Technologies and Contributors +# See LICENSE from frappe.core.doctype.user_permission.user_permission import add_user_permissions, remove_applicable from frappe.permissions import has_user_permission from frappe.core.doctype.doctype.test_doctype import new_doctype diff --git a/frappe/core/doctype/user_permission/user_permission.py b/frappe/core/doctype/user_permission/user_permission.py index de242efe10..5201ffef8d 100644 --- a/frappe/core/doctype/user_permission/user_permission.py +++ b/frappe/core/doctype/user_permission/user_permission.py @@ -1,5 +1,4 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Frappe Technologies and contributors +# Copyright (c) 2021, Frappe Technologies and contributors # For license information, please see license.txt import frappe, json @@ -179,14 +178,16 @@ def check_applicable_doc_perm(user, doctype, docname): @frappe.whitelist() def clear_user_permissions(user, for_doctype): - frappe.only_for('System Manager') - total = frappe.db.count('User Permission', filters = dict(user=user, allow=for_doctype)) + frappe.only_for("System Manager") + total = frappe.db.count("User Permission", {"user": user, "allow": for_doctype}) + if total: frappe.db.delete("User Permission", { + "allow": for_doctype, "user": user, - "allow": for_doctype }) frappe.clear_cache() + return total @frappe.whitelist() @@ -228,37 +229,35 @@ def insert_user_perm(user, doctype, docname, is_default=0, hide_descendants=0, a user_perm.is_default = is_default user_perm.hide_descendants = hide_descendants if applicable: - user_perm.applicable_for = applicable + user_perm.applicable_for = applicable user_perm.apply_to_all_doctypes = 0 else: user_perm.apply_to_all_doctypes = 1 user_perm.insert() def remove_applicable(perm_applied_docs, user, doctype, docname): - for applicable_for in perm_applied_docs: frappe.db.delete("User Permission", { - "user": user, "applicable_for": applicable_for, + "for_value": docname, "allow": doctype, - "for_value": docname + "user": user, }) -def remove_apply_to_all(user, doctype, docname): +def remove_apply_to_all(user, doctype, docname): frappe.db.delete("User Permission", { - "user": user, "apply_to_all_doctypes": 1, + "for_value": docname, "allow": doctype, - "for_value": docname + "user": user, }) def update_applicable(already_applied, to_apply, user, doctype, docname): for applied in already_applied: if applied not in to_apply: - frappe.db.delete("User Permission", { - "user": user, "applicable_for": applied, + "for_value": docname, "allow": doctype, - "for_value": docname + "user": user, }) diff --git a/frappe/custom/doctype/customize_form/customize_form.py b/frappe/custom/doctype/customize_form/customize_form.py index 5431511d42..8de194fb00 100644 --- a/frappe/custom/doctype/customize_form/customize_form.py +++ b/frappe/custom/doctype/customize_form/customize_form.py @@ -1,5 +1,5 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors -# MIT License. See license.txt +# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors +# MIT License. See LICENSE """ Customize Form is a Single DocType used to mask the Property Setter @@ -18,13 +18,12 @@ from frappe.custom.doctype.property_setter.property_setter import delete_propert from frappe.model.docfield import supports_translation from frappe.core.doctype.doctype.doctype import validate_series + class CustomizeForm(Document): def on_update(self): - frappe.db.delete("Singles", { - "doctype": "Customize Form" - }) + frappe.db.delete("Singles", {"doctype": "Customize Form"}) frappe.db.delete("Customize Form Field") - + @frappe.whitelist() def fetch_to_customize(self): self.clear_existing_doc() diff --git a/frappe/desk/doctype/desktop_icon/desktop_icon.py b/frappe/desk/doctype/desktop_icon/desktop_icon.py index 283ffad967..28c5a670cb 100644 --- a/frappe/desk/doctype/desktop_icon/desktop_icon.py +++ b/frappe/desk/doctype/desktop_icon/desktop_icon.py @@ -197,9 +197,7 @@ def set_desktop_icons(visible_list, ignore_duplicate=True): # clear all custom only if setup is not complete if not int(frappe.defaults.get_defaults().setup_complete or 0): - frappe.db.delete("Desktop Icon", { - "standard": 0 - }) + frappe.db.delete("Desktop Icon", {"standard": 0}) # set standard as blocked and hidden if setting first active domain if not frappe.flags.keep_desktop_icons: diff --git a/frappe/desk/doctype/event/event.py b/frappe/desk/doctype/event/event.py index 2182ec4c9f..e7e7be530b 100644 --- a/frappe/desk/doctype/event/event.py +++ b/frappe/desk/doctype/event/event.py @@ -338,12 +338,8 @@ def delete_events(ref_type, ref_name, delete_event=False): total_participants = frappe.get_all("Event Participants", filters={"parenttype": "Event", "parent": participation.parent}) if len(total_participants) <= 1: - frappe.db.delete("Event", { - "name": participation.parent - }) - frappe.db.delete("Event Participants", { - "name": participation.name - }) + frappe.db.delete("Event", {"name": participation.parent}) + frappe.db.delete("Event Participants", {"name": participation.name}) # Close events if ends_on or repeat_till is less than now_datetime def set_status_of_events(): diff --git a/frappe/desk/doctype/route_history/route_history.py b/frappe/desk/doctype/route_history/route_history.py index 4303b7554d..a179119861 100644 --- a/frappe/desk/doctype/route_history/route_history.py +++ b/frappe/desk/doctype/route_history/route_history.py @@ -1,5 +1,4 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2018, Frappe Technologies and contributors +# Copyright (c) 2021, Frappe Technologies and contributors # For license information, please see license.txt import frappe @@ -8,6 +7,7 @@ from frappe.model.document import Document class RouteHistory(Document): pass + def flush_old_route_records(): """Deletes all route records except last 500 records per user""" diff --git a/frappe/desk/doctype/tag/tag.py b/frappe/desk/doctype/tag/tag.py index 626a2db085..2341d721e2 100644 --- a/frappe/desk/doctype/tag/tag.py +++ b/frappe/desk/doctype/tag/tag.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and contributors # For license information, please see license.txt @@ -124,7 +123,7 @@ def delete_tags_for_document(doc): return frappe.db.delete("Tag Link", { - "document_type": doc.doctype, + "document_type": doc.doctype, "document_name": doc.name }) diff --git a/frappe/model/__init__.py b/frappe/model/__init__.py index 6b38b383bf..79b41936c3 100644 --- a/frappe/model/__init__.py +++ b/frappe/model/__init__.py @@ -151,11 +151,12 @@ def delete_fields(args_dict, delete=0): fields = args_dict[dt] if not fields: continue - + frappe.db.delete("DocField", { "parent": dt, - "fieldname": ("in", fields) + "fieldname": ("in", fields), }) + # Delete the data/column only if delete is specified if not delete: continue @@ -163,7 +164,7 @@ def delete_fields(args_dict, delete=0): if frappe.db.get_value("DocType", dt, "issingle"): frappe.db.delete("Singles", { "doctype": dt, - "field": ("in", fields) + "field": ("in", fields), }) else: existing_fields = frappe.db.multisql({ diff --git a/frappe/model/delete_doc.py b/frappe/model/delete_doc.py index 44e675f3c5..fbbf1a4852 100644 --- a/frappe/model/delete_doc.py +++ b/frappe/model/delete_doc.py @@ -64,24 +64,14 @@ def delete_doc(doctype=None, name=None, force=0, ignore_doctypes=None, for_reloa update_flags(doc, flags, ignore_permissions) check_permission_and_not_submitted(doc) - frappe.db.delete("Custom Field", { - "dt": name - }) - frappe.db.delete("Client Script", { - "dt": name - }) - frappe.db.delete("Property Setter", { - "doc_type": name - }) - frappe.db.delete("Report", { - "ref_doctype": name - }) - frappe.db.delete("Custom DocPerm", { - "parent": name - }) - frappe.db.delete("__global_search", { - "doctype": name - }) + + frappe.db.delete("Custom Field", {"dt": name}) + frappe.db.delete("Client Script", {"dt": name}) + frappe.db.delete("Property Setter", {"doc_type": name}) + frappe.db.delete("Report", {"ref_doctype": name}) + frappe.db.delete("Custom DocPerm", {"parent": name}) + frappe.db.delete("__global_search", {"doctype": name}) + delete_from_table(doctype, name, ignore_doctypes, None) if frappe.conf.developer_mode and not doc.custom and not ( @@ -172,13 +162,9 @@ def update_naming_series(doc): def delete_from_table(doctype, name, ignore_doctypes, doc): if doctype!="DocType" and doctype==name: - frappe.db.delete("Singles", { - "doctype": name - }) + frappe.db.delete("Singles", {"doctype": name}) else: - frappe.db.delete(doctype, { - "name": name - }) + frappe.db.delete(doctype, {"name": name}) # get child tables if doc: tables = [d.options for d in doc.meta.get_table_fields()] @@ -356,7 +342,7 @@ def clear_timeline_references(link_doctype, link_name): "link_doctype": link_doctype, "link_name": link_name }) - + def insert_feed(doc): if ( frappe.flags.in_install diff --git a/frappe/patches/v11_0/apply_customization_to_custom_doctype.py b/frappe/patches/v11_0/apply_customization_to_custom_doctype.py index c01db50d5e..7e84c5ae24 100644 --- a/frappe/patches/v11_0/apply_customization_to_custom_doctype.py +++ b/frappe/patches/v11_0/apply_customization_to_custom_doctype.py @@ -28,9 +28,8 @@ def execute(): for prop in property_setters: property_setter_map[prop.field_name] = prop - frappe.db.delete("Property Setter", { - "name": prop.name - }) + frappe.db.delete("Property Setter", {"name": prop.name}) + meta = frappe.get_meta(doctype.name) for df in meta.fields: @@ -51,7 +50,6 @@ def execute(): df = frappe.new_doc('DocField', meta, 'fields') df.update(cf) meta.fields.append(df) - frappe.db.delete("Custom Field", { - "name": cf.name - }) + frappe.db.delete("Custom Field", {"name": cf.name}) + meta.save() diff --git a/frappe/patches/v11_0/sync_stripe_settings_before_migrate.py b/frappe/patches/v11_0/sync_stripe_settings_before_migrate.py index 5c949fbb1d..901ab66bfd 100644 --- a/frappe/patches/v11_0/sync_stripe_settings_before_migrate.py +++ b/frappe/patches/v11_0/sync_stripe_settings_before_migrate.py @@ -17,6 +17,4 @@ def execute(): settings.secret_key = secret_key settings.save(ignore_permissions=True) - frappe.db.delete("Singles", { - "doctype": "Stripe Settings" - }) \ No newline at end of file + frappe.db.delete("Singles", {"doctype": "Stripe Settings"}) diff --git a/frappe/patches/v12_0/delete_feedback_request_if_exists.py b/frappe/patches/v12_0/delete_feedback_request_if_exists.py index bc3c7b8f97..c1bf46b14a 100644 --- a/frappe/patches/v12_0/delete_feedback_request_if_exists.py +++ b/frappe/patches/v12_0/delete_feedback_request_if_exists.py @@ -2,6 +2,4 @@ import frappe def execute(): - frappe.db.delete("DocType", { - "name": "Feedback Request" - }) \ No newline at end of file + frappe.db.delete("DocType", {"name": "Feedback Request"}) diff --git a/frappe/patches/v12_0/set_primary_key_in_series.py b/frappe/patches/v12_0/set_primary_key_in_series.py index e8d3abdde1..b2139f63b6 100644 --- a/frappe/patches/v12_0/set_primary_key_in_series.py +++ b/frappe/patches/v12_0/set_primary_key_in_series.py @@ -11,6 +11,7 @@ def execute(): name having count(name) > 1 ''', as_dict=True) + for row in duplicate_keys: frappe.db.delete("Series", { "name": row.name @@ -18,4 +19,5 @@ def execute(): if row.current: frappe.db.sql('insert into `tabSeries`(`name`, `current`) values (%(name)s, %(current)s)', row) frappe.db.commit() + frappe.db.sql('ALTER table `tabSeries` ADD PRIMARY KEY IF NOT EXISTS (name)') diff --git a/frappe/patches/v12_0/setup_comments_from_communications.py b/frappe/patches/v12_0/setup_comments_from_communications.py index b72d02720e..11e02965f1 100644 --- a/frappe/patches/v12_0/setup_comments_from_communications.py +++ b/frappe/patches/v12_0/setup_comments_from_communications.py @@ -31,4 +31,4 @@ def execute(): # clean up frappe.db.delete("Communication", { "communication_type": "Comment" - }) \ No newline at end of file + }) diff --git a/frappe/patches/v13_0/remove_twilio_settings.py b/frappe/patches/v13_0/remove_twilio_settings.py index 8b7e5e02c2..7efaf876e2 100644 --- a/frappe/patches/v13_0/remove_twilio_settings.py +++ b/frappe/patches/v13_0/remove_twilio_settings.py @@ -19,4 +19,4 @@ def execute(): def twilio_settings_doctype_in_integrations() -> bool: """Check Twilio Settings doctype exists in integrations module or not. """ - return frappe.db.exists("DocType", {'name': 'Twilio Settings', 'module': 'Integrations'}) \ No newline at end of file + return frappe.db.exists("DocType", {'name': 'Twilio Settings', 'module': 'Integrations'}) diff --git a/frappe/permissions.py b/frappe/permissions.py index 82f27452fb..33aef4ab41 100644 --- a/frappe/permissions.py +++ b/frappe/permissions.py @@ -7,11 +7,11 @@ import frappe.share from frappe import _, msgprint from frappe.utils import cint + rights = ("select", "read", "write", "create", "delete", "submit", "cancel", "amend", "print", "email", "report", "import", "export", "set_user_permissions", "share") - def check_admin_or_system_manager(user=None): if not user: user = frappe.session.user diff --git a/frappe/sessions.py b/frappe/sessions.py index 0202931e70..4f769ea88f 100644 --- a/frappe/sessions.py +++ b/frappe/sessions.py @@ -76,6 +76,7 @@ def get_sessions_to_clear(user=None, keep_current=False, device=None): def delete_session(sid=None, user=None, reason="Session Expired"): from frappe.core.doctype.activity_log.feed import logout_feed + frappe.cache().hdel("session", sid) frappe.cache().hdel("last_db_session_update", sid) if sid and not user: diff --git a/frappe/tests/test_commands.py b/frappe/tests/test_commands.py index 54103f0151..f687f70228 100644 --- a/frappe/tests/test_commands.py +++ b/frappe/tests/test_commands.py @@ -433,6 +433,6 @@ class TestCommands(BaseTestCommands): for output in ["legacy", "plain", "table", "json"]: self.execute(f"bench version -f {output}") self.assertEqual(self.returncode, 0) - + self.execute("bench version -f invalid") self.assertEqual(self.returncode, 2) diff --git a/frappe/utils/error.py b/frappe/utils/error.py index d83cbb49ea..05b578d7e8 100644 --- a/frappe/utils/error.py +++ b/frappe/utils/error.py @@ -176,9 +176,9 @@ def collect_error_snapshots(): def clear_old_snapshots(): """Clear snapshots that are older than a month""" - + frappe.db.sql("""delete from `tabError Snapshot` - where creation < (NOW() - INTERVAL '1' MONTH)""") + where creation < (NOW() - INTERVAL '1' MONTH)""") path = get_error_snapshot_path() today = datetime.datetime.now() diff --git a/frappe/utils/testutils.py b/frappe/utils/testutils.py index 5c5bfa7976..9a2b2da791 100644 --- a/frappe/utils/testutils.py +++ b/frappe/utils/testutils.py @@ -12,7 +12,5 @@ def add_custom_field(doctype, fieldname, fieldtype='Data', options=None): }).insert() def clear_custom_fields(doctype): - frappe.db.delete("Custom Field", { - "dt": doctype - }) + frappe.db.delete("Custom Field", {"dt": doctype}) frappe.clear_cache(doctype=doctype) diff --git a/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py b/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py index 54d2c2e446..63ba96d138 100644 --- a/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py +++ b/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py @@ -330,6 +330,7 @@ def remove_unverified_record(): AND `creation` < (NOW() - INTERVAL '7' DAY)""" ) + @frappe.whitelist(allow_guest=True) def confirm_deletion(email, name, host_name): if not verify_request(): diff --git a/frappe/workflow/doctype/workflow_action/workflow_action.py b/frappe/workflow/doctype/workflow_action/workflow_action.py index e3b89ba0e5..5eedc27d9c 100644 --- a/frappe/workflow/doctype/workflow_action/workflow_action.py +++ b/frappe/workflow/doctype/workflow_action/workflow_action.py @@ -139,7 +139,7 @@ def clear_old_workflow_actions(doc, user=None): "user": ("!=", user), "status": "Open" }) - + def update_completed_workflow_actions(doc, user=None): user = user if user else frappe.session.user frappe.db.sql("""UPDATE `tabWorkflow Action` SET `status`='Completed', `completed_by`=%s From f7b551fe33581b05ef061d03345dbbc6990a9e5b Mon Sep 17 00:00:00 2001 From: shariquerik Date: Wed, 28 Jul 2021 18:56:34 +0530 Subject: [PATCH 47/86] fix: updated router.js and undo inbox_view changes --- frappe/public/js/frappe/router.js | 20 +++++++++++++++---- .../js/frappe/views/inbox/inbox_view.js | 2 +- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/frappe/public/js/frappe/router.js b/frappe/public/js/frappe/router.js index 3f8ce7fee2..5920846819 100644 --- a/frappe/public/js/frappe/router.js +++ b/frappe/public/js/frappe/router.js @@ -170,8 +170,13 @@ frappe.router = { } else { standard_route = ['List', doctype_route.doctype, frappe.utils.to_title_case(route[2])]; if (route[3]) { - // calendar / kanban / dashboard / folder name - standard_route.push([...route].splice(3, route.length)); + if (Array.isArray(route)) { + // folder + standard_route.push(...route.splice(3, route.length)); + } else { + // calendar / kanban / dashboard name + standard_route.push(route[3]); + } } } return standard_route; @@ -302,8 +307,15 @@ frappe.router = { if (route[2] && route[2] !== 'list' && !$.isPlainObject(route[2])) { new_route = [this.slug(route[1]), 'view', route[2].toLowerCase()]; - // calendar / inbox / file folder - if (route[3]) new_route.push([...route].slice(3, route.length)); + if (route[3]) { + if (Array.isArray(route)) { + // file folder + new_route.push(...route.splice(3, route.length)); + } else { + // calendar / inbox + new_route.push(route[3]); + } + } } else { if ($.isPlainObject(route[2])) { frappe.route_options = route[2]; diff --git a/frappe/public/js/frappe/views/inbox/inbox_view.js b/frappe/public/js/frappe/views/inbox/inbox_view.js index e928327a43..8b53bd49a9 100644 --- a/frappe/public/js/frappe/views/inbox/inbox_view.js +++ b/frappe/public/js/frappe/views/inbox/inbox_view.js @@ -16,7 +16,7 @@ frappe.views.InboxView = class InboxView extends frappe.views.ListView { } frappe.set_route("List", "Communication", "Inbox", email_account); return true; - } else if (!route[3] || (route[3][0] !== "All Accounts" && !is_valid(route[3][0]))) { + } else if (!route[3] || (route[3] !== "All Accounts" && !is_valid(route[3]))) { frappe.throw(__('No email account associated with the User. Please add an account under User > Email Inbox.')); } return false; From a836b2de0c041fe17ca2530a5bf856719ad4f51c Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 28 Jul 2021 18:57:51 +0530 Subject: [PATCH 48/86] perf: Use truncate instead of drop Changed DML DROP statements to use DDL TRUNCATE for better performance wherever possible --- frappe/core/doctype/error_log/error_log.py | 2 +- frappe/utils/install.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/frappe/core/doctype/error_log/error_log.py b/frappe/core/doctype/error_log/error_log.py index 081b4e879d..3d66253b08 100644 --- a/frappe/core/doctype/error_log/error_log.py +++ b/frappe/core/doctype/error_log/error_log.py @@ -20,4 +20,4 @@ def set_old_logs_as_seen(): def clear_error_logs(): '''Flush all Error Logs''' frappe.only_for('System Manager') - frappe.db.delete("Error Log") \ No newline at end of file + frappe.db.truncate("Error Log") diff --git a/frappe/utils/install.py b/frappe/utils/install.py index e6d4386ebe..3d6a2fed97 100644 --- a/frappe/utils/install.py +++ b/frappe/utils/install.py @@ -111,9 +111,9 @@ def before_tests(): # don't run before tests if any other app is installed return - frappe.db.delete("Custom Field") - frappe.db.delete("Event") - frappe.db.commit() + frappe.db.truncate("Custom Field") + frappe.db.truncate("Event") + frappe.clear_cache() # complete setup if missing From 7dac03cea01f534ccaf3df01d34ffc71ecca4d05 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 28 Jul 2021 18:59:18 +0530 Subject: [PATCH 49/86] fix: Re-introduce code erreneously taken out in previous commits Due to bulk updations, some statements were missed out/got deleted. This commit re-introduces them. --- frappe/core/doctype/report/test_report.py | 2 +- .../doctype/user_permission/test_user_permission.py | 12 +++++++----- frappe/desk/doctype/route_history/route_history.py | 11 +++++------ 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/frappe/core/doctype/report/test_report.py b/frappe/core/doctype/report/test_report.py index 1bf9893bd7..9c953db1f0 100644 --- a/frappe/core/doctype/report/test_report.py +++ b/frappe/core/doctype/report/test_report.py @@ -81,12 +81,12 @@ class TestReport(unittest.TestCase): self.assertDictEqual({'name': 'Administrator', 'user_type': 'System User', 'email': 'admin@example.com'}, admin_dict) def test_report_permissions(self): - frappe.set_user('test@example.com') frappe.db.delete("Has Role", { "parent": frappe.session.user, "role": "Test Has Role" }) + frappe.db.commit() if not frappe.db.exists('Role', 'Test Has Role'): role = frappe.get_doc({ 'doctype': 'Role', diff --git a/frappe/core/doctype/user_permission/test_user_permission.py b/frappe/core/doctype/user_permission/test_user_permission.py index 1e48de334b..85db846982 100644 --- a/frappe/core/doctype/user_permission/test_user_permission.py +++ b/frappe/core/doctype/user_permission/test_user_permission.py @@ -9,13 +9,15 @@ import unittest class TestUserPermission(unittest.TestCase): def setUp(self): - + test_users = ( + "test_bulk_creation_update@example.com", + "test_user_perm1@example.com", + "nested_doc_user@example.com", + ) frappe.db.delete("User Permission", { - "user": ("in", ("test_bulk_creation_update@example.com", - "test_user_perm1@example.com", - "nested_doc_user@example.com")) + "user": ("in", test_users) }) - + frappe.delete_doc_if_exists("DocType", "Person") frappe.db.sql_ddl("DROP TABLE IF EXISTS `tabPerson`") frappe.delete_doc_if_exists("DocType", "Doc A") frappe.db.sql_ddl("DROP TABLE IF EXISTS `tabDoc A`") diff --git a/frappe/desk/doctype/route_history/route_history.py b/frappe/desk/doctype/route_history/route_history.py index a179119861..95872440c7 100644 --- a/frappe/desk/doctype/route_history/route_history.py +++ b/frappe/desk/doctype/route_history/route_history.py @@ -24,15 +24,14 @@ def flush_old_route_records(): for user in users: user = user[0] last_record_to_keep = frappe.db.get_all('Route History', - filters={ - 'user': user, - }, + filters={'user': user}, limit=1, limit_start=500, fields=['modified'], - order_by='modified desc') + order_by='modified desc' + ) frappe.db.delete("Route History", { - "modified": last_record_to_keep[0].modified, + "modified": ("<=", last_record_to_keep[0].modified), "user": user - }) \ No newline at end of file + }) From 3ca7fa77967a49b1d391cecd01e33a45d4e42012 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 27 Jul 2021 01:12:19 +0530 Subject: [PATCH 50/86] refactor(minor): Make frappe.db.delete DRY-er --- frappe/database/database.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index 8b4fd3ad9e..ee2d062a81 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -958,20 +958,17 @@ class Database(object): Doctype name can be passed directly, it will be pre-pended with `tab`. """ - if kwargs: - filters = filters or kwargs.get("conditions") + values = () + filters = filters or kwargs.get("conditions") + table = doctype if doctype.startswith("__") else f"tab{doctype}" + query = f"DELETE FROM `{table}`" if "debug" not in kwargs: kwargs["debug"] = debug - if not filters: - table = doctype if doctype.startswith("__") else f"tab{doctype}" - query = f"DELETE FROM `{table}`" - return self.sql(query, **kwargs) - - table = doctype if doctype.startswith("__") else f"tab{doctype}" - conditions, values = self.build_conditions(filters) - query = f"DELETE FROM `{table}` WHERE {conditions}" + if filters: + conditions, values = self.build_conditions(filters) + query = f"{query} WHERE {conditions}" return self.sql(query, values, **kwargs) From d63affc73251cd565bdc5e232eafd4a77f20a6a0 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 27 Jul 2021 02:12:19 +0530 Subject: [PATCH 51/86] refactor(minor): Use ORM instead of raw delete queries Modified query building for "IN" statements, as well as older condition builders to use frappe.db.delete --- frappe/defaults.py | 23 +++++++------------ frappe/desk/doctype/todo/todo.py | 13 +++++++---- frappe/email/queue.py | 10 ++------ .../v12_0/set_primary_key_in_series.py | 3 ++- 4 files changed, 20 insertions(+), 29 deletions(-) diff --git a/frappe/defaults.py b/frappe/defaults.py index 794d30a0c8..d4c338388d 100644 --- a/frappe/defaults.py +++ b/frappe/defaults.py @@ -154,29 +154,23 @@ def clear_default(key=None, value=None, parent=None, name=None, parenttype=None) :param name: Default ID. :param parenttype: Clear defaults table for a particular type e.g. **User**. """ - conditions = [] - values = [] + filters = {} if name: - conditions.append("name=%s") - values.append(name) + filters.update({"name": name}) else: if key: - conditions.append("defkey=%s") - values.append(key) + filters.update({"defkey": key}) if value: - conditions.append("defvalue=%s") - values.append(value) + filters.update({"defvalue": value}) if parent: - conditions.append("parent=%s") - values.append(parent) + filters.update({"parent": parent}) if parenttype: - conditions.append("parenttype=%s") - values.append(parenttype) + filters.update({"parenttype": parenttype}) if parent: clear_defaults_cache(parent) @@ -184,11 +178,10 @@ def clear_default(key=None, value=None, parent=None, name=None, parenttype=None) clear_defaults_cache("__default") clear_defaults_cache("__global") - if not conditions: + if not filters: raise Exception("[clear_default] No key specified.") - frappe.db.sql("""delete from tabDefaultValue where {0}""".format(" and ".join(conditions)), - tuple(values)) + frappe.db.delete("DefaultValue", filters) _clear_cache(parent) diff --git a/frappe/desk/doctype/todo/todo.py b/frappe/desk/doctype/todo/todo.py index 34fb3bb905..01eed41b4f 100644 --- a/frappe/desk/doctype/todo/todo.py +++ b/frappe/desk/doctype/todo/todo.py @@ -39,11 +39,7 @@ class ToDo(Document): self.update_in_reference() def on_trash(self): - # unlink todo from linked comments - frappe.db.delete("Communication Link", { - "link_doctype": self.doctype, - "link_name": self.name - }) + self.delete_communication_links() self.update_in_reference() def add_assign_comment(self, text, comment_type): @@ -52,6 +48,13 @@ class ToDo(Document): frappe.get_doc(self.reference_type, self.reference_name).add_comment(comment_type, text) + def delete_communication_links(self): + # unlink todo from linked comments + return frappe.db.delete("Communication Link", { + "link_doctype": self.doctype, + "link_name": self.name + }) + def update_in_reference(self): if not (self.reference_type and self.reference_name): return diff --git a/frappe/email/queue.py b/frappe/email/queue.py index 40f1c7be3a..ef59302bab 100755 --- a/frappe/email/queue.py +++ b/frappe/email/queue.py @@ -173,14 +173,8 @@ def clear_outbox(days=None): WHERE `priority`=0 AND `modified` < (NOW() - INTERVAL '{0}' DAY)""".format(days)) if email_queues: - # TODO: email_queues IN frappe.db.sql - frappe.db.sql("""DELETE FROM `tabEmail Queue` WHERE `name` IN ({0})""".format( - ','.join(['%s']*len(email_queues) - )), tuple(email_queues)) - - frappe.db.sql("""DELETE FROM `tabEmail Queue Recipient` WHERE `parent` IN ({0})""".format( - ','.join(['%s']*len(email_queues) - )), tuple(email_queues)) + frappe.db.delete("Email Queue", {"name": ("in", email_queues)}) + frappe.db.delete("Email Queue Recipient", {"parent": ("in", email_queues)}) def set_expiry_for_email_queue(): ''' Mark emails as expire that has not sent for 7 days. diff --git a/frappe/patches/v12_0/set_primary_key_in_series.py b/frappe/patches/v12_0/set_primary_key_in_series.py index b2139f63b6..83a903fc2d 100644 --- a/frappe/patches/v12_0/set_primary_key_in_series.py +++ b/frappe/patches/v12_0/set_primary_key_in_series.py @@ -2,7 +2,8 @@ import frappe def execute(): #if current = 0, simply delete the key as it'll be recreated on first entry - frappe.db.sql('delete from `tabSeries` where current = 0') + frappe.db.delete("Series", {"current": 0}) + duplicate_keys = frappe.db.sql(''' SELECT name, max(current) as current from From dfccae524618cd56b91e6da9f1091ea933ef6816 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 29 Jul 2021 02:12:19 +0530 Subject: [PATCH 52/86] perf(minor): Permission Manager remove API Delete using the filters directly instead of selecting rows in one query and deleting those rows in another. DBMS would have to scan the table twice prior --- frappe/core/page/permission_manager/permission_manager.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/frappe/core/page/permission_manager/permission_manager.py b/frappe/core/page/permission_manager/permission_manager.py index af8973811a..2a99283dda 100644 --- a/frappe/core/page/permission_manager/permission_manager.py +++ b/frappe/core/page/permission_manager/permission_manager.py @@ -110,11 +110,9 @@ def remove(doctype, role, permlevel): frappe.only_for("System Manager") setup_custom_perms(doctype) - name = frappe.get_value('Custom DocPerm', dict(parent=doctype, role=role, permlevel=permlevel)) - frappe.db.delete("Custom DocPerm", { - "name": name - }) - if not frappe.get_all('Custom DocPerm', dict(parent=doctype)): + frappe.db.delete("Custom DocPerm", {"parent": doctype, "role": role, "permlevel": permlevel}) + + if not frappe.get_all('Custom DocPerm', {"parent": doctype}): frappe.throw(_('There must be atleast one permission rule.'), title=_('Cannot Remove')) validate_permissions_for_doctype(doctype, for_remove=True, alert=True) From b40721de4b488025d484fa8b9abe5217778dd60e Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 29 Jul 2021 12:53:43 +0530 Subject: [PATCH 53/86] style: Remove extra whitespace --- frappe/database/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index ee2d062a81..2070ba676b 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -968,7 +968,7 @@ class Database(object): if filters: conditions, values = self.build_conditions(filters) - query = f"{query} WHERE {conditions}" + query = f"{query} WHERE {conditions}" return self.sql(query, values, **kwargs) From 83d442d39df2b4097cfb58aad2bf7fd4ae5e9a3f Mon Sep 17 00:00:00 2001 From: shariquerik Date: Thu, 29 Jul 2021 16:01:25 +0530 Subject: [PATCH 54/86] fix: using slice instead of splice --- frappe/public/js/frappe/router.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/public/js/frappe/router.js b/frappe/public/js/frappe/router.js index 5920846819..84af5ca4cd 100644 --- a/frappe/public/js/frappe/router.js +++ b/frappe/public/js/frappe/router.js @@ -172,7 +172,7 @@ frappe.router = { if (route[3]) { if (Array.isArray(route)) { // folder - standard_route.push(...route.splice(3, route.length)); + standard_route.push(...route.slice(3, route.length)); } else { // calendar / kanban / dashboard name standard_route.push(route[3]); @@ -310,7 +310,7 @@ frappe.router = { if (route[3]) { if (Array.isArray(route)) { // file folder - new_route.push(...route.splice(3, route.length)); + new_route.push(...route.slice(3, route.length)); } else { // calendar / inbox new_route.push(route[3]); From 5bb01511aa3e30aa7994ca2dfac0cf0d266d13e6 Mon Sep 17 00:00:00 2001 From: shariquerik Date: Thu, 29 Jul 2021 17:50:43 +0530 Subject: [PATCH 55/86] fix: new-doc-1 not found error --- frappe/public/js/frappe/form/dashboard.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/form/dashboard.js b/frappe/public/js/frappe/form/dashboard.js index b2b0c11d54..420704149b 100644 --- a/frappe/public/js/frappe/form/dashboard.js +++ b/frappe/public/js/frappe/form/dashboard.js @@ -204,7 +204,7 @@ frappe.ui.form.Dashboard = class FormDashboard { $(this).removeClass('hidden'); } }); - this.set_open_count(); + !this.frm.is_new() && this.set_open_count(); } init_data() { From ff973477a3e429ac82d8e5cb95ea0377b48b80be Mon Sep 17 00:00:00 2001 From: leela Date: Thu, 29 Jul 2021 20:32:03 +0530 Subject: [PATCH 56/86] fix: Use bench path as default bench_id --- frappe/tests/test_background_jobs.py | 6 +++--- frappe/utils/__init__.py | 2 +- frappe/utils/background_jobs.py | 20 ++++++++++---------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/frappe/tests/test_background_jobs.py b/frappe/tests/test_background_jobs.py index 48e0dd2ee9..188f3e166f 100644 --- a/frappe/tests/test_background_jobs.py +++ b/frappe/tests/test_background_jobs.py @@ -4,7 +4,7 @@ from rq import Queue import frappe from frappe.core.page.background_jobs.background_jobs import remove_failed_jobs -from frappe.utils.background_jobs import get_redis_conn, rename_queue +from frappe.utils.background_jobs import get_redis_conn, generate_qname import time @@ -17,14 +17,14 @@ class TestBackgroundJobs(unittest.TestCase): queues = Queue.all(conn) for queue in queues: - if queue.name == rename_queue("short"): + if queue.name == generate_qname("short"): fail_registry = queue.failed_job_registry self.assertGreater(fail_registry.count, 0) remove_failed_jobs() for queue in queues: - if queue.name == rename_queue("short"): + if queue.name == generate_qname("short"): fail_registry = queue.failed_job_registry self.assertEqual(fail_registry.count, 0) diff --git a/frappe/utils/__init__.py b/frappe/utils/__init__.py index 80c6cda98c..68366eb234 100644 --- a/frappe/utils/__init__.py +++ b/frappe/utils/__init__.py @@ -384,7 +384,7 @@ def get_bench_path(): return os.path.realpath(os.path.join(os.path.dirname(frappe.__file__), '..', '..', '..')) def get_bench_id(): - return frappe.get_conf().get('bench_id', 'DefaultBench') + return frappe.get_conf().get('bench_id', get_bench_path().strip('/').replace('/', '-')) def get_site_id(site=None): return f"{site or frappe.local.site}@{get_bench_id()}" diff --git a/frappe/utils/background_jobs.py b/frappe/utils/background_jobs.py index 4241c95c5d..f0bd06aff4 100755 --- a/frappe/utils/background_jobs.py +++ b/frappe/utils/background_jobs.py @@ -145,7 +145,7 @@ def start_worker(queue=None, quiet = False, rq_username=None, rq_password=None): # empty init is required to get redis_queue from common_site_config.json redis_connection = get_redis_conn(username=rq_username, password=rq_password) queues = get_queue_list(queue, build_queue_name=True) - queue_name = queue and rename_queue(queue) + queue_name = queue and generate_qname(queue) if os.environ.get('CI'): setup_loghandlers('ERROR') @@ -206,7 +206,7 @@ def get_queue_list(queue_list=None, build_queue_name=False): validate_queue(queue, default_queue_list) else: queue_list = default_queue_list - return [rename_queue(q) for q in queue_list] if build_queue_name else queue_list + return [generate_qname(qtype) for qtype in queue_list] if build_queue_name else queue_list def get_workers(queue): '''Returns a list of Worker objects tied to a queue object''' @@ -222,10 +222,10 @@ def get_running_jobs_in_queue(queue): jobs.append(current_job) return jobs -def get_queue(queue, is_async=True): +def get_queue(qtype, is_async=True): '''Returns a Queue object tied to a redis connection''' - validate_queue(queue) - return Queue(rename_queue(queue), connection=get_redis_conn(), is_async=is_async) + validate_queue(qtype) + return Queue(generate_qname(qtype), connection=get_redis_conn(), is_async=is_async) def validate_queue(queue, default_queue_list=None): if not default_queue_list: @@ -274,17 +274,17 @@ def get_queues() -> List[Queue]: queues = Queue.all(connection=get_redis_conn()) return [q for q in queues if is_queue_accessible(q)] -def rename_queue(qname: str) -> str: - """Rename qname by adding bench name as prefix. +def generate_qname(qtype: str) -> str: + """Generate qname by combining bench ID and queue type. - Renamed queues are useful to define namespaces of customers. + qnames are useful to define namespaces of customers. """ - return f"{get_bench_id()}:{qname}" + return f"{get_bench_id()}:{qtype}" def is_queue_accessible(qobj: Queue) -> bool: """Checks whether queue is relate to current bench or not. """ - accessible_queues = [rename_queue(q) for q in list(queue_timeout)] + accessible_queues = [generate_qname(q) for q in list(queue_timeout)] return qobj.name in accessible_queues def enqueue_test_job(): From bf82d909afb1b80daa09e093450e12a507b1fc99 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Thu, 29 Jul 2021 23:14:25 +0530 Subject: [PATCH 57/86] fix: multiple recorder ui fixes - fix broken list view - syntax error in html showing template strings in output. - UX of toggling rows. Click anywhere on same row to toggle instead of a button. --- frappe/public/js/frappe/recorder/RequestDetail.vue | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/frappe/public/js/frappe/recorder/RequestDetail.vue b/frappe/public/js/frappe/recorder/RequestDetail.vue index 2e995bca39..471306cba5 100644 --- a/frappe/public/js/frappe/recorder/RequestDetail.vue +++ b/frappe/public/js/frappe/recorder/RequestDetail.vue @@ -64,7 +64,7 @@
-
+
{{ call.index }}
{{ call.query }}
@@ -76,16 +76,13 @@
{{ call.exact_copies }}
- +
{{ __("SQL Query") }} #{{ call.index }} -
- -
@@ -116,7 +113,7 @@
-
+