diff --git a/frappe/__init__.py b/frappe/__init__.py index 1c978945c7..b4728f9ac3 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -28,6 +28,8 @@ from .exceptions import * from .utils.jinja import (get_jenv, get_template, render_template, get_email_from_template, get_jloader) from .utils.lazy_loader import lazy_import +from frappe.query_builder import get_query_builder + # Lazy imports faker = lazy_import('faker') @@ -118,6 +120,7 @@ def set_user_lang(user, user_language=None): # local-globals db = local("db") +qb = local("qb") conf = local("conf") form = form_dict = local("form_dict") request = local("request") @@ -202,6 +205,7 @@ def init(site, sites_path=None, new_site=False): local.form_dict = _dict() local.session = _dict() local.dev_server = _dev_server + local.qb = get_query_builder(local.conf.db_type or "mariadb") setup_module_map() diff --git a/frappe/api.py b/frappe/api.py index 36d51e894c..636c6b2888 100644 --- a/frappe/api.py +++ b/frappe/api.py @@ -82,7 +82,7 @@ def handle(): if frappe.local.request.method=="PUT": data = get_request_form_data() - doc = frappe.get_doc(doctype, name) + doc = frappe.get_doc(doctype, name, for_update=True) if "flags" in data: del data["flags"] diff --git a/frappe/cache_manager.py b/frappe/cache_manager.py index 9f09f26be8..c17ae583ed 100644 --- a/frappe/cache_manager.py +++ b/frappe/cache_manager.py @@ -141,17 +141,13 @@ def build_table_count_cache(): return _cache = frappe.cache() - data = frappe.db.multisql({ - "mariadb": """ - SELECT table_name AS name, - table_rows AS count - FROM information_schema.tables""", - "postgres": """ - SELECT "relname" AS name, - "n_tup_ins" AS count - FROM "pg_stat_all_tables" - """ - }, as_dict=1) + table_name = frappe.qb.Field("table_name").as_("name") + table_rows = frappe.qb.Field("table_rows").as_("count") + information_schema = frappe.qb.Schema("information_schema") + + query = frappe.qb.from_(information_schema.tables).select(table_name, table_rows) + + data = frappe.db.sql(query, as_dict=1) counts = {d.get('name').lstrip('tab'): d.get('count', None) for d in data} _cache.set_value("information_schema:counts", counts) diff --git a/frappe/commands/__init__.py b/frappe/commands/__init__.py index be9d107025..9ed333d034 100644 --- a/frappe/commands/__init__.py +++ b/frappe/commands/__init__.py @@ -102,7 +102,9 @@ def get_commands(): from .site import commands as site_commands from .translate import commands as translate_commands from .utils import commands as utils_commands + from .redis import commands as redis_commands - return list(set(scheduler_commands + site_commands + translate_commands + utils_commands)) + all_commands = scheduler_commands + site_commands + translate_commands + utils_commands + redis_commands + return list(set(all_commands)) commands = get_commands() diff --git a/frappe/commands/redis.py b/frappe/commands/redis.py new file mode 100644 index 0000000000..38a46c2142 --- /dev/null +++ b/frappe/commands/redis.py @@ -0,0 +1,53 @@ +import os + +import click + +import frappe +from frappe.utils.rq import RedisQueue +from frappe.installer import update_site_config + +@click.command('create-rq-users') +@click.option('--set-admin-password', is_flag=True, default=False, help='Set new Redis admin(default user) password') +@click.option('--use-rq-auth', is_flag=True, default=False, help='Enable Redis authentication for sites') +def create_rq_users(set_admin_password=False, use_rq_auth=False): + """Create Redis Queue users and add to acl and app configs. + + acl config file will be used by redis server while starting the server + and app config is used by app while connecting to redis server. + """ + acl_file_path = os.path.abspath('../config/redis_queue.acl') + + with frappe.init_site(): + acl_list, user_credentials = RedisQueue.gen_acl_list( + set_admin_password=set_admin_password) + + with open(acl_file_path, 'w') as f: + f.writelines([acl+'\n' for acl in acl_list]) + + sites_path = os.getcwd() + common_site_config_path = os.path.join(sites_path, 'common_site_config.json') + update_site_config("rq_username", user_credentials['bench'][0], validate=False, + site_config_path=common_site_config_path) + update_site_config("rq_password", user_credentials['bench'][1], validate=False, + site_config_path=common_site_config_path) + update_site_config("use_rq_auth", use_rq_auth, validate=False, + site_config_path=common_site_config_path) + + click.secho('* ACL and site configs are updated with new user credentials. ' + 'Please restart Redis Queue server to enable namespaces.', + fg='green') + + if set_admin_password: + env_key = 'RQ_ADMIN_PASWORD' + click.secho('* Redis admin password is successfully set up. ' + 'Include below line in .bashrc file for system to use', + fg='green') + click.secho(f"`export {env_key}={user_credentials['default'][1]}`") + click.secho('NOTE: Please save the admin password as you ' + 'can not access redis server without the password', + fg='yellow') + + +commands = [ + create_rq_users +] diff --git a/frappe/commands/scheduler.py b/frappe/commands/scheduler.py index d69ebb3024..f82473fd55 100755 --- a/frappe/commands/scheduler.py +++ b/frappe/commands/scheduler.py @@ -172,9 +172,13 @@ def start_scheduler(): @click.command('worker') @click.option('--queue', type=str) @click.option('--quiet', is_flag = True, default = False, help = 'Hide Log Outputs') -def start_worker(queue, quiet = False): +@click.option('-u', '--rq-username', default=None, help='Redis ACL user') +@click.option('-p', '--rq-password', default=None, help='Redis ACL user password') +def start_worker(queue, quiet = False, rq_username=None, rq_password=None): + """Site is used to find redis credentals. + """ from frappe.utils.background_jobs import start_worker - start_worker(queue, quiet = quiet) + start_worker(queue, quiet = quiet, rq_username=rq_username, rq_password=rq_password) @click.command('ready-for-migration') @click.option('--site', help='site name') diff --git a/frappe/config/__init__.py b/frappe/config/__init__.py index 62a877be24..e7f0f1a763 100644 --- a/frappe/config/__init__.py +++ b/frappe/config/__init__.py @@ -39,18 +39,13 @@ def get_modules_from_app(app): ) def get_all_empty_tables_by_module(): - empty_tables = set(r[0] for r in frappe.db.multisql({ - "mariadb": """ - SELECT table_name - FROM information_schema.tables - WHERE table_rows = 0 and table_schema = "{}" - """.format(frappe.conf.db_name), - "postgres": """ - SELECT "relname" as "table_name" - FROM "pg_stat_all_tables" - WHERE n_tup_ins = 0 - """ - })) + table_rows = frappe.qb.Field("table_rows") + table_name = frappe.qb.Field("table_name") + information_schema = frappe.qb.Schema("information_schema") + + query = frappe.qb.from_(information_schema.tables).select(table_name).where(table_rows == 0) + + empty_tables = {r[0] for r in frappe.db.sql(query)} results = frappe.get_all("DocType", fields=["name", "module"]) empty_tables_by_module = {} diff --git a/frappe/core/doctype/activity_log/feed.py b/frappe/core/doctype/activity_log/feed.py index caa3cae613..19d7b77184 100644 --- a/frappe/core/doctype/activity_log/feed.py +++ b/frappe/core/doctype/activity_log/feed.py @@ -29,10 +29,12 @@ def update_feed(doc, method=None): name = feed.name or doc.name # delete earlier feed - frappe.db.sql("""delete from `tabActivity Log` - where - reference_doctype=%s and reference_name=%s - and link_doctype=%s""", (doctype, name,feed.link_doctype)) + frappe.db.delete("Activity Log", { + "reference_doctype": doctype, + "reference_name": name, + "link_doctype": feed.link_doctype + }) + frappe.get_doc({ "doctype": "Activity Log", "reference_doctype": doctype, diff --git a/frappe/core/doctype/communication/mixins.py b/frappe/core/doctype/communication/mixins.py index 09a8a0ac22..52cd370890 100644 --- a/frappe/core/doctype/communication/mixins.py +++ b/frappe/core/doctype/communication/mixins.py @@ -3,6 +3,7 @@ from frappe import _ from frappe.core.utils import get_parent_doc from frappe.utils import parse_addr, get_formatted_email, get_url from frappe.email.doctype.email_account.email_account import EmailAccount +from frappe.desk.doctype.todo.todo import ToDo class CommunicationEmailMixin: """Mixin class to handle communication mails. @@ -76,6 +77,7 @@ class CommunicationEmailMixin: if is_inbound_mail_communcation: cc.append(self.get_owner()) cc = set(cc) - {self.sender_mailid} + cc.update(self.get_assignees()) cc = set(cc) - set(self.filter_thread_notification_disbled_users(cc)) cc = cc - set(self.mail_recipients(is_inbound_mail_communcation=is_inbound_mail_communcation)) @@ -201,6 +203,13 @@ class CommunicationEmailMixin: self.mail_cc(is_inbound_mail_communcation = is_inbound_mail_communcation, include_sender=include_sender) return set(all_ids) - set(final_ids) + def get_assignees(self): + """Get owners of the reference document. + """ + filters = {'status': 'Open', 'reference_name': self.reference_name, + 'reference_type': self.reference_doctype} + return ToDo.get_owners(filters) + @staticmethod def filter_thread_notification_disbled_users(emails): """Filter users based on notifications for email threads setting is disabled. diff --git a/frappe/core/doctype/doctype/doctype.json b/frappe/core/doctype/doctype/doctype.json index 7f93d3130a..6a427f71e1 100644 --- a/frappe/core/doctype/doctype/doctype.json +++ b/frappe/core/doctype/doctype/doctype.json @@ -76,6 +76,7 @@ "index_web_pages_for_search", "route", "is_published_field", + "website_search_field", "advanced", "engine" ], @@ -547,6 +548,12 @@ { "fieldname": "column_break_51", "fieldtype": "Column Break" + }, + { + "depends_on": "has_web_view", + "fieldname": "website_search_field", + "fieldtype": "Data", + "label": "Website Search Field" } ], "icon": "fa fa-bolt", @@ -628,7 +635,7 @@ "link_fieldname": "reference_doctype" } ], - "modified": "2021-04-16 12:26:41.031135", + "modified": "2021-06-17 23:31:44.974199", "modified_by": "Administrator", "module": "Core", "name": "DocType", @@ -662,4 +669,4 @@ "sort_field": "modified", "sort_order": "DESC", "track_changes": 1 -} +} \ No newline at end of file diff --git a/frappe/core/doctype/doctype/doctype.py b/frappe/core/doctype/doctype/doctype.py index 3cdc45ea08..d2f62d0a15 100644 --- a/frappe/core/doctype/doctype/doctype.py +++ b/frappe/core/doctype/doctype/doctype.py @@ -396,10 +396,7 @@ class DocType(Document): frappe.db.sql("""update tabSingles set value=%s where doctype=%s and field='name' and value = %s""", (new, new, old)) else: - frappe.db.multisql({ - "mariadb": f"RENAME TABLE `tab{old}` TO `tab{new}`", - "postgres": f"ALTER TABLE `tab{old}` RENAME TO `tab{new}`" - }) + frappe.db.rename_table(old, new) frappe.db.commit() # Do not rename and move files and folders for custom doctype @@ -927,6 +924,13 @@ def validate_fields(meta): if meta.is_published_field not in fieldname_list: frappe.throw(_("Is Published Field must be a valid fieldname"), InvalidFieldNameError) + def check_website_search_field(meta): + if not meta.website_search_field: + return + + if meta.website_search_field not in fieldname_list: + frappe.throw(_("Website Search Field must be a valid fieldname"), InvalidFieldNameError) + def check_timeline_field(meta): if not meta.timeline_field: return @@ -1046,6 +1050,7 @@ def validate_fields(meta): check_title_field(meta) check_timeline_field(meta) check_is_published_field(meta) + check_website_search_field(meta) check_sort_field(meta) check_image_field(meta) diff --git a/frappe/core/doctype/doctype/test_doctype.py b/frappe/core/doctype/doctype/test_doctype.py index 4b6d0e4794..1e1a01a685 100644 --- a/frappe/core/doctype/doctype/test_doctype.py +++ b/frappe/core/doctype/doctype/test_doctype.py @@ -348,7 +348,6 @@ class TestDocType(unittest.TestCase): dump_docs = json.dumps(docs.get('docs')) cancel_all_linked_docs(dump_docs) data_link_doc.cancel() - data_doc.name = '{}-CAN-0'.format(data_doc.name) data_doc.load_from_db() self.assertEqual(data_link_doc.docstatus, 2) self.assertEqual(data_doc.docstatus, 2) @@ -372,7 +371,7 @@ class TestDocType(unittest.TestCase): for data in link_doc.get('permissions'): data.submit = 1 data.cancel = 1 - link_doc.insert(ignore_if_duplicate=True) + link_doc.insert() #create first parent doctype test_doc_1 = new_doctype('Test Doctype 1') @@ -387,7 +386,7 @@ class TestDocType(unittest.TestCase): for data in test_doc_1.get('permissions'): data.submit = 1 data.cancel = 1 - test_doc_1.insert(ignore_if_duplicate=True) + test_doc_1.insert() #crete second parent doctype doc = new_doctype('Test Doctype 2') @@ -402,7 +401,7 @@ class TestDocType(unittest.TestCase): for data in link_doc.get('permissions'): data.submit = 1 data.cancel = 1 - doc.insert(ignore_if_duplicate=True) + doc.insert() # create doctype data data_link_doc_1 = frappe.new_doc('Test Linked Doctype 1') @@ -433,7 +432,6 @@ class TestDocType(unittest.TestCase): # checking that doc for Test Doctype 2 is not canceled self.assertRaises(frappe.LinkExistsError, data_link_doc_1.cancel) - data_doc_2.name = '{}-CAN-0'.format(data_doc_2.name) data_doc.load_from_db() data_doc_2.load_from_db() self.assertEqual(data_link_doc_1.docstatus, 2) diff --git a/frappe/core/doctype/domain_settings/domain_settings.py b/frappe/core/doctype/domain_settings/domain_settings.py index 7ad0aeff21..a8c7c6a747 100644 --- a/frappe/core/doctype/domain_settings/domain_settings.py +++ b/frappe/core/doctype/domain_settings/domain_settings.py @@ -34,7 +34,7 @@ class DomainSettings(Document): all_domains = list((frappe.get_hooks('domains') or {})) def remove_role(role): - frappe.db.sql('delete from `tabHas Role` where role=%s', role) + frappe.db.delete("Has Role", {"role": role}) frappe.set_value('Role', role, 'disabled', 1) for domain in all_domains: diff --git a/frappe/core/doctype/error_log/error_log.py b/frappe/core/doctype/error_log/error_log.py index 8223238c57..3d66253b08 100644 --- a/frappe/core/doctype/error_log/error_log.py +++ b/frappe/core/doctype/error_log/error_log.py @@ -20,4 +20,4 @@ def set_old_logs_as_seen(): def clear_error_logs(): '''Flush all Error Logs''' frappe.only_for('System Manager') - frappe.db.sql('''DELETE FROM `tabError Log`''') \ No newline at end of file + frappe.db.truncate("Error Log") diff --git a/frappe/core/doctype/report/test_report.py b/frappe/core/doctype/report/test_report.py index 9d0c0b9af0..9c953db1f0 100644 --- a/frappe/core/doctype/report/test_report.py +++ b/frappe/core/doctype/report/test_report.py @@ -82,9 +82,11 @@ class TestReport(unittest.TestCase): def test_report_permissions(self): frappe.set_user('test@example.com') - frappe.db.sql("""delete from `tabHas Role` where parent = %s - and role = 'Test Has Role'""", frappe.session.user, auto_commit=1) - + frappe.db.delete("Has Role", { + "parent": frappe.session.user, + "role": "Test Has Role" + }) + frappe.db.commit() if not frappe.db.exists('Role', 'Test Has Role'): role = frappe.get_doc({ 'doctype': 'Role', diff --git a/frappe/core/doctype/role/role.py b/frappe/core/doctype/role/role.py index 02482c75ca..28b444e1e7 100644 --- a/frappe/core/doctype/role/role.py +++ b/frappe/core/doctype/role/role.py @@ -38,7 +38,7 @@ class Role(Document): self.set(key, 0) def remove_roles(self): - frappe.db.sql("delete from `tabHas Role` where role = %s", self.name) + frappe.db.delete("Has Role", {"role": self.name}) frappe.clear_cache() def on_update(self): diff --git a/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py b/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py index 59089d12ad..b6515b1e79 100644 --- a/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py +++ b/frappe/core/doctype/scheduled_job_type/scheduled_job_type.py @@ -1,5 +1,4 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2019, Frappe Technologies and contributors +# Copyright (c) 2021, Frappe Technologies and contributors # For license information, please see license.txt import json @@ -110,7 +109,7 @@ class ScheduledJobType(Document): return 'long' if ('Long' in self.frequency) else 'default' def on_trash(self): - frappe.db.sql('delete from `tabScheduled Job Log` where scheduled_job_type=%s', self.name) + frappe.db.delete("Scheduled Job Log", {"scheduled_job_type": self.name}) @frappe.whitelist() diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py index 53d1c9ffe5..5d799f8ee9 100644 --- a/frappe/core/doctype/user/user.py +++ b/frappe/core/doctype/user/user.py @@ -368,17 +368,15 @@ class User(Document): frappe.local.login_manager.logout(user=self.name) # delete todos - frappe.db.sql("""DELETE FROM `tabToDo` WHERE `owner`=%s""", (self.name,)) + frappe.db.delete("ToDo", {"owner": self.name}) frappe.db.sql("""UPDATE `tabToDo` SET `assigned_by`=NULL WHERE `assigned_by`=%s""", (self.name,)) # delete events - frappe.db.sql("""delete from `tabEvent` where owner=%s - and event_type='Private'""", (self.name,)) + frappe.db.delete("Event", {"owner": self.name, "event_type": "Private"}) # delete shares - frappe.db.sql("""delete from `tabDocShare` where user=%s""", self.name) - + frappe.db.delete("DocShare", {"user": self.name}) # delete messages frappe.db.sql("""delete from `tabCommunication` where communication_type in ('Chat', 'Notification') diff --git a/frappe/core/doctype/user_permission/test_user_permission.py b/frappe/core/doctype/user_permission/test_user_permission.py index 1a442b53e7..85db846982 100644 --- a/frappe/core/doctype/user_permission/test_user_permission.py +++ b/frappe/core/doctype/user_permission/test_user_permission.py @@ -1,6 +1,5 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Frappe Technologies and Contributors -# See license.txt +# Copyright (c) 2021, Frappe Technologies and Contributors +# See LICENSE from frappe.core.doctype.user_permission.user_permission import add_user_permissions, remove_applicable from frappe.permissions import has_user_permission from frappe.core.doctype.doctype.test_doctype import new_doctype @@ -10,11 +9,14 @@ import unittest class TestUserPermission(unittest.TestCase): def setUp(self): - frappe.db.sql("""DELETE FROM `tabUser Permission` - WHERE `user` in ( - 'test_bulk_creation_update@example.com', - 'test_user_perm1@example.com', - 'nested_doc_user@example.com')""") + test_users = ( + "test_bulk_creation_update@example.com", + "test_user_perm1@example.com", + "nested_doc_user@example.com", + ) + frappe.db.delete("User Permission", { + "user": ("in", test_users) + }) frappe.delete_doc_if_exists("DocType", "Person") frappe.db.sql_ddl("DROP TABLE IF EXISTS `tabPerson`") frappe.delete_doc_if_exists("DocType", "Doc A") diff --git a/frappe/core/doctype/user_permission/user_permission.py b/frappe/core/doctype/user_permission/user_permission.py index 4aa5797c7f..5201ffef8d 100644 --- a/frappe/core/doctype/user_permission/user_permission.py +++ b/frappe/core/doctype/user_permission/user_permission.py @@ -1,5 +1,4 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Frappe Technologies and contributors +# Copyright (c) 2021, Frappe Technologies and contributors # For license information, please see license.txt import frappe, json @@ -179,11 +178,16 @@ def check_applicable_doc_perm(user, doctype, docname): @frappe.whitelist() def clear_user_permissions(user, for_doctype): - frappe.only_for('System Manager') - total = frappe.db.count('User Permission', filters = dict(user=user, allow=for_doctype)) + frappe.only_for("System Manager") + total = frappe.db.count("User Permission", {"user": user, "allow": for_doctype}) + if total: - frappe.db.sql('DELETE FROM `tabUser Permission` WHERE `user`=%s AND `allow`=%s', (user, for_doctype)) + frappe.db.delete("User Permission", { + "allow": for_doctype, + "user": user, + }) frappe.clear_cache() + return total @frappe.whitelist() @@ -225,7 +229,7 @@ def insert_user_perm(user, doctype, docname, is_default=0, hide_descendants=0, a user_perm.is_default = is_default user_perm.hide_descendants = hide_descendants if applicable: - user_perm.applicable_for = applicable + user_perm.applicable_for = applicable user_perm.apply_to_all_doctypes = 0 else: user_perm.apply_to_all_doctypes = 1 @@ -233,27 +237,27 @@ def insert_user_perm(user, doctype, docname, is_default=0, hide_descendants=0, a def remove_applicable(perm_applied_docs, user, doctype, docname): for applicable_for in perm_applied_docs: - frappe.db.sql("""DELETE FROM `tabUser Permission` - WHERE `user`=%s - AND `applicable_for`=%s - AND `allow`=%s - AND `for_value`=%s - """, (user, applicable_for, doctype, docname)) + frappe.db.delete("User Permission", { + "applicable_for": applicable_for, + "for_value": docname, + "allow": doctype, + "user": user, + }) def remove_apply_to_all(user, doctype, docname): - frappe.db.sql("""DELETE from `tabUser Permission` - WHERE `user`=%s - AND `apply_to_all_doctypes`=1 - AND `allow`=%s - AND `for_value`=%s - """,(user, doctype, docname)) + frappe.db.delete("User Permission", { + "apply_to_all_doctypes": 1, + "for_value": docname, + "allow": doctype, + "user": user, + }) def update_applicable(already_applied, to_apply, user, doctype, docname): for applied in already_applied: if applied not in to_apply: - frappe.db.sql("""DELETE FROM `tabUser Permission` - WHERE `user`=%s - AND `applicable_for`=%s - AND `allow`=%s - AND `for_value`=%s - """,(user, applied, doctype, docname)) + frappe.db.delete("User Permission", { + "applicable_for": applied, + "for_value": docname, + "allow": doctype, + "user": user, + }) diff --git a/frappe/core/page/background_jobs/background_jobs.py b/frappe/core/page/background_jobs/background_jobs.py index 847b23bd3e..1f3555e351 100644 --- a/frappe/core/page/background_jobs/background_jobs.py +++ b/frappe/core/page/background_jobs/background_jobs.py @@ -4,12 +4,12 @@ import json from typing import TYPE_CHECKING, Dict, List -from rq import Queue, Worker +from rq import Worker import frappe from frappe import _ from frappe.utils import convert_utc_to_user_timezone, format_datetime -from frappe.utils.background_jobs import get_redis_conn +from frappe.utils.background_jobs import get_redis_conn, get_queues from frappe.utils.scheduler import is_scheduler_inactive if TYPE_CHECKING: @@ -29,7 +29,7 @@ def get_info(show_failed=False) -> List[Dict]: show_failed = json.loads(show_failed) conn = get_redis_conn() - queues = Queue.all(conn) + queues = get_queues() workers = Worker.all(conn) jobs = [] @@ -75,7 +75,7 @@ def get_info(show_failed=False) -> List[Dict]: @frappe.whitelist() def remove_failed_jobs(): conn = get_redis_conn() - queues = Queue.all(conn) + queues = get_queues() for queue in queues: fail_registry = queue.failed_job_registry for job_id in fail_registry.get_job_ids(): diff --git a/frappe/core/page/permission_manager/permission_manager.py b/frappe/core/page/permission_manager/permission_manager.py index 15c7cb55ae..2a99283dda 100644 --- a/frappe/core/page/permission_manager/permission_manager.py +++ b/frappe/core/page/permission_manager/permission_manager.py @@ -92,14 +92,14 @@ def update(doctype, role, permlevel, ptype, value=None): """Update role permission params Args: - doctype (str): Name of the DocType to update params for - role (str): Role to be updated for, eg "Website Manager". - permlevel (int): perm level the provided rule applies to - ptype (str): permission type, example "read", "delete", etc. - value (None, optional): value for ptype, None indicates False + doctype (str): Name of the DocType to update params for + role (str): Role to be updated for, eg "Website Manager". + permlevel (int): perm level the provided rule applies to + ptype (str): permission type, example "read", "delete", etc. + value (None, optional): value for ptype, None indicates False Returns: - str: Refresh flag is permission is updated successfully + str: Refresh flag is permission is updated successfully """ frappe.only_for("System Manager") out = update_permission_property(doctype, role, permlevel, ptype, value) @@ -110,10 +110,9 @@ def remove(doctype, role, permlevel): frappe.only_for("System Manager") setup_custom_perms(doctype) - name = frappe.get_value('Custom DocPerm', dict(parent=doctype, role=role, permlevel=permlevel)) + frappe.db.delete("Custom DocPerm", {"parent": doctype, "role": role, "permlevel": permlevel}) - frappe.db.sql('delete from `tabCustom DocPerm` where name=%s', name) - if not frappe.get_all('Custom DocPerm', dict(parent=doctype)): + if not frappe.get_all('Custom DocPerm', {"parent": doctype}): frappe.throw(_('There must be atleast one permission rule.'), title=_('Cannot Remove')) validate_permissions_for_doctype(doctype, for_remove=True, alert=True) diff --git a/frappe/custom/doctype/custom_field/custom_field.py b/frappe/custom/doctype/custom_field/custom_field.py index 7e6ea1875a..e266455f7a 100644 --- a/frappe/custom/doctype/custom_field/custom_field.py +++ b/frappe/custom/doctype/custom_field/custom_field.py @@ -85,12 +85,10 @@ class CustomField(Document): frappe.bold(self.label))) # delete property setter entries - frappe.db.sql("""\ - DELETE FROM `tabProperty Setter` - WHERE doc_type = %s - AND field_name = %s""", - (self.dt, self.fieldname)) - + frappe.db.delete("Property Setter", { + "doc_type": self.dt, + "field_name": self.fieldname + }) frappe.clear_cache(doctype=self.dt) def validate_insert_after(self, meta): diff --git a/frappe/custom/doctype/customize_form/customize_form.py b/frappe/custom/doctype/customize_form/customize_form.py index 1b8977acc4..8de194fb00 100644 --- a/frappe/custom/doctype/customize_form/customize_form.py +++ b/frappe/custom/doctype/customize_form/customize_form.py @@ -1,5 +1,5 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors -# MIT License. See license.txt +# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors +# MIT License. See LICENSE """ Customize Form is a Single DocType used to mask the Property Setter @@ -18,10 +18,11 @@ from frappe.custom.doctype.property_setter.property_setter import delete_propert from frappe.model.docfield import supports_translation from frappe.core.doctype.doctype.doctype import validate_series + class CustomizeForm(Document): def on_update(self): - frappe.db.sql("delete from tabSingles where doctype='Customize Form'") - frappe.db.sql("delete from `tabCustomize Form Field`") + frappe.db.delete("Singles", {"doctype": "Customize Form"}) + frappe.db.delete("Customize Form Field") @frappe.whitelist() def fetch_to_customize(self): diff --git a/frappe/database/database.py b/frappe/database/database.py index 6012e47445..b1dec95139 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -6,6 +6,7 @@ import re import time +from typing import Dict, List, Union import frappe import datetime import frappe.defaults @@ -13,7 +14,7 @@ import frappe.model.meta from frappe import _ from time import time -from frappe.utils import now, getdate, cast_fieldtype, get_datetime +from frappe.utils import now, getdate, cast_fieldtype, get_datetime, get_table_name from frappe.model.utils.link_count import flush_local_link_count @@ -103,6 +104,7 @@ class Database(object): {"name": "a%", "owner":"test@example.com"}) """ + query = str(query) if re.search(r'ifnull\(', query, flags=re.IGNORECASE): # replaces ifnull in query with coalesce query = re.sub(r'ifnull\(', 'coalesce(', query, flags=re.IGNORECASE) @@ -951,15 +953,37 @@ class Database(object): query = sql_dict.get(current_dialect) return self.sql(query, values, **kwargs) - def delete(self, doctype, conditions, debug=False): - if conditions: - conditions, values = self.build_conditions(conditions) - return self.sql("DELETE FROM `tab{doctype}` where {conditions}".format( - doctype=doctype, - conditions=conditions - ), values, debug=debug) - else: - frappe.throw(_('No conditions provided')) + def delete(self, doctype: str, filters: Union[Dict, List] = None, debug=False, **kwargs): + """Delete rows from a table in site which match the passed filters. This + does trigger DocType hooks. Simply runs a DELETE query in the database. + + Doctype name can be passed directly, it will be pre-pended with `tab`. + """ + values = () + filters = filters or kwargs.get("conditions") + table = get_table_name(doctype) + query = f"DELETE FROM `{table}`" + + if "debug" not in kwargs: + kwargs["debug"] = debug + + if filters: + conditions, values = self.build_conditions(filters) + query = f"{query} WHERE {conditions}" + + return self.sql(query, values, **kwargs) + + def truncate(self, doctype: str): + """Truncate a table in the database. This runs a DDL command `TRUNCATE TABLE`. + This cannot be rolled back. + + Doctype name can be passed directly, it will be pre-pended with `tab`. + """ + table = doctype if doctype.startswith("__") else f"tab{doctype}" + return self.sql_ddl(f"truncate `{table}`") + + def clear_table(self, doctype): + return self.truncate(doctype) def get_last_created(self, doctype): last_record = self.get_all(doctype, ('creation'), limit=1, order_by='creation desc') @@ -968,9 +992,6 @@ class Database(object): else: return None - def clear_table(self, doctype): - self.sql('truncate `tab{}`'.format(doctype)) - def log_touched_tables(self, query, values=None): if values: query = frappe.safe_decode(self._cursor.mogrify(query, values)) @@ -1021,6 +1042,7 @@ class Database(object): ), tuple(insert_list)) insert_list = [] + def enqueue_jobs_after_commit(): from frappe.utils.background_jobs import execute_job, get_queue diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index 879c8394d7..5dd6d9e58a 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -1,3 +1,5 @@ +from typing import List, Tuple, Union + import pymysql from pymysql.constants import ER, FIELD_TYPE from pymysql.converters import conversions, escape_string @@ -5,7 +7,7 @@ from pymysql.converters import conversions, escape_string import frappe from frappe.database.database import Database from frappe.database.mariadb.schema import MariaDBTable -from frappe.utils import UnicodeWithAttrs, cstr, get_datetime +from frappe.utils import UnicodeWithAttrs, cstr, get_datetime, get_table_name class MariaDBDatabase(Database): @@ -123,6 +125,19 @@ class MariaDBDatabase(Database): def is_type_datetime(code): return code in (pymysql.DATE, pymysql.DATETIME) + def rename_table(self, old_name: str, new_name: str) -> Union[List, Tuple]: + old_name = get_table_name(old_name) + new_name = get_table_name(new_name) + return self.sql(f"RENAME TABLE `{old_name}` TO `{new_name}`") + + def describe(self, doctype: str) -> Union[List, Tuple]: + table_name = get_table_name(doctype) + return self.sql(f"DESC `{table_name}`") + + def change_column_type(self, table: str, column: str, type: str) -> Union[List, Tuple]: + table_name = get_table_name(table) + return self.sql(f"ALTER TABLE `{table_name}` MODIFY `{column}` {type} NOT NULL") + # exception types @staticmethod def is_deadlocked(e): diff --git a/frappe/database/mariadb/framework_mariadb.sql b/frappe/database/mariadb/framework_mariadb.sql index a52efd01e3..f8841e9417 100644 --- a/frappe/database/mariadb/framework_mariadb.sql +++ b/frappe/database/mariadb/framework_mariadb.sql @@ -220,6 +220,7 @@ CREATE TABLE `tabDocType` ( `allow_guest_to_view` int(1) NOT NULL DEFAULT 0, `route` varchar(255) DEFAULT NULL, `is_published_field` varchar(255) DEFAULT NULL, + `website_search_field` varchar(255) DEFAULT NULL, `email_append_to` int(1) NOT NULL DEFAULT 0, `subject_field` varchar(255) DEFAULT NULL, `sender_field` varchar(255) DEFAULT NULL, diff --git a/frappe/database/postgres/database.py b/frappe/database/postgres/database.py index 8235277e30..0b73c8b44b 100644 --- a/frappe/database/postgres/database.py +++ b/frappe/database/postgres/database.py @@ -1,12 +1,14 @@ import re -import frappe +from typing import List, Tuple, Union + import psycopg2 import psycopg2.extensions -from frappe.utils import cstr from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT +import frappe from frappe.database.database import Database from frappe.database.postgres.schema import PostgresTable +from frappe.utils import cstr, get_table_name # cast decimals as floats DEC2FLOAT = psycopg2.extensions.new_type( @@ -170,6 +172,19 @@ class PostgresDatabase(Database): def is_data_too_long(e): return e.pgcode == '22001' + def rename_table(self, old_name: str, new_name: str) -> Union[List, Tuple]: + old_name = get_table_name(old_name) + new_name = get_table_name(new_name) + return self.sql(f"ALTER TABLE `{old_name}` RENAME TO `{new_name}`") + + def describe(self, doctype: str)-> Union[List, Tuple]: + table_name = get_table_name(doctype) + return self.sql(f"SELECT COLUMN_NAME FROM information_schema.COLUMNS WHERE TABLE_NAME = '{table_name}'") + + def change_column_type(self, table: str, column: str, type: str) -> Union[List, Tuple]: + table_name = get_table_name(table) + return self.sql(f'ALTER TABLE "{table_name}" ALTER COLUMN "{column}" TYPE {type}') + def create_auth_table(self): self.sql_ddl("""create table if not exists "__Auth" ( "doctype" VARCHAR(140) NOT NULL, @@ -297,6 +312,7 @@ class PostgresDatabase(Database): def modify_query(query): """"Modifies query according to the requirements of postgres""" # replace ` with " for definitions + query = str(query) query = query.replace('`', '"') query = replace_locate_with_strpos(query) # select from requires "" diff --git a/frappe/database/postgres/framework_postgres.sql b/frappe/database/postgres/framework_postgres.sql index eeb0eecd3f..a4e94aa326 100644 --- a/frappe/database/postgres/framework_postgres.sql +++ b/frappe/database/postgres/framework_postgres.sql @@ -225,6 +225,7 @@ CREATE TABLE "tabDocType" ( "allow_guest_to_view" smallint NOT NULL DEFAULT 0, "route" varchar(255) DEFAULT NULL, "is_published_field" varchar(255) DEFAULT NULL, + "website_search_field" varchar(255) DEFAULT NULL, "email_append_to" smallint NOT NULL DEFAULT 0, "subject_field" varchar(255) DEFAULT NULL, "sender_field" varchar(255) DEFAULT NULL, diff --git a/frappe/defaults.py b/frappe/defaults.py index fde48d71ff..d4c338388d 100644 --- a/frappe/defaults.py +++ b/frappe/defaults.py @@ -124,11 +124,10 @@ def set_default(key, value, parent, parenttype="__default"): where defkey=%s and parent=%s for update''', (key, parent)): - frappe.db.sql(""" - delete from - `tabDefaultValue` - where - defkey=%s and parent=%s""", (key, parent)) + frappe.db.delete("DefaultValue", { + "defkey": key, + "parent": parent + }) if value != None: add_default(key, value, parent) else: @@ -155,29 +154,23 @@ def clear_default(key=None, value=None, parent=None, name=None, parenttype=None) :param name: Default ID. :param parenttype: Clear defaults table for a particular type e.g. **User**. """ - conditions = [] - values = [] + filters = {} if name: - conditions.append("name=%s") - values.append(name) + filters.update({"name": name}) else: if key: - conditions.append("defkey=%s") - values.append(key) + filters.update({"defkey": key}) if value: - conditions.append("defvalue=%s") - values.append(value) + filters.update({"defvalue": value}) if parent: - conditions.append("parent=%s") - values.append(parent) + filters.update({"parent": parent}) if parenttype: - conditions.append("parenttype=%s") - values.append(parenttype) + filters.update({"parenttype": parenttype}) if parent: clear_defaults_cache(parent) @@ -185,11 +178,10 @@ def clear_default(key=None, value=None, parent=None, name=None, parenttype=None) clear_defaults_cache("__default") clear_defaults_cache("__global") - if not conditions: + if not filters: raise Exception("[clear_default] No key specified.") - frappe.db.sql("""delete from tabDefaultValue where {0}""".format(" and ".join(conditions)), - tuple(values)) + frappe.db.delete("DefaultValue", filters) _clear_cache(parent) diff --git a/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py b/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py index 78d133b2d5..9f10522b12 100644 --- a/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py +++ b/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py @@ -64,7 +64,7 @@ class TestDashboardChart(unittest.TestCase): if frappe.db.exists('Dashboard Chart', 'Test Empty Dashboard Chart'): frappe.delete_doc('Dashboard Chart', 'Test Empty Dashboard Chart') - frappe.db.sql('delete from `tabError Log`') + frappe.db.delete("Error Log") frappe.get_doc(dict( doctype = 'Dashboard Chart', @@ -94,7 +94,7 @@ class TestDashboardChart(unittest.TestCase): if frappe.db.exists('Dashboard Chart', 'Test Empty Dashboard Chart 2'): frappe.delete_doc('Dashboard Chart', 'Test Empty Dashboard Chart 2') - frappe.db.sql('delete from `tabError Log`') + frappe.db.delete("Error Log") # create one data point frappe.get_doc(dict(doctype = 'Error Log', creation = '2018-06-01 00:00:00')).insert() diff --git a/frappe/desk/doctype/desktop_icon/desktop_icon.py b/frappe/desk/doctype/desktop_icon/desktop_icon.py index 81a79cdb09..28c5a670cb 100644 --- a/frappe/desk/doctype/desktop_icon/desktop_icon.py +++ b/frappe/desk/doctype/desktop_icon/desktop_icon.py @@ -197,7 +197,7 @@ def set_desktop_icons(visible_list, ignore_duplicate=True): # clear all custom only if setup is not complete if not int(frappe.defaults.get_defaults().setup_complete or 0): - frappe.db.sql('delete from `tabDesktop Icon` where standard=0') + frappe.db.delete("Desktop Icon", {"standard": 0}) # set standard as blocked and hidden if setting first active domain if not frappe.flags.keep_desktop_icons: diff --git a/frappe/desk/doctype/event/event.py b/frappe/desk/doctype/event/event.py index 57c89eaf2e..e7e7be530b 100644 --- a/frappe/desk/doctype/event/event.py +++ b/frappe/desk/doctype/event/event.py @@ -338,9 +338,8 @@ def delete_events(ref_type, ref_name, delete_event=False): total_participants = frappe.get_all("Event Participants", filters={"parenttype": "Event", "parent": participation.parent}) if len(total_participants) <= 1: - frappe.db.sql("DELETE FROM `tabEvent` WHERE `name` = %(name)s", {'name': participation.parent}) - - frappe.db.sql("DELETE FROM `tabEvent Participants ` WHERE `name` = %(name)s", {'name': participation.name}) + frappe.db.delete("Event", {"name": participation.parent}) + frappe.db.delete("Event Participants", {"name": participation.name}) # Close events if ends_on or repeat_till is less than now_datetime def set_status_of_events(): diff --git a/frappe/desk/doctype/route_history/route_history.py b/frappe/desk/doctype/route_history/route_history.py index b82077f485..95872440c7 100644 --- a/frappe/desk/doctype/route_history/route_history.py +++ b/frappe/desk/doctype/route_history/route_history.py @@ -1,5 +1,4 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2018, Frappe Technologies and contributors +# Copyright (c) 2021, Frappe Technologies and contributors # For license information, please see license.txt import frappe @@ -8,6 +7,7 @@ from frappe.model.document import Document class RouteHistory(Document): pass + def flush_old_route_records(): """Deletes all route records except last 500 records per user""" @@ -24,19 +24,14 @@ def flush_old_route_records(): for user in users: user = user[0] last_record_to_keep = frappe.db.get_all('Route History', - filters={ - 'user': user, - }, + filters={'user': user}, limit=1, limit_start=500, fields=['modified'], - order_by='modified desc') + order_by='modified desc' + ) - frappe.db.sql(''' - DELETE - FROM `tabRoute History` - WHERE `modified` <= %(modified)s and `user`=%(modified)s - ''', { - "modified": last_record_to_keep[0].modified, + frappe.db.delete("Route History", { + "modified": ("<=", last_record_to_keep[0].modified), "user": user - }) \ No newline at end of file + }) diff --git a/frappe/desk/doctype/tag/tag.py b/frappe/desk/doctype/tag/tag.py index 4ea5c9cd7e..2341d721e2 100644 --- a/frappe/desk/doctype/tag/tag.py +++ b/frappe/desk/doctype/tag/tag.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (c) 2019, Frappe Technologies and contributors # For license information, please see license.txt @@ -123,7 +122,10 @@ def delete_tags_for_document(doc): if not frappe.db.table_exists("Tag Link"): return - frappe.db.sql("""DELETE FROM `tabTag Link` WHERE `document_type`=%s AND `document_name`=%s""", (doc.doctype, doc.name)) + frappe.db.delete("Tag Link", { + "document_type": doc.doctype, + "document_name": doc.name + }) def update_tags(doc, tags): """ @@ -161,7 +163,11 @@ def get_deleted_tags(new_tags, existing_tags): return list(set(existing_tags) - set(new_tags)) def delete_tag_for_document(dt, dn, tag): - frappe.db.sql("""DELETE FROM `tabTag Link` WHERE `document_type`=%s AND `document_name`=%s AND tag=%s""", (dt, dn, tag)) + frappe.db.delete("Tag Link", { + "document_type": dt, + "document_name": dn, + "tag": tag + }) @frappe.whitelist() def get_documents_for_tag(tag): diff --git a/frappe/desk/doctype/todo/todo.py b/frappe/desk/doctype/todo/todo.py index 4696563445..09297b4e5e 100644 --- a/frappe/desk/doctype/todo/todo.py +++ b/frappe/desk/doctype/todo/todo.py @@ -5,11 +5,13 @@ import frappe import json from frappe.model.document import Document -from frappe.utils import get_fullname +from frappe.utils import get_fullname, parse_addr exclude_from_linked_with = True class ToDo(Document): + DocType = 'ToDo' + def validate(self): self._assignment = None if self.is_new(): @@ -39,13 +41,7 @@ class ToDo(Document): self.update_in_reference() def on_trash(self): - # unlink todo from linked comments - frappe.db.sql(""" - delete from `tabCommunication Link` - where link_doctype=%(doctype)s and link_name=%(name)s""", { - "doctype": self.doctype, "name": self.name - }) - + self.delete_communication_links() self.update_in_reference() def add_assign_comment(self, text, comment_type): @@ -54,6 +50,13 @@ class ToDo(Document): frappe.get_doc(self.reference_type, self.reference_name).add_comment(comment_type, text) + def delete_communication_links(self): + # unlink todo from linked comments + return frappe.db.delete("Communication Link", { + "link_doctype": self.doctype, + "link_name": self.name + }) + def update_in_reference(self): if not (self.reference_type and self.reference_name): return @@ -84,6 +87,13 @@ class ToDo(Document): else: raise + @classmethod + def get_owners(cls, filters=None): + """Returns list of owners after applying filters on todo's. + """ + rows = frappe.get_all(cls.DocType, filters=filters or {}, fields=['owner']) + return [parse_addr(row.owner)[1] for row in rows if row.owner] + # NOTE: todo is viewable if a user is an owner, or set as assigned_to value, or has any role that is allowed to access ToDo doctype. def on_doctype_update(): frappe.db.add_index("ToDo", ["reference_type", "reference_name"]) diff --git a/frappe/email/doctype/unhandled_email/unhandled_email.py b/frappe/email/doctype/unhandled_email/unhandled_email.py index 6414dbece3..b445c98aa6 100644 --- a/frappe/email/doctype/unhandled_email/unhandled_email.py +++ b/frappe/email/doctype/unhandled_email/unhandled_email.py @@ -10,5 +10,6 @@ class UnhandledEmail(Document): def remove_old_unhandled_emails(): - frappe.db.sql("""DELETE FROM `tabUnhandled Email` - WHERE creation < %s""", frappe.utils.add_days(frappe.utils.nowdate(), -30)) + frappe.db.delete("Unhandled Email", { + "creation": ("<", frappe.utils.add_days(frappe.utils.nowdate(), -30)) + }) \ No newline at end of file diff --git a/frappe/email/queue.py b/frappe/email/queue.py index 885a306cfb..ef59302bab 100755 --- a/frappe/email/queue.py +++ b/frappe/email/queue.py @@ -173,13 +173,8 @@ def clear_outbox(days=None): WHERE `priority`=0 AND `modified` < (NOW() - INTERVAL '{0}' DAY)""".format(days)) if email_queues: - frappe.db.sql("""DELETE FROM `tabEmail Queue` WHERE `name` IN ({0})""".format( - ','.join(['%s']*len(email_queues) - )), tuple(email_queues)) - - frappe.db.sql("""DELETE FROM `tabEmail Queue Recipient` WHERE `parent` IN ({0})""".format( - ','.join(['%s']*len(email_queues) - )), tuple(email_queues)) + frappe.db.delete("Email Queue", {"name": ("in", email_queues)}) + frappe.db.delete("Email Queue Recipient", {"parent": ("in", email_queues)}) def set_expiry_for_email_queue(): ''' Mark emails as expire that has not sent for 7 days. diff --git a/frappe/model/__init__.py b/frappe/model/__init__.py index 75122f5aba..4aa8fb9000 100644 --- a/frappe/model/__init__.py +++ b/frappe/model/__init__.py @@ -152,32 +152,22 @@ def delete_fields(args_dict, delete=0): if not fields: continue - frappe.db.sql(""" - DELETE FROM `tabDocField` - WHERE parent='%s' AND fieldname IN (%s) - """ % (dt, ", ".join(["'{}'".format(f) for f in fields]))) + frappe.db.delete("DocField", { + "parent": dt, + "fieldname": ("in", fields), + }) # Delete the data/column only if delete is specified if not delete: continue if frappe.db.get_value("DocType", dt, "issingle"): - frappe.db.sql(""" - DELETE FROM `tabSingles` - WHERE doctype='%s' AND field IN (%s) - """ % (dt, ", ".join("'{}'".format(f) for f in fields))) + frappe.db.delete("Singles", { + "doctype": dt, + "field": ("in", fields), + }) else: - existing_fields = frappe.db.multisql({ - "mariadb": "DESC `tab%s`" % dt, - "postgres": """ - SELECT - COLUMN_NAME - FROM - information_schema.COLUMNS - WHERE - TABLE_NAME = 'tab%s'; - """ % dt, - }) + existing_fields = frappe.db.describe(dt) existing_fields = existing_fields and [e[0] for e in existing_fields] or [] fields_need_to_delete = set(fields) & set(existing_fields) if not fields_need_to_delete: diff --git a/frappe/model/delete_doc.py b/frappe/model/delete_doc.py index cc88cfa106..fbbf1a4852 100644 --- a/frappe/model/delete_doc.py +++ b/frappe/model/delete_doc.py @@ -65,12 +65,12 @@ def delete_doc(doctype=None, name=None, force=0, ignore_doctypes=None, for_reloa update_flags(doc, flags, ignore_permissions) check_permission_and_not_submitted(doc) - frappe.db.sql("delete from `tabCustom Field` where dt = %s", name) - frappe.db.sql("delete from `tabClient Script` where dt = %s", name) - frappe.db.sql("delete from `tabProperty Setter` where doc_type = %s", name) - frappe.db.sql("delete from `tabReport` where ref_doctype=%s", name) - frappe.db.sql("delete from `tabCustom DocPerm` where parent=%s", name) - frappe.db.sql("delete from `__global_search` where doctype=%s", name) + frappe.db.delete("Custom Field", {"dt": name}) + frappe.db.delete("Client Script", {"dt": name}) + frappe.db.delete("Property Setter", {"doc_type": name}) + frappe.db.delete("Report", {"ref_doctype": name}) + frappe.db.delete("Custom DocPerm", {"parent": name}) + frappe.db.delete("__global_search", {"doctype": name}) delete_from_table(doctype, name, ignore_doctypes, None) @@ -162,10 +162,9 @@ def update_naming_series(doc): def delete_from_table(doctype, name, ignore_doctypes, doc): if doctype!="DocType" and doctype==name: - frappe.db.sql("delete from `tabSingles` where `doctype`=%s", name) + frappe.db.delete("Singles", {"doctype": name}) else: - frappe.db.sql("delete from `tab{0}` where `name`=%s".format(doctype), name) - + frappe.db.delete(doctype, {"name": name}) # get child tables if doc: tables = [d.options for d in doc.meta.get_table_fields()] @@ -339,8 +338,10 @@ def clear_references(doctype, reference_doctype, reference_name, (reference_doctype, reference_name)) def clear_timeline_references(link_doctype, link_name): - frappe.db.sql("""DELETE FROM `tabCommunication Link` - WHERE `tabCommunication Link`.link_doctype=%s AND `tabCommunication Link`.link_name=%s""", (link_doctype, link_name)) + frappe.db.delete("Communication Link", { + "link_doctype": link_doctype, + "link_name": link_name + }) def insert_feed(doc): if ( diff --git a/frappe/model/document.py b/frappe/model/document.py index e974ae2a3e..b44d95716e 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -5,7 +5,7 @@ import time from frappe import _, msgprint, is_whitelisted from frappe.utils import flt, cstr, now, get_datetime_str, file_lock, date_diff from frappe.model.base_document import BaseDocument, get_controller -from frappe.model.naming import set_new_name, gen_new_name_for_cancelled_doc +from frappe.model.naming import set_new_name from werkzeug.exceptions import NotFound, Forbidden import hashlib, json from frappe.model import optional_fields, table_fields @@ -390,10 +390,11 @@ class Document(BaseDocument): else: # no rows found, delete all rows - frappe.db.sql("""delete from `tab{0}` where parent=%s - and parenttype=%s and parentfield=%s""".format(df.options), - (self.name, self.doctype, fieldname)) - + frappe.db.delete(df.options, { + "parent": self.name, + "parenttype": self.doctype, + "parentfield": fieldname + }) def get_doc_before_save(self): return getattr(self, '_doc_before_save', None) @@ -451,7 +452,9 @@ class Document(BaseDocument): def update_single(self, d): """Updates values for Single type Document in `tabSingles`.""" - frappe.db.sql("""delete from `tabSingles` where doctype=%s""", self.doctype) + frappe.db.delete("Singles", { + "doctype": self.doctype + }) for field, value in d.items(): if field != "doctype": frappe.db.sql("""insert into `tabSingles` (doctype, field, value) @@ -705,6 +708,7 @@ class Document(BaseDocument): else: tmp = frappe.db.sql("""select modified, docstatus from `tab{0}` where name = %s for update""".format(self.doctype), self.name, as_dict=True) + if not tmp: frappe.throw(_("Record does not exist")) else: @@ -915,12 +919,8 @@ class Document(BaseDocument): @whitelist.__func__ def _cancel(self): - """Cancel the document. Sets `docstatus` = 2, then saves. - """ + """Cancel the document. Sets `docstatus` = 2, then saves.""" self.docstatus = 2 - new_name = gen_new_name_for_cancelled_doc(self) - frappe.rename_doc(self.doctype, self.name, new_name, force=True, show_alert=False) - self.name = new_name self.save() @whitelist.__func__ diff --git a/frappe/model/naming.py b/frappe/model/naming.py index 6dff0aaff6..fe136adce8 100644 --- a/frappe/model/naming.py +++ b/frappe/model/naming.py @@ -28,7 +28,7 @@ def set_new_name(doc): doc.name = None if getattr(doc, "amended_from", None): - doc.name = _get_amended_name(doc) + _set_amended_name(doc) return elif getattr(doc.meta, "issingle", False): @@ -221,15 +221,6 @@ def revert_series_if_last(key, name, doc=None): * prefix = #### and hashes = 2021 (hash doesn't exist) * will search hash in key then accordingly get prefix = "" """ - if hasattr(doc, 'amended_from'): - # do not revert if doc is amended, since cancelled docs still exist - if doc.docstatus != 2 and doc.amended_from: - return - - # for first cancelled doc - if doc.docstatus == 2 and not doc.amended_from: - name, _ = NameParser.parse_docname(doc.name, sep='-CAN-') - if ".#" in key: prefix, hashes = key.rsplit(".", 1) if "#" not in hashes: @@ -312,9 +303,16 @@ def append_number_if_name_exists(doctype, value, fieldname="name", separator="-" return value -def _get_amended_name(doc): - name, _ = NameParser(doc).parse_amended_from() - return name +def _set_amended_name(doc): + am_id = 1 + am_prefix = doc.amended_from + if frappe.db.get_value(doc.doctype, doc.amended_from, "amended_from"): + am_id = cint(doc.amended_from.split("-")[-1]) + 1 + am_prefix = "-".join(doc.amended_from.split("-")[:-1]) # except the last hyphen + + doc.name = am_prefix + "-" + str(am_id) + return doc.name + def _field_autoname(autoname, doc, skip_slicing=None): """ @@ -325,6 +323,7 @@ def _field_autoname(autoname, doc, skip_slicing=None): name = (cstr(doc.get(fieldname)) or "").strip() return name + def _prompt_autoname(autoname, doc): """ Generate a name using Prompt option. This simply means the user will have to set the name manually. @@ -355,61 +354,3 @@ def _format_autoname(autoname, doc): name = re.sub(r"(\{[\w | #]+\})", get_param_value_for_match, autoname_value) return name - -class NameParser: - """Parse document name and return all the parts of it. - - NOTE: It handles cancellend and amended doc parsing for now. It can be expanded. - """ - def __init__(self, doc): - self.doc = doc - - def parse_name(self): - if not hasattr(self.doc, "amended_from"): - return (self.doc.name, None, None) - - #If document is cancelled document - if hasattr(self.doc, "amended_from") and self.doc.docstatus == 2: - return self.parse_docname(self.doc.name, sep='-CAN-') - return self.parse_docname(self.doc.name) - - def parse_amended_from(self): - if not getattr(self.doc, 'amended_from', None): - return (None, None) - return self.parse_docname(self.doc.amended_from, '-CAN-') - - @classmethod - def parse_docname(cls, name, sep='-'): - split_list = name.rsplit(sep, 1) - - if len(split_list) == 1: - return (name, None) - return (split_list[0], split_list[1]) - -def get_cancelled_doc_latest_counter(tname, docname): - """Get the latest counter used for cancelled docs of given docname. - """ - name_prefix = f'{docname}-CAN-' - - rows = frappe.db.sql(""" - select - name - from `tab{tname}` - where - name like %(name_prefix)s and docstatus=2 - """.format(tname=tname), {'name_prefix': name_prefix+'%'}, as_dict=1) - - if not rows: - return -1 - return max([int(row.name.replace(name_prefix, '') or -1) for row in rows]) - -def gen_new_name_for_cancelled_doc(doc): - """Generate a new name for cancelled document. - """ - if getattr(doc, "amended_from", None): - name, _ = NameParser(doc).parse_amended_from() - else: - name = doc.name - - counter = get_cancelled_doc_latest_counter(doc.doctype, name) - return f'{name}-CAN-{counter+1}' diff --git a/frappe/patches.txt b/frappe/patches.txt index a9c5807df0..493c4dc9f6 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -180,4 +180,4 @@ frappe.patches.v12_0.rename_uploaded_files_with_proper_name frappe.patches.v13_0.queryreport_columns frappe.patches.v13_0.jinja_hook frappe.patches.v13_0.update_notification_channel_if_empty -frappe.patches.v13_0.rename_cancelled_docs +frappe.patches.v14_0.drop_data_import_legacy diff --git a/frappe/patches/v11_0/apply_customization_to_custom_doctype.py b/frappe/patches/v11_0/apply_customization_to_custom_doctype.py index 49b68ed240..7e84c5ae24 100644 --- a/frappe/patches/v11_0/apply_customization_to_custom_doctype.py +++ b/frappe/patches/v11_0/apply_customization_to_custom_doctype.py @@ -28,7 +28,7 @@ def execute(): for prop in property_setters: property_setter_map[prop.field_name] = prop - frappe.db.sql('DELETE FROM `tabProperty Setter` WHERE `name`=%s', prop.name) + frappe.db.delete("Property Setter", {"name": prop.name}) meta = frappe.get_meta(doctype.name) @@ -50,6 +50,6 @@ def execute(): df = frappe.new_doc('DocField', meta, 'fields') df.update(cf) meta.fields.append(df) - frappe.db.sql('DELETE FROM `tabCustom Field` WHERE name=%s', cf.name) + frappe.db.delete("Custom Field", {"name": cf.name}) meta.save() diff --git a/frappe/patches/v11_0/sync_stripe_settings_before_migrate.py b/frappe/patches/v11_0/sync_stripe_settings_before_migrate.py index a8e9bd4de1..901ab66bfd 100644 --- a/frappe/patches/v11_0/sync_stripe_settings_before_migrate.py +++ b/frappe/patches/v11_0/sync_stripe_settings_before_migrate.py @@ -17,4 +17,4 @@ def execute(): settings.secret_key = secret_key settings.save(ignore_permissions=True) - frappe.db.sql("""DELETE FROM tabSingles WHERE doctype='Stripe Settings'""") \ No newline at end of file + frappe.db.delete("Singles", {"doctype": "Stripe Settings"}) diff --git a/frappe/patches/v12_0/delete_feedback_request_if_exists.py b/frappe/patches/v12_0/delete_feedback_request_if_exists.py index fdbcecfc5a..c1bf46b14a 100644 --- a/frappe/patches/v12_0/delete_feedback_request_if_exists.py +++ b/frappe/patches/v12_0/delete_feedback_request_if_exists.py @@ -2,7 +2,4 @@ import frappe def execute(): - frappe.db.sql(''' - DELETE from `tabDocType` - WHERE name = 'Feedback Request' - ''') \ No newline at end of file + frappe.db.delete("DocType", {"name": "Feedback Request"}) diff --git a/frappe/patches/v12_0/remove_deprecated_fields_from_doctype.py b/frappe/patches/v12_0/remove_deprecated_fields_from_doctype.py index 60599066e6..9c9a79ccbf 100644 --- a/frappe/patches/v12_0/remove_deprecated_fields_from_doctype.py +++ b/frappe/patches/v12_0/remove_deprecated_fields_from_doctype.py @@ -8,7 +8,6 @@ def execute(): 'DocType': ['hide_heading', 'image_view', 'read_only_onload'] }, delete=1) - frappe.db.sql(''' - DELETE from `tabProperty Setter` - WHERE property = 'read_only_onload' - ''') + frappe.db.delete("Property Setter", { + "property": "read_only_onload" + }) \ No newline at end of file diff --git a/frappe/patches/v12_0/set_correct_assign_value_in_docs.py b/frappe/patches/v12_0/set_correct_assign_value_in_docs.py index 65a635c170..90766b5f64 100644 --- a/frappe/patches/v12_0/set_correct_assign_value_in_docs.py +++ b/frappe/patches/v12_0/set_correct_assign_value_in_docs.py @@ -1,32 +1,29 @@ import frappe +from frappe.query_builder.functions import GroupConcat, Coalesce def execute(): - frappe.reload_doc('desk', 'doctype', 'todo') + frappe.reload_doc("desk", "doctype", "todo") - query = ''' - SELECT - name, reference_type, reference_name, {} as assignees - FROM - `tabToDo` - WHERE - COALESCE(reference_type, '') != '' AND - COALESCE(reference_name, '') != '' AND - status != 'Cancelled' - GROUP BY - reference_type, reference_name - ''' + ToDo = frappe.qb.Table("ToDo") + assignees = GroupConcat("owner").distinct().as_("assignees") - assignments = frappe.db.multisql({ - 'mariadb': query.format('GROUP_CONCAT(DISTINCT `owner`)'), - 'postgres': query.format('STRING_AGG(DISTINCT "owner", ",")') - }, as_dict=True) + query = ( + frappe.qb.from_(ToDo) + .select(ToDo.name, ToDo.reference_type, assignees) + .where(Coalesce(ToDo.reference_type, "") != "") + .where(Coalesce(ToDo.reference_name, "") != "") + .where(ToDo.status != "Cancelled") + .groupby(ToDo.reference_type, ToDo.reference_name) + ) + + assignments = frappe.db.sql(query, as_dict=True) for doc in assignments: - assignments = doc.assignees.split(',') + assignments = doc.assignees.split(",") frappe.db.set_value( doc.reference_type, doc.reference_name, - '_assign', + "_assign", frappe.as_json(assignments), update_modified=False - ) + ) \ No newline at end of file diff --git a/frappe/patches/v12_0/set_primary_key_in_series.py b/frappe/patches/v12_0/set_primary_key_in_series.py index e5ed2204ba..83a903fc2d 100644 --- a/frappe/patches/v12_0/set_primary_key_in_series.py +++ b/frappe/patches/v12_0/set_primary_key_in_series.py @@ -1,21 +1,24 @@ import frappe def execute(): - #if current = 0, simply delete the key as it'll be recreated on first entry - frappe.db.sql('delete from `tabSeries` where current = 0') - duplicate_keys = frappe.db.sql(''' - SELECT name, max(current) as current - from - `tabSeries` - group by - name - having count(name) > 1 - ''', as_dict=True) - for row in duplicate_keys: - frappe.db.sql('delete from `tabSeries` where name = %(key)s', { - 'key': row.name - }) - if row.current: - frappe.db.sql('insert into `tabSeries`(`name`, `current`) values (%(name)s, %(current)s)', row) - frappe.db.commit() - frappe.db.sql('ALTER table `tabSeries` ADD PRIMARY KEY IF NOT EXISTS (name)') + #if current = 0, simply delete the key as it'll be recreated on first entry + frappe.db.delete("Series", {"current": 0}) + + duplicate_keys = frappe.db.sql(''' + SELECT name, max(current) as current + from + `tabSeries` + group by + name + having count(name) > 1 + ''', as_dict=True) + + for row in duplicate_keys: + frappe.db.delete("Series", { + "name": row.name + }) + if row.current: + frappe.db.sql('insert into `tabSeries`(`name`, `current`) values (%(name)s, %(current)s)', row) + frappe.db.commit() + + frappe.db.sql('ALTER table `tabSeries` ADD PRIMARY KEY IF NOT EXISTS (name)') diff --git a/frappe/patches/v12_0/setup_comments_from_communications.py b/frappe/patches/v12_0/setup_comments_from_communications.py index 039ceeff35..11e02965f1 100644 --- a/frappe/patches/v12_0/setup_comments_from_communications.py +++ b/frappe/patches/v12_0/setup_comments_from_communications.py @@ -29,4 +29,6 @@ def execute(): frappe.db.auto_commit_on_many_writes = False # clean up - frappe.db.sql("delete from `tabCommunication` where communication_type = 'Comment'") + frappe.db.delete("Communication", { + "communication_type": "Comment" + }) diff --git a/frappe/patches/v13_0/increase_password_length.py b/frappe/patches/v13_0/increase_password_length.py index 1bb1979051..62ca2ed779 100644 --- a/frappe/patches/v13_0/increase_password_length.py +++ b/frappe/patches/v13_0/increase_password_length.py @@ -1,7 +1,4 @@ import frappe def execute(): - frappe.db.multisql({ - "mariadb": "ALTER TABLE `__Auth` MODIFY `password` TEXT NOT NULL", - "postgres": 'ALTER TABLE "__Auth" ALTER COLUMN "password" TYPE TEXT' - }) + frappe.db.change_column_type(table="__Auth", column="password", type="TEXT") diff --git a/frappe/patches/v13_0/remove_twilio_settings.py b/frappe/patches/v13_0/remove_twilio_settings.py index 363cbdd4b6..7efaf876e2 100644 --- a/frappe/patches/v13_0/remove_twilio_settings.py +++ b/frappe/patches/v13_0/remove_twilio_settings.py @@ -12,7 +12,9 @@ def execute(): frappe.delete_doc_if_exists('DocType', 'Twilio Number Group') if twilio_settings_doctype_in_integrations(): frappe.delete_doc_if_exists('DocType', 'Twilio Settings') - frappe.db.sql("delete from `tabSingles` where `doctype`=%s", 'Twilio Settings') + frappe.db.delete("Singles", { + "doctype": "Twilio Settings" + }) def twilio_settings_doctype_in_integrations() -> bool: """Check Twilio Settings doctype exists in integrations module or not. diff --git a/frappe/patches/v13_0/rename_cancelled_docs.py b/frappe/patches/v13_0/rename_cancelled_docs.py deleted file mode 100644 index 2e99a6f3cd..0000000000 --- a/frappe/patches/v13_0/rename_cancelled_docs.py +++ /dev/null @@ -1,27 +0,0 @@ -import frappe -from frappe.model.naming import NameParser -from frappe.model.rename_doc import rename_doc - -def execute(): - """Rename already cancelled documents by adding `CAN-X` postfix instead of `-X`. - """ - for doctype in frappe.db.get_all('DocType'): - doctype = frappe.get_doc('DocType', doctype.name) - if doctype.is_submittable and frappe.db.table_exists(doctype.name): - cancelled_docs = frappe.db.get_all(doctype.name, ['amended_from', 'name'], {'docstatus':2}) - - for doc in cancelled_docs: - if '-CAN-' in doc.name: - continue - - current_name = doc.name - - if getattr(doc, "amended_from", None): - orig_name, counter = NameParser.parse_docname(doc.name) - else: - orig_name, counter = doc.name, 0 - new_name = f'{orig_name}-CAN-{counter or 0}' - - print(f"Renaming {doctype.name} record from {current_name} to {new_name}") - rename_doc(doctype.name, current_name, new_name, ignore_permissions=True, show_alert=False) - frappe.db.commit() diff --git a/frappe/patches/v14_0/__init__.py b/frappe/patches/v14_0/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/frappe/patches/v14_0/drop_data_import_legacy.py b/frappe/patches/v14_0/drop_data_import_legacy.py new file mode 100644 index 0000000000..2037930c9f --- /dev/null +++ b/frappe/patches/v14_0/drop_data_import_legacy.py @@ -0,0 +1,22 @@ +import frappe +import click + + +def execute(): + doctype = "Data Import Legacy" + table = frappe.utils.get_table_name(doctype) + + # delete the doctype record to avoid broken links + frappe.db.delete("DocType", {"name": doctype}) + + # leaving table in database for manual cleanup + click.secho( + f"`{doctype}` has been deprecated. The DocType is deleted, but the data still" + " exists on the database. If this data is worth recovering, you may export it" + f" using\n\n\tbench --site {frappe.local.site} backup -i '{doctype}'\n\nAfter" + " this, the table will continue to persist in the database, until you choose" + " to remove it yourself. If you want to drop the table, you may run\n\n\tbench" + f" --site {frappe.local.site} execute frappe.db.sql --args \"('DROP TABLE IF" + f" EXISTS `{table}`', )\"\n", + fg="yellow", + ) diff --git a/frappe/permissions.py b/frappe/permissions.py index 07b4a2e68f..33aef4ab41 100644 --- a/frappe/permissions.py +++ b/frappe/permissions.py @@ -7,9 +7,11 @@ import frappe.share from frappe import _, msgprint from frappe.utils import cint + rights = ("select", "read", "write", "create", "delete", "submit", "cancel", "amend", "print", "email", "report", "import", "export", "set_user_permissions", "share") + def check_admin_or_system_manager(user=None): if not user: user = frappe.session.user @@ -516,8 +518,7 @@ def reset_perms(doctype): """Reset permissions for given doctype.""" from frappe.desk.notifications import delete_notification_count_for delete_notification_count_for(doctype) - - frappe.db.sql("""delete from `tabCustom DocPerm` where parent=%s""", doctype) + frappe.db.delete("Custom DocPerm", {"parent": doctype}) def get_linked_doctypes(dt): return list(set([dt] + [d.options for d in diff --git a/frappe/public/html/print_template.html b/frappe/public/html/print_template.html index f63a20377f..e2ff9c9c76 100644 --- a/frappe/public/html/print_template.html +++ b/frappe/public/html/print_template.html @@ -7,7 +7,7 @@
| # | {% for col in columns %} {% if col.name && col._id !== "_check" %}{% for row in data %} |
|---|---|
| + {{ row._index + 1 }} + | {% for col in columns %} {% if col.name && col._id !== "_check" %} diff --git a/frappe/query_builder/__init__.py b/frappe/query_builder/__init__.py new file mode 100644 index 0000000000..798c34b6cc --- /dev/null +++ b/frappe/query_builder/__init__.py @@ -0,0 +1 @@ +from frappe.query_builder.utils import get_query_builder diff --git a/frappe/query_builder/builder.py b/frappe/query_builder/builder.py new file mode 100644 index 0000000000..da1533fb1a --- /dev/null +++ b/frappe/query_builder/builder.py @@ -0,0 +1,55 @@ +from pypika import MySQLQuery, Order, PostgreSQLQuery, terms +from pypika.queries import Schema, Table +from frappe.utils import get_table_name + + +class Base: + terms = terms + desc = Order.desc + Schema = Schema + + @staticmethod + def Table(table_name: str, *args, **kwargs) -> Table: + table_name = get_table_name(table_name) + return Table(table_name, *args, **kwargs) + + +class MariaDB(Base, MySQLQuery): + Field = terms.Field + + @classmethod + def from_(cls, table, *args, **kwargs): + if isinstance(table, str): + table = cls.Table(table) + return super().from_(table, *args, **kwargs) + + +class Postgres(Base, PostgreSQLQuery): + field_translation = {"table_name": "relname", "table_rows": "n_tup_ins"} + schema_translation = {"tables": "pg_stat_all_tables"} + # TODO: Find a better way to do this + # These are interdependent query changes that need fixing. These + # translations happen in the same query. But there is no check to see if + # the Fields are changed only when a particular `information_schema` schema + # is used. Replacing them is not straightforward because the "from_" + # function can not see the arguments passed to the "select" function as + # they are two different objects. The quick fix used here is to replace the + # Field names in the "Field" function. + + @classmethod + def Field(cls, field_name, *args, **kwargs): + if field_name in cls.field_translation: + field_name = cls.field_translation[field_name] + return terms.Field(field_name, *args, **kwargs) + + @classmethod + def from_(cls, table, *args, **kwargs): + if isinstance(table, Table): + if table._schema: + if table._schema._name == "information_schema": + table = cls.schema_translation[table._table_name] + + elif isinstance(table, str): + table = cls.Table(table) + + return super().from_(table, *args, **kwargs) diff --git a/frappe/query_builder/custom.py b/frappe/query_builder/custom.py new file mode 100644 index 0000000000..5aaed463d9 --- /dev/null +++ b/frappe/query_builder/custom.py @@ -0,0 +1,83 @@ +from typing import Optional + +from pypika.functions import DistinctOptionFunction +from pypika.utils import builder + +import frappe + + +class GROUP_CONCAT(DistinctOptionFunction): + def __init__(self, column: str, alias: Optional[str] = None): + """[ Implements the group concat function read more about it at https://www.geeksforgeeks.org/mysql-group_concat-function ] + Args: + column (str): [ name of the column you want to concat] + alias (Optional[str], optional): [ is this an alias? ]. Defaults to None. + """ + super(GROUP_CONCAT, self).__init__("GROUP_CONCAT", column, alias=alias) + + +class STRING_AGG(DistinctOptionFunction): + def __init__(self, column: str, separator: str = ",", alias: Optional[str] = None): + """[ Implements the group concat function read more about it at https://docs.microsoft.com/en-us/sql/t-sql/functions/string-agg-transact-sql?view=sql-server-ver15 ] + + Args: + column (str): [ name of the column you want to concat ] + separator (str, optional): [separator to be used]. Defaults to ",". + alias (Optional[str], optional): [description]. Defaults to None. + """ + super(STRING_AGG, self).__init__("STRING_AGG", column, separator, alias=alias) + + +class MATCH(DistinctOptionFunction): + def __init__(self, column: str, *args, **kwargs): + """[ Implementation of Match Against read more about it https://dev.mysql.com/doc/refman/8.0/en/fulltext-search.html#function_match ] + + Args: + column (str):[ column to search in ] + """ + alias = kwargs.get("alias") + super(MATCH, self).__init__(" MATCH", column, *args, alias=alias) + self._Against = False + + def get_function_sql(self, **kwargs): + s = super(DistinctOptionFunction, self).get_function_sql(**kwargs) + + if self._Against: + return f"{s} AGAINST ({frappe.db.escape(f'+{self._Against}*')} IN BOOLEAN MODE)" + return s + + @builder + def Against(self, text: str): + """[ Text that has to be searched against ] + + Args: + text (str): [ the text string that we match it against ] + """ + self._Against = text + + +class TO_TSVECTOR(DistinctOptionFunction): + def __init__(self, column: str, *args, **kwargs): + """[ Implementation of TO_TSVECTOR read more about it https://www.postgresql.org/docs/9.1/textsearch-controls.html] + + Args: + column (str): [ column to search in ] + """ + alias = kwargs.get("alias") + super(TO_TSVECTOR, self).__init__("TO_TSVECTOR", column, *args, alias=alias) + self._PLAINTO_TSQUERY = False + + def get_function_sql(self, **kwargs): + s = super(DistinctOptionFunction, self).get_function_sql(**kwargs) + if self._PLAINTO_TSQUERY: + return f"{s} @@ PLAINTO_TSQUERY({frappe.db.escape(self._PLAINTO_TSQUERY)})" + return s + + @builder + def Against(self, text: str): + """[ Text that has to be searched against ] + + Args: + text (str): [ the text string that we match it against ] + """ + self._PLAINTO_TSQUERY = text diff --git a/frappe/query_builder/functions.py b/frappe/query_builder/functions.py new file mode 100644 index 0000000000..5ccb266945 --- /dev/null +++ b/frappe/query_builder/functions.py @@ -0,0 +1,17 @@ +from pypika.functions import * +from frappe.query_builder.utils import ImportMapper, db_type_is +from frappe.query_builder.custom import GROUP_CONCAT, STRING_AGG, MATCH, TO_TSVECTOR + +GroupConcat = ImportMapper( + { + db_type_is.MARIADB: GROUP_CONCAT, + db_type_is.POSTGRES: STRING_AGG + } +) + +Match = ImportMapper( + { + db_type_is.MARIADB: MATCH, + db_type_is.POSTGRES: TO_TSVECTOR + } +) diff --git a/frappe/query_builder/utils.py b/frappe/query_builder/utils.py new file mode 100644 index 0000000000..b52a3606e8 --- /dev/null +++ b/frappe/query_builder/utils.py @@ -0,0 +1,34 @@ +from enum import Enum +from typing import Any, Callable, Dict + +from pypika import Query + +import frappe +from .builder import MariaDB, Postgres + + +class db_type_is(Enum): + MARIADB = "mariadb" + POSTGRES = "postgres" + +class ImportMapper: + def __init__(self, func_map: Dict[db_type_is, Callable]) -> None: + self.func_map = func_map + + def __call__(self, *args: Any, **kwds: Any) -> Callable: + db = db_type_is(frappe.conf.db_type or "mariadb") + return self.func_map[db](*args, **kwds) + + +def get_query_builder(type_of_db: str) -> Query: + """[return the query builder object] + + Args: + type_of_db (str): [string value of the db used] + + Returns: + Query: [Query object] + """ + db = db_type_is(type_of_db) + picks = {db_type_is.MARIADB: MariaDB, db_type_is.POSTGRES: Postgres} + return picks[db] diff --git a/frappe/search/website_search.py b/frappe/search/website_search.py index 49bdade936..70962e8172 100644 --- a/frappe/search/website_search.py +++ b/frappe/search/website_search.py @@ -35,10 +35,12 @@ class WebsiteSearch(FullTextSearch): if getattr(self, "_items_to_index", False): return self._items_to_index - routes = get_static_pages_from_all_apps() + slugs_with_web_view() - self._items_to_index = [] + + routes = get_static_pages_from_all_apps() + slugs_with_web_view(self._items_to_index) + + for i, route in enumerate(routes): update_progress_bar("Retrieving Routes", i, len(routes)) self._items_to_index += [self.get_document_to_index(route)] @@ -85,16 +87,23 @@ class WebsiteSearch(FullTextSearch): ) -def slugs_with_web_view(): +def slugs_with_web_view(_items_to_index): all_routes = [] filters = { "has_web_view": 1, "allow_guest_to_view": 1, "index_web_pages_for_search": 1} - fields = ["name", "is_published_field"] + fields = ["name", "is_published_field", 'website_search_field'] doctype_with_web_views = frappe.get_all("DocType", filters=filters, fields=fields) for doctype in doctype_with_web_views: if doctype.is_published_field: - routes = frappe.get_all(doctype.name, filters={doctype.is_published_field: 1}, fields="route") - all_routes += [route.route for route in routes] + docs = frappe.get_all(doctype.name, filters={doctype.is_published_field: 1}, fields=["route", doctype.website_search_field, 'title']) + if doctype.website_search_field: + for doc in docs: + content = frappe.utils.md_to_html(getattr(doc, doctype.website_search_field)) + soup = BeautifulSoup(content, "html.parser") + text_content = soup.text if soup else "" + _items_to_index += [frappe._dict(title=doc.title, content=text_content, path=doc.route)] + else: + all_routes += [route.route for route in docs] return all_routes diff --git a/frappe/sessions.py b/frappe/sessions.py index 0b469616b8..4f769ea88f 100644 --- a/frappe/sessions.py +++ b/frappe/sessions.py @@ -84,7 +84,7 @@ def delete_session(sid=None, user=None, reason="Session Expired"): if user_details: user = user_details[0].get("user") logout_feed(user, reason) - frappe.db.sql("""delete from tabSessions where sid=%s""", sid) + frappe.db.delete("Sessions", {"sid": sid}) frappe.db.commit() def clear_all_sessions(reason=None): diff --git a/frappe/test_runner.py b/frappe/test_runner.py index 0c30fbbd00..8112362f34 100644 --- a/frappe/test_runner.py +++ b/frappe/test_runner.py @@ -56,6 +56,7 @@ def main(app=None, module=None, doctype=None, verbose=False, tests=(), frappe.clear_cache() frappe.utils.scheduler.disable_scheduler() set_test_email_config() + frappe.conf.update({'bench_id': 'test_bench', 'use_rq_auth': False}) if not frappe.flags.skip_before_tests: if verbose: diff --git a/frappe/tests/test_background_jobs.py b/frappe/tests/test_background_jobs.py index 88783f14f1..188f3e166f 100644 --- a/frappe/tests/test_background_jobs.py +++ b/frappe/tests/test_background_jobs.py @@ -4,7 +4,7 @@ from rq import Queue import frappe from frappe.core.page.background_jobs.background_jobs import remove_failed_jobs -from frappe.utils.background_jobs import get_redis_conn +from frappe.utils.background_jobs import get_redis_conn, generate_qname import time @@ -17,14 +17,14 @@ class TestBackgroundJobs(unittest.TestCase): queues = Queue.all(conn) for queue in queues: - if queue.name == "short": + if queue.name == generate_qname("short"): fail_registry = queue.failed_job_registry self.assertGreater(fail_registry.count, 0) remove_failed_jobs() for queue in queues: - if queue.name == "short": + if queue.name == generate_qname("short"): fail_registry = queue.failed_job_registry self.assertEqual(fail_registry.count, 0) diff --git a/frappe/tests/test_commands.py b/frappe/tests/test_commands.py index 54103f0151..f687f70228 100644 --- a/frappe/tests/test_commands.py +++ b/frappe/tests/test_commands.py @@ -433,6 +433,6 @@ class TestCommands(BaseTestCommands): for output in ["legacy", "plain", "table", "json"]: self.execute(f"bench version -f {output}") self.assertEqual(self.returncode, 0) - + self.execute("bench version -f invalid") self.assertEqual(self.returncode, 2) diff --git a/frappe/tests/test_db.py b/frappe/tests/test_db.py index 04c9a525b1..044ce455d9 100644 --- a/frappe/tests/test_db.py +++ b/frappe/tests/test_db.py @@ -12,6 +12,8 @@ from frappe.custom.doctype.custom_field.custom_field import create_custom_field from frappe.utils import random_string from frappe.utils.testutils import clear_custom_fields +from .test_query_builder import run_only_if, db_type_is + class TestDB(unittest.TestCase): def test_get_value(self): @@ -146,7 +148,7 @@ class TestDB(unittest.TestCase): # Create documents under that doctype and query them via ORM for _ in range(10): - docfields = { key.lower(): random_string(10) for key in fields } + docfields = {key.lower(): random_string(10) for key in fields} doc = frappe.get_doc({"doctype": test_doctype, "description": random_string(20), **docfields}) doc.insert() created_docs.append(doc.name) @@ -189,3 +191,98 @@ class TestDB(unittest.TestCase): for doc in created_docs: frappe.delete_doc(test_doctype, doc) clear_custom_fields(test_doctype) + +@run_only_if(db_type_is.MARIADB) +class TestDDLCommandsMaria(unittest.TestCase): + test_table_name = "TestNotes" + + def setUp(self) -> None: + frappe.db.commit() + frappe.db.sql( + f""" + CREATE TABLE `tab{self.test_table_name}` (`id` INT NULL,PRIMARY KEY (`id`)); + """ + ) + + def tearDown(self) -> None: + frappe.db.sql(f"DROP TABLE tab{self.test_table_name};") + self.test_table_name = "TestNotes" + + def test_rename(self) -> None: + new_table_name = f"{self.test_table_name}_new" + frappe.db.rename_table(self.test_table_name, new_table_name) + check_exists = frappe.db.sql( + f""" + SELECT * FROM INFORMATION_SCHEMA.TABLES + WHERE TABLE_NAME = N'tab{new_table_name}'; + """ + ) + self.assertGreater(len(check_exists), 0) + self.assertIn(f"tab{new_table_name}", check_exists[0]) + + # * so this table is deleted after the rename + self.test_table_name = new_table_name + + def test_describe(self) -> None: + self.assertEqual( + (("id", "int(11)", "NO", "PRI", None, ""),), + frappe.db.describe(self.test_table_name), + ) + + def test_change_type(self) -> None: + frappe.db.change_column_type("TestNotes", "id", "varchar(255)") + test_table_description = frappe.db.sql(f"DESC tab{self.test_table_name};") + self.assertGreater(len(test_table_description), 0) + self.assertIn("varchar(255)", test_table_description[0]) + + +@run_only_if(db_type_is.POSTGRES) +class TestDDLCommandsPost(unittest.TestCase): + test_table_name = "TestNotes" + + def setUp(self) -> None: + frappe.db.sql( + f""" + CREATE TABLE "tab{self.test_table_name}" ("id" INT NULL,PRIMARY KEY ("id")) + """ + ) + + def tearDown(self) -> None: + frappe.db.sql(f'DROP TABLE "tab{self.test_table_name}"') + self.test_table_name = "TestNotes" + + def test_rename(self) -> None: + new_table_name = f"{self.test_table_name}_new" + frappe.db.rename_table(self.test_table_name, new_table_name) + check_exists = frappe.db.sql( + f""" + SELECT EXISTS ( + SELECT FROM information_schema.tables + WHERE table_name = 'tab{new_table_name}' + ); + """ + ) + self.assertTrue(check_exists[0][0]) + + # * so this table is deleted after the rename + self.test_table_name = new_table_name + + def test_describe(self) -> None: + self.assertEqual([("id",)], frappe.db.describe(self.test_table_name)) + + def test_change_type(self) -> None: + frappe.db.change_column_type(self.test_table_name, "id", "varchar(255)") + check_change = frappe.db.sql( + f""" + SELECT + table_name, + column_name, + data_type + FROM + information_schema.columns + WHERE + table_name = 'tab{self.test_table_name}' + """ + ) + self.assertGreater(len(check_change), 0) + self.assertIn("character varying", check_change[0]) diff --git a/frappe/tests/test_naming.py b/frappe/tests/test_naming.py index 78071a4120..557993882f 100644 --- a/frappe/tests/test_naming.py +++ b/frappe/tests/test_naming.py @@ -116,37 +116,3 @@ class TestNaming(unittest.TestCase): self.assertEqual(current_index.get('current'), 2) frappe.db.sql("""delete from `tabSeries` where name = %s""", series) - - def test_naming_for_cancelled_and_amended_doc(self): - submittable_doctype = frappe.get_doc({ - "doctype": "DocType", - "module": "Core", - "custom": 1, - "is_submittable": 1, - "permissions": [{ - "role": "System Manager", - "read": 1 - }], - "name": 'Submittable Doctype' - }).insert(ignore_if_duplicate=True) - - doc = frappe.new_doc('Submittable Doctype') - doc.save() - original_name = doc.name - - doc.submit() - doc.cancel() - cancelled_name = doc.name - self.assertEqual(cancelled_name, "{}-CAN-0".format(original_name)) - - amended_doc = frappe.copy_doc(doc) - amended_doc.docstatus = 0 - amended_doc.amended_from = doc.name - amended_doc.save() - self.assertEqual(amended_doc.name, original_name) - - amended_doc.submit() - amended_doc.cancel() - self.assertEqual(amended_doc.name, "{}-CAN-1".format(original_name)) - - submittable_doctype.delete() diff --git a/frappe/tests/test_query_builder.py b/frappe/tests/test_query_builder.py new file mode 100644 index 0000000000..d155dd95db --- /dev/null +++ b/frappe/tests/test_query_builder.py @@ -0,0 +1,74 @@ +import unittest +from typing import Callable + +import frappe +from frappe.query_builder.functions import GroupConcat, Match +from frappe.query_builder.utils import db_type_is + + +def run_only_if(dbtype: db_type_is) -> Callable: + return unittest.skipIf( + db_type_is(frappe.conf.db_type) != dbtype, f"Only runs for {dbtype.value}" + ) + + +@run_only_if(db_type_is.MARIADB) +class TestCustomFunctionsMariaDB(unittest.TestCase): + def test_concat(self): + self.assertEqual("GROUP_CONCAT('Notes')", GroupConcat("Notes").get_sql()) + + def test_match(self): + query = Match("Notes").Against("text") + self.assertEqual( + " MATCH('Notes') AGAINST ('+text*' IN BOOLEAN MODE)", query.get_sql() + ) + + +@run_only_if(db_type_is.POSTGRES) +class TestCustomFunctionsPostgres(unittest.TestCase): + def test_concat(self): + self.assertEqual("STRING_AGG('Notes',',')", GroupConcat("Notes").get_sql()) + + def test_match(self): + query = Match("Notes").Against("text") + self.assertEqual( + "TO_TSVECTOR('Notes') @@ PLAINTO_TSQUERY('text')", query.get_sql() + ) + + +class TestBuilderBase(object): + def test_adding_tabs(self): + self.assertEqual("tabNotes", frappe.qb.Table("Notes").get_sql()) + self.assertEqual("__Auth", frappe.qb.Table("__Auth").get_sql()) + + +@run_only_if(db_type_is.MARIADB) +class TestBuilderMaria(unittest.TestCase, TestBuilderBase): + def test_adding_tabs_in_from(self): + self.assertEqual( + "SELECT * FROM `tabNotes`", frappe.qb.from_("Notes").select("*").get_sql() + ) + self.assertEqual( + "SELECT * FROM `__Auth`", frappe.qb.from_("__Auth").select("*").get_sql() + ) + + +@run_only_if(db_type_is.POSTGRES) +class TestBuilderPostgres(unittest.TestCase, TestBuilderBase): + def test_adding_tabs_in_from(self): + self.assertEqual( + 'SELECT * FROM "tabNotes"', frappe.qb.from_("Notes").select("*").get_sql() + ) + self.assertEqual( + 'SELECT * FROM "__Auth"', frappe.qb.from_("__Auth").select("*").get_sql() + ) + + def test_replace_tables(self): + info_schema = frappe.qb.Schema("information_schema") + self.assertEqual( + 'SELECT * FROM "pg_stat_all_tables"', + frappe.qb.from_(info_schema.tables).select("*").get_sql(), + ) + + def test_replace_fields_post(self): + self.assertEqual("relname", frappe.qb.Field("table_name").get_sql()) diff --git a/frappe/tests/test_redis.py b/frappe/tests/test_redis.py new file mode 100644 index 0000000000..72af1ac699 --- /dev/null +++ b/frappe/tests/test_redis.py @@ -0,0 +1,70 @@ +import unittest +import functools + +import redis + +import frappe +from frappe.utils import get_bench_id +from frappe.utils.rq import RedisQueue +from frappe.utils.background_jobs import get_redis_conn + +def version_tuple(version): + return tuple(map(int, (version.split(".")))) + +def skip_if_redis_version_lt(version): + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + conn = get_redis_conn() + redis_version = conn.execute_command('info')['redis_version'] + if version_tuple(redis_version) < version_tuple(version): + return + return func(*args, **kwargs) + return wrapper + return decorator + +class TestRedisAuth(unittest.TestCase): + @skip_if_redis_version_lt('6.0') + def test_rq_gen_acllist(self): + """Make sure that ACL list is genrated + """ + acl_list = RedisQueue.gen_acl_list() + self.assertEqual(acl_list[1]['bench'][0], get_bench_id()) + + @skip_if_redis_version_lt('6.0') + def test_adding_redis_user(self): + acl_list = RedisQueue.gen_acl_list() + username, password = acl_list[1]['bench'] + conn = get_redis_conn() + + conn.acl_deluser(username) + _ = RedisQueue(conn).add_user(username, password) + self.assertTrue(conn.acl_getuser(username)) + conn.acl_deluser(username) + + @skip_if_redis_version_lt('6.0') + def test_rq_namespace(self): + """Make sure that user can access only their respective namespace. + """ + # Current bench ID + bench_id = frappe.conf.get('bench_id') + conn = get_redis_conn() + conn.set('rq:queue:test_bench1:abc', 'value') + conn.set(f'rq:queue:{bench_id}:abc', 'value') + + # Create new Redis Queue user + tmp_bench_id = 'test_bench1' + username, password = tmp_bench_id, 'password1' + conn.acl_deluser(username) + frappe.conf.update({'bench_id': tmp_bench_id}) + _ = RedisQueue(conn).add_user(username, password) + test_bench1_conn = RedisQueue.get_connection(username, password) + + self.assertEqual(test_bench1_conn.get('rq:queue:test_bench1:abc'), b'value') + + # User should not be able to access queues apart from their bench queues + with self.assertRaises(redis.exceptions.NoPermissionError): + test_bench1_conn.get(f'rq:queue:{bench_id}:abc') + + frappe.conf.update({'bench_id': bench_id}) + conn.acl_deluser(username) diff --git a/frappe/utils/__init__.py b/frappe/utils/__init__.py index af9d5de1ee..b97585aa04 100644 --- a/frappe/utils/__init__.py +++ b/frappe/utils/__init__.py @@ -383,6 +383,12 @@ def get_files_path(*path, **kwargs): def get_bench_path(): return os.path.realpath(os.path.join(os.path.dirname(frappe.__file__), '..', '..', '..')) +def get_bench_id(): + return frappe.get_conf().get('bench_id', get_bench_path().strip('/').replace('/', '-')) + +def get_site_id(site=None): + return f"{site or frappe.local.site}@{get_bench_id()}" + def get_backups_path(): return get_site_path("private", "backups") @@ -843,3 +849,6 @@ def groupby_metric(iterable: typing.Dict[str, list], key: str): for item in items: records.setdefault(item[key], {}).setdefault(category, []).append(item) return records + +def get_table_name(table_name: str) -> str: + return f"tab{table_name}" if not table_name.startswith("__") else table_name diff --git a/frappe/utils/background_jobs.py b/frappe/utils/background_jobs.py index 8456835ca7..f0bd06aff4 100755 --- a/frappe/utils/background_jobs.py +++ b/frappe/utils/background_jobs.py @@ -1,13 +1,21 @@ +import os +import socket +import time +from uuid import uuid4 +from collections import defaultdict + + import redis +from typing import List from rq import Connection, Queue, Worker from rq.logutils import setup_loghandlers -from frappe.utils import cstr -from collections import defaultdict + import frappe -import os, socket, time from frappe import _ -from uuid import uuid4 import frappe.monitor +from frappe.utils import cstr, get_bench_id +from frappe.utils.rq import RedisQueue +from frappe.utils.commands import log default_timeout = 300 @@ -131,21 +139,22 @@ def execute_job(site, method, event, job_name, kwargs, user=None, is_async=True, if is_async: frappe.destroy() -def start_worker(queue=None, quiet = False): +def start_worker(queue=None, quiet = False, rq_username=None, rq_password=None): '''Wrapper to start rq worker. Connects to redis and monitors these queues.''' with frappe.init_site(): # empty init is required to get redis_queue from common_site_config.json - redis_connection = get_redis_conn() + redis_connection = get_redis_conn(username=rq_username, password=rq_password) + queues = get_queue_list(queue, build_queue_name=True) + queue_name = queue and generate_qname(queue) if os.environ.get('CI'): setup_loghandlers('ERROR') with Connection(redis_connection): - queues = get_queue_list(queue) logging_level = "INFO" if quiet: logging_level = "WARNING" - Worker(queues, name=get_worker_name(queue)).work(logging_level = logging_level) + Worker(queues, name=get_worker_name(queue_name)).work(logging_level = logging_level) def get_worker_name(queue): '''When limiting worker to a specific queue, also append queue name to default worker name''' @@ -186,7 +195,7 @@ def get_jobs(site=None, queue=None, key='method'): return jobs_per_site -def get_queue_list(queue_list=None): +def get_queue_list(queue_list=None, build_queue_name=False): '''Defines possible queues. Also wraps a given queue in a list after validating.''' default_queue_list = list(queue_timeout) if queue_list: @@ -195,11 +204,9 @@ def get_queue_list(queue_list=None): for queue in queue_list: validate_queue(queue, default_queue_list) - - return queue_list - else: - return default_queue_list + queue_list = default_queue_list + return [generate_qname(qtype) for qtype in queue_list] if build_queue_name else queue_list def get_workers(queue): '''Returns a list of Worker objects tied to a queue object''' @@ -215,10 +222,10 @@ def get_running_jobs_in_queue(queue): jobs.append(current_job) return jobs -def get_queue(queue, is_async=True): +def get_queue(qtype, is_async=True): '''Returns a Queue object tied to a redis connection''' - validate_queue(queue) - return Queue(queue, connection=get_redis_conn(), is_async=is_async) + validate_queue(qtype) + return Queue(generate_qname(qtype), connection=get_redis_conn(), is_async=is_async) def validate_queue(queue, default_queue_list=None): if not default_queue_list: @@ -227,7 +234,7 @@ def validate_queue(queue, default_queue_list=None): if queue not in default_queue_list: frappe.throw(_("Queue should be one of {0}").format(', '.join(default_queue_list))) -def get_redis_conn(): +def get_redis_conn(username=None, password=None): if not hasattr(frappe.local, 'conf'): raise Exception('You need to call frappe.init') @@ -236,11 +243,50 @@ def get_redis_conn(): global redis_connection - if not redis_connection: - redis_connection = redis.from_url(frappe.local.conf.redis_queue) + cred = frappe._dict() + if frappe.conf.get('use_rq_auth'): + if username: + cred['username'] = username + cred['password'] = password + else: + cred['username'] = frappe.get_site_config().rq_username or get_bench_id() + cred['password'] = frappe.get_site_config().rq_password + + elif os.environ.get('RQ_ADMIN_PASWORD'): + cred['username'] = 'default' + cred['password'] = os.environ.get('RQ_ADMIN_PASWORD') + try: + redis_connection = RedisQueue.get_connection(**cred) + except (redis.exceptions.AuthenticationError, redis.exceptions.ResponseError): + log(f'Wrong credentials used for {cred.username or "default user"}. ' + 'You can reset credentials using `bench create-rq-users` CLI and restart the server', + colour='red') + raise + except Exception: + log(f'Please make sure that Redis Queue runs @ {frappe.get_conf().redis_queue}', colour='red') + raise return redis_connection +def get_queues() -> List[Queue]: + """Get all the queues linked to the current bench. + """ + queues = Queue.all(connection=get_redis_conn()) + return [q for q in queues if is_queue_accessible(q)] + +def generate_qname(qtype: str) -> str: + """Generate qname by combining bench ID and queue type. + + qnames are useful to define namespaces of customers. + """ + return f"{get_bench_id()}:{qtype}" + +def is_queue_accessible(qobj: Queue) -> bool: + """Checks whether queue is relate to current bench or not. + """ + accessible_queues = [generate_qname(q) for q in list(queue_timeout)] + return qobj.name in accessible_queues + def enqueue_test_job(): enqueue('frappe.utils.background_jobs.test_job', s=100) diff --git a/frappe/utils/backups.py b/frappe/utils/backups.py index 908be52452..f13710dcfe 100644 --- a/frappe/utils/backups.py +++ b/frappe/utils/backups.py @@ -116,16 +116,16 @@ class BackupGenerator: def setup_backup_tables(self): """Sets self.backup_includes, self.backup_excludes based on passed args""" - existing_doctypes = set([x.name for x in frappe.get_all("DocType")]) + existing_tables = frappe.db.get_tables() def get_tables(doctypes): tables = [] for doctype in doctypes: - if doctype and doctype in existing_doctypes: - if doctype.startswith("tab"): - tables.append(doctype) - else: - tables.append("tab" + doctype) + if not doctype: + continue + table = frappe.utils.get_table_name(doctype) + if table in existing_tables: + tables.append(table) return tables passed_tables = { diff --git a/frappe/utils/error.py b/frappe/utils/error.py index 07e34674fe..05b578d7e8 100644 --- a/frappe/utils/error.py +++ b/frappe/utils/error.py @@ -176,6 +176,7 @@ def collect_error_snapshots(): def clear_old_snapshots(): """Clear snapshots that are older than a month""" + frappe.db.sql("""delete from `tabError Snapshot` where creation < (NOW() - INTERVAL '1' MONTH)""") diff --git a/frappe/utils/global_search.py b/frappe/utils/global_search.py index 8fa2ea474f..072e3a7c62 100644 --- a/frappe/utils/global_search.py +++ b/frappe/utils/global_search.py @@ -23,8 +23,7 @@ def reset(): Deletes all data in __global_search :return: """ - frappe.db.sql('DELETE FROM `__global_search`') - + frappe.db.delete("__global_search") def get_doctypes_with_global_search(with_child_tables=True): """ @@ -146,10 +145,9 @@ def rebuild_for_doctype(doctype): def delete_global_search_records_for_doctype(doctype): - frappe.db.sql('''DELETE - FROM `__global_search` - WHERE doctype = %s''', doctype, as_dict=True) - + frappe.db.delete("__global_search", { + "doctype": doctype + }) def get_selected_fields(meta, global_search_fields): fieldnames = [df.fieldname for df in global_search_fields] @@ -399,12 +397,10 @@ def delete_for_document(doc): been deleted :param doc: Deleted document """ - - frappe.db.sql('''DELETE - FROM `__global_search` - WHERE doctype = %s - AND name = %s''', (doc.doctype, doc.name), as_dict=True) - + frappe.db.delete("__global_search", { + "doctype": doc.doctype, + "name": doc.name + }) @frappe.whitelist() def search(text, start=0, limit=20, doctype=""): @@ -415,51 +411,41 @@ def search(text, start=0, limit=20, doctype=""): :param limit: number of results to return, default 20 :return: Array of result objects """ - from frappe.desk.doctype.global_search_settings.global_search_settings import get_doctypes_for_global_search + from frappe.desk.doctype.global_search_settings.global_search_settings import ( + get_doctypes_for_global_search, + ) + from frappe.query_builder.functions import Match results = [] sorted_results = [] allowed_doctypes = get_doctypes_for_global_search() - for text in set(text.split('&')): + for text in set(text.split("&")): text = text.strip() if not text: continue - conditions = '1=1' - offset = '' - - mariadb_text = frappe.db.escape('+' + text + '*') - - mariadb_fields = '`doctype`, `name`, `content`, MATCH (`content`) AGAINST ({} IN BOOLEAN MODE) AS rank'.format(mariadb_text) - postgres_fields = '`doctype`, `name`, `content`, TO_TSVECTOR("content") @@ PLAINTO_TSQUERY({}) AS rank'.format(frappe.db.escape(text)) - - values = {} + global_search = frappe.qb.Table("__global_search") + rank = Match(global_search.content).Against(text).as_("rank") + query = ( + frappe.qb.from_(global_search) + .select( + global_search.doctype, global_search.name, global_search.content, rank + ) + .orderby("rank", order=frappe.qb.desc) + .limit(limit) + ) if doctype: - conditions = '`doctype` = %(doctype)s' - values['doctype'] = doctype + query = query.where(global_search.doctype == doctype) elif allowed_doctypes: - conditions = '`doctype` IN %(allowed_doctypes)s' - values['allowed_doctypes'] = tuple(allowed_doctypes) + query = query.where(global_search.doctype.isin(allowed_doctypes)) - if int(start) > 0: - offset = 'OFFSET {}'.format(start) + if start > 0: + query = query.offset(start) - common_query = """ - SELECT {fields} - FROM `__global_search` - WHERE {conditions} - ORDER BY rank DESC - LIMIT {limit} - {offset} - """ - - result = frappe.db.multisql({ - 'mariadb': common_query.format(fields=mariadb_fields, conditions=conditions, limit=limit, offset=offset), - 'postgres': common_query.format(fields=postgres_fields, conditions=conditions, limit=limit, offset=offset) - }, values=values, as_dict=True) + result = frappe.db.sql(query, as_dict=True) results.extend(result) @@ -470,7 +456,9 @@ def search(text, start=0, limit=20, doctype=""): try: meta = frappe.get_meta(r.doctype) if meta.image_field: - r.image = frappe.db.get_value(r.doctype, r.name, meta.image_field) + r.image = frappe.db.get_value( + r.doctype, r.name, meta.image_field + ) except Exception: frappe.clear_messages() diff --git a/frappe/utils/install.py b/frappe/utils/install.py index 91d8f04eb4..3d6a2fed97 100644 --- a/frappe/utils/install.py +++ b/frappe/utils/install.py @@ -111,9 +111,9 @@ def before_tests(): # don't run before tests if any other app is installed return - frappe.db.sql("delete from `tabCustom Field`") - frappe.db.sql("delete from `tabEvent`") - frappe.db.commit() + frappe.db.truncate("Custom Field") + frappe.db.truncate("Event") + frappe.clear_cache() # complete setup if missing diff --git a/frappe/utils/nestedset.py b/frappe/utils/nestedset.py index 3c024c40e4..4a65140449 100644 --- a/frappe/utils/nestedset.py +++ b/frappe/utils/nestedset.py @@ -57,7 +57,7 @@ def update_add_node(doc, parent, parent_field): # get the last sibling of the parent if parent: - left, right = frappe.db.sql("select lft, rgt from `tab{0}` where name=%s" + left, right = frappe.db.sql("select lft, rgt from `tab{0}` where name=%s for update" .format(doctype), parent)[0] validate_loop(doc.doctype, doc.name, left, right) else: # root @@ -89,7 +89,7 @@ def update_move_node(doc, parent_field): if parent: new_parent = frappe.db.sql("""select lft, rgt from `tab{0}` - where name = %s""".format(doc.doctype), parent, as_dict=1)[0] + where name = %s for update""".format(doc.doctype), parent, as_dict=1)[0] validate_loop(doc.doctype, doc.name, new_parent.lft, new_parent.rgt) @@ -108,7 +108,7 @@ def update_move_node(doc, parent_field): if parent: new_parent = frappe.db.sql("""select lft, rgt from `tab%s` - where name = %s""" % (doc.doctype, '%s'), parent, as_dict=1)[0] + where name = %s for update""" % (doc.doctype, '%s'), parent, as_dict=1)[0] # set parent lft, rgt diff --git a/frappe/utils/password.py b/frappe/utils/password.py index 428f2e9577..a097c58b31 100644 --- a/frappe/utils/password.py +++ b/frappe/utils/password.py @@ -65,11 +65,11 @@ def set_encrypted_password(doctype, name, pwd, fieldname='password'): def remove_encrypted_password(doctype, name, fieldname='password'): - frappe.db.sql( - 'DELETE FROM `__Auth` WHERE doctype = %s and name = %s and fieldname = %s', - values=[doctype, name, fieldname] - ) - + frappe.db.delete("__Auth", { + "doctype": doctype, + "name": name, + "fieldname": fieldname + }) def check_password(user, pwd, doctype='User', fieldname='password', delete_tracker_cache=True): '''Checks if user and password are correct, else raises frappe.AuthenticationError''' @@ -131,8 +131,10 @@ def update_password(user, pwd, doctype='User', fieldname='password', logout_all_ def delete_all_passwords_for(doctype, name): try: - frappe.db.sql("""delete from `__Auth` where `doctype`=%(doctype)s and `name`=%(name)s""", - { 'doctype': doctype, 'name': name }) + frappe.db.delete("__Auth", { + "doctype": doctype, + "name": name + }) except Exception as e: if not frappe.db.is_missing_column(e): raise diff --git a/frappe/utils/rq.py b/frappe/utils/rq.py new file mode 100644 index 0000000000..b344b0caa5 --- /dev/null +++ b/frappe/utils/rq.py @@ -0,0 +1,83 @@ +import redis + +import frappe +from frappe.utils import get_bench_id, random_string + +class RedisQueue: + def __init__(self, conn): + self.conn = conn + + def add_user(self, username, password=None): + """Create or update the user. + """ + password = password or self.conn.acl_genpass() + user_settings = self.get_new_user_settings(username, password) + is_created = self.conn.acl_setuser(**user_settings) + return frappe._dict(user_settings) if is_created else {} + + @classmethod + def get_connection(cls, username=None, password=None): + rq_url = frappe.local.conf.redis_queue + domain = rq_url.split("redis://", 1)[-1] + url = (username and f"redis://{username}:{password or ''}@{domain}") or rq_url + conn = redis.from_url(url) + conn.ping() + return conn + + @classmethod + def new(cls, username='default', password=None): + return cls(cls.get_connection(username, password)) + + @classmethod + def set_admin_password(cls, cur_password=None, new_password=None, reset_passwords=False): + username = 'default' + conn = cls.get_connection(username, cur_password) + password = '+'+(new_password or conn.acl_genpass()) + conn.acl_setuser( + username=username, enabled=True, reset_passwords=reset_passwords, passwords=password + ) + return password[1:] + + @classmethod + def get_new_user_settings(cls, username, password): + d = {} + d['username'] = username + d['passwords'] = '+'+password + d['reset_keys'] = True + d['enabled'] = True + d['keys'] = cls.get_acl_key_rules() + d['commands'] = cls.get_acl_command_rules() + return d + + @classmethod + def get_acl_key_rules(cls, include_key_prefix=False): + """FIXME: Find better way + """ + rules = ['rq:[^q]*', 'rq:queues', f'rq:queue:{get_bench_id()}:*'] + if include_key_prefix: + return ['~'+pattern for pattern in rules] + return rules + + @classmethod + def get_acl_command_rules(cls): + return ['+@all', '-@admin'] + + @classmethod + def gen_acl_list(cls, set_admin_password=False): + """Generate list of ACL users needed for this branch. + + This list contains default ACL user and the bench ACL user(used by all sites incase of ACL is enabled). + """ + bench_username = get_bench_id() + bench_user_rules = cls.get_acl_key_rules(include_key_prefix=True) + cls.get_acl_command_rules() + bench_user_rule_str = ' '.join(bench_user_rules).strip() + bench_user_password = random_string(20) + + default_username = 'default' + _default_user_password = random_string(20) if set_admin_password else '' + default_user_password = '>'+_default_user_password if _default_user_password else 'nopass' + + return [ + f'user {default_username} on {default_user_password} ~* &* +@all', + f'user {bench_username} on >{bench_user_password} {bench_user_rule_str}' + ], {'bench': (bench_username, bench_user_password), 'default': (default_username, _default_user_password)} diff --git a/frappe/utils/testutils.py b/frappe/utils/testutils.py index c451d090f1..9a2b2da791 100644 --- a/frappe/utils/testutils.py +++ b/frappe/utils/testutils.py @@ -12,5 +12,5 @@ def add_custom_field(doctype, fieldname, fieldtype='Data', options=None): }).insert() def clear_custom_fields(doctype): - frappe.db.sql('delete from `tabCustom Field` where dt=%s', doctype) + frappe.db.delete("Custom Field", {"dt": doctype}) frappe.clear_cache(doctype=doctype) diff --git a/frappe/website/doctype/web_form/web_form.py b/frappe/website/doctype/web_form/web_form.py index 6e4200b54b..32f7e030a6 100644 --- a/frappe/website/doctype/web_form/web_form.py +++ b/frappe/website/doctype/web_form/web_form.py @@ -542,7 +542,7 @@ def get_form_data(doctype, docname=None, web_form_name=None): # For Table fields, server-side processing for meta for field in out.web_form.web_form_fields: if field.fieldtype == "Table": - field.fields = get_in_list_view_fields(field.options) + field.fields = frappe.get_meta(field.options).fields out.update({field.fieldname: field.fields}) if field.fieldtype == "Link": diff --git a/frappe/website/report/website_analytics/website_analytics.py b/frappe/website/report/website_analytics/website_analytics.py index d141972679..6cc0fb7f97 100644 --- a/frappe/website/report/website_analytics/website_analytics.py +++ b/frappe/website/report/website_analytics/website_analytics.py @@ -1,11 +1,14 @@ # Copyright (c) 2013, Frappe Technologies and contributors # For license information, please see license.txt -import frappe from datetime import datetime + +import frappe +from frappe.query_builder.functions import Coalesce, Count from frappe.utils import getdate from frappe.utils.dateutils import get_dates_from_timegrain + def execute(filters=None): return WebsiteAnalytics(filters).run() @@ -56,33 +59,21 @@ class WebsiteAnalytics(object): ] def get_data(self): - pg_query = """ - SELECT - path, - COUNT(*) as count, - COUNT(CASE WHEN CAST(is_unique as Integer) = 1 THEN 1 END) as unique_count - FROM `tabWeb Page View` - WHERE coalesce("tabWeb Page View".creation, '0001-01-01') BETWEEN %s AND %s - GROUP BY path - ORDER BY count desc - """ + WebPageView = frappe.qb.Table("Web Page View") + count_all = Count("*").as_("count") + case = frappe.qb.terms.Case().when(WebPageView.is_unique == "1", "1") + count_is_unique = Count(case).as_("unique_count") - mariadb_query = """ - SELECT - path, - COUNT(*) as count, - COUNT(CASE WHEN is_unique = 1 THEN 1 END) as unique_count - FROM `tabWeb Page View` - WHERE creation BETWEEN %s AND %s - GROUP BY path - ORDER BY count desc - """ - - data = frappe.db.multisql({ - "mariadb": mariadb_query, - "postgres": pg_query - }, (self.filters.from_date, self.filters.to_date)) - return data + query = ( + frappe.qb.from_(WebPageView) + .select("path", count_all, count_is_unique) + .where( + Coalesce(WebPageView.creation, "0001-01-01")[self.filters.from_date:self.filters.to_date] + ) + .groupby(WebPageView.path) + .orderby("count", Order=frappe.qb.desc) + ) + return frappe.db.sql(query) def _get_query_for_mariadb(self): filters_range = self.filters.range diff --git a/frappe/website/utils.py b/frappe/website/utils.py index e218afe8c6..472e86d8f5 100644 --- a/frappe/website/utils.py +++ b/frappe/website/utils.py @@ -488,11 +488,12 @@ def set_content_type(response, data, path): return data def add_preload_headers(response): - from bs4 import BeautifulSoup + from bs4 import BeautifulSoup, SoupStrainer try: preload = [] - soup = BeautifulSoup(response.data, "lxml") + strainer = SoupStrainer(re.compile("script|link")) + soup = BeautifulSoup(response.data, "lxml", parse_only=strainer) for elem in soup.find_all('script', src=re.compile(".*")): preload.append(("script", elem.get("src"))) diff --git a/frappe/workflow/doctype/workflow_action/workflow_action.py b/frappe/workflow/doctype/workflow_action/workflow_action.py index b70ffb2406..5eedc27d9c 100644 --- a/frappe/workflow/doctype/workflow_action/workflow_action.py +++ b/frappe/workflow/doctype/workflow_action/workflow_action.py @@ -133,9 +133,12 @@ def return_link_expired_page(doc, doc_workflow_state): def clear_old_workflow_actions(doc, user=None): user = user if user else frappe.session.user - frappe.db.sql("""DELETE FROM `tabWorkflow Action` - WHERE `reference_doctype`=%s AND `reference_name`=%s AND `user`!=%s AND `status`='Open'""", - (doc.get('doctype'), doc.get('name'), user)) + frappe.db.delete("Workflow Action", { + "reference_doctype": doc.get("doctype"), + "reference_name": doc.get("name"), + "user": ("!=", user), + "status": "Open" + }) def update_completed_workflow_actions(doc, user=None): user = user if user else frappe.session.user @@ -253,11 +256,10 @@ def is_workflow_action_already_created(doc): def clear_workflow_actions(doctype, name): if not (doctype and name): return - - frappe.db.sql('''delete from `tabWorkflow Action` - where reference_doctype=%s and reference_name=%s''', - (doctype, name)) - + frappe.db.delete("Workflow Action", { + "reference_doctype": doctype, + "reference_name": name + }) def get_doc_workflow_state(doc): workflow_name = get_workflow_name(doc.get('doctype')) workflow_state_field = get_workflow_state_field(workflow_name) diff --git a/requirements.txt b/requirements.txt index 0791f01b27..51327953d5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -49,6 +49,7 @@ pyngrok~=5.0.5 pyOpenSSL~=20.0.1 pyotp~=2.6.0 PyPDF2~=1.26.0 +PyPika~=0.48.6 pypng~=0.0.20 PyQRCode~=1.2.1 python-dateutil~=2.8.1