From 4754ab71d1cf3099b66b63bf77a770a16c64736f Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sun, 9 May 2021 19:44:58 +0530 Subject: [PATCH 01/14] perf(minor): remove unnecessary comprehensions - remove several unnecessary comprehensions from functions that accept a generator. - Using `[x for x in iter]` causes a list to be built first then passed to the outer function. - `any` and `all` can take generator instead. This makes memory usage O(1) and actually makes these functions short-circuiting. E.g. if the first condition fails then `all` will immediately return false instead of evaluating all the entries. - `sum`, `min`, `max` => memory usage become O(1) - `list`, `set`, `.join()` => roughly halves memory usage, as list is not required to be built. - lastly, it's two fewer characters to read/think about. --- frappe/automation/doctype/auto_repeat/auto_repeat.py | 2 +- frappe/contacts/address_and_contact.py | 2 +- frappe/contacts/doctype/address/address.py | 2 +- frappe/core/doctype/data_import/importer.py | 12 ++++++------ frappe/core/doctype/data_import_legacy/importer.py | 4 ++-- frappe/core/doctype/doctype/doctype.py | 6 +++--- frappe/core/doctype/domain/domain.py | 4 ++-- frappe/core/doctype/user/user.py | 2 +- frappe/core/doctype/user_type/user_type.py | 2 +- .../custom/doctype/customize_form/customize_form.py | 2 +- frappe/database/database.py | 4 ++-- .../global_search_settings/global_search_settings.py | 4 ++-- frappe/desk/doctype/tag/tag.py | 2 +- frappe/email/doctype/email_account/email_account.py | 6 +++--- frappe/email/doctype/newsletter/test_newsletter.py | 2 +- frappe/email/inbox.py | 2 +- frappe/installer.py | 2 +- .../doctype/ldap_settings/ldap_settings.py | 2 +- frappe/model/__init__.py | 4 ++-- frappe/model/base_document.py | 4 ++-- frappe/model/db_query.py | 10 +++++----- frappe/model/meta.py | 2 +- frappe/model/rename_doc.py | 2 +- frappe/permissions.py | 2 +- frappe/tests/test_commands.py | 8 ++++---- frappe/translate.py | 4 ++-- frappe/utils/__init__.py | 4 ++-- frappe/utils/backups.py | 6 +++--- frappe/utils/bot.py | 8 ++++---- frappe/utils/jinja_globals.py | 4 ++-- frappe/website/doctype/web_form/web_form.py | 2 +- .../doctype/website_slideshow/website_slideshow.py | 2 +- 32 files changed, 62 insertions(+), 62 deletions(-) diff --git a/frappe/automation/doctype/auto_repeat/auto_repeat.py b/frappe/automation/doctype/auto_repeat/auto_repeat.py index bf05baf5b6..37285ad571 100644 --- a/frappe/automation/doctype/auto_repeat/auto_repeat.py +++ b/frappe/automation/doctype/auto_repeat/auto_repeat.py @@ -334,7 +334,7 @@ class AutoRepeat(Document): if self.reference_doctype and self.reference_document: res = get_contacts_linking_to(self.reference_doctype, self.reference_document, fields=['email_id']) res += get_contacts_linked_from(self.reference_doctype, self.reference_document, fields=['email_id']) - email_ids = list(set([d.email_id for d in res])) + email_ids = set(d.email_id for d in res) if not email_ids: frappe.msgprint(_('No contacts linked to document'), alert=True) else: diff --git a/frappe/contacts/address_and_contact.py b/frappe/contacts/address_and_contact.py index 3ca9547188..43aa39f678 100644 --- a/frappe/contacts/address_and_contact.py +++ b/frappe/contacts/address_and_contact.py @@ -154,7 +154,7 @@ def filter_dynamic_link_doctypes(doctype, txt, searchfield, start, page_len, fil doctypes = frappe.db.get_all("DocField", filters=filters, fields=["parent"], distinct=True, as_list=True) - doctypes = tuple([d for d in doctypes if re.search(txt+".*", _(d[0]), re.IGNORECASE)]) + doctypes = tuple(d for d in doctypes if re.search(txt+".*", _(d[0]), re.IGNORECASE)) filters.update({ "dt": ("not in", [d[0] for d in doctypes]) diff --git a/frappe/contacts/doctype/address/address.py b/frappe/contacts/doctype/address/address.py index 84b925d50e..a6fd7b1755 100644 --- a/frappe/contacts/doctype/address/address.py +++ b/frappe/contacts/doctype/address/address.py @@ -263,7 +263,7 @@ def address_query(doctype, txt, searchfield, start, page_len, filters): def get_condensed_address(doc): fields = ["address_title", "address_line1", "address_line2", "city", "county", "state", "country"] - return ", ".join([doc.get(d) for d in fields if doc.get(d)]) + return ", ".join(doc.get(d) for d in fields if doc.get(d)) def update_preferred_address(address, field): frappe.db.set_value('Address', address, field, 0) diff --git a/frappe/core/doctype/data_import/importer.py b/frappe/core/doctype/data_import/importer.py index 720fe1dda7..841319dd60 100644 --- a/frappe/core/doctype/data_import/importer.py +++ b/frappe/core/doctype/data_import/importer.py @@ -450,7 +450,7 @@ class ImportFile: for row in data_without_first_row: row_values = row.get_values(parent_column_indexes) # if the row is blank, it's a child row doc - if all([v in INVALID_VALUES for v in row_values]): + if all(v in INVALID_VALUES for v in row_values): rows.append(row) continue # if we encounter a row which has values in parent columns, @@ -607,7 +607,7 @@ class Row: if df.fieldtype == "Select": select_options = get_select_options(df) if select_options and value not in select_options: - options_string = ", ".join([frappe.bold(d) for d in select_options]) + options_string = ", ".join(frappe.bold(d) for d in select_options) msg = _("Value must be one of {0}").format(options_string) self.warnings.append( {"row": self.row_number, "field": df_as_json(df), "message": msg,} @@ -903,7 +903,7 @@ class Column: if self.df.fieldtype == "Link": # find all values that dont exist - values = list(set([cstr(v) for v in self.column_values[1:] if v])) + values = list(set(cstr(v) for v in self.column_values[1:] if v)) exists = [ d.name for d in frappe.db.get_all(self.df.options, filters={"name": ("in", values)}) ] @@ -939,11 +939,11 @@ class Column: elif self.df.fieldtype == "Select": options = get_select_options(self.df) if options: - values = list(set([cstr(v) for v in self.column_values[1:] if v])) + values = list(set(cstr(v) for v in self.column_values[1:] if v)) invalid = list(set(values) - set(options)) if invalid: - valid_values = ", ".join([frappe.bold(o) for o in options]) - invalid_values = ", ".join([frappe.bold(i) for i in invalid]) + valid_values = ", ".join(frappe.bold(o) for o in options) + invalid_values = ", ".join(frappe.bold(i) for i in invalid) self.warnings.append( { "col": self.column_number, diff --git a/frappe/core/doctype/data_import_legacy/importer.py b/frappe/core/doctype/data_import_legacy/importer.py index 35569c7186..b4be57be4f 100644 --- a/frappe/core/doctype/data_import_legacy/importer.py +++ b/frappe/core/doctype/data_import_legacy/importer.py @@ -181,7 +181,7 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False, if d.get("name") and d["name"].startswith('"'): d["name"] = d["name"][1:-1] - if sum([0 if not val else 1 for val in d.values()]): + if sum(0 if not val else 1 for val in d.values()): d['doctype'] = dt if dt == doctype: doc.update(d) @@ -537,6 +537,6 @@ def get_parent_field(doctype, parenttype): def delete_child_rows(rows, doctype): """delete child rows for all parents""" - for p in list(set([r[1] for r in rows])): + for p in list(set(r[1] for r in rows)): if p: frappe.db.sql("""delete from `tab{0}` where parent=%s""".format(doctype), p) diff --git a/frappe/core/doctype/doctype/doctype.py b/frappe/core/doctype/doctype/doctype.py index aff07836c0..582c5d3d89 100644 --- a/frappe/core/doctype/doctype/doctype.py +++ b/frappe/core/doctype/doctype/doctype.py @@ -196,7 +196,7 @@ class DocType(Document): self.flags.update_fields_to_fetch_queries = [] - if set(old_fields_to_fetch) != set([df.fieldname for df in new_meta.get_fields_to_fetch()]): + if set(old_fields_to_fetch) != set(df.fieldname for df in new_meta.get_fields_to_fetch()): for df in new_meta.get_fields_to_fetch(): if df.fieldname not in old_fields_to_fetch: link_fieldname, source_fieldname = df.fetch_from.split('.', 1) @@ -765,7 +765,7 @@ def validate_fields(meta): invalid_fields = ('doctype',) if fieldname in invalid_fields: frappe.throw(_("{0}: Fieldname cannot be one of {1}") - .format(docname, ", ".join([frappe.bold(d) for d in invalid_fields]))) + .format(docname, ", ".join(frappe.bold(d) for d in invalid_fields))) def check_unique_fieldname(docname, fieldname): duplicates = list(filter(None, map(lambda df: df.fieldname==fieldname and str(df.idx) or None, fields))) @@ -999,7 +999,7 @@ def validate_fields(meta): if docfield.options and (docfield.options not in data_field_options): df_str = frappe.bold(_(docfield.label)) text_str = _("{0} is an invalid Data field.").format(df_str) + "
" * 2 + _("Only Options allowed for Data field are:") + "
" - df_options_str = "" + df_options_str = "" frappe.msgprint(text_str + df_options_str, title="Invalid Data Field", raise_exception=True) diff --git a/frappe/core/doctype/domain/domain.py b/frappe/core/doctype/domain/domain.py index a4e9f503ab..4f320578fb 100644 --- a/frappe/core/doctype/domain/domain.py +++ b/frappe/core/doctype/domain/domain.py @@ -111,7 +111,7 @@ class Domain(Document): # enable frappe.db.sql('''update `tabPortal Menu Item` set enabled=1 - where route in ({0})'''.format(', '.join(['"{0}"'.format(d) for d in self.data.allow_sidebar_items]))) + where route in ({0})'''.format(', '.join('"{0}"'.format(d) for d in self.data.allow_sidebar_items))) if self.data.remove_sidebar_items: # disable all @@ -119,4 +119,4 @@ class Domain(Document): # enable frappe.db.sql('''update `tabPortal Menu Item` set enabled=0 - where route in ({0})'''.format(', '.join(['"{0}"'.format(d) for d in self.data.remove_sidebar_items]))) + where route in ({0})'''.format(', '.join('"{0}"'.format(d) for d in self.data.remove_sidebar_items))) diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py index a4d13a57e0..979a300a4d 100644 --- a/frappe/core/doctype/user/user.py +++ b/frappe/core/doctype/user/user.py @@ -935,7 +935,7 @@ def user_query(doctype, txt, searchfield, start, page_len, filters): LIMIT %(page_len)s OFFSET %(start)s """.format( user_type_condition = user_type_condition, - standard_users=", ".join([frappe.db.escape(u) for u in STANDARD_USERS]), + standard_users=", ".join(frappe.db.escape(u) for u in STANDARD_USERS), key=searchfield, fcond=get_filters_cond(doctype, filters, conditions), mcond=get_match_cond(doctype) diff --git a/frappe/core/doctype/user_type/user_type.py b/frappe/core/doctype/user_type/user_type.py index 0e8b692416..b3965d9b84 100644 --- a/frappe/core/doctype/user_type/user_type.py +++ b/frappe/core/doctype/user_type/user_type.py @@ -114,7 +114,7 @@ class UserType(Document): self.select_doctypes = [] select_doctypes = [] - user_doctypes = tuple([row.document_type for row in self.user_doctypes]) + user_doctypes = tuple(row.document_type for row in self.user_doctypes) for doctype in user_doctypes: doc = frappe.get_meta(doctype) diff --git a/frappe/custom/doctype/customize_form/customize_form.py b/frappe/custom/doctype/customize_form/customize_form.py index be0dded99c..17fd8ada05 100644 --- a/frappe/custom/doctype/customize_form/customize_form.py +++ b/frappe/custom/doctype/customize_form/customize_form.py @@ -356,7 +356,7 @@ class CustomizeForm(Document): def delete_custom_fields(self): meta = frappe.get_meta(self.doc_type) - fields_to_remove = (set([df.fieldname for df in meta.get("fields")]) + fields_to_remove = (set(df.fieldname for df in meta.get("fields")) - set(df.fieldname for df in self.get("fields"))) for fieldname in fields_to_remove: diff --git a/frappe/database/database.py b/frappe/database/database.py index c9c1ec3909..9ff3d25320 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -344,7 +344,7 @@ class Database(object): values[key] = value[1] if isinstance(value[1], (tuple, list)): # value is a list in tuple ("in", ("A", "B")) - _rhs = " ({0})".format(", ".join([self.escape(v) for v in value[1]])) + _rhs = " ({0})".format(", ".join(self.escape(v) for v in value[1])) del values[key] if _operator not in ["=", "!=", ">", ">=", "<", "<=", "like", "in", "not in", "not like"]: @@ -1019,7 +1019,7 @@ class Database(object): :params values: list of list of values """ insert_list = [] - fields = ", ".join(["`"+field+"`" for field in fields]) + fields = ", ".join("`"+field+"`" for field in fields) for idx, value in enumerate(values): insert_list.append(tuple(value)) diff --git a/frappe/desk/doctype/global_search_settings/global_search_settings.py b/frappe/desk/doctype/global_search_settings/global_search_settings.py index 85c9687ab3..f483376ed0 100644 --- a/frappe/desk/doctype/global_search_settings/global_search_settings.py +++ b/frappe/desk/doctype/global_search_settings/global_search_settings.py @@ -22,7 +22,7 @@ class GlobalSearchSettings(Document): dts.append(dt.document_type) if core_dts: - core_dts = (", ".join([frappe.bold(dt) for dt in core_dts])) + core_dts = (", ".join(frappe.bold(dt) for dt in core_dts)) frappe.throw(_("Core Modules {0} cannot be searched in Global Search.").format(core_dts)) if repeated_dts: @@ -61,7 +61,7 @@ def update_global_search_doctypes(): if search_doctypes.get(domain): global_search_doctypes.extend(search_doctypes.get(domain)) - doctype_list = set([dt.name for dt in frappe.get_all("DocType")]) + doctype_list = set(dt.name for dt in frappe.get_all("DocType")) allowed_in_global_search = [] for dt in global_search_doctypes: diff --git a/frappe/desk/doctype/tag/tag.py b/frappe/desk/doctype/tag/tag.py index 7e016ee91b..3ec3f05ac0 100644 --- a/frappe/desk/doctype/tag/tag.py +++ b/frappe/desk/doctype/tag/tag.py @@ -132,7 +132,7 @@ def update_tags(doc, tags): :param doc: Document to be added to global tags """ - new_tags = list(set([tag.strip() for tag in tags.split(",") if tag])) + new_tags = list(set(tag.strip() for tag in tags.split(",") if tag)) for tag in new_tags: if not frappe.db.exists("Tag Link", {"parenttype": doc.doctype, "parent": doc.name, "tag": tag}): diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index 36b662bb39..769dd57481 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -594,7 +594,7 @@ class EmailAccount(Document): # get email account user and set communication as seen users = frappe.get_all("User Email", filters={ "email_account": self.name }, fields=["parent"]) - users = list(set([ user.get("parent") for user in users ])) + users = list(set(user.get("parent") for user in users)) communication._seen = json.dumps(users) communication.flags.in_receive = True @@ -851,8 +851,8 @@ class EmailAccount(Document): email_server.update_flag(uid_list=uid_list) # mark communication as read - docnames = ",".join([ "'%s'"%flag.get("communication") for flag in flags \ - if flag.get("action") == "Read" ]) + docnames = ",".join("'%s'"%flag.get("communication") for flag in flags \ + if flag.get("action") == "Read") self.set_communication_seen_status(docnames, seen=1) # mark communication as unread diff --git a/frappe/email/doctype/newsletter/test_newsletter.py b/frappe/email/doctype/newsletter/test_newsletter.py index bd8fadc29c..073b46af44 100644 --- a/frappe/email/doctype/newsletter/test_newsletter.py +++ b/frappe/email/doctype/newsletter/test_newsletter.py @@ -44,7 +44,7 @@ class TestNewsletter(unittest.TestCase): email_queue_list = [frappe.get_doc("Email Queue", e.name) for e in frappe.get_all("Email Queue")] self.assertEqual(len(email_queue_list), 4) - recipients = set([e.recipients[0].recipient for e in email_queue_list]) + recipients = set(e.recipients[0].recipient for e in email_queue_list) self.assertTrue(set(emails).issubset(recipients)) def test_unsubscribe(self): diff --git a/frappe/email/inbox.py b/frappe/email/inbox.py index 395a2d3e2d..08eef5f7bf 100644 --- a/frappe/email/inbox.py +++ b/frappe/email/inbox.py @@ -18,7 +18,7 @@ def get_email_accounts(user=None): "all_accounts": "" } - all_accounts = ",".join([ account.get("email_account") for account in accounts ]) + all_accounts = ",".join(account.get("email_account") for account in accounts) if len(accounts) > 1: email_accounts.append({ "email_account": all_accounts, diff --git a/frappe/installer.py b/frappe/installer.py index 0cd5b136ae..cc5a6a6734 100755 --- a/frappe/installer.py +++ b/frappe/installer.py @@ -540,7 +540,7 @@ def is_downgrade(sql_file_path, verbose=False): def is_partial(sql_file_path): with open(sql_file_path) as f: - header = " ".join([f.readline() for _ in range(5)]) + header = " ".join(f.readline() for _ in range(5)) if "Partial Backup" in header: return True return False diff --git a/frappe/integrations/doctype/ldap_settings/ldap_settings.py b/frappe/integrations/doctype/ldap_settings/ldap_settings.py index 80dfef2693..ef72d03cde 100644 --- a/frappe/integrations/doctype/ldap_settings/ldap_settings.py +++ b/frappe/integrations/doctype/ldap_settings/ldap_settings.py @@ -80,7 +80,7 @@ class LDAPSettings(Document): def sync_roles(self, user, additional_groups=None): - current_roles = set([d.role for d in user.get("roles")]) + current_roles = set(d.role for d in user.get("roles")) needed_roles = set() needed_roles.add(self.default_role) diff --git a/frappe/model/__init__.py b/frappe/model/__init__.py index 205b451336..d9cbe7313e 100644 --- a/frappe/model/__init__.py +++ b/frappe/model/__init__.py @@ -166,7 +166,7 @@ def delete_fields(args_dict, delete=0): frappe.db.sql(""" DELETE FROM `tabSingles` WHERE doctype='%s' AND field IN (%s) - """ % (dt, ", ".join(["'{}'".format(f) for f in fields]))) + """ % (dt, ", ".join("'{}'".format(f) for f in fields))) else: existing_fields = frappe.db.multisql({ "mariadb": "DESC `tab%s`" % dt, @@ -189,7 +189,7 @@ def delete_fields(args_dict, delete=0): frappe.db.commit() query = "ALTER TABLE `tab%s` " % dt + \ - ", ".join(["DROP COLUMN `%s`" % f for f in fields_need_to_delete]) + ", ".join("DROP COLUMN `%s`" % f for f in fields_need_to_delete) frappe.db.sql(query) if frappe.db.db_type == 'postgres': diff --git a/frappe/model/base_document.py b/frappe/model/base_document.py index 154a091b8a..34c329175a 100644 --- a/frappe/model/base_document.py +++ b/frappe/model/base_document.py @@ -358,7 +358,7 @@ class BaseDocument(object): frappe.db.sql("""INSERT INTO `tab{doctype}` ({columns}) VALUES ({values})""".format( doctype = self.doctype, - columns = ", ".join(["`"+c+"`" for c in columns]), + columns = ", ".join("`"+c+"`" for c in columns), values = ", ".join(["%s"] * len(columns)) ), list(d.values())) except Exception as e: @@ -401,7 +401,7 @@ class BaseDocument(object): frappe.db.sql("""UPDATE `tab{doctype}` SET {values} WHERE `name`=%s""".format( doctype = self.doctype, - values = ", ".join(["`"+c+"`=%s" for c in columns]) + values = ", ".join("`"+c+"`=%s" for c in columns) ), list(d.values()) + [name]) except Exception as e: if frappe.db.is_unique_key_violation(e): diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py index e0c3406c46..8460880794 100644 --- a/frappe/model/db_query.py +++ b/frappe/model/db_query.py @@ -620,7 +620,7 @@ class DatabaseQuery(object): def get_share_condition(self): return """`tab{0}`.name in ({1})""".format(self.doctype, ", ".join(["%s"] * len(self.shared))) % \ - tuple([frappe.db.escape(s, percent=False) for s in self.shared]) + tuple(frappe.db.escape(s, percent=False) for s in self.shared) def add_user_permissions(self, user_permissions): meta = frappe.get_meta(self.doctype) @@ -726,8 +726,8 @@ class DatabaseQuery(object): # `idx desc, modified desc` # will covert to # `tabItem`.`idx` desc, `tabItem`.`modified` desc - args.order_by = ', '.join(['`tab{0}`.`{1}` {2}'.format(self.doctype, - f.split()[0].strip(), f.split()[1].strip()) for f in meta.sort_field.split(',')]) + args.order_by = ', '.join('`tab{0}`.`{1}` {2}'.format(self.doctype, + f.split()[0].strip(), f.split()[1].strip()) for f in meta.sort_field.split(',')) else: sort_field = meta.sort_field or 'modified' sort_order = (meta.sort_field and meta.sort_order) or 'desc' @@ -807,8 +807,8 @@ def get_order_by(doctype, meta): # `idx desc, modified desc` # will covert to # `tabItem`.`idx` desc, `tabItem`.`modified` desc - order_by = ', '.join(['`tab{0}`.`{1}` {2}'.format(doctype, - f.split()[0].strip(), f.split()[1].strip()) for f in meta.sort_field.split(',')]) + order_by = ', '.join('`tab{0}`.`{1}` {2}'.format(doctype, + f.split()[0].strip(), f.split()[1].strip()) for f in meta.sort_field.split(',')) else: sort_field = meta.sort_field or 'modified' sort_order = (meta.sort_field and meta.sort_order) or 'desc' diff --git a/frappe/model/meta.py b/frappe/model/meta.py index 66e8b08d92..1c3e7f60fb 100644 --- a/frappe/model/meta.py +++ b/frappe/model/meta.py @@ -667,7 +667,7 @@ def trim_tables(doctype=None): and not f.startswith("_")] if columns_to_remove: print(doctype, "columns removed:", columns_to_remove) - columns_to_remove = ", ".join(["drop `{0}`".format(c) for c in columns_to_remove]) + columns_to_remove = ", ".join("drop `{0}`".format(c) for c in columns_to_remove) query = """alter table `tab{doctype}` {columns}""".format( doctype=doctype, columns=columns_to_remove) frappe.db.sql_ddl(query) diff --git a/frappe/model/rename_doc.py b/frappe/model/rename_doc.py index 2c9dc5d823..1567ac544b 100644 --- a/frappe/model/rename_doc.py +++ b/frappe/model/rename_doc.py @@ -144,7 +144,7 @@ def update_user_settings(old, new, link_fields): if not link_fields: return # find the user settings for the linked doctypes - linked_doctypes = set([d.parent for d in link_fields if not d.issingle]) + linked_doctypes = set(d.parent for d in link_fields if not d.issingle) user_settings_details = frappe.db.sql('''SELECT `user`, `doctype`, `data` FROM `__UserSettings` WHERE `data` like %s diff --git a/frappe/permissions.py b/frappe/permissions.py index 19f101aab5..c0ff7fc590 100644 --- a/frappe/permissions.py +++ b/frappe/permissions.py @@ -312,7 +312,7 @@ def has_controller_permissions(doc, ptype, user=None): return None def get_doctypes_with_read(): - return list(set([p.parent if type(p.parent) == str else p.parent.encode('UTF8') for p in get_valid_perms()])) + return list(set(p.parent if type(p.parent) == str else p.parent.encode('UTF8') for p in get_valid_perms())) def get_valid_perms(doctype=None, user=None): '''Get valid permissions for the current user from DocPerm and Custom DocPerm''' diff --git a/frappe/tests/test_commands.py b/frappe/tests/test_commands.py index b6cd0b575c..07bdf8791e 100644 --- a/frappe/tests/test_commands.py +++ b/frappe/tests/test_commands.py @@ -80,7 +80,7 @@ def exists_in_backup(doctypes, file): ) with gzip.open(file, "rb") as f: content = f.read().decode("utf8") - return all([predicate.format(doctype).lower() in content.lower() for doctype in doctypes]) + return all(predicate.format(doctype).lower() in content.lower() for doctype in doctypes) class BaseTestCommands(unittest.TestCase): @@ -355,12 +355,12 @@ class TestCommands(BaseTestCommands): # test 2: bare functionality for single site self.execute("bench --site {site} list-apps") self.assertEqual(self.returncode, 0) - list_apps = set([ + list_apps = set( _x.split()[0] for _x in self.stdout.split("\n") - ]) + ) doctype = frappe.get_single("Installed Applications").installed_applications if doctype: - installed_apps = set([x.app_name for x in doctype]) + installed_apps = set(x.app_name for x in doctype) else: installed_apps = set(frappe.get_installed_apps()) self.assertSetEqual(list_apps, installed_apps) diff --git a/frappe/translate.py b/frappe/translate.py index 1d8b1234c7..caa7bcfea6 100644 --- a/frappe/translate.py +++ b/frappe/translate.py @@ -283,8 +283,8 @@ def clear_cache(): def get_messages_for_app(app, deduplicate=True): """Returns all messages (list) for a specified `app`""" messages = [] - modules = ", ".join(['"{}"'.format(m.title().replace("_", " ")) \ - for m in frappe.local.app_modules[app]]) + modules = ", ".join('"{}"'.format(m.title().replace("_", " ")) \ + for m in frappe.local.app_modules[app]) # doctypes if modules: diff --git a/frappe/utils/__init__.py b/frappe/utils/__init__.py index 1da4cd3bae..db0f39b3f5 100644 --- a/frappe/utils/__init__.py +++ b/frappe/utils/__init__.py @@ -186,7 +186,7 @@ def random_string(length): """generate a random string""" import string from random import choice - return ''.join([choice(string.ascii_letters + string.digits) for i in range(length)]) + return ''.join(choice(string.ascii_letters + string.digits) for i in range(length)) def has_gravatar(email): @@ -305,7 +305,7 @@ def make_esc(esc_chars): """ Function generator for Escaping special characters """ - return lambda s: ''.join(['\\' + c if c in esc_chars else c for c in s]) + return lambda s: ''.join('\\' + c if c in esc_chars else c for c in s) # esc / unescape characters -- used for command line def esc(s, esc_chars): diff --git a/frappe/utils/backups.py b/frappe/utils/backups.py index b21efc5e89..908be52452 100644 --- a/frappe/utils/backups.py +++ b/frappe/utils/backups.py @@ -307,8 +307,8 @@ class BackupGenerator: backup_summary = self.get_summary() print("Backup Summary for {0} at {1}".format(frappe.local.site, now())) - title = max([len(x) for x in backup_summary]) - path = max([len(x["path"]) for x in backup_summary.values()]) + title = max(len(x) for x in backup_summary) + path = max(len(x["path"]) for x in backup_summary.values()) for _type, info in backup_summary.items(): template = "{{0:{0}}}: {{1:{1}}} {{2}}".format(title, path) @@ -381,7 +381,7 @@ class BackupGenerator: "", ]) - generated_header = "\n".join([f"-- {x}" for x in database_header_content]) + "\n" + generated_header = "\n".join(f"-- {x}" for x in database_header_content) + "\n" with gzip.open(args.backup_path_db, "wt") as f: f.write(generated_header) diff --git a/frappe/utils/bot.py b/frappe/utils/bot.py index 45e1bd5a4e..68f49c47ed 100644 --- a/frappe/utils/bot.py +++ b/frappe/utils/bot.py @@ -40,10 +40,10 @@ class BotParser(object): def format_list(self, data): '''Format list as markdown''' - return _('I found these: ') + ', '.join([' [{title}](/app/Form/{doctype}/{name})'.format( + return _('I found these: ') + ', '.join(' [{title}](/app/Form/{doctype}/{name})'.format( title = d.title or d.name, doctype=self.get_doctype(), - name=d.name) for d in data]) + name=d.name) for d in data) def get_doctype(self): '''returns the doctype name from self.tables''' @@ -58,8 +58,8 @@ class ShowNotificationBot(BotParser): if open_items: return ("Following items need your attention:\n\n" - + "\n\n".join(["{0} [{1}](/app/List/{1})".format(d[1], d[0]) - for d in open_items if d[1] > 0])) + + "\n\n".join("{0} [{1}](/app/List/{1})".format(d[1], d[0]) + for d in open_items if d[1] > 0)) else: return 'Take it easy, nothing urgent needs your attention' diff --git a/frappe/utils/jinja_globals.py b/frappe/utils/jinja_globals.py index d8c5cb79f1..8eedca5335 100644 --- a/frappe/utils/jinja_globals.py +++ b/frappe/utils/jinja_globals.py @@ -14,10 +14,10 @@ def resolve_class(classes): return classes if isinstance(classes, (list, tuple)): - return " ".join([resolve_class(c) for c in classes]).strip() + return " ".join(resolve_class(c) for c in classes).strip() if isinstance(classes, dict): - return " ".join([classname for classname in classes if classes[classname]]).strip() + return " ".join(classname for classname in classes if classes[classname]).strip() return classes diff --git a/frappe/website/doctype/web_form/web_form.py b/frappe/website/doctype/web_form/web_form.py index f78aaac934..5ff2165527 100644 --- a/frappe/website/doctype/web_form/web_form.py +++ b/frappe/website/doctype/web_form/web_form.py @@ -346,7 +346,7 @@ def get_context(context): if missing: frappe.throw(_('Mandatory Information missing:') + '

' - + '
'.join(['{0} ({1})'.format(d.label, d.fieldtype) for d in missing])) + + '
'.join('{0} ({1})'.format(d.label, d.fieldtype) for d in missing)) def allow_website_search_indexing(self): return False diff --git a/frappe/website/doctype/website_slideshow/website_slideshow.py b/frappe/website/doctype/website_slideshow/website_slideshow.py index 90f62d1bb1..03e26bdcfd 100644 --- a/frappe/website/doctype/website_slideshow/website_slideshow.py +++ b/frappe/website/doctype/website_slideshow/website_slideshow.py @@ -23,7 +23,7 @@ class WebsiteSlideshow(Document): files = map(lambda row: row.image, self.slideshow_items) if files: result = frappe.get_all("File", filters={ "file_url":("in", list(files)) }, fields="is_private") - if any([file.is_private for file in result]): + if any(file.is_private for file in result): frappe.throw(_("All Images attached to Website Slideshow should be public")) def get_slideshow(doc): From cbbc85424bd1ce715f813f4349b6a63211b74b96 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Wed, 12 May 2021 16:46:58 +0530 Subject: [PATCH 02/14] fix: use set instead of list for new_ags No dependent functionalities expect a list. Co-authored-by: gavin --- frappe/desk/doctype/tag/tag.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/desk/doctype/tag/tag.py b/frappe/desk/doctype/tag/tag.py index 3ec3f05ac0..a85b264cee 100644 --- a/frappe/desk/doctype/tag/tag.py +++ b/frappe/desk/doctype/tag/tag.py @@ -132,7 +132,7 @@ def update_tags(doc, tags): :param doc: Document to be added to global tags """ - new_tags = list(set(tag.strip() for tag in tags.split(",") if tag)) + new_tags = {tag.strip() for tag in tags.split(",") if tag} for tag in new_tags: if not frappe.db.exists("Tag Link", {"parenttype": doc.doctype, "parent": doc.name, "tag": tag}): @@ -187,4 +187,4 @@ def get_documents_for_tag(tag): @frappe.whitelist() def get_tags_list_for_awesomebar(): - return [t.name for t in frappe.get_list("Tag")] \ No newline at end of file + return [t.name for t in frappe.get_list("Tag")] From d8a4cf896b3eca5132d1e23d6da40491005a9f1b Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Wed, 12 May 2021 16:56:16 +0530 Subject: [PATCH 03/14] perf: prefer set builder notation over constructor --- frappe/automation/doctype/auto_repeat/auto_repeat.py | 2 +- frappe/core/doctype/data_import/importer.py | 4 ++-- .../doctype/global_search_settings/global_search_settings.py | 2 +- frappe/email/doctype/email_account/email_account.py | 2 +- frappe/email/doctype/newsletter/test_newsletter.py | 2 +- frappe/model/rename_doc.py | 2 +- frappe/permissions.py | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/frappe/automation/doctype/auto_repeat/auto_repeat.py b/frappe/automation/doctype/auto_repeat/auto_repeat.py index 37285ad571..dfad85d0dd 100644 --- a/frappe/automation/doctype/auto_repeat/auto_repeat.py +++ b/frappe/automation/doctype/auto_repeat/auto_repeat.py @@ -334,7 +334,7 @@ class AutoRepeat(Document): if self.reference_doctype and self.reference_document: res = get_contacts_linking_to(self.reference_doctype, self.reference_document, fields=['email_id']) res += get_contacts_linked_from(self.reference_doctype, self.reference_document, fields=['email_id']) - email_ids = set(d.email_id for d in res) + email_ids = {d.email_id for d in res} if not email_ids: frappe.msgprint(_('No contacts linked to document'), alert=True) else: diff --git a/frappe/core/doctype/data_import/importer.py b/frappe/core/doctype/data_import/importer.py index 841319dd60..fed6d2cb06 100644 --- a/frappe/core/doctype/data_import/importer.py +++ b/frappe/core/doctype/data_import/importer.py @@ -903,7 +903,7 @@ class Column: if self.df.fieldtype == "Link": # find all values that dont exist - values = list(set(cstr(v) for v in self.column_values[1:] if v)) + values = list({cstr(v) for v in self.column_values[1:] if v}) exists = [ d.name for d in frappe.db.get_all(self.df.options, filters={"name": ("in", values)}) ] @@ -939,7 +939,7 @@ class Column: elif self.df.fieldtype == "Select": options = get_select_options(self.df) if options: - values = list(set(cstr(v) for v in self.column_values[1:] if v)) + values = list({cstr(v) for v in self.column_values[1:] if v}) invalid = list(set(values) - set(options)) if invalid: valid_values = ", ".join(frappe.bold(o) for o in options) diff --git a/frappe/desk/doctype/global_search_settings/global_search_settings.py b/frappe/desk/doctype/global_search_settings/global_search_settings.py index f483376ed0..fead118e75 100644 --- a/frappe/desk/doctype/global_search_settings/global_search_settings.py +++ b/frappe/desk/doctype/global_search_settings/global_search_settings.py @@ -61,7 +61,7 @@ def update_global_search_doctypes(): if search_doctypes.get(domain): global_search_doctypes.extend(search_doctypes.get(domain)) - doctype_list = set(dt.name for dt in frappe.get_all("DocType")) + doctype_list = {dt.name for dt in frappe.get_all("DocType")} allowed_in_global_search = [] for dt in global_search_doctypes: diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index 769dd57481..786a185585 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -594,7 +594,7 @@ class EmailAccount(Document): # get email account user and set communication as seen users = frappe.get_all("User Email", filters={ "email_account": self.name }, fields=["parent"]) - users = list(set(user.get("parent") for user in users)) + users = list({user.get("parent") for user in users}) communication._seen = json.dumps(users) communication.flags.in_receive = True diff --git a/frappe/email/doctype/newsletter/test_newsletter.py b/frappe/email/doctype/newsletter/test_newsletter.py index 073b46af44..4b9a0749d5 100644 --- a/frappe/email/doctype/newsletter/test_newsletter.py +++ b/frappe/email/doctype/newsletter/test_newsletter.py @@ -44,7 +44,7 @@ class TestNewsletter(unittest.TestCase): email_queue_list = [frappe.get_doc("Email Queue", e.name) for e in frappe.get_all("Email Queue")] self.assertEqual(len(email_queue_list), 4) - recipients = set(e.recipients[0].recipient for e in email_queue_list) + recipients = {e.recipients[0].recipient for e in email_queue_list} self.assertTrue(set(emails).issubset(recipients)) def test_unsubscribe(self): diff --git a/frappe/model/rename_doc.py b/frappe/model/rename_doc.py index 1567ac544b..2c2550f39d 100644 --- a/frappe/model/rename_doc.py +++ b/frappe/model/rename_doc.py @@ -144,7 +144,7 @@ def update_user_settings(old, new, link_fields): if not link_fields: return # find the user settings for the linked doctypes - linked_doctypes = set(d.parent for d in link_fields if not d.issingle) + linked_doctypes = {d.parent for d in link_fields if not d.issingle} user_settings_details = frappe.db.sql('''SELECT `user`, `doctype`, `data` FROM `__UserSettings` WHERE `data` like %s diff --git a/frappe/permissions.py b/frappe/permissions.py index c0ff7fc590..c17f4b6bf1 100644 --- a/frappe/permissions.py +++ b/frappe/permissions.py @@ -312,7 +312,7 @@ def has_controller_permissions(doc, ptype, user=None): return None def get_doctypes_with_read(): - return list(set(p.parent if type(p.parent) == str else p.parent.encode('UTF8') for p in get_valid_perms())) + return list({p.parent if type(p.parent) == str else p.parent.encode('UTF8') for p in get_valid_perms()}) def get_valid_perms(doctype=None, user=None): '''Get valid permissions for the current user from DocPerm and Custom DocPerm''' From 6d8c691dddc502baf160b28d05c1d5baa8da8e35 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Wed, 12 May 2021 19:22:31 +0530 Subject: [PATCH 04/14] refactor: use list instead of tuple constructor --- frappe/core/doctype/user_type/user_type.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/core/doctype/user_type/user_type.py b/frappe/core/doctype/user_type/user_type.py index b3965d9b84..b3e61610cc 100644 --- a/frappe/core/doctype/user_type/user_type.py +++ b/frappe/core/doctype/user_type/user_type.py @@ -114,7 +114,7 @@ class UserType(Document): self.select_doctypes = [] select_doctypes = [] - user_doctypes = tuple(row.document_type for row in self.user_doctypes) + user_doctypes = [row.document_type for row in self.user_doctypes] for doctype in user_doctypes: doc = frappe.get_meta(doctype) @@ -267,4 +267,4 @@ def apply_permissions_for_non_standard_user_type(doc, method=None): user_doc.update_children() add_user_permission(doc.doctype, doc.name, doc.get(data[1])) else: - frappe.db.set_value('User Permission', perm_data[0], 'user', doc.get(data[1])) \ No newline at end of file + frappe.db.set_value('User Permission', perm_data[0], 'user', doc.get(data[1])) From d7eea226094145a4d45e4fbc6a7ef8bdfe2fad55 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Wed, 12 May 2021 19:25:21 +0530 Subject: [PATCH 05/14] chore: misc translation/semgrep fixes --- frappe/core/doctype/data_import/importer.py | 5 +---- frappe/utils/bot.py | 2 +- frappe/utils/jinja_globals.py | 4 ++-- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/frappe/core/doctype/data_import/importer.py b/frappe/core/doctype/data_import/importer.py index fed6d2cb06..6337d919eb 100644 --- a/frappe/core/doctype/data_import/importer.py +++ b/frappe/core/doctype/data_import/importer.py @@ -929,10 +929,7 @@ class Column: self.warnings.append( { "col": self.column_number, - "message": _( - "Date format could not be determined from the values in" - " this column. Defaulting to yyyy-mm-dd." - ), + "message": _("Date format could not be determined from the values in this column. Defaulting to yyyy-mm-dd."), "type": "info", } ) diff --git a/frappe/utils/bot.py b/frappe/utils/bot.py index 68f49c47ed..33ec139b64 100644 --- a/frappe/utils/bot.py +++ b/frappe/utils/bot.py @@ -40,7 +40,7 @@ class BotParser(object): def format_list(self, data): '''Format list as markdown''' - return _('I found these: ') + ', '.join(' [{title}](/app/Form/{doctype}/{name})'.format( + return _('I found these:') + ' ' + ', '.join(' [{title}](/app/Form/{doctype}/{name})'.format( title = d.title or d.name, doctype=self.get_doctype(), name=d.name) for d in data) diff --git a/frappe/utils/jinja_globals.py b/frappe/utils/jinja_globals.py index 8eedca5335..906d557326 100644 --- a/frappe/utils/jinja_globals.py +++ b/frappe/utils/jinja_globals.py @@ -38,13 +38,13 @@ def web_block(template, values=None, **kwargs): def web_blocks(blocks): - from frappe import throw, _dict + from frappe import throw, _dict, _ from frappe.website.doctype.web_page.web_page import get_web_blocks_html web_blocks = [] for block in blocks: if not block.get("template"): - throw("Web Template is not specified") + throw(_("Web Template is not specified")) doc = _dict( { From 8b1b5c707b11f150d85d791354a7ebadccd27f43 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Thu, 13 May 2021 16:41:20 +0530 Subject: [PATCH 06/14] fix: remove unnecessary casting from set to list --- frappe/core/doctype/data_import/importer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/core/doctype/data_import/importer.py b/frappe/core/doctype/data_import/importer.py index 6337d919eb..6856fde4d0 100644 --- a/frappe/core/doctype/data_import/importer.py +++ b/frappe/core/doctype/data_import/importer.py @@ -936,8 +936,8 @@ class Column: elif self.df.fieldtype == "Select": options = get_select_options(self.df) if options: - values = list({cstr(v) for v in self.column_values[1:] if v}) - invalid = list(set(values) - set(options)) + values = {cstr(v) for v in self.column_values[1:] if v} + invalid = values - set(options) if invalid: valid_values = ", ".join(frappe.bold(o) for o in options) invalid_values = ", ".join(frappe.bold(i) for i in invalid) From cd6f353175aaa01accd141828028a0da4133b0ae Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 1 Jun 2021 01:12:19 +0530 Subject: [PATCH 07/14] chore: Remove dead and sort imports --- .../doctype/email_account/email_account.py | 35 ++++++++----------- 1 file changed, 14 insertions(+), 21 deletions(-) diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index f9925c0ed0..0df3c031ff 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -1,32 +1,25 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors # For license information, please see license.txt -import frappe +import email.utils +import functools import imaplib -import re -import json import socket import time -import functools - -import email.utils - -from frappe import _, are_emails_muted -from frappe.model.document import Document -from frappe.utils import (validate_email_address, cint, cstr, get_datetime, - DATE_FORMAT, strip, comma_or, sanitize_html, add_days, parse_addr) -from frappe.utils.user import is_system_user -from frappe.utils.jinja import render_template -from frappe.email.smtp import SMTPServer -from frappe.email.receive import EmailServer, InboundMail, SentEmailInInboxError -from poplib import error_proto -from dateutil.relativedelta import relativedelta from datetime import datetime, timedelta +from poplib import error_proto + +import frappe +from frappe import _, are_emails_muted from frappe.desk.form import assign_to -from frappe.utils.user import get_system_managers -from frappe.utils.background_jobs import enqueue, get_jobs -from frappe.utils.html_utils import clean_email_html -from frappe.utils.error import raise_error_on_no_output +from frappe.email.receive import EmailServer, InboundMail, SentEmailInInboxError +from frappe.email.smtp import SMTPServer from frappe.email.utils import get_port +from frappe.model.document import Document +from frappe.utils import cint, comma_or, cstr, parse_addr, validate_email_address +from frappe.utils.background_jobs import enqueue, get_jobs +from frappe.utils.error import raise_error_on_no_output +from frappe.utils.jinja import render_template +from frappe.utils.user import get_system_managers OUTGOING_EMAIL_ACCOUNT_MISSING = _("Please setup default Email Account from Setup > Email > Email Account") From b0bb0cf2c4864a1b97edf507a5733183c56c81e7 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 1 Jun 2021 01:32:19 +0530 Subject: [PATCH 08/14] refactor(minor): Use set comprehension instead of difference --- frappe/custom/doctype/customize_form/customize_form.py | 3 +-- .../doctype/global_search_settings/global_search_settings.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/frappe/custom/doctype/customize_form/customize_form.py b/frappe/custom/doctype/customize_form/customize_form.py index 9d613b207b..d397d8899d 100644 --- a/frappe/custom/doctype/customize_form/customize_form.py +++ b/frappe/custom/doctype/customize_form/customize_form.py @@ -355,8 +355,7 @@ class CustomizeForm(Document): def delete_custom_fields(self): meta = frappe.get_meta(self.doc_type) - fields_to_remove = (set(df.fieldname for df in meta.get("fields")) - - set(df.fieldname for df in self.get("fields"))) + fields_to_remove = {df.fieldname for df in meta.get("fields") if df not in self.get("fields")} for fieldname in fields_to_remove: df = meta.get("fields", {"fieldname": fieldname})[0] diff --git a/frappe/desk/doctype/global_search_settings/global_search_settings.py b/frappe/desk/doctype/global_search_settings/global_search_settings.py index f48e964959..9112349c1b 100644 --- a/frappe/desk/doctype/global_search_settings/global_search_settings.py +++ b/frappe/desk/doctype/global_search_settings/global_search_settings.py @@ -21,7 +21,7 @@ class GlobalSearchSettings(Document): dts.append(dt.document_type) if core_dts: - core_dts = (", ".join(frappe.bold(dt) for dt in core_dts)) + core_dts = ", ".join(frappe.bold(dt) for dt in core_dts) frappe.throw(_("Core Modules {0} cannot be searched in Global Search.").format(core_dts)) if repeated_dts: From 974bf463df4542b660061bf3756d83364dbe53a1 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 2 Jun 2021 22:38:53 +0530 Subject: [PATCH 09/14] refactor: Simplify string representations * Simplify existing %s, .format representations to f-string * Simplify logic * Remove redundant tokens --- frappe/model/db_query.py | 88 ++++++++++++++++------------------------ 1 file changed, 36 insertions(+), 52 deletions(-) diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py index ee1b04f07a..a94b43e517 100644 --- a/frappe/model/db_query.py +++ b/frappe/model/db_query.py @@ -56,10 +56,8 @@ class DatabaseQuery(object): if fields: self.fields = fields else: - if pluck: - self.fields = ["`tab{0}`.`{1}`".format(self.doctype, pluck)] - else: - self.fields = ["`tab{0}`.`name`".format(self.doctype)] + pluck = pluck or "name" + self.fields = [f"`tab{self.doctype}`.`{pluck}`"] if start: limit_start = start if page_length: limit_page_length = page_length @@ -157,11 +155,10 @@ class DatabaseQuery(object): # left join parent, child tables for child in self.tables[1:]: - args.tables += " {join} {child} on ({child}.parent = {main}.name)".format(join=self.join, - child=child, main=self.tables[0]) + args.tables += f" {self.join} {child} on ({child}.parent = {self.tables[0]}.name)" if self.grouped_or_conditions: - self.conditions.append("({0})".format(" or ".join(self.grouped_or_conditions))) + self.conditions.append(f"({' or '.join(self.grouped_or_conditions)})") args.conditions = ' and '.join(self.conditions) @@ -186,9 +183,9 @@ class DatabaseQuery(object): fields.append(field) elif "as" in field.lower().split(" "): col, _, new = field.split() - fields.append("`{0}` as {1}".format(col, new)) + fields.append(f"`{col}` as {new}") else: - fields.append("`{0}`".format(field)) + fields.append(f"`{field}`") args.fields = ", ".join(fields) @@ -260,10 +257,10 @@ class DatabaseQuery(object): if any(keyword in field.lower().split() for keyword in blacklisted_keywords): _raise_exception() - if any("({0}".format(keyword) in field.lower() for keyword in blacklisted_keywords): + if any(f"({keyword}" in field.lower() for keyword in blacklisted_keywords): _raise_exception() - if any("{0}(".format(keyword) in field.lower() for keyword in blacklisted_functions): + if any(f"{keyword}(" in field.lower() for keyword in blacklisted_functions): _raise_exception() if '@' in field.lower(): @@ -326,7 +323,7 @@ class DatabaseQuery(object): if len(self.tables) > 1: for idx, field in enumerate(self.fields): if '.' not in field and not _in_standard_sql_methods(field): - self.fields[idx] = '{0}.{1}'.format(self.tables[0], field) + self.fields[idx] = f"{self.tables[0]}.{field}" def get_table_columns(self): try: @@ -375,7 +372,7 @@ class DatabaseQuery(object): if not self.flags.ignore_permissions: match_conditions = self.build_match_conditions() if match_conditions: - self.conditions.append("(" + match_conditions + ")") + self.conditions.append(f"({match_conditions})") def build_filter_conditions(self, filters, conditions, ignore_permissions=None): """build conditions from user filters""" @@ -407,8 +404,7 @@ class DatabaseQuery(object): if 'ifnull(' in f.fieldname: column_name = f.fieldname else: - column_name = '{tname}.{fname}'.format(tname=tname, - fname=f.fieldname) + column_name = f"{tname}.{f.fieldname}" can_be_null = True @@ -450,7 +446,7 @@ class DatabaseQuery(object): fallback = "''" value = [frappe.db.escape((v.name or '').strip(), percent=False) for v in result] if len(value): - value = "({0})".format(", ".join(value)) + value = f"({', '.join(value)})" else: value = "('')" # changing operator to IN as the above code fetches all the parent / child values and convert into tuple @@ -466,7 +462,7 @@ class DatabaseQuery(object): fallback = "''" value = [frappe.db.escape((v or '').strip(), percent=False) for v in values] if len(value): - value = "({0})".format(", ".join(value)) + value = f"({', '.join(value)})" else: value = "('')" else: @@ -503,7 +499,7 @@ class DatabaseQuery(object): can_be_null = True if 'ifnull' not in column_name: - column_name = 'ifnull({}, {})'.format(column_name, fallback) + column_name = f'ifnull({column_name}, {fallback})' elif df and df.fieldtype=="Date": value = frappe.db.format_date(f.value) @@ -540,7 +536,7 @@ class DatabaseQuery(object): # escape value if isinstance(value, str) and not f.operator.lower() == 'between': - value = "{0}".format(frappe.db.escape(value, percent=False)) + value = f"{frappe.db.escape(value, percent=False)}" if (self.ignore_ifnull or not can_be_null @@ -548,13 +544,9 @@ class DatabaseQuery(object): or 'ifnull(' in column_name.lower()): if f.operator.lower() == 'like' and frappe.conf.get('db_type') == 'postgres': f.operator = 'ilike' - condition = '{column_name} {operator} {value}'.format( - column_name=column_name, operator=f.operator, - value=value) + condition = f'{column_name} {f.operator} {value}' else: - condition = 'ifnull({column_name}, {fallback}) {operator} {value}'.format( - column_name=column_name, fallback=fallback, operator=f.operator, - value=value) + condition = f'ifnull({column_name}, {fallback}) {f.operator} {value}' return condition @@ -585,8 +577,10 @@ class DatabaseQuery(object): else: #if has if_owner permission skip user perm check if role_permissions.get("has_if_owner_enabled") and role_permissions.get("if_owner", {}): - self.match_conditions.append("`tab{0}`.`owner` = {1}".format(self.doctype, - frappe.db.escape(self.user, percent=False))) + self.match_conditions.append( + f"`tab{self.doctype}`.`owner` = {frappe.db.escape(self.user, percent=False)}" + ) + # add user permission only if role has read perm elif role_permissions.get("read") or role_permissions.get("select"): # get user permissions @@ -605,8 +599,7 @@ class DatabaseQuery(object): # share is an OR condition, if there is a role permission if not only_if_shared and self.shared and conditions: - conditions = "({conditions}) or ({shared_condition})".format( - conditions=conditions, shared_condition=self.get_share_condition()) + conditions = f"({conditions}) or ({self.get_share_condition()})" return conditions @@ -614,8 +607,7 @@ class DatabaseQuery(object): return self.match_filters def get_share_condition(self): - return """`tab{0}`.name in ({1})""".format(self.doctype, ", ".join(["%s"] * len(self.shared))) % \ - tuple(frappe.db.escape(s, percent=False) for s in self.shared) + return f"`tab{self.doctype}`.name in ({', '.join(frappe.db.escape(s, percent=False) for s in self.shared)})" def add_user_permissions(self, user_permissions): meta = frappe.get_meta(self.doctype) @@ -640,9 +632,7 @@ class DatabaseQuery(object): if frappe.get_system_settings("apply_strict_user_permissions"): condition = "" else: - empty_value_condition = "ifnull(`tab{doctype}`.`{fieldname}`, '')=''".format( - doctype=self.doctype, fieldname=df.get('fieldname') - ) + empty_value_condition = f"ifnull(`tab{self.doctype}`.`{df.get('fieldname')}`, '')=''" condition = empty_value_condition + " or " for permission in user_permission_values: @@ -664,14 +654,9 @@ class DatabaseQuery(object): docs.append(permission.get('doc')) if docs: - condition += "`tab{doctype}`.`{fieldname}` in ({values})".format( - doctype=self.doctype, - fieldname=df.get('fieldname'), - values=", ".join( - [(frappe.db.escape(doc, percent=False)) for doc in docs]) - ) - - match_conditions.append("({condition})".format(condition=condition)) + values = ", ".join(frappe.db.escape(doc, percent=False) for doc in docs) + condition += f"`tab{self.doctype}`.`{df.get('fieldname')}` in ({values})" + match_conditions.append(f"({condition})") match_filters[df.get('options')] = docs if match_conditions: @@ -721,17 +706,17 @@ class DatabaseQuery(object): # `idx desc, modified desc` # will covert to # `tabItem`.`idx` desc, `tabItem`.`modified` desc - args.order_by = ', '.join('`tab{0}`.`{1}` {2}'.format(self.doctype, - f.split()[0].strip(), f.split()[1].strip()) for f in meta.sort_field.split(',')) + args.order_by = ', '.join( + f"`tab{self.doctype}`.`{f.split()[0].strip()}` {f.split()[1].strip()}" for f in meta.sort_field.split(',') + ) else: sort_field = meta.sort_field or 'modified' sort_order = (meta.sort_field and meta.sort_order) or 'desc' - - args.order_by = "`tab{0}`.`{1}` {2}".format(self.doctype, sort_field or "modified", sort_order or "desc") + args.order_by = f"`tab{self.doctype}`.`{sort_field or 'modified'}` {sort_order or 'desc'}" # draft docs always on top if hasattr(meta, 'is_submittable') and meta.is_submittable: - args.order_by = "`tab{0}`.docstatus asc, {1}".format(self.doctype, args.order_by) + args.order_by = f"`tab{self.doctype}`.docstatus asc, {args.order_by}" def validate_order_by_and_group_by(self, parameters): """Check order by, group by so that atleast one column is selected and does not have subquery""" @@ -802,17 +787,16 @@ def get_order_by(doctype, meta): # `idx desc, modified desc` # will covert to # `tabItem`.`idx` desc, `tabItem`.`modified` desc - order_by = ', '.join('`tab{0}`.`{1}` {2}'.format(doctype, - f.split()[0].strip(), f.split()[1].strip()) for f in meta.sort_field.split(',')) + order_by = ', '.join(f"`tab{doctype}`.`{f.split()[0].strip()}` {f.split()[1].strip()}" for f in meta.sort_field.split(',')) + else: sort_field = meta.sort_field or 'modified' sort_order = (meta.sort_field and meta.sort_order) or 'desc' - - order_by = "`tab{0}`.`{1}` {2}".format(doctype, sort_field or "modified", sort_order or "desc") + order_by = f"`tab{doctype}`.`{sort_field or 'modified'}` {sort_order or 'desc'}" # draft docs always on top if meta.is_submittable: - order_by = "`tab{0}`.docstatus asc, {1}".format(doctype, order_by) + order_by = f"`tab{doctype}`.docstatus asc, {order_by}" return order_by From ee75fb38cce9d1256e6be4115b4c3304b5498b4d Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 2 Jun 2021 21:38:53 +0530 Subject: [PATCH 10/14] style: Restructured if blocks in db_query --- frappe/model/db_query.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py index a94b43e517..46b7db002b 100644 --- a/frappe/model/db_query.py +++ b/frappe/model/db_query.py @@ -43,8 +43,14 @@ class DatabaseQuery(object): # filters and fields swappable # its hard to remember what comes first - if (isinstance(fields, dict) - or (isinstance(fields, list) and fields and isinstance(fields[0], list))): + if ( + isinstance(fields, dict) + or ( + fields + and isinstance(fields, list) + and isinstance(fields[0], list) + ) + ): # if fields is given as dict/list of list, its probably filters filters, fields = fields, filters @@ -308,8 +314,7 @@ class DatabaseQuery(object): doctype = table_name[4:-1] ptype = 'select' if frappe.only_has_select_perm(doctype) else 'read' - if (not self.flags.ignore_permissions) and\ - (not frappe.has_permission(doctype, ptype=ptype)): + if not self.flags.ignore_permissions and not frappe.has_permission(doctype, ptype=ptype): frappe.flags.error_message = _('Insufficient Permission for {0}').format(frappe.bold(doctype)) raise frappe.PermissionError(doctype) @@ -538,10 +543,12 @@ class DatabaseQuery(object): if isinstance(value, str) and not f.operator.lower() == 'between': value = f"{frappe.db.escape(value, percent=False)}" - if (self.ignore_ifnull + if ( + self.ignore_ifnull or not can_be_null or (f.value and f.operator.lower() in ('=', 'like')) - or 'ifnull(' in column_name.lower()): + or 'ifnull(' in column_name.lower() + ): if f.operator.lower() == 'like' and frappe.conf.get('db_type') == 'postgres': f.operator = 'ilike' condition = f'{column_name} {f.operator} {value}' @@ -564,10 +571,12 @@ class DatabaseQuery(object): role_permissions = frappe.permissions.get_role_permissions(meta, user=self.user) self.shared = frappe.share.get_shared(self.doctype, self.user) - if (not meta.istable and + if ( + not meta.istable and not (role_permissions.get("select") or role_permissions.get("read")) and not self.flags.ignore_permissions and - not has_any_user_permission_for_doctype(self.doctype, self.user, self.reference_doctype)): + not has_any_user_permission_for_doctype(self.doctype, self.user, self.reference_doctype) + ): only_if_shared = True if not self.shared: frappe.throw(_("No permission to read {0}").format(self.doctype), frappe.PermissionError) @@ -640,9 +649,7 @@ class DatabaseQuery(object): docs.append(permission.get('doc')) # append docs based on user permission applicable on reference doctype - # this is useful when getting list of docs from a link field - # in this case parent doctype of the link # will be the reference doctype From 888aa8ad849fd6ca4f0c1a5a8cc3d8aa61fbc1d5 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 2 Jun 2021 21:38:53 +0530 Subject: [PATCH 11/14] refactor(minor): DatabaseQuery.extract_tables --- frappe/model/db_query.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py index 46b7db002b..cb0b8daf16 100644 --- a/frappe/model/db_query.py +++ b/frappe/model/db_query.py @@ -290,22 +290,30 @@ class DatabaseQuery(object): def extract_tables(self): """extract tables from fields""" - self.tables = ['`tab' + self.doctype + '`'] - + self.tables = [f"`tab{self.doctype}`"] + sql_functions = [ + "dayofyear(", + "extract(", + "locate(", + "strpos(", + "count(", + "sum(", + "avg(", + ] # add tables from fields if self.fields: - for f in self.fields: - if ( not ("tab" in f and "." in f) ) or ("locate(" in f) or ("strpos(" in f) or \ - ("count(" in f) or ("avg(" in f) or ("sum(" in f) or ("extract(" in f) or ("dayofyear(" in f): + for field in self.fields: + if ("tab" not in field and "." not in field) or any(x for x in sql_functions if x in field): continue - table_name = f.split('.')[0] + table_name = field.split('.')[0] + if table_name.lower().startswith('group_concat('): table_name = table_name[13:] if table_name.lower().startswith('ifnull('): table_name = table_name[7:] if not table_name[0]=='`': - table_name = '`' + table_name + '`' + table_name = f"`{table_name}`" if not table_name in self.tables: self.append_table(table_name) From f92c5d989839745b3f951c8430c1ca72feca2ec2 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 2 Jun 2021 01:12:19 +0530 Subject: [PATCH 12/14] refactor: Unnecessary/illogical check because cint(False) gives 0 --- frappe/model/db_query.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py index cb0b8daf16..c27d49c920 100644 --- a/frappe/model/db_query.py +++ b/frappe/model/db_query.py @@ -74,7 +74,7 @@ class DatabaseQuery(object): self.docstatus = docstatus or [] self.group_by = group_by self.order_by = order_by - self.limit_start = 0 if (limit_start is False) else cint(limit_start) + self.limit_start = cint(limit_start) self.limit_page_length = cint(limit_page_length) if limit_page_length else None self.with_childnames = with_childnames self.debug = debug From 1b6c41cbfc18302460069235c8a9be5e5b9a4d29 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 11 Jun 2021 19:44:22 +0530 Subject: [PATCH 13/14] fix: Don't re-define pluck bug Introduced via 974bf463df4542b660061bf3756d83364dbe53a1 --- frappe/model/db_query.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py index c27d49c920..8d13b9cb20 100644 --- a/frappe/model/db_query.py +++ b/frappe/model/db_query.py @@ -62,8 +62,7 @@ class DatabaseQuery(object): if fields: self.fields = fields else: - pluck = pluck or "name" - self.fields = [f"`tab{self.doctype}`.`{pluck}`"] + self.fields = [f"`tab{self.doctype}`.`{pluck or 'name'}`"] if start: limit_start = start if page_length: limit_page_length = page_length From 254b8ae1b3b27f5ccf028577988b1f68b8132f03 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 11 Jun 2021 21:07:16 +0530 Subject: [PATCH 14/14] fix: Use set difference instead of comprehension Bug introduced via b0bb0cf2c4864a1b97edf507a5733183c56c81e7 due to improper comprehension definition --- frappe/custom/doctype/customize_form/customize_form.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/frappe/custom/doctype/customize_form/customize_form.py b/frappe/custom/doctype/customize_form/customize_form.py index d397d8899d..1b8977acc4 100644 --- a/frappe/custom/doctype/customize_form/customize_form.py +++ b/frappe/custom/doctype/customize_form/customize_form.py @@ -355,8 +355,9 @@ class CustomizeForm(Document): def delete_custom_fields(self): meta = frappe.get_meta(self.doc_type) - fields_to_remove = {df.fieldname for df in meta.get("fields") if df not in self.get("fields")} - + fields_to_remove = ( + {df.fieldname for df in meta.get("fields")} - {df.fieldname for df in self.get("fields")} + ) for fieldname in fields_to_remove: df = meta.get("fields", {"fieldname": fieldname})[0] if df.get("is_custom_field"):