From 1952920add38964450ae12121e3f82f628780be8 Mon Sep 17 00:00:00 2001 From: hrwx Date: Tue, 15 Jun 2021 22:53:24 +0530 Subject: [PATCH 001/246] feat: Convert datetime field values to system timezone Co-authored-by: Sahil Khan --- frappe/boot.py | 12 +++- frappe/core/doctype/user/user.js | 7 ++- frappe/core/doctype/user/user.py | 8 +++ .../js/frappe/form/controls/datetime.js | 44 ++++++++++++- frappe/public/js/frappe/utils/datetime.js | 61 +++++++++++-------- 5 files changed, 101 insertions(+), 31 deletions(-) diff --git a/frappe/boot.py b/frappe/boot.py index 0589e32ac8..feace1a66d 100644 --- a/frappe/boot.py +++ b/frappe/boot.py @@ -17,6 +17,7 @@ from frappe.social.doctype.energy_point_log.energy_point_log import get_energy_p from frappe.model.base_document import get_controller from frappe.social.doctype.post.post import frequently_visited_links from frappe.core.doctype.navbar_settings.navbar_settings import get_navbar_settings, get_app_logo +from frappe.utils import get_time_zone def get_bootinfo(): """build and return boot info""" @@ -58,6 +59,7 @@ def get_bootinfo(): bootinfo.home_folder = frappe.db.get_value("File", {"is_home_folder": 1}) bootinfo.navbar_settings = get_navbar_settings() bootinfo.notification_settings = get_notification_settings() + set_time_zone(bootinfo) # ipinfo if frappe.session.data.get('ipinfo'): @@ -220,8 +222,8 @@ def load_translations(bootinfo): bootinfo["__messages"] = messages def get_user_info(): - user_info = frappe.db.get_all('User', fields=['`name`', 'full_name as fullname', 'user_image as image', - 'gender', 'email', 'username', 'bio', 'location', 'interest', 'banner_image', 'allowed_in_mentions', 'user_type'], + user_info = frappe.db.get_all('User', fields=['`name`', 'full_name as fullname', 'user_image as image', 'gender', + 'email', 'username', 'bio', 'location', 'interest', 'banner_image', 'allowed_in_mentions', 'user_type', 'time_zone'], filters=dict(enabled=1)) user_info_map = {d.name: d for d in user_info} @@ -324,3 +326,9 @@ def get_desk_settings(): def get_notification_settings(): return frappe.get_cached_doc('Notification Settings', frappe.session.user) + +def set_time_zone(bootinfo): + bootinfo.time_zone = { + "system_time_zone": get_time_zone(), + "user_time_zone": bootinfo.get("user_info", {}).get(frappe.session.user, {}).get("time_zone", None) or get_time_zone() + } diff --git a/frappe/core/doctype/user/user.js b/frappe/core/doctype/user/user.js index 8c5b89c5fc..7c9e00d6bc 100644 --- a/frappe/core/doctype/user/user.js +++ b/frappe/core/doctype/user/user.js @@ -77,7 +77,12 @@ frappe.ui.form.on('User', { } }, refresh: function(frm) { - var doc = frm.doc; + let doc = frm.doc; + + if (frm.is_new()) { + frm.set_value("time_zone", frappe.sys_defaults.time_zone); + } + if (in_list(['System User', 'Website User'], frm.doc.user_type) && !frm.is_new() && !frm.roles_editor && frm.can_edit_roles) { frm.reload_doc(); diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py index 13063b8fd2..93963c6015 100644 --- a/frappe/core/doctype/user/user.py +++ b/frappe/core/doctype/user/user.py @@ -80,6 +80,7 @@ class User(Document): self.validate_roles() self.validate_allowed_modules() self.validate_user_image() + self.set_time_zone() if self.language == "Loading...": self.language = None @@ -594,6 +595,13 @@ class User(Document): return user + def set_time_zone(self): + from frappe.utils import get_time_zone + + if not self.time_zone: + frappe.msgprint(_("User Time Zone was not set, defaulting to System Time Zone."), title=_("User Time Zone")) + self.time_zone = get_time_zone() + @frappe.whitelist() def get_timezones(): import pytz diff --git a/frappe/public/js/frappe/form/controls/datetime.js b/frappe/public/js/frappe/form/controls/datetime.js index 341a933066..c634aac293 100644 --- a/frappe/public/js/frappe/form/controls/datetime.js +++ b/frappe/public/js/frappe/form/controls/datetime.js @@ -1,4 +1,16 @@ frappe.ui.form.ControlDatetime = class ControlDatetime extends frappe.ui.form.ControlDate { + set_formatted_input(value) { + if (this.timepicker_only) return; + if (!this.datepicker) return; + if (!value) { + this.datepicker.clear(); + return; + } else if (value === "Today") { + value = this.get_now_date(); + } + + this.$input && this.$input.val(this.format_for_input(value)); + } set_date_options() { super.set_date_options(); this.today_text = __("Now"); @@ -14,10 +26,36 @@ frappe.ui.form.ControlDatetime = class ControlDatetime extends frappe.ui.form.Co get_now_date() { return frappe.datetime.now_datetime(true); } + parse(value) { + if (value) { + value = frappe.datetime.user_to_str(value, false); + + if (!frappe.datetime.is_timezone_same()) { + value = frappe.datetime.convert_to_system_tz(value, true); + } + + return value; + } + } + format_for_input(value) { + if (!value) return ""; + + let m = frappe.datetime.is_timezone_same(); + if (!frappe.datetime.is_timezone_same()) { + m = frappe.datetime.convert_to_user_tz(value, true) + value = frappe.datetime.convert_to_user_tz(value, true); + } + + return frappe.datetime.str_to_user(value, false); + } set_description() { - const { description } = this.df; - const { time_zone } = frappe.sys_defaults; - if (!this.df.hide_timezone && !frappe.datetime.is_timezone_same()) { + const description = this.df.description; + const time_zone = frappe.boot.time_zone ? frappe.boot.time_zone.user_time_zone : frappe.sys_defaults.time_zone; + + if (!this.df.hide_timezone) { + // Always show the timezone when rendering the Datetime field since the datetime value will + // always be in system_time_zone rather then local time. + if (!description) { this.df.description = time_zone; } else if (!description.includes(time_zone)) { diff --git a/frappe/public/js/frappe/utils/datetime.js b/frappe/public/js/frappe/utils/datetime.js index 0c6fea2986..3d00762c94 100644 --- a/frappe/public/js/frappe/utils/datetime.js +++ b/frappe/public/js/frappe/utils/datetime.js @@ -13,33 +13,44 @@ frappe.provide("frappe.datetime"); $.extend(frappe.datetime, { convert_to_user_tz: function(date, format) { // format defaults to true - if(frappe.sys_defaults.time_zone) { - var date_obj = moment.tz(date, frappe.sys_defaults.time_zone).local(); + // Converts the datetime string to system time zone first since the database only stores datetime in + // system time zone and then convert the string to user time zone(from User doctype). + let date_obj = null; + if (frappe.boot.time_zone && frappe.boot.time_zone.system_time_zone && frappe.boot.time_zone.user_time_zone) { + date_obj = moment.tz(date, frappe.boot.time_zone.system_time_zone) + .clone() + .tz(frappe.boot.time_zone.user_time_zone); } else { - var date_obj = moment(date); + date_obj = moment(date); } - return (format===false) ? date_obj : date_obj.format(frappe.defaultDatetimeFormat); + return format===false ? date_obj : date_obj.format(frappe.defaultDatetimeFormat); }, convert_to_system_tz: function(date, format) { // format defaults to true - - if(frappe.sys_defaults.time_zone) { - var date_obj = moment(date).tz(frappe.sys_defaults.time_zone); + // Converts the datetime string to user time zone (from User doctype) first since this fn is called in datetime which accepts datetime + // in user time zone then convert the string to user time zone. + // This is done so that only one timezone is present in database and we do not end up storing local timezone since it changes + // as per the location of user. + let date_obj = null; + if (frappe.boot.time_zone && frappe.boot.time_zone.system_time_zone && frappe.boot.time_zone.user_time_zone) { + date_obj = moment.tz(date, frappe.boot.time_zone.user_time_zone) + .clone() + .tz(frappe.boot.time_zone.system_time_zone); } else { - var date_obj = moment(date); + date_obj = moment(date); } - return (format===false) ? date_obj : date_obj.format(frappe.defaultDatetimeFormat); + return format===false ? date_obj : date_obj.format(frappe.defaultDatetimeFormat); }, is_timezone_same: function() { - if(frappe.sys_defaults.time_zone) { - return moment().tz(frappe.sys_defaults.time_zone).utcOffset() === moment().utcOffset(); - } else { - return true; + if (frappe.boot.time_zone && frappe.boot.time_zone.system_time_zone && frappe.boot.time_zone.user_time_zone) { + return moment().tz(frappe.boot.time_zone.system_time_zone).utcOffset() === moment().tz(frappe.boot.time_zone.user_time_zone).utcOffset(); } + + return true; }, str_to_obj: function(d) { @@ -186,18 +197,18 @@ $.extend(frappe.datetime, { }, _date: function(format, as_obj = false) { - const time_zone = frappe.sys_defaults && frappe.sys_defaults.time_zone; - let date; - if (time_zone) { - date = moment.tz(time_zone); - } else { - date = moment(); - } - if (as_obj) { - return frappe.datetime.moment_to_date_obj(date); - } else { - return date.format(format); - } + /** + * Whenever we are getting now_date/datetime, always make sure dates are fetched using usertime zone. + * This is to make sure that time is as per user time zone set in User doctype, If a user had to change the timezone, + * we will end up having multiple timezone by not honouring timezone in User doctype. + * This will make sure that at any point we know which timezone the user if following and not have random timezone + * when the timezone of the local machine changes. + */ + let time_zone = frappe.boot.time_zone.user_time_zone || frappe.boot.time_zone.system_time_zone; + if (!time_zone) time_zone = frappe.sys_defaults.time_zone; + + let date = moment.tz(time_zone); + return as_obj ? frappe.datetime.moment_to_date_obj(date) : date.format(format); }, moment_to_date_obj: function(moment) { From 02640b791f24a9d089e8a65852bcf2ca0f12bf9e Mon Sep 17 00:00:00 2001 From: hrwx Date: Tue, 15 Jun 2021 23:31:30 +0530 Subject: [PATCH 002/246] fix: check if frappe.sys_defaults.time_zone exists --- .../core/doctype/system_settings/system_settings.js | 6 ++++++ frappe/core/doctype/user/user.js | 6 ++++++ frappe/core/doctype/user/user.py | 1 - frappe/public/js/frappe/form/controls/datetime.js | 11 ++++++----- frappe/public/js/frappe/form/controls/time.js | 2 +- frappe/public/js/frappe/form/formatters.js | 8 ++------ frappe/public/js/frappe/utils/datetime.js | 7 +++---- 7 files changed, 24 insertions(+), 17 deletions(-) diff --git a/frappe/core/doctype/system_settings/system_settings.js b/frappe/core/doctype/system_settings/system_settings.js index c0c9074cbc..aefe3786bd 100644 --- a/frappe/core/doctype/system_settings/system_settings.js +++ b/frappe/core/doctype/system_settings/system_settings.js @@ -32,5 +32,11 @@ frappe.ui.form.on("System Settings", { frm.set_value('prepared_report_expiry_period', 7); } } + }, + after_save: function(frm) { + if (frappe.boot.time_zone && frappe.boot.time_zone.system_time_zone !== frm.doc.time_zone) { + // Clear cache after saving to refresh the values of time_zone + frappe.ui.toolbar.clear_cache(); + } } }); diff --git a/frappe/core/doctype/user/user.js b/frappe/core/doctype/user/user.js index 7c9e00d6bc..819684cdfe 100644 --- a/frappe/core/doctype/user/user.js +++ b/frappe/core/doctype/user/user.js @@ -271,6 +271,12 @@ frappe.ui.form.on('User', { } } }); + }, + after_save: function(frm) { + if (frappe.boot.time_zone && frappe.boot.time_zone.user_time_zone !== frm.doc.time_zone) { + // Clear cache after saving to refresh the values of time_zone + frappe.ui.toolbar.clear_cache(); + } } }); diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py index 93963c6015..cf2b045c6d 100644 --- a/frappe/core/doctype/user/user.py +++ b/frappe/core/doctype/user/user.py @@ -599,7 +599,6 @@ class User(Document): from frappe.utils import get_time_zone if not self.time_zone: - frappe.msgprint(_("User Time Zone was not set, defaulting to System Time Zone."), title=_("User Time Zone")) self.time_zone = get_time_zone() @frappe.whitelist() diff --git a/frappe/public/js/frappe/form/controls/datetime.js b/frappe/public/js/frappe/form/controls/datetime.js index c634aac293..a8b19c4ece 100644 --- a/frappe/public/js/frappe/form/controls/datetime.js +++ b/frappe/public/js/frappe/form/controls/datetime.js @@ -30,7 +30,7 @@ frappe.ui.form.ControlDatetime = class ControlDatetime extends frappe.ui.form.Co if (value) { value = frappe.datetime.user_to_str(value, false); - if (!frappe.datetime.is_timezone_same()) { + if (!frappe.datetime.is_system_time_zone()) { value = frappe.datetime.convert_to_system_tz(value, true); } @@ -40,9 +40,7 @@ frappe.ui.form.ControlDatetime = class ControlDatetime extends frappe.ui.form.Co format_for_input(value) { if (!value) return ""; - let m = frappe.datetime.is_timezone_same(); - if (!frappe.datetime.is_timezone_same()) { - m = frappe.datetime.convert_to_user_tz(value, true) + if (!frappe.datetime.is_system_time_zone()) { value = frappe.datetime.convert_to_user_tz(value, true); } @@ -50,7 +48,7 @@ frappe.ui.form.ControlDatetime = class ControlDatetime extends frappe.ui.form.Co } set_description() { const description = this.df.description; - const time_zone = frappe.boot.time_zone ? frappe.boot.time_zone.user_time_zone : frappe.sys_defaults.time_zone; + const time_zone = this.get_user_time_zone(); if (!this.df.hide_timezone) { // Always show the timezone when rendering the Datetime field since the datetime value will @@ -64,6 +62,9 @@ frappe.ui.form.ControlDatetime = class ControlDatetime extends frappe.ui.form.Co } super.set_description(); } + get_user_time_zone() { + return frappe.boot.time_zone ? frappe.boot.time_zone.user_time_zone : frappe.sys_defaults.time_zone; + } set_datepicker() { super.set_datepicker(); if (this.datepicker.opts.timeFormat.indexOf('s') == -1) { diff --git a/frappe/public/js/frappe/form/controls/time.js b/frappe/public/js/frappe/form/controls/time.js index a7b6645681..f7fcc4c618 100644 --- a/frappe/public/js/frappe/form/controls/time.js +++ b/frappe/public/js/frappe/form/controls/time.js @@ -71,7 +71,7 @@ frappe.ui.form.ControlTime = class ControlTime extends frappe.ui.form.ControlDat set_description() { const { description } = this.df; const { time_zone } = frappe.sys_defaults; - if (!frappe.datetime.is_timezone_same()) { + if (!frappe.datetime.is_system_time_zone()) { if (!description) { this.df.description = time_zone; } else if (!description.includes(time_zone)) { diff --git a/frappe/public/js/frappe/form/formatters.js b/frappe/public/js/frappe/form/formatters.js index b9a838688d..caecd65336 100644 --- a/frappe/public/js/frappe/form/formatters.js +++ b/frappe/public/js/frappe/form/formatters.js @@ -167,12 +167,8 @@ frappe.form.formatters = { }, Datetime: function(value) { if(value) { - var m = moment(frappe.datetime.convert_to_user_tz(value)); - if(frappe.boot.sysdefaults.time_zone) { - m = m.tz(frappe.boot.sysdefaults.time_zone); - } - return m.format(frappe.boot.sysdefaults.date_format.toUpperCase() - + ' ' + frappe.boot.sysdefaults.time_format); + return moment(frappe.datetime.convert_to_user_tz(value)) + .format(frappe.boot.sysdefaults.date_format.toUpperCase() + ' ' + frappe.boot.sysdefaults.time_format); } else { return ""; } diff --git a/frappe/public/js/frappe/utils/datetime.js b/frappe/public/js/frappe/utils/datetime.js index 3d00762c94..ae7308cf7b 100644 --- a/frappe/public/js/frappe/utils/datetime.js +++ b/frappe/public/js/frappe/utils/datetime.js @@ -45,7 +45,7 @@ $.extend(frappe.datetime, { return format===false ? date_obj : date_obj.format(frappe.defaultDatetimeFormat); }, - is_timezone_same: function() { + is_system_time_zone: function() { if (frappe.boot.time_zone && frappe.boot.time_zone.system_time_zone && frappe.boot.time_zone.user_time_zone) { return moment().tz(frappe.boot.time_zone.system_time_zone).utcOffset() === moment().tz(frappe.boot.time_zone.user_time_zone).utcOffset(); } @@ -204,10 +204,9 @@ $.extend(frappe.datetime, { * This will make sure that at any point we know which timezone the user if following and not have random timezone * when the timezone of the local machine changes. */ - let time_zone = frappe.boot.time_zone.user_time_zone || frappe.boot.time_zone.system_time_zone; - if (!time_zone) time_zone = frappe.sys_defaults.time_zone; - + let time_zone = frappe.boot.time_zone ? frappe.boot.time_zone.user_time_zone || frappe.boot.time_zone.system_time_zone : frappe.sys_defaults.time_zone; let date = moment.tz(time_zone); + return as_obj ? frappe.datetime.moment_to_date_obj(date) : date.format(format); }, From 8a8fbad0f3232c113043b9a86b0a6571a77f88d1 Mon Sep 17 00:00:00 2001 From: leela Date: Wed, 25 Aug 2021 09:16:11 +0530 Subject: [PATCH 003/246] refactor: getting submitted linked reference docs --- frappe/desk/form/linked_with.py | 344 +++++++++++++++++++++++++------ frappe/tests/test_linked_with.py | 113 ++++++++++ 2 files changed, 399 insertions(+), 58 deletions(-) create mode 100644 frappe/tests/test_linked_with.py diff --git a/frappe/desk/form/linked_with.py b/frappe/desk/form/linked_with.py index ae48b7fc6b..0b5fe58f93 100644 --- a/frappe/desk/form/linked_with.py +++ b/frappe/desk/form/linked_with.py @@ -2,6 +2,9 @@ # MIT License. See license.txt import json from collections import defaultdict +from os import link +import itertools +from typing import List import frappe import frappe.desk.form.load @@ -12,69 +15,296 @@ from frappe.modules import load_doctype_module @frappe.whitelist() -def get_submitted_linked_docs(doctype, name, docs=None, visited=None): +def get_submitted_linked_docs(doctype: str, name: str) -> List[tuple]: + """ Get all the nested submitted documents those are present in referencing tables (dependent tables). + + :param doctype: Document type + :param name: Name of the document + + Usecase: + * User should be able to cancel the linked documents along with the one user trying to cancel. + + Case1: If document sd1-n1 (document name n1 from sumittable doctype sd1) is linked to sd2-n2 and sd2-n2 is linked to sd3-n3, + Getting submittable linked docs of `sd1-n1`should give both sd2-n2 and sd3-n3. + Case2: If document sd1-n1 (document name n1 from sumittable doctype sd1) is linked to d2-n2 and d2-n2 is linked to sd3-n3, + Getting submittable linked docs of `sd1-n1`should give None. (because d2-n2 is not a submittable doctype) + Case3: If document sd1-n1 (document name n1 from submittable doctype sd1) is linked to d2-n2 & sd2-n2. d2-n2 is linked to sd3-n3. + Getting submittable linked docs of `sd1-n1`should give sd2-n2. + + Logic: + ----- + 1. We can find linked documents only if we know how the doctypes are related. + 2. As we need only submittable documents, we can limit doctype relations search to submittable doctypes by + finding the relationships(Foreign key references) across submittable doctypes. + 3. Searching for links is going to be a tree like structure where at every level, + you will be finding documents using parent document and parent document links. """ - Get all nested submitted linked doctype linkinfo + tree = SubmittableDocumentTree(doctype, name) + visited_documents = tree.get_all_children() + docs = [] - Arguments: - doctype (str) - The doctype for which get all linked doctypes - name (str) - The docname for which get all linked doctypes + for dt, names in visited_documents.items(): + docs.extend([{'doctype': dt, 'name': name, 'docstatus': 1} for name in names]) - Keyword Arguments: - docs (list of dict) - (Optional) Get list of dictionary for linked doctype. - - Returns: - dict - Return list of documents and link count - """ - - if not docs: - docs = [] - - if not visited: - visited = {} - - if doctype not in visited: - visited[doctype] = [] - - if name in visited[doctype]: - return - - linkinfo = get_linked_doctypes(doctype) - linked_docs = get_linked_docs(doctype, name, linkinfo) - - link_count = 0 - visited[doctype].append(name) - - for link_doctype, link_names in linked_docs.items(): - - for link in link_names: - if link['name'] == name: - continue - - docinfo = link.update({"doctype": link_doctype}) - validated_doc = validate_linked_doc(docinfo) - - if not validated_doc: - continue - - link_count += 1 - - links = get_submitted_linked_docs(link_doctype, link.name, docs, visited) - if links: - docs.append({ - "doctype": link_doctype, - "name": link.name, - "docstatus": link.docstatus, - "link_count": links.get("count") - }) - - # sort linked documents by ascending number of links - docs.sort(key=lambda doc: doc.get("link_count")) return { "docs": docs, - "count": link_count + "count": len(docs) } +class SubmittableDocumentTree: + def __init__(self, doctype: str, name: str): + """Construct a tree for the submitable linked documents. + + * Node has properties like doctype and docnames. Represented as Node(doctype, docnames). + * Nodes are linked by doctype relationships like table, link and dynamic links. + * Node is referenced(linked) by many other documents and those are the child nodes. + + NOTE: child document is a property of child node (not same as Frappe child docs of a table field). + """ + self.root_doctype = doctype + self.root_docname = name + + # Documents those are yet to be visited for linked documents. + self.to_be_visited_documents = {doctype: [name]} + self.visited_documents = defaultdict(list) + + self._submittable_doctypes = None # All submittable doctypes in the system + self._references_across_doctypes = None # doctype wise links/references + + def get_all_children(self): + """Get all nodes of a tree except the root node (all the nested submitted + documents those are present in referencing tables (dependent tables). + """ + while self.to_be_visited_documents: + next_level_children = defaultdict(list) + for parent_dt in list(self.to_be_visited_documents): + parent_docs = self.to_be_visited_documents.get(parent_dt) + if not parent_docs: + del self.to_be_visited_documents[parent_dt] + continue + + child_docs = self.get_next_level_children(parent_dt, parent_docs) + self.visited_documents[parent_dt].extend(parent_docs) + for linked_dt, linked_names in child_docs.items(): + not_visited_child_docs = set(linked_names) - set(self.visited_documents.get(linked_dt, [])) + next_level_children[linked_dt].extend(not_visited_child_docs) + + self.to_be_visited_documents = next_level_children + + # Remove root node from visited documents + if self.root_docname in self.visited_documents.get(self.root_doctype, []): + self.visited_documents[self.root_doctype].remove(self.root_docname) + + return self.visited_documents + + def get_next_level_children(self, parent_dt, parent_names): + """Get immediate children of a Node(parent_dt, parent_names) + """ + referencing_fields = self.get_doctype_references(parent_dt) + + child_docs = defaultdict(list) + for field in referencing_fields: + links = get_referencing_documents(parent_dt, parent_names.copy(), field, get_parent_if_child_table_doc=True, + parent_filters=[('docstatus', '=', 1)], allowed_parents=self.get_link_sources()) or {} + for dt, names in links.items(): + child_docs[dt].extend(names) + return child_docs + + def get_doctype_references(self, doctype): + """Get references for a given document. + """ + if self._references_across_doctypes is None: + get_links_to = self.get_document_sources() + limit_link_doctypes = self.get_link_sources() + self._references_across_doctypes = get_references_across_doctypes( + get_links_to, limit_link_doctypes) + return self._references_across_doctypes.get(doctype, []) + + def get_document_sources(self): + """Returns list of doctypes from where we access submittable documents. + """ + return list(set(self.get_link_sources() + [self.root_doctype])) + + def get_link_sources(self): + """limit doctype links to these doctypes. + """ + return list(set(self.get_submittable_doctypes()) - set(get_exempted_doctypes() or [])) + + def get_submittable_doctypes(self) -> List[str]: + """Returns list of submittable doctypes. + """ + if not self._submittable_doctypes: + self._submittable_doctypes = frappe.db.get_list('DocType', {'is_submittable': 1}, pluck='name') + return self._submittable_doctypes + + +def get_child_tables_of_doctypes(doctypes: List[str]=None): + """Returns child tables by doctype. + """ + filters=[['fieldtype','=', 'Table']] + filters_for_docfield = filters + filters_for_customfield = filters + + if doctypes: + filters_for_docfield = filters + [['parent', 'in', tuple(doctypes)]] + filters_for_customfield = filters + [['dt', 'in', tuple(doctypes)]] + + links = frappe.get_all("DocField", + fields=["parent", "fieldname", "options as child_table"], + filters=filters_for_docfield, + as_list=1) + + links+= frappe.get_all("Custom Field", + fields=["dt as parent", "fieldname", "options as child_table"], + filters=filters_for_customfield, + as_list=1) + + child_tables_by_doctype = defaultdict(list) + for doctype, fieldname, child_table in links: + child_tables_by_doctype[doctype].append( + {'doctype': doctype, 'fieldname': fieldname, 'child_table': child_table}) + return child_tables_by_doctype + + +def get_references_across_doctypes(to_doctypes: List[str]=None, limit_link_doctypes: List[str]=None) -> List: + """Find doctype wise foreign key references. + + :param to_doctypes: Get links of these doctypes. + :param limit_link_doctypes: limit links to these doctypes. + + * Include child table, link and dynamic link references. + """ + if limit_link_doctypes: + child_tables_by_doctype = get_child_tables_of_doctypes(limit_link_doctypes) + all_child_tables = [each['child_table'] for each in itertools.chain(*child_tables_by_doctype.values())] + limit_link_doctypes = limit_link_doctypes + all_child_tables + else: + child_tables_by_doctype = get_child_tables_of_doctypes() + all_child_tables = [each['child_table'] for each in itertools.chain(*child_tables_by_doctype.values())] + + references_by_link_fields = get_references_across_doctypes_by_link_field(to_doctypes, limit_link_doctypes) + references_by_dlink_fields = get_references_across_doctypes_by_dynamic_link_field(to_doctypes, limit_link_doctypes) + + references = references_by_link_fields.copy() + for k, v in references_by_dlink_fields.items(): + references.setdefault(k, []).extend(v) + + for doctype, links in references.items(): + for link in links: + link['is_child'] = (link['doctype'] in all_child_tables) + return references + + +def get_references_across_doctypes_by_link_field(to_doctypes: List[str]=None, limit_link_doctypes: List[str]=None): + """Find doctype wise foreign key references based on link fields. + + :param to_doctypes: Get links to these doctypes. + :param limit_link_doctypes: limit links to these doctypes. + """ + filters=[['fieldtype','=', 'Link']] + + if to_doctypes: + filters += [['options', 'in', tuple(to_doctypes)]] + + filters_for_docfield = filters[:] + filters_for_customfield = filters[:] + + if limit_link_doctypes: + filters_for_docfield += [['parent', 'in', tuple(limit_link_doctypes)]] + filters_for_customfield += [['dt', 'in', tuple(limit_link_doctypes)]] + + links = frappe.get_all("DocField", + fields=["parent", "fieldname", "options as linked_to"], + filters=filters_for_docfield, + as_list=1) + + links+= frappe.get_all("Custom Field", + fields=["dt as parent", "fieldname", "options as linked_to"], + filters=filters_for_customfield, + as_list=1) + + links_by_doctype = defaultdict(list) + for doctype, fieldname, linked_to in links: + links_by_doctype[linked_to].append({'doctype': doctype, 'fieldname': fieldname}) + return links_by_doctype + + +def get_references_across_doctypes_by_dynamic_link_field(to_doctypes: List[str]=None, limit_link_doctypes: List[str]=None): + """Find doctype wise foreign key references based on dynamic link fields. + + :param to_doctypes: Get links to these doctypes. + :param limit_link_doctypes: limit links to these doctypes. + """ + + filters=[['fieldtype','=', 'Dynamic Link']] + + filters_for_docfield = filters[:] + filters_for_customfield = filters[:] + + if limit_link_doctypes: + filters_for_docfield += [['parent', 'in', tuple(limit_link_doctypes)]] + filters_for_customfield += [['dt', 'in', tuple(limit_link_doctypes)]] + + # find dynamic links of parents + links = frappe.get_all("DocField", + fields=["parent as doctype", "fieldname", "options as doctype_fieldname"], + filters=filters_for_docfield, + as_list=1) + + links += frappe.get_all("Custom Field", + fields=["dt as doctype", "fieldname", "options as doctype_fieldname"], + filters=filters_for_customfield, + as_list=1) + + links_by_doctype = defaultdict(list) + for doctype, fieldname, doctype_fieldname in links: + try: + filters = [[doctype_fieldname, 'in', to_doctypes]] if to_doctypes else [] + for linked_to in frappe.db.get_all(doctype, pluck=doctype_fieldname, filters = filters, distinct=1): + if linked_to: + links_by_doctype[linked_to].append({'doctype': doctype, 'fieldname': fieldname, 'doctype_fieldname': doctype_fieldname}) + except frappe.db.ProgrammingError as e: + # TODO: FIXME + continue + return links_by_doctype + +def get_referencing_documents(reference_doctype: str, reference_names: List[str], + link_info: dict, get_parent_if_child_table_doc: bool=True, + parent_filters: List[list]=None, child_filters=None, allowed_parents=None): + """Get linked documents based on link_info. + + :param reference_doctype: reference doctype to find links + :param reference_names: reference document names to find links for + :param link_info: linking details to get the linked documents + Ex: {'doctype': 'Purchase Invoice Advance', 'fieldname': 'reference_name', + 'doctype_fieldname': 'reference_type', 'is_child': True} + :param get_parent_if_child_table_doc: Get parent record incase linked document is a child table record. + :param parent_filters: filters to apply on if not a child table. + :param child_filters: apply filters if it is a child table. + :param allowed_parents: list of parents allowed in case of get_parent_if_child_table_doc + is enabled. + """ + from_table = link_info['doctype'] + filters = [[link_info['fieldname'], 'in', tuple(reference_names)]] + if link_info.get('doctype_fieldname'): + filters.append([link_info['doctype_fieldname'], '=', reference_doctype]) + + if not link_info.get('is_child'): + filters.extend(parent_filters or []) + return {from_table: frappe.db.get_all(from_table, filters, pluck='name')} + + + filters.extend(child_filters or []) + res = frappe.db.get_all(from_table, filters = filters, fields = ['name', 'parenttype', 'parent']) + documents = defaultdict(list) + + for parent, rows in itertools.groupby(res, key = lambda row: row['parenttype']): + if allowed_parents and parent not in allowed_parents: + continue + filters = (parent_filters or []) + [['name', 'in', tuple([row.parent for row in rows])]] + documents[parent].extend(frappe.db.get_all(parent, filters=filters, pluck='name') or []) + return documents + @frappe.whitelist() def cancel_all_linked_docs(docs, ignore_doctypes_on_cancel_all=[]): @@ -107,7 +337,6 @@ def validate_linked_doc(docinfo, ignore_doctypes_on_cancel_all=[]): Returns: bool: True if linked document passes all validations, else False """ - #ignore doctype to cancel if docinfo.get("doctype") in ignore_doctypes_on_cancel_all: return False @@ -130,7 +359,6 @@ def validate_linked_doc(docinfo, ignore_doctypes_on_cancel_all=[]): def get_exempted_doctypes(): """ Get list of doctypes exempted from being auto-cancelled """ - auto_cancel_exempt_doctypes = [] for doctypes in frappe.get_hooks('auto_cancel_exempted_doctypes'): auto_cancel_exempt_doctypes.append(doctypes) diff --git a/frappe/tests/test_linked_with.py b/frappe/tests/test_linked_with.py new file mode 100644 index 0000000000..64da8e51e0 --- /dev/null +++ b/frappe/tests/test_linked_with.py @@ -0,0 +1,113 @@ +import frappe, unittest +from frappe.core.doctype.doctype.test_doctype import new_doctype +from frappe.desk.form import linked_with + + +class TestLinkedWith(unittest.TestCase): + def setUp(self): + parent_doc = new_doctype("Parent Doc") + parent_doc.is_submittable = 1 + parent_doc.insert() + + child_doc1 = new_doctype("Child Doc1", + fields=[ + { + "label": "Parent Doc", + "fieldname": "parent_doc", + "fieldtype": "Link", + "options": "Parent Doc" + }, + { + "label": "Reference field", + "fieldname": "reference_name", + "fieldtype": "Dynamic Link", + "options": "reference_doctype" + }, + { + "label": "Reference Doctype", + "fieldname": "reference_doctype", + "fieldtype": "Link", + "options": "DocType" + } + + ], unique=0) + child_doc1.is_submittable = 1 + child_doc1.insert() + + child_doc2 = new_doctype("Child Doc2", + fields=[ + { + "label": "Parent Doc", + "fieldname": "parent_doc", + "fieldtype": "Link", + "options": "Parent Doc" + }, + { + "label": "Child Doc1", + "fieldname": "child_doc1", + "fieldtype": "Link", + "options": "Child Doc1" + } + + ], unique=0) + child_doc2.is_submittable = 1 + child_doc2.insert() + + def tearDown(self): + for doctype in ['Parent Doc', 'Child Doc1', 'Child Doc2']: + frappe.delete_doc("DocType", doctype) + + def test_get_doctype_references_by_link_field(self): + references = linked_with.get_references_across_doctypes_by_link_field(to_doctypes = ['Parent Doc']) + self.assertEqual(len(references['Parent Doc']), 3) + self.assertIn({'doctype': 'Child Doc1', 'fieldname': 'parent_doc'}, references['Parent Doc']) + self.assertIn({'doctype': 'Child Doc2', 'fieldname': 'parent_doc'}, references['Parent Doc']) + + references = linked_with.get_references_across_doctypes_by_link_field(to_doctypes = ['Child Doc1']) + self.assertEqual(len(references['Child Doc1']), 2) + self.assertIn({'doctype': 'Child Doc2', 'fieldname': 'child_doc1'}, references['Child Doc1']) + + references = linked_with.get_references_across_doctypes_by_link_field( + to_doctypes = ['Child Doc1', 'Parent Doc'], limit_link_doctypes=['Child Doc1']) + self.assertEqual(len(references['Child Doc1']), 1) + self.assertEqual(len(references['Parent Doc']), 1) + self.assertIn({'doctype': 'Child Doc1', 'fieldname': 'parent_doc'}, references['Parent Doc']) + + def test_get_doctype_references_by_dlink_field(self): + references = linked_with.get_references_across_doctypes_by_dynamic_link_field( + to_doctypes = ['Parent Doc'], limit_link_doctypes = ['Parent Doc', 'Child Doc1', 'Child Doc2']) + self.assertFalse(references) + + parent_record = frappe.get_doc({'doctype': 'Parent Doc'}).insert() + + child_record = frappe.get_doc({ + 'doctype': 'Child Doc1', + 'reference_doctype': 'Parent Doc', + 'reference_name': parent_record.name + }).insert() + + references = linked_with.get_references_across_doctypes_by_dynamic_link_field( + to_doctypes = ['Parent Doc'], limit_link_doctypes = ['Parent Doc', 'Child Doc1', 'Child Doc2']) + + self.assertEqual(len(references['Parent Doc']), 1) + self.assertEqual(references['Parent Doc'][0]['doctype'], 'Child Doc1') + self.assertEqual(references['Parent Doc'][0]['doctype_fieldname'], 'reference_doctype') + + child_record.delete() + parent_record.delete() + + def test_get_submitted_linked_docs(self): + parent_record = frappe.get_doc({'doctype': 'Parent Doc'}).insert() + + child_record = frappe.get_doc({ + 'doctype': 'Child Doc1', + 'reference_doctype': 'Parent Doc', + 'reference_name': parent_record.name, + 'docstatus': 1 + }).insert() + + linked_docs = linked_with.get_submitted_linked_docs(parent_record.doctype, parent_record.name)["docs"] + self.assertIn(child_record.name,linked_docs[0]['name']) + child_record.cancel() + child_record.delete() + parent_record.delete() From f608fbeddd8ae9ee21b38a4c4a5615d8c7bc99fc Mon Sep 17 00:00:00 2001 From: mtraeber Date: Tue, 16 Mar 2021 16:05:28 +0100 Subject: [PATCH 004/246] feat: sync mutliple IMAP folders in `Email Account` When working with IMAP accounts, frappe should allow the user to choose multiple folders to look for new mails. This helps users to separate their frappe-related email from other conversations. Use cases range from sieve filters in the mail server that stuff incoming mail in various mail folders to people manually sorting their e-mail. In both cases, we can have different import policies for different folders, and we can avoid importing unrelated email. Created a new child table `IMAP Folder` with following fields: - Folder Name (user-modifiable) - Append To (user-modifiable) - UIDVALIDITY (hidden) - UIDNEXT (hidden) Doctype `Email Account` and `receive.py` code adjusted so that emails with the changes are processed correctly and Frappe only logs in to the imap server once per sync. Created a patch that copies the data from the old fields into the new child table with `INBOX` as default `folder_name`. This keeps existing setups working without manual changes. The original fields - uidvalidity - uidnext - append_to are still available for the pop3 setups. In IMAP, these fields are hidden user and not used. Added a test case in `Email Account` that validates data to make sure a IMAP folder is provided if the use_imap is true. Also added some code formatting changes in email_account.js to get rid of sider checks failures that block this change --- .../doctype/communication/communication.json | 8 ++ .../doctype/communication/communication.py | 1 + .../communication/test_communication.py | 1 + .../doctype/email_account/email_account.js | 30 +++++--- .../doctype/email_account/email_account.json | 17 ++++- .../doctype/email_account/email_account.py | 73 ++++++++++++------- .../email_account/test_email_account.py | 16 ++++ .../doctype/email_account/test_records.json | 3 +- frappe/email/doctype/imap_folder/__init__.py | 0 .../doctype/imap_folder/imap_folder.json | 53 ++++++++++++++ .../email/doctype/imap_folder/imap_folder.py | 8 ++ frappe/email/receive.py | 54 ++++++++------ frappe/email/test_smtp.py | 8 +- frappe/patches.txt | 1 + frappe/patches/v14_0/copy_mail_data.py | 33 +++++++++ 15 files changed, 241 insertions(+), 65 deletions(-) create mode 100644 frappe/email/doctype/imap_folder/__init__.py create mode 100644 frappe/email/doctype/imap_folder/imap_folder.json create mode 100644 frappe/email/doctype/imap_folder/imap_folder.py create mode 100644 frappe/patches/v14_0/copy_mail_data.py diff --git a/frappe/core/doctype/communication/communication.json b/frappe/core/doctype/communication/communication.json index 849df66a5f..9e154146b3 100644 --- a/frappe/core/doctype/communication/communication.json +++ b/frappe/core/doctype/communication/communication.json @@ -51,6 +51,7 @@ "email_inbox", "message_id", "uid", + "imap_folder", "email_status", "has_attachment", "feedback_section", @@ -382,6 +383,13 @@ "label": "Timeline Links", "options": "Communication Link", "permlevel": 2 + }, + { + "fieldname": "imap_folder", + "fieldtype": "Data", + "hidden": 1, + "label": "IMAP Folder", + "read_only": 1 } ], "icon": "fa fa-comment", diff --git a/frappe/core/doctype/communication/communication.py b/frappe/core/doctype/communication/communication.py index 66bb3909da..5714d122eb 100644 --- a/frappe/core/doctype/communication/communication.py +++ b/frappe/core/doctype/communication/communication.py @@ -42,6 +42,7 @@ class Communication(Document, CommunicationEmailMixin): "action": "Read", "communication": self.name, "uid": self.uid, + "imap_folder": self.imap_folder, "email_account": self.email_account }).insert(ignore_permissions=True) frappe.db.commit() diff --git a/frappe/core/doctype/communication/test_communication.py b/frappe/core/doctype/communication/test_communication.py index b0c8e1fcee..f26e70771b 100644 --- a/frappe/core/doctype/communication/test_communication.py +++ b/frappe/core/doctype/communication/test_communication.py @@ -291,6 +291,7 @@ def create_email_account(): "unreplied_for_mins": 20, "send_notification_to": "test_comm@example.com", "pop3_server": "pop.test.example.com", + "imap_folder": [{"folder_name": "INBOX", "append_to": "ToDo"}], "no_remaining":"0", "enable_automatic_linking": 1 }).insert(ignore_permissions=True) diff --git a/frappe/email/doctype/email_account/email_account.js b/frappe/email/doctype/email_account/email_account.js index 277bf43eb6..54f0d2372d 100644 --- a/frappe/email/doctype/email_account/email_account.js +++ b/frappe/email/doctype/email_account/email_account.js @@ -109,6 +109,15 @@ frappe.ui.form.on("Email Account", { onload: function(frm) { frm.set_df_property("append_to", "only_select", true); frm.set_query("append_to", "frappe.email.doctype.email_account.email_account.get_append_to"); + frm.set_query("append_to", "imap_folder", function() { + return { + query: "frappe.email.doctype.email_account.email_account.get_append_to" + }; + }); + if (frm.doc.__islocal) { + frm.add_child("imap_folder", {"folder_name": "INBOX"}); + frm.refresh_field("imap_folder"); + } }, refresh: function(frm) { @@ -117,7 +126,7 @@ frappe.ui.form.on("Email Account", { frm.events.notify_if_unreplied(frm); frm.events.show_gmail_message_for_less_secure_apps(frm); - if(frappe.route_flags.delete_user_from_locals && frappe.route_flags.linked_user) { + if (frappe.route_flags.delete_user_from_locals && frappe.route_flags.linked_user) { delete frappe.route_flags.delete_user_from_locals; delete locals['User'][frappe.route_flags.linked_user]; } @@ -125,7 +134,7 @@ frappe.ui.form.on("Email Account", { show_gmail_message_for_less_secure_apps: function(frm) { frm.dashboard.clear_headline(); - if(frm.doc.service==="GMail") { + if (frm.doc.service==="GMail") { frm.dashboard.set_headline_alert('Gmail will only work if you allow access for less secure \ apps in Gmail settings. Read this for details'); @@ -137,8 +146,8 @@ frappe.ui.form.on("Email Account", { frm.events.update_domain(frm); }, - update_domain: function(frm){ - if (!frm.doc.email_id && !frm.doc.service){ + update_domain: function(frm) { + if (!frm.doc.email_id && !frm.doc.service) { return; } @@ -148,7 +157,7 @@ frappe.ui.form.on("Email Account", { args: { "email_id": frm.doc.email_id }, - callback: function (r) { + callback: function(r) { if (r.message) { frm.events.set_domain_fields(frm, r.message); } @@ -157,7 +166,7 @@ frappe.ui.form.on("Email Account", { }, set_domain_fields: function(frm, args) { - if(!args){ + if (!args) { args = frappe.route_flags.set_domain_values? frappe.route_options: {}; } @@ -172,10 +181,8 @@ frappe.ui.form.on("Email Account", { email_sync_option: function(frm) { // confirm if the ALL sync option is selected - if(frm.doc.email_sync_option == "ALL"){ - var msg = __("You are selecting Sync Option as ALL, It will resync all \ - read as well as unread message from server. This may also cause the duplication\ - of Communication (emails)."); + if (frm.doc.email_sync_option == "ALL") { + var msg = __("You are selecting Sync Option as ALL, It will resync all read as well as unread message from server. This may also cause the duplication of Communication (emails)."); frappe.confirm(msg, null, function() { frm.set_value("email_sync_option", "UNSEEN"); }); @@ -184,8 +191,7 @@ frappe.ui.form.on("Email Account", { warn_autoreply_on_incoming: function(frm) { if (frm.doc.enable_incoming && frm.doc.enable_auto_reply && frm.doc.__islocal) { - var msg = __("Enabling auto reply on an incoming email account will send automated replies \ - to all the synchronized emails. Do you wish to continue?"); + var msg = __("Enabling auto reply on an incoming email account will send automated replies to all the synchronized emails. Do you wish to continue?"); frappe.confirm(msg, null, function() { frm.set_value("enable_auto_reply", 0); frappe.show_alert({message: __("Disabled Auto Reply"), indicator: "blue"}); diff --git a/frappe/email/doctype/email_account/email_account.json b/frappe/email/doctype/email_account/email_account.json index e20f38c74a..bf9a79529b 100644 --- a/frappe/email/doctype/email_account/email_account.json +++ b/frappe/email/doctype/email_account/email_account.json @@ -31,6 +31,8 @@ "attachment_limit", "email_sync_option", "initial_sync_count", + "section_break_25", + "imap_folder", "section_break_12", "append_emails_to_sent_folder", "append_to", @@ -204,7 +206,7 @@ "label": "Attachment Limit (MB)" }, { - "depends_on": "enable_incoming", + "depends_on": "eval: doc.enable_incoming && !doc.use_imap", "description": "Append as communication against this DocType (must have fields, \"Status\", \"Subject\")", "fieldname": "append_to", "fieldtype": "Link", @@ -562,6 +564,18 @@ "fieldname": "account_section", "fieldtype": "Section Break", "label": "Account" + }, + { + "depends_on": "eval: doc.use_imap && doc.enable_incoming", + "fieldname": "imap_folder", + "fieldtype": "Table", + "label": "IMAP Folder", + "options": "IMAP Folder" + }, + { + "fieldname": "section_break_25", + "fieldtype": "Section Break", + "label": "IMAP Details" } ], "icon": "fa fa-inbox", @@ -571,6 +585,7 @@ "modified_by": "Administrator", "module": "Email", "name": "Email Account", + "naming_rule": "By fieldname", "owner": "Administrator", "permissions": [ { diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index d90c56d90d..d5b683c9e3 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -67,6 +67,10 @@ class EmailAccount(Document): else: self.login_id = None + # validate the imap settings + if self.enable_incoming and self.use_imap and len(self.imap_folder) <= 0: + frappe.throw(_("You need to set one IMAP folder for {0}").format(frappe.bold(self.email_id))) + duplicate_email_account = frappe.get_all("Email Account", filters={ "email_id": self.email_id, "name": ("!=", self.name) @@ -100,10 +104,11 @@ class EmailAccount(Document): for e in self.get_unreplied_notification_emails(): validate_email_address(e, True) - if self.enable_incoming and self.append_to: - valid_doctypes = [d[0] for d in get_append_to()] - if self.append_to not in valid_doctypes: - frappe.throw(_("Append To can be one of {0}").format(comma_or(valid_doctypes))) + for folder in self.imap_folder: + if self.enable_incoming and folder.append_to: + valid_doctypes = [d[0] for d in get_append_to()] + if folder.append_to not in valid_doctypes: + frappe.throw(_("Append To can be one of {0}").format(comma_or(valid_doctypes))) def validate_smtp_conn(self): if not self.smtp_server: @@ -177,13 +182,13 @@ class EmailAccount(Document): return None args = frappe._dict({ + "email_account_name": self.email_account_name, "email_account": self.name, "host": self.email_server, "use_ssl": self.use_ssl, "username": getattr(self, "login_id", None) or self.email_id, "use_imap": self.use_imap, "email_sync_rule": email_sync_rule, - "uid_validity": self.uidvalidity, "incoming_port": get_port(self), "initial_sync_count": self.initial_sync_count or 100 }) @@ -457,6 +462,14 @@ class EmailAccount(Document): """retrive and return inbound mails. """ + mails = [] + + def process_mail(messages): + for index, message in enumerate(messages.get("latest_messages", [])): + uid = messages['uid_list'][index] if messages.get('uid_list') else None + seen_status = 1 if messages.get('seen_status', {}).get(uid) == 'SEEN' else 0 + mails.append(InboundMail(message, self, uid, seen_status)) + if frappe.local.flags.in_test: return [InboundMail(msg, self) for msg in test_mails or []] @@ -466,17 +479,23 @@ class EmailAccount(Document): email_sync_rule = self.build_email_sync_rule() try: email_server = self.get_incoming_server(in_receive=True, email_sync_rule=email_sync_rule) - messages = email_server.get_messages() or {} + if self.use_imap: + # process all given imap folder + for folder in self.imap_folder: + email_server.select_imap_folder(folder.folder_name) + email_server.settings['uid_validity'] = folder.uidvalidity + messages = email_server.get_messages(folder=folder.folder_name) or {} + process_mail(messages) + else: + # process the pop3 account + messages = email_server.get_messages() or {} + process_mail(messages) + # close connection to mailserver + email_server.logout() except Exception: frappe.log_error(title=_("Error while connecting to email account {0}").format(self.name)) return [] - mails = [] - for index, message in enumerate(messages.get("latest_messages", [])): - uid = messages['uid_list'][index] if messages.get('uid_list') else None - seen_status = 1 if messages.get('seen_status', {}).get(uid)=='SEEN' else 0 - mails.append(InboundMail(message, self, uid, seen_status)) - return mails def handle_bad_emails(self, uid, raw, reason): @@ -547,23 +566,22 @@ class EmailAccount(Document): else: return self.email_sync_option or "UNSEEN" - def mark_emails_as_read_unread(self): + def mark_emails_as_read_unread(self, email_server=None, folder_name="INBOX"): """ mark Email Flag Queue of self.email_account mails as read""" - if not self.use_imap: return - flags = frappe.db.sql("""select name, communication, uid, action from - `tabEmail Flag Queue` where is_completed=0 and email_account={email_account} - """.format(email_account=frappe.db.escape(self.name)), as_dict=True) + flags = frappe.db.sql("""select name, communication, uid, action, imap_folder from + `tabEmail Flag Queue` where is_completed=0 and email_account={email_account} and imap_folder={folder_name} + """.format(email_account=frappe.db.escape(self.name),folder_name=frappe.db.escape(folder_name)), as_dict=True) uid_list = { flag.get("uid", None): flag.get("action", "Read") for flag in flags } if flags and uid_list: - email_server = self.get_incoming_server() + if not email_server: + email_server = self.get_incoming_server() if not email_server: return - - email_server.update_flag(uid_list=uid_list) + email_server.update_flag(folder_name, uid_list=uid_list) # mark communication as read docnames = ",".join("'%s'"%flag.get("communication") for flag in flags \ @@ -651,15 +669,19 @@ def test_internet(host="8.8.8.8", port=53, timeout=3): def notify_unreplied(): """Sends email notifications if there are unreplied Communications and `notify_if_unreplied` is set as true.""" - for email_account in frappe.get_all("Email Account", "name", filters={"enable_incoming": 1, "notify_if_unreplied": 1}): email_account = frappe.get_doc("Email Account", email_account.name) - if email_account.append_to: + if email_account.use_imap: + append_to = [folder.get("append_to") for folder in email_account.imap_folder] + else: + append_to = email_account.append_to + + if append_to: # get open communications younger than x mins, for given doctype for comm in frappe.get_all("Communication", "name", filters=[ {"sent_or_received": "Received"}, - {"reference_doctype": email_account.append_to}, + {"reference_doctype": ("in", append_to)}, {"unread_notification_sent": 0}, {"email_account":email_account.name}, {"creation": ("<", datetime.now() - timedelta(seconds = (email_account.unreplied_for_mins or 30) * 60))}, @@ -702,9 +724,6 @@ def pull_from_email_account(email_account): email_account = frappe.get_doc("Email Account", email_account) email_account.receive() - # mark Email Flag Queue mail as read - email_account.mark_emails_as_read_unread() - def get_max_email_uid(email_account): # get maximum uid of emails max_uid = 1 @@ -800,4 +819,4 @@ def set_email_password(email_account, user, password): frappe.db.rollback() return False - return True \ No newline at end of file + return True diff --git a/frappe/email/doctype/email_account/test_email_account.py b/frappe/email/doctype/email_account/test_email_account.py index 21dc4b84c4..51f722a7af 100644 --- a/frappe/email/doctype/email_account/test_email_account.py +++ b/frappe/email/doctype/email_account/test_email_account.py @@ -229,6 +229,22 @@ class TestEmailAccount(unittest.TestCase): email_account.handle_bad_emails(uid=-1, raw=mail_content, reason="Testing") self.assertTrue(frappe.db.get_value("Unhandled Email", {'message_id': message_id})) + def test_imap_folder(self): + # assert tests if imap_folder >= 1 and imap is checked + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + + self.assertTrue(email_account.use_imap) + self.assertTrue(email_account.enable_incoming) + self.assertTrue(len(email_account.imap_folder) > 0) + + def test_imap_folder_missing(self): + # Test the Exception in validate() that verifies the imap_folder list + email_account = frappe.get_doc("Email Account", "_Test Email Account 1") + email_account.imap_folder = [] + + with self.assertRaises(Exception): + email_account.validate() + class TestInboundMail(unittest.TestCase): @classmethod def setUpClass(cls): diff --git a/frappe/email/doctype/email_account/test_records.json b/frappe/email/doctype/email_account/test_records.json index 15ca2a886e..450895d7a6 100644 --- a/frappe/email/doctype/email_account/test_records.json +++ b/frappe/email/doctype/email_account/test_records.json @@ -4,7 +4,6 @@ "is_global": 1, "doctype": "Email Account", "domain":"example.com", - "append_to": "ToDo", "email_account_name": "_Test Email Account 1", "enable_outgoing": 1, "smtp_server": "test.example.com", @@ -20,6 +19,8 @@ "send_notification_to": "test_unreplied@example.com", "pop3_server": "pop.test.example.com", "no_remaining":"0", + "append_to": "ToDo", + "imap_folder": [{"folder_name": "INBOX", "append_to": "ToDo"}], "track_email_status": 1 }, { diff --git a/frappe/email/doctype/imap_folder/__init__.py b/frappe/email/doctype/imap_folder/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/frappe/email/doctype/imap_folder/imap_folder.json b/frappe/email/doctype/imap_folder/imap_folder.json new file mode 100644 index 0000000000..bab50dea39 --- /dev/null +++ b/frappe/email/doctype/imap_folder/imap_folder.json @@ -0,0 +1,53 @@ +{ + "actions": [], + "creation": "2021-09-21 11:38:13.521979", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "folder_name", + "append_to", + "uidvalidity", + "uidnext" + ], + "fields": [ + { + "fieldname": "folder_name", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Folder Name", + "reqd": 1 + }, + { + "fieldname": "append_to", + "fieldtype": "Link", + "in_list_view": 1, + "label": "Append To", + "options": "DocType" + }, + { + "fieldname": "uidvalidity", + "fieldtype": "Data", + "hidden": 1, + "label": "UIDVALIDITY" + }, + { + "fieldname": "uidnext", + "fieldtype": "Data", + "hidden": 1, + "label": "UIDNEXT" + } + ], + "index_web_pages_for_search": 1, + "istable": 1, + "links": [], + "modified": "2021-09-21 11:53:00.811236", + "modified_by": "Administrator", + "module": "Email", + "name": "IMAP Folder", + "owner": "Administrator", + "permissions": [], + "sort_field": "modified", + "sort_order": "DESC", + "track_changes": 1 +} diff --git a/frappe/email/doctype/imap_folder/imap_folder.py b/frappe/email/doctype/imap_folder/imap_folder.py new file mode 100644 index 0000000000..b0bb36b677 --- /dev/null +++ b/frappe/email/doctype/imap_folder/imap_folder.py @@ -0,0 +1,8 @@ +# Copyright (c) 2021, Frappe Technologies and contributors +# For license information, please see license.txt + +# import frappe +from frappe.model.document import Document + +class IMAPFolder(Document): + pass diff --git a/frappe/email/receive.py b/frappe/email/receive.py index a755ec5e74..d2b4376dfd 100644 --- a/frappe/email/receive.py +++ b/frappe/email/receive.py @@ -111,7 +111,17 @@ class EmailServer: frappe.msgprint(_('Invalid User Name or Support Password. Please rectify and try again.')) raise - def get_messages(self): + def select_imap_folder(self, folder): + self.imap.select(folder) + + def logout(self): + if cint(self.settings.use_imap): + self.imap.logout() + else: + self.pop.quit() + return + + def get_messages(self, folder="INBOX"): """Returns new email messages in a list.""" if not (self.check_mails() or self.connect()): return [] @@ -126,7 +136,8 @@ class EmailServer: self.latest_messages = [] self.seen_status = {} self.uid_reindexed = False - uid_list = email_list = self.get_new_mails() + + uid_list = email_list = self.get_new_mails(folder) if not email_list: return @@ -160,13 +171,6 @@ class EmailServer: else: raise - finally: - # no matter the exception, pop should quit if connected - if cint(self.settings.use_imap): - self.imap.logout() - else: - self.pop.quit() - out = { "latest_messages": self.latest_messages } if self.settings.use_imap: out.update({ @@ -177,15 +181,15 @@ class EmailServer: return out - def get_new_mails(self): + def get_new_mails(self, folder): """Return list of new mails""" if cint(self.settings.use_imap): email_list = [] - self.check_imap_uidvalidity() + self.check_imap_uidvalidity(folder) readonly = False if self.settings.email_sync_rule == "UNSEEN" else True - self.imap.select("Inbox", readonly=readonly) + self.imap.select(folder, readonly=readonly) response, message = self.imap.uid('search', None, self.settings.email_sync_rule) if message[0]: email_list = message[0].split() @@ -194,11 +198,11 @@ class EmailServer: return email_list - def check_imap_uidvalidity(self): + def check_imap_uidvalidity(self, folder): # compare the UIDVALIDITY of email account and imap server uid_validity = self.settings.uid_validity - response, message = self.imap.status("Inbox", "(UIDVALIDITY UIDNEXT)") + response, message = self.imap.status(folder, "(UIDVALIDITY UIDNEXT)") current_uid_validity = self.parse_imap_response("UIDVALIDITY", message[0]) or 0 uidnext = int(self.parse_imap_response("UIDNEXT", message[0]) or "1") @@ -210,10 +214,18 @@ class EmailServer: """update `tabCommunication` set uid=-1 where communication_medium='Email' and email_account=%s""", (self.settings.email_account,) ) - frappe.db.sql( - """update `tabEmail Account` set uidvalidity=%s, uidnext=%s where - name=%s""", (current_uid_validity, uidnext, self.settings.email_account) - ) + if self.settings.use_imap: + # new update for the IMAP Folder DoctType + frappe.db.sql( + """update `tabIMAP Folder` set uidvalidity=%s, uidnext=%s where + parent=%s and folder_name=%s""", + (current_uid_validity, uidnext, self.settings.email_account_name, folder) + ) + else: + frappe.db.sql( + """update `tabEmail Account` set uidvalidity=%s, uidnext=%s where + name=%s""", (current_uid_validity, uidnext, self.settings.email_account) + ) # uid validity not found pulling emails for first time if not uid_validity: @@ -232,6 +244,7 @@ class EmailServer: def parse_imap_response(self, cmd, response): pattern = r"(?<={cmd} )[0-9]*".format(cmd=cmd) match = re.search(pattern, response.decode('utf-8'), re.U | re.I) + if match: return match.group(0) else: @@ -340,16 +353,15 @@ class EmailServer: return error_msg - def update_flag(self, uid_list={}): + def update_flag(self, folder, uid_list={}): """ set all uids mails the flag as seen """ - if not uid_list: return if not self.connect(): return - self.imap.select("Inbox") + self.imap.select(folder) for uid, operation in uid_list.items(): if not uid: continue diff --git a/frappe/email/test_smtp.py b/frappe/email/test_smtp.py index 58e4fdd8a6..127bdd44ce 100644 --- a/frappe/email/test_smtp.py +++ b/frappe/email/test_smtp.py @@ -16,11 +16,12 @@ class TestSMTP(unittest.TestCase): make_server(port, 0, 1) def test_get_email_account(self): - existing_email_accounts = frappe.get_all("Email Account", fields = ["name", "enable_outgoing", "default_outgoing", "append_to"]) + existing_email_accounts = frappe.get_all("Email Account", fields = ["name", "enable_outgoing", "default_outgoing","append_to", "use_imap"]) unset_details = { "enable_outgoing": 0, "default_outgoing": 0, - "append_to": None + "append_to": None, + "use_imap": 0 } for email_account in existing_email_accounts: frappe.db.set_value('Email Account', email_account['name'], unset_details) @@ -60,7 +61,8 @@ def create_email_account(email_id, password, enable_outgoing, default_outgoing=0 "enable_incoming": 1, "append_to":append_to, "is_dummy_password": 1, - "smtp_server": "localhost" + "smtp_server": "localhost", + "use_imap": 0 } email_account = frappe.new_doc('Email Account') diff --git a/frappe/patches.txt b/frappe/patches.txt index 41ca1a1724..37fa5379d6 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -183,3 +183,4 @@ frappe.patches.v13_0.update_notification_channel_if_empty frappe.patches.v14_0.drop_data_import_legacy frappe.patches.v14_0.rename_cancelled_documents frappe.patches.v14_0.update_workspace2 # 25.08.2021 +frappe.patches.v14_0.copy_mail_data #08.03.21 diff --git a/frappe/patches/v14_0/copy_mail_data.py b/frappe/patches/v14_0/copy_mail_data.py new file mode 100644 index 0000000000..d3a4baca88 --- /dev/null +++ b/frappe/patches/v14_0/copy_mail_data.py @@ -0,0 +1,33 @@ +from __future__ import unicode_literals +import frappe + + +def execute(): + # patch for all Email Account with the flag use_imap + for email_account in frappe.get_list("Email Account", filters={"enable_incoming": 1, "use_imap": 1}): + # get all data from Email Account + doc = frappe.get_doc("Email Account", email_account.name) + + imap_list = [folder.folder_name for folder in doc.imap_folder] + # and append the old data to the child table + if doc.uidvalidity or doc.uidnext and "INBOX" not in imap_list: + doc.append("imap_folder", { + "folder_name": "INBOX", + "append_to": doc.append_to, + "uid_validity": doc.uidvalidity, + "uidnext": doc.uidnext, + }) + + doc.save() + + frappe.db.sql( + """ + update + `tabEmail Flag Queue` + set + imap_folder = "INBOX" + where + email_account = '%s' + and imap_folder is NULL + """ % (doc.name) + ) From 8ea6158690e405a46f683c240f62052ed3701fb1 Mon Sep 17 00:00:00 2001 From: Aradhya-Tripathi Date: Thu, 14 Oct 2021 16:13:49 +0530 Subject: [PATCH 005/246] refactor: removed aggregation functions at db level --- frappe/database/database.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index e98cc22f41..88c3241303 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -766,18 +766,6 @@ class Database(object): except Exception: return None - def min(self, dt, fieldname, filters=None, **kwargs): - return self.query.build_conditions(dt, filters=filters).select(Min(Column(fieldname))).run(**kwargs)[0][0] or 0 - - def max(self, dt, fieldname, filters=None, **kwargs): - return self.query.build_conditions(dt, filters=filters).select(Max(Column(fieldname))).run(**kwargs)[0][0] or 0 - - def avg(self, dt, fieldname, filters=None, **kwargs): - return self.query.build_conditions(dt, filters=filters).select(Avg(Column(fieldname))).run(**kwargs)[0][0] or 0 - - def sum(self, dt, fieldname, filters=None, **kwargs): - return self.query.build_conditions(dt, filters=filters).select(Sum(Column(fieldname))).run(**kwargs)[0][0] or 0 - def count(self, dt, filters=None, debug=False, cache=False): """Returns `COUNT(*)` for given DocType and filters.""" if cache and not filters: From 18e2ab7e084447749dd189103a42ab8337ad497e Mon Sep 17 00:00:00 2001 From: Aradhya-Tripathi Date: Thu, 14 Oct 2021 16:54:44 +0530 Subject: [PATCH 006/246] refactor: moved aggregation functions from safe_exec --- frappe/utils/safe_exec.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/frappe/utils/safe_exec.py b/frappe/utils/safe_exec.py index 8f6b33a838..a439ef0a8b 100644 --- a/frappe/utils/safe_exec.py +++ b/frappe/utils/safe_exec.py @@ -163,10 +163,6 @@ def get_safe_globals(): get_default=frappe.db.get_default, exists=frappe.db.exists, count=frappe.db.count, - min=frappe.db.min, - max=frappe.db.max, - avg=frappe.db.avg, - sum=frappe.db.sum, escape=frappe.db.escape, sql=read_sql, commit=frappe.db.commit, From 44216958f9276b9c6ba11db834523e58767533de Mon Sep 17 00:00:00 2001 From: mtraeber Date: Thu, 28 Oct 2021 11:08:10 +0200 Subject: [PATCH 007/246] setting `use_imap` flag --- frappe/email/doctype/email_account/test_email_account.py | 1 + 1 file changed, 1 insertion(+) diff --git a/frappe/email/doctype/email_account/test_email_account.py b/frappe/email/doctype/email_account/test_email_account.py index 51f722a7af..6d26f9f070 100644 --- a/frappe/email/doctype/email_account/test_email_account.py +++ b/frappe/email/doctype/email_account/test_email_account.py @@ -25,6 +25,7 @@ class TestEmailAccount(unittest.TestCase): email_account = frappe.get_doc("Email Account", "_Test Email Account 1") email_account.db_set("enable_incoming", 1) email_account.db_set("enable_auto_reply", 1) + email_account.db_set("use_imap", 1) @classmethod def tearDownClass(cls): From 151768d1d9c8b1b6dbe06d2c5d4c450f0630c7e0 Mon Sep 17 00:00:00 2001 From: mtraeber Date: Wed, 3 Nov 2021 09:36:00 +0100 Subject: [PATCH 008/246] added method param again after merge --- frappe/email/receive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/email/receive.py b/frappe/email/receive.py index bba31b122f..e7989660b8 100644 --- a/frappe/email/receive.py +++ b/frappe/email/receive.py @@ -353,7 +353,7 @@ class EmailServer: return error_msg - def update_flag(self, uid_list=None): + def update_flag(self, folder, uid_list=None): """ set all uids mails the flag as seen """ if not uid_list: return From 1f70c27e9f3ec11ce04bd857d7cae6f2ff2356bc Mon Sep 17 00:00:00 2001 From: hrwx Date: Mon, 15 Nov 2021 14:33:28 +0000 Subject: [PATCH 009/246] chore: rename timezone keys --- frappe/boot.py | 4 ++-- .../system_settings/system_settings.js | 6 ++--- frappe/core/doctype/user/user.js | 6 ++--- frappe/core/doctype/user/user.py | 14 +++++------- .../js/frappe/form/controls/datetime.js | 2 +- frappe/public/js/frappe/utils/datetime.js | 22 +++++++++++-------- 6 files changed, 26 insertions(+), 28 deletions(-) diff --git a/frappe/boot.py b/frappe/boot.py index fd7564d75a..e671d8b37d 100644 --- a/frappe/boot.py +++ b/frappe/boot.py @@ -329,6 +329,6 @@ def get_notification_settings(): def set_time_zone(bootinfo): bootinfo.time_zone = { - "system_time_zone": get_time_zone(), - "user_time_zone": bootinfo.get("user_info", {}).get(frappe.session.user, {}).get("time_zone", None) or get_time_zone() + "system": get_time_zone(), + "user": bootinfo.get("user_info", {}).get(frappe.session.user, {}).get("time_zone", None) or get_time_zone() } diff --git a/frappe/core/doctype/system_settings/system_settings.js b/frappe/core/doctype/system_settings/system_settings.js index aefe3786bd..0164a1a683 100644 --- a/frappe/core/doctype/system_settings/system_settings.js +++ b/frappe/core/doctype/system_settings/system_settings.js @@ -34,9 +34,7 @@ frappe.ui.form.on("System Settings", { } }, after_save: function(frm) { - if (frappe.boot.time_zone && frappe.boot.time_zone.system_time_zone !== frm.doc.time_zone) { - // Clear cache after saving to refresh the values of time_zone - frappe.ui.toolbar.clear_cache(); - } + // Clear cache after saving to refresh the values of boot. + frappe.ui.toolbar.clear_cache(); } }); diff --git a/frappe/core/doctype/user/user.js b/frappe/core/doctype/user/user.js index 48dc2d1672..681080b2b3 100644 --- a/frappe/core/doctype/user/user.js +++ b/frappe/core/doctype/user/user.js @@ -274,10 +274,8 @@ frappe.ui.form.on('User', { }); }, after_save: function(frm) { - if (frappe.boot.time_zone && frappe.boot.time_zone.user_time_zone !== frm.doc.time_zone) { - // Clear cache after saving to refresh the values of time_zone - frappe.ui.toolbar.clear_cache(); - } + // Clear cache after saving to refresh the values of boot. + frappe.ui.toolbar.clear_cache(); } }); diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py index fd19f4d82e..76bdbbbeb8 100644 --- a/frappe/core/doctype/user/user.py +++ b/frappe/core/doctype/user/user.py @@ -7,7 +7,7 @@ import frappe.defaults import frappe.permissions from frappe.model.document import Document from frappe.utils import (cint, flt, has_gravatar, escape_html, format_datetime, - now_datetime, get_formatted_email, today) + now_datetime, get_formatted_email, today, get_time_zone) from frappe import throw, msgprint, _ from frappe.utils.password import update_password as _update_password, check_password, get_password_reset_limit from frappe.desk.notifications import clear_notifications @@ -231,11 +231,11 @@ class User(Document): def validate_share(self, docshare): pass # if docshare.user == self.name: - # if self.user_type=="System User": - # if docshare.share != 1: - # frappe.throw(_("Sorry! User should have complete access to their own record.")) - # else: - # frappe.throw(_("Sorry! Sharing with Website User is prohibited.")) + # if self.user_type=="System User": + # if docshare.share != 1: + # frappe.throw(_("Sorry! User should have complete access to their own record.")) + # else: + # frappe.throw(_("Sorry! Sharing with Website User is prohibited.")) def send_password_notification(self, new_password): try: @@ -592,8 +592,6 @@ class User(Document): return user def set_time_zone(self): - from frappe.utils import get_time_zone - if not self.time_zone: self.time_zone = get_time_zone() diff --git a/frappe/public/js/frappe/form/controls/datetime.js b/frappe/public/js/frappe/form/controls/datetime.js index b69f40e9c4..3142f1bf0f 100644 --- a/frappe/public/js/frappe/form/controls/datetime.js +++ b/frappe/public/js/frappe/form/controls/datetime.js @@ -63,7 +63,7 @@ frappe.ui.form.ControlDatetime = class ControlDatetime extends frappe.ui.form.Co super.set_description(); } get_user_time_zone() { - return frappe.boot.time_zone ? frappe.boot.time_zone.user_time_zone : frappe.sys_defaults.time_zone; + return frappe.boot.time_zone ? frappe.boot.time_zone.user : frappe.sys_defaults.time_zone; } set_datepicker() { super.set_datepicker(); diff --git a/frappe/public/js/frappe/utils/datetime.js b/frappe/public/js/frappe/utils/datetime.js index 71fdbbb897..c85cbd42e7 100644 --- a/frappe/public/js/frappe/utils/datetime.js +++ b/frappe/public/js/frappe/utils/datetime.js @@ -16,10 +16,10 @@ $.extend(frappe.datetime, { // Converts the datetime string to system time zone first since the database only stores datetime in // system time zone and then convert the string to user time zone(from User doctype). let date_obj = null; - if (frappe.boot.time_zone && frappe.boot.time_zone.system_time_zone && frappe.boot.time_zone.user_time_zone) { - date_obj = moment.tz(date, frappe.boot.time_zone.system_time_zone) + if (frappe.boot.time_zone && frappe.boot.time_zone.system && frappe.boot.time_zone.user) { + date_obj = moment.tz(date, frappe.boot.time_zone.system) .clone() - .tz(frappe.boot.time_zone.user_time_zone); + .tz(frappe.boot.time_zone.user); } else { date_obj = moment(date); } @@ -34,10 +34,10 @@ $.extend(frappe.datetime, { // This is done so that only one timezone is present in database and we do not end up storing local timezone since it changes // as per the location of user. let date_obj = null; - if (frappe.boot.time_zone && frappe.boot.time_zone.system_time_zone && frappe.boot.time_zone.user_time_zone) { - date_obj = moment.tz(date, frappe.boot.time_zone.user_time_zone) + if (frappe.boot.time_zone && frappe.boot.time_zone.system && frappe.boot.time_zone.user) { + date_obj = moment.tz(date, frappe.boot.time_zone.user) .clone() - .tz(frappe.boot.time_zone.system_time_zone); + .tz(frappe.boot.time_zone.system); } else { date_obj = moment(date); } @@ -46,13 +46,17 @@ $.extend(frappe.datetime, { }, is_system_time_zone: function() { - if (frappe.boot.time_zone && frappe.boot.time_zone.system_time_zone && frappe.boot.time_zone.user_time_zone) { - return moment().tz(frappe.boot.time_zone.system_time_zone).utcOffset() === moment().tz(frappe.boot.time_zone.user_time_zone).utcOffset(); + if (frappe.boot.time_zone && frappe.boot.time_zone.system && frappe.boot.time_zone.user) { + return moment().tz(frappe.boot.time_zone.system).utcOffset() === moment().tz(frappe.boot.time_zone.user).utcOffset(); } return true; }, + is_timezone_same: function() { + return frappe.datetime.is_system_time_zone(); + }, + str_to_obj: function(d) { return moment(d, frappe.defaultDatetimeFormat)._d; }, @@ -204,7 +208,7 @@ $.extend(frappe.datetime, { * This will make sure that at any point we know which timezone the user if following and not have random timezone * when the timezone of the local machine changes. */ - let time_zone = frappe.boot.time_zone ? frappe.boot.time_zone.user_time_zone || frappe.boot.time_zone.system_time_zone : frappe.sys_defaults.time_zone; + let time_zone = frappe.boot.time_zone ? frappe.boot.time_zone.user || frappe.boot.time_zone.system : frappe.sys_defaults.time_zone; let date = moment.tz(time_zone); return as_obj ? frappe.datetime.moment_to_date_obj(date) : date.format(format); From 576efed7f5a9cd29d01e746bcd506bde4292a811 Mon Sep 17 00:00:00 2001 From: hrwx Date: Mon, 15 Nov 2021 14:35:56 +0000 Subject: [PATCH 010/246] chore: move timezone clear cache to on_update --- frappe/core/doctype/system_settings/system_settings.js | 8 +++++--- frappe/core/doctype/user/user.js | 8 +++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/frappe/core/doctype/system_settings/system_settings.js b/frappe/core/doctype/system_settings/system_settings.js index 0164a1a683..4eeab0274b 100644 --- a/frappe/core/doctype/system_settings/system_settings.js +++ b/frappe/core/doctype/system_settings/system_settings.js @@ -33,8 +33,10 @@ frappe.ui.form.on("System Settings", { } } }, - after_save: function(frm) { - // Clear cache after saving to refresh the values of boot. - frappe.ui.toolbar.clear_cache(); + on_update: function(frm) { + if (frappe.boot.time_zone && frappe.boot.time_zone.system !== frm.doc.time_zone) { + // Clear cache after saving to refresh the values of boot. + frappe.ui.toolbar.clear_cache(); + } } }); diff --git a/frappe/core/doctype/user/user.js b/frappe/core/doctype/user/user.js index 681080b2b3..79c2665a05 100644 --- a/frappe/core/doctype/user/user.js +++ b/frappe/core/doctype/user/user.js @@ -273,9 +273,11 @@ frappe.ui.form.on('User', { } }); }, - after_save: function(frm) { - // Clear cache after saving to refresh the values of boot. - frappe.ui.toolbar.clear_cache(); + on_update: function(frm) { + if (frappe.boot.time_zone && frappe.boot.time_zone.user !== frm.doc.time_zone) { + // Clear cache after saving to refresh the values of boot. + frappe.ui.toolbar.clear_cache(); + } } }); From 008954aed875063959e0e8dd9c235b59da14d80f Mon Sep 17 00:00:00 2001 From: Aradhya-Tripathi Date: Tue, 16 Nov 2021 15:12:18 +0530 Subject: [PATCH 011/246] feat: added Pseudocolumn to qb utils --- frappe/query_builder/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/frappe/query_builder/utils.py b/frappe/query_builder/utils.py index 386ddda751..b7d22fc354 100644 --- a/frappe/query_builder/utils.py +++ b/frappe/query_builder/utils.py @@ -8,6 +8,7 @@ from pypika.queries import Column import frappe from .builder import MariaDB, Postgres +from pypika.terms import PseudoColumn class db_type_is(Enum): From 6a8515af85374ccbdc9a877aa391ac86ce515170 Mon Sep 17 00:00:00 2001 From: Aradhya-Tripathi Date: Tue, 16 Nov 2021 15:12:39 +0530 Subject: [PATCH 012/246] feat: Added pluck to get_values --- frappe/database/database.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index a7dd9b6b66..09d19f21de 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -335,7 +335,7 @@ class Database(object): return self.get_value(doctype, filters, "*", as_dict=as_dict, cache=cache) def get_value(self, doctype, filters=None, fieldname="name", ignore=None, as_dict=False, - debug=False, order_by=None, cache=False, for_update=False, run=True): + debug=False, order_by=None, cache=False, for_update=False, run=True, **kwargs): """Returns a document property or list of properties. :param doctype: DocType name. @@ -362,7 +362,7 @@ class Database(object): """ ret = self.get_values(doctype, filters, fieldname, ignore, as_dict, debug, - order_by, cache=cache, for_update=for_update, run=run) + order_by, cache=cache, for_update=for_update, run=run, **kwargs) if not run: return ret @@ -370,7 +370,7 @@ class Database(object): return ((len(ret[0]) > 1 or as_dict) and ret[0] or ret[0][0]) if ret else None def get_values(self, doctype, filters=None, fieldname="name", ignore=None, as_dict=False, - debug=False, order_by=None, update=None, cache=False, for_update=False, run=True): + debug=False, order_by=None, update=None, cache=False, for_update=False, run=True, **kwargs): """Returns multiple document properties. :param doctype: DocType name. @@ -396,7 +396,7 @@ class Database(object): if isinstance(filters, list): order_by = order_by or "modified_desc" - out = self._get_value_for_many_names(doctype, filters, fieldname, debug=debug, run=run) + out = self._get_value_for_many_names(doctype, filters, fieldname, debug=debug, run=run, **kwargs) else: fields = fieldname @@ -408,9 +408,11 @@ class Database(object): if (filters is not None) and (filters!=doctype or doctype=="DocType"): try: - order_by = order_by or "modified" + if not kwargs.get("no_order"): + order_by = order_by or "modified" + kwargs.pop("no_order", None) out = self._get_values_from_table( - fields, filters, doctype, as_dict, debug, order_by, update, for_update=for_update, run=run + fields, filters, doctype, as_dict, debug, order_by, update, for_update=for_update, run=run, **kwargs ) except Exception as e: if ignore and (frappe.db.is_missing_column(e) or frappe.db.is_table_missing(e)): @@ -540,7 +542,7 @@ class Database(object): return self.get_single_value(*args, **kwargs) def _get_values_from_table(self, fields, filters, doctype, as_dict, debug, order_by=None, - update=None, for_update=False, run=True): + update=None, for_update=False, run=True, **kwargs): field_objects = [] if not isinstance(fields, Criterion): @@ -563,17 +565,17 @@ class Database(object): if fields=="*": query = criterion.select(fields) as_dict = True - r = self.sql(query, as_dict=as_dict, debug=debug, update=update, run=run) + r = self.sql(query, as_dict=as_dict, debug=debug, update=update, run=run, **kwargs) return r - def _get_value_for_many_names(self, doctype, names, field, debug=False, run=True): + def _get_value_for_many_names(self, doctype, names, field, debug=False, run=True, **kwargs): names = list(filter(None, names)) if names: return self.get_all(doctype, fields=['name', field], filters=[['name', 'in', names]], - debug=debug, as_list=1, run=run) + debug=debug, as_list=1, run=run, **kwargs), else: return {} From 09e7af70abd0cc5d7421fc1e9621e8b5d933fa21 Mon Sep 17 00:00:00 2001 From: Aradhya-Tripathi Date: Tue, 16 Nov 2021 15:12:58 +0530 Subject: [PATCH 013/246] refactor: converted raw queries --- .../doctype/custom_field/custom_field.py | 5 +-- frappe/sessions.py | 43 ++++++++++++++----- 2 files changed, 34 insertions(+), 14 deletions(-) diff --git a/frappe/custom/doctype/custom_field/custom_field.py b/frappe/custom/doctype/custom_field/custom_field.py index 8c22d3c45c..8f7b21dd24 100644 --- a/frappe/custom/doctype/custom_field/custom_field.py +++ b/frappe/custom/doctype/custom_field/custom_field.py @@ -8,6 +8,7 @@ from frappe import _ from frappe.model.document import Document from frappe.model.docfield import supports_translation from frappe.model import core_doctypes_list +from frappe.query_builder.functions import IfNull class CustomField(Document): def autoname(self): @@ -115,9 +116,7 @@ def get_fields_label(doctype=None): def create_custom_field_if_values_exist(doctype, df): df = frappe._dict(df) if df.fieldname in frappe.db.get_table_columns(doctype) and \ - frappe.db.sql("""select count(*) from `tab{doctype}` - where ifnull({fieldname},'')!=''""".format(doctype=doctype, fieldname=df.fieldname))[0][0]: - + frappe.db.count(dt=doctype, filters=IfNull(df.fieldname, "") != ""): create_custom_field(doctype, df) def create_custom_field(doctype, df, ignore_validate=False): diff --git a/frappe/sessions.py b/frappe/sessions.py index 9a0f19df80..9b96435093 100644 --- a/frappe/sessions.py +++ b/frappe/sessions.py @@ -17,6 +17,8 @@ import redis from urllib.parse import unquote from frappe.cache_manager import clear_user_cache from frappe.query_builder import Order, DocType +from frappe.query_builder.utils import PseudoColumn +from frappe.query_builder.functions import Now @frappe.whitelist() @@ -97,12 +99,23 @@ def clear_all_sessions(reason=None): def get_expired_sessions(): '''Returns list of expired sessions''' + sessions = DocType("Sessions") + expired = [] for device in ("desktop", "mobile"): - expired += frappe.db.sql_list("""SELECT `sid` - FROM `tabSessions` - WHERE (NOW() - `lastupdate`) > %s - AND device = %s""", (get_expiry_period_for_query(device), device)) + expired.extend( + frappe.db.get_values( + sessions, + filters=( + PseudoColumn(f"({Now() - sessions.lastupdate})") + > get_expiry_period_for_query(device) + ) + & (sessions.device == device), + fieldname="sid", + no_order=True, + pluck=True, + ) + ) return expired @@ -305,14 +318,22 @@ class Session: return data and data.data def get_session_data_from_db(self): - self.device = frappe.db.sql('SELECT `device` FROM `tabSessions` WHERE `sid`=%s', self.sid) - self.device = self.device and self.device[0][0] or 'desktop' + sessions = DocType("Sessions") - rec = frappe.db.sql(""" - SELECT `user`, `sessiondata` - FROM `tabSessions` WHERE `sid`=%s AND - (NOW() - lastupdate) < %s - """, (self.sid, get_expiry_period_for_query(self.device))) + self.device = frappe.db.get_values( + sessions, filters=sessions.sid == self.sid, fieldname="device", no_order=True, + ) + self.device = self.device and self.device[0][0] or 'desktop' + rec = frappe.db.get_values( + sessions, + filters=(sessions.sid == self.sid) + & ( + PseudoColumn(f"({Now() - sessions.lastupdate})") + < get_expiry_period_for_query(self.device) + ), + fieldname=["user", "sessiondata"], + no_order=True, + ) if rec: data = frappe._dict(frappe.safe_eval(rec and rec[0][1] or '{}')) From b081496912e3821e6f8581686908572451220308 Mon Sep 17 00:00:00 2001 From: Aradhya-Tripathi Date: Tue, 16 Nov 2021 22:44:51 +0530 Subject: [PATCH 014/246] fix: added additional conditions when filters are None --- frappe/database/database.py | 4 ++-- frappe/database/query.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index 09d19f21de..18f5b4dda9 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -85,7 +85,7 @@ class Database(object): def sql(self, query, values=(), as_dict = 0, as_list = 0, formatted = 0, debug=0, ignore_ddl=0, as_utf8=0, auto_commit=0, update=None, - explain=False, run=True, pluck=False): + explain=False, run=True, pluck=False, **kwargs): """Execute a SQL query and fetch all rows. :param query: SQL query. @@ -553,7 +553,7 @@ class Database(object): field_objects.append(field) criterion = self.query.build_conditions( - table=doctype, filters=filters, orderby=order_by, for_update=for_update + table=doctype, filters=filters, orderby=order_by, for_update=for_update, **kwargs, ) if isinstance(fields, (list, tuple)): query = criterion.select(*field_objects) diff --git a/frappe/database/query.py b/frappe/database/query.py index 3545efb412..7341a7eb78 100644 --- a/frappe/database/query.py +++ b/frappe/database/query.py @@ -224,6 +224,7 @@ class Query: """ conditions = self.get_condition(table, **kwargs) if not filters: + conditions = self.add_conditions(conditions, **kwargs) return conditions for key in filters: From a3c992bce4e0165a92953b5db8e32c641525cf66 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Tue, 16 Nov 2021 23:41:23 +0530 Subject: [PATCH 015/246] fix: fixed order in get_values tests --- frappe/tests/test_db.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/frappe/tests/test_db.py b/frappe/tests/test_db.py index 9077655dc9..19ff73fc95 100644 --- a/frappe/tests/test_db.py +++ b/frappe/tests/test_db.py @@ -24,8 +24,8 @@ class TestDB(unittest.TestCase): self.assertNotEqual(frappe.db.get_value("User", {"name": ["!=", "Guest"]}), "Guest") self.assertEqual(frappe.db.get_value("User", {"name": ["<", "Adn"]}), "Administrator") self.assertEqual(frappe.db.get_value("User", {"name": ["<=", "Administrator"]}), "Administrator") - self.assertEqual(frappe.db.get_value("User", {}, ["Max(name)"]), frappe.db.sql("SELECT Max(name) FROM tabUser")[0][0]) - self.assertEqual(frappe.db.get_value("User", {}, "Min(name)"), frappe.db.sql("SELECT Min(name) FROM tabUser")[0][0]) + self.assertEqual(frappe.db.get_value("User", {}, ["Max(name)"], no_order=True), frappe.db.sql("SELECT Max(name) FROM tabUser")[0][0]) + self.assertEqual(frappe.db.get_value("User", {}, "Min(name)", no_order=True), frappe.db.sql("SELECT Min(name) FROM tabUser")[0][0]) self.assertIn("for update", frappe.db.get_value("User", Field("name") == "Administrator", for_update=True, run=False).lower()) self.assertEqual(frappe.db.sql("""SELECT name FROM `tabUser` WHERE name > 's' ORDER BY MODIFIED DESC""")[0][0], @@ -34,8 +34,15 @@ class TestDB(unittest.TestCase): self.assertEqual(frappe.db.sql("""SELECT name FROM `tabUser` WHERE name >= 't' ORDER BY MODIFIED DESC""")[0][0], frappe.db.get_value("User", {"name": [">=", "t"]})) - self.assertIn("concat_ws", frappe.db.get_value("User", filters={"name": "Administrator"}, fieldname=Concat_ws(" ", "LastName"), run=False).lower()) - + self.assertIn( + "concat_ws", + frappe.db.get_value( + "User", + filters={"name": "Administrator"}, + fieldname=Concat_ws(" ", "LastName"), + run=False, + ).lower(), + ) def test_set_value(self): todo1 = frappe.get_doc(dict(doctype='ToDo', description = 'test_set_value 1')).insert() todo2 = frappe.get_doc(dict(doctype='ToDo', description = 'test_set_value 2')).insert() From fcd480b2a82b036bb021bee391ce5e703caaff12 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Thu, 18 Nov 2021 09:56:41 +0530 Subject: [PATCH 016/246] refactor: Converted queries in translate --- frappe/translate.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/frappe/translate.py b/frappe/translate.py index 03720f115d..2905af6490 100644 --- a/frappe/translate.py +++ b/frappe/translate.py @@ -457,17 +457,26 @@ def get_messages_from_workflow(doctype=None, app_name=None): workflows.extend(frappe.get_all('Workflow', filters=fixture.get('filters'))) messages = [] + document_state = DocType("Workflow Document State") for w in workflows: - states = frappe.db.sql( - 'select distinct state from `tabWorkflow Document State` where parent=%s', - (w['name'],), as_dict=True) - + states = frappe.db.get_values( + document_state, + filters=document_state.parent == w["name"], + fieldname="state", + distinct=True, + as_dict=True, + no_order=True, + ) messages.extend([('Workflow: ' + w['name'], state['state']) for state in states if is_translatable(state['state'])]) - - states = frappe.db.sql( - 'select distinct message from `tabWorkflow Document State` where parent=%s and message is not null', - (w['name'],), as_dict=True) - + states = frappe.db.get_values( + document_state, + filters=(document_state.parent == w["name"]) + & (document_state.message.isnotnull()), + fieldname="message", + distinct=True, + no_order=True, + as_dict=True, + ) messages.extend([("Workflow: " + w['name'], state['message']) for state in states if is_translatable(state['message'])]) From ae59fd7c58495cccb2eeb0fef37923b3a6015f13 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Fri, 19 Nov 2021 22:58:32 +0530 Subject: [PATCH 017/246] feat: added no_order to execute --- frappe/database/database.py | 10 +++++++--- frappe/model/db_query.py | 6 +++++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index 6703ae0ff3..6de8c9e72a 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -395,7 +395,6 @@ class Database(object): return self.value_cache[(doctype, filters, fieldname)] if isinstance(filters, list): - order_by = order_by or "modified_desc" out = self._get_value_for_many_names(doctype, filters, fieldname, debug=debug, run=run, **kwargs) else: @@ -571,10 +570,15 @@ class Database(object): def _get_value_for_many_names(self, doctype, names, field, debug=False, run=True, **kwargs): names = list(filter(None, names)) if names: - return self.get_all(doctype, + return self.get_all( + doctype, fields=field, filters=names, - debug=debug, as_list=1, run=run) + debug=debug, + as_list=1, + run=run, + **kwargs, + ) else: return {} diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py index 6181832363..a6c502d129 100644 --- a/frappe/model/db_query.py +++ b/frappe/model/db_query.py @@ -35,7 +35,7 @@ class DatabaseQuery(object): join='left join', distinct=False, start=None, page_length=None, limit=None, ignore_ifnull=False, save_user_settings=False, save_user_settings_fields=False, update=None, add_total_row=None, user_settings=None, reference_doctype=None, - run=True, strict=True, pluck=None, ignore_ddl=False, parent_doctype=None) -> List: + run=True, strict=True, pluck=None, ignore_ddl=False, parent_doctype=None, no_order=False) -> List: if not ignore_permissions and \ not frappe.has_permission(self.doctype, "select", user=user, parent_doctype=parent_doctype) and \ not frappe.has_permission(self.doctype, "read", user=user, parent_doctype=parent_doctype): @@ -90,6 +90,7 @@ class DatabaseQuery(object): self.run = run self.strict = strict self.ignore_ddl = ignore_ddl + self.no_order = no_order # for contextual user permission check # to determine which user permission is applicable on link field of specific doctype @@ -128,6 +129,9 @@ class DatabaseQuery(object): args.fields = 'distinct ' + args.fields args.order_by = '' # TODO: recheck for alternative + if self.no_order: + args.order_by = "" + query = """select %(fields)s from %(tables)s %(conditions)s From ed7b3f54a7453d2be9b8f25b8738649f377afcf5 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sat, 20 Nov 2021 13:21:01 +0530 Subject: [PATCH 018/246] refactor: converted more queries --- frappe/translate.py | 11 ++++++++--- frappe/twofactor.py | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/frappe/translate.py b/frappe/translate.py index 2905af6490..23072d064c 100644 --- a/frappe/translate.py +++ b/frappe/translate.py @@ -480,9 +480,14 @@ def get_messages_from_workflow(doctype=None, app_name=None): messages.extend([("Workflow: " + w['name'], state['message']) for state in states if is_translatable(state['message'])]) - actions = frappe.db.sql( - 'select distinct action from `tabWorkflow Transition` where parent=%s', - (w['name'],), as_dict=True) + actions = frappe.db.get_values( + "Workflow Transition", + filters={"parent": w["name"]}, + fieldname="action", + as_dict=True, + distinct=True, + no_order=True, + ) messages.extend([("Workflow: " + w['name'], action['action']) \ for action in actions if is_translatable(action['action'])]) diff --git a/frappe/twofactor.py b/frappe/twofactor.py index 6ae53ed717..bd49d588b0 100644 --- a/frappe/twofactor.py +++ b/frappe/twofactor.py @@ -417,4 +417,4 @@ def reset_otp_secret(user): enqueue(method=frappe.sendmail, queue='short', timeout=300, event=None, is_async=True, job_name=None, now=False, **email_args) return frappe.msgprint(_("OTP Secret has been reset. Re-registration will be required on next login.")) else: - return frappe.throw(_("OTP secret can only be reset by the Administrator.")) + return frappe.throw(_("OTP secret can only be reset by the Administrator.")) \ No newline at end of file From 1bdff9f3bdd7c579ba5d4a089b5f7362cf0247d1 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sat, 20 Nov 2021 21:33:03 +0530 Subject: [PATCH 019/246] refactor: converted queries in init --- frappe/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index 4218aa113b..7db70ed39b 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -790,7 +790,9 @@ def has_website_permission(doc=None, ptype='read', user=None, verbose=False, doc def is_table(doctype): """Returns True if `istable` property (indicating child Table) is set for given DocType.""" def get_tables(): - return db.sql_list("select name from tabDocType where istable=1") + return db.get_values( + "DocType", filters={"istable": 1}, no_order=True, pluck=True + ) tables = cache().get_value("is_table", get_tables) return doctype in tables From ae68ad53a7778dc6df97d887ef2c5bacaea66d5e Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sat, 20 Nov 2021 23:51:06 +0530 Subject: [PATCH 020/246] feat: Added permissions to database API --- frappe/database/database.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/frappe/database/database.py b/frappe/database/database.py index 6de8c9e72a..6d6ed467b6 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -116,6 +116,10 @@ class Database(object): if not run: return query + if not kwargs.get("ignore_permissions", True): + tables = self.get_tables_from_query(query) + self.check_permissions(doctype=tables, **kwargs) + if re.search(r'ifnull\(', query, flags=re.IGNORECASE): # replaces ifnull in query with coalesce query = re.sub(r'ifnull\(', 'coalesce(', query, flags=re.IGNORECASE) @@ -260,6 +264,24 @@ class Database(object): self.commit() self.sql(query, debug=debug) + @staticmethod + def check_permissions(doctype, **kwargs): + kwargs.pop("ignore_permissions") + if isinstance(doctype, str): + doctype = [doctype] + + for dt in doctype: + dt = re.sub("tab", "", dt) + if not frappe.has_permission( + dt, "select", **kwargs + ) and not frappe.has_permission(dt, "read", **kwargs): + frappe.flags.error_message = _('Insufficient Permission for {0}').format(frappe.bold(dt)) + raise frappe.PermissionError(dt) + + @staticmethod + def get_tables_from_query(query: str): + return [table for table in re.findall(r"\w+", query) if table.startswith("tab")] + def check_transaction_status(self, query): """Raises exception if more than 20,000 `INSERT`, `UPDATE` queries are executed in one transaction. This is to ensure that writes are always flushed otherwise this From aede12d8b7af65b85269273da1deff1dd3400539 Mon Sep 17 00:00:00 2001 From: Jannat Patel Date: Tue, 23 Nov 2021 10:05:22 +0530 Subject: [PATCH 021/246] feat: letter head tour --- .../form_tour/letter_head/letter_head.json | 53 +++++++++++++++++++ frappe/public/js/frappe/form/form_tour.js | 40 ++++++++++++-- 2 files changed, 89 insertions(+), 4 deletions(-) create mode 100644 frappe/printing/form_tour/letter_head/letter_head.json diff --git a/frappe/printing/form_tour/letter_head/letter_head.json b/frappe/printing/form_tour/letter_head/letter_head.json new file mode 100644 index 0000000000..66730b4e38 --- /dev/null +++ b/frappe/printing/form_tour/letter_head/letter_head.json @@ -0,0 +1,53 @@ +{ + "creation": "2021-11-22 15:26:53.878805", + "docstatus": 0, + "doctype": "Form Tour", + "idx": 0, + "is_standard": 1, + "modified": "2021-11-22 15:26:53.878805", + "modified_by": "Administrator", + "module": "Printing", + "name": "Letter Head", + "owner": "Administrator", + "reference_doctype": "Letter Head", + "save_on_complete": 1, + "steps": [ + { + "description": "Let's name your first Letter Head with your company's name", + "field": "", + "fieldname": "letter_head_name", + "fieldtype": "Data", + "has_next_condition": 0, + "is_table_field": 0, + "label": "Letter Head Name", + "parent_field": "", + "position": "Right", + "title": "Letter Head Name" + }, + { + "description": "Select the image containing only header part of your letter Head.", + "field": "", + "fieldname": "image", + "fieldtype": "Attach Image", + "has_next_condition": 0, + "is_table_field": 0, + "label": "Image", + "parent_field": "", + "position": "Right", + "title": "Image" + }, + { + "description": "You can mark the Letter Head as default", + "field": "", + "fieldname": "is_default", + "fieldtype": "Check", + "has_next_condition": 0, + "is_table_field": 0, + "label": "Default Letter Head", + "parent_field": "", + "position": "Right", + "title": "Default Letter Head" + } + ], + "title": "Letter Head" +} \ No newline at end of file diff --git a/frappe/public/js/frappe/form/form_tour.js b/frappe/public/js/frappe/form/form_tour.js index 0694aa634a..9bae776d66 100644 --- a/frappe/public/js/frappe/form/form_tour.js +++ b/frappe/public/js/frappe/form/form_tour.js @@ -42,7 +42,7 @@ frappe.ui.form.FormTour = class FormTour { this.tour = { steps: frappe.tour[this.frm.doctype] }; } } - + if (on_finish) this.on_finish = on_finish; this.init_driver(); @@ -65,9 +65,10 @@ frappe.ui.form.FormTour = class FormTour { const driver_step = this.get_step(step, on_next); this.driver_steps.push(driver_step); - + if (step.fieldtype == 'Table') this.handle_table_step(step); if (step.is_table_field) this.handle_child_table_step(step); + //if (step.fieldtype == 'Attach Image') this.handle_attach_image_steps(step); }); if (this.tour.save_on_complete) { @@ -139,7 +140,7 @@ frappe.ui.form.FormTour = class FormTour { const is_next_field_in_curr_table = next_step.parent_field == curr_step.field; if (!is_next_field_in_curr_table) return; - + const rows = this.frm.doc[curr_step.fieldname]; const table_has_rows = rows && rows.length > 0; if (table_has_rows) { @@ -242,6 +243,7 @@ frappe.ui.form.FormTour = class FormTour { } add_step_to_save() { + console.log("save") const page_id = `[id="page-${this.frm.doctype}"]`; const $save_btn = `${page_id} .standard-actions .primary-action`; const save_step = { @@ -262,4 +264,34 @@ frappe.ui.form.FormTour = class FormTour { this.driver_steps.push(save_step); frappe.ui.form.on(this.frm.doctype, 'after_save', () => this.on_finish && this.on_finish()); } -}; \ No newline at end of file + + handle_attach_image_steps() { + $('.btn-attach').one('click', () => { + frappe.utils.sleep(300) + setTimeout(() => { + const modal_element = $(".file-uploader").closest(".modal-content"); + modal_element.css("z-index", "1000004 !important"); + const attach_dialog_step = { + element: modal_element[0], + allowClose: false, + overlayClickNext: false, + popover: { + title: __("Select an Image"), + description: "", + position: "left", + doneBtnText: __("Next") + } + }; + + this.driver_steps.splice(this.driver.currentStep + 1, 0, attach_dialog_step); + this.update_driver_steps(); // need to define again, since driver.js only considers steps which are inside DOM + frappe.utils.sleep(300).then(() => this.driver.start(this.driver.currentStep + 1)); + console.log('click', this.driver_steps) + }, 1000); + + modal_element.on('hidden.bs.modal', () => { + this.driver.moveNext(); + }) + }) + } +}; From 30278a393936ae4397c9b758ba6b5aab8d6b65ef Mon Sep 17 00:00:00 2001 From: Aradhya Date: Tue, 23 Nov 2021 13:29:59 +0530 Subject: [PATCH 022/246] refactor: removed no_order to support order_by None as valid input --- frappe/database/database.py | 7 +++---- frappe/model/db_query.py | 16 +++++++--------- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index 6d6ed467b6..6978c31182 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -392,7 +392,7 @@ class Database(object): return ((len(ret[0]) > 1 or as_dict) and ret[0] or ret[0][0]) if ret else None def get_values(self, doctype, filters=None, fieldname="name", ignore=None, as_dict=False, - debug=False, order_by=None, update=None, cache=False, for_update=False, run=True, **kwargs): + debug=False, order_by="default_ordering", update=None, cache=False, for_update=False, run=True, **kwargs): """Returns multiple document properties. :param doctype: DocType name. @@ -429,9 +429,8 @@ class Database(object): if (filters is not None) and (filters!=doctype or doctype=="DocType"): try: - if not kwargs.get("no_order"): - order_by = order_by or "modified" - kwargs.pop("no_order", None) + if order_by: + order_by = "modified" if order_by == "default_ordering" else order_by out = self._get_values_from_table( fields, filters, doctype, as_dict, debug, order_by, update, for_update=for_update, run=run, **kwargs ) diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py index a6c502d129..c94819a8c3 100644 --- a/frappe/model/db_query.py +++ b/frappe/model/db_query.py @@ -29,13 +29,13 @@ class DatabaseQuery(object): self.reference_doctype = None def execute(self, fields=None, filters=None, or_filters=None, - docstatus=None, group_by=None, order_by=None, limit_start=False, + docstatus=None, group_by=None, order_by="default_ordering", limit_start=False, limit_page_length=None, as_list=False, with_childnames=False, debug=False, ignore_permissions=False, user=None, with_comment_count=False, join='left join', distinct=False, start=None, page_length=None, limit=None, ignore_ifnull=False, save_user_settings=False, save_user_settings_fields=False, update=None, add_total_row=None, user_settings=None, reference_doctype=None, - run=True, strict=True, pluck=None, ignore_ddl=False, parent_doctype=None, no_order=False) -> List: + run=True, strict=True, pluck=None, ignore_ddl=False, parent_doctype=None) -> List: if not ignore_permissions and \ not frappe.has_permission(self.doctype, "select", user=user, parent_doctype=parent_doctype) and \ not frappe.has_permission(self.doctype, "read", user=user, parent_doctype=parent_doctype): @@ -90,7 +90,6 @@ class DatabaseQuery(object): self.run = run self.strict = strict self.ignore_ddl = ignore_ddl - self.no_order = no_order # for contextual user permission check # to determine which user permission is applicable on link field of specific doctype @@ -129,9 +128,6 @@ class DatabaseQuery(object): args.fields = 'distinct ' + args.fields args.order_by = '' # TODO: recheck for alternative - if self.no_order: - args.order_by = "" - query = """select %(fields)s from %(tables)s %(conditions)s @@ -707,7 +703,7 @@ class DatabaseQuery(object): def set_order_by(self, args): meta = frappe.get_meta(self.doctype) - if self.order_by: + if self.order_by and self.order_by != "default_ordering": args.order_by = self.order_by else: args.order_by = "" @@ -733,11 +729,13 @@ class DatabaseQuery(object): else: sort_field = meta.sort_field or 'modified' sort_order = (meta.sort_field and meta.sort_order) or 'desc' - args.order_by = f"`tab{self.doctype}`.`{sort_field or 'modified'}` {sort_order or 'desc'}" + if self.order_by: + args.order_by = f"`tab{self.doctype}`.`{sort_field or 'modified'}` {sort_order or 'desc'}" # draft docs always on top if hasattr(meta, 'is_submittable') and meta.is_submittable: - args.order_by = f"`tab{self.doctype}`.docstatus asc, {args.order_by}" + if self.order_by: + args.order_by = f"`tab{self.doctype}`.docstatus asc, {args.order_by}" def validate_order_by_and_group_by(self, parameters): """Check order by, group by so that atleast one column is selected and does not have subquery""" From 9ccf467acb917d32d45e193e13b838262937c6de Mon Sep 17 00:00:00 2001 From: Aradhya Date: Tue, 23 Nov 2021 13:30:42 +0530 Subject: [PATCH 023/246] fix: fixed no order arg in converted queries --- frappe/__init__.py | 2 +- frappe/sessions.py | 6 +++--- frappe/tests/test_db.py | 4 ++-- frappe/translate.py | 6 +++--- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index 7db70ed39b..f3e36c3b3b 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -791,7 +791,7 @@ def is_table(doctype): """Returns True if `istable` property (indicating child Table) is set for given DocType.""" def get_tables(): return db.get_values( - "DocType", filters={"istable": 1}, no_order=True, pluck=True + "DocType", filters={"istable": 1}, order_by=None, pluck=True ) tables = cache().get_value("is_table", get_tables) diff --git a/frappe/sessions.py b/frappe/sessions.py index cbe20cffb3..9034d94f72 100644 --- a/frappe/sessions.py +++ b/frappe/sessions.py @@ -112,7 +112,7 @@ def get_expired_sessions(): ) & (sessions.device == device), fieldname="sid", - no_order=True, + order_by=None, pluck=True, ) ) @@ -323,7 +323,7 @@ class Session: sessions = DocType("Sessions") self.device = frappe.db.get_values( - sessions, filters=sessions.sid == self.sid, fieldname="device", no_order=True, + sessions, filters=sessions.sid == self.sid, fieldname="device", order_by=None, ) self.device = self.device and self.device[0][0] or 'desktop' rec = frappe.db.get_values( @@ -334,7 +334,7 @@ class Session: < get_expiry_period_for_query(self.device) ), fieldname=["user", "sessiondata"], - no_order=True, + order_by=None, ) if rec: diff --git a/frappe/tests/test_db.py b/frappe/tests/test_db.py index b39bedc4e5..472528b174 100644 --- a/frappe/tests/test_db.py +++ b/frappe/tests/test_db.py @@ -24,8 +24,8 @@ class TestDB(unittest.TestCase): self.assertNotEqual(frappe.db.get_value("User", {"name": ["!=", "Guest"]}), "Guest") self.assertEqual(frappe.db.get_value("User", {"name": ["<", "Adn"]}), "Administrator") self.assertEqual(frappe.db.get_value("User", {"name": ["<=", "Administrator"]}), "Administrator") - self.assertEqual(frappe.db.get_value("User", {}, ["Max(name)"], no_order=True), frappe.db.sql("SELECT Max(name) FROM tabUser")[0][0]) - self.assertEqual(frappe.db.get_value("User", {}, "Min(name)", no_order=True), frappe.db.sql("SELECT Min(name) FROM tabUser")[0][0]) + self.assertEqual(frappe.db.get_value("User", {}, ["Max(name)"], order_by=True), frappe.db.sql("SELECT Max(name) FROM tabUser")[0][0]) + self.assertEqual(frappe.db.get_value("User", {}, "Min(name)", order_by=True), frappe.db.sql("SELECT Min(name) FROM tabUser")[0][0]) self.assertIn("for update", frappe.db.get_value("User", Field("name") == "Administrator", for_update=True, run=False).lower()) self.assertEqual(frappe.db.sql("""SELECT name FROM `tabUser` WHERE name > 's' ORDER BY MODIFIED DESC""")[0][0], diff --git a/frappe/translate.py b/frappe/translate.py index 23072d064c..c5ef24bc2a 100644 --- a/frappe/translate.py +++ b/frappe/translate.py @@ -465,7 +465,7 @@ def get_messages_from_workflow(doctype=None, app_name=None): fieldname="state", distinct=True, as_dict=True, - no_order=True, + order_by=None, ) messages.extend([('Workflow: ' + w['name'], state['state']) for state in states if is_translatable(state['state'])]) states = frappe.db.get_values( @@ -474,7 +474,7 @@ def get_messages_from_workflow(doctype=None, app_name=None): & (document_state.message.isnotnull()), fieldname="message", distinct=True, - no_order=True, + order_by=None, as_dict=True, ) messages.extend([("Workflow: " + w['name'], state['message']) @@ -486,7 +486,7 @@ def get_messages_from_workflow(doctype=None, app_name=None): fieldname="action", as_dict=True, distinct=True, - no_order=True, + order_by=None, ) messages.extend([("Workflow: " + w['name'], action['action']) \ From c4f76c056890f07dd9ed526ef9d0859fe96742a3 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Tue, 23 Nov 2021 14:21:20 +0530 Subject: [PATCH 024/246] fix: fixed order_by arg in get_value --- frappe/database/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index 6978c31182..35d054e5ca 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -357,7 +357,7 @@ class Database(object): return self.get_value(doctype, filters, "*", as_dict=as_dict, cache=cache) def get_value(self, doctype, filters=None, fieldname="name", ignore=None, as_dict=False, - debug=False, order_by=None, cache=False, for_update=False, run=True, **kwargs): + debug=False, order_by="default_ordering", cache=False, for_update=False, run=True, **kwargs): """Returns a document property or list of properties. :param doctype: DocType name. From 50db6d5dd4774830ccffbfe24d9465fc66092d52 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Tue, 23 Nov 2021 14:47:03 +0530 Subject: [PATCH 025/246] refactor: refactored query in sessions --- frappe/sessions.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/frappe/sessions.py b/frappe/sessions.py index 9034d94f72..ce7950c24e 100644 --- a/frappe/sessions.py +++ b/frappe/sessions.py @@ -324,8 +324,7 @@ class Session: self.device = frappe.db.get_values( sessions, filters=sessions.sid == self.sid, fieldname="device", order_by=None, - ) - self.device = self.device and self.device[0][0] or 'desktop' + ) or 'desktop' rec = frappe.db.get_values( sessions, filters=(sessions.sid == self.sid) From 71a21f5d7d260a74183478fc712ab7111ed178f1 Mon Sep 17 00:00:00 2001 From: Jannat Patel Date: Tue, 23 Nov 2021 15:21:46 +0530 Subject: [PATCH 026/246] feat: customizations onboarding --- frappe/core/form_tour/doctype/doctype.json | 53 +++++++++++++ .../form_tour/custom_field/custom_field.json | 77 +++++++++++++++++++ .../customization/customization.json | 41 ++++++++++ .../custom_doctype/custom_doctype.json | 21 +++++ .../custom_field/custom_field.json | 21 +++++ .../naming_series/naming_series.json | 20 +++++ .../print_format/print_format.json | 21 +++++ .../role_permissions/role_permissions.json | 20 +++++ .../onboarding_step/workflows/workflows.json | 20 +++++ 9 files changed, 294 insertions(+) create mode 100644 frappe/core/form_tour/doctype/doctype.json create mode 100644 frappe/custom/form_tour/custom_field/custom_field.json create mode 100644 frappe/custom/module_onboarding/customization/customization.json create mode 100644 frappe/custom/onboarding_step/custom_doctype/custom_doctype.json create mode 100644 frappe/custom/onboarding_step/custom_field/custom_field.json create mode 100644 frappe/custom/onboarding_step/naming_series/naming_series.json create mode 100644 frappe/custom/onboarding_step/print_format/print_format.json create mode 100644 frappe/custom/onboarding_step/role_permissions/role_permissions.json create mode 100644 frappe/custom/onboarding_step/workflows/workflows.json diff --git a/frappe/core/form_tour/doctype/doctype.json b/frappe/core/form_tour/doctype/doctype.json new file mode 100644 index 0000000000..866d3ea508 --- /dev/null +++ b/frappe/core/form_tour/doctype/doctype.json @@ -0,0 +1,53 @@ +{ + "creation": "2021-11-23 12:38:52.807353", + "docstatus": 0, + "doctype": "Form Tour", + "idx": 0, + "is_standard": 1, + "modified": "2021-11-23 15:13:02.794031", + "modified_by": "Administrator", + "module": "Core", + "name": "Doctype", + "owner": "Administrator", + "reference_doctype": "DocType", + "save_on_complete": 1, + "steps": [ + { + "description": "Select a Module to which this Doctype would belong", + "field": "", + "fieldname": "module", + "fieldtype": "Link", + "has_next_condition": 0, + "is_table_field": 0, + "label": "Module", + "parent_field": "", + "position": "Right", + "title": "Module" + }, + { + "description": "Check this to make the Docytpe as Custom", + "field": "", + "fieldname": "custom", + "fieldtype": "Check", + "has_next_condition": 0, + "is_table_field": 0, + "label": "Custom?", + "parent_field": "", + "position": "Left", + "title": "Custom " + }, + { + "description": "Add fields to this Custom Doctype", + "field": "", + "fieldname": "fields", + "fieldtype": "Table", + "has_next_condition": 0, + "is_table_field": 0, + "label": "Fields", + "parent_field": "", + "position": "Bottom", + "title": "Fields" + } + ], + "title": "Doctype" +} \ No newline at end of file diff --git a/frappe/custom/form_tour/custom_field/custom_field.json b/frappe/custom/form_tour/custom_field/custom_field.json new file mode 100644 index 0000000000..58656a3386 --- /dev/null +++ b/frappe/custom/form_tour/custom_field/custom_field.json @@ -0,0 +1,77 @@ +{ + "creation": "2021-11-23 12:22:32.922700", + "docstatus": 0, + "doctype": "Form Tour", + "idx": 0, + "is_standard": 1, + "modified": "2021-11-23 12:28:09.923397", + "modified_by": "Administrator", + "module": "Custom", + "name": "Custom Field", + "owner": "Administrator", + "reference_doctype": "Custom Field", + "save_on_complete": 1, + "steps": [ + { + "description": "Select a Document for which you want the Custom Field", + "field": "", + "fieldname": "dt", + "fieldtype": "Link", + "has_next_condition": 0, + "is_table_field": 0, + "label": "Document", + "parent_field": "", + "position": "Right", + "title": "Document" + }, + { + "description": "Enter a Label for this field", + "field": "", + "fieldname": "label", + "fieldtype": "Data", + "has_next_condition": 0, + "is_table_field": 0, + "label": "Label", + "parent_field": "", + "position": "Right", + "title": "Label" + }, + { + "description": "Select an appropriate Field Type that suits your requirements", + "field": "", + "fieldname": "fieldtype", + "fieldtype": "Select", + "has_next_condition": 0, + "is_table_field": 0, + "label": "Field Type", + "parent_field": "", + "position": "Left", + "title": "Field Type" + }, + { + "description": "Select the label after which you want to insert new field.", + "field": "", + "fieldname": "insert_after", + "fieldtype": "Select", + "has_next_condition": 0, + "is_table_field": 0, + "label": "Insert After", + "parent_field": "", + "position": "Right", + "title": "Insert After" + }, + { + "description": "Check this to make it a mandatory field", + "field": "", + "fieldname": "reqd", + "fieldtype": "Check", + "has_next_condition": 0, + "is_table_field": 0, + "label": "Is Mandatory Field", + "parent_field": "", + "position": "Left", + "title": "Is Mandatory Field" + } + ], + "title": "Custom Field" +} \ No newline at end of file diff --git a/frappe/custom/module_onboarding/customization/customization.json b/frappe/custom/module_onboarding/customization/customization.json new file mode 100644 index 0000000000..db2654443d --- /dev/null +++ b/frappe/custom/module_onboarding/customization/customization.json @@ -0,0 +1,41 @@ +{ + "allow_roles": [ + { + "role": "All" + } + ], + "creation": "2021-11-23 12:21:11.384229", + "docstatus": 0, + "doctype": "Module Onboarding", + "documentation_url": "https://docs.erpnext.com/docs/v13/user/manual/en/customize-erpnext", + "idx": 0, + "is_complete": 0, + "modified": "2021-11-23 15:04:15.826436", + "modified_by": "Administrator", + "module": "Custom", + "name": "Customization", + "owner": "Administrator", + "steps": [ + { + "step": "Custom Field" + }, + { + "step": "Custom Doctype" + }, + { + "step": "Naming Series" + }, + { + "step": "Workflows" + }, + { + "step": "Role Permissions" + }, + { + "step": "Print Format" + } + ], + "subtitle": "Custom Field, Custom Doctype, Naming Series, Role Permission, Workflow, Print Formats, Reports", + "success_message": "Customization onboarding is all done!", + "title": "Customization" +} \ No newline at end of file diff --git a/frappe/custom/onboarding_step/custom_doctype/custom_doctype.json b/frappe/custom/onboarding_step/custom_doctype/custom_doctype.json new file mode 100644 index 0000000000..1f8601abee --- /dev/null +++ b/frappe/custom/onboarding_step/custom_doctype/custom_doctype.json @@ -0,0 +1,21 @@ +{ + "action": "Create Entry", + "action_label": "Learn more about creating new DocTypes", + "creation": "2021-11-23 12:30:04.407568", + "description": "A DocType (Document Type) is used to insert forms in ERPNext. Forms such as Customer, Orders, and Invoices are Doctypes in the backend. You can also create new DocTypes to create new forms in ERPNext as per your business needs.", + "docstatus": 0, + "doctype": "Onboarding Step", + "idx": 0, + "is_complete": 0, + "is_single": 0, + "is_skipped": 0, + "modified": "2021-11-23 12:30:04.407568", + "modified_by": "Administrator", + "name": "Custom Doctype", + "owner": "Administrator", + "reference_document": "DocType", + "show_form_tour": 1, + "show_full_form": 1, + "title": "Custom Document Types", + "validate_action": 1 +} \ No newline at end of file diff --git a/frappe/custom/onboarding_step/custom_field/custom_field.json b/frappe/custom/onboarding_step/custom_field/custom_field.json new file mode 100644 index 0000000000..4044cf2456 --- /dev/null +++ b/frappe/custom/onboarding_step/custom_field/custom_field.json @@ -0,0 +1,21 @@ +{ + "action": "Create Entry", + "action_label": "Learn how to add Custom Fields", + "creation": "2021-11-23 12:21:09.479808", + "description": "Every form in ERPNext has a standard set of fields. If you need to capture some information, but there is no standard Field available for it, you can insert Custom Field for it.\n\nOnce custom fields are added, you can use them for reports and analytics charts as well.\n", + "docstatus": 0, + "doctype": "Onboarding Step", + "idx": 0, + "is_complete": 0, + "is_single": 0, + "is_skipped": 0, + "modified": "2021-11-23 12:21:09.479808", + "modified_by": "Administrator", + "name": "Custom Field", + "owner": "Administrator", + "reference_document": "Custom Field", + "show_form_tour": 1, + "show_full_form": 1, + "title": "Create Custom Fields", + "validate_action": 1 +} \ No newline at end of file diff --git a/frappe/custom/onboarding_step/naming_series/naming_series.json b/frappe/custom/onboarding_step/naming_series/naming_series.json new file mode 100644 index 0000000000..d24bf340c2 --- /dev/null +++ b/frappe/custom/onboarding_step/naming_series/naming_series.json @@ -0,0 +1,20 @@ +{ + "action": "Watch Video", + "creation": "2021-11-23 13:57:45.091427", + "description": "Each document created in ERPNext can have a unique ID generated for it, using a prefix defined for it. Though each document has some prefix pre-configured, you can further customize it using tools like Naming Series Tool and Document Naming Rule.\n", + "docstatus": 0, + "doctype": "Onboarding Step", + "idx": 0, + "is_complete": 0, + "is_single": 0, + "is_skipped": 0, + "modified": "2021-11-23 13:57:45.091427", + "modified_by": "Administrator", + "name": "Naming Series", + "owner": "Administrator", + "show_form_tour": 0, + "show_full_form": 0, + "title": "Setup Naming Series", + "validate_action": 1, + "video_url": "https://youtu.be/IGyISSfI1qU" +} \ No newline at end of file diff --git a/frappe/custom/onboarding_step/print_format/print_format.json b/frappe/custom/onboarding_step/print_format/print_format.json new file mode 100644 index 0000000000..681ef85b95 --- /dev/null +++ b/frappe/custom/onboarding_step/print_format/print_format.json @@ -0,0 +1,21 @@ +{ + "action": "Create Entry", + "action_label": "Learn about Standard and Custom Print Formats", + "creation": "2021-11-23 15:04:12.728513", + "description": "Print Formats allow you can define looks for documents when printed or converted to PDF. You can also create a custom Print Format using drag-and-drop tools.", + "docstatus": 0, + "doctype": "Onboarding Step", + "idx": 0, + "is_complete": 0, + "is_single": 0, + "is_skipped": 0, + "modified": "2021-11-23 15:04:12.728513", + "modified_by": "Administrator", + "name": "Print Format", + "owner": "Administrator", + "reference_document": "Print Format", + "show_form_tour": 1, + "show_full_form": 1, + "title": "Customize Print Formats", + "validate_action": 1 +} \ No newline at end of file diff --git a/frappe/custom/onboarding_step/role_permissions/role_permissions.json b/frappe/custom/onboarding_step/role_permissions/role_permissions.json new file mode 100644 index 0000000000..9d903b8bab --- /dev/null +++ b/frappe/custom/onboarding_step/role_permissions/role_permissions.json @@ -0,0 +1,20 @@ +{ + "action": "Watch Video", + "creation": "2021-11-23 14:00:27.208500", + "description": "In ERPNext, you can add your Employees as Users, and give them restricted access. Tools like Role Permission and User Permission allow you to define rules which give restricted access to the user to masters and transactions.", + "docstatus": 0, + "doctype": "Onboarding Step", + "idx": 0, + "is_complete": 0, + "is_single": 0, + "is_skipped": 0, + "modified": "2021-11-23 14:00:27.208500", + "modified_by": "Administrator", + "name": "Role Permissions", + "owner": "Administrator", + "show_form_tour": 0, + "show_full_form": 0, + "title": "Setup Limited Access for a User", + "validate_action": 1, + "video_url": "https://youtu.be/g3mk45o1zAg" +} \ No newline at end of file diff --git a/frappe/custom/onboarding_step/workflows/workflows.json b/frappe/custom/onboarding_step/workflows/workflows.json new file mode 100644 index 0000000000..060e9ae87d --- /dev/null +++ b/frappe/custom/onboarding_step/workflows/workflows.json @@ -0,0 +1,20 @@ +{ + "action": "Watch Video", + "creation": "2021-11-23 13:58:58.530044", + "description": "Workflows allow you to define custom rules for the approval process of a particular document in ERPNext. You can also set complex Workflow Rules and set approval conditions.", + "docstatus": 0, + "doctype": "Onboarding Step", + "idx": 0, + "is_complete": 0, + "is_single": 0, + "is_skipped": 0, + "modified": "2021-11-23 13:58:58.530044", + "modified_by": "Administrator", + "name": "Workflows", + "owner": "Administrator", + "show_form_tour": 0, + "show_full_form": 0, + "title": "Setup Approval Workflows", + "validate_action": 1, + "video_url": "https://youtu.be/yObJUg9FxFs" +} \ No newline at end of file From 59f54aa60c1b1ed5365d4d54353c36d4216a160a Mon Sep 17 00:00:00 2001 From: Aradhya Date: Tue, 23 Nov 2021 14:52:05 +0530 Subject: [PATCH 027/246] refactor: changed default_ordering to keep_default_ordering --- frappe/database/database.py | 6 +++--- frappe/sessions.py | 3 ++- frappe/tests/test_db.py | 4 ++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index 35d054e5ca..8f5a0d1a1f 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -357,7 +357,7 @@ class Database(object): return self.get_value(doctype, filters, "*", as_dict=as_dict, cache=cache) def get_value(self, doctype, filters=None, fieldname="name", ignore=None, as_dict=False, - debug=False, order_by="default_ordering", cache=False, for_update=False, run=True, **kwargs): + debug=False, order_by="KEEP_DEFAULT_ORDERING", cache=False, for_update=False, run=True, **kwargs): """Returns a document property or list of properties. :param doctype: DocType name. @@ -392,7 +392,7 @@ class Database(object): return ((len(ret[0]) > 1 or as_dict) and ret[0] or ret[0][0]) if ret else None def get_values(self, doctype, filters=None, fieldname="name", ignore=None, as_dict=False, - debug=False, order_by="default_ordering", update=None, cache=False, for_update=False, run=True, **kwargs): + debug=False, order_by="KEEP_DEFAULT_ORDERING", update=None, cache=False, for_update=False, run=True, **kwargs): """Returns multiple document properties. :param doctype: DocType name. @@ -430,7 +430,7 @@ class Database(object): if (filters is not None) and (filters!=doctype or doctype=="DocType"): try: if order_by: - order_by = "modified" if order_by == "default_ordering" else order_by + order_by = "modified" if order_by == "KEEP_DEFAULT_ORDERING" else order_by out = self._get_values_from_table( fields, filters, doctype, as_dict, debug, order_by, update, for_update=for_update, run=run, **kwargs ) diff --git a/frappe/sessions.py b/frappe/sessions.py index ce7950c24e..6a192ee010 100644 --- a/frappe/sessions.py +++ b/frappe/sessions.py @@ -324,7 +324,8 @@ class Session: self.device = frappe.db.get_values( sessions, filters=sessions.sid == self.sid, fieldname="device", order_by=None, - ) or 'desktop' + ) + self.device = self.device and self.device[0][0] or "desktop" rec = frappe.db.get_values( sessions, filters=(sessions.sid == self.sid) diff --git a/frappe/tests/test_db.py b/frappe/tests/test_db.py index 472528b174..60c8db6ab6 100644 --- a/frappe/tests/test_db.py +++ b/frappe/tests/test_db.py @@ -24,8 +24,8 @@ class TestDB(unittest.TestCase): self.assertNotEqual(frappe.db.get_value("User", {"name": ["!=", "Guest"]}), "Guest") self.assertEqual(frappe.db.get_value("User", {"name": ["<", "Adn"]}), "Administrator") self.assertEqual(frappe.db.get_value("User", {"name": ["<=", "Administrator"]}), "Administrator") - self.assertEqual(frappe.db.get_value("User", {}, ["Max(name)"], order_by=True), frappe.db.sql("SELECT Max(name) FROM tabUser")[0][0]) - self.assertEqual(frappe.db.get_value("User", {}, "Min(name)", order_by=True), frappe.db.sql("SELECT Min(name) FROM tabUser")[0][0]) + self.assertEqual(frappe.db.get_value("User", {}, ["Max(name)"], order_by=None), frappe.db.sql("SELECT Max(name) FROM tabUser")[0][0]) + self.assertEqual(frappe.db.get_value("User", {}, "Min(name)", order_by=None), frappe.db.sql("SELECT Min(name) FROM tabUser")[0][0]) self.assertIn("for update", frappe.db.get_value("User", Field("name") == "Administrator", for_update=True, run=False).lower()) self.assertEqual(frappe.db.sql("""SELECT name FROM `tabUser` WHERE name > 's' ORDER BY MODIFIED DESC""")[0][0], From d55d19ea5053f695ad29e8ed9db92df508ff161b Mon Sep 17 00:00:00 2001 From: Aradhya Date: Tue, 23 Nov 2021 15:44:41 +0530 Subject: [PATCH 028/246] refactor: using throw instead of raise --- frappe/database/database.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index 8f5a0d1a1f..1cf434947e 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -275,8 +275,7 @@ class Database(object): if not frappe.has_permission( dt, "select", **kwargs ) and not frappe.has_permission(dt, "read", **kwargs): - frappe.flags.error_message = _('Insufficient Permission for {0}').format(frappe.bold(dt)) - raise frappe.PermissionError(dt) + frappe.throw(_(f"Insufficient Permission for {frappe.bold(dt)}")) @staticmethod def get_tables_from_query(query: str): From 08d94d991fd06c071c83edfde9cb107b385bd533 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Tue, 23 Nov 2021 15:49:16 +0530 Subject: [PATCH 029/246] refactor: changed query in sessions --- frappe/sessions.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/frappe/sessions.py b/frappe/sessions.py index 6a192ee010..f0609cd74e 100644 --- a/frappe/sessions.py +++ b/frappe/sessions.py @@ -322,10 +322,9 @@ class Session: def get_session_data_from_db(self): sessions = DocType("Sessions") - self.device = frappe.db.get_values( + self.device = frappe.db.get_value( sessions, filters=sessions.sid == self.sid, fieldname="device", order_by=None, - ) - self.device = self.device and self.device[0][0] or "desktop" + ) or "desktop" rec = frappe.db.get_values( sessions, filters=(sessions.sid == self.sid) From 7a0c06b46a09ebdd1721d9d5146bfe7f6d8bb6f1 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Tue, 23 Nov 2021 15:56:55 +0530 Subject: [PATCH 030/246] fix: fixed semgrep issues --- frappe/database/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index 1cf434947e..52ce173cee 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -275,7 +275,7 @@ class Database(object): if not frappe.has_permission( dt, "select", **kwargs ) and not frappe.has_permission(dt, "read", **kwargs): - frappe.throw(_(f"Insufficient Permission for {frappe.bold(dt)}")) + frappe.throw(_("Insufficient Permission for {0}").format(frappe.bold(dt))) @staticmethod def get_tables_from_query(query: str): From f947fb9cf24922746eab35a85cebcb0f2ce4a3e6 Mon Sep 17 00:00:00 2001 From: Jannat Patel Date: Tue, 23 Nov 2021 17:36:08 +0530 Subject: [PATCH 031/246] fix: attach image modal for tour --- frappe/public/js/frappe/form/form_tour.js | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/frappe/public/js/frappe/form/form_tour.js b/frappe/public/js/frappe/form/form_tour.js index 9bae776d66..7946495b45 100644 --- a/frappe/public/js/frappe/form/form_tour.js +++ b/frappe/public/js/frappe/form/form_tour.js @@ -68,7 +68,7 @@ frappe.ui.form.FormTour = class FormTour { if (step.fieldtype == 'Table') this.handle_table_step(step); if (step.is_table_field) this.handle_child_table_step(step); - //if (step.fieldtype == 'Attach Image') this.handle_attach_image_steps(step); + if (step.fieldtype == 'Attach Image') this.handle_attach_image_steps(step); }); if (this.tour.save_on_complete) { @@ -243,7 +243,6 @@ frappe.ui.form.FormTour = class FormTour { } add_step_to_save() { - console.log("save") const page_id = `[id="page-${this.frm.doctype}"]`; const $save_btn = `${page_id} .standard-actions .primary-action`; const save_step = { @@ -267,10 +266,8 @@ frappe.ui.form.FormTour = class FormTour { handle_attach_image_steps() { $('.btn-attach').one('click', () => { - frappe.utils.sleep(300) setTimeout(() => { const modal_element = $(".file-uploader").closest(".modal-content"); - modal_element.css("z-index", "1000004 !important"); const attach_dialog_step = { element: modal_element[0], allowClose: false, @@ -285,13 +282,14 @@ frappe.ui.form.FormTour = class FormTour { this.driver_steps.splice(this.driver.currentStep + 1, 0, attach_dialog_step); this.update_driver_steps(); // need to define again, since driver.js only considers steps which are inside DOM - frappe.utils.sleep(300).then(() => this.driver.start(this.driver.currentStep + 1)); - console.log('click', this.driver_steps) - }, 1000); - - modal_element.on('hidden.bs.modal', () => { this.driver.moveNext(); - }) - }) + this.driver.overlay.refresh(); + + modal_element.closest('.modal').on('hidden.bs.modal', () => { + this.driver.moveNext(); + }); + + }, 500); + }); } }; From e0712171a6b09a2523a057baa2626c6f25b70eff Mon Sep 17 00:00:00 2001 From: mtraeber Date: Tue, 23 Nov 2021 15:54:00 +0100 Subject: [PATCH 032/246] fix: frappe-linter, all queries in email_account.py and receive.py rewritten according to these specifications: https://frappeframework.com/docs/user/en/api/query-builder --- .../doctype/email_account/email_account.py | 44 ++++++++++++------- frappe/email/receive.py | 32 ++++++++------ 2 files changed, 47 insertions(+), 29 deletions(-) diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index d5b683c9e3..59db7d04c7 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -549,7 +549,11 @@ class EmailAccount(Document): def on_trash(self): """Clear communications where email account is linked""" - frappe.db.sql("update `tabCommunication` set email_account='' where email_account=%s", self.name) + Communication = frappe.qb.from_("Communication") + frappe.qb.update(Communication) \ + .set(Communication.email_account == "") \ + .where(email_account == self.name).run() + remove_user_email_inbox(email_account=self.name) def after_rename(self, old, new, merge=False): @@ -571,9 +575,15 @@ class EmailAccount(Document): if not self.use_imap: return - flags = frappe.db.sql("""select name, communication, uid, action, imap_folder from - `tabEmail Flag Queue` where is_completed=0 and email_account={email_account} and imap_folder={folder_name} - """.format(email_account=frappe.db.escape(self.name),folder_name=frappe.db.escape(folder_name)), as_dict=True) + EmailFlagQueue = frappe.qb.DocType("Email Flag Queue") + flags = ( + frappe.qb.from_(EmailFlagQueue) + .select(EmailFlagQueue.name, EmailFlagQueue.communication, EmailFlagQueue.uid, + EmailFlagQueue.action, EmailFlagQueue.imap_folder) + .where(EmailFlagQueue.is_completed == 0) + .where(EmailFlagQueue.email_account == frappe.db.escape(self.name)) + .where(EmailFlagQueue.folder_name == frappe.db.escape(folder_name)) + ).run(as_dict=True) uid_list = { flag.get("uid", None): flag.get("action", "Read") for flag in flags } if flags and uid_list: @@ -594,16 +604,20 @@ class EmailAccount(Document): self.set_communication_seen_status(docnames, seen=0) docnames = ",".join([ "'%s'"%flag.get("name") for flag in flags ]) - frappe.db.sql(""" update `tabEmail Flag Queue` set is_completed=1 - where name in ({docnames})""".format(docnames=docnames)) + + EmailFlagQueue = frappe.qb.DocType("Email Flag Queue") + frappe.qb.update(EmailFlagQueue) \ + .set(EmailFlagQueue.is_completed, 1) \ + .where(EmailFlagQueue.name.isin(docnames)).run() def set_communication_seen_status(self, docnames, seen=0): """ mark Email Flag Queue of self.email_account mails as read""" if not docnames: return - - frappe.db.sql(""" update `tabCommunication` set seen={seen} - where name in ({docnames})""".format(docnames=docnames, seen=seen)) + Communication = frappe.qb.from_("Communication") + frappe.qb.update(Communication) \ + .set(Communication.seen == seen) \ + .where(Communication.name.isin(docnames)).run() def check_automatic_linking_email_account(self): if self.enable_automatic_linking: @@ -780,12 +794,12 @@ def setup_user_email_inbox(email_account, awaiting_password, email_id, enable_ou update_user_email_settings = True if update_user_email_settings: - frappe.db.sql("""UPDATE `tabUser Email` SET awaiting_password = %(awaiting_password)s, - enable_outgoing = %(enable_outgoing)s WHERE email_account = %(email_account)s""", { - "email_account": email_account, - "enable_outgoing": enable_outgoing, - "awaiting_password": awaiting_password or 0 - }) + UserEmail = frappe.qb.from_("User Email") + frappe.qb.update(UserEmail) \ + .set(UserEmail.awaiting_password == awaiting_password or 0) \ + .set(UserEmail.enable_outgoing == enable_outgoing) \ + .where(UserEmail.email_account == email_account).run() + else: users = " and ".join([frappe.bold(user.get("name")) for user in user_names]) frappe.msgprint(_("Enabled email inbox for user {0}").format(users)) diff --git a/frappe/email/receive.py b/frappe/email/receive.py index e7989660b8..4f4ed6d48e 100644 --- a/frappe/email/receive.py +++ b/frappe/email/receive.py @@ -210,22 +210,26 @@ class EmailServer: if not uid_validity or uid_validity != current_uid_validity: # uidvalidity changed & all email uids are reindexed by server - frappe.db.sql( - """update `tabCommunication` set uid=-1 where communication_medium='Email' - and email_account=%s""", (self.settings.email_account,) - ) + Communication = frappe.qb.DocType("Communication") + frappe.qb.update(Communication) \ + .set(Communication.uid, -1) \ + .where(Communication.communication_medium == "Email") \ + .where(Communication.email_account == self.settings.email_account).run() + if self.settings.use_imap: - # new update for the IMAP Folder DoctType - frappe.db.sql( - """update `tabIMAP Folder` set uidvalidity=%s, uidnext=%s where - parent=%s and folder_name=%s""", - (current_uid_validity, uidnext, self.settings.email_account_name, folder) - ) + # new update for the IMAP Folder DocType + IMAPFolder = frappe.qb.DocType("IMAP Folder") + frappe.qb.update(IMAPFolder) \ + .set(IMAPFolder.uidvalidity, current_uid_validity) \ + .set(IMAPFolder.uidnext, uidnext) \ + .where(IMAPFolder.parent == self.settings.email_account_name) \ + .where(IMAPFolder.folder_name == folder).run() else: - frappe.db.sql( - """update `tabEmail Account` set uidvalidity=%s, uidnext=%s where - name=%s""", (current_uid_validity, uidnext, self.settings.email_account) - ) + EmailAccount = frappe.qb.DocType("Email Account") + frappe.qb.update(EmailAccount) \ + .set(EmailAccount.uidvalidity, current_uid_validity) \ + .set(EmailAccount.uidnext, uidnext) \ + .where(EmailAccount.name == self.settings.email_account_name).run() # uid validity not found pulling emails for first time if not uid_validity: From 88e17771851f03bfd57f477cf3e8ec6a15e5f936 Mon Sep 17 00:00:00 2001 From: mtraeber Date: Tue, 23 Nov 2021 16:05:56 +0100 Subject: [PATCH 033/246] fix: frappe-linter, query rewritten --- frappe/patches/v14_0/copy_mail_data.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/frappe/patches/v14_0/copy_mail_data.py b/frappe/patches/v14_0/copy_mail_data.py index d3a4baca88..8780ab8630 100644 --- a/frappe/patches/v14_0/copy_mail_data.py +++ b/frappe/patches/v14_0/copy_mail_data.py @@ -19,15 +19,8 @@ def execute(): }) doc.save() - - frappe.db.sql( - """ - update - `tabEmail Flag Queue` - set - imap_folder = "INBOX" - where - email_account = '%s' - and imap_folder is NULL - """ % (doc.name) - ) + EmailFlagQueue = frappe.qb.DocType("Email Flag Queue") + frappe.qb.update(EmailFlagQueue) \ + .set(EmailFlagQueue.imap_folder, "INBOX") \ + .where(EmailFlagQueue.email_account == doc.name) \ + .where(EmailFlagQueue.imap_folder.isnull()).run() From dbfb959e0c9ef85e37456aa9ae0c72ef919cb1c9 Mon Sep 17 00:00:00 2001 From: mtraeber Date: Tue, 23 Nov 2021 16:13:23 +0100 Subject: [PATCH 034/246] fix: sider --- frappe/email/doctype/email_account/email_account.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index 59db7d04c7..3f163547f3 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -552,7 +552,7 @@ class EmailAccount(Document): Communication = frappe.qb.from_("Communication") frappe.qb.update(Communication) \ .set(Communication.email_account == "") \ - .where(email_account == self.name).run() + .where(Communication.email_account == self.name).run() remove_user_email_inbox(email_account=self.name) @@ -578,8 +578,7 @@ class EmailAccount(Document): EmailFlagQueue = frappe.qb.DocType("Email Flag Queue") flags = ( frappe.qb.from_(EmailFlagQueue) - .select(EmailFlagQueue.name, EmailFlagQueue.communication, EmailFlagQueue.uid, - EmailFlagQueue.action, EmailFlagQueue.imap_folder) + .select(EmailFlagQueue.name, EmailFlagQueue.communication, EmailFlagQueue.uid, EmailFlagQueue.action, EmailFlagQueue.imap_folder) .where(EmailFlagQueue.is_completed == 0) .where(EmailFlagQueue.email_account == frappe.db.escape(self.name)) .where(EmailFlagQueue.folder_name == frappe.db.escape(folder_name)) From 9f1f3e345a438bbc25a97ddf45aaa4901bbd69d8 Mon Sep 17 00:00:00 2001 From: mtraeber Date: Tue, 23 Nov 2021 16:19:00 +0100 Subject: [PATCH 035/246] fix: sider --- frappe/email/doctype/email_account/email_account.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index 3f163547f3..18c9ca1737 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -575,13 +575,13 @@ class EmailAccount(Document): if not self.use_imap: return - EmailFlagQueue = frappe.qb.DocType("Email Flag Queue") + EmailFlagQ = frappe.qb.DocType("Email Flag Queue") flags = ( - frappe.qb.from_(EmailFlagQueue) - .select(EmailFlagQueue.name, EmailFlagQueue.communication, EmailFlagQueue.uid, EmailFlagQueue.action, EmailFlagQueue.imap_folder) - .where(EmailFlagQueue.is_completed == 0) - .where(EmailFlagQueue.email_account == frappe.db.escape(self.name)) - .where(EmailFlagQueue.folder_name == frappe.db.escape(folder_name)) + frappe.qb.from_(EmailFlagQ) + .select(EmailFlagQ.name, EmailFlagQ.communication, EmailFlagQ.uid, EmailFlagQ.action, EmailFlagQ.imap_folder) + .where(EmailFlagQ.is_completed == 0) + .where(EmailFlagQ.email_account == frappe.db.escape(self.name)) + .where(EmailFlagQ.folder_name == frappe.db.escape(folder_name)) ).run(as_dict=True) uid_list = { flag.get("uid", None): flag.get("action", "Read") for flag in flags } From 8ad52df6a47d77500aebbfca641d3f30e704917a Mon Sep 17 00:00:00 2001 From: Jannat Patel Date: Wed, 24 Nov 2021 11:57:56 +0530 Subject: [PATCH 036/246] feat: first document and include name field check in form tour --- frappe/desk/doctype/form_tour/form_tour.json | 24 ++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/frappe/desk/doctype/form_tour/form_tour.json b/frappe/desk/doctype/form_tour/form_tour.json index e4ea528fcc..494a17bafb 100644 --- a/frappe/desk/doctype/form_tour/form_tour.json +++ b/frappe/desk/doctype/form_tour/form_tour.json @@ -9,8 +9,11 @@ "title", "reference_doctype", "module", + "column_break_6", "is_standard", "save_on_complete", + "first_document", + "include_name_field", "section_break_3", "steps" ], @@ -62,11 +65,28 @@ "label": "Module", "options": "Module Def", "read_only": 1 + }, + { + "fieldname": "column_break_6", + "fieldtype": "Column Break" + }, + { + "default": "0", + "fieldname": "first_document", + "fieldtype": "Check", + "label": "Show First Document Tour" + }, + { + "default": "0", + "depends_on": "eval:!doc.first_document", + "fieldname": "include_name_field", + "fieldtype": "Check", + "label": "Include Name Field" } ], "index_web_pages_for_search": 1, "links": [], - "modified": "2021-06-06 20:32:54.068774", + "modified": "2021-11-24 10:12:23.365136", "modified_by": "Administrator", "module": "Desk", "name": "Form Tour", @@ -88,4 +108,4 @@ "sort_field": "modified", "sort_order": "DESC", "track_changes": 1 -} \ No newline at end of file +} From 303edc8c0499eadb8e171132bbe820d011666819 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Wed, 24 Nov 2021 15:53:03 +0530 Subject: [PATCH 037/246] refactor: added get_sql function to query class (encapsulation++) --- frappe/database/database.py | 41 +++++--------------- frappe/database/query.py | 75 +++++++++++++++++++++++++++++++++---- 2 files changed, 78 insertions(+), 38 deletions(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index 52ce173cee..4ac86b76f2 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -116,10 +116,6 @@ class Database(object): if not run: return query - if not kwargs.get("ignore_permissions", True): - tables = self.get_tables_from_query(query) - self.check_permissions(doctype=tables, **kwargs) - if re.search(r'ifnull\(', query, flags=re.IGNORECASE): # replaces ifnull in query with coalesce query = re.sub(r'ifnull\(', 'coalesce(', query, flags=re.IGNORECASE) @@ -264,22 +260,6 @@ class Database(object): self.commit() self.sql(query, debug=debug) - @staticmethod - def check_permissions(doctype, **kwargs): - kwargs.pop("ignore_permissions") - if isinstance(doctype, str): - doctype = [doctype] - - for dt in doctype: - dt = re.sub("tab", "", dt) - if not frappe.has_permission( - dt, "select", **kwargs - ) and not frappe.has_permission(dt, "read", **kwargs): - frappe.throw(_("Insufficient Permission for {0}").format(frappe.bold(dt))) - - @staticmethod - def get_tables_from_query(query: str): - return [table for table in re.findall(r"\w+", query) if table.startswith("tab")] def check_transaction_status(self, query): """Raises exception if more than 20,000 `INSERT`, `UPDATE` queries are @@ -571,19 +551,18 @@ class Database(object): else: field_objects.append(field) - criterion = self.query.build_conditions( - table=doctype, filters=filters, orderby=order_by, for_update=for_update, **kwargs, + query = self.query.get_sql( + table=doctype, + filters=filters, + orderby=order_by, + for_update=for_update, + field_objects=field_objects, + fields=fields, + **kwargs, ) - if isinstance(fields, (list, tuple)): - query = criterion.select(*field_objects) + if fields=="*": + as_dict = True - elif isinstance(fields, Criterion): - query = criterion.select(fields) - - else: - if fields=="*": - query = criterion.select(fields) - as_dict = True r = self.sql(query, as_dict=as_dict, debug=debug, update=update, run=run, **kwargs) return r diff --git a/frappe/database/query.py b/frappe/database/query.py index 7341a7eb78..c962fc3675 100644 --- a/frappe/database/query.py +++ b/frappe/database/query.py @@ -1,4 +1,5 @@ import operator +import re from typing import Any, Dict, List, Tuple, Union import frappe @@ -246,7 +247,12 @@ class Query: conditions = self.add_conditions(conditions, **kwargs) return conditions - def build_conditions(self, table: str, filters: Union[Dict[str, Union[str, int]], str, int] = None, **kwargs) -> frappe.qb: + def build_conditions( + self, + table: str, + filters: Union[Dict[str, Union[str, int]], str, int] = None, + **kwargs + ) -> frappe.qb: """Build conditions for sql query Args: @@ -256,13 +262,68 @@ class Query: Returns: frappe.qb: frappe.qb conditions object """ - if isinstance(filters, Criterion): - return self.criterion_query(table, filters, **kwargs) - if isinstance(filters, int) or isinstance(filters, str): filters = {"name": str(filters)} - if isinstance(filters, (list, tuple)): - return self.misc_query(table, filters, **kwargs) + if isinstance(filters, Criterion): + criterion = self.criterion_query(table, filters, **kwargs) - return self.dict_query(filters=filters, table=table, **kwargs) + elif isinstance(filters, (list, tuple)): + criterion = self.misc_query(table, filters, **kwargs) + + else: + criterion = self.dict_query(filters=filters, table=table, **kwargs) + + return criterion + + def get_sql( + self, + table: str, + fields: Union[List, Tuple], + filters: Union[Dict[str, Union[str, int]], str, int] = None, + **kwargs + ): + criterion = self.build_conditions(table, filters, **kwargs) + if isinstance(fields, (list, tuple)): + query = criterion.select(*kwargs.get("field_objects")) + + elif isinstance(fields, Criterion): + query = criterion.select(fields) + + else: + if fields=="*": + query = criterion.select(fields) + + return query + + +class Permission: + @classmethod + def check_permissions(cls, query, **kwargs): + if not isinstance(query, str): + query = query.get_sql() + + doctype = cls.get_tables_from_query(query) + if isinstance(doctype, str): + doctype = [doctype] + + for dt in doctype: + dt = re.sub("tab", "", dt) + if not frappe.has_permission( + dt, + "select", + user=kwargs.get("user"), + parent_doctype=kwargs.get("parent_doctype"), + ) and not frappe.has_permission( + dt, + "read", + user=kwargs.get("user"), + parent_doctype=kwargs.get("parent_doctype"), + ): + frappe.throw( + _("Insufficient Permission for {0}").format(frappe.bold(dt)) + ) + + @staticmethod + def get_tables_from_query(query: str): + return [table for table in re.findall(r"\w+", query) if table.startswith("tab")] From c9b05f7d95a51d39b65cc46f5603294de81f6ce5 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Wed, 24 Nov 2021 16:10:29 +0530 Subject: [PATCH 038/246] fix: fixed as_dict in get_values --- frappe/database/database.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index 4ac86b76f2..4b1f835218 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -560,7 +560,11 @@ class Database(object): fields=fields, **kwargs, ) - if fields=="*": + if ( + fields == "*" + and not isinstance(fields, (list, tuple)) + and not isinstance(fields, Criterion) + ): as_dict = True r = self.sql(query, as_dict=as_dict, debug=debug, update=update, run=run, **kwargs) From 7323689654714cd30fe552aa87579fc0946b612d Mon Sep 17 00:00:00 2001 From: Jannat Patel Date: Wed, 24 Nov 2021 16:13:04 +0530 Subject: [PATCH 039/246] fix: tour of existing document --- frappe/core/form_tour/doctype/doctype.json | 4 ++- frappe/desk/doctype/form_tour/form_tour.js | 16 +++++++++- frappe/desk/doctype/form_tour/form_tour.json | 5 +-- frappe/public/js/frappe/form/form_tour.js | 2 -- .../js/frappe/widgets/onboarding_widget.js | 32 +++++++++++++++++-- 5 files changed, 51 insertions(+), 8 deletions(-) diff --git a/frappe/core/form_tour/doctype/doctype.json b/frappe/core/form_tour/doctype/doctype.json index 866d3ea508..3b77241201 100644 --- a/frappe/core/form_tour/doctype/doctype.json +++ b/frappe/core/form_tour/doctype/doctype.json @@ -2,9 +2,11 @@ "creation": "2021-11-23 12:38:52.807353", "docstatus": 0, "doctype": "Form Tour", + "first_document": 0, "idx": 0, + "include_name_field": 1, "is_standard": 1, - "modified": "2021-11-23 15:13:02.794031", + "modified": "2021-11-24 12:35:44.895630", "modified_by": "Administrator", "module": "Core", "name": "Doctype", diff --git a/frappe/desk/doctype/form_tour/form_tour.js b/frappe/desk/doctype/form_tour/form_tour.js index 8d70dcd3dc..2c6355749b 100644 --- a/frappe/desk/doctype/form_tour/form_tour.js +++ b/frappe/desk/doctype/form_tour/form_tour.js @@ -15,10 +15,13 @@ frappe.ui.form.on('Form Tour', { frm.add_custom_button(__('Show Tour'), async () => { const issingle = await check_if_single(frm.doc.reference_doctype); + const name = await get_first_document(frm.doc.reference_doctype); let route_changed = null; if (issingle) { route_changed = frappe.set_route('Form', frm.doc.reference_doctype); + } else if(frm.doc.first_document) { + route_changed = frappe.set_route('Form', frm.doc.reference_doctype, name); } else { route_changed = frappe.set_route('Form', frm.doc.reference_doctype, 'new'); } @@ -120,4 +123,15 @@ function get_child_field(child_table, child_name, fieldname) { async function check_if_single(doctype) { const { message } = await frappe.db.get_value('DocType', doctype, 'issingle'); return message.issingle || 0; -} \ No newline at end of file +} + +async function get_first_document(doctype) { + let docname; + + await frappe.db.get_list(doctype).then(res => { + if (Array.isArray(res) && res.length) + docname = res[0].name + }); + + return docname || 'new'; +} diff --git a/frappe/desk/doctype/form_tour/form_tour.json b/frappe/desk/doctype/form_tour/form_tour.json index 494a17bafb..6f3bd56a4e 100644 --- a/frappe/desk/doctype/form_tour/form_tour.json +++ b/frappe/desk/doctype/form_tour/form_tour.json @@ -86,10 +86,11 @@ ], "index_web_pages_for_search": 1, "links": [], - "modified": "2021-11-24 10:12:23.365136", + "modified": "2021-11-24 12:03:45.449311", "modified_by": "Administrator", "module": "Desk", "name": "Form Tour", + "naming_rule": "By fieldname", "owner": "Administrator", "permissions": [ { @@ -108,4 +109,4 @@ "sort_field": "modified", "sort_order": "DESC", "track_changes": 1 -} +} \ No newline at end of file diff --git a/frappe/public/js/frappe/form/form_tour.js b/frappe/public/js/frappe/form/form_tour.js index 9bae776d66..3aca58a159 100644 --- a/frappe/public/js/frappe/form/form_tour.js +++ b/frappe/public/js/frappe/form/form_tour.js @@ -243,7 +243,6 @@ frappe.ui.form.FormTour = class FormTour { } add_step_to_save() { - console.log("save") const page_id = `[id="page-${this.frm.doctype}"]`; const $save_btn = `${page_id} .standard-actions .primary-action`; const save_step = { @@ -286,7 +285,6 @@ frappe.ui.form.FormTour = class FormTour { this.driver_steps.splice(this.driver.currentStep + 1, 0, attach_dialog_step); this.update_driver_steps(); // need to define again, since driver.js only considers steps which are inside DOM frappe.utils.sleep(300).then(() => this.driver.start(this.driver.currentStep + 1)); - console.log('click', this.driver_steps) }, 1000); modal_element.on('hidden.bs.modal', () => { diff --git a/frappe/public/js/frappe/widgets/onboarding_widget.js b/frappe/public/js/frappe/widgets/onboarding_widget.js index 7237de2fb6..9827e0911d 100644 --- a/frappe/public/js/frappe/widgets/onboarding_widget.js +++ b/frappe/public/js/frappe/widgets/onboarding_widget.js @@ -304,8 +304,9 @@ export default class OnboardingWidget extends Widget { frappe.set_route("Form", step.reference_document); } - create_entry(step) { + async create_entry(step) { let current_route = frappe.get_route(); + let docname = await this.get_first_document(step.reference_document); frappe.route_hooks = {}; frappe.route_hooks.after_load = (frm) => { @@ -313,7 +314,19 @@ export default class OnboardingWidget extends Widget { frappe.msgprint({ message: __("Awesome, now try making an entry yourself"), title: __("Great"), + primary_action: { + action: () => { + frappe.set_route(current_route).then(() => { + this.mark_complete(step); + }); + }, + label: __("Continue"), + } }); + + frappe.msg_dialog.custom_onhide = () => { + this.mark_complete(step); + }; }; frm.tour .init({ on_finish }) @@ -351,7 +364,7 @@ export default class OnboardingWidget extends Widget { frappe.route_hooks.after_save = callback; } - frappe.set_route('Form', step.reference_document, 'new'); + frappe.set_route('Form', step.reference_document, docname); } show_quick_entry(step) { @@ -552,4 +565,19 @@ export default class OnboardingWidget extends Widget { } }); } + + async get_first_document(doctype) { + const { message } = await frappe.db.get_value('Form Tour', { 'reference_doctype': doctype }, ["first_document"]) + let docname; + + if (message.first_document) { + await frappe.db.get_list(doctype).then(res => { + if (Array.isArray(res) && res.length) + docname = res[0].name + }); + } + + + return docname || 'new'; + } } From 6a045f69f5d606fcf0c64aff9212b38dda1a25d8 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Wed, 24 Nov 2021 16:55:45 +0530 Subject: [PATCH 040/246] fix: fixed translate import --- frappe/database/query.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frappe/database/query.py b/frappe/database/query.py index c962fc3675..69328cb206 100644 --- a/frappe/database/query.py +++ b/frappe/database/query.py @@ -3,7 +3,8 @@ import re from typing import Any, Dict, List, Tuple, Union import frappe -from frappe.query_builder import Criterion, Order, Field +from frappe import _ +from frappe.query_builder import Criterion, Field, Order def like(key: str, value: str) -> frappe.qb: From 64be2a343efded59830f6193933e487f586b7d1b Mon Sep 17 00:00:00 2001 From: Summayya Date: Wed, 24 Nov 2021 18:49:28 +0530 Subject: [PATCH 041/246] fix: add condition to check allow_import --- frappe/public/js/frappe/list/list_view.js | 4 ++-- frappe/public/js/frappe/model/model.js | 11 +++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/frappe/public/js/frappe/list/list_view.js b/frappe/public/js/frappe/list/list_view.js index 64530e15ef..0104fd44c1 100644 --- a/frappe/public/js/frappe/list/list_view.js +++ b/frappe/public/js/frappe/list/list_view.js @@ -1484,8 +1484,8 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList { get_menu_items() { const doctype = this.doctype; const items = []; - - if (frappe.model.can_import(doctype)) { + + if (frappe.model.can_import(this.meta)) { items.push({ label: __("Import"), action: () => diff --git a/frappe/public/js/frappe/model/model.js b/frappe/public/js/frappe/model/model.js index 9e394a7433..37eba8d612 100644 --- a/frappe/public/js/frappe/model/model.js +++ b/frappe/public/js/frappe/model/model.js @@ -318,11 +318,14 @@ $.extend(frappe.model, { }, can_import: function(doctype, frm) { - // system manager can always import - if(frappe.user_roles.includes("System Manager")) return true; + if(doctype.allow_import) { + // system manager can always import + if(frappe.user_roles.includes("System Manager")) return true; - if(frm) return frm.perm[0].import===1; - return frappe.boot.user.can_import.indexOf(doctype)!==-1; + if(frm) return frm.perm[0].import===1; + return frappe.boot.user.can_import.indexOf(doctype)!==-1; + } + else return false; }, can_export: function(doctype, frm) { From 7cc6da212219e59a1d9be45b68b9d64db1b665f0 Mon Sep 17 00:00:00 2001 From: Jannat Patel Date: Wed, 24 Nov 2021 19:43:12 +0530 Subject: [PATCH 042/246] fix: sort result by creation --- frappe/desk/doctype/form_tour/form_tour.js | 2 +- frappe/public/js/frappe/widgets/onboarding_widget.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/desk/doctype/form_tour/form_tour.js b/frappe/desk/doctype/form_tour/form_tour.js index 2c6355749b..4aa1f7607f 100644 --- a/frappe/desk/doctype/form_tour/form_tour.js +++ b/frappe/desk/doctype/form_tour/form_tour.js @@ -128,7 +128,7 @@ async function check_if_single(doctype) { async function get_first_document(doctype) { let docname; - await frappe.db.get_list(doctype).then(res => { + await frappe.db.get_list(doctype, { order_by: "creation" }).then(res => { if (Array.isArray(res) && res.length) docname = res[0].name }); diff --git a/frappe/public/js/frappe/widgets/onboarding_widget.js b/frappe/public/js/frappe/widgets/onboarding_widget.js index 9827e0911d..1de44a89d5 100644 --- a/frappe/public/js/frappe/widgets/onboarding_widget.js +++ b/frappe/public/js/frappe/widgets/onboarding_widget.js @@ -571,7 +571,7 @@ export default class OnboardingWidget extends Widget { let docname; if (message.first_document) { - await frappe.db.get_list(doctype).then(res => { + await frappe.db.get_list(doctype, { order_by: "creation" }).then(res => { if (Array.isArray(res) && res.length) docname = res[0].name }); From 5b9c4e57816a1d13eaf2282b4b86c2cfdda172c4 Mon Sep 17 00:00:00 2001 From: Jannat Patel Date: Wed, 24 Nov 2021 20:11:29 +0530 Subject: [PATCH 043/246] fix: space and semicolon --- frappe/desk/doctype/form_tour/form_tour.js | 4 ++-- frappe/public/js/frappe/widgets/onboarding_widget.js | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/frappe/desk/doctype/form_tour/form_tour.js b/frappe/desk/doctype/form_tour/form_tour.js index 4aa1f7607f..6a7c736fac 100644 --- a/frappe/desk/doctype/form_tour/form_tour.js +++ b/frappe/desk/doctype/form_tour/form_tour.js @@ -20,7 +20,7 @@ frappe.ui.form.on('Form Tour', { if (issingle) { route_changed = frappe.set_route('Form', frm.doc.reference_doctype); - } else if(frm.doc.first_document) { + } else if (frm.doc.first_document) { route_changed = frappe.set_route('Form', frm.doc.reference_doctype, name); } else { route_changed = frappe.set_route('Form', frm.doc.reference_doctype, 'new'); @@ -130,7 +130,7 @@ async function get_first_document(doctype) { await frappe.db.get_list(doctype, { order_by: "creation" }).then(res => { if (Array.isArray(res) && res.length) - docname = res[0].name + docname = res[0].name; }); return docname || 'new'; diff --git a/frappe/public/js/frappe/widgets/onboarding_widget.js b/frappe/public/js/frappe/widgets/onboarding_widget.js index 1de44a89d5..110d617f73 100644 --- a/frappe/public/js/frappe/widgets/onboarding_widget.js +++ b/frappe/public/js/frappe/widgets/onboarding_widget.js @@ -567,13 +567,13 @@ export default class OnboardingWidget extends Widget { } async get_first_document(doctype) { - const { message } = await frappe.db.get_value('Form Tour', { 'reference_doctype': doctype }, ["first_document"]) + const { message } = await frappe.db.get_value('Form Tour', { 'reference_doctype': doctype }, ["first_document"]); let docname; if (message.first_document) { await frappe.db.get_list(doctype, { order_by: "creation" }).then(res => { if (Array.isArray(res) && res.length) - docname = res[0].name + docname = res[0].name; }); } From 088fdca74db53ffe48cc2357e85e2ee263bb5456 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 22 Nov 2021 13:03:38 +0530 Subject: [PATCH 044/246] refactor: bench browse * Manage colours through click * Standardize the command's behaviour for consistency :') * Use click instead of webbrowser module * Simplify logic --- frappe/commands/site.py | 50 ++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 26 deletions(-) diff --git a/frappe/commands/site.py b/frappe/commands/site.py index 3c7f2f5525..fb8d7e0a6c 100755 --- a/frappe/commands/site.py +++ b/frappe/commands/site.py @@ -829,39 +829,37 @@ def publish_realtime(context, event, message, room, user, doctype, docname, afte @pass_context def browse(context, site, user=None): '''Opens the site on web browser''' - from frappe.auth import LoginManager - from frappe.auth import CookieManager - import webbrowser + from frappe.auth import CookieManager, LoginManager - site = context.sites[0] if context.sites else site + site = get_site(context, raise_err=False) or site if not site: - click.echo('''Please provide site name\n\nUsage:\n\tbench browse [site-name]\nor\n\tbench --site [site-name] browse''') - return + raise SiteNotSpecifiedError - site = site.lower() + if site not in frappe.utils.get_sites(): + click.echo(f"\nSite named {click.style(site, bold=True)} doesn't exist\n", err=True) + sys.exit(1) - if site in frappe.utils.get_sites(): - frappe.init(site=site) - frappe.connect() + frappe.init(site=site) + frappe.connect() - sid = '' - if user: - if frappe.conf.developer_mode or user == "Administrator": - frappe.utils.set_request(path="/") - frappe.local.cookie_manager = CookieManager() - frappe.local.login_manager = LoginManager() - frappe.local.login_manager.login_as(user) - sid = f'/app?sid={frappe.session.sid}' - else: - print("Please enable developer mode to login as a user") + sid = '' + if user: + if frappe.conf.developer_mode or user == "Administrator": + frappe.utils.set_request(path="/") + frappe.local.cookie_manager = CookieManager() + frappe.local.login_manager = LoginManager() + frappe.local.login_manager.login_as(user) + sid = f'/app?sid={frappe.session.sid}' + else: + click.echo("Please enable developer mode to login as a user") - url = f'{frappe.utils.get_site_url(site)}{sid}' - if user == "Administrator": - print(f'Login URL: {url}') - webbrowser.open(url, new=2) - else: - click.echo("\nSite named \033[1m{}\033[0m doesn't exist\n".format(site)) + url = f'{frappe.utils.get_site_url(site)}{sid}' + + if user == "Administrator": + click.echo(f'Login URL: {url}') + + click.launch(url) @click.command('start-recording') From 5914f35bb761ec2d25c014eb349e7d96c727d490 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 23 Nov 2021 11:49:19 +0530 Subject: [PATCH 045/246] fix: Change site archive path to archived/sites --- frappe/commands/site.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/frappe/commands/site.py b/frappe/commands/site.py index fb8d7e0a6c..6d3ed1af16 100755 --- a/frappe/commands/site.py +++ b/frappe/commands/site.py @@ -447,11 +447,10 @@ def disable_user(context, email): @pass_context def migrate(context, skip_failing=False, skip_search_index=False): "Run patches, sync schema and rebuild files/translations" - import re from frappe.migrate import migrate for site in context.sites: - print('Migrating', site) + click.secho(f"Migrating {site}", fg="green") frappe.init(site=site) frappe.connect() try: @@ -697,8 +696,7 @@ def _drop_site(site, root_login='root', root_password=None, archived_sites_path= drop_user_and_database(frappe.conf.db_name, root_login, root_password) - if not archived_sites_path: - archived_sites_path = os.path.join(frappe.get_app_path('frappe'), '..', '..', '..', 'archived_sites') + archived_sites_path = archived_sites_path or os.path.join(frappe.get_app_path('frappe'), '..', '..', '..', 'archived', 'sites') if not os.path.exists(archived_sites_path): os.mkdir(archived_sites_path) From 0657524888c2d148be4976d21b353f2f4e321cbe Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 25 Nov 2021 12:49:10 +0530 Subject: [PATCH 046/246] refactor: Retire color in favour of click.secho * Add type hints for Command test suite * Remove support utils - dead code --- frappe/tests/test_commands.py | 41 +++++++---------------------------- 1 file changed, 8 insertions(+), 33 deletions(-) diff --git a/frappe/tests/test_commands.py b/frappe/tests/test_commands.py index 94389cd7a3..14ed77eeeb 100644 --- a/frappe/tests/test_commands.py +++ b/frappe/tests/test_commands.py @@ -7,7 +7,7 @@ import os import shlex import shutil import subprocess -import sys +from typing import List import unittest import glob @@ -18,37 +18,11 @@ from frappe.installer import add_to_installed_apps, remove_app from frappe.utils import add_to_date, get_bench_relative_path, now from frappe.utils.backups import fetch_latest_backups - -# TODO: check frappe.cli.coloured_output to set coloured output! -def supports_color(): - """ - Returns True if the running system's terminal supports color, and False - otherwise. - """ - plat = sys.platform - supported_platform = plat != 'Pocket PC' and (plat != 'win32' or 'ANSICON' in os.environ) - # isatty is not always implemented, #6223. - is_a_tty = hasattr(sys.stdout, 'isatty') and sys.stdout.isatty() - return supported_platform and is_a_tty +# imports - third party imports +import click -class color(dict): - nc = "\033[0m" - blue = "\033[94m" - green = "\033[92m" - yellow = "\033[93m" - red = "\033[91m" - silver = "\033[90m" - - def __getattr__(self, key): - if supports_color(): - ret = self.get(key) - else: - ret = "" - return ret - - -def clean(value): +def clean(value) -> str: """Strips and converts bytes to str Args: @@ -64,7 +38,7 @@ def clean(value): return value -def missing_in_backup(doctypes, file): +def missing_in_backup(doctypes: List, file: os.PathLike) -> List: """Returns list of missing doctypes in the backup. Args: @@ -86,7 +60,7 @@ def missing_in_backup(doctypes, file): if predicate.format(doctype).lower() not in content] -def exists_in_backup(doctypes, file): +def exists_in_backup(doctypes: List, file: os.PathLike) -> bool: """Checks if the list of doctypes exist in the database.sql.gz file supplied Args: @@ -118,7 +92,8 @@ class BaseTestCommands(unittest.TestCase): kwargs = site self.command = " ".join(command.split()).format(**kwargs) - print("{0}$ {1}{2}".format(color.silver, self.command, color.nc)) + click.secho(self.command, fg="bright_black") + command = shlex.split(self.command) self._proc = subprocess.run(command, input=cmd_input, stdout=subprocess.PIPE, stderr=subprocess.PIPE) self.stdout = clean(self._proc.stdout) From f04b2157f199da71a53d0986dfba56e075a4fecd Mon Sep 17 00:00:00 2001 From: Aradhya Date: Thu, 25 Nov 2021 13:07:38 +0530 Subject: [PATCH 047/246] refactor: getting rid of kwargs --- frappe/database/database.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index 4b1f835218..bfdaebde75 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -85,7 +85,7 @@ class Database(object): def sql(self, query, values=(), as_dict = 0, as_list = 0, formatted = 0, debug=0, ignore_ddl=0, as_utf8=0, auto_commit=0, update=None, - explain=False, run=True, pluck=False, **kwargs): + explain=False, run=True, pluck=False): """Execute a SQL query and fetch all rows. :param query: SQL query. @@ -336,7 +336,7 @@ class Database(object): return self.get_value(doctype, filters, "*", as_dict=as_dict, cache=cache) def get_value(self, doctype, filters=None, fieldname="name", ignore=None, as_dict=False, - debug=False, order_by="KEEP_DEFAULT_ORDERING", cache=False, for_update=False, run=True, **kwargs): + debug=False, order_by="KEEP_DEFAULT_ORDERING", cache=False, for_update=False, run=True): """Returns a document property or list of properties. :param doctype: DocType name. @@ -363,7 +363,7 @@ class Database(object): """ ret = self.get_values(doctype, filters, fieldname, ignore, as_dict, debug, - order_by, cache=cache, for_update=for_update, run=run, **kwargs) + order_by, cache=cache, for_update=for_update, run=run) if not run: return ret @@ -371,7 +371,7 @@ class Database(object): return ((len(ret[0]) > 1 or as_dict) and ret[0] or ret[0][0]) if ret else None def get_values(self, doctype, filters=None, fieldname="name", ignore=None, as_dict=False, - debug=False, order_by="KEEP_DEFAULT_ORDERING", update=None, cache=False, for_update=False, run=True, **kwargs): + debug=False, order_by="KEEP_DEFAULT_ORDERING", update=None, cache=False, for_update=False, run=True): """Returns multiple document properties. :param doctype: DocType name. @@ -396,7 +396,7 @@ class Database(object): return self.value_cache[(doctype, filters, fieldname)] if isinstance(filters, list): - out = self._get_value_for_many_names(doctype, filters, fieldname, debug=debug, run=run, **kwargs) + out = self._get_value_for_many_names(doctype, filters, fieldname, order_by, debug=debug, run=run) else: fields = fieldname @@ -411,7 +411,7 @@ class Database(object): if order_by: order_by = "modified" if order_by == "KEEP_DEFAULT_ORDERING" else order_by out = self._get_values_from_table( - fields, filters, doctype, as_dict, debug, order_by, update, for_update=for_update, run=run, **kwargs + fields, filters, doctype, as_dict, debug, order_by, update, for_update=for_update, run=run ) except Exception as e: if ignore and (frappe.db.is_missing_column(e) or frappe.db.is_table_missing(e)): @@ -570,17 +570,17 @@ class Database(object): r = self.sql(query, as_dict=as_dict, debug=debug, update=update, run=run, **kwargs) return r - def _get_value_for_many_names(self, doctype, names, field, debug=False, run=True, **kwargs): + def _get_value_for_many_names(self, doctype, names, field, order_by, debug=False, run=True): names = list(filter(None, names)) if names: return self.get_all( doctype, fields=field, filters=names, + order_by=order_by, debug=debug, as_list=1, run=run, - **kwargs, ) else: return {} From 435cf503facde7ba4b89866a896e06ac78d3e2e0 Mon Sep 17 00:00:00 2001 From: Summayya Date: Thu, 25 Nov 2021 15:00:22 +0530 Subject: [PATCH 048/246] refactor: rremove else statement --- frappe/public/js/frappe/model/model.js | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/frappe/public/js/frappe/model/model.js b/frappe/public/js/frappe/model/model.js index 37eba8d612..badd1859e3 100644 --- a/frappe/public/js/frappe/model/model.js +++ b/frappe/public/js/frappe/model/model.js @@ -318,14 +318,13 @@ $.extend(frappe.model, { }, can_import: function(doctype, frm) { - if(doctype.allow_import) { - // system manager can always import - if(frappe.user_roles.includes("System Manager")) return true; + if (!doctype.allow_import) return false; - if(frm) return frm.perm[0].import===1; - return frappe.boot.user.can_import.indexOf(doctype)!==-1; - } - else return false; + // system manager can always import + if (frappe.user_roles.includes("System Manager")) return true; + + if (frm) return frm.perm[0].import===1; + return frappe.boot.user.can_import.indexOf(doctype.name)!==-1; }, can_export: function(doctype, frm) { From fc65c2cd3666a1d6fb146fdd8aa4a325a5d9cfb4 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Thu, 25 Nov 2021 16:20:06 +0530 Subject: [PATCH 049/246] fix: fixed pluck in execute --- frappe/database/database.py | 69 +++++++++++++++++++++++++++++-------- 1 file changed, 54 insertions(+), 15 deletions(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index bfdaebde75..411587aa7e 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -336,7 +336,7 @@ class Database(object): return self.get_value(doctype, filters, "*", as_dict=as_dict, cache=cache) def get_value(self, doctype, filters=None, fieldname="name", ignore=None, as_dict=False, - debug=False, order_by="KEEP_DEFAULT_ORDERING", cache=False, for_update=False, run=True): + debug=False, order_by="KEEP_DEFAULT_ORDERING", cache=False, for_update=False, run=True, pluck=False): """Returns a document property or list of properties. :param doctype: DocType name. @@ -363,7 +363,7 @@ class Database(object): """ ret = self.get_values(doctype, filters, fieldname, ignore, as_dict, debug, - order_by, cache=cache, for_update=for_update, run=run) + order_by, cache=cache, for_update=for_update, run=run, pluck=pluck) if not run: return ret @@ -371,7 +371,8 @@ class Database(object): return ((len(ret[0]) > 1 or as_dict) and ret[0] or ret[0][0]) if ret else None def get_values(self, doctype, filters=None, fieldname="name", ignore=None, as_dict=False, - debug=False, order_by="KEEP_DEFAULT_ORDERING", update=None, cache=False, for_update=False, run=True): + debug=False, order_by="KEEP_DEFAULT_ORDERING", update=None, cache=False, for_update=False, + run=True, pluck=False): """Returns multiple document properties. :param doctype: DocType name. @@ -396,7 +397,7 @@ class Database(object): return self.value_cache[(doctype, filters, fieldname)] if isinstance(filters, list): - out = self._get_value_for_many_names(doctype, filters, fieldname, order_by, debug=debug, run=run) + out = self._get_value_for_many_names(doctype, filters, fieldname, order_by, debug=debug, run=run, pluck=pluck) else: fields = fieldname @@ -411,7 +412,16 @@ class Database(object): if order_by: order_by = "modified" if order_by == "KEEP_DEFAULT_ORDERING" else order_by out = self._get_values_from_table( - fields, filters, doctype, as_dict, debug, order_by, update, for_update=for_update, run=run + fields, + filters, + doctype, + as_dict, + debug, + order_by, + update, + for_update=for_update, + run=run, + pluck=pluck, ) except Exception as e: if ignore and (frappe.db.is_missing_column(e) or frappe.db.is_table_missing(e)): @@ -424,14 +434,24 @@ class Database(object): else: raise else: - out = self.get_values_from_single(fields, filters, doctype, as_dict, debug, update, run=run) + out = self.get_values_from_single(fields, filters, doctype, as_dict, debug, update, run=run, pluck=pluck) if cache and isinstance(filters, str): self.value_cache[(doctype, filters, fieldname)] = out return out - def get_values_from_single(self, fields, filters, doctype, as_dict=False, debug=False, update=None, run=True): + def get_values_from_single( + self, + fields, + filters, + doctype, + as_dict=False, + debug=False, + update=None, + run=True, + pluck=False, + ): """Get values from `tabSingles` (Single DocTypes) (internal). :param fields: List of fields, @@ -457,10 +477,16 @@ class Database(object): return [map(values.get, fields)] else: - r = self.sql("""select field, value + r = self.sql( + """select field, value from `tabSingles` where field in (%s) and doctype=%s""" - % (', '.join(['%s'] * len(fields)), '%s'), - tuple(fields) + (doctype,), as_dict=False, debug=debug, run=run) + % (", ".join(["%s"] * len(fields)), "%s"), + tuple(fields) + (doctype,), + as_dict=False, + debug=debug, + run=run, + pluck=pluck, + ) if not run: return r if as_dict: @@ -540,8 +566,19 @@ class Database(object): """Alias for get_single_value""" return self.get_single_value(*args, **kwargs) - def _get_values_from_table(self, fields, filters, doctype, as_dict, debug, order_by=None, - update=None, for_update=False, run=True, **kwargs): + def _get_values_from_table( + self, + fields, + filters, + doctype, + as_dict, + debug, + order_by=None, + update=None, + for_update=False, + run=True, + pluck=False, + ): field_objects = [] if not isinstance(fields, Criterion): @@ -558,7 +595,6 @@ class Database(object): for_update=for_update, field_objects=field_objects, fields=fields, - **kwargs, ) if ( fields == "*" @@ -567,10 +603,12 @@ class Database(object): ): as_dict = True - r = self.sql(query, as_dict=as_dict, debug=debug, update=update, run=run, **kwargs) + r = self.sql( + query, as_dict=as_dict, debug=debug, update=update, run=run, pluck=pluck + ) return r - def _get_value_for_many_names(self, doctype, names, field, order_by, debug=False, run=True): + def _get_value_for_many_names(self, doctype, names, field, order_by, debug=False, run=True, pluck=False): names = list(filter(None, names)) if names: return self.get_all( @@ -578,6 +616,7 @@ class Database(object): fields=field, filters=names, order_by=order_by, + pluck=pluck, debug=debug, as_list=1, run=run, From 0ea0f7dfa3fd7f4282e35e0a9fe03ce3c4016d21 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Thu, 25 Nov 2021 16:39:44 +0530 Subject: [PATCH 050/246] fix: fixed default ordering in execute --- frappe/model/db_query.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/model/db_query.py b/frappe/model/db_query.py index c94819a8c3..16c0d18d9f 100644 --- a/frappe/model/db_query.py +++ b/frappe/model/db_query.py @@ -29,7 +29,7 @@ class DatabaseQuery(object): self.reference_doctype = None def execute(self, fields=None, filters=None, or_filters=None, - docstatus=None, group_by=None, order_by="default_ordering", limit_start=False, + docstatus=None, group_by=None, order_by="KEEP_DEFAULT_ORDERING", limit_start=False, limit_page_length=None, as_list=False, with_childnames=False, debug=False, ignore_permissions=False, user=None, with_comment_count=False, join='left join', distinct=False, start=None, page_length=None, limit=None, @@ -703,7 +703,7 @@ class DatabaseQuery(object): def set_order_by(self, args): meta = frappe.get_meta(self.doctype) - if self.order_by and self.order_by != "default_ordering": + if self.order_by and self.order_by != "KEEP_DEFAULT_ORDERING": args.order_by = self.order_by else: args.order_by = "" From 83272fd1ecfc818a1d39a8c3b9d226fff67d893d Mon Sep 17 00:00:00 2001 From: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> Date: Thu, 25 Nov 2021 16:47:58 +0530 Subject: [PATCH 051/246] style: Remove extra tabs Co-authored-by: gavin --- frappe/public/js/frappe/list/list_view.js | 1 - 1 file changed, 1 deletion(-) diff --git a/frappe/public/js/frappe/list/list_view.js b/frappe/public/js/frappe/list/list_view.js index 0104fd44c1..9218217162 100644 --- a/frappe/public/js/frappe/list/list_view.js +++ b/frappe/public/js/frappe/list/list_view.js @@ -1484,7 +1484,6 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList { get_menu_items() { const doctype = this.doctype; const items = []; - if (frappe.model.can_import(this.meta)) { items.push({ label: __("Import"), From c13e1838d7afec30fe1b018ccd9aff7c9bd0db70 Mon Sep 17 00:00:00 2001 From: Jannat Patel Date: Thu, 25 Nov 2021 17:57:07 +0530 Subject: [PATCH 052/246] fix: module load issue, tour description and save field overlay --- frappe/core/form_tour/doctype/doctype.json | 11 ++++++----- frappe/public/js/frappe/form/form_tour.js | 3 ++- frappe/utils/install.py | 2 ++ 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/frappe/core/form_tour/doctype/doctype.json b/frappe/core/form_tour/doctype/doctype.json index e38b9640cc..391d3ecf40 100644 --- a/frappe/core/form_tour/doctype/doctype.json +++ b/frappe/core/form_tour/doctype/doctype.json @@ -6,7 +6,7 @@ "idx": 0, "include_name_field": 1, "is_standard": 1, - "modified": "2021-11-24 17:25:18.317075", + "modified": "2021-11-25 17:03:01.646360", "modified_by": "Administrator", "module": "Core", "name": "Doctype", @@ -15,7 +15,7 @@ "save_on_complete": 1, "steps": [ { - "description": "Select a Module to which this Doctype would belong", + "description": "Select a Module to which this DocType would belong", "field": "", "fieldname": "module", "fieldtype": "Link", @@ -27,19 +27,20 @@ "title": "Module" }, { - "description": "Check this to make the Docytpe as Custom", + "description": "Check this to make the DocType as Custom", "field": "", "fieldname": "custom", "fieldtype": "Check", - "has_next_condition": 0, + "has_next_condition": 1, "is_table_field": 0, "label": "Custom?", + "next_step_condition": "eval: doc.custom", "parent_field": "", "position": "Left", "title": "Custom " }, { - "description": "Add fields to this Custom Doctype", + "description": "A Field (or a docfield) defines a property of a DocType. You can define the column name, label, datatype and more for DocFields. For instance, a ToDo doctype has fields description, status and priority. These ultimately become columns in the database table tabToDo.", "field": "", "fieldname": "fields", "fieldtype": "Table", diff --git a/frappe/public/js/frappe/form/form_tour.js b/frappe/public/js/frappe/form/form_tour.js index 9b93d160ed..84c7afdef4 100644 --- a/frappe/public/js/frappe/form/form_tour.js +++ b/frappe/public/js/frappe/form/form_tour.js @@ -18,6 +18,7 @@ frappe.ui.form.FormTour = class FormTour { // if last step is to save, then attach a listener to save button if (step.options.is_save_step) { $(step.options.element).one('click', () => this.driver.reset()); + this.driver.overlay.refresh(); } // focus on input @@ -54,7 +55,7 @@ frappe.ui.form.FormTour = class FormTour { include_name_field() { const name_step = { - "description": "Enter a name", + "description": `Enter a name for this ${this.frm.doctype}`, "fieldname": "__newname", "title": "Name", "position": "right", diff --git a/frappe/utils/install.py b/frappe/utils/install.py index 1cc94a68d6..2bb58a6b4c 100644 --- a/frappe/utils/install.py +++ b/frappe/utils/install.py @@ -5,6 +5,8 @@ import getpass from frappe.utils.password import update_password def before_install(): + frappe.reload_doc("desk", "doctype", "form_tour_step") + frappe.reload_doc("desk", "doctype", "form_tour") frappe.reload_doc("core", "doctype", "docfield") frappe.reload_doc("core", "doctype", "docperm") frappe.reload_doc("core", "doctype", "doctype_action") From 171e8b06f83ea79a7b76dd6edf5252c1684fc076 Mon Sep 17 00:00:00 2001 From: Manuel <57345036+mtraeber@users.noreply.github.com> Date: Thu, 25 Nov 2021 13:30:45 +0100 Subject: [PATCH 053/246] Update frappe/email/doctype/email_account/email_account.py Co-authored-by: gavin --- frappe/email/doctype/email_account/email_account.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index 18c9ca1737..1e5d1dc0ba 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -549,7 +549,7 @@ class EmailAccount(Document): def on_trash(self): """Clear communications where email account is linked""" - Communication = frappe.qb.from_("Communication") + Communication = frappe.qb.DocType("Communication") frappe.qb.update(Communication) \ .set(Communication.email_account == "") \ .where(Communication.email_account == self.name).run() From 4d2f9157ae3342e816481fa55d77d676663568e3 Mon Sep 17 00:00:00 2001 From: Summayya Date: Thu, 25 Nov 2021 20:25:04 +0530 Subject: [PATCH 054/246] refactor: add meta as separate parameter --- frappe/public/js/frappe/list/list_view.js | 2 +- frappe/public/js/frappe/model/model.js | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/frappe/public/js/frappe/list/list_view.js b/frappe/public/js/frappe/list/list_view.js index 0104fd44c1..582aaa2ec7 100644 --- a/frappe/public/js/frappe/list/list_view.js +++ b/frappe/public/js/frappe/list/list_view.js @@ -1485,7 +1485,7 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList { const doctype = this.doctype; const items = []; - if (frappe.model.can_import(this.meta)) { + if (frappe.model.can_import(doctype, null, this.meta)) { items.push({ label: __("Import"), action: () => diff --git a/frappe/public/js/frappe/model/model.js b/frappe/public/js/frappe/model/model.js index badd1859e3..041905408a 100644 --- a/frappe/public/js/frappe/model/model.js +++ b/frappe/public/js/frappe/model/model.js @@ -317,14 +317,14 @@ $.extend(frappe.model, { return doc && doc.__last_sync_on && ((new Date() - doc.__last_sync_on)) < 5000; }, - can_import: function(doctype, frm) { - if (!doctype.allow_import) return false; + can_import: function(doctype, frm, meta=null) { + if (meta && !meta.allow_import) return false; // system manager can always import if (frappe.user_roles.includes("System Manager")) return true; if (frm) return frm.perm[0].import===1; - return frappe.boot.user.can_import.indexOf(doctype.name)!==-1; + return frappe.boot.user.can_import.indexOf(doctype)!==-1; }, can_export: function(doctype, frm) { From 48111964ad67de96ce30b961ddcd6357d73f74a0 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Thu, 25 Nov 2021 22:06:50 +0530 Subject: [PATCH 055/246] fix: Remove unnecessary style file for note --- frappe/desk/doctype/note/note.css | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 frappe/desk/doctype/note/note.css diff --git a/frappe/desk/doctype/note/note.css b/frappe/desk/doctype/note/note.css deleted file mode 100644 index b5026d2e46..0000000000 --- a/frappe/desk/doctype/note/note.css +++ /dev/null @@ -1,3 +0,0 @@ -.like-disabled-input{ - background-color: #fff; -} \ No newline at end of file From fdab567fc2340cafe6ab88c53d0596c29adec272 Mon Sep 17 00:00:00 2001 From: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> Date: Fri, 26 Nov 2021 08:47:54 +0530 Subject: [PATCH 056/246] style: Remove unnecessary tabs --- frappe/public/js/frappe/list/list_view.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/list/list_view.js b/frappe/public/js/frappe/list/list_view.js index 582aaa2ec7..3c9f1e39fb 100644 --- a/frappe/public/js/frappe/list/list_view.js +++ b/frappe/public/js/frappe/list/list_view.js @@ -1484,7 +1484,7 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList { get_menu_items() { const doctype = this.doctype; const items = []; - + if (frappe.model.can_import(doctype, null, this.meta)) { items.push({ label: __("Import"), From ba6d96855b909fe9660453b7daf3a420b398610e Mon Sep 17 00:00:00 2001 From: Sagar Sharma Date: Wed, 10 Nov 2021 20:52:13 +0530 Subject: [PATCH 057/246] fix: Image(link) render as text in print format (cherry picked from commit df3692b51e66771c9308a9c7c744c943c1f95752) --- frappe/templates/print_formats/standard_macros.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/templates/print_formats/standard_macros.html b/frappe/templates/print_formats/standard_macros.html index 9986e45999..580a41f959 100644 --- a/frappe/templates/print_formats/standard_macros.html +++ b/frappe/templates/print_formats/standard_macros.html @@ -153,7 +153,7 @@ data-fieldname="{{ df.fieldname }}" data-fieldtype="{{ df.fieldtype }}" {% elif df.fieldtype=="Signature" %} - {% elif df.fieldtype in ("Attach", "Attach Image") and frappe.utils.is_image(doc[df.fieldname]) %} + {% elif df.fieldtype in ("Attach", "Attach Image") %} {% elif df.fieldtype=="HTML" %} From d57fe8f7990cd370be3c3c121c5c7398aedb08b9 Mon Sep 17 00:00:00 2001 From: Manuel <57345036+mtraeber@users.noreply.github.com> Date: Fri, 26 Nov 2021 08:08:18 +0100 Subject: [PATCH 058/246] Update frappe/email/doctype/email_account/email_account.py Co-authored-by: Abhishek Saxena <33656173+saxenabhishek@users.noreply.github.com> --- frappe/email/doctype/email_account/email_account.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index 1e5d1dc0ba..52623fb358 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -551,7 +551,7 @@ class EmailAccount(Document): """Clear communications where email account is linked""" Communication = frappe.qb.DocType("Communication") frappe.qb.update(Communication) \ - .set(Communication.email_account == "") \ + .set(Communication.email_account, "") \ .where(Communication.email_account == self.name).run() remove_user_email_inbox(email_account=self.name) From 22434d065cf3a4a24f6e0659fa1669892019512c Mon Sep 17 00:00:00 2001 From: Aradhya Date: Fri, 26 Nov 2021 13:09:48 +0530 Subject: [PATCH 059/246] feat: Added aggregation functions to qb functions refactor: changed args to aggregation funcs to match db level aggregation funcs --- frappe/query_builder/functions.py | 39 +++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/frappe/query_builder/functions.py b/frappe/query_builder/functions.py index 39c67178c2..ddb10831c5 100644 --- a/frappe/query_builder/functions.py +++ b/frappe/query_builder/functions.py @@ -2,6 +2,8 @@ from pypika.functions import * from pypika.terms import Function from frappe.query_builder.utils import ImportMapper, db_type_is from frappe.query_builder.custom import GROUP_CONCAT, STRING_AGG, MATCH, TO_TSVECTOR +from frappe.database.query import Query +from .utils import Column class Concat_ws(Function): @@ -22,3 +24,40 @@ Match = ImportMapper( db_type_is.POSTGRES: TO_TSVECTOR } ) + + +def max(dt, fieldname, filters=None, **kwargs): + return ( + Query() + .build_conditions(dt, filters) + .select(Max(Column(fieldname))) + .run(**kwargs)[0][0] + or 0 + ) + +def min(dt, fieldname, filters=None, **kwargs): + return ( + Query() + .build_conditions(dt, filters) + .select(Min(Column(fieldname))) + .run(**kwargs)[0][0] + or 0 + ) + +def avg(dt, fieldname, filters=None, **kwargs): + return ( + Query() + .build_conditions(dt, filters) + .select(Avg(Column(fieldname))) + .run(**kwargs)[0][0] + or 0 + ) + +def sum(dt, fieldname, filters=None, **kwargs): + return ( + Query() + .build_conditions(dt, filters) + .select(Sum(Column(fieldname))) + .run(**kwargs)[0][0] + or 0 + ) \ No newline at end of file From 971a581359c8514f0112626a5cd6b6fdce758ee2 Mon Sep 17 00:00:00 2001 From: Jannat Patel Date: Fri, 26 Nov 2021 14:01:57 +0530 Subject: [PATCH 060/246] fix: order of reload for doctypes --- frappe/utils/install.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/utils/install.py b/frappe/utils/install.py index 2bb58a6b4c..c8c118ede8 100644 --- a/frappe/utils/install.py +++ b/frappe/utils/install.py @@ -5,12 +5,12 @@ import getpass from frappe.utils.password import update_password def before_install(): - frappe.reload_doc("desk", "doctype", "form_tour_step") - frappe.reload_doc("desk", "doctype", "form_tour") frappe.reload_doc("core", "doctype", "docfield") frappe.reload_doc("core", "doctype", "docperm") frappe.reload_doc("core", "doctype", "doctype_action") frappe.reload_doc("core", "doctype", "doctype_link") + frappe.reload_doc("desk", "doctype", "form_tour_step") + frappe.reload_doc("desk", "doctype", "form_tour") frappe.reload_doc("core", "doctype", "doctype") def after_install(): From 9a190c145ed389d01b9d1dbc945ab78b39bbf6a6 Mon Sep 17 00:00:00 2001 From: Jannat Patel Date: Fri, 26 Nov 2021 14:19:23 +0530 Subject: [PATCH 061/246] fix: translatable strings --- frappe/templates/styles/discussion_style.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/templates/styles/discussion_style.css b/frappe/templates/styles/discussion_style.css index f1dab60589..975376c484 100644 --- a/frappe/templates/styles/discussion_style.css +++ b/frappe/templates/styles/discussion_style.css @@ -37,7 +37,7 @@ } .no-discussions { - width: 500px; + width: 80%; margin: 0 auto; text-align: center; } From e3bdf110061c4e36fd934e205303d559884247b3 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Fri, 26 Nov 2021 14:19:59 +0530 Subject: [PATCH 062/246] refactor: moved aggregation functions to Query Builder --- frappe/__init__.py | 7 ++++++- frappe/query_builder/__init__.py | 2 +- frappe/query_builder/utils.py | 11 +++++++++++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index 895bdcaddc..a87f930be6 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -28,7 +28,11 @@ from .exceptions import * from .utils.jinja import (get_jenv, get_template, render_template, get_email_from_template, get_jloader) from .utils.lazy_loader import lazy_import -from frappe.query_builder import get_query_builder, patch_query_execute +from frappe.query_builder import ( + get_query_builder, + patch_query_execute, + patch_query_aggregation, +) __version__ = '14.0.0-dev' @@ -211,6 +215,7 @@ def init(site, sites_path=None, new_site=False): setup_module_map() patch_query_execute() + patch_query_aggregation() local.initialised = True diff --git a/frappe/query_builder/__init__.py b/frappe/query_builder/__init__.py index 4a1fe8fb84..9c7432142f 100644 --- a/frappe/query_builder/__init__.py +++ b/frappe/query_builder/__init__.py @@ -1,2 +1,2 @@ from pypika import * -from frappe.query_builder.utils import Column, DocType, get_query_builder, patch_query_execute +from frappe.query_builder.utils import Column, DocType, get_query_builder, patch_query_execute, patch_query_aggregation diff --git a/frappe/query_builder/utils.py b/frappe/query_builder/utils.py index 386ddda751..e2916a859c 100644 --- a/frappe/query_builder/utils.py +++ b/frappe/query_builder/utils.py @@ -66,3 +66,14 @@ def patch_query_execute(): raise BuilderIdentificationFailed builder_class.run = execute_query + + +def patch_query_aggregation(): + """Patch aggregation functions to frappe.qb + """ + from frappe.query_builder.functions import max, min, avg, sum + + frappe.qb.max = max + frappe.qb.min = min + frappe.qb.avg = avg + frappe.qb.sum = sum \ No newline at end of file From b60452ca18dce578f1d097361bdf3abbb40cea13 Mon Sep 17 00:00:00 2001 From: Jannat Patel Date: Fri, 26 Nov 2021 15:13:24 +0530 Subject: [PATCH 063/246] fix: removed unrelated code --- frappe/templates/styles/discussion_style.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/templates/styles/discussion_style.css b/frappe/templates/styles/discussion_style.css index 975376c484..f1dab60589 100644 --- a/frappe/templates/styles/discussion_style.css +++ b/frappe/templates/styles/discussion_style.css @@ -37,7 +37,7 @@ } .no-discussions { - width: 80%; + width: 500px; margin: 0 auto; text-align: center; } From b5c73648dc84cd4d8238080e80757e6a15472f29 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Fri, 26 Nov 2021 15:44:14 +0530 Subject: [PATCH 064/246] refactor: made DRY-er functions --- frappe/query_builder/functions.py | 38 ++++++++++--------------------- frappe/query_builder/utils.py | 10 ++++---- 2 files changed, 17 insertions(+), 31 deletions(-) diff --git a/frappe/query_builder/functions.py b/frappe/query_builder/functions.py index ddb10831c5..c98df775b7 100644 --- a/frappe/query_builder/functions.py +++ b/frappe/query_builder/functions.py @@ -26,38 +26,24 @@ Match = ImportMapper( ) -def max(dt, fieldname, filters=None, **kwargs): +def _aggregate(function, dt, fieldname, filters, **kwargs): return ( Query() .build_conditions(dt, filters) - .select(Max(Column(fieldname))) + .select(function(Column(fieldname))) .run(**kwargs)[0][0] or 0 ) -def min(dt, fieldname, filters=None, **kwargs): - return ( - Query() - .build_conditions(dt, filters) - .select(Min(Column(fieldname))) - .run(**kwargs)[0][0] - or 0 - ) -def avg(dt, fieldname, filters=None, **kwargs): - return ( - Query() - .build_conditions(dt, filters) - .select(Avg(Column(fieldname))) - .run(**kwargs)[0][0] - or 0 - ) +def _max(dt, fieldname, filters=None, **kwargs): + return _aggregate(Max, dt, fieldname, filters, **kwargs) -def sum(dt, fieldname, filters=None, **kwargs): - return ( - Query() - .build_conditions(dt, filters) - .select(Sum(Column(fieldname))) - .run(**kwargs)[0][0] - or 0 - ) \ No newline at end of file +def _min(dt, fieldname, filters=None, **kwargs): + return _aggregate(Min, dt, fieldname, filters, **kwargs) + +def _avg(dt, fieldname, filters=None, **kwargs): + return _aggregate(Avg, dt, fieldname, filters, **kwargs) + +def _sum(dt, fieldname, filters=None, **kwargs): + return _aggregate(Sum, dt, fieldname, filters, **kwargs) \ No newline at end of file diff --git a/frappe/query_builder/utils.py b/frappe/query_builder/utils.py index e2916a859c..08768aa11e 100644 --- a/frappe/query_builder/utils.py +++ b/frappe/query_builder/utils.py @@ -71,9 +71,9 @@ def patch_query_execute(): def patch_query_aggregation(): """Patch aggregation functions to frappe.qb """ - from frappe.query_builder.functions import max, min, avg, sum + from frappe.query_builder.functions import _max, _min, _avg, _sum - frappe.qb.max = max - frappe.qb.min = min - frappe.qb.avg = avg - frappe.qb.sum = sum \ No newline at end of file + frappe.qb.max = _max + frappe.qb.min = _min + frappe.qb.avg = _avg + frappe.qb.sum = _sum \ No newline at end of file From 7e35dc4913ee61425bf6ba4c4d09037d9f893033 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Fri, 26 Nov 2021 16:30:34 +0530 Subject: [PATCH 065/246] fix: add frappe.as_json for safe_exec scripts --- frappe/utils/safe_exec.py | 1 + 1 file changed, 1 insertion(+) diff --git a/frappe/utils/safe_exec.py b/frappe/utils/safe_exec.py index cc9662b4eb..00b7822104 100644 --- a/frappe/utils/safe_exec.py +++ b/frappe/utils/safe_exec.py @@ -85,6 +85,7 @@ def get_safe_globals(): loads=json.loads, dumps=json.dumps ), + as_json=frappe.as_json, dict=dict, log=frappe.log, _dict=frappe._dict, From 5a6d7ee191149291dc2b84391ca8fddeb9543388 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 26 Nov 2021 17:29:44 +0530 Subject: [PATCH 066/246] refactor: get_mapping_module doesn't need to access to instance --- .../data_migration_plan.py | 33 ++++++++++--------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.py b/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.py index 94ed77e2ec..d13912b431 100644 --- a/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.py +++ b/frappe/data_migration/doctype/data_migration_plan/data_migration_plan.py @@ -1,5 +1,4 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2017, Frappe Technologies and contributors +# Copyright (c) 2021, Frappe Technologies and contributors # License: MIT. See LICENSE import frappe @@ -8,6 +7,20 @@ from frappe.modules.export_file import export_to_files, create_init_py from frappe.custom.doctype.custom_field.custom_field import create_custom_field from frappe.model.document import Document + +def get_mapping_module(module, mapping_name): + app_name = frappe.db.get_value("Module Def", module, "app_name") + mapping_name = frappe.scrub(mapping_name) + module = frappe.scrub(module) + + try: + return frappe.get_module( + f"{app_name}.{module}.data_migration_mapping.{mapping_name}" + ) + except ImportError: + return None + + class DataMigrationPlan(Document): def on_update(self): # update custom fields in mappings @@ -54,26 +67,14 @@ class DataMigrationPlan(Document): frappe.flags.ignore_in_install = False def pre_process_doc(self, mapping_name, doc): - module = self.get_mapping_module(mapping_name) + module = get_mapping_module(self.module, mapping_name) if module and hasattr(module, 'pre_process'): return module.pre_process(doc) return doc def post_process_doc(self, mapping_name, local_doc=None, remote_doc=None): - module = self.get_mapping_module(mapping_name) + module = get_mapping_module(self.module, mapping_name) if module and hasattr(module, 'post_process'): return module.post_process(local_doc=local_doc, remote_doc=remote_doc) - - def get_mapping_module(self, mapping_name): - try: - module_def = frappe.get_doc("Module Def", self.module) - module = frappe.get_module('{app}.{module}.data_migration_mapping.{mapping_name}'.format( - app= module_def.app_name, - module=frappe.scrub(self.module), - mapping_name=frappe.scrub(mapping_name) - )) - return module - except ImportError: - return None From 171ff3ba4c8c4d3cb5f23dd1ed6bf8df2f845e6c Mon Sep 17 00:00:00 2001 From: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> Date: Fri, 26 Nov 2021 19:16:40 +0530 Subject: [PATCH 067/246] fix: Make strings translatable --- frappe/public/js/frappe/form/form_tour.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/public/js/frappe/form/form_tour.js b/frappe/public/js/frappe/form/form_tour.js index 84c7afdef4..7fefb59ac6 100644 --- a/frappe/public/js/frappe/form/form_tour.js +++ b/frappe/public/js/frappe/form/form_tour.js @@ -55,9 +55,9 @@ frappe.ui.form.FormTour = class FormTour { include_name_field() { const name_step = { - "description": `Enter a name for this ${this.frm.doctype}`, + "description": __("Enter a name for this {0}", [this.frm.doctype]), "fieldname": "__newname", - "title": "Name", + "title": __("Document Name"), "position": "right", "is_table_field": 0 }; From 6b9b28825987b8af04e070623631e7a3f61876c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20W=C4=99glowski-Hodur?= Date: Fri, 26 Nov 2021 14:50:37 +0100 Subject: [PATCH 068/246] fix: Correct the polish translations to correctly show buttons in Frappe framework. (#15093) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Michał Węglowski-Hodur --- frappe/translations/pl.csv | 100 ++++++++++++++++++------------------- 1 file changed, 50 insertions(+), 50 deletions(-) diff --git a/frappe/translations/pl.csv b/frappe/translations/pl.csv index 3fa85687f1..7e6b10385e 100644 --- a/frappe/translations/pl.csv +++ b/frappe/translations/pl.csv @@ -27,7 +27,7 @@ Assign To,Przypisano do, Attachment,Załącznik, Attachments,Załączniki, Author,Autor, -Auto Repeat,Auto Repeat, +Auto Repeat,Powtarzanie automatyczne, Base URL,Podstawowy adres URL, Based On,Bazujący na, Beginner,Początkujący, @@ -58,10 +58,10 @@ Custom?,Niestandardowy?, Date Format,Format daty, Datetime,Data-czas, Day,Dzień, -Default Letter Head,Domyślny nagłówek pisma, +Default Letter Head,Nagłówek domyślny, Defaults,Wartości domyślne, Delivery Status,Status dostawy, -Department,Departament, +Department,Dział, Details,Szczegóły, Document Name,Nazwa dokumentu, Document Status,Stan dokumentu, @@ -69,21 +69,21 @@ Document Type,Typ Dokumentu, Domain,Domena, Domains,Domeny, Draft,Wersja robocza, -Edit,Edytować, +Edit,Edycja, Email Account,Konto e-mail, Email Address,Adres e-mail, Email ID,ID e-mail, Email Sent,Wiadomość wysłana, Email Template,Szablon e-maila, -Enable,Włączyć, +Enable,Włącz, Enabled,Aktywny, -End Date,Data zakonczenia, -Error Code: {0},Kod błędu {0}, +End Date,Data zakończenia, +Error Code: {0},Kod błędu: {0}, Error Log,Dziennik błędów, Event,Wydarzenie, -Expand All,Rozwiń wszystkie, -Fail,Zawieść, -Failed,Nieudane, +Expand All,Rozwiń wszystko, +Fail,Nie powiodło się., +Failed,Nie powiodło się., Fax,Faks, Feedback,Informacja zwrotna, Female,Kobieta, @@ -114,25 +114,25 @@ Inactive,Nieaktywny, Insert,Wstaw, Interests,Zainteresowania, Introduction,Wprowadzenie, -Is Active,Jest aktywny, -Is Completed,Jest zakończony, -Is Default,Jest domyślny, +Is Active,Aktywny, +Is Completed,Zakończony, +Is Default,Domyślny, Kanban Board,Kanban Board, -Label,etykieta, -Language Name,Nazwa Język, +Label,Etykieta, +Language Name,Język, Last Name,Nazwisko, Leaderboard,Tabela liderów, Letter Head,Nagłówek, Level,Poziom, Limit,Limit, -Log,Log, +Log,Dziennik, Logs,Dzienniki, Low,Niski, -Maintenance Manager,Menager Konserwacji, +Maintenance Manager,Menedżer Konserwacji, Maintenance User,Użytkownik Konserwacji, Male,Mężczyzna, -Mandatory,Obowiązkowe, -Mapping,Mapowanie, +Mandatory,Wymagane, +Mapping,Odwzorowanie, Mapping Type,Typ odwzorowania, Medium,Średni, Meeting,Spotkanie, @@ -144,8 +144,8 @@ Monday,Poniedziałek, Monthly,Miesięcznie, More,Więcej, More Information,Więcej informacji, -More...,Jeszcze..., -Move,ruch, +More...,Więcej..., +Move,Przenieś, My Account,Moje Konto, New Address,Nowy adres, New Contact,Nowy kontakt, @@ -159,7 +159,7 @@ Not Permitted,Niedozwolone, Not active,Nieaktywny, Notes,Notatki, Number,Numer, -Online,online, +Online,Online, Operation,Operacja, Options,Opcje, Other,Inne, @@ -180,7 +180,7 @@ Please set Email Address,Proszę ustawić adres e-mail, Portal,Portal, Portal Settings,Ustawienia, Preview,Podgląd, -Primary,Podstawowy, +Primary,Główny, Print Format,Format Druku, Print Settings,Ustawienia drukowania, Print taxes with zero amount,Drukowanie podatków z zerową kwotą, @@ -196,17 +196,17 @@ Range,Przedział, Rating,Ocena, Received,Otrzymano, Recipients,Adresaci, -Redirect URL,przekierowanie, -Reference,Referencja, -Reference Date,Data Odniesienia, +Redirect URL,Adres przekierowania, +Reference,Odnośnik, +Reference Date,Data odnośnika, Reference Document,Dokument referencyjny, -Reference Document Type,Oznaczenie typu dokumentu, -Reference Owner,Odniesienie Właściciel, -Reference Type,Typ Odniesienia, -Refresh Token,Odśwież Reklamowe, +Reference Document Type,Typ dokumentu referencyjnego, +Reference Owner,Właściciel odnośnika, +Reference Type,Typ odnośnika, +Refresh Token,Odśwież token, Region,Region, Rejected,Odrzucono, -Reopen,Otworzyć na nowo, +Reopen,Otwórz ponownie, Replied,Odpowiedziane, Report,Raport, Report Builder,Kreator raportów, @@ -217,37 +217,37 @@ Role,Rola, Route,Trasa, Sales Manager,Menadżer Sprzedaży, Sales Master Manager,Główny Menadżer Sprzedaży, -Sales User,Sprzedaż użytkownika, +Sales User,Użytkownik Sprzedaży, Salutation,Forma grzecznościowa, -Sample,Próba, +Sample,Próbka, Saturday,Sobota, -Saved,Zapisane, +Saved,Zapisano, Scan Barcode,Skanuj kod kreskowy, Scheduled,Zaplanowane, Search,Szukaj, -Secret Key,Sekretny klucz, +Secret Key,Klucz tajny, Select,Wybierz, Select DocType,Wybierz DocType, Send Now,Wyślij teraz, Sent,Wysłano, -Series {0} already used in {1},Seria {0} już zostały użyte w {1}, +Series {0} already used in {1},Seria {0} już została użyte w {1}, Service,Usługa, Set as Default,Ustaw jako domyślne, Settings,Ustawienia, Shipping,Wysyłka, -Short Name,Skrócona nazwa, +Short Name,Nazwa skrócona, Slideshow,Pokaz slajdów, -Some information is missing,Niektóre informacje brakuje, +Some information is missing,Brakuje wymaganych informacji., Source,Źródło, -Source Name,Źródło Nazwa, +Source Name,Nazwa źródła, Standard,Standard, -Start Date,Data startu, +Start Date,Data rozpoczęcia, Start Import,Rozpocznij importowanie, State,Stan, Stopped,Zatrzymany, Subject,Temat, Submit,Zatwierdź, -Successful,Udany, +Successful,Zakończono pomyślnie., Summary,Podsumowanie, Sunday,Niedziela, System Manager,System Manager, @@ -256,15 +256,15 @@ Task,Zadanie, Tax Category,Kategoria podatku, Test,Test, Thank you,Dziękuję, -The page you are looking for is missing. This could be because it is moved or there is a typo in the link.,"Strona, której szukasz nie brakuje. To może być dlatego, że porusza się lub jest literówka w linku.", -Timespan,Okres czasu, +The page you are looking for is missing. This could be because it is moved or there is a typo in the link.,"Strona, której szukasz nie istnieje. Być może została przeniesiona, bądź odnośnik jest nieprawidłowy.", +Timespan,Okres, To,Do, To Date,Do daty, Tools,Narzędzia, Traceback,Traceback, URL,URL, Unsubscribed,Nie zarejestrowany, -Use Sandbox,Korzystanie Sandbox, +Use Sandbox,Użyj piaskownicy, User,Użytkownik, User ID,ID Użytkownika, Users,Użytkownicy, @@ -291,14 +291,14 @@ old_parent,old_parent, 'In List View' not allowed for type {0} in row {1},Pole 'W widoku listy' nie jest dozwolone dla typu {0} w lini {1}, 'Recipients' not specified,"Odbiorcy" nie podano, (Ctrl + G),(Ctrl + G), -** Failed: {0} to {1}: {2},** Nie udało: {0} {1}: {2}, +** Failed: {0} to {1}: {2},** Nie udało się: {0} {1}: {2}, **Currency** Master,** Waluta ** Główna, 0 - Draft; 1 - Submitted; 2 - Cancelled,0 - Projekt; 1 - Wysłane; 2 - Anulowane, -0 is highest,0 jest nawyższe, -1 Currency = [?] Fraction\nFor e.g. 1 USD = 100 Cent,1 jednostka Walutowa = [?] części zdawkowych. Na przykład 1 zł = 100 groszy, +0 is highest,0 jest najwyższą wartością, +1 Currency = [?] Fraction\nFor e.g. 1 USD = 100 Cent,1 jednostka Walutowa = [?] części dziesiętnych. Na przykład 1 zł = 100 groszy, 1 comment,1 komentarz, 1 hour ago,1 godzinę temu, -1 minute ago,1 minuta temu, +1 minute ago,1 minutę temu, 1 month ago,1 miesiąc temu, 1 year ago,1 rok temu, ; not allowed in condition,; Niedozwolony w stanie, @@ -4095,7 +4095,7 @@ Browser,Przeglądarka, Browser Version,Wersja przeglądarki, Web Template Field,Pole szablonu sieci Web, Section,Sekcja, -Hide,Ukryć, +Hide,Ukryj, Enable In App Website Tracking,Włącz śledzenie witryn w aplikacji, Enable Google Indexing,Włącz indeksowanie Google, "To use Google Indexing, enable Google Settings.","Aby korzystać z indeksowania Google, włącz Ustawienia Google .", @@ -4215,7 +4215,7 @@ since yesterday,od wczoraj, since last week,od zeszłego tygodnia, since last month,od ostatniego miesiąca, since last year,od zeszłego roku, -Show,Pokazać, +Show,Pokaż, New Number Card,Nowa karta z numerem, Your Shortcuts,Twoje skróty, You haven't added any Dashboard Charts or Number Cards yet.,Nie dodałeś jeszcze żadnych wykresów ani kart z numerami., From 58ca570f67e47fc3704a5932a14a773e6c87b832 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Fri, 26 Nov 2021 20:19:47 +0530 Subject: [PATCH 069/246] fix: Selection color in dark mode --- frappe/public/scss/desk/dark.scss | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/frappe/public/scss/desk/dark.scss b/frappe/public/scss/desk/dark.scss index 35cdffc91a..f894704ca2 100644 --- a/frappe/public/scss/desk/dark.scss +++ b/frappe/public/scss/desk/dark.scss @@ -161,4 +161,9 @@ --right-arrow-svg: url("data: image/svg+xml;utf8, "); --left-arrow-svg: url("data: image/svg+xml;utf8, "); + + ::selection { + color: var(--text-color); + background: var(--gray-500); + } } From ae31a0d6fbca7e947de5f212a26bbdb7e0fcfcbe Mon Sep 17 00:00:00 2001 From: hrwx Date: Sat, 27 Nov 2021 00:24:21 +0000 Subject: [PATCH 070/246] feat: add option to disable notification --- .../system_settings/system_settings.json | 17 +++++++++++++++-- frappe/utils/change_log.py | 19 ++++++++++--------- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/frappe/core/doctype/system_settings/system_settings.json b/frappe/core/doctype/system_settings/system_settings.json index 82e88d2477..d4b7272dd3 100644 --- a/frappe/core/doctype/system_settings/system_settings.json +++ b/frappe/core/doctype/system_settings/system_settings.json @@ -66,7 +66,9 @@ "attach_view_link", "prepared_report_section", "enable_prepared_report_auto_deletion", - "prepared_report_expiry_period" + "prepared_report_expiry_period", + "system_updates_section", + "disable_system_update_notification" ], "fields": [ { @@ -462,12 +464,23 @@ "fieldname": "encrypt_backup", "fieldtype": "Check", "label": "Encrypt Backups" + }, + { + "fieldname": "system_updates_section", + "fieldtype": "Section Break", + "label": "System Updates" + }, + { + "default": "0", + "fieldname": "disable_system_update_notification", + "fieldtype": "Check", + "label": "Disable System Update Notification" } ], "icon": "fa fa-cog", "issingle": 1, "links": [], - "modified": "2021-10-21 19:24:15.232430", + "modified": "2021-11-27 01:17:05.228959", "modified_by": "Administrator", "module": "Core", "name": "System Settings", diff --git a/frappe/utils/change_log.py b/frappe/utils/change_log.py index 109778b87b..b4f54e1b20 100644 --- a/frappe/utils/change_log.py +++ b/frappe/utils/change_log.py @@ -155,11 +155,11 @@ def check_for_update(): for update_type in updates: if github_version.__dict__[update_type] > instance_version.__dict__[update_type]: updates[update_type].append(frappe._dict( - current_version = str(instance_version), + current_version = str(instance_version), available_version = str(github_version), - org_name = org_name, - app_name = app, - title = apps[app]['title'], + org_name = org_name, + app_name = app, + title = apps[app]['title'], )) break if github_version.__dict__[update_type] < instance_version.__dict__[update_type]: break @@ -242,10 +242,11 @@ def add_message_to_redis(update_json): @frappe.whitelist() def show_update_popup(): cache = frappe.cache() - user = frappe.session.user + user = frappe.session.user + system_settings = frappe.get_single("System Settings") update_info = cache.get_value("update-info") - if not update_info: + if not update_info or system_settings.disable_system_update_notification: return updates = json.loads(update_info) @@ -259,9 +260,9 @@ def show_update_popup(): app = frappe._dict(app) release_links += "{title}: v{available_version}
".format( available_version = app.available_version, - org_name = app.org_name, - app_name = app.app_name, - title = app.title + org_name = app.org_name, + app_name = app.app_name, + title = app.title ) if release_links: message = _("New {} releases for the following apps are available").format(_(update_type)) From 5339008fef88cf691a28b0b1b3e23f721ff635d7 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Sat, 27 Nov 2021 14:35:58 +0530 Subject: [PATCH 071/246] feat: Added patch for replacing db level aggregation calls --- frappe/patches/v14_0/remove_db_aggregation.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 frappe/patches/v14_0/remove_db_aggregation.py diff --git a/frappe/patches/v14_0/remove_db_aggregation.py b/frappe/patches/v14_0/remove_db_aggregation.py new file mode 100644 index 0000000000..373cfaf7db --- /dev/null +++ b/frappe/patches/v14_0/remove_db_aggregation.py @@ -0,0 +1,25 @@ +import frappe +import re + + +def execute(): + _sub_aggregation("frappe.db.max", "frappe.qb.max") + _sub_aggregation("frappe.db.min", "frappe.qb.min") + _sub_aggregation("frappe.db.sum", "frappe.qb.sum") + _sub_aggregation("frappe.db.avg", "frappe.qb.avg") + + +def _sub_aggregation(function, subtitution): + scripts = frappe.get_all( + "Server Script", + filters={"script": ("like", f"%{function}%")}, + fields=["name", "script"], + ) + for script in scripts: + script.update( + {"script": re.sub(f"{function}", f"{subtitution}", script["script"])} + ) + for script in scripts: + frappe.db.update( + "Server Script", {"name": script["name"]}, "script", script["script"] + ) From b1d0e574a0947fdb4133763f63f51677dc9c43ec Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sat, 27 Nov 2021 17:17:05 +0530 Subject: [PATCH 072/246] feat(REST): OR filters in REST API --- frappe/client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frappe/client.py b/frappe/client.py index a3ed0fa37d..6641e471af 100644 --- a/frappe/client.py +++ b/frappe/client.py @@ -18,7 +18,7 @@ Requests via FrappeClient are also handled here. @frappe.whitelist() def get_list(doctype, fields=None, filters=None, order_by=None, - limit_start=None, limit_page_length=20, parent=None, debug=False, as_dict=True): + limit_start=None, limit_page_length=20, parent=None, debug=False, as_dict=True, or_filters=None): '''Returns a list of records by filters, fields, ordering and limit :param doctype: DocType of the data to be queried @@ -34,6 +34,7 @@ def get_list(doctype, fields=None, filters=None, order_by=None, doctype=doctype, fields=fields, filters=filters, + or_filters=or_filters, order_by=order_by, limit_start=limit_start, limit_page_length=limit_page_length, From 88c3d92662d4cf05eb1fe4aa1815fdb0f90d53a9 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sat, 27 Nov 2021 19:23:37 +0530 Subject: [PATCH 073/246] fix: return self after submit/cancel `Document.save` returns self but `submit` and `cancel` don't. change: For sake of consistency and better support for `run_method` via REST API, return the document. ref: https://github.com/frappe/frappe/issues/14869 --- frappe/model/document.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/frappe/model/document.py b/frappe/model/document.py index 411d447d0f..6c85090ea9 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -915,7 +915,7 @@ class Document(BaseDocument): def _submit(self): """Submit the document. Sets `docstatus` = 1, then saves.""" self.docstatus = 1 - self.save() + return self.save() @whitelist.__func__ def _cancel(self): @@ -925,17 +925,17 @@ class Document(BaseDocument): new_name = gen_new_name_for_cancelled_doc(self) frappe.rename_doc(self.doctype, self.name, new_name, force=True, show_alert=False) self.name = new_name - self.save() + return self.save() @whitelist.__func__ def submit(self): """Submit the document. Sets `docstatus` = 1, then saves.""" - self._submit() + return self._submit() @whitelist.__func__ def cancel(self): """Cancel the document. Sets `docstatus` = 2, then saves.""" - self._cancel() + return self._cancel() def delete(self, ignore_permissions=False): """Delete document.""" From 615db9b55eaa4ffde45e513594d1b8e9fd7940b3 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Sun, 28 Nov 2021 09:44:05 +0530 Subject: [PATCH 074/246] style: Fix sider alerts --- frappe/desk/form/linked_with.py | 3 +-- frappe/tests/test_linked_with.py | 4 +++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/frappe/desk/form/linked_with.py b/frappe/desk/form/linked_with.py index f44a350263..cbf459e8ae 100644 --- a/frappe/desk/form/linked_with.py +++ b/frappe/desk/form/linked_with.py @@ -2,7 +2,6 @@ # License: MIT. See LICENSE import json from collections import defaultdict -from os import link import itertools from typing import List @@ -263,7 +262,7 @@ def get_references_across_doctypes_by_dynamic_link_field(to_doctypes: List[str]= for linked_to in frappe.db.get_all(doctype, pluck=doctype_fieldname, filters = filters, distinct=1): if linked_to: links_by_doctype[linked_to].append({'doctype': doctype, 'fieldname': fieldname, 'doctype_fieldname': doctype_fieldname}) - except frappe.db.ProgrammingError as e: + except frappe.db.ProgrammingError: # TODO: FIXME continue return links_by_doctype diff --git a/frappe/tests/test_linked_with.py b/frappe/tests/test_linked_with.py index 64da8e51e0..ec461c7d5f 100644 --- a/frappe/tests/test_linked_with.py +++ b/frappe/tests/test_linked_with.py @@ -1,4 +1,6 @@ -import frappe, unittest +import unittest + +import frappe from frappe.core.doctype.doctype.test_doctype import new_doctype from frappe.desk.form import linked_with From 491ca2b7c7d7dd710dde3fbc0e98724d37ca9230 Mon Sep 17 00:00:00 2001 From: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> Date: Sun, 28 Nov 2021 09:57:55 +0530 Subject: [PATCH 075/246] chore: Fix CODEOWNERS formatting --- CODEOWNERS | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/CODEOWNERS b/CODEOWNERS index 69ca578b6c..f7d759c123 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -3,18 +3,18 @@ # These owners will be the default owners for everything in # the repo. Unless a later match takes precedence, -* @frappe/frappe-review-team -templates/ @surajshetty3416 -www/ @surajshetty3416 -integrations/ @leela -patches/ @surajshetty3416 @gavindsouza -email/ @leela -event_streaming/ @ruchamahabal -data_import* @netchampfaris -core/ @surajshetty3416 +* @frappe/frappe-review-team +templates/ @surajshetty3416 +www/ @surajshetty3416 +integrations/ @leela +patches/ @surajshetty3416 @gavindsouza +email/ @leela +event_streaming/ @ruchamahabal +data_import* @netchampfaris +core/ @surajshetty3416 database @gavindsouza model @gavindsouza -requirements.txt @gavindsouza -query_builder/ @gavindsouza -commands/ @gavindsouza +requirements.txt @gavindsouza +query_builder/ @gavindsouza +commands/ @gavindsouza workspace @shariquerik From 4cae147aed134c358ba1b1ab9eb854aa17d45206 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Mon, 29 Nov 2021 09:23:52 +0530 Subject: [PATCH 076/246] fix: remove duplicate parent when child item option selected (backport #15101) (#15110) Co-authored-by: Bhavesh Maheshwari --- frappe/public/js/frappe/form/multi_select_dialog.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/form/multi_select_dialog.js b/frappe/public/js/frappe/form/multi_select_dialog.js index 37b7e08a80..161e4196b0 100644 --- a/frappe/public/js/frappe/form/multi_select_dialog.js +++ b/frappe/public/js/frappe/form/multi_select_dialog.js @@ -325,7 +325,9 @@ frappe.ui.form.MultiSelectDialog = class MultiSelectDialog { let parent_names = this.child_datatable.rowmanager.checkMap.reduce((parent_names, checked, index) => { if (checked == 1) { const parent_name = this.child_results[index].parent; - parent_names.push(parent_name); + if (!parent_names.includes(parent_name)) { + parent_names.push(parent_name); + } } return parent_names; }, []); From 2efed9d12e9c5785e7c45c2bf49035c171cea8d4 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 1 Sep 2021 15:24:16 +0530 Subject: [PATCH 077/246] refactor: convert doctype user db.sql calls --- frappe/core/doctype/user/user.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py index b127cf5f0c..c00541ab5a 100644 --- a/frappe/core/doctype/user/user.py +++ b/frappe/core/doctype/user/user.py @@ -698,13 +698,11 @@ def has_email_account(email): @frappe.whitelist(allow_guest=False) def get_email_awaiting(user): - waiting = frappe.db.sql("""select email_account,email_id - from `tabUser Email` - where awaiting_password = 1 - and parent = %(user)s""", {"user":user}, as_dict=1) + waiting = frappe.get_all("User Email", fields=["email_account", "email_id"], filters={"awaiting_password": 1, "parent": user}) if waiting: return waiting else: + # TODO frappe.db.sql("""update `tabUser Email` set awaiting_password =0 where parent = %(user)s""",{"user":user}) From ad1842e69d9ae9ba39c0cd5a224286f1a0267a9f Mon Sep 17 00:00:00 2001 From: abhishek Date: Thu, 7 Oct 2021 13:24:21 +0530 Subject: [PATCH 078/246] refactor: convert doctype doctype db.sql call change_modified_of_parent() --- frappe/core/doctype/doctype/doctype.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/core/doctype/doctype/doctype.py b/frappe/core/doctype/doctype/doctype.py index 738fb73a34..1c8c1f9217 100644 --- a/frappe/core/doctype/doctype/doctype.py +++ b/frappe/core/doctype/doctype/doctype.py @@ -253,7 +253,7 @@ class DocType(Document): parent_list = frappe.db.get_all('DocField', 'parent', dict(fieldtype=['in', frappe.model.table_fields], options=self.name)) for p in parent_list: - frappe.db.sql('UPDATE `tabDocType` SET modified=%s WHERE `name`=%s', (now(), p.parent)) + frappe.db.update("DocType", p.parent, {}, for_update=False) def scrub_field_names(self): """Sluggify fieldnames if not set from Label.""" From 7e84529dacc25eeafe87fc2b076639653c220e0b Mon Sep 17 00:00:00 2001 From: abhishek Date: Thu, 7 Oct 2021 13:42:39 +0530 Subject: [PATCH 079/246] refactor: convert doctype file db.sql calls update_existing_file_docs() --- frappe/core/doctype/file/file.py | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/frappe/core/doctype/file/file.py b/frappe/core/doctype/file/file.py index 0021240106..af9c8a48fa 100755 --- a/frappe/core/doctype/file/file.py +++ b/frappe/core/doctype/file/file.py @@ -940,20 +940,14 @@ def get_files_by_search_text(text): def update_existing_file_docs(doc): # Update is private and file url of all file docs that point to the same file - frappe.db.sql(""" - UPDATE `tabFile` - SET - file_url = %(file_url)s, - is_private = %(is_private)s - WHERE - content_hash = %(content_hash)s - and name != %(file_name)s - """, dict( - file_url=doc.file_url, - is_private=doc.is_private, - content_hash=doc.content_hash, - file_name=doc.name - )) + file_doctype = frappe.qb.DocType("File") + ( + frappe.qb.update(file_doctype) + .set(file_doctype.file_url, doc.file_url) + .set(file_doctype.is_private, doc.is_private) + .where(file_doctype.content_hash == doc.content_hash) + .where(file_doctype.name != doc.name) + ).run() def attach_files_to_document(doc, event): """ Runs on on_update hook of all documents. From f7dcd781fb4632c3f757b1a1ca031addd1c4f922 Mon Sep 17 00:00:00 2001 From: abhishek Date: Thu, 7 Oct 2021 13:57:10 +0530 Subject: [PATCH 080/246] refactor: convert doctype scheduled job type db.sql calls class TestScheduledJobType --- .../core/doctype/scheduled_job_type/test_scheduled_job_type.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/core/doctype/scheduled_job_type/test_scheduled_job_type.py b/frappe/core/doctype/scheduled_job_type/test_scheduled_job_type.py index dc3353b176..a11966c47e 100644 --- a/frappe/core/doctype/scheduled_job_type/test_scheduled_job_type.py +++ b/frappe/core/doctype/scheduled_job_type/test_scheduled_job_type.py @@ -10,7 +10,7 @@ from frappe.core.doctype.scheduled_job_type.scheduled_job_type import sync_jobs class TestScheduledJobType(unittest.TestCase): def setUp(self): frappe.db.rollback() - frappe.db.sql('truncate `tabScheduled Job Type`') + frappe.db.truncate("Scheduled Job Type") sync_jobs() frappe.db.commit() From 5409be0a3440f1f2447ed249be79721975f0803f Mon Sep 17 00:00:00 2001 From: abhishek Date: Thu, 7 Oct 2021 13:58:56 +0530 Subject: [PATCH 081/246] refactor: convert doctype server_script db.sql setUpClass() tearDownClass() --- frappe/core/doctype/server_script/test_server_script.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/core/doctype/server_script/test_server_script.py b/frappe/core/doctype/server_script/test_server_script.py index 3c091fec0b..bc92061f42 100644 --- a/frappe/core/doctype/server_script/test_server_script.py +++ b/frappe/core/doctype/server_script/test_server_script.py @@ -76,7 +76,7 @@ class TestServerScript(unittest.TestCase): @classmethod def setUpClass(cls): frappe.db.commit() - frappe.db.sql('truncate `tabServer Script`') + frappe.db.truncate("Server Script") frappe.get_doc('User', 'Administrator').add_roles('Script Manager') for script in scripts: script_doc = frappe.get_doc(doctype ='Server Script') @@ -88,7 +88,7 @@ class TestServerScript(unittest.TestCase): @classmethod def tearDownClass(cls): frappe.db.commit() - frappe.db.sql('truncate `tabServer Script`') + frappe.db.truncate("Server Script") frappe.cache().delete_value('server_script_map') def setUp(self): From 1bb13fcee3d85d420fe17599f9a7f5202cebff72 Mon Sep 17 00:00:00 2001 From: abhishek Date: Thu, 7 Oct 2021 15:24:50 +0530 Subject: [PATCH 082/246] refactor: convert doctype transaction log db.sql get_current_index() --- .../core/doctype/transaction_log/transaction_log.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/frappe/core/doctype/transaction_log/transaction_log.py b/frappe/core/doctype/transaction_log/transaction_log.py index 6dc4340277..0a480f6660 100644 --- a/frappe/core/doctype/transaction_log/transaction_log.py +++ b/frappe/core/doctype/transaction_log/transaction_log.py @@ -9,6 +9,7 @@ from frappe.model.document import Document from frappe.query_builder import DocType from frappe.utils import cint, now_datetime + class TransactionLog(Document): def before_insert(self): index = get_current_index() @@ -29,18 +30,15 @@ class TransactionLog(Document): def hash_line(self): sha = hashlib.sha256() sha.update( - frappe.safe_encode(str(self.row_index)) + \ - frappe.safe_encode(str(self.timestamp)) + \ - frappe.safe_encode(str(self.data)) + frappe.safe_encode(str(self.row_index)) + + frappe.safe_encode(str(self.timestamp)) + + frappe.safe_encode(str(self.data)) ) return sha.hexdigest() def hash_chain(self): sha = hashlib.sha256() - sha.update( - frappe.safe_encode(str(self.transaction_hash)) + \ - frappe.safe_encode(str(self.previous_hash)) - ) + sha.update(frappe.safe_encode(str(self.transaction_hash)) + frappe.safe_encode(str(self.previous_hash))) return sha.hexdigest() From d74f985728c9f44a76c21b011b7ac42e04662247 Mon Sep 17 00:00:00 2001 From: abhishek Date: Thu, 7 Oct 2021 16:35:08 +0530 Subject: [PATCH 083/246] refactor: convert doctype user db.sql calls has_desk_access() --- frappe/core/doctype/user/user.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py index c00541ab5a..9694c11b87 100644 --- a/frappe/core/doctype/user/user.py +++ b/frappe/core/doctype/user/user.py @@ -213,15 +213,19 @@ class User(Document): user_type_doc.update_modules_in_user(self) def has_desk_access(self): - '''Return true if any of the set roles has desk access''' + """Return true if any of the set roles has desk access""" if not self.roles: return False - return len(frappe.db.sql("""select name - from `tabRole` where desk_access=1 - and name in ({0}) limit 1""".format(', '.join(['%s'] * len(self.roles))), - [d.role for d in self.roles])) - + role_table = frappe.qb.DocType("Role") + return len( + frappe.qb.from_(role_table) + .select(role_table.name) + .where(role_table.desk_access == 1) + .where(role_table.name.isin([d.role for d in self.roles])) + .limit(1) + .run() + ) def share_with_self(self): frappe.share.add(self.doctype, self.name, self.name, write=1, share=1, From e503d8117303eb787c0099f02d91b89eb23b6ce4 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Mon, 29 Nov 2021 13:07:31 +0530 Subject: [PATCH 084/246] perf: reduced no. of db calls --- frappe/patches/v14_0/remove_db_aggregation.py | 39 ++++++++++++------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/frappe/patches/v14_0/remove_db_aggregation.py b/frappe/patches/v14_0/remove_db_aggregation.py index 373cfaf7db..231acccc6c 100644 --- a/frappe/patches/v14_0/remove_db_aggregation.py +++ b/frappe/patches/v14_0/remove_db_aggregation.py @@ -1,24 +1,37 @@ +from frappe.query_builder import DocType import frappe import re def execute(): - _sub_aggregation("frappe.db.max", "frappe.qb.max") - _sub_aggregation("frappe.db.min", "frappe.qb.min") - _sub_aggregation("frappe.db.sum", "frappe.qb.sum") - _sub_aggregation("frappe.db.avg", "frappe.qb.avg") + sub_aggregation() -def _sub_aggregation(function, subtitution): - scripts = frappe.get_all( - "Server Script", - filters={"script": ("like", f"%{function}%")}, - fields=["name", "script"], - ) - for script in scripts: - script.update( - {"script": re.sub(f"{function}", f"{subtitution}", script["script"])} +def sub_aggregation(): + server_scripts = DocType("Server Script") + scripts = ( + frappe.qb.from_(server_scripts) + .where( + server_scripts.script.like("%frappe.db.max%") + | server_scripts.script.like("%frappe.db.min%") + | server_scripts.script.like("%frappe.db.sum%") + | server_scripts.script.like("%frappe.db.avg%") ) + .select("name", "script") + .run(as_dict=True) + ) + + def _sub_aggregation(scripts, function, substitution): + for script in scripts: + script.update( + {"script": re.sub(f"{function}", f"{substitution}", script["script"])} + ) + + _sub_aggregation(scripts, "frappe.db.max", "frappe.qb.max") + _sub_aggregation(scripts, "frappe.db.min", "frappe.qb.min") + _sub_aggregation(scripts, "frappe.db.sum", "frappe.qb.sum") + _sub_aggregation(scripts, "frappe.db.avg", "frappe.qb.avg") + for script in scripts: frappe.db.update( "Server Script", {"name": script["name"]}, "script", script["script"] From 43fc713dd7ef881f8f76632b266965d94c98de21 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 29 Nov 2021 13:46:47 +0530 Subject: [PATCH 085/246] refactor: Remove Aggregation methods from DB API * Make it DRY & make it "better" * Add to patches.txt so the patch runs :') * Style fixes - tabs > spaces for consistency, removed unnecessary blocks --- frappe/patches.txt | 1 + frappe/patches/v14_0/remove_db_aggregation.py | 54 +++++++++---------- 2 files changed, 25 insertions(+), 30 deletions(-) diff --git a/frappe/patches.txt b/frappe/patches.txt index b230c336b4..3078159c3d 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -186,3 +186,4 @@ frappe.patches.v14_0.rename_cancelled_documents frappe.patches.v14_0.copy_mail_data #08.03.21 frappe.patches.v14_0.update_workspace2 # 20.09.2021 frappe.patches.v14_0.update_github_endpoints #08-11-2021 +frappe.patches.v14_0.remove_db_aggregation diff --git a/frappe/patches/v14_0/remove_db_aggregation.py b/frappe/patches/v14_0/remove_db_aggregation.py index 231acccc6c..25a170f362 100644 --- a/frappe/patches/v14_0/remove_db_aggregation.py +++ b/frappe/patches/v14_0/remove_db_aggregation.py @@ -1,38 +1,32 @@ -from frappe.query_builder import DocType -import frappe import re +import frappe +from frappe.query_builder import DocType + def execute(): - sub_aggregation() + """Replace temporarily available Database Aggregate APIs on frappe (develop) + APIs changed: + * frappe.db.max => frappe.qb.max + * frappe.db.min => frappe.qb.min + * frappe.db.sum => frappe.qb.sum + * frappe.db.avg => frappe.qb.avg + """ + ServerScript = DocType("Server Script") + server_scripts = frappe.qb.from_(ServerScript).where( + ServerScript.script.like("%frappe.db.max(%") + | ServerScript.script.like("%frappe.db.min(%") + | ServerScript.script.like("%frappe.db.sum(%") + | ServerScript.script.like("%frappe.db.avg(%") + ).select( + "name", "script" + ).run(as_dict=True) -def sub_aggregation(): - server_scripts = DocType("Server Script") - scripts = ( - frappe.qb.from_(server_scripts) - .where( - server_scripts.script.like("%frappe.db.max%") - | server_scripts.script.like("%frappe.db.min%") - | server_scripts.script.like("%frappe.db.sum%") - | server_scripts.script.like("%frappe.db.avg%") - ) - .select("name", "script") - .run(as_dict=True) - ) + for server_script in server_scripts: + name, script = server_script["name"], server_script["script"] - def _sub_aggregation(scripts, function, substitution): - for script in scripts: - script.update( - {"script": re.sub(f"{function}", f"{substitution}", script["script"])} - ) + for agg in ["avg", "max", "min", "sum"]: + script = re.sub(f"frappe.db.{agg}(", f"frappe.qb.{agg}(", script) - _sub_aggregation(scripts, "frappe.db.max", "frappe.qb.max") - _sub_aggregation(scripts, "frappe.db.min", "frappe.qb.min") - _sub_aggregation(scripts, "frappe.db.sum", "frappe.qb.sum") - _sub_aggregation(scripts, "frappe.db.avg", "frappe.qb.avg") - - for script in scripts: - frappe.db.update( - "Server Script", {"name": script["name"]}, "script", script["script"] - ) + frappe.db.update("Server Script", name, "script", script) From 0e32d52e3a0a75500ac59f8fe5c875b37138009f Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Mon, 29 Nov 2021 16:38:15 +0530 Subject: [PATCH 086/246] fix: Use separate API to insert route history --- frappe/deferred_insert.py | 1 - .../doctype/route_history/route_history.py | 20 +++++++++++++++++++ frappe/public/js/frappe/router_history.js | 12 +++++------ 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/frappe/deferred_insert.py b/frappe/deferred_insert.py index 499fc5e41b..b1338a73b0 100644 --- a/frappe/deferred_insert.py +++ b/frappe/deferred_insert.py @@ -5,7 +5,6 @@ from frappe.utils import cstr queue_prefix = 'insert_queue_for_' -@frappe.whitelist() def deferred_insert(doctype, records): frappe.cache().rpush(queue_prefix + doctype, records) diff --git a/frappe/desk/doctype/route_history/route_history.py b/frappe/desk/doctype/route_history/route_history.py index 01184fcc3a..489a3bd50a 100644 --- a/frappe/desk/doctype/route_history/route_history.py +++ b/frappe/desk/doctype/route_history/route_history.py @@ -1,9 +1,13 @@ # Copyright (c) 2021, Frappe Technologies and contributors # License: MIT. See LICENSE +import json + import frappe +from frappe.deferred_insert import deferred_insert from frappe.model.document import Document + class RouteHistory(Document): pass @@ -35,3 +39,19 @@ def flush_old_route_records(): "modified": ("<=", last_record_to_keep[0].modified), "user": user }) + +@frappe.whitelist() +def deferred_insert_route_history(routes): + routes_record = [] + + if isinstance(routes, str): + routes = json.loads(routes) + + for route_doc in routes: + routes_record.append({ + "user": frappe.session.user, + "route": route_doc.get("route"), + "creation": route_doc.get("creation") + }) + + deferred_insert("Route History", json.dumps(routes_record)) diff --git a/frappe/public/js/frappe/router_history.js b/frappe/public/js/frappe/router_history.js index c64c3fc9f2..1fda6f4a1b 100644 --- a/frappe/public/js/frappe/router_history.js +++ b/frappe/public/js/frappe/router_history.js @@ -5,13 +5,14 @@ const save_routes = frappe.utils.debounce(() => { if (frappe.session.user === 'Guest') return; const routes = frappe.route_history_queue; frappe.route_history_queue = []; - - frappe.xcall('frappe.deferred_insert.deferred_insert', { - 'doctype': 'Route History', - 'records': routes + + if (!routes.length) return; + + frappe.xcall('frappe.desk.doctype.route_history.route_history.deferred_insert_route_history', { + 'routes': routes }).catch(() => { frappe.route_history_queue.concat(routes); - }); + }); }, 10000); @@ -19,7 +20,6 @@ frappe.router.on('change', () => { const route = frappe.get_route(); if (is_route_useful(route)) { frappe.route_history_queue.push({ - 'user': frappe.session.user, 'creation': frappe.datetime.now_datetime(), 'route': frappe.get_route_str() }); From aa40abadb1277636e1f79d1ff787a7d615e218d5 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Mon, 29 Nov 2021 17:06:29 +0530 Subject: [PATCH 087/246] refactor: Simplify method naming --- frappe/desk/doctype/route_history/route_history.py | 6 +++--- frappe/public/js/frappe/router_history.js | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/frappe/desk/doctype/route_history/route_history.py b/frappe/desk/doctype/route_history/route_history.py index 489a3bd50a..fc87312950 100644 --- a/frappe/desk/doctype/route_history/route_history.py +++ b/frappe/desk/doctype/route_history/route_history.py @@ -4,7 +4,7 @@ import json import frappe -from frappe.deferred_insert import deferred_insert +from frappe.deferred_insert import deferred_insert as _deferred_insert from frappe.model.document import Document @@ -41,7 +41,7 @@ def flush_old_route_records(): }) @frappe.whitelist() -def deferred_insert_route_history(routes): +def deferred_insert(routes): routes_record = [] if isinstance(routes, str): @@ -54,4 +54,4 @@ def deferred_insert_route_history(routes): "creation": route_doc.get("creation") }) - deferred_insert("Route History", json.dumps(routes_record)) + _deferred_insert("Route History", json.dumps(routes_record)) diff --git a/frappe/public/js/frappe/router_history.js b/frappe/public/js/frappe/router_history.js index 1fda6f4a1b..fb2d5790da 100644 --- a/frappe/public/js/frappe/router_history.js +++ b/frappe/public/js/frappe/router_history.js @@ -8,7 +8,7 @@ const save_routes = frappe.utils.debounce(() => { if (!routes.length) return; - frappe.xcall('frappe.desk.doctype.route_history.route_history.deferred_insert_route_history', { + frappe.xcall('frappe.desk.doctype.route_history.route_history.deferred_insert', { 'routes': routes }).catch(() => { frappe.route_history_queue.concat(routes); From 2a08f35836efa010c3a9c100ec62f9a497cae9ca Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sat, 27 Nov 2021 19:10:45 +0530 Subject: [PATCH 088/246] fix: allow cancelling by PUT docstatus=2 --- frappe/model/document.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/frappe/model/document.py b/frappe/model/document.py index 6c85090ea9..fcdadf48e6 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -306,6 +306,9 @@ class Document(BaseDocument): self.check_permission("write", "save") + if self.docstatus == 2: + self._rename_doc_on_cancel() + self.set_user_and_timestamp() self.set_docstatus() self.check_if_latest() @@ -922,9 +925,6 @@ class Document(BaseDocument): """Cancel the document. Sets `docstatus` = 2, then saves. """ self.docstatus = 2 - new_name = gen_new_name_for_cancelled_doc(self) - frappe.rename_doc(self.doctype, self.name, new_name, force=True, show_alert=False) - self.name = new_name return self.save() @whitelist.__func__ @@ -1355,6 +1355,11 @@ class Document(BaseDocument): from frappe.desk.doctype.tag.tag import DocTags return DocTags(self.doctype).get_tags(self.name).split(",")[1:] + def _rename_doc_on_cancel(self): + new_name = gen_new_name_for_cancelled_doc(self) + frappe.rename_doc(self.doctype, self.name, new_name, force=True, show_alert=False) + self.name = new_name + def __repr__(self): name = self.name or "unsaved" doctype = self.__class__.__name__ From e751b07bde74046aeba9520ffd843f66e28336dc Mon Sep 17 00:00:00 2001 From: hrwx Date: Mon, 29 Nov 2021 16:49:46 +0000 Subject: [PATCH 089/246] chore: make system timezone readonly --- frappe/core/doctype/system_settings/system_settings.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frappe/core/doctype/system_settings/system_settings.json b/frappe/core/doctype/system_settings/system_settings.json index 82e88d2477..2a06f58845 100644 --- a/frappe/core/doctype/system_settings/system_settings.json +++ b/frappe/core/doctype/system_settings/system_settings.json @@ -95,6 +95,7 @@ "fieldname": "time_zone", "fieldtype": "Select", "label": "Time Zone", + "read_only": 1, "reqd": 1 }, { @@ -467,7 +468,7 @@ "icon": "fa fa-cog", "issingle": 1, "links": [], - "modified": "2021-10-21 19:24:15.232430", + "modified": "2021-11-29 17:49:20.950033", "modified_by": "Administrator", "module": "Core", "name": "System Settings", From db4476fc5fb8e0f99b5a9126c1f63def831db522 Mon Sep 17 00:00:00 2001 From: hrwx Date: Mon, 29 Nov 2021 17:28:40 +0000 Subject: [PATCH 090/246] fix: reduce api call --- .../system_settings/system_settings.json | 3 ++- frappe/public/js/frappe/desk.js | 2 ++ frappe/utils/change_log.py | 21 +++++++++---------- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/frappe/core/doctype/system_settings/system_settings.json b/frappe/core/doctype/system_settings/system_settings.json index d4b7272dd3..dcec9b13c2 100644 --- a/frappe/core/doctype/system_settings/system_settings.json +++ b/frappe/core/doctype/system_settings/system_settings.json @@ -466,6 +466,7 @@ "label": "Encrypt Backups" }, { + "collapsible": 1, "fieldname": "system_updates_section", "fieldtype": "Section Break", "label": "System Updates" @@ -480,7 +481,7 @@ "icon": "fa fa-cog", "issingle": 1, "links": [], - "modified": "2021-11-27 01:17:05.228959", + "modified": "2021-11-29 18:09:53.601629", "modified_by": "Administrator", "module": "Core", "name": "System Settings", diff --git a/frappe/public/js/frappe/desk.js b/frappe/public/js/frappe/desk.js index 2855c6ae7c..4563875b91 100644 --- a/frappe/public/js/frappe/desk.js +++ b/frappe/public/js/frappe/desk.js @@ -531,6 +531,8 @@ frappe.Application = class Application { } show_update_available() { + if (frappe.boot.sysdefaults.disable_system_update_notification) return; + frappe.call({ "method": "frappe.utils.change_log.show_update_popup" }); diff --git a/frappe/utils/change_log.py b/frappe/utils/change_log.py index b4f54e1b20..5888166d5d 100644 --- a/frappe/utils/change_log.py +++ b/frappe/utils/change_log.py @@ -155,11 +155,11 @@ def check_for_update(): for update_type in updates: if github_version.__dict__[update_type] > instance_version.__dict__[update_type]: updates[update_type].append(frappe._dict( - current_version = str(instance_version), - available_version = str(github_version), - org_name = org_name, - app_name = app, - title = apps[app]['title'], + current_version=str(instance_version), + available_version=str(github_version), + org_name=org_name, + app_name=app, + title=apps[app]['title'], )) break if github_version.__dict__[update_type] < instance_version.__dict__[update_type]: break @@ -243,10 +243,9 @@ def add_message_to_redis(update_json): def show_update_popup(): cache = frappe.cache() user = frappe.session.user - system_settings = frappe.get_single("System Settings") update_info = cache.get_value("update-info") - if not update_info or system_settings.disable_system_update_notification: + if not update_info: return updates = json.loads(update_info) @@ -259,10 +258,10 @@ def show_update_popup(): for app in updates[update_type]: app = frappe._dict(app) release_links += "{title}: v{available_version}
".format( - available_version = app.available_version, - org_name = app.org_name, - app_name = app.app_name, - title = app.title + available_version=app.available_version, + org_name=app.org_name, + app_name=app.app_name, + title=app.title ) if release_links: message = _("New {} releases for the following apps are available").format(_(update_type)) From 7047cb83011f6472cd5686651aa177b2cc8990c1 Mon Sep 17 00:00:00 2001 From: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> Date: Tue, 30 Nov 2021 08:33:31 +0530 Subject: [PATCH 091/246] refactor: Deferred insert for route history (#15120) --- .../doctype/route_history/route_history.py | 19 ++++++++----------- frappe/public/js/frappe/router_history.js | 4 ++-- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/frappe/desk/doctype/route_history/route_history.py b/frappe/desk/doctype/route_history/route_history.py index fc87312950..a49d5d5418 100644 --- a/frappe/desk/doctype/route_history/route_history.py +++ b/frappe/desk/doctype/route_history/route_history.py @@ -42,16 +42,13 @@ def flush_old_route_records(): @frappe.whitelist() def deferred_insert(routes): - routes_record = [] - - if isinstance(routes, str): - routes = json.loads(routes) - - for route_doc in routes: - routes_record.append({ + routes = [ + { "user": frappe.session.user, - "route": route_doc.get("route"), - "creation": route_doc.get("creation") - }) + "route": route.get("route"), + "creation": route.get("creation"), + } + for route in frappe.parse_json(routes) + ] - _deferred_insert("Route History", json.dumps(routes_record)) + _deferred_insert("Route History", json.dumps(routes)) diff --git a/frappe/public/js/frappe/router_history.js b/frappe/public/js/frappe/router_history.js index fb2d5790da..14b936f5e8 100644 --- a/frappe/public/js/frappe/router_history.js +++ b/frappe/public/js/frappe/router_history.js @@ -4,10 +4,10 @@ const routes_to_skip = ['Form', 'social', 'setup-wizard', 'recorder']; const save_routes = frappe.utils.debounce(() => { if (frappe.session.user === 'Guest') return; const routes = frappe.route_history_queue; - frappe.route_history_queue = []; - if (!routes.length) return; + frappe.route_history_queue = []; + frappe.xcall('frappe.desk.doctype.route_history.route_history.deferred_insert', { 'routes': routes }).catch(() => { From 72fa9a3c1980cbaf29d307928382a5c466a4aa51 Mon Sep 17 00:00:00 2001 From: leela Date: Tue, 14 Sep 2021 16:35:15 +0530 Subject: [PATCH 092/246] fix: Move site setup into background to fix timeouts Provide an option to trigger site setup as a background task. --- frappe/desk/page/setup_wizard/setup_wizard.js | 5 ++++ frappe/desk/page/setup_wizard/setup_wizard.py | 23 +++++++++++++++---- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/frappe/desk/page/setup_wizard/setup_wizard.js b/frappe/desk/page/setup_wizard/setup_wizard.js index f44a57e339..7e90bc01ad 100644 --- a/frappe/desk/page/setup_wizard/setup_wizard.js +++ b/frappe/desk/page/setup_wizard/setup_wizard.js @@ -197,6 +197,8 @@ frappe.setup.SetupWizard = class SetupWizard extends frappe.ui.Slides { callback: (r) => { if (r.message.status === 'ok') { this.post_setup_success(); + } else if (r.message.status === 'registered') { + this.update_setup_message(__("starting the setup...")); } else if (r.message.fail !== undefined) { this.abort_setup(r.message.fail); } @@ -238,6 +240,9 @@ frappe.setup.SetupWizard = class SetupWizard extends frappe.ui.Slides { if (data.fail_msg) { this.abort_setup(data.fail_msg); } + if (data.status === 'ok') { + this.post_setup_success(); + } }) } diff --git a/frappe/desk/page/setup_wizard/setup_wizard.py b/frappe/desk/page/setup_wizard/setup_wizard.py index c729c1d78b..83a5e16009 100755 --- a/frappe/desk/page/setup_wizard/setup_wizard.py +++ b/frappe/desk/page/setup_wizard/setup_wizard.py @@ -54,9 +54,17 @@ def setup_complete(args): return {'status': 'ok'} args = parse_args(args) - stages = get_setup_stages(args) + is_background_task = frappe.conf.get('trigger_site_setup_in_background') + if is_background_task: + process_setup_stages.enqueue(stages=stages, user_input=args, is_background_task=True) + return {'status': 'registered'} + else: + return process_setup_stages(stages, args) + +@frappe.task() +def process_setup_stages(stages, user_input, is_background_task=False): try: frappe.flags.in_setup_wizard = True current_task = None @@ -68,11 +76,16 @@ def setup_complete(args): current_task = task task.get('fn')(task.get('args')) except Exception: - handle_setup_exception(args) - return {'status': 'fail', 'fail': current_task.get('fail_msg')} + handle_setup_exception(user_input) + if not is_background_task: + return {'status': 'fail', 'fail': current_task.get('fail_msg')} + frappe.publish_realtime('setup_task', + {'status': 'fail', "fail_msg": current_task.get('fail_msg')}, user=frappe.session.user) else: - run_setup_success(args) - return {'status': 'ok'} + run_setup_success(user_input) + if not is_background_task: + return {'status': 'ok'} + frappe.publish_realtime('setup_task', {"status": 'ok'}, user=frappe.session.user) finally: frappe.flags.in_setup_wizard = False From 124407a70a9d18659e7c9032987f9a53b1942e4d Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Tue, 30 Nov 2021 11:15:52 +0530 Subject: [PATCH 093/246] fix: update EPS records when renaming doc --- frappe/model/rename_doc.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/frappe/model/rename_doc.py b/frappe/model/rename_doc.py index ee9044b73e..1db89493f2 100644 --- a/frappe/model/rename_doc.py +++ b/frappe/model/rename_doc.py @@ -78,6 +78,8 @@ def rename_doc(doctype, old, new, force=False, merge=False, ignore_permissions=F rename_versions(doctype, old, new) + rename_eps_records(doctype, old, new) + # call after_rename new_doc = frappe.get_doc(doctype, new) @@ -177,6 +179,16 @@ def rename_versions(doctype, old, new): frappe.db.sql("""UPDATE `tabVersion` SET `docname`=%s WHERE `ref_doctype`=%s AND `docname`=%s""", (new, doctype, old)) +def rename_eps_records(doctype, old, new): + epl = frappe.qb.DocType("Energy Point Log") + (frappe.qb.update(epl) + .set(epl.reference_name, new) + .where( + (epl.reference_doctype == doctype) + & (epl.reference_name == old) + ) + ).run() + def rename_parent_and_child(doctype, old, new, meta): # rename the doc frappe.db.sql("UPDATE `tab{0}` SET `name`={1} WHERE `name`={1}".format(doctype, '%s'), (new, old)) From 67b0293e6fc20eb449c79e26f67c4eb4d9341d46 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Tue, 30 Nov 2021 12:33:29 +0530 Subject: [PATCH 094/246] fix: Pretty date rendering based on user-timezone --- frappe/public/js/frappe/utils/datetime.js | 25 +++++++++++--------- frappe/public/js/frappe/utils/pretty_date.js | 2 +- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/frappe/public/js/frappe/utils/datetime.js b/frappe/public/js/frappe/utils/datetime.js index c85cbd42e7..165636bad4 100644 --- a/frappe/public/js/frappe/utils/datetime.js +++ b/frappe/public/js/frappe/utils/datetime.js @@ -134,20 +134,23 @@ $.extend(frappe.datetime, { }, str_to_user: function(val, only_time = false) { - if(!val) return ""; + if (!val) return ""; + const user_time_fmt = frappe.datetime.get_user_time_fmt(); + let date_obj = moment(val); + let user_format = user_time_fmt; - var user_time_fmt = frappe.datetime.get_user_time_fmt(); - if(only_time) { - return moment(val, frappe.defaultTimeFormat) - .format(user_time_fmt); - } - - var user_date_fmt = frappe.datetime.get_user_date_fmt().toUpperCase(); - if(typeof val !== "string" || val.indexOf(" ")===-1) { - return moment(val).format(user_date_fmt); + if (only_time) { + date_obj = moment(val, frappe.defaultTimeFormat); } else { - return moment(val, "YYYY-MM-DD HH:mm:ss").format(user_date_fmt + " " + user_time_fmt); + let user_date_fmt = frappe.datetime.get_user_date_fmt().toUpperCase(); + if (typeof val !== "string" || val.indexOf(" ")===-1) { + date_obj = moment(val); + } else { + date_obj = moment(val, "YYYY-MM-DD HH:mm:ss"); + user_format = user_date_fmt + " " + user_time_fmt; + } } + return date_obj.tz(frappe.boot.time_zone.user).format(user_format); }, get_datetime_as_string: function(d) { diff --git a/frappe/public/js/frappe/utils/pretty_date.js b/frappe/public/js/frappe/utils/pretty_date.js index 3ebe2c1ae2..a5279682ce 100644 --- a/frappe/public/js/frappe/utils/pretty_date.js +++ b/frappe/public/js/frappe/utils/pretty_date.js @@ -6,7 +6,7 @@ function prettyDate(date, mini) { date = new Date((date || "").replace(/-/g, "/").replace(/[TZ]/g, " ").replace(/\.[0-9]*/, "")); } - let diff = (((new Date()).getTime() - date.getTime()) / 1000); + let diff = (((new Date(frappe.datetime.now_datetime())).getTime() - date.getTime()) / 1000); let day_diff = Math.floor(diff / 86400); if (isNaN(day_diff) || day_diff < 0) return ''; From 8eea80e14ec4e86f619bb752f2ce81177596e459 Mon Sep 17 00:00:00 2001 From: abhishek Date: Mon, 25 Oct 2021 14:13:14 +0530 Subject: [PATCH 095/246] refactor: convert doctype user db.sql calls --- frappe/core/doctype/user/user.py | 56 +++++++++++++------------------- 1 file changed, 23 insertions(+), 33 deletions(-) diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py index 9694c11b87..9561dcb737 100644 --- a/frappe/core/doctype/user/user.py +++ b/frappe/core/doctype/user/user.py @@ -217,15 +217,8 @@ class User(Document): if not self.roles: return False - role_table = frappe.qb.DocType("Role") - return len( - frappe.qb.from_(role_table) - .select(role_table.name) - .where(role_table.desk_access == 1) - .where(role_table.name.isin([d.role for d in self.roles])) - .limit(1) - .run() - ) + role_table = DocType("Role") + return frappe.db.count(role_table, ((role_table.desk_access == 1) & (role_table.name.isin([d.role for d in self.roles])))) def share_with_self(self): frappe.share.add(self.doctype, self.name, self.name, write=1, share=1, @@ -283,12 +276,20 @@ class User(Document): return link def get_other_system_managers(self): - return frappe.db.sql("""select distinct `user`.`name` from `tabHas Role` as `user_role`, `tabUser` as `user` - where user_role.role='System Manager' - and `user`.docstatus<2 - and `user`.enabled=1 - and `user_role`.parent = `user`.name - and `user_role`.parent not in ('Administrator', %s) limit 1""", (self.name,)) + user_doctype = DocType("User").as_("user") + user_role_doctype = DocType("Has Role").as_("user_role") + return ( + frappe.qb.from_(user_doctype) + .from_(user_role_doctype) + .select(user_doctype.name) + .where(user_role_doctype.role == 'System Manager') + .where(user_doctype.docstatus < 2) + .where(user_doctype.enabled == 1) + .where(user_role_doctype.parent == user_doctype.name) + .where(user_role_doctype.parent.notin(["Administrator", self.name])) + .limit(1) + .distinct() + ).run() def get_fullname(self): """get first_name space last_name""" @@ -362,8 +363,8 @@ class User(Document): # delete todos frappe.db.delete("ToDo", {"owner": self.name}) - frappe.db.sql("""UPDATE `tabToDo` SET `assigned_by`=NULL WHERE `assigned_by`=%s""", - (self.name,)) + todo_table = DocType("ToDo") + frappe.qb.update(todo_table).set(todo_table.assigned_by, None).where(todo_table.assigned_by == self.name).run() # delete events frappe.db.delete("Event", {"owner": self.name, "event_type": "Private"}) @@ -429,10 +430,7 @@ class User(Document): frappe.rename_doc("Notification Settings", old_name, new_name, force=True, show_alert=False) # set email - table = DocType("User") - frappe.qb.update(table).where( - table.name == new_name - ).set("email", new_name).run() + frappe.db.update("User", new_name, "email", new_name) def append_roles(self, *roles): """Add roles to user""" @@ -706,22 +704,15 @@ def get_email_awaiting(user): if waiting: return waiting else: - # TODO - frappe.db.sql("""update `tabUser Email` - set awaiting_password =0 - where parent = %(user)s""",{"user":user}) + user_email_table = DocType("User Email") + frappe.qb.update(user_email_table).set(user_email_table.user_email_table, 0).where(user_email_table.parent == user).run() return False def ask_pass_update(): # update the sys defaults as to awaiting users from frappe.utils import set_default - doctype = DocType("User Email") - users = frappe.qb.from_(doctype).where(doctype.awaiting_password == 1).select( - doctype.parent.as_("user") - ).distinct().run(as_dict=True) - - password_list = [ user.get("user") for user in users ] + password_list = frappe.get_all("User Email", filters={"awaiting_password": True}, pluck="parent", distinct=True) set_default("email_user_password", u','.join(password_list)) def _get_user_for_update_password(key, old_password): @@ -889,8 +880,7 @@ def get_active_users(): def get_website_users(): """Returns total no. of website users""" - return frappe.db.sql("""select count(*) from `tabUser` - where enabled = 1 and user_type = 'Website User'""")[0][0] + return frappe.db.count("User", filters={"enabled": True, "user_type": "Website User"}) def get_active_website_users(): """Returns No. of website users who logged in, in the last 3 days""" From 1bb8dcf1767508088e593e61122da506456ec752 Mon Sep 17 00:00:00 2001 From: abhishek Date: Sun, 14 Nov 2021 14:40:53 +0530 Subject: [PATCH 096/246] refactor: convert core notifications db.sql calls get_unseen_likes() get_unread_emails() --- frappe/core/notifications.py | 52 +++++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/frappe/core/notifications.py b/frappe/core/notifications.py index b43d424df5..d091983d00 100644 --- a/frappe/core/notifications.py +++ b/frappe/core/notifications.py @@ -2,6 +2,9 @@ # License: MIT. See LICENSE import frappe +from frappe.query_builder import DocType, Interval +from frappe.query_builder.functions import Now + def get_notification_config(): return { @@ -39,28 +42,33 @@ def get_todays_events(as_list=False): def get_unseen_likes(): """Returns count of unseen likes""" - return frappe.db.sql("""select count(*) from `tabComment` - where - comment_type='Like' - and modified >= (NOW() - INTERVAL '1' YEAR) - and owner is not null and owner!=%(user)s - and reference_owner=%(user)s - and seen=0 - """, {"user": frappe.session.user})[0][0] + + comment_doctype = DocType("Comment") + return frappe.db.count(comment_doctype, + filters=( + (comment_doctype.comment_type == "Like") + & (comment_doctype.modified >= Now() - Interval(years=1)) + & (comment_doctype.owner.notnull()) + & (comment_doctype.owner != frappe.session.user) + & (comment_doctype.reference_owner == frappe.session.user) + & (comment_doctype.seen == 0) + ) + ) + def get_unread_emails(): - "returns unread emails for a user" + "returns count of unread emails for a user" - return frappe.db.sql("""\ - SELECT count(*) - FROM `tabCommunication` - WHERE communication_type='Communication' - AND communication_medium='Email' - AND sent_or_received='Received' - AND email_status not in ('Spam', 'Trash') - AND email_account in ( - SELECT distinct email_account from `tabUser Email` WHERE parent=%(user)s - ) - AND modified >= (NOW() - INTERVAL '1' YEAR) - AND seen=0 - """, {"user": frappe.session.user})[0][0] + communication_doctype = DocType("Communication") + user_doctype = DocType("User") + distinct_email_accounts = frappe.qb.from_(user_doctype).select(user_doctype.email_account).where(user_doctype.parent == frappe.session.user).distinct() + + return frappe.db.count(communication_doctype, filters=( + (communication_doctype.communication_type == "Communication") + & (communication_doctype.communication_medium == "Email") + & (communication_doctype.sent_or_received == "Received") + & (communication_doctype.email_status.notin(["spam", "Trash"])) + & (communication_doctype.email_account.isin(distinct_email_accounts)) + & (communication_doctype.modified >= Now() - Interval(years=1)) + & (communication_doctype.seen == 0) + )) From 9f183e6cf2d709457e371674b7a38661644a6dfd Mon Sep 17 00:00:00 2001 From: saxenabhishek Date: Tue, 30 Nov 2021 14:16:13 +0530 Subject: [PATCH 097/246] style: follow simple indenting standard --- frappe/core/doctype/user/user.py | 6 +++++- frappe/core/notifications.py | 7 ++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py index 9561dcb737..f36553593b 100644 --- a/frappe/core/doctype/user/user.py +++ b/frappe/core/doctype/user/user.py @@ -364,7 +364,11 @@ class User(Document): # delete todos frappe.db.delete("ToDo", {"owner": self.name}) todo_table = DocType("ToDo") - frappe.qb.update(todo_table).set(todo_table.assigned_by, None).where(todo_table.assigned_by == self.name).run() + ( + frappe.qb.update(todo_table) + .set(todo_table.assigned_by, None) + .where(todo_table.assigned_by == self.name) + ).run() # delete events frappe.db.delete("Event", {"owner": self.name, "event_type": "Private"}) diff --git a/frappe/core/notifications.py b/frappe/core/notifications.py index d091983d00..a69c644383 100644 --- a/frappe/core/notifications.py +++ b/frappe/core/notifications.py @@ -61,7 +61,12 @@ def get_unread_emails(): communication_doctype = DocType("Communication") user_doctype = DocType("User") - distinct_email_accounts = frappe.qb.from_(user_doctype).select(user_doctype.email_account).where(user_doctype.parent == frappe.session.user).distinct() + distinct_email_accounts = ( + frappe.qb.from_(user_doctype) + .select(user_doctype.email_account) + .where(user_doctype.parent == frappe.session.user) + .distinct() + ) return frappe.db.count(communication_doctype, filters=( (communication_doctype.communication_type == "Communication") From 23ce93be73a404326e43bcaad02d0da37d03bc47 Mon Sep 17 00:00:00 2001 From: Rucha Mahabal Date: Tue, 30 Nov 2021 15:28:16 +0530 Subject: [PATCH 098/246] fix: reset lft rgt in copy doc --- frappe/public/js/frappe/model/create_new.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/model/create_new.js b/frappe/public/js/frappe/model/create_new.js index 1b09a451eb..d10af1932e 100644 --- a/frappe/public/js/frappe/model/create_new.js +++ b/frappe/public/js/frappe/model/create_new.js @@ -178,7 +178,7 @@ $.extend(frappe.model, { user_default = user_defaults[0]; } } - + if (!user_default) { user_default = frappe.defaults.get_user_default(df.fieldname); } else if ( @@ -351,6 +351,8 @@ $.extend(frappe.model, { newdoc.creation = ""; newdoc.modified_by = user; newdoc.modified = ""; + newdoc.lft = null; + newdoc.rgt = null; return newdoc; }, From 2410dc66c197d3fce2d0256c46382122e6028519 Mon Sep 17 00:00:00 2001 From: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> Date: Tue, 30 Nov 2021 15:36:06 +0530 Subject: [PATCH 099/246] style: Fix indentation --- frappe/core/notifications.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/frappe/core/notifications.py b/frappe/core/notifications.py index a69c644383..939cf52911 100644 --- a/frappe/core/notifications.py +++ b/frappe/core/notifications.py @@ -45,15 +45,15 @@ def get_unseen_likes(): comment_doctype = DocType("Comment") return frappe.db.count(comment_doctype, - filters=( - (comment_doctype.comment_type == "Like") - & (comment_doctype.modified >= Now() - Interval(years=1)) - & (comment_doctype.owner.notnull()) - & (comment_doctype.owner != frappe.session.user) - & (comment_doctype.reference_owner == frappe.session.user) - & (comment_doctype.seen == 0) - ) - ) + filters=( + (comment_doctype.comment_type == "Like") + & (comment_doctype.modified >= Now() - Interval(years=1)) + & (comment_doctype.owner.notnull()) + & (comment_doctype.owner != frappe.session.user) + & (comment_doctype.reference_owner == frappe.session.user) + & (comment_doctype.seen == 0) + ) + ) def get_unread_emails(): @@ -68,7 +68,8 @@ def get_unread_emails(): .distinct() ) - return frappe.db.count(communication_doctype, filters=( + return frappe.db.count(communication_doctype, + filters=( (communication_doctype.communication_type == "Communication") & (communication_doctype.communication_medium == "Email") & (communication_doctype.sent_or_received == "Received") @@ -76,4 +77,5 @@ def get_unread_emails(): & (communication_doctype.email_account.isin(distinct_email_accounts)) & (communication_doctype.modified >= Now() - Interval(years=1)) & (communication_doctype.seen == 0) - )) + ) + ) From dc522395c80fa9e320b63bf23827974ef0ef3488 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Tue, 30 Nov 2021 16:49:27 +0530 Subject: [PATCH 100/246] test: Fix test case for datetime --- cypress/integration/datetime.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cypress/integration/datetime.js b/cypress/integration/datetime.js index b310526c7c..ef1952dc94 100644 --- a/cypress/integration/datetime.js +++ b/cypress/integration/datetime.js @@ -92,15 +92,15 @@ context('Control Date, Time and DateTime', () => { date_format: 'dd.mm.yyyy', time_format: 'HH:mm:ss', value: ' 02.12.2019 11:00:12', - doc_value: '2019-12-02 11:00:12', - input_value: '02.12.2019 11:00:12' + doc_value: '2019-12-02 00:30:12', // system timezone (America/New_York) + input_value: '02.12.2019 11:00:12' // admin timezone (Asia/Kolkata) }, { date_format: 'mm-dd-yyyy', time_format: 'HH:mm', value: ' 12-02-2019 11:00:00', - doc_value: '2019-12-02 11:00:00', - input_value: '12-02-2019 11:00' + doc_value: '2019-12-02 00:30:00', // system timezone (America/New_York) + input_value: '12-02-2019 11:00' // admin timezone (Asia/Kolkata) } ]; datetime_formats.forEach(d => { From 62247431ca11efa09529c061f57d5329bc758a7f Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Tue, 30 Nov 2021 17:38:47 +0530 Subject: [PATCH 101/246] fix: scrolling issues after minimize global search (backport #15127) (#15131) Co-authored-by: Bhavesh Maheshwari --- frappe/public/scss/common/modal.scss | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frappe/public/scss/common/modal.scss b/frappe/public/scss/common/modal.scss index 54843290fc..ec582591f2 100644 --- a/frappe/public/scss/common/modal.scss +++ b/frappe/public/scss/common/modal.scss @@ -152,6 +152,8 @@ body.modal-open[style^="padding-right"] { .modal-minimize { position: initial; + height: 0; + width: 0; .modal-dialog { position: fixed; From 5715bbda5077349575cb227e5576c0c193d0c638 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Tue, 30 Nov 2021 22:50:43 +0530 Subject: [PATCH 102/246] fix: multiple time global search minimize screen freeze (backport #15133) (#15134) Co-authored-by: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> Co-authored-by: Bhavesh Maheshwari <34086262+bhavesh95863@users.noreply.github.com> --- frappe/public/js/frappe/ui/dialog.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/frappe/public/js/frappe/ui/dialog.js b/frappe/public/js/frappe/ui/dialog.js index b1a22c8929..e2e51ce501 100644 --- a/frappe/public/js/frappe/ui/dialog.js +++ b/frappe/public/js/frappe/ui/dialog.js @@ -198,6 +198,11 @@ frappe.ui.Dialog = class Dialog extends frappe.ui.FieldGroup { this.$wrapper.removeClass('modal-minimize'); + if (this.minimizable && this.is_minimized) { + $(".modal-backdrop").toggle(); + this.is_minimized = false; + } + // clear any message this.clear_message(); From d622b96e64312bd24d8606b4ad3511d6d88d5e92 Mon Sep 17 00:00:00 2001 From: Jannat Patel Date: Wed, 1 Dec 2021 10:54:15 +0530 Subject: [PATCH 103/246] fix: request for account deletion flow --- .../emails/account_deletion_notification.html | 3 +++ .../personal_data_deletion_request.py | 17 +++++++++++-- .../website_settings/website_settings.json | 25 ++++++++++++++++--- .../website_settings/website_settings.py | 4 +++ .../request_to_delete_data.js | 15 +++++++++-- .../request_to_delete_data.json | 10 +++++--- frappe/www/me.html | 3 +++ 7 files changed, 66 insertions(+), 11 deletions(-) create mode 100644 frappe/templates/emails/account_deletion_notification.html diff --git a/frappe/templates/emails/account_deletion_notification.html b/frappe/templates/emails/account_deletion_notification.html new file mode 100644 index 0000000000..17d6aa3c93 --- /dev/null +++ b/frappe/templates/emails/account_deletion_notification.html @@ -0,0 +1,3 @@ +

{{_("Dear User,")}}

+

{{_("As per your request, your account and data on {0} associated with email {1} has been permanently deleted").format(app_name, email)}}.

+ diff --git a/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py b/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py index 8a72fa269f..c3e0d22063 100644 --- a/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py +++ b/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py @@ -75,7 +75,7 @@ class PersonalDataDeletionRequest(Document): frappe.sendmail( recipients=self.email, - subject=_("Confirm Deletion of Data"), + subject=_("Confirm Deletion of Account"), template="delete_data_confirmation", args={ "email": self.email, @@ -83,7 +83,7 @@ class PersonalDataDeletionRequest(Document): "host_name": frappe.local.site, "link": url, }, - header=[_("Confirm Deletion of Data"), "green"], + header=[_("Confirm Deletion of Account"), "green"], ) def notify_system_managers(self): @@ -109,6 +109,7 @@ class PersonalDataDeletionRequest(Document): self.validate_data_anonymization() self.disable_user() self.anonymize_data() + self.notify_user_after_deletion() def anonymize_data(self): return frappe.enqueue_doc( @@ -120,6 +121,18 @@ class PersonalDataDeletionRequest(Document): now=frappe.flags.in_test, ) + def notify_user_after_deletion(self): + frappe.sendmail( + recipients=self.email, + subject=_("Your account has been deleted"), + template="account_deletion_notification", + args={ + "email": self.email, + "app_name": frappe.db.get_single_value("Website Settings", "app_name"), + }, + header=[_("Your account has been deleted"), "green"], + ) + def add_deletion_steps(self): if self.deletion_steps: return diff --git a/frappe/website/doctype/website_settings/website_settings.json b/frappe/website/doctype/website_settings/website_settings.json index 48f097e525..f39147d555 100644 --- a/frappe/website/doctype/website_settings/website_settings.json +++ b/frappe/website/doctype/website_settings/website_settings.json @@ -63,7 +63,10 @@ "subdomain", "head_html", "robots_txt", - "route_redirects" + "route_redirects", + "account_deletion_settings_section", + "show_account_deletion_link", + "account_deletion_sla" ], "fields": [ { @@ -386,6 +389,22 @@ "fieldname": "app_logo", "fieldtype": "Attach Image", "label": "App Logo" + }, + { + "fieldname": "account_deletion_settings_section", + "fieldtype": "Section Break", + "label": "Account Deletion Settings" + }, + { + "fieldname": "account_deletion_sla", + "fieldtype": "Int", + "label": "Account Deletion SLA (Days)" + }, + { + "default": "0", + "fieldname": "show_account_deletion_link", + "fieldtype": "Check", + "label": "Show Account Deletion Link in My Account Page" } ], "icon": "fa fa-cog", @@ -394,7 +413,7 @@ "issingle": 1, "links": [], "max_attachments": 10, - "modified": "2021-08-23 21:39:51.702248", + "modified": "2021-12-01 10:15:17.403155", "modified_by": "Administrator", "module": "Website", "name": "Website Settings", @@ -418,4 +437,4 @@ "sort_field": "modified", "sort_order": "ASC", "track_changes": 1 -} +} \ No newline at end of file diff --git a/frappe/website/doctype/website_settings/website_settings.py b/frappe/website/doctype/website_settings/website_settings.py index 7a744eaf53..02eed9cc6d 100644 --- a/frappe/website/doctype/website_settings/website_settings.py +++ b/frappe/website/doctype/website_settings/website_settings.py @@ -177,3 +177,7 @@ def get_items(parentfield): t['child_items'].append(d) break return top_items + +@frappe.whitelist(allow_guest=True) +def get_account_deletion_sla(): + return frappe.db.get_single_value("Website Settings", "account_deletion_sla") diff --git a/frappe/website/web_form/request_to_delete_data/request_to_delete_data.js b/frappe/website/web_form/request_to_delete_data/request_to_delete_data.js index 7da3f1fb41..9d19e36a05 100644 --- a/frappe/website/web_form/request_to_delete_data/request_to_delete_data.js +++ b/frappe/website/web_form/request_to_delete_data/request_to_delete_data.js @@ -1,3 +1,14 @@ frappe.ready(function() { - // bind events here -}); \ No newline at end of file + frappe.web_form.after_load = () => { + frappe.call({ + method: "frappe.website.doctype.website_settings.website_settings.get_account_deletion_sla", + callback: (data) => { + if (data.message) { + const intro_wrapper = $('#introduction .ql-editor.read-mode'); + const sla_description = `
Note: Your request for account deletion will be fulfilled within ${data.message} days.`; + intro_wrapper.html(intro_wrapper.html() + sla_description); + } + } + }) + } +}); diff --git a/frappe/website/web_form/request_to_delete_data/request_to_delete_data.json b/frappe/website/web_form/request_to_delete_data/request_to_delete_data.json index b0180d833c..1113297df6 100644 --- a/frappe/website/web_form/request_to_delete_data/request_to_delete_data.json +++ b/frappe/website/web_form/request_to_delete_data/request_to_delete_data.json @@ -10,24 +10,26 @@ "amount_based_on_field": 0, "apply_document_permissions": 0, "button_label": "Submit", + "client_script": "", "creation": "2019-01-25 14:24:12.588810", "currency": "INR", + "custom_css": "[data-doctype=\"Web Form\"] {\n width: 50%;\n margin: 6rem auto;\n}", "doc_type": "Personal Data Deletion Request", "docstatus": 0, "doctype": "Web Form", "idx": 0, - "introduction_text": "

Send a request to delete your personally identifiable information (PII) that is stored on our system. You will receive an email to verify your request. Once the request is verified we will take care of deleting your PII. If you just want to check what PII we have stored, you can request your data.

", + "introduction_text": "

Send a request to delete your account and personally identifiable information (PII) that is stored on our system. You will receive an email to verify your request. Once the request is verified we will take care of deleting your PII. If you just want to check what PII we have stored, you can request your data.

", "is_standard": 1, "login_required": 0, "max_attachment_size": 0, - "modified": "2021-03-25 11:08:49.580621", + "modified": "2021-11-30 17:56:03.099870", "modified_by": "Administrator", "module": "Website", "name": "request-to-delete-data", "owner": "Administrator", "payment_button_label": "Buy Now", "published": 1, - "route": "request-to-delete-data", + "route": "request-for-account-deletion", "route_to_success_link": 0, "show_attachments": 0, "show_in_grid": 0, @@ -35,7 +37,7 @@ "sidebar_items": [], "success_message": "An email to verify your request has been sent to your email address. Please verify your request to complete the process.", "success_url": "/", - "title": "Request to Delete Data", + "title": "Request for Account Deletion", "web_form_fields": [ { "allow_read_on_all_link_options": 0, diff --git a/frappe/www/me.html b/frappe/www/me.html index eb97c566d8..4f9a59cac5 100644 --- a/frappe/www/me.html +++ b/frappe/www/me.html @@ -10,6 +10,9 @@
  • {{ _("Reset Password") }}
  • {{ _("Edit Profile") }}
  • {{ _("Manage Third Party Apps") }}
    • + {% if frappe.db.get_single_value("Website Settings", "show_account_deletion_link") %} +
  • {{ _("Request for Account Deletion") }}
    • + {% endif %} From a86f8d9640a7d01f88cc2d2beeac52572c00aa98 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Wed, 1 Dec 2021 11:27:23 +0530 Subject: [PATCH 104/246] fix: Do not guess timezone for only time - Time value should be consistent across timezones - Only worry about timezone when dealing with datetime --- frappe/public/js/frappe/utils/datetime.js | 26 ++++++++++++----------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/frappe/public/js/frappe/utils/datetime.js b/frappe/public/js/frappe/utils/datetime.js index 165636bad4..2fcfd75c5e 100644 --- a/frappe/public/js/frappe/utils/datetime.js +++ b/frappe/public/js/frappe/utils/datetime.js @@ -17,14 +17,15 @@ $.extend(frappe.datetime, { // system time zone and then convert the string to user time zone(from User doctype). let date_obj = null; if (frappe.boot.time_zone && frappe.boot.time_zone.system && frappe.boot.time_zone.user) { - date_obj = moment.tz(date, frappe.boot.time_zone.system) + date_obj = moment(date) + .tz(frappe.boot.time_zone.system) .clone() .tz(frappe.boot.time_zone.user); } else { date_obj = moment(date); } - return format===false ? date_obj : date_obj.format(frappe.defaultDatetimeFormat); + return format === false ? date_obj : date_obj.format(frappe.defaultDatetimeFormat); }, convert_to_system_tz: function(date, format) { @@ -113,11 +114,11 @@ $.extend(frappe.datetime, { return moment().endOf("quarter").format(); }, - year_start: function(){ + year_start: function() { return moment().startOf("year").format(); }, - year_end: function(){ + year_end: function() { return moment().endOf("year").format(); }, @@ -135,22 +136,23 @@ $.extend(frappe.datetime, { str_to_user: function(val, only_time = false) { if (!val) return ""; + const user_date_fmt = frappe.datetime.get_user_date_fmt().toUpperCase(); const user_time_fmt = frappe.datetime.get_user_time_fmt(); - let date_obj = moment(val); let user_format = user_time_fmt; if (only_time) { - date_obj = moment(val, frappe.defaultTimeFormat); + let date_obj = moment(val, frappe.defaultTimeFormat); + return date_obj.format(user_format); } else { - let user_date_fmt = frappe.datetime.get_user_date_fmt().toUpperCase(); - if (typeof val !== "string" || val.indexOf(" ")===-1) { - date_obj = moment(val); + let date_obj = moment(val); + if (typeof val !== "string" || val.indexOf(" ") === -1) { + user_format = user_date_fmt; } else { date_obj = moment(val, "YYYY-MM-DD HH:mm:ss"); user_format = user_date_fmt + " " + user_time_fmt; } + return date_obj.tz(frappe.boot.time_zone.user).format(user_format); } - return date_obj.tz(frappe.boot.time_zone.user).format(user_format); }, get_datetime_as_string: function(d) { @@ -217,9 +219,9 @@ $.extend(frappe.datetime, { return as_obj ? frappe.datetime.moment_to_date_obj(date) : date.format(format); }, - moment_to_date_obj: function(moment) { + moment_to_date_obj: function(moment_obj) { const date_obj = new Date(); - const date_array = moment.toArray(); + const date_array = moment_obj.toArray(); date_obj.setFullYear(date_array[0]); date_obj.setMonth(date_array[1]); date_obj.setDate(date_array[2]); From 515dca29fda58c4979989d0590f59d30c6a6c595 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Wed, 1 Dec 2021 12:57:15 +0530 Subject: [PATCH 105/246] fix: Remove unnecessary code - frappe.datetime.str_to_user(value, false) takes care of tz conversion --- frappe/public/js/frappe/form/controls/datetime.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/frappe/public/js/frappe/form/controls/datetime.js b/frappe/public/js/frappe/form/controls/datetime.js index 3142f1bf0f..d8c67ce2e9 100644 --- a/frappe/public/js/frappe/form/controls/datetime.js +++ b/frappe/public/js/frappe/form/controls/datetime.js @@ -40,9 +40,6 @@ frappe.ui.form.ControlDatetime = class ControlDatetime extends frappe.ui.form.Co format_for_input(value) { if (!value) return ""; - if (!frappe.datetime.is_system_time_zone()) { - value = frappe.datetime.convert_to_user_tz(value, true); - } return frappe.datetime.str_to_user(value, false); } From e2bf192633624669693d39a09bc9ac4ccb0cc940 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Wed, 1 Dec 2021 12:59:32 +0530 Subject: [PATCH 106/246] fix: Select date of datepicker while doing set_formatted_input - To avoid empty value when clicking datetime for first time --- frappe/public/js/frappe/form/controls/datetime.js | 1 + 1 file changed, 1 insertion(+) diff --git a/frappe/public/js/frappe/form/controls/datetime.js b/frappe/public/js/frappe/form/controls/datetime.js index d8c67ce2e9..c7efc3e2c3 100644 --- a/frappe/public/js/frappe/form/controls/datetime.js +++ b/frappe/public/js/frappe/form/controls/datetime.js @@ -10,6 +10,7 @@ frappe.ui.form.ControlDatetime = class ControlDatetime extends frappe.ui.form.Co } this.$input && this.$input.val(this.format_for_input(value)); + this.datepicker.selectDate(frappe.datetime.str_to_obj(value)); } set_date_options() { super.set_date_options(); From 62073d65a0aa948902c6f2f556a3f4971e7e72d7 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Wed, 1 Dec 2021 13:10:49 +0530 Subject: [PATCH 107/246] fix: Set now date based on system timezone - because set_formatted_value will convert value to user timezone down the line --- frappe/public/js/frappe/form/controls/date.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/public/js/frappe/form/controls/date.js b/frappe/public/js/frappe/form/controls/date.js index 9ad81c7e46..ba94531b0f 100644 --- a/frappe/public/js/frappe/form/controls/date.js +++ b/frappe/public/js/frappe/form/controls/date.js @@ -53,7 +53,7 @@ frappe.ui.form.ControlDate = class ControlDate extends frappe.ui.form.ControlDat let date_format = sysdefaults && sysdefaults.date_format ? sysdefaults.date_format : 'yyyy-mm-dd'; - let now_date = new Date(); + let now_date = new Date(this.get_now_date()); this.today_text = __("Today"); this.date_format = frappe.defaultDateFormat; @@ -112,7 +112,7 @@ frappe.ui.form.ControlDate = class ControlDate extends frappe.ui.form.ControlDat this.datepicker.update('position', position); } get_now_date() { - return frappe.datetime.now_date(true); + return frappe.datetime.convert_to_system_tz(frappe.datetime.now_date(true)); } set_t_for_today() { var me = this; From db101e5ec396ae1562608ed9a159c9f0877eab37 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Tue, 30 Nov 2021 16:25:20 +0530 Subject: [PATCH 108/246] fix(ux): ensure max_attachments is more than no of attach fields --- frappe/core/doctype/doctype/doctype.js | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/frappe/core/doctype/doctype/doctype.js b/frappe/core/doctype/doctype/doctype.js index 262a6efd90..291fb237a9 100644 --- a/frappe/core/doctype/doctype/doctype.js +++ b/frappe/core/doctype/doctype/doctype.js @@ -129,7 +129,23 @@ frappe.ui.form.on('DocType', { } frm.set_df_property('fields', 'reqd', frm.doc.autoname !== 'Prompt'); - } + }, + + max_attachments: function(frm) { + if (!frm.doc.max_attachments) { + return; + } + const is_attach_field = (f) => ["Attach", "Attach Image"].includes(f.fieldtype); + const no_of_attach_fields = frm.doc.fields.filter(is_attach_field).length; + + if (no_of_attach_fields > frm.doc.max_attachments) { + frm.set_value("max_attachments", no_of_attach_fields); + const label = frm.get_docfield("max_attachments").label; + frappe.show_alert( + __("Number of attachment fields are more than {}, limit updated to {}.", [label, no_of_attach_fields])); + } + }, + }); frappe.ui.form.on("DocField", { @@ -217,5 +233,9 @@ frappe.ui.form.on("DocField", { $doctype_select.val(curr_value.doctype); update_fieldname_options(); } + }, + + fieldtype: function(frm) { + frm.trigger("max_attachments"); } }); From 2f6b57cc0ae2b87859bdcca8755f0e764f0ea16a Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Tue, 30 Nov 2021 12:18:52 +0530 Subject: [PATCH 109/246] fix(ux): validate max_attachment on doctype controller --- frappe/core/doctype/doctype/doctype.py | 17 +++++++++++++++++ frappe/model/__init__.py | 5 +++++ 2 files changed, 22 insertions(+) diff --git a/frappe/core/doctype/doctype/doctype.py b/frappe/core/doctype/doctype/doctype.py index 1c8c1f9217..a6a81cb195 100644 --- a/frappe/core/doctype/doctype/doctype.py +++ b/frappe/core/doctype/doctype/doctype.py @@ -75,6 +75,7 @@ class DocType(Document): self.make_repeatable() self.validate_nestedset() self.validate_website() + self.ensure_minimum_max_attachment_limit() validate_links_table_fieldnames(self) if not self.is_new(): @@ -246,6 +247,22 @@ class DocType(Document): # clear website cache clear_cache() + def ensure_minimum_max_attachment_limit(self): + """Ensure that max_attachments is *at least* bigger than number of attach fields.""" + from frappe.model import attachment_fieldtypes + + + if not self.max_attachments: + return + + total_attach_fields = len([d for d in self.fields if d.fieldtype in attachment_fieldtypes]) + if total_attach_fields > self.max_attachments: + self.max_attachments = total_attach_fields + field_label = frappe.bold(self.meta.get_field("max_attachments").label) + frappe.msgprint(_("Number of attachment fields are more than {}, limit updated to {}.") + .format(field_label, total_attach_fields), + title=_("Insufficient attachment limit"), alert=True) + def change_modified_of_parent(self): """Change the timestamp of parent DocType if the current one is a child to clear caches.""" if frappe.flags.in_import: diff --git a/frappe/model/__init__.py b/frappe/model/__init__.py index b460db29a7..b50a0304a5 100644 --- a/frappe/model/__init__.py +++ b/frappe/model/__init__.py @@ -38,6 +38,11 @@ data_fieldtypes = ( 'Icon' ) +attachment_fieldtypes = ( + 'Attach', + 'Attach Image', +) + no_value_fields = ( 'Section Break', 'Column Break', From 3a34bfc52035b2ff23091d827d8f41a8c7f49f30 Mon Sep 17 00:00:00 2001 From: Manuel <57345036+mtraeber@users.noreply.github.com> Date: Wed, 1 Dec 2021 13:41:27 +0100 Subject: [PATCH 110/246] Fix: copy_email_data patch #15106 (#15115) Reload DocType `Email Account` in patch. Changed `modified` in json files. Removed linking of `imap_folder` in `Email Flag Queue` this connection seems not necessary at this point. Also removed all parts that create this connection. --- .../doctype/communication/communication.json | 2 +- .../doctype/communication/communication.py | 1 - .../doctype/email_account/email_account.json | 2 +- .../doctype/email_account/email_account.py | 3 +- .../email_flag_queue/email_flag_queue.json | 256 ++++-------------- frappe/patches/v14_0/copy_mail_data.py | 6 +- 6 files changed, 59 insertions(+), 211 deletions(-) diff --git a/frappe/core/doctype/communication/communication.json b/frappe/core/doctype/communication/communication.json index 9e154146b3..175c64b9eb 100644 --- a/frappe/core/doctype/communication/communication.json +++ b/frappe/core/doctype/communication/communication.json @@ -395,7 +395,7 @@ "icon": "fa fa-comment", "idx": 1, "links": [], - "modified": "2021-03-25 09:44:28.963538", + "modified": "2021-11-30 09:03:25.728637", "modified_by": "Administrator", "module": "Core", "name": "Communication", diff --git a/frappe/core/doctype/communication/communication.py b/frappe/core/doctype/communication/communication.py index f2fbc26a22..3a78a6a599 100644 --- a/frappe/core/doctype/communication/communication.py +++ b/frappe/core/doctype/communication/communication.py @@ -42,7 +42,6 @@ class Communication(Document, CommunicationEmailMixin): "action": "Read", "communication": self.name, "uid": self.uid, - "imap_folder": self.imap_folder, "email_account": self.email_account }).insert(ignore_permissions=True) frappe.db.commit() diff --git a/frappe/email/doctype/email_account/email_account.json b/frappe/email/doctype/email_account/email_account.json index bf9a79529b..65053bab3d 100644 --- a/frappe/email/doctype/email_account/email_account.json +++ b/frappe/email/doctype/email_account/email_account.json @@ -581,7 +581,7 @@ "icon": "fa fa-inbox", "index_web_pages_for_search": 1, "links": [], - "modified": "2021-09-21 16:44:25.728637", + "modified": "2021-11-30 09:03:25.728637", "modified_by": "Administrator", "module": "Email", "name": "Email Account", diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index 52623fb358..ef1d49302f 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -578,10 +578,9 @@ class EmailAccount(Document): EmailFlagQ = frappe.qb.DocType("Email Flag Queue") flags = ( frappe.qb.from_(EmailFlagQ) - .select(EmailFlagQ.name, EmailFlagQ.communication, EmailFlagQ.uid, EmailFlagQ.action, EmailFlagQ.imap_folder) + .select(EmailFlagQ.name, EmailFlagQ.communication, EmailFlagQ.uid, EmailFlagQ.action) .where(EmailFlagQ.is_completed == 0) .where(EmailFlagQ.email_account == frappe.db.escape(self.name)) - .where(EmailFlagQ.folder_name == frappe.db.escape(folder_name)) ).run(as_dict=True) uid_list = { flag.get("uid", None): flag.get("action", "Read") for flag in flags } diff --git a/frappe/email/doctype/email_flag_queue/email_flag_queue.json b/frappe/email/doctype/email_flag_queue/email_flag_queue.json index 165e8f9ea9..14b1ec4f53 100644 --- a/frappe/email/doctype/email_flag_queue/email_flag_queue.json +++ b/frappe/email/doctype/email_flag_queue/email_flag_queue.json @@ -1,213 +1,67 @@ { - "allow_copy": 1, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, - "creation": "2016-04-20 15:29:39.785172", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 0, + "actions": [], + "allow_copy": 1, + "creation": "2016-04-20 15:29:39.785172", + "doctype": "DocType", + "engine": "InnoDB", + "field_order": [ + "is_completed", + "communication", + "action", + "email_account", + "uid" + ], "fields": [ { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "is_completed", - "fieldtype": "Check", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Is Completed", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "default": "0", + "fieldname": "is_completed", + "fieldtype": "Check", + "label": "Is Completed", + "read_only": 1 + }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "communication", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Communication", - "length": 0, - "no_copy": 0, - "options": "", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "communication", + "fieldtype": "Data", + "label": "Communication" + }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "action", - "fieldtype": "Select", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Action", - "length": 0, - "no_copy": 0, - "options": "Read\nUnread", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "action", + "fieldtype": "Select", + "label": "Action", + "options": "Read\nUnread" + }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "email_account", - "fieldtype": "Data", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Email Account", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "email_account", + "fieldtype": "Data", + "hidden": 1, + "label": "Email Account" + }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "uid", - "fieldtype": "Data", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "UID", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "fieldname": "uid", + "fieldtype": "Data", + "hidden": 1, + "label": "UID" } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 1, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2017-09-20 15:27:12.142079", - "modified_by": "Administrator", - "module": "Email", - "name": "Email Flag Queue", - "name_case": "", - "owner": "Administrator", + ], + "in_create": 1, + "links": [], + "modified": "2021-11-30 09:51:34.489932", + "modified_by": "Administrator", + "module": "Email", + "name": "Email Flag Queue", + "owner": "Administrator", "permissions": [ { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 0, - "delete": 1, - "email": 1, - "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "set_user_permissions": 0, - "share": 1, - "submit": 0, - "write": 0 + "delete": 1, + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1 } - ], - "quick_entry": 0, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 0, - "track_seen": 0 + ], + "sort_field": "modified", + "sort_order": "DESC" } \ No newline at end of file diff --git a/frappe/patches/v14_0/copy_mail_data.py b/frappe/patches/v14_0/copy_mail_data.py index 8780ab8630..362d23d0e1 100644 --- a/frappe/patches/v14_0/copy_mail_data.py +++ b/frappe/patches/v14_0/copy_mail_data.py @@ -3,6 +3,7 @@ import frappe def execute(): + frappe.reload_doc("email", "doctype", "email_account") # patch for all Email Account with the flag use_imap for email_account in frappe.get_list("Email Account", filters={"enable_incoming": 1, "use_imap": 1}): # get all data from Email Account @@ -19,8 +20,3 @@ def execute(): }) doc.save() - EmailFlagQueue = frappe.qb.DocType("Email Flag Queue") - frappe.qb.update(EmailFlagQueue) \ - .set(EmailFlagQueue.imap_folder, "INBOX") \ - .where(EmailFlagQueue.email_account == doc.name) \ - .where(EmailFlagQueue.imap_folder.isnull()).run() From 121333874a07d18eb4e3aff83496d975c3f03bc3 Mon Sep 17 00:00:00 2001 From: Jannat Patel Date: Wed, 1 Dec 2021 18:54:01 +0530 Subject: [PATCH 111/246] fix: send email immediately --- frappe/templates/emails/delete_data_confirmation.html | 4 ++-- .../personal_data_deletion_request.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/frappe/templates/emails/delete_data_confirmation.html b/frappe/templates/emails/delete_data_confirmation.html index 126d8bcb4b..a5794abf96 100644 --- a/frappe/templates/emails/delete_data_confirmation.html +++ b/frappe/templates/emails/delete_data_confirmation.html @@ -7,6 +7,6 @@ {{ _("Confirm Request") }}

    - {% set verification_link = '{{ _("Verification Link") }}' %} + {% set verification_link = ' _("Verification Link") ' %} {{_("You can also copy-paste this {0} to your browser").format(verification_link) }} -

    \ No newline at end of file +

    diff --git a/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py b/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py index c3e0d22063..460f2cad53 100644 --- a/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py +++ b/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py @@ -131,6 +131,7 @@ class PersonalDataDeletionRequest(Document): "app_name": frappe.db.get_single_value("Website Settings", "app_name"), }, header=[_("Your account has been deleted"), "green"], + now=True ) def add_deletion_steps(self): From f33be7592f21de80c55a48824c7e94885a3198cb Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Wed, 1 Dec 2021 18:59:03 +0530 Subject: [PATCH 112/246] refactor: common JS controller for DocType and customize form --- frappe/core/doctype/doctype/doctype.js | 28 ++----------------- .../doctype/customize_form/customize_form.js | 1 + frappe/public/js/form.bundle.js | 2 +- frappe/public/js/frappe/doctype/index.js | 23 +++++++++++++++ 4 files changed, 27 insertions(+), 27 deletions(-) create mode 100644 frappe/public/js/frappe/doctype/index.js diff --git a/frappe/core/doctype/doctype/doctype.js b/frappe/core/doctype/doctype/doctype.js index 291fb237a9..1c52070063 100644 --- a/frappe/core/doctype/doctype/doctype.js +++ b/frappe/core/doctype/doctype/doctype.js @@ -1,16 +1,6 @@ // Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors // MIT License. See license.txt -// ------------- -// Menu Display -// ------------- - -// $(cur_frm.wrapper).on("grid-row-render", function(e, grid_row) { -// if(grid_row.doc && grid_row.doc.fieldtype=="Section Break") { -// $(grid_row.row).css({"font-weight": "bold"}); -// } -// }) - frappe.ui.form.on('DocType', { refresh: function(frm) { frm.set_query('role', 'permissions', function(doc) { @@ -130,22 +120,6 @@ frappe.ui.form.on('DocType', { frm.set_df_property('fields', 'reqd', frm.doc.autoname !== 'Prompt'); }, - - max_attachments: function(frm) { - if (!frm.doc.max_attachments) { - return; - } - const is_attach_field = (f) => ["Attach", "Attach Image"].includes(f.fieldtype); - const no_of_attach_fields = frm.doc.fields.filter(is_attach_field).length; - - if (no_of_attach_fields > frm.doc.max_attachments) { - frm.set_value("max_attachments", no_of_attach_fields); - const label = frm.get_docfield("max_attachments").label; - frappe.show_alert( - __("Number of attachment fields are more than {}, limit updated to {}.", [label, no_of_attach_fields])); - } - }, - }); frappe.ui.form.on("DocField", { @@ -239,3 +213,5 @@ frappe.ui.form.on("DocField", { frm.trigger("max_attachments"); } }); + +extend_cscript(cur_frm.cscript, new frappe.model.DocTypeController({frm: cur_frm})); diff --git a/frappe/custom/doctype/customize_form/customize_form.js b/frappe/custom/doctype/customize_form/customize_form.js index 4e00456f0d..8ca6e0e54e 100644 --- a/frappe/custom/doctype/customize_form/customize_form.js +++ b/frappe/custom/doctype/customize_form/customize_form.js @@ -332,3 +332,4 @@ frappe.customize_form.clear_locals_and_refresh = function(frm) { frm.refresh(); } +extend_cscript(cur_frm.cscript, new frappe.model.DocTypeController({frm: cur_frm})); diff --git a/frappe/public/js/form.bundle.js b/frappe/public/js/form.bundle.js index 5bed5c2cb8..2719535599 100644 --- a/frappe/public/js/form.bundle.js +++ b/frappe/public/js/form.bundle.js @@ -14,4 +14,4 @@ import "./frappe/form/controls/control.js"; import "./frappe/views/formview.js"; import "./frappe/form/form.js"; import "./frappe/meta_tag.js"; - +import "./frappe/doctype/" diff --git a/frappe/public/js/frappe/doctype/index.js b/frappe/public/js/frappe/doctype/index.js new file mode 100644 index 0000000000..9fe8957c60 --- /dev/null +++ b/frappe/public/js/frappe/doctype/index.js @@ -0,0 +1,23 @@ +frappe.provide("frappe.model"); + +/* + Common class for handling client side interactions that + apply to both DocType form and customize form. +*/ +frappe.model.DocTypeController = class DocTypeController extends frappe.ui.form.Controller { + + max_attachments() { + if (!this.frm.doc.max_attachments) { + return; + } + const is_attach_field = (f) => ["Attach", "Attach Image"].includes(f.fieldtype); + const no_of_attach_fields = this.frm.doc.fields.filter(is_attach_field).length; + + if (no_of_attach_fields > this.frm.doc.max_attachments) { + this.frm.set_value("max_attachments", no_of_attach_fields); + const label = this.frm.get_docfield("max_attachments").label; + frappe.show_alert( + __("Number of attachment fields are more than {}, limit updated to {}.", [label, no_of_attach_fields])); + } + } +} From abd6eafbad3321b71636940e20ee6549c12507aa Mon Sep 17 00:00:00 2001 From: Jannat Patel Date: Wed, 1 Dec 2021 19:44:19 +0530 Subject: [PATCH 113/246] fix: email content --- frappe/templates/emails/account_deletion_notification.html | 3 ++- frappe/templates/emails/delete_data_confirmation.html | 2 +- .../personal_data_deletion_request.py | 6 +++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/frappe/templates/emails/account_deletion_notification.html b/frappe/templates/emails/account_deletion_notification.html index 17d6aa3c93..2e8d8b2e3a 100644 --- a/frappe/templates/emails/account_deletion_notification.html +++ b/frappe/templates/emails/account_deletion_notification.html @@ -1,3 +1,4 @@

    {{_("Dear User,")}}

    -

    {{_("As per your request, your account and data on {0} associated with email {1} has been permanently deleted").format(app_name, email)}}.

    +
    +

    {{_("As per your request, your account and data on {0} associated with email {1} has been permanently deleted").format(host_name, email)}}.

    diff --git a/frappe/templates/emails/delete_data_confirmation.html b/frappe/templates/emails/delete_data_confirmation.html index a5794abf96..bd81f92f40 100644 --- a/frappe/templates/emails/delete_data_confirmation.html +++ b/frappe/templates/emails/delete_data_confirmation.html @@ -7,6 +7,6 @@ {{ _("Confirm Request") }}

    - {% set verification_link = ' _("Verification Link") ' %} + {% set verification_link = ' {0} '.format(_("Verification Link")) %} {{_("You can also copy-paste this {0} to your browser").format(verification_link) }}

    diff --git a/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py b/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py index 460f2cad53..ef3856ad25 100644 --- a/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py +++ b/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py @@ -80,7 +80,7 @@ class PersonalDataDeletionRequest(Document): args={ "email": self.email, "name": self.name, - "host_name": frappe.local.site, + "host_name": frappe.utils.get_url(), "link": url, }, header=[_("Confirm Deletion of Account"), "green"], @@ -128,7 +128,7 @@ class PersonalDataDeletionRequest(Document): template="account_deletion_notification", args={ "email": self.email, - "app_name": frappe.db.get_single_value("Website Settings", "app_name"), + "host_name": frappe.utils.get_url(), }, header=[_("Your account has been deleted"), "green"], now=True @@ -351,7 +351,7 @@ def confirm_deletion(email, name, host_name): return doc = frappe.get_doc("Personal Data Deletion Request", name) - host_name = frappe.local.site + host_name = frappe.utils.get_url() if doc.status == "Pending Verification": doc.status = "Pending Approval" From d26899662929d5d86cda26ab037f69a930599334 Mon Sep 17 00:00:00 2001 From: Jannat Patel Date: Wed, 1 Dec 2021 21:18:52 +0530 Subject: [PATCH 114/246] fix: test and semicolon --- .../test_personal_data_deletion_request.py | 2 +- .../web_form/request_to_delete_data/request_to_delete_data.js | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/frappe/website/doctype/personal_data_deletion_request/test_personal_data_deletion_request.py b/frappe/website/doctype/personal_data_deletion_request/test_personal_data_deletion_request.py index 240e37adef..8fc8f38512 100644 --- a/frappe/website/doctype/personal_data_deletion_request/test_personal_data_deletion_request.py +++ b/frappe/website/doctype/personal_data_deletion_request/test_personal_data_deletion_request.py @@ -24,7 +24,7 @@ class TestPersonalDataDeletionRequest(unittest.TestCase): email_queue = frappe.get_all("Email Queue", fields=["*"], order_by="creation desc", limit=1) self.assertEqual(self.delete_request.status, "Pending Verification") - self.assertTrue("Subject: Confirm Deletion of Data" in email_queue[0].message) + self.assertTrue("Subject: Confirm Deletion of Account" in email_queue[0].message) def test_anonymized_data(self): self.delete_request.status = "Pending Approval" diff --git a/frappe/website/web_form/request_to_delete_data/request_to_delete_data.js b/frappe/website/web_form/request_to_delete_data/request_to_delete_data.js index 9d19e36a05..9279302628 100644 --- a/frappe/website/web_form/request_to_delete_data/request_to_delete_data.js +++ b/frappe/website/web_form/request_to_delete_data/request_to_delete_data.js @@ -9,6 +9,6 @@ frappe.ready(function() { intro_wrapper.html(intro_wrapper.html() + sla_description); } } - }) - } + }); + }; }); From 9e6cefba6c280dbbe800b9428212627d55ff24ac Mon Sep 17 00:00:00 2001 From: Rucha Mahabal Date: Thu, 2 Dec 2021 10:01:20 +0530 Subject: [PATCH 115/246] fix: Add fallback option for time format when system defaults are not set --- frappe/public/js/frappe/form/formatters.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/form/formatters.js b/frappe/public/js/frappe/form/formatters.js index 0a3c032a40..7e1f3fed06 100644 --- a/frappe/public/js/frappe/form/formatters.js +++ b/frappe/public/js/frappe/form/formatters.js @@ -172,7 +172,7 @@ frappe.form.formatters = { m = m.tz(frappe.boot.sysdefaults.time_zone); } return m.format(frappe.boot.sysdefaults.date_format.toUpperCase() - + ' ' + frappe.boot.sysdefaults.time_format); + + ' ' + (frappe.boot.sysdefaults.time_format || 'HH:mm:ss')); } else { return ""; } From ba094e76d1fe0b4c093e4b4e0c16fce05c277687 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Thu, 2 Dec 2021 10:58:36 +0530 Subject: [PATCH 116/246] fix: Add `xcall` to Frappe's web bundle (backport #15146) (#15152) Co-authored-by: Omar Younis --- frappe/website/js/website.js | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/frappe/website/js/website.js b/frappe/website/js/website.js index 824b9ae3bf..079924d014 100644 --- a/frappe/website/js/website.js +++ b/frappe/website/js/website.js @@ -46,6 +46,20 @@ $.extend(frappe, { hide_message: function() { $('.message-overlay').remove(); }, + xcall: function(method, params) { + return new Promise((resolve, reject) => { + frappe.call({ + method: method, + args: params, + callback: (r) => { + resolve(r.message); + }, + error: (r) => { + reject(r.message); + } + }); + }); + }, call: function(opts) { // opts = {"method": "PYTHON MODULE STRING", "args": {}, "callback": function(r) {}} if (typeof arguments[0]==='string') { From ecca7f287ef48575dacd380c42dcd4c957156c31 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Thu, 2 Dec 2021 11:09:33 +0530 Subject: [PATCH 117/246] chore: better message for system updates (#15154) closes https://github.com/frappe/frappe/issues/10997 [skip ci] --- frappe/utils/response.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/utils/response.py b/frappe/utils/response.py index 3dc6fa0c80..104c48527c 100644 --- a/frappe/utils/response.py +++ b/frappe/utils/response.py @@ -220,6 +220,6 @@ def send_private_file(path): def handle_session_stopped(): from frappe.website.serve import get_response frappe.respond_as_web_page(_("Updating"), - _("Your system is being updated. Please refresh again after a few moments."), + _("The system is being updated. Please refresh again after a few moments."), http_status_code=503, indicator_color='orange', fullpage = True, primary_action=None) return get_response("message", http_status_code=503) From 2fe2db5f944157c5cc7fe163b9859e04ab63669c Mon Sep 17 00:00:00 2001 From: Jannat Patel Date: Thu, 2 Dec 2021 11:17:31 +0530 Subject: [PATCH 118/246] fix: form tour field in onboarding step --- .../desk/doctype/onboarding_step/onboarding_step.js | 11 +++++++++++ .../desk/doctype/onboarding_step/onboarding_step.json | 11 ++++++++++- frappe/public/js/frappe/widgets/onboarding_widget.js | 6 ++++-- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/frappe/desk/doctype/onboarding_step/onboarding_step.js b/frappe/desk/doctype/onboarding_step/onboarding_step.js index 793e044d98..3c9bbab9ac 100644 --- a/frappe/desk/doctype/onboarding_step/onboarding_step.js +++ b/frappe/desk/doctype/onboarding_step/onboarding_step.js @@ -2,6 +2,17 @@ // For license information, please see license.txt frappe.ui.form.on("Onboarding Step", { + + setup: function(frm) { + frm.set_query("form_tour", function() { + return { + filters: { + reference_doctype: frm.doc.reference_document + } + }; + }); + }, + refresh: function(frm) { frappe.boot.developer_mode && frm.set_intro( diff --git a/frappe/desk/doctype/onboarding_step/onboarding_step.json b/frappe/desk/doctype/onboarding_step/onboarding_step.json index f71e821f65..b5d7851eca 100644 --- a/frappe/desk/doctype/onboarding_step/onboarding_step.json +++ b/frappe/desk/doctype/onboarding_step/onboarding_step.json @@ -20,6 +20,7 @@ "reference_document", "show_full_form", "show_form_tour", + "form_tour", "is_single", "reference_report", "report_reference_doctype", @@ -206,13 +207,21 @@ "fieldname": "show_form_tour", "fieldtype": "Check", "label": "Show Form Tour" + }, + { + "depends_on": "show_form_tour", + "fieldname": "form_tour", + "fieldtype": "Link", + "label": "Form Tour", + "options": "Form Tour" } ], "links": [], - "modified": "2020-10-30 14:54:06.646513", + "modified": "2021-12-02 10:56:04.448580", "modified_by": "Administrator", "module": "Desk", "name": "Onboarding Step", + "naming_rule": "Set by user", "owner": "Administrator", "permissions": [ { diff --git a/frappe/public/js/frappe/widgets/onboarding_widget.js b/frappe/public/js/frappe/widgets/onboarding_widget.js index 110d617f73..0c06bd3203 100644 --- a/frappe/public/js/frappe/widgets/onboarding_widget.js +++ b/frappe/public/js/frappe/widgets/onboarding_widget.js @@ -234,8 +234,9 @@ export default class OnboardingWidget extends Widget { }, }); }; + const tour_name = step.form_tour; frm.tour - .init({ on_finish }) + .init({ tour_name, on_finish }) .then(() => frm.tour.start()); }; @@ -328,8 +329,9 @@ export default class OnboardingWidget extends Widget { this.mark_complete(step); }; }; + const tour_name = step.form_tour frm.tour - .init({ on_finish }) + .init({ tour_name, on_finish }) .then(() => frm.tour.start()); }; From a9b433e81944d626c6a4b18b84a9a619bcf19a20 Mon Sep 17 00:00:00 2001 From: Jannat Patel Date: Thu, 2 Dec 2021 11:42:49 +0530 Subject: [PATCH 119/246] fix: semicolon --- frappe/public/js/frappe/widgets/onboarding_widget.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/widgets/onboarding_widget.js b/frappe/public/js/frappe/widgets/onboarding_widget.js index 0c06bd3203..7d379d4531 100644 --- a/frappe/public/js/frappe/widgets/onboarding_widget.js +++ b/frappe/public/js/frappe/widgets/onboarding_widget.js @@ -329,7 +329,7 @@ export default class OnboardingWidget extends Widget { this.mark_complete(step); }; }; - const tour_name = step.form_tour + const tour_name = step.form_tour; frm.tour .init({ tour_name, on_finish }) .then(() => frm.tour.start()); From 66321b075ef06dd73f4d6244f7fa1e99e940d065 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Thu, 2 Dec 2021 16:19:23 +0530 Subject: [PATCH 120/246] fix: show "Button" field in user grid config --- frappe/public/js/frappe/form/grid_row.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/form/grid_row.js b/frappe/public/js/frappe/form/grid_row.js index 311a5b7a1e..5903a46683 100644 --- a/frappe/public/js/frappe/form/grid_row.js +++ b/frappe/public/js/frappe/form/grid_row.js @@ -368,8 +368,13 @@ export default class GridRow { prepare_columns_for_dialog(selected_fields) { let fields = []; + const blocked_fields = frappe.model.no_value_type; + const always_allow = ["Button"]; + + const show_field = (f) => always_allow.includes(f) || !blocked_fields.includes(f); + this.docfields.forEach(column => { - if (!column.hidden && !in_list(frappe.model.no_value_type, column.fieldtype)) { + if (!column.hidden && show_field(column.fieldtype)) { fields.push({ label: column.label, value: column.fieldname, From e0fadef11baf91ac7647538c13e56dd2e0b63dde Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Thu, 2 Dec 2021 16:19:38 +0530 Subject: [PATCH 121/246] feat: reset user grid config to default --- frappe/public/js/frappe/form/grid_row.js | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/frappe/public/js/frappe/form/grid_row.js b/frappe/public/js/frappe/form/grid_row.js index 5903a46683..dff5c216ee 100644 --- a/frappe/public/js/frappe/form/grid_row.js +++ b/frappe/public/js/frappe/form/grid_row.js @@ -291,6 +291,11 @@ export default class GridRow { this.grid_settings_dialog.hide(); }); + this.grid_settings_dialog.set_secondary_action_label(__("Reset to default")); + this.grid_settings_dialog.set_secondary_action(() => { + this.reset_user_settings_for_grid(); + this.grid_settings_dialog.hide(); + }); } setup_columns_for_dialog() { @@ -515,6 +520,14 @@ export default class GridRow { }); } + reset_user_settings_for_grid() { + frappe.model.user_settings.save(this.frm.doctype, 'GridView', null) + .then((r) => { + frappe.model.user_settings[this.frm.doctype] = r.message || r; + this.grid.reset_grid(); + }); + } + setup_columns() { this.focus_set = false; From 51c4738a04ae5e2aa7bc23bd0fb685f691174cb7 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Thu, 2 Dec 2021 18:01:43 +0530 Subject: [PATCH 122/246] fix(newsletter): use md_to_html instead of markdown because valid html is valid markdown and markdown method doesn't convert markdown if it encounters some html tags --- frappe/email/doctype/newsletter/newsletter.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/frappe/email/doctype/newsletter/newsletter.py b/frappe/email/doctype/newsletter/newsletter.py index 7c0e2dfe87..12fe160c9d 100644 --- a/frappe/email/doctype/newsletter/newsletter.py +++ b/frappe/email/doctype/newsletter/newsletter.py @@ -186,12 +186,13 @@ class Newsletter(WebsiteGenerator): frappe.db.auto_commit_on_many_writes = is_auto_commit_set def get_message(self) -> str: - if self.content_type == "HTML": - return frappe.render_template(self.message_html, {"doc": self.as_dict()}) + message = self.message if self.content_type == "Markdown": - return frappe.utils.markdown(self.message_md) - # fallback to Rich Text - return self.message + message = frappe.utils.md_to_html(self.message_md) + if self.content_type == "HTML": + message = self.message_html + + return frappe.render_template(message, {"doc": self.as_dict()}) def get_recipients(self) -> List[str]: """Get recipients from Email Group""" From 3bdca7192434646014b2abe2251124afdc1e9908 Mon Sep 17 00:00:00 2001 From: Rucha Mahabal Date: Fri, 3 Dec 2021 10:37:30 +0530 Subject: [PATCH 123/246] fix: delete Event Producer Last Update on trash event of Event Producer --- .../event_streaming/doctype/event_producer/event_producer.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/frappe/event_streaming/doctype/event_producer/event_producer.py b/frappe/event_streaming/doctype/event_producer/event_producer.py index 05771a89d3..a6c2a257fa 100644 --- a/frappe/event_streaming/doctype/event_producer/event_producer.py +++ b/frappe/event_streaming/doctype/event_producer/event_producer.py @@ -54,6 +54,11 @@ class EventProducer(Document): self.db_set('incoming_change', 0) self.reload() + def on_trash(self): + last_update = frappe.db.get_value('Event Producer Last Update', dict(event_producer=self.name)) + if last_update: + frappe.delete_doc('Event Producer Last Update', last_update) + def check_url(self): valid_url_schemes = ("http", "https") frappe.utils.validate_url(self.producer_url, throw=True, valid_schemes=valid_url_schemes) From aea21c2fe68bcd5ba8700f2cf62ff12881a63526 Mon Sep 17 00:00:00 2001 From: Shariq Ansari Date: Fri, 3 Dec 2021 12:48:36 +0530 Subject: [PATCH 124/246] fix: show unique records in list view --- frappe/public/js/frappe/list/base_list.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frappe/public/js/frappe/list/base_list.js b/frappe/public/js/frappe/list/base_list.js index 03e20ee6f5..f8d83019c1 100644 --- a/frappe/public/js/frappe/list/base_list.js +++ b/frappe/public/js/frappe/list/base_list.js @@ -492,6 +492,8 @@ frappe.views.BaseList = class BaseList { } else { this.data = this.data.concat(data); } + + this.data = this.data.uniqBy((d) => d.name); } freeze() { From 2ca687dac8e591afe2021c1e877819d53a2c31cb Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Fri, 3 Dec 2021 13:26:13 +0530 Subject: [PATCH 125/246] fix: Fix timezone conversions - User to System & vice-versa - Fixed infinite loop of setting datepicker value --- frappe/public/js/frappe/form/controls/date.js | 9 ++++++--- frappe/public/js/frappe/form/controls/datetime.js | 9 +++++++-- frappe/public/js/frappe/utils/datetime.js | 13 +++++++------ 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/frappe/public/js/frappe/form/controls/date.js b/frappe/public/js/frappe/form/controls/date.js index ba94531b0f..b964740ace 100644 --- a/frappe/public/js/frappe/form/controls/date.js +++ b/frappe/public/js/frappe/form/controls/date.js @@ -53,8 +53,6 @@ frappe.ui.form.ControlDate = class ControlDate extends frappe.ui.form.ControlDat let date_format = sysdefaults && sysdefaults.date_format ? sysdefaults.date_format : 'yyyy-mm-dd'; - let now_date = new Date(this.get_now_date()); - this.today_text = __("Today"); this.date_format = frappe.defaultDateFormat; this.datepicker_options = { @@ -62,7 +60,7 @@ frappe.ui.form.ControlDate = class ControlDate extends frappe.ui.form.ControlDat autoClose: true, todayButton: true, dateFormat: date_format, - startDate: now_date, + startDate: this.get_start_date(), keyboardNav: false, onSelect: () => { this.$input.trigger('change'); @@ -76,6 +74,11 @@ frappe.ui.form.ControlDate = class ControlDate extends frappe.ui.form.ControlDat } }; } + + get_start_date() { + return new Date(this.get_now_date()); + } + set_datepicker() { this.$input.datepicker(this.datepicker_options); this.datepicker = this.$input.data('datepicker'); diff --git a/frappe/public/js/frappe/form/controls/datetime.js b/frappe/public/js/frappe/form/controls/datetime.js index c7efc3e2c3..5d0ecb9fe7 100644 --- a/frappe/public/js/frappe/form/controls/datetime.js +++ b/frappe/public/js/frappe/form/controls/datetime.js @@ -8,9 +8,14 @@ frappe.ui.form.ControlDatetime = class ControlDatetime extends frappe.ui.form.Co } else if (value === "Today") { value = this.get_now_date(); } + value = this.format_for_input(value); + this.$input && this.$input.val(value); + this.datepicker.selectDate(frappe.datetime.user_to_obj(value)); + } - this.$input && this.$input.val(this.format_for_input(value)); - this.datepicker.selectDate(frappe.datetime.str_to_obj(value)); + get_start_date() { + let value = frappe.datetime.convert_to_user_tz(this.value); + return frappe.datetime.str_to_obj(value); } set_date_options() { super.set_date_options(); diff --git a/frappe/public/js/frappe/utils/datetime.js b/frappe/public/js/frappe/utils/datetime.js index 2fcfd75c5e..7bb6076b72 100644 --- a/frappe/public/js/frappe/utils/datetime.js +++ b/frappe/public/js/frappe/utils/datetime.js @@ -17,8 +17,7 @@ $.extend(frappe.datetime, { // system time zone and then convert the string to user time zone(from User doctype). let date_obj = null; if (frappe.boot.time_zone && frappe.boot.time_zone.system && frappe.boot.time_zone.user) { - date_obj = moment(date) - .tz(frappe.boot.time_zone.system) + date_obj = moment.tz(date, frappe.boot.time_zone.system) .clone() .tz(frappe.boot.time_zone.user); } else { @@ -144,14 +143,16 @@ $.extend(frappe.datetime, { let date_obj = moment(val, frappe.defaultTimeFormat); return date_obj.format(user_format); } else { - let date_obj = moment(val); + let date_obj = moment.tz(val, frappe.boot.time_zone.system); if (typeof val !== "string" || val.indexOf(" ") === -1) { user_format = user_date_fmt; } else { - date_obj = moment(val, "YYYY-MM-DD HH:mm:ss"); user_format = user_date_fmt + " " + user_time_fmt; } - return date_obj.tz(frappe.boot.time_zone.user).format(user_format); + return date_obj + .clone() + .tz(frappe.boot.time_zone.user) + .format(user_format); } }, @@ -207,7 +208,7 @@ $.extend(frappe.datetime, { _date: function(format, as_obj = false) { /** - * Whenever we are getting now_date/datetime, always make sure dates are fetched using usertime zone. + * Whenever we are getting now_date/datetime, always make sure dates are fetched using user time zone. * This is to make sure that time is as per user time zone set in User doctype, If a user had to change the timezone, * we will end up having multiple timezone by not honouring timezone in User doctype. * This will make sure that at any point we know which timezone the user if following and not have random timezone From 8a84ae4f0812d2dc5700b7e7011fa96141be204f Mon Sep 17 00:00:00 2001 From: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> Date: Fri, 3 Dec 2021 13:57:57 +0530 Subject: [PATCH 126/246] ci: Use node version 14 to avoid node-sass failure in patch testing build (#15176) --- .github/workflows/patch-mariadb-tests.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/patch-mariadb-tests.yml b/.github/workflows/patch-mariadb-tests.yml index 52fa987994..c8294886a0 100644 --- a/.github/workflows/patch-mariadb-tests.yml +++ b/.github/workflows/patch-mariadb-tests.yml @@ -32,6 +32,12 @@ jobs: with: python-version: '3.9' + - name: Setup Node + uses: actions/setup-node@v2 + with: + node-version: 14 + check-latest: true + - name: Check if build should be run id: check-build run: | From e862ae83da1e19abea889fcfe5e6366975201547 Mon Sep 17 00:00:00 2001 From: Aradhya Date: Thu, 2 Dec 2021 21:07:06 +0530 Subject: [PATCH 127/246] fix: fixed list of Field objects as fields in get_values tests: added test for list of field objects --- frappe/database/database.py | 4 ++-- frappe/tests/test_db.py | 28 ++++++++++++++++++++++++---- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index f489cea7de..64f09c1835 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -583,7 +583,7 @@ class Database(object): if not isinstance(fields, Criterion): for field in fields: - if "(" in field or " as " in field: + if "(" in str(field) or " as " in str(field): field_objects.append(PseudoColumn(field)) else: field_objects.append(field) @@ -842,7 +842,7 @@ class Database(object): cache_count = frappe.cache().get_value('doctype:count:{}'.format(dt)) if cache_count is not None: return cache_count - query = self.query.build_conditions(table=dt, filters=filters).select(Count("*")) + query = self.query.get_sql(table=dt, filters=filters, fields=Count("*")) if filters: count = self.sql(query, debug=debug)[0][0] return count diff --git a/frappe/tests/test_db.py b/frappe/tests/test_db.py index 60c8db6ab6..6501d753ff 100644 --- a/frappe/tests/test_db.py +++ b/frappe/tests/test_db.py @@ -24,10 +24,30 @@ class TestDB(unittest.TestCase): self.assertNotEqual(frappe.db.get_value("User", {"name": ["!=", "Guest"]}), "Guest") self.assertEqual(frappe.db.get_value("User", {"name": ["<", "Adn"]}), "Administrator") self.assertEqual(frappe.db.get_value("User", {"name": ["<=", "Administrator"]}), "Administrator") - self.assertEqual(frappe.db.get_value("User", {}, ["Max(name)"], order_by=None), frappe.db.sql("SELECT Max(name) FROM tabUser")[0][0]) - self.assertEqual(frappe.db.get_value("User", {}, "Min(name)", order_by=None), frappe.db.sql("SELECT Min(name) FROM tabUser")[0][0]) - self.assertIn("for update", frappe.db.get_value("User", Field("name") == "Administrator", for_update=True, run=False).lower()) - + self.assertEqual( + frappe.db.get_value("User", {}, ["Max(name)"], order_by=None), + frappe.db.sql("SELECT Max(name) FROM tabUser")[0][0], + ) + self.assertEqual( + frappe.db.get_value("User", {}, "Min(name)", order_by=None), + frappe.db.sql("SELECT Min(name) FROM tabUser")[0][0], + ) + self.assertIn( + "for update", + frappe.db.get_value( + "User", Field("name") == "Administrator", for_update=True, run=False + ).lower(), + ) + doctype = frappe.qb.DocType("User") + self.assertEqual( + frappe.qb.from_(doctype).select(doctype.name, doctype.email).run(), + frappe.db.get_values( + doctype, + filters={}, + fieldname=[doctype.name, doctype.email], + order_by=None, + ), + ) self.assertEqual(frappe.db.sql("""SELECT name FROM `tabUser` WHERE name > 's' ORDER BY MODIFIED DESC""")[0][0], frappe.db.get_value("User", {"name": [">", "s"]})) From 6f7d030e82ae3b2664f56a6fd9989cfbfc1f1648 Mon Sep 17 00:00:00 2001 From: Saqib Ansari Date: Fri, 3 Dec 2021 16:06:23 +0530 Subject: [PATCH 128/246] fix: IndexError while handling sql timeout error --- frappe/database/database.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index f489cea7de..ea56acff27 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -171,10 +171,10 @@ class Database(object): frappe.errprint(query) elif self.is_deadlocked(e): - raise frappe.QueryDeadlockError + raise frappe.QueryDeadlockError(e) elif self.is_timedout(e): - raise frappe.QueryTimeoutError + raise frappe.QueryTimeoutError(e) if ignore_ddl and (self.is_missing_column(e) or self.is_missing_table(e) or self.cant_drop_field_or_key(e)): pass From db951d2369d8ca3a13f9f5667706b4a002185983 Mon Sep 17 00:00:00 2001 From: Summayya Hashmani <58825865+sumaiya2908@users.noreply.github.com> Date: Fri, 3 Dec 2021 17:22:07 +0530 Subject: [PATCH 129/246] frefactor: padd separate padding (#15178) Co-authored-by: Summayya Co-authored-by: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> --- frappe/public/scss/website/index.scss | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frappe/public/scss/website/index.scss b/frappe/public/scss/website/index.scss index 9c84e99a5a..58f5ca79c6 100644 --- a/frappe/public/scss/website/index.scss +++ b/frappe/public/scss/website/index.scss @@ -266,7 +266,8 @@ h5.modal-title { .login-content.container { background-color: var(--fg-color); - padding: 45px 0px; + padding-bottom: 45px; + padding-top: 45px; box-shadow: var(--shadow-base); border-radius: var(--border-radius-md); max-width: 400px; From c5df17e3561c083a68f477ffe56403747aaf4d9c Mon Sep 17 00:00:00 2001 From: Raffael Meyer <14891507+barredterra@users.noreply.github.com> Date: Fri, 3 Dec 2021 15:34:41 +0100 Subject: [PATCH 130/246] fix: cannot uninstall app with virtual doctype (#15136) * Update installer.py * fix: Drop table only if it exists * revert: "Update installer.py" This reverts commit 0e8370ede8a9c2b1c0687e5c216ecf67566da0f5. Co-authored-by: Suraj Shetty --- frappe/installer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/installer.py b/frappe/installer.py index 9eed44ea15..cd6526c788 100755 --- a/frappe/installer.py +++ b/frappe/installer.py @@ -324,7 +324,7 @@ def _delete_doctypes(doctypes: List[str], dry_run: bool) -> None: print(f"* dropping Table for '{doctype}'...") if not dry_run: frappe.delete_doc("DocType", doctype, ignore_on_trash=True) - frappe.db.sql_ddl(f"drop table `tab{doctype}`") + frappe.db.sql_ddl(f"DROP TABLE IF EXISTS `tab{doctype}`") def post_install(rebuild_website=False): From 850cd54b890856af30403542289facdeec42fad4 Mon Sep 17 00:00:00 2001 From: barredterra <14891507+barredterra@users.noreply.github.com> Date: Fri, 3 Dec 2021 17:07:16 +0100 Subject: [PATCH 131/246] refactor: module profile --- .../doctype/module_profile/module_profile.js | 17 +- .../module_profile/module_profile.json | 10 +- .../doctype/role_profile/role_profile.json | 227 +++++------------- frappe/core/doctype/user/user.js | 4 +- frappe/public/js/frappe/module_editor.js | 76 +++--- 5 files changed, 133 insertions(+), 201 deletions(-) diff --git a/frappe/core/doctype/module_profile/module_profile.js b/frappe/core/doctype/module_profile/module_profile.js index 9c92042dda..57b563157c 100644 --- a/frappe/core/doctype/module_profile/module_profile.js +++ b/frappe/core/doctype/module_profile/module_profile.js @@ -1,19 +1,24 @@ // Copyright (c) 2020, Frappe Technologies and contributors // For license information, please see license.txt -frappe.ui.form.on('Module Profile', { - refresh: function(frm) { +frappe.ui.form.on("Module Profile", { + refresh: function (frm) { + debugger; if (has_common(frappe.user_roles, ["Administrator", "System Manager"])) { if (!frm.module_editor && frm.doc.__onload && frm.doc.__onload.all_modules) { - let module_area = $('
    ') - .appendTo(frm.fields_dict.module_html.wrapper); - + const module_area = $(frm.fields_dict.module_html.wrapper); frm.module_editor = new frappe.ModuleEditor(frm, module_area); } } if (frm.module_editor) { - frm.module_editor.refresh(); + frm.module_editor.show(); + } + }, + + validate: function (frm) { + if (frm.module_editor) { + frm.module_editor.set_modules_in_table(); } } }); diff --git a/frappe/core/doctype/module_profile/module_profile.json b/frappe/core/doctype/module_profile/module_profile.json index 0e4e56962e..32bc757427 100644 --- a/frappe/core/doctype/module_profile/module_profile.json +++ b/frappe/core/doctype/module_profile/module_profile.json @@ -34,11 +34,17 @@ } ], "index_web_pages_for_search": 1, - "links": [], - "modified": "2021-01-03 15:36:52.622696", + "links": [ + { + "link_doctype": "User", + "link_fieldname": "module_profile" + } + ], + "modified": "2021-12-03 15:47:21.296443", "modified_by": "Administrator", "module": "Core", "name": "Module Profile", + "naming_rule": "By fieldname", "owner": "Administrator", "permissions": [ { diff --git a/frappe/core/doctype/role_profile/role_profile.json b/frappe/core/doctype/role_profile/role_profile.json index 4b3f35aa57..7cd60a16d1 100644 --- a/frappe/core/doctype/role_profile/role_profile.json +++ b/frappe/core/doctype/role_profile/role_profile.json @@ -1,175 +1,80 @@ { - "allow_copy": 0, - "allow_guest_to_view": 0, - "allow_import": 0, - "allow_rename": 0, - "autoname": "role_profile", - "beta": 0, - "creation": "2017-08-31 04:16:38.764465", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", + "actions": [], + "autoname": "role_profile", + "creation": "2017-08-31 04:16:38.764465", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "role_profile", + "roles_html", + "roles" + ], "fields": [ { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "role_profile", - "fieldtype": "Data", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 1, - "in_standard_filter": 0, - "label": "Role Name", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, + "fieldname": "role_profile", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Role Name", + "reqd": 1, "unique": 1 - }, + }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "roles_html", - "fieldtype": "HTML", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Roles HTML", - "length": 0, - "no_copy": 0, - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "roles_html", + "fieldtype": "HTML", + "label": "Roles HTML", + "read_only": 1 + }, { - "allow_bulk_edit": 0, - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "roles", - "fieldtype": "Table", - "hidden": 1, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_global_search": 0, - "in_list_view": 0, - "in_standard_filter": 0, - "label": "Roles Assigned", - "length": 0, - "no_copy": 0, - "options": "Has Role", - "permlevel": 1, - "precision": "", - "print_hide": 1, - "print_hide_if_no_value": 0, - "read_only": 1, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 0, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "fieldname": "roles", + "fieldtype": "Table", + "hidden": 1, + "label": "Roles Assigned", + "options": "Has Role", + "permlevel": 1, + "print_hide": 1, + "read_only": 1 } - ], - "has_web_view": 0, - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - "is_submittable": 0, - "issingle": 0, - "istable": 0, - "max_attachments": 0, - "modified": "2017-10-17 11:05:11.183066", - "modified_by": "Administrator", - "module": "Core", - "name": "Role Profile", - "name_case": "", - "owner": "Administrator", + ], + "links": [ + { + "link_doctype": "User", + "link_fieldname": "role_profile_name" + } + ], + "modified": "2021-12-03 15:45:45.270963", + "modified_by": "Administrator", + "module": "Core", + "name": "Role Profile", + "naming_rule": "Expression (old style)", + "owner": "Administrator", "permissions": [ { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 0, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "set_user_permissions": 0, - "share": 1, - "submit": 0, + "create": 1, + "delete": 1, + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1, "write": 1 - }, + }, { - "amend": 0, - "apply_user_permissions": 0, - "cancel": 0, - "create": 0, - "delete": 0, - "email": 1, - "export": 1, - "if_owner": 0, - "import": 0, - "permlevel": 1, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "set_user_permissions": 0, - "share": 1, - "submit": 0, + "email": 1, + "export": 1, + "permlevel": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1, "write": 1 } - ], - "quick_entry": 0, - "read_only": 0, - "read_only_onload": 0, - "show_name_in_global_search": 0, - "sort_field": "modified", - "sort_order": "DESC", - "title_field": "role_profile", - "track_changes": 1, - "track_seen": 0 + ], + "sort_field": "modified", + "sort_order": "DESC", + "title_field": "role_profile", + "track_changes": 1 } \ No newline at end of file diff --git a/frappe/core/doctype/user/user.js b/frappe/core/doctype/user/user.js index 2ce7413aa7..5b3a1affd9 100644 --- a/frappe/core/doctype/user/user.js +++ b/frappe/core/doctype/user/user.js @@ -50,7 +50,7 @@ frappe.ui.form.on('User', { let d = frm.add_child("block_modules"); d.module = v.module; }); - frm.module_editor && frm.module_editor.refresh(); + frm.module_editor && frm.module_editor.show(); } }); } @@ -180,7 +180,7 @@ frappe.ui.form.on('User', { frm.roles_editor.show(); } - frm.module_editor && frm.module_editor.refresh(); + frm.module_editor && frm.module_editor.show(); if(frappe.session.user==doc.name) { // update display settings diff --git a/frappe/public/js/frappe/module_editor.js b/frappe/public/js/frappe/module_editor.js index 5e2ca4bc83..ff0cfc2426 100644 --- a/frappe/public/js/frappe/module_editor.js +++ b/frappe/public/js/frappe/module_editor.js @@ -1,38 +1,54 @@ frappe.ModuleEditor = class ModuleEditor { constructor(frm, wrapper) { - this.wrapper = $('
    ').appendTo(wrapper); this.frm = frm; - this.make(); - } - make() { - var me = this; - this.frm.doc.__onload.all_modules.forEach(function(m) { - $(repl('
    \ -
    ', {module: m})).appendTo(me.wrapper); - }); - this.bind(); - } - refresh() { - var me = this; - this.wrapper.find(".block-module-check").prop("checked", true); - $.each(this.frm.doc.block_modules, function(i, d) { - me.wrapper.find(".block-module-check[data-module='"+ d.module +"']").prop("checked", false); + this.wrapper = wrapper; + const block_modules = this.frm.doc.block_modules.map(row => row.module); + this.multicheck = frappe.ui.form.make_control({ + parent: wrapper, + df: { + fieldname: "block_modules", + fieldtype: "MultiCheck", + select_all: true, + columns: 3, + get_data: () => { + return this.frm.doc.__onload.all_modules.map(module => { + return { + label: __(module), + value: module, + checked: !block_modules.includes(module), + }; + }); + }, + on_change: () => { + this.set_modules_in_table(); + this.frm.dirty(); + } + }, + render_input: true }); } - bind() { - var me = this; - this.wrapper.on("change", ".block-module-check", function() { - var module = $(this).attr('data-module'); - if ($(this).prop("checked")) { - // remove from block_modules - me.frm.doc.block_modules = $.map(me.frm.doc.block_modules || [], function(d) { - if (d.module != module) { - return d; - } - }); - } else { - me.frm.add_child("block_modules", {"module": module}); + + show() { + const block_modules = this.frm.doc.block_modules.map(row => row.module); + const all_modules = this.frm.doc.__onload.all_modules; + this.multicheck.selected_options = all_modules.filter(m => !block_modules.includes(m)); + this.multicheck.refresh_input(); + } + + set_modules_in_table() { + let block_modules = this.frm.doc.block_modules || []; + let unchecked_options = this.multicheck.get_unchecked_options(); + + block_modules.map(module_doc => { + if (!unchecked_options.includes(module_doc.module)) { + frappe.model.clear_doc(module_doc.doctype, module_doc.name); + } + }); + + unchecked_options.map(module => { + if (!block_modules.find(d => d.module === module)) { + let module_doc = frappe.model.add_child(this.frm.doc, "Block Module", "block_modules"); + module_doc.module = module; } }); } From 504f8743c9ac818f1cacc7c1424340ebcc24c5a9 Mon Sep 17 00:00:00 2001 From: barredterra <14891507+barredterra@users.noreply.github.com> Date: Fri, 3 Dec 2021 17:24:57 +0100 Subject: [PATCH 132/246] fix: remove debugger --- frappe/core/doctype/module_profile/module_profile.js | 1 - 1 file changed, 1 deletion(-) diff --git a/frappe/core/doctype/module_profile/module_profile.js b/frappe/core/doctype/module_profile/module_profile.js index 57b563157c..3714d31ade 100644 --- a/frappe/core/doctype/module_profile/module_profile.js +++ b/frappe/core/doctype/module_profile/module_profile.js @@ -3,7 +3,6 @@ frappe.ui.form.on("Module Profile", { refresh: function (frm) { - debugger; if (has_common(frappe.user_roles, ["Administrator", "System Manager"])) { if (!frm.module_editor && frm.doc.__onload && frm.doc.__onload.all_modules) { const module_area = $(frm.fields_dict.module_html.wrapper); From 8ead1d9c487b5f7f3e86016b0404a700cc11ba8c Mon Sep 17 00:00:00 2001 From: this-gavagai Date: Mon, 6 Dec 2021 13:13:43 +0545 Subject: [PATCH 133/246] fix: Clarified docstatus transition exceptions (#15194) * [fix] Clarified docstatus transition exceptions Exceptions issued by the document.py `check_docstatus_transition` method are potentially very misleading. In cases where an invalid docstatus is used, users receive an confusing exception stating "Cannot change docstatus from 0 to 2" or "Cannot change docstatus from 1 to 0". This PR adds an additional exception message when an invalid docstatus is used. * fix: Clarified docstatus transition exceptions Added additional clarifications to exception messages --- frappe/model/document.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/frappe/model/document.py b/frappe/model/document.py index fcdadf48e6..2260406125 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -750,8 +750,10 @@ class Document(BaseDocument): elif self.docstatus==1: self._action = "submit" self.check_permission("submit") + elif self.docstatus==2: + raise frappe.DocstatusTransitionError(_("Cannot change docstatus from 0 (Draft) to 2 (Cancelled)")) else: - raise frappe.DocstatusTransitionError(_("Cannot change docstatus from 0 to 2")) + raise frappe.ValidationError(_("Invalid docstatus"), self.docstatus) elif docstatus==1: if self.docstatus==1: @@ -760,8 +762,10 @@ class Document(BaseDocument): elif self.docstatus==2: self._action = "cancel" self.check_permission("cancel") + elif self.docstatus==0: + raise frappe.DocstatusTransitionError(_("Cannot change docstatus from 1 (Submitted) to 0 (Draft)")) else: - raise frappe.DocstatusTransitionError(_("Cannot change docstatus from 1 to 0")) + raise frappe.ValidationError(_("Invalid docstatus"), self.docstatus) elif docstatus==2: raise frappe.ValidationError(_("Cannot edit cancelled document")) From 3243fb2083593fdc42b59f609935a758ae399bad Mon Sep 17 00:00:00 2001 From: Aradhya Date: Mon, 6 Dec 2021 13:04:27 +0530 Subject: [PATCH 134/246] fix: misc fixes --- frappe/database/database.py | 17 ++++++++--------- frappe/database/query.py | 5 ++--- frappe/query_builder/builder.py | 33 +++++++++++---------------------- 3 files changed, 21 insertions(+), 34 deletions(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index 6a4e781b44..0f325a746e 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -511,14 +511,10 @@ class Database(object): # Get coulmn and value of the single doctype Accounts Settings account_settings = frappe.db.get_singles_dict("Accounts Settings") """ - result = self.sql(""" - SELECT field, value - FROM `tabSingles` - WHERE doctype = %s - """, doctype) - + result = self.query.get_sql( + "Singles", filters={"doctype": doctype}, fields=["field", "value"] + ).run() dict_ = frappe._dict(result) - return dict_ @staticmethod @@ -547,8 +543,11 @@ class Database(object): if fieldname in self.value_cache[doctype]: return self.value_cache[doctype][fieldname] - val = self.sql("""select `value` from - `tabSingles` where `doctype`=%s and `field`=%s""", (doctype, fieldname)) + val = self.query.get_sql( + table="Singles", + filters={"doctype": doctype, "field": fieldname}, + fields="value", + ).run() val = val[0][0] if val else None df = frappe.get_meta(doctype).get_field(fieldname) diff --git a/frappe/database/query.py b/frappe/database/query.py index 69328cb206..6d2be5fa25 100644 --- a/frappe/database/query.py +++ b/frappe/database/query.py @@ -286,14 +286,13 @@ class Query: ): criterion = self.build_conditions(table, filters, **kwargs) if isinstance(fields, (list, tuple)): - query = criterion.select(*kwargs.get("field_objects")) + query = criterion.select(*kwargs.get("field_objects", fields)) elif isinstance(fields, Criterion): query = criterion.select(fields) else: - if fields=="*": - query = criterion.select(fields) + query = criterion.select(fields) return query diff --git a/frappe/query_builder/builder.py b/frappe/query_builder/builder.py index 630cfea222..a65d50fdeb 100644 --- a/frappe/query_builder/builder.py +++ b/frappe/query_builder/builder.py @@ -18,16 +18,6 @@ class Base: table_name = get_table_name(table_name) return Table(table_name, *args, **kwargs) - -class MariaDB(Base, MySQLQuery): - Field = terms.Field - - @classmethod - def from_(cls, table, *args, **kwargs): - if isinstance(table, str): - table = cls.DocType(table) - return super().from_(table, *args, **kwargs) - @classmethod def into(cls, table, *args, **kwargs): if isinstance(table, str): @@ -40,6 +30,17 @@ class MariaDB(Base, MySQLQuery): table = cls.DocType(table) return super().update(table, *args, **kwargs) + +class MariaDB(Base, MySQLQuery): + Field = terms.Field + + @classmethod + def from_(cls, table, *args, **kwargs): + if isinstance(table, str): + table = cls.DocType(table) + return super().from_(table, *args, **kwargs) + + class Postgres(Base, PostgreSQLQuery): field_translation = {"table_name": "relname", "table_rows": "n_tup_ins"} schema_translation = {"tables": "pg_stat_all_tables"} @@ -69,15 +70,3 @@ class Postgres(Base, PostgreSQLQuery): table = cls.DocType(table) return super().from_(table, *args, **kwargs) - - @classmethod - def into(cls, table, *args, **kwargs): - if isinstance(table, str): - table = cls.DocType(table) - return super().into(table, *args, **kwargs) - - @classmethod - def update(cls, table, *args, **kwargs): - if isinstance(table, str): - table = cls.DocType(table) - return super().update(table, *args, **kwargs) \ No newline at end of file From a574c1ba88cd76d74065bfbcbf2c9c78dd208d0b Mon Sep 17 00:00:00 2001 From: saxenabhishek Date: Thu, 2 Dec 2021 23:20:40 +0530 Subject: [PATCH 135/246] chore: patching ValueWrapper --- frappe/query_builder/terms.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 frappe/query_builder/terms.py diff --git a/frappe/query_builder/terms.py b/frappe/query_builder/terms.py new file mode 100644 index 0000000000..c221dcb28e --- /dev/null +++ b/frappe/query_builder/terms.py @@ -0,0 +1,27 @@ +from typing import Any, Optional, Dict +from pypika.terms import ValueWrapper +from pypika.utils import format_alias_sql + + +class NamedParameterWrapper(): + def __init__(self, parameters: Dict[str, Any]): + self.parameters = parameters + + def update_parameters(self, param_key: Any, param_value: Any, **kwargs): + self.parameters[param_key[1:]] = param_value + + def get_sql(self, **kwargs): + return f'@param{len(self.parameters) + 1}' + + +class ParameterizedValueWrapper(ValueWrapper): + def get_sql(self, quote_char: Optional[str] = None, secondary_quote_char: str = "'", param_wrapper= None, **kwargs: Any) -> str: + if param_wrapper is None: + sql = self.get_value_sql(quote_char=quote_char, secondary_quote_char=secondary_quote_char, **kwargs) + return format_alias_sql(sql, self.alias, quote_char=quote_char, **kwargs) + else: + value_sql = self.get_value_sql(quote_char=quote_char, **kwargs) + param_sql = param_wrapper.get_sql(**kwargs) + param_wrapper.update_parameters(param_key=param_sql, param_value=value_sql, **kwargs) + + return format_alias_sql(param_sql, self.alias, quote_char=quote_char, **kwargs) \ No newline at end of file From 9fdacedfc80889c81c4887c2f2f2581e5d0d56f6 Mon Sep 17 00:00:00 2001 From: saxenabhishek Date: Thu, 2 Dec 2021 23:23:25 +0530 Subject: [PATCH 136/246] feat: sanitise frappe.qb --- frappe/query_builder/__init__.py | 5 +++++ frappe/query_builder/terms.py | 8 ++++---- frappe/query_builder/utils.py | 14 ++++++++++---- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/frappe/query_builder/__init__.py b/frappe/query_builder/__init__.py index 9c7432142f..06d499678f 100644 --- a/frappe/query_builder/__init__.py +++ b/frappe/query_builder/__init__.py @@ -1,2 +1,7 @@ +from frappe.query_builder.terms import ParameterizedValueWrapper +import pypika + +pypika.terms.ValueWrapper = ParameterizedValueWrapper + from pypika import * from frappe.query_builder.utils import Column, DocType, get_query_builder, patch_query_execute, patch_query_aggregation diff --git a/frappe/query_builder/terms.py b/frappe/query_builder/terms.py index c221dcb28e..c09d9595fb 100644 --- a/frappe/query_builder/terms.py +++ b/frappe/query_builder/terms.py @@ -1,4 +1,4 @@ -from typing import Any, Optional, Dict +from typing import Any, List, Optional, Dict from pypika.terms import ValueWrapper from pypika.utils import format_alias_sql @@ -8,10 +8,10 @@ class NamedParameterWrapper(): self.parameters = parameters def update_parameters(self, param_key: Any, param_value: Any, **kwargs): - self.parameters[param_key[1:]] = param_value + self.parameters[param_key[2:-2]] = param_value def get_sql(self, **kwargs): - return f'@param{len(self.parameters) + 1}' + return f'%(param{len(self.parameters) + 1})s' class ParameterizedValueWrapper(ValueWrapper): @@ -20,7 +20,7 @@ class ParameterizedValueWrapper(ValueWrapper): sql = self.get_value_sql(quote_char=quote_char, secondary_quote_char=secondary_quote_char, **kwargs) return format_alias_sql(sql, self.alias, quote_char=quote_char, **kwargs) else: - value_sql = self.get_value_sql(quote_char=quote_char, **kwargs) + value_sql = self.get_value_sql(quote_char=quote_char, **kwargs) if not isinstance(self.value,int) else self.value param_sql = param_wrapper.get_sql(**kwargs) param_wrapper.update_parameters(param_key=param_sql, param_value=value_sql, **kwargs) diff --git a/frappe/query_builder/utils.py b/frappe/query_builder/utils.py index a7f52df012..7922825725 100644 --- a/frappe/query_builder/utils.py +++ b/frappe/query_builder/utils.py @@ -10,6 +10,7 @@ import frappe from .builder import MariaDB, Postgres from pypika.terms import PseudoColumn +from frappe.query_builder.terms import NamedParameterWrapper class db_type_is(Enum): MARIADB = "mariadb" @@ -53,12 +54,16 @@ def patch_query_execute(): This excludes the use of `frappe.db.sql` method while executing the query object """ - def execute_query(query, *args, **kwargs): - query = str(query) + query, params = prepare_query(query) + return frappe.db.sql(query, params, *args, **kwargs) + + def prepare_query(query): + params = {} + query = query.get_sql(param_wrapper = NamedParameterWrapper(params)) if frappe.flags.in_safe_exec and not query.lower().strip().startswith("select"): raise frappe.PermissionError('Only SELECT SQL allowed in scripting') - return frappe.db.sql(query, *args, **kwargs) + return query, params query_class = get_attr(str(frappe.qb).split("'")[1]) builder_class = get_type_hints(query_class._builder).get('return') @@ -67,6 +72,7 @@ def patch_query_execute(): raise BuilderIdentificationFailed builder_class.run = execute_query + builder_class.walk = prepare_query def patch_query_aggregation(): @@ -77,4 +83,4 @@ def patch_query_aggregation(): frappe.qb.max = _max frappe.qb.min = _min frappe.qb.avg = _avg - frappe.qb.sum = _sum \ No newline at end of file + frappe.qb.sum = _sum From 6120b4b3c1dbf17bc783be16ba41a5c73d5c5df1 Mon Sep 17 00:00:00 2001 From: saxenabhishek Date: Sat, 4 Dec 2021 20:12:48 +0530 Subject: [PATCH 137/246] fix: extend named parameters to frappe.qb.function --- frappe/query_builder/__init__.py | 3 ++- frappe/query_builder/terms.py | 28 +++++++++++++++++++++++++--- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/frappe/query_builder/__init__.py b/frappe/query_builder/__init__.py index 06d499678f..bf7be84c51 100644 --- a/frappe/query_builder/__init__.py +++ b/frappe/query_builder/__init__.py @@ -1,7 +1,8 @@ -from frappe.query_builder.terms import ParameterizedValueWrapper +from frappe.query_builder.terms import ParameterizedValueWrapper, ParameterizedFunction import pypika pypika.terms.ValueWrapper = ParameterizedValueWrapper +pypika.terms.Function = ParameterizedFunction from pypika import * from frappe.query_builder.utils import Column, DocType, get_query_builder, patch_query_execute, patch_query_aggregation diff --git a/frappe/query_builder/terms.py b/frappe/query_builder/terms.py index c09d9595fb..2032cd8497 100644 --- a/frappe/query_builder/terms.py +++ b/frappe/query_builder/terms.py @@ -1,5 +1,6 @@ -from typing import Any, List, Optional, Dict -from pypika.terms import ValueWrapper +from typing import Any, Dict, Optional + +from pypika.terms import Function, ValueWrapper from pypika.utils import format_alias_sql @@ -23,5 +24,26 @@ class ParameterizedValueWrapper(ValueWrapper): value_sql = self.get_value_sql(quote_char=quote_char, **kwargs) if not isinstance(self.value,int) else self.value param_sql = param_wrapper.get_sql(**kwargs) param_wrapper.update_parameters(param_key=param_sql, param_value=value_sql, **kwargs) + return format_alias_sql(param_sql, self.alias, quote_char=quote_char, **kwargs) - return format_alias_sql(param_sql, self.alias, quote_char=quote_char, **kwargs) \ No newline at end of file + +class ParameterizedFunction(Function): + def get_sql(self, **kwargs: Any) -> str: + with_alias = kwargs.pop("with_alias", False) + with_namespace = kwargs.pop("with_namespace", False) + quote_char = kwargs.pop("quote_char", None) + dialect = kwargs.pop("dialect", None) + param_wrapper = kwargs.pop("param_wrapper", None) + + function_sql = self.get_function_sql(with_namespace=with_namespace, quote_char=quote_char, param_wrapper=param_wrapper, dialect=dialect) + + if self.schema is not None: + function_sql = "{schema}.{function}".format( + schema=self.schema.get_sql(quote_char=quote_char, dialect=dialect, **kwargs), + function=function_sql, + ) + + if with_alias: + return format_alias_sql(function_sql, self.alias, quote_char=quote_char, **kwargs) + + return function_sql From aa855afe089d209d2a4796c8497d21ed5d12578a Mon Sep 17 00:00:00 2001 From: saxenabhishek Date: Mon, 6 Dec 2021 12:12:56 +0530 Subject: [PATCH 138/246] test: test for patches through walk --- frappe/tests/test_query_builder.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/frappe/tests/test_query_builder.py b/frappe/tests/test_query_builder.py index 7a0935a63b..1d63d2041c 100644 --- a/frappe/tests/test_query_builder.py +++ b/frappe/tests/test_query_builder.py @@ -2,7 +2,7 @@ import unittest from typing import Callable import frappe -from frappe.query_builder.functions import GroupConcat, Match +from frappe.query_builder.functions import Coalesce, GroupConcat, Match from frappe.query_builder.utils import db_type_is @@ -49,6 +49,25 @@ class TestBuilderBase(object): self.assertIsInstance(query.run, Callable) self.assertIsInstance(data, list) + def test_walk(self): + DocType = frappe.qb.DocType('DocType') + query = ( + frappe.qb.from_(DocType) + .select(DocType.name) + .where((DocType.owner == "Administrator' --") + & (Coalesce(DocType.search_fields == "subject")) + ) + ) + self.assertTrue("walk" in dir(query)) + query, params = query.walk() + + self.assertIn("%(param1)s", query) + self.assertIn("%(param2)s", query) + self.assertIn("param1",params) + self.assertEqual(params["param1"],"Administrator' --") + self.assertEqual(params["param2"],"subject") + + @run_only_if(db_type_is.MARIADB) class TestBuilderMaria(unittest.TestCase, TestBuilderBase): def test_adding_tabs_in_from(self): @@ -59,7 +78,6 @@ class TestBuilderMaria(unittest.TestCase, TestBuilderBase): "SELECT * FROM `__Auth`", frappe.qb.from_("__Auth").select("*").get_sql() ) - @run_only_if(db_type_is.POSTGRES) class TestBuilderPostgres(unittest.TestCase, TestBuilderBase): def test_adding_tabs_in_from(self): From 8340639afdeab4414ae54af67ef0b4e23d5a8a64 Mon Sep 17 00:00:00 2001 From: Rucha Mahabal Date: Mon, 6 Dec 2021 14:51:37 +0530 Subject: [PATCH 139/246] fix: Allow Fetch From for a different link field of the same DocType --- frappe/core/doctype/doctype/doctype.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/frappe/core/doctype/doctype/doctype.js b/frappe/core/doctype/doctype/doctype.js index 1c52070063..b907ebc0bc 100644 --- a/frappe/core/doctype/doctype/doctype.js +++ b/frappe/core/doctype/doctype/doctype.js @@ -143,11 +143,10 @@ frappe.ui.form.on("DocField", { curr_value.doctype = doctype; curr_value.fieldname = fieldname; } - let curr_df_link_doctype = row.fieldtype == "Link" ? row.options : null; let doctypes = frm.doc.fields .filter(df => df.fieldtype == "Link") - .filter(df => df.options && df.options != curr_df_link_doctype) + .filter(df => df.options && df.fieldname != row.fieldname) .map(df => ({ label: `${df.options} (${df.fieldname})`, value: df.fieldname From da4160e2dd0c9c47e207b51689d9a276221b50af Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Fri, 3 Dec 2021 18:22:25 +0530 Subject: [PATCH 140/246] fix: Newsletter enhancements and fixes - Organize fields into sections - Buttons for Send now and Scheduled sending - Buttons to Send test email and to Check broken links - Remove Test section --- frappe/email/doctype/newsletter/newsletter.js | 144 ++++++++++++------ .../email/doctype/newsletter/newsletter.json | 110 +++++++------ frappe/email/doctype/newsletter/newsletter.py | 31 +++- .../newsletter/templates/newsletter.html | 4 +- 4 files changed, 194 insertions(+), 95 deletions(-) diff --git a/frappe/email/doctype/newsletter/newsletter.js b/frappe/email/doctype/newsletter/newsletter.js index 3277d8e9ee..a7cbcf702a 100644 --- a/frappe/email/doctype/newsletter/newsletter.js +++ b/frappe/email/doctype/newsletter/newsletter.js @@ -4,69 +4,123 @@ frappe.ui.form.on('Newsletter', { refresh(frm) { let doc = frm.doc; - if (!doc.__islocal && !cint(doc.email_sent) && !doc.__unsaved - && in_list(frappe.boot.user.can_write, doc.doctype)) { - frm.add_custom_button(__('Send Now'), function() { - frappe.confirm(__("Do you really want to send this email newsletter?"), function() { - frm.call('send_emails').then(() => { - frm.refresh(); - }); + let can_write = in_list(frappe.boot.user.can_write, doc.doctype); + if (!frm.is_new() && !frm.is_dirty() && !doc.email_sent && can_write) { + frm.add_custom_button(__('Send a test email'), () => { + frm.events.send_test_email(frm); + }, __('Preview')); + + frm.add_custom_button(__('Check broken links'), () => { + frm.call('find_broken_links').then(r => { + let links = r.message; + if (links) { + let html = '
      ' + links.map(link => `
    • ${link}
      • `).join('') + '
    '; + frappe.msgprint({ + title: __("Broken Links"), + message: __("Following links are broken in the email content: {0}", [html]), + indicator: "red" + }) + } else { + frappe.msgprint({ + title: _("No Broken Links"), + message: _("No broken links found in the email content"), + indicator: "green" + }) + } + }) + }, __('Preview')); + + frm.add_custom_button(__('Send now'), () => { + frappe.confirm(__("Do you really want to send this email newsletter?"), function () { + frm.call('send_emails').then(() => frm.refresh()); }); - }, "fa fa-play", "btn-success"); + }, __('Send')); + + frm.add_custom_button(__('Schedule sending'), () => { + frm.events.schedule_send_dialog(frm); + }, __('Send')); } frm.events.setup_dashboard(frm); - if (doc.__islocal && !doc.send_from) { + if (frm.is_new() && !doc.sender_email) { let { fullname, email } = frappe.user_info(doc.owner); - frm.set_value('send_from', `${fullname} <${email}>`); + frm.set_value('sender_email', email); + frm.set_value('sender_name', fullname); } }, - onload_post_render(frm) { - frm.trigger('setup_schedule_send'); - }, - - setup_schedule_send(frm) { - let today = new Date(); - - // setting datepicker options to set min date & min time - today.setHours(today.getHours() + 1 ); - frm.get_field('schedule_send').$input.datepicker({ - maxMinutes: 0, - minDate: today, - timeFormat: 'hh:00:00', - onSelect: function (fd, d, picker) { - if (!d) return; - var date = d.toDateString(); - if (date === today.toDateString()) { - picker.update({ - minHours: (today.getHours() + 1) - }); - } else { - picker.update({ - minHours: 0 - }); - } - frm.get_field('schedule_send').$input.trigger('change'); + schedule_send_dialog(frm) { + let hours = frappe.utils.range(24); + let time_slots = hours.map(hour => { + return `${(hour + '').padStart(2, '0')}:00`; + }); + let d = new frappe.ui.Dialog({ + title: __('Schedule Newsletter'), + fields: [ + { + label: __('Date'), + fieldname: 'date', + fieldtype: 'Date', + options: { + minDate: new Date() + } + }, + { + label: __('Time'), + fieldname: 'time', + fieldtype: 'Select', + options: time_slots, + }, + ], + primary_action_label: __('Schedule'), + primary_action({ date, time }) { + frm.set_value('schedule_sending', 1); + frm.set_value('schedule_send', `${date} ${time}`); + d.hide(); } }); + if (frm.doc.schedule_sending) { + let parts = frm.doc.schedule_send.split(' '); + if (parts.length === 2) { + let [date, time] = parts; + d.set_value('date', date); + d.set_value('time', time); + } + } + d.show(); + }, - - const $tp = frm.get_field('schedule_send').datepicker.timepicker; - $tp.$minutes.parent().css('display', 'none'); - $tp.$minutesText.css('display', 'none'); - $tp.$minutesText.prev().css('display', 'none'); - $tp.$seconds.parent().css('display', 'none'); + send_test_email(frm) { + let d = new frappe.ui.Dialog({ + title: __('Send Test Email'), + fields: [ + { + label: __('Email'), + fieldname: 'email', + fieldtype: 'Data', + options: 'Email', + } + ], + primary_action_label: __('Send'), + primary_action({ email }) { + d.get_primary_btn().text(__('Sending...')).prop('disabled', true); + frm.call('send_test_email', { email }) + .then(() => { + d.get_primary_btn().text(__('Send again')).prop('disabled', false); + }); + } + }); + d.show(); }, setup_dashboard(frm) { - if(!frm.doc.__islocal && cint(frm.doc.email_sent) + if (!frm.doc.__islocal && cint(frm.doc.email_sent) && frm.doc.__onload && frm.doc.__onload.status_count) { var stat = frm.doc.__onload.status_count; var total = frm.doc.scheduled_to_send; - if(total) { - $.each(stat, function(k, v) { + if (total) { + $.each(stat, function (k, v) { stat[k] = flt(v * 100 / total, 2) + '%'; }); diff --git a/frappe/email/doctype/newsletter/newsletter.json b/frappe/email/doctype/newsletter/newsletter.json index dcd19ed33c..ccb2ca8181 100644 --- a/frappe/email/doctype/newsletter/newsletter.json +++ b/frappe/email/doctype/newsletter/newsletter.json @@ -7,29 +7,33 @@ "document_type": "Other", "engine": "InnoDB", "field_order": [ + "from_section", + "sender_name", + "column_break_5", + "sender_email", + "column_break_7", "send_from", - "schedule_sending", - "schedule_send", "recipients", "email_group", "email_sent", - "newsletter_content", + "subject_section", "subject", + "preview_text", + "newsletter_content", "content_type", "message", "message_md", "message_html", - "section_break_13", "send_unsubscribe_link", "send_attachments", - "column_break_9", - "published", "send_webview_link", - "route", - "test_the_newsletter", - "test_email_id", - "test_send", - "scheduled_to_send" + "schedule_settings_section", + "scheduled_to_send", + "schedule_sending", + "schedule_send", + "publish_as_a_web_page_section", + "published", + "route" ], "fields": [ { @@ -43,7 +47,8 @@ "fieldname": "send_from", "fieldtype": "Data", "ignore_xss_filter": 1, - "label": "Sender" + "label": "Sender", + "read_only": 1 }, { "default": "0", @@ -89,30 +94,9 @@ { "fieldname": "route", "fieldtype": "Data", - "hidden": 1, "label": "Route", "read_only": 1 }, - { - "collapsible": 1, - "fieldname": "test_the_newsletter", - "fieldtype": "Section Break", - "label": "Testing" - }, - { - "description": "A Lead with this Email Address should exist", - "fieldname": "test_email_id", - "fieldtype": "Data", - "label": "Test Email Address", - "options": "Email" - }, - { - "depends_on": "eval: doc.test_email_id", - "fieldname": "test_send", - "fieldtype": "Button", - "label": "Test", - "options": "test_send" - }, { "fieldname": "scheduled_to_send", "fieldtype": "Int", @@ -122,13 +106,14 @@ { "fieldname": "recipients", "fieldtype": "Section Break", - "label": "Recipients" + "label": "To" }, { "depends_on": "eval: doc.schedule_sending", "fieldname": "schedule_send", "fieldtype": "Datetime", - "label": "Schedule Send", + "label": "Send Email At", + "read_only": 1, "read_only_depends_on": "eval: doc.email_sent" }, { @@ -161,13 +146,9 @@ "default": "0", "fieldname": "schedule_sending", "fieldtype": "Check", - "label": "Schedule Sending", + "label": "Schedule sending at a later time", "read_only_depends_on": "eval: doc.email_sent" }, - { - "fieldname": "column_break_9", - "fieldtype": "Column Break" - }, { "default": "0", "depends_on": "published", @@ -176,8 +157,51 @@ "label": "Send Web View Link" }, { - "fieldname": "section_break_13", - "fieldtype": "Section Break" + "fieldname": "from_section", + "fieldtype": "Section Break", + "label": "From" + }, + { + "fieldname": "sender_name", + "fieldtype": "Data", + "label": "Sender Name" + }, + { + "fieldname": "sender_email", + "fieldtype": "Data", + "label": "Sender Email", + "options": "Email", + "reqd": 1 + }, + { + "fieldname": "column_break_5", + "fieldtype": "Column Break" + }, + { + "fieldname": "column_break_7", + "fieldtype": "Column Break" + }, + { + "fieldname": "subject_section", + "fieldtype": "Section Break", + "label": "Subject" + }, + { + "description": "Preview Text appears in the inbox after the subject line", + "fieldname": "preview_text", + "fieldtype": "Data", + "label": "Preview Text" + }, + { + "fieldname": "publish_as_a_web_page_section", + "fieldtype": "Section Break", + "label": "Publish as a web page" + }, + { + "depends_on": "schedule_sending", + "fieldname": "schedule_settings_section", + "fieldtype": "Section Break", + "label": "Scheduled Sending" } ], "has_web_view": 1, @@ -187,7 +211,7 @@ "is_published_field": "published", "links": [], "max_attachments": 3, - "modified": "2021-02-22 14:33:56.095380", + "modified": "2021-12-03 17:50:12.028162", "modified_by": "Administrator", "module": "Email", "name": "Newsletter", diff --git a/frappe/email/doctype/newsletter/newsletter.py b/frappe/email/doctype/newsletter/newsletter.py index 12fe160c9d..acaa1dbcc1 100644 --- a/frappe/email/doctype/newsletter/newsletter.py +++ b/frappe/email/doctype/newsletter/newsletter.py @@ -30,10 +30,30 @@ class Newsletter(WebsiteGenerator): return self._recipients @frappe.whitelist() - def test_send(self): - test_emails = frappe.utils.split_emails(self.test_email_id) + def send_test_email(self, email): + test_emails = frappe.utils.split_emails(email) self.queue_all(test_emails=test_emails) - frappe.msgprint(_("Test email sent to {0}").format(self.test_email_id)) + frappe.msgprint(_("Test email sent to {0}").format(email), alert=True) + + @frappe.whitelist() + def find_broken_links(self): + from bs4 import BeautifulSoup + import requests + + html = self.get_message() + soup = BeautifulSoup(html, "html.parser") + links = soup.find_all("a") + images = soup.find_all("img") + broken_links = [] + for el in links + images: + url = el.attrs.get("href") or el.attrs.get("src") + try: + response = requests.head(url, verify=False, timeout=5) + if response.status_code >= 400: + broken_links.append(url) + except: + broken_links.append(url) + return broken_links @frappe.whitelist() def send_emails(self): @@ -75,8 +95,9 @@ class Newsletter(WebsiteGenerator): def validate_sender_address(self): """Validate self.send_from is a valid email address or not. """ - if self.send_from: - frappe.utils.validate_email_address(self.send_from, throw=True) + if self.sender_email: + frappe.utils.validate_email_address(self.sender_email, throw=True) + self.send_from = f"{self.sender_name} <{self.sender_email}>" if self.sender_name else self.sender_email def validate_recipient_address(self): """Validate if self.newsletter_recipients are all valid email addresses or not. diff --git a/frappe/email/doctype/newsletter/templates/newsletter.html b/frappe/email/doctype/newsletter/templates/newsletter.html index 733c7df6af..11ee19a550 100644 --- a/frappe/email/doctype/newsletter/templates/newsletter.html +++ b/frappe/email/doctype/newsletter/templates/newsletter.html @@ -36,7 +36,7 @@

    - {{ doc.message }} + {{ doc.get_message() }}
    @@ -51,7 +51,7 @@
    {% for attachment in attachments %}

    - + {{ attachment.file_name }}

    From f6379fdf40ff75e93bd672218a7171697e5e95ec Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Fri, 3 Dec 2021 18:23:28 +0530 Subject: [PATCH 141/246] fix: allow options in datepicker via df.options ability to customize datepicker options via df.options --- frappe/public/js/frappe/form/controls/date.js | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/form/controls/date.js b/frappe/public/js/frappe/form/controls/date.js index 9ad81c7e46..b9945060cd 100644 --- a/frappe/public/js/frappe/form/controls/date.js +++ b/frappe/public/js/frappe/form/controls/date.js @@ -73,7 +73,8 @@ frappe.ui.form.ControlDate = class ControlDate extends frappe.ui.form.ControlDat .text(this.today_text); this.update_datepicker_position(); - } + }, + ...(this.get_df_options()) }; } set_datepicker() { @@ -150,4 +151,19 @@ frappe.ui.form.ControlDate = class ControlDate extends frappe.ui.form.ControlDat } return value; } + get_df_options() { + let options = {}; + let df_options = this.df.options || ''; + if (typeof df_options === 'string') { + try { + options = JSON.parse(df_options); + } catch (error) { + console.warn(`Invalid JSON in options of "${this.df.fieldname}"`); + } + } + else if (typeof df_options === 'object') { + options = df_options; + } + return options; + } }; From 742d5e2e0691107657497b0574cef8c110782011 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Fri, 3 Dec 2021 18:23:51 +0530 Subject: [PATCH 142/246] fix: utility method to create a range of values mimics python's range function --- frappe/public/js/frappe/utils/utils.js | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/frappe/public/js/frappe/utils/utils.js b/frappe/public/js/frappe/utils/utils.js index 2dbad5427d..5f546c22da 100644 --- a/frappe/public/js/frappe/utils/utils.js +++ b/frappe/public/js/frappe/utils/utils.js @@ -1376,5 +1376,18 @@ Object.assign(frappe.utils, { return array; } return undefined; + }, + + // simple implementation of python's range + range(start, end) { + if (!end) { + end = start; + start = 0; + } + let arr = []; + for (let i = start; i < end; i++) { + arr.push(i); + } + return arr; } }); From 02759631b498ebed638b2764585027378e4326ca Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Mon, 6 Dec 2021 15:35:41 +0530 Subject: [PATCH 143/246] fix: handle falsy return values in document methods problem: if a whitelisted document method returns a falsy value like `[]`, `{}`, `0` then response.message is not set and not returned in the response. this change checks if the return value is `None` and falsy values are returned properly in the response --- frappe/handler.py | 2 +- frappe/model/document.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/frappe/handler.py b/frappe/handler.py index 42c17261b4..35063ee9d6 100755 --- a/frappe/handler.py +++ b/frappe/handler.py @@ -255,7 +255,7 @@ def run_doc_method(method, docs=None, dt=None, dn=None, arg=None, args=None): response = doc.run_method(method, **args) frappe.response.docs.append(doc) - if not response: + if response is None: return # build output as csv diff --git a/frappe/model/document.py b/frappe/model/document.py index 2260406125..bbba9b1492 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -1130,12 +1130,16 @@ class Document(BaseDocument): collated in one dict and returned. Ideally, don't return values in hookable methods, set properties in the document.""" def add_to_return_value(self, new_return_value): + if new_return_value is None: + self._return_value = self.get("_return_value") + return + if isinstance(new_return_value, dict): if not self.get("_return_value"): self._return_value = {} self._return_value.update(new_return_value) else: - self._return_value = new_return_value or self.get("_return_value") + self._return_value = new_return_value def compose(fn, *hooks): def runner(self, method, *args, **kwargs): From 0d3bac55284ae3c6bff085211ea573d69e124e0a Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Mon, 6 Dec 2021 15:36:30 +0530 Subject: [PATCH 144/246] fix(ux): Show broken links as dashboard message --- frappe/email/doctype/newsletter/newsletter.js | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/frappe/email/doctype/newsletter/newsletter.js b/frappe/email/doctype/newsletter/newsletter.js index a7cbcf702a..e8978b8d0b 100644 --- a/frappe/email/doctype/newsletter/newsletter.js +++ b/frappe/email/doctype/newsletter/newsletter.js @@ -11,21 +11,18 @@ frappe.ui.form.on('Newsletter', { }, __('Preview')); frm.add_custom_button(__('Check broken links'), () => { + frm.dashboard.set_headline(__('Checking broken links...')); frm.call('find_broken_links').then(r => { + frm.dashboard.set_headline(''); let links = r.message; - if (links) { + if (links && links.length) { let html = '
      ' + links.map(link => `
    • ${link}
      • `).join('') + '
    '; - frappe.msgprint({ - title: __("Broken Links"), - message: __("Following links are broken in the email content: {0}", [html]), - indicator: "red" - }) + frm.dashboard.set_headline(__("Following links are broken in the email content: {0}", [html])); } else { - frappe.msgprint({ - title: _("No Broken Links"), - message: _("No broken links found in the email content"), - indicator: "green" - }) + frm.dashboard.set_headline(__("No broken links found in the email content")); + setTimeout(() => { + frm.dashboard.set_headline(''); + }, 3000); } }) }, __('Preview')); From 9bdb5f2eb2404a4a6db855659d7c8f55cdb9b518 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Mon, 6 Dec 2021 17:04:24 +0530 Subject: [PATCH 145/246] fix: Explicit attachments table The newsletter content may contain images that get "attached" to the newsletter document. If this is the case, you can't selectively include attachments in the newsletter as it attaches all the attachments. An explicit attachments table solves this problem. --- .../email/doctype/email_queue/email_queue.py | 14 ++++++--- .../email/doctype/newsletter/newsletter.json | 23 +++++--------- frappe/email/doctype/newsletter/newsletter.py | 13 +------- .../doctype/newsletter_attachment/__init__.py | 0 .../newsletter_attachment.json | 31 +++++++++++++++++++ .../newsletter_attachment.py | 8 +++++ 6 files changed, 58 insertions(+), 31 deletions(-) create mode 100644 frappe/email/doctype/newsletter_attachment/__init__.py create mode 100644 frappe/email/doctype/newsletter_attachment/newsletter_attachment.json create mode 100644 frappe/email/doctype/newsletter_attachment/newsletter_attachment.py diff --git a/frappe/email/doctype/email_queue/email_queue.py b/frappe/email/doctype/email_queue/email_queue.py index 4489a68cac..077a5dd40b 100644 --- a/frappe/email/doctype/email_queue/email_queue.py +++ b/frappe/email/doctype/email_queue/email_queue.py @@ -283,9 +283,14 @@ class SendMailContext: if attachment.get('fcontent'): continue - fid = attachment.get("fid") - if fid: - _file = frappe.get_doc("File", fid) + file_filters = {} + if attachment.get('fid'): + file_filters['name'] = attachment.get('fid') + elif attachment.get('file_url'): + file_filters['file_url'] = attachment.get('file_url') + + if file_filters: + _file = frappe.get_doc("File", file_filters) fcontent = _file.get_content() attachment.update({ 'fname': _file.file_name, @@ -293,6 +298,7 @@ class SendMailContext: 'parent': message_obj }) attachment.pop("fid", None) + attachment.pop("file_url", None) add_attachment(**attachment) elif attachment.get("print_format_attachment") == 1: @@ -503,7 +509,7 @@ class QueueBuilder: if self._attachments: # store attachments with fid or print format details, to be attached on-demand later for att in self._attachments: - if att.get('fid'): + if att.get('fid') or att.get('file_url'): attachments.append(att) elif att.get("print_format_attachment") == 1: if not att.get('lang', None): diff --git a/frappe/email/doctype/newsletter/newsletter.json b/frappe/email/doctype/newsletter/newsletter.json index ccb2ca8181..9d35671042 100644 --- a/frappe/email/doctype/newsletter/newsletter.json +++ b/frappe/email/doctype/newsletter/newsletter.json @@ -18,14 +18,13 @@ "email_sent", "subject_section", "subject", - "preview_text", "newsletter_content", "content_type", "message", "message_md", "message_html", "send_unsubscribe_link", - "send_attachments", + "attachments", "send_webview_link", "schedule_settings_section", "scheduled_to_send", @@ -116,12 +115,6 @@ "read_only": 1, "read_only_depends_on": "eval: doc.email_sent" }, - { - "default": "0", - "fieldname": "send_attachments", - "fieldtype": "Check", - "label": "Send Attachments" - }, { "fieldname": "content_type", "fieldtype": "Select", @@ -186,12 +179,6 @@ "fieldtype": "Section Break", "label": "Subject" }, - { - "description": "Preview Text appears in the inbox after the subject line", - "fieldname": "preview_text", - "fieldtype": "Data", - "label": "Preview Text" - }, { "fieldname": "publish_as_a_web_page_section", "fieldtype": "Section Break", @@ -202,6 +189,12 @@ "fieldname": "schedule_settings_section", "fieldtype": "Section Break", "label": "Scheduled Sending" + }, + { + "fieldname": "attachments", + "fieldtype": "Table", + "label": "Attachments", + "options": "Newsletter Attachment" } ], "has_web_view": 1, @@ -211,7 +204,7 @@ "is_published_field": "published", "links": [], "max_attachments": 3, - "modified": "2021-12-03 17:50:12.028162", + "modified": "2021-12-06 17:01:32.353153", "modified_by": "Administrator", "module": "Email", "name": "Newsletter", diff --git a/frappe/email/doctype/newsletter/newsletter.py b/frappe/email/doctype/newsletter/newsletter.py index acaa1dbcc1..b2146ed100 100644 --- a/frappe/email/doctype/newsletter/newsletter.py +++ b/frappe/email/doctype/newsletter/newsletter.py @@ -164,18 +164,7 @@ class Newsletter(WebsiteGenerator): def get_newsletter_attachments(self) -> List[Dict[str, str]]: """Get list of attachments on current Newsletter """ - attachments = [] - - if self.send_attachments: - files = frappe.get_all( - "File", - filters={"attached_to_doctype": "Newsletter", "attached_to_name": self.name}, - order_by="creation desc", - pluck="name", - ) - attachments.extend({"fid": file} for file in files) - - return attachments + return [{"file_url": row.attachment} for row in self.attachments] def send_newsletter(self, emails: List[str]): """Trigger email generation for `emails` and add it in Email Queue. diff --git a/frappe/email/doctype/newsletter_attachment/__init__.py b/frappe/email/doctype/newsletter_attachment/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/frappe/email/doctype/newsletter_attachment/newsletter_attachment.json b/frappe/email/doctype/newsletter_attachment/newsletter_attachment.json new file mode 100644 index 0000000000..e2add0ed64 --- /dev/null +++ b/frappe/email/doctype/newsletter_attachment/newsletter_attachment.json @@ -0,0 +1,31 @@ +{ + "actions": [], + "allow_rename": 1, + "creation": "2021-12-06 16:37:40.652468", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "attachment" + ], + "fields": [ + { + "fieldname": "attachment", + "fieldtype": "Attach", + "in_list_view": 1, + "label": "Attachment", + "reqd": 1 + } + ], + "index_web_pages_for_search": 1, + "istable": 1, + "links": [], + "modified": "2021-12-06 16:37:47.481057", + "modified_by": "Administrator", + "module": "Email", + "name": "Newsletter Attachment", + "owner": "Administrator", + "permissions": [], + "sort_field": "modified", + "sort_order": "DESC" +} \ No newline at end of file diff --git a/frappe/email/doctype/newsletter_attachment/newsletter_attachment.py b/frappe/email/doctype/newsletter_attachment/newsletter_attachment.py new file mode 100644 index 0000000000..7842badbe1 --- /dev/null +++ b/frappe/email/doctype/newsletter_attachment/newsletter_attachment.py @@ -0,0 +1,8 @@ +# Copyright (c) 2021, Frappe Technologies and contributors +# For license information, please see license.txt + +# import frappe +from frappe.model.document import Document + +class NewsletterAttachment(Document): + pass From 3caa93c2c438b09cb1b386730ba3a5f9bae9abf1 Mon Sep 17 00:00:00 2001 From: Jannat Patel Date: Tue, 7 Dec 2021 11:14:11 +0530 Subject: [PATCH 146/246] chore: sla tracking --- frappe/hooks.py | 3 ++- .../personal_data_deletion_request.js | 8 +++++++ .../personal_data_deletion_request.json | 23 ++++++++++++++++++- .../personal_data_deletion_request.py | 23 ++++++++++++++++++- 4 files changed, 54 insertions(+), 3 deletions(-) diff --git a/frappe/hooks.py b/frappe/hooks.py index 8bca5c066c..c204b01356 100644 --- a/frappe/hooks.py +++ b/frappe/hooks.py @@ -240,7 +240,8 @@ scheduler_events = { "frappe.automation.doctype.auto_repeat.auto_repeat.set_auto_repeat_as_completed", "frappe.email.doctype.unhandled_email.unhandled_email.remove_old_unhandled_emails", "frappe.core.doctype.prepared_report.prepared_report.delete_expired_prepared_reports", - "frappe.core.doctype.log_settings.log_settings.run_log_clean_up" + "frappe.core.doctype.log_settings.log_settings.run_log_clean_up", + "frappe.website.doctype.personal_data_deletion_request.personal_data_deletion_request.update_sla" ], "daily_long": [ "frappe.integrations.doctype.dropbox_settings.dropbox_settings.take_backups_daily", diff --git a/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.js b/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.js index 1eb2e02f49..a6cb0b234f 100644 --- a/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.js +++ b/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.js @@ -18,5 +18,13 @@ frappe.ui.form.on("Personal Data Deletion Request", { }); }); } + }, + + before_load: function(frm) { + frappe.db.get_single_value("Website Settings", "account_deletion_sla").then((data) => { + if (data < 1) { + frm.set_df_property("sla_status", "hidden", 1); + } + }); } }); diff --git a/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.json b/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.json index 0cb11068f5..dd4ae54a90 100644 --- a/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.json +++ b/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.json @@ -7,6 +7,9 @@ "field_order": [ "email", "status", + "column_break_3", + "sla_status", + "section_break_5", "anonymization_matrix", "deletion_steps" ], @@ -42,10 +45,28 @@ "fieldtype": "Table", "label": "Deletion Steps ", "options": "Personal Data Deletion Step" + }, + { + "default": "Open", + "fieldname": "sla_status", + "fieldtype": "Select", + "in_list_view": 1, + "in_standard_filter": 1, + "label": "SLA Status", + "options": "Open\nFulfilled\nFailed", + "read_only": 1 + }, + { + "fieldname": "column_break_3", + "fieldtype": "Column Break" + }, + { + "fieldname": "section_break_5", + "fieldtype": "Section Break" } ], "links": [], - "modified": "2021-04-23 13:25:53.629308", + "modified": "2021-12-07 10:48:06.194408", "modified_by": "Administrator", "module": "Website", "name": "Personal Data Deletion Request", diff --git a/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py b/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py index ef3856ad25..6d943eb103 100644 --- a/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py +++ b/frappe/website/doctype/personal_data_deletion_request/personal_data_deletion_request.py @@ -7,7 +7,7 @@ import re import frappe from frappe import _ from frappe.model.document import Document -from frappe.utils import get_fullname +from frappe.utils import get_fullname, date_diff, get_datetime from frappe.utils.user import get_system_managers from frappe.utils.verified_command import get_signed_params, verify_request import json @@ -280,6 +280,13 @@ class PersonalDataDeletionRequest(Document): frappe.rename_doc("User", email, anon, force=True, show_alert=False) self.db_set("status", "Deleted") + account_deletion_sla = frappe.db.get_single_value("Website Settings", "account_deletion_sla") + if account_deletion_sla > 0 and self.sla_status == "Open": + if date_diff(get_datetime(), self.creation) > account_deletion_sla: + self.db_set("sla_status", "Failed") + elif date_diff(get_datetime(), self.creation) <= account_deletion_sla: + self.db_set("sla_status", "Fulfilled") + if commit: frappe.db.commit() @@ -344,6 +351,20 @@ def remove_unverified_record(): AND `creation` < (NOW() - INTERVAL '7' DAY)""" ) +def update_sla(): + account_deletion_sla = frappe.db.get_single_value("Website Settings", "account_deletion_sla") + if account_deletion_sla < 1: + return + + requests = frappe.get_all("Personal Data Deletion Request", + filters = { + "sla_status": "Open" + }, + fields = ["name", "creation", "status"]) + + for request in requests: + if date_diff(get_datetime(), request.creation) > account_deletion_sla and request.status != "Deleted": + frappe.db.set_value("Personal Data Deletion Request", request.name, "sla_status", "Failed") @frappe.whitelist(allow_guest=True) def confirm_deletion(email, name, host_name): From 330677bb0a8960ab4624c3813bc15e1a6091544c Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Tue, 7 Dec 2021 15:40:55 +0530 Subject: [PATCH 147/246] fix: better sending status - show email sending progress in form dashboard --- frappe/email/doctype/newsletter/newsletter.js | 46 +++++++++++++++++++ frappe/email/doctype/newsletter/newsletter.py | 17 +++++++ 2 files changed, 63 insertions(+) diff --git a/frappe/email/doctype/newsletter/newsletter.js b/frappe/email/doctype/newsletter/newsletter.js index e8978b8d0b..d68736a00f 100644 --- a/frappe/email/doctype/newsletter/newsletter.js +++ b/frappe/email/doctype/newsletter/newsletter.js @@ -39,6 +39,7 @@ frappe.ui.form.on('Newsletter', { } frm.events.setup_dashboard(frm); + frm.events.setup_sending_status(frm); if (frm.is_new() && !doc.sender_email) { let { fullname, email } = frappe.user_info(doc.owner); @@ -145,5 +146,50 @@ frappe.ui.form.on('Newsletter', { ]); } } + }, + + setup_sending_status(frm) { + frm.call('get_sending_status').then(r => { + if (r.message) { + frm.events.update_sending_progress(frm, r.message.sent, r.message.total); + } + if (r.message.sent >= r.message.total) { + return; + } + if (frm.sending_status) return; + + frm.sending_status = setInterval(() => { + if (frm.doc.email_sent && frm.$wrapper.is(':visible')) { + frm.call('get_sending_status').then(r => { + if (r.message) { + let { sent, total } = r.message; + frm.events.update_sending_progress(frm, sent, total); + + if (sent >= total) { + clearInterval(frm.sending_status); + frm.sending_status = null; + return; + } + } + }); + } + }, 5000); + }); + }, + + update_sending_progress(frm, sent, total) { + if (sent >= total) { + frm.dashboard.hide_progress(); + return; + } + frm.dashboard.show_progress(__('Sending emails'), sent * 100 / total, __("{0} of {1} sent", [sent, total])); + }, + + on_hide(frm) { + if (frm.sending_status) { + clearInterval(frm.sending_status); + frm.sending_status = null; + } + }, } }); diff --git a/frappe/email/doctype/newsletter/newsletter.py b/frappe/email/doctype/newsletter/newsletter.py index b2146ed100..d84953db93 100644 --- a/frappe/email/doctype/newsletter/newsletter.py +++ b/frappe/email/doctype/newsletter/newsletter.py @@ -29,6 +29,23 @@ class Newsletter(WebsiteGenerator): self._recipients = self.get_recipients() return self._recipients + @frappe.whitelist() + def get_sending_status(self): + count_by_status = frappe.get_all("Email Queue", + filters={"reference_doctype": self.doctype, "reference_name": self.name}, + fields=["status", "count(name) as count"], + group_by="status", + order_by="status" + ) + sent = 0 + total = 0 + for row in count_by_status: + if row.status == "Sent": + sent = row.count + total += row.count + + return {'sent': sent, 'total': total} + @frappe.whitelist() def send_test_email(self, email): test_emails = frappe.utils.split_emails(email) From 1bb3c2d3f4797b76c400beb95982f7d51be0dde7 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Tue, 7 Dec 2021 15:41:38 +0530 Subject: [PATCH 148/246] fix: add on_hide event in form can be used to clearing events, for e.g., clearing setInterval --- frappe/public/js/frappe/form/form.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js index 27281d8927..e789b7241c 100644 --- a/frappe/public/js/frappe/form/form.js +++ b/frappe/public/js/frappe/form/form.js @@ -75,6 +75,10 @@ frappe.ui.form.Form = class FrappeForm { this.page = this.wrapper.page; this.layout_main = this.page.main.get(0); + this.$wrapper.on("hide", () => { + this.script_manager.trigger("on_hide"); + }); + this.toolbar = new frappe.ui.form.Toolbar({ frm: this, page: this.page From de7d0337a60be9ca3d6e59ab4d6141a595e8f2fe Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Tue, 7 Dec 2021 15:48:35 +0530 Subject: [PATCH 149/246] fix: various newsletter form ux fixes - Cancel Scheduling button - Show dashboard message if newsletter is scheduled --- frappe/email/doctype/newsletter/newsletter.js | 30 +++++++++++++++++-- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/frappe/email/doctype/newsletter/newsletter.js b/frappe/email/doctype/newsletter/newsletter.js index d68736a00f..72b8fa6993 100644 --- a/frappe/email/doctype/newsletter/newsletter.js +++ b/frappe/email/doctype/newsletter/newsletter.js @@ -28,7 +28,13 @@ frappe.ui.form.on('Newsletter', { }, __('Preview')); frm.add_custom_button(__('Send now'), () => { - frappe.confirm(__("Do you really want to send this email newsletter?"), function () { + if (frm.doc.schedule_send) { + frappe.confirm(__("This newsletter was scheduled to send on a later date. Are you sure you want to send it now?"), function () { + frm.call('send_emails').then(() => frm.refresh()); + }); + return; + } + frappe.confirm(__("Are you sure you want to send this newsletter now?"), function () { frm.call('send_emails').then(() => frm.refresh()); }); }, __('Send')); @@ -46,6 +52,8 @@ frappe.ui.form.on('Newsletter', { frm.set_value('sender_email', email); frm.set_value('sender_name', fullname); } + + frm.trigger('update_schedule_message'); }, schedule_send_dialog(frm) { @@ -74,8 +82,16 @@ frappe.ui.form.on('Newsletter', { primary_action_label: __('Schedule'), primary_action({ date, time }) { frm.set_value('schedule_sending', 1); - frm.set_value('schedule_send', `${date} ${time}`); + frm.set_value('schedule_send', `${date} ${time}:00`); d.hide(); + frm.save(); + }, + secondary_action_label: __('Cancel Scheduling'), + secondary_action() { + frm.set_value('schedule_sending', 0); + frm.set_value('schedule_send', ''); + d.hide(); + frm.save(); } }); if (frm.doc.schedule_sending) { @@ -83,7 +99,7 @@ frappe.ui.form.on('Newsletter', { if (parts.length === 2) { let [date, time] = parts; d.set_value('date', date); - d.set_value('time', time); + d.set_value('time', time.slice(0, 5)); } } d.show(); @@ -191,5 +207,13 @@ frappe.ui.form.on('Newsletter', { frm.sending_status = null; } }, + + update_schedule_message(frm) { + if (!frm.doc.email_sent && frm.doc.schedule_send) { + let datetime = frappe.datetime.global_date_format(frm.doc.schedule_send); + frm.dashboard.set_headline_alert(__('This newsletter is scheduled to be sent on {0}', [datetime.bold()])); + } else { + frm.dashboard.clear_headline(); + } } }); From 606a0d3809f868a5f55e3be02f5264b9929d6f26 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Tue, 7 Dec 2021 15:52:25 +0530 Subject: [PATCH 150/246] fix: various fixes - show published newsletters in list view - show published newsletter as web page - show status section after newsletter is sent - add email_sent_at and total_recipients field --- .../email/doctype/newsletter/newsletter.json | 45 +++++- frappe/email/doctype/newsletter/newsletter.py | 103 ++++---------- .../newsletter/templates/newsletter.html | 10 +- .../newsletter_email_group.json | 134 +++++------------- 4 files changed, 103 insertions(+), 189 deletions(-) diff --git a/frappe/email/doctype/newsletter/newsletter.json b/frappe/email/doctype/newsletter/newsletter.json index 9d35671042..baabd4991e 100644 --- a/frappe/email/doctype/newsletter/newsletter.json +++ b/frappe/email/doctype/newsletter/newsletter.json @@ -7,6 +7,12 @@ "document_type": "Other", "engine": "InnoDB", "field_order": [ + "status_section", + "email_sent_at", + "column_break_3", + "total_recipients", + "column_break_12", + "email_sent", "from_section", "sender_name", "column_break_5", @@ -15,7 +21,6 @@ "send_from", "recipients", "email_group", - "email_sent", "subject_section", "subject", "newsletter_content", @@ -23,8 +28,8 @@ "message", "message_md", "message_html", - "send_unsubscribe_link", "attachments", + "send_unsubscribe_link", "send_webview_link", "schedule_settings_section", "scheduled_to_send", @@ -39,8 +44,9 @@ "fieldname": "email_group", "fieldtype": "Table", "in_standard_filter": 1, - "label": "Email Group", - "options": "Newsletter Email Group" + "label": "Audience", + "options": "Newsletter Email Group", + "reqd": 1 }, { "fieldname": "send_from", @@ -53,6 +59,7 @@ "default": "0", "fieldname": "email_sent", "fieldtype": "Check", + "hidden": 1, "label": "Email Sent", "no_copy": 1, "read_only": 1 @@ -91,6 +98,7 @@ "label": "Published" }, { + "depends_on": "published", "fieldname": "route", "fieldtype": "Data", "label": "Route", @@ -144,7 +152,6 @@ }, { "default": "0", - "depends_on": "published", "fieldname": "send_webview_link", "fieldtype": "Check", "label": "Send Web View Link" @@ -195,6 +202,32 @@ "fieldtype": "Table", "label": "Attachments", "options": "Newsletter Attachment" + }, + { + "fieldname": "email_sent_at", + "fieldtype": "Datetime", + "label": "Email Sent At", + "read_only": 1 + }, + { + "fieldname": "total_recipients", + "fieldtype": "Int", + "label": "Total Recipients", + "read_only": 1 + }, + { + "depends_on": "email_sent", + "fieldname": "status_section", + "fieldtype": "Section Break", + "label": "Status" + }, + { + "fieldname": "column_break_12", + "fieldtype": "Column Break" + }, + { + "fieldname": "column_break_3", + "fieldtype": "Column Break" } ], "has_web_view": 1, @@ -204,7 +237,7 @@ "is_published_field": "published", "links": [], "max_attachments": 3, - "modified": "2021-12-06 17:01:32.353153", + "modified": "2021-12-06 20:09:37.963141", "modified_by": "Administrator", "module": "Email", "name": "Newsletter", diff --git a/frappe/email/doctype/newsletter/newsletter.py b/frappe/email/doctype/newsletter/newsletter.py index d84953db93..aa6fa2c40a 100644 --- a/frappe/email/doctype/newsletter/newsletter.py +++ b/frappe/email/doctype/newsletter/newsletter.py @@ -15,13 +15,11 @@ from .exceptions import NewsletterAlreadySentError, NoRecipientFoundError, Newsl class Newsletter(WebsiteGenerator): - def onload(self): - self.setup_newsletter_status() - def validate(self): self.route = f"newsletters/{self.name}" self.validate_sender_address() self.validate_recipient_address() + self.validate_publishing() @property def newsletter_recipients(self) -> List[str]: @@ -48,8 +46,8 @@ class Newsletter(WebsiteGenerator): @frappe.whitelist() def send_test_email(self, email): - test_emails = frappe.utils.split_emails(email) - self.queue_all(test_emails=test_emails) + test_emails = frappe.utils.validate_email_address(email, throw=True) + self.send_newsletter(emails=test_emails) frappe.msgprint(_("Test email sent to {0}").format(email), alert=True) @frappe.whitelist() @@ -74,22 +72,11 @@ class Newsletter(WebsiteGenerator): @frappe.whitelist() def send_emails(self): - """send emails to leads and customers""" + """queue sending emails to recipients""" + self.schedule_sending = False + self.schedule_send = None self.queue_all() - frappe.msgprint(_("Email queued to {0} recipients").format(len(self.newsletter_recipients))) - - def setup_newsletter_status(self): - """Setup analytical status for current Newsletter. Can be accessible from desk. - """ - if self.email_sent: - status_count = frappe.get_all("Email Queue", - filters={"reference_doctype": self.doctype, "reference_name": self.name}, - fields=["status", "count(name)"], - group_by="status", - order_by="status", - as_list=True, - ) - self.get("__onload").status_count = dict(status_count) + frappe.msgprint(_("Email queued to {0} recipients").format(self.total_recipients)) def validate_send(self): """Validate if Newsletter can be sent. @@ -122,6 +109,10 @@ class Newsletter(WebsiteGenerator): for recipient in self.newsletter_recipients: frappe.utils.validate_email_address(recipient, throw=True) + def validate_publishing(self): + if self.send_webview_link and not self.published: + frappe.throw(_("Newsletter must be published to send webview link in email")) + def get_linked_email_queue(self) -> List[str]: """Get list of email queue linked to this newsletter. """ @@ -154,29 +145,19 @@ class Newsletter(WebsiteGenerator): x for x in self.newsletter_recipients if x not in self.get_success_recipients() ] - def queue_all(self, test_emails: List[str] = None): - """Queue Newsletter to all the recipients generated from the `Email Group` - table - - Args: - test_email (List[str], optional): Send test Newsletter to the passed set of emails. - Defaults to None. + def queue_all(self): + """Queue Newsletter to all the recipients generated from the `Email Group` table """ - if test_emails: - for test_email in test_emails: - frappe.utils.validate_email_address(test_email, throw=True) - else: - self.validate() - self.validate_send() + self.validate() + self.validate_send() - newsletter_recipients = test_emails or self.get_pending_recipients() - self.send_newsletter(emails=newsletter_recipients) + recipients = self.get_pending_recipients() + self.send_newsletter(emails=recipients) - if not test_emails: - self.email_sent = True - self.schedule_send = frappe.utils.now_datetime() - self.scheduled_to_send = len(newsletter_recipients) - self.save() + self.email_sent = True + self.email_sent_at = frappe.utils.now() + self.total_recipients = len(recipients) + self.save() def get_newsletter_attachments(self) -> List[Dict[str, str]]: """Get list of attachments on current Newsletter @@ -251,21 +232,6 @@ class Newsletter(WebsiteGenerator): }, ) - def get_context(self, context): - newsletters = get_newsletter_list("Newsletter", None, None, 0) - if newsletters: - newsletter_list = [d.name for d in newsletters] - if self.name not in newsletter_list: - frappe.redirect_to_message( - _("Permission Error"), _("You are not permitted to view the newsletter.") - ) - frappe.local.flags.redirect_location = frappe.local.response.location - raise frappe.Redirect - else: - context.attachments = self.get_attachments() - context.no_cache = 1 - context.show_sidebar = True - @frappe.whitelist(allow_guest=True) def confirmed_unsubscribe(email, group): @@ -348,35 +314,14 @@ def confirm_subscription(email, email_group=_("Website")): def get_list_context(context=None): context.update({ - "show_sidebar": True, "show_search": True, - 'no_breadcrumbs': True, - "title": _("Newsletter"), - "get_list": get_newsletter_list, + "no_breadcrumbs": True, + "title": _("Newsletters"), + "filters": {"published": 1}, "row_template": "email/doctype/newsletter/templates/newsletter_row.html", }) -def get_newsletter_list(doctype, txt, filters, limit_start, limit_page_length=20, order_by="modified"): - email_group_list = frappe.db.sql('''SELECT eg.name - FROM `tabEmail Group` eg, `tabEmail Group Member` egm - WHERE egm.unsubscribed=0 - AND eg.name=egm.email_group - AND egm.email = %s''', frappe.session.user) - email_group_list = [d[0] for d in email_group_list] - - if email_group_list: - return frappe.db.sql('''SELECT n.name, n.subject, n.message, n.modified - FROM `tabNewsletter` n, `tabNewsletter Email Group` neg - WHERE n.name = neg.parent - AND n.email_sent=1 - AND n.published=1 - AND neg.email_group in ({0}) - ORDER BY n.modified DESC LIMIT {1} OFFSET {2} - '''.format(','.join(['%s'] * len(email_group_list)), - limit_page_length, limit_start), email_group_list, as_dict=1) - - def send_scheduled_email(): """Send scheduled newsletter to the recipients.""" scheduled_newsletter = frappe.get_all( diff --git a/frappe/email/doctype/newsletter/templates/newsletter.html b/frappe/email/doctype/newsletter/templates/newsletter.html index 11ee19a550..1244f4c49a 100644 --- a/frappe/email/doctype/newsletter/templates/newsletter.html +++ b/frappe/email/doctype/newsletter/templates/newsletter.html @@ -1,6 +1,6 @@ {% extends "templates/web.html" %} -{% block title %} {{ _("Newsletter") }} {% endblock %} +{% block title %} {{ doc.subject }} {% endblock %} {% block page_content %}