From 476e625261ab4954b7429978b24840e52967c9c0 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 16 Jul 2020 16:26:20 +0530 Subject: [PATCH 01/57] fix: Add site validation for logger --- frappe/__init__.py | 4 ++-- frappe/app.py | 5 +++-- frappe/utils/logger.py | 4 ++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index d644d2a473..8c24980189 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -1559,10 +1559,10 @@ def get_doctype_app(doctype): loggers = {} log_level = None -def logger(module=None, with_more_info=False): +def logger(module=None, with_more_info=False, _site=None): '''Returns a python logger that uses StreamHandler''' from frappe.utils.logger import get_logger - return get_logger(module=module, with_more_info=with_more_info) + return get_logger(module=module, with_more_info=with_more_info, _site=_site) def log_error(message=None, title=_("Error")): '''Log error to Error Log''' diff --git a/frappe/app.py b/frappe/app.py index 57db867882..83a9b8c149 100644 --- a/frappe/app.py +++ b/frappe/app.py @@ -99,8 +99,9 @@ def application(request): frappe.monitor.stop(response) frappe.recorder.dump() - frappe.logger("frappe.web").info({ - "site": get_site_name(request.host), + _site = get_site_name(request.host) + frappe.logger("frappe.web", _site=_site).info({ + "site": _site, "remote_addr": getattr(request, "remote_addr", "NOTFOUND"), "base_url": getattr(request, "base_url", "NOTFOUND"), "full_path": getattr(request, "full_path", "NOTFOUND"), diff --git a/frappe/utils/logger.py b/frappe/utils/logger.py index 89e3711b0f..bea83297d2 100755 --- a/frappe/utils/logger.py +++ b/frappe/utils/logger.py @@ -17,7 +17,7 @@ default_log_level = logging.DEBUG site = getattr(frappe.local, 'site', None) -def get_logger(module, with_more_info=False): +def get_logger(module, with_more_info=False, _site=None): global site if module in frappe.loggers: return frappe.loggers[module] @@ -38,7 +38,7 @@ def get_logger(module, with_more_info=False): handler = RotatingFileHandler(LOG_FILENAME, maxBytes=100_000, backupCount=20) logger.addHandler(handler) # - if site: + if site == _site: SITELOG_FILENAME = os.path.join(site, 'logs', logfile) site_handler = RotatingFileHandler(SITELOG_FILENAME, maxBytes=100_000, backupCount=20) site_handler.setFormatter(formatter) From e92a612ba231eebb8dbe7ac42d24ac002a89fbe1 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 16 Jul 2020 16:26:53 +0530 Subject: [PATCH 02/57] docs(get_logger): add docstring --- frappe/utils/logger.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/frappe/utils/logger.py b/frappe/utils/logger.py index bea83297d2..afe4335aea 100755 --- a/frappe/utils/logger.py +++ b/frappe/utils/logger.py @@ -18,7 +18,24 @@ site = getattr(frappe.local, 'site', None) def get_logger(module, with_more_info=False, _site=None): + """Application Logger for your given module + + Args: + module (str): Name of your logger and consequently your log file. + with_more_info (bool, optional): Will log the form dict using the SiteContextFilter. Defaults to False. + _site (str, optional): If set, validates the current site context with the passed value. The `frappe.web` logger uses this to determine that the application is logging information related to the logger called. Defaults to None. + + Returns: + : Returns a Python logger object with Site and Bench level logging capabilities. + """ global site + + def allow_site(): + allow = False + if site: allow = True + if _site: allow = site == _site + return allow + if module in frappe.loggers: return frappe.loggers[module] @@ -37,8 +54,8 @@ def get_logger(module, with_more_info=False, _site=None): formatter = logging.Formatter('%(asctime)s %(levelname)s %(name)s %(message)s') handler = RotatingFileHandler(LOG_FILENAME, maxBytes=100_000, backupCount=20) logger.addHandler(handler) -# - if site == _site: + + if allow_site(): SITELOG_FILENAME = os.path.join(site, 'logs', logfile) site_handler = RotatingFileHandler(SITELOG_FILENAME, maxBytes=100_000, backupCount=20) site_handler.setFormatter(formatter) From c87d598480e006a88a4374d0ab17d15429701441 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 17 Jul 2020 17:35:08 +0530 Subject: [PATCH 03/57] refactor: less word do trick Co-authored-by: Chinmay Pai --- frappe/utils/logger.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/frappe/utils/logger.py b/frappe/utils/logger.py index afe4335aea..9254c0aedd 100755 --- a/frappe/utils/logger.py +++ b/frappe/utils/logger.py @@ -31,10 +31,7 @@ def get_logger(module, with_more_info=False, _site=None): global site def allow_site(): - allow = False - if site: allow = True - if _site: allow = site == _site - return allow + return (_site and _site == site) or bool(site) if module in frappe.loggers: return frappe.loggers[module] From ad1152f8e8367f8d7c46d0e4e9a14ee9e2d0df9b Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 27 Jul 2020 17:13:18 +0530 Subject: [PATCH 04/57] feat: override site name via _site added filter parameter for logger --- frappe/__init__.py | 4 ++-- frappe/utils/logger.py | 16 +++++++++------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index 8c24980189..504ac96eb9 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -1559,10 +1559,10 @@ def get_doctype_app(doctype): loggers = {} log_level = None -def logger(module=None, with_more_info=False, _site=None): +def logger(module=None, with_more_info=False, _site=None, filter=None): '''Returns a python logger that uses StreamHandler''' from frappe.utils.logger import get_logger - return get_logger(module=module, with_more_info=with_more_info, _site=_site) + return get_logger(module=module, with_more_info=with_more_info, _site=_site, filter=filter) def log_error(message=None, title=_("Error")): '''Log error to Error Log''' diff --git a/frappe/utils/logger.py b/frappe/utils/logger.py index 9254c0aedd..56fcf9857a 100755 --- a/frappe/utils/logger.py +++ b/frappe/utils/logger.py @@ -17,22 +17,20 @@ default_log_level = logging.DEBUG site = getattr(frappe.local, 'site', None) -def get_logger(module, with_more_info=False, _site=None): +def get_logger(module, with_more_info=False, _site=None, filter=None): """Application Logger for your given module Args: module (str): Name of your logger and consequently your log file. with_more_info (bool, optional): Will log the form dict using the SiteContextFilter. Defaults to False. - _site (str, optional): If set, validates the current site context with the passed value. The `frappe.web` logger uses this to determine that the application is logging information related to the logger called. Defaults to None. + _site (str, optional): Overrides the global site variable set in this scope. Defaults to None. + filter (function, optional): Add a filter function for your logger Returns: : Returns a Python logger object with Site and Bench level logging capabilities. """ global site - def allow_site(): - return (_site and _site == site) or bool(site) - if module in frappe.loggers: return frappe.loggers[module] @@ -42,6 +40,7 @@ def get_logger(module, with_more_info=False, _site=None): logfile = module + '.log' site = getattr(frappe.local, 'site', None) + equivalent_site = _site or site LOG_FILENAME = os.path.join('..', 'logs', logfile) logger = logging.getLogger(module) @@ -52,8 +51,8 @@ def get_logger(module, with_more_info=False, _site=None): handler = RotatingFileHandler(LOG_FILENAME, maxBytes=100_000, backupCount=20) logger.addHandler(handler) - if allow_site(): - SITELOG_FILENAME = os.path.join(site, 'logs', logfile) + if equivalent_site: + SITELOG_FILENAME = os.path.join(equivalent_site, 'logs', logfile) site_handler = RotatingFileHandler(SITELOG_FILENAME, maxBytes=100_000, backupCount=20) site_handler.setFormatter(formatter) logger.addHandler(site_handler) @@ -61,6 +60,9 @@ def get_logger(module, with_more_info=False, _site=None): if with_more_info: handler.addFilter(SiteContextFilter()) + if filter: + logger.addFilter(filter) + handler.setFormatter(formatter) frappe.loggers[module] = logger From 1345c2feb311bf7abb06b3f1969c4a2a43dc9b94 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 27 Jul 2020 17:31:12 +0530 Subject: [PATCH 05/57] feat: Allow_site flag in frappe.logger --- frappe/__init__.py | 4 ++-- frappe/app.py | 2 +- frappe/utils/logger.py | 22 +++++++++++++--------- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index 504ac96eb9..7ea95ec3db 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -1559,10 +1559,10 @@ def get_doctype_app(doctype): loggers = {} log_level = None -def logger(module=None, with_more_info=False, _site=None, filter=None): +def logger(module=None, with_more_info=False, allow_site=True, filter=None): '''Returns a python logger that uses StreamHandler''' from frappe.utils.logger import get_logger - return get_logger(module=module, with_more_info=with_more_info, _site=_site, filter=filter) + return get_logger(module=module, with_more_info=with_more_info, allow_site=allow_site, filter=filter) def log_error(message=None, title=_("Error")): '''Log error to Error Log''' diff --git a/frappe/app.py b/frappe/app.py index 83a9b8c149..100b10e9ef 100644 --- a/frappe/app.py +++ b/frappe/app.py @@ -100,7 +100,7 @@ def application(request): frappe.recorder.dump() _site = get_site_name(request.host) - frappe.logger("frappe.web", _site=_site).info({ + frappe.logger("frappe.web", allow_site=_site).info({ "site": _site, "remote_addr": getattr(request, "remote_addr", "NOTFOUND"), "base_url": getattr(request, "base_url", "NOTFOUND"), diff --git a/frappe/utils/logger.py b/frappe/utils/logger.py index 56fcf9857a..e7058ce42d 100755 --- a/frappe/utils/logger.py +++ b/frappe/utils/logger.py @@ -14,22 +14,20 @@ import frappe default_log_level = logging.DEBUG -site = getattr(frappe.local, 'site', None) -def get_logger(module, with_more_info=False, _site=None, filter=None): +def get_logger(module, with_more_info=False, allow_site=True, filter=None): """Application Logger for your given module Args: module (str): Name of your logger and consequently your log file. with_more_info (bool, optional): Will log the form dict using the SiteContextFilter. Defaults to False. - _site (str, optional): Overrides the global site variable set in this scope. Defaults to None. + allow_site ((str, bool), optional): Pass site name to explicitly log under it's logs. If True and unspecified, guesses which site the logs would be saved under. Defaults to True. filter (function, optional): Add a filter function for your logger Returns: : Returns a Python logger object with Site and Bench level logging capabilities. """ - global site if module in frappe.loggers: return frappe.loggers[module] @@ -39,8 +37,14 @@ def get_logger(module, with_more_info=False, _site=None, filter=None): with_more_info = True logfile = module + '.log' - site = getattr(frappe.local, 'site', None) - equivalent_site = _site or site + + if allow_site == True: + site = getattr(frappe.local, 'site', None) + elif allow_site: + site = allow_site + else: + site = False + LOG_FILENAME = os.path.join('..', 'logs', logfile) logger = logging.getLogger(module) @@ -51,8 +55,8 @@ def get_logger(module, with_more_info=False, _site=None, filter=None): handler = RotatingFileHandler(LOG_FILENAME, maxBytes=100_000, backupCount=20) logger.addHandler(handler) - if equivalent_site: - SITELOG_FILENAME = os.path.join(equivalent_site, 'logs', logfile) + if site: + SITELOG_FILENAME = os.path.join(site, 'logs', logfile) site_handler = RotatingFileHandler(SITELOG_FILENAME, maxBytes=100_000, backupCount=20) site_handler.setFormatter(formatter) logger.addHandler(site_handler) @@ -73,7 +77,7 @@ class SiteContextFilter(logging.Filter): """This is a filter which injects request information (if available) into the log.""" def filter(self, record): if "Form Dict" not in text_type(record.msg): - record.msg = text_type(record.msg) + "\nSite: {0}\nForm Dict: {1}".format(site, getattr(frappe.local, 'form_dict', None)) + record.msg = text_type(record.msg) + "\nSite: {0}\nForm Dict: {1}".format(getattr(frappe.local, 'site', None), getattr(frappe.local, 'form_dict', None)) return True def set_log_level(level): From ae3ee5b2eb5c64fa00f4461ad1a11841ced65c54 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 29 Jul 2020 13:33:04 +0530 Subject: [PATCH 06/57] fix: save frappe loggers site wise --- frappe/utils/logger.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/frappe/utils/logger.py b/frappe/utils/logger.py index e7058ce42d..c38e0e6997 100755 --- a/frappe/utils/logger.py +++ b/frappe/utils/logger.py @@ -29,8 +29,18 @@ def get_logger(module, with_more_info=False, allow_site=True, filter=None): : Returns a Python logger object with Site and Bench level logging capabilities. """ + if allow_site == True: + site = getattr(frappe.local, 'site', None) + elif allow_site: + site = allow_site + else: + site = False + if module in frappe.loggers: - return frappe.loggers[module] + try: + return frappe.loggers[module][site or "all"] + except: + pass if not module: module = "frappe" @@ -38,12 +48,6 @@ def get_logger(module, with_more_info=False, allow_site=True, filter=None): logfile = module + '.log' - if allow_site == True: - site = getattr(frappe.local, 'site', None) - elif allow_site: - site = allow_site - else: - site = False LOG_FILENAME = os.path.join('..', 'logs', logfile) @@ -69,7 +73,11 @@ def get_logger(module, with_more_info=False, allow_site=True, filter=None): handler.setFormatter(formatter) - frappe.loggers[module] = logger + try: + frappe.loggers[module][site or "all"] = logger + except KeyError: + frappe.loggers[module] = {} + frappe.loggers[module][site or "all"] = logger return logger From 9cef385005e68221cc2d4b628f9080b81a434da2 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Thu, 30 Jul 2020 14:37:22 +0530 Subject: [PATCH 07/57] refactor: refer to frappe.boot.single_types instead of db call --- .../public/js/frappe/widgets/widget_dialog.js | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/frappe/public/js/frappe/widgets/widget_dialog.js b/frappe/public/js/frappe/widgets/widget_dialog.js index 054159116f..1089d7e629 100644 --- a/frappe/public/js/frappe/widgets/widget_dialog.js +++ b/frappe/public/js/frappe/widgets/widget_dialog.js @@ -174,18 +174,12 @@ class ShortcutDialog extends WidgetDialog { onchange: () => { if (this.dialog.get_value("type") == "DocType") { let doctype = this.dialog.get_value("link_to"); - - doctype && - frappe.db - .get_value("DocType", doctype, "issingle") - .then((res) => { - if (res.message && res.message.issingle) { - this.hide_filters(); - } else { - this.setup_filter(doctype); - this.show_filters(); - } - }); + if (frappe.boot.single_types.includes("Stock Settings")) { + this.hide_filters(); + } else { + this.setup_filter(doctype); + this.show_filters(); + } } else { this.hide_filters(); } From 2ad15ee9f75dca2391e45b83f5298b1e7e90e163 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Thu, 30 Jul 2020 14:57:01 +0530 Subject: [PATCH 08/57] fix: use doctype variable --- frappe/public/js/frappe/widgets/widget_dialog.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/widgets/widget_dialog.js b/frappe/public/js/frappe/widgets/widget_dialog.js index 1089d7e629..e3b4cc1d8c 100644 --- a/frappe/public/js/frappe/widgets/widget_dialog.js +++ b/frappe/public/js/frappe/widgets/widget_dialog.js @@ -174,7 +174,7 @@ class ShortcutDialog extends WidgetDialog { onchange: () => { if (this.dialog.get_value("type") == "DocType") { let doctype = this.dialog.get_value("link_to"); - if (frappe.boot.single_types.includes("Stock Settings")) { + if (frappe.boot.single_types.includes(doctype)) { this.hide_filters(); } else { this.setup_filter(doctype); From 7427fb9cd3b9589d8366f330b968a9f5e8c6f310 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Thu, 30 Jul 2020 15:56:38 +0530 Subject: [PATCH 09/57] fix: Dont validate skipped columns --- frappe/core/doctype/data_import/importer.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/frappe/core/doctype/data_import/importer.py b/frappe/core/doctype/data_import/importer.py index 910e42af1a..14626eb5e3 100644 --- a/frappe/core/doctype/data_import/importer.py +++ b/frappe/core/doctype/data_import/importer.py @@ -955,6 +955,9 @@ class Column: if not self.df: return + if self.skip_import: + return + if self.df.fieldtype == 'Link': # find all values that dont exist values = list(set([cstr(v) for v in self.column_values[1:] if v])) From 84a15407a733a57ce5d1b13c2c1fefd80f12b198 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Thu, 30 Jul 2020 16:06:10 +0530 Subject: [PATCH 10/57] fix: Respect hostname config in sitemap.xml --- frappe/www/sitemap.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frappe/www/sitemap.py b/frappe/www/sitemap.py index 8b93270ab5..f8f03c45f8 100644 --- a/frappe/www/sitemap.py +++ b/frappe/www/sitemap.py @@ -5,7 +5,7 @@ from __future__ import unicode_literals import frappe from frappe.model.document import get_controller -from frappe.utils import get_request_site_address, get_datetime, nowdate +from frappe.utils import get_datetime, nowdate, get_url from frappe.website.router import get_pages, get_all_page_context_from_doctypes from six import iteritems from six.moves.urllib.parse import quote, urljoin @@ -25,13 +25,13 @@ def get_context(context): for route, page in iteritems(get_pages()): if page.sitemap: links.append({ - "loc": urljoin(host, quote(page.name.encode("utf-8"))), + "loc": get_url(quote(page.name.encode("utf-8"))), "lastmod": nowdate() }) for route, data in iteritems(get_public_pages_from_doctypes()): links.append({ - "loc": urljoin(host, quote((route or "").encode("utf-8"))), + "loc": get_url(quote((route or "").encode("utf-8"))), "lastmod": get_datetime(data.get("modified")).strftime("%Y-%m-%d") }) From 7bf69c62c1cc942d394d40484502c51b96e98b9f Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 30 Jul 2020 17:47:01 +0530 Subject: [PATCH 11/57] fix: suffix logger name with site --- frappe/utils/logger.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/frappe/utils/logger.py b/frappe/utils/logger.py index c38e0e6997..1fbd54dd38 100755 --- a/frappe/utils/logger.py +++ b/frappe/utils/logger.py @@ -36,9 +36,11 @@ def get_logger(module, with_more_info=False, allow_site=True, filter=None): else: site = False - if module in frappe.loggers: + LOGGER_NAME = "{}-{}".format(module, site or "all") + + if LOGGER_NAME in frappe.loggers: try: - return frappe.loggers[module][site or "all"] + return frappe.loggers[LOGGER_NAME] except: pass @@ -51,7 +53,7 @@ def get_logger(module, with_more_info=False, allow_site=True, filter=None): LOG_FILENAME = os.path.join('..', 'logs', logfile) - logger = logging.getLogger(module) + logger = logging.getLogger(LOGGER_NAME) logger.setLevel(frappe.log_level or default_log_level) logger.propagate = False @@ -73,11 +75,7 @@ def get_logger(module, with_more_info=False, allow_site=True, filter=None): handler.setFormatter(formatter) - try: - frappe.loggers[module][site or "all"] = logger - except KeyError: - frappe.loggers[module] = {} - frappe.loggers[module][site or "all"] = logger + frappe.loggers[LOGGER_NAME] = logger return logger From 090b81f2f268447581eac496bb245af3a17224c5 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 30 Jul 2020 17:59:01 +0530 Subject: [PATCH 12/57] style: black + restructure --- frappe/utils/logger.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/frappe/utils/logger.py b/frappe/utils/logger.py index 1fbd54dd38..79ea93e3bd 100755 --- a/frappe/utils/logger.py +++ b/frappe/utils/logger.py @@ -30,7 +30,7 @@ def get_logger(module, with_more_info=False, allow_site=True, filter=None): """ if allow_site == True: - site = getattr(frappe.local, 'site', None) + site = getattr(frappe.local, "site", None) elif allow_site: site = allow_site else: @@ -48,21 +48,21 @@ def get_logger(module, with_more_info=False, allow_site=True, filter=None): module = "frappe" with_more_info = True - logfile = module + '.log' + logfile = module + ".log" - - LOG_FILENAME = os.path.join('..', 'logs', logfile) + LOG_FILENAME = os.path.join("..", "logs", logfile) logger = logging.getLogger(LOGGER_NAME) logger.setLevel(frappe.log_level or default_log_level) logger.propagate = False - formatter = logging.Formatter('%(asctime)s %(levelname)s %(name)s %(message)s') + formatter = logging.Formatter("%(asctime)s %(levelname)s %(name)s %(message)s") handler = RotatingFileHandler(LOG_FILENAME, maxBytes=100_000, backupCount=20) + handler.setFormatter(formatter) logger.addHandler(handler) if site: - SITELOG_FILENAME = os.path.join(site, 'logs', logfile) + SITELOG_FILENAME = os.path.join(site, "logs", logfile) site_handler = RotatingFileHandler(SITELOG_FILENAME, maxBytes=100_000, backupCount=20) site_handler.setFormatter(formatter) logger.addHandler(site_handler) @@ -73,20 +73,23 @@ def get_logger(module, with_more_info=False, allow_site=True, filter=None): if filter: logger.addFilter(filter) - handler.setFormatter(formatter) - frappe.loggers[LOGGER_NAME] = logger return logger + class SiteContextFilter(logging.Filter): """This is a filter which injects request information (if available) into the log.""" + def filter(self, record): if "Form Dict" not in text_type(record.msg): - record.msg = text_type(record.msg) + "\nSite: {0}\nForm Dict: {1}".format(getattr(frappe.local, 'site', None), getattr(frappe.local, 'form_dict', None)) + site = getattr(frappe.local, "site", None) + form_dict = getattr(frappe.local, "form_dict", None) + record.msg = text_type(record.msg) + "\nSite: {0}\nForm Dict: {1}".format(site, form_dict) return True + def set_log_level(level): - '''Use this method to set log level to something other than the default DEBUG''' - frappe.log_level = getattr(logging, (level or '').upper(), None) or default_log_level + """Use this method to set log level to something other than the default DEBUG""" + frappe.log_level = getattr(logging, (level or "").upper(), None) or default_log_level frappe.loggers = {} From fac7f359ae0b49064003e7424989579d580b7535 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 30 Jul 2020 18:49:59 +0530 Subject: [PATCH 13/57] fix: original log format + EAFP --- frappe/utils/logger.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/frappe/utils/logger.py b/frappe/utils/logger.py index 79ea93e3bd..46269fe909 100755 --- a/frappe/utils/logger.py +++ b/frappe/utils/logger.py @@ -38,26 +38,24 @@ def get_logger(module, with_more_info=False, allow_site=True, filter=None): LOGGER_NAME = "{}-{}".format(module, site or "all") - if LOGGER_NAME in frappe.loggers: - try: - return frappe.loggers[LOGGER_NAME] - except: - pass + try: + return frappe.loggers[LOGGER_NAME] + except KeyError: + pass if not module: module = "frappe" with_more_info = True logfile = module + ".log" - - LOG_FILENAME = os.path.join("..", "logs", logfile) + log_filename = os.path.join("..", "logs", logfile) logger = logging.getLogger(LOGGER_NAME) logger.setLevel(frappe.log_level or default_log_level) logger.propagate = False - formatter = logging.Formatter("%(asctime)s %(levelname)s %(name)s %(message)s") - handler = RotatingFileHandler(LOG_FILENAME, maxBytes=100_000, backupCount=20) + formatter = logging.Formatter("%(asctime)s %(levelname)s {0} %(message)s".format(module)) + handler = RotatingFileHandler(log_filename, maxBytes=100_000, backupCount=20) handler.setFormatter(formatter) logger.addHandler(handler) From faa9a5466eacc4d07e19286309f36aeb8468f7ec Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 30 Jul 2020 19:04:04 +0530 Subject: [PATCH 14/57] fix: Slider --- frappe/utils/logger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/utils/logger.py b/frappe/utils/logger.py index 46269fe909..6d24eef9d3 100755 --- a/frappe/utils/logger.py +++ b/frappe/utils/logger.py @@ -29,7 +29,7 @@ def get_logger(module, with_more_info=False, allow_site=True, filter=None): : Returns a Python logger object with Site and Bench level logging capabilities. """ - if allow_site == True: + if allow_site is True: site = getattr(frappe.local, "site", None) elif allow_site: site = allow_site From 444e8225a46188137028f99cb12efcab195559b5 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Thu, 30 Jul 2020 19:50:36 +0530 Subject: [PATCH 15/57] fix: setup filters only if doctype is available --- frappe/public/js/frappe/widgets/widget_dialog.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/public/js/frappe/widgets/widget_dialog.js b/frappe/public/js/frappe/widgets/widget_dialog.js index e3b4cc1d8c..2ca26d59e1 100644 --- a/frappe/public/js/frappe/widgets/widget_dialog.js +++ b/frappe/public/js/frappe/widgets/widget_dialog.js @@ -174,9 +174,9 @@ class ShortcutDialog extends WidgetDialog { onchange: () => { if (this.dialog.get_value("type") == "DocType") { let doctype = this.dialog.get_value("link_to"); - if (frappe.boot.single_types.includes(doctype)) { + if (doctype && frappe.boot.single_types.includes(doctype)) { this.hide_filters(); - } else { + } else if (doctype) { this.setup_filter(doctype); this.show_filters(); } From 94116ae93c416958619e98aa47ad979481ec9dd1 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 31 Jul 2020 16:07:51 +0530 Subject: [PATCH 16/57] feat(logger): allow max_size and file_count params --- frappe/__init__.py | 4 ++-- frappe/utils/logger.py | 20 +++++++++++--------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index 7ea95ec3db..45f40f7783 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -1559,10 +1559,10 @@ def get_doctype_app(doctype): loggers = {} log_level = None -def logger(module=None, with_more_info=False, allow_site=True, filter=None): +def logger(module=None, with_more_info=False, allow_site=True, filter=None, max_size=100_000, file_count=20): '''Returns a python logger that uses StreamHandler''' from frappe.utils.logger import get_logger - return get_logger(module=module, with_more_info=with_more_info, allow_site=allow_site, filter=filter) + return get_logger(module=module, with_more_info=with_more_info, allow_site=allow_site, filter=filter, max_size=max_size, file_count=file_count) def log_error(message=None, title=_("Error")): '''Log error to Error Log''' diff --git a/frappe/utils/logger.py b/frappe/utils/logger.py index 6d24eef9d3..b3295cccdf 100755 --- a/frappe/utils/logger.py +++ b/frappe/utils/logger.py @@ -16,14 +16,16 @@ import frappe default_log_level = logging.DEBUG -def get_logger(module, with_more_info=False, allow_site=True, filter=None): +def get_logger(module, with_more_info=False, allow_site=True, filter=None, max_size=100_000, file_count=20): """Application Logger for your given module Args: module (str): Name of your logger and consequently your log file. with_more_info (bool, optional): Will log the form dict using the SiteContextFilter. Defaults to False. allow_site ((str, bool), optional): Pass site name to explicitly log under it's logs. If True and unspecified, guesses which site the logs would be saved under. Defaults to True. - filter (function, optional): Add a filter function for your logger + filter (function, optional): Add a filter function for your logger. Defaults to None. + max_size (int, optional): Max file size of each log file in kB. Defaults to 100_000. + file_count (int, optional): Max count of log files to be retained via Log Rotation. Defaults to 20. Returns: : Returns a Python logger object with Site and Bench level logging capabilities. @@ -36,10 +38,10 @@ def get_logger(module, with_more_info=False, allow_site=True, filter=None): else: site = False - LOGGER_NAME = "{}-{}".format(module, site or "all") + logger_name = "{0}-{1}".format(module, site or "all") try: - return frappe.loggers[LOGGER_NAME] + return frappe.loggers[logger_name] except KeyError: pass @@ -50,18 +52,18 @@ def get_logger(module, with_more_info=False, allow_site=True, filter=None): logfile = module + ".log" log_filename = os.path.join("..", "logs", logfile) - logger = logging.getLogger(LOGGER_NAME) + logger = logging.getLogger(logger_name) logger.setLevel(frappe.log_level or default_log_level) logger.propagate = False formatter = logging.Formatter("%(asctime)s %(levelname)s {0} %(message)s".format(module)) - handler = RotatingFileHandler(log_filename, maxBytes=100_000, backupCount=20) + handler = RotatingFileHandler(log_filename, maxBytes=max_size, backupCount=file_count) handler.setFormatter(formatter) logger.addHandler(handler) if site: - SITELOG_FILENAME = os.path.join(site, "logs", logfile) - site_handler = RotatingFileHandler(SITELOG_FILENAME, maxBytes=100_000, backupCount=20) + sitelog_filename = os.path.join(site, "logs", logfile) + site_handler = RotatingFileHandler(sitelog_filename, maxBytes=max_size, backupCount=file_count) site_handler.setFormatter(formatter) logger.addHandler(site_handler) @@ -71,7 +73,7 @@ def get_logger(module, with_more_info=False, allow_site=True, filter=None): if filter: logger.addFilter(filter) - frappe.loggers[LOGGER_NAME] = logger + frappe.loggers[logger_name] = logger return logger From f04ce5dd7a7957d46ef12d2d5e2216bb93e753e1 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 31 Jul 2020 16:11:39 +0530 Subject: [PATCH 17/57] chore: Typo --- frappe/utils/logger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/utils/logger.py b/frappe/utils/logger.py index b3295cccdf..451211b74b 100755 --- a/frappe/utils/logger.py +++ b/frappe/utils/logger.py @@ -24,7 +24,7 @@ def get_logger(module, with_more_info=False, allow_site=True, filter=None, max_s with_more_info (bool, optional): Will log the form dict using the SiteContextFilter. Defaults to False. allow_site ((str, bool), optional): Pass site name to explicitly log under it's logs. If True and unspecified, guesses which site the logs would be saved under. Defaults to True. filter (function, optional): Add a filter function for your logger. Defaults to None. - max_size (int, optional): Max file size of each log file in kB. Defaults to 100_000. + max_size (int, optional): Max file size of each log file in bytes. Defaults to 100_000. file_count (int, optional): Max count of log files to be retained via Log Rotation. Defaults to 20. Returns: From a87531e1d0a808f69ce9d12a7e2fc9a9bb962641 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 31 Jul 2020 17:26:37 +0530 Subject: [PATCH 18/57] fix(get_logger): allow None for module --- frappe/utils/logger.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/utils/logger.py b/frappe/utils/logger.py index 451211b74b..398cdd0bfc 100755 --- a/frappe/utils/logger.py +++ b/frappe/utils/logger.py @@ -16,11 +16,11 @@ import frappe default_log_level = logging.DEBUG -def get_logger(module, with_more_info=False, allow_site=True, filter=None, max_size=100_000, file_count=20): +def get_logger(module=None, with_more_info=False, allow_site=True, filter=None, max_size=100_000, file_count=20): """Application Logger for your given module Args: - module (str): Name of your logger and consequently your log file. + module (str, optional): Name of your logger and consequently your log file. Defaults to None. with_more_info (bool, optional): Will log the form dict using the SiteContextFilter. Defaults to False. allow_site ((str, bool), optional): Pass site name to explicitly log under it's logs. If True and unspecified, guesses which site the logs would be saved under. Defaults to True. filter (function, optional): Add a filter function for your logger. Defaults to None. From c423eb7288623fb770216d712e923ee96c22e88f Mon Sep 17 00:00:00 2001 From: prssanna Date: Fri, 31 Jul 2020 18:05:23 +0530 Subject: [PATCH 19/57] fix: text in MultiSelectList doesn't get truncated --- frappe/public/js/frappe/form/controls/multiselect_list.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/form/controls/multiselect_list.js b/frappe/public/js/frappe/form/controls/multiselect_list.js index cd86bdd767..2a7ee5cb10 100644 --- a/frappe/public/js/frappe/form/controls/multiselect_list.js +++ b/frappe/public/js/frappe/form/controls/multiselect_list.js @@ -3,7 +3,7 @@ frappe.ui.form.ControlMultiSelectList = frappe.ui.form.ControlData.extend({ let template = `
- +
  • From eab96ab904d9f39739111f8c486d0cecf4240638 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Sat, 1 Aug 2020 13:42:33 +0530 Subject: [PATCH 20/57] Revert "fix(image-with-blur): Set width, height in CSS" --- frappe/templates/includes/image_with_blur.html | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/frappe/templates/includes/image_with_blur.html b/frappe/templates/includes/image_with_blur.html index dbddb9d8ef..20b0380b88 100644 --- a/frappe/templates/includes/image_with_blur.html +++ b/frappe/templates/includes/image_with_blur.html @@ -1,9 +1,7 @@ {%- set res = frappe.utils.get_thumbnail_base64_for_image(src) if src else false -%} {%- if res and res['base64'].startswith('data:') -%} {{ alt or '' }} + alt="{{ alt or '' }}" width="{{ res['width'] }}" height="{{ res['height'] }}" data-src="{{ src or '' }}" /> {%- else -%} {{ alt or '' }} {%- endif -%} From 0bc2e6e9c808126feee66b6da7b592fe451ab7ae Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Mon, 3 Aug 2020 10:44:51 +0530 Subject: [PATCH 21/57] chore: Run test in verbose mode --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 63895675ea..3ae844bc3a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,12 +31,12 @@ matrix: - name: "Python 3.7 MariaDB" python: 3.7 env: DB=mariadb TYPE=server - script: bench --site test_site run-tests --coverage + script: bench --site test_site --versbose run-tests --coverage - name: "Python 3.7 PostgreSQL" python: 3.7 env: DB=postgres TYPE=server - script: bench --site test_site run-tests --coverage + script: bench --site test_site --versbose run-tests --coverage - name: "Cypress" python: 3.7 From 817e90d17111b4618f4e3c44e623222e62255f51 Mon Sep 17 00:00:00 2001 From: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> Date: Mon, 3 Aug 2020 10:54:20 +0530 Subject: [PATCH 22/57] fix: Typo --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3ae844bc3a..eb35e0cca2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,12 +31,12 @@ matrix: - name: "Python 3.7 MariaDB" python: 3.7 env: DB=mariadb TYPE=server - script: bench --site test_site --versbose run-tests --coverage + script: bench --site test_site --verbose run-tests --coverage - name: "Python 3.7 PostgreSQL" python: 3.7 env: DB=postgres TYPE=server - script: bench --site test_site --versbose run-tests --coverage + script: bench --site test_site --verbose run-tests --coverage - name: "Cypress" python: 3.7 From f0729ff6a8a58239e68c5c0810becc3ec27b056e Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Mon, 3 Aug 2020 12:00:44 +0530 Subject: [PATCH 23/57] test: Reset workflow doc --- frappe/workflow/doctype/workflow/test_workflow.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/frappe/workflow/doctype/workflow/test_workflow.py b/frappe/workflow/doctype/workflow/test_workflow.py index 2719bc7cf0..b8c40d4d3b 100644 --- a/frappe/workflow/doctype/workflow/test_workflow.py +++ b/frappe/workflow/doctype/workflow/test_workflow.py @@ -78,7 +78,7 @@ class TestWorkflow(unittest.TestCase): frappe.set_user('test2@example.com') doc = self.test_default_condition() - workflow_actions = frappe.get_all('Workflow Action', fields=['status']) + workflow_actions = frappe.get_all('Workflow Action', fields=['status', 'reference_name']) self.assertEqual(len(workflow_actions), 1) # test if status of workflow actions are updated on approval @@ -102,6 +102,9 @@ class TestWorkflow(unittest.TestCase): todo.reload() self.assertEqual(todo.docstatus, 1) + self.workflow.states[1].doc_status = 0 + self.workflow.save() + def test_if_workflow_set_on_action(self): self.workflow.states[1].doc_status = 1 self.workflow.save() @@ -111,6 +114,9 @@ class TestWorkflow(unittest.TestCase): self.assertEqual(todo.docstatus, 1) self.assertEqual(todo.workflow_state, 'Approved') + self.workflow.states[1].doc_status = 0 + self.workflow.save() + def create_todo_workflow(): if frappe.db.exists('Workflow', 'Test ToDo'): return frappe.get_doc('Workflow', 'Test ToDo').save(ignore_permissions=True) From 9445c115ecdf9ffbf5386adaf279935648a96754 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Mon, 3 Aug 2020 15:51:30 +0530 Subject: [PATCH 24/57] fix: check if is_standard is "Yes" (#11180) --- frappe/core/doctype/report/report.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/core/doctype/report/report.js b/frappe/core/doctype/report/report.js index b4c30f5bbd..c410e9aa1a 100644 --- a/frappe/core/doctype/report/report.js +++ b/frappe/core/doctype/report/report.js @@ -1,6 +1,6 @@ frappe.ui.form.on('Report', { refresh: function(frm) { - if (frm.doc.is_standard && !frappe.boot.developer_mode) { + if (frm.doc.is_standard === "Yes" && !frappe.boot.developer_mode) { // make the document read-only frm.set_read_only(); } From 4b93f75eba40625bba5b83cca2470aa179e3f082 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 3 Aug 2020 17:23:32 +0530 Subject: [PATCH 25/57] fix: Log site only if exists --- frappe/app.py | 5 ++--- frappe/utils/logger.py | 3 ++- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/frappe/app.py b/frappe/app.py index 100b10e9ef..450e3b5f17 100644 --- a/frappe/app.py +++ b/frappe/app.py @@ -99,9 +99,8 @@ def application(request): frappe.monitor.stop(response) frappe.recorder.dump() - _site = get_site_name(request.host) - frappe.logger("frappe.web", allow_site=_site).info({ - "site": _site, + frappe.logger("frappe.web", allow_site=frappe.local.site).info({ + "site": get_site_name(request.host), "remote_addr": getattr(request, "remote_addr", "NOTFOUND"), "base_url": getattr(request, "base_url", "NOTFOUND"), "full_path": getattr(request, "full_path", "NOTFOUND"), diff --git a/frappe/utils/logger.py b/frappe/utils/logger.py index 398cdd0bfc..364ffa776d 100755 --- a/frappe/utils/logger.py +++ b/frappe/utils/logger.py @@ -11,6 +11,7 @@ from six import text_type # imports - module imports import frappe +from frappe.utils import get_sites default_log_level = logging.DEBUG @@ -33,7 +34,7 @@ def get_logger(module=None, with_more_info=False, allow_site=True, filter=None, if allow_site is True: site = getattr(frappe.local, "site", None) - elif allow_site: + elif allow_site in get_sites(): site = allow_site else: site = False From 00a8edf910b4c328aeb69f674db31439e206fd54 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Mon, 3 Aug 2020 23:53:01 +0530 Subject: [PATCH 26/57] test: Make User records before executing test --- frappe/workflow/doctype/workflow/test_workflow.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/frappe/workflow/doctype/workflow/test_workflow.py b/frappe/workflow/doctype/workflow/test_workflow.py index b8c40d4d3b..fc12832e57 100644 --- a/frappe/workflow/doctype/workflow/test_workflow.py +++ b/frappe/workflow/doctype/workflow/test_workflow.py @@ -5,7 +5,10 @@ from __future__ import unicode_literals import frappe import unittest from frappe.utils import random_string -from frappe.model.workflow import apply_workflow, WorkflowTransitionError, WorkflowPermissionError, get_common_transition_actions +from frappe.model.workflow import apply_workflow, WorkflowTransitionError, WorkflowPermissionError, get_common_transition_actions, get_workflow +from frappe.test_runner import make_test_records + +make_test_records("User") class TestWorkflow(unittest.TestCase): def setUp(self): From e555970485467b69cddcbc7fce990f94c722f1cc Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Mon, 3 Aug 2020 23:53:44 +0530 Subject: [PATCH 27/57] chore: Comment out possibly problematic test --- .../event_producer/test_event_producer.py | 80 +++++++++---------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/frappe/event_streaming/doctype/event_producer/test_event_producer.py b/frappe/event_streaming/doctype/event_producer/test_event_producer.py index 0d414f5d63..cda4ca8bf6 100644 --- a/frappe/event_streaming/doctype/event_producer/test_event_producer.py +++ b/frappe/event_streaming/doctype/event_producer/test_event_producer.py @@ -203,49 +203,49 @@ class TestEventProducer(unittest.TestCase): reset_configuration(producer_url) - def test_inner_mapping(self): - producer = get_remote_site() - event_producer = frappe.get_doc('Event Producer', producer_url) - event_producer.producer_doctypes = [] - inner_mapping = [ - { - 'local_fieldname':'role_name', - 'remote_fieldname':'title' - } - ] - inner_map = get_mapping('Role to Note Dependency Creation', 'Role', 'Note', inner_mapping) - mapping = [ - { - 'local_fieldname':'description', - 'remote_fieldname':'content', - }, - { - 'local_fieldname': 'role', - 'remote_fieldname': 'title', - 'mapping_type': 'Document', - 'mapping': inner_map, - 'remote_value_filters': json.dumps({'title': 'title'}) - } - ] - event_producer.append('producer_doctypes', { - 'ref_doctype': 'ToDo', - 'use_same_name': 1, - 'has_mapping': 1, - 'mapping': get_mapping('ToDo to Note Mapping', 'ToDo', 'Note', mapping) - }) - event_producer.save() + # def test_inner_mapping(self): + # producer = get_remote_site() + # event_producer = frappe.get_doc('Event Producer', producer_url) + # event_producer.producer_doctypes = [] + # inner_mapping = [ + # { + # 'local_fieldname':'role_name', + # 'remote_fieldname':'title' + # } + # ] + # inner_map = get_mapping('Role to Note Dependency Creation', 'Role', 'Note', inner_mapping) + # mapping = [ + # { + # 'local_fieldname':'description', + # 'remote_fieldname':'content', + # }, + # { + # 'local_fieldname': 'role', + # 'remote_fieldname': 'title', + # 'mapping_type': 'Document', + # 'mapping': inner_map, + # 'remote_value_filters': json.dumps({'title': 'title'}) + # } + # ] + # event_producer.append('producer_doctypes', { + # 'ref_doctype': 'ToDo', + # 'use_same_name': 1, + # 'has_mapping': 1, + # 'mapping': get_mapping('ToDo to Note Mapping', 'ToDo', 'Note', mapping) + # }) + # event_producer.save() - producer_note = frappe._dict(doctype='Note', title='Inner Mapping Tester', content='Test Inner Mapping') - delete_on_remote_if_exists(producer, 'Note', {'title': producer_note.title}) - producer_note = producer.insert(producer_note) - self.pull_producer_data() + # producer_note = frappe._dict(doctype='Note', title='Inner Mapping Tester', content='Test Inner Mapping') + # delete_on_remote_if_exists(producer, 'Note', {'title': producer_note.title}) + # producer_note = producer.insert(producer_note) + # self.pull_producer_data() - #check dependency inserted - self.assertTrue(frappe.db.exists('Role', {'role_name': producer_note.title})) - #check doc inserted - self.assertTrue(frappe.db.exists('ToDo', {'description': producer_note.content})) + # #check dependency inserted + # self.assertTrue(frappe.db.exists('Role', {'role_name': producer_note.title})) + # #check doc inserted + # self.assertTrue(frappe.db.exists('ToDo', {'description': producer_note.content})) - reset_configuration(producer_url) + # reset_configuration(producer_url) def insert_into_producer(producer, description): From 88b68c6cf51061b4b1a37b93b5426279fc1dfe9d Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Tue, 4 Aug 2020 00:11:48 +0530 Subject: [PATCH 28/57] chore: Remove unwanted changes --- .travis.yml | 4 ++-- frappe/workflow/doctype/workflow/test_workflow.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index eb35e0cca2..63895675ea 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,12 +31,12 @@ matrix: - name: "Python 3.7 MariaDB" python: 3.7 env: DB=mariadb TYPE=server - script: bench --site test_site --verbose run-tests --coverage + script: bench --site test_site run-tests --coverage - name: "Python 3.7 PostgreSQL" python: 3.7 env: DB=postgres TYPE=server - script: bench --site test_site --verbose run-tests --coverage + script: bench --site test_site run-tests --coverage - name: "Cypress" python: 3.7 diff --git a/frappe/workflow/doctype/workflow/test_workflow.py b/frappe/workflow/doctype/workflow/test_workflow.py index fc12832e57..253f95dab7 100644 --- a/frappe/workflow/doctype/workflow/test_workflow.py +++ b/frappe/workflow/doctype/workflow/test_workflow.py @@ -5,7 +5,7 @@ from __future__ import unicode_literals import frappe import unittest from frappe.utils import random_string -from frappe.model.workflow import apply_workflow, WorkflowTransitionError, WorkflowPermissionError, get_common_transition_actions, get_workflow +from frappe.model.workflow import apply_workflow, WorkflowTransitionError, WorkflowPermissionError, get_common_transition_actions from frappe.test_runner import make_test_records make_test_records("User") From 89dfd82134d02d1a88444d02f858be820371129b Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Tue, 4 Aug 2020 00:36:42 +0530 Subject: [PATCH 29/57] chore: Add debug statement - Uncomment innocent test --- .../event_producer/test_event_producer.py | 80 +++++++++---------- .../doctype/workflow/test_workflow.py | 3 + 2 files changed, 43 insertions(+), 40 deletions(-) diff --git a/frappe/event_streaming/doctype/event_producer/test_event_producer.py b/frappe/event_streaming/doctype/event_producer/test_event_producer.py index cda4ca8bf6..0d414f5d63 100644 --- a/frappe/event_streaming/doctype/event_producer/test_event_producer.py +++ b/frappe/event_streaming/doctype/event_producer/test_event_producer.py @@ -203,49 +203,49 @@ class TestEventProducer(unittest.TestCase): reset_configuration(producer_url) - # def test_inner_mapping(self): - # producer = get_remote_site() - # event_producer = frappe.get_doc('Event Producer', producer_url) - # event_producer.producer_doctypes = [] - # inner_mapping = [ - # { - # 'local_fieldname':'role_name', - # 'remote_fieldname':'title' - # } - # ] - # inner_map = get_mapping('Role to Note Dependency Creation', 'Role', 'Note', inner_mapping) - # mapping = [ - # { - # 'local_fieldname':'description', - # 'remote_fieldname':'content', - # }, - # { - # 'local_fieldname': 'role', - # 'remote_fieldname': 'title', - # 'mapping_type': 'Document', - # 'mapping': inner_map, - # 'remote_value_filters': json.dumps({'title': 'title'}) - # } - # ] - # event_producer.append('producer_doctypes', { - # 'ref_doctype': 'ToDo', - # 'use_same_name': 1, - # 'has_mapping': 1, - # 'mapping': get_mapping('ToDo to Note Mapping', 'ToDo', 'Note', mapping) - # }) - # event_producer.save() + def test_inner_mapping(self): + producer = get_remote_site() + event_producer = frappe.get_doc('Event Producer', producer_url) + event_producer.producer_doctypes = [] + inner_mapping = [ + { + 'local_fieldname':'role_name', + 'remote_fieldname':'title' + } + ] + inner_map = get_mapping('Role to Note Dependency Creation', 'Role', 'Note', inner_mapping) + mapping = [ + { + 'local_fieldname':'description', + 'remote_fieldname':'content', + }, + { + 'local_fieldname': 'role', + 'remote_fieldname': 'title', + 'mapping_type': 'Document', + 'mapping': inner_map, + 'remote_value_filters': json.dumps({'title': 'title'}) + } + ] + event_producer.append('producer_doctypes', { + 'ref_doctype': 'ToDo', + 'use_same_name': 1, + 'has_mapping': 1, + 'mapping': get_mapping('ToDo to Note Mapping', 'ToDo', 'Note', mapping) + }) + event_producer.save() - # producer_note = frappe._dict(doctype='Note', title='Inner Mapping Tester', content='Test Inner Mapping') - # delete_on_remote_if_exists(producer, 'Note', {'title': producer_note.title}) - # producer_note = producer.insert(producer_note) - # self.pull_producer_data() + producer_note = frappe._dict(doctype='Note', title='Inner Mapping Tester', content='Test Inner Mapping') + delete_on_remote_if_exists(producer, 'Note', {'title': producer_note.title}) + producer_note = producer.insert(producer_note) + self.pull_producer_data() - # #check dependency inserted - # self.assertTrue(frappe.db.exists('Role', {'role_name': producer_note.title})) - # #check doc inserted - # self.assertTrue(frappe.db.exists('ToDo', {'description': producer_note.content})) + #check dependency inserted + self.assertTrue(frappe.db.exists('Role', {'role_name': producer_note.title})) + #check doc inserted + self.assertTrue(frappe.db.exists('ToDo', {'description': producer_note.content})) - # reset_configuration(producer_url) + reset_configuration(producer_url) def insert_into_producer(producer, description): diff --git a/frappe/workflow/doctype/workflow/test_workflow.py b/frappe/workflow/doctype/workflow/test_workflow.py index 253f95dab7..f94193fab9 100644 --- a/frappe/workflow/doctype/workflow/test_workflow.py +++ b/frappe/workflow/doctype/workflow/test_workflow.py @@ -30,6 +30,9 @@ class TestWorkflow(unittest.TestCase): def test_approve(self, doc=None): '''test simple workflow''' todo = doc or self.test_default_condition() + print(frappe.db.sql("select name from `tabRole`")) + print('------------') + print(frappe.get_roles(), frappe.session.user) apply_workflow(todo, 'Approve') # default condition is set From 6e9169a0aadc6b05af83432293297220e633631d Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Tue, 4 Aug 2020 10:12:41 +0530 Subject: [PATCH 30/57] test: Remove debug statements & clear-cache --- frappe/workflow/doctype/workflow/test_workflow.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/frappe/workflow/doctype/workflow/test_workflow.py b/frappe/workflow/doctype/workflow/test_workflow.py index f94193fab9..fb42e6fd6e 100644 --- a/frappe/workflow/doctype/workflow/test_workflow.py +++ b/frappe/workflow/doctype/workflow/test_workflow.py @@ -30,9 +30,6 @@ class TestWorkflow(unittest.TestCase): def test_approve(self, doc=None): '''test simple workflow''' todo = doc or self.test_default_condition() - print(frappe.db.sql("select name from `tabRole`")) - print('------------') - print(frappe.get_roles(), frappe.session.user) apply_workflow(todo, 'Approve') # default condition is set @@ -129,6 +126,7 @@ def create_todo_workflow(): else: frappe.get_doc(dict(doctype='Role', role_name='Test Approver')).insert(ignore_if_duplicate=True) + frappe.cache().hdel('roles', 'Administrator') workflow = frappe.new_doc('Workflow') workflow.workflow_name = 'Test ToDo' workflow.document_type = 'ToDo' From 744a9413fdc7ffee3ac26883add9a950c3f0ab19 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Tue, 4 Aug 2020 10:36:16 +0530 Subject: [PATCH 31/57] fix: Use common function to get_roles --- frappe/__init__.py | 6 +----- frappe/workflow/doctype/workflow/test_workflow.py | 1 - 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index d644d2a473..f91317584d 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -437,11 +437,7 @@ def get_roles(username=None): if not local.session: return ["Guest"] - if username: - import frappe.permissions - return frappe.permissions.get_roles(username) - else: - return get_user().get_roles() + return frappe.permissions.get_roles(username or local.session.user) def get_request_header(key, default=None): """Return HTTP request header. diff --git a/frappe/workflow/doctype/workflow/test_workflow.py b/frappe/workflow/doctype/workflow/test_workflow.py index fb42e6fd6e..253f95dab7 100644 --- a/frappe/workflow/doctype/workflow/test_workflow.py +++ b/frappe/workflow/doctype/workflow/test_workflow.py @@ -126,7 +126,6 @@ def create_todo_workflow(): else: frappe.get_doc(dict(doctype='Role', role_name='Test Approver')).insert(ignore_if_duplicate=True) - frappe.cache().hdel('roles', 'Administrator') workflow = frappe.new_doc('Workflow') workflow.workflow_name = 'Test ToDo' workflow.document_type = 'ToDo' From 3c3b1037eaf9b7e92a52dd7c391d18f969d06102 Mon Sep 17 00:00:00 2001 From: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> Date: Tue, 4 Aug 2020 10:44:45 +0530 Subject: [PATCH 32/57] fix: Import frappe.permissions --- frappe/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index f91317584d..31ce00e440 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -436,7 +436,7 @@ def get_roles(username=None): """Returns roles of current user.""" if not local.session: return ["Guest"] - + import frappe.permissions return frappe.permissions.get_roles(username or local.session.user) def get_request_header(key, default=None): From 86a9ceb29b93a0df1db233bb461adbcd0b005739 Mon Sep 17 00:00:00 2001 From: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> Date: Tue, 4 Aug 2020 11:16:18 +0530 Subject: [PATCH 33/57] test: Clear role cache --- frappe/workflow/doctype/workflow/test_workflow.py | 1 + 1 file changed, 1 insertion(+) diff --git a/frappe/workflow/doctype/workflow/test_workflow.py b/frappe/workflow/doctype/workflow/test_workflow.py index 253f95dab7..d7f7939d3d 100644 --- a/frappe/workflow/doctype/workflow/test_workflow.py +++ b/frappe/workflow/doctype/workflow/test_workflow.py @@ -126,6 +126,7 @@ def create_todo_workflow(): else: frappe.get_doc(dict(doctype='Role', role_name='Test Approver')).insert(ignore_if_duplicate=True) + frappe.cache().hdel('roles', frappe.session.user) workflow = frappe.new_doc('Workflow') workflow.workflow_name = 'Test ToDo' workflow.document_type = 'ToDo' From 645331799af44feb4629ccc50f214278579be200 Mon Sep 17 00:00:00 2001 From: Prssanna Desai Date: Tue, 4 Aug 2020 12:17:33 +0530 Subject: [PATCH 34/57] fix: whitelist functions called through search widget (#11153) --- frappe/core/doctype/user_permission/user_permission.py | 1 + .../permitted_documents_for_user.py | 1 + frappe/desk/doctype/dashboard_chart/dashboard_chart.py | 1 + frappe/desk/doctype/number_card/number_card.py | 4 ++++ frappe/email/__init__.py | 1 + 5 files changed, 8 insertions(+) diff --git a/frappe/core/doctype/user_permission/user_permission.py b/frappe/core/doctype/user_permission/user_permission.py index 48d4fcb5d4..325c41622d 100644 --- a/frappe/core/doctype/user_permission/user_permission.py +++ b/frappe/core/doctype/user_permission/user_permission.py @@ -119,6 +119,7 @@ def user_permission_exists(user, allow, for_value, applicable_for=None): return has_same_user_permission +@frappe.whitelist() def get_applicable_for_doctype_list(doctype, txt, searchfield, start, page_len, filters): linked_doctypes_map = get_linked_doctypes(doctype, True) diff --git a/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.py b/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.py index 8b2d1e01fa..5ca21b811e 100644 --- a/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.py +++ b/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.py @@ -41,6 +41,7 @@ def get_columns_and_fields(doctype): return columns, fields +@frappe.whitelist() def query_doctypes(doctype, txt, searchfield, start, page_len, filters): user = filters.get("user") user_perms = frappe.utils.user.UserPermissions(user) diff --git a/frappe/desk/doctype/dashboard_chart/dashboard_chart.py b/frappe/desk/doctype/dashboard_chart/dashboard_chart.py index a25e0cab9e..7547efa234 100644 --- a/frappe/desk/doctype/dashboard_chart/dashboard_chart.py +++ b/frappe/desk/doctype/dashboard_chart/dashboard_chart.py @@ -344,6 +344,7 @@ def get_year_ending(date): # last day of this month return add_to_date(date, days=-1) +@frappe.whitelist() def get_charts_for_user(doctype, txt, searchfield, start, page_len, filters): or_filters = {'owner': frappe.session.user, 'is_public': 1} return frappe.db.get_list('Dashboard Chart', diff --git a/frappe/desk/doctype/number_card/number_card.py b/frappe/desk/doctype/number_card/number_card.py index 3eb08ec00a..f44893d950 100644 --- a/frappe/desk/doctype/number_card/number_card.py +++ b/frappe/desk/doctype/number_card/number_card.py @@ -124,11 +124,15 @@ def create_number_card(args): doc.insert(ignore_permissions=True) return doc +@frappe.whitelist() def get_cards_for_user(doctype, txt, searchfield, start, page_len, filters): meta = frappe.get_meta(doctype) searchfields = meta.get_search_fields() search_conditions = [] + if not frappe.db.exists('DocType', doctype): + return + if txt: for field in searchfields: search_conditions.append('`tab{doctype}`.`{field}` like %(txt)s'.format(field=field, doctype=doctype, txt=txt)) diff --git a/frappe/email/__init__.py b/frappe/email/__init__.py index d58b35040e..0b874a03bb 100644 --- a/frappe/email/__init__.py +++ b/frappe/email/__init__.py @@ -57,6 +57,7 @@ def relink(name, reference_doctype=None, reference_name=None): communication_type = "Communication" and name = %s""", (reference_doctype, reference_name, name)) +@frappe.whitelist() def get_communication_doctype(doctype, txt, searchfield, start, page_len, filters): user_perms = frappe.utils.user.UserPermissions(frappe.session.user) user_perms.build_permissions() From 9b267027b5ce2c0674a4dd1cb476d5b80b873632 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Tue, 4 Aug 2020 13:25:49 +0530 Subject: [PATCH 35/57] feat: don't load user translation in make_dict_from_messages (#11120) --- frappe/translate.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/frappe/translate.py b/frappe/translate.py index 2fc2d3f328..ee1f8fa5ff 100644 --- a/frappe/translate.py +++ b/frappe/translate.py @@ -119,7 +119,7 @@ def get_dict(fortype, name=None): messages += frappe.db.sql("select 'Role:', name from tabRole") messages += frappe.db.sql("select 'Module:', name from `tabModule Def`") - message_dict = make_dict_from_messages(messages) + message_dict = make_dict_from_messages(messages, load_user_translation=False) message_dict.update(get_dict_from_hooks(fortype, name)) # remove untranslated message_dict = {k:v for k, v in iteritems(message_dict) if k!=v} @@ -127,6 +127,7 @@ def get_dict(fortype, name=None): cache.hset("translation_assets", frappe.local.lang, translation_assets, shared=True) translation_map = translation_assets[asset_key] + if fortype == "boot": translation_map.update(get_user_translations(frappe.local.lang)) @@ -144,14 +145,17 @@ def get_dict_from_hooks(fortype, name): return translated_dict -def make_dict_from_messages(messages, full_dict=None): +def make_dict_from_messages(messages, full_dict=None, load_user_translation=True): """Returns translated messages as a dict in Language specified in `frappe.local.lang` :param messages: List of untranslated messages """ out = {} if full_dict==None: - full_dict = get_full_dict(frappe.local.lang) + if load_user_translation: + full_dict = get_full_dict(frappe.local.lang) + else: + full_dict = load_lang(frappe.local.lang) for m in messages: if m[1] in full_dict: @@ -189,11 +193,9 @@ def get_full_dict(lang): try: # get user specific transaltion data user_translations = get_user_translations(lang) - except Exception: - user_translations = None - - if user_translations: frappe.local.lang_full_dict.update(user_translations) + except Exception: + pass return frappe.local.lang_full_dict From d5622f541021e9ecfa682bf30977baeb3e48f89b Mon Sep 17 00:00:00 2001 From: Prssanna Desai Date: Tue, 4 Aug 2020 17:38:48 +0530 Subject: [PATCH 36/57] fix: Permission query in Dashboard Chart (#10981) Co-authored-by: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> --- .../dashboard_chart/dashboard_chart.py | 25 ++++++++++++++----- .../desk/doctype/number_card/number_card.py | 14 +++++++---- 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/frappe/desk/doctype/dashboard_chart/dashboard_chart.py b/frappe/desk/doctype/dashboard_chart/dashboard_chart.py index 7547efa234..88c9623945 100644 --- a/frappe/desk/doctype/dashboard_chart/dashboard_chart.py +++ b/frappe/desk/doctype/dashboard_chart/dashboard_chart.py @@ -28,15 +28,28 @@ def get_permission_query_conditions(user): if "System Manager" in roles: return None - allowed_doctypes = ['"%s"' % doctype for doctype in frappe.permissions.get_doctypes_with_read()] - allowed_reports = ['"%s"' % key if type(key) == str else key.encode('UTF8') for key in get_allowed_reports()] + doctype_condition = False + report_condition = False + + allowed_doctypes = [frappe.db.escape(doctype) for doctype in frappe.permissions.get_doctypes_with_read()] + allowed_reports = [frappe.db.escape(key) if type(key) == str else key.encode('UTF8') for key in get_allowed_reports()] + + if allowed_doctypes: + doctype_condition = '`tabDashboard Chart`.`document_type` in ({allowed_doctypes})'.format( + allowed_doctypes=','.join(allowed_doctypes)) + if allowed_reports: + report_condition = '`tabDashboard Chart`.`report_name` in ({allowed_reports})'.format( + allowed_reports=','.join(allowed_reports)) return ''' - `tabDashboard Chart`.`document_type` in ({allowed_doctypes}) - or `tabDashboard Chart`.`report_name` in ({allowed_reports}) + (`tabDashboard Chart`.`chart_type` in ('Count', 'Sum', 'Average') + and {doctype_condition}) + or + (`tabDashboard Chart`.`chart_type` = 'Report' + and {report_condition}) '''.format( - allowed_doctypes=','.join(allowed_doctypes), - allowed_reports=','.join(allowed_reports) + doctype_condition=doctype_condition, + report_condition=report_condition ) diff --git a/frappe/desk/doctype/number_card/number_card.py b/frappe/desk/doctype/number_card/number_card.py index f44893d950..68ed79e64b 100644 --- a/frappe/desk/doctype/number_card/number_card.py +++ b/frappe/desk/doctype/number_card/number_card.py @@ -32,13 +32,17 @@ def get_permission_query_conditions(user=None): if "System Manager" in roles: return None - allowed_doctypes = ['"%s"' % doctype for doctype in frappe.permissions.get_doctypes_with_read()] + doctype_condition = False + + allowed_doctypes = [frappe.db.escape(doctype) for doctype in frappe.permissions.get_doctypes_with_read()] + + if allowed_doctypes: + doctype_condition = '`tabNumber Card`.`document_type` in ({allowed_doctypes})'.format( + allowed_doctypes=','.join(allowed_doctypes)) return ''' - `tabNumber Card`.`document_type` in ({allowed_doctypes}) - '''.format( - allowed_doctypes=','.join(allowed_doctypes) - ) + {doctype_condition} + '''.format(doctype_condition=doctype_condition) def has_permission(doc, ptype, user): roles = frappe.get_roles(user) From dd157ba82255727e0095925eaf8d64fbfe89b275 Mon Sep 17 00:00:00 2001 From: gavin Date: Tue, 4 Aug 2020 17:39:35 +0530 Subject: [PATCH 37/57] fix: dont show git errors in terminal (#11064) Co-authored-by: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> --- frappe/utils/change_log.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/frappe/utils/change_log.py b/frappe/utils/change_log.py index 53c83cfe99..c75b3289db 100644 --- a/frappe/utils/change_log.py +++ b/frappe/utils/change_log.py @@ -118,8 +118,9 @@ def get_versions(): def get_app_branch(app): '''Returns branch of an app''' try: + null_stream = open(os.devnull, 'wb') result = subprocess.check_output('cd ../apps/{0} && git rev-parse --abbrev-ref HEAD'.format(app), - shell=True) + shell=True, stdin=null_stream, stderr=null_stream) result = safe_decode(result) result = result.strip() return result @@ -128,8 +129,9 @@ def get_app_branch(app): def get_app_last_commit_ref(app): try: + null_stream = open(os.devnull, 'wb') result = subprocess.check_output('cd ../apps/{0} && git rev-parse HEAD --short 7'.format(app), - shell=True) + shell=True, stdin=null_stream, stderr=null_stream) result = safe_decode(result) result = result.strip() return result From 34fdf63b9097d5ad0cfb366121242c0f8480e460 Mon Sep 17 00:00:00 2001 From: Glen Whitney Date: Tue, 4 Aug 2020 05:10:06 -0700 Subject: [PATCH 38/57] fix(Link Card): refresh incomplete_dependencies as needed (#11169) When _prepare_item in Workspace.get_cards notices that the dependencies for an item have been completed, it should clear the incomplete_dependencies field in case a value has been saved in that attribute. Fixes #11168. --- frappe/desk/desktop.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frappe/desk/desktop.py b/frappe/desk/desktop.py index 68b57a93d4..ae9d070976 100644 --- a/frappe/desk/desktop.py +++ b/frappe/desk/desktop.py @@ -221,6 +221,8 @@ class Workspace: incomplete_dependencies = [d for d in item.dependencies if not _doctype_contains_a_record(d)] if len(incomplete_dependencies): item.incomplete_dependencies = incomplete_dependencies + else: + item.incomplete_dependencies = "" if item.onboard: # Mark Spotlights for initial From f5c61d04da27d8258caad8631c5f7f4edf3ffa85 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Tue, 4 Aug 2020 17:40:18 +0530 Subject: [PATCH 39/57] fix: Patch to generate theme files in public folder (#11148) Co-authored-by: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> --- frappe/patches.txt | 1 + .../generate_theme_files_in_public_folder.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 frappe/patches/v13_0/generate_theme_files_in_public_folder.py diff --git a/frappe/patches.txt b/frappe/patches.txt index b207a325cb..75750ab59c 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -296,3 +296,4 @@ frappe.patches.v13_0.update_duration_options frappe.patches.v13_0.replace_old_data_import # 2020-06-24 frappe.patches.v13_0.create_custom_dashboards_cards_and_charts frappe.patches.v13_0.rename_is_custom_field_in_dashboard_chart +frappe.patches.v13_0.generate_theme_files_in_public_folder diff --git a/frappe/patches/v13_0/generate_theme_files_in_public_folder.py b/frappe/patches/v13_0/generate_theme_files_in_public_folder.py new file mode 100644 index 0000000000..c5a64780cd --- /dev/null +++ b/frappe/patches/v13_0/generate_theme_files_in_public_folder.py @@ -0,0 +1,15 @@ +# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors +# MIT License. See license.txt + +from __future__ import unicode_literals +import frappe + + +def execute(): + themes = frappe.db.get_all( + "Website Theme", filters={"theme_url": ("not like", "/files/website_theme/%")} + ) + for theme in themes: + doc = frappe.get_doc("Website Theme", theme.name) + doc.generate_bootstrap_theme() + doc.save() From 0748312e3950516bb08c76f200359757cd2c6759 Mon Sep 17 00:00:00 2001 From: Aditya Hase Date: Tue, 4 Aug 2020 17:41:34 +0530 Subject: [PATCH 40/57] fix(werkzeug): Avoid duplicate lines from werkzeug in bench start logs (#11171) Co-authored-by: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> --- frappe/app.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frappe/app.py b/frappe/app.py index 450e3b5f17..725bec183a 100644 --- a/frappe/app.py +++ b/frappe/app.py @@ -256,9 +256,11 @@ def serve(port=8000, profile=False, no_reload=False, no_threading=False, site=No 'SERVER_NAME': 'localhost:8000' } + log = logging.getLogger('werkzeug') + log.propagate = False + in_test_env = os.environ.get('CI') if in_test_env: - log = logging.getLogger('werkzeug') log.setLevel(logging.ERROR) run_simple('0.0.0.0', int(port), application, From cc22350b04f73b6321e1fac6ccf62f57ee9776fb Mon Sep 17 00:00:00 2001 From: Rucha Mahabal Date: Tue, 4 Aug 2020 17:42:47 +0530 Subject: [PATCH 41/57] fix: remove Video DocType from Core module (#11151) Co-authored-by: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> --- frappe/automation/desk_page/tools/tools.json | 5 +- frappe/core/doctype/video/__init__.py | 0 frappe/core/doctype/video/test_video.py | 10 -- frappe/core/doctype/video/video.js | 8 -- frappe/core/doctype/video/video.json | 106 ------------------- frappe/core/doctype/video/video.py | 10 -- frappe/public/js/frappe/views/breadcrumbs.js | 1 - 7 files changed, 3 insertions(+), 137 deletions(-) delete mode 100644 frappe/core/doctype/video/__init__.py delete mode 100644 frappe/core/doctype/video/test_video.py delete mode 100644 frappe/core/doctype/video/video.js delete mode 100644 frappe/core/doctype/video/video.json delete mode 100644 frappe/core/doctype/video/video.py diff --git a/frappe/automation/desk_page/tools/tools.json b/frappe/automation/desk_page/tools/tools.json index 2164a4ce38..3fbaf62d02 100644 --- a/frappe/automation/desk_page/tools/tools.json +++ b/frappe/automation/desk_page/tools/tools.json @@ -3,7 +3,7 @@ { "hidden": 0, "label": "Tools", - "links": "[\n {\n \"description\": \"Documents assigned to you and by you.\",\n \"label\": \"To Do\",\n \"name\": \"ToDo\",\n \"onboard\": 1,\n \"type\": \"doctype\"\n },\n {\n \"description\": \"Event and other calendars.\",\n \"label\": \"Calendar\",\n \"link\": \"List/Event/Calendar\",\n \"name\": \"Event\",\n \"onboard\": 1,\n \"type\": \"doctype\"\n },\n {\n \"description\": \"Private and public Notes.\",\n \"label\": \"Note\",\n \"name\": \"Note\",\n \"onboard\": 1,\n \"type\": \"doctype\"\n },\n {\n \"label\": \"Files\",\n \"name\": \"File\",\n \"type\": \"doctype\"\n },\n {\n \"label\": \"Video\",\n \"name\": \"Video\",\n \"type\": \"doctype\"\n },\n {\n \"description\": \"Activity log of all users.\",\n \"label\": \"Activity\",\n \"name\": \"activity\",\n \"type\": \"page\"\n }\n]" + "links": "[\n {\n \"description\": \"Documents assigned to you and by you.\",\n \"label\": \"To Do\",\n \"name\": \"ToDo\",\n \"onboard\": 1,\n \"type\": \"doctype\"\n },\n {\n \"description\": \"Event and other calendars.\",\n \"label\": \"Calendar\",\n \"link\": \"List/Event/Calendar\",\n \"name\": \"Event\",\n \"onboard\": 1,\n \"type\": \"doctype\"\n },\n {\n \"description\": \"Private and public Notes.\",\n \"label\": \"Note\",\n \"name\": \"Note\",\n \"onboard\": 1,\n \"type\": \"doctype\"\n },\n {\n \"label\": \"Files\",\n \"name\": \"File\",\n \"type\": \"doctype\"\n },\n {\n \"description\": \"Activity log of all users.\",\n \"label\": \"Activity\",\n \"name\": \"activity\",\n \"type\": \"page\"\n }\n]" }, { "hidden": 0, @@ -29,10 +29,11 @@ "docstatus": 0, "doctype": "Desk Page", "extends_another_page": 0, + "hide_custom": 0, "idx": 0, "is_standard": 1, "label": "Tools", - "modified": "2020-04-20 18:21:14.152537", + "modified": "2020-07-21 19:32:18.480700", "modified_by": "Administrator", "module": "Automation", "name": "Tools", diff --git a/frappe/core/doctype/video/__init__.py b/frappe/core/doctype/video/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/core/doctype/video/test_video.py b/frappe/core/doctype/video/test_video.py deleted file mode 100644 index 0bed1e98d6..0000000000 --- a/frappe/core/doctype/video/test_video.py +++ /dev/null @@ -1,10 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2020, Frappe Technologies and Contributors -# See license.txt -from __future__ import unicode_literals - -# import frappe -import unittest - -class TestVideo(unittest.TestCase): - pass diff --git a/frappe/core/doctype/video/video.js b/frappe/core/doctype/video/video.js deleted file mode 100644 index 36ea240a36..0000000000 --- a/frappe/core/doctype/video/video.js +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright (c) 2020, Frappe Technologies and contributors -// For license information, please see license.txt - -frappe.ui.form.on('Video', { - // refresh: function(frm) { - - // } -}); diff --git a/frappe/core/doctype/video/video.json b/frappe/core/doctype/video/video.json deleted file mode 100644 index 26a407c05c..0000000000 --- a/frappe/core/doctype/video/video.json +++ /dev/null @@ -1,106 +0,0 @@ -{ - "actions": [], - "allow_import": 1, - "allow_rename": 1, - "autoname": "field:title", - "creation": "2018-10-17 05:47:13.087395", - "doctype": "DocType", - "editable_grid": 1, - "engine": "InnoDB", - "field_order": [ - "title", - "provider", - "url", - "column_break_4", - "publish_date", - "duration", - "section_break_7", - "description" - ], - "fields": [ - { - "fieldname": "title", - "fieldtype": "Data", - "in_list_view": 1, - "label": "Title", - "reqd": 1, - "unique": 1 - }, - { - "fieldname": "provider", - "fieldtype": "Select", - "in_list_view": 1, - "label": "Provider", - "options": "YouTube\nVimeo", - "reqd": 1 - }, - { - "fieldname": "url", - "fieldtype": "Data", - "in_list_view": 1, - "label": "URL", - "reqd": 1 - }, - { - "fieldname": "column_break_4", - "fieldtype": "Column Break" - }, - { - "fieldname": "publish_date", - "fieldtype": "Date", - "label": "Publish Date" - }, - { - "fieldname": "duration", - "fieldtype": "Data", - "label": "Duration" - }, - { - "fieldname": "section_break_7", - "fieldtype": "Section Break" - }, - { - "fieldname": "description", - "fieldtype": "Text Editor", - "in_list_view": 1, - "label": "Description", - "reqd": 1 - } - ], - "links": [], - "modified": "2020-04-22 12:09:49.057403", - "modified_by": "Administrator", - "module": "Core", - "name": "Video", - "owner": "Administrator", - "permissions": [ - { - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "if_owner": 1, - "print": 1, - "read": 1, - "report": 1, - "role": "All", - "share": 1, - "write": 1 - }, - { - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "share": 1, - "write": 1 - } - ], - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1 -} \ No newline at end of file diff --git a/frappe/core/doctype/video/video.py b/frappe/core/doctype/video/video.py deleted file mode 100644 index fdbd3a1abe..0000000000 --- a/frappe/core/doctype/video/video.py +++ /dev/null @@ -1,10 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2020, Frappe Technologies and contributors -# For license information, please see license.txt - -from __future__ import unicode_literals -# import frappe -from frappe.model.document import Document - -class Video(Document): - pass diff --git a/frappe/public/js/frappe/views/breadcrumbs.js b/frappe/public/js/frappe/views/breadcrumbs.js index 0058310e3f..dbe591f6fd 100644 --- a/frappe/public/js/frappe/views/breadcrumbs.js +++ b/frappe/public/js/frappe/views/breadcrumbs.js @@ -6,7 +6,6 @@ frappe.breadcrumbs = { preferred: { "File": "", - "Video": "", "Dashboard": "Customization", "Dashboard Chart": "Customization", "Dashboard Chart Source": "Customization" From 5d4feb5dbb264611d2d257eea8f8f45b9636fbae Mon Sep 17 00:00:00 2001 From: gavin Date: Tue, 4 Aug 2020 17:45:14 +0530 Subject: [PATCH 42/57] feat: Convenience APIs for tags (#11088) Co-authored-by: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> --- frappe/model/document.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/frappe/model/document.py b/frappe/model/document.py index 69a781d6d1..316c576f55 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -1307,6 +1307,16 @@ class Document(BaseDocument): users = set([assignment.owner for assignment in assignments]) return users + def add_tag(self, tag): + """Add a Tag to this document""" + from frappe.desk.doctype.tag.tag import DocTags + DocTags(self.doctype).add(self.name, tag) + + def get_tags(self): + """Return a list of Tags attached to this document""" + from frappe.desk.doctype.tag.tag import DocTags + return DocTags(self.doctype).get_tags(self.name).split(",")[1:] + def execute_action(doctype, name, action, **kwargs): """Execute an action on a document (called by background worker)""" doc = frappe.get_doc(doctype, name) From 805e415502164c49aeef5a6332208ef2075c258d Mon Sep 17 00:00:00 2001 From: Prssanna Desai Date: Wed, 5 Aug 2020 14:57:48 +0530 Subject: [PATCH 43/57] fix: Dashboard fixes (#11186) Co-authored-by: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> --- frappe/desk/doctype/dashboard_chart/dashboard_chart.js | 6 ------ frappe/desk/doctype/dashboard_chart/dashboard_chart.py | 2 +- frappe/desk/doctype/number_card/number_card.py | 2 +- frappe/public/js/frappe/widgets/utils.js | 10 ++++++++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/frappe/desk/doctype/dashboard_chart/dashboard_chart.js b/frappe/desk/doctype/dashboard_chart/dashboard_chart.js index 8d89cc2f31..7f26bd9101 100644 --- a/frappe/desk/doctype/dashboard_chart/dashboard_chart.js +++ b/frappe/desk/doctype/dashboard_chart/dashboard_chart.js @@ -23,7 +23,6 @@ frappe.ui.form.on('Dashboard Chart', { frm.chart_filters = null; if (!frappe.boot.developer_mode && frm.doc.is_standard) { - frm.set_df_property('chart_options_section', 'hidden', 1); frm.disable_form(); } @@ -57,11 +56,6 @@ frappe.ui.form.on('Dashboard Chart', { if (frm.doc.report_name) { frm.trigger('set_chart_report_filters'); } - - if (!frappe.boot.developer_mode) { - frm.set_df_property("custom_options", "hidden", 1); - } - }, is_standard: function(frm) { diff --git a/frappe/desk/doctype/dashboard_chart/dashboard_chart.py b/frappe/desk/doctype/dashboard_chart/dashboard_chart.py index 88c9623945..7be97f8e24 100644 --- a/frappe/desk/doctype/dashboard_chart/dashboard_chart.py +++ b/frappe/desk/doctype/dashboard_chart/dashboard_chart.py @@ -143,7 +143,7 @@ def add_chart_to_dashboard(args): dashboard_link = frappe.new_doc('Dashboard Chart Link') dashboard_link.chart = args.chart_name or args.name - if args.set_standard: + if args.set_standard and dashboard.is_standard: chart = frappe.get_doc('Dashboard Chart', dashboard_link.chart) chart.is_standard = 1 chart.module = dashboard.module diff --git a/frappe/desk/doctype/number_card/number_card.py b/frappe/desk/doctype/number_card/number_card.py index 68ed79e64b..8b79cf4052 100644 --- a/frappe/desk/doctype/number_card/number_card.py +++ b/frappe/desk/doctype/number_card/number_card.py @@ -180,7 +180,7 @@ def add_card_to_dashboard(args): dashboard_link = frappe.new_doc('Number Card Link') dashboard_link.card = args.name - if args.set_standard: + if args.set_standard and dashboard.is_standard: card = frappe.get_doc('Number Card', dashboard_link.card) card.is_standard = 1 card.module = dashboard.module diff --git a/frappe/public/js/frappe/widgets/utils.js b/frappe/public/js/frappe/widgets/utils.js index 22b3167977..4599b4adc8 100644 --- a/frappe/public/js/frappe/widgets/utils.js +++ b/frappe/public/js/frappe/widgets/utils.js @@ -122,7 +122,8 @@ function shorten_number(number, country) { const number_system = get_number_system(country); let x = Math.abs(Math.round(number)); for (const map of number_system) { - if (x >= map.divisor) { + const condition = map.condition ? map.condition(x) : x >= map.divisor; + if (condition) { return Math.round(number/map.divisor) + ' ' + map.symbol; } } @@ -152,9 +153,14 @@ function get_number_system(country) { { divisor: 1.0e+6, symbol: 'M' + }, + { + divisor: 1.0e+3, + symbol: 'K', + condition: (num) => num.toFixed().length > 5 }] }; return number_system_map[country]; } -export { generate_route, generate_grid, build_summary_item, shorten_number }; \ No newline at end of file +export { generate_route, generate_grid, build_summary_item, shorten_number }; From 4225b55041c4da41a961bf5ca13c5f339b81e3ae Mon Sep 17 00:00:00 2001 From: Rushabh Mehta Date: Wed, 5 Aug 2020 15:33:30 +0530 Subject: [PATCH 44/57] feat(minor): Allow web form to follow document permissions --- frappe/website/doctype/web_form/web_form.json | 11 ++++- frappe/website/doctype/web_form/web_form.py | 45 ++++++++++--------- 2 files changed, 33 insertions(+), 23 deletions(-) diff --git a/frappe/website/doctype/web_form/web_form.json b/frappe/website/doctype/web_form/web_form.json index e6b7632e4d..77786734e7 100644 --- a/frappe/website/doctype/web_form/web_form.json +++ b/frappe/website/doctype/web_form/web_form.json @@ -16,6 +16,7 @@ "route_to_success_link", "allow_edit", "allow_multiple", + "apply_document_permissions", "show_in_grid", "allow_delete", "allow_print", @@ -346,14 +347,20 @@ "fieldname": "custom_css_section", "fieldtype": "Section Break", "label": "Custom CSS" + }, + { + "default": "0", + "fieldname": "apply_document_permissions", + "fieldtype": "Check", + "label": "Apply Document Permissions" } ], "has_web_view": 1, "icon": "icon-edit", "is_published_field": "published", "links": [], - "modified": "2019-12-24 14:15:43.497431", - "modified_by": "faris@erpnext.com", + "modified": "2020-06-30 21:49:18.237443", + "modified_by": "Administrator", "module": "Website", "name": "Web Form", "owner": "Administrator", diff --git a/frappe/website/doctype/web_form/web_form.py b/frappe/website/doctype/web_form/web_form.py index 4fc3c99037..bb0e03ddae 100644 --- a/frappe/website/doctype/web_form/web_form.py +++ b/frappe/website/doctype/web_form/web_form.py @@ -134,7 +134,7 @@ def get_context(context): if frappe.session.user == "Guest" and frappe.form_dict.name: frappe.throw(_("You need to be logged in to access this {0}.").format(self.doc_type), frappe.PermissionError) - if frappe.form_dict.name and not has_web_form_permission(self.doc_type, frappe.form_dict.name): + if frappe.form_dict.name and not self.has_web_form_permission(self.doc_type, frappe.form_dict.name): frappe.throw(_("You don't have the permissions to access this document"), frappe.PermissionError) self.reset_field_parent() @@ -347,6 +347,27 @@ def get_context(context): frappe.throw(_('Mandatory Information missing:') + '

    ' + '
    '.join(['{0} ({1})'.format(d.label, d.fieldtype) for d in missing])) + def has_web_form_permission(self, doctype, name, ptype='read'): + if frappe.session.user=="Guest": + return False + + if self.apply_document_permissions: + return frappe.get_doc(doctype, name).has_permission() + + # owner matches + elif frappe.db.get_value(doctype, name, "owner")==frappe.session.user: + return True + + elif frappe.has_website_permission(name, ptype=ptype, doctype=doctype): + return True + + elif check_webform_perm(doctype, name): + return True + + else: + return False + + @frappe.whitelist(allow_guest=True) def accept(web_form, data, docname=None, for_payment=False): @@ -395,7 +416,7 @@ def accept(web_form, data, docname=None, for_payment=False): doc.run_method('validate_payment') if doc.name: - if has_web_form_permission(doc.doctype, doc.name, "write"): + if web_form.has_web_form_permission(doc.doctype, doc.name, "write"): doc.save(ignore_permissions=True) else: # only if permissions are present @@ -482,24 +503,6 @@ def delete_multiple(web_form_name, docnames): raise frappe.PermissionError("You do not have permisssion to delete " + ", ".join(restricted_docnames)) -def has_web_form_permission(doctype, name, ptype='read'): - if frappe.session.user=="Guest": - return False - - # owner matches - elif frappe.db.get_value(doctype, name, "owner")==frappe.session.user: - return True - - elif frappe.has_website_permission(name, ptype=ptype, doctype=doctype): - return True - - elif check_webform_perm(doctype, name): - return True - - else: - return False - - def check_webform_perm(doctype, name): doc = frappe.get_doc(doctype, name) if hasattr(doc, "has_webform_permission"): @@ -536,7 +539,7 @@ def get_form_data(doctype, docname=None, web_form_name=None): if docname: doc = frappe.get_doc(doctype, docname) - if has_web_form_permission(doctype, docname, ptype='read'): + if web_form.has_web_form_permission(doctype, docname, ptype='read'): out.doc = doc else: frappe.throw(_("Not permitted"), frappe.PermissionError) From cd1ab8e23c4a1de4bbd5137c34a864af53788237 Mon Sep 17 00:00:00 2001 From: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> Date: Wed, 5 Aug 2020 19:41:41 +0530 Subject: [PATCH 45/57] feat: Add validate_and_sanitize_search_inputs decorator (#11194) * feat: Add validate_and_sanitize_search_inputs decorator Co-authored-by: Nabin Hait Co-authored-by: Prssanna Desai Co-authored-by: Shivam Mishra * refactor: Move validate_and_sanitize_search_inputs to init * refactor: Move validate_and_sanitize_search_inputs to search.py * test: validate_and_sanitize_search_inputs decorator * chore: Add wrapt module * refactor: Use @wrapt to define validate_and_sanitize_search_inputs decorator * test: Add a case to make sure frappe.call works as well Co-authored-by: Nabin Hait Co-authored-by: Prssanna Desai Co-authored-by: Shivam Mishra --- frappe/__init__.py | 4 ++++ frappe/desk/search.py | 13 +++++++++++++ frappe/tests/test_search.py | 28 ++++++++++++++++++++++++++++ requirements.txt | 3 ++- 4 files changed, 47 insertions(+), 1 deletion(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index 45f40f7783..9e744c2c07 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -1707,3 +1707,7 @@ def mock(type, size=1, locale='en'): from frappe.chat.util import squashify return squashify(results) + +def validate_and_sanitize_search_inputs(fn): + from frappe.desk.search import validate_and_sanitize_search_inputs as func + return func(fn) diff --git a/frappe/desk/search.py b/frappe/desk/search.py index b4b54b4b6e..798e499bb9 100644 --- a/frappe/desk/search.py +++ b/frappe/desk/search.py @@ -10,6 +10,7 @@ from frappe.handler import is_whitelisted from frappe import _ from six import string_types import re +import wrapt UNTRANSLATED_DOCTYPES = ["DocType", "Role"] @@ -206,3 +207,15 @@ def scrub_custom_query(query, key, txt): if '%s' in query: query = query.replace('%s', ((txt or '') + '%')) return query + +@wrapt.decorator +def validate_and_sanitize_search_inputs(fn, instance, args, kwargs): + kwargs.update(dict(zip(fn.__code__.co_varnames, args))) + sanitize_searchfield(kwargs['searchfield']) + kwargs['start'] = cint(kwargs['start']) + kwargs['page_len'] = cint(kwargs['page_len']) + + if kwargs['doctype'] and not frappe.db.exists('DocType', kwargs['doctype']): + return [] + + return fn(**kwargs) \ No newline at end of file diff --git a/frappe/tests/test_search.py b/frappe/tests/test_search.py index 5cab827218..192fe9dd25 100644 --- a/frappe/tests/test_search.py +++ b/frappe/tests/test_search.py @@ -50,3 +50,31 @@ class TestSearch(unittest.TestCase): def tearDown(self): frappe.local.lang = 'en' + + def test_validate_and_sanitize_search_inputs(self): + + # should raise error if searchfield is injectable + self.assertRaises(frappe.DataError, + get_data, *('User', 'Random', 'select * from tabSessions) --', '1', '10', dict())) + + # page_len and start should be converted to int + self.assertListEqual(get_data('User', 'Random', 'email', 'name or (select * from tabSessions)', '10', dict()), + ['User', 'Random', 'email', 0, 10, {}]) + self.assertListEqual(get_data('User', 'Random', 'email', page_len='2', start='10', filters=dict()), + ['User', 'Random', 'email', 10, 2, {}]) + + # DocType can be passed as None which should be accepted + self.assertListEqual(get_data(None, 'Random', 'email', '2', '10', dict()), + [None, 'Random', 'email', 2, 10, {}]) + + # return empty string if passed doctype is invalid + self.assertListEqual(get_data("Random DocType", 'Random', 'email', '2', '10', dict()), []) + + # should not fail if function is called via frappe.call with extra arguments + args = ("Random DocType", 'Random', 'email', '2', '10', dict()) + kwargs = {'as_dict': False} + self.assertListEqual(frappe.call('frappe.tests.test_search.get_data', *args, **kwargs), []) + +@frappe.validate_and_sanitize_search_inputs +def get_data(doctype, txt, searchfield, start, page_len, filters): + return [doctype, txt, searchfield, start, page_len, filters] \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 33cd292bf6..2995553541 100644 --- a/requirements.txt +++ b/requirements.txt @@ -69,4 +69,5 @@ Whoosh==2.7.4 xlrd==1.2.0 zxcvbn-python==4.4.24 pycryptodome==3.9.8 -paytmchecksum==1.7.0 \ No newline at end of file +paytmchecksum==1.7.0 +wrapt==1.10.11 \ No newline at end of file From 21a0cafe6209334906441c07c7f2e085afc82ade Mon Sep 17 00:00:00 2001 From: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> Date: Wed, 5 Aug 2020 19:42:19 +0530 Subject: [PATCH 46/57] fix: Validate and sanitise whitelisted search methods using decorator (#11197) * fix: Validate and sanitize whitelisted search methods using decorator Co-authored-by: Prssanna Desai Co-authored-by: Shivam Mishra * refactor: Replace decorator name Co-authored-by: Prssanna Desai Co-authored-by: Shivam Mishra Co-authored-by: Prssanna Desai Co-authored-by: Shivam Mishra --- frappe/automation/doctype/auto_repeat/auto_repeat.py | 1 + frappe/contacts/address_and_contact.py | 1 + frappe/contacts/doctype/address/address.py | 1 + frappe/contacts/doctype/contact/contact.py | 1 + frappe/core/doctype/user/user.py | 1 + frappe/core/doctype/user_permission/user_permission.py | 1 + .../permitted_documents_for_user/permitted_documents_for_user.py | 1 + frappe/desk/doctype/dashboard_chart/dashboard_chart.py | 1 + frappe/desk/doctype/number_card/number_card.py | 1 + frappe/email/__init__.py | 1 + 10 files changed, 10 insertions(+) diff --git a/frappe/automation/doctype/auto_repeat/auto_repeat.py b/frappe/automation/doctype/auto_repeat/auto_repeat.py index a946fcc81c..c09e347e71 100644 --- a/frappe/automation/doctype/auto_repeat/auto_repeat.py +++ b/frappe/automation/doctype/auto_repeat/auto_repeat.py @@ -374,6 +374,7 @@ def make_auto_repeat(doctype, docname, frequency = 'Daily', start_date = None, e # method for reference_doctype filter @frappe.whitelist() +@frappe.validate_and_sanitize_search_inputs def get_auto_repeat_doctypes(doctype, txt, searchfield, start, page_len, filters): res = frappe.db.get_all('Property Setter', { 'property': 'allow_auto_repeat', diff --git a/frappe/contacts/address_and_contact.py b/frappe/contacts/address_and_contact.py index 51f13fb1a1..3ca9547188 100644 --- a/frappe/contacts/address_and_contact.py +++ b/frappe/contacts/address_and_contact.py @@ -147,6 +147,7 @@ def delete_contact_and_address(doctype, docname): doc.delete() @frappe.whitelist() +@frappe.validate_and_sanitize_search_inputs def filter_dynamic_link_doctypes(doctype, txt, searchfield, start, page_len, filters): if not txt: txt = "" diff --git a/frappe/contacts/doctype/address/address.py b/frappe/contacts/doctype/address/address.py index 57dea8284c..e82ab9b26e 100644 --- a/frappe/contacts/doctype/address/address.py +++ b/frappe/contacts/doctype/address/address.py @@ -231,6 +231,7 @@ def get_company_address(company): return ret @frappe.whitelist() +@frappe.validate_and_sanitize_search_inputs def address_query(doctype, txt, searchfield, start, page_len, filters): from frappe.desk.reportview import get_match_cond diff --git a/frappe/contacts/doctype/contact/contact.py b/frappe/contacts/doctype/contact/contact.py index 8240940d2f..f82946dc5e 100644 --- a/frappe/contacts/doctype/contact/contact.py +++ b/frappe/contacts/doctype/contact/contact.py @@ -183,6 +183,7 @@ def update_contact(doc, method): contact.save(ignore_permissions=True) @frappe.whitelist() +@frappe.validate_and_sanitize_search_inputs def contact_query(doctype, txt, searchfield, start, page_len, filters): from frappe.desk.reportview import get_match_cond diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py index 64bff32189..ef2de679ae 100644 --- a/frappe/core/doctype/user/user.py +++ b/frappe/core/doctype/user/user.py @@ -812,6 +812,7 @@ def reset_password(user): return 'not found' @frappe.whitelist() +@frappe.validate_and_sanitize_search_inputs def user_query(doctype, txt, searchfield, start, page_len, filters): from frappe.desk.reportview import get_match_cond diff --git a/frappe/core/doctype/user_permission/user_permission.py b/frappe/core/doctype/user_permission/user_permission.py index 325c41622d..ba14583c2f 100644 --- a/frappe/core/doctype/user_permission/user_permission.py +++ b/frappe/core/doctype/user_permission/user_permission.py @@ -120,6 +120,7 @@ def user_permission_exists(user, allow, for_value, applicable_for=None): return has_same_user_permission @frappe.whitelist() +@frappe.validate_and_sanitize_search_inputs def get_applicable_for_doctype_list(doctype, txt, searchfield, start, page_len, filters): linked_doctypes_map = get_linked_doctypes(doctype, True) diff --git a/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.py b/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.py index 5ca21b811e..97209cd8ea 100644 --- a/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.py +++ b/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.py @@ -42,6 +42,7 @@ def get_columns_and_fields(doctype): return columns, fields @frappe.whitelist() +@frappe.validate_and_sanitize_search_inputs def query_doctypes(doctype, txt, searchfield, start, page_len, filters): user = filters.get("user") user_perms = frappe.utils.user.UserPermissions(user) diff --git a/frappe/desk/doctype/dashboard_chart/dashboard_chart.py b/frappe/desk/doctype/dashboard_chart/dashboard_chart.py index 7be97f8e24..4ea61ec6a9 100644 --- a/frappe/desk/doctype/dashboard_chart/dashboard_chart.py +++ b/frappe/desk/doctype/dashboard_chart/dashboard_chart.py @@ -358,6 +358,7 @@ def get_year_ending(date): return add_to_date(date, days=-1) @frappe.whitelist() +@frappe.validate_and_sanitize_search_inputs def get_charts_for_user(doctype, txt, searchfield, start, page_len, filters): or_filters = {'owner': frappe.session.user, 'is_public': 1} return frappe.db.get_list('Dashboard Chart', diff --git a/frappe/desk/doctype/number_card/number_card.py b/frappe/desk/doctype/number_card/number_card.py index 8b79cf4052..d4a2b00c57 100644 --- a/frappe/desk/doctype/number_card/number_card.py +++ b/frappe/desk/doctype/number_card/number_card.py @@ -129,6 +129,7 @@ def create_number_card(args): return doc @frappe.whitelist() +@frappe.validate_and_sanitize_search_inputs def get_cards_for_user(doctype, txt, searchfield, start, page_len, filters): meta = frappe.get_meta(doctype) searchfields = meta.get_search_fields() diff --git a/frappe/email/__init__.py b/frappe/email/__init__.py index 0b874a03bb..b05aef7639 100644 --- a/frappe/email/__init__.py +++ b/frappe/email/__init__.py @@ -58,6 +58,7 @@ def relink(name, reference_doctype=None, reference_name=None): name = %s""", (reference_doctype, reference_name, name)) @frappe.whitelist() +@frappe.validate_and_sanitize_search_inputs def get_communication_doctype(doctype, txt, searchfield, start, page_len, filters): user_perms = frappe.utils.user.UserPermissions(frappe.session.user) user_perms.build_permissions() From 34c3d57767611db622779efe5cf0454b8a866f92 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Wed, 5 Aug 2020 18:31:11 +0000 Subject: [PATCH 47/57] feat: validate json when saving desk page (#11095) Co-authored-by: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> --- frappe/desk/doctype/desk_page/desk_page.py | 9 +++++++++ frappe/utils/data.py | 9 +++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/frappe/desk/doctype/desk_page/desk_page.py b/frappe/desk/doctype/desk_page/desk_page.py index f14535cb5f..cc2db53481 100644 --- a/frappe/desk/doctype/desk_page/desk_page.py +++ b/frappe/desk/doctype/desk_page/desk_page.py @@ -5,14 +5,23 @@ from __future__ import unicode_literals import frappe from frappe import _ +from frappe.utils.data import validate_json_string from frappe.modules.export_file import export_to_files from frappe.model.document import Document class DeskPage(Document): def validate(self): + self.validate_cards_json() if (self.is_standard and not frappe.conf.developer_mode and not disable_saving_as_standard()): frappe.throw(_("You need to be in developer mode to edit this document")) + def validate_cards_json(self): + for card in self.cards: + try: + validate_json_string(card.links) + except frappe.ValidationError: + frappe.throw(_("Invalid JSON in card links for {0}").format(frappe.bold(card.label))) + def on_update(self): if disable_saving_as_standard(): return diff --git a/frappe/utils/data.py b/frappe/utils/data.py index d1b409d1fc..bda201f083 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -8,6 +8,7 @@ import frappe from dateutil.parser._parser import ParserError import subprocess import operator +import json import re, datetime, math, time import babel.dates from babel.core import UnknownLocaleError @@ -1236,8 +1237,6 @@ def is_subset(list_a, list_b): def generate_hash(*args, **kwargs): return frappe.generate_hash(*args, **kwargs) - - def guess_date_format(date_string): DATE_FORMATS = [ r"%d-%m-%Y", @@ -1310,3 +1309,9 @@ def guess_date_format(date_string): if date_format and time_format: return (date_format + ' ' + time_format).strip() + +def validate_json_string(string): + try: + json.loads(string) + except (TypeError, ValueError): + raise frappe.ValidationError \ No newline at end of file From ef4a28e10c0de0083c57278b615e587600b64e3b Mon Sep 17 00:00:00 2001 From: Mangesh-Khairnar Date: Thu, 6 Aug 2020 09:32:04 +0530 Subject: [PATCH 48/57] fix: add a distinct identifier for public files backup (#11201) --- frappe/integrations/offsite_backup_utils.py | 2 +- frappe/utils/backups.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/integrations/offsite_backup_utils.py b/frappe/integrations/offsite_backup_utils.py index 9de176b2d0..0b3c47064e 100644 --- a/frappe/integrations/offsite_backup_utils.py +++ b/frappe/integrations/offsite_backup_utils.py @@ -53,7 +53,7 @@ def get_latest_backup_file(with_files=False): latest_site_config = get_latest('*.json') if with_files: - latest_public_file_bak = get_latest('*-files.tar') + latest_public_file_bak = get_latest('*-public-files.tar') latest_private_file_bak = get_latest('*-private-files.tar') return latest_file, latest_site_config, latest_public_file_bak, latest_private_file_bak diff --git a/frappe/utils/backups.py b/frappe/utils/backups.py index 7eb00ceccd..08b911c9ae 100644 --- a/frappe/utils/backups.py +++ b/frappe/utils/backups.py @@ -82,7 +82,7 @@ class BackupGenerator: def set_backup_file_name(self): #Generate a random name using today's date and a 8 digit random number for_db = self.todays_date + "-" + self.site_slug + "-database.sql.gz" - for_public_files = self.todays_date + "-" + self.site_slug + "-files.tar" + for_public_files = self.todays_date + "-" + self.site_slug + "-public-files.tar" for_private_files = self.todays_date + "-" + self.site_slug + "-private-files.tar" backup_path = get_backup_path() From 4f37f9a4528ba50b82f2000a4381c8ff813ff138 Mon Sep 17 00:00:00 2001 From: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> Date: Thu, 6 Aug 2020 09:33:39 +0530 Subject: [PATCH 49/57] test: Try committing after Role creation --- frappe/workflow/doctype/workflow/test_workflow.py | 1 + 1 file changed, 1 insertion(+) diff --git a/frappe/workflow/doctype/workflow/test_workflow.py b/frappe/workflow/doctype/workflow/test_workflow.py index d7f7939d3d..84adc3a096 100644 --- a/frappe/workflow/doctype/workflow/test_workflow.py +++ b/frappe/workflow/doctype/workflow/test_workflow.py @@ -126,6 +126,7 @@ def create_todo_workflow(): else: frappe.get_doc(dict(doctype='Role', role_name='Test Approver')).insert(ignore_if_duplicate=True) + frappe.db.commit() frappe.cache().hdel('roles', frappe.session.user) workflow = frappe.new_doc('Workflow') workflow.workflow_name = 'Test ToDo' From b92ce9a245ad9582a833b4d0ce62bf1230e99a68 Mon Sep 17 00:00:00 2001 From: Deepesh Garg <42651287+deepeshgarg007@users.noreply.github.com> Date: Fri, 7 Aug 2020 09:25:37 +0530 Subject: [PATCH 50/57] feat: Allow sending emails via Server Script (#11162) Co-authored-by: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> --- frappe/utils/safe_exec.py | 1 + 1 file changed, 1 insertion(+) diff --git a/frappe/utils/safe_exec.py b/frappe/utils/safe_exec.py index 826aed6082..d97e770d88 100644 --- a/frappe/utils/safe_exec.py +++ b/frappe/utils/safe_exec.py @@ -70,6 +70,7 @@ def get_safe_globals(): render_template=frappe.render_template, msgprint=frappe.msgprint, throw=frappe.throw, + sendmail = frappe.sendmail, user=user, get_fullname=frappe.utils.get_fullname, From 5b3fa971b54be7641b369d30a6c68f8acf290a13 Mon Sep 17 00:00:00 2001 From: gavin Date: Fri, 7 Aug 2020 13:14:29 +0530 Subject: [PATCH 51/57] Revert "fix: add a distinct identifier for public files backup (#11201)" This reverts commit ef4a28e10c0de0083c57278b615e587600b64e3b. --- frappe/integrations/offsite_backup_utils.py | 2 +- frappe/utils/backups.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/integrations/offsite_backup_utils.py b/frappe/integrations/offsite_backup_utils.py index 0b3c47064e..9de176b2d0 100644 --- a/frappe/integrations/offsite_backup_utils.py +++ b/frappe/integrations/offsite_backup_utils.py @@ -53,7 +53,7 @@ def get_latest_backup_file(with_files=False): latest_site_config = get_latest('*.json') if with_files: - latest_public_file_bak = get_latest('*-public-files.tar') + latest_public_file_bak = get_latest('*-files.tar') latest_private_file_bak = get_latest('*-private-files.tar') return latest_file, latest_site_config, latest_public_file_bak, latest_private_file_bak diff --git a/frappe/utils/backups.py b/frappe/utils/backups.py index 08b911c9ae..7eb00ceccd 100644 --- a/frappe/utils/backups.py +++ b/frappe/utils/backups.py @@ -82,7 +82,7 @@ class BackupGenerator: def set_backup_file_name(self): #Generate a random name using today's date and a 8 digit random number for_db = self.todays_date + "-" + self.site_slug + "-database.sql.gz" - for_public_files = self.todays_date + "-" + self.site_slug + "-public-files.tar" + for_public_files = self.todays_date + "-" + self.site_slug + "-files.tar" for_private_files = self.todays_date + "-" + self.site_slug + "-private-files.tar" backup_path = get_backup_path() From ce9c23ba609e5f04cf15aba195a78b81b5a7508d Mon Sep 17 00:00:00 2001 From: Prssanna Desai Date: Fri, 7 Aug 2020 14:48:59 +0530 Subject: [PATCH 52/57] fix: return handler in _handler (#11215) --- frappe/public/js/frappe/form/script_manager.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/form/script_manager.js b/frappe/public/js/frappe/form/script_manager.js index 1238bf141c..4456c7d39c 100644 --- a/frappe/public/js/frappe/form/script_manager.js +++ b/frappe/public/js/frappe/form/script_manager.js @@ -19,7 +19,7 @@ frappe.ui.form.on = frappe.ui.form.on_change = function(doctype, fieldname, hand let _handler = (...args) => { try { - handler(...args); + return handler(...args); } catch (error) { console.error(handler); throw error; From 80733d4124b5ed5dc0a82d7a77b5f6593dd4b67b Mon Sep 17 00:00:00 2001 From: Shridhar Patil Date: Fri, 7 Aug 2020 21:52:04 +0530 Subject: [PATCH 53/57] fix: raise incorrect site path error instead of sys.exit (#11206) --- frappe/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index 9e744c2c07..fdf6079220 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -231,8 +231,7 @@ def get_site_config(sites_path=None, site_path=None): if os.path.exists(site_config): config.update(get_file_json(site_config)) elif local.site and not local.flags.new_site: - print("Site {0} does not exist".format(local.site)) - sys.exit(1) + raise IncorrectSitePath("{0} does not exist".format(local.site)) return _dict(config) From 4a331016319471e07a2fabac7312bcb8347176b2 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Sat, 8 Aug 2020 19:47:39 +0530 Subject: [PATCH 54/57] fix: Check if frappe.local has "request" to avoid attribute error --- frappe/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/auth.py b/frappe/auth.py index 64fea36748..998e97fe24 100644 --- a/frappe/auth.py +++ b/frappe/auth.py @@ -338,7 +338,7 @@ class CookieManager: self.set_cookie("country", frappe.session.session_country) def set_cookie(self, key, value, expires=None, secure=False, httponly=False, samesite="Lax"): - if not secure: + if not secure and hasattr(frappe.local, 'request'): secure = frappe.local.request.scheme == "https" self.cookies[key] = { "value": value, From 52d6925c49c7acee15189fcde58b92a55a17f97c Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Sat, 8 Aug 2020 19:47:54 +0530 Subject: [PATCH 55/57] test: Fallback to site URL if host_name is none --- .../doctype/data_migration_run/test_data_migration_run.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/data_migration/doctype/data_migration_run/test_data_migration_run.py b/frappe/data_migration/doctype/data_migration_run/test_data_migration_run.py index ba4a255b97..c6c3ea138c 100644 --- a/frappe/data_migration/doctype/data_migration_run/test_data_migration_run.py +++ b/frappe/data_migration/doctype/data_migration_run/test_data_migration_run.py @@ -108,7 +108,7 @@ def create_plan(): 'connector_name': 'Local Connector', 'connector_type': 'Frappe', # connect to same host. - 'hostname': frappe.conf.host_name, + 'hostname': frappe.conf.host_name or frappe.utils.get_site_url(frappe.local.site), 'username': 'Administrator', - 'password': 'admin' + 'password': frappe.conf.get("admin_password") or 'admin' }).insert(ignore_if_duplicate=True) From 7d8695c7d348155ab337619df1a208ffc3e6ec76 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Sat, 8 Aug 2020 23:46:45 +0530 Subject: [PATCH 56/57] fix: Provide fallback array for test_mails to avoid unexpected trace in travis --- frappe/email/doctype/email_account/email_account.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/email/doctype/email_account/email_account.py b/frappe/email/doctype/email_account/email_account.py index cf8c6e80c6..29cd890bf1 100755 --- a/frappe/email/doctype/email_account/email_account.py +++ b/frappe/email/doctype/email_account/email_account.py @@ -251,7 +251,7 @@ class EmailAccount(Document): email_server = None if frappe.local.flags.in_test: - incoming_mails = test_mails + incoming_mails = test_mails or [] else: email_sync_rule = self.build_email_sync_rule() From 0919dbeccd8c7367f0a26e2af3ebf75b22814f9a Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Sun, 9 Aug 2020 00:06:25 +0530 Subject: [PATCH 57/57] test: Manually delete jobs instead of deleting job type - This removes unwanted traceback of "Scheduled Job Type" not found while running tests --- frappe/tests/test_scheduler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/tests/test_scheduler.py b/frappe/tests/test_scheduler.py index e554fd23be..30818e6f17 100644 --- a/frappe/tests/test_scheduler.py +++ b/frappe/tests/test_scheduler.py @@ -49,7 +49,7 @@ class TestScheduler(TestCase): # 2nd job not loaded self.assertFalse(job.enqueue()) - job.delete() + frappe.db.sql('DELETE FROM `tabScheduled Job Log` WHERE `scheduled_job_type`=%s', job.name) def test_is_dormant(self): self.assertTrue(is_dormant(check_time= get_datetime('2100-01-01 00:00:00')))