From f057260e90a5e8c7888ac9b62431bb19d09063ef Mon Sep 17 00:00:00 2001 From: prssanna Date: Tue, 14 Jul 2020 17:49:34 +0530 Subject: [PATCH 001/230] fix: reorder result according to custom columns for script reports --- frappe/desk/query_report.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/frappe/desk/query_report.py b/frappe/desk/query_report.py index 0edfd57d4f..d0a32ef076 100644 --- a/frappe/desk/query_report.py +++ b/frappe/desk/query_report.py @@ -67,8 +67,7 @@ def generate_report_result(report, filters=None, user=None, custom_columns=None) # Reordered columns columns = json.loads(report.custom_columns) - if report.report_type == 'Query Report': - result = reorder_data_for_custom_columns(columns, query_columns, result) + result = reorder_data_for_custom_columns(columns, query_columns, result, report.report_type) result = add_data_to_custom_columns(columns, result) @@ -216,15 +215,21 @@ def add_data_to_custom_columns(columns, result): return data -def reorder_data_for_custom_columns(custom_columns, columns, result): +def reorder_data_for_custom_columns(custom_columns, columns, result, report_type): + custom_column_labels = [col["label"] for col in custom_columns] + + if report_type == 'Query Report': + original_column_labels = [col.split(":")[0] for col in columns] + else: + original_column_labels = [col["label"] for col in columns] + reordered_result = [] - columns = [col.split(":")[0] for col in columns] for res in result: r = [] - for col in custom_columns: + for col_name in custom_column_labels: try: - idx = columns.index(col.get("label")) + idx = original_column_labels.index(col_name) r.append(res[idx]) except ValueError: pass From 476e625261ab4954b7429978b24840e52967c9c0 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 16 Jul 2020 16:26:20 +0530 Subject: [PATCH 002/230] fix: Add site validation for logger --- frappe/__init__.py | 4 ++-- frappe/app.py | 5 +++-- frappe/utils/logger.py | 4 ++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index d644d2a473..8c24980189 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -1559,10 +1559,10 @@ def get_doctype_app(doctype): loggers = {} log_level = None -def logger(module=None, with_more_info=False): +def logger(module=None, with_more_info=False, _site=None): '''Returns a python logger that uses StreamHandler''' from frappe.utils.logger import get_logger - return get_logger(module=module, with_more_info=with_more_info) + return get_logger(module=module, with_more_info=with_more_info, _site=_site) def log_error(message=None, title=_("Error")): '''Log error to Error Log''' diff --git a/frappe/app.py b/frappe/app.py index 57db867882..83a9b8c149 100644 --- a/frappe/app.py +++ b/frappe/app.py @@ -99,8 +99,9 @@ def application(request): frappe.monitor.stop(response) frappe.recorder.dump() - frappe.logger("frappe.web").info({ - "site": get_site_name(request.host), + _site = get_site_name(request.host) + frappe.logger("frappe.web", _site=_site).info({ + "site": _site, "remote_addr": getattr(request, "remote_addr", "NOTFOUND"), "base_url": getattr(request, "base_url", "NOTFOUND"), "full_path": getattr(request, "full_path", "NOTFOUND"), diff --git a/frappe/utils/logger.py b/frappe/utils/logger.py index 89e3711b0f..bea83297d2 100755 --- a/frappe/utils/logger.py +++ b/frappe/utils/logger.py @@ -17,7 +17,7 @@ default_log_level = logging.DEBUG site = getattr(frappe.local, 'site', None) -def get_logger(module, with_more_info=False): +def get_logger(module, with_more_info=False, _site=None): global site if module in frappe.loggers: return frappe.loggers[module] @@ -38,7 +38,7 @@ def get_logger(module, with_more_info=False): handler = RotatingFileHandler(LOG_FILENAME, maxBytes=100_000, backupCount=20) logger.addHandler(handler) # - if site: + if site == _site: SITELOG_FILENAME = os.path.join(site, 'logs', logfile) site_handler = RotatingFileHandler(SITELOG_FILENAME, maxBytes=100_000, backupCount=20) site_handler.setFormatter(formatter) From e92a612ba231eebb8dbe7ac42d24ac002a89fbe1 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 16 Jul 2020 16:26:53 +0530 Subject: [PATCH 003/230] docs(get_logger): add docstring --- frappe/utils/logger.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/frappe/utils/logger.py b/frappe/utils/logger.py index bea83297d2..afe4335aea 100755 --- a/frappe/utils/logger.py +++ b/frappe/utils/logger.py @@ -18,7 +18,24 @@ site = getattr(frappe.local, 'site', None) def get_logger(module, with_more_info=False, _site=None): + """Application Logger for your given module + + Args: + module (str): Name of your logger and consequently your log file. + with_more_info (bool, optional): Will log the form dict using the SiteContextFilter. Defaults to False. + _site (str, optional): If set, validates the current site context with the passed value. The `frappe.web` logger uses this to determine that the application is logging information related to the logger called. Defaults to None. + + Returns: + : Returns a Python logger object with Site and Bench level logging capabilities. + """ global site + + def allow_site(): + allow = False + if site: allow = True + if _site: allow = site == _site + return allow + if module in frappe.loggers: return frappe.loggers[module] @@ -37,8 +54,8 @@ def get_logger(module, with_more_info=False, _site=None): formatter = logging.Formatter('%(asctime)s %(levelname)s %(name)s %(message)s') handler = RotatingFileHandler(LOG_FILENAME, maxBytes=100_000, backupCount=20) logger.addHandler(handler) -# - if site == _site: + + if allow_site(): SITELOG_FILENAME = os.path.join(site, 'logs', logfile) site_handler = RotatingFileHandler(SITELOG_FILENAME, maxBytes=100_000, backupCount=20) site_handler.setFormatter(formatter) From c87d598480e006a88a4374d0ab17d15429701441 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 17 Jul 2020 17:35:08 +0530 Subject: [PATCH 004/230] refactor: less word do trick Co-authored-by: Chinmay Pai --- frappe/utils/logger.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/frappe/utils/logger.py b/frappe/utils/logger.py index afe4335aea..9254c0aedd 100755 --- a/frappe/utils/logger.py +++ b/frappe/utils/logger.py @@ -31,10 +31,7 @@ def get_logger(module, with_more_info=False, _site=None): global site def allow_site(): - allow = False - if site: allow = True - if _site: allow = site == _site - return allow + return (_site and _site == site) or bool(site) if module in frappe.loggers: return frappe.loggers[module] From a2b2c9c926b1fd2be8192499dc8a21b250019eaa Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Wed, 22 Jul 2020 17:54:48 +0530 Subject: [PATCH 005/230] fix: Validate select values in Column --- frappe/core/doctype/data_import/importer.py | 29 ++++++++++++++++----- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/frappe/core/doctype/data_import/importer.py b/frappe/core/doctype/data_import/importer.py index 910e42af1a..729f4e8956 100644 --- a/frappe/core/doctype/data_import/importer.py +++ b/frappe/core/doctype/data_import/importer.py @@ -615,7 +615,7 @@ class Row: def validate_value(self, value, col): df = col.df if df.fieldtype == "Select": - select_options = [d for d in (df.options or '').split('\n') if d] + select_options = get_select_options(df) if select_options and value not in select_options: options_string = ", ".join([frappe.bold(d) for d in select_options]) msg = _("Value must be one of {0}").format(options_string) @@ -971,12 +971,23 @@ class Column: # guess date format self.date_format = self.guess_date_format_for_column() if not self.date_format: - self.date_format = '%Y-%m-%d' - self.warnings.append({ - 'col': self.column_number, - 'message': _("Date format could not determined from the values in this column. Defaulting to yyyy-mm-dd."), - 'type': 'info' - }) + elif self.df.fieldtype == "Select": + options = get_select_options(self.df) + if options: + values = list(set([cstr(v) for v in self.column_values[1:] if v])) + invalid = list(set(values) - set(options)) + if invalid: + valid_values = ", ".join([frappe.bold(o) for o in options]) + invalid_values = ", ".join([frappe.bold(i) for i in invalid]) + self.warnings.append( + { + "col": self.column_number, + "message": ( + "The following values are invalid: {0}. Values must be" + " one of {1}".format(invalid_values, valid_values) + ), + } + ) def as_dict(self): d = frappe._dict() @@ -1170,3 +1181,7 @@ def df_as_json(df): 'parent': df.parent, 'default': df.default } + + +def get_select_options(df): + return [d for d in (df.options or "").split("\n") if d] From 5dcb910ae9e040f76b1e7e18354e31c2dd7ce729 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Wed, 22 Jul 2020 17:57:01 +0530 Subject: [PATCH 006/230] fix: Set default date format for parsed date values --- frappe/core/doctype/data_import/importer.py | 32 ++++++++++++++++----- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/frappe/core/doctype/data_import/importer.py b/frappe/core/doctype/data_import/importer.py index 729f4e8956..47113d35cd 100644 --- a/frappe/core/doctype/data_import/importer.py +++ b/frappe/core/doctype/data_import/importer.py @@ -7,7 +7,7 @@ import io import frappe import timeit import json -from datetime import datetime +from datetime import datetime, date from frappe import _ from frappe.utils import cint, flt, update_progress_bar, cstr from frappe.utils.csvutils import read_csv_content, get_csv_content_from_google_sheets @@ -668,7 +668,7 @@ class Row: def parse_value(self, value, col): df = col.df - if isinstance(value, datetime) and df.fieldtype in ["Date", "Datetime"]: + if isinstance(value, (datetime, date)) and df.fieldtype in ["Date", "Datetime"]: return value value = cstr(value) @@ -689,7 +689,7 @@ class Row: return value def get_date(self, value, column): - if isinstance(value, datetime): + if isinstance(value, (datetime, date)): return value date_format = column.date_format @@ -918,13 +918,20 @@ class Column: self.skip_import = skip_import def guess_date_format_for_column(self): - """ Guesses date format for a column by parsing all the values in the column, + """Guesses date format for a column by parsing all the values in the column, getting the date format and then returning the one which has the maximum frequency """ - date_formats = [ - frappe.utils.guess_date_format(d) for d in self.column_values if isinstance(d, str) - ] + def guess_date_format(d): + if isinstance(d, (datetime, date)): + if self.df.fieldtype == "Date": + return "%Y-%m-%d" + if self.df.fieldtype == "Datetime": + return "%Y-%m-%d %H:%M:%S" + if isinstance(d, str): + return frappe.utils.guess_date_format(d) + + date_formats = [guess_date_format(d) for d in self.column_values] date_formats = [d for d in date_formats if d] if not date_formats: return @@ -971,6 +978,17 @@ class Column: # guess date format self.date_format = self.guess_date_format_for_column() if not self.date_format: + self.date_format = "%Y-%m-%d" + self.warnings.append( + { + "col": self.column_number, + "message": _( + "Date format could not be determined from the values in" + " this column. Defaulting to yyyy-mm-dd." + ), + "type": "info", + } + ) elif self.df.fieldtype == "Select": options = get_select_options(self.df) if options: From 015ab803a36355cd02df0e640dba0897515d8ebf Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Wed, 22 Jul 2020 17:57:13 +0530 Subject: [PATCH 007/230] style: formatting --- frappe/core/doctype/data_import/importer.py | 63 ++++++++++----------- 1 file changed, 31 insertions(+), 32 deletions(-) diff --git a/frappe/core/doctype/data_import/importer.py b/frappe/core/doctype/data_import/importer.py index 47113d35cd..6bc8b0586f 100644 --- a/frappe/core/doctype/data_import/importer.py +++ b/frappe/core/doctype/data_import/importer.py @@ -233,7 +233,7 @@ class Importer: return updated_doc else: # throw if no changes - frappe.throw('No changes to update') + frappe.throw("No changes to update") def get_eta(self, current, total, processing_time): self.last_eta = getattr(self, "last_eta", 0) @@ -322,7 +322,7 @@ class ImportFile: if isinstance(file, frappe.string_types): if frappe.db.exists("File", {"file_url": file}): self.file_doc = frappe.get_doc("File", {"file_url": file}) - elif 'docs.google.com/spreadsheets' in file: + elif "docs.google.com/spreadsheets" in file: self.google_sheets_url = file elif os.path.exists(file): self.file_path = file @@ -348,7 +348,7 @@ class ImportFile: elif self.google_sheets_url: content = get_csv_content_from_google_sheets(self.google_sheets_url) - extension = 'csv' + extension = "csv" if not content: frappe.throw(_("Invalid or corrupted content for import")) @@ -620,11 +620,7 @@ class Row: options_string = ", ".join([frappe.bold(d) for d in select_options]) msg = _("Value must be one of {0}").format(options_string) self.warnings.append( - { - "row": self.row_number, - "field": df_as_json(df), - "message": msg, - } + {"row": self.row_number, "field": df_as_json(df), "message": msg,} ) return @@ -635,11 +631,7 @@ class Row: frappe.bold(value), frappe.bold(df.options) ) self.warnings.append( - { - "row": self.row_number, - "field": df_as_json(df), - "message": msg, - } + {"row": self.row_number, "field": df_as_json(df), "message": msg,} ) return elif df.fieldtype in ["Date", "Datetime"]: @@ -786,9 +778,7 @@ class Header(Row): for j, header in enumerate(row): column_values = [get_item_at_index(r, j) for r in raw_data] map_to_field = column_to_field_map.get(str(j)) - column = Column( - j, header, self.doctype, column_values, map_to_field, self.seen - ) + column = Column(j, header, self.doctype, column_values, map_to_field, self.seen) self.seen.append(header) self.columns.append(column) @@ -962,18 +952,26 @@ class Column: if not self.df: return - if self.df.fieldtype == 'Link': + if self.df.fieldtype == "Link": # find all values that dont exist values = list(set([cstr(v) for v in self.column_values[1:] if v])) - exists = [d.name for d in frappe.db.get_all(self.df.options, filters={'name': ('in', values)})] + exists = [ + d.name for d in frappe.db.get_all(self.df.options, filters={"name": ("in", values)}) + ] not_exists = list(set(values) - set(exists)) if not_exists: - missing_values = ', '.join(not_exists) - self.warnings.append({ - 'col': self.column_number, - 'message': "The following values do not exist for {}: {}".format(self.df.options, missing_values), - 'type': 'warning' - }) + missing_values = ", ".join(not_exists) + self.warnings.append( + { + "col": self.column_number, + "message": ( + "The following values do not exist for {}: {}".format( + self.df.options, missing_values + ) + ), + "type": "warning", + } + ) elif self.df.fieldtype in ("Date", "Time", "Datetime"): # guess date format self.date_format = self.guess_date_format_for_column() @@ -1016,7 +1014,7 @@ class Column: d.map_to_field = self.map_to_field d.date_format = self.date_format d.df = self.df - if hasattr(self.df, 'is_child_table_field'): + if hasattr(self.df, "is_child_table_field"): d.is_child_table_field = self.df.is_child_table_field d.child_table_df = self.df.child_table_df d.skip_import = self.skip_import @@ -1096,7 +1094,7 @@ def build_fields_dict_for_column_matching(parent_doctype): # other fields fields = get_standard_fields(doctype) + frappe.get_meta(doctype).fields for df in fields: - label = (df.label or '').strip() + label = (df.label or "").strip() fieldtype = df.fieldtype or "Data" parent = df.parent or parent_doctype if fieldtype not in no_value_fields: @@ -1190,14 +1188,15 @@ def get_user_format(date_format): .replace("%d", "dd") ) + def df_as_json(df): return { - 'fieldname': df.fieldname, - 'fieldtype': df.fieldtype, - 'label': df.label, - 'options': df.options, - 'parent': df.parent, - 'default': df.default + "fieldname": df.fieldname, + "fieldtype": df.fieldtype, + "label": df.label, + "options": df.options, + "parent": df.parent, + "default": df.default, } From e34e48bb2f503c20899b4f4e6f56ca74ffbbb1ee Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Wed, 22 Jul 2020 17:57:57 +0530 Subject: [PATCH 008/230] fix: Show Column Name in warnings header --- frappe/core/doctype/data_import/data_import.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/frappe/core/doctype/data_import/data_import.js b/frappe/core/doctype/data_import/data_import.js index 6a922618cb..0e827a42d8 100644 --- a/frappe/core/doctype/data_import/data_import.js +++ b/frappe/core/doctype/data_import/data_import.js @@ -317,6 +317,7 @@ frappe.ui.form.on('Data Import', { }, show_import_warnings(frm, preview_data) { + let columns = preview_data.columns; let warnings = JSON.parse(frm.doc.template_warnings || '[]'); warnings = warnings.concat(preview_data.warnings || []); @@ -367,11 +368,13 @@ frappe.ui.form.on('Data Import', { .map(warning => { let header = ''; if (warning.col) { - header = __('Column {0}', [warning.col]); + let column_number = `${__('Column {0}', [warning.col])}`; + let column_header = columns[warning.col].header_title; + header = `${column_number} (${column_header})`; } return `
-
${header}
+
${header}
${warning.message}
`; From de07437015dd3a21cb1a4db1bc0fdf68230a35a3 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Wed, 22 Jul 2020 17:58:16 +0530 Subject: [PATCH 009/230] fix: Show Pending imports as Not Started --- frappe/core/doctype/data_import/data_import_list.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frappe/core/doctype/data_import/data_import_list.js b/frappe/core/doctype/data_import/data_import_list.js index 1dee4319f9..0eb05aa354 100644 --- a/frappe/core/doctype/data_import/data_import_list.js +++ b/frappe/core/doctype/data_import/data_import_list.js @@ -17,6 +17,7 @@ frappe.listview_settings['Data Import'] = { get_indicator: function(doc) { var colors = { 'Pending': 'orange', + 'Not Started': 'orange', 'Partial Success': 'orange', 'Success': 'green', 'In Progress': 'orange', @@ -26,6 +27,9 @@ frappe.listview_settings['Data Import'] = { if (imports_in_progress.includes(doc.name)) { status = 'In Progress'; } + if (status == 'Pending') { + status = 'Not Started'; + } return [__(status), colors[status], 'status,=,' + doc.status]; }, formatters: { From b0786fd5e5ade05555f16d4e0a1b50a6bbe611d0 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Wed, 22 Jul 2020 17:58:36 +0530 Subject: [PATCH 010/230] fix: Human friendly time format --- frappe/public/js/frappe/data_import/import_preview.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/data_import/import_preview.js b/frappe/public/js/frappe/data_import/import_preview.js index 4edcb87aeb..6c17cb4351 100644 --- a/frappe/public/js/frappe/data_import/import_preview.js +++ b/frappe/public/js/frappe/data_import/import_preview.js @@ -98,6 +98,9 @@ frappe.data_import.ImportPreview = class ImportPreview { .replace('%y', 'yy') .replace('%m', 'mm') .replace('%d', 'dd') + .replace('%H', 'HH') + .replace('%M', 'mm') + .replace('%S', 'ss') : null; let column_title = ` @@ -354,4 +357,4 @@ function get_fields_as_options(doctype, column_map) { }); }) ); -} \ No newline at end of file +} From 4b7e36f90da10a8c49fdbb827ea922e1f548e1e5 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Wed, 22 Jul 2020 19:30:17 +0530 Subject: [PATCH 011/230] fix: Handle child table row additions --- frappe/core/doctype/data_import/importer.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/frappe/core/doctype/data_import/importer.py b/frappe/core/doctype/data_import/importer.py index 6bc8b0586f..34e53488e5 100644 --- a/frappe/core/doctype/data_import/importer.py +++ b/frappe/core/doctype/data_import/importer.py @@ -602,12 +602,20 @@ class Row: is_table = frappe.get_meta(doctype).istable is_update = self.import_type == UPDATE - if is_table and is_update and doc.get("name") in INVALID_VALUES: - # for table rows being inserted in update - # create a new doc with defaults set - new_doc = frappe.new_doc(doctype, as_dict=True) - new_doc.update(doc) - doc = new_doc + if is_table and is_update: + # check if the row already exists + # if yes, fetch the original doc so that it is not updated + # if no, create a new doc + id_field = get_id_field(doctype) + id_value = doc.get(id_field.fieldname) + if id_value and frappe.db.exists(doctype, id_value): + doc = frappe.get_doc(doctype, id_value) + else: + # for table rows being inserted in update + # create a new doc with defaults set + new_doc = frappe.new_doc(doctype, as_dict=True) + new_doc.update(doc) + doc = new_doc self.check_mandatory_fields(doctype, doc, table_df) return doc From dd038a0cc201f0b5061864b8960892a419b4c0d0 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Wed, 22 Jul 2020 19:30:41 +0530 Subject: [PATCH 012/230] fix: Show autoname column instead of ID in export --- .../js/frappe/data_import/data_exporter.js | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/frappe/public/js/frappe/data_import/data_exporter.js b/frappe/public/js/frappe/data_import/data_exporter.js index f6af338235..8651f2083d 100644 --- a/frappe/public/js/frappe/data_import/data_exporter.js +++ b/frappe/public/js/frappe/data_import/data_exporter.js @@ -274,23 +274,29 @@ frappe.data_import.DataExporter = class DataExporter { ? this.column_map[child_fieldname] : this.column_map[doctype]; - let is_field_mandatory = df => (df.fieldname === 'name' && !child_fieldname) - || (df.reqd && this.exporting_for == 'Insert New Records'); + let is_field_mandatory = df => { + if (df.reqd && this.exporting_for == 'Insert New Records') { + return true; + } + if (autoname_field && df.fieldname == autoname_field.fieldname) { + return true; + } + if (df.fieldname === 'name') { + return true; + } + return false; + } return fields .filter(df => { - if (autoname_field && df.fieldname === autoname_field.fieldname) { + if (autoname_field && df.fieldname === 'name') { return false; } return true; }) .map(df => { - let label = __(df.label); - if (autoname_field && df.fieldname === 'name') { - label = label + ` (${__(autoname_field.label)})`; - } return { - label, + label: __(df.label), value: df.fieldname, danger: is_field_mandatory(df), checked: false, From 5558b20834fa0800da42a6dbab71b1e308c8e863 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Wed, 22 Jul 2020 19:52:56 +0530 Subject: [PATCH 013/230] style: missing semicolon --- frappe/public/js/frappe/data_import/data_exporter.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/data_import/data_exporter.js b/frappe/public/js/frappe/data_import/data_exporter.js index 8651f2083d..dee4839b34 100644 --- a/frappe/public/js/frappe/data_import/data_exporter.js +++ b/frappe/public/js/frappe/data_import/data_exporter.js @@ -285,7 +285,7 @@ frappe.data_import.DataExporter = class DataExporter { return true; } return false; - } + }; return fields .filter(df => { From b0b53e03f6a602c5cddcf2c3de5db580dba197af Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 23 Jul 2020 19:01:18 +0530 Subject: [PATCH 014/230] feat: Add Bulk restore API --- .../deleted_document/deleted_document.py | 34 +++++++++++++++++-- frappe/exceptions.py | 1 + 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/frappe/core/doctype/deleted_document/deleted_document.py b/frappe/core/doctype/deleted_document/deleted_document.py index bc2962ab3f..a1786e0a3e 100644 --- a/frappe/core/doctype/deleted_document/deleted_document.py +++ b/frappe/core/doctype/deleted_document/deleted_document.py @@ -4,6 +4,7 @@ from __future__ import unicode_literals import frappe, json +from frappe.desk.doctype.bulk_update.bulk_update import show_progress from frappe.model.document import Document from frappe import _ @@ -11,9 +12,14 @@ class DeletedDocument(Document): pass @frappe.whitelist() -def restore(name): +def restore(name, alert=True): deleted = frappe.get_doc('Deleted Document', name) + + if deleted.restored: + frappe.throw(_("Document {0} Already Restored".format(name)), exc=frappe.DocumentAlreadyRestored) + doc = frappe.get_doc(json.loads(deleted.data)) + try: doc.insert() except frappe.DocstatusTransitionError: @@ -27,4 +33,28 @@ def restore(name): deleted.restored = 1 deleted.db_update() - frappe.msgprint(_('Document Restored')) + if alert: + frappe.msgprint(_('Document Restored')) + + +@frappe.whitelist() +def bulk_restore(docnames): + docnames = frappe.parse_json(docnames) + doctype = "Deleted Document" + restored = [] + failed = [] + + for i, d in enumerate(docnames): + try: + restore(d, alert=False) + message = _('Restoring {0}').format(doctype) + frappe.db.commit() + show_progress(docnames, message, i+1, d) + restored.append(d) + except frappe.DocumentAlreadyRestored: + failed.append(d) + except Exception as e: + failed.append(d) + frappe.db.rollback() + + return restored \ No newline at end of file diff --git a/frappe/exceptions.py b/frappe/exceptions.py index 8ebda9c7b8..88428b875c 100644 --- a/frappe/exceptions.py +++ b/frappe/exceptions.py @@ -104,6 +104,7 @@ class IncompatibleApp(ValidationError): pass class InvalidDates(ValidationError): pass class DataTooLongException(ValidationError): pass class FileAlreadyAttachedException(Exception): pass +class DocumentAlreadyRestored(Exception): pass # OAuth exceptions class InvalidAuthorizationHeader(CSRFTokenError): pass class InvalidAuthorizationPrefix(CSRFTokenError): pass From 70e966d5b691b8b9d3889fb21e65d828d35549af Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 23 Jul 2020 19:01:51 +0530 Subject: [PATCH 015/230] style: formatting + dropped unused variables --- .../deleted_document/deleted_document.py | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/frappe/core/doctype/deleted_document/deleted_document.py b/frappe/core/doctype/deleted_document/deleted_document.py index a1786e0a3e..636e3c4f37 100644 --- a/frappe/core/doctype/deleted_document/deleted_document.py +++ b/frappe/core/doctype/deleted_document/deleted_document.py @@ -3,14 +3,17 @@ # For license information, please see license.txt from __future__ import unicode_literals -import frappe, json +import frappe +import json from frappe.desk.doctype.bulk_update.bulk_update import show_progress from frappe.model.document import Document from frappe import _ + class DeletedDocument(Document): pass + @frappe.whitelist() def restore(name, alert=True): deleted = frappe.get_doc('Deleted Document', name) @@ -40,21 +43,20 @@ def restore(name, alert=True): @frappe.whitelist() def bulk_restore(docnames): docnames = frappe.parse_json(docnames) - doctype = "Deleted Document" + message = _('Restoring Deleted Document') restored = [] - failed = [] for i, d in enumerate(docnames): try: + show_progress(docnames, message, i + 1, d) restore(d, alert=False) - message = _('Restoring {0}').format(doctype) frappe.db.commit() - show_progress(docnames, message, i+1, d) restored.append(d) + except frappe.DocumentAlreadyRestored: - failed.append(d) - except Exception as e: - failed.append(d) + pass + + except Exception: frappe.db.rollback() - return restored \ No newline at end of file + return restored From 9dd0304773c10fb363351cf1e0f07634082ac624 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 23 Jul 2020 19:02:25 +0530 Subject: [PATCH 016/230] feat: Bulk Restore action under Deleted Document --- .../deleted_document/deleted_document_list.js | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 frappe/core/doctype/deleted_document/deleted_document_list.js diff --git a/frappe/core/doctype/deleted_document/deleted_document_list.js b/frappe/core/doctype/deleted_document/deleted_document_list.js new file mode 100644 index 0000000000..eb5a9e087d --- /dev/null +++ b/frappe/core/doctype/deleted_document/deleted_document_list.js @@ -0,0 +1,33 @@ +frappe.listview_settings["Deleted Document"] = { + onload: function (doclist) { + const action = () => { + const selected_docs = doclist.get_checked_items(); + if (selected_docs.length > 0) { + let docnames = selected_docs.map((doc) => { + return doc.name; + }); + frappe.call({ + method: + "frappe.core.doctype.deleted_document.deleted_document.bulk_restore", + args: { docnames: docnames }, + callback: function (r) { + if (r.message) { + let num = r.message.length; + frappe.msgprint( + __("{0} Document{1} {2} Restored", [ + num, + num == 1 ? "" : "s", + num == 1 ? "was" : "were", + ]) + ); + if (num > 0) { + doclist.refresh(); + } + } + }, + }); + } + }; + doclist.page.add_actions_menu_item(__("Restore"), action, false); + }, +}; From 0fdb7953bb8b86b7fabe3f34cbf9987bf9d107c9 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 23 Jul 2020 19:12:08 +0530 Subject: [PATCH 017/230] fix: translation syntax and style fixes --- frappe/core/doctype/deleted_document/deleted_document.py | 2 +- .../core/doctype/deleted_document/deleted_document_list.js | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/frappe/core/doctype/deleted_document/deleted_document.py b/frappe/core/doctype/deleted_document/deleted_document.py index 636e3c4f37..598bd00cf6 100644 --- a/frappe/core/doctype/deleted_document/deleted_document.py +++ b/frappe/core/doctype/deleted_document/deleted_document.py @@ -19,7 +19,7 @@ def restore(name, alert=True): deleted = frappe.get_doc('Deleted Document', name) if deleted.restored: - frappe.throw(_("Document {0} Already Restored".format(name)), exc=frappe.DocumentAlreadyRestored) + frappe.throw(_("Document {0} Already Restored").format(name), exc=frappe.DocumentAlreadyRestored) doc = frappe.get_doc(json.loads(deleted.data)) diff --git a/frappe/core/doctype/deleted_document/deleted_document_list.js b/frappe/core/doctype/deleted_document/deleted_document_list.js index eb5a9e087d..12ed62f6de 100644 --- a/frappe/core/doctype/deleted_document/deleted_document_list.js +++ b/frappe/core/doctype/deleted_document/deleted_document_list.js @@ -7,9 +7,8 @@ frappe.listview_settings["Deleted Document"] = { return doc.name; }); frappe.call({ - method: - "frappe.core.doctype.deleted_document.deleted_document.bulk_restore", - args: { docnames: docnames }, + method: "frappe.core.doctype.deleted_document.deleted_document.bulk_restore", + args: { "docnames": docnames }, callback: function (r) { if (r.message) { let num = r.message.length; From 47b0fb3de4882787a910938feff605f880cc44b3 Mon Sep 17 00:00:00 2001 From: Anupam K Date: Fri, 24 Jul 2020 03:31:58 +0530 Subject: [PATCH 018/230] fix: print format custom button --- .../doctype/print_format/print_format.js | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/frappe/printing/doctype/print_format/print_format.js b/frappe/printing/doctype/print_format/print_format.js index 252c706e51..bb0ba27e38 100644 --- a/frappe/printing/doctype/print_format/print_format.js +++ b/frappe/printing/doctype/print_format/print_format.js @@ -35,13 +35,20 @@ frappe.ui.form.on("Print Format", { else if (frm.doc.custom_format && !frm.doc.raw_printing) { frm.set_df_property("html", "reqd", 1); } - frm.add_custom_button(__("Make Default"), function () { - frappe.call({ - method: "frappe.printing.doctype.print_format.print_format.make_default", - args: { - name: frm.doc.name - } - }) + frappe.db.get_value('DocType', frm.doc.doc_type, ['default_print_format', 'custom'], (r) => { + if (r.default_print_format != frm.doc.name && r.custom) { + frm.add_custom_button(__("Set as Default"), function () { + frappe.call({ + method: "frappe.printing.doctype.print_format.print_format.make_default", + args: { + name: frm.doc.name + }, + callback: function() { + frm.refresh(); + } + }); + }); + } }); } }, From ac1f82af9aeb457a202bb8a2907f3aed1df3e7aa Mon Sep 17 00:00:00 2001 From: gavin Date: Fri, 24 Jul 2020 11:41:10 +0530 Subject: [PATCH 019/230] fix: translation syntax Co-authored-by: Shivam Mishra --- .../deleted_document/deleted_document_list.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/frappe/core/doctype/deleted_document/deleted_document_list.js b/frappe/core/doctype/deleted_document/deleted_document_list.js index 12ed62f6de..4443dc883d 100644 --- a/frappe/core/doctype/deleted_document/deleted_document_list.js +++ b/frappe/core/doctype/deleted_document/deleted_document_list.js @@ -12,13 +12,13 @@ frappe.listview_settings["Deleted Document"] = { callback: function (r) { if (r.message) { let num = r.message.length; - frappe.msgprint( - __("{0} Document{1} {2} Restored", [ - num, - num == 1 ? "" : "s", - num == 1 ? "was" : "were", - ]) - ); + let message; + if (num === 1) { + message = "{0} Document was Restored" + } else { + message = "{0} Documents were Restored" + } + frappe.msgprint(__(message, [num]) if (num > 0) { doclist.refresh(); } From cfa286c5b03a01d9a6898f4bc90058c1470a08d1 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 24 Jul 2020 11:58:50 +0530 Subject: [PATCH 020/230] style: formatting, syntax and spacing --- .../core/doctype/deleted_document/deleted_document_list.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frappe/core/doctype/deleted_document/deleted_document_list.js b/frappe/core/doctype/deleted_document/deleted_document_list.js index 4443dc883d..a49d8125e3 100644 --- a/frappe/core/doctype/deleted_document/deleted_document_list.js +++ b/frappe/core/doctype/deleted_document/deleted_document_list.js @@ -14,11 +14,11 @@ frappe.listview_settings["Deleted Document"] = { let num = r.message.length; let message; if (num === 1) { - message = "{0} Document was Restored" + message = "{0} Document was Restored"; } else { - message = "{0} Documents were Restored" + message = "{0} Documents were Restored"; } - frappe.msgprint(__(message, [num]) + frappe.msgprint(__(message, [num])); if (num > 0) { doclist.refresh(); } From 85f8c712dcfba4f9c643e9d2f1dc4efd85aff010 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 24 Jul 2020 11:59:29 +0530 Subject: [PATCH 021/230] fix: Show files restore summary --- .../deleted_document/deleted_document.py | 31 ++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/frappe/core/doctype/deleted_document/deleted_document.py b/frappe/core/doctype/deleted_document/deleted_document.py index 598bd00cf6..e637f4f000 100644 --- a/frappe/core/doctype/deleted_document/deleted_document.py +++ b/frappe/core/doctype/deleted_document/deleted_document.py @@ -45,6 +45,8 @@ def bulk_restore(docnames): docnames = frappe.parse_json(docnames) message = _('Restoring Deleted Document') restored = [] + invalid = [] + failed = [] for i, d in enumerate(docnames): try: @@ -54,9 +56,36 @@ def bulk_restore(docnames): restored.append(d) except frappe.DocumentAlreadyRestored: - pass + frappe.message_log.pop() + invalid.append(d) except Exception: + failed.append(d) frappe.db.rollback() + frappe.message_log.pop() + + if failed or invalid: + tail = "" + + restored_data = "" + restored_head = _("Documents restored successfully") + "
    " + for docname in restored: + restored_data += "
  • {0}
    • ".format(docname) + restored_body = restored_head + restored_data + tail + + invalid_data = "" + invalid_head = _("Documents that were already Restored") + "
      " + for docname in invalid: + invalid_data += "
    • {0}
      • ".format(docname) + invalid_body = invalid_head + invalid_data + tail + + failed_data = "" + failed_head = _("Documents that Failed to Restore") + "
        " + for docname in failed: + failed_data += "
      • {0}
        • ".format(docname) + failed_body = failed_head + failed_data + tail + + summary = (restored_body if restored else "") + (invalid_body if invalid else "") + (failed_body if failed else "") + frappe.msgprint(summary, title="Document Restoration Summary", indicator="orange", is_minimizable=True) return restored From 9d24389c096f1dd2c4edfdfd588f9192bf94e437 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 24 Jul 2020 12:21:02 +0530 Subject: [PATCH 022/230] style: commonify summary generation --- .../deleted_document/deleted_document.py | 35 +++++++------------ 1 file changed, 13 insertions(+), 22 deletions(-) diff --git a/frappe/core/doctype/deleted_document/deleted_document.py b/frappe/core/doctype/deleted_document/deleted_document.py index e637f4f000..d8f39ca139 100644 --- a/frappe/core/doctype/deleted_document/deleted_document.py +++ b/frappe/core/doctype/deleted_document/deleted_document.py @@ -44,9 +44,7 @@ def restore(name, alert=True): def bulk_restore(docnames): docnames = frappe.parse_json(docnames) message = _('Restoring Deleted Document') - restored = [] - invalid = [] - failed = [] + restored, invalid, failed = [], [], [] for i, d in enumerate(docnames): try: @@ -60,32 +58,25 @@ def bulk_restore(docnames): invalid.append(d) except Exception: + frappe.message_log.pop() failed.append(d) frappe.db.rollback() - frappe.message_log.pop() if failed or invalid: - tail = "
      " + def body(docnames): + href = "
    • {0}
      • " + return "
        " + "".join([href.format(docname) for docname in docnames]) - restored_data = "" - restored_head = _("Documents restored successfully") + "
          " - for docname in restored: - restored_data += "
        • {0}
          • ".format(docname) - restored_body = restored_head + restored_data + tail + def message(title, docnames): + if docnames: + return _(title) + body(docnames) + "
        " + return "" - invalid_data = "" - invalid_head = _("Documents that were already Restored") + "
          " - for docname in invalid: - invalid_data += "
        • {0}
          • ".format(docname) - invalid_body = invalid_head + invalid_data + tail + restored_summary = message("Documents restored successfully", restored) + invalid_summary = message("Documents that were already Restored", invalid) + failed_summary = message("Documents that Failed to Restore", failed) - failed_data = "" - failed_head = _("Documents that Failed to Restore") + "
            " - for docname in failed: - failed_data += "
          • {0}
            • ".format(docname) - failed_body = failed_head + failed_data + tail - - summary = (restored_body if restored else "") + (invalid_body if invalid else "") + (failed_body if failed else "") + summary = restored_summary + invalid_summary + failed_summary frappe.msgprint(summary, title="Document Restoration Summary", indicator="orange", is_minimizable=True) return restored From af8771d5dccd8c882f74d5cca84e3275a6ae84a2 Mon Sep 17 00:00:00 2001 From: Saqib Ansari Date: Mon, 27 Jul 2020 15:51:44 +0530 Subject: [PATCH 023/230] fix: throw message for invalid date filter --- frappe/utils/data.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index d1b409d1fc..f36fa656b0 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -94,7 +94,10 @@ def add_to_date(date, years=0, months=0, weeks=0, days=0, hours=0, minutes=0, se as_string = True if " " in date: as_datetime = True - date = parser.parse(date) + try: + date = parser.parse(date) + except: + frappe.throw(frappe._("Please select a valid date filter"), title=frappe._("Invalid Date")) date = date + relativedelta(years=years, months=months, weeks=weeks, days=days, hours=hours, minutes=minutes, seconds=seconds) From ad1152f8e8367f8d7c46d0e4e9a14ee9e2d0df9b Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 27 Jul 2020 17:13:18 +0530 Subject: [PATCH 024/230] feat: override site name via _site added filter parameter for logger --- frappe/__init__.py | 4 ++-- frappe/utils/logger.py | 16 +++++++++------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index 8c24980189..504ac96eb9 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -1559,10 +1559,10 @@ def get_doctype_app(doctype): loggers = {} log_level = None -def logger(module=None, with_more_info=False, _site=None): +def logger(module=None, with_more_info=False, _site=None, filter=None): '''Returns a python logger that uses StreamHandler''' from frappe.utils.logger import get_logger - return get_logger(module=module, with_more_info=with_more_info, _site=_site) + return get_logger(module=module, with_more_info=with_more_info, _site=_site, filter=filter) def log_error(message=None, title=_("Error")): '''Log error to Error Log''' diff --git a/frappe/utils/logger.py b/frappe/utils/logger.py index 9254c0aedd..56fcf9857a 100755 --- a/frappe/utils/logger.py +++ b/frappe/utils/logger.py @@ -17,22 +17,20 @@ default_log_level = logging.DEBUG site = getattr(frappe.local, 'site', None) -def get_logger(module, with_more_info=False, _site=None): +def get_logger(module, with_more_info=False, _site=None, filter=None): """Application Logger for your given module Args: module (str): Name of your logger and consequently your log file. with_more_info (bool, optional): Will log the form dict using the SiteContextFilter. Defaults to False. - _site (str, optional): If set, validates the current site context with the passed value. The `frappe.web` logger uses this to determine that the application is logging information related to the logger called. Defaults to None. + _site (str, optional): Overrides the global site variable set in this scope. Defaults to None. + filter (function, optional): Add a filter function for your logger Returns: : Returns a Python logger object with Site and Bench level logging capabilities. """ global site - def allow_site(): - return (_site and _site == site) or bool(site) - if module in frappe.loggers: return frappe.loggers[module] @@ -42,6 +40,7 @@ def get_logger(module, with_more_info=False, _site=None): logfile = module + '.log' site = getattr(frappe.local, 'site', None) + equivalent_site = _site or site LOG_FILENAME = os.path.join('..', 'logs', logfile) logger = logging.getLogger(module) @@ -52,8 +51,8 @@ def get_logger(module, with_more_info=False, _site=None): handler = RotatingFileHandler(LOG_FILENAME, maxBytes=100_000, backupCount=20) logger.addHandler(handler) - if allow_site(): - SITELOG_FILENAME = os.path.join(site, 'logs', logfile) + if equivalent_site: + SITELOG_FILENAME = os.path.join(equivalent_site, 'logs', logfile) site_handler = RotatingFileHandler(SITELOG_FILENAME, maxBytes=100_000, backupCount=20) site_handler.setFormatter(formatter) logger.addHandler(site_handler) @@ -61,6 +60,9 @@ def get_logger(module, with_more_info=False, _site=None): if with_more_info: handler.addFilter(SiteContextFilter()) + if filter: + logger.addFilter(filter) + handler.setFormatter(formatter) frappe.loggers[module] = logger From 1345c2feb311bf7abb06b3f1969c4a2a43dc9b94 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 27 Jul 2020 17:31:12 +0530 Subject: [PATCH 025/230] feat: Allow_site flag in frappe.logger --- frappe/__init__.py | 4 ++-- frappe/app.py | 2 +- frappe/utils/logger.py | 22 +++++++++++++--------- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index 504ac96eb9..7ea95ec3db 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -1559,10 +1559,10 @@ def get_doctype_app(doctype): loggers = {} log_level = None -def logger(module=None, with_more_info=False, _site=None, filter=None): +def logger(module=None, with_more_info=False, allow_site=True, filter=None): '''Returns a python logger that uses StreamHandler''' from frappe.utils.logger import get_logger - return get_logger(module=module, with_more_info=with_more_info, _site=_site, filter=filter) + return get_logger(module=module, with_more_info=with_more_info, allow_site=allow_site, filter=filter) def log_error(message=None, title=_("Error")): '''Log error to Error Log''' diff --git a/frappe/app.py b/frappe/app.py index 83a9b8c149..100b10e9ef 100644 --- a/frappe/app.py +++ b/frappe/app.py @@ -100,7 +100,7 @@ def application(request): frappe.recorder.dump() _site = get_site_name(request.host) - frappe.logger("frappe.web", _site=_site).info({ + frappe.logger("frappe.web", allow_site=_site).info({ "site": _site, "remote_addr": getattr(request, "remote_addr", "NOTFOUND"), "base_url": getattr(request, "base_url", "NOTFOUND"), diff --git a/frappe/utils/logger.py b/frappe/utils/logger.py index 56fcf9857a..e7058ce42d 100755 --- a/frappe/utils/logger.py +++ b/frappe/utils/logger.py @@ -14,22 +14,20 @@ import frappe default_log_level = logging.DEBUG -site = getattr(frappe.local, 'site', None) -def get_logger(module, with_more_info=False, _site=None, filter=None): +def get_logger(module, with_more_info=False, allow_site=True, filter=None): """Application Logger for your given module Args: module (str): Name of your logger and consequently your log file. with_more_info (bool, optional): Will log the form dict using the SiteContextFilter. Defaults to False. - _site (str, optional): Overrides the global site variable set in this scope. Defaults to None. + allow_site ((str, bool), optional): Pass site name to explicitly log under it's logs. If True and unspecified, guesses which site the logs would be saved under. Defaults to True. filter (function, optional): Add a filter function for your logger Returns: : Returns a Python logger object with Site and Bench level logging capabilities. """ - global site if module in frappe.loggers: return frappe.loggers[module] @@ -39,8 +37,14 @@ def get_logger(module, with_more_info=False, _site=None, filter=None): with_more_info = True logfile = module + '.log' - site = getattr(frappe.local, 'site', None) - equivalent_site = _site or site + + if allow_site == True: + site = getattr(frappe.local, 'site', None) + elif allow_site: + site = allow_site + else: + site = False + LOG_FILENAME = os.path.join('..', 'logs', logfile) logger = logging.getLogger(module) @@ -51,8 +55,8 @@ def get_logger(module, with_more_info=False, _site=None, filter=None): handler = RotatingFileHandler(LOG_FILENAME, maxBytes=100_000, backupCount=20) logger.addHandler(handler) - if equivalent_site: - SITELOG_FILENAME = os.path.join(equivalent_site, 'logs', logfile) + if site: + SITELOG_FILENAME = os.path.join(site, 'logs', logfile) site_handler = RotatingFileHandler(SITELOG_FILENAME, maxBytes=100_000, backupCount=20) site_handler.setFormatter(formatter) logger.addHandler(site_handler) @@ -73,7 +77,7 @@ class SiteContextFilter(logging.Filter): """This is a filter which injects request information (if available) into the log.""" def filter(self, record): if "Form Dict" not in text_type(record.msg): - record.msg = text_type(record.msg) + "\nSite: {0}\nForm Dict: {1}".format(site, getattr(frappe.local, 'form_dict', None)) + record.msg = text_type(record.msg) + "\nSite: {0}\nForm Dict: {1}".format(getattr(frappe.local, 'site', None), getattr(frappe.local, 'form_dict', None)) return True def set_log_level(level): From dc46450f89a2d95163eda8002f047ba3e6b04630 Mon Sep 17 00:00:00 2001 From: prssanna Date: Mon, 27 Jul 2020 18:14:02 +0530 Subject: [PATCH 026/230] fix: push fieldtype in invalid conditions map --- frappe/public/js/frappe/ui/filters/filter.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/ui/filters/filter.js b/frappe/public/js/frappe/ui/filters/filter.js index bfc0b1449d..4dedfb32fe 100644 --- a/frappe/public/js/frappe/ui/filters/filter.js +++ b/frappe/public/js/frappe/ui/filters/filter.js @@ -57,7 +57,7 @@ frappe.ui.Filter = class { this.conditions.push([key, __(`{0}`, [filter.label])]); for (let fieldtype of Object.keys(this.invalid_condition_map)) { if (!filter.valid_for_fieldtypes.includes(fieldtype)) { - this.invalid_condition_map[fieldtype].push(filter.label); + this.invalid_condition_map[fieldtype].push(key); } } } From 59ab9249cc3a9e7abeb5f6e599133f705ea6d8e6 Mon Sep 17 00:00:00 2001 From: Anupam K Date: Mon, 27 Jul 2020 19:40:11 +0530 Subject: [PATCH 027/230] fix: print format custom button --- frappe/printing/doctype/print_format/print_format.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/printing/doctype/print_format/print_format.js b/frappe/printing/doctype/print_format/print_format.js index bb0ba27e38..e6599b2496 100644 --- a/frappe/printing/doctype/print_format/print_format.js +++ b/frappe/printing/doctype/print_format/print_format.js @@ -35,8 +35,8 @@ frappe.ui.form.on("Print Format", { else if (frm.doc.custom_format && !frm.doc.raw_printing) { frm.set_df_property("html", "reqd", 1); } - frappe.db.get_value('DocType', frm.doc.doc_type, ['default_print_format', 'custom'], (r) => { - if (r.default_print_format != frm.doc.name && r.custom) { + frappe.db.get_value('DocType', frm.doc.doc_type, 'default_print_format', (r) => { + if (r.default_print_format != frm.doc.name) { frm.add_custom_button(__("Set as Default"), function () { frappe.call({ method: "frappe.printing.doctype.print_format.print_format.make_default", From 786625a92a94536b749f0b02acc9fc4cf56516c7 Mon Sep 17 00:00:00 2001 From: Rucha Mahabal Date: Mon, 27 Jul 2020 21:34:56 +0530 Subject: [PATCH 028/230] fix: regex for youtube video help link --- frappe/public/js/frappe/utils/help.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/utils/help.js b/frappe/public/js/frappe/utils/help.js index 336d1ef526..f88849d212 100644 --- a/frappe/public/js/frappe/utils/help.js +++ b/frappe/public/js/frappe/utils/help.js @@ -17,7 +17,7 @@ frappe.help.show = function(doctype) { frappe.help.show_video = function(youtube_id, title) { if (frappe.utils.is_url(youtube_id)) { - const expression = '(?:youtube.com/(?:[^/]+/.+/|(?:v|e(?:mbed)?)/|.*[?&]v=)|youtu.be/)([^\"&?/s]{11})'; + const expression = '(?:youtube.com/(?:[^/]+/.+/|(?:v|e(?:mbed)?)/|.*[?&]v=)|youtu.be/)([^\"&?\\s]{11})'; youtube_id = youtube_id.match(expression)[1]; } From cb6208b19be27f72e05c017ef395588755a1aba1 Mon Sep 17 00:00:00 2001 From: Mangesh-Khairnar Date: Mon, 27 Jul 2020 22:49:42 +0530 Subject: [PATCH 029/230] fix(field-group): strip html to check for null values --- frappe/public/js/frappe/ui/field_group.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/ui/field_group.js b/frappe/public/js/frappe/ui/field_group.js index 4d827354d8..ec7234c9bb 100644 --- a/frappe/public/js/frappe/ui/field_group.js +++ b/frappe/public/js/frappe/ui/field_group.js @@ -86,7 +86,7 @@ frappe.ui.FieldGroup = frappe.ui.form.Layout.extend({ var f = this.fields_dict[key]; if(f.get_value) { var v = f.get_value(); - if(f.df.reqd && is_null(v)) + if(f.df.reqd && is_null(strip_html(cstr(v)))) errors.push(__(f.df.label)); if(!is_null(v)) ret[f.df.fieldname] = v; From 7383a55c9adb9af62590ba18738609784b4ea8b3 Mon Sep 17 00:00:00 2001 From: Rucha Mahabal Date: Tue, 28 Jul 2020 11:29:22 +0530 Subject: [PATCH 030/230] fix: remove useless escape character --- frappe/public/js/frappe/utils/help.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/utils/help.js b/frappe/public/js/frappe/utils/help.js index f88849d212..ec6f7c8158 100644 --- a/frappe/public/js/frappe/utils/help.js +++ b/frappe/public/js/frappe/utils/help.js @@ -17,7 +17,7 @@ frappe.help.show = function(doctype) { frappe.help.show_video = function(youtube_id, title) { if (frappe.utils.is_url(youtube_id)) { - const expression = '(?:youtube.com/(?:[^/]+/.+/|(?:v|e(?:mbed)?)/|.*[?&]v=)|youtu.be/)([^\"&?\\s]{11})'; + const expression = '(?:youtube.com/(?:[^/]+/.+/|(?:v|e(?:mbed)?)/|.*[?&]v=)|youtu.be/)([^"&?\\s]{11})'; youtube_id = youtube_id.match(expression)[1]; } From d9c8b6b66281946ee504c240c84a8819e60bc903 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Tue, 28 Jul 2020 13:09:24 +0530 Subject: [PATCH 031/230] fix: Don't render meta tags if no value --- frappe/templates/includes/meta_block.html | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/frappe/templates/includes/meta_block.html b/frappe/templates/includes/meta_block.html index 3d867aaef3..1aef8a160e 100644 --- a/frappe/templates/includes/meta_block.html +++ b/frappe/templates/includes/meta_block.html @@ -1,5 +1,8 @@ {%- if metatags -%} {%- for name in metatags %} - +{%- set content = metatags.get(name, '') -%} +{%- if content -%} + +{%- endif -%} {%- endfor -%} {%- endif -%} From 1903e169dfa464704f2360bd7e1ccb8c03962424 Mon Sep 17 00:00:00 2001 From: gavin Date: Wed, 29 Jul 2020 11:16:24 +0530 Subject: [PATCH 032/230] fix: Update translations ref: https://github.com/frappe/frappe/pull/11094#discussion_r462050750 --- .../doctype/deleted_document/deleted_document_list.js | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/frappe/core/doctype/deleted_document/deleted_document_list.js b/frappe/core/doctype/deleted_document/deleted_document_list.js index a49d8125e3..ac21d0ff42 100644 --- a/frappe/core/doctype/deleted_document/deleted_document_list.js +++ b/frappe/core/doctype/deleted_document/deleted_document_list.js @@ -13,12 +13,14 @@ frappe.listview_settings["Deleted Document"] = { if (r.message) { let num = r.message.length; let message; - if (num === 1) { - message = "{0} Document was Restored"; + if (num === 0) { + message = __("No Documents were Restored"); + } else if (num === 1) { + message = __("Document was Restored"); } else { - message = "{0} Documents were Restored"; + message = __("{0} Documents were Restored", [num]); } - frappe.msgprint(__(message, [num])); + frappe.msgprint(message); if (num > 0) { doclist.refresh(); } From ae3ee5b2eb5c64fa00f4461ad1a11841ced65c54 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 29 Jul 2020 13:33:04 +0530 Subject: [PATCH 033/230] fix: save frappe loggers site wise --- frappe/utils/logger.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/frappe/utils/logger.py b/frappe/utils/logger.py index e7058ce42d..c38e0e6997 100755 --- a/frappe/utils/logger.py +++ b/frappe/utils/logger.py @@ -29,8 +29,18 @@ def get_logger(module, with_more_info=False, allow_site=True, filter=None): : Returns a Python logger object with Site and Bench level logging capabilities. """ + if allow_site == True: + site = getattr(frappe.local, 'site', None) + elif allow_site: + site = allow_site + else: + site = False + if module in frappe.loggers: - return frappe.loggers[module] + try: + return frappe.loggers[module][site or "all"] + except: + pass if not module: module = "frappe" @@ -38,12 +48,6 @@ def get_logger(module, with_more_info=False, allow_site=True, filter=None): logfile = module + '.log' - if allow_site == True: - site = getattr(frappe.local, 'site', None) - elif allow_site: - site = allow_site - else: - site = False LOG_FILENAME = os.path.join('..', 'logs', logfile) @@ -69,7 +73,11 @@ def get_logger(module, with_more_info=False, allow_site=True, filter=None): handler.setFormatter(formatter) - frappe.loggers[module] = logger + try: + frappe.loggers[module][site or "all"] = logger + except KeyError: + frappe.loggers[module] = {} + frappe.loggers[module][site or "all"] = logger return logger From d3e4a97b6fc96f2003c1fad3c14a99b105ade5ed Mon Sep 17 00:00:00 2001 From: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> Date: Wed, 29 Jul 2020 17:17:53 +0530 Subject: [PATCH 034/230] test: Make email account is email account test (#11139) --- frappe/core/doctype/server_script/test_server_script.py | 7 ++++--- frappe/email/doctype/email_account/test_email_account.py | 5 ++++- frappe/model/workflow.py | 2 +- frappe/tests/test_naming.py | 1 + frappe/workflow/doctype/workflow/test_workflow.py | 3 ++- 5 files changed, 12 insertions(+), 6 deletions(-) diff --git a/frappe/core/doctype/server_script/test_server_script.py b/frappe/core/doctype/server_script/test_server_script.py index 3e6b7a3a98..5c12858e8a 100644 --- a/frappe/core/doctype/server_script/test_server_script.py +++ b/frappe/core/doctype/server_script/test_server_script.py @@ -52,9 +52,10 @@ class TestServerScript(unittest.TestCase): frappe.db.commit() - # @classmethod - # def tearDownClass(cls): - # frappe.db.sql('truncate `tabServer Script`') + @classmethod + def tearDownClass(cls): + frappe.db.commit() + frappe.db.sql('truncate `tabServer Script`') def setUp(self): frappe.cache().delete_value('server_script_map') diff --git a/frappe/email/doctype/email_account/test_email_account.py b/frappe/email/doctype/email_account/test_email_account.py index 29b54d7f8b..f87ee32bb1 100644 --- a/frappe/email/doctype/email_account/test_email_account.py +++ b/frappe/email/doctype/email_account/test_email_account.py @@ -5,7 +5,10 @@ from __future__ import unicode_literals import frappe, os import unittest, email -test_records = frappe.get_test_records('Email Account') +from frappe.test_runner import make_test_records + +make_test_records("User") +make_test_records("Email Account") from frappe.core.doctype.communication.email import make from frappe.desk.form.load import get_attachments diff --git a/frappe/model/workflow.py b/frappe/model/workflow.py index ea563dfc13..32919b3333 100644 --- a/frappe/model/workflow.py +++ b/frappe/model/workflow.py @@ -307,4 +307,4 @@ def set_workflow_state_on_action(doc, workflow_name, action): for state in workflow.states: if state.doc_status == docstatus: doc.set(workflow_state_field, state.state) - return \ No newline at end of file + return diff --git a/frappe/tests/test_naming.py b/frappe/tests/test_naming.py index 90ab6a6a94..b47fb809ca 100644 --- a/frappe/tests/test_naming.py +++ b/frappe/tests/test_naming.py @@ -88,6 +88,7 @@ class TestNaming(unittest.TestCase): series = 'TEST-' key = 'TEST-' name = 'TEST-00003' + frappe.db.sql("DELETE FROM `tabSeries` WHERE `name`=%s", series) frappe.db.sql("""INSERT INTO `tabSeries` (name, current) values (%s, 3)""", (series,)) revert_series_if_last(key, name) count = frappe.db.sql("""SELECT current from `tabSeries` where name = %s""", series, as_dict=True)[0] diff --git a/frappe/workflow/doctype/workflow/test_workflow.py b/frappe/workflow/doctype/workflow/test_workflow.py index 9999df40cb..2719bc7cf0 100644 --- a/frappe/workflow/doctype/workflow/test_workflow.py +++ b/frappe/workflow/doctype/workflow/test_workflow.py @@ -10,6 +10,7 @@ from frappe.model.workflow import apply_workflow, WorkflowTransitionError, Workf class TestWorkflow(unittest.TestCase): def setUp(self): frappe.db.sql('DELETE FROM `tabToDo`') + frappe.db.sql("DELETE FROM `tabHas Role` WHERE `role`='Test Approver'") if not getattr(self, 'workflow', None): self.workflow = create_todo_workflow() frappe.set_user('Administrator') @@ -146,4 +147,4 @@ def create_todo_workflow(): return workflow def create_new_todo(): - return frappe.get_doc(dict(doctype='ToDo', description='workflow ' + random_string(10))).insert() \ No newline at end of file + return frappe.get_doc(dict(doctype='ToDo', description='workflow ' + random_string(10))).insert() From 219fbc8e88d8505391fe8259b0fbeda86844f42d Mon Sep 17 00:00:00 2001 From: Matteo D Date: Wed, 29 Jul 2020 17:28:46 +0200 Subject: [PATCH 035/230] fix: Added AWS S3 missing regions (#11109) --- .gitignore | 5 ++++- .../doctype/s3_backup_settings/s3_backup_settings.json | 8 ++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 7df673c1f1..900ae1c7b7 100644 --- a/.gitignore +++ b/.gitignore @@ -188,4 +188,7 @@ typings/ # cypress cypress/screenshots -cypress/videos \ No newline at end of file +cypress/videos + +# JetBrains IDEs +.idea/ diff --git a/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.json b/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.json index 830afbae53..123bb21e88 100755 --- a/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.json +++ b/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.json @@ -74,11 +74,11 @@ }, { "default": "us-east-1", - "description": "See https://docs.aws.amazon.com/de_de/general/latest/gr/rande.html#s3_region for details.", + "description": "See https://docs.aws.amazon.com/general/latest/gr/s3.html for details.", "fieldname": "region", "fieldtype": "Select", "label": "Region", - "options": "us-east-1\nus-east-2\nus-west-1\nus-west-2\nap-south-1\nap-southeast-1\nap-southeast-2\nap-northeast-1\nap-northeast-2\nap-northeast-3\nca-central-1\ncn-north-1\ncn-northwest-1\neu-central-1\neu-west-1\neu-west-2\neu-west-3\neu-north-1\nsa-east-1" + "options": "us-east-1\nus-east-2\nus-west-1\nus-west-2\naf-south-1\nap-east-1\nap-south-1\nap-southeast-1\nap-southeast-2\nap-northeast-1\nap-northeast-2\nap-northeast-3\nca-central-1\ncn-north-1\ncn-northwest-1\neu-central-1\neu-west-1\neu-west-2\neu-west-3\neu-south-1\neu-north-1\nme-south-1\nsa-east-1" }, { "fieldname": "endpoint_url", @@ -151,7 +151,7 @@ "hide_toolbar": 1, "issingle": 1, "links": [], - "modified": "2020-04-13 20:57:24.432183", + "modified": "2020-07-27 17:27:21.400000", "modified_by": "Administrator", "module": "Integrations", "name": "S3 Backup Settings", @@ -172,4 +172,4 @@ "sort_field": "modified", "sort_order": "DESC", "track_changes": 1 -} \ No newline at end of file +} From 8d18fb43230c80b40c00732850b5631c4346d54a Mon Sep 17 00:00:00 2001 From: Raffael Meyer <14891507+barredterra@users.noreply.github.com> Date: Wed, 29 Jul 2020 17:48:39 +0200 Subject: [PATCH 036/230] fix(oauth provider): parse cookies correctly (#11066) --- frappe/oauth.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/frappe/oauth.py b/frappe/oauth.py index 4dc50366be..122c806072 100644 --- a/frappe/oauth.py +++ b/frappe/oauth.py @@ -4,6 +4,7 @@ import pytz from frappe import _ from frappe.auth import LoginManager +from http import cookies from oauthlib.oauth2.rfc6749.tokens import BearerToken from oauthlib.oauth2.rfc6749.grant_types import AuthorizationCodeGrant, ImplicitGrant, ResourceOwnerPasswordCredentialsGrant, ClientCredentialsGrant, RefreshTokenGrant from oauthlib.oauth2 import RequestValidator @@ -130,15 +131,12 @@ class OAuthWebRequestValidator(RequestValidator): oac.scopes = get_url_delimiter().join(request.scopes) oac.redirect_uri_bound_to_authorization_code = request.redirect_uri oac.client = client_id - oac.user = unquote(cookie_dict['user_id']) + oac.user = unquote(cookie_dict['user_id'].value) oac.authorization_code = code['code'] oac.save(ignore_permissions=True) frappe.db.commit() def authenticate_client(self, request, *args, **kwargs): - - cookie_dict = get_cookie_dict_from_headers(request) - #Get ClientID in URL if request.client_id: oc = frappe.get_doc("OAuth Client", request.client_id) @@ -155,7 +153,9 @@ class OAuthWebRequestValidator(RequestValidator): except Exception as e: print("Failed body authentication: Application %s does not exist".format(cid=request.client_id)) - return frappe.session.user == unquote(cookie_dict.get('user_id', "Guest")) + cookie_dict = get_cookie_dict_from_headers(request) + user_id = unquote(cookie_dict['user_id']) if 'user_id' in cookie_dict else "Guest" + return frappe.session.user == user_id def authenticate_client_id(self, client_id, request, *args, **kwargs): cli_id = frappe.db.get_value('OAuth Client', client_id, 'name') @@ -400,13 +400,10 @@ class OAuthWebRequestValidator(RequestValidator): return True def get_cookie_dict_from_headers(r): + cookie = cookies.BaseCookie() if r.headers.get('Cookie'): - cookie = r.headers.get('Cookie') - cookie = cookie.split("; ") - cookie_dict = {k:v for k,v in (x.split('=') for x in cookie)} - return cookie_dict - else: - return {} + cookie.load(r.headers.get('Cookie')) + return cookie def calculate_at_hash(access_token, hash_alg): """Helper method for calculating an access token From 9cef385005e68221cc2d4b628f9080b81a434da2 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Thu, 30 Jul 2020 14:37:22 +0530 Subject: [PATCH 037/230] refactor: refer to frappe.boot.single_types instead of db call --- .../public/js/frappe/widgets/widget_dialog.js | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/frappe/public/js/frappe/widgets/widget_dialog.js b/frappe/public/js/frappe/widgets/widget_dialog.js index 054159116f..1089d7e629 100644 --- a/frappe/public/js/frappe/widgets/widget_dialog.js +++ b/frappe/public/js/frappe/widgets/widget_dialog.js @@ -174,18 +174,12 @@ class ShortcutDialog extends WidgetDialog { onchange: () => { if (this.dialog.get_value("type") == "DocType") { let doctype = this.dialog.get_value("link_to"); - - doctype && - frappe.db - .get_value("DocType", doctype, "issingle") - .then((res) => { - if (res.message && res.message.issingle) { - this.hide_filters(); - } else { - this.setup_filter(doctype); - this.show_filters(); - } - }); + if (frappe.boot.single_types.includes("Stock Settings")) { + this.hide_filters(); + } else { + this.setup_filter(doctype); + this.show_filters(); + } } else { this.hide_filters(); } From 2ad15ee9f75dca2391e45b83f5298b1e7e90e163 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Thu, 30 Jul 2020 14:57:01 +0530 Subject: [PATCH 038/230] fix: use doctype variable --- frappe/public/js/frappe/widgets/widget_dialog.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/widgets/widget_dialog.js b/frappe/public/js/frappe/widgets/widget_dialog.js index 1089d7e629..e3b4cc1d8c 100644 --- a/frappe/public/js/frappe/widgets/widget_dialog.js +++ b/frappe/public/js/frappe/widgets/widget_dialog.js @@ -174,7 +174,7 @@ class ShortcutDialog extends WidgetDialog { onchange: () => { if (this.dialog.get_value("type") == "DocType") { let doctype = this.dialog.get_value("link_to"); - if (frappe.boot.single_types.includes("Stock Settings")) { + if (frappe.boot.single_types.includes(doctype)) { this.hide_filters(); } else { this.setup_filter(doctype); From 7427fb9cd3b9589d8366f330b968a9f5e8c6f310 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Thu, 30 Jul 2020 15:56:38 +0530 Subject: [PATCH 039/230] fix: Dont validate skipped columns --- frappe/core/doctype/data_import/importer.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/frappe/core/doctype/data_import/importer.py b/frappe/core/doctype/data_import/importer.py index 910e42af1a..14626eb5e3 100644 --- a/frappe/core/doctype/data_import/importer.py +++ b/frappe/core/doctype/data_import/importer.py @@ -955,6 +955,9 @@ class Column: if not self.df: return + if self.skip_import: + return + if self.df.fieldtype == 'Link': # find all values that dont exist values = list(set([cstr(v) for v in self.column_values[1:] if v])) From 84a15407a733a57ce5d1b13c2c1fefd80f12b198 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Thu, 30 Jul 2020 16:06:10 +0530 Subject: [PATCH 040/230] fix: Respect hostname config in sitemap.xml --- frappe/www/sitemap.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frappe/www/sitemap.py b/frappe/www/sitemap.py index 8b93270ab5..f8f03c45f8 100644 --- a/frappe/www/sitemap.py +++ b/frappe/www/sitemap.py @@ -5,7 +5,7 @@ from __future__ import unicode_literals import frappe from frappe.model.document import get_controller -from frappe.utils import get_request_site_address, get_datetime, nowdate +from frappe.utils import get_datetime, nowdate, get_url from frappe.website.router import get_pages, get_all_page_context_from_doctypes from six import iteritems from six.moves.urllib.parse import quote, urljoin @@ -25,13 +25,13 @@ def get_context(context): for route, page in iteritems(get_pages()): if page.sitemap: links.append({ - "loc": urljoin(host, quote(page.name.encode("utf-8"))), + "loc": get_url(quote(page.name.encode("utf-8"))), "lastmod": nowdate() }) for route, data in iteritems(get_public_pages_from_doctypes()): links.append({ - "loc": urljoin(host, quote((route or "").encode("utf-8"))), + "loc": get_url(quote((route or "").encode("utf-8"))), "lastmod": get_datetime(data.get("modified")).strftime("%Y-%m-%d") }) From 7bf69c62c1cc942d394d40484502c51b96e98b9f Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 30 Jul 2020 17:47:01 +0530 Subject: [PATCH 041/230] fix: suffix logger name with site --- frappe/utils/logger.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/frappe/utils/logger.py b/frappe/utils/logger.py index c38e0e6997..1fbd54dd38 100755 --- a/frappe/utils/logger.py +++ b/frappe/utils/logger.py @@ -36,9 +36,11 @@ def get_logger(module, with_more_info=False, allow_site=True, filter=None): else: site = False - if module in frappe.loggers: + LOGGER_NAME = "{}-{}".format(module, site or "all") + + if LOGGER_NAME in frappe.loggers: try: - return frappe.loggers[module][site or "all"] + return frappe.loggers[LOGGER_NAME] except: pass @@ -51,7 +53,7 @@ def get_logger(module, with_more_info=False, allow_site=True, filter=None): LOG_FILENAME = os.path.join('..', 'logs', logfile) - logger = logging.getLogger(module) + logger = logging.getLogger(LOGGER_NAME) logger.setLevel(frappe.log_level or default_log_level) logger.propagate = False @@ -73,11 +75,7 @@ def get_logger(module, with_more_info=False, allow_site=True, filter=None): handler.setFormatter(formatter) - try: - frappe.loggers[module][site or "all"] = logger - except KeyError: - frappe.loggers[module] = {} - frappe.loggers[module][site or "all"] = logger + frappe.loggers[LOGGER_NAME] = logger return logger From 090b81f2f268447581eac496bb245af3a17224c5 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 30 Jul 2020 17:59:01 +0530 Subject: [PATCH 042/230] style: black + restructure --- frappe/utils/logger.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/frappe/utils/logger.py b/frappe/utils/logger.py index 1fbd54dd38..79ea93e3bd 100755 --- a/frappe/utils/logger.py +++ b/frappe/utils/logger.py @@ -30,7 +30,7 @@ def get_logger(module, with_more_info=False, allow_site=True, filter=None): """ if allow_site == True: - site = getattr(frappe.local, 'site', None) + site = getattr(frappe.local, "site", None) elif allow_site: site = allow_site else: @@ -48,21 +48,21 @@ def get_logger(module, with_more_info=False, allow_site=True, filter=None): module = "frappe" with_more_info = True - logfile = module + '.log' + logfile = module + ".log" - - LOG_FILENAME = os.path.join('..', 'logs', logfile) + LOG_FILENAME = os.path.join("..", "logs", logfile) logger = logging.getLogger(LOGGER_NAME) logger.setLevel(frappe.log_level or default_log_level) logger.propagate = False - formatter = logging.Formatter('%(asctime)s %(levelname)s %(name)s %(message)s') + formatter = logging.Formatter("%(asctime)s %(levelname)s %(name)s %(message)s") handler = RotatingFileHandler(LOG_FILENAME, maxBytes=100_000, backupCount=20) + handler.setFormatter(formatter) logger.addHandler(handler) if site: - SITELOG_FILENAME = os.path.join(site, 'logs', logfile) + SITELOG_FILENAME = os.path.join(site, "logs", logfile) site_handler = RotatingFileHandler(SITELOG_FILENAME, maxBytes=100_000, backupCount=20) site_handler.setFormatter(formatter) logger.addHandler(site_handler) @@ -73,20 +73,23 @@ def get_logger(module, with_more_info=False, allow_site=True, filter=None): if filter: logger.addFilter(filter) - handler.setFormatter(formatter) - frappe.loggers[LOGGER_NAME] = logger return logger + class SiteContextFilter(logging.Filter): """This is a filter which injects request information (if available) into the log.""" + def filter(self, record): if "Form Dict" not in text_type(record.msg): - record.msg = text_type(record.msg) + "\nSite: {0}\nForm Dict: {1}".format(getattr(frappe.local, 'site', None), getattr(frappe.local, 'form_dict', None)) + site = getattr(frappe.local, "site", None) + form_dict = getattr(frappe.local, "form_dict", None) + record.msg = text_type(record.msg) + "\nSite: {0}\nForm Dict: {1}".format(site, form_dict) return True + def set_log_level(level): - '''Use this method to set log level to something other than the default DEBUG''' - frappe.log_level = getattr(logging, (level or '').upper(), None) or default_log_level + """Use this method to set log level to something other than the default DEBUG""" + frappe.log_level = getattr(logging, (level or "").upper(), None) or default_log_level frappe.loggers = {} From fac7f359ae0b49064003e7424989579d580b7535 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 30 Jul 2020 18:49:59 +0530 Subject: [PATCH 043/230] fix: original log format + EAFP --- frappe/utils/logger.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/frappe/utils/logger.py b/frappe/utils/logger.py index 79ea93e3bd..46269fe909 100755 --- a/frappe/utils/logger.py +++ b/frappe/utils/logger.py @@ -38,26 +38,24 @@ def get_logger(module, with_more_info=False, allow_site=True, filter=None): LOGGER_NAME = "{}-{}".format(module, site or "all") - if LOGGER_NAME in frappe.loggers: - try: - return frappe.loggers[LOGGER_NAME] - except: - pass + try: + return frappe.loggers[LOGGER_NAME] + except KeyError: + pass if not module: module = "frappe" with_more_info = True logfile = module + ".log" - - LOG_FILENAME = os.path.join("..", "logs", logfile) + log_filename = os.path.join("..", "logs", logfile) logger = logging.getLogger(LOGGER_NAME) logger.setLevel(frappe.log_level or default_log_level) logger.propagate = False - formatter = logging.Formatter("%(asctime)s %(levelname)s %(name)s %(message)s") - handler = RotatingFileHandler(LOG_FILENAME, maxBytes=100_000, backupCount=20) + formatter = logging.Formatter("%(asctime)s %(levelname)s {0} %(message)s".format(module)) + handler = RotatingFileHandler(log_filename, maxBytes=100_000, backupCount=20) handler.setFormatter(formatter) logger.addHandler(handler) From faa9a5466eacc4d07e19286309f36aeb8468f7ec Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 30 Jul 2020 19:04:04 +0530 Subject: [PATCH 044/230] fix: Slider --- frappe/utils/logger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/utils/logger.py b/frappe/utils/logger.py index 46269fe909..6d24eef9d3 100755 --- a/frappe/utils/logger.py +++ b/frappe/utils/logger.py @@ -29,7 +29,7 @@ def get_logger(module, with_more_info=False, allow_site=True, filter=None): : Returns a Python logger object with Site and Bench level logging capabilities. """ - if allow_site == True: + if allow_site is True: site = getattr(frappe.local, "site", None) elif allow_site: site = allow_site From 444e8225a46188137028f99cb12efcab195559b5 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Thu, 30 Jul 2020 19:50:36 +0530 Subject: [PATCH 045/230] fix: setup filters only if doctype is available --- frappe/public/js/frappe/widgets/widget_dialog.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/public/js/frappe/widgets/widget_dialog.js b/frappe/public/js/frappe/widgets/widget_dialog.js index e3b4cc1d8c..2ca26d59e1 100644 --- a/frappe/public/js/frappe/widgets/widget_dialog.js +++ b/frappe/public/js/frappe/widgets/widget_dialog.js @@ -174,9 +174,9 @@ class ShortcutDialog extends WidgetDialog { onchange: () => { if (this.dialog.get_value("type") == "DocType") { let doctype = this.dialog.get_value("link_to"); - if (frappe.boot.single_types.includes(doctype)) { + if (doctype && frappe.boot.single_types.includes(doctype)) { this.hide_filters(); - } else { + } else if (doctype) { this.setup_filter(doctype); this.show_filters(); } From 0c16533cd21eb5ee3b95297fcab32cb7e7dd70f6 Mon Sep 17 00:00:00 2001 From: Mangesh-Khairnar Date: Fri, 31 Jul 2020 12:32:17 +0530 Subject: [PATCH 046/230] fix(field-group): consider mandatory separately for text editor --- frappe/public/js/frappe/ui/field_group.js | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/frappe/public/js/frappe/ui/field_group.js b/frappe/public/js/frappe/ui/field_group.js index ec7234c9bb..d432e553f1 100644 --- a/frappe/public/js/frappe/ui/field_group.js +++ b/frappe/public/js/frappe/ui/field_group.js @@ -82,17 +82,22 @@ frappe.ui.FieldGroup = frappe.ui.form.Layout.extend({ get_values: function(ignore_errors) { var ret = {}; var errors = []; - for(var key in this.fields_dict) { + for (var key in this.fields_dict) { var f = this.fields_dict[key]; - if(f.get_value) { + if (f.get_value) { var v = f.get_value(); - if(f.df.reqd && is_null(strip_html(cstr(v)))) + if (f.df.reqd && is_null(v)) errors.push(__(f.df.label)); - if(!is_null(v)) ret[f.df.fieldname] = v; + if (f.df.reqd + && f.df.fieldtype === 'Text Editor' + && is_null(strip_html(cstr(v)))) + errors.push(__(f.df.label)); + + if (!is_null(v)) ret[f.df.fieldname] = v; } } - if(errors.length && !ignore_errors) { + if (errors.length && !ignore_errors) { frappe.msgprint({ title: __('Missing Values Required'), message: __('Following fields have missing values:') + From 94116ae93c416958619e98aa47ad979481ec9dd1 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 31 Jul 2020 16:07:51 +0530 Subject: [PATCH 047/230] feat(logger): allow max_size and file_count params --- frappe/__init__.py | 4 ++-- frappe/utils/logger.py | 20 +++++++++++--------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index 7ea95ec3db..45f40f7783 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -1559,10 +1559,10 @@ def get_doctype_app(doctype): loggers = {} log_level = None -def logger(module=None, with_more_info=False, allow_site=True, filter=None): +def logger(module=None, with_more_info=False, allow_site=True, filter=None, max_size=100_000, file_count=20): '''Returns a python logger that uses StreamHandler''' from frappe.utils.logger import get_logger - return get_logger(module=module, with_more_info=with_more_info, allow_site=allow_site, filter=filter) + return get_logger(module=module, with_more_info=with_more_info, allow_site=allow_site, filter=filter, max_size=max_size, file_count=file_count) def log_error(message=None, title=_("Error")): '''Log error to Error Log''' diff --git a/frappe/utils/logger.py b/frappe/utils/logger.py index 6d24eef9d3..b3295cccdf 100755 --- a/frappe/utils/logger.py +++ b/frappe/utils/logger.py @@ -16,14 +16,16 @@ import frappe default_log_level = logging.DEBUG -def get_logger(module, with_more_info=False, allow_site=True, filter=None): +def get_logger(module, with_more_info=False, allow_site=True, filter=None, max_size=100_000, file_count=20): """Application Logger for your given module Args: module (str): Name of your logger and consequently your log file. with_more_info (bool, optional): Will log the form dict using the SiteContextFilter. Defaults to False. allow_site ((str, bool), optional): Pass site name to explicitly log under it's logs. If True and unspecified, guesses which site the logs would be saved under. Defaults to True. - filter (function, optional): Add a filter function for your logger + filter (function, optional): Add a filter function for your logger. Defaults to None. + max_size (int, optional): Max file size of each log file in kB. Defaults to 100_000. + file_count (int, optional): Max count of log files to be retained via Log Rotation. Defaults to 20. Returns: : Returns a Python logger object with Site and Bench level logging capabilities. @@ -36,10 +38,10 @@ def get_logger(module, with_more_info=False, allow_site=True, filter=None): else: site = False - LOGGER_NAME = "{}-{}".format(module, site or "all") + logger_name = "{0}-{1}".format(module, site or "all") try: - return frappe.loggers[LOGGER_NAME] + return frappe.loggers[logger_name] except KeyError: pass @@ -50,18 +52,18 @@ def get_logger(module, with_more_info=False, allow_site=True, filter=None): logfile = module + ".log" log_filename = os.path.join("..", "logs", logfile) - logger = logging.getLogger(LOGGER_NAME) + logger = logging.getLogger(logger_name) logger.setLevel(frappe.log_level or default_log_level) logger.propagate = False formatter = logging.Formatter("%(asctime)s %(levelname)s {0} %(message)s".format(module)) - handler = RotatingFileHandler(log_filename, maxBytes=100_000, backupCount=20) + handler = RotatingFileHandler(log_filename, maxBytes=max_size, backupCount=file_count) handler.setFormatter(formatter) logger.addHandler(handler) if site: - SITELOG_FILENAME = os.path.join(site, "logs", logfile) - site_handler = RotatingFileHandler(SITELOG_FILENAME, maxBytes=100_000, backupCount=20) + sitelog_filename = os.path.join(site, "logs", logfile) + site_handler = RotatingFileHandler(sitelog_filename, maxBytes=max_size, backupCount=file_count) site_handler.setFormatter(formatter) logger.addHandler(site_handler) @@ -71,7 +73,7 @@ def get_logger(module, with_more_info=False, allow_site=True, filter=None): if filter: logger.addFilter(filter) - frappe.loggers[LOGGER_NAME] = logger + frappe.loggers[logger_name] = logger return logger From f04ce5dd7a7957d46ef12d2d5e2216bb93e753e1 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 31 Jul 2020 16:11:39 +0530 Subject: [PATCH 048/230] chore: Typo --- frappe/utils/logger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/utils/logger.py b/frappe/utils/logger.py index b3295cccdf..451211b74b 100755 --- a/frappe/utils/logger.py +++ b/frappe/utils/logger.py @@ -24,7 +24,7 @@ def get_logger(module, with_more_info=False, allow_site=True, filter=None, max_s with_more_info (bool, optional): Will log the form dict using the SiteContextFilter. Defaults to False. allow_site ((str, bool), optional): Pass site name to explicitly log under it's logs. If True and unspecified, guesses which site the logs would be saved under. Defaults to True. filter (function, optional): Add a filter function for your logger. Defaults to None. - max_size (int, optional): Max file size of each log file in kB. Defaults to 100_000. + max_size (int, optional): Max file size of each log file in bytes. Defaults to 100_000. file_count (int, optional): Max count of log files to be retained via Log Rotation. Defaults to 20. Returns: From a87531e1d0a808f69ce9d12a7e2fc9a9bb962641 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 31 Jul 2020 17:26:37 +0530 Subject: [PATCH 049/230] fix(get_logger): allow None for module --- frappe/utils/logger.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/utils/logger.py b/frappe/utils/logger.py index 451211b74b..398cdd0bfc 100755 --- a/frappe/utils/logger.py +++ b/frappe/utils/logger.py @@ -16,11 +16,11 @@ import frappe default_log_level = logging.DEBUG -def get_logger(module, with_more_info=False, allow_site=True, filter=None, max_size=100_000, file_count=20): +def get_logger(module=None, with_more_info=False, allow_site=True, filter=None, max_size=100_000, file_count=20): """Application Logger for your given module Args: - module (str): Name of your logger and consequently your log file. + module (str, optional): Name of your logger and consequently your log file. Defaults to None. with_more_info (bool, optional): Will log the form dict using the SiteContextFilter. Defaults to False. allow_site ((str, bool), optional): Pass site name to explicitly log under it's logs. If True and unspecified, guesses which site the logs would be saved under. Defaults to True. filter (function, optional): Add a filter function for your logger. Defaults to None. From c423eb7288623fb770216d712e923ee96c22e88f Mon Sep 17 00:00:00 2001 From: prssanna Date: Fri, 31 Jul 2020 18:05:23 +0530 Subject: [PATCH 050/230] fix: text in MultiSelectList doesn't get truncated --- frappe/public/js/frappe/form/controls/multiselect_list.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/form/controls/multiselect_list.js b/frappe/public/js/frappe/form/controls/multiselect_list.js index cd86bdd767..2a7ee5cb10 100644 --- a/frappe/public/js/frappe/form/controls/multiselect_list.js +++ b/frappe/public/js/frappe/form/controls/multiselect_list.js @@ -3,7 +3,7 @@ frappe.ui.form.ControlMultiSelectList = frappe.ui.form.ControlData.extend({ let template = `
            - +
            • From eab96ab904d9f39739111f8c486d0cecf4240638 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Sat, 1 Aug 2020 13:42:33 +0530 Subject: [PATCH 051/230] Revert "fix(image-with-blur): Set width, height in CSS" --- frappe/templates/includes/image_with_blur.html | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/frappe/templates/includes/image_with_blur.html b/frappe/templates/includes/image_with_blur.html index dbddb9d8ef..20b0380b88 100644 --- a/frappe/templates/includes/image_with_blur.html +++ b/frappe/templates/includes/image_with_blur.html @@ -1,9 +1,7 @@ {%- set res = frappe.utils.get_thumbnail_base64_for_image(src) if src else false -%} {%- if res and res['base64'].startswith('data:') -%} {{ alt or '' }} + alt="{{ alt or '' }}" width="{{ res['width'] }}" height="{{ res['height'] }}" data-src="{{ src or '' }}" /> {%- else -%} {{ alt or '' }} {%- endif -%} From 0bc2e6e9c808126feee66b6da7b592fe451ab7ae Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Mon, 3 Aug 2020 10:44:51 +0530 Subject: [PATCH 052/230] chore: Run test in verbose mode --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 63895675ea..3ae844bc3a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,12 +31,12 @@ matrix: - name: "Python 3.7 MariaDB" python: 3.7 env: DB=mariadb TYPE=server - script: bench --site test_site run-tests --coverage + script: bench --site test_site --versbose run-tests --coverage - name: "Python 3.7 PostgreSQL" python: 3.7 env: DB=postgres TYPE=server - script: bench --site test_site run-tests --coverage + script: bench --site test_site --versbose run-tests --coverage - name: "Cypress" python: 3.7 From 817e90d17111b4618f4e3c44e623222e62255f51 Mon Sep 17 00:00:00 2001 From: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> Date: Mon, 3 Aug 2020 10:54:20 +0530 Subject: [PATCH 053/230] fix: Typo --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3ae844bc3a..eb35e0cca2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,12 +31,12 @@ matrix: - name: "Python 3.7 MariaDB" python: 3.7 env: DB=mariadb TYPE=server - script: bench --site test_site --versbose run-tests --coverage + script: bench --site test_site --verbose run-tests --coverage - name: "Python 3.7 PostgreSQL" python: 3.7 env: DB=postgres TYPE=server - script: bench --site test_site --versbose run-tests --coverage + script: bench --site test_site --verbose run-tests --coverage - name: "Cypress" python: 3.7 From f0729ff6a8a58239e68c5c0810becc3ec27b056e Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Mon, 3 Aug 2020 12:00:44 +0530 Subject: [PATCH 054/230] test: Reset workflow doc --- frappe/workflow/doctype/workflow/test_workflow.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/frappe/workflow/doctype/workflow/test_workflow.py b/frappe/workflow/doctype/workflow/test_workflow.py index 2719bc7cf0..b8c40d4d3b 100644 --- a/frappe/workflow/doctype/workflow/test_workflow.py +++ b/frappe/workflow/doctype/workflow/test_workflow.py @@ -78,7 +78,7 @@ class TestWorkflow(unittest.TestCase): frappe.set_user('test2@example.com') doc = self.test_default_condition() - workflow_actions = frappe.get_all('Workflow Action', fields=['status']) + workflow_actions = frappe.get_all('Workflow Action', fields=['status', 'reference_name']) self.assertEqual(len(workflow_actions), 1) # test if status of workflow actions are updated on approval @@ -102,6 +102,9 @@ class TestWorkflow(unittest.TestCase): todo.reload() self.assertEqual(todo.docstatus, 1) + self.workflow.states[1].doc_status = 0 + self.workflow.save() + def test_if_workflow_set_on_action(self): self.workflow.states[1].doc_status = 1 self.workflow.save() @@ -111,6 +114,9 @@ class TestWorkflow(unittest.TestCase): self.assertEqual(todo.docstatus, 1) self.assertEqual(todo.workflow_state, 'Approved') + self.workflow.states[1].doc_status = 0 + self.workflow.save() + def create_todo_workflow(): if frappe.db.exists('Workflow', 'Test ToDo'): return frappe.get_doc('Workflow', 'Test ToDo').save(ignore_permissions=True) From 9445c115ecdf9ffbf5386adaf279935648a96754 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Mon, 3 Aug 2020 15:51:30 +0530 Subject: [PATCH 055/230] fix: check if is_standard is "Yes" (#11180) --- frappe/core/doctype/report/report.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/core/doctype/report/report.js b/frappe/core/doctype/report/report.js index b4c30f5bbd..c410e9aa1a 100644 --- a/frappe/core/doctype/report/report.js +++ b/frappe/core/doctype/report/report.js @@ -1,6 +1,6 @@ frappe.ui.form.on('Report', { refresh: function(frm) { - if (frm.doc.is_standard && !frappe.boot.developer_mode) { + if (frm.doc.is_standard === "Yes" && !frappe.boot.developer_mode) { // make the document read-only frm.set_read_only(); } From 4b93f75eba40625bba5b83cca2470aa179e3f082 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 3 Aug 2020 17:23:32 +0530 Subject: [PATCH 056/230] fix: Log site only if exists --- frappe/app.py | 5 ++--- frappe/utils/logger.py | 3 ++- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/frappe/app.py b/frappe/app.py index 100b10e9ef..450e3b5f17 100644 --- a/frappe/app.py +++ b/frappe/app.py @@ -99,9 +99,8 @@ def application(request): frappe.monitor.stop(response) frappe.recorder.dump() - _site = get_site_name(request.host) - frappe.logger("frappe.web", allow_site=_site).info({ - "site": _site, + frappe.logger("frappe.web", allow_site=frappe.local.site).info({ + "site": get_site_name(request.host), "remote_addr": getattr(request, "remote_addr", "NOTFOUND"), "base_url": getattr(request, "base_url", "NOTFOUND"), "full_path": getattr(request, "full_path", "NOTFOUND"), diff --git a/frappe/utils/logger.py b/frappe/utils/logger.py index 398cdd0bfc..364ffa776d 100755 --- a/frappe/utils/logger.py +++ b/frappe/utils/logger.py @@ -11,6 +11,7 @@ from six import text_type # imports - module imports import frappe +from frappe.utils import get_sites default_log_level = logging.DEBUG @@ -33,7 +34,7 @@ def get_logger(module=None, with_more_info=False, allow_site=True, filter=None, if allow_site is True: site = getattr(frappe.local, "site", None) - elif allow_site: + elif allow_site in get_sites(): site = allow_site else: site = False From 00a8edf910b4c328aeb69f674db31439e206fd54 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Mon, 3 Aug 2020 23:53:01 +0530 Subject: [PATCH 057/230] test: Make User records before executing test --- frappe/workflow/doctype/workflow/test_workflow.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/frappe/workflow/doctype/workflow/test_workflow.py b/frappe/workflow/doctype/workflow/test_workflow.py index b8c40d4d3b..fc12832e57 100644 --- a/frappe/workflow/doctype/workflow/test_workflow.py +++ b/frappe/workflow/doctype/workflow/test_workflow.py @@ -5,7 +5,10 @@ from __future__ import unicode_literals import frappe import unittest from frappe.utils import random_string -from frappe.model.workflow import apply_workflow, WorkflowTransitionError, WorkflowPermissionError, get_common_transition_actions +from frappe.model.workflow import apply_workflow, WorkflowTransitionError, WorkflowPermissionError, get_common_transition_actions, get_workflow +from frappe.test_runner import make_test_records + +make_test_records("User") class TestWorkflow(unittest.TestCase): def setUp(self): From e555970485467b69cddcbc7fce990f94c722f1cc Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Mon, 3 Aug 2020 23:53:44 +0530 Subject: [PATCH 058/230] chore: Comment out possibly problematic test --- .../event_producer/test_event_producer.py | 80 +++++++++---------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/frappe/event_streaming/doctype/event_producer/test_event_producer.py b/frappe/event_streaming/doctype/event_producer/test_event_producer.py index 0d414f5d63..cda4ca8bf6 100644 --- a/frappe/event_streaming/doctype/event_producer/test_event_producer.py +++ b/frappe/event_streaming/doctype/event_producer/test_event_producer.py @@ -203,49 +203,49 @@ class TestEventProducer(unittest.TestCase): reset_configuration(producer_url) - def test_inner_mapping(self): - producer = get_remote_site() - event_producer = frappe.get_doc('Event Producer', producer_url) - event_producer.producer_doctypes = [] - inner_mapping = [ - { - 'local_fieldname':'role_name', - 'remote_fieldname':'title' - } - ] - inner_map = get_mapping('Role to Note Dependency Creation', 'Role', 'Note', inner_mapping) - mapping = [ - { - 'local_fieldname':'description', - 'remote_fieldname':'content', - }, - { - 'local_fieldname': 'role', - 'remote_fieldname': 'title', - 'mapping_type': 'Document', - 'mapping': inner_map, - 'remote_value_filters': json.dumps({'title': 'title'}) - } - ] - event_producer.append('producer_doctypes', { - 'ref_doctype': 'ToDo', - 'use_same_name': 1, - 'has_mapping': 1, - 'mapping': get_mapping('ToDo to Note Mapping', 'ToDo', 'Note', mapping) - }) - event_producer.save() + # def test_inner_mapping(self): + # producer = get_remote_site() + # event_producer = frappe.get_doc('Event Producer', producer_url) + # event_producer.producer_doctypes = [] + # inner_mapping = [ + # { + # 'local_fieldname':'role_name', + # 'remote_fieldname':'title' + # } + # ] + # inner_map = get_mapping('Role to Note Dependency Creation', 'Role', 'Note', inner_mapping) + # mapping = [ + # { + # 'local_fieldname':'description', + # 'remote_fieldname':'content', + # }, + # { + # 'local_fieldname': 'role', + # 'remote_fieldname': 'title', + # 'mapping_type': 'Document', + # 'mapping': inner_map, + # 'remote_value_filters': json.dumps({'title': 'title'}) + # } + # ] + # event_producer.append('producer_doctypes', { + # 'ref_doctype': 'ToDo', + # 'use_same_name': 1, + # 'has_mapping': 1, + # 'mapping': get_mapping('ToDo to Note Mapping', 'ToDo', 'Note', mapping) + # }) + # event_producer.save() - producer_note = frappe._dict(doctype='Note', title='Inner Mapping Tester', content='Test Inner Mapping') - delete_on_remote_if_exists(producer, 'Note', {'title': producer_note.title}) - producer_note = producer.insert(producer_note) - self.pull_producer_data() + # producer_note = frappe._dict(doctype='Note', title='Inner Mapping Tester', content='Test Inner Mapping') + # delete_on_remote_if_exists(producer, 'Note', {'title': producer_note.title}) + # producer_note = producer.insert(producer_note) + # self.pull_producer_data() - #check dependency inserted - self.assertTrue(frappe.db.exists('Role', {'role_name': producer_note.title})) - #check doc inserted - self.assertTrue(frappe.db.exists('ToDo', {'description': producer_note.content})) + # #check dependency inserted + # self.assertTrue(frappe.db.exists('Role', {'role_name': producer_note.title})) + # #check doc inserted + # self.assertTrue(frappe.db.exists('ToDo', {'description': producer_note.content})) - reset_configuration(producer_url) + # reset_configuration(producer_url) def insert_into_producer(producer, description): From 88b68c6cf51061b4b1a37b93b5426279fc1dfe9d Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Tue, 4 Aug 2020 00:11:48 +0530 Subject: [PATCH 059/230] chore: Remove unwanted changes --- .travis.yml | 4 ++-- frappe/workflow/doctype/workflow/test_workflow.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index eb35e0cca2..63895675ea 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,12 +31,12 @@ matrix: - name: "Python 3.7 MariaDB" python: 3.7 env: DB=mariadb TYPE=server - script: bench --site test_site --verbose run-tests --coverage + script: bench --site test_site run-tests --coverage - name: "Python 3.7 PostgreSQL" python: 3.7 env: DB=postgres TYPE=server - script: bench --site test_site --verbose run-tests --coverage + script: bench --site test_site run-tests --coverage - name: "Cypress" python: 3.7 diff --git a/frappe/workflow/doctype/workflow/test_workflow.py b/frappe/workflow/doctype/workflow/test_workflow.py index fc12832e57..253f95dab7 100644 --- a/frappe/workflow/doctype/workflow/test_workflow.py +++ b/frappe/workflow/doctype/workflow/test_workflow.py @@ -5,7 +5,7 @@ from __future__ import unicode_literals import frappe import unittest from frappe.utils import random_string -from frappe.model.workflow import apply_workflow, WorkflowTransitionError, WorkflowPermissionError, get_common_transition_actions, get_workflow +from frappe.model.workflow import apply_workflow, WorkflowTransitionError, WorkflowPermissionError, get_common_transition_actions from frappe.test_runner import make_test_records make_test_records("User") From 89dfd82134d02d1a88444d02f858be820371129b Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Tue, 4 Aug 2020 00:36:42 +0530 Subject: [PATCH 060/230] chore: Add debug statement - Uncomment innocent test --- .../event_producer/test_event_producer.py | 80 +++++++++---------- .../doctype/workflow/test_workflow.py | 3 + 2 files changed, 43 insertions(+), 40 deletions(-) diff --git a/frappe/event_streaming/doctype/event_producer/test_event_producer.py b/frappe/event_streaming/doctype/event_producer/test_event_producer.py index cda4ca8bf6..0d414f5d63 100644 --- a/frappe/event_streaming/doctype/event_producer/test_event_producer.py +++ b/frappe/event_streaming/doctype/event_producer/test_event_producer.py @@ -203,49 +203,49 @@ class TestEventProducer(unittest.TestCase): reset_configuration(producer_url) - # def test_inner_mapping(self): - # producer = get_remote_site() - # event_producer = frappe.get_doc('Event Producer', producer_url) - # event_producer.producer_doctypes = [] - # inner_mapping = [ - # { - # 'local_fieldname':'role_name', - # 'remote_fieldname':'title' - # } - # ] - # inner_map = get_mapping('Role to Note Dependency Creation', 'Role', 'Note', inner_mapping) - # mapping = [ - # { - # 'local_fieldname':'description', - # 'remote_fieldname':'content', - # }, - # { - # 'local_fieldname': 'role', - # 'remote_fieldname': 'title', - # 'mapping_type': 'Document', - # 'mapping': inner_map, - # 'remote_value_filters': json.dumps({'title': 'title'}) - # } - # ] - # event_producer.append('producer_doctypes', { - # 'ref_doctype': 'ToDo', - # 'use_same_name': 1, - # 'has_mapping': 1, - # 'mapping': get_mapping('ToDo to Note Mapping', 'ToDo', 'Note', mapping) - # }) - # event_producer.save() + def test_inner_mapping(self): + producer = get_remote_site() + event_producer = frappe.get_doc('Event Producer', producer_url) + event_producer.producer_doctypes = [] + inner_mapping = [ + { + 'local_fieldname':'role_name', + 'remote_fieldname':'title' + } + ] + inner_map = get_mapping('Role to Note Dependency Creation', 'Role', 'Note', inner_mapping) + mapping = [ + { + 'local_fieldname':'description', + 'remote_fieldname':'content', + }, + { + 'local_fieldname': 'role', + 'remote_fieldname': 'title', + 'mapping_type': 'Document', + 'mapping': inner_map, + 'remote_value_filters': json.dumps({'title': 'title'}) + } + ] + event_producer.append('producer_doctypes', { + 'ref_doctype': 'ToDo', + 'use_same_name': 1, + 'has_mapping': 1, + 'mapping': get_mapping('ToDo to Note Mapping', 'ToDo', 'Note', mapping) + }) + event_producer.save() - # producer_note = frappe._dict(doctype='Note', title='Inner Mapping Tester', content='Test Inner Mapping') - # delete_on_remote_if_exists(producer, 'Note', {'title': producer_note.title}) - # producer_note = producer.insert(producer_note) - # self.pull_producer_data() + producer_note = frappe._dict(doctype='Note', title='Inner Mapping Tester', content='Test Inner Mapping') + delete_on_remote_if_exists(producer, 'Note', {'title': producer_note.title}) + producer_note = producer.insert(producer_note) + self.pull_producer_data() - # #check dependency inserted - # self.assertTrue(frappe.db.exists('Role', {'role_name': producer_note.title})) - # #check doc inserted - # self.assertTrue(frappe.db.exists('ToDo', {'description': producer_note.content})) + #check dependency inserted + self.assertTrue(frappe.db.exists('Role', {'role_name': producer_note.title})) + #check doc inserted + self.assertTrue(frappe.db.exists('ToDo', {'description': producer_note.content})) - # reset_configuration(producer_url) + reset_configuration(producer_url) def insert_into_producer(producer, description): diff --git a/frappe/workflow/doctype/workflow/test_workflow.py b/frappe/workflow/doctype/workflow/test_workflow.py index 253f95dab7..f94193fab9 100644 --- a/frappe/workflow/doctype/workflow/test_workflow.py +++ b/frappe/workflow/doctype/workflow/test_workflow.py @@ -30,6 +30,9 @@ class TestWorkflow(unittest.TestCase): def test_approve(self, doc=None): '''test simple workflow''' todo = doc or self.test_default_condition() + print(frappe.db.sql("select name from `tabRole`")) + print('------------') + print(frappe.get_roles(), frappe.session.user) apply_workflow(todo, 'Approve') # default condition is set From 6e9169a0aadc6b05af83432293297220e633631d Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Tue, 4 Aug 2020 10:12:41 +0530 Subject: [PATCH 061/230] test: Remove debug statements & clear-cache --- frappe/workflow/doctype/workflow/test_workflow.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/frappe/workflow/doctype/workflow/test_workflow.py b/frappe/workflow/doctype/workflow/test_workflow.py index f94193fab9..fb42e6fd6e 100644 --- a/frappe/workflow/doctype/workflow/test_workflow.py +++ b/frappe/workflow/doctype/workflow/test_workflow.py @@ -30,9 +30,6 @@ class TestWorkflow(unittest.TestCase): def test_approve(self, doc=None): '''test simple workflow''' todo = doc or self.test_default_condition() - print(frappe.db.sql("select name from `tabRole`")) - print('------------') - print(frappe.get_roles(), frappe.session.user) apply_workflow(todo, 'Approve') # default condition is set @@ -129,6 +126,7 @@ def create_todo_workflow(): else: frappe.get_doc(dict(doctype='Role', role_name='Test Approver')).insert(ignore_if_duplicate=True) + frappe.cache().hdel('roles', 'Administrator') workflow = frappe.new_doc('Workflow') workflow.workflow_name = 'Test ToDo' workflow.document_type = 'ToDo' From 744a9413fdc7ffee3ac26883add9a950c3f0ab19 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Tue, 4 Aug 2020 10:36:16 +0530 Subject: [PATCH 062/230] fix: Use common function to get_roles --- frappe/__init__.py | 6 +----- frappe/workflow/doctype/workflow/test_workflow.py | 1 - 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index d644d2a473..f91317584d 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -437,11 +437,7 @@ def get_roles(username=None): if not local.session: return ["Guest"] - if username: - import frappe.permissions - return frappe.permissions.get_roles(username) - else: - return get_user().get_roles() + return frappe.permissions.get_roles(username or local.session.user) def get_request_header(key, default=None): """Return HTTP request header. diff --git a/frappe/workflow/doctype/workflow/test_workflow.py b/frappe/workflow/doctype/workflow/test_workflow.py index fb42e6fd6e..253f95dab7 100644 --- a/frappe/workflow/doctype/workflow/test_workflow.py +++ b/frappe/workflow/doctype/workflow/test_workflow.py @@ -126,7 +126,6 @@ def create_todo_workflow(): else: frappe.get_doc(dict(doctype='Role', role_name='Test Approver')).insert(ignore_if_duplicate=True) - frappe.cache().hdel('roles', 'Administrator') workflow = frappe.new_doc('Workflow') workflow.workflow_name = 'Test ToDo' workflow.document_type = 'ToDo' From 3c3b1037eaf9b7e92a52dd7c391d18f969d06102 Mon Sep 17 00:00:00 2001 From: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> Date: Tue, 4 Aug 2020 10:44:45 +0530 Subject: [PATCH 063/230] fix: Import frappe.permissions --- frappe/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index f91317584d..31ce00e440 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -436,7 +436,7 @@ def get_roles(username=None): """Returns roles of current user.""" if not local.session: return ["Guest"] - + import frappe.permissions return frappe.permissions.get_roles(username or local.session.user) def get_request_header(key, default=None): From 86a9ceb29b93a0df1db233bb461adbcd0b005739 Mon Sep 17 00:00:00 2001 From: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> Date: Tue, 4 Aug 2020 11:16:18 +0530 Subject: [PATCH 064/230] test: Clear role cache --- frappe/workflow/doctype/workflow/test_workflow.py | 1 + 1 file changed, 1 insertion(+) diff --git a/frappe/workflow/doctype/workflow/test_workflow.py b/frappe/workflow/doctype/workflow/test_workflow.py index 253f95dab7..d7f7939d3d 100644 --- a/frappe/workflow/doctype/workflow/test_workflow.py +++ b/frappe/workflow/doctype/workflow/test_workflow.py @@ -126,6 +126,7 @@ def create_todo_workflow(): else: frappe.get_doc(dict(doctype='Role', role_name='Test Approver')).insert(ignore_if_duplicate=True) + frappe.cache().hdel('roles', frappe.session.user) workflow = frappe.new_doc('Workflow') workflow.workflow_name = 'Test ToDo' workflow.document_type = 'ToDo' From 645331799af44feb4629ccc50f214278579be200 Mon Sep 17 00:00:00 2001 From: Prssanna Desai Date: Tue, 4 Aug 2020 12:17:33 +0530 Subject: [PATCH 065/230] fix: whitelist functions called through search widget (#11153) --- frappe/core/doctype/user_permission/user_permission.py | 1 + .../permitted_documents_for_user.py | 1 + frappe/desk/doctype/dashboard_chart/dashboard_chart.py | 1 + frappe/desk/doctype/number_card/number_card.py | 4 ++++ frappe/email/__init__.py | 1 + 5 files changed, 8 insertions(+) diff --git a/frappe/core/doctype/user_permission/user_permission.py b/frappe/core/doctype/user_permission/user_permission.py index 48d4fcb5d4..325c41622d 100644 --- a/frappe/core/doctype/user_permission/user_permission.py +++ b/frappe/core/doctype/user_permission/user_permission.py @@ -119,6 +119,7 @@ def user_permission_exists(user, allow, for_value, applicable_for=None): return has_same_user_permission +@frappe.whitelist() def get_applicable_for_doctype_list(doctype, txt, searchfield, start, page_len, filters): linked_doctypes_map = get_linked_doctypes(doctype, True) diff --git a/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.py b/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.py index 8b2d1e01fa..5ca21b811e 100644 --- a/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.py +++ b/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.py @@ -41,6 +41,7 @@ def get_columns_and_fields(doctype): return columns, fields +@frappe.whitelist() def query_doctypes(doctype, txt, searchfield, start, page_len, filters): user = filters.get("user") user_perms = frappe.utils.user.UserPermissions(user) diff --git a/frappe/desk/doctype/dashboard_chart/dashboard_chart.py b/frappe/desk/doctype/dashboard_chart/dashboard_chart.py index a25e0cab9e..7547efa234 100644 --- a/frappe/desk/doctype/dashboard_chart/dashboard_chart.py +++ b/frappe/desk/doctype/dashboard_chart/dashboard_chart.py @@ -344,6 +344,7 @@ def get_year_ending(date): # last day of this month return add_to_date(date, days=-1) +@frappe.whitelist() def get_charts_for_user(doctype, txt, searchfield, start, page_len, filters): or_filters = {'owner': frappe.session.user, 'is_public': 1} return frappe.db.get_list('Dashboard Chart', diff --git a/frappe/desk/doctype/number_card/number_card.py b/frappe/desk/doctype/number_card/number_card.py index 3eb08ec00a..f44893d950 100644 --- a/frappe/desk/doctype/number_card/number_card.py +++ b/frappe/desk/doctype/number_card/number_card.py @@ -124,11 +124,15 @@ def create_number_card(args): doc.insert(ignore_permissions=True) return doc +@frappe.whitelist() def get_cards_for_user(doctype, txt, searchfield, start, page_len, filters): meta = frappe.get_meta(doctype) searchfields = meta.get_search_fields() search_conditions = [] + if not frappe.db.exists('DocType', doctype): + return + if txt: for field in searchfields: search_conditions.append('`tab{doctype}`.`{field}` like %(txt)s'.format(field=field, doctype=doctype, txt=txt)) diff --git a/frappe/email/__init__.py b/frappe/email/__init__.py index d58b35040e..0b874a03bb 100644 --- a/frappe/email/__init__.py +++ b/frappe/email/__init__.py @@ -57,6 +57,7 @@ def relink(name, reference_doctype=None, reference_name=None): communication_type = "Communication" and name = %s""", (reference_doctype, reference_name, name)) +@frappe.whitelist() def get_communication_doctype(doctype, txt, searchfield, start, page_len, filters): user_perms = frappe.utils.user.UserPermissions(frappe.session.user) user_perms.build_permissions() From 9b267027b5ce2c0674a4dd1cb476d5b80b873632 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Tue, 4 Aug 2020 13:25:49 +0530 Subject: [PATCH 066/230] feat: don't load user translation in make_dict_from_messages (#11120) --- frappe/translate.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/frappe/translate.py b/frappe/translate.py index 2fc2d3f328..ee1f8fa5ff 100644 --- a/frappe/translate.py +++ b/frappe/translate.py @@ -119,7 +119,7 @@ def get_dict(fortype, name=None): messages += frappe.db.sql("select 'Role:', name from tabRole") messages += frappe.db.sql("select 'Module:', name from `tabModule Def`") - message_dict = make_dict_from_messages(messages) + message_dict = make_dict_from_messages(messages, load_user_translation=False) message_dict.update(get_dict_from_hooks(fortype, name)) # remove untranslated message_dict = {k:v for k, v in iteritems(message_dict) if k!=v} @@ -127,6 +127,7 @@ def get_dict(fortype, name=None): cache.hset("translation_assets", frappe.local.lang, translation_assets, shared=True) translation_map = translation_assets[asset_key] + if fortype == "boot": translation_map.update(get_user_translations(frappe.local.lang)) @@ -144,14 +145,17 @@ def get_dict_from_hooks(fortype, name): return translated_dict -def make_dict_from_messages(messages, full_dict=None): +def make_dict_from_messages(messages, full_dict=None, load_user_translation=True): """Returns translated messages as a dict in Language specified in `frappe.local.lang` :param messages: List of untranslated messages """ out = {} if full_dict==None: - full_dict = get_full_dict(frappe.local.lang) + if load_user_translation: + full_dict = get_full_dict(frappe.local.lang) + else: + full_dict = load_lang(frappe.local.lang) for m in messages: if m[1] in full_dict: @@ -189,11 +193,9 @@ def get_full_dict(lang): try: # get user specific transaltion data user_translations = get_user_translations(lang) - except Exception: - user_translations = None - - if user_translations: frappe.local.lang_full_dict.update(user_translations) + except Exception: + pass return frappe.local.lang_full_dict From d5622f541021e9ecfa682bf30977baeb3e48f89b Mon Sep 17 00:00:00 2001 From: Prssanna Desai Date: Tue, 4 Aug 2020 17:38:48 +0530 Subject: [PATCH 067/230] fix: Permission query in Dashboard Chart (#10981) Co-authored-by: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> --- .../dashboard_chart/dashboard_chart.py | 25 ++++++++++++++----- .../desk/doctype/number_card/number_card.py | 14 +++++++---- 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/frappe/desk/doctype/dashboard_chart/dashboard_chart.py b/frappe/desk/doctype/dashboard_chart/dashboard_chart.py index 7547efa234..88c9623945 100644 --- a/frappe/desk/doctype/dashboard_chart/dashboard_chart.py +++ b/frappe/desk/doctype/dashboard_chart/dashboard_chart.py @@ -28,15 +28,28 @@ def get_permission_query_conditions(user): if "System Manager" in roles: return None - allowed_doctypes = ['"%s"' % doctype for doctype in frappe.permissions.get_doctypes_with_read()] - allowed_reports = ['"%s"' % key if type(key) == str else key.encode('UTF8') for key in get_allowed_reports()] + doctype_condition = False + report_condition = False + + allowed_doctypes = [frappe.db.escape(doctype) for doctype in frappe.permissions.get_doctypes_with_read()] + allowed_reports = [frappe.db.escape(key) if type(key) == str else key.encode('UTF8') for key in get_allowed_reports()] + + if allowed_doctypes: + doctype_condition = '`tabDashboard Chart`.`document_type` in ({allowed_doctypes})'.format( + allowed_doctypes=','.join(allowed_doctypes)) + if allowed_reports: + report_condition = '`tabDashboard Chart`.`report_name` in ({allowed_reports})'.format( + allowed_reports=','.join(allowed_reports)) return ''' - `tabDashboard Chart`.`document_type` in ({allowed_doctypes}) - or `tabDashboard Chart`.`report_name` in ({allowed_reports}) + (`tabDashboard Chart`.`chart_type` in ('Count', 'Sum', 'Average') + and {doctype_condition}) + or + (`tabDashboard Chart`.`chart_type` = 'Report' + and {report_condition}) '''.format( - allowed_doctypes=','.join(allowed_doctypes), - allowed_reports=','.join(allowed_reports) + doctype_condition=doctype_condition, + report_condition=report_condition ) diff --git a/frappe/desk/doctype/number_card/number_card.py b/frappe/desk/doctype/number_card/number_card.py index f44893d950..68ed79e64b 100644 --- a/frappe/desk/doctype/number_card/number_card.py +++ b/frappe/desk/doctype/number_card/number_card.py @@ -32,13 +32,17 @@ def get_permission_query_conditions(user=None): if "System Manager" in roles: return None - allowed_doctypes = ['"%s"' % doctype for doctype in frappe.permissions.get_doctypes_with_read()] + doctype_condition = False + + allowed_doctypes = [frappe.db.escape(doctype) for doctype in frappe.permissions.get_doctypes_with_read()] + + if allowed_doctypes: + doctype_condition = '`tabNumber Card`.`document_type` in ({allowed_doctypes})'.format( + allowed_doctypes=','.join(allowed_doctypes)) return ''' - `tabNumber Card`.`document_type` in ({allowed_doctypes}) - '''.format( - allowed_doctypes=','.join(allowed_doctypes) - ) + {doctype_condition} + '''.format(doctype_condition=doctype_condition) def has_permission(doc, ptype, user): roles = frappe.get_roles(user) From dd157ba82255727e0095925eaf8d64fbfe89b275 Mon Sep 17 00:00:00 2001 From: gavin Date: Tue, 4 Aug 2020 17:39:35 +0530 Subject: [PATCH 068/230] fix: dont show git errors in terminal (#11064) Co-authored-by: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> --- frappe/utils/change_log.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/frappe/utils/change_log.py b/frappe/utils/change_log.py index 53c83cfe99..c75b3289db 100644 --- a/frappe/utils/change_log.py +++ b/frappe/utils/change_log.py @@ -118,8 +118,9 @@ def get_versions(): def get_app_branch(app): '''Returns branch of an app''' try: + null_stream = open(os.devnull, 'wb') result = subprocess.check_output('cd ../apps/{0} && git rev-parse --abbrev-ref HEAD'.format(app), - shell=True) + shell=True, stdin=null_stream, stderr=null_stream) result = safe_decode(result) result = result.strip() return result @@ -128,8 +129,9 @@ def get_app_branch(app): def get_app_last_commit_ref(app): try: + null_stream = open(os.devnull, 'wb') result = subprocess.check_output('cd ../apps/{0} && git rev-parse HEAD --short 7'.format(app), - shell=True) + shell=True, stdin=null_stream, stderr=null_stream) result = safe_decode(result) result = result.strip() return result From 34fdf63b9097d5ad0cfb366121242c0f8480e460 Mon Sep 17 00:00:00 2001 From: Glen Whitney Date: Tue, 4 Aug 2020 05:10:06 -0700 Subject: [PATCH 069/230] fix(Link Card): refresh incomplete_dependencies as needed (#11169) When _prepare_item in Workspace.get_cards notices that the dependencies for an item have been completed, it should clear the incomplete_dependencies field in case a value has been saved in that attribute. Fixes #11168. --- frappe/desk/desktop.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frappe/desk/desktop.py b/frappe/desk/desktop.py index 68b57a93d4..ae9d070976 100644 --- a/frappe/desk/desktop.py +++ b/frappe/desk/desktop.py @@ -221,6 +221,8 @@ class Workspace: incomplete_dependencies = [d for d in item.dependencies if not _doctype_contains_a_record(d)] if len(incomplete_dependencies): item.incomplete_dependencies = incomplete_dependencies + else: + item.incomplete_dependencies = "" if item.onboard: # Mark Spotlights for initial From f5c61d04da27d8258caad8631c5f7f4edf3ffa85 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Tue, 4 Aug 2020 17:40:18 +0530 Subject: [PATCH 070/230] fix: Patch to generate theme files in public folder (#11148) Co-authored-by: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> --- frappe/patches.txt | 1 + .../generate_theme_files_in_public_folder.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 frappe/patches/v13_0/generate_theme_files_in_public_folder.py diff --git a/frappe/patches.txt b/frappe/patches.txt index b207a325cb..75750ab59c 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -296,3 +296,4 @@ frappe.patches.v13_0.update_duration_options frappe.patches.v13_0.replace_old_data_import # 2020-06-24 frappe.patches.v13_0.create_custom_dashboards_cards_and_charts frappe.patches.v13_0.rename_is_custom_field_in_dashboard_chart +frappe.patches.v13_0.generate_theme_files_in_public_folder diff --git a/frappe/patches/v13_0/generate_theme_files_in_public_folder.py b/frappe/patches/v13_0/generate_theme_files_in_public_folder.py new file mode 100644 index 0000000000..c5a64780cd --- /dev/null +++ b/frappe/patches/v13_0/generate_theme_files_in_public_folder.py @@ -0,0 +1,15 @@ +# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors +# MIT License. See license.txt + +from __future__ import unicode_literals +import frappe + + +def execute(): + themes = frappe.db.get_all( + "Website Theme", filters={"theme_url": ("not like", "/files/website_theme/%")} + ) + for theme in themes: + doc = frappe.get_doc("Website Theme", theme.name) + doc.generate_bootstrap_theme() + doc.save() From 0748312e3950516bb08c76f200359757cd2c6759 Mon Sep 17 00:00:00 2001 From: Aditya Hase Date: Tue, 4 Aug 2020 17:41:34 +0530 Subject: [PATCH 071/230] fix(werkzeug): Avoid duplicate lines from werkzeug in bench start logs (#11171) Co-authored-by: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> --- frappe/app.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frappe/app.py b/frappe/app.py index 450e3b5f17..725bec183a 100644 --- a/frappe/app.py +++ b/frappe/app.py @@ -256,9 +256,11 @@ def serve(port=8000, profile=False, no_reload=False, no_threading=False, site=No 'SERVER_NAME': 'localhost:8000' } + log = logging.getLogger('werkzeug') + log.propagate = False + in_test_env = os.environ.get('CI') if in_test_env: - log = logging.getLogger('werkzeug') log.setLevel(logging.ERROR) run_simple('0.0.0.0', int(port), application, From cc22350b04f73b6321e1fac6ccf62f57ee9776fb Mon Sep 17 00:00:00 2001 From: Rucha Mahabal Date: Tue, 4 Aug 2020 17:42:47 +0530 Subject: [PATCH 072/230] fix: remove Video DocType from Core module (#11151) Co-authored-by: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> --- frappe/automation/desk_page/tools/tools.json | 5 +- frappe/core/doctype/video/__init__.py | 0 frappe/core/doctype/video/test_video.py | 10 -- frappe/core/doctype/video/video.js | 8 -- frappe/core/doctype/video/video.json | 106 ------------------- frappe/core/doctype/video/video.py | 10 -- frappe/public/js/frappe/views/breadcrumbs.js | 1 - 7 files changed, 3 insertions(+), 137 deletions(-) delete mode 100644 frappe/core/doctype/video/__init__.py delete mode 100644 frappe/core/doctype/video/test_video.py delete mode 100644 frappe/core/doctype/video/video.js delete mode 100644 frappe/core/doctype/video/video.json delete mode 100644 frappe/core/doctype/video/video.py diff --git a/frappe/automation/desk_page/tools/tools.json b/frappe/automation/desk_page/tools/tools.json index 2164a4ce38..3fbaf62d02 100644 --- a/frappe/automation/desk_page/tools/tools.json +++ b/frappe/automation/desk_page/tools/tools.json @@ -3,7 +3,7 @@ { "hidden": 0, "label": "Tools", - "links": "[\n {\n \"description\": \"Documents assigned to you and by you.\",\n \"label\": \"To Do\",\n \"name\": \"ToDo\",\n \"onboard\": 1,\n \"type\": \"doctype\"\n },\n {\n \"description\": \"Event and other calendars.\",\n \"label\": \"Calendar\",\n \"link\": \"List/Event/Calendar\",\n \"name\": \"Event\",\n \"onboard\": 1,\n \"type\": \"doctype\"\n },\n {\n \"description\": \"Private and public Notes.\",\n \"label\": \"Note\",\n \"name\": \"Note\",\n \"onboard\": 1,\n \"type\": \"doctype\"\n },\n {\n \"label\": \"Files\",\n \"name\": \"File\",\n \"type\": \"doctype\"\n },\n {\n \"label\": \"Video\",\n \"name\": \"Video\",\n \"type\": \"doctype\"\n },\n {\n \"description\": \"Activity log of all users.\",\n \"label\": \"Activity\",\n \"name\": \"activity\",\n \"type\": \"page\"\n }\n]" + "links": "[\n {\n \"description\": \"Documents assigned to you and by you.\",\n \"label\": \"To Do\",\n \"name\": \"ToDo\",\n \"onboard\": 1,\n \"type\": \"doctype\"\n },\n {\n \"description\": \"Event and other calendars.\",\n \"label\": \"Calendar\",\n \"link\": \"List/Event/Calendar\",\n \"name\": \"Event\",\n \"onboard\": 1,\n \"type\": \"doctype\"\n },\n {\n \"description\": \"Private and public Notes.\",\n \"label\": \"Note\",\n \"name\": \"Note\",\n \"onboard\": 1,\n \"type\": \"doctype\"\n },\n {\n \"label\": \"Files\",\n \"name\": \"File\",\n \"type\": \"doctype\"\n },\n {\n \"description\": \"Activity log of all users.\",\n \"label\": \"Activity\",\n \"name\": \"activity\",\n \"type\": \"page\"\n }\n]" }, { "hidden": 0, @@ -29,10 +29,11 @@ "docstatus": 0, "doctype": "Desk Page", "extends_another_page": 0, + "hide_custom": 0, "idx": 0, "is_standard": 1, "label": "Tools", - "modified": "2020-04-20 18:21:14.152537", + "modified": "2020-07-21 19:32:18.480700", "modified_by": "Administrator", "module": "Automation", "name": "Tools", diff --git a/frappe/core/doctype/video/__init__.py b/frappe/core/doctype/video/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/frappe/core/doctype/video/test_video.py b/frappe/core/doctype/video/test_video.py deleted file mode 100644 index 0bed1e98d6..0000000000 --- a/frappe/core/doctype/video/test_video.py +++ /dev/null @@ -1,10 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2020, Frappe Technologies and Contributors -# See license.txt -from __future__ import unicode_literals - -# import frappe -import unittest - -class TestVideo(unittest.TestCase): - pass diff --git a/frappe/core/doctype/video/video.js b/frappe/core/doctype/video/video.js deleted file mode 100644 index 36ea240a36..0000000000 --- a/frappe/core/doctype/video/video.js +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright (c) 2020, Frappe Technologies and contributors -// For license information, please see license.txt - -frappe.ui.form.on('Video', { - // refresh: function(frm) { - - // } -}); diff --git a/frappe/core/doctype/video/video.json b/frappe/core/doctype/video/video.json deleted file mode 100644 index 26a407c05c..0000000000 --- a/frappe/core/doctype/video/video.json +++ /dev/null @@ -1,106 +0,0 @@ -{ - "actions": [], - "allow_import": 1, - "allow_rename": 1, - "autoname": "field:title", - "creation": "2018-10-17 05:47:13.087395", - "doctype": "DocType", - "editable_grid": 1, - "engine": "InnoDB", - "field_order": [ - "title", - "provider", - "url", - "column_break_4", - "publish_date", - "duration", - "section_break_7", - "description" - ], - "fields": [ - { - "fieldname": "title", - "fieldtype": "Data", - "in_list_view": 1, - "label": "Title", - "reqd": 1, - "unique": 1 - }, - { - "fieldname": "provider", - "fieldtype": "Select", - "in_list_view": 1, - "label": "Provider", - "options": "YouTube\nVimeo", - "reqd": 1 - }, - { - "fieldname": "url", - "fieldtype": "Data", - "in_list_view": 1, - "label": "URL", - "reqd": 1 - }, - { - "fieldname": "column_break_4", - "fieldtype": "Column Break" - }, - { - "fieldname": "publish_date", - "fieldtype": "Date", - "label": "Publish Date" - }, - { - "fieldname": "duration", - "fieldtype": "Data", - "label": "Duration" - }, - { - "fieldname": "section_break_7", - "fieldtype": "Section Break" - }, - { - "fieldname": "description", - "fieldtype": "Text Editor", - "in_list_view": 1, - "label": "Description", - "reqd": 1 - } - ], - "links": [], - "modified": "2020-04-22 12:09:49.057403", - "modified_by": "Administrator", - "module": "Core", - "name": "Video", - "owner": "Administrator", - "permissions": [ - { - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "if_owner": 1, - "print": 1, - "read": 1, - "report": 1, - "role": "All", - "share": 1, - "write": 1 - }, - { - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "share": 1, - "write": 1 - } - ], - "sort_field": "modified", - "sort_order": "DESC", - "track_changes": 1 -} \ No newline at end of file diff --git a/frappe/core/doctype/video/video.py b/frappe/core/doctype/video/video.py deleted file mode 100644 index fdbd3a1abe..0000000000 --- a/frappe/core/doctype/video/video.py +++ /dev/null @@ -1,10 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2020, Frappe Technologies and contributors -# For license information, please see license.txt - -from __future__ import unicode_literals -# import frappe -from frappe.model.document import Document - -class Video(Document): - pass diff --git a/frappe/public/js/frappe/views/breadcrumbs.js b/frappe/public/js/frappe/views/breadcrumbs.js index 0058310e3f..dbe591f6fd 100644 --- a/frappe/public/js/frappe/views/breadcrumbs.js +++ b/frappe/public/js/frappe/views/breadcrumbs.js @@ -6,7 +6,6 @@ frappe.breadcrumbs = { preferred: { "File": "", - "Video": "", "Dashboard": "Customization", "Dashboard Chart": "Customization", "Dashboard Chart Source": "Customization" From 5d4feb5dbb264611d2d257eea8f8f45b9636fbae Mon Sep 17 00:00:00 2001 From: gavin Date: Tue, 4 Aug 2020 17:45:14 +0530 Subject: [PATCH 073/230] feat: Convenience APIs for tags (#11088) Co-authored-by: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> --- frappe/model/document.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/frappe/model/document.py b/frappe/model/document.py index 69a781d6d1..316c576f55 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -1307,6 +1307,16 @@ class Document(BaseDocument): users = set([assignment.owner for assignment in assignments]) return users + def add_tag(self, tag): + """Add a Tag to this document""" + from frappe.desk.doctype.tag.tag import DocTags + DocTags(self.doctype).add(self.name, tag) + + def get_tags(self): + """Return a list of Tags attached to this document""" + from frappe.desk.doctype.tag.tag import DocTags + return DocTags(self.doctype).get_tags(self.name).split(",")[1:] + def execute_action(doctype, name, action, **kwargs): """Execute an action on a document (called by background worker)""" doc = frappe.get_doc(doctype, name) From 805e415502164c49aeef5a6332208ef2075c258d Mon Sep 17 00:00:00 2001 From: Prssanna Desai Date: Wed, 5 Aug 2020 14:57:48 +0530 Subject: [PATCH 074/230] fix: Dashboard fixes (#11186) Co-authored-by: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> --- frappe/desk/doctype/dashboard_chart/dashboard_chart.js | 6 ------ frappe/desk/doctype/dashboard_chart/dashboard_chart.py | 2 +- frappe/desk/doctype/number_card/number_card.py | 2 +- frappe/public/js/frappe/widgets/utils.js | 10 ++++++++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/frappe/desk/doctype/dashboard_chart/dashboard_chart.js b/frappe/desk/doctype/dashboard_chart/dashboard_chart.js index 8d89cc2f31..7f26bd9101 100644 --- a/frappe/desk/doctype/dashboard_chart/dashboard_chart.js +++ b/frappe/desk/doctype/dashboard_chart/dashboard_chart.js @@ -23,7 +23,6 @@ frappe.ui.form.on('Dashboard Chart', { frm.chart_filters = null; if (!frappe.boot.developer_mode && frm.doc.is_standard) { - frm.set_df_property('chart_options_section', 'hidden', 1); frm.disable_form(); } @@ -57,11 +56,6 @@ frappe.ui.form.on('Dashboard Chart', { if (frm.doc.report_name) { frm.trigger('set_chart_report_filters'); } - - if (!frappe.boot.developer_mode) { - frm.set_df_property("custom_options", "hidden", 1); - } - }, is_standard: function(frm) { diff --git a/frappe/desk/doctype/dashboard_chart/dashboard_chart.py b/frappe/desk/doctype/dashboard_chart/dashboard_chart.py index 88c9623945..7be97f8e24 100644 --- a/frappe/desk/doctype/dashboard_chart/dashboard_chart.py +++ b/frappe/desk/doctype/dashboard_chart/dashboard_chart.py @@ -143,7 +143,7 @@ def add_chart_to_dashboard(args): dashboard_link = frappe.new_doc('Dashboard Chart Link') dashboard_link.chart = args.chart_name or args.name - if args.set_standard: + if args.set_standard and dashboard.is_standard: chart = frappe.get_doc('Dashboard Chart', dashboard_link.chart) chart.is_standard = 1 chart.module = dashboard.module diff --git a/frappe/desk/doctype/number_card/number_card.py b/frappe/desk/doctype/number_card/number_card.py index 68ed79e64b..8b79cf4052 100644 --- a/frappe/desk/doctype/number_card/number_card.py +++ b/frappe/desk/doctype/number_card/number_card.py @@ -180,7 +180,7 @@ def add_card_to_dashboard(args): dashboard_link = frappe.new_doc('Number Card Link') dashboard_link.card = args.name - if args.set_standard: + if args.set_standard and dashboard.is_standard: card = frappe.get_doc('Number Card', dashboard_link.card) card.is_standard = 1 card.module = dashboard.module diff --git a/frappe/public/js/frappe/widgets/utils.js b/frappe/public/js/frappe/widgets/utils.js index 22b3167977..4599b4adc8 100644 --- a/frappe/public/js/frappe/widgets/utils.js +++ b/frappe/public/js/frappe/widgets/utils.js @@ -122,7 +122,8 @@ function shorten_number(number, country) { const number_system = get_number_system(country); let x = Math.abs(Math.round(number)); for (const map of number_system) { - if (x >= map.divisor) { + const condition = map.condition ? map.condition(x) : x >= map.divisor; + if (condition) { return Math.round(number/map.divisor) + ' ' + map.symbol; } } @@ -152,9 +153,14 @@ function get_number_system(country) { { divisor: 1.0e+6, symbol: 'M' + }, + { + divisor: 1.0e+3, + symbol: 'K', + condition: (num) => num.toFixed().length > 5 }] }; return number_system_map[country]; } -export { generate_route, generate_grid, build_summary_item, shorten_number }; \ No newline at end of file +export { generate_route, generate_grid, build_summary_item, shorten_number }; From 4225b55041c4da41a961bf5ca13c5f339b81e3ae Mon Sep 17 00:00:00 2001 From: Rushabh Mehta Date: Wed, 5 Aug 2020 15:33:30 +0530 Subject: [PATCH 075/230] feat(minor): Allow web form to follow document permissions --- frappe/website/doctype/web_form/web_form.json | 11 ++++- frappe/website/doctype/web_form/web_form.py | 45 ++++++++++--------- 2 files changed, 33 insertions(+), 23 deletions(-) diff --git a/frappe/website/doctype/web_form/web_form.json b/frappe/website/doctype/web_form/web_form.json index e6b7632e4d..77786734e7 100644 --- a/frappe/website/doctype/web_form/web_form.json +++ b/frappe/website/doctype/web_form/web_form.json @@ -16,6 +16,7 @@ "route_to_success_link", "allow_edit", "allow_multiple", + "apply_document_permissions", "show_in_grid", "allow_delete", "allow_print", @@ -346,14 +347,20 @@ "fieldname": "custom_css_section", "fieldtype": "Section Break", "label": "Custom CSS" + }, + { + "default": "0", + "fieldname": "apply_document_permissions", + "fieldtype": "Check", + "label": "Apply Document Permissions" } ], "has_web_view": 1, "icon": "icon-edit", "is_published_field": "published", "links": [], - "modified": "2019-12-24 14:15:43.497431", - "modified_by": "faris@erpnext.com", + "modified": "2020-06-30 21:49:18.237443", + "modified_by": "Administrator", "module": "Website", "name": "Web Form", "owner": "Administrator", diff --git a/frappe/website/doctype/web_form/web_form.py b/frappe/website/doctype/web_form/web_form.py index 4fc3c99037..bb0e03ddae 100644 --- a/frappe/website/doctype/web_form/web_form.py +++ b/frappe/website/doctype/web_form/web_form.py @@ -134,7 +134,7 @@ def get_context(context): if frappe.session.user == "Guest" and frappe.form_dict.name: frappe.throw(_("You need to be logged in to access this {0}.").format(self.doc_type), frappe.PermissionError) - if frappe.form_dict.name and not has_web_form_permission(self.doc_type, frappe.form_dict.name): + if frappe.form_dict.name and not self.has_web_form_permission(self.doc_type, frappe.form_dict.name): frappe.throw(_("You don't have the permissions to access this document"), frappe.PermissionError) self.reset_field_parent() @@ -347,6 +347,27 @@ def get_context(context): frappe.throw(_('Mandatory Information missing:') + '

              ' + '
              '.join(['{0} ({1})'.format(d.label, d.fieldtype) for d in missing])) + def has_web_form_permission(self, doctype, name, ptype='read'): + if frappe.session.user=="Guest": + return False + + if self.apply_document_permissions: + return frappe.get_doc(doctype, name).has_permission() + + # owner matches + elif frappe.db.get_value(doctype, name, "owner")==frappe.session.user: + return True + + elif frappe.has_website_permission(name, ptype=ptype, doctype=doctype): + return True + + elif check_webform_perm(doctype, name): + return True + + else: + return False + + @frappe.whitelist(allow_guest=True) def accept(web_form, data, docname=None, for_payment=False): @@ -395,7 +416,7 @@ def accept(web_form, data, docname=None, for_payment=False): doc.run_method('validate_payment') if doc.name: - if has_web_form_permission(doc.doctype, doc.name, "write"): + if web_form.has_web_form_permission(doc.doctype, doc.name, "write"): doc.save(ignore_permissions=True) else: # only if permissions are present @@ -482,24 +503,6 @@ def delete_multiple(web_form_name, docnames): raise frappe.PermissionError("You do not have permisssion to delete " + ", ".join(restricted_docnames)) -def has_web_form_permission(doctype, name, ptype='read'): - if frappe.session.user=="Guest": - return False - - # owner matches - elif frappe.db.get_value(doctype, name, "owner")==frappe.session.user: - return True - - elif frappe.has_website_permission(name, ptype=ptype, doctype=doctype): - return True - - elif check_webform_perm(doctype, name): - return True - - else: - return False - - def check_webform_perm(doctype, name): doc = frappe.get_doc(doctype, name) if hasattr(doc, "has_webform_permission"): @@ -536,7 +539,7 @@ def get_form_data(doctype, docname=None, web_form_name=None): if docname: doc = frappe.get_doc(doctype, docname) - if has_web_form_permission(doctype, docname, ptype='read'): + if web_form.has_web_form_permission(doctype, docname, ptype='read'): out.doc = doc else: frappe.throw(_("Not permitted"), frappe.PermissionError) From cd1ab8e23c4a1de4bbd5137c34a864af53788237 Mon Sep 17 00:00:00 2001 From: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> Date: Wed, 5 Aug 2020 19:41:41 +0530 Subject: [PATCH 076/230] feat: Add validate_and_sanitize_search_inputs decorator (#11194) * feat: Add validate_and_sanitize_search_inputs decorator Co-authored-by: Nabin Hait Co-authored-by: Prssanna Desai Co-authored-by: Shivam Mishra * refactor: Move validate_and_sanitize_search_inputs to init * refactor: Move validate_and_sanitize_search_inputs to search.py * test: validate_and_sanitize_search_inputs decorator * chore: Add wrapt module * refactor: Use @wrapt to define validate_and_sanitize_search_inputs decorator * test: Add a case to make sure frappe.call works as well Co-authored-by: Nabin Hait Co-authored-by: Prssanna Desai Co-authored-by: Shivam Mishra --- frappe/__init__.py | 4 ++++ frappe/desk/search.py | 13 +++++++++++++ frappe/tests/test_search.py | 28 ++++++++++++++++++++++++++++ requirements.txt | 3 ++- 4 files changed, 47 insertions(+), 1 deletion(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index 45f40f7783..9e744c2c07 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -1707,3 +1707,7 @@ def mock(type, size=1, locale='en'): from frappe.chat.util import squashify return squashify(results) + +def validate_and_sanitize_search_inputs(fn): + from frappe.desk.search import validate_and_sanitize_search_inputs as func + return func(fn) diff --git a/frappe/desk/search.py b/frappe/desk/search.py index b4b54b4b6e..798e499bb9 100644 --- a/frappe/desk/search.py +++ b/frappe/desk/search.py @@ -10,6 +10,7 @@ from frappe.handler import is_whitelisted from frappe import _ from six import string_types import re +import wrapt UNTRANSLATED_DOCTYPES = ["DocType", "Role"] @@ -206,3 +207,15 @@ def scrub_custom_query(query, key, txt): if '%s' in query: query = query.replace('%s', ((txt or '') + '%')) return query + +@wrapt.decorator +def validate_and_sanitize_search_inputs(fn, instance, args, kwargs): + kwargs.update(dict(zip(fn.__code__.co_varnames, args))) + sanitize_searchfield(kwargs['searchfield']) + kwargs['start'] = cint(kwargs['start']) + kwargs['page_len'] = cint(kwargs['page_len']) + + if kwargs['doctype'] and not frappe.db.exists('DocType', kwargs['doctype']): + return [] + + return fn(**kwargs) \ No newline at end of file diff --git a/frappe/tests/test_search.py b/frappe/tests/test_search.py index 5cab827218..192fe9dd25 100644 --- a/frappe/tests/test_search.py +++ b/frappe/tests/test_search.py @@ -50,3 +50,31 @@ class TestSearch(unittest.TestCase): def tearDown(self): frappe.local.lang = 'en' + + def test_validate_and_sanitize_search_inputs(self): + + # should raise error if searchfield is injectable + self.assertRaises(frappe.DataError, + get_data, *('User', 'Random', 'select * from tabSessions) --', '1', '10', dict())) + + # page_len and start should be converted to int + self.assertListEqual(get_data('User', 'Random', 'email', 'name or (select * from tabSessions)', '10', dict()), + ['User', 'Random', 'email', 0, 10, {}]) + self.assertListEqual(get_data('User', 'Random', 'email', page_len='2', start='10', filters=dict()), + ['User', 'Random', 'email', 10, 2, {}]) + + # DocType can be passed as None which should be accepted + self.assertListEqual(get_data(None, 'Random', 'email', '2', '10', dict()), + [None, 'Random', 'email', 2, 10, {}]) + + # return empty string if passed doctype is invalid + self.assertListEqual(get_data("Random DocType", 'Random', 'email', '2', '10', dict()), []) + + # should not fail if function is called via frappe.call with extra arguments + args = ("Random DocType", 'Random', 'email', '2', '10', dict()) + kwargs = {'as_dict': False} + self.assertListEqual(frappe.call('frappe.tests.test_search.get_data', *args, **kwargs), []) + +@frappe.validate_and_sanitize_search_inputs +def get_data(doctype, txt, searchfield, start, page_len, filters): + return [doctype, txt, searchfield, start, page_len, filters] \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 33cd292bf6..2995553541 100644 --- a/requirements.txt +++ b/requirements.txt @@ -69,4 +69,5 @@ Whoosh==2.7.4 xlrd==1.2.0 zxcvbn-python==4.4.24 pycryptodome==3.9.8 -paytmchecksum==1.7.0 \ No newline at end of file +paytmchecksum==1.7.0 +wrapt==1.10.11 \ No newline at end of file From 21a0cafe6209334906441c07c7f2e085afc82ade Mon Sep 17 00:00:00 2001 From: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> Date: Wed, 5 Aug 2020 19:42:19 +0530 Subject: [PATCH 077/230] fix: Validate and sanitise whitelisted search methods using decorator (#11197) * fix: Validate and sanitize whitelisted search methods using decorator Co-authored-by: Prssanna Desai Co-authored-by: Shivam Mishra * refactor: Replace decorator name Co-authored-by: Prssanna Desai Co-authored-by: Shivam Mishra Co-authored-by: Prssanna Desai Co-authored-by: Shivam Mishra --- frappe/automation/doctype/auto_repeat/auto_repeat.py | 1 + frappe/contacts/address_and_contact.py | 1 + frappe/contacts/doctype/address/address.py | 1 + frappe/contacts/doctype/contact/contact.py | 1 + frappe/core/doctype/user/user.py | 1 + frappe/core/doctype/user_permission/user_permission.py | 1 + .../permitted_documents_for_user/permitted_documents_for_user.py | 1 + frappe/desk/doctype/dashboard_chart/dashboard_chart.py | 1 + frappe/desk/doctype/number_card/number_card.py | 1 + frappe/email/__init__.py | 1 + 10 files changed, 10 insertions(+) diff --git a/frappe/automation/doctype/auto_repeat/auto_repeat.py b/frappe/automation/doctype/auto_repeat/auto_repeat.py index a946fcc81c..c09e347e71 100644 --- a/frappe/automation/doctype/auto_repeat/auto_repeat.py +++ b/frappe/automation/doctype/auto_repeat/auto_repeat.py @@ -374,6 +374,7 @@ def make_auto_repeat(doctype, docname, frequency = 'Daily', start_date = None, e # method for reference_doctype filter @frappe.whitelist() +@frappe.validate_and_sanitize_search_inputs def get_auto_repeat_doctypes(doctype, txt, searchfield, start, page_len, filters): res = frappe.db.get_all('Property Setter', { 'property': 'allow_auto_repeat', diff --git a/frappe/contacts/address_and_contact.py b/frappe/contacts/address_and_contact.py index 51f13fb1a1..3ca9547188 100644 --- a/frappe/contacts/address_and_contact.py +++ b/frappe/contacts/address_and_contact.py @@ -147,6 +147,7 @@ def delete_contact_and_address(doctype, docname): doc.delete() @frappe.whitelist() +@frappe.validate_and_sanitize_search_inputs def filter_dynamic_link_doctypes(doctype, txt, searchfield, start, page_len, filters): if not txt: txt = "" diff --git a/frappe/contacts/doctype/address/address.py b/frappe/contacts/doctype/address/address.py index 57dea8284c..e82ab9b26e 100644 --- a/frappe/contacts/doctype/address/address.py +++ b/frappe/contacts/doctype/address/address.py @@ -231,6 +231,7 @@ def get_company_address(company): return ret @frappe.whitelist() +@frappe.validate_and_sanitize_search_inputs def address_query(doctype, txt, searchfield, start, page_len, filters): from frappe.desk.reportview import get_match_cond diff --git a/frappe/contacts/doctype/contact/contact.py b/frappe/contacts/doctype/contact/contact.py index 8240940d2f..f82946dc5e 100644 --- a/frappe/contacts/doctype/contact/contact.py +++ b/frappe/contacts/doctype/contact/contact.py @@ -183,6 +183,7 @@ def update_contact(doc, method): contact.save(ignore_permissions=True) @frappe.whitelist() +@frappe.validate_and_sanitize_search_inputs def contact_query(doctype, txt, searchfield, start, page_len, filters): from frappe.desk.reportview import get_match_cond diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py index 64bff32189..ef2de679ae 100644 --- a/frappe/core/doctype/user/user.py +++ b/frappe/core/doctype/user/user.py @@ -812,6 +812,7 @@ def reset_password(user): return 'not found' @frappe.whitelist() +@frappe.validate_and_sanitize_search_inputs def user_query(doctype, txt, searchfield, start, page_len, filters): from frappe.desk.reportview import get_match_cond diff --git a/frappe/core/doctype/user_permission/user_permission.py b/frappe/core/doctype/user_permission/user_permission.py index 325c41622d..ba14583c2f 100644 --- a/frappe/core/doctype/user_permission/user_permission.py +++ b/frappe/core/doctype/user_permission/user_permission.py @@ -120,6 +120,7 @@ def user_permission_exists(user, allow, for_value, applicable_for=None): return has_same_user_permission @frappe.whitelist() +@frappe.validate_and_sanitize_search_inputs def get_applicable_for_doctype_list(doctype, txt, searchfield, start, page_len, filters): linked_doctypes_map = get_linked_doctypes(doctype, True) diff --git a/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.py b/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.py index 5ca21b811e..97209cd8ea 100644 --- a/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.py +++ b/frappe/core/report/permitted_documents_for_user/permitted_documents_for_user.py @@ -42,6 +42,7 @@ def get_columns_and_fields(doctype): return columns, fields @frappe.whitelist() +@frappe.validate_and_sanitize_search_inputs def query_doctypes(doctype, txt, searchfield, start, page_len, filters): user = filters.get("user") user_perms = frappe.utils.user.UserPermissions(user) diff --git a/frappe/desk/doctype/dashboard_chart/dashboard_chart.py b/frappe/desk/doctype/dashboard_chart/dashboard_chart.py index 7be97f8e24..4ea61ec6a9 100644 --- a/frappe/desk/doctype/dashboard_chart/dashboard_chart.py +++ b/frappe/desk/doctype/dashboard_chart/dashboard_chart.py @@ -358,6 +358,7 @@ def get_year_ending(date): return add_to_date(date, days=-1) @frappe.whitelist() +@frappe.validate_and_sanitize_search_inputs def get_charts_for_user(doctype, txt, searchfield, start, page_len, filters): or_filters = {'owner': frappe.session.user, 'is_public': 1} return frappe.db.get_list('Dashboard Chart', diff --git a/frappe/desk/doctype/number_card/number_card.py b/frappe/desk/doctype/number_card/number_card.py index 8b79cf4052..d4a2b00c57 100644 --- a/frappe/desk/doctype/number_card/number_card.py +++ b/frappe/desk/doctype/number_card/number_card.py @@ -129,6 +129,7 @@ def create_number_card(args): return doc @frappe.whitelist() +@frappe.validate_and_sanitize_search_inputs def get_cards_for_user(doctype, txt, searchfield, start, page_len, filters): meta = frappe.get_meta(doctype) searchfields = meta.get_search_fields() diff --git a/frappe/email/__init__.py b/frappe/email/__init__.py index 0b874a03bb..b05aef7639 100644 --- a/frappe/email/__init__.py +++ b/frappe/email/__init__.py @@ -58,6 +58,7 @@ def relink(name, reference_doctype=None, reference_name=None): name = %s""", (reference_doctype, reference_name, name)) @frappe.whitelist() +@frappe.validate_and_sanitize_search_inputs def get_communication_doctype(doctype, txt, searchfield, start, page_len, filters): user_perms = frappe.utils.user.UserPermissions(frappe.session.user) user_perms.build_permissions() From 34c3d57767611db622779efe5cf0454b8a866f92 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Wed, 5 Aug 2020 18:31:11 +0000 Subject: [PATCH 078/230] feat: validate json when saving desk page (#11095) Co-authored-by: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> --- frappe/desk/doctype/desk_page/desk_page.py | 9 +++++++++ frappe/utils/data.py | 9 +++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/frappe/desk/doctype/desk_page/desk_page.py b/frappe/desk/doctype/desk_page/desk_page.py index f14535cb5f..cc2db53481 100644 --- a/frappe/desk/doctype/desk_page/desk_page.py +++ b/frappe/desk/doctype/desk_page/desk_page.py @@ -5,14 +5,23 @@ from __future__ import unicode_literals import frappe from frappe import _ +from frappe.utils.data import validate_json_string from frappe.modules.export_file import export_to_files from frappe.model.document import Document class DeskPage(Document): def validate(self): + self.validate_cards_json() if (self.is_standard and not frappe.conf.developer_mode and not disable_saving_as_standard()): frappe.throw(_("You need to be in developer mode to edit this document")) + def validate_cards_json(self): + for card in self.cards: + try: + validate_json_string(card.links) + except frappe.ValidationError: + frappe.throw(_("Invalid JSON in card links for {0}").format(frappe.bold(card.label))) + def on_update(self): if disable_saving_as_standard(): return diff --git a/frappe/utils/data.py b/frappe/utils/data.py index d1b409d1fc..bda201f083 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -8,6 +8,7 @@ import frappe from dateutil.parser._parser import ParserError import subprocess import operator +import json import re, datetime, math, time import babel.dates from babel.core import UnknownLocaleError @@ -1236,8 +1237,6 @@ def is_subset(list_a, list_b): def generate_hash(*args, **kwargs): return frappe.generate_hash(*args, **kwargs) - - def guess_date_format(date_string): DATE_FORMATS = [ r"%d-%m-%Y", @@ -1310,3 +1309,9 @@ def guess_date_format(date_string): if date_format and time_format: return (date_format + ' ' + time_format).strip() + +def validate_json_string(string): + try: + json.loads(string) + except (TypeError, ValueError): + raise frappe.ValidationError \ No newline at end of file From ef4a28e10c0de0083c57278b615e587600b64e3b Mon Sep 17 00:00:00 2001 From: Mangesh-Khairnar Date: Thu, 6 Aug 2020 09:32:04 +0530 Subject: [PATCH 079/230] fix: add a distinct identifier for public files backup (#11201) --- frappe/integrations/offsite_backup_utils.py | 2 +- frappe/utils/backups.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/integrations/offsite_backup_utils.py b/frappe/integrations/offsite_backup_utils.py index 9de176b2d0..0b3c47064e 100644 --- a/frappe/integrations/offsite_backup_utils.py +++ b/frappe/integrations/offsite_backup_utils.py @@ -53,7 +53,7 @@ def get_latest_backup_file(with_files=False): latest_site_config = get_latest('*.json') if with_files: - latest_public_file_bak = get_latest('*-files.tar') + latest_public_file_bak = get_latest('*-public-files.tar') latest_private_file_bak = get_latest('*-private-files.tar') return latest_file, latest_site_config, latest_public_file_bak, latest_private_file_bak diff --git a/frappe/utils/backups.py b/frappe/utils/backups.py index 7eb00ceccd..08b911c9ae 100644 --- a/frappe/utils/backups.py +++ b/frappe/utils/backups.py @@ -82,7 +82,7 @@ class BackupGenerator: def set_backup_file_name(self): #Generate a random name using today's date and a 8 digit random number for_db = self.todays_date + "-" + self.site_slug + "-database.sql.gz" - for_public_files = self.todays_date + "-" + self.site_slug + "-files.tar" + for_public_files = self.todays_date + "-" + self.site_slug + "-public-files.tar" for_private_files = self.todays_date + "-" + self.site_slug + "-private-files.tar" backup_path = get_backup_path() From 4f37f9a4528ba50b82f2000a4381c8ff813ff138 Mon Sep 17 00:00:00 2001 From: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> Date: Thu, 6 Aug 2020 09:33:39 +0530 Subject: [PATCH 080/230] test: Try committing after Role creation --- frappe/workflow/doctype/workflow/test_workflow.py | 1 + 1 file changed, 1 insertion(+) diff --git a/frappe/workflow/doctype/workflow/test_workflow.py b/frappe/workflow/doctype/workflow/test_workflow.py index d7f7939d3d..84adc3a096 100644 --- a/frappe/workflow/doctype/workflow/test_workflow.py +++ b/frappe/workflow/doctype/workflow/test_workflow.py @@ -126,6 +126,7 @@ def create_todo_workflow(): else: frappe.get_doc(dict(doctype='Role', role_name='Test Approver')).insert(ignore_if_duplicate=True) + frappe.db.commit() frappe.cache().hdel('roles', frappe.session.user) workflow = frappe.new_doc('Workflow') workflow.workflow_name = 'Test ToDo' From 91d0e47f1dacf544533dc840b848cb5a59646a6a Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Wed, 17 Jun 2020 20:03:38 +0530 Subject: [PATCH 081/230] feat: optimize while storing index --- frappe/modules/full_text_search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/modules/full_text_search.py b/frappe/modules/full_text_search.py index fce9983907..a9b93a1265 100644 --- a/frappe/modules/full_text_search.py +++ b/frappe/modules/full_text_search.py @@ -67,7 +67,7 @@ def build_index(index_name, documents): title=document.title, path=document.path, content=document.content ) - writer.commit() + writer.commit(optimize=True) def search(index_name, text, scope=None, limit=20): From aa0cf2bad4ba4a0ed749e0187bab7f39fcb5cd92 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Wed, 17 Jun 2020 20:05:08 +0530 Subject: [PATCH 082/230] feat: added re-index method --- frappe/modules/full_text_search.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/frappe/modules/full_text_search.py b/frappe/modules/full_text_search.py index a9b93a1265..d06d925d08 100644 --- a/frappe/modules/full_text_search.py +++ b/frappe/modules/full_text_search.py @@ -101,6 +101,26 @@ def search(index_name, text, scope=None, limit=20): return out +def reindex_path(index_name, path): + document = get_document_to_index(path): + reindex(index_name, document) + +def reindex(index_name, document): + index_dir = get_index_path(index_name) + ix = open_dir(index_dir) + + with ix.searcher() as searcher: + writer = ix.writer() + + # Remove the index of the particular file + writer.delete_by_term('path', document.path) + + # that wasn't indexed before. So index it! + writer.add_document( + title=document.title, path=document.path, content=document.content + ) + + writer.commit(optimize=True) def get_index_path(index_name): return frappe.get_site_path("indexes", index_name) From d5d32770136fc71e6d0de8bbc3a291b825d916a0 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Wed, 17 Jun 2020 20:05:55 +0530 Subject: [PATCH 083/230] feat: move search portable --- frappe/templates/doc.html | 66 +-------------------------- frappe/website/js/website.js | 86 +++++++++++++++++++++++++++++++++++- 2 files changed, 87 insertions(+), 65 deletions(-) diff --git a/frappe/templates/doc.html b/frappe/templates/doc.html index 3e1cc5509a..1c70aace98 100644 --- a/frappe/templates/doc.html +++ b/frappe/templates/doc.html @@ -22,7 +22,7 @@
            -
            +