From 2ea74dee36f1bbd094a7fa967fc50be26ba5e09d Mon Sep 17 00:00:00 2001 From: "Chinmay D. Pai" Date: Tue, 26 May 2020 18:47:17 +0530 Subject: [PATCH 001/245] fix: check for whitelist before calling from search search widget takes query as an input, but does not check whether the query function that is called is whitelisted, basically allowing anyone logged-in to call any function regardless of the whitelist. Signed-off-by: Chinmay D. Pai --- frappe/desk/search.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frappe/desk/search.py b/frappe/desk/search.py index c70b650945..1da2a3d0b5 100644 --- a/frappe/desk/search.py +++ b/frappe/desk/search.py @@ -6,6 +6,7 @@ from __future__ import unicode_literals import frappe, json from frappe.utils import cstr, unique, cint from frappe.permissions import has_permission +from frappe.handler import is_whitelisted from frappe import _ from six import string_types import re @@ -74,6 +75,7 @@ def search_widget(doctype, txt, query=None, searchfield=None, start=0, if query and query.split()[0].lower()!="select": # by method + is_whitelisted(query) frappe.response["values"] = frappe.call(query, doctype, txt, searchfield, start, page_length, filters, as_dict=as_dict) elif not query and doctype in standard_queries: From c4d4fc3574dcb0f716fc8ad5edb162f171a990fe Mon Sep 17 00:00:00 2001 From: "Chinmay D. Pai" Date: Tue, 26 May 2020 18:49:12 +0530 Subject: [PATCH 002/245] chore: add standard queries hooks to whitelist standard queries are used within the search widget, and now require to be whitelisted before they can be executed through the search widget. Signed-off-by: Chinmay D. Pai --- frappe/core/doctype/user/user.py | 1 + 1 file changed, 1 insertion(+) diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py index 0c5ebc3ede..f571240454 100644 --- a/frappe/core/doctype/user/user.py +++ b/frappe/core/doctype/user/user.py @@ -811,6 +811,7 @@ def reset_password(user): frappe.clear_messages() return 'not found' +@frappe.whitelist() def user_query(doctype, txt, searchfield, start, page_len, filters): from frappe.desk.reportview import get_match_cond From 59a1ee6a823ad6d29f4cb8951304ae67947f1941 Mon Sep 17 00:00:00 2001 From: prssanna Date: Thu, 4 Jun 2020 18:49:01 +0530 Subject: [PATCH 003/245] feat: date range in leaderboard --- frappe/desk/leaderboard.py | 13 +++-- frappe/desk/page/leaderboard/leaderboard.js | 57 ++++++++++----------- 2 files changed, 36 insertions(+), 34 deletions(-) diff --git a/frappe/desk/leaderboard.py b/frappe/desk/leaderboard.py index 1ebf32febe..e5654c853f 100644 --- a/frappe/desk/leaderboard.py +++ b/frappe/desk/leaderboard.py @@ -14,13 +14,16 @@ def get_leaderboards(): return leaderboards @frappe.whitelist() -def get_energy_point_leaderboard(from_date, company = None, field = None, limit = None): +def get_energy_point_leaderboard(date_range, company = None, field = None, limit = None): + filters = [ + ['type', '!=', 'Review'], + ] + if date_range: + date_range = frappe.parse_json(date_range) + filters.append(['creation', 'between', [date_range[0], date_range[1]]]) energy_point_users = frappe.db.get_all('Energy Point Log', fields = ['user as name', 'sum(points) as value'], - filters = [ - ['type', '!=', 'Review'], - ['creation', '>', from_date] - ], + filters = filters, group_by = 'user', order_by = 'value desc' ) diff --git a/frappe/desk/page/leaderboard/leaderboard.js b/frappe/desk/page/leaderboard/leaderboard.js index 4472a2978a..ecec7f4805 100644 --- a/frappe/desk/page/leaderboard/leaderboard.js +++ b/frappe/desk/page/leaderboard/leaderboard.js @@ -49,7 +49,7 @@ class Leaderboard { this.timespans = [ "This Week", "This Month", "This Quarter", "This Year", "Last Week", "Last Month", "Last Quarter", "Last Year", - "All Time", "Select From Date" + "All Time", "Select Date Range" ]; // for saving current selected filters @@ -113,7 +113,7 @@ class Leaderboard { return {"label": __(d), value: d }; }) ); - this.create_from_date_field(); + this.create_date_range_field(); this.type_select = this.page.add_select(__("Field"), this.options.selected_filter.map(d => { @@ -123,12 +123,12 @@ class Leaderboard { this.timespan_select.on("change", (e) => { this.options.selected_timespan = e.currentTarget.value; - if (this.options.selected_timespan === 'Select From Date') { - this.from_date_field.show(); + if (this.options.selected_timespan === 'Select Date Range') { + this.date_range_field.show(); } else { - this.from_date_field.hide(); - this.make_request(); + this.date_range_field.hide(); } + this.make_request(); }); this.type_select.on("change", (e) => { @@ -137,21 +137,21 @@ class Leaderboard { }); } - create_from_date_field() { + create_date_range_field() { let timespan_field = $(this.parent).find(`.frappe-control[data-original-title='Timespan']`); - this.from_date_field = $(`
`).insertAfter(timespan_field).hide(); + this.date_range_field = $(`
`).insertAfter(timespan_field).hide(); let date_field = frappe.ui.form.make_control({ df: { - fieldtype: 'Date', - fieldname: 'selected_from_date', - placeholder: frappe.datetime.month_start(), - default: frappe.datetime.month_start(), + fieldtype: 'DateRange', + fieldname: 'selected_date_range', + placeholder: "Date Range", + default: [frappe.datetime.month_start(), frappe.datetime.now_date()], input_class: 'input-sm', reqd: 1, change: () => { - this.selected_from_date = date_field.get_value(); - if (this.selected_from_date) this.make_request(); + this.selected_date_range = date_field.get_value(); + if (this.selected_date_range) this.make_request(); } }, parent: $(this.parent).find('.from-date-field'), @@ -225,7 +225,7 @@ class Leaderboard { frappe.call( this.leaderboard_config[this.options.selected_doctype].method, { - 'from_date': this.get_from_date(), + 'date_range': this.get_date_range(), 'company': this.options.selected_company, 'field': this.options.selected_filter_item, 'limit': this.leaderboard_limit, @@ -375,23 +375,22 @@ class Leaderboard { `); } - get_from_date() { + get_date_range() { let timespan = this.options.selected_timespan.toLowerCase(); let current_date = frappe.datetime.now_date(); - let get_from_date = { - "this week": frappe.datetime.week_start(), - "this month": frappe.datetime.month_start(), - "this quarter": frappe.datetime.quarter_start(), - "this year": frappe.datetime.year_start(), - "last week": frappe.datetime.add_days(current_date, -7), - "last month": frappe.datetime.add_months(current_date, -1), - "last quarter": frappe.datetime.add_months(current_date, -3), - "last year": frappe.datetime.add_months(current_date, -12), - "all time": "", - "select from date": this.selected_from_date || frappe.datetime.month_start() + let date_range_map = { + "this week": [frappe.datetime.week_start(), frappe.datetime.now_date()], + "this month": [frappe.datetime.month_start(), frappe.datetime.now_date()], + "this quarter": [frappe.datetime.quarter_start(), frappe.datetime.now_date()], + "this year": [frappe.datetime.year_start(), frappe.datetime.now_date()], + "last week": [frappe.datetime.add_days(current_date, -7), frappe.datetime.now_date()], + "last month": [frappe.datetime.add_months(current_date, -1), frappe.datetime.now_date()], + "last quarter": [frappe.datetime.add_months(current_date, -3), frappe.datetime.now_date()], + "last year": [frappe.datetime.add_months(current_date, -12), frappe.datetime.now_date()], + "all time": null, + "select date range": this.selected_date_range || [frappe.datetime.month_start(), frappe.datetime.now_date()] } - - return get_from_date[timespan]; + return date_range_map[timespan]; } } From 67e1198d064ec6e0b838f2d242cb119e2b906c8e Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 12 Jun 2020 16:24:41 +0530 Subject: [PATCH 004/245] feat: prompt in case of site downgrades on restore --- frappe/commands/site.py | 12 +++++++++--- frappe/installer.py | 27 +++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/frappe/commands/site.py b/frappe/commands/site.py index 28e61282eb..c327d85af0 100755 --- a/frappe/commands/site.py +++ b/frappe/commands/site.py @@ -108,12 +108,14 @@ def _new_site(db_name, site, mariadb_root_username=None, mariadb_root_password=N @click.option('--install-app', multiple=True, help='Install app after installation') @click.option('--with-public-files', help='Restores the public files of the site, given path to its tar file') @click.option('--with-private-files', help='Restores the private files of the site, given path to its tar file') +@click.option('--force', is_flag=True, default=False, help='Use a bit of force to get the job done') @pass_context def restore(context, sql_file_path, mariadb_root_username=None, mariadb_root_password=None, db_name=None, verbose=None, install_app=None, admin_password=None, force=None, with_public_files=None, with_private_files=None): "Restore site database from an sql file" - from frappe.installer import extract_sql_gzip, extract_tar_files - # Extract the gzip file if user has passed *.sql.gz file instead of *.sql file + from frappe.installer import extract_sql_gzip, extract_tar_files, is_downgrade + force = context.force or force + # Extract the gzip file if user has passed *.sql.gz file instead of *.sql file if not os.path.exists(sql_file_path): base_path = '..' sql_file_path = os.path.join(base_path, sql_file_path) @@ -125,7 +127,6 @@ def restore(context, sql_file_path, mariadb_root_username=None, mariadb_root_pas else: base_path = '.' - if sql_file_path.endswith('sql.gz'): decompressed_file_name = extract_sql_gzip(os.path.abspath(sql_file_path)) else: @@ -133,6 +134,11 @@ def restore(context, sql_file_path, mariadb_root_username=None, mariadb_root_pas site = get_site(context) frappe.init(site=site) + + # dont allow downgrading to older versions of frappe without force + if not force and is_downgrade(decompressed_file_name): + click.confirm("Downgrading sites may lead to a broken site. Do you wish to continue?", abort=True) + _new_site(frappe.conf.db_name, site, mariadb_root_username=mariadb_root_username, mariadb_root_password=mariadb_root_password, admin_password=admin_password, verbose=context.verbose, install_apps=install_app, source_sql=decompressed_file_name, diff --git a/frappe/installer.py b/frappe/installer.py index 4fc19b282a..38a8aadc3b 100755 --- a/frappe/installer.py +++ b/frappe/installer.py @@ -326,3 +326,30 @@ def extract_tar_files(site_name, file_path, folder_name): frappe.destroy() return tar_path + +def is_downgrade(sql_file_path): + """checks if input db backup will get downgraded on current bench""" + from semantic_version import Version + from frappe.utils.change_log import get_app_branch + head = "INSERT INTO `tabInstalled Application` VALUES" + + with open(sql_file_path) as f: + for line in f: + if head in line: + # ('2056588823','2020-05-11 18:21:31.488367','2020-06-12 11:49:31.079506','Administrator','Administrator',0,'Installed Applications','installed_applications','Installed Applications',1,'frappe','v10.1.71-74 (3c50d5e) (v10.x.x)','v10.x.x'),('855c640b8e','2020-05-11 18:21:31.488367','2020-06-12 11:49:31.079506','Administrator','Administrator',0,'Installed Applications','installed_applications','Installed Applications',2,'press','0.0.1','master') + line = line.strip().lstrip(head).rstrip(";").strip() + # [('frappe', '12.x.x-develop ()', 'develop'), ('press', '0.0.1', 'master')] + all_apps = [ x[-3:] for x in frappe.safe_eval(line) ] + + for app in all_apps: + app_name = app[0] + app_version = app[1].split(" ")[0] + + if app_name == "frappe": + try: + current_version = Version(frappe.__version__) + backup_version = Version(app_version[1:] if app_version[0] is "v" else app_version) + except ValueError: + return False + + return backup_version > current_version From 13ca526be8bc35a27a7a2d4ac1971ecf0d2391af Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 12 Jun 2020 16:36:50 +0530 Subject: [PATCH 005/245] chore: Sider --- frappe/installer.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/frappe/installer.py b/frappe/installer.py index 38a8aadc3b..962a9e40e5 100755 --- a/frappe/installer.py +++ b/frappe/installer.py @@ -330,7 +330,6 @@ def extract_tar_files(site_name, file_path, folder_name): def is_downgrade(sql_file_path): """checks if input db backup will get downgraded on current bench""" from semantic_version import Version - from frappe.utils.change_log import get_app_branch head = "INSERT INTO `tabInstalled Application` VALUES" with open(sql_file_path) as f: @@ -348,7 +347,7 @@ def is_downgrade(sql_file_path): if app_name == "frappe": try: current_version = Version(frappe.__version__) - backup_version = Version(app_version[1:] if app_version[0] is "v" else app_version) + backup_version = Version(app_version[1:] if app_version[0] == "v" else app_version) except ValueError: return False From a72f8088972c7154e3400e59449de5bd2c5023e5 Mon Sep 17 00:00:00 2001 From: "Chinmay D. Pai" Date: Thu, 18 Jun 2020 18:09:27 +0530 Subject: [PATCH 006/245] fix: make cookies more secure * add HTTPOnly to sid, so sid will no longer be accessible through javascript. * mark all cookies Secure by default over HTTPS. * set SameSite to Strict for all cookies by default, preventing cross-origin cookie access. * remove redundant publish_realtime for setting csrf_token on the client-side. Signed-off-by: Chinmay D. Pai --- frappe/auth.py | 21 ++++++++++++++++----- frappe/public/js/frappe/desk.js | 9 --------- frappe/public/js/frappe/request.js | 4 ++-- frappe/sessions.py | 7 ------- frappe/website/js/website.js | 9 ++------- 5 files changed, 20 insertions(+), 30 deletions(-) diff --git a/frappe/auth.py b/frappe/auth.py index 1353acf10f..ab3624bee8 100644 --- a/frappe/auth.py +++ b/frappe/auth.py @@ -333,12 +333,20 @@ class CookieManager: # sid expires in 3 days expires = datetime.datetime.now() + datetime.timedelta(days=3) if frappe.session.sid: - self.cookies["sid"] = {"value": frappe.session.sid, "expires": expires} + self.set_cookie("sid", frappe.session.sid, expires=expires, httponly=True) if frappe.session.session_country: - self.cookies["country"] = {"value": frappe.session.get("session_country")} + self.set_cookie("country", frappe.session.session_country) - def set_cookie(self, key, value, expires=None): - self.cookies[key] = {"value": value, "expires": expires} + def set_cookie(self, key, value, expires=None, secure=False, httponly=False, samesite="Strict"): + if not secure: + secure = frappe.local.request.scheme == "https" + self.cookies[key] = { + "value": value, + "expires": expires, + "secure": secure, + "httponly": httponly, + "samesite": samesite + } def delete_cookie(self, to_delete): if not isinstance(to_delete, (list, tuple)): @@ -349,7 +357,10 @@ class CookieManager: def flush_cookies(self, response): for key, opts in self.cookies.items(): response.set_cookie(key, quote((opts.get("value") or "").encode('utf-8')), - expires=opts.get("expires")) + expires=opts.get("expires"), + secure=opts.get("secure"), + httponly=opts.get("httponly"), + samesite=opts.get("samesite")) # expires yesterday! expires = datetime.datetime.now() + datetime.timedelta(days=-1) diff --git a/frappe/public/js/frappe/desk.js b/frappe/public/js/frappe/desk.js index 79a78717cb..2e80dbfd85 100644 --- a/frappe/public/js/frappe/desk.js +++ b/frappe/public/js/frappe/desk.js @@ -101,15 +101,6 @@ frappe.Application = Class.extend({ frappe.ui.startup_setup_dialog.show(); } - // listen to csrf_update - frappe.realtime.on("csrf_generated", function(data) { - // handles the case when a user logs in again from another tab - // and it leads to invalid request in the current tab - if (data.csrf_token && data.sid===frappe.get_cookie("sid")) { - frappe.csrf_token = data.csrf_token; - } - }); - frappe.realtime.on("version-update", function() { var dialog = frappe.msgprint({ message:__("The application has been updated to a new version, please refresh this page"), diff --git a/frappe/public/js/frappe/request.js b/frappe/public/js/frappe/request.js index ea4de99249..e378499f35 100644 --- a/frappe/public/js/frappe/request.js +++ b/frappe/public/js/frappe/request.js @@ -125,7 +125,7 @@ frappe.request.call = function(opts) { message: __('The resource you are looking for is not available')}); }, 403: function(xhr) { - if (frappe.get_cookie('sid')==='Guest') { + if (frappe.session.user === 'Guest') { // session expired frappe.app.handle_session_expired(); } @@ -320,7 +320,7 @@ frappe.request.cleanup = function(opts, r) { if(r) { // session expired? - Guest has no business here! - if(r.session_expired || frappe.get_cookie("sid")==="Guest") { + if (r.session_expired || frappe.session.user === "Guest") { frappe.app.handle_session_expired(); return; } diff --git a/frappe/sessions.py b/frappe/sessions.py index d317d6caf3..7a018bb0aa 100644 --- a/frappe/sessions.py +++ b/frappe/sessions.py @@ -172,13 +172,6 @@ def generate_csrf_token(): frappe.local.session.data.csrf_token = frappe.generate_hash() frappe.local.session_obj.update(force=True) - # send sid and csrf token to the user - # handles the case when a user logs in again from another tab - # and it leads to invalid request in the current tab - frappe.publish_realtime(event="csrf_generated", - message={"sid": frappe.local.session.sid, "csrf_token": frappe.local.session.data.csrf_token}, - user=frappe.session.user, after_commit=True) - class Session: def __init__(self, user, resume=False, full_name=None, user_type=None): self.sid = cstr(frappe.form_dict.get('sid') or diff --git a/frappe/website/js/website.js b/frappe/website/js/website.js index b52a3fef63..f2c142fe4c 100644 --- a/frappe/website/js/website.js +++ b/frappe/website/js/website.js @@ -183,10 +183,6 @@ $.extend(frappe, { .html('

' +text+'
').appendTo(document.body); }, - get_sid: function() { - var sid = frappe.get_cookie("sid"); - return sid && sid !== "Guest"; - }, send_message: function(opts, btn) { return frappe.call({ type: "POST", @@ -212,8 +208,7 @@ $.extend(frappe, { }); }, render_user: function() { - var sid = frappe.get_cookie("sid"); - if(sid && sid!=="Guest") { + if (frappe.is_user_logged_in()) { $(".btn-login-area").toggle(false); $(".logged-in").toggle(true); $(".full-name").html(frappe.get_cookie("full_name")); @@ -323,7 +318,7 @@ $.extend(frappe, { return $(".navbar .search, .sidebar .search"); }, is_user_logged_in: function() { - return frappe.get_cookie("sid") && frappe.get_cookie("sid") !== "Guest"; + return frappe.session.user !== "Guest"; }, add_switch_to_desk: function() { $('.switch-to-desk').removeClass('hidden'); From a29fab70ac5653c81bda5d076a06db83789e47d0 Mon Sep 17 00:00:00 2001 From: "Chinmay D. Pai" Date: Thu, 18 Jun 2020 23:14:19 +0530 Subject: [PATCH 007/245] chore: improve logic for checking logged in user frappe.session.user sometimes gets set after the function runs, so current implementation would return undefined !== "Guest" as True, causing Guest user to be set as a logged-in user. Signed-off-by: Chinmay D. Pai --- frappe/website/js/website.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/website/js/website.js b/frappe/website/js/website.js index f2c142fe4c..cd9609cfd3 100644 --- a/frappe/website/js/website.js +++ b/frappe/website/js/website.js @@ -318,7 +318,7 @@ $.extend(frappe, { return $(".navbar .search, .sidebar .search"); }, is_user_logged_in: function() { - return frappe.session.user !== "Guest"; + return frappe.get_cookie("user_id") !== "Guest" && frappe.session.user !== "Guest"; }, add_switch_to_desk: function() { $('.switch-to-desk').removeClass('hidden'); From 2afe0cdaa48c88de5ad884f180eba9b3ae2d5215 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Mon, 22 Jun 2020 09:01:37 +0530 Subject: [PATCH 008/245] fix: Column mapping --- .../core/doctype/data_import/data_import.js | 4 +- frappe/core/doctype/data_import/importer.py | 6 ++- .../js/frappe/data_import/import_preview.js | 37 ++++++++++++++++--- 3 files changed, 39 insertions(+), 8 deletions(-) diff --git a/frappe/core/doctype/data_import/data_import.js b/frappe/core/doctype/data_import/data_import.js index 81a7bc9705..9ccf43436b 100644 --- a/frappe/core/doctype/data_import/data_import.js +++ b/frappe/core/doctype/data_import/data_import.js @@ -301,8 +301,8 @@ frappe.ui.form.on('Data Import', { events: { remap_column(changed_map) { let template_options = JSON.parse(frm.doc.template_options || '{}'); - template_options.remap_column = template_options.remap_column || {}; - Object.assign(template_options.remap_column, changed_map); + template_options.column_to_field_map = template_options.column_to_field_map || {}; + Object.assign(template_options.column_to_field_map, changed_map); frm.set_value('template_options', JSON.stringify(template_options)); frm.save().then(() => frm.trigger('import_file')); } diff --git a/frappe/core/doctype/data_import/importer.py b/frappe/core/doctype/data_import/importer.py index 4761652c70..09849dd5b9 100644 --- a/frappe/core/doctype/data_import/importer.py +++ b/frappe/core/doctype/data_import/importer.py @@ -766,8 +766,9 @@ class Header(Row): for j, header in enumerate(row): column_values = [get_item_at_index(r, j) for r in raw_data] + map_to_field = column_to_field_map.get(str(j)) column = Column( - j, header, self.doctype, column_values, column_to_field_map.get(header), self.seen + j, header, self.doctype, column_values, map_to_field, self.seen ) self.seen.append(header) self.columns.append(column) @@ -944,6 +945,9 @@ class Column: d.map_to_field = self.map_to_field d.date_format = self.date_format d.df = self.df + if hasattr(self.df, 'is_child_table_field'): + d.is_child_table_field = self.df.is_child_table_field + d.child_table_df = self.df.child_table_df d.skip_import = self.skip_import d.warnings = self.warnings return d diff --git a/frappe/public/js/frappe/data_import/import_preview.js b/frappe/public/js/frappe/data_import/import_preview.js index 7cf8431456..4edcb87aeb 100644 --- a/frappe/public/js/frappe/data_import/import_preview.js +++ b/frappe/public/js/frappe/data_import/import_preview.js @@ -245,11 +245,12 @@ frappe.data_import.ImportPreview = class ImportPreview { let fieldname; if (!df) { fieldname = null; + } else if (col.map_to_field) { + fieldname = col.map_to_field; + } else if (col.is_child_table_field) { + fieldname = `${col.child_table_df.fieldname}.${df.fieldname}`; } else { - fieldname = - df.parent === this.doctype - ? df.fieldname - : `${df.parent}:${df.fieldname}`; + fieldname = df.fieldname; } return [ { @@ -272,7 +273,7 @@ frappe.data_import.ImportPreview = class ImportPreview { label: __("Don't Import"), value: "Don't Import" } - ].concat(column_picker_fields.get_fields_as_options()), + ].concat(get_fields_as_options(this.doctype, column_picker_fields)), default: fieldname || "Don't Import", change() { changed.push(i); @@ -328,3 +329,29 @@ frappe.data_import.ImportPreview = class ImportPreview { }); } }; + +function get_fields_as_options(doctype, column_map) { + let keys = [doctype]; + frappe.meta.get_table_fields(doctype).forEach(df => { + keys.push(df.fieldname); + }); + // flatten array + return [].concat( + ...keys.map(key => { + return column_map[key].map(df => { + let label = df.label; + let value = df.fieldname; + if (doctype !== key) { + let table_field = frappe.meta.get_docfield(doctype, key); + label = `${df.label} (${table_field.label})`; + value = `${table_field.fieldname}.${df.fieldname}`; + } + return { + label, + value, + description: value + }; + }); + }) + ); +} \ No newline at end of file From b502bfff07da834745bde3018e4a2ebc316d3a4f Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Mon, 22 Jun 2020 09:02:07 +0530 Subject: [PATCH 009/245] fix: Serialize df manually --- frappe/core/doctype/data_import/importer.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/frappe/core/doctype/data_import/importer.py b/frappe/core/doctype/data_import/importer.py index 09849dd5b9..13684aaad3 100644 --- a/frappe/core/doctype/data_import/importer.py +++ b/frappe/core/doctype/data_import/importer.py @@ -607,7 +607,7 @@ class Row: self.warnings.append( { "row": self.row_number, - "field": df.as_dict(convert_dates_to_str=True), + "field": df_as_json(df), "message": msg, } ) @@ -622,7 +622,7 @@ class Row: self.warnings.append( { "row": self.row_number, - "field": df.as_dict(convert_dates_to_str=True), + "field": df_as_json(df), "message": msg, } ) @@ -635,7 +635,7 @@ class Row: { "row": self.row_number, "col": col.column_number, - "field": df.as_dict(convert_dates_to_str=True), + "field": df_as_json(df), "message": _("Value {0} must in {1} format").format( frappe.bold(value), frappe.bold(get_user_format(col.date_format)) ), @@ -646,7 +646,7 @@ class Row: return value def link_exists(self, value, df): - key = df.options + "::" + value + key = df.options + "::" + cstr(value) if Row.link_values_exist_map.get(key) is None: Row.link_values_exist_map[key] = frappe.db.exists(df.options, value) return Row.link_values_exist_map.get(key) @@ -1117,3 +1117,13 @@ def get_user_format(date_format): .replace("%m", "mm") .replace("%d", "dd") ) + +def df_as_json(df): + return { + 'fieldname': df.fieldname, + 'fieldtype': df.fieldtype, + 'label': df.label, + 'options': df.options, + 'parent': df.parent, + 'default': df.default + } From bbdc5e0db897f7fe7e5ce09b4a917142fe7e56be Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Mon, 22 Jun 2020 09:03:00 +0530 Subject: [PATCH 010/245] fix(exporter): Count for child table filters - Extract count method from list_view.js for reuse --- .../js/frappe/data_import/data_exporter.js | 8 +++--- frappe/public/js/frappe/db.js | 26 ++++++++++++++----- frappe/public/js/frappe/list/list_view.js | 24 +++-------------- 3 files changed, 27 insertions(+), 31 deletions(-) diff --git a/frappe/public/js/frappe/data_import/data_exporter.js b/frappe/public/js/frappe/data_import/data_exporter.js index 735237189d..6ee4e907fc 100644 --- a/frappe/public/js/frappe/data_import/data_exporter.js +++ b/frappe/public/js/frappe/data_import/data_exporter.js @@ -285,11 +285,9 @@ frappe.data_import.DataExporter = class DataExporter { } get_filters() { - return this.filter_group.get_filters().reduce((acc, filter) => { - return Object.assign(acc, { - [filter[1]]: [filter[2], filter[3]] - }); - }, {}); + return this.filter_group.get_filters().map(filter => { + return filter.slice(0, 4) + }); } get_multicheck_options(doctype, child_fieldname = null) { diff --git a/frappe/public/js/frappe/db.js b/frappe/public/js/frappe/db.js index 1b6fb0e438..cf716c67e5 100644 --- a/frappe/public/js/frappe/db.js +++ b/frappe/public/js/frappe/db.js @@ -91,12 +91,26 @@ frappe.db = { }); }, count: function(doctype, args={}) { - return new Promise(resolve => { - frappe.call({ - method: 'frappe.client.get_count', - type: 'GET', - args: Object.assign(args, { doctype }) - }).then(r => resolve(r.message)); + let filters = args.filters || {}; + const with_child_table_filter = Array.isArray(filters) && filters.some(filter => { + return filter[0] !== doctype; + }); + + const fields = [ + // cannot break this line as it adds extra \n's and \t's which breaks the query + `count(${with_child_table_filter ? 'distinct': ''} ${frappe.model.get_full_column_name('name', doctype)}) AS total_count` + ]; + + return frappe.call({ + type: 'GET', + method: 'frappe.desk.reportview.get', + args: { + doctype, + filters, + fields, + } + }).then(r => { + return r.message.values[0][0]; }); }, get_link_options(doctype, txt = '', filters={}) { diff --git a/frappe/public/js/frappe/list/list_view.js b/frappe/public/js/frappe/list/list_view.js index 9e1ba1b9bd..8da73b0dec 100644 --- a/frappe/public/js/frappe/list/list_view.js +++ b/frappe/public/js/frappe/list/list_view.js @@ -760,26 +760,10 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList { let current_count = this.data.length; let count_without_children = this.data.uniqBy(d => d.name).length; - const filters = this.get_filters_for_args(); - const with_child_table_filter = filters.some(filter => { - return filter[0] !== this.doctype; - }); - - const fields = [ - // cannot break this line as it adds extra \n's and \t's which breaks the query - `count(${with_child_table_filter ? 'distinct': ''}${frappe.model.get_full_column_name('name', this.doctype)}) AS total_count` - ]; - - return frappe.call({ - type: 'GET', - method: this.method, - args: { - doctype: this.doctype, - filters, - fields, - } - }).then(r => { - this.total_count = r.message.values[0][0] || current_count; + return frappe.db.count(this.doctype, { + filters: this.get_filters_for_args() + }).then(total_count => { + this.total_count = total_count || current_count; let str = __('{0} of {1}', [current_count, this.total_count]); if (count_without_children !== current_count) { str = __('{0} of {1} ({2} rows with children)', [count_without_children, this.total_count, current_count]); From 010dbc402e4685aaf83a138c704c723a31c6c61a Mon Sep 17 00:00:00 2001 From: Mangesh-Khairnar Date: Mon, 22 Jun 2020 16:29:44 +0530 Subject: [PATCH 011/245] fix(google-calendar): reset sync token incase of invalid next sync token --- .../doctype/google_calendar/google_calendar.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/frappe/integrations/doctype/google_calendar/google_calendar.py b/frappe/integrations/doctype/google_calendar/google_calendar.py index fa2eea6ce1..c43f867711 100644 --- a/frappe/integrations/doctype/google_calendar/google_calendar.py +++ b/frappe/integrations/doctype/google_calendar/google_calendar.py @@ -12,6 +12,7 @@ from frappe import _ from frappe.model.document import Document from frappe.utils import get_request_site_address from googleapiclient.errors import HttpError +from frappe.utils.password import set_encrypted_password from frappe.utils import add_days, get_datetime, get_weekdays, now_datetime, add_to_date, get_time_zone from dateutil import parser from datetime import datetime, timedelta @@ -218,15 +219,23 @@ def sync_events_from_google_calendar(g_calendar, method=None, page_length=10): events = google_calendar.events().list(calendarId=account.google_calendar_id, maxResults=page_length, singleEvents=False, showDeleted=True, syncToken=sync_token).execute() except HttpError as err: - frappe.throw(_("Google Calendar - Could not fetch event from Google Calendar, error code {0}.").format(err.resp.status)) + msg = _("Google Calendar - Could not fetch event from Google Calendar, error code {0}.").format(err.resp.status) + + if err.resp.status == 410: + set_encrypted_password("Google Calendar", account.name, "", "next_sync_token") + frappe.db.commit() + msg += _(' Sync token was invalid and has been resetted, Retry syncing.') + frappe.msgprint(msg, title='Invalid Sync Token', indicator='blue') + else: + frappe.throw(msg) for event in events.get("items", []): results.append(event) if not events.get("nextPageToken"): if events.get("nextSyncToken"): - frappe.db.set_value("Google Calendar", account.name, "next_sync_token", events.get("nextSyncToken")) - frappe.db.commit() + account.next_sync_token = events.get("nextSyncToken") + account.save() break for idx, event in enumerate(results): From 25667cde1ed17deb9ee5f4cec6dac0deba6d56d7 Mon Sep 17 00:00:00 2001 From: Mangesh-Khairnar Date: Mon, 22 Jun 2020 16:43:27 +0530 Subject: [PATCH 012/245] fix(google-calendar): pass page token to load the next set of events --- .../doctype/google_calendar/google_calendar.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/frappe/integrations/doctype/google_calendar/google_calendar.py b/frappe/integrations/doctype/google_calendar/google_calendar.py index c43f867711..d1cd62f84f 100644 --- a/frappe/integrations/doctype/google_calendar/google_calendar.py +++ b/frappe/integrations/doctype/google_calendar/google_calendar.py @@ -199,7 +199,7 @@ def check_google_calendar(account, google_calendar): except HttpError as err: frappe.throw(_("Google Calendar - Could not create Calendar for {0}, error code {1}.").format(account.name, err.resp.status)) -def sync_events_from_google_calendar(g_calendar, method=None, page_length=10): +def sync_events_from_google_calendar(g_calendar, method=None): """ Syncs Events from Google Calendar in Framework Calendar. Google Calendar returns nextSyncToken when all the events in Google Calendar are fetched. @@ -211,13 +211,14 @@ def sync_events_from_google_calendar(g_calendar, method=None, page_length=10): if not account.pull_from_google_calendar: return + sync_token = account.get_password(fieldname="next_sync_token", raise_exception=False) or None + events = frappe._dict() results = [] while True: try: # API Response listed at EOF - sync_token = account.get_password(fieldname="next_sync_token", raise_exception=False) or None - events = google_calendar.events().list(calendarId=account.google_calendar_id, maxResults=page_length, - singleEvents=False, showDeleted=True, syncToken=sync_token).execute() + events = google_calendar.events().list(calendarId=account.google_calendar_id, maxResults=2000, + pageToken=events.get("nextPageToken"), singleEvents=False, showDeleted=True, syncToken=sync_token).execute() except HttpError as err: msg = _("Google Calendar - Could not fetch event from Google Calendar, error code {0}.").format(err.resp.status) From aae9133594a915b19506144749674b567730b398 Mon Sep 17 00:00:00 2001 From: "Chinmay D. Pai" Date: Wed, 24 Jun 2020 11:15:24 +0530 Subject: [PATCH 013/245] fix: disallow access to signup page when disabled signup page is still accessible through the URL even when the button is not shown. this disables access to the page when signups are disabled. Signed-off-by: Chinmay D. Pai --- frappe/www/login.html | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/frappe/www/login.html b/frappe/www/login.html index ebbff748ec..f1131db8cf 100644 --- a/frappe/www/login.html +++ b/frappe/www/login.html @@ -70,20 +70,29 @@
From 5b828adab7d56d09d1745ff74c2870223b78d7a0 Mon Sep 17 00:00:00 2001 From: Chinmay Pai Date: Wed, 24 Jun 2020 11:31:10 +0530 Subject: [PATCH 014/245] chore: use default alignment for elements Co-authored-by: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> --- frappe/www/login.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/www/login.html b/frappe/www/login.html index f1131db8cf..d57f126022 100644 --- a/frappe/www/login.html +++ b/frappe/www/login.html @@ -85,8 +85,8 @@
{{_("Signup Disabled")}}
-

{{_("Signups have been disabled for this website.")}}

-
{{ _("Home") }}
+

{{_("Signups have been disabled for this website.")}}

+ {%- endif -%}
From 711f8231bc0642120d7e01f650fae050fa6c8da1 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Wed, 24 Jun 2020 13:36:15 +0530 Subject: [PATCH 015/245] fix(patch): Drop DI Legacy if exists - Ignore patch if already run --- frappe/patches.txt | 2 +- frappe/patches/v13_0/replace_old_data_import.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/frappe/patches.txt b/frappe/patches.txt index a03d31918b..f883d2a7bb 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -289,4 +289,4 @@ execute:frappe.delete_doc("DocType", "Onboarding Slide Field") execute:frappe.delete_doc("DocType", "Onboarding Slide Help Link") frappe.patches.v13_0.update_date_filters_in_user_settings frappe.patches.v13_0.update_duration_options -frappe.patches.v13_0.replace_old_data_import +frappe.patches.v13_0.replace_old_data_import # 2020-06-24 diff --git a/frappe/patches/v13_0/replace_old_data_import.py b/frappe/patches/v13_0/replace_old_data_import.py index 1c00ae5f34..469703e172 100644 --- a/frappe/patches/v13_0/replace_old_data_import.py +++ b/frappe/patches/v13_0/replace_old_data_import.py @@ -6,6 +6,10 @@ import frappe def execute(): + if not frappe.db.exists("DocType", "Data Import Beta"): + return + + frappe.db.sql("DROP TABLE IF EXISTS `tabData Import Legacy`") frappe.rename_doc('DocType', 'Data Import', 'Data Import Legacy') frappe.db.commit() frappe.db.sql("DROP TABLE IF EXISTS `tabData Import`") From 5e31a6886ce55d5ae3ca4e8933685316028eef70 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Wed, 24 Jun 2020 15:06:25 +0530 Subject: [PATCH 016/245] fix: Data Exporter - Set options as mandatory only when inserting new records - Hide Select Mandatory button during update --- .../core/doctype/data_import/data_import.js | 28 +++++------ .../js/frappe/data_import/data_exporter.js | 49 ++++--------------- 2 files changed, 22 insertions(+), 55 deletions(-) diff --git a/frappe/core/doctype/data_import/data_import.js b/frappe/core/doctype/data_import/data_import.js index 9ccf43436b..6a922618cb 100644 --- a/frappe/core/doctype/data_import/data_import.js +++ b/frappe/core/doctype/data_import/data_import.js @@ -102,6 +102,10 @@ frappe.ui.form.on('Data Import', { }, update_primary_action(frm) { + if (frm.is_dirty()) { + frm.enable_save(); + return; + } frm.disable_save(); if (frm.doc.status !== 'Success') { if (!frm.is_new() && (frm.has_import_file())) { @@ -199,20 +203,12 @@ frappe.ui.form.on('Data Import', { }, download_template(frm) { - if ( - frm.data_exporter && - frm.data_exporter.doctype === frm.doc.reference_doctype - ) { - frm.data_exporter.exporting_for = frm.doc.import_type; - frm.data_exporter.dialog.show(); - } else { - frappe.require('/assets/js/data_import_tools.min.js', () => { - frm.data_exporter = new frappe.data_import.DataExporter( - frm.doc.reference_doctype, - frm.doc.import_type - ); - }); - } + frappe.require('/assets/js/data_import_tools.min.js', () => { + frm.data_exporter = new frappe.data_import.DataExporter( + frm.doc.reference_doctype, + frm.doc.import_type + ); + }); }, reference_doctype(frm) { @@ -435,10 +431,10 @@ frappe.ui.form.on('Data Import', { .join(''); let id = frappe.dom.get_unique_id(); html = `${messages} - -
+ 
${log.exception}
diff --git a/frappe/public/js/frappe/data_import/data_exporter.js b/frappe/public/js/frappe/data_import/data_exporter.js index 6ee4e907fc..238e916c34 100644 --- a/frappe/public/js/frappe/data_import/data_exporter.js +++ b/frappe/public/js/frappe/data_import/data_exporter.js @@ -13,36 +13,6 @@ frappe.data_import.DataExporter = class DataExporter { this.dialog = new frappe.ui.Dialog({ title: __('Export Data'), fields: [ - { - fieldtype: 'Select', - fieldname: 'exporting_for', - label: __('Exporting For'), - options: [ - { - label: __('Insert New Records'), - value: 'Insert New Records' - }, - { - label: __('Update Existing Records'), - value: 'Update Existing Records' - } - ], - change: () => { - let exporting_for = this.dialog.get_value('exporting_for'); - this.dialog.set_value( - 'export_records', - exporting_for === 'Insert New Records' ? 'blank_template' : 'all' - ); - - // Force ID field to be exported when updating existing records - let id_field = this.dialog.get_field(this.doctype).options[0]; - if (id_field.value === 'name' && id_field.$checkbox) { - id_field.$checkbox - .find('input') - .prop('disabled', exporting_for === 'Update Existing Records'); - } - } - }, { fieldtype: 'Select', fieldname: 'export_records', @@ -65,7 +35,7 @@ frappe.data_import.DataExporter = class DataExporter { value: 'blank_template' } ], - default: 'blank_template', + default: this.exporting_for === 'Insert New Records' ? 'blank_template' : 'all', change: () => { this.update_record_count_message(); } @@ -119,10 +89,6 @@ frappe.data_import.DataExporter = class DataExporter { on_page_show: () => this.select_mandatory() }); - if (this.exporting_for) { - this.dialog.set_value('exporting_for', this.exporting_for); - } - this.make_filter_area(); this.make_select_all_buttons(); this.update_record_count_message(); @@ -172,15 +138,17 @@ frappe.data_import.DataExporter = class DataExporter { } make_select_all_buttons() { + let for_insert = this.exporting_for === 'Insert New Records'; + let section_title = for_insert ? __('Select Fields To Insert') : __('Select Fields To Update'); let $select_all_buttons = $(`
-
${__('Select fields to export')}
+
${section_title}
- + `: ''} @@ -306,6 +274,9 @@ frappe.data_import.DataExporter = class DataExporter { ? this.column_map[child_fieldname] : this.column_map[doctype]; + let is_field_mandatory = df => (df.fieldname === 'name' && !child_fieldname) + || (df.reqd && this.exporting_for == 'Insert New Records') + return fields .filter(df => { if (autoname_field && df.fieldname === autoname_field.fieldname) { @@ -321,7 +292,7 @@ frappe.data_import.DataExporter = class DataExporter { return { label, value: df.fieldname, - danger: df.reqd, + danger: is_field_mandatory(df), checked: false, description: `${df.fieldname} ${df.reqd ? __('(Mandatory)') : ''}` }; From 8d8c9a348b93226c94b12124660ae116689baf4b Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Wed, 24 Jun 2020 15:07:23 +0530 Subject: [PATCH 017/245] fix: Data Import form - Hide Download Template button for new doc - Hide Refresh Google Sheet button if doc is dirty --- frappe/core/doctype/data_import/data_import.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/frappe/core/doctype/data_import/data_import.json b/frappe/core/doctype/data_import/data_import.json index 177252ea22..8b1b6c4e07 100644 --- a/frappe/core/doctype/data_import/data_import.json +++ b/frappe/core/doctype/data_import/data_import.json @@ -119,7 +119,7 @@ { "fieldname": "import_warnings_section", "fieldtype": "Section Break", - "label": "Warnings" + "label": "Import File Errors and Warnings" }, { "fieldname": "import_warnings", @@ -127,7 +127,7 @@ "label": "Import Warnings" }, { - "depends_on": "reference_doctype", + "depends_on": "eval:!doc.__islocal", "fieldname": "download_template", "fieldtype": "Button", "label": "Download Template" @@ -159,7 +159,7 @@ "label": "Import from Google Sheets" }, { - "depends_on": "eval:doc.google_sheets_url", + "depends_on": "eval:doc.google_sheets_url && !doc.__unsaved", "fieldname": "refresh_google_sheet", "fieldtype": "Button", "label": "Refresh Google Sheet" @@ -167,7 +167,7 @@ ], "hide_toolbar": 1, "links": [], - "modified": "2020-06-18 16:05:54.211034", + "modified": "2020-06-24 14:33:03.173876", "modified_by": "Administrator", "module": "Core", "name": "Data Import", From ebc3c26f7d52d76066012737799561f80e32c636 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Wed, 24 Jun 2020 15:09:54 +0530 Subject: [PATCH 018/245] fix: Data Import for Update - Update doc only if changed else throw error - Check for mandatory only for inserting --- frappe/core/doctype/data_import/importer.py | 84 +++++++++++++-------- 1 file changed, 51 insertions(+), 33 deletions(-) diff --git a/frappe/core/doctype/data_import/importer.py b/frappe/core/doctype/data_import/importer.py index 13684aaad3..11228b834b 100644 --- a/frappe/core/doctype/data_import/importer.py +++ b/frappe/core/doctype/data_import/importer.py @@ -16,6 +16,7 @@ from frappe.utils.xlsxutils import ( read_xls_file_from_attached_file, ) from frappe.model import no_value_fields, table_fields as table_fieldtypes +from frappe.core.doctype.version.version import get_diff INVALID_VALUES = ("", None) MAX_ROWS_IN_PREVIEW = 10 @@ -216,14 +217,22 @@ class Importer: def update_record(self, doc): id_field = get_id_field(self.doctype) existing_doc = frappe.get_doc(self.doctype, doc.get(id_field.fieldname)) - existing_doc.flags.updater_reference = { - "doctype": self.data_import.doctype, - "docname": self.data_import.name, - "label": _("via Data Import"), - } - existing_doc.update(doc) - existing_doc.save() - return existing_doc + + updated_doc = frappe.get_doc(self.doctype, doc.get(id_field.fieldname)) + updated_doc.update(doc) + + if get_diff(existing_doc, updated_doc): + # update doc if there are changes + updated_doc.flags.updater_reference = { + "doctype": self.data_import.doctype, + "docname": self.data_import.name, + "label": _("via Data Import"), + } + updated_doc.save() + return updated_doc + else: + # throw if no changes + frappe.throw('No changes to update') def get_eta(self, current, total, processing_time): self.last_eta = getattr(self, "last_eta", 0) @@ -306,8 +315,9 @@ class ImportFile: ) self.column_to_field_map = self.template_options.column_to_field_map self.import_type = import_type + self.warnings = [] - self.file_doc = self.file_path = None + self.file_doc = self.file_path = self.google_sheets_url = None if isinstance(file, frappe.string_types): if frappe.db.exists("File", {"file_url": file}): self.file_doc = frappe.get_doc("File", {"file_url": file}) @@ -462,38 +472,46 @@ class ImportFile: parent_doc[table_df.fieldname].append(child_doc) doc = parent_doc - # check if there is atleast one row for mandatory table fields - meta = frappe.get_meta(self.doctype) - mandatory_table_fields = [ - df - for df in meta.fields - if df.fieldtype in table_fieldtypes - and df.reqd - and len(doc.get(df.fieldname, [])) == 0 - ] - if len(mandatory_table_fields) == 1: - self.warnings.append( - { - "row": first_row.row_number, - "message": _("There should be atleast one row for {0} table").format( - mandatory_table_fields[0].label - ), - } - ) - elif mandatory_table_fields: - fields_string = ", ".join([df.label for df in mandatory_table_fields]) - message = _("There should be atleast one row for the following tables: {0}").format( - fields_string - ) - self.warnings.append({"row": first_row.row_number, "message": message}) + + if self.import_type == INSERT: + # check if there is atleast one row for mandatory table fields + meta = frappe.get_meta(self.doctype) + mandatory_table_fields = [ + df + for df in meta.fields + if df.fieldtype in table_fieldtypes + and df.reqd + and len(doc.get(df.fieldname, [])) == 0 + ] + if len(mandatory_table_fields) == 1: + self.warnings.append( + { + "row": first_row.row_number, + "message": _("There should be atleast one row for {0} table").format( + frappe.bold(mandatory_table_fields[0].label) + ), + } + ) + elif mandatory_table_fields: + fields_string = ", ".join([df.label for df in mandatory_table_fields]) + message = _("There should be atleast one row for the following tables: {0}").format( + fields_string + ) + self.warnings.append({"row": first_row.row_number, "message": message}) return doc, rows, data[len(rows) :] def get_warnings(self): warnings = [] + + # ImportFile warnings + warnings += self.warnings + + # Column warnings for col in self.header.columns: warnings += col.warnings + # Row warnings for row in self.data: warnings += row.warnings From f3943ed663d9849ef87bf358948aa505e9174f46 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Wed, 24 Jun 2020 15:10:21 +0530 Subject: [PATCH 019/245] fix: Column - Check if values exists in Link Column --- frappe/core/doctype/data_import/importer.py | 26 +++++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/frappe/core/doctype/data_import/importer.py b/frappe/core/doctype/data_import/importer.py index 11228b834b..6e530f93db 100644 --- a/frappe/core/doctype/data_import/importer.py +++ b/frappe/core/doctype/data_import/importer.py @@ -843,7 +843,7 @@ class Column: self.meta = frappe.get_meta(doctype) self.parse() - self.parse_date_format() + self.validate_values() def parse(self): header_title = self.header_title @@ -916,10 +916,6 @@ class Column: self.df = df self.skip_import = skip_import - def parse_date_format(self): - if self.df and self.df.fieldtype in ("Date", "Time", "Datetime"): - self.date_format = self.guess_date_format_for_column() - def guess_date_format_for_column(self): """ Guesses date format for a column by parsing all the values in the column, getting the date format and then returning the one which has the maximum frequency @@ -954,6 +950,26 @@ class Column: return max_occurred_date_format + def validate_values(self): + if not self.df: + return + + if self.df.fieldtype == 'Link': + # find all values that dont exist + values = list(set([v for v in self.column_values[1:] if v])) + exists = [d.name for d in frappe.db.get_all(self.df.options, filters={'name': ('in', values)})] + not_exists = list(set(values) - set(exists)) + if not_exists: + missing_values = ', '.join(not_exists) + self.warnings.append({ + 'col': self.column_number, + 'message': "The following values do not exist for {}: {}".format(self.df.options, missing_values), + 'type': 'warning' + }) + elif self.df.fieldtype in ("Date", "Time", "Datetime"): + # guess date format + self.date_format = self.guess_date_format_for_column() + def as_dict(self): d = frappe._dict() d.index = self.index From 68b4591f10ed970f3cfe8215d046ff1c4d17e1f0 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Wed, 24 Jun 2020 15:10:29 +0530 Subject: [PATCH 020/245] fix: Update yarn.lock --- yarn.lock | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/yarn.lock b/yarn.lock index d858972218..251aa893b5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -252,18 +252,6 @@ "@types/events" "*" "@types/node" "*" -"@types/bunyan@*": - version "1.8.6" - resolved "https://registry.yarnpkg.com/@types/bunyan/-/bunyan-1.8.6.tgz#6527641cca30bedec5feb9ab527b7803b8000582" - integrity sha512-YiozPOOsS6bIuz31ilYqR5SlLif4TBWsousN2aCWLi5233nZSX19tFbcQUPdR7xJ8ypPyxkCGNxg0CIV5n9qxQ== - dependencies: - "@types/node" "*" - -"@types/color-name@^1.1.1": - version "1.1.1" - resolved "https://registry.yarnpkg.com/@types/color-name/-/color-name-1.1.1.tgz#1c1261bbeaa10a8055bbc5d8ab84b7b2afc846a0" - integrity sha512-rr+OQyAjxze7GgWrSaJwydHStIhHq2lvY3BOC2Mj7KnzI7XK0Uw1TOOdI9lDoajEbSWLiYgoo4f1R51erQfhPQ== - "@types/debug@^4.1.4": version "4.1.5" resolved "https://registry.yarnpkg.com/@types/debug/-/debug-4.1.5.tgz#b14efa8852b7768d898906613c23f688713e02cd" @@ -3529,7 +3517,7 @@ lodash.uniq@^4.5.0: resolved "https://registry.yarnpkg.com/lodash.uniq/-/lodash.uniq-4.5.0.tgz#d0225373aeb652adc1bc82e4945339a842754773" integrity sha1-0CJTc662Uq3BvILklFM5qEJ1R3M= -lodash@4.17.15, lodash@^4.0.0, lodash@^4.17.10, lodash@^4.17.11, lodash@^4.17.14, lodash@^4.17.15, lodash@^4.17.4, lodash@^4.17.5, lodash@~4.17.10: +lodash@4.17.15, lodash@^4.0.0, lodash@^4.17.10, lodash@^4.17.14, lodash@^4.17.15, lodash@^4.17.4, lodash@^4.17.5, lodash@~4.17.10: version "4.17.15" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.15.tgz#b447f6670a0455bbfeedd11392eff330ea097548" integrity sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A== @@ -4803,11 +4791,6 @@ prettier@^1.19.1: resolved "https://registry.yarnpkg.com/prettier/-/prettier-1.19.1.tgz#f7d7f5ff8a9cd872a7be4ca142095956a60797cb" integrity sha512-s7PoyDv/II1ObgQunCbB9PdLmUcBZcnWOcxDh7O0N/UwDEsHyqkW+Qh28jW+mVuCdx7gLB0BotYI1Y6uI9iyew== -pretty-hrtime@^1.0.3: - version "1.0.3" - resolved "https://registry.yarnpkg.com/pretty-hrtime/-/pretty-hrtime-1.0.3.tgz#b7e3ea42435a4c9b2759d99e0f201eb195802ee1" - integrity sha1-t+PqQkNaTJsnWdmeDyAesZWALuE= - process-nextick-args@~1.0.6: version "1.0.7" resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-1.0.7.tgz#150e20b756590ad3f91093f25a4f2ad8bff30ba3" From 8343105093af811351c09d7283b4a4ab819044ac Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Wed, 24 Jun 2020 15:18:22 +0530 Subject: [PATCH 021/245] fix: Handle empty column_to_field_map --- frappe/core/doctype/data_import/importer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frappe/core/doctype/data_import/importer.py b/frappe/core/doctype/data_import/importer.py index 6e530f93db..ec3cccc1b1 100644 --- a/frappe/core/doctype/data_import/importer.py +++ b/frappe/core/doctype/data_import/importer.py @@ -773,11 +773,12 @@ class Row: class Header(Row): - def __init__(self, index, row, doctype, raw_data, column_to_field_map): + def __init__(self, index, row, doctype, raw_data, column_to_field_map=None): self.index = index self.row_number = index + 1 self.data = row self.doctype = doctype + column_to_field_map = column_to_field_map or frappe._dict() self.seen = [] self.columns = [] From c53c0c43f4080620f69ffd7efe8809988002dd05 Mon Sep 17 00:00:00 2001 From: "Chinmay D. Pai" Date: Wed, 24 Jun 2020 19:45:53 +0530 Subject: [PATCH 022/245] fix: allow darkgrey indicator for disabled signup Signed-off-by: Chinmay D. Pai --- frappe/templates/includes/login/login.js | 2 +- frappe/www/login.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/templates/includes/login/login.js b/frappe/templates/includes/login/login.js index 37b8acb328..9fa22468d3 100644 --- a/frappe/templates/includes/login/login.js +++ b/frappe/templates/includes/login/login.js @@ -97,7 +97,7 @@ login.reset_sections = function(hide) { $("section.for-forgot").toggle(false); $("section.for-signup").toggle(false); } - $('section .indicator').each(function() { + $('section:not(.signup-disabled) .indicator').each(function() { $(this).removeClass().addClass('indicator').addClass('blue') .text($(this).attr('data-text')); }); diff --git a/frappe/www/login.html b/frappe/www/login.html index d57f126022..9d00892b18 100644 --- a/frappe/www/login.html +++ b/frappe/www/login.html @@ -68,7 +68,7 @@ {{ _("Forgot Password?") }}

-