diff --git a/frappe/core/doctype/data_import/data_import.js b/frappe/core/doctype/data_import/data_import.js index 81a7bc9705..6a922618cb 100644 --- a/frappe/core/doctype/data_import/data_import.js +++ b/frappe/core/doctype/data_import/data_import.js @@ -102,6 +102,10 @@ frappe.ui.form.on('Data Import', { }, update_primary_action(frm) { + if (frm.is_dirty()) { + frm.enable_save(); + return; + } frm.disable_save(); if (frm.doc.status !== 'Success') { if (!frm.is_new() && (frm.has_import_file())) { @@ -199,20 +203,12 @@ frappe.ui.form.on('Data Import', { }, download_template(frm) { - if ( - frm.data_exporter && - frm.data_exporter.doctype === frm.doc.reference_doctype - ) { - frm.data_exporter.exporting_for = frm.doc.import_type; - frm.data_exporter.dialog.show(); - } else { - frappe.require('/assets/js/data_import_tools.min.js', () => { - frm.data_exporter = new frappe.data_import.DataExporter( - frm.doc.reference_doctype, - frm.doc.import_type - ); - }); - } + frappe.require('/assets/js/data_import_tools.min.js', () => { + frm.data_exporter = new frappe.data_import.DataExporter( + frm.doc.reference_doctype, + frm.doc.import_type + ); + }); }, reference_doctype(frm) { @@ -301,8 +297,8 @@ frappe.ui.form.on('Data Import', { events: { remap_column(changed_map) { let template_options = JSON.parse(frm.doc.template_options || '{}'); - template_options.remap_column = template_options.remap_column || {}; - Object.assign(template_options.remap_column, changed_map); + template_options.column_to_field_map = template_options.column_to_field_map || {}; + Object.assign(template_options.column_to_field_map, changed_map); frm.set_value('template_options', JSON.stringify(template_options)); frm.save().then(() => frm.trigger('import_file')); } @@ -435,10 +431,10 @@ frappe.ui.form.on('Data Import', { .join(''); let id = frappe.dom.get_unique_id(); html = `${messages} - -
+
${log.exception}
diff --git a/frappe/core/doctype/data_import/data_import.json b/frappe/core/doctype/data_import/data_import.json index 177252ea22..8b1b6c4e07 100644 --- a/frappe/core/doctype/data_import/data_import.json +++ b/frappe/core/doctype/data_import/data_import.json @@ -119,7 +119,7 @@ { "fieldname": "import_warnings_section", "fieldtype": "Section Break", - "label": "Warnings" + "label": "Import File Errors and Warnings" }, { "fieldname": "import_warnings", @@ -127,7 +127,7 @@ "label": "Import Warnings" }, { - "depends_on": "reference_doctype", + "depends_on": "eval:!doc.__islocal", "fieldname": "download_template", "fieldtype": "Button", "label": "Download Template" @@ -159,7 +159,7 @@ "label": "Import from Google Sheets" }, { - "depends_on": "eval:doc.google_sheets_url", + "depends_on": "eval:doc.google_sheets_url && !doc.__unsaved", "fieldname": "refresh_google_sheet", "fieldtype": "Button", "label": "Refresh Google Sheet" @@ -167,7 +167,7 @@ ], "hide_toolbar": 1, "links": [], - "modified": "2020-06-18 16:05:54.211034", + "modified": "2020-06-24 14:33:03.173876", "modified_by": "Administrator", "module": "Core", "name": "Data Import", diff --git a/frappe/core/doctype/data_import/importer.py b/frappe/core/doctype/data_import/importer.py index 4761652c70..ec3cccc1b1 100644 --- a/frappe/core/doctype/data_import/importer.py +++ b/frappe/core/doctype/data_import/importer.py @@ -16,6 +16,7 @@ from frappe.utils.xlsxutils import ( read_xls_file_from_attached_file, ) from frappe.model import no_value_fields, table_fields as table_fieldtypes +from frappe.core.doctype.version.version import get_diff INVALID_VALUES = ("", None) MAX_ROWS_IN_PREVIEW = 10 @@ -216,14 +217,22 @@ class Importer: def update_record(self, doc): id_field = get_id_field(self.doctype) existing_doc = frappe.get_doc(self.doctype, doc.get(id_field.fieldname)) - existing_doc.flags.updater_reference = { - "doctype": self.data_import.doctype, - "docname": self.data_import.name, - "label": _("via Data Import"), - } - existing_doc.update(doc) - existing_doc.save() - return existing_doc + + updated_doc = frappe.get_doc(self.doctype, doc.get(id_field.fieldname)) + updated_doc.update(doc) + + if get_diff(existing_doc, updated_doc): + # update doc if there are changes + updated_doc.flags.updater_reference = { + "doctype": self.data_import.doctype, + "docname": self.data_import.name, + "label": _("via Data Import"), + } + updated_doc.save() + return updated_doc + else: + # throw if no changes + frappe.throw('No changes to update') def get_eta(self, current, total, processing_time): self.last_eta = getattr(self, "last_eta", 0) @@ -306,8 +315,9 @@ class ImportFile: ) self.column_to_field_map = self.template_options.column_to_field_map self.import_type = import_type + self.warnings = [] - self.file_doc = self.file_path = None + self.file_doc = self.file_path = self.google_sheets_url = None if isinstance(file, frappe.string_types): if frappe.db.exists("File", {"file_url": file}): self.file_doc = frappe.get_doc("File", {"file_url": file}) @@ -462,38 +472,46 @@ class ImportFile: parent_doc[table_df.fieldname].append(child_doc) doc = parent_doc - # check if there is atleast one row for mandatory table fields - meta = frappe.get_meta(self.doctype) - mandatory_table_fields = [ - df - for df in meta.fields - if df.fieldtype in table_fieldtypes - and df.reqd - and len(doc.get(df.fieldname, [])) == 0 - ] - if len(mandatory_table_fields) == 1: - self.warnings.append( - { - "row": first_row.row_number, - "message": _("There should be atleast one row for {0} table").format( - mandatory_table_fields[0].label - ), - } - ) - elif mandatory_table_fields: - fields_string = ", ".join([df.label for df in mandatory_table_fields]) - message = _("There should be atleast one row for the following tables: {0}").format( - fields_string - ) - self.warnings.append({"row": first_row.row_number, "message": message}) + + if self.import_type == INSERT: + # check if there is atleast one row for mandatory table fields + meta = frappe.get_meta(self.doctype) + mandatory_table_fields = [ + df + for df in meta.fields + if df.fieldtype in table_fieldtypes + and df.reqd + and len(doc.get(df.fieldname, [])) == 0 + ] + if len(mandatory_table_fields) == 1: + self.warnings.append( + { + "row": first_row.row_number, + "message": _("There should be atleast one row for {0} table").format( + frappe.bold(mandatory_table_fields[0].label) + ), + } + ) + elif mandatory_table_fields: + fields_string = ", ".join([df.label for df in mandatory_table_fields]) + message = _("There should be atleast one row for the following tables: {0}").format( + fields_string + ) + self.warnings.append({"row": first_row.row_number, "message": message}) return doc, rows, data[len(rows) :] def get_warnings(self): warnings = [] + + # ImportFile warnings + warnings += self.warnings + + # Column warnings for col in self.header.columns: warnings += col.warnings + # Row warnings for row in self.data: warnings += row.warnings @@ -607,7 +625,7 @@ class Row: self.warnings.append( { "row": self.row_number, - "field": df.as_dict(convert_dates_to_str=True), + "field": df_as_json(df), "message": msg, } ) @@ -622,7 +640,7 @@ class Row: self.warnings.append( { "row": self.row_number, - "field": df.as_dict(convert_dates_to_str=True), + "field": df_as_json(df), "message": msg, } ) @@ -635,7 +653,7 @@ class Row: { "row": self.row_number, "col": col.column_number, - "field": df.as_dict(convert_dates_to_str=True), + "field": df_as_json(df), "message": _("Value {0} must in {1} format").format( frappe.bold(value), frappe.bold(get_user_format(col.date_format)) ), @@ -646,7 +664,7 @@ class Row: return value def link_exists(self, value, df): - key = df.options + "::" + value + key = df.options + "::" + cstr(value) if Row.link_values_exist_map.get(key) is None: Row.link_values_exist_map[key] = frappe.db.exists(df.options, value) return Row.link_values_exist_map.get(key) @@ -755,19 +773,21 @@ class Row: class Header(Row): - def __init__(self, index, row, doctype, raw_data, column_to_field_map): + def __init__(self, index, row, doctype, raw_data, column_to_field_map=None): self.index = index self.row_number = index + 1 self.data = row self.doctype = doctype + column_to_field_map = column_to_field_map or frappe._dict() self.seen = [] self.columns = [] for j, header in enumerate(row): column_values = [get_item_at_index(r, j) for r in raw_data] + map_to_field = column_to_field_map.get(str(j)) column = Column( - j, header, self.doctype, column_values, column_to_field_map.get(header), self.seen + j, header, self.doctype, column_values, map_to_field, self.seen ) self.seen.append(header) self.columns.append(column) @@ -824,7 +844,7 @@ class Column: self.meta = frappe.get_meta(doctype) self.parse() - self.parse_date_format() + self.validate_values() def parse(self): header_title = self.header_title @@ -897,10 +917,6 @@ class Column: self.df = df self.skip_import = skip_import - def parse_date_format(self): - if self.df and self.df.fieldtype in ("Date", "Time", "Datetime"): - self.date_format = self.guess_date_format_for_column() - def guess_date_format_for_column(self): """ Guesses date format for a column by parsing all the values in the column, getting the date format and then returning the one which has the maximum frequency @@ -935,6 +951,26 @@ class Column: return max_occurred_date_format + def validate_values(self): + if not self.df: + return + + if self.df.fieldtype == 'Link': + # find all values that dont exist + values = list(set([v for v in self.column_values[1:] if v])) + exists = [d.name for d in frappe.db.get_all(self.df.options, filters={'name': ('in', values)})] + not_exists = list(set(values) - set(exists)) + if not_exists: + missing_values = ', '.join(not_exists) + self.warnings.append({ + 'col': self.column_number, + 'message': "The following values do not exist for {}: {}".format(self.df.options, missing_values), + 'type': 'warning' + }) + elif self.df.fieldtype in ("Date", "Time", "Datetime"): + # guess date format + self.date_format = self.guess_date_format_for_column() + def as_dict(self): d = frappe._dict() d.index = self.index @@ -944,6 +980,9 @@ class Column: d.map_to_field = self.map_to_field d.date_format = self.date_format d.df = self.df + if hasattr(self.df, 'is_child_table_field'): + d.is_child_table_field = self.df.is_child_table_field + d.child_table_df = self.df.child_table_df d.skip_import = self.skip_import d.warnings = self.warnings return d @@ -1113,3 +1152,13 @@ def get_user_format(date_format): .replace("%m", "mm") .replace("%d", "dd") ) + +def df_as_json(df): + return { + 'fieldname': df.fieldname, + 'fieldtype': df.fieldtype, + 'label': df.label, + 'options': df.options, + 'parent': df.parent, + 'default': df.default + } diff --git a/frappe/desk/doctype/dashboard_chart/dashboard_chart.py b/frappe/desk/doctype/dashboard_chart/dashboard_chart.py index a5c5504db2..4ad6943e0b 100644 --- a/frappe/desk/doctype/dashboard_chart/dashboard_chart.py +++ b/frappe/desk/doctype/dashboard_chart/dashboard_chart.py @@ -260,9 +260,7 @@ def get_result(data, timegrain, from_date, to_date): start_date = getdate(from_date) end_date = getdate(to_date) - result = [] - if timegrain == 'Daily': - result.append([start_date, 0.0]) + result = [[start_date, 0.0]] while start_date < end_date: next_date = get_next_expected_date(start_date, timegrain) @@ -280,11 +278,8 @@ def get_result(data, timegrain, from_date, to_date): def get_next_expected_date(date, timegrain): next_date = None - if timegrain=='Daily': - next_date = add_to_date(date, days=1) - else: - # given date is always assumed to be the period ending date - next_date = get_period_ending(add_to_date(date, days=1), timegrain) + # given date is always assumed to be the period ending date + next_date = get_period_ending(add_to_date(date, days=1), timegrain) return getdate(next_date) def get_period_ending(date, timegrain): diff --git a/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py b/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py index 1a300e471a..5e39998e62 100644 --- a/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py +++ b/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py @@ -4,13 +4,12 @@ from __future__ import unicode_literals import unittest, frappe -from frappe.utils import getdate, formatdate +from frappe.utils import getdate, formatdate, get_last_day from frappe.desk.doctype.dashboard_chart.dashboard_chart import (get, get_period_ending) from datetime import datetime from dateutil.relativedelta import relativedelta -import calendar class TestDashboardChart(unittest.TestCase): def test_period_ending(self): @@ -53,16 +52,18 @@ class TestDashboardChart(unittest.TestCase): cur_date = datetime.now() - relativedelta(years=1) - result = get(chart_name ='Test Dashboard Chart', refresh = 1) - for idx in range(13): - month = datetime(int(cur_date.year), int(cur_date.strftime('%m')), int(calendar.monthrange(cur_date.year, cur_date.month)[1])) + result = get(chart_name='Test Dashboard Chart', refresh=1) + self.assertEqual(result.get('labels')[0], formatdate(cur_date.strftime('%Y-%m-%d'))) + + if formatdate(cur_date.strftime('%Y-%m-%d')) == formatdate(get_last_day(cur_date).strftime('%Y-%m-%d')): + cur_date += relativedelta(months=1) + + for idx in range(1, 13): + month = get_last_day(cur_date) month = formatdate(month.strftime('%Y-%m-%d')) self.assertEqual(result.get('labels')[idx], month) cur_date += relativedelta(months=1) - # self.assertEqual(result.get('datasets')[0].get('values')[:-1], - # [44, 28, 8, 11, 2, 6, 18, 6, 4, 5, 15, 13]) - frappe.db.rollback() def test_empty_dashboard_chart(self): @@ -79,15 +80,20 @@ class TestDashboardChart(unittest.TestCase): based_on = 'creation', timespan = 'Last Year', time_interval = 'Monthly', - filters_json = '{}', + filters_json = '[]', timeseries = 1 )).insert() cur_date = datetime.now() - relativedelta(years=1) - result = get(chart_name ='Test Empty Dashboard Chart', refresh = 1) - for idx in range(13): - month = datetime(int(cur_date.year), int(cur_date.strftime('%m')), int(calendar.monthrange(cur_date.year, cur_date.month)[1])) + result = get(chart_name ='Test Empty Dashboard Chart', refresh=1) + self.assertEqual(result.get('labels')[0], formatdate(cur_date.strftime('%Y-%m-%d'))) + + if formatdate(cur_date.strftime('%Y-%m-%d')) == formatdate(get_last_day(cur_date).strftime('%Y-%m-%d')): + cur_date += relativedelta(months=1) + + for idx in range(1, 13): + month = get_last_day(cur_date) month = formatdate(month.strftime('%Y-%m-%d')) self.assertEqual(result.get('labels')[idx], month) cur_date += relativedelta(months=1) @@ -111,15 +117,20 @@ class TestDashboardChart(unittest.TestCase): based_on = 'creation', timespan = 'Last Year', time_interval = 'Monthly', - filters_json = '{}', + filters_json = '[]', timeseries = 1 )).insert() cur_date = datetime.now() - relativedelta(years=1) result = get(chart_name ='Test Empty Dashboard Chart 2', refresh = 1) - for idx in range(13): - month = datetime(int(cur_date.year), int(cur_date.strftime('%m')), int(calendar.monthrange(cur_date.year, cur_date.month)[1])) + self.assertEqual(result.get('labels')[0], formatdate(cur_date.strftime('%Y-%m-%d'))) + + if formatdate(cur_date.strftime('%Y-%m-%d')) == formatdate(get_last_day(cur_date).strftime('%Y-%m-%d')): + cur_date += relativedelta(months=1) + + for idx in range(1, 13): + month = get_last_day(cur_date) month = formatdate(month.strftime('%Y-%m-%d')) self.assertEqual(result.get('labels')[idx], month) cur_date += relativedelta(months=1) @@ -141,7 +152,7 @@ class TestDashboardChart(unittest.TestCase): chart_type = 'Group By', document_type = 'ToDo', group_by_based_on = 'status', - filters_json = '{}', + filters_json = '[]', )).insert() result = get(chart_name ='Test Group By Dashboard Chart', refresh = 1) @@ -168,7 +179,7 @@ class TestDashboardChart(unittest.TestCase): time_interval = 'Daily', from_date = datetime(2019, 1, 6), to_date = datetime(2019, 1, 11), - filters_json = '{}', + filters_json = '[]', timeseries = 1 )).insert() @@ -200,22 +211,24 @@ class TestDashboardChart(unittest.TestCase): time_interval = 'Weekly', from_date = datetime(2018, 12, 30), to_date = datetime(2019, 1, 15), - filters_json = '{}', + filters_json = '[]', timeseries = 1 )).insert() result = get(chart_name ='Test Weekly Dashboard Chart', refresh = 1) - self.assertEqual(result.get('datasets')[0].get('values'), [200.0, 800.0, 0.0]) - self.assertEqual(result.get('labels'), [formatdate('2019-01-06'), formatdate('2019-01-13'), formatdate('2019-01-20')]) + self.assertEqual(result.get('datasets')[0].get('values'), [50.0, 300.0, 800.0, 0.0]) + self.assertEqual(result.get('labels'), [formatdate('2018-12-30'), formatdate('2019-01-06'), formatdate('2019-01-13'), formatdate('2019-01-20')]) frappe.db.rollback() def insert_test_records(): - create_new_communication(datetime(2019, 1, 10), 100) + create_new_communication(datetime(2018, 12, 30), 50) + create_new_communication(datetime(2019, 1, 4), 100) create_new_communication(datetime(2019, 1, 6), 200) create_new_communication(datetime(2019, 1, 7), 400) create_new_communication(datetime(2019, 1, 8), 300) + create_new_communication(datetime(2019, 1, 10), 100) def create_new_communication(date, rating): communication = { diff --git a/frappe/email/queue.py b/frappe/email/queue.py index ce512de276..8bffc108b9 100755 --- a/frappe/email/queue.py +++ b/frappe/email/queue.py @@ -347,7 +347,7 @@ def flush(from_test=False): if not smtpserver: smtpserver = SMTPServer() smtpserver_dict[email.sender] = smtpserver - + if from_test: send_one(email.name, smtpserver, auto_commit) else: @@ -390,12 +390,12 @@ def send_one(email, smtpserver=None, auto_commit=True, now=False): where name=%s for update''', email, as_dict=True) - + if len(email): email = email[0] else: return - + recipients_list = frappe.db.sql('''select name, recipient, status from `tabEmail Queue Recipient` where parent=%s''', email.name, as_dict=1) @@ -417,6 +417,8 @@ def send_one(email, smtpserver=None, auto_commit=True, now=False): if email.communication: frappe.get_doc('Communication', email.communication).set_delivery_status(commit=auto_commit) + email_sent_to_any_recipient = None + try: message = None diff --git a/frappe/hooks.py b/frappe/hooks.py index c207e61de5..1f209f00a2 100644 --- a/frappe/hooks.py +++ b/frappe/hooks.py @@ -12,7 +12,7 @@ source_link = "https://github.com/frappe/frappe" app_license = "MIT" app_logo_url = '/assets/frappe/images/frappe-framework-logo.png' -develop_version = '12.x.x-develop' +develop_version = '13.x.x-develop' app_email = "info@frappe.io" diff --git a/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.py b/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.py index 6cfd3646b2..c8b007ba7b 100755 --- a/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.py +++ b/frappe/integrations/doctype/s3_backup_settings/s3_backup_settings.py @@ -19,6 +19,9 @@ from botocore.exceptions import ClientError class S3BackupSettings(Document): def validate(self): + if not self.enabled: + return + if not self.endpoint_url: self.endpoint_url = 'https://s3.amazonaws.com' conn = boto3.client( diff --git a/frappe/patches.txt b/frappe/patches.txt index a03d31918b..f883d2a7bb 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -289,4 +289,4 @@ execute:frappe.delete_doc("DocType", "Onboarding Slide Field") execute:frappe.delete_doc("DocType", "Onboarding Slide Help Link") frappe.patches.v13_0.update_date_filters_in_user_settings frappe.patches.v13_0.update_duration_options -frappe.patches.v13_0.replace_old_data_import +frappe.patches.v13_0.replace_old_data_import # 2020-06-24 diff --git a/frappe/patches/v13_0/replace_old_data_import.py b/frappe/patches/v13_0/replace_old_data_import.py index 1c00ae5f34..f3eed6253c 100644 --- a/frappe/patches/v13_0/replace_old_data_import.py +++ b/frappe/patches/v13_0/replace_old_data_import.py @@ -6,9 +6,11 @@ import frappe def execute(): + if not frappe.db.exists("DocType", "Data Import Beta"): + return + + frappe.db.sql("DROP TABLE IF EXISTS `tabData Import Legacy`") frappe.rename_doc('DocType', 'Data Import', 'Data Import Legacy') frappe.db.commit() frappe.db.sql("DROP TABLE IF EXISTS `tabData Import`") - frappe.reload_doc("core", "doctype", "data_import") - frappe.get_doc("DocType", "Data Import").on_update() - frappe.delete_doc_if_exists("DocType", "Data Import Beta") + frappe.rename_doc('DocType', 'Data Import Beta', 'Data Import') diff --git a/frappe/public/js/frappe/data_import/data_exporter.js b/frappe/public/js/frappe/data_import/data_exporter.js index 735237189d..f6af338235 100644 --- a/frappe/public/js/frappe/data_import/data_exporter.js +++ b/frappe/public/js/frappe/data_import/data_exporter.js @@ -13,36 +13,6 @@ frappe.data_import.DataExporter = class DataExporter { this.dialog = new frappe.ui.Dialog({ title: __('Export Data'), fields: [ - { - fieldtype: 'Select', - fieldname: 'exporting_for', - label: __('Exporting For'), - options: [ - { - label: __('Insert New Records'), - value: 'Insert New Records' - }, - { - label: __('Update Existing Records'), - value: 'Update Existing Records' - } - ], - change: () => { - let exporting_for = this.dialog.get_value('exporting_for'); - this.dialog.set_value( - 'export_records', - exporting_for === 'Insert New Records' ? 'blank_template' : 'all' - ); - - // Force ID field to be exported when updating existing records - let id_field = this.dialog.get_field(this.doctype).options[0]; - if (id_field.value === 'name' && id_field.$checkbox) { - id_field.$checkbox - .find('input') - .prop('disabled', exporting_for === 'Update Existing Records'); - } - } - }, { fieldtype: 'Select', fieldname: 'export_records', @@ -65,7 +35,7 @@ frappe.data_import.DataExporter = class DataExporter { value: 'blank_template' } ], - default: 'blank_template', + default: this.exporting_for === 'Insert New Records' ? 'blank_template' : 'all', change: () => { this.update_record_count_message(); } @@ -119,10 +89,6 @@ frappe.data_import.DataExporter = class DataExporter { on_page_show: () => this.select_mandatory() }); - if (this.exporting_for) { - this.dialog.set_value('exporting_for', this.exporting_for); - } - this.make_filter_area(); this.make_select_all_buttons(); this.update_record_count_message(); @@ -172,15 +138,17 @@ frappe.data_import.DataExporter = class DataExporter { } make_select_all_buttons() { + let for_insert = this.exporting_for === 'Insert New Records'; + let section_title = for_insert ? __('Select Fields To Insert') : __('Select Fields To Update'); let $select_all_buttons = $(`
-
${__('Select fields to export')}
+
${section_title}
- + `: ''} @@ -285,11 +253,9 @@ frappe.data_import.DataExporter = class DataExporter { } get_filters() { - return this.filter_group.get_filters().reduce((acc, filter) => { - return Object.assign(acc, { - [filter[1]]: [filter[2], filter[3]] - }); - }, {}); + return this.filter_group.get_filters().map(filter => { + return filter.slice(0, 4); + }); } get_multicheck_options(doctype, child_fieldname = null) { @@ -308,6 +274,9 @@ frappe.data_import.DataExporter = class DataExporter { ? this.column_map[child_fieldname] : this.column_map[doctype]; + let is_field_mandatory = df => (df.fieldname === 'name' && !child_fieldname) + || (df.reqd && this.exporting_for == 'Insert New Records'); + return fields .filter(df => { if (autoname_field && df.fieldname === autoname_field.fieldname) { @@ -323,7 +292,7 @@ frappe.data_import.DataExporter = class DataExporter { return { label, value: df.fieldname, - danger: df.reqd, + danger: is_field_mandatory(df), checked: false, description: `${df.fieldname} ${df.reqd ? __('(Mandatory)') : ''}` }; diff --git a/frappe/public/js/frappe/data_import/import_preview.js b/frappe/public/js/frappe/data_import/import_preview.js index 7cf8431456..4edcb87aeb 100644 --- a/frappe/public/js/frappe/data_import/import_preview.js +++ b/frappe/public/js/frappe/data_import/import_preview.js @@ -245,11 +245,12 @@ frappe.data_import.ImportPreview = class ImportPreview { let fieldname; if (!df) { fieldname = null; + } else if (col.map_to_field) { + fieldname = col.map_to_field; + } else if (col.is_child_table_field) { + fieldname = `${col.child_table_df.fieldname}.${df.fieldname}`; } else { - fieldname = - df.parent === this.doctype - ? df.fieldname - : `${df.parent}:${df.fieldname}`; + fieldname = df.fieldname; } return [ { @@ -272,7 +273,7 @@ frappe.data_import.ImportPreview = class ImportPreview { label: __("Don't Import"), value: "Don't Import" } - ].concat(column_picker_fields.get_fields_as_options()), + ].concat(get_fields_as_options(this.doctype, column_picker_fields)), default: fieldname || "Don't Import", change() { changed.push(i); @@ -328,3 +329,29 @@ frappe.data_import.ImportPreview = class ImportPreview { }); } }; + +function get_fields_as_options(doctype, column_map) { + let keys = [doctype]; + frappe.meta.get_table_fields(doctype).forEach(df => { + keys.push(df.fieldname); + }); + // flatten array + return [].concat( + ...keys.map(key => { + return column_map[key].map(df => { + let label = df.label; + let value = df.fieldname; + if (doctype !== key) { + let table_field = frappe.meta.get_docfield(doctype, key); + label = `${df.label} (${table_field.label})`; + value = `${table_field.fieldname}.${df.fieldname}`; + } + return { + label, + value, + description: value + }; + }); + }) + ); +} \ No newline at end of file diff --git a/frappe/public/js/frappe/db.js b/frappe/public/js/frappe/db.js index 1b6fb0e438..cf716c67e5 100644 --- a/frappe/public/js/frappe/db.js +++ b/frappe/public/js/frappe/db.js @@ -91,12 +91,26 @@ frappe.db = { }); }, count: function(doctype, args={}) { - return new Promise(resolve => { - frappe.call({ - method: 'frappe.client.get_count', - type: 'GET', - args: Object.assign(args, { doctype }) - }).then(r => resolve(r.message)); + let filters = args.filters || {}; + const with_child_table_filter = Array.isArray(filters) && filters.some(filter => { + return filter[0] !== doctype; + }); + + const fields = [ + // cannot break this line as it adds extra \n's and \t's which breaks the query + `count(${with_child_table_filter ? 'distinct': ''} ${frappe.model.get_full_column_name('name', doctype)}) AS total_count` + ]; + + return frappe.call({ + type: 'GET', + method: 'frappe.desk.reportview.get', + args: { + doctype, + filters, + fields, + } + }).then(r => { + return r.message.values[0][0]; }); }, get_link_options(doctype, txt = '', filters={}) { diff --git a/frappe/public/js/frappe/list/list_view.js b/frappe/public/js/frappe/list/list_view.js index 9e1ba1b9bd..8da73b0dec 100644 --- a/frappe/public/js/frappe/list/list_view.js +++ b/frappe/public/js/frappe/list/list_view.js @@ -760,26 +760,10 @@ frappe.views.ListView = class ListView extends frappe.views.BaseList { let current_count = this.data.length; let count_without_children = this.data.uniqBy(d => d.name).length; - const filters = this.get_filters_for_args(); - const with_child_table_filter = filters.some(filter => { - return filter[0] !== this.doctype; - }); - - const fields = [ - // cannot break this line as it adds extra \n's and \t's which breaks the query - `count(${with_child_table_filter ? 'distinct': ''}${frappe.model.get_full_column_name('name', this.doctype)}) AS total_count` - ]; - - return frappe.call({ - type: 'GET', - method: this.method, - args: { - doctype: this.doctype, - filters, - fields, - } - }).then(r => { - this.total_count = r.message.values[0][0] || current_count; + return frappe.db.count(this.doctype, { + filters: this.get_filters_for_args() + }).then(total_count => { + this.total_count = total_count || current_count; let str = __('{0} of {1}', [current_count, this.total_count]); if (count_without_children !== current_count) { str = __('{0} of {1} ({2} rows with children)', [count_without_children, this.total_count, current_count]); diff --git a/frappe/utils/csvutils.py b/frappe/utils/csvutils.py index 1d326bd1c5..f73f80582a 100644 --- a/frappe/utils/csvutils.py +++ b/frappe/utils/csvutils.py @@ -175,12 +175,18 @@ def getlink(doctype, name): return '%(name)s' % locals() def get_csv_content_from_google_sheets(url): + # https://docs.google.com/spreadsheets/d/{sheetid}}/edit#gid={gid} validate_google_sheets_url(url) - + # get gid, defaults to first sheet + if "gid=" in url: + gid = url.rsplit('gid=', 1)[1] + else: + gid = 0 # remove /edit path url = url.rsplit('/edit', 1)[0] - # add /export path, defaults to first sheet - url = url + '/export?format=csv&gid=0' + # add /export path, + url = url + '/export?format=csv&gid={0}'.format(gid) + headers = { 'Accept': 'text/csv' }