fix: Use db insert for logs
This commit is contained in:
Deepesh Garg
parent
cafd513c49
commit
2ca5c390d0
5 changed files with 229 additions and 250 deletions
|
|
@ -278,14 +278,15 @@ frappe.ui.form.on('Data Import', {
|
|||
}
|
||||
})
|
||||
.then(r => {
|
||||
let preview_data = r.message.preview_data;
|
||||
let import_log = r.message.import_log;
|
||||
frm.events.show_import_preview(frm, preview_data, import_log);
|
||||
let preview_data = r.message;
|
||||
frm.events.show_import_preview(frm, preview_data);
|
||||
frm.events.show_import_warnings(frm, preview_data);
|
||||
});
|
||||
},
|
||||
|
||||
show_import_preview(frm, preview_data, import_log) {
|
||||
show_import_preview(frm, preview_data) {
|
||||
let import_log = preview_data.import_log;
|
||||
|
||||
if (
|
||||
frm.import_preview &&
|
||||
frm.import_preview.doctype === frm.doc.reference_doctype
|
||||
|
|
|
|||
|
|
@ -1,227 +1,197 @@
|
|||
{
|
||||
"actions": [],
|
||||
"autoname": "format:{reference_doctype} Import on {creation}",
|
||||
"beta": 1,
|
||||
"creation": "2019-08-04 14:16:08.318714",
|
||||
"doctype": "DocType",
|
||||
"editable_grid": 1,
|
||||
"engine": "InnoDB",
|
||||
"field_order": [
|
||||
"reference_doctype",
|
||||
"import_type",
|
||||
"download_template",
|
||||
"import_file",
|
||||
"html_5",
|
||||
"google_sheets_url",
|
||||
"refresh_google_sheet",
|
||||
"column_break_5",
|
||||
"status",
|
||||
"submit_after_import",
|
||||
"mute_emails",
|
||||
"template_options",
|
||||
"import_warnings_section",
|
||||
"template_warnings",
|
||||
"import_warnings",
|
||||
"section_import_preview",
|
||||
"import_preview",
|
||||
"import_log_section",
|
||||
"show_failed_logs",
|
||||
"import_log_preview"
|
||||
],
|
||||
"fields": [
|
||||
{
|
||||
"fieldname": "reference_doctype",
|
||||
"fieldtype": "Link",
|
||||
"in_list_view": 1,
|
||||
"label": "Document Type",
|
||||
"options": "DocType",
|
||||
"reqd": 1,
|
||||
"set_only_once": 1,
|
||||
"show_days": 1,
|
||||
"show_seconds": 1
|
||||
},
|
||||
{
|
||||
"fieldname": "import_type",
|
||||
"fieldtype": "Select",
|
||||
"in_list_view": 1,
|
||||
"label": "Import Type",
|
||||
"options": "\nInsert New Records\nUpdate Existing Records",
|
||||
"reqd": 1,
|
||||
"set_only_once": 1,
|
||||
"show_days": 1,
|
||||
"show_seconds": 1
|
||||
},
|
||||
{
|
||||
"depends_on": "eval:!doc.__islocal",
|
||||
"fieldname": "import_file",
|
||||
"fieldtype": "Attach",
|
||||
"in_list_view": 1,
|
||||
"label": "Import File",
|
||||
"read_only_depends_on": "eval: ['Success', 'Partial Success', 'Partially Completed'].includes(doc.status)",
|
||||
"show_days": 1,
|
||||
"show_seconds": 1
|
||||
},
|
||||
{
|
||||
"fieldname": "import_preview",
|
||||
"fieldtype": "HTML",
|
||||
"label": "Import Preview",
|
||||
"show_days": 1,
|
||||
"show_seconds": 1
|
||||
},
|
||||
{
|
||||
"fieldname": "section_import_preview",
|
||||
"fieldtype": "Section Break",
|
||||
"label": "Preview",
|
||||
"show_days": 1,
|
||||
"show_seconds": 1
|
||||
},
|
||||
{
|
||||
"fieldname": "column_break_5",
|
||||
"fieldtype": "Column Break",
|
||||
"show_days": 1,
|
||||
"show_seconds": 1
|
||||
},
|
||||
{
|
||||
"fieldname": "template_options",
|
||||
"fieldtype": "Code",
|
||||
"hidden": 1,
|
||||
"label": "Template Options",
|
||||
"options": "JSON",
|
||||
"read_only": 1,
|
||||
"show_days": 1,
|
||||
"show_seconds": 1
|
||||
},
|
||||
{
|
||||
"fieldname": "import_log_section",
|
||||
"fieldtype": "Section Break",
|
||||
"label": "Import Log",
|
||||
"show_days": 1,
|
||||
"show_seconds": 1
|
||||
},
|
||||
{
|
||||
"fieldname": "import_log_preview",
|
||||
"fieldtype": "HTML",
|
||||
"label": "Import Log Preview",
|
||||
"show_days": 1,
|
||||
"show_seconds": 1
|
||||
},
|
||||
{
|
||||
"default": "Pending",
|
||||
"fieldname": "status",
|
||||
"fieldtype": "Select",
|
||||
"hidden": 1,
|
||||
"label": "Status",
|
||||
"options": "Pending\nSuccess\nPartial Success\nError",
|
||||
"read_only": 1,
|
||||
"show_days": 1,
|
||||
"show_seconds": 1
|
||||
},
|
||||
{
|
||||
"fieldname": "template_warnings",
|
||||
"fieldtype": "Code",
|
||||
"hidden": 1,
|
||||
"label": "Template Warnings",
|
||||
"options": "JSON",
|
||||
"show_days": 1,
|
||||
"show_seconds": 1
|
||||
},
|
||||
{
|
||||
"default": "0",
|
||||
"fieldname": "submit_after_import",
|
||||
"fieldtype": "Check",
|
||||
"label": "Submit After Import",
|
||||
"set_only_once": 1,
|
||||
"show_days": 1,
|
||||
"show_seconds": 1
|
||||
},
|
||||
{
|
||||
"fieldname": "import_warnings_section",
|
||||
"fieldtype": "Section Break",
|
||||
"label": "Import File Errors and Warnings",
|
||||
"show_days": 1,
|
||||
"show_seconds": 1
|
||||
},
|
||||
{
|
||||
"fieldname": "import_warnings",
|
||||
"fieldtype": "HTML",
|
||||
"label": "Import Warnings",
|
||||
"show_days": 1,
|
||||
"show_seconds": 1
|
||||
},
|
||||
{
|
||||
"depends_on": "eval:!doc.__islocal",
|
||||
"fieldname": "download_template",
|
||||
"fieldtype": "Button",
|
||||
"label": "Download Template",
|
||||
"show_days": 1,
|
||||
"show_seconds": 1
|
||||
},
|
||||
{
|
||||
"default": "1",
|
||||
"fieldname": "mute_emails",
|
||||
"fieldtype": "Check",
|
||||
"label": "Don't Send Emails",
|
||||
"set_only_once": 1,
|
||||
"show_days": 1,
|
||||
"show_seconds": 1
|
||||
},
|
||||
{
|
||||
"default": "0",
|
||||
"fieldname": "show_failed_logs",
|
||||
"fieldtype": "Check",
|
||||
"label": "Show Failed Logs",
|
||||
"show_days": 1,
|
||||
"show_seconds": 1
|
||||
},
|
||||
{
|
||||
"depends_on": "eval:!doc.__islocal && !doc.import_file",
|
||||
"fieldname": "html_5",
|
||||
"fieldtype": "HTML",
|
||||
"options": "<h5 class=\"text-muted uppercase\">Or</h5>",
|
||||
"show_days": 1,
|
||||
"show_seconds": 1
|
||||
},
|
||||
{
|
||||
"depends_on": "eval:!doc.__islocal && !doc.import_file\n",
|
||||
"description": "Must be a publicly accessible Google Sheets URL",
|
||||
"fieldname": "google_sheets_url",
|
||||
"fieldtype": "Data",
|
||||
"label": "Import from Google Sheets",
|
||||
"read_only_depends_on": "eval: ['Success', 'Partial Success'].includes(doc.status)",
|
||||
"show_days": 1,
|
||||
"show_seconds": 1
|
||||
},
|
||||
{
|
||||
"depends_on": "eval:doc.google_sheets_url && !doc.__unsaved && ['Success', 'Partial Success'].includes(doc.status)",
|
||||
"fieldname": "refresh_google_sheet",
|
||||
"fieldtype": "Button",
|
||||
"label": "Refresh Google Sheet",
|
||||
"show_days": 1,
|
||||
"show_seconds": 1
|
||||
}
|
||||
],
|
||||
"hide_toolbar": 1,
|
||||
"links": [],
|
||||
"modified": "2022-01-31 10:50:13.015426",
|
||||
"modified_by": "Administrator",
|
||||
"module": "Core",
|
||||
"name": "Data Import",
|
||||
"owner": "Administrator",
|
||||
"permissions": [
|
||||
{
|
||||
"create": 1,
|
||||
"delete": 1,
|
||||
"email": 1,
|
||||
"export": 1,
|
||||
"print": 1,
|
||||
"read": 1,
|
||||
"report": 1,
|
||||
"role": "System Manager",
|
||||
"share": 1,
|
||||
"write": 1
|
||||
}
|
||||
],
|
||||
"sort_field": "modified",
|
||||
"sort_order": "DESC",
|
||||
"track_changes": 1
|
||||
"actions": [],
|
||||
"autoname": "format:{reference_doctype} Import on {creation}",
|
||||
"beta": 1,
|
||||
"creation": "2019-08-04 14:16:08.318714",
|
||||
"doctype": "DocType",
|
||||
"editable_grid": 1,
|
||||
"engine": "InnoDB",
|
||||
"field_order": [
|
||||
"reference_doctype",
|
||||
"import_type",
|
||||
"download_template",
|
||||
"import_file",
|
||||
"payload_count",
|
||||
"html_5",
|
||||
"google_sheets_url",
|
||||
"refresh_google_sheet",
|
||||
"column_break_5",
|
||||
"status",
|
||||
"submit_after_import",
|
||||
"mute_emails",
|
||||
"template_options",
|
||||
"import_warnings_section",
|
||||
"template_warnings",
|
||||
"import_warnings",
|
||||
"section_import_preview",
|
||||
"import_preview",
|
||||
"import_log_section",
|
||||
"show_failed_logs",
|
||||
"import_log_preview"
|
||||
],
|
||||
"fields": [
|
||||
{
|
||||
"fieldname": "reference_doctype",
|
||||
"fieldtype": "Link",
|
||||
"in_list_view": 1,
|
||||
"label": "Document Type",
|
||||
"options": "DocType",
|
||||
"reqd": 1,
|
||||
"set_only_once": 1
|
||||
},
|
||||
{
|
||||
"fieldname": "import_type",
|
||||
"fieldtype": "Select",
|
||||
"in_list_view": 1,
|
||||
"label": "Import Type",
|
||||
"options": "\nInsert New Records\nUpdate Existing Records",
|
||||
"reqd": 1,
|
||||
"set_only_once": 1
|
||||
},
|
||||
{
|
||||
"depends_on": "eval:!doc.__islocal",
|
||||
"fieldname": "import_file",
|
||||
"fieldtype": "Attach",
|
||||
"in_list_view": 1,
|
||||
"label": "Import File",
|
||||
"read_only_depends_on": "eval: ['Success', 'Partial Success'].includes(doc.status)"
|
||||
},
|
||||
{
|
||||
"fieldname": "import_preview",
|
||||
"fieldtype": "HTML",
|
||||
"label": "Import Preview"
|
||||
},
|
||||
{
|
||||
"fieldname": "section_import_preview",
|
||||
"fieldtype": "Section Break",
|
||||
"label": "Preview"
|
||||
},
|
||||
{
|
||||
"fieldname": "column_break_5",
|
||||
"fieldtype": "Column Break"
|
||||
},
|
||||
{
|
||||
"fieldname": "template_options",
|
||||
"fieldtype": "Code",
|
||||
"hidden": 1,
|
||||
"label": "Template Options",
|
||||
"options": "JSON",
|
||||
"read_only": 1
|
||||
},
|
||||
{
|
||||
"fieldname": "import_log_section",
|
||||
"fieldtype": "Section Break",
|
||||
"label": "Import Log"
|
||||
},
|
||||
{
|
||||
"fieldname": "import_log_preview",
|
||||
"fieldtype": "HTML",
|
||||
"label": "Import Log Preview"
|
||||
},
|
||||
{
|
||||
"default": "Pending",
|
||||
"fieldname": "status",
|
||||
"fieldtype": "Select",
|
||||
"hidden": 1,
|
||||
"label": "Status",
|
||||
"options": "Pending\nSuccess\nPartial Success\nError",
|
||||
"read_only": 1
|
||||
},
|
||||
{
|
||||
"fieldname": "template_warnings",
|
||||
"fieldtype": "Code",
|
||||
"hidden": 1,
|
||||
"label": "Template Warnings",
|
||||
"options": "JSON"
|
||||
},
|
||||
{
|
||||
"default": "0",
|
||||
"fieldname": "submit_after_import",
|
||||
"fieldtype": "Check",
|
||||
"label": "Submit After Import",
|
||||
"set_only_once": 1
|
||||
},
|
||||
{
|
||||
"fieldname": "import_warnings_section",
|
||||
"fieldtype": "Section Break",
|
||||
"label": "Import File Errors and Warnings"
|
||||
},
|
||||
{
|
||||
"fieldname": "import_warnings",
|
||||
"fieldtype": "HTML",
|
||||
"label": "Import Warnings"
|
||||
},
|
||||
{
|
||||
"depends_on": "eval:!doc.__islocal",
|
||||
"fieldname": "download_template",
|
||||
"fieldtype": "Button",
|
||||
"label": "Download Template"
|
||||
},
|
||||
{
|
||||
"default": "1",
|
||||
"fieldname": "mute_emails",
|
||||
"fieldtype": "Check",
|
||||
"label": "Don't Send Emails",
|
||||
"set_only_once": 1
|
||||
},
|
||||
{
|
||||
"default": "0",
|
||||
"fieldname": "show_failed_logs",
|
||||
"fieldtype": "Check",
|
||||
"label": "Show Failed Logs"
|
||||
},
|
||||
{
|
||||
"depends_on": "eval:!doc.__islocal && !doc.import_file",
|
||||
"fieldname": "html_5",
|
||||
"fieldtype": "HTML",
|
||||
"options": "<h5 class=\"text-muted uppercase\">Or</h5>"
|
||||
},
|
||||
{
|
||||
"depends_on": "eval:!doc.__islocal && !doc.import_file\n",
|
||||
"description": "Must be a publicly accessible Google Sheets URL",
|
||||
"fieldname": "google_sheets_url",
|
||||
"fieldtype": "Data",
|
||||
"label": "Import from Google Sheets",
|
||||
"read_only_depends_on": "eval: ['Success', 'Partial Success'].includes(doc.status)"
|
||||
},
|
||||
{
|
||||
"depends_on": "eval:doc.google_sheets_url && !doc.__unsaved && ['Success', 'Partial Success'].includes(doc.status)",
|
||||
"fieldname": "refresh_google_sheet",
|
||||
"fieldtype": "Button",
|
||||
"label": "Refresh Google Sheet"
|
||||
},
|
||||
{
|
||||
"fieldname": "payload_count",
|
||||
"fieldtype": "Int",
|
||||
"hidden": 1,
|
||||
"label": "Payload Count",
|
||||
"read_only": 1
|
||||
}
|
||||
],
|
||||
"hide_toolbar": 1,
|
||||
"links": [],
|
||||
"modified": "2022-02-01 20:08:37.624914",
|
||||
"modified_by": "Administrator",
|
||||
"module": "Core",
|
||||
"name": "Data Import",
|
||||
"naming_rule": "Expression",
|
||||
"owner": "Administrator",
|
||||
"permissions": [
|
||||
{
|
||||
"create": 1,
|
||||
"delete": 1,
|
||||
"email": 1,
|
||||
"export": 1,
|
||||
"print": 1,
|
||||
"read": 1,
|
||||
"report": 1,
|
||||
"role": "System Manager",
|
||||
"share": 1,
|
||||
"write": 1
|
||||
}
|
||||
],
|
||||
"sort_field": "modified",
|
||||
"sort_order": "DESC",
|
||||
"states": [],
|
||||
"track_changes": 1
|
||||
}
|
||||
|
|
@ -28,6 +28,7 @@ class DataImport(Document):
|
|||
|
||||
self.validate_import_file()
|
||||
self.validate_google_sheets_url()
|
||||
self.set_payload_count()
|
||||
|
||||
def validate_import_file(self):
|
||||
if self.import_file:
|
||||
|
|
@ -39,6 +40,12 @@ class DataImport(Document):
|
|||
return
|
||||
validate_google_sheets_url(self.google_sheets_url)
|
||||
|
||||
def set_payload_count(self):
|
||||
if self.import_file:
|
||||
i = self.get_importer()
|
||||
payloads = i.import_file.get_payloads_for_import()
|
||||
self.payload_count = len(payloads)
|
||||
|
||||
@frappe.whitelist()
|
||||
def get_preview_from_template(self, import_file=None, google_sheets_url=None):
|
||||
if import_file:
|
||||
|
|
@ -90,21 +97,10 @@ class DataImport(Document):
|
|||
|
||||
@frappe.whitelist()
|
||||
def get_preview_from_template(data_import, import_file=None, google_sheets_url=None):
|
||||
preview_data = frappe.get_doc("Data Import", data_import).get_preview_from_template(
|
||||
return frappe.get_doc("Data Import", data_import).get_preview_from_template(
|
||||
import_file, google_sheets_url
|
||||
)
|
||||
|
||||
# get first 10 import log if any
|
||||
import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes"],
|
||||
filters={"data_import": data_import},
|
||||
order_by="log_index", limit=10)
|
||||
|
||||
return {
|
||||
'preview_data': preview_data,
|
||||
'import_log': import_log
|
||||
}
|
||||
|
||||
|
||||
@frappe.whitelist()
|
||||
def form_start_import(data_import):
|
||||
return frappe.get_doc("Data Import", data_import).start_import()
|
||||
|
|
@ -172,14 +168,15 @@ def get_import_status(data_import_name):
|
|||
filters={'data_import': data_import_name},
|
||||
group_by='success')
|
||||
|
||||
total_payload_count = frappe.db.get_value('Data Import', data_import_name, 'payload_count')
|
||||
|
||||
for log in logs:
|
||||
if log.get('success'):
|
||||
import_status['success'] = log.get('count')
|
||||
else:
|
||||
import_status['failed'] = log.get('count')
|
||||
|
||||
import_status['total_records'] = cint(import_status.get('success')) + \
|
||||
cint(import_status.get('failed'))
|
||||
import_status['total_records'] = total_payload_count
|
||||
|
||||
return import_status
|
||||
|
||||
|
|
|
|||
|
|
@ -47,7 +47,13 @@ class Importer:
|
|||
)
|
||||
|
||||
def get_data_for_import_preview(self):
|
||||
return self.import_file.get_data_for_import_preview()
|
||||
out = self.import_file.get_data_for_import_preview()
|
||||
|
||||
out.import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success"],
|
||||
filters={"data_import": self.data_import.name},
|
||||
order_by="log_index", limit=10)
|
||||
|
||||
return out
|
||||
|
||||
def before_import(self):
|
||||
# set user lang for translations
|
||||
|
|
@ -85,7 +91,7 @@ class Importer:
|
|||
log_index = 0
|
||||
|
||||
# Do not remove rows in case of retry after an error or pending data import
|
||||
if self.data_import.status == "Partial Success" and len(import_log) > len(payloads):
|
||||
if self.data_import.status == "Partial Success" and len(import_log) >= self.data_import.payload_count:
|
||||
# remove previous failures from import log only in case of retry after partial success
|
||||
import_log = [log for log in import_log if log.get("success")]
|
||||
|
||||
|
|
@ -93,7 +99,7 @@ class Importer:
|
|||
imported_rows = []
|
||||
for log in import_log:
|
||||
log = frappe._dict(log)
|
||||
if log.success or len(import_log) < len(payloads):
|
||||
if log.success or len(import_log) < self.data_import.payload_count:
|
||||
imported_rows += json.loads(log.row_indexes)
|
||||
|
||||
log_index = log.log_index
|
||||
|
|
@ -182,6 +188,11 @@ class Importer:
|
|||
frappe.db.commit()
|
||||
log_index += 1
|
||||
|
||||
# Logs are db inserted directly so will have to be fetched again
|
||||
import_log = frappe.db.get_all("Data Import Log", fields=["row_indexes", "success", "log_index"],
|
||||
filters={"data_import": self.data_import.name},
|
||||
order_by="log_index") or []
|
||||
|
||||
# set status
|
||||
failures = [log for log in import_log if not log.get("success")]
|
||||
if len(failures) == total_payload_count:
|
||||
|
|
@ -1217,7 +1228,7 @@ def get_select_options(df):
|
|||
return [d for d in (df.options or "").split("\n") if d]
|
||||
|
||||
def create_import_log(data_import, log_index, log_details):
|
||||
return frappe.get_doc({
|
||||
frappe.get_doc({
|
||||
'doctype': 'Data Import Log',
|
||||
'log_index': log_index,
|
||||
'success': log_details.get('success'),
|
||||
|
|
@ -1226,6 +1237,6 @@ def create_import_log(data_import, log_index, log_details):
|
|||
'docname': log_details.get('docname'),
|
||||
'messages': json.dumps(log_details.get('messages', '[]')),
|
||||
'exception': log_details.get('exception')
|
||||
}).insert(ignore_permissions=True)
|
||||
}).db_insert()
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
"creation": "2021-12-25 16:12:20.205889",
|
||||
"doctype": "DocType",
|
||||
"editable_grid": 1,
|
||||
"engine": "InnoDB",
|
||||
"engine": "MyISAM",
|
||||
"field_order": [
|
||||
"data_import",
|
||||
"row_indexes",
|
||||
|
|
@ -60,7 +60,7 @@
|
|||
"in_create": 1,
|
||||
"index_web_pages_for_search": 1,
|
||||
"links": [],
|
||||
"modified": "2021-12-27 11:19:19.646076",
|
||||
"modified": "2021-12-28 11:19:19.646076",
|
||||
"modified_by": "Administrator",
|
||||
"module": "Core",
|
||||
"name": "Data Import Log",
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue