From 2fb27a7ceb9f2aa575832b68c8562d84b3939b37 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Fri, 15 May 2020 13:23:53 +0530 Subject: [PATCH 001/485] refactor: explicitly hide amend button in toolbar --- frappe/public/js/frappe/form/toolbar.js | 29 +++++++++++++++---------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/frappe/public/js/frappe/form/toolbar.js b/frappe/public/js/frappe/form/toolbar.js index 528c874935..6f475fa9e5 100644 --- a/frappe/public/js/frappe/form/toolbar.js +++ b/frappe/public/js/frappe/form/toolbar.js @@ -374,19 +374,24 @@ frappe.ui.form.Toolbar = Class.extend({ var status = this.get_action_status(); if (status) { - if (status !== this.current_status) { - if (status === 'Amend') { - let doc = this.frm.doc; - frappe.xcall('frappe.client.is_document_amended', { - 'doctype': doc.doctype, - 'docname': doc.name - }).then(is_amended => { - if (is_amended) return; - this.set_page_actions(status); - }); - } else { + // When moving from a page with status amend to another page with status amend + // We need to check if document is already amened specifcally and hide + // or clear the menu actions accordingly + + if (status !== this.current_status || status === 'Amend') { + let doc = this.frm.doc; + frappe.xcall('frappe.client.is_document_amended', { + 'doctype': doc.doctype, + 'docname': doc.name + }).then(is_amended => { + if (is_amended) { + this.page.clear_actions(); + return; + } this.set_page_actions(status); - } + }); + } else { + this.set_page_actions(status); } } else { this.page.clear_actions(); From bfaf2fcf7740de19c8568235e80c5eb3ac3cfbf4 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Fri, 15 May 2020 13:24:06 +0530 Subject: [PATCH 002/485] feat: do not allow amending already amended docs --- frappe/public/js/frappe/form/form.js | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js index 01dfbf81f9..58a1be73c1 100644 --- a/frappe/public/js/frappe/form/form.js +++ b/frappe/public/js/frappe/form/form.js @@ -780,15 +780,24 @@ frappe.ui.form.Form = class FrappeForm { frappe.msgprint(__('"amended_from" field must be present to do an amendment.')); return; } - this.validate_form_action("Amend"); - var me = this; - var fn = function(newdoc) { - newdoc.amended_from = me.docname; - if(me.fields_dict && me.fields_dict['amendment_date']) - newdoc.amendment_date = frappe.datetime.obj_to_str(new Date()); - }; - this.copy_doc(fn, 1); - frappe.utils.play_sound("click"); + + frappe.xcall('frappe.client.is_document_amended', { + 'doctype': this.doc.doctype, + 'docname': this.doc.name + }).then(is_amended => { + if (is_amended) { + frappe.throw(__('This document is already amended, you cannot ammend it again')) + }; + this.validate_form_action("Amend"); + var me = this; + var fn = function(newdoc) { + newdoc.amended_from = me.docname; + if(me.fields_dict && me.fields_dict['amendment_date']) + newdoc.amendment_date = frappe.datetime.obj_to_str(new Date()); + }; + this.copy_doc(fn, 1); + frappe.utils.play_sound("click"); + }); } validate_form_action(action, resolve) { From e4729b00008449e0f96136fe4bfd1f80280b01e7 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Sat, 16 May 2020 13:29:44 +0530 Subject: [PATCH 003/485] feat: added FileAlreadyAttachedException --- frappe/exceptions.py | 1 + 1 file changed, 1 insertion(+) diff --git a/frappe/exceptions.py b/frappe/exceptions.py index 5a1181f31e..fe8c1895d7 100644 --- a/frappe/exceptions.py +++ b/frappe/exceptions.py @@ -98,6 +98,7 @@ class InvalidColumnName(ValidationError): pass class IncompatibleApp(ValidationError): pass class InvalidDates(ValidationError): pass class DataTooLongException(ValidationError): pass +class FileAlreadyAttachedException(Exception): pass # OAuth exceptions class InvalidAuthorizationHeader(CSRFTokenError): pass class InvalidAuthorizationPrefix(CSRFTokenError): pass From d837535740456cff5e1f65bdbad7e67936e99f47 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Sat, 16 May 2020 13:30:15 +0530 Subject: [PATCH 004/485] fix: raise explicit exception not conflicting with NameError --- frappe/core/doctype/file/file.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/core/doctype/file/file.py b/frappe/core/doctype/file/file.py index b35abfa861..fedd35aa4f 100755 --- a/frappe/core/doctype/file/file.py +++ b/frappe/core/doctype/file/file.py @@ -180,11 +180,11 @@ class File(Document): if duplicate_file: duplicate_file_doc = frappe.get_cached_doc('File', duplicate_file.name) if duplicate_file_doc.exists_on_disk(): - # if it is attached to a document then throw DuplicateEntryError + # if it is attached to a document then throw FileAlreadyAttachedException if self.attached_to_doctype and self.attached_to_name: self.duplicate_entry = duplicate_file.name frappe.throw(_("Same file has already been attached to the record"), - frappe.DuplicateEntryError) + frappe.FileAlreadyAttachedException) # else just use the url, to avoid uploading a duplicate else: self.file_url = duplicate_file.file_url From af0cccf5cd518e257386ec87808c5e9ef4deeaed Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Sat, 16 May 2020 13:52:30 +0530 Subject: [PATCH 005/485] refactor: delete doc directly without checking for comments --- frappe/core/doctype/file/file.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/frappe/core/doctype/file/file.py b/frappe/core/doctype/file/file.py index fedd35aa4f..0f57ed0a5e 100755 --- a/frappe/core/doctype/file/file.py +++ b/frappe/core/doctype/file/file.py @@ -704,7 +704,12 @@ def remove_all(dt, dn, from_delete=False): try: for fid in frappe.db.sql_list("""select name from `tabFile` where attached_to_doctype=%s and attached_to_name=%s""", (dt, dn)): - remove_file(fid=fid, attached_to_doctype=dt, attached_to_name=dn, from_delete=from_delete) + if from_delete: + # If deleting a doc, directly delete files + frappe.delete_doc("File", fid, ignore_permissions=True) + else: + # Removes file and adds a comment in the document it is attached to + remove_file(fid=fid, attached_to_doctype=dt, attached_to_name=dn, from_delete=from_delete) except Exception as e: if e.args[0]!=1054: raise # (temp till for patched) From aef2da89b9a91e15df9499ea9908ff4e3464641e Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Sat, 16 May 2020 13:59:16 +0530 Subject: [PATCH 006/485] style: linting fixes for sider --- frappe/public/js/frappe/form/form.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frappe/public/js/frappe/form/form.js b/frappe/public/js/frappe/form/form.js index 58a1be73c1..8a498a69a2 100644 --- a/frappe/public/js/frappe/form/form.js +++ b/frappe/public/js/frappe/form/form.js @@ -786,13 +786,13 @@ frappe.ui.form.Form = class FrappeForm { 'docname': this.doc.name }).then(is_amended => { if (is_amended) { - frappe.throw(__('This document is already amended, you cannot ammend it again')) - }; + frappe.throw(__('This document is already amended, you cannot ammend it again')); + } this.validate_form_action("Amend"); var me = this; var fn = function(newdoc) { newdoc.amended_from = me.docname; - if(me.fields_dict && me.fields_dict['amendment_date']) + if (me.fields_dict && me.fields_dict['amendment_date']) newdoc.amendment_date = frappe.datetime.obj_to_str(new Date()); }; this.copy_doc(fn, 1); From 0a4f1e66bccea0fa7b479d1024b5524be9f8b1f2 Mon Sep 17 00:00:00 2001 From: Himanshu Warekar Date: Fri, 22 May 2020 13:42:42 +0530 Subject: [PATCH 007/485] fix: filter table multiselect datalist --- frappe/public/js/frappe/form/controls/link.js | 13 +++++++++---- .../js/frappe/form/controls/table_multiselect.js | 15 ++++++++++++++- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/frappe/public/js/frappe/form/controls/link.js b/frappe/public/js/frappe/form/controls/link.js index 9d8241f5a7..c847fb8f44 100644 --- a/frappe/public/js/frappe/form/controls/link.js +++ b/frappe/public/js/frappe/form/controls/link.js @@ -117,9 +117,6 @@ frappe.ui.form.ControlLink = frappe.ui.form.ControlData.extend({ value: item.value }; }, - filter: function() { - return true; - }, item: function (item) { var d = this.get_item(item.value); if(!d.label) { d.label = d.value; } @@ -140,6 +137,8 @@ frappe.ui.form.ControlLink = frappe.ui.form.ControlData.extend({ } }); + this.filter_awesomplete(this.awesomplete); + this.$input.on("input", frappe.utils.debounce(function(e) { var doctype = me.get_options(); if(!doctype) return; @@ -467,10 +466,16 @@ frappe.ui.form.ControlLink = frappe.ui.form.ControlData.extend({ for(var i=0; i < fl.length; i++) { frappe.model.set_value(df.parent, docname, fl[i], fetch_values[i], df.fieldtype); } + }, + + filter_awesomplete: function(awesomplete) { + awesomplete.filter = function() { + return true; + } } }); -if(Awesomplete) { +if (Awesomplete) { Awesomplete.prototype.get_item = function(value) { return this._list.find(function(item) { return item.value === value; diff --git a/frappe/public/js/frappe/form/controls/table_multiselect.js b/frappe/public/js/frappe/form/controls/table_multiselect.js index a75a947e3f..5764cb05e6 100644 --- a/frappe/public/js/frappe/form/controls/table_multiselect.js +++ b/frappe/public/js/frappe/form/controls/table_multiselect.js @@ -11,6 +11,8 @@ frappe.ui.form.ControlTableMultiSelect = frappe.ui.form.ControlLink.extend({ // used as an internal model to store values this.rows = []; + // used as an internal model to filter awesomplete values + this._rows_list = [] this.$input_area.on('click', (e) => { if (e.target === this.$input_area.get(0)) { @@ -61,7 +63,7 @@ frappe.ui.form.ControlTableMultiSelect = frappe.ui.form.ControlLink.extend({ }); } } - + this._rows_list = this.rows.map(row => row[link_field.fieldname]); return this.rows; }, validate(value) { @@ -141,4 +143,15 @@ frappe.ui.form.ControlTableMultiSelect = frappe.ui.form.ControlLink.extend({ } return this._link_field; }, + filter_awesomplete: function(awesomplete) { + let me = this; + + awesomplete.filter = function(item) { + if (in_list(me._rows_list, item.value)) { + return false + } + + return true + } + } }); From 30b59881e33d4419b18d967e685f3e638dd2c082 Mon Sep 17 00:00:00 2001 From: Himanshu Warekar Date: Fri, 22 May 2020 14:18:48 +0530 Subject: [PATCH 008/485] fix: sider fixes --- frappe/public/js/frappe/form/controls/link.js | 2 +- .../public/js/frappe/form/controls/table_multiselect.js | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/frappe/public/js/frappe/form/controls/link.js b/frappe/public/js/frappe/form/controls/link.js index c847fb8f44..6bbc32acbc 100644 --- a/frappe/public/js/frappe/form/controls/link.js +++ b/frappe/public/js/frappe/form/controls/link.js @@ -471,7 +471,7 @@ frappe.ui.form.ControlLink = frappe.ui.form.ControlData.extend({ filter_awesomplete: function(awesomplete) { awesomplete.filter = function() { return true; - } + }; } }); diff --git a/frappe/public/js/frappe/form/controls/table_multiselect.js b/frappe/public/js/frappe/form/controls/table_multiselect.js index 5764cb05e6..f31fac6b69 100644 --- a/frappe/public/js/frappe/form/controls/table_multiselect.js +++ b/frappe/public/js/frappe/form/controls/table_multiselect.js @@ -12,7 +12,7 @@ frappe.ui.form.ControlTableMultiSelect = frappe.ui.form.ControlLink.extend({ // used as an internal model to store values this.rows = []; // used as an internal model to filter awesomplete values - this._rows_list = [] + this._rows_list = []; this.$input_area.on('click', (e) => { if (e.target === this.$input_area.get(0)) { @@ -148,10 +148,10 @@ frappe.ui.form.ControlTableMultiSelect = frappe.ui.form.ControlLink.extend({ awesomplete.filter = function(item) { if (in_list(me._rows_list, item.value)) { - return false + return false; } - return true - } + return true; + }; } }); From 2ea74dee36f1bbd094a7fa967fc50be26ba5e09d Mon Sep 17 00:00:00 2001 From: "Chinmay D. Pai" Date: Tue, 26 May 2020 18:47:17 +0530 Subject: [PATCH 009/485] fix: check for whitelist before calling from search search widget takes query as an input, but does not check whether the query function that is called is whitelisted, basically allowing anyone logged-in to call any function regardless of the whitelist. Signed-off-by: Chinmay D. Pai --- frappe/desk/search.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frappe/desk/search.py b/frappe/desk/search.py index c70b650945..1da2a3d0b5 100644 --- a/frappe/desk/search.py +++ b/frappe/desk/search.py @@ -6,6 +6,7 @@ from __future__ import unicode_literals import frappe, json from frappe.utils import cstr, unique, cint from frappe.permissions import has_permission +from frappe.handler import is_whitelisted from frappe import _ from six import string_types import re @@ -74,6 +75,7 @@ def search_widget(doctype, txt, query=None, searchfield=None, start=0, if query and query.split()[0].lower()!="select": # by method + is_whitelisted(query) frappe.response["values"] = frappe.call(query, doctype, txt, searchfield, start, page_length, filters, as_dict=as_dict) elif not query and doctype in standard_queries: From c4d4fc3574dcb0f716fc8ad5edb162f171a990fe Mon Sep 17 00:00:00 2001 From: "Chinmay D. Pai" Date: Tue, 26 May 2020 18:49:12 +0530 Subject: [PATCH 010/485] chore: add standard queries hooks to whitelist standard queries are used within the search widget, and now require to be whitelisted before they can be executed through the search widget. Signed-off-by: Chinmay D. Pai --- frappe/core/doctype/user/user.py | 1 + 1 file changed, 1 insertion(+) diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py index 0c5ebc3ede..f571240454 100644 --- a/frappe/core/doctype/user/user.py +++ b/frappe/core/doctype/user/user.py @@ -811,6 +811,7 @@ def reset_password(user): frappe.clear_messages() return 'not found' +@frappe.whitelist() def user_query(doctype, txt, searchfield, start, page_len, filters): from frappe.desk.reportview import get_match_cond From 2321a75f948a152d69d3cb517690ca68f436931d Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Thu, 30 Apr 2020 13:01:46 +0530 Subject: [PATCH 011/485] fix: Handle duplicate columns --- .../core/doctype/data_import/exporter_new.py | 24 +++++++--- .../core/doctype/data_import/importer_new.py | 45 +++++++++++++++---- 2 files changed, 54 insertions(+), 15 deletions(-) diff --git a/frappe/core/doctype/data_import/exporter_new.py b/frappe/core/doctype/data_import/exporter_new.py index 85f933be69..d0c3572f2e 100644 --- a/frappe/core/doctype/data_import/exporter_new.py +++ b/frappe/core/doctype/data_import/exporter_new.py @@ -230,13 +230,25 @@ class Exporter: return [i for i, df in enumerate(self.fields) if df.parent == doctype] def add_header(self): - def get_label(df): - if df.parent == self.doctype: - return df.label - else: - return "{0} ({1})".format(df.label, df.parent) - header = [get_label(df) for df in self.fields] + header = [] + for df in self.fields: + is_parent = df.parent == self.doctype + if is_parent: + label = df.label + else: + label = "{0} ({1})".format(df.label, df.parent) + + if label in header: + # this label is already in the header, + # which means two fields with the same label + # add the fieldname to avoid clash + if is_parent: + label = '{0} ({1})'.format(df.label, df.fieldname) + else: + label = '{0} ({1}) ({2})'.format(df.label, df.fieldname, df.parent) + header.append(label) + self.csv_array.append(header) def add_data(self): diff --git a/frappe/core/doctype/data_import/importer_new.py b/frappe/core/doctype/data_import/importer_new.py index 040e9fabc4..383d8bc366 100644 --- a/frappe/core/doctype/data_import/importer_new.py +++ b/frappe/core/doctype/data_import/importer_new.py @@ -175,6 +175,7 @@ class Importer: def parse_columns_from_header_row(self): remap_column = self.template_options.remap_column columns = [] + seen = [] df_by_labels_and_fieldnames = self.build_fields_dict_for_column_matching() @@ -203,7 +204,17 @@ class Importer: else: skip_import = False - if fieldname == "Don't Import": + if header_title in seen: + self.warnings.append( + { + "col": column_number, + "message": _("Skipping Duplicate Column {0}").format(frappe.bold(header_title)), + "type": "info", + } + ) + df = None + skip_import = True + elif fieldname == "Don't Import": skip_import = True self.warnings.append( { @@ -236,6 +247,7 @@ class Importer: index=i, ) ) + seen.append(header_title) return columns @@ -278,17 +290,32 @@ class Importer: fieldtype = df.fieldtype or "Data" parent = df.parent or self.doctype if fieldtype not in no_value_fields: - # label as key - label = ( - df.label if self.doctype == doctype else "{0} ({1})".format(df.label, parent) - ) - out[label] = df - # fieldname as key if self.doctype == doctype: + # for parent doctypes keys will be + # Label + # label + # Label (label) + if not out.get(df.label): + # if Label is already set, don't set it again + # in case of duplicate column headers + out[df.label] = df out[df.fieldname] = df + label_with_fieldname = "{0} ({1})".format(df.label, df.fieldname) + out[label_with_fieldname] = df else: - key = "{0}:{1}".format(doctype, df.fieldname) - out[key] = df + # for child doctypes keys will be + # Label (Child DocType) + # Child DocType:label + # Label (label) (Child DocType) + label = "{0} ({1})".format(df.label, parent) + fieldname = "{0}:{1}".format(doctype, df.fieldname) + label_with_fieldname = "{0} ({1}) ({2})".format(df.label, df.fieldname, parent) + if not out.get(label): + # if Label is already set, don't set it again + # in case of duplicate column headers + out[label] = df + out[fieldname] = df + out[label_with_fieldname] = df # if autoname is based on field # add an entry for "ID (Autoname Field)" From 7ec51923320ca80ddd0283fcd8867088ea7fc96d Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Thu, 30 Apr 2020 13:01:51 +0530 Subject: [PATCH 012/485] fix: Style --- frappe/core/doctype/data_import/importer_new.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/frappe/core/doctype/data_import/importer_new.py b/frappe/core/doctype/data_import/importer_new.py index 383d8bc366..cbb2ee482b 100644 --- a/frappe/core/doctype/data_import/importer_new.py +++ b/frappe/core/doctype/data_import/importer_new.py @@ -378,16 +378,8 @@ class Importer: value = cstr(value) # convert boolean values to 0 or 1 - if df.fieldtype == "Check" and value.lower().strip() in [ - "t", - "f", - "true", - "false", - "yes", - "no", - "y", - "n", - ]: + valid_check_values = ["t", "f", "true", "false", "yes", "no", "y", "n"] + if df.fieldtype == "Check" and value.lower().strip() in valid_check_values: value = value.lower().strip() value = 1 if value in ["t", "true", "y", "yes"] else 0 From 4888eb96945f3aaa6cf5011b698b8ed74d012ced Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Sun, 3 May 2020 15:19:52 +0530 Subject: [PATCH 013/485] fix: Handle Insert / Update case in exporter --- .../data_import_beta/data_import_beta.js | 23 +----- .../js/frappe/data_import/data_exporter.js | 72 ++++++++++++++----- 2 files changed, 58 insertions(+), 37 deletions(-) diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.js b/frappe/core/doctype/data_import_beta/data_import_beta.js index 527dbd7d0c..72404c74f4 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.js +++ b/frappe/core/doctype/data_import_beta/data_import_beta.js @@ -193,33 +193,16 @@ frappe.ui.form.on('Data Import Beta', { frm.data_exporter && frm.data_exporter.doctype === frm.doc.reference_doctype ) { + frm.data_exporter.exporting_for = frm.doc.import_type; frm.data_exporter.dialog.show(); - set_export_records(); } else { frappe.require('/assets/js/data_import_tools.min.js', () => { frm.data_exporter = new frappe.data_import.DataExporter( - frm.doc.reference_doctype + frm.doc.reference_doctype, + frm.doc.import_type ); - set_export_records(); }); } - - function set_export_records() { - if (frm.doc.import_type === 'Insert New Records') { - frm.data_exporter.dialog.set_value('export_records', 'blank_template'); - } else { - frm.data_exporter.dialog.set_value('export_records', 'all'); - } - // Force ID field to be exported when updating existing records - let id_field = frm.data_exporter.dialog.get_field( - frm.doc.reference_doctype - ).options[0]; - if (id_field.value === 'name' && id_field.$checkbox) { - id_field.$checkbox - .find('input') - .prop('disabled', frm.doc.import_type === 'Update Existing Records'); - } - } }, reference_doctype(frm) { diff --git a/frappe/public/js/frappe/data_import/data_exporter.js b/frappe/public/js/frappe/data_import/data_exporter.js index d0bf794df6..7a5efc6cf0 100644 --- a/frappe/public/js/frappe/data_import/data_exporter.js +++ b/frappe/public/js/frappe/data_import/data_exporter.js @@ -2,8 +2,9 @@ import ColumnPickerFields from './column_picker_fields'; frappe.provide('frappe.data_import'); frappe.data_import.DataExporter = class DataExporter { - constructor(doctype) { + constructor(doctype, exporting_for) { this.doctype = doctype; + this.exporting_for = exporting_for; frappe.model.with_doctype(doctype, () => { this.make_dialog(); }); @@ -13,6 +14,36 @@ frappe.data_import.DataExporter = class DataExporter { this.dialog = new frappe.ui.Dialog({ title: __('Export Data'), fields: [ + { + fieldtype: 'Select', + fieldname: 'exporting_for', + label: __('Exporting For'), + options: [ + { + label: __('Insert New Records'), + value: 'Insert New Records' + }, + { + label: __('Update Existing Records'), + value: 'Update Existing Records' + } + ], + change: () => { + let exporting_for = this.dialog.get_value('exporting_for'); + this.dialog.set_value( + 'export_records', + exporting_for === 'Insert New Records' ? 'blank_template' : 'all' + ); + + // Force ID field to be exported when updating existing records + let id_field = this.dialog.get_field(this.doctype).options[0]; + if (id_field.value === 'name' && id_field.$checkbox) { + id_field.$checkbox + .find('input') + .prop('disabled', exporting_for === 'Update Existing Records'); + } + } + }, { fieldtype: 'Select', fieldname: 'export_records', @@ -67,27 +98,30 @@ frappe.data_import.DataExporter = class DataExporter { on_change: () => this.update_primary_action(), options: this.get_multicheck_options(this.doctype) }, - ...frappe.meta.get_table_fields(this.doctype) - .map(df => { - let doctype = df.options; - let label = df.reqd - ? __('{0} (1 row mandatory)', [doctype]) - : __(doctype); - return { - label, - fieldname: doctype, - fieldtype: 'MultiCheck', - columns: 2, - on_change: () => this.update_primary_action(), - options: this.get_multicheck_options(doctype) - }; - }) + ...frappe.meta.get_table_fields(this.doctype).map(df => { + let doctype = df.options; + let label = df.reqd + ? __('{0} (1 row mandatory)', [doctype]) + : __(doctype); + return { + label, + fieldname: doctype, + fieldtype: 'MultiCheck', + columns: 2, + on_change: () => this.update_primary_action(), + options: this.get_multicheck_options(doctype) + }; + }) ], primary_action_label: __('Export'), primary_action: values => this.export_records(values), on_page_show: () => this.select_mandatory() }); + if (this.exporting_for) { + this.dialog.set_value('exporting_for', this.exporting_for); + } + this.make_filter_area(); this.make_select_all_buttons(); this.update_record_count_message(); @@ -192,8 +226,12 @@ frappe.data_import.DataExporter = class DataExporter { } unselect_all() { + let update_existing_records = + this.dialog.get_value('exporting_for') == 'Update Existing Records'; this.dialog.$wrapper - .find(':checkbox') + .find( + `:checkbox${update_existing_records ? ':not([data-unit=name])' : ''}` + ) .prop('checked', false) .trigger('change'); } From fe1a55110d3862d951c7fd6c04920f23b6ea8ab7 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Mon, 18 May 2020 07:32:34 +0530 Subject: [PATCH 014/485] fix: Data Export - Handle multiple child tables with the same doctype - Refactor exporter to fix child rows export --- .../core/doctype/data_import/exporter_new.py | 252 +++++++++--------- .../data_import/column_picker_fields.js | 28 -- .../js/frappe/data_import/data_exporter.js | 66 ++++- 3 files changed, 177 insertions(+), 169 deletions(-) delete mode 100644 frappe/public/js/frappe/data_import/column_picker_fields.js diff --git a/frappe/core/doctype/data_import/exporter_new.py b/frappe/core/doctype/data_import/exporter_new.py index d0c3572f2e..744a050c1c 100644 --- a/frappe/core/doctype/data_import/exporter_new.py +++ b/frappe/core/doctype/data_import/exporter_new.py @@ -3,7 +3,11 @@ # MIT License. See license.txt import frappe -from frappe.model import display_fieldtypes, no_value_fields, table_fields +from frappe.model import ( + display_fieldtypes, + no_value_fields, + table_fields as table_fieldtypes, +) from frappe.utils.csvutils import build_csv_response from frappe.utils.xlsxutils import build_xlsx_response from .importer_new import INVALID_VALUES @@ -38,8 +42,8 @@ class Exporter: self.csv_array = [] # fields that get exported - # can be All, Mandatory or User Selected Fields - self.fields = self.get_all_exportable_fields() + self.exportable_fields = self.get_all_exportable_fields() + self.fields = self.serialize_exportable_fields() self.add_header() if export_data: @@ -49,47 +53,51 @@ class Exporter: self.add_data() def get_all_exportable_fields(self): - return self.get_exportable_parent_fields() + self.get_exportable_children_fields() + child_table_fields = [ + df.fieldname for df in self.meta.fields if df.fieldtype in table_fieldtypes + ] - def get_exportable_parent_fields(self): - parent_fields = self.get_exportable_fields(self.doctype) + def is_exportable(df): + return df and df.fieldtype not in (display_fieldtypes + no_value_fields) - # if autoname is based on field - # then merge ID and the field column title as "ID (Autoname Field)" - autoname = self.meta.autoname - if autoname and autoname.startswith("field:"): - fieldname = autoname[len("field:") :] - autoname_field = self.meta.get_field(fieldname) - if autoname_field: - name_field = parent_fields[0] - name_field.label = "ID ({})".format(autoname_field.label) - # remove the autoname field as it is a duplicate of ID field - parent_fields = [ - df for df in parent_fields if df.fieldname != autoname_field.fieldname - ] + meta = frappe.get_meta(self.doctype) + exportable_fields = frappe._dict({}) - return parent_fields + for key, fieldnames in self.export_fields.items(): + if key == self.doctype: + # parent fields + exportable_fields[key] = self.get_exportable_fields(key, fieldnames) - def get_exportable_children_fields(self): - child_table_fields = [df for df in self.meta.fields if df.fieldtype in table_fields] - if self.export_fields == "Mandatory": - child_table_fields = [df for df in child_table_fields if df.reqd] + elif key in child_table_fields: + # child fields + child_df = meta.get_field(key) + child_doctype = child_df.options + exportable_fields[key] = self.get_exportable_fields(child_doctype, fieldnames) - children = [df.options for df in child_table_fields] - children_fields = [] - for child in children: - children_fields += self.get_exportable_fields(child) + return exportable_fields - return children_fields + def serialize_exportable_fields(self): + fields = [] + for key, exportable_fields in self.exportable_fields.items(): + for _df in exportable_fields: + # make a copy of df dict to avoid reference mutation + if isinstance(_df, frappe.core.doctype.docfield.docfield.DocField): + df = _df.as_dict() + else: + df = _df.copy() - def get_exportable_fields(self, doctype): + df.is_child_table_field = key != self.doctype + if df.is_child_table_field: + df.child_table_df = self.meta.get_field(key) + fields.append(df) + return fields + + def get_exportable_fields(self, doctype, fieldnames): meta = frappe.get_meta(doctype) def is_exportable(df): return df and df.fieldtype not in (display_fieldtypes + no_value_fields) - # filter out invalid fieldtypes - all_fields = [df for df in meta.fields if is_exportable(df)] # add name field name_field = frappe._dict( { @@ -100,30 +108,55 @@ class Exporter: "parent": doctype, } ) - all_fields = [name_field] + all_fields - if self.export_fields == "Mandatory": - fields = [df for df in all_fields if df.reqd] + fields = [meta.get_field(fieldname) for fieldname in fieldnames] + fields = [df for df in fields if is_exportable(df)] - if self.export_fields == "All": - fields = list(all_fields) - - elif isinstance(self.export_fields, dict): - fields_to_export = self.export_fields.get(doctype, []) - fields = [meta.get_field(fieldname) for fieldname in fields_to_export] - fields = [df for df in fields if is_exportable(df)] - if 'name' in fields_to_export: - fields = [name_field] + fields + if "name" in fieldnames: + fields = [name_field] + fields return fields or [] def get_data_to_export(self): frappe.permissions.can_export(self.doctype, raise_exception=True) + data_to_export = [] - def get_column_name(df): - return "`tab{0}`.`{1}`".format(df.parent, df.fieldname) + table_fields = [f for f in self.exportable_fields if f != self.doctype] + data = self.get_data_as_docs() - fields = [get_column_name(df) for df in self.fields] + for doc in data: + rows = [] + rows = self.add_data_row(self.doctype, None, doc, rows, 0) + + if table_fields: + # add child table data + for f in table_fields: + for i, child_row in enumerate(doc[f]): + table_df = self.meta.get_field(f) + child_doctype = table_df.options + rows = self.add_data_row(child_doctype, child_row.parentfield, child_row, rows, i) + + data_to_export += rows + + return data_to_export + + def add_data_row(self, doctype, parentfield, doc, rows, row_idx): + if len(rows) < row_idx + 1: + rows.append([""] * len(self.fields)) + + row = rows[row_idx] + + for i, df in enumerate(self.fields): + if df.parent == doctype: + if df.is_child_table_field and df.child_table_df.fieldname != parentfield: + continue + row[i] = doc.get(df.fieldname, "") + + return rows + + def get_data_as_docs(self): + format_column_name = lambda df: "`tab{0}`.`{1}`".format(df.parent, df.fieldname) + fields = [format_column_name(df) for df in self.fields] filters = self.export_filters if self.meta.is_nested_set(): @@ -131,94 +164,51 @@ class Exporter: else: order_by = "`tab{0}`.`creation` DESC".format(self.doctype) - data = frappe.db.get_list( + parent_fields = [ + format_column_name(df) for df in self.fields if df.parent == self.doctype + ] + parent_data = frappe.db.get_list( self.doctype, filters=filters, - fields=fields, + fields=["name"] + parent_fields, limit_page_length=self.export_page_length, order_by=order_by, - as_list=1, + as_list=0, ) + parent_names = [p.name for p in parent_data] - data = self.remove_duplicate_values(data) - data = self.remove_row_gaps(data) - data = self.remove_empty_rows(data) - # data = self.remove_values_from_name_column(data) - - return data - - def remove_duplicate_values(self, data): - out = [] - - doctypes = set([df.parent for df in self.fields]) - - def name_exists_in_column_before_row(name, column_index, row_index): - column_values = [row[column_index] for i, row in enumerate(data) if i < row_index] - return name in column_values - - for i, row in enumerate(data): - # first row is fine - if i == 0: - out.append(row) + child_data = {} + for key in self.exportable_fields: + if key == self.doctype: continue + child_table_df = self.meta.get_field(key) + child_table_doctype = child_table_df.options + child_fields = ["name", "idx", "parent", "parentfield"] + list( + set( + [format_column_name(df) for df in self.fields if df.parent == child_table_doctype] + ) + ) + data = frappe.db.get_list( + child_table_doctype, + filters={ + "parent": ("in", parent_names), + "parentfield": child_table_df.fieldname, + "parenttype": self.doctype, + }, + fields=child_fields, + order_by="idx asc", + as_list=0, + ) + child_data[key] = data - row = list(row) - for doctype in doctypes: - name_index = self.get_name_column_index(doctype) - name = row[name_index] - column_indexes = self.get_column_indexes(doctype) + return self.merge_data(parent_data, child_data) - if name_exists_in_column_before_row(name, name_index, i): - # remove the values from the row - row = [None if i in column_indexes else d for i, d in enumerate(row)] + def merge_data(self, parent_data, child_data): + for doc in parent_data: + for table_field, table_rows in child_data.items(): + doc[table_field] = [row for row in table_rows if row.parent == doc.name] - out.append(row) - - return out - - def remove_row_gaps(self, data): - doctypes = set([df.parent for df in self.fields if df.parent != self.doctype]) - - def get_nearest_empty_row_index(col_index, row_index): - col_values = [row[col_index] for row in data] - i = row_index - 1 - while not col_values[i]: - i = i - 1 - out = i + 1 - if row_index != out: - return out - - for i, row in enumerate(data): - # if this is the row that contains parent values then skip - if row[0]: - continue - - for doctype in doctypes: - name_index = self.get_name_column_index(doctype) - name = row[name_index] - column_indexes = self.get_column_indexes(doctype) - - if not name: - continue - - row_index = get_nearest_empty_row_index(name_index, i) - if row_index: - for col_index in column_indexes: - data[row_index][col_index] = row[col_index] - row[col_index] = None - - return data - - # pylint: disable=R0201 - def remove_empty_rows(self, data): - return [row for row in data if any(v not in INVALID_VALUES for v in row)] - - def remove_values_from_name_column(self, data): - out = [] - name_columns = [i for i, df in enumerate(self.fields) if df.fieldname == "name"] - for row in data: - out.append(["" if i in name_columns else value for i, value in enumerate(row)]) - return out + return parent_data def get_name_column_index(self, doctype): for i, df in enumerate(self.fields): @@ -233,20 +223,20 @@ class Exporter: header = [] for df in self.fields: - is_parent = df.parent == self.doctype + is_parent = not df.is_child_table_field if is_parent: label = df.label else: - label = "{0} ({1})".format(df.label, df.parent) + label = "{0} ({1})".format(df.label, df.child_table_df.label) if label in header: # this label is already in the header, # which means two fields with the same label # add the fieldname to avoid clash if is_parent: - label = '{0} ({1})'.format(df.label, df.fieldname) + label = "{0}".format(df.fieldname) else: - label = '{0} ({1}) ({2})'.format(df.label, df.fieldname, df.parent) + label = "{0}.{1}".format(df.child_table_df.fieldname, df.fieldname) header.append(label) self.csv_array.append(header) @@ -267,9 +257,9 @@ class Exporter: return csv_array def build_response(self): - if self.file_type == 'CSV': + if self.file_type == "CSV": self.build_csv_response() - elif self.file_type == 'Excel': + elif self.file_type == "Excel": self.build_xlsx_response() def build_csv_response(self): diff --git a/frappe/public/js/frappe/data_import/column_picker_fields.js b/frappe/public/js/frappe/data_import/column_picker_fields.js deleted file mode 100644 index 36cbf3c413..0000000000 --- a/frappe/public/js/frappe/data_import/column_picker_fields.js +++ /dev/null @@ -1,28 +0,0 @@ -export default class ColumnPickerFields extends frappe.views.ReportView { - show() {} - - get_fields_as_options() { - let column_map = this.get_columns_for_picker(); - let doctypes = [this.doctype].concat( - ...frappe.meta.get_table_fields(this.doctype).map(df => df.options) - ); - // flatten array - return [].concat( - ...doctypes.map(doctype => { - return column_map[doctype].map(df => { - let label = df.label; - let value = df.fieldname; - if (this.doctype !== doctype) { - label = `${df.label} (${doctype})`; - value = `${doctype}:${df.fieldname}`; - } - return { - label, - value, - description: value - }; - }); - }) - ); - } -} diff --git a/frappe/public/js/frappe/data_import/data_exporter.js b/frappe/public/js/frappe/data_import/data_exporter.js index 7a5efc6cf0..8276be6670 100644 --- a/frappe/public/js/frappe/data_import/data_exporter.js +++ b/frappe/public/js/frappe/data_import/data_exporter.js @@ -1,4 +1,3 @@ -import ColumnPickerFields from './column_picker_fields'; frappe.provide('frappe.data_import'); frappe.data_import.DataExporter = class DataExporter { @@ -100,16 +99,20 @@ frappe.data_import.DataExporter = class DataExporter { }, ...frappe.meta.get_table_fields(this.doctype).map(df => { let doctype = df.options; + let child_fieldname = df.fieldname; let label = df.reqd - ? __('{0} (1 row mandatory)', [doctype]) - : __(doctype); + ? __('{0} ({1}) (1 row mandatory)', [ + df.label || df.fieldname, + doctype + ]) + : __('{0} ({1})', [df.label || df.fieldname, doctype]); return { label, - fieldname: doctype, + fieldname: child_fieldname, fieldtype: 'MultiCheck', columns: 2, on_change: () => this.update_primary_action(), - options: this.get_multicheck_options(doctype) + options: this.get_multicheck_options(doctype, child_fieldname) }; }) ], @@ -291,11 +294,9 @@ frappe.data_import.DataExporter = class DataExporter { }, {}); } - get_multicheck_options(doctype) { + get_multicheck_options(doctype, child_fieldname = null) { if (!this.column_map) { - this.column_map = new ColumnPickerFields({ - doctype: this.doctype - }).get_columns_for_picker(); + this.column_map = get_columns_for_picker(this.doctype); } let autoname_field = null; @@ -305,7 +306,11 @@ frappe.data_import.DataExporter = class DataExporter { autoname_field = frappe.meta.get_field(doctype, fieldname); } - return this.column_map[doctype] + let fields = child_fieldname + ? this.column_map[child_fieldname] + : this.column_map[doctype]; + + return fields .filter(df => { if (autoname_field && df.fieldname === autoname_field.fieldname) { return false; @@ -327,3 +332,44 @@ frappe.data_import.DataExporter = class DataExporter { }); } }; + +function get_columns_for_picker(doctype) { + let out = {}; + + const standard_fields_filter = df => + !in_list(frappe.model.no_value_type, df.fieldtype); + + // parent + let doctype_fields = frappe.meta + .get_docfields(doctype) + .filter(standard_fields_filter); + + out[doctype] = [ + { + label: __('ID'), + fieldname: 'name', + fieldtype: 'Data', + reqd: 1 + } + ].concat(doctype_fields); + + // children + const table_fields = frappe.meta.get_table_fields(doctype); + table_fields.forEach(df => { + const cdt = df.options; + const child_table_fields = frappe.meta + .get_docfields(cdt) + .filter(standard_fields_filter); + + out[df.fieldname] = [ + { + label: __('ID'), + fieldname: 'name', + fieldtype: 'Data', + reqd: 1 + } + ].concat(child_table_fields); + }); + + return out; +} From 11ec3442a3d8032ac78e4149c8be8285b321a651 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Thu, 28 May 2020 19:24:04 +0530 Subject: [PATCH 015/485] feat: remove writer introduction from bog settings --- .../website/doctype/blog_settings/blog_settings.json | 8 +------- .../doctype/blog_settings/test_blog_settings.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 7 deletions(-) create mode 100644 frappe/website/doctype/blog_settings/test_blog_settings.py diff --git a/frappe/website/doctype/blog_settings/blog_settings.json b/frappe/website/doctype/blog_settings/blog_settings.json index f0e51de170..a05c0ea5f3 100644 --- a/frappe/website/doctype/blog_settings/blog_settings.json +++ b/frappe/website/doctype/blog_settings/blog_settings.json @@ -7,7 +7,6 @@ "field_order": [ "blog_title", "blog_introduction", - "writers_introduction", "section_break_4", "social_share_settings" ], @@ -22,11 +21,6 @@ "fieldtype": "Small Text", "label": "Blog Introduction" }, - { - "fieldname": "writers_introduction", - "fieldtype": "Small Text", - "label": "Writers Introduction" - }, { "collapsible": 1, "fieldname": "section_break_4", @@ -43,7 +37,7 @@ "idx": 1, "issingle": 1, "links": [], - "modified": "2020-05-04 09:10:41.815238", + "modified": "2020-05-28 18:49:08.356608", "modified_by": "Administrator", "module": "Website", "name": "Blog Settings", diff --git a/frappe/website/doctype/blog_settings/test_blog_settings.py b/frappe/website/doctype/blog_settings/test_blog_settings.py new file mode 100644 index 0000000000..e4ddb85c4b --- /dev/null +++ b/frappe/website/doctype/blog_settings/test_blog_settings.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020, Frappe Technologies and Contributors +# See license.txt +from __future__ import unicode_literals + +# import frappe +import unittest + +class TestBlogSettings(unittest.TestCase): + pass From 892149af99ae4b81e6005b2b25dfbcf0dc4a9209 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Thu, 28 May 2020 19:24:31 +0530 Subject: [PATCH 016/485] feat: give precedence to template --- frappe/website/render.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/frappe/website/render.py b/frappe/website/render.py index c1bca3f5c5..f137a28eaa 100644 --- a/frappe/website/render.py +++ b/frappe/website/render.py @@ -214,14 +214,13 @@ def build_page(path): context = get_context(path) - if context.source: - html = frappe.render_template(context.source, context) - - elif context.template: + if context.template: if path.endswith('min.js'): html = frappe.get_jloader().get_source(frappe.get_jenv(), context.template)[0] else: html = frappe.get_template(context.template).render(context) + elif context.source: + html = frappe.render_template(context.source, context) if '{index}' in html: html = html.replace('{index}', get_toc(context.route)) From 8b08a7d71d8418891869411cbaf7be525f35730e Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Thu, 28 May 2020 19:25:10 +0530 Subject: [PATCH 017/485] refactor: blogger doctype * remove blog post count * Set image field --- frappe/website/doctype/blog_post/blog_post.py | 11 ++++------- frappe/website/doctype/blogger/blogger.json | 16 ++++------------ 2 files changed, 8 insertions(+), 19 deletions(-) diff --git a/frappe/website/doctype/blog_post/blog_post.py b/frappe/website/doctype/blog_post/blog_post.py index 4596c60710..5516d17780 100644 --- a/frappe/website/doctype/blog_post/blog_post.py +++ b/frappe/website/doctype/blog_post/blog_post.py @@ -39,11 +39,6 @@ class BlogPost(WebsiteGenerator): if self.published and not self.published_on: self.published_on = today() - # update posts - frappe.db.sql("""UPDATE `tabBlogger` SET `posts`=(SELECT COUNT(*) FROM `tabBlog Post` - WHERE IFNULL(`blogger`,'')=`tabBlogger`.`name`) - WHERE `name`=%s""", (self.blogger,)) - self.set_read_time() def on_update(self): @@ -133,8 +128,9 @@ class BlogPost(WebsiteGenerator): def get_list_context(context=None): list_context = frappe._dict( - template = "templates/includes/blog/blog.html", + template = "/templates/includes/blog/blog.html", get_list = get_blog_list, + no_breadcrumbs = True, hide_filters = True, children = get_children(), # show_search = True, @@ -161,7 +157,8 @@ def get_list_context(context=None): else: list_context.parents = [{"name": _("Home"), "route": "/"}] - list_context.update(frappe.get_doc("Blog Settings", "Blog Settings").as_dict(no_default_fields=True)) + list_context.update(frappe.get_doc("Blog Settings").as_dict(no_default_fields=True)) + return list_context def get_children(): diff --git a/frappe/website/doctype/blogger/blogger.json b/frappe/website/doctype/blogger/blogger.json index b8165a5908..f7494e7ec5 100644 --- a/frappe/website/doctype/blogger/blogger.json +++ b/frappe/website/doctype/blogger/blogger.json @@ -13,8 +13,7 @@ "full_name", "user", "bio", - "avatar", - "posts" + "avatar" ], "fields": [ { @@ -51,20 +50,13 @@ }, { "fieldname": "avatar", - "fieldtype": "Attach", + "fieldtype": "Attach Image", "label": "Avatar" - }, - { - "fieldname": "posts", - "fieldtype": "Int", - "in_list_view": 1, - "label": "Posts", - "no_copy": 1, - "read_only": 1 } ], "icon": "fa fa-user", "idx": 1, + "image_field": "avatar", "links": [ { "link_doctype": "Blog Post", @@ -72,7 +64,7 @@ } ], "max_attachments": 1, - "modified": "2020-04-19 08:21:09.684300", + "modified": "2020-05-28 19:22:40.959895", "modified_by": "Administrator", "module": "Website", "name": "Blogger", From ea728d158f630b8f8d3a1a64f9c3d2dd49b994d9 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Thu, 28 May 2020 22:07:22 +0530 Subject: [PATCH 018/485] feat: added image extra small option --- frappe/public/scss/website-image.scss | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/frappe/public/scss/website-image.scss b/frappe/public/scss/website-image.scss index 8c32e821fe..d416c05650 100644 --- a/frappe/public/scss/website-image.scss +++ b/frappe/public/scss/website-image.scss @@ -55,6 +55,12 @@ img:after { width: 100%; } +.website-image-extra-small { + @include website-image; + width: 2.5rem; + height: 2.5rem; +} + .website-image-small { @include website-image; width: 5rem; From 5ea0688e40c27473f69743d7a0f1d210ebf7a1ba Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Thu, 28 May 2020 22:07:37 +0530 Subject: [PATCH 019/485] feat: added blog.scss --- frappe/public/scss/blog.scss | 48 +++++++++++++++++++++++++++++++++ frappe/public/scss/website.scss | 1 + 2 files changed, 49 insertions(+) create mode 100644 frappe/public/scss/blog.scss diff --git a/frappe/public/scss/blog.scss b/frappe/public/scss/blog.scss new file mode 100644 index 0000000000..86c5b18ceb --- /dev/null +++ b/frappe/public/scss/blog.scss @@ -0,0 +1,48 @@ +.blog-list-item { + @extend .col-12; + + &:not(.featured) { + @extend .col-md-4; + + .blog-list-body { + min-height: 14rem; + } + } + + .blog-list-cover { + @extend .col-12; + } + + .blog-list-body { + @extend .col-12; + @extend .mt-3; + } + + .post-cover-container { + min-height: 12rem; + overflow: hidden; + border-radius: $border-radius; + + img { + min-height: 12rem; + width: 100%; + } + } + + &.featured { + .blog-list-cover { + @extend .col-12; + @extend .col-md-8; + } + + .blog-list-body { + @extend .col-12; + @extend .col-md-4; + @extend .mt-0; + } + + .post-cover-container { + max-height: 22rem; + } + } +} \ No newline at end of file diff --git a/frappe/public/scss/website.scss b/frappe/public/scss/website.scss index 0149ac0d0a..152551c8d1 100644 --- a/frappe/public/scss/website.scss +++ b/frappe/public/scss/website.scss @@ -5,6 +5,7 @@ @import 'multilevel-dropdown'; @import 'website-image'; @import 'page-builder'; +@import 'blog'; @import 'markdown'; @import 'sidebar'; From 78cb3a9c178964bebe0270234b4690fc9b7fb7c7 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Thu, 28 May 2020 22:08:08 +0530 Subject: [PATCH 020/485] feat: updated blog listing layout --- frappe/templates/includes/blog/blog.html | 32 ++++++-- frappe/templates/includes/blog/blog_list.html | 25 +++++++ .../blog_post/templates/blog_post_row.html | 73 ++++++++++--------- 3 files changed, 91 insertions(+), 39 deletions(-) create mode 100644 frappe/templates/includes/blog/blog_list.html diff --git a/frappe/templates/includes/blog/blog.html b/frappe/templates/includes/blog/blog.html index 5afaeb6ab8..a841bff627 100644 --- a/frappe/templates/includes/blog/blog.html +++ b/frappe/templates/includes/blog/blog.html @@ -1,15 +1,35 @@ {% extends "templates/web.html" %} - {% block title %}{{ blog_title or _("Blog") }}{% endblock %} -{% block header %}

{{ blog_title or _("Blog") }}

{% endblock %} {% block hero %}{% endblock %} {% block page_content %} - - + +{{ web_blocks([ + { + 'template': "Hero", + 'values': { + 'title': blog_title, + 'subtitle': blog_introduction, + }, + 'add_container': 0, + 'add_top_padding': 0, + 'add_bottom_padding': 0, + 'css_class': "py-5" + } + ]) +}} +
-
- {% include "templates/includes/list/list.html" %} +
+ {% if not result -%} +
+ {{ no_result_message or _("Nothing to show") }} +
+ {% else %} + {% for item in result %} + {{ item }} + {% endfor %} + {% endif %}
{% endblock %} diff --git a/frappe/templates/includes/blog/blog_list.html b/frappe/templates/includes/blog/blog_list.html new file mode 100644 index 0000000000..cfe25f3682 --- /dev/null +++ b/frappe/templates/includes/blog/blog_list.html @@ -0,0 +1,25 @@ +{% if sub_title %} +

{{ sub_title }}

+{% endif %} +{% if not result -%} +
+ {{ no_result_message or _("Nothing to show") }} +
+{% else %} +
+ + {% if result_heading_template %}{% include result_heading_template %}{% endif %} + +
+ {% for item in result %} + {{ item }} + {% endfor %} +
+ +
+{%- endif %} diff --git a/frappe/website/doctype/blog_post/templates/blog_post_row.html b/frappe/website/doctype/blog_post/templates/blog_post_row.html index dffe0ef81d..d197494ae5 100644 --- a/frappe/website/doctype/blog_post/templates/blog_post_row.html +++ b/frappe/website/doctype/blog_post/templates/blog_post_row.html @@ -1,38 +1,45 @@ {%- set post = doc -%} -
-
-
-
-
-
-
{{ post.category.title }}
-

{{ post.title }}

-

{{ post.intro }}

-
-
- {{ post.full_name }} - · - {{ frappe.format_date(post.published_on) }} - {% if post.comments %} - · - {% if post.comments == 1 %} - {{ _('1 comment') }} - {% else %} - {{ _('{0} comments').format(post.comments) }} - {% endif %} - {% endif %} - {% if post.read_time %} - · - {{ _('{0} min read').format(post.read_time) }} - {% endif %} -
-
-
- {% if post.cover_image %} - {{post.title}} - Cover Image - {% endif %} + +{%- if post.featured -%} + {% set col_class="featured" %} +{%- else -%} + {% set col_class="" %} +{%- endif -%} + +
+
+ {% if post.cover_image %} +
+ + {{post.title}} - Cover Image + +
+ {% endif %} +
+
+
+
+ {%- if post.featured -%} + {{ _('Featured') }} · + {%- endif -%} + {{ post.category.title }} +
+ {%- if post.featured -%} +

{{ post.title }}

+ {%- else -%} +
{{ post.title }}
+ {%- endif -%} +

{{ post.intro }}

+
+
+ +
+ {{ post.full_name }} +
+ {{ frappe.format_date(post.published_on) }} + {% if post.read_time %} · {{ post.read_time }} min read {% endif %}
-
+
\ No newline at end of file From 5f4cf98ee20f5aac96fb1cbb947469414cefe44d Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Thu, 28 May 2020 22:09:19 +0530 Subject: [PATCH 021/485] feat: added featured checkbox and validation --- frappe/website/doctype/blog_post/blog_post.json | 14 +++++++++++--- frappe/website/doctype/blog_post/blog_post.py | 10 ++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/frappe/website/doctype/blog_post/blog_post.json b/frappe/website/doctype/blog_post/blog_post.json index 3d24879c62..3fb18af2cd 100644 --- a/frappe/website/doctype/blog_post/blog_post.json +++ b/frappe/website/doctype/blog_post/blog_post.json @@ -10,12 +10,13 @@ "title", "published_on", "published", - "read_time", + "featured", "disable_comments", "column_break_3", "blog_category", "blogger", "route", + "read_time", "section_break_5", "blog_intro", "content_type", @@ -143,7 +144,8 @@ { "fieldname": "meta_image", "fieldtype": "Attach Image", - "label": "Meta Image" + "label": "Meta Image", + "mandatory_depends_on": "eval:doc.featured" }, { "fieldname": "section_break_20", @@ -167,6 +169,12 @@ "fieldtype": "Int", "label": "Read Time", "read_only": 1 + }, + { + "default": "0", + "fieldname": "featured", + "fieldtype": "Check", + "label": "Featured" } ], "has_web_view": 1, @@ -175,7 +183,7 @@ "is_published_field": "published", "links": [], "max_attachments": 5, - "modified": "2020-04-30 17:32:41.055883", + "modified": "2020-05-28 21:40:26.068480", "modified_by": "Administrator", "module": "Website", "name": "Blog Post", diff --git a/frappe/website/doctype/blog_post/blog_post.py b/frappe/website/doctype/blog_post/blog_post.py index 5516d17780..0492173ad6 100644 --- a/frappe/website/doctype/blog_post/blog_post.py +++ b/frappe/website/doctype/blog_post/blog_post.py @@ -39,8 +39,18 @@ class BlogPost(WebsiteGenerator): if self.published and not self.published_on: self.published_on = today() + if self.featured: + if not self.meta_image: + frappe.throw(_("A featured post must have a cover image")) + self.reset_featured_for_other_blogs() + self.set_read_time() + def reset_featured_for_other_blogs(self): + all_posts = frappe.get_all("Blog Post", {"featured": 1}) + for post in all_posts: + frappe.db.set_value("Blog Post", post.name, "featured", 0) + def on_update(self): super(BlogPost, self).on_update() clear_cache("writers") From 54cd0191aeee4c68354da79279491b95af8bab4c Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Thu, 28 May 2020 22:10:29 +0530 Subject: [PATCH 022/485] feat: update query for new layout --- frappe/website/doctype/blog_post/blog_post.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/frappe/website/doctype/blog_post/blog_post.py b/frappe/website/doctype/blog_post/blog_post.py index 0492173ad6..efa69c55bd 100644 --- a/frappe/website/doctype/blog_post/blog_post.py +++ b/frappe/website/doctype/blog_post/blog_post.py @@ -208,6 +208,9 @@ def get_blog_list(doctype, txt=None, filters=None, limit_start=0, limit_page_len select t1.title, t1.name, t1.blog_category, t1.route, t1.published_on, t1.read_time, t1.published_on as creation, + t1.read_time as read_time, + t1.featured as featured, + t1.meta_image as cover_image, t1.content as content, t1.content_type as content_type, t1.content_html as content_html, @@ -223,7 +226,7 @@ def get_blog_list(doctype, txt=None, filters=None, limit_start=0, limit_page_len where ifnull(t1.published,0)=1 and t1.blogger = t2.name %(condition)s - order by published_on desc, name asc + order by featured desc, published_on desc, name asc limit %(start)s, %(page_len)s""" % { "start": limit_start, "page_len": limit_page_length, "condition": (" and " + " and ".join(conditions)) if conditions else "" @@ -232,9 +235,9 @@ def get_blog_list(doctype, txt=None, filters=None, limit_start=0, limit_page_len posts = frappe.db.sql(query, as_dict=1) for post in posts: - post.content = get_html_content_based_on_type(post, 'content', post.content_type) - post.cover_image = find_first_image(post.content) + if not post.cover_image: + post.cover_image = find_first_image(post.content) post.published = global_date_format(post.creation) post.content = strip_html_tags(post.content) @@ -247,7 +250,7 @@ def get_blog_list(doctype, txt=None, filters=None, limit_start=0, limit_page_len post.avatar = post.avatar or "" post.category = frappe.db.get_value('Blog Category', post.blog_category, - ['route', 'title'], as_dict=True) + ['name', 'route', 'title'], as_dict=True) if post.avatar and (not "http:" in post.avatar and not "https:" in post.avatar) and not post.avatar.startswith("/"): post.avatar = "/" + post.avatar From 806c2ac0b6c6ec4b0c584e56de14aac49b6aedc7 Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Thu, 28 May 2020 23:12:23 +0530 Subject: [PATCH 023/485] fix: Move Importer/Exporter to Data Import Beta --- .../exporter_new.py => data_import_beta/exporter.py} | 0 .../doctype/{data_import => data_import_beta}/importer_new.py | 0 .../test_exporter_new.py => data_import_beta/test_exporter.py} | 2 +- .../test_importer_new.py => data_import_beta/test_importer.py} | 2 +- 4 files changed, 2 insertions(+), 2 deletions(-) rename frappe/core/doctype/{data_import/exporter_new.py => data_import_beta/exporter.py} (100%) rename frappe/core/doctype/{data_import => data_import_beta}/importer_new.py (100%) rename frappe/core/doctype/{data_import/test_exporter_new.py => data_import_beta/test_exporter.py} (94%) rename frappe/core/doctype/{data_import/test_importer_new.py => data_import_beta/test_importer.py} (97%) diff --git a/frappe/core/doctype/data_import/exporter_new.py b/frappe/core/doctype/data_import_beta/exporter.py similarity index 100% rename from frappe/core/doctype/data_import/exporter_new.py rename to frappe/core/doctype/data_import_beta/exporter.py diff --git a/frappe/core/doctype/data_import/importer_new.py b/frappe/core/doctype/data_import_beta/importer_new.py similarity index 100% rename from frappe/core/doctype/data_import/importer_new.py rename to frappe/core/doctype/data_import_beta/importer_new.py diff --git a/frappe/core/doctype/data_import/test_exporter_new.py b/frappe/core/doctype/data_import_beta/test_exporter.py similarity index 94% rename from frappe/core/doctype/data_import/test_exporter_new.py rename to frappe/core/doctype/data_import_beta/test_exporter.py index 0d3aedb033..8e63019ecc 100644 --- a/frappe/core/doctype/data_import/test_exporter_new.py +++ b/frappe/core/doctype/data_import_beta/test_exporter.py @@ -5,7 +5,7 @@ from __future__ import unicode_literals import unittest import frappe -from frappe.core.doctype.data_import.exporter_new import Exporter +from frappe.core.doctype.data_import_beta.exporter import Exporter class TestExporter(unittest.TestCase): diff --git a/frappe/core/doctype/data_import/test_importer_new.py b/frappe/core/doctype/data_import_beta/test_importer.py similarity index 97% rename from frappe/core/doctype/data_import/test_importer_new.py rename to frappe/core/doctype/data_import_beta/test_importer.py index d6349daa55..d1db8f84ce 100644 --- a/frappe/core/doctype/data_import/test_importer_new.py +++ b/frappe/core/doctype/data_import_beta/test_importer.py @@ -6,7 +6,7 @@ from __future__ import unicode_literals import datetime import unittest import frappe -from frappe.core.doctype.data_import.importer_new import Importer +from frappe.core.doctype.data_import_beta.importer import Importer content_empty_rows = '''title,start_date,idx,show_title ,,, From 429ff2ef87160c0b3e026aedf6383efa9528f33b Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Thu, 28 May 2020 23:16:46 +0530 Subject: [PATCH 024/485] fix: Refactor Data Import - Break Importer into classes ImportFile, Row, Column, Header - Show warnings section before import preview --- frappe/cache_manager.py | 2 +- .../data_import_beta/data_import_beta.css | 3 + .../data_import_beta/data_import_beta.js | 14 +- .../data_import_beta/data_import_beta.json | 80 +- .../data_import_beta/data_import_beta.py | 18 +- .../doctype/data_import_beta/importer_new.py | 1789 +++++++++-------- .../js/frappe/data_import/data_exporter.js | 26 +- .../js/frappe/data_import/import_preview.js | 6 +- frappe/public/less/form.less | 7 +- frappe/utils/data.py | 72 + 10 files changed, 1078 insertions(+), 939 deletions(-) create mode 100644 frappe/core/doctype/data_import_beta/data_import_beta.css diff --git a/frappe/cache_manager.py b/frappe/cache_manager.py index 4560680653..92d12289c6 100644 --- a/frappe/cache_manager.py +++ b/frappe/cache_manager.py @@ -24,7 +24,7 @@ user_cache_keys = ("bootinfo", "user_recent", "roles", "user_doc", "lang", "has_role:Page", "has_role:Report") doctype_cache_keys = ("meta", "form_meta", "table_columns", "last_modified", - "linked_doctypes", 'notifications', 'workflow' ,'energy_point_rule_map') + "linked_doctypes", 'notifications', 'workflow' ,'energy_point_rule_map', 'data_import_column_header_map') def clear_user_cache(user=None): diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.css b/frappe/core/doctype/data_import_beta/data_import_beta.css new file mode 100644 index 0000000000..5206540a33 --- /dev/null +++ b/frappe/core/doctype/data_import_beta/data_import_beta.css @@ -0,0 +1,3 @@ +.warnings .warning { + margin-bottom: 40px; +} diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.js b/frappe/core/doctype/data_import_beta/data_import_beta.js index 72404c74f4..1c648621d8 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.js +++ b/frappe/core/doctype/data_import_beta/data_import_beta.js @@ -57,7 +57,7 @@ frappe.ui.form.on('Data Import Beta', { frm.set_query('reference_doctype', () => { return { filters: { - allow_import: 1 + name: ['in', frappe.boot.user.can_import] } }; }); @@ -236,7 +236,7 @@ frappe.ui.form.on('Data Import Beta', { frm .call({ method: 'get_preview_from_template', - args: { data_import: frm.doc.name }, + args: { data_import: frm.doc.name, import_file: frm.doc.import_file }, error_handlers: { TimestampMismatchError() { // ignore this error @@ -331,8 +331,8 @@ frappe.ui.form.on('Data Import Beta', { }) .join(''); return ` -
-
${__('Row {0}', [row_number])}
+
+
${__('Row {0}', [row_number])}
    ${message}
`; @@ -346,8 +346,8 @@ frappe.ui.form.on('Data Import Beta', { header = __('Column {0}', [warning.col]); } return ` -
-
${header}
+
+
${header}
${warning.message}
`; @@ -355,7 +355,7 @@ frappe.ui.form.on('Data Import Beta', { .join(''); frm.get_field('import_warnings').$wrapper.html(`
-
${html}
+
${html}
`); }, diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.json b/frappe/core/doctype/data_import_beta/data_import_beta.json index 777af0a071..8876d2246a 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.json +++ b/frappe/core/doctype/data_import_beta/data_import_beta.json @@ -16,11 +16,11 @@ "submit_after_import", "mute_emails", "template_options", - "section_import_preview", - "import_preview", "import_warnings_section", "template_warnings", "import_warnings", + "section_import_preview", + "import_preview", "import_log_section", "import_log", "show_failed_logs", @@ -34,7 +34,9 @@ "label": "Document Type", "options": "DocType", "reqd": 1, - "set_only_once": 1 + "set_only_once": 1, + "show_days": 1, + "show_seconds": 1 }, { "fieldname": "import_type", @@ -43,28 +45,38 @@ "label": "Import Type", "options": "\nInsert New Records\nUpdate Existing Records", "reqd": 1, - "set_only_once": 1 + "set_only_once": 1, + "show_days": 1, + "show_seconds": 1 }, { "depends_on": "eval:!doc.__islocal", "fieldname": "import_file", "fieldtype": "Attach", "in_list_view": 1, - "label": "Import File" + "label": "Import File", + "show_days": 1, + "show_seconds": 1 }, { "fieldname": "import_preview", "fieldtype": "HTML", - "label": "Import Preview" + "label": "Import Preview", + "show_days": 1, + "show_seconds": 1 }, { "fieldname": "section_import_preview", "fieldtype": "Section Break", - "label": "Preview" + "label": "Preview", + "show_days": 1, + "show_seconds": 1 }, { "fieldname": "column_break_5", - "fieldtype": "Column Break" + "fieldtype": "Column Break", + "show_days": 1, + "show_seconds": 1 }, { "fieldname": "template_options", @@ -72,23 +84,31 @@ "hidden": 1, "label": "Template Options", "options": "JSON", - "read_only": 1 + "read_only": 1, + "show_days": 1, + "show_seconds": 1 }, { "fieldname": "import_log", "fieldtype": "Code", "label": "Import Log", - "options": "JSON" + "options": "JSON", + "show_days": 1, + "show_seconds": 1 }, { "fieldname": "import_log_section", "fieldtype": "Section Break", - "label": "Import Log" + "label": "Import Log", + "show_days": 1, + "show_seconds": 1 }, { "fieldname": "import_log_preview", "fieldtype": "HTML", - "label": "Import Log Preview" + "label": "Import Log Preview", + "show_days": 1, + "show_seconds": 1 }, { "default": "Pending", @@ -97,56 +117,72 @@ "hidden": 1, "label": "Status", "options": "Pending\nSuccess\nPartial Success\nError", - "read_only": 1 + "read_only": 1, + "show_days": 1, + "show_seconds": 1 }, { "fieldname": "template_warnings", "fieldtype": "Code", "hidden": 1, "label": "Template Warnings", - "options": "JSON" + "options": "JSON", + "show_days": 1, + "show_seconds": 1 }, { "default": "0", "fieldname": "submit_after_import", "fieldtype": "Check", "label": "Submit After Import", - "set_only_once": 1 + "set_only_once": 1, + "show_days": 1, + "show_seconds": 1 }, { "fieldname": "import_warnings_section", "fieldtype": "Section Break", - "label": "Warnings" + "label": "Warnings", + "show_days": 1, + "show_seconds": 1 }, { "fieldname": "import_warnings", "fieldtype": "HTML", - "label": "Import Warnings" + "label": "Import Warnings", + "show_days": 1, + "show_seconds": 1 }, { "depends_on": "reference_doctype", "fieldname": "download_template", "fieldtype": "Button", - "label": "Download Template" + "label": "Download Template", + "show_days": 1, + "show_seconds": 1 }, { "default": "1", "fieldname": "mute_emails", "fieldtype": "Check", "label": "Don't Send Emails", - "set_only_once": 1 + "set_only_once": 1, + "show_days": 1, + "show_seconds": 1 }, { "default": "0", "fieldname": "show_failed_logs", "fieldtype": "Check", - "label": "Show Failed Logs" + "label": "Show Failed Logs", + "show_days": 1, + "show_seconds": 1 } ], "hide_toolbar": 1, "links": [], - "modified": "2020-02-17 15:35:04.386098", - "modified_by": "faris@erpnext.com", + "modified": "2020-05-28 22:11:38.266208", + "modified_by": "Administrator", "module": "Core", "name": "Data Import Beta", "owner": "Administrator", diff --git a/frappe/core/doctype/data_import_beta/data_import_beta.py b/frappe/core/doctype/data_import_beta/data_import_beta.py index 8f12bd20ed..23e0681011 100644 --- a/frappe/core/doctype/data_import_beta/data_import_beta.py +++ b/frappe/core/doctype/data_import_beta/data_import_beta.py @@ -5,8 +5,9 @@ from __future__ import unicode_literals import frappe from frappe.model.document import Document -from frappe.core.doctype.data_import.importer_new import Importer -from frappe.core.doctype.data_import.exporter_new import Exporter + +from frappe.core.doctype.data_import_beta.importer import Importer +from frappe.core.doctype.data_import_beta.exporter import Exporter from frappe.core.page.background_jobs.background_jobs import get_info from frappe.utils.background_jobs import enqueue from frappe import _ @@ -25,7 +26,10 @@ class DataImportBeta(Document): # validate template self.get_importer() - def get_preview_from_template(self): + def get_preview_from_template(self, import_file=None): + if import_file: + self.import_file = import_file + if not self.import_file: return @@ -62,8 +66,8 @@ class DataImportBeta(Document): @frappe.whitelist() -def get_preview_from_template(data_import): - return frappe.get_doc("Data Import Beta", data_import).get_preview_from_template() +def get_preview_from_template(data_import, import_file): + return frappe.get_doc("Data Import Beta", data_import).get_preview_from_template(import_file) @frappe.whitelist() @@ -81,8 +85,8 @@ def start_import(data_import): frappe.db.rollback() data_import.db_set("status", "Error") frappe.log_error(title=data_import.name) - frappe.db.commit() - frappe.publish_realtime("data_import_refresh", {"data_import": data_import.name}) + + frappe.publish_realtime("data_import_refresh", {"data_import": data_import.name}) @frappe.whitelist() diff --git a/frappe/core/doctype/data_import_beta/importer_new.py b/frappe/core/doctype/data_import_beta/importer_new.py index cbb2ee482b..02721fb93f 100644 --- a/frappe/core/doctype/data_import_beta/importer_new.py +++ b/frappe/core/doctype/data_import_beta/importer_new.py @@ -1,12 +1,12 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and Contributors +# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -import io +from __future__ import unicode_literals import os -import json -import timeit +import io import frappe +import timeit +import json from datetime import datetime from frappe import _ from frappe.utils import cint, flt, update_progress_bar, cstr, DATETIME_FORMAT @@ -15,65 +15,406 @@ from frappe.utils.xlsxutils import ( read_xlsx_file_from_attached_file, read_xls_file_from_attached_file, ) -from frappe.model import no_value_fields, table_fields +from frappe.model import no_value_fields, table_fields as table_fieldtypes -INVALID_VALUES = ["", None] +INVALID_VALUES = ("", None) MAX_ROWS_IN_PREVIEW = 10 INSERT = "Insert New Records" UPDATE = "Update Existing Records" -# pylint: disable=R0201 + class Importer: - def __init__( - self, doctype, data_import=None, file_path=None, content=None, console=False - ): + def __init__(self, doctype, data_import=None, import_type=None, console=False): self.doctype = doctype - self.template_options = frappe._dict({"remap_column": {}}) self.console = console - if data_import: - self.data_import = data_import - if self.data_import.template_options: - template_options = frappe.parse_json(self.data_import.template_options) - self.template_options.update(template_options) - self.import_type = self.data_import.import_type + self.data_import = data_import + if not self.data_import: + self.data_import = frappe.get_doc(doctype="Data Import Beta") + if import_type: + self.data_import.import_type = import_type + + self.template_options = frappe.parse_json(self.data_import.template_options or "{}") + self.import_type = self.data_import.import_type + + self.import_file = ImportFile( + doctype, data_import.import_file, self.template_options, self.import_type + ) + + def get_data_for_import_preview(self): + return self.import_file.get_data_for_import_preview() + + def before_import(self): + # set user lang for translations + frappe.cache().hdel("lang", frappe.session.user) + frappe.set_user_lang(frappe.session.user) + + # set flags + frappe.flags.in_import = True + frappe.flags.mute_emails = self.data_import.mute_emails + + self.data_import.db_set("template_warnings", "") + + def import_data(self): + self.before_import() + + # parse docs from rows + payloads = self.import_file.get_payloads_for_import() + + # dont import if there are non-ignorable warnings + warnings = self.import_file.get_warnings() + warnings = [w for w in warnings if w.get("type") != "info"] + + print(warnings) + + if warnings: + if self.console: + self.print_grouped_warnings(warnings) + else: + self.data_import.db_set("template_warnings", json.dumps(warnings)) + return + + # setup import log + if self.data_import.import_log: + import_log = frappe.parse_json(self.data_import.import_log) else: - self.data_import = None + import_log = [] - self.import_type = self.import_type or INSERT + # remove previous failures from import log + import_log = [l for l in import_log if l.get("success") == True] - self.header_row = None - self.data = None - # used to store date formats guessed from data rows per column - self._guessed_date_formats = {} - # used to store eta during import - self.last_eta = 0 - # used to collect warnings during template parsing - # and show them to user - self.warnings = [] - self.meta = frappe.get_meta(doctype) - self.prepare_content(file_path, content) + # get successfully imported rows + imported_rows = [] + for log in import_log: + log = frappe._dict(log) + if log.success: + imported_rows += log.row_indexes + + # start import + total_payload_count = len(payloads) + batch_size = frappe.conf.data_import_batch_size or 1000 + + for batch_index, batched_payloads in enumerate( + frappe.utils.create_batch(payloads, batch_size) + ): + for i, payload in enumerate(batched_payloads): + doc = payload.doc + row_indexes = [row.row_number for row in payload.rows] + current_index = (i + 1) + (batch_index * batch_size) + + if set(row_indexes).intersection(set(imported_rows)): + print("Skipping imported rows", row_indexes) + if total_payload_count > 5: + frappe.publish_realtime( + "data_import_progress", + { + "current": current_index, + "total": total_payload_count, + "skipping": True, + "data_import": self.data_import.name, + }, + ) + continue + + try: + start = timeit.default_timer() + doc = self.process_doc(doc) + processing_time = timeit.default_timer() - start + eta = self.get_eta(current_index, total_payload_count, processing_time) + + if self.console: + update_progress_bar( + "Importing {0} records".format(total_payload_count), + current_index, + total_payload_count, + ) + elif total_payload_count > 5: + frappe.publish_realtime( + "data_import_progress", + { + "current": current_index, + "total": total_payload_count, + "docname": doc.name, + "data_import": self.data_import.name, + "success": True, + "row_indexes": row_indexes, + "eta": eta, + }, + ) + + import_log.append( + frappe._dict(success=True, docname=doc.name, row_indexes=row_indexes) + ) + # commit after every successful import + frappe.db.commit() + + except Exception: + import_log.append( + frappe._dict( + success=False, + exception=frappe.get_traceback(), + messages=frappe.local.message_log, + row_indexes=row_indexes, + ) + ) + frappe.clear_messages() + # rollback if exception + frappe.db.rollback() + + # set status + failures = [l for l in import_log if l.get("success") == False] + if len(failures) == total_payload_count: + status = "Pending" + elif len(failures) > 0: + status = "Partial Success" + else: + status = "Success" + + if self.console: + self.print_import_log(import_log) + else: + self.data_import.db_set("status", status) + self.data_import.db_set("import_log", json.dumps(import_log)) + + self.after_import() + + return import_log + + def after_import(self): + frappe.flags.in_import = False + frappe.flags.mute_emails = False + + def process_doc(self, doc): + if self.import_type == INSERT: + return self.insert_record(doc) + elif self.import_type == UPDATE: + return self.update_record(doc) + + def insert_record(self, doc): + meta = frappe.get_meta(self.doctype) + new_doc = frappe.new_doc(self.doctype) + new_doc.update(doc) + + if (meta.autoname or "").lower() != "prompt": + # name can only be set directly if autoname is prompt + new_doc.set("name", None) + + new_doc.flags.updater_reference = { + "doctype": self.data_import.doctype, + "docname": self.data_import.name, + "label": _("via Data Import"), + } + + new_doc.insert() + if meta.is_submittable and self.data_import.submit_after_import: + new_doc.submit() + return new_doc + + def update_record(self, doc): + existing_doc = frappe.get_doc(self.doctype, doc["name"]) + existing_doc.flags.updater_reference = { + "doctype": self.data_import.doctype, + "docname": self.data_import.name, + "label": _("via Data Import"), + } + existing_doc.update(doc) + existing_doc.save() + return existing_doc + + def get_eta(self, current, total, processing_time): + self.last_eta = getattr(self, "last_eta", 0) + remaining = total - current + eta = processing_time * remaining + if not self.last_eta or eta < self.last_eta: + self.last_eta = eta + return self.last_eta + + +class ImportFile: + def __init__(self, doctype, file, template_options=None, import_type=None): + self.doctype = doctype + self.template_options = template_options or frappe._dict( + column_to_field_map=frappe._dict() + ) + self.column_to_field_map = self.template_options.column_to_field_map + self.import_type = import_type + + self.file_doc = self.file_path = None + if isinstance(file, frappe.model.document.Document) and file.doctype == "File": + self.file_doc = file + elif isinstance(file, frappe.string_types): + if frappe.db.exists("File", {"file_url": file}): + self.file_doc = frappe.get_doc("File", {"file_url": file}) + elif os.path.exists(file): + self.file_path = file + + if not self.file_doc and not self.file_path: + frappe.throw(_("Invalid template file for import")) + + self.raw_data = self.get_data_from_template_file() self.parse_data_from_template() - def prepare_content(self, file_path, content): + def get_data_from_template_file(self): + content = None extension = None - if self.data_import and self.data_import.import_file: - file_doc = frappe.get_doc("File", {"file_url": self.data_import.import_file}) - parts = file_doc.get_extension() + + if self.file_doc: + parts = self.file_doc.get_extension() extension = parts[1] - content = file_doc.get_content() + content = self.file_doc.get_content() extension = extension.lstrip(".") - if file_path: - content, extension = self.read_file(file_path) + elif self.file_path: + content, extension = self.read_file(self.file_path) + + if not content: + frappe.throw(_("Invalid or corrupted content for import")) if not extension: extension = "csv" if content: - self.read_content(content, extension) + return self.read_content(content, extension) - self.validate_template_content() + def parse_data_from_template(self): + header = None + data = [] + + for i, row in enumerate(self.raw_data): + if all(v in INVALID_VALUES for v in row): + # empty row + continue + + if not header: + header = Header(i, row, self.doctype, self.raw_data, self.column_to_field_map) + else: + row_obj = Row(i, row, self.doctype, header, self.import_type) + data.append(row_obj) + + self.header = header + self.columns = self.header.columns + self.data = data + + if len(data) <= 1: + frappe.throw( + _("Import template should contain a Header and atleast one row."), + title=_("Template Error"), + ) + + def get_data_for_import_preview(self): + """Adds a serial number column as the first column""" + + columns = [frappe._dict({"header_title": "Sr. No", "skip_import": True})] + columns += [col.as_dict() for col in self.columns] + data = [[row.row_number] + row.as_list() for row in self.data] + + warnings = self.get_warnings() + + out = frappe._dict() + out.data = data + out.columns = columns + out.warnings = warnings + total_number_of_rows = len(out.data) + if total_number_of_rows > MAX_ROWS_IN_PREVIEW: + out.data = out.data[:MAX_ROWS_IN_PREVIEW] + out.max_rows_exceeded = True + out.max_rows_in_preview = MAX_ROWS_IN_PREVIEW + out.total_number_of_rows = total_number_of_rows + return out + + def get_payloads_for_import(self): + payloads = [] + # make a copy + data = list(self.data) + while data: + doc, rows, data = self.parse_next_row_for_import(data) + payloads.append(frappe._dict(doc=doc, rows=rows)) + return payloads + + def parse_next_row_for_import(self, data): + """ + Parses rows that make up a doc. A doc maybe built from a single row or multiple rows. + Returns the doc, rows, and data without the rows. + """ + doctypes = self.header.doctypes + + # first row is included by default + first_row = data[0] + rows = [first_row] + + # if there are child doctypes, find the subsequent rows + if len(doctypes) > 1: + # subsequent rows either dont have any parent value set + # or have the same value as the parent row + # we include a row if either of conditions match + parent_column_indexes = self.header.get_column_indexes(self.doctype) + parent_row_values = first_row.get_values(parent_column_indexes) + + data_without_first_row = data[1:] + for row in data_without_first_row: + row_values = row.get_values(parent_column_indexes) + # if the row is blank, it's a child row doc + if all([v in INVALID_VALUES for v in row_values]): + rows.append(row) + continue + # if the row has same values as parent row, it's a child row doc + if row_values == parent_row_values: + rows.append(row) + continue + # if any of those conditions dont match, it's the next doc + break + + parsed_docs = {} + parent_doc = None + for row in rows: + for doctype, table_df in doctypes: + if doctype == self.doctype and not parent_doc: + parent_doc = row.parse_doc(doctype) + + if doctype != self.doctype and table_df: + child_doc = row.parse_doc(doctype, parent_doc, table_df) + parent_doc[table_df.fieldname] = parent_doc.get(table_df.fieldname, []) + parent_doc[table_df.fieldname].append(child_doc) + + doc = parent_doc + # check if there is atleast one row for mandatory table fields + meta = frappe.get_meta(self.doctype) + mandatory_table_fields = [ + df + for df in meta.fields + if df.fieldtype in table_fieldtypes + and df.reqd + and len(doc.get(df.fieldname, [])) == 0 + ] + if len(mandatory_table_fields) == 1: + self.warnings.append( + { + "row": first_row.row_number, + "message": _("There should be atleast one row for {0} table").format( + mandatory_table_fields[0].label + ), + } + ) + elif mandatory_table_fields: + fields_string = ", ".join([df.label for df in mandatory_table_fields]) + message = _("There should be atleast one row for the following tables: {0}").format( + fields_string + ) + self.warnings.append({"row": first_row.row_number, "message": message}) + + return doc, rows, data[len(rows) :] + + def get_warnings(self): + warnings = [] + for col in self.header.columns: + warnings += col.warnings + + for row in self.data: + warnings += row.warnings + + return warnings + + ###### def read_file(self, file_path): extn = file_path.split(".")[1] @@ -98,18 +439,10 @@ class Importer: elif extension == "xls": data = read_xls_file_from_attached_file(content) - data = self.remove_empty_rows_and_columns(data) - - if len(data) <= 1: - frappe.throw( - _("Import template should contain a Header and atleast one row."), title=error_title - ) - - self.header_row = data[0] - self.data = data[1:] + return data def validate_template_content(self): - column_count = len(self.header_row) + column_count = len(self.columns) if any([len(row) != column_count and len(row) != 0 for row in self.data]): frappe.throw( _("Number of columns does not match with data"), title=_("Invalid Template") @@ -151,45 +484,324 @@ class Importer: return data_without_empty_rows_and_columns - def get_data_for_import_preview(self): - out = frappe._dict() - out.data = list(self.rows) - out.columns = self.columns - out.warnings = self.warnings - total_number_of_rows = len(out.data) - if total_number_of_rows > MAX_ROWS_IN_PREVIEW: - out.data = out.data[:MAX_ROWS_IN_PREVIEW] - out.max_rows_exceeded = True - out.max_rows_in_preview = MAX_ROWS_IN_PREVIEW - out.total_number_of_rows = total_number_of_rows - return out - def parse_data_from_template(self): - columns = self.parse_columns_from_header_row() - columns = self.detect_date_formats(columns) - columns, data = self.add_serial_no_column(columns, self.data) +class Row: + link_values_exist_map = {} - self.columns = columns - self.rows = data + def __init__(self, index, row, doctype, header, import_type): + self.index = index + self.row_number = index + 1 + self.doctype = doctype + self.data = row + self.header = header + self.import_type = import_type + self.warnings = [] - def parse_columns_from_header_row(self): - remap_column = self.template_options.remap_column - columns = [] - seen = [] + len_row = len(self.data) + len_columns = len(self.header.columns) + if len_row != len_columns: + less_than_columns = len_row < len_columns + message = ( + "Row has less values than columns" + if less_than_columns + else "Row has more values than columns" + ) + self.warnings.append( + {"row": self.row_number, "message": message,} + ) - df_by_labels_and_fieldnames = self.build_fields_dict_for_column_matching() + def parse_doc(self, doctype, parent_doc=None, table_df=None): + col_indexes = self.header.get_column_indexes(doctype, table_df) + values = self.get_values(col_indexes) + columns = self.header.get_columns(col_indexes) + doc = self._parse_doc(doctype, columns, values, parent_doc, table_df) + return doc - for i, header_title in enumerate(self.header_row): - header_row_index = str(i) - column_number = str(i + 1) - skip_import = False - fieldname = remap_column.get(header_row_index) + def _parse_doc(self, doctype, columns, values, parent_doc=None, table_df=None): + doc = frappe._dict() + if self.import_type == INSERT: + # new_doc returns a dict with default values set + doc = frappe.new_doc( + doctype, + parent_doc=parent_doc, + parentfield=table_df.fieldname if table_df else None, + as_dict=True, + ) - if fieldname and fieldname != "Don't Import": - df = df_by_labels_and_fieldnames.get(fieldname) + # remove standard fields and __islocal + for key in frappe.model.default_fields + ("__islocal",): + doc.pop(key, None) + + for col, value in zip(columns, values): + df = col.df + if value in INVALID_VALUES: + value = None + + if value is not None: + value = self.validate_value(value, col) + + if value is not None: + doc[df.fieldname] = self.parse_value(value, col) + + is_table = frappe.get_meta(doctype).istable + is_update = self.import_type == UPDATE + if is_table and is_update and doc.get("name") in INVALID_VALUES: + # for table rows being inserted in update + # create a new doc with defaults set + new_doc = frappe.new_doc(doctype, as_dict=True) + new_doc.update(doc) + doc = new_doc + + self.check_mandatory_fields(doctype, doc) + return doc + + def validate_value(self, value, col): + df = col.df + if df.fieldtype == "Select": + select_options = df.get_select_options() + if select_options and value not in select_options: + options_string = ", ".join([frappe.bold(d) for d in select_options]) + msg = _("Value must be one of {0}").format(options_string) + self.warnings.append( + { + "row": self.row_number, + "field": df.as_dict(convert_dates_to_str=True), + "message": msg, + } + ) + return + + elif df.fieldtype == "Link": + exists = self.link_exists(value, df) + if not exists: + msg = _("Value {0} missing for {1}").format( + frappe.bold(value), frappe.bold(df.options) + ) + self.warnings.append( + { + "row": self.row_number, + "field": df.as_dict(convert_dates_to_str=True), + "message": msg, + } + ) + return + elif df.fieldtype in ["Date", "Datetime"]: + value = self.get_date(value, col) + if isinstance(value, frappe.string_types): + # value was not parsed as datetime object + self.warnings.append( + { + "row": self.row_number, + "col": col.column_number, + "field": df.as_dict(convert_dates_to_str=True), + "message": _("Value {0} must in {1} format").format( + frappe.bold(value), frappe.bold(get_user_format(col.date_format)) + ), + } + ) + return + + return value + + def link_exists(self, value, df): + key = df.options + "::" + value + if Row.link_values_exist_map.get(key) is None: + Row.link_values_exist_map[key] = frappe.db.exists(df.options, value) + return Row.link_values_exist_map.get(key) + + def parse_value(self, value, col): + df = col.df + if isinstance(value, datetime) and df.fieldtype in ["Date", "Datetime"]: + return value + + value = cstr(value) + + # convert boolean values to 0 or 1 + valid_check_values = ["t", "f", "true", "false", "yes", "no", "y", "n"] + if df.fieldtype == "Check" and value.lower().strip() in valid_check_values: + value = value.lower().strip() + value = 1 if value in ["t", "true", "y", "yes"] else 0 + + if df.fieldtype in ["Int", "Check"]: + value = cint(value) + elif df.fieldtype in ["Float", "Percent", "Currency"]: + value = flt(value) + elif df.fieldtype in ["Date", "Datetime"]: + value = self.get_date(value, col) + + return value + + def get_date(self, value, column): + date_format = column.date_format + if date_format: + try: + return datetime.strptime(value, date_format) + except ValueError: + # ignore date values that dont match the format + # import will break for these values later + pass + return value + + def check_mandatory_fields(self, doctype, doc): + """If import type is Insert: + Check for mandatory fields (except table fields) in doc + if import type is Update: + Check for name field or autoname field in doc + """ + meta = frappe.get_meta(doctype) + if self.import_type == UPDATE: + if meta.istable: + # when updating records with table rows, + # there are two scenarios: + # 1. if row 'name' is provided in the template + # the table row will be updated + # 2. if row 'name' is not provided + # then a new row will be added + # so we dont need to check for mandatory + return + + id_field = self.get_id_field(doctype) + if doc.get(id_field.fieldname) in INVALID_VALUES: + self.warnings.append( + { + "row": self.row_number, + "message": _("{0} is a mandatory field").format(id_field.label), + } + ) + return + + fields = [ + df + for df in meta.fields + if df.fieldtype not in table_fieldtypes + and df.reqd + and doc.get(df.fieldname) in INVALID_VALUES + ] + + if not fields: + return + + if len(fields) == 1: + self.warnings.append( + { + "row": self.row_number, + "message": _("{0} is a mandatory field").format(fields[0].label), + } + ) + else: + fields_string = ", ".join([df.label for df in fields]) + self.warnings.append( + { + "row": self.row_number, + "message": _("{0} are mandatory fields").format(fields_string), + } + ) + + def get_id_field(self, doctype): + autoname_field = self.get_autoname_field(doctype) + if autoname_field: + return autoname_field + return frappe._dict({"label": "ID", "fieldname": "name", "fieldtype": "Data"}) + + def get_autoname_field(self, doctype): + meta = frappe.get_meta(doctype) + if meta.autoname and meta.autoname.startswith("field:"): + fieldname = meta.autoname[len("field:") :] + return meta.get_field(fieldname) + + def get_values(self, indexes): + return [self.data[i] for i in indexes] + + def get(self, index): + return self.data[index] + + def as_list(self): + return self.data + + +class Header(Row): + def __init__(self, index, row, doctype, raw_data, column_to_field_map): + self.index = index + self.row_number = index + 1 + self.data = row + self.doctype = doctype + + self.seen = [] + self.columns = [] + + for j, header in enumerate(row): + column_values = [get_item_at_index(r, j) for r in raw_data] + column = Column( + j, header, self.doctype, column_values, column_to_field_map.get(header), self.seen + ) + self.seen.append(header) + self.columns.append(column) + + doctypes = [] + for col in self.columns: + if not col.df: + continue + if col.df.parent == self.doctype: + doctypes.append((col.df.parent, None)) + else: + doctypes.append((col.df.parent, col.df.child_table_df)) + + self.doctypes = sorted( + list(set(doctypes)), key=lambda x: -1 if x[0] == self.doctype else 1 + ) + + def get_column_indexes(self, doctype, tablefield=None): + return [ + col.index + for col in self.columns + if not col.skip_import and col.df and col.df.parent == doctype + ] + + def get_columns(self, indexes): + return [self.columns[i] for i in indexes] + + def get_docfields(self, indexes): + return [col.df for col in self.get_columns(indexes)] + + +class Column: + seen = [] + fields_column_map = {} + + def __init__(self, index, header, doctype, column_values, map_to_field=None, seen=[]): + self.index = index + self.column_number = index + 1 + self.doctype = doctype + self.header_title = header + self.column_values = column_values + self.map_to_field = map_to_field + self.seen = seen + + self.date_format = None + self.df = None + self.skip_import = None + self.warnings = [] + + self.meta = frappe.get_meta(doctype) + self.parse() + self.parse_date_format() + + def parse(self): + # df_by_labels_and_fieldnames = Column.build_fields_dict_for_column_matching( + # self.doctype + # ) + + header_title = self.header_title + header_row_index = str(self.index) + column_number = str(self.column_number) + skip_import = False + + if self.map_to_field and self.map_to_field != "Don't Import": + df = get_df_for_column_header(self.doctype, self.map_to_field) + # df = df_by_labels_and_fieldnames.get(self.map_to_field) + if df: self.warnings.append( { - "col": column_number, "message": _("Mapping column {0} to field {1}").format( frappe.bold(header_title or "Untitled Column"), frappe.bold(df.label) ), @@ -197,138 +809,129 @@ class Importer: } ) else: - df = df_by_labels_and_fieldnames.get(header_title) - - if not df: - skip_import = True - else: - skip_import = False - - if header_title in seen: self.warnings.append( { - "col": column_number, - "message": _("Skipping Duplicate Column {0}").format(frappe.bold(header_title)), - "type": "info", - } - ) - df = None - skip_import = True - elif fieldname == "Don't Import": - skip_import = True - self.warnings.append( - { - "col": column_number, - "message": _("Skipping column {0}").format(frappe.bold(header_title)), - "type": "info", - } - ) - elif header_title and not df: - self.warnings.append( - { - "col": column_number, - "message": _("Cannot match column {0} with any field").format( - frappe.bold(header_title) + "message": _("Could not map column {0} to field {1}").format( + column_number, self.map_to_field ), "type": "info", } ) - elif not header_title and not df: - self.warnings.append( - {"col": column_number, "message": _("Skipping Untitled Column"), "type": "info"} - ) + else: + df = get_df_for_column_header(self.doctype, header_title) + # df = df_by_labels_and_fieldnames.get(header_title) - columns.append( - frappe._dict( - df=df, - skip_import=skip_import, - header_title=header_title, - column_number=column_number, - index=i, - ) - ) - seen.append(header_title) + if not df: + skip_import = True + else: + skip_import = False - return columns - - def build_fields_dict_for_column_matching(self): - """ - Build a dict with various keys to match with column headers and value as docfield - The keys can be label or fieldname - { - 'Customer': df1, - 'customer': df1, - 'Due Date': df2, - 'due_date': df2, - 'Item Code (Sales Invoice Item)': df3, - 'Sales Invoice Item:item_code': df3, - } - """ - out = {} - - table_doctypes = [df.options for df in self.meta.get_table_fields()] - doctypes = table_doctypes + [self.doctype] - for doctype in doctypes: - # name field - name_key = "ID" if self.doctype == doctype else "ID ({})".format(doctype) - name_df = frappe._dict( + if header_title in self.seen: + self.warnings.append( { - "fieldtype": "Data", - "fieldname": "name", - "label": "ID", - "reqd": self.import_type == UPDATE, - "parent": doctype, + "col": column_number, + "message": _("Skipping Duplicate Column {0}").format(frappe.bold(header_title)), + "type": "info", } ) - out[name_key] = name_df - out["name"] = name_df + df = None + skip_import = True + elif self.map_to_field == "Don't Import": + skip_import = True + self.warnings.append( + { + "col": column_number, + "message": _("Skipping column {0}").format(frappe.bold(header_title)), + "type": "info", + } + ) + elif header_title and not df: + self.warnings.append( + { + "col": column_number, + "message": _("Cannot match column {0} with any field").format( + frappe.bold(header_title) + ), + "type": "info", + } + ) + elif not header_title and not df: + self.warnings.append( + {"col": column_number, "message": _("Skipping Untitled Column"), "type": "info"} + ) - # other fields - meta = frappe.get_meta(doctype) - fields = self.get_standard_fields(doctype) + meta.fields - for df in fields: - fieldtype = df.fieldtype or "Data" - parent = df.parent or self.doctype - if fieldtype not in no_value_fields: - if self.doctype == doctype: - # for parent doctypes keys will be - # Label - # label - # Label (label) - if not out.get(df.label): - # if Label is already set, don't set it again - # in case of duplicate column headers - out[df.label] = df - out[df.fieldname] = df - label_with_fieldname = "{0} ({1})".format(df.label, df.fieldname) - out[label_with_fieldname] = df - else: - # for child doctypes keys will be - # Label (Child DocType) - # Child DocType:label - # Label (label) (Child DocType) - label = "{0} ({1})".format(df.label, parent) - fieldname = "{0}:{1}".format(doctype, df.fieldname) - label_with_fieldname = "{0} ({1}) ({2})".format(df.label, df.fieldname, parent) - if not out.get(label): - # if Label is already set, don't set it again - # in case of duplicate column headers - out[label] = df - out[fieldname] = df - out[label_with_fieldname] = df + self.df = df + self.skip_import = skip_import - # if autoname is based on field - # add an entry for "ID (Autoname Field)" - autoname_field = self.get_autoname_field(self.doctype) - if autoname_field: - out["ID ({})".format(autoname_field.label)] = autoname_field - # ID field should also map to the autoname field - out["ID"] = autoname_field - out["name"] = autoname_field + def parse_date_format(self): + if self.df and self.df.fieldtype in ("Date", "Time", "Datetime"): + self.date_format = self.guess_date_format_for_column() - return out + def guess_date_format_for_column(self): + """ Guesses date format for a column by parsing the first 100 values in the column, + getting the date format and then returning the one which has the maximum frequency + """ + PARSE_ROW_COUNT = 100 - def get_standard_fields(self, doctype): + date_formats = [ + frappe.utils.guess_date_format(d) for d in self.column_values if isinstance(d, str) + ] + date_formats = [d for d in date_formats if d] + if not date_formats: + return + + unique_date_formats = set(date_formats) + print(unique_date_formats) + max_occurred_date_format = max(unique_date_formats, key=date_formats.count) + + # fmt: off + message = _("The column {0} has {1} different date formats. Automatically setting {2} as the default format as it is the most common. Please change other values in this column to this format.") + # fmt: on + user_date_format = get_user_format(max_occurred_date_format) + self.warnings.append( + { + "col": self.column_number, + "message": message.format( + frappe.bold(self.header_title), + len(unique_date_formats), + frappe.bold(user_date_format), + ), + "type": "info", + } + ) + + return max_occurred_date_format + + def as_dict(self): + d = frappe._dict() + d.index = self.index + d.column_number = self.column_number + d.doctype = self.doctype + d.header_title = self.header_title + d.column_values = self.column_values + d.map_to_field = self.map_to_field + d.date_format = self.date_format + d.df = self.df + d.skip_import = self.skip_import + d.warnings = self.warnings + return d + + +def build_fields_dict_for_column_matching(parent_doctype): + """ + Build a dict with various keys to match with column headers and value as docfield + The keys can be label or fieldname + { + 'Customer': df1, + 'customer': df1, + 'Due Date': df2, + 'due_date': df2, + 'Item Code (Sales Invoice Item)': df3, + 'Sales Invoice Item:item_code': df3, + } + """ + + def get_standard_fields(doctype): meta = frappe.get_meta(doctype) if meta.istable: standard_fields = [ @@ -350,714 +953,124 @@ class Importer: out.append(df) return out - def detect_date_formats(self, columns): - for col in columns: - if col.df and col.df.fieldtype in ["Date", "Time", "Datetime"]: - col.date_format = self.guess_date_format_for_column(col, columns) - return columns + parent_meta = frappe.get_meta(parent_doctype) + out = {} - def add_serial_no_column(self, columns, data): - columns_with_serial_no = [ - frappe._dict({"header_title": "Sr. No", "skip_import": True}) - ] + columns + # doctypes and fieldname if it is a child doctype + doctypes = [[parent_doctype, None]] + [ + [df.options, df] for df in parent_meta.get_table_fields() + ] - # update index for each column - for i, col in enumerate(columns_with_serial_no): - col.index = i + for doctype, table_df in doctypes: + # name field + name_by_label = ( + "ID" if doctype == parent_doctype else "ID ({0})".format(table_df.label) + ) + name_by_fieldname = ( + "name" if doctype == parent_doctype else "{0}.name".format(table_df.fieldname) + ) + name_df = frappe._dict( + { + "fieldtype": "Data", + "fieldname": "name", + "label": "ID", + "reqd": 1, # self.import_type == UPDATE, + "parent": doctype, + } + ) - data_with_serial_no = [] - for i, row in enumerate(data): - data_with_serial_no.append([self.row_index_map[i] + 1] + row) + if doctype != parent_doctype: + name_df.is_child_table_field = True + name_df.child_table_df = table_df - return columns_with_serial_no, data_with_serial_no + out[name_by_label] = name_df + out[name_by_fieldname] = name_df - def parse_value(self, value, df): - if isinstance(value, datetime) and df.fieldtype in ["Date", "Datetime"]: - return value - - value = cstr(value) - - # convert boolean values to 0 or 1 - valid_check_values = ["t", "f", "true", "false", "yes", "no", "y", "n"] - if df.fieldtype == "Check" and value.lower().strip() in valid_check_values: - value = value.lower().strip() - value = 1 if value in ["t", "true", "y", "yes"] else 0 - - if df.fieldtype in ["Int", "Check"]: - value = cint(value) - elif df.fieldtype in ["Float", "Percent", "Currency"]: - value = flt(value) - elif df.fieldtype in ["Date", "Datetime"]: - value = self.parse_date_format(value, df) - - return value - - def parse_date_format(self, value, df): - date_format = self.get_date_format_for_df(df) or DATETIME_FORMAT - try: - return datetime.strptime(value, date_format) - except ValueError: - # ignore date values that dont match the format - # import will break for these values later - pass - return value - - def get_date_format_for_df(self, df): - return self._guessed_date_formats.get(df.parent + df.fieldname) - - def guess_date_format_for_column(self, column, columns): - """ Guesses date format for a column by parsing the first 10 values in the column, - getting the date format and then returning the one which has the maximum frequency - """ - PARSE_ROW_COUNT = 10 - - df = column.df - key = df.parent + df.fieldname - - if not self._guessed_date_formats.get(key): - matches = [col for col in columns if col.df == df] - if not matches: - self._guessed_date_formats[key] = None - return - - column = matches[0] - column_index = column.index - - date_values = [ - row[column_index] for row in self.data[:PARSE_ROW_COUNT] if row[column_index] - ] - date_formats = [ - guess_date_format(d) if isinstance(d, str) else None for d in date_values - ] - if not date_formats: - return - max_occurred_date_format = max(set(date_formats), key=date_formats.count) - self._guessed_date_formats[key] = max_occurred_date_format - - return self._guessed_date_formats[key] - - def import_data(self): - # set user lang for translations - frappe.cache().hdel("lang", frappe.session.user) - frappe.set_user_lang(frappe.session.user) - - if not self.console: - self.data_import.db_set("template_warnings", "") - - # set flags - frappe.flags.in_import = True - frappe.flags.mute_emails = self.data_import.mute_emails - - # prepare a map for missing link field values - self.prepare_missing_link_field_values() - - # parse docs from rows - payloads = self.get_payloads_for_import() - - # dont import if there are non-ignorable warnings - warnings = [w for w in self.warnings if w.get("type") != "info"] - if warnings: - if self.console: - self.print_grouped_warnings(warnings) - else: - self.data_import.db_set("template_warnings", json.dumps(warnings)) - frappe.publish_realtime( - "data_import_refresh", {"data_import": self.data_import.name} - ) - return - - # setup import log - if self.data_import.import_log: - import_log = frappe.parse_json(self.data_import.import_log) - else: - import_log = [] - - # remove previous failures from import log - import_log = [l for l in import_log if l.get("success") == True] - - # get successfully imported rows - imported_rows = [] - for log in import_log: - log = frappe._dict(log) - if log.success: - imported_rows += log.row_indexes - - # start import - total_payload_count = len(payloads) - batch_size = frappe.conf.data_import_batch_size or 1000 - - for batch_index, batched_payloads in enumerate( - frappe.utils.create_batch(payloads, batch_size) - ): - for i, payload in enumerate(batched_payloads): - doc = payload.doc - row_indexes = [row[0] for row in payload.rows] - current_index = (i + 1) + (batch_index * batch_size) - - if set(row_indexes).intersection(set(imported_rows)): - print("Skipping imported rows", row_indexes) - if total_payload_count > 5: - frappe.publish_realtime( - "data_import_progress", - { - "current": current_index, - "total": total_payload_count, - "skipping": True, - "data_import": self.data_import.name, - }, - ) - continue - - try: - start = timeit.default_timer() - doc = self.process_doc(doc) - processing_time = timeit.default_timer() - start - eta = self.get_eta(current_index, total_payload_count, processing_time) - - if total_payload_count > 5: - frappe.publish_realtime( - "data_import_progress", - { - "current": current_index, - "total": total_payload_count, - "docname": doc.name, - "data_import": self.data_import.name, - "success": True, - "row_indexes": row_indexes, - "eta": eta, - }, - ) - if self.console: - update_progress_bar( - "Importing {0} records".format(total_payload_count), - current_index, - total_payload_count, - ) - import_log.append( - frappe._dict(success=True, docname=doc.name, row_indexes=row_indexes) + # other fields + fields = get_standard_fields(doctype) + frappe.get_meta(doctype).fields + for df in fields: + fieldtype = df.fieldtype or "Data" + parent = df.parent or parent_doctype + if fieldtype not in no_value_fields: + if parent_doctype == doctype: + # for parent doctypes keys will be + # Label + # label + # Label (label) + if not out.get(df.label): + # if Label is already set, don't set it again + # in case of duplicate column headers + out[df.label] = df + out[df.fieldname] = df + label_with_fieldname = "{0} ({1})".format(df.label, df.fieldname) + out[label_with_fieldname] = df + else: + # in case there are multiple table fields with the same doctype + # for child doctypes keys will be + # Label (Table Field Label) + # table_field.fieldname + table_fields = parent_meta.get( + "fields", {"fieldtype": ["in", table_fieldtypes], "options": parent} ) - # commit after every successful import - frappe.db.commit() + for table_field in table_fields: + by_label = "{0} ({1})".format(df.label, table_field.label) + by_fieldname = "{0}.{1}".format(table_field.fieldname, df.fieldname) - except Exception: - import_log.append( - frappe._dict( - success=False, - exception=frappe.get_traceback(), - messages=frappe.local.message_log, - row_indexes=row_indexes, - ) - ) - frappe.clear_messages() - # rollback if exception - frappe.db.rollback() + # create a new df object to avoid mutation problems + if isinstance(df, dict): + new_df = frappe._dict(df.copy()) + else: + new_df = df.as_dict() - # set status - failures = [l for l in import_log if l.get("success") == False] - if len(failures) == total_payload_count: - status = "Pending" - elif len(failures) > 0: - status = "Partial Success" - else: - status = "Success" + new_df.is_child_table_field = True + new_df.child_table_df = table_field + out[by_label] = new_df + out[by_fieldname] = new_df - if self.console: - self.print_import_log(import_log) - else: - self.data_import.db_set("status", status) - self.data_import.db_set("import_log", json.dumps(import_log)) + # if autoname is based on field + # add an entry for "ID (Autoname Field)" + autoname_field = get_autoname_field(parent_doctype) + if autoname_field: + out["ID ({})".format(autoname_field.label)] = autoname_field + # ID field should also map to the autoname field + out["ID"] = autoname_field + out["name"] = autoname_field - frappe.flags.in_import = False - frappe.flags.mute_emails = False - frappe.publish_realtime("data_import_refresh", {"data_import": self.data_import.name}) + return out - return import_log - def get_payloads_for_import(self): - payloads = [] - # make a copy - data = list(self.rows) - while data: - doc, rows, data = self.parse_next_row_for_import(data) - payloads.append(frappe._dict(doc=doc, rows=rows)) - return payloads +def get_df_for_column_header(doctype, header): + def build_fields_dict_for_doctype(): + return build_fields_dict_for_column_matching(doctype) - def parse_next_row_for_import(self, data): - """ - Parses rows that make up a doc. A doc maybe built from a single row or multiple rows. - Returns the doc, rows, and data without the rows. - """ - doctypes = set([col.df.parent for col in self.columns if col.df and col.df.parent]) + df_by_labels_and_fieldname = frappe.cache().hget( + "data_import_column_header_map", doctype, generator=build_fields_dict_for_doctype + ) + return df_by_labels_and_fieldname.get(header) - # first row is included by default - first_row = data[0] - rows = [first_row] - # if there are child doctypes, find the subsequent rows - if len(doctypes) > 1: - # subsequent rows either dont have any parent value set - # or have the same value as the parent row - # we include a row if either of conditions match - parent_column_indexes = [ - col.index - for col in self.columns - if not col.skip_import and col.df and col.df.parent == self.doctype - ] - parent_row_values = [first_row[i] for i in parent_column_indexes] +# utilities - data_without_first_row = data[1:] - for row in data_without_first_row: - row_values = [row[i] for i in parent_column_indexes] - # if the row is blank, it's a child row doc - if all([v in INVALID_VALUES for v in row_values]): - rows.append(row) - continue - # if the row has same values as parent row, it's a child row doc - if row_values == parent_row_values: - rows.append(row) - continue - # if any of those conditions dont match, it's the next doc - break - def get_column_indexes(doctype): - return [ - col.index - for col in self.columns - if not col.skip_import and col.df and col.df.parent == doctype - ] +def get_autoname_field(doctype): + meta = frappe.get_meta(doctype) + if meta.autoname and meta.autoname.startswith("field:"): + fieldname = meta.autoname[len("field:") :] + return meta.get_field(fieldname) - def validate_value(value, df): - if df.fieldtype == "Select": - select_options = df.get_select_options() - if select_options and value not in select_options: - options_string = ", ".join([frappe.bold(d) for d in select_options]) - msg = _("Value must be one of {0}").format(options_string) - self.warnings.append( - { - "row": row_number, - "field": df.as_dict(convert_dates_to_str=True), - "message": msg, - } - ) - return - elif df.fieldtype == "Link": - d = self.get_missing_link_field_values(df.options) - if value in d.missing_values and not d.one_mandatory: - msg = _("Value {0} missing for {1}").format( - frappe.bold(value), frappe.bold(df.options) - ) - self.warnings.append( - { - "row": row_number, - "field": df.as_dict(convert_dates_to_str=True), - "message": msg, - } - ) - return value +def get_item_at_index(_list, i, default=None): + try: + a = _list[i] + except IndexError: + a = default + return a - return value - def parse_doc(doctype, docfields, values, row_number): - doc = frappe._dict() - if self.import_type == INSERT: - # new_doc returns a dict with default values set - doc = frappe.new_doc(doctype, as_dict=True) - - # remove standard fields and __islocal - for key in frappe.model.default_fields + ("__islocal",): - doc.pop(key, None) - - for df, value in zip(docfields, values): - if value in INVALID_VALUES: - value = None - - if value is not None: - value = validate_value(value, df) - - if value is not None: - doc[df.fieldname] = self.parse_value(value, df) - - is_table = frappe.get_meta(doctype).istable - is_update = self.import_type == UPDATE - if is_table and is_update and doc.get("name") in INVALID_VALUES: - # for table rows being inserted in update - # create a new doc with defaults set - new_doc = frappe.new_doc(doctype, as_dict=True) - new_doc.update(doc) - doc = new_doc - - check_mandatory_fields(doctype, doc, row_number) - return doc - - def check_mandatory_fields(doctype, doc, row_number): - """If import type is Insert: - Check for mandatory fields (except table fields) in doc - if import type is Update: - Check for name field or autoname field in doc - """ - meta = frappe.get_meta(doctype) - if self.import_type == UPDATE: - if meta.istable: - # when updating records with table rows, - # there are two scenarios: - # 1. if row 'name' is provided in the template - # the table row will be updated - # 2. if row 'name' is not provided - # then a new row will be added - # so we dont need to check for mandatory - return - - id_field = self.get_id_field(doctype) - if doc.get(id_field.fieldname) in INVALID_VALUES: - self.warnings.append( - { - "row": row_number, - "message": _("{0} is a mandatory field").format(id_field.label), - } - ) - return - - fields = [ - df - for df in meta.fields - if df.fieldtype not in table_fields - and df.reqd - and doc.get(df.fieldname) in INVALID_VALUES - ] - - if not fields: - return - - if len(fields) == 1: - self.warnings.append( - { - "row": row_number, - "message": _("{0} is a mandatory field").format(fields[0].label), - } - ) - else: - fields_string = ", ".join([df.label for df in fields]) - self.warnings.append( - {"row": row_number, "message": _("{0} are mandatory fields").format(fields_string)} - ) - - parsed_docs = {} - for row in rows: - for doctype in doctypes: - if doctype == self.doctype and parsed_docs.get(doctype): - # if parent doc is already parsed from the first row - # then skip - continue - - row_number = row[0] - column_indexes = get_column_indexes(doctype) - values = [row[i] for i in column_indexes] - - if all(v in INVALID_VALUES for v in values): - # skip values if all of them are empty - continue - - columns = [self.columns[i] for i in column_indexes] - docfields = [col.df for col in columns] - doc = parse_doc(doctype, docfields, values, row_number) - parsed_docs[doctype] = parsed_docs.get(doctype, []) - parsed_docs[doctype].append(doc) - - # build the doc with children - doc = {} - for doctype, docs in parsed_docs.items(): - if doctype == self.doctype: - doc.update(docs[0]) - else: - table_dfs = self.meta.get( - "fields", {"options": doctype, "fieldtype": ["in", table_fields]} - ) - if table_dfs: - table_field = table_dfs[0] - doc[table_field.fieldname] = docs - - # check if there is atleast one row for mandatory table fields - mandatory_table_fields = [ - df - for df in self.meta.fields - if df.fieldtype in table_fields and df.reqd and len(doc.get(df.fieldname, [])) == 0 - ] - if len(mandatory_table_fields) == 1: - self.warnings.append( - { - "row": first_row[0], - "message": _("There should be atleast one row for {0} table").format( - mandatory_table_fields[0].label - ), - } - ) - elif mandatory_table_fields: - fields_string = ", ".join([df.label for df in mandatory_table_fields]) - message = _("There should be atleast one row for the following tables: {0}").format( - fields_string - ) - self.warnings.append({"row": first_row[0], "message": message}) - - return doc, rows, data[len(rows) :] - - def process_doc(self, doc): - if self.import_type == INSERT: - return self.insert_record(doc) - elif self.import_type == UPDATE: - return self.update_record(doc) - - def insert_record(self, doc): - self.create_missing_linked_records(doc) - - new_doc = frappe.new_doc(self.doctype) - new_doc.update(doc) - # name shouldn't be set when inserting a new record - new_doc.set("name", None) - new_doc.insert() - if self.meta.is_submittable and self.data_import.submit_after_import: - new_doc.submit() - return new_doc - - def create_missing_linked_records(self, doc): - """ - Finds fields that are of type Link, and creates the corresponding - document automatically if it has only one mandatory field - """ - link_values = [] - - def get_link_fields(doc, doctype): - for fieldname, value in doc.items(): - meta = frappe.get_meta(doctype) - df = meta.get_field(fieldname) - if not df: - continue - if df.fieldtype == "Link" and value not in INVALID_VALUES: - link_values.append([df.options, value]) - elif df.fieldtype in table_fields: - for row in value: - get_link_fields(row, df.options) - - get_link_fields(doc, self.doctype) - - for link_doctype, link_value in link_values: - d = self.missing_link_values.get(link_doctype) - if d and d.one_mandatory and link_value in d.missing_values: - # find the autoname field - autoname_field = self.get_autoname_field(link_doctype) - name_field = autoname_field.fieldname if autoname_field else "name" - new_doc = frappe.new_doc(link_doctype) - new_doc.set(name_field, link_value) - new_doc.insert() - d.missing_values.remove(link_value) - - def update_record(self, doc): - id_fieldname = self.get_id_fieldname(self.doctype) - id_value = doc[id_fieldname] - existing_doc = frappe.get_doc(self.doctype, id_value) - existing_doc.flags.updater_reference = { - "doctype": self.data_import.doctype, - "docname": self.data_import.name, - "label": _("via Data Import"), - } - existing_doc.update(doc) - existing_doc.save() - return existing_doc - - def export_errored_rows(self): - from frappe.utils.csvutils import build_csv_response - - if not self.data_import: - return - - import_log = frappe.parse_json(self.data_import.import_log or "[]") - failures = [l for l in import_log if l.get("success") == False] - row_indexes = [] - for f in failures: - row_indexes.extend(f.get("row_indexes", [])) - - # de duplicate - row_indexes = list(set(row_indexes)) - row_indexes.sort() - - header_row = [col.header_title for col in self.columns[1:]] - rows = [header_row] - rows += [row[1:] for row in self.rows if row[0] in row_indexes] - - build_csv_response(rows, self.doctype) - - def get_missing_link_field_values(self, doctype): - return self.missing_link_values.get(doctype, {}) - - def prepare_missing_link_field_values(self): - columns = self.columns - rows = self.rows - link_column_indexes = [ - col.index for col in columns if col.df and col.df.fieldtype == "Link" - ] - - self.missing_link_values = {} - for index in link_column_indexes: - col = columns[index] - column_values = [row[index] for row in rows] - values = set([v for v in column_values if v not in INVALID_VALUES]) - doctype = col.df.options - - missing_values = [value for value in values if not frappe.db.exists(doctype, value)] - if self.missing_link_values.get(doctype): - self.missing_link_values[doctype].missing_values += missing_values - else: - self.missing_link_values[doctype] = frappe._dict( - missing_values=missing_values, - one_mandatory=self.has_one_mandatory_field(doctype), - df=col.df, - ) - - def get_eta(self, current, total, processing_time): - remaining = total - current - eta = processing_time * remaining - if not self.last_eta or eta < self.last_eta: - self.last_eta = eta - return self.last_eta - - def has_one_mandatory_field(self, doctype): - meta = frappe.get_meta(doctype) - # get mandatory fields with default not set - mandatory_fields = [df for df in meta.fields if df.reqd and not df.default] - mandatory_fields_count = len(mandatory_fields) - if meta.autoname and meta.autoname.lower() == "prompt": - mandatory_fields_count += 1 - return mandatory_fields_count == 1 - - def get_id_fieldname(self, doctype): - return self.get_id_field(doctype).fieldname - - def get_id_field(self, doctype): - autoname_field = self.get_autoname_field(doctype) - if autoname_field: - return autoname_field - return frappe._dict({"label": "ID", "fieldname": "name", "fieldtype": "Data"}) - - def get_autoname_field(self, doctype): - meta = frappe.get_meta(doctype) - if meta.autoname and meta.autoname.startswith("field:"): - fieldname = meta.autoname[len("field:") :] - return meta.get_field(fieldname) - - def print_grouped_warnings(self, warnings): - warnings_by_row = {} - other_warnings = [] - for w in warnings: - if w.get("row"): - warnings_by_row.setdefault(w.get("row"), []).append(w) - else: - other_warnings.append(w) - - for row_number, warnings in warnings_by_row.items(): - print("Row {0}".format(row_number)) - for w in warnings: - print(w.get("message")) - - for w in other_warnings: - print(w.get("message")) - - def print_import_log(self, import_log): - failed_records = [l for l in import_log if not l.success] - successful_records = [l for l in import_log if l.success] - - if successful_records: - print( - "Successfully imported {0} records out of {1}".format( - len(successful_records), len(import_log) - ) - ) - - if failed_records: - print("Failed to import {0} records".format(len(failed_records))) - file_name = "{0}_import_on_{1}.txt".format(self.doctype, frappe.utils.now()) - print("Check {0} for errors".format(os.path.join("sites", file_name))) - text = "" - for w in failed_records: - text += "Row Indexes: {0}\n".format(str(w.get("row_indexes", []))) - text += "Messages:\n{0}\n".format("\n".join(w.get("messages", []))) - text += "Traceback:\n{0}\n\n".format(w.get("exception")) - - with open(file_name, "w") as f: - f.write(text) - - -DATE_FORMATS = [ - r"%d-%m-%Y", - r"%m-%d-%Y", - r"%Y-%m-%d", - r"%d-%m-%y", - r"%m-%d-%y", - r"%y-%m-%d", - r"%d/%m/%Y", - r"%m/%d/%Y", - r"%Y/%m/%d", - r"%d/%m/%y", - r"%m/%d/%y", - r"%y/%m/%d", - r"%d.%m.%Y", - r"%m.%d.%Y", - r"%Y.%m.%d", - r"%d.%m.%y", - r"%m.%d.%y", - r"%y.%m.%d", -] - -TIME_FORMATS = [ - r"%H:%M:%S.%f", - r"%H:%M:%S", - r"%H:%M", - r"%I:%M:%S.%f %p", - r"%I:%M:%S %p", - r"%I:%M %p", -] - - -def guess_date_format(date_string): - date_string = date_string.strip() - - _date = None - _time = None - - if " " in date_string: - _date, _time = date_string.split(" ", 1) - else: - _date = date_string - - date_format = None - time_format = None - - for f in DATE_FORMATS: - try: - # if date is parsed without any exception - # capture the date format - datetime.strptime(_date, f) - date_format = f - break - except ValueError: - pass - - if _time: - for f in TIME_FORMATS: - try: - # if time is parsed without any exception - # capture the time format - datetime.strptime(_time, f) - time_format = f - break - except ValueError: - pass - - full_format = date_format - if time_format: - full_format += " " + time_format - return full_format - - -def import_data(doctype, file_path): - i = Importer(doctype, file_path) - i.import_data() +def get_user_format(date_format): + return ( + date_format.replace("%Y", "yyyy") + .replace("%y", "yy") + .replace("%m", "mm") + .replace("%d", "dd") + ) diff --git a/frappe/public/js/frappe/data_import/data_exporter.js b/frappe/public/js/frappe/data_import/data_exporter.js index 8276be6670..21f0b78a25 100644 --- a/frappe/public/js/frappe/data_import/data_exporter.js +++ b/frappe/public/js/frappe/data_import/data_exporter.js @@ -202,16 +202,16 @@ frappe.data_import.DataExporter = class DataExporter { } select_mandatory() { - let mandatory_table_doctypes = frappe.meta + let mandatory_table_fields = frappe.meta .get_table_fields(this.doctype) .filter(df => df.reqd) - .map(df => df.options); - mandatory_table_doctypes.push(this.doctype); + .map(df => df.fieldname); + mandatory_table_fields.push(this.doctype); let multicheck_fields = this.dialog.fields .filter(df => df.fieldtype === 'MultiCheck') .map(df => df.fieldname) - .filter(doctype => mandatory_table_doctypes.includes(doctype)); + .filter(doctype => mandatory_table_fields.includes(doctype)); let checkboxes = [].concat( ...multicheck_fields.map(fieldname => { @@ -333,16 +333,24 @@ frappe.data_import.DataExporter = class DataExporter { } }; -function get_columns_for_picker(doctype) { +export function get_columns_for_picker(doctype) { let out = {}; - const standard_fields_filter = df => - !in_list(frappe.model.no_value_type, df.fieldtype); + const exportable_fields = df => { + let keep = true; + if (frappe.model.no_value_type.includes(df.fieldtype)) { + keep = false; + } + if (['lft', 'rgt'].includes(df.fieldname)) { + keep = false; + } + return keep; + }; // parent let doctype_fields = frappe.meta .get_docfields(doctype) - .filter(standard_fields_filter); + .filter(exportable_fields); out[doctype] = [ { @@ -359,7 +367,7 @@ function get_columns_for_picker(doctype) { const cdt = df.options; const child_table_fields = frappe.meta .get_docfields(cdt) - .filter(standard_fields_filter); + .filter(exportable_fields); out[df.fieldname] = [ { diff --git a/frappe/public/js/frappe/data_import/import_preview.js b/frappe/public/js/frappe/data_import/import_preview.js index 27d81b75b7..7cf8431456 100644 --- a/frappe/public/js/frappe/data_import/import_preview.js +++ b/frappe/public/js/frappe/data_import/import_preview.js @@ -1,5 +1,5 @@ import DataTable from 'frappe-datatable'; -import ColumnPickerFields from './column_picker_fields'; +import { get_columns_for_picker } from './data_exporter'; frappe.provide('frappe.data_import'); @@ -236,9 +236,7 @@ frappe.data_import.ImportPreview = class ImportPreview { } show_column_mapper() { - let column_picker_fields = new ColumnPickerFields({ - doctype: this.doctype - }); + let column_picker_fields = get_columns_for_picker(this.doctype); let changed = []; let fields = this.preview_data.columns.map((col, i) => { let df = col.df; diff --git a/frappe/public/less/form.less b/frappe/public/less/form.less index df0334c14f..cd391c1f10 100644 --- a/frappe/public/less/form.less +++ b/frappe/public/less/form.less @@ -249,6 +249,7 @@ } .progress-message { + font-feature-settings: "tnum" 1; margin-top: 0px; } } @@ -1011,7 +1012,7 @@ body[data-route^="Form/Communication"] textarea[data-fieldname="subject"] { .map-columns .form-section { padding: 0 7px 7px; - border-bottom: none; + border-top: none; .clearfix { display: none; @@ -1021,3 +1022,7 @@ body[data-route^="Form/Communication"] textarea[data-fieldname="subject"] { .map-columns .form-section:first-child { padding-top: 7px; } + +.table-preview { + margin-top: 12px; +} diff --git a/frappe/utils/data.py b/frappe/utils/data.py index 7e991f472e..0d946c01a8 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -1185,3 +1185,75 @@ def is_subset(list_a, list_b): def generate_hash(*args, **kwargs): return frappe.generate_hash(*args, **kwargs) + + + +def guess_date_format(date_string): + DATE_FORMATS = [ + r"%d-%m-%Y", + r"%m-%d-%Y", + r"%Y-%m-%d", + r"%d-%m-%y", + r"%m-%d-%y", + r"%y-%m-%d", + r"%d/%m/%Y", + r"%m/%d/%Y", + r"%Y/%m/%d", + r"%d/%m/%y", + r"%m/%d/%y", + r"%y/%m/%d", + r"%d.%m.%Y", + r"%m.%d.%Y", + r"%Y.%m.%d", + r"%d.%m.%y", + r"%m.%d.%y", + r"%y.%m.%d", + ] + + TIME_FORMATS = [ + r"%H:%M:%S.%f", + r"%H:%M:%S", + r"%H:%M", + r"%I:%M:%S.%f %p", + r"%I:%M:%S %p", + r"%I:%M %p", + ] + + date_string = date_string.strip() + + _date = None + _time = None + + if " " in date_string: + _date, _time = date_string.split(" ", 1) + else: + _date = date_string + + date_format = None + time_format = None + + for f in DATE_FORMATS: + try: + # if date is parsed without any exception + # capture the date format + datetime.datetime.strptime(_date, f) + date_format = f + break + except ValueError: + pass + + if _time: + for f in TIME_FORMATS: + try: + # if time is parsed without any exception + # capture the time format + datetime.datetime.strptime(_time, f) + time_format = f + break + except ValueError: + pass + + full_format = date_format + if time_format: + full_format += " " + time_format + return full_format From 5a7ff2a446b095ec6a1bcfc4192b57483a17c92f Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Thu, 28 May 2020 23:17:14 +0530 Subject: [PATCH 025/485] fix: Rename importer_new to importer --- .../doctype/data_import_beta/{importer_new.py => importer.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename frappe/core/doctype/data_import_beta/{importer_new.py => importer.py} (100%) diff --git a/frappe/core/doctype/data_import_beta/importer_new.py b/frappe/core/doctype/data_import_beta/importer.py similarity index 100% rename from frappe/core/doctype/data_import_beta/importer_new.py rename to frappe/core/doctype/data_import_beta/importer.py From b7dfe7969dd6b2851844b1e9c090cbb6447748de Mon Sep 17 00:00:00 2001 From: Faris Ansari Date: Thu, 28 May 2020 23:18:08 +0530 Subject: [PATCH 026/485] fix: Create version for document creation So that we can track if documents were created by Data Import --- frappe/core/doctype/version/version.py | 11 +++++ frappe/model/document.py | 7 ++- .../public/js/frappe/form/footer/timeline.js | 48 ++++++++++++------- 3 files changed, 48 insertions(+), 18 deletions(-) diff --git a/frappe/core/doctype/version/version.py b/frappe/core/doctype/version/version.py index 216cdb1716..7654db4ae5 100644 --- a/frappe/core/doctype/version/version.py +++ b/frappe/core/doctype/version/version.py @@ -21,6 +21,17 @@ class Version(Document): else: return False + def for_insert(self, doc): + updater_reference = doc.flags.updater_reference + data = { + 'creation': doc.creation, + 'updater_reference': updater_reference, + 'created_by': doc.owner + } + self.ref_doctype = doc.doctype + self.docname = doc.name + self.data = frappe.as_json(data) + def get_data(self): return json.loads(self.data) diff --git a/frappe/model/document.py b/frappe/model/document.py index 843cb421fe..db56e1e395 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -961,7 +961,7 @@ class Document(BaseDocument): update_global_search(self) - if getattr(self.meta, 'track_changes', False) and self._doc_before_save and not self.flags.ignore_version: + if getattr(self.meta, 'track_changes', False) and not self.flags.ignore_version and not self.doctype == 'Version': self.save_version() self.run_method('on_change') @@ -1059,7 +1059,10 @@ class Document(BaseDocument): def save_version(self): """Save version info""" version = frappe.new_doc('Version') - if version.set_diff(self._doc_before_save, self): + if not self._doc_before_save: + version.for_insert(self) + version.insert(ignore_permissions=True) + elif version.set_diff(self._doc_before_save, self): version.insert(ignore_permissions=True) if not frappe.flags.in_migrate: follow_document(self.doctype, self.name, frappe.session.user) diff --git a/frappe/public/js/frappe/form/footer/timeline.js b/frappe/public/js/frappe/form/footer/timeline.js index bb44408c2a..7821a04c50 100644 --- a/frappe/public/js/frappe/form/footer/timeline.js +++ b/frappe/public/js/frappe/form/footer/timeline.js @@ -205,16 +205,18 @@ frappe.ui.form.Timeline = class Timeline {
').appendTo(me.list); } - // created - me.render_timeline_item({ - content: __("created"), - comment_type: "Created", - communication_type: "Comment", - sender: this.frm.doc.owner, - communication_date: this.frm.doc.creation, - creation: this.frm.doc.creation, - frm: this.frm - }); + // if a created comment is not added, add the default one + if (!timeline.find(comment => comment.comment_type === 'Created')) { + me.render_timeline_item({ + content: __("created"), + comment_type: "Created", + communication_type: "Comment", + sender: this.frm.doc.owner, + communication_date: this.frm.doc.creation, + creation: this.frm.doc.creation, + frm: this.frm + }); + } this.wrapper.find(".is-email").prop("checked", this.last_type==="Email").change(); @@ -564,12 +566,17 @@ frappe.ui.form.Timeline = class Timeline { let updater_reference = data.updater_reference; if (!$.isEmptyObject(updater_reference)) { let label = updater_reference.label || __('via {0}', [updater_reference.doctype]); - updater_reference_link = frappe.utils.get_form_link( - updater_reference.doctype, - updater_reference.docname, - true, - label - ); + let { doctype, docname } = updater_reference; + if (doctype && docname) { + updater_reference_link = frappe.utils.get_form_link( + doctype, + docname, + true, + label + ); + } else { + updater_reference_link = label; + } } // value changed in parent @@ -677,6 +684,15 @@ frappe.ui.form.Timeline = class Timeline { } } }); + + // creation by updater reference + if (data.creation && data.created_by) { + if (updater_reference_link) { + out.push(me.get_version_comment(version, __('created {0}', [updater_reference_link]), 'Created')); + } else { + out.push(me.get_version_comment(version, __('created'), 'Created')); + } + } }); } From 33857269efd594298d2f2da9186101b65c93f5a1 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Fri, 29 May 2020 10:35:47 +0530 Subject: [PATCH 027/485] feat: updated styles for blog post --- frappe/public/scss/blog.scss | 43 +++++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/frappe/public/scss/blog.scss b/frappe/public/scss/blog.scss index 86c5b18ceb..b24fb631fd 100644 --- a/frappe/public/scss/blog.scss +++ b/frappe/public/scss/blog.scss @@ -7,6 +7,10 @@ .blog-list-body { min-height: 14rem; } + + .post-cover-container { + height: 12rem; + } } .blog-list-cover { @@ -19,7 +23,6 @@ } .post-cover-container { - min-height: 12rem; overflow: hidden; border-radius: $border-radius; @@ -45,4 +48,42 @@ max-height: 22rem; } } +} + +.blog-container { + max-width: 840px; + font-size: 18px; + margin: 0px auto; + font-family: Inter, sans-serif; + + .blog-intro { + font-size: 1.25rem; + font-weight: 300; + } + + .blog-content { + p { + line-height: 1.8; + margin-bottom: 1.2rem; + } + + h1, h2, h3 { + margin-top: 2rem; + } + + h4, h5, h6 { + margin-top: 1.5rem; + } + + img { + border-radius: $border-radius; + } + + blockquote{ + font-weight: 300; + padding: 1.2rem 1.8rem; + border-left: 8px solid $gray-300 ; + background: $gray-100; + } + } } \ No newline at end of file From ee5b7aa84ef5135934c1537ec0503de4093e62eb Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Fri, 29 May 2020 10:36:15 +0530 Subject: [PATCH 028/485] refactor: remove breadcrumbs from blog post --- frappe/website/doctype/blog_post/blog_post.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frappe/website/doctype/blog_post/blog_post.py b/frappe/website/doctype/blog_post/blog_post.py index efa69c55bd..c3f78bff26 100644 --- a/frappe/website/doctype/blog_post/blog_post.py +++ b/frappe/website/doctype/blog_post/blog_post.py @@ -63,6 +63,8 @@ class BlogPost(WebsiteGenerator): if not cint(self.published): raise Exception("This blog has not been published yet!") + context.no_breadcrumbs = True + # temp fields context.full_name = get_fullname(self.owner) context.updated = global_date_format(self.published_on) From 6ffe45e5ce729e92c497666bf9c7b83618e8f787 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Fri, 29 May 2020 10:37:42 +0530 Subject: [PATCH 029/485] refactor: simplify social links --- frappe/website/doctype/blog_post/blog_post.py | 26 ++++++-------- .../blog_post/templates/blog_post.html | 36 +++---------------- .../doctype/blog_settings/blog_settings.json | 20 +++++------ 3 files changed, 24 insertions(+), 58 deletions(-) diff --git a/frappe/website/doctype/blog_post/blog_post.py b/frappe/website/doctype/blog_post/blog_post.py index c3f78bff26..9551c26e15 100644 --- a/frappe/website/doctype/blog_post/blog_post.py +++ b/frappe/website/doctype/blog_post/blog_post.py @@ -99,25 +99,19 @@ class BlogPost(WebsiteGenerator): def fetch_social_links_info(self): + if not frappe.db.get_single_value("Blog Settings", "enable_social_sharing", cache=True): + return [] + url = frappe.local.site + "/" +self.route - social_url_map = { - "twitter": "https://twitter.com/intent/tweet?text=" +self.title + "&url=" + url, - "facebook": "https://www.facebook.com/sharer.php?u=" + url, - "linkedin": "https://www.linkedin.com/sharing/share-offsite/?url=" + url, - "email": "mailto:?subject=" + self.title + "&body=" + url, - } - social_link = [] - for link in frappe.get_cached_doc("Blog Settings").social_share_settings: - social_media = link.social_link_type + social_links = [ + { "icon": "twitter", "link": "https://twitter.com/intent/tweet?text=" + self.title + "&url=" + url }, + { "icon": "facebook", "link": "https://www.facebook.com/sharer.php?u=" + url }, + { "icon": "linkedin", "link": "https://www.linkedin.com/sharing/share-offsite/?url=" + url }, + { "icon": "envelope", "link": "mailto:?subject=" + self.title + "&body=" + url } + ] - social_link.append({ - 'icon': social_media if not social_media == 'email' else 'envelope', - 'url': social_url_map.get(social_media), - 'color': link.color, - 'background': link.background_color - }) - return social_link + return social_links def load_comments(self, context): context.comment_list = get_comment_list(self.doctype, self.name) diff --git a/frappe/website/doctype/blog_post/templates/blog_post.html b/frappe/website/doctype/blog_post/templates/blog_post.html index 12e5ccf2d7..f1c99dd0fd 100644 --- a/frappe/website/doctype/blog_post/templates/blog_post.html +++ b/frappe/website/doctype/blog_post/templates/blog_post.html @@ -24,13 +24,12 @@ {% endif %} {% if social_links %} - {% endif %}
@@ -54,31 +53,4 @@ -{% endblock %} - -{% block style %} - -{% endblock %} +{% endblock %} \ No newline at end of file diff --git a/frappe/website/doctype/blog_settings/blog_settings.json b/frappe/website/doctype/blog_settings/blog_settings.json index a05c0ea5f3..63dae06c68 100644 --- a/frappe/website/doctype/blog_settings/blog_settings.json +++ b/frappe/website/doctype/blog_settings/blog_settings.json @@ -7,8 +7,8 @@ "field_order": [ "blog_title", "blog_introduction", - "section_break_4", - "social_share_settings" + "column_break", + "enable_social_sharing" ], "fields": [ { @@ -22,22 +22,22 @@ "label": "Blog Introduction" }, { - "collapsible": 1, - "fieldname": "section_break_4", - "fieldtype": "Section Break" + "default": "0", + "fieldname": "enable_social_sharing", + "fieldtype": "Check", + "label": "Enable Social Sharing" }, { - "fieldname": "social_share_settings", - "fieldtype": "Table", - "label": "Social Share Settings", - "options": "Social Link Settings" + "collapsible": 1, + "fieldname": "column_break", + "fieldtype": "Column Break" } ], "icon": "fa fa-cog", "idx": 1, "issingle": 1, "links": [], - "modified": "2020-05-28 18:49:08.356608", + "modified": "2020-05-28 22:34:46.679003", "modified_by": "Administrator", "module": "Website", "name": "Blog Settings", From b60fffc60492646f2124dce7ea92629c98a9c086 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Fri, 29 May 2020 10:38:13 +0530 Subject: [PATCH 030/485] feat: updated layout for blog and blogger --- frappe/templates/includes/blog/blogger.html | 2 +- .../blog_post/templates/blog_post.html | 42 +++++++++++-------- 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/frappe/templates/includes/blog/blogger.html b/frappe/templates/includes/blog/blogger.html index 68df22786d..136d2cdaa9 100644 --- a/frappe/templates/includes/blog/blogger.html +++ b/frappe/templates/includes/blog/blogger.html @@ -1,7 +1,7 @@ {% from "frappe/templates/includes/macros.html" import square_image_with_fallback %}
- {{ square_image_with_fallback(src=blogger_info.avatar, size='72px', alt=blogger_info.full_name, class='align-self-start mr-3 rounded') }} + {{ square_image_with_fallback(src=blogger_info.avatar, size='90px', alt=blogger_info.full_name, class='align-self-start mr-3 rounded') }}
{{ blogger_info.full_name }} diff --git a/frappe/website/doctype/blog_post/templates/blog_post.html b/frappe/website/doctype/blog_post/templates/blog_post.html index f1c99dd0fd..f0eb505d61 100644 --- a/frappe/website/doctype/blog_post/templates/blog_post.html +++ b/frappe/website/doctype/blog_post/templates/blog_post.html @@ -8,22 +8,30 @@
-
- -

{{ title }}

-

- {{ blog_intro }} -

-
- -
- {{ frappe.format_date(published_on) }} - {% if read_time %} - · - {{ read_time }} min read - {% endif %} - {% if social_links %} -
+ {% if not hide_cta %} + {{ web_blocks([ + { + 'template': "Section With CTA", + 'values': cta, + 'add_container': 0, + 'add_top_padding': 0, + 'add_bottom_padding': 0, + 'css_class': "py-5" + } + ]) + }} + {% endif %} {% if blogger_info %}
diff --git a/frappe/website/doctype/blog_settings/blog_settings.json b/frappe/website/doctype/blog_settings/blog_settings.json index 63dae06c68..a1c9591837 100644 --- a/frappe/website/doctype/blog_settings/blog_settings.json +++ b/frappe/website/doctype/blog_settings/blog_settings.json @@ -8,7 +8,16 @@ "blog_title", "blog_introduction", "column_break", - "enable_social_sharing" + "enable_social_sharing", + "show_cta_in_blog", + "cta_section", + "title", + "subtitle", + "cta_label", + "cta_url", + "column_break_11", + "cta_description", + "show_confetti" ], "fields": [ { @@ -31,13 +40,64 @@ "collapsible": 1, "fieldname": "column_break", "fieldtype": "Column Break" + }, + { + "default": "0", + "fieldname": "show_cta_in_blog", + "fieldtype": "Check", + "label": "Show CTA in Blog" + }, + { + "depends_on": "eval:doc.show_cta_in_blog", + "fieldname": "cta_section", + "fieldtype": "Section Break", + "label": "CTA" + }, + { + "fieldname": "title", + "fieldtype": "Data", + "label": "Title", + "mandatory_depends_on": "eval:doc.show_cta_in_blog" + }, + { + "fieldname": "subtitle", + "fieldtype": "Data", + "label": "Subtitle", + "mandatory_depends_on": "eval:doc.show_cta_in_blog" + }, + { + "fieldname": "cta_label", + "fieldtype": "Data", + "label": "CTA Label", + "mandatory_depends_on": "eval:doc.show_cta_in_blog" + }, + { + "fieldname": "cta_url", + "fieldtype": "Data", + "label": "CTA URL", + "mandatory_depends_on": "eval:doc.show_cta_in_blog" + }, + { + "fieldname": "column_break_11", + "fieldtype": "Column Break" + }, + { + "fieldname": "cta_description", + "fieldtype": "Data", + "label": "CTA Description" + }, + { + "default": "0", + "fieldname": "show_confetti", + "fieldtype": "Check", + "label": "Show Confetti" } ], "icon": "fa fa-cog", "idx": 1, "issingle": 1, "links": [], - "modified": "2020-05-28 22:34:46.679003", + "modified": "2020-05-29 16:39:44.018100", "modified_by": "Administrator", "module": "Website", "name": "Blog Settings", From 3de8549b0b67f1bfe8ca4ad4b63c9a73d0105f47 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Mon, 1 Jun 2020 13:42:23 +0530 Subject: [PATCH 043/485] refactor: allow blogger to read blog settings --- frappe/website/doctype/blog_settings/blog_settings.json | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/frappe/website/doctype/blog_settings/blog_settings.json b/frappe/website/doctype/blog_settings/blog_settings.json index a1c9591837..1b96054705 100644 --- a/frappe/website/doctype/blog_settings/blog_settings.json +++ b/frappe/website/doctype/blog_settings/blog_settings.json @@ -97,7 +97,7 @@ "idx": 1, "issingle": 1, "links": [], - "modified": "2020-05-29 16:39:44.018100", + "modified": "2020-06-01 13:41:44.150987", "modified_by": "Administrator", "module": "Website", "name": "Blog Settings", @@ -111,6 +111,13 @@ "role": "Website Manager", "share": 1, "write": 1 + }, + { + "email": 1, + "print": 1, + "read": 1, + "role": "Blogger", + "share": 1 } ], "sort_field": "modified", From 7dfb7e0646b1c49ee547fd2cd1f6c547e8c79cb7 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Mon, 1 Jun 2020 13:53:03 +0530 Subject: [PATCH 044/485] refactor: remove cover image --- frappe/public/scss/blog.scss | 6 +++--- .../blog_post/templates/blog_post_row.html | 16 ++++++++-------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/frappe/public/scss/blog.scss b/frappe/public/scss/blog.scss index aa6c3b543c..f857270292 100644 --- a/frappe/public/scss/blog.scss +++ b/frappe/public/scss/blog.scss @@ -15,13 +15,13 @@ border-radius: $border-radius; img { - min-height: 12rem; + min-height: 10rem; width: 100%; } } .default-cover { - background: $gray-300; + background: $gray-200; color: $gray-600; display: flex; flex-direction: column; @@ -34,7 +34,7 @@ @extend .col-md-4; .blog-list-body { - min-height: 12rem; + min-height: 14rem; } .post-cover-container { diff --git a/frappe/website/doctype/blog_post/templates/blog_post_row.html b/frappe/website/doctype/blog_post/templates/blog_post_row.html index 5987532c98..a80525c761 100644 --- a/frappe/website/doctype/blog_post/templates/blog_post_row.html +++ b/frappe/website/doctype/blog_post/templates/blog_post_row.html @@ -2,15 +2,15 @@
- {% if post.cover_image %} - + + {% if post.cover_image %} {{post.title}} - Cover Image - - {% else %} -
-
{{ post.title }}
-
- {% endif %} + {% else %} +
+
{{ post.title }}
+
+ {% endif %} +
From 299200da190b1663dc854e26fdc8f81dd8d2fb31 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Mon, 1 Jun 2020 14:13:45 +0530 Subject: [PATCH 045/485] refactor: macro for avatar --- frappe/templates/includes/blog/blogger.html | 2 +- frappe/templates/includes/comments/comment.html | 2 +- frappe/templates/includes/macros.html | 16 ++-------------- 3 files changed, 4 insertions(+), 16 deletions(-) diff --git a/frappe/templates/includes/blog/blogger.html b/frappe/templates/includes/blog/blogger.html index 136d2cdaa9..ef8f8257e8 100644 --- a/frappe/templates/includes/blog/blogger.html +++ b/frappe/templates/includes/blog/blogger.html @@ -1,7 +1,7 @@ {% from "frappe/templates/includes/macros.html" import square_image_with_fallback %}
- {{ square_image_with_fallback(src=blogger_info.avatar, size='90px', alt=blogger_info.full_name, class='align-self-start mr-3 rounded') }} + {{ square_image_with_fallback(src=blogger_info.avatar, size='small', alt=blogger_info.full_name, class='align-self-start mr-3 rounded') }}
{{ blogger_info.full_name }} diff --git a/frappe/templates/includes/comments/comment.html b/frappe/templates/includes/comments/comment.html index 3fe3d7df58..1deb49bb3e 100644 --- a/frappe/templates/includes/comments/comment.html +++ b/frappe/templates/includes/comments/comment.html @@ -1,7 +1,7 @@ {% from "frappe/templates/includes/macros.html" import square_image_with_fallback %}
- {{ square_image_with_fallback(src=frappe.get_gravatar(comment.comment_email or comment.sender), size='48px', alt=comment.sender_full_name, class='align-self-start mr-3') }} + {{ square_image_with_fallback(src=frappe.get_gravatar(comment.comment_email or comment.sender), size='extra-small', alt=comment.sender_full_name, class='align-self-start mr-3') }}
diff --git a/frappe/templates/includes/macros.html b/frappe/templates/includes/macros.html index 3e822b8bf3..767bd59ec9 100644 --- a/frappe/templates/includes/macros.html +++ b/frappe/templates/includes/macros.html @@ -1,18 +1,6 @@ -{% macro square_image_with_fallback(src=None, size=None, alt=None, class="") %} +{% macro square_image_with_fallback(src=None, size='small', alt=None, class="") %} {% if src %} -{{ alt or '' }} + {% else %}
{% endif %} From e868fac37961e1e505f5fa97575bb033035d4f31 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Mon, 1 Jun 2020 15:19:33 +0530 Subject: [PATCH 046/485] refactor: simpler blog layout with cards --- frappe/public/scss/blog.scss | 66 +++---------------- .../blog_post/templates/blog_post_row.html | 57 +++++++--------- 2 files changed, 33 insertions(+), 90 deletions(-) diff --git a/frappe/public/scss/blog.scss b/frappe/public/scss/blog.scss index f857270292..64a27b3193 100644 --- a/frappe/public/scss/blog.scss +++ b/frappe/public/scss/blog.scss @@ -1,62 +1,12 @@ -.blog-list-item { - @extend .col-12; +.card-img-top { + width: 100%; + overflow: hidden; + height: 12rem; + width: auto; - .blog-list-cover { - @extend .col-12; - } - - .blog-list-body { - @extend .col-12; - @extend .mt-3; - } - - .post-cover-container { - overflow: hidden; - border-radius: $border-radius; - - img { - min-height: 10rem; - width: 100%; - } - } - - .default-cover { - background: $gray-200; - color: $gray-600; - display: flex; - flex-direction: column; - justify-content: end; - height: 100%; - padding: 1rem; - } - - &:not(.featured) { - @extend .col-md-4; - - .blog-list-body { - min-height: 14rem; - } - - .post-cover-container { - height: 10rem; - } - } - - &.featured { - .blog-list-cover { - @extend .col-12; - @extend .col-md-8; - } - - .blog-list-body { - @extend .col-12; - @extend .col-md-4; - @extend .mt-0; - } - - .post-cover-container { - max-height: 22rem; - } + img { + width: 100%; + min-height: 100%; } } diff --git a/frappe/website/doctype/blog_post/templates/blog_post_row.html b/frappe/website/doctype/blog_post/templates/blog_post_row.html index a80525c761..bd5b0b382b 100644 --- a/frappe/website/doctype/blog_post/templates/blog_post_row.html +++ b/frappe/website/doctype/blog_post/templates/blog_post_row.html @@ -1,42 +1,35 @@ {%- set post = doc -%} -
-
-
- - {% if post.cover_image %} - {{post.title}} - Cover Image - {% else %} -
-
{{ post.title }}
-
- {% endif %} -
+
+
+
+ {{post.title}} - Cover Image
-
-
-
-
+
+
+
+ {%- if post.featured -%} + {{ _('Featured') }} · + {%- endif -%} + {{ post.category.title }} +
{%- if post.featured -%} - {{ _('Featured') }} · +
{{ post.title }}
+ {%- else -%} +
{{ post.title }}
{%- endif -%} - {{ post.category.title }} +

{{ post.intro }}

- {%- if post.featured -%} -

{{ post.title }}

- {%- else -%} -
{{ post.title }}
- {%- endif -%} -

{{ post.intro }}

-
-
- -
- {{ post.full_name }} -
- {{ frappe.format_date(post.published_on) }} - {% if post.read_time %} · {{ post.read_time }} min read {% endif %} +
+ +
+ {{ post.full_name }} +
+ {{ frappe.format_date(post.published_on) }} + {% if post.read_time %} · {{ post.read_time }} min read {% endif %} +
+
\ No newline at end of file From f38e081d04a722a4f38571762037b9eda9eaf8ea Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Mon, 1 Jun 2020 17:52:18 +0530 Subject: [PATCH 047/485] refactor: tweak typography --- frappe/public/scss/blog.scss | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/frappe/public/scss/blog.scss b/frappe/public/scss/blog.scss index 64a27b3193..9b78e47e48 100644 --- a/frappe/public/scss/blog.scss +++ b/frappe/public/scss/blog.scss @@ -12,7 +12,7 @@ .blog-container { max-width: 840px; - font-size: 18px; + font-size: 16px; margin: 0px auto; font-family: Inter, sans-serif; @@ -23,10 +23,34 @@ .blog-content { p { - line-height: 1.8; + line-height: 1.625; margin-bottom: 1.2rem; } + h1 { + font-size: 2rem; + } + + h1.blog-header { + font-size: 2.5rem; + } + + h2 { + font-size: 1.75rem; + } + + h3 { + font-size: 1.5rem; + } + + h4 { + font-size: 1.25rem; + } + + h5, h6 { + font-size: 1rem; + } + h1, h2, h3 { margin-top: 2rem; } From 9e358261f1baef3eb84cd43eeb4ab9a41667cde7 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Mon, 1 Jun 2020 18:32:29 +0530 Subject: [PATCH 048/485] feat: added section small cta --- frappe/public/scss/page-builder.scss | 44 +++++++++++++++++++ .../section_with_small_cta/__init__.py | 0 .../section_with_small_cta.html | 11 +++++ .../section_with_small_cta.json | 37 ++++++++++++++++ 4 files changed, 92 insertions(+) create mode 100644 frappe/website/web_template/section_with_small_cta/__init__.py create mode 100644 frappe/website/web_template/section_with_small_cta/section_with_small_cta.html create mode 100644 frappe/website/web_template/section_with_small_cta/section_with_small_cta.json diff --git a/frappe/public/scss/page-builder.scss b/frappe/public/scss/page-builder.scss index a028e34158..cc96a2751c 100644 --- a/frappe/public/scss/page-builder.scss +++ b/frappe/public/scss/page-builder.scss @@ -224,6 +224,50 @@ } } +.section-small-cta { + padding: 1.8rem; + background-color: lighten($primary, 42%); + border-radius: 0.75rem; + display: flex; + flex-direction: column; + text-align: center; + + @include media-breakpoint-up(sm) { + flex-direction: column; + text-align: left; + } + + @include media-breakpoint-up(md) { + flex-direction: row; + justify-content: space-between; + + div { + align-self: center; + } + } + + .title { + max-width: 36rem; + font-size: $font-size-xl; + font-weight: 800; + line-height: 1.25; + @include media-breakpoint-up(md) { + font-size: $font-size-2xl; + } + } + .subtitle { + max-width: 36rem; + font-size: $font-size-base; + color: $gray-900; + margin-bottom: 1.2rem; + + @include media-breakpoint-up(md) { + font-size: $font-size-lg; + margin-bottom: 0px; + } + } +} + .section-cta-container { position: relative; .confetti { diff --git a/frappe/website/web_template/section_with_small_cta/__init__.py b/frappe/website/web_template/section_with_small_cta/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/frappe/website/web_template/section_with_small_cta/section_with_small_cta.html b/frappe/website/web_template/section_with_small_cta/section_with_small_cta.html new file mode 100644 index 0000000000..e2f8f5d7f6 --- /dev/null +++ b/frappe/website/web_template/section_with_small_cta/section_with_small_cta.html @@ -0,0 +1,11 @@ +
+
+
+

{{ title }}

+

{{ subtitle }}

+
+ +
+
diff --git a/frappe/website/web_template/section_with_small_cta/section_with_small_cta.json b/frappe/website/web_template/section_with_small_cta/section_with_small_cta.json new file mode 100644 index 0000000000..d392830473 --- /dev/null +++ b/frappe/website/web_template/section_with_small_cta/section_with_small_cta.json @@ -0,0 +1,37 @@ +{ + "creation": "2020-06-01 15:56:38.002136", + "docstatus": 0, + "doctype": "Web Template", + "fields": [ + { + "fieldname": "title", + "fieldtype": "Data", + "label": "Title", + "reqd": 0 + }, + { + "fieldname": "subtitle", + "fieldtype": "Small Text", + "label": "Subtitle", + "reqd": 0 + }, + { + "fieldname": "cta_label", + "fieldtype": "Data", + "label": "CTA Label", + "reqd": 0 + }, + { + "fieldname": "cta_url", + "fieldtype": "Data", + "label": "CTA URL", + "reqd": 0 + } + ], + "idx": 0, + "modified": "2020-06-01 17:51:23.073342", + "modified_by": "Administrator", + "name": "Section with Small CTA", + "owner": "Administrator", + "standard": 1 +} \ No newline at end of file From b00fb8bd21dc111c3301ea9789ebef446c2df777 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Mon, 1 Jun 2020 18:33:03 +0530 Subject: [PATCH 049/485] refactor: replace cta with small cta --- frappe/website/doctype/blog_post/blog_post.py | 4 +--- .../blog_post/templates/blog_post.html | 2 +- .../doctype/blog_settings/blog_settings.json | 19 +++---------------- 3 files changed, 5 insertions(+), 20 deletions(-) diff --git a/frappe/website/doctype/blog_post/blog_post.py b/frappe/website/doctype/blog_post/blog_post.py index dcd712b634..a696ebc8c2 100644 --- a/frappe/website/doctype/blog_post/blog_post.py +++ b/frappe/website/doctype/blog_post/blog_post.py @@ -112,9 +112,7 @@ class BlogPost(WebsiteGenerator): "title": blog_settings.title, "subtitle": blog_settings.subtitle, "cta_label": blog_settings.cta_label, - "cta_url": blog_settings.cta_url, - "cta_description": blog_settings.cta_description, - "show_confetti": blog_settings.show_confetti + "cta_url": blog_settings.cta_url } return {} diff --git a/frappe/website/doctype/blog_post/templates/blog_post.html b/frappe/website/doctype/blog_post/templates/blog_post.html index f38c3db4b4..862380de47 100644 --- a/frappe/website/doctype/blog_post/templates/blog_post.html +++ b/frappe/website/doctype/blog_post/templates/blog_post.html @@ -48,7 +48,7 @@ {% if not hide_cta %} {{ web_blocks([ { - 'template': "Section With CTA", + 'template': "Section With Small CTA", 'values': cta, 'add_container': 0, 'add_top_padding': 0, diff --git a/frappe/website/doctype/blog_settings/blog_settings.json b/frappe/website/doctype/blog_settings/blog_settings.json index 1b96054705..73ea3ce877 100644 --- a/frappe/website/doctype/blog_settings/blog_settings.json +++ b/frappe/website/doctype/blog_settings/blog_settings.json @@ -13,11 +13,9 @@ "cta_section", "title", "subtitle", - "cta_label", - "cta_url", "column_break_11", - "cta_description", - "show_confetti" + "cta_label", + "cta_url" ], "fields": [ { @@ -80,24 +78,13 @@ { "fieldname": "column_break_11", "fieldtype": "Column Break" - }, - { - "fieldname": "cta_description", - "fieldtype": "Data", - "label": "CTA Description" - }, - { - "default": "0", - "fieldname": "show_confetti", - "fieldtype": "Check", - "label": "Show Confetti" } ], "icon": "fa fa-cog", "idx": 1, "issingle": 1, "links": [], - "modified": "2020-06-01 13:41:44.150987", + "modified": "2020-06-01 15:57:21.564652", "modified_by": "Administrator", "module": "Website", "name": "Blog Settings", From 882b200f32c475ab5fa26b6c9b7129d23d5f6988 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Mon, 1 Jun 2020 18:33:25 +0530 Subject: [PATCH 050/485] feat: updated blog typography --- frappe/public/scss/blog.scss | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/frappe/public/scss/blog.scss b/frappe/public/scss/blog.scss index 9b78e47e48..f0b8d9e34f 100644 --- a/frappe/public/scss/blog.scss +++ b/frappe/public/scss/blog.scss @@ -23,7 +23,7 @@ .blog-content { p { - line-height: 1.625; + line-height: 1.8; margin-bottom: 1.2rem; } @@ -47,15 +47,20 @@ font-size: 1.25rem; } - h5, h6 { + h5, + h6 { font-size: 1rem; } - h1, h2, h3 { + h1, + h2, + h3 { margin-top: 2rem; } - h4, h5, h6 { + h4, + h5, + h6 { margin-top: 1.5rem; } @@ -63,11 +68,11 @@ border-radius: $border-radius; } - blockquote{ + blockquote { font-weight: 300; padding: 1.2rem 1.8rem; - border-left: 8px solid $gray-300 ; + border-left: 8px solid $gray-300; background: $gray-100; } } -} \ No newline at end of file +} From 8e815552f648ca295befaa002b522588e32fd5fc Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Mon, 1 Jun 2020 19:15:00 +0530 Subject: [PATCH 051/485] feat: use blog_post_list as default --- frappe/model/meta.py | 3 +++ frappe/templates/includes/blog/blog_list.html | 25 ------------------- frappe/website/doctype/blog_post/blog_post.py | 1 - .../blog_post/templates/blog_post_list.html} | 0 frappe/www/list.py | 3 +++ 5 files changed, 6 insertions(+), 26 deletions(-) delete mode 100644 frappe/templates/includes/blog/blog_list.html rename frappe/{templates/includes/blog/blog.html => website/doctype/blog_post/templates/blog_post_list.html} (100%) diff --git a/frappe/model/meta.py b/frappe/model/meta.py index 0c5ec75597..1cc3abba5b 100644 --- a/frappe/model/meta.py +++ b/frappe/model/meta.py @@ -483,6 +483,9 @@ class Meta(Document): def get_row_template(self): return self.get_web_template(suffix='_row') + def get_list_template(self): + return self.get_web_template(suffix='_list') + def get_web_template(self, suffix=''): '''Returns the relative path of the row template for this doctype''' module_name = frappe.scrub(self.module) diff --git a/frappe/templates/includes/blog/blog_list.html b/frappe/templates/includes/blog/blog_list.html deleted file mode 100644 index cfe25f3682..0000000000 --- a/frappe/templates/includes/blog/blog_list.html +++ /dev/null @@ -1,25 +0,0 @@ -{% if sub_title %} -

{{ sub_title }}

-{% endif %} -{% if not result -%} -
- {{ no_result_message or _("Nothing to show") }} -
-{% else %} -
- - {% if result_heading_template %}{% include result_heading_template %}{% endif %} - -
- {% for item in result %} - {{ item }} - {% endfor %} -
- -
-{%- endif %} diff --git a/frappe/website/doctype/blog_post/blog_post.py b/frappe/website/doctype/blog_post/blog_post.py index a696ebc8c2..8a0e2323fc 100644 --- a/frappe/website/doctype/blog_post/blog_post.py +++ b/frappe/website/doctype/blog_post/blog_post.py @@ -153,7 +153,6 @@ class BlogPost(WebsiteGenerator): def get_list_context(context=None): list_context = frappe._dict( - template = "/templates/includes/blog/blog.html", get_list = get_blog_list, no_breadcrumbs = True, hide_filters = True, diff --git a/frappe/templates/includes/blog/blog.html b/frappe/website/doctype/blog_post/templates/blog_post_list.html similarity index 100% rename from frappe/templates/includes/blog/blog.html rename to frappe/website/doctype/blog_post/templates/blog_post_list.html diff --git a/frappe/www/list.py b/frappe/www/list.py index 313505b729..03171a74bd 100644 --- a/frappe/www/list.py +++ b/frappe/www/list.py @@ -171,6 +171,9 @@ def get_list_context(context, doctype, web_form_name=None): if not meta.custom and not list_context.row_template: list_context.row_template = meta.get_row_template() + if not meta.custom and not list_context.list_template: + list_context.template = meta.get_list_template() + return list_context def get_list(doctype, txt, filters, limit_start, limit_page_length=20, ignore_permissions=False, From 89b8eea57b50832dc46701b11eefd607ae1d2f57 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Mon, 1 Jun 2020 19:29:17 +0530 Subject: [PATCH 052/485] feat: move social share after content --- frappe/website/doctype/blog_post/blog_post.py | 1 + .../blog_post/templates/blog_post.html | 31 ++++++++++++------- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/frappe/website/doctype/blog_post/blog_post.py b/frappe/website/doctype/blog_post/blog_post.py index 8a0e2323fc..beffcdca25 100644 --- a/frappe/website/doctype/blog_post/blog_post.py +++ b/frappe/website/doctype/blog_post/blog_post.py @@ -75,6 +75,7 @@ class BlogPost(WebsiteGenerator): context.updated = global_date_format(self.published_on) context.social_links = self.fetch_social_links_info() context.cta = self.fetch_cta() + context.enable_cta = not self.hide_cta and frappe.db.get_single_value("Blog Settings", "show_cta_in_blog", cache=True) if self.blogger: context.blogger_info = frappe.get_doc("Blogger", self.blogger).as_dict() diff --git a/frappe/website/doctype/blog_post/templates/blog_post.html b/frappe/website/doctype/blog_post/templates/blog_post.html index 862380de47..e7e1e5db95 100644 --- a/frappe/website/doctype/blog_post/templates/blog_post.html +++ b/frappe/website/doctype/blog_post/templates/blog_post.html @@ -18,11 +18,6 @@

{{ blog_intro }}

- {%-if featured -%} -
- {{ _("Featured") }} -
- {%- endif -%}
{{ frappe.format_date(published_on) }} @@ -32,11 +27,11 @@ {% endif %}
- {% if social_links %} - {% for link in social_links %} - - {% endfor %} - {% endif %} + {%-if featured -%} +
+ {{ _("Featured") }} +
+ {%- endif -%}
@@ -45,7 +40,7 @@
- {% if not hide_cta %} + {% if enable_cta %} {{ web_blocks([ { 'template': "Section With Small CTA", @@ -53,11 +48,23 @@ 'add_container': 0, 'add_top_padding': 0, 'add_bottom_padding': 0, - 'css_class': "py-5" + 'css_class': "my-5" } ]) }} {% endif %} +
+
+ Published on {{ frappe.format_date(published_on) }} +
+
+ {% if social_links %} + {% for link in social_links %} + + {% endfor %} + {% endif %} +
+
{% if blogger_info %}
From 5d9d06a7a2c6dd0c67c9d26fd5414433ffd31257 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Mon, 1 Jun 2020 19:51:43 +0530 Subject: [PATCH 053/485] refactor: minor blog styling --- frappe/public/scss/blog.scss | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/frappe/public/scss/blog.scss b/frappe/public/scss/blog.scss index f0b8d9e34f..999df7bde0 100644 --- a/frappe/public/scss/blog.scss +++ b/frappe/public/scss/blog.scss @@ -11,8 +11,8 @@ } .blog-container { - max-width: 840px; font-size: 16px; + max-width: 800px; margin: 0px auto; font-family: Inter, sans-serif; @@ -22,11 +22,16 @@ } .blog-content { + line-height: 1.8; + p { - line-height: 1.8; margin-bottom: 1.2rem; } + ul { + margin-top: 1rem; + } + h1 { font-size: 2rem; } @@ -69,10 +74,15 @@ } blockquote { - font-weight: 300; - padding: 1.2rem 1.8rem; - border-left: 8px solid $gray-300; - background: $gray-100; + margin: 2rem auto; + width: 90%; + padding-top: 2rem; + padding-bottom: 2rem; + border-top: 1px solid $gray-200; + border-bottom: 1px solid $gray-200; + text-align: center; + font-size: 1.2rem; + font-weight: 500; } } } From 190ed209747ed0cd8495402e9966a4ea992be983 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Mon, 1 Jun 2020 19:55:37 +0530 Subject: [PATCH 054/485] fix: top margin for ordered list --- frappe/public/scss/blog.scss | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/scss/blog.scss b/frappe/public/scss/blog.scss index 999df7bde0..7ecdca3879 100644 --- a/frappe/public/scss/blog.scss +++ b/frappe/public/scss/blog.scss @@ -28,7 +28,7 @@ margin-bottom: 1.2rem; } - ul { + ul, ol { margin-top: 1rem; } From 9a997764a125ad268e2b24834821fcd83a1f414d Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Mon, 1 Jun 2020 20:03:10 +0530 Subject: [PATCH 055/485] refactor: use margin instead of padding --- frappe/website/doctype/blog_post/templates/blog_post.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/website/doctype/blog_post/templates/blog_post.html b/frappe/website/doctype/blog_post/templates/blog_post.html index e7e1e5db95..df0300ff23 100644 --- a/frappe/website/doctype/blog_post/templates/blog_post.html +++ b/frappe/website/doctype/blog_post/templates/blog_post.html @@ -8,7 +8,7 @@
-
+
{{ _('Blog') }} / From 315f01833ced5d61a4be9ce070e8363c11c08458 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Mon, 1 Jun 2020 20:23:28 +0530 Subject: [PATCH 056/485] fix: fonts --- frappe/public/scss/blog.scss | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/public/scss/blog.scss b/frappe/public/scss/blog.scss index 7ecdca3879..16266b5da2 100644 --- a/frappe/public/scss/blog.scss +++ b/frappe/public/scss/blog.scss @@ -14,7 +14,6 @@ font-size: 16px; max-width: 800px; margin: 0px auto; - font-family: Inter, sans-serif; .blog-intro { font-size: 1.25rem; @@ -66,6 +65,7 @@ h4, h5, h6 { + font-weight: 600; margin-top: 1.5rem; } From 6f678e26cd5e063475358b81ff303a978daa421d Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Mon, 1 Jun 2020 20:27:20 +0530 Subject: [PATCH 057/485] refactor: fonts --- frappe/public/scss/blog.scss | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/frappe/public/scss/blog.scss b/frappe/public/scss/blog.scss index 16266b5da2..92a86d6c95 100644 --- a/frappe/public/scss/blog.scss +++ b/frappe/public/scss/blog.scss @@ -32,41 +32,40 @@ } h1 { + margin-top: 2rem; + font-weight: 800; font-size: 2rem; } h1.blog-header { + margin-top: 2rem; + font-weight: 800; font-size: 2.5rem; } h2 { + margin-top: 2rem; + font-weight: 700; font-size: 1.75rem; } h3 { + margin-top: 2rem; + font-weight: 600; font-size: 1.5rem; } h4 { + font-weight: 500; + margin-top: 1.5rem; font-size: 1.25rem; } h5, h6 { - font-size: 1rem; - } - - h1, - h2, - h3 { - margin-top: 2rem; - } - - h4, - h5, - h6 { - font-weight: 600; + font-weight: 500; margin-top: 1.5rem; + font-size: 1rem; } img { From 8e1830de33964a44e39ac7485f2abd810e2941b4 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Mon, 1 Jun 2020 20:46:20 +0530 Subject: [PATCH 058/485] refactor: remove featured tag --- .../blog_post/templates/blog_post.html | 21 ++++++------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/frappe/website/doctype/blog_post/templates/blog_post.html b/frappe/website/doctype/blog_post/templates/blog_post.html index df0300ff23..2bb4b6025b 100644 --- a/frappe/website/doctype/blog_post/templates/blog_post.html +++ b/frappe/website/doctype/blog_post/templates/blog_post.html @@ -18,21 +18,12 @@

{{ blog_intro }}

-
-
- {{ frappe.format_date(published_on) }} - {% if read_time %} - · - {{ read_time }} min read - {% endif %} -
-
- {%-if featured -%} -
- {{ _("Featured") }} -
- {%- endif -%} -
+
+ {{ frappe.format_date(published_on) }} + {% if read_time %} + · + {{ read_time }} min read + {% endif %}
From 5cbb72a0eedce3bf6b368a1f5aae034ce2ebb241 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Mon, 1 Jun 2020 20:46:29 +0530 Subject: [PATCH 059/485] refactor: style for hr and blockquote --- frappe/public/scss/blog.scss | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/frappe/public/scss/blog.scss b/frappe/public/scss/blog.scss index 92a86d6c95..888d699457 100644 --- a/frappe/public/scss/blog.scss +++ b/frappe/public/scss/blog.scss @@ -31,6 +31,10 @@ margin-top: 1rem; } + hr { + margin: 2rem auto; + } + h1 { margin-top: 2rem; font-weight: 800; @@ -76,9 +80,11 @@ margin: 2rem auto; width: 90%; padding-top: 2rem; - padding-bottom: 2rem; + padding-bottom: 0.8rem; + border-top: 1px solid $gray-200; border-bottom: 1px solid $gray-200; + text-align: center; font-size: 1.2rem; font-weight: 500; From fb74a095385154bac1ab64f82eedad49bc7b24a6 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Tue, 2 Jun 2020 13:08:35 +0530 Subject: [PATCH 060/485] refactor: use custom rules instead of utility classes --- frappe/public/scss/blog.scss | 58 ++++++++++++++++++- .../blog_post/templates/blog_post.html | 8 +-- .../blog_post/templates/blog_post_list.html | 2 +- .../blog_post/templates/blog_post_row.html | 8 +-- 4 files changed, 66 insertions(+), 10 deletions(-) diff --git a/frappe/public/scss/blog.scss b/frappe/public/scss/blog.scss index 888d699457..b2ddece217 100644 --- a/frappe/public/scss/blog.scss +++ b/frappe/public/scss/blog.scss @@ -1,3 +1,43 @@ +.blog-list { + display: flex; + flex-wrap: wrap; + margin-right: -15px; + margin-left: -15px; +} + +.blog-card { + margin-bottom: 3rem; + position: relative; + width: 100%; + padding-right: 15px; + padding-left: 15px; + + flex: 0 0 33.33333%; + max-width: 33.33333%; + + &.featured { + flex: 0 0 66.66667%; + max-width: 66.66667%; + } + + .card-body { + display: flex; + flex-direction: column; + justify-content: space-between; + } + + .blog-card-footer { + display: flex; + align-items: center; + margin-top: 0.5rem; + + .avatar { + margin-right: 0.5rem; + border-radius: 50%; + } + } +} + .card-img-top { width: 100%; overflow: hidden; @@ -15,6 +55,22 @@ max-width: 800px; margin: 0px auto; + .blog-content { + margin-bottom: 1rem; + + .blog-header { + margin-bottom: 3rem; + margin-top: 3rem; + } + } + + .blog-footer { + display: flex; + justify-content: space-between; + color: #8d99a6; + margin-top: 3rem; + } + .blog-intro { font-size: 1.25rem; font-weight: 300; @@ -41,7 +97,7 @@ font-size: 2rem; } - h1.blog-header { + h1.blog-title { margin-top: 2rem; font-weight: 800; font-size: 2.5rem; diff --git a/frappe/website/doctype/blog_post/templates/blog_post.html b/frappe/website/doctype/blog_post/templates/blog_post.html index 2bb4b6025b..d335cb6613 100644 --- a/frappe/website/doctype/blog_post/templates/blog_post.html +++ b/frappe/website/doctype/blog_post/templates/blog_post.html @@ -6,15 +6,15 @@ {% block page_content %}
-
+
-
+
-

{{ title }}

+

{{ title }}

{{ blog_intro }}

@@ -44,7 +44,7 @@ ]) }} {% endif %} -
+
{% endblock %} {% block page_sidebar %} {% include "templates/includes/web_sidebar.html" %} {% endblock %} diff --git a/frappe/www/contact.html b/frappe/www/contact.html index 5433ccf253..91f14985b2 100644 --- a/frappe/www/contact.html +++ b/frappe/www/contact.html @@ -1,6 +1,7 @@ {% extends "templates/web.html" %} -{% block title %}{{ heading or "Contact Us"}}{% endblock %} +{% set title = heading or "Contact Us" %} +{% block header %}

{{ heading or "Contact Us" }}

{% endblock %} {% block page_content %}