Data Import Tool in excel file format (#3231)

* Data Import in excel file format

* Include test case and minor fixes

* typos

frappe/core/page/data_import_tool/README.md

@@ -1 +1 @@
-Bulk import / update of data via file upload in CSV.
+Bulk import / update of data via file upload in Excel or CSV.

frappe/core/page/data_import_tool/data_import_main.html

@@ -46,11 +46,21 @@
                 <h6 class="text-muted">{%= __("Recommended bulk editing records via import, or understanding the import format.") %}</h6>
             </div>
         </div>
+        <div class="row">
+            <div class="col-sm-4">
+                <div class="checkbox" style="margin: 5px 0px;">
+                    <label>
+                        <input type="checkbox" class="excel-check" data-fieldname="excel_check" checked>
+                            <small>{%= __("Download in Excel File Format") %}</small>
+                    </label>
+                </div>
+            </div>
+        </div>
     </div>
     <div>
         <hr style="margin-top: 50px;">
         <h3>{%= __("Import") %}</h3>
-        <p class="text-muted">{%= __("Update the template and save in CSV (Comma Separate Values) format before attaching.") %}</p>
+        <p class="text-muted">{%= __("Update the template and save in downloaded format before attaching.") %}</p>
         <div class="row">
             <div class="col-md-6">
 				<br>

frappe/core/page/data_import_tool/data_import_tool.js

@@ -99,7 +99,9 @@ frappe.DataImportTool = Class.extend({
 			parent_doctype: doctype,
 			select_columns: JSON.stringify(columns),
 			with_data: with_data ? 'Yes' : 'No',
-			all_doctypes: 'Yes'
+			all_doctypes: 'Yes',
+			from_data_import: 'Yes',
+			excel_format: this.page.main.find(".excel-check").is(":checked") ? 'Yes' : 'No'
 		}
 	},
 	make_upload: function() {
@@ -113,7 +115,8 @@ frappe.DataImportTool = Class.extend({
 					ignore_encoding_errors: me.page.main.find('[name="ignore_encoding_errors"]').prop("checked"),
 					overwrite: !me.page.main.find('[name="always_insert"]').prop("checked"),
 					update_only: me.page.main.find('[name="update_only"]').prop("checked"),
-					no_email: me.page.main.find('[name="no_email"]').prop("checked")
+					no_email: me.page.main.find('[name="no_email"]').prop("checked"),
+					from_data_import: 'Yes'
 				}
 			},
 			args: {

frappe/core/page/data_import_tool/exporter.py

@@ -6,7 +6,7 @@ from __future__ import unicode_literals
 import frappe, json
 from frappe import _
 import frappe.permissions
-import re
+import re, csv, os
 from frappe.utils.csvutils import UnicodeWriter
 from frappe.utils import cstr, formatdate, format_datetime
 from  frappe.core.page.data_import_tool.data_import_tool import get_data_keys
@@ -22,7 +22,8 @@ reflags = {
 }
 
 @frappe.whitelist()
-def get_template(doctype=None, parent_doctype=None, all_doctypes="No", with_data="No", select_columns=None):
+def get_template(doctype=None, parent_doctype=None, all_doctypes="No", with_data="No", select_columns=None,
+	from_data_import="No", excel_format="No"):
 	all_doctypes = all_doctypes=="Yes"
 	if select_columns:
 		select_columns = json.loads(select_columns);
@@ -280,7 +281,26 @@ def get_template(doctype=None, parent_doctype=None, all_doctypes="No", with_data
 	add_field_headings()
 	add_data()
 
-	# write out response as a type csv
-	frappe.response['result'] = cstr(w.getvalue())
-	frappe.response['type'] = 'csv'
-	frappe.response['doctype'] = doctype
+	if from_data_import == "Yes" and excel_format == "Yes":
+		filename = frappe.generate_hash("", 10)
+		with open(filename, 'wb') as f:
+		    f.write(cstr(w.getvalue()).encode("utf-8"))
+		f = open(filename)
+		reader = csv.reader(f)
+
+		from frappe.utils.xlsxutils import make_xlsx
+		xlsx_file = make_xlsx(reader, "Data Import Template")
+
+		f.close()
+		os.remove(filename)
+
+		# write out response as a xlsx type
+		frappe.response['filename'] = doctype + '.xlsx'
+		frappe.response['filecontent'] = xlsx_file.getvalue()
+		frappe.response['type'] = 'binary'
+
+	else:
+		# write out response as a type csv
+		frappe.response['result'] = cstr(w.getvalue())
+		frappe.response['type'] = 'csv'
+		frappe.response['doctype'] = doctype

frappe/core/page/data_import_tool/importer.py

@@ -5,7 +5,7 @@ from __future__ import unicode_literals, print_function
 
 from six.moves import range
 import requests
-import frappe, json
+import frappe, json, os
 import frappe.permissions
 import frappe.async
 
@@ -20,7 +20,7 @@ from frappe.core.page.data_import_tool.data_import_tool import get_data_keys
 
 @frappe.whitelist()
 def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False, no_email=True, overwrite=None,
-	update_only = None, ignore_links=False, pre_process=None, via_console=False):
+	update_only = None, ignore_links=False, pre_process=None, via_console=False, from_data_import="No"):
 	"""upload data"""
 
 	frappe.flags.in_import = True
@@ -37,11 +37,11 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False,
 		no_email = False
 	if params.get('update_only'):
 		update_only = True
+	if params.get('from_data_import'):
+		from_data_import = params.get('from_data_import')
 
 	frappe.flags.mute_emails = no_email
 
-	from frappe.utils.csvutils import read_csv_content_from_uploaded_file
-
 	def get_data_keys_definition():
 		return get_data_keys()
 
@@ -207,7 +207,23 @@ def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False,
 
 	# header
 	if not rows:
-		rows = read_csv_content_from_uploaded_file(ignore_encoding_errors)
+		from frappe.utils.file_manager import save_uploaded
+		file_doc = save_uploaded(dt=None, dn="Data Import", folder='Home', is_private=1)
+		filename, file_extension = os.path.splitext(file_doc.file_name)
+
+		if file_extension == '.xlsx' and from_data_import == 'Yes':
+			from frappe.utils.xlsxutils import read_xlsx_file_from_attached_file
+			rows = read_xlsx_file_from_attached_file(file_id=file_doc.name)
+
+		elif file_extension == '.csv':
+			from frappe.utils.file_manager import get_file
+			from frappe.utils.csvutils import read_csv_content
+			fname, fcontent = get_file(file_doc.names)
+			rows = read_csv_content(fcontent, ignore_encoding_errors)
+
+		else:
+			frappe.throw(_("Unsupported File Format"))
+
 	start_row = get_start_row()
 	header = rows[:start_row]
 	data = rows[start_row:]

frappe/tests/test_data_import.py

@@ -85,3 +85,14 @@ class TestDataImport(unittest.TestCase):
 		importer.upload(content)
 
 		ev = frappe.get_doc("Event", {"subject":"__Test Event with children"})
+
+	def test_excel_import(self):
+		if frappe.db.exists("Event", "EV00001"):
+			frappe.delete_doc("Event", "EV00001")
+
+		exporter.get_template("Event", all_doctypes="No", with_data="No", from_data_import="Yes", excel_format="Yes")
+		from frappe.utils.xlsxutils import read_xlsx_file_from_attached_file
+		content = read_xlsx_file_from_attached_file(fcontent=frappe.response.filecontent)
+		content.append(["", "EV00001", "_test", "Private", "05-11-2017 13:51:48", "0", "0", "", "1", "blue"])
+		importer.upload(content)
+		self.assertTrue(frappe.db.get_value("Event", "EV00001", "subject"), "_test")

frappe/utils/xlsxutils.py

@@ -8,6 +8,7 @@ from frappe.utils import encode, cstr, cint, flt, comma_or
 import openpyxl
 from cStringIO import StringIO
 from openpyxl.styles import Font
+from openpyxl import load_workbook
 
 
 # return xlsx file object
@@ -22,7 +23,7 @@ def make_xlsx(data, sheet_name):
 	for row in data:
 		clean_row = []
 		for item in row:
-			if isinstance(item, basestring):
+			if isinstance(item, basestring) and sheet_name != "Data Import Template":
 				value = handle_html(item)
 			else:
 				value = item
@@ -36,7 +37,6 @@ def make_xlsx(data, sheet_name):
 
 
 def handle_html(data):
-	# import html2text
 	from html2text import unescape, HTML2Text
 
 	h = HTML2Text()
@@ -53,3 +53,24 @@ def handle_html(data):
 		return value[0]
 	else:
 		return value[1]
+
+
+def read_xlsx_file_from_attached_file(file_id=None, fcontent=None):
+	if file_id:
+		from frappe.utils.file_manager import get_file_path
+		filename = get_file_path(file_id)
+	elif fcontent:
+		from io import BytesIO
+		filename = BytesIO(fcontent)
+	else:
+		return
+
+	rows = []
+	wb1 = load_workbook(filename=filename, read_only=True)
+	ws1 = wb1.active
+	for row in ws1.iter_rows():
+		tmp_list = []
+		for cell in row:
+			tmp_list.append(cell.value)
+		rows.append(tmp_list)
+	return rows