From de7e288875ac2382fa3f477ef8e416ab667e0966 Mon Sep 17 00:00:00 2001
From: Rushabh Mehta <rmehta@gmail.com>
Date: Thu, 11 Aug 2016 13:26:51 +0530
Subject: [PATCH] [feature] select columns in data import tool

---
 frappe/change_log/v7/v7_0_18                  |  1 -
 frappe/change_log/v7/v7_0_18.md               |  2 +
 .../data_import_tool/data_import_main.html    | 30 ++++++--
 .../page/data_import_tool/data_import_tool.js | 71 ++++++++++++++-----
 .../data_import_tool_columns.html             | 21 ++++++
 frappe/core/page/data_import_tool/exporter.py | 42 +++++++----
 frappe/public/js/frappe/form/control.js       |  3 +
 frappe/public/js/frappe/form/formatters.js    |  3 +-
 frappe/public/js/frappe/misc/utils.js         |  9 +++
 frappe/public/js/frappe/upload.js             |  2 +-
 frappe/public/js/lib/microtemplate.js         | 12 +++-
 frappe/tests/test_data_import.py              |  5 +-
 12 files changed, 157 insertions(+), 44 deletions(-)
 delete mode 100644 frappe/change_log/v7/v7_0_18
 create mode 100644 frappe/change_log/v7/v7_0_18.md
 create mode 100644 frappe/core/page/data_import_tool/data_import_tool_columns.html

diff --git a/frappe/change_log/v7/v7_0_18 b/frappe/change_log/v7/v7_0_18
deleted file mode 100644
index 51fdd6ef3e..0000000000
--- a/frappe/change_log/v7/v7_0_18
+++ /dev/null
@@ -1 +0,0 @@
-- Ability to add multiple sessions to users. Edit user record and edit "Simultaneous Sessions"
\ No newline at end of file
diff --git a/frappe/change_log/v7/v7_0_18.md b/frappe/change_log/v7/v7_0_18.md
new file mode 100644
index 0000000000..5e59c024c2
--- /dev/null
+++ b/frappe/change_log/v7/v7_0_18.md
@@ -0,0 +1,2 @@
+- New Feature: Ability to add multiple sessions to users. Edit **User** record and edit "Simultaneous Sessions"
+- New Feature: Select columns to export and import in **Data Import Tool**
\ No newline at end of file
diff --git a/frappe/core/page/data_import_tool/data_import_main.html b/frappe/core/page/data_import_tool/data_import_main.html
index 13991252bd..beae98338b 100644
--- a/frappe/core/page/data_import_tool/data_import_main.html
+++ b/frappe/core/page/data_import_tool/data_import_main.html
@@ -14,19 +14,32 @@
             <br>
         </div>
     </div>
-    <div class="export-import-section hide">
-        <div class="row" style="max-width: 700px;">
+    <div class="export-import-section hide"  style="max-width: 700px;">
+		<h4>{{ __("1. Select Columns") }}</h4>
+        <p>
+			<a class="btn btn-default btn-xs btn-select-all" style="margin-right: 7px;">
+            {%= __("Select All") %}</a>
+			<a class="btn btn-default btn-xs btn-select-mandatory" style="margin-right: 7px;">
+            {%= __("Select Mandatory") %}</a>
+			<a class="btn btn-default btn-xs btn-unselect-all">
+            {%= __("Unselect All") %}</a>
+		</p>
+		<div class="select-columns">
+		</div>
+		<br>
+		<h4>{{ __("2. Download") }}</h4>
+		<div class="row">
             <div class="col-sm-4">
-                <p><a class="btn btn-default btn-sm btn-download-template">
+                <p><a class="btn btn-primary btn-xs btn-download-template">
                     {%= __("Download Blank Template") %}</a></p>
             </div>
             <div class="col-sm-8">
                 <h6 class="text-muted">{%= __("Recommended for inserting new records.") %}</h6>
             </div>
         </div>
-        <div class="row" style="max-width: 700px;">
+        <div class="row">
             <div class="col-sm-4">
-                <p><a class="btn btn-default btn-sm btn-download-data">
+                <p><a class="btn btn-primary btn-xs btn-download-data">
                     {%= __("Download with Data") %}</a></p>
             </div>
             <div class="col-sm-8">
@@ -35,12 +48,17 @@
         </div>
     </div>
     <div>
-        <hr>
+        <hr style="margin-top: 50px;">
         <h3>{%= __("Import") %}</h3>
         <p class="text-muted">{%= __("Update the template and save in CSV (Comma Separate Values) format before attaching.") %}</p>
         <div class="row">
             <div class="col-md-6">
+				<br>
+				<h4>{{ __("1. Select File") }}</h4>
                 <div class="upload-area"></div>
+				<br>
+
+				<h4>{{ __("2. Upload") }}</h4>
                 <div class="checkbox">
                     <label>
                         <input type="checkbox" name="always_insert">
diff --git a/frappe/core/page/data_import_tool/data_import_tool.js b/frappe/core/page/data_import_tool/data_import_tool.js
index e3e01c6fec..f1094ae472 100644
--- a/frappe/core/page/data_import_tool/data_import_tool.js
+++ b/frappe/core/page/data_import_tool/data_import_tool.js
@@ -34,28 +34,65 @@ frappe.DataImportTool = Class.extend({
 		$(frappe.render_template("data_import_main", this)).appendTo(this.page.main);
 
 		this.select = this.page.main.find("select.doctype");
+		this.select_columns = this.page.main.find('.select-columns');
 		this.select.on("change", function() {
 			me.doctype = $(this).val();
-			me.page.main.find(".export-import-section").toggleClass(!!me.doctype);
-			if(me.doctype) {
-				me.set_btn_links();
-				// set button links
-			}
-		});
-	},
-	set_btn_links: function() {
-		var doctype = encodeURIComponent(this.doctype);
-		this.page.main.find(".btn-download-template").attr("href",
-			"/api/method/frappe.core.page.data_import_tool.exporter.get_template?"
-			+ "doctype=" + doctype
-			+ "&parent_doctype=" + doctype
-			+ "&with_data=No&all_doctypes=Yes");
+			frappe.model.with_doctype(me.doctype, function() {
+				me.page.main.find(".export-import-section").toggleClass(!!me.doctype);
+				if(me.doctype) {
 
-		this.page.main.find(".btn-download-data").attr("href",
-			"/api/method/frappe.core.page.data_import_tool.exporter.get_template?"
+					// render select columns
+					var doctype_list = [frappe.get_doc('DocType', me.doctype)];
+					frappe.meta.get_table_fields(me.doctype).forEach(function(df) {
+						doctype_list.push(frappe.get_doc('DocType', df.options));
+					});
+
+					$(frappe.render_template("data_import_tool_columns", {doctype_list: doctype_list}))
+						.appendTo(me.select_columns.empty());
+				}
+			});
+		});
+
+		this.page.main.find('.btn-select-all').on('click', function() {
+			me.select_columns.find('.select-column-check').prop('checked', true);
+		});
+
+		this.page.main.find('.btn-unselect-all').on('click', function() {
+			me.select_columns.find('.select-column-check').prop('checked', false);
+		});
+
+		this.page.main.find('.btn-select-mandatory').on('click', function() {
+			me.select_columns.find('.select-column-check').prop('checked', false);
+			me.select_columns.find('.select-column-check[data-reqd="1"]').prop('checked', true);
+		});
+
+		this.page.main.find(".btn-download-template").on('click', function() {
+			window.open(me.get_export_url(false));
+		});
+
+		this.page.main.find(".btn-download-data").on('click', function() {
+			window.open(me.get_export_url(true));
+		});
+
+	},
+	get_export_url: function(with_data) {
+		var doctype = this.select.val();
+		var columns = {};
+
+		this.select_columns.find('.select-column-check:checked').each(function() {
+			var _doctype = $(this).attr('data-doctype');
+			var _fieldname = $(this).attr('data-fieldname');
+			if(!columns[_doctype]) {
+				columns[_doctype] = [];
+			}
+			columns[_doctype].push(_fieldname);
+		});
+
+		return "/api/method/frappe.core.page.data_import_tool.exporter.get_template?"
 			+ "doctype=" + doctype
 			+ "&parent_doctype=" + doctype
-			+ "&with_data=Yes&all_doctypes=Yes");
+			+ "&select_columns=" + JSON.stringify(columns)
+			+ "&with_data="+ (with_data ? 'Yes' : 'No')+"&all_doctypes=Yes";
 	},
 	make_upload: function() {
 		var me = this;
diff --git a/frappe/core/page/data_import_tool/data_import_tool_columns.html b/frappe/core/page/data_import_tool/data_import_tool_columns.html
new file mode 100644
index 0000000000..d5a28181c7
--- /dev/null
+++ b/frappe/core/page/data_import_tool/data_import_tool_columns.html
@@ -0,0 +1,21 @@
+<div style="margin: 15px 0px;">
+{% for doctype in doctype_list %}
+	<h5 style="margin-top: 25px; margin-bottom: 5px;">{{ doctype.name }}</h5>
+	<div class="row">
+		{% for f in doctype.fields %}
+		{% if (frappe.model.no_value_type.indexOf(f.fieldtype)===-1) %}
+		<div class="col-sm-4">
+            <div class="checkbox" style="margin: 5px 0px;">
+                <label>
+                    <input type="checkbox" class="select-column-check"
+						data-fieldname="{{ f.fieldname }}" data-reqd="{{ f.reqd }}"
+						data-doctype="{{ doctype.name }}" checked>
+						<small>{{ __(f.label) }}</small>
+                </label>
+            </div>
+		</div>
+		{% endif %}
+		{% endfor %}
+	</div>
+{% endfor %}
+</div>
diff --git a/frappe/core/page/data_import_tool/exporter.py b/frappe/core/page/data_import_tool/exporter.py
index c3baafdd75..eb47455a1d 100644
--- a/frappe/core/page/data_import_tool/exporter.py
+++ b/frappe/core/page/data_import_tool/exporter.py
@@ -3,12 +3,12 @@
 
 from __future__ import unicode_literals
 
-import frappe, json, os
+import frappe, json
 from frappe import _
 import frappe.permissions
 import re
 from frappe.utils.csvutils import UnicodeWriter
-from frappe.utils import cstr, cint, flt, formatdate, format_datetime
+from frappe.utils import cstr, formatdate, format_datetime
 from  frappe.core.page.data_import_tool.data_import_tool import get_data_keys
 
 reflags = {
@@ -22,8 +22,10 @@ reflags = {
 }
 
 @frappe.whitelist()
-def get_template(doctype=None, parent_doctype=None, all_doctypes="No", with_data="No"):
+def get_template(doctype=None, parent_doctype=None, all_doctypes="No", with_data="No", select_columns=None):
 	all_doctypes = all_doctypes=="Yes"
+	if select_columns:
+		select_columns = json.loads(select_columns);
 	docs_to_export = {}
 	if doctype:
 		if isinstance(doctype, basestring):
@@ -84,6 +86,7 @@ def get_template(doctype=None, parent_doctype=None, all_doctypes="No", with_data
 
 			append_field_column(frappe._dict({
 				"fieldname": "name",
+				"parent": dt,
 				"label": "ID",
 				"fieldtype": "Data",
 				"reqd": 1,
@@ -105,16 +108,27 @@ def get_template(doctype=None, parent_doctype=None, all_doctypes="No", with_data
 
 		column_start_end[dt].end = len(columns) + 1
 
-	def append_field_column(docfield, mandatory):
-		if docfield and ((mandatory and docfield.reqd) or not (mandatory or docfield.reqd)) \
-			and (docfield.fieldname not in ('parenttype', 'trash_reason')) and not docfield.hidden:
-			tablerow.append("")
-			fieldrow.append(docfield.fieldname)
-			labelrow.append(_(docfield.label))
-			mandatoryrow.append(docfield.reqd and 'Yes' or 'No')
-			typerow.append(docfield.fieldtype)
-			inforow.append(getinforow(docfield))
-			columns.append(docfield.fieldname)
+	def append_field_column(docfield, for_mandatory):
+		if not docfield:
+			return
+		if for_mandatory and not docfield.reqd:
+			return
+		if not for_mandatory and docfield.reqd:
+			return
+		if docfield.fieldname in ('parenttype', 'trash_reason'):
+			return
+		if docfield.hidden:
+			return
+		if select_columns and docfield.fieldname not in select_columns.get(docfield.parent, []):
+			return
+
+		tablerow.append("")
+		fieldrow.append(docfield.fieldname)
+		labelrow.append(_(docfield.label))
+		mandatoryrow.append(docfield.reqd and 'Yes' or 'No')
+		typerow.append(docfield.fieldtype)
+		inforow.append(getinforow(docfield))
+		columns.append(docfield.fieldname)
 
 	def append_empty_field_column():
 		tablerow.append("~")
@@ -238,6 +252,8 @@ def get_template(doctype=None, parent_doctype=None, all_doctypes="No", with_data
 	inforow = [_('Info:'), '']
 	columns = [key]
 
+
+
 	build_field_columns(doctype)
 	if all_doctypes:
 		for d in child_doctypes:
diff --git a/frappe/public/js/frappe/form/control.js b/frappe/public/js/frappe/form/control.js
index fadde6fc9e..1bd20aee61 100644
--- a/frappe/public/js/frappe/form/control.js
+++ b/frappe/public/js/frappe/form/control.js
@@ -842,6 +842,7 @@ frappe.ui.form.ControlAttach = frappe.ui.form.ControlData.extend({
 			me.frm.attachments.remove_attachment_by_filename(me.value, function() {
 				me.parse_validate_and_set_in_model(null);
 				me.refresh();
+				me.frm.save();
 			});
 		} else {
 			this.dataurl = null;
@@ -918,6 +919,7 @@ frappe.ui.form.ControlAttach = frappe.ui.form.ControlData.extend({
 				if(selected) {
 					me.parse_validate_and_set_in_model(selected);
 					me.dialog.hide();
+					me.frm.save();
 				} else {
 					msgprint(__("Please attach a file or set a URL"));
 				}
@@ -988,6 +990,7 @@ frappe.ui.form.ControlAttach = frappe.ui.form.ControlData.extend({
 			this.parse_validate_and_set_in_model(attachment.file_url);
 			this.refresh();
 			this.frm.attachments.update_attachment(attachment);
+			this.frm.save();
 		} else {
 			this.value = this.get_value();
 			this.refresh();
diff --git a/frappe/public/js/frappe/form/formatters.js b/frappe/public/js/frappe/form/formatters.js
index 02bf07b5c9..44c3c3f74e 100644
--- a/frappe/public/js/frappe/form/formatters.js
+++ b/frappe/public/js/frappe/form/formatters.js
@@ -64,6 +64,7 @@ frappe.form.formatters = {
 	},
 	Link: function(value, docfield, options, doc) {
 		var doctype = docfield._options || docfield.options;
+		var original_value = value;
 		if(value && value.match(/^['"].*['"]$/)) {
 			value.replace(/^.(.*).$/, "$1");
 		}
@@ -85,7 +86,7 @@ frappe.form.formatters = {
 		} else if(docfield && doctype) {
 			return repl('<a class="grey" href="#Form/%(doctype)s/%(name)s" data-doctype="%(doctype)s">%(label)s</a>', {
 				doctype: encodeURIComponent(doctype),
-				name: encodeURIComponent(value),
+				name: encodeURIComponent(original_value),
 				label: __(options && options.label || value)
 			});
 		} else {
diff --git a/frappe/public/js/frappe/misc/utils.js b/frappe/public/js/frappe/misc/utils.js
index dded7ff7a7..c8c75192af 100644
--- a/frappe/public/js/frappe/misc/utils.js
+++ b/frappe/public/js/frappe/misc/utils.js
@@ -4,6 +4,15 @@
 frappe.provide('frappe.utils');
 
 frappe.utils = {
+	get_random: function(len) {
+	    var text = "";
+	    var possible = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
+
+	    for( var i=0; i < len; i++ )
+	        text += possible.charAt(Math.floor(Math.random() * possible.length));
+
+	    return text;
+	},
 	get_file_link: function(filename) {
 		filename = cstr(filename);
 		if(frappe.utils.is_url(filename)) {
diff --git a/frappe/public/js/frappe/upload.js b/frappe/public/js/frappe/upload.js
index 8f95e0226e..463aec5f77 100644
--- a/frappe/public/js/frappe/upload.js
+++ b/frappe/public/js/frappe/upload.js
@@ -170,7 +170,7 @@ frappe.upload = {
 					return;
 				}
 				var attachment = r.message;
-				opts.callback(attachment, r);
+				opts.callback && opts.callback(attachment, r);
 				$(document).trigger("upload_complete", attachment);
 			},
 			error: function(r) {
diff --git a/frappe/public/js/lib/microtemplate.js b/frappe/public/js/lib/microtemplate.js
index 26884265b4..deb8a067fc 100644
--- a/frappe/public/js/lib/microtemplate.js
+++ b/frappe/public/js/lib/microtemplate.js
@@ -13,7 +13,7 @@ frappe.template.compile = function(str, name) {
 
 		// replace jinja style tags
 		str = str.replace(/{{/g, "{%=").replace(/}}/g, "%}");
-		
+
 		// {% if not test %} --> {% if (!test) { %}
 		str = str.replace(/{%\s?if\s?\s?not\s?([^\(][^%{]+)\s?%}/g, "{% if (! $1) { %}")
 
@@ -22,7 +22,13 @@ frappe.template.compile = function(str, name) {
 
 		// {% for item in list %}
 		//       --> {% for (var i=0, len=list.length; i<len; i++) {  var item = list[i]; %}
-		str = str.replace(/{%\s?for\s([a-z]+)\sin\s([a-z]+)\s?%}/g, "{% for (var i=0, len=$2.length; i<len; i++) { var $1 = $2[i]; %}");
+		function replacer(match, p1, p2, offset, string) {
+			var i = frappe.utils.get_random(3);
+			var len = frappe.utils.get_random(3);
+			return "{% for (var "+i+"=0, "+len+"="+p2+".length; "+i+"<"+len+"; "+i+"++) { var "
+				+p1+" = "+p2+"["+i+"]; %}";
+		}
+		str = str.replace(/{%\s?for\s([a-z]+)\sin\s([a-z._]+)\s?%}/g, replacer);
 
 		// {% endfor %} --> {% } %}
 		str = str.replace(/{%\s?endif\s?%}/g, "{% }; %}");
@@ -46,7 +52,7 @@ frappe.template.compile = function(str, name) {
 	          .split("\r").join("\\'")
 	      + "');}return _p.join('');";
 
-  		frappe.template.debug[str] = fn_str;
+		  frappe.template.debug[name] = fn_str;
 		try {
 			frappe.template.compiled[key] = new Function("obj", fn_str);
 		} catch (e) {
diff --git a/frappe/tests/test_data_import.py b/frappe/tests/test_data_import.py
index af7cc4fc15..69940063f6 100644
--- a/frappe/tests/test_data_import.py
+++ b/frappe/tests/test_data_import.py
@@ -76,12 +76,13 @@ class TestDataImport(unittest.TestCase):
 	def test_import_with_children(self):
 		exporter.get_template("Event", all_doctypes="Yes", with_data="No")
 		content = read_csv_content(frappe.response.result)
+
 		content.append([None] * len(content[-2]))
-		content[-1][2] = "__Test Event"
+		content[-1][2] = "__Test Event with children"
 		content[-1][3] = "Private"
 		content[-1][4] = "2014-01-01 10:00:00.000000"
 		content[-1][content[15].index("role")] = "System Manager"
 		importer.upload(content)
 
-		ev = frappe.get_doc("Event", {"subject":"__Test Event"})
+		ev = frappe.get_doc("Event", {"subject":"__Test Event with children"})
 		self.assertTrue("System Manager" in [d.role for d in ev.roles])