From b35719f1faa26b3e86cbccff1bdb22e99e1fa2e2 Mon Sep 17 00:00:00 2001 From: Rushabh Mehta Date: Fri, 11 May 2012 15:54:46 +0530 Subject: [PATCH] data import tool and cleanup of files, handler and other minor --- css/legacy/body.css | 1 + js/wn/app.js | 2 +- js/wn/upload.js | 2 +- .../doctype/doctype_mapper/doctype_mapper.py | 2 +- .../page/data_import_tool/data_import_tool.js | 79 +++++++-- .../page/data_import_tool/data_import_tool.py | 165 ++++++++++++++---- py/webnotes/__init__.py | 17 +- py/webnotes/handler.py | 69 ++------ py/webnotes/model/code.py | 4 +- py/webnotes/model/doc.py | 41 ++--- py/webnotes/utils/file_manager.py | 31 ++-- 11 files changed, 256 insertions(+), 157 deletions(-) diff --git a/css/legacy/body.css b/css/legacy/body.css index 774b523005..13247062e3 100644 --- a/css/legacy/body.css +++ b/css/legacy/body.css @@ -92,6 +92,7 @@ li { } hr { + clear: both; margin: 18px 0; border: 0; border-top: 1px solid #e5e5e5; diff --git a/js/wn/app.js b/js/wn/app.js index 700766c40c..418fe1a87a 100644 --- a/js/wn/app.js +++ b/js/wn/app.js @@ -38,7 +38,7 @@ wn.Application = Class.extend({ }) } else { // clear sid cookie - document.cookie = "sid=Guest;expires=Thu, 01 Jan 1970 00:00:00 GMT; path=/" + //document.cookie = "sid=Guest;expires=Thu, 01 Jan 1970 00:00:00 GMT; path=/" this.startup(); //wn.views.pageview.show(window.home_page); } diff --git a/js/wn/upload.js b/js/wn/upload.js index 7c4d8c2473..bfbf0f68c4 100644 --- a/js/wn/upload.js +++ b/js/wn/upload.js @@ -6,7 +6,7 @@ wn.upload = { style="width:0px; height:0px; border:0px">\
\ -
\ +

\ \
', { id: id, diff --git a/py/core/doctype/doctype_mapper/doctype_mapper.py b/py/core/doctype/doctype_mapper/doctype_mapper.py index 8af627fd35..acca86f58f 100644 --- a/py/core/doctype/doctype_mapper/doctype_mapper.py +++ b/py/core/doctype/doctype_mapper/doctype_mapper.py @@ -25,7 +25,7 @@ import webnotes from webnotes.utils import cint, cstr, default_fields, flt, formatdate, get_defaults, getdate, now, nowdate, replace_newlines, set_default from webnotes.model import db_exists, default_fields -from webnotes.model.doc import Document, addchild, removechild, getchildren, make_autoname, SuperDocType +from webnotes.model.doc import Document, addchild, getchildren, make_autoname from webnotes.model.doclist import getlist from webnotes.model.code import get_obj from webnotes import session, form, msgprint, errprint diff --git a/py/core/page/data_import_tool/data_import_tool.js b/py/core/page/data_import_tool/data_import_tool.js index 7386680308..b10285d7c4 100644 --- a/py/core/page/data_import_tool/data_import_tool.js +++ b/py/core/page/data_import_tool/data_import_tool.js @@ -6,17 +6,40 @@ wn.pages['data-import-tool'].onload = function(wrapper) { $(wrapper).find('.layout-main-section').append('

1. Download Template

\
\ -

Download a template for importing a table

\ -

\ -

\ +

Download a template for importing a table.

\ +

\ +

\ + \ + Download with data\ +

\ +

\
\
\

2. Import Data

\ -

Attach file to import data

\ +

Attach .csv file to import data

\

\

\ '); + + $(wrapper).find('.layout-side-section').append('

Help


\ +

Date Format:

\ +

Dates must be in format "YYYY-MM-DD", for example, \ + 31st Jan 2012 must be "2012-01-31"

\ +

Importing non-English data:

\ +

While uploading non English files ensure that the encoding is UTF-8.

\ +

Microsoft Excel Users:\ +

    \ +
  1. In Excel, save the file in CSV (Comma Delimited) format
    2. \ +
  2. Open this saved file in Notepad
    3. \ +
  3. Click on File -> Save As
    4. \ +
  4. File Name: <your filename>.csv
    \ + Save as type: Text Documents (*.txt)
    \ + Encoding: UTF-8\ +
    5. \ +
  5. Click on Save
    6. \ +
\ +

') $select = $(wrapper).find('[name="dit-doctype"]'); @@ -32,20 +55,37 @@ wn.pages['data-import-tool'].onload = function(wrapper) { $select.change(function() { var val = $(this).val() if(val!='Select...') { - $('.dit-download').empty(); + $('#dit-download').empty(); // get options wn.call({ method:'core.page.data_import_tool.data_import_tool.get_doctype_options', args: {doctype: val}, callback: function(r) { - $('

Select Template:

').appendTo('.dit-download'); - + $('

Select Template:

').appendTo('#dit-download'); + var with_data = $('[name="dit-with-data"]:checked').length ? 'Yes' : 'No'; // download link $.each(r.message, function(i, v) { - $(''+v+'
') - .appendTo('.dit-download'); + if(i==0) + $('Main Table:
').appendTo('#dit-download'); + if(i==1) + $('
Child Tables:
').appendTo('#dit-download'); + $('') + .html(v) + .data('doctype', v) + .click(function() { + window.location.href = repl(wn.request.url + + '?cmd=%(cmd)s&doctype=%(doctype)s' + + '&parent_doctype=%(parent_doctype)s&with_data=%(with_data)s', + { + cmd: 'core.page.data_import_tool.data_import_tool.get_template', + doctype: $(this).data('doctype'), + parent_doctype: $('[name="dit-doctype"]').val(), + with_data: $('[name="dit-with-data"]:checked').length ? 'Yes' : 'No' + }); + }) + .appendTo('#dit-download'); + $('#dit-download').append('
'); }) } }) @@ -66,8 +106,25 @@ wn.pages['data-import-tool'].onload = function(wrapper) { if(v.substr(0,5)=='Error') { $p.css('color', 'red'); } + if(v.substr(0,8)=='Inserted') { + $p.css('color', 'green'); + } + if(v.substr(0,7)=='Updated') { + $p.css('color', 'green'); + } }); } }); + + // add overwrite option + $(' Overwrite

') + .insertBefore('#dit-upload-area form input[type="submit"]') + + // rename button + $('#dit-upload-area form input[type="submit"]') + .attr('value', 'Upload and Import') + .click(function() { + $('#dit-output').html('Performing hardcore import process....') + }); } \ No newline at end of file diff --git a/py/core/page/data_import_tool/data_import_tool.py b/py/core/page/data_import_tool/data_import_tool.py index 5ee26af584..bee6328db8 100644 --- a/py/core/page/data_import_tool/data_import_tool.py +++ b/py/core/page/data_import_tool/data_import_tool.py @@ -12,25 +12,24 @@ def get_doctype_options(): import webnotes.model.doctype return [doctype] + filter(None, map(lambda d: \ d.doctype=='DocField' and d.fieldtype=='Table' and d.options or None, - webnotes.model.doctype.get(doctype))) + webnotes.model.doctype.get(doctype, form=0))) data_separator = '----Start entering data below this line----' -@webnotes.whitelist() +doctype_dl = None + +@webnotes.whitelist(allow_roles=['System Manager', 'Administrator']) def get_template(): import webnotes, csv from cStringIO import StringIO import webnotes.model.doctype + global doctype_dl doctype = webnotes.form_dict['doctype'] parentdoctype = webnotes.form_dict.get('parent_doctype') - doclist = webnotes.model.doctype.get(doctype) - tablefields = [f[0] for f in webnotes.conn.sql('desc `tab%s`' % doctype)] - - def getdocfield(fieldname): - l = [d for d in doclist if d.doctype=='DocField' and d.fieldname==fieldname] - return l and l[0] or None + doctype_dl = webnotes.model.doctype.get(doctype) + tablecolumns = [f[0] for f in webnotes.conn.sql('desc `tab%s`' % doctype)] def getinforow(docfield): """make info comment""" @@ -53,7 +52,7 @@ def get_template(): w.writerow(['Upload Template for: %s' % doctype]) if parentdoctype != doctype: - w.writerow(['This is a child table for %s' % parentdoctype]) + w.writerow(['This is a child table for: %s' % parentdoctype]) key = 'parent' else: w.writerow(['']) @@ -64,25 +63,26 @@ def get_template(): mandatoryrow = ['Mandatory:', 'Yes'] typerow = ['Type:', 'Data (text)'] inforow = ['Info:', 'ID'] - - # get all mandatory fields - for t in tablefields: + columns = [key] + + def append_row(t, mandatory): docfield = getdocfield(t) - if docfield and docfield.reqd: + if docfield and ((mandatory and docfield.reqd) or (not mandatory and not docfield.reqd)) \ + and (t not in ('parenttype', 'trash_reason')): fieldrow.append(t) - mandatoryrow.append('Yes') - typerow.append(docfield.fieldtype) + mandatoryrow.append(docfield.reqd and 'Yes' or 'No') + typerow.append(docfield.fieldtype) inforow.append(getinforow(docfield)) + columns.append(t) + + # get all mandatory fields + for t in tablecolumns: + append_row(t, True) # all non mandatory fields - for t in tablefields: - docfield = getdocfield(t) - if docfield and not docfield.reqd: - fieldrow.append(t) - mandatoryrow.append('No') - typerow.append(docfield.fieldtype) - inforow.append(getinforow(docfield)) - + for t in tablecolumns: + append_row(t, False) + w.writerow(fieldrow) w.writerow(mandatoryrow) w.writerow(typerow) @@ -90,43 +90,134 @@ def get_template(): w.writerow([data_separator]) + if webnotes.form_dict.get('with_data')=='Yes': + data = webnotes.conn.sql("""select * from `tab%s` where docstatus<2""" % doctype, as_dict=1) + for d in data: + row = [''] + for c in columns: + val = d.get(c, '') + if type(val) is unicode: + val = val.encode('utf-8') + row.append(val) + w.writerow(row) + # write out response as a type csv webnotes.response['result'] = tobj.getvalue() webnotes.response['type'] = 'csv' webnotes.response['doctype'] = doctype - -@webnotes.whitelist() + +def getdocfield(fieldname): + """get docfield from doclist of doctype""" + l = [d for d in doctype_dl if d.doctype=='DocField' and d.fieldname==fieldname] + return l and l[0] or None + +@webnotes.whitelist(allow_roles=['System Manager', 'Administrator']) def upload(): """upload data""" import csv + global doctype_dl from webnotes.utils.file_manager import get_uploaded_content + import webnotes.model.doctype from webnotes.model.doc import Document - from webnotes.model.doclist import DocList - + fname, fcontent = get_uploaded_content() + overwrite = webnotes.form_dict.get('overwrite') ret, rows = [], [] - for row in csv.reader(fcontent.splitlines()): - rows.append([c.strip() for c in row]) + try: + reader = csv.reader(fcontent.splitlines()) + # decode everything + for row in reader: + rows.append([unicode(c.strip(), 'utf-8') for c in row]) + except Exception, e: + webnotes.msgprint("Not a valid Comma Separated Value (CSV File)") + raise e + # doctype doctype = rows[0][0].split(':')[1].strip() + doctype_dl = webnotes.model.doctype.get(doctype, form=0) + + parentdoctype = None + if len(rows[1]) > 0 and ':' in rows[1][0]: + parentdoctype = rows[1][0].split(':')[1].strip() # columns columns = rows[3][1:] + + if parentdoctype and overwrite: + delete_child_rows(rows, doctype) for row in rows[8:]: d = dict(zip(columns, row[1:])) d['doctype'] = doctype - - try: - if not webnotes.conn.exists(doctype, d['name']): - d['__islocal'] = 1 - DocList([Document(fielddata = d)]).save() - ret.append('Uploaded ' + row[1]) + try: + check_record(d, parentdoctype) + if parentdoctype: + # child doc + doc = Document(doctype) + doc.fields.update(d) + doc.save() + ret.append('Inserted row for %s at #%s' % (getlink(parentdoctype, doc.parent), + str(doc.idx))) + else: + ret.append(import_doc(d, doctype, overwrite)) except Exception, e: ret.append('Error for ' + row[1] + ': ' + str(e)) - + webnotes.errprint(webnotes.getTraceback()) return ret - + +def check_record(d, parentdoctype): + """check for mandatory, select options, dates. these should ideally be in doclist""" + if parentdoctype and not d.get('parent'): + raise Exception, "parent is required." + + for key in d: + docfield = getdocfield(key) + val = d[key] + if docfield: + if docfield.reqd and (val=='' or val==None): + raise Exception, "%s is mandatory." % key + + if docfield.fieldtype=='Select': + if docfield.options.startswith('link:'): + if val: + link_doctype = docfield.options.split(':')[1] + if not webnotes.conn.exists(link_doctype, val): + raise Exception, "%s must be a valid %s" % (key, link_doctype) + else: + if val not in docfield.options.split('\n'): + raise Exception, "%s must be one of:" % key + + if docfield.fieldtype=='Date': + import datetime + datetime.datetime.strptime(val, '%Y-%m-%d') + +def getlink(doctype, name): + return '%(name)s' % locals() + +def delete_child_rows(rows, doctype): + """delete child rows for all parents""" + import webnotes + for p in list(set([r[1] for r in rows[8:]])): + webnotes.conn.sql("""delete from `tab%s` where parent=%s""" % (doctype, '%s'), p) + +def import_doc(d, doctype, overwrite): + """import main (non child) document""" + import webnotes + from webnotes.model.doc import Document + from webnotes.model.doclist import DocList + + if webnotes.conn.exists(doctype, d['name']): + if overwrite: + doc = Document(doctype, d['name']) + doc.fields.update(d) + DocList([doc]).save() + return 'Updated ' + getlink(doctype, d['name']) + else: + return 'Ignored ' + getlink(doctype, d['name']) + ' (exists)' + else: + d['__islocal'] = 1 + DocList([Document(fielddata = d)]).save() + return 'Inserted ' + getlink(doctype, d['name']) \ No newline at end of file diff --git a/py/webnotes/__init__.py b/py/webnotes/__init__.py index 265afae3ca..822e9e3fef 100644 --- a/py/webnotes/__init__.py +++ b/py/webnotes/__init__.py @@ -95,7 +95,7 @@ def msgprint(msg, small=0, raise_exception=0, as_table=False): message_log.append((small and '__small:' or '')+cstr(msg or '')) if raise_exception: - raise ValidationError + raise ValidationError, msg def is_apache_user(): import os @@ -190,12 +190,14 @@ def get_db_password(db_name): whitelisted = [] guest_methods = [] -def whitelist(allow_guest=False): +def whitelist(allow_guest=False, allow_roles=[]): """ decorator for whitelisting a function Note: if the function is allowed to be accessed by a guest user, it must explicitly be marked as allow_guest=True + + for specific roles, set allow_roles = ['Administrator'] etc. """ def innerfn(fn): global whitelisted, guest_methods @@ -204,6 +206,17 @@ def whitelist(allow_guest=False): if allow_guest: guest_methods.append(fn) + if allow_roles: + roles = get_roles() + allowed = False + for role in allow_roles: + if role in roles: + allowed = True + break + + if not allowed: + raise PermissionError, "Method not allowed" + return fn return innerfn diff --git a/py/webnotes/handler.py b/py/webnotes/handler.py index 31c823b1cc..04e3717cfe 100755 --- a/py/webnotes/handler.py +++ b/py/webnotes/handler.py @@ -154,58 +154,8 @@ def uploadfile(): if not webnotes.response.get('result'): webnotes.response['result'] = """""" % (webnotes.form_dict.get('_id'), - json.dumps(ret), - json.dumps(webnotes.message_log), - json.dumps(webnotes.debug_log)) - -# File upload (from scripts) -# ------------------------------------------------------------------------------------ - -@webnotes.whitelist() -def upload_many(): - from webnotes.model.code import get_obj - - # pass it on to upload_many method in Control Panel - cp = get_obj('Control Panel') - cp.upload_many(webnotes.form) - - webnotes.response['result'] = """ - -%s -%s""" % (cp.upload_callback(webnotes.form), '\n----\n'.join(webnotes.message_log).replace("'", "\'"), '\n----\n'.join(webnotes.debug_log).replace("'", "\'").replace("\n","
")) - webnotes.response['type'] = 'iframe' - - -@webnotes.whitelist() -def get_file(): - import webnotes - import webnotes.utils.file_manager - form = webnotes.form - - res = webnotes.utils.file_manager.get_file(form.getvalue('fname')) - if res: - webnotes.response['type'] = 'download' - webnotes.response['filename'] = res[0] - - if hasattr(res[1], 'tostring'): - webnotes.response['filecontent'] = res[1].tostring() - else: - webnotes.response['filecontent'] = res[1] - else: - webnotes.msgprint('[get_file] Unknown file name') + """ % (webnotes.form_dict.get('_id'), + json.dumps(ret)) @webnotes.whitelist(allow_guest=True) def reset_password(): @@ -318,12 +268,25 @@ def print_csv(): print webnotes.response['result'] def print_iframe(): + import json print "Content-Type: text/html" print if webnotes.response.get('result'): print webnotes.response['result'] if webnotes.debug_log: - print '''''' % ('-------'.join(webnotes.debug_log).replace('"', '').replace('\n','')) + print """ + """ % (json.dumps(webnotes.message_log), json.dumps(webnotes.debug_log)) def print_raw(): import mimetypes diff --git a/py/webnotes/model/code.py b/py/webnotes/model/code.py index fea49ee4dd..f6bc3f7795 100644 --- a/py/webnotes/model/code.py +++ b/py/webnotes/model/code.py @@ -39,7 +39,7 @@ import webnotes from webnotes.utils import add_days, add_months, add_years, cint, cstr, date_diff, default_fields, flt, fmt_money, formatdate, generate_hash, getTraceback, get_defaults, get_first_day, get_last_day, getdate, has_common, month_name, now, nowdate, replace_newlines, sendmail, set_default, str_esc_quote, user_format, validate_email_add from webnotes.model import db_exists -from webnotes.model.doc import Document, addchild, removechild, getchildren, make_autoname, SuperDocType +from webnotes.model.doc import Document, addchild, getchildren, make_autoname from webnotes.model.utils import getlist from webnotes.model.code import get_obj, get_server_obj, run_server_obj, updatedb, check_syntax from webnotes import session, form, is_testing, msgprint, errprint @@ -70,7 +70,7 @@ def execute(code, doc=None, doclist=[]): # -------------------------------------------------- from webnotes.utils import add_days, add_months, add_years, cint, cstr, date_diff, default_fields, flt, fmt_money, formatdate, generate_hash, getTraceback, get_defaults, get_first_day, get_last_day, getdate, has_common, month_name, now, nowdate, replace_newlines, sendmail, set_default, str_esc_quote, user_format, validate_email_add from webnotes.model import db_exists - from webnotes.model.doc import Document, addchild, removechild, getchildren + from webnotes.model.doc import Document, addchild, getchildren from webnotes.model.utils import getlist from webnotes import session, form, msgprint, errprint diff --git a/py/webnotes/model/doc.py b/py/webnotes/model/doc.py index 67a7b09197..7849c265a7 100755 --- a/py/webnotes/model/doc.py +++ b/py/webnotes/model/doc.py @@ -29,19 +29,6 @@ import webnotes.model.meta from webnotes.utils import * -# actually should be "BaseDocType" - deprecated. Only for v160 -class SuperDocType: - def __init__(self): - pass - - def __getattr__(self, name): - if self.__dict__.has_key(name): - return self.__dict__[name] - elif self.super and hasattr(self.super, name): - return getattr(self.super, name) - else: - raise AttributeError, 'BaseDocType Attribute Error' - class Document: """ The wn(meta-data)framework equivalent of a Database Record. @@ -400,6 +387,9 @@ class Document: # add missing parentinfo (if reqd) if self.parent and not (self.parenttype and self.parentfield): self.update_parentinfo() + + if self.parent and not self.idx: + self.set_idx() # if required, make new if new or (not new and self.fields.get('__islocal')) and (not res.get('issingle')): @@ -430,6 +420,12 @@ class Document: self.parenttype = tmp[0][0] self.parentfield = tmp[0][1] + def set_idx(self): + """set idx""" + self.idx = (webnotes.conn.sql("""select max(idx) from `tab%s` + where parent=%s and parentfield=%s""" % (self.doctype, '%s', '%s'), + (self.parent, self.parentfield))[0][0] or 0) + 1 + # check permissions # --------------------------------------------------------------------------- @@ -559,20 +555,6 @@ def addchild(parent, fieldname, childtype = '', local=0, doclist=None): d.save(1) """ return parent.addchild(fieldname, childtype, local, doclist) - -# Remove Child -# ------------ - -def removechild(d, is_local = 0): - """ - Sets the docstatus of the object d to 2 (deleted) and appends an 'old_parent:' to the parent name - """ - if not is_local: - set(d, 'docstatus', 2) - set(d, 'parent', 'old_parent:' + d.parent) - else: - d.parent = 'old_parent:' + d.parent - d.docstatus = 2 # Naming # ------ @@ -644,11 +626,12 @@ def getchildren(name, childtype, field='', parenttype='', from_doctype=0, prefix if field: tmp = ' and parentfield="%s" ' % field - if parenttype: + if parenttype: tmp = ' and parenttype="%s" ' % parenttype try: - dataset = webnotes.conn.sql("select * from `%s%s` where parent='%s' %s order by idx" % (prefix, childtype, name, tmp)) + dataset = webnotes.conn.sql("select * from `%s%s` where parent='%s' %s order by idx" \ + % (prefix, childtype, name, tmp)) desc = webnotes.conn.get_description() except Exception, e: if prefix=='arc' and e.args[0]==1146: diff --git a/py/webnotes/utils/file_manager.py b/py/webnotes/utils/file_manager.py index a3312c1a05..d63d5b751a 100644 --- a/py/webnotes/utils/file_manager.py +++ b/py/webnotes/utils/file_manager.py @@ -70,26 +70,17 @@ def add_file_list(dt, dn, fname, fid): """ udpate file_list attribute of the record """ - try: - # get the old file_list - fl = webnotes.conn.get_value(dt, dn, 'file_list') or '' - if fl: - fl += '\n' - - # add new file id - fl += fname + ',' + fid - - # save - webnotes.conn.set_value(dt, dn, 'file_list', fl) - - return True + fl = webnotes.conn.get_value(dt, dn, 'file_list') or '' + if fl: + fl += '\n' + + # add new file id + fl += fname + ',' + fid - except Exception, e: - webnotes.response['result'] = """ -""" % str(e) + # save + webnotes.conn.set_value(dt, dn, 'file_list', fl) + return True def remove_all(dt, dn): """remove all files in a transaction""" @@ -141,10 +132,10 @@ def get_uploaded_content(): webnotes.uploaded_filename, webnotes.uploaded_content = i.filename, i.file.read() return webnotes.uploaded_filename, webnotes.uploaded_content else: - webnotes.response['result'] = """""" % js_fail + webnotes.msgprint('No File'); return None, None -def save_uploaded(js_okay='window.parent.msgprint("File Upload Successful")', js_fail=''): +def save_uploaded(): import webnotes.utils webnotes.response['type'] = 'iframe'