diff --git a/.travis.yml b/.travis.yml index 1acc716307..19f80eb69b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ python: services: - mysql - + before_script: - mysql -u root -ptravis -e 'CREATE DATABASE test_frappe' - echo "USE mysql;\nCREATE USER 'test_frappe'@'localhost' IDENTIFIED BY 'test_frappe';\nFLUSH PRIVILEGES;\n" | mysql -u root -ptravis diff --git a/frappe/__init__.py b/frappe/__init__.py index 7c74557347..bb0707c1a9 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -17,7 +17,7 @@ from faker import Faker from .exceptions import * from .utils.jinja import get_jenv, get_template, render_template, get_email_from_template -__version__ = '10.1.23' +__version__ = '10.1.24' __title__ = "Frappe Framework" local = Local() diff --git a/frappe/core/doctype/activity_log/activity_log.py b/frappe/core/doctype/activity_log/activity_log.py index 33e444650b..94a4803279 100644 --- a/frappe/core/doctype/activity_log/activity_log.py +++ b/frappe/core/doctype/activity_log/activity_log.py @@ -41,7 +41,7 @@ def add_authentication_log(subject, user, operation="Login", status="Success"): "status": status, "subject": subject, "operation": operation, - }).insert(ignore_permissions=True) + }).insert(ignore_permissions=True, ignore_links=True) def clear_authentication_logs(): """clear 100 day old authentication logs""" diff --git a/frappe/core/doctype/data_import/exporter.py b/frappe/core/doctype/data_import/exporter.py new file mode 100644 index 0000000000..2a267c122f --- /dev/null +++ b/frappe/core/doctype/data_import/exporter.py @@ -0,0 +1,313 @@ +# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors +# MIT License. See license.txt + +from __future__ import unicode_literals + +import frappe, json +from frappe import _ +import frappe.permissions +import re, csv, os, sys +from frappe.utils.csvutils import UnicodeWriter +from frappe.utils import cstr, formatdate, format_datetime +from frappe.core.doctype.data_import.importer import get_data_keys +from six import string_types + +reflags = { + "I":re.I, + "L":re.L, + "M":re.M, + "U":re.U, + "S":re.S, + "X":re.X, + "D": re.DEBUG +} + +@frappe.whitelist() +def get_template(doctype=None, parent_doctype=None, all_doctypes="No", with_data="No", select_columns=None, + from_data_import="No", excel_format="No"): + all_doctypes = all_doctypes=="Yes" + if select_columns: + select_columns = json.loads(select_columns); + docs_to_export = {} + if doctype: + if isinstance(doctype, string_types): + doctype = [doctype]; + if len(doctype) > 1: + docs_to_export = doctype[1] + doctype = doctype[0] + + if not parent_doctype: + parent_doctype = doctype + + column_start_end = {} + + if all_doctypes: + child_doctypes = [] + for df in frappe.get_meta(doctype).get_table_fields(): + child_doctypes.append(dict(doctype=df.options, parentfield=df.fieldname)) + + def get_data_keys_definition(): + return get_data_keys() + + def add_main_header(): + w.writerow([_('Data Import Template')]) + w.writerow([get_data_keys_definition().main_table, doctype]) + + if parent_doctype != doctype: + w.writerow([get_data_keys_definition().parent_table, parent_doctype]) + else: + w.writerow(['']) + + w.writerow(['']) + w.writerow([_('Notes:')]) + w.writerow([_('Please do not change the template headings.')]) + w.writerow([_('First data column must be blank.')]) + w.writerow([_('If you are uploading new records, leave the "name" (ID) column blank.')]) + w.writerow([_('If you are uploading new records, "Naming Series" becomes mandatory, if present.')]) + w.writerow([_('Only mandatory fields are necessary for new records. You can delete non-mandatory columns if you wish.')]) + w.writerow([_('For updating, you can update only selective columns.')]) + w.writerow([_('You can only upload upto 5000 records in one go. (may be less in some cases)')]) + if key == "parent": + w.writerow([_('"Parent" signifies the parent table in which this row must be added')]) + w.writerow([_('If you are updating, please select "Overwrite" else existing rows will not be deleted.')]) + + def build_field_columns(dt, parentfield=None): + meta = frappe.get_meta(dt) + + # build list of valid docfields + tablecolumns = [] + for f in frappe.db.sql('desc `tab%s`' % dt): + field = meta.get_field(f[0]) + if field and ((select_columns and f[0] in select_columns[dt]) or not select_columns): + tablecolumns.append(field) + + tablecolumns.sort(key = lambda a: int(a.idx)) + + _column_start_end = frappe._dict(start=0) + + if dt==doctype: + _column_start_end = frappe._dict(start=0) + else: + _column_start_end = frappe._dict(start=len(columns)) + + append_field_column(frappe._dict({ + "fieldname": "name", + "parent": dt, + "label": "ID", + "fieldtype": "Data", + "reqd": 1, + "idx": 0, + "info": _("Leave blank for new records") + }), True) + + for docfield in tablecolumns: + append_field_column(docfield, True) + + # all non mandatory fields + for docfield in tablecolumns: + append_field_column(docfield, False) + + # if there is one column, add a blank column (?) + if len(columns)-_column_start_end.start == 1: + append_empty_field_column() + + # append DocType name + tablerow[_column_start_end.start + 1] = dt + + if parentfield: + tablerow[_column_start_end.start + 2] = parentfield + + _column_start_end.end = len(columns) + 1 + + column_start_end[(dt, parentfield)] = _column_start_end + + def append_field_column(docfield, for_mandatory): + if not docfield: + return + if for_mandatory and not docfield.reqd: + return + if not for_mandatory and docfield.reqd: + return + if docfield.fieldname in ('parenttype', 'trash_reason'): + return + if docfield.hidden: + return + if select_columns and docfield.fieldname not in select_columns.get(docfield.parent, []): + return + + tablerow.append("") + fieldrow.append(docfield.fieldname) + labelrow.append(_(docfield.label)) + mandatoryrow.append(docfield.reqd and 'Yes' or 'No') + typerow.append(docfield.fieldtype) + inforow.append(getinforow(docfield)) + columns.append(docfield.fieldname) + + def append_empty_field_column(): + tablerow.append("~") + fieldrow.append("~") + labelrow.append("") + mandatoryrow.append("") + typerow.append("") + inforow.append("") + columns.append("") + + def getinforow(docfield): + """make info comment for options, links etc.""" + if docfield.fieldtype == 'Select': + if not docfield.options: + return '' + else: + return _("One of") + ': %s' % ', '.join(filter(None, docfield.options.split('\n'))) + elif docfield.fieldtype == 'Link': + return 'Valid %s' % docfield.options + elif docfield.fieldtype == 'Int': + return 'Integer' + elif docfield.fieldtype == "Check": + return "0 or 1" + elif docfield.fieldtype in ["Date", "Datetime"]: + return cstr(frappe.defaults.get_defaults().date_format) + elif hasattr(docfield, "info"): + return docfield.info + else: + return '' + + def add_field_headings(): + w.writerow(tablerow) + w.writerow(labelrow) + w.writerow(fieldrow) + w.writerow(mandatoryrow) + w.writerow(typerow) + w.writerow(inforow) + w.writerow([get_data_keys_definition().data_separator]) + + def add_data(): + def add_data_row(row_group, dt, parentfield, doc, rowidx): + d = doc.copy() + meta = frappe.get_meta(dt) + if all_doctypes: + d.name = '"'+ d.name+'"' + + if len(row_group) < rowidx + 1: + row_group.append([""] * (len(columns) + 1)) + row = row_group[rowidx] + + _column_start_end = column_start_end.get((dt, parentfield)) + + if _column_start_end: + for i, c in enumerate(columns[_column_start_end.start:_column_start_end.end]): + df = meta.get_field(c) + fieldtype = df.fieldtype if df else "Data" + value = d.get(c, "") + if value: + if fieldtype == "Date": + value = formatdate(value) + elif fieldtype == "Datetime": + value = format_datetime(value) + + row[_column_start_end.start + i + 1] = value + + if with_data=='Yes': + frappe.permissions.can_export(parent_doctype, raise_exception=True) + + # sort nested set doctypes by `lft asc` + order_by = None + table_columns = frappe.db.get_table_columns(parent_doctype) + if 'lft' in table_columns and 'rgt' in table_columns: + order_by = '`tab{doctype}`.`lft` asc'.format(doctype=parent_doctype) + + # get permitted data only + data = frappe.get_list(doctype, fields=["*"], limit_page_length=None, order_by=order_by) + + for doc in data: + op = docs_to_export.get("op") + names = docs_to_export.get("name") + + if names and op: + if op == '=' and doc.name not in names: + continue + elif op == '!=' and doc.name in names: + continue + elif names: + try: + sflags = docs_to_export.get("flags", "I,U").upper() + flags = 0 + for a in re.split('\W+',sflags): + flags = flags | reflags.get(a,0) + + c = re.compile(names, flags) + m = c.match(doc.name) + if not m: + continue + except: + if doc.name not in names: + continue + # add main table + row_group = [] + + add_data_row(row_group, doctype, None, doc, 0) + + if all_doctypes: + # add child tables + for c in child_doctypes: + for ci, child in enumerate(frappe.db.sql("""select * from `tab{0}` + where parent=%s and parentfield=%s order by idx""".format(c['doctype']), + (doc.name, c['parentfield']), as_dict=1)): + add_data_row(row_group, c['doctype'], c['parentfield'], child, ci) + + for row in row_group: + w.writerow(row) + + w = UnicodeWriter() + key = 'parent' if parent_doctype != doctype else 'name' + + add_main_header() + + w.writerow(['']) + tablerow = [get_data_keys_definition().doctype, ""] + labelrow = [_("Column Labels:"), "ID"] + fieldrow = [get_data_keys_definition().columns, key] + mandatoryrow = [_("Mandatory:"), _("Yes")] + typerow = [_('Type:'), 'Data (text)'] + inforow = [_('Info:'), ''] + columns = [key] + + build_field_columns(doctype) + + if all_doctypes: + for d in child_doctypes: + append_empty_field_column() + if (select_columns and select_columns.get(d['doctype'], None)) or not select_columns: + # if atleast one column is selected for this doctype + build_field_columns(d['doctype'], d['parentfield']) + + add_field_headings() + add_data() + + if from_data_import == "Yes" and excel_format == "Yes": + filename = frappe.generate_hash("", 10) + with open(filename, 'wb') as f: + f.write(cstr(w.getvalue()).encode("utf-8")) + f = open(filename) + + # increase the field limit in case of larger fields + # works for Python 2.x and 3.x + csv.field_size_limit(sys.maxsize) + reader = csv.reader(f) + + from frappe.utils.xlsxutils import make_xlsx + xlsx_file = make_xlsx(reader, "Data Import Template") + + f.close() + os.remove(filename) + + # write out response as a xlsx type + frappe.response['filename'] = doctype + '.xlsx' + frappe.response['filecontent'] = xlsx_file.getvalue() + frappe.response['type'] = 'binary' + + else: + # write out response as a type csv + frappe.response['result'] = cstr(w.getvalue()) + frappe.response['type'] = 'csv' + frappe.response['doctype'] = doctype diff --git a/frappe/core/doctype/version/version.json b/frappe/core/doctype/version/version.json index 37294355ee..57cd54b6d2 100644 --- a/frappe/core/doctype/version/version.json +++ b/frappe/core/doctype/version/version.json @@ -188,7 +188,7 @@ "issingle": 0, "istable": 0, "max_attachments": 0, - "modified": "2016-12-29 14:39:45.926836", + "modified": "2018-04-10 14:39:45.926836", "modified_by": "Administrator", "module": "Core", "name": "Version", diff --git a/frappe/core/doctype/version/version.py b/frappe/core/doctype/version/version.py index 671eb8c597..9d945fed11 100644 --- a/frappe/core/doctype/version/version.py +++ b/frappe/core/doctype/version/version.py @@ -85,4 +85,7 @@ def get_diff(old, new, for_child=False): return out else: - return None \ No newline at end of file + return None + +def on_doctype_update(): + frappe.db.add_index("Version", ["ref_doctype", "docname"]) \ No newline at end of file diff --git a/frappe/desk/doctype/event/event.py b/frappe/desk/doctype/event/event.py index 0e97d78fb7..b99b0453f6 100644 --- a/frappe/desk/doctype/event/event.py +++ b/frappe/desk/doctype/event/event.py @@ -27,7 +27,7 @@ class Event(Document): # this scenario doesn't make sense i.e. it starts and ends at the same second! self.ends_on = None - if getdate(self.starts_on) == getdate(self.ends_on) and self.repeat_on == "Every Day": + if getdate(self.starts_on) != getdate(self.ends_on) and self.repeat_on == "Every Day": frappe.msgprint(frappe._("Every day events should finish on the same day."), raise_exception=True) def get_permission_query_conditions(user): diff --git a/frappe/model/db_schema.py b/frappe/model/db_schema.py index 6990b3d637..f521221936 100644 --- a/frappe/model/db_schema.py +++ b/frappe/model/db_schema.py @@ -110,7 +110,8 @@ class DbTable: for col in columns: if len(col.fieldname) >= 64: - frappe.throw(_("Fieldname is limited to 64 characters ({0})").format(frappe.bold(col.fieldname))) + frappe.throw(_("Fieldname is limited to 64 characters ({0})") + .format(frappe.bold(col.fieldname))) if col.fieldtype in type_map and type_map[col.fieldtype][0]=="varchar": @@ -119,33 +120,35 @@ class DbTable: if not (1 <= new_length <= 1000): frappe.throw(_("Length of {0} should be between 1 and 1000").format(col.fieldname)) - try: - # check for truncation - max_length = frappe.db.sql("""select max(char_length(`{fieldname}`)) from `tab{doctype}`"""\ - .format(fieldname=col.fieldname, doctype=self.doctype)) + current_col = self.current_columns.get(col.fieldname, {}) + if not current_col: + continue + current_type = self.current_columns[col.fieldname]["type"] + current_length = re.findall('varchar\(([\d]+)\)', current_type) + if not current_length: + # case when the field is no longer a varchar + continue + current_length = current_length[0] + if cint(current_length) != cint(new_length): + try: + # check for truncation + max_length = frappe.db.sql("""select max(char_length(`{fieldname}`)) from `tab{doctype}`"""\ + .format(fieldname=col.fieldname, doctype=self.doctype)) - except pymysql.InternalError as e: - if e.args[0] == ER.BAD_FIELD_ERROR: - # Unknown column 'column_name' in 'field list' - continue + except pymysql.InternalError as e: + if e.args[0] == ER.BAD_FIELD_ERROR: + # Unknown column 'column_name' in 'field list' + continue - else: - raise + else: + raise - if max_length and max_length[0][0] and max_length[0][0] > new_length: - current_type = self.current_columns[col.fieldname]["type"] - current_length = re.findall('varchar\(([\d]+)\)', current_type) - if not current_length: - # case when the field is no longer a varchar - continue + if max_length and max_length[0][0] and max_length[0][0] > new_length: + if col.fieldname in self.columns: + self.columns[col.fieldname].length = current_length - current_length = current_length[0] - - if col.fieldname in self.columns: - self.columns[col.fieldname].length = current_length - - frappe.msgprint(_("Reverting length to {0} for '{1}' in '{2}'; Setting the length as {3} will cause truncation of data.")\ - .format(current_length, col.fieldname, self.doctype, new_length)) + frappe.msgprint(_("Reverting length to {0} for '{1}' in '{2}'; Setting the length as {3} will cause truncation of data.")\ + .format(current_length, col.fieldname, self.doctype, new_length)) def sync(self): @@ -180,7 +183,8 @@ class DbTable: parentfield varchar({varchar_len}), parenttype varchar({varchar_len}), idx int(8) not null default '0', - %sindex parent(parent)) + %sindex parent(parent), + index modified(modified)) ENGINE={engine} ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 diff --git a/frappe/model/delete_doc.py b/frappe/model/delete_doc.py index 979bcc1885..b452073467 100644 --- a/frappe/model/delete_doc.py +++ b/frappe/model/delete_doc.py @@ -133,7 +133,7 @@ def delete_from_table(doctype, name, ignore_doctypes, doc): if doctype!="DocType" and doctype==name: frappe.db.sql("delete from `tabSingles` where doctype=%s", name) else: - frappe.db.sql("delete from `tab%s` where name=%s" % (frappe.db.escape(doctype), "%s"), (name,)) + frappe.db.sql("delete from `tab{0}` where name=%s".format(doctype), name) # get child tables if doc: @@ -193,13 +193,20 @@ def check_if_doc_is_linked(doc, method="Delete"): # don't check for communication and todo! continue - if item and ((item.parent or item.name) != doc.name) \ - and ((method=="Delete" and item.docstatus<2) or (method=="Cancel" and item.docstatus==1)): - # raise exception only if - # linked to an non-cancelled doc when deleting - # or linked to a submitted doc when cancelling + if not item: + continue + elif (method != "Delete" or item.docstatus == 2) and (method != "Cancel" or item.docstatus != 1): + # don't raise exception if not + # linked to a non-cancelled doc when deleting or to a submitted doc when cancelling + continue + elif link_dt == doc.doctype and (item.parent or item.name) == doc.name: + # don't raise exception if not + # linked to same item or doc having same name as the item + continue + else: reference_docname = item.parent or item.name raise_link_exists_exception(doc, linked_doctype, reference_docname) + else: if frappe.db.get_value(link_dt, None, link_field) == doc.name: raise_link_exists_exception(doc, link_dt, link_dt) diff --git a/frappe/model/document.py b/frappe/model/document.py index 0734938631..a39824a35e 100644 --- a/frappe/model/document.py +++ b/frappe/model/document.py @@ -185,7 +185,7 @@ class Document(BaseDocument): frappe.flags.error_message = _('Insufficient Permission for {0}').format(self.doctype) raise frappe.PermissionError - def insert(self, ignore_permissions=None, ignore_if_duplicate=False, ignore_mandatory=None): + def insert(self, ignore_permissions=None, ignore_links=None, ignore_if_duplicate=False, ignore_mandatory=None): """Insert the document in the database (as a new document). This will check for user permissions and execute `before_insert`, `validate`, `on_update`, `after_insert` methods if they are written. @@ -199,6 +199,9 @@ class Document(BaseDocument): if ignore_permissions!=None: self.flags.ignore_permissions = ignore_permissions + if ignore_links!=None: + self.flags.ignore_links = ignore_links + if ignore_mandatory!=None: self.flags.ignore_mandatory = ignore_mandatory diff --git a/frappe/utils/data.py b/frappe/utils/data.py index b0729959df..2f5cfbdb8f 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -26,9 +26,9 @@ def getdate(string_date=None): """ Coverts string date (yyyy-mm-dd) to datetime.date object """ + if not string_date: return get_datetime().date() - if isinstance(string_date, datetime.datetime): return string_date.date() @@ -38,7 +38,6 @@ def getdate(string_date=None): # dateutil parser does not agree with dates like 0000-00-00 if not string_date or string_date=="0000-00-00": return None - return parser.parse(string_date).date() def get_datetime(datetime_str=None): @@ -199,7 +198,6 @@ def get_time(time_str): def get_datetime_str(datetime_obj): if isinstance(datetime_obj, string_types): datetime_obj = get_datetime(datetime_obj) - return datetime_obj.strftime(DATETIME_FORMAT) def get_user_format(): @@ -226,11 +224,11 @@ def formatdate(string_date=None, format_string=None): date = getdate(string_date) if not format_string: format_string = get_user_format().replace("mm", "MM") - try: formatted_date = babel.dates.format_date(date, format_string, locale=(frappe.local.lang or "").replace("-", "_")) except UnknownLocaleError: - formatted_date = date.strftime("%Y-%m-%d") + format_string = format_string.replace("MM", "%m").replace("dd", "%d").replace("yyyy", "%Y") + formatted_date = date.strftime(format_string) return formatted_date def format_time(txt):