diff --git a/frappe/core/doctype/doctype/doctype.py b/frappe/core/doctype/doctype/doctype.py index da1b184cc1..2c8cd240ee 100644 --- a/frappe/core/doctype/doctype/doctype.py +++ b/frappe/core/doctype/doctype/doctype.py @@ -24,6 +24,7 @@ from frappe.modules import make_boilerplate, get_doc_path from frappe.database.schema import validate_column_name, validate_column_length from frappe.model.docfield import supports_translation from frappe.modules.import_file import get_file_path +from frappe.model.meta import Meta class InvalidFieldNameError(frappe.ValidationError): pass @@ -275,7 +276,7 @@ class DocType(Document): """Update database schema, make controller templates if `custom` is not set and clear cache.""" self.delete_duplicate_custom_fields() try: - frappe.db.updatedb(self.name, self) + frappe.db.updatedb(self.name, Meta(self)) except Exception as e: print("\n\nThere was an issue while migrating the DocType: {}\n".format(self.name)) raise e diff --git a/frappe/custom/doctype/customize_form/customize_form.py b/frappe/custom/doctype/customize_form/customize_form.py index 8d47a075ba..3259085781 100644 --- a/frappe/custom/doctype/customize_form/customize_form.py +++ b/frappe/custom/doctype/customize_form/customize_form.py @@ -166,7 +166,6 @@ class CustomizeForm(Document): self.flags.update_db = False self.flags.rebuild_doctype_for_global_search = False - self.set_property_setters() self.update_custom_fields() self.set_name_translation() @@ -362,13 +361,49 @@ class CustomizeForm(Document): def validate_fieldtype_change(self, df, old_value, new_value): allowed = False + self.check_length_for_fieldtypes = [] for allowed_changes in allowed_fieldtype_change: if (old_value in allowed_changes and new_value in allowed_changes): allowed = True + if frappe.db.type_map.get(old_value)[1] > frappe.db.type_map.get(new_value)[1]: + self.check_length_for_fieldtypes.append({'df': df, 'old_value': old_value}) + self.validate_fieldtype_length() + else: + self.flags.update_db = True break if not allowed: frappe.throw(_("Fieldtype cannot be changed from {0} to {1} in row {2}").format(old_value, new_value, df.idx)) + def validate_fieldtype_length(self): + for field in self.check_length_for_fieldtypes: + df = field.get('df') + max_length = frappe.db.type_map.get(df.fieldtype)[1] + fieldname = df.fieldname + docs = frappe.db.sql(''' + SELECT name, {fieldname}, LENGTH({fieldname}) AS len + FROM `tab{doctype}` + WHERE LENGTH({fieldname}) > {max_length} + '''.format( + fieldname=fieldname, + doctype=self.doc_type, + max_length=max_length + ), as_dict=True) + links = [] + label = df.label + for doc in docs: + links.append(frappe.utils.get_link_to_form(self.doc_type, doc.name)) + links_str = ', '.join(links) + + if docs: + frappe.throw(_('Value for field {0} is too long in {1}. Length should be lesser than {2} characters') + .format( + frappe.bold(label), + links_str, + frappe.bold(max_length) + ), title=_('Data Too Long'), is_minimizable=len(docs) > 1) + + self.flags.update_db = True + def reset_to_defaults(self): if not self.doc_type: return diff --git a/frappe/database/postgres/database.py b/frappe/database/postgres/database.py index 243d0f934e..e30ef3293f 100644 --- a/frappe/database/postgres/database.py +++ b/frappe/database/postgres/database.py @@ -92,7 +92,7 @@ class PostgresDatabase(Database): # pylint: disable=W0221 def sql(self, *args, **kwargs): - if len(args): + if args: # since tuple is immutable args = list(args) args[0] = modify_query(args[0]) @@ -276,13 +276,13 @@ class PostgresDatabase(Database): # pylint: disable=W1401 return self.sql(''' SELECT a.column_name AS name, - CASE a.data_type + CASE LOWER(a.data_type) WHEN 'character varying' THEN CONCAT('varchar(', a.character_maximum_length ,')') - WHEN 'timestamp without TIME zone' THEN 'timestamp' + WHEN 'timestamp without time zone' THEN 'timestamp' ELSE a.data_type END AS type, COUNT(b.indexdef) AS Index, - COALESCE(a.column_default, NULL) AS default, + SPLIT_PART(COALESCE(a.column_default, NULL), '::', 1) AS default, BOOL_OR(b.unique) AS unique FROM information_schema.columns a LEFT JOIN diff --git a/frappe/database/schema.py b/frappe/database/schema.py index 88cda9340b..28e055f382 100644 --- a/frappe/database/schema.py +++ b/frappe/database/schema.py @@ -13,7 +13,7 @@ class DBTable: def __init__(self, doctype, meta=None): self.doctype = doctype self.table_name = 'tab{}'.format(doctype) - self.meta = meta or frappe.get_meta(doctype) + self.meta = meta or frappe.get_meta(doctype, False) self.columns = {} self.current_columns = {} @@ -65,64 +65,35 @@ class DBTable: """ get columns from docfields and custom fields """ - fl = frappe.db.sql("SELECT * FROM `tabDocField` WHERE parent = %s", self.doctype, as_dict = 1) - lengths = {} - precisions = {} - uniques = {} + fields = self.meta.get_fieldnames_with_value(True) # optional fields like _comments - if not self.meta.istable: + if not self.meta.get('istable'): for fieldname in frappe.db.OPTIONAL_COLUMNS: - fl.append({ + fields.append({ "fieldname": fieldname, "fieldtype": "Text" }) # add _seen column if track_seen - if getattr(self.meta, 'track_seen', False): - fl.append({ + if self.meta.get('track_seen'): + fields.append({ 'fieldname': '_seen', 'fieldtype': 'Text' }) - if (not frappe.flags.in_install_db - and (frappe.flags.in_install != "frappe" - or frappe.flags.ignore_in_install)): - custom_fl = frappe.db.sql(""" - SELECT * FROM `tabCustom Field` - WHERE dt = %s AND docstatus < 2 - """, (self.doctype,), as_dict=1) - if custom_fl: fl += custom_fl - - # apply length, precision and unique from property setters - for ps in frappe.get_all("Property Setter", - fields=["field_name", "property", "value"], - filters={ - "doc_type": self.doctype, - "doctype_or_field": "DocField", - "property": ["in", ["precision", "length", "unique"]] - }): - - if ps.property=="length": - lengths[ps.field_name] = cint(ps.value) - - elif ps.property=="precision": - precisions[ps.field_name] = cint(ps.value) - - elif ps.property=="unique": - uniques[ps.field_name] = cint(ps.value) - - for f in fl: - self.columns[f['fieldname']] = DbColumn(self, - f['fieldname'], - f['fieldtype'], - lengths.get(f["fieldname"]) or f.get('length'), - f.get('default'), - f.get('search_index'), - f.get('options'), - uniques.get(f["fieldname"], - f.get('unique')), - precisions.get(f['fieldname']) or f.get('precision')) + for field in fields: + self.columns[field.get('fieldname')] = DbColumn( + self, + field.get('fieldname'), + field.get('fieldtype'), + field.get('length'), + field.get('default'), + field.get('search_index'), + field.get('options'), + field.get('unique'), + field.get('precision') + ) def validate(self): """Check if change in varchar length isn't truncating the columns""" diff --git a/frappe/model/meta.py b/frappe/model/meta.py index 1938a4a96c..5065684311 100644 --- a/frappe/model/meta.py +++ b/frappe/model/meta.py @@ -165,7 +165,7 @@ class Meta(Document): def get_valid_columns(self): if not hasattr(self, "_valid_columns"): - if self.name in ("DocType", "DocField", "DocPerm", 'DocType Action', 'DocType Link', "Property Setter"): + if self.name in ("DocType", "DocField", "DocPerm", 'DocType Action', 'DocType Link'): self._valid_columns = get_table_columns(self.name) else: self._valid_columns = self.default_fields + \ @@ -290,17 +290,20 @@ class Meta(Document): return get_workflow_name(self.name) def add_custom_fields(self): - try: - self.extend("fields", frappe.db.sql("""SELECT * FROM `tabCustom Field` - WHERE dt = %s AND docstatus < 2""", (self.name,), as_dict=1, - update={"is_custom_field": 1})) - except Exception as e: - if frappe.db.is_table_missing(e): - return - else: - raise + if not frappe.db.table_exists('Custom Field'): + return + + custom_fields = frappe.db.sql(""" + SELECT * FROM `tabCustom Field` + WHERE dt = %s AND docstatus < 2 + """, (self.name,), as_dict=1, update={"is_custom_field": 1}) + + self.extend("fields", custom_fields) def apply_property_setters(self): + if not frappe.db.table_exists('Property Setter'): + return + property_setters = frappe.db.sql("""select * from `tabProperty Setter` where doc_type=%s""", (self.name,), as_dict=1) @@ -378,8 +381,9 @@ class Meta(Document): if custom_perms: self.permissions = [Document(d) for d in custom_perms] - def get_fieldnames_with_value(self): - return [df.fieldname for df in self.fields if df.fieldtype not in no_value_fields] + def get_fieldnames_with_value(self, with_field_meta=False): + return [df if with_field_meta else df.fieldname \ + for df in self.fields if df.fieldtype not in no_value_fields] def get_fields_to_check_permissions(self, user_permission_doctypes): diff --git a/frappe/tests/test_db_update.py b/frappe/tests/test_db_update.py new file mode 100644 index 0000000000..34378de3af --- /dev/null +++ b/frappe/tests/test_db_update.py @@ -0,0 +1,70 @@ +import unittest +import frappe + +from frappe.core.utils import find +from frappe.custom.doctype.property_setter.property_setter import make_property_setter + + +class TestDBUpdate(unittest.TestCase): + def test_db_update(self): + doctype = 'User' + frappe.reload_doctype('User', force=True) + frappe.model.meta.trim_tables('User') + make_property_setter(doctype, 'bio', 'fieldtype', 'Text', 'Data') + make_property_setter(doctype, 'enabled', 'default', '1', 'Int') + + frappe.db.updatedb(doctype) + + field_defs = get_field_defs(doctype) + table_columns = frappe.db.get_table_columns_description('tab{}'.format(doctype)) + + self.assertEqual(len(field_defs), len(table_columns)) + + for field_def in field_defs: + fieldname = field_def.get('fieldname') + table_column = find(table_columns, lambda d: d.get('name') == fieldname) + + fieldtype = get_fieldtype_from_def(field_def) + + fallback_default = '0' if field_def.get('fieldtype') in frappe.model.numeric_fieldtypes else 'NULL' + default = field_def.default if field_def.default is not None else fallback_default + + self.assertEqual(fieldtype, table_column.type) + self.assertIn(table_column.default or 'NULL', [default, "'{}'".format(default)]) + +def get_fieldtype_from_def(field_def): + fieldtuple = frappe.db.type_map.get(field_def.fieldtype, ('', 0)) + fieldtype = fieldtuple[0] + if fieldtype in ('varchar', 'datetime', 'int'): + fieldtype += '({})'.format(field_def.length or fieldtuple[1]) + return fieldtype + +def get_field_defs(doctype): + meta = frappe.get_meta(doctype, cached=False) + field_defs = meta.get_fieldnames_with_value(True) + field_defs += get_other_fields_meta(meta) + return field_defs + +def get_other_fields_meta(meta): + default_fields_map = { + 'name': ('Data', 0), + 'owner': ('Data', 0), + 'parent': ('Data', 0), + 'parentfield': ('Data', 0), + 'modified_by': ('Data', 0), + 'parenttype': ('Data', 0), + 'creation': ('Datetime', 0), + 'modified': ('Datetime', 0), + 'idx': ('Int', 8), + 'docstatus': ('Check', 0) + } + + optional_fields = frappe.db.OPTIONAL_COLUMNS + if meta.track_seen: + optional_fields.append('_seen') + + optional_fields_map = {field: ('Text', 0) for field in optional_fields} + fields = dict(default_fields_map, **optional_fields_map) + field_map = [frappe._dict({'fieldname': field, 'fieldtype': _type, 'length': _length}) for field, (_type, _length) in fields.items()] + + return field_map \ No newline at end of file