From 61270ba2107e432b797071ac95c208ff0803f3b8 Mon Sep 17 00:00:00 2001 From: prssanna Date: Fri, 6 Mar 2020 19:46:20 +0530 Subject: [PATCH 01/15] feat: change fieldtype in db with customize form --- .../doctype/customize_form/customize_form.py | 32 +++++++++++++++++-- frappe/database/schema.py | 8 +++-- 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/frappe/custom/doctype/customize_form/customize_form.py b/frappe/custom/doctype/customize_form/customize_form.py index 8d47a075ba..b35bfddf76 100644 --- a/frappe/custom/doctype/customize_form/customize_form.py +++ b/frappe/custom/doctype/customize_form/customize_form.py @@ -166,11 +166,10 @@ class CustomizeForm(Document): self.flags.update_db = False self.flags.rebuild_doctype_for_global_search = False - + validate_fields_for_doctype(self.doc_type) self.set_property_setters() self.update_custom_fields() self.set_name_translation() - validate_fields_for_doctype(self.doc_type) if self.flags.update_db: frappe.db.updatedb(self.doc_type) @@ -362,13 +361,42 @@ class CustomizeForm(Document): def validate_fieldtype_change(self, df, old_value, new_value): allowed = False + self.check_length_for_fieldtypes = [] for allowed_changes in allowed_fieldtype_change: if (old_value in allowed_changes and new_value in allowed_changes): allowed = True + self.flags.update_db = True + if frappe.db.type_map.get(old_value)[1] > frappe.db.type_map.get(new_value)[1]: + + self.check_length_for_fieldtypes.append({'df': df, 'old_value': old_value}) + self.validate_fieldtype_length() break if not allowed: frappe.throw(_("Fieldtype cannot be changed from {0} to {1} in row {2}").format(old_value, new_value, df.idx)) + def validate_fieldtype_length(self): + for field in self.check_length_for_fieldtypes: + df = field.get('df') + max_length = frappe.db.type_map.get(df.fieldtype)[1] + fieldname = df.fieldname + docs = frappe.db.sql('''select name, {fieldname}, length({fieldname}) as len + from `tab{doctype}` + where length({fieldname}) > {max_length} + '''.format( + fieldname = fieldname, + doctype = self.doc_type, + max_length = max_length + ), as_dict = True) + links = [] + label = df.label + for doc in docs: + links.append(frappe.utils.get_link_to_form(self.doc_type, doc.name)) + links_str = ', '.join(links) + + if len(docs): + frappe.throw(_('Value for field {label} is too long in {links_str}. Length should be lesser than {max_length}') + .format(label=label, max_length=max_length, links_str=links_str), title='Data Too Long') + def reset_to_defaults(self): if not self.doc_type: return diff --git a/frappe/database/schema.py b/frappe/database/schema.py index 88cda9340b..73bdf32761 100644 --- a/frappe/database/schema.py +++ b/frappe/database/schema.py @@ -69,6 +69,7 @@ class DBTable: lengths = {} precisions = {} uniques = {} + fieldtype = '' # optional fields like _comments if not self.meta.istable: @@ -100,7 +101,7 @@ class DBTable: filters={ "doc_type": self.doctype, "doctype_or_field": "DocField", - "property": ["in", ["precision", "length", "unique"]] + "property": ["in", ["precision", "length", "unique", "fieldtype"]] }): if ps.property=="length": @@ -112,10 +113,13 @@ class DBTable: elif ps.property=="unique": uniques[ps.field_name] = cint(ps.value) + if ps.property=="fieldtype": + fieldtype = ps.value + for f in fl: self.columns[f['fieldname']] = DbColumn(self, f['fieldname'], - f['fieldtype'], + fieldtype or f['fieldtype'], lengths.get(f["fieldname"]) or f.get('length'), f.get('default'), f.get('search_index'), From 6ed6ab2907cb908767d7149bb605ba19a140461a Mon Sep 17 00:00:00 2001 From: prssanna Date: Sat, 7 Mar 2020 15:15:12 +0530 Subject: [PATCH 02/15] fix: update db type map --- frappe/custom/doctype/customize_form/customize_form.py | 10 ++++++---- frappe/database/mariadb/database.py | 4 ++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/frappe/custom/doctype/customize_form/customize_form.py b/frappe/custom/doctype/customize_form/customize_form.py index b35bfddf76..a38e8f4fa7 100644 --- a/frappe/custom/doctype/customize_form/customize_form.py +++ b/frappe/custom/doctype/customize_form/customize_form.py @@ -166,10 +166,10 @@ class CustomizeForm(Document): self.flags.update_db = False self.flags.rebuild_doctype_for_global_search = False - validate_fields_for_doctype(self.doc_type) self.set_property_setters() self.update_custom_fields() self.set_name_translation() + validate_fields_for_doctype(self.doc_type) if self.flags.update_db: frappe.db.updatedb(self.doc_type) @@ -365,11 +365,11 @@ class CustomizeForm(Document): for allowed_changes in allowed_fieldtype_change: if (old_value in allowed_changes and new_value in allowed_changes): allowed = True - self.flags.update_db = True if frappe.db.type_map.get(old_value)[1] > frappe.db.type_map.get(new_value)[1]: - self.check_length_for_fieldtypes.append({'df': df, 'old_value': old_value}) self.validate_fieldtype_length() + else: + self.flags.update_db = True break if not allowed: frappe.throw(_("Fieldtype cannot be changed from {0} to {1} in row {2}").format(old_value, new_value, df.idx)) @@ -394,8 +394,10 @@ class CustomizeForm(Document): links_str = ', '.join(links) if len(docs): - frappe.throw(_('Value for field {label} is too long in {links_str}. Length should be lesser than {max_length}') + frappe.throw(_('Value for field {label} is too long in {links_str}. Length should be lesser than {max_length} characters') .format(label=label, max_length=max_length, links_str=links_str), title='Data Too Long') + else: + self.flags.update_db = True def reset_to_defaults(self): if not self.doc_type: diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index cd053569f0..926425f857 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -33,7 +33,7 @@ class MariaDBDatabase(Database): 'Float': ('decimal', '18,6'), 'Percent': ('decimal', '18,6'), 'Check': ('int', '1'), - 'Small Text': ('text', ''), + 'Small Text': ('text', 65535), 'Long Text': ('longtext', ''), 'Code': ('longtext', ''), 'Text Editor': ('longtext', ''), @@ -42,7 +42,7 @@ class MariaDBDatabase(Database): 'Date': ('date', ''), 'Datetime': ('datetime', '6'), 'Time': ('time', '6'), - 'Text': ('text', ''), + 'Text': ('text', 65535), 'Data': ('varchar', self.VARCHAR_LEN), 'Link': ('varchar', self.VARCHAR_LEN), 'Dynamic Link': ('varchar', self.VARCHAR_LEN), From 316abd5df2a5012019af774f1954a270c52308ec Mon Sep 17 00:00:00 2001 From: prssanna Date: Fri, 13 Mar 2020 15:14:54 +0530 Subject: [PATCH 03/15] fix: code formatting --- .../custom/doctype/customize_form/customize_form.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/frappe/custom/doctype/customize_form/customize_form.py b/frappe/custom/doctype/customize_form/customize_form.py index a38e8f4fa7..61d00984fd 100644 --- a/frappe/custom/doctype/customize_form/customize_form.py +++ b/frappe/custom/doctype/customize_form/customize_form.py @@ -379,9 +379,10 @@ class CustomizeForm(Document): df = field.get('df') max_length = frappe.db.type_map.get(df.fieldtype)[1] fieldname = df.fieldname - docs = frappe.db.sql('''select name, {fieldname}, length({fieldname}) as len - from `tab{doctype}` - where length({fieldname}) > {max_length} + docs = frappe.db.sql(''' + SELECT name, {fieldname}, LENGTH({fieldname}) AS len + FROM `tab{doctype}` + WHERE LENGTH({fieldname}) > {max_length} '''.format( fieldname = fieldname, doctype = self.doc_type, @@ -394,8 +395,8 @@ class CustomizeForm(Document): links_str = ', '.join(links) if len(docs): - frappe.throw(_('Value for field {label} is too long in {links_str}. Length should be lesser than {max_length} characters') - .format(label=label, max_length=max_length, links_str=links_str), title='Data Too Long') + frappe.throw(_('Value for field {0} is too long in {1}. Length should be lesser than {2} characters') + .format(frappe.bold(label), links_str, frappe.bold(max_length)), title='Data Too Long') else: self.flags.update_db = True From c4d5b2a289d6821cca5400d5a54d91c7542609bd Mon Sep 17 00:00:00 2001 From: prssanna Date: Fri, 13 Mar 2020 15:23:17 +0530 Subject: [PATCH 04/15] fix: set is_minimizable as true for error dialog --- frappe/custom/doctype/customize_form/customize_form.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/custom/doctype/customize_form/customize_form.py b/frappe/custom/doctype/customize_form/customize_form.py index 61d00984fd..0cef12fd7e 100644 --- a/frappe/custom/doctype/customize_form/customize_form.py +++ b/frappe/custom/doctype/customize_form/customize_form.py @@ -396,7 +396,7 @@ class CustomizeForm(Document): if len(docs): frappe.throw(_('Value for field {0} is too long in {1}. Length should be lesser than {2} characters') - .format(frappe.bold(label), links_str, frappe.bold(max_length)), title='Data Too Long') + .format(frappe.bold(label), links_str, frappe.bold(max_length)), title='Data Too Long', is_minimizable=True) else: self.flags.update_db = True From 8865fc78b134cb8483d930945f4d7eba81299f63 Mon Sep 17 00:00:00 2001 From: Prssanna Desai Date: Fri, 13 Mar 2020 17:18:02 +0530 Subject: [PATCH 05/15] fix: translate dialog title Co-Authored-By: Suraj Shetty <13928957+surajshetty3416@users.noreply.github.com> --- frappe/custom/doctype/customize_form/customize_form.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/frappe/custom/doctype/customize_form/customize_form.py b/frappe/custom/doctype/customize_form/customize_form.py index 0cef12fd7e..c8d1bb18b4 100644 --- a/frappe/custom/doctype/customize_form/customize_form.py +++ b/frappe/custom/doctype/customize_form/customize_form.py @@ -394,11 +394,11 @@ class CustomizeForm(Document): links.append(frappe.utils.get_link_to_form(self.doc_type, doc.name)) links_str = ', '.join(links) - if len(docs): - frappe.throw(_('Value for field {0} is too long in {1}. Length should be lesser than {2} characters') - .format(frappe.bold(label), links_str, frappe.bold(max_length)), title='Data Too Long', is_minimizable=True) - else: - self.flags.update_db = True + if docs: + frappe.throw(_('Value for field {0} is too long in {1}. Length should be lesser than {2} characters') + .format(frappe.bold(label), links_str, frappe.bold(max_length)), title=_('Data Too Long'), is_minimizable=len(docs) > 1) + + self.flags.update_db = True def reset_to_defaults(self): if not self.doc_type: From af7e040d1a89b0e06fab8368701354d84defea47 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Sat, 21 Mar 2020 12:30:03 +0530 Subject: [PATCH 06/15] refactor: Get doc fields directly from meta --- frappe/core/doctype/doctype/doctype.py | 2 +- frappe/database/mariadb/database.py | 4 +- frappe/database/schema.py | 61 ++++++-------------------- frappe/model/meta.py | 4 +- 4 files changed, 18 insertions(+), 53 deletions(-) diff --git a/frappe/core/doctype/doctype/doctype.py b/frappe/core/doctype/doctype/doctype.py index da1b184cc1..0024995573 100644 --- a/frappe/core/doctype/doctype/doctype.py +++ b/frappe/core/doctype/doctype/doctype.py @@ -275,7 +275,7 @@ class DocType(Document): """Update database schema, make controller templates if `custom` is not set and clear cache.""" self.delete_duplicate_custom_fields() try: - frappe.db.updatedb(self.name, self) + frappe.db.updatedb(self.name) except Exception as e: print("\n\nThere was an issue while migrating the DocType: {}\n".format(self.name)) raise e diff --git a/frappe/database/mariadb/database.py b/frappe/database/mariadb/database.py index 926425f857..cd053569f0 100644 --- a/frappe/database/mariadb/database.py +++ b/frappe/database/mariadb/database.py @@ -33,7 +33,7 @@ class MariaDBDatabase(Database): 'Float': ('decimal', '18,6'), 'Percent': ('decimal', '18,6'), 'Check': ('int', '1'), - 'Small Text': ('text', 65535), + 'Small Text': ('text', ''), 'Long Text': ('longtext', ''), 'Code': ('longtext', ''), 'Text Editor': ('longtext', ''), @@ -42,7 +42,7 @@ class MariaDBDatabase(Database): 'Date': ('date', ''), 'Datetime': ('datetime', '6'), 'Time': ('time', '6'), - 'Text': ('text', 65535), + 'Text': ('text', ''), 'Data': ('varchar', self.VARCHAR_LEN), 'Link': ('varchar', self.VARCHAR_LEN), 'Dynamic Link': ('varchar', self.VARCHAR_LEN), diff --git a/frappe/database/schema.py b/frappe/database/schema.py index 73bdf32761..a9170feeb0 100644 --- a/frappe/database/schema.py +++ b/frappe/database/schema.py @@ -65,68 +65,33 @@ class DBTable: """ get columns from docfields and custom fields """ - fl = frappe.db.sql("SELECT * FROM `tabDocField` WHERE parent = %s", self.doctype, as_dict = 1) - lengths = {} - precisions = {} - uniques = {} - fieldtype = '' + fields = self.meta.get_fieldnames_with_value(True) # optional fields like _comments if not self.meta.istable: for fieldname in frappe.db.OPTIONAL_COLUMNS: - fl.append({ + fields.append({ "fieldname": fieldname, "fieldtype": "Text" }) # add _seen column if track_seen if getattr(self.meta, 'track_seen', False): - fl.append({ + fields.append({ 'fieldname': '_seen', 'fieldtype': 'Text' }) - if (not frappe.flags.in_install_db - and (frappe.flags.in_install != "frappe" - or frappe.flags.ignore_in_install)): - custom_fl = frappe.db.sql(""" - SELECT * FROM `tabCustom Field` - WHERE dt = %s AND docstatus < 2 - """, (self.doctype,), as_dict=1) - if custom_fl: fl += custom_fl - - # apply length, precision and unique from property setters - for ps in frappe.get_all("Property Setter", - fields=["field_name", "property", "value"], - filters={ - "doc_type": self.doctype, - "doctype_or_field": "DocField", - "property": ["in", ["precision", "length", "unique", "fieldtype"]] - }): - - if ps.property=="length": - lengths[ps.field_name] = cint(ps.value) - - elif ps.property=="precision": - precisions[ps.field_name] = cint(ps.value) - - elif ps.property=="unique": - uniques[ps.field_name] = cint(ps.value) - - if ps.property=="fieldtype": - fieldtype = ps.value - - for f in fl: - self.columns[f['fieldname']] = DbColumn(self, - f['fieldname'], - fieldtype or f['fieldtype'], - lengths.get(f["fieldname"]) or f.get('length'), - f.get('default'), - f.get('search_index'), - f.get('options'), - uniques.get(f["fieldname"], - f.get('unique')), - precisions.get(f['fieldname']) or f.get('precision')) + for field in fields: + self.columns[field.get('fieldname')] = DbColumn(self, + field.get('fieldname'), + field.get('fieldtype'), + field.get('length'), + field.get('default'), + field.get('search_index'), + field.get('options'), + field.get('unique'), + field.get('precision')) def validate(self): """Check if change in varchar length isn't truncating the columns""" diff --git a/frappe/model/meta.py b/frappe/model/meta.py index 1938a4a96c..495d3a6713 100644 --- a/frappe/model/meta.py +++ b/frappe/model/meta.py @@ -378,8 +378,8 @@ class Meta(Document): if custom_perms: self.permissions = [Document(d) for d in custom_perms] - def get_fieldnames_with_value(self): - return [df.fieldname for df in self.fields if df.fieldtype not in no_value_fields] + def get_fieldnames_with_value(self, with_field_meta=False): + return [df if with_meta else df.fieldname for df in self.fields if df.fieldtype not in no_value_fields] def get_fields_to_check_permissions(self, user_permission_doctypes): From 739f0618717719473769539c88d7410134fd7ff4 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Sat, 21 Mar 2020 12:30:54 +0530 Subject: [PATCH 07/15] test: Add test for frappe.db.updatedb --- frappe/tests/test_db_update.py | 73 ++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 frappe/tests/test_db_update.py diff --git a/frappe/tests/test_db_update.py b/frappe/tests/test_db_update.py new file mode 100644 index 0000000000..680c4c0672 --- /dev/null +++ b/frappe/tests/test_db_update.py @@ -0,0 +1,73 @@ +import unittest +import frappe + +from frappe.core.utils import find +from frappe.custom.doctype.property_setter.property_setter import make_property_setter + + +class TestDBUpdate(unittest.TestCase): + def test_db_update(self): + doctype = 'User' + frappe.reload_doctype('User', force=True) + frappe.model.meta.trim_tables('User') + make_property_setter(doctype, 'bio', 'fieldtype', 'Text', 'Data') + make_property_setter(doctype, 'enabled', 'default', '1', 'Int') + + meta = frappe.get_meta(doctype, cached=False) + + frappe.db.updatedb(doctype) + + field_defs = get_field_defs(doctype) + table_columns = frappe.db.get_table_columns_description('tab{}'.format(doctype)) + + self.assertEqual(len(field_defs), len(table_columns)) + + for field_def in field_defs: + fieldname = field_def.get('fieldname') + table_column = find(table_columns, lambda d: d.get('name') == fieldname) + + fieldtype = get_fieldtype_from_def(field_def) + + default = field_def.default if field_def.default is not None else 'NULL' + + self.assertEquals(fieldtype, table_column.type) + self.assertIn(table_column.default or 'NULL', [default, "'{}'".format(default), '0']) + + +def get_fieldtype_from_def(field_def): + fieldtuple = frappe.db.type_map.get(field_def.fieldtype, ('', 0)) + fieldtype = fieldtuple[0] + if fieldtype in ('varchar', 'datetime', 'int'): + fieldtype += '({})'.format(field_def.length or fieldtuple[1]) + return fieldtype + +def get_field_defs(doctype): + meta = frappe.get_meta(doctype, cached=False) + field_defs = meta.get_fieldnames_with_value(True) + field_defs += get_other_fields_meta(meta) + return field_defs + +def get_other_fields_meta(meta): + from frappe.model import optional_fields + default_fields_map = { + 'name': ('Data', 0), + 'owner': ('Data', 0), + 'parent': ('Data', 0), + 'parentfield': ('Data', 0), + 'modified_by': ('Data', 0), + 'parenttype': ('Data', 0), + 'creation': ('Datetime', 0), + 'modified': ('Datetime', 0), + 'idx': ('Int', 8), + 'docstatus': ('Int', 1) + } + + optional_fields = frappe.db.OPTIONAL_COLUMNS + if meta.track_seen: + optional_fields.append('_seen') + + optional_fields_map = {field: ('Text', 0) for field in optional_fields} + fields = dict(default_fields_map, **optional_fields_map) + field_map = [frappe._dict({'fieldname': field, 'fieldtype': _type, 'length': _length}) for field, (_type, _length) in fields.items()] + + return field_map \ No newline at end of file From 4d758bf191c0b37b141a21451eb1d6bc022cce9f Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Sat, 21 Mar 2020 12:51:02 +0530 Subject: [PATCH 08/15] fix: Typo --- frappe/model/meta.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/model/meta.py b/frappe/model/meta.py index 495d3a6713..c981745b99 100644 --- a/frappe/model/meta.py +++ b/frappe/model/meta.py @@ -379,7 +379,7 @@ class Meta(Document): self.permissions = [Document(d) for d in custom_perms] def get_fieldnames_with_value(self, with_field_meta=False): - return [df if with_meta else df.fieldname for df in self.fields if df.fieldtype not in no_value_fields] + return [df if with_field_meta else df.fieldname for df in self.fields if df.fieldtype not in no_value_fields] def get_fields_to_check_permissions(self, user_permission_doctypes): From d495e9c3d4897b45745860f2fa03a608f017593f Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Sat, 21 Mar 2020 13:03:02 +0530 Subject: [PATCH 09/15] fix: Use .get to fetch attribute value --- frappe/database/schema.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/database/schema.py b/frappe/database/schema.py index a9170feeb0..42bbcf0b22 100644 --- a/frappe/database/schema.py +++ b/frappe/database/schema.py @@ -68,7 +68,7 @@ class DBTable: fields = self.meta.get_fieldnames_with_value(True) # optional fields like _comments - if not self.meta.istable: + if not self.meta.get('istable'): for fieldname in frappe.db.OPTIONAL_COLUMNS: fields.append({ "fieldname": fieldname, @@ -76,7 +76,7 @@ class DBTable: }) # add _seen column if track_seen - if getattr(self.meta, 'track_seen', False): + if self.meta.get('track_seen'): fields.append({ 'fieldname': '_seen', 'fieldtype': 'Text' From 4c8d209c4cd04d77fa2e30055e49fc7c284087c0 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Sat, 21 Mar 2020 15:23:24 +0530 Subject: [PATCH 10/15] fix: Pass Meta instance to updatedb - Check if Property Setter exists --- frappe/core/doctype/doctype/doctype.py | 3 ++- frappe/model/meta.py | 21 +++++++++++---------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/frappe/core/doctype/doctype/doctype.py b/frappe/core/doctype/doctype/doctype.py index 0024995573..2c8cd240ee 100644 --- a/frappe/core/doctype/doctype/doctype.py +++ b/frappe/core/doctype/doctype/doctype.py @@ -24,6 +24,7 @@ from frappe.modules import make_boilerplate, get_doc_path from frappe.database.schema import validate_column_name, validate_column_length from frappe.model.docfield import supports_translation from frappe.modules.import_file import get_file_path +from frappe.model.meta import Meta class InvalidFieldNameError(frappe.ValidationError): pass @@ -275,7 +276,7 @@ class DocType(Document): """Update database schema, make controller templates if `custom` is not set and clear cache.""" self.delete_duplicate_custom_fields() try: - frappe.db.updatedb(self.name) + frappe.db.updatedb(self.name, Meta(self)) except Exception as e: print("\n\nThere was an issue while migrating the DocType: {}\n".format(self.name)) raise e diff --git a/frappe/model/meta.py b/frappe/model/meta.py index c981745b99..016e624f37 100644 --- a/frappe/model/meta.py +++ b/frappe/model/meta.py @@ -165,7 +165,7 @@ class Meta(Document): def get_valid_columns(self): if not hasattr(self, "_valid_columns"): - if self.name in ("DocType", "DocField", "DocPerm", 'DocType Action', 'DocType Link', "Property Setter"): + if self.name in ("DocType", "DocField", "DocPerm", 'DocType Action', 'DocType Link'): self._valid_columns = get_table_columns(self.name) else: self._valid_columns = self.default_fields + \ @@ -290,17 +290,17 @@ class Meta(Document): return get_workflow_name(self.name) def add_custom_fields(self): - try: - self.extend("fields", frappe.db.sql("""SELECT * FROM `tabCustom Field` - WHERE dt = %s AND docstatus < 2""", (self.name,), as_dict=1, - update={"is_custom_field": 1})) - except Exception as e: - if frappe.db.is_table_missing(e): - return - else: - raise + if not frappe.db.table_exists('Custom Field'): + return + + self.extend("fields", frappe.db.sql("""SELECT * FROM `tabCustom Field` + WHERE dt = %s AND docstatus < 2""", (self.name,), as_dict=1, + update={"is_custom_field": 1})) def apply_property_setters(self): + if not frappe.db.table_exists('Property Setter'): + return + property_setters = frappe.db.sql("""select * from `tabProperty Setter` where doc_type=%s""", (self.name,), as_dict=1) @@ -371,6 +371,7 @@ class Meta(Document): if frappe.flags.in_patch or frappe.flags.in_install: return + if not self.istable and self.name not in ('DocType', 'DocField', 'DocPerm', 'Custom DocPerm'): custom_perms = frappe.get_all('Custom DocPerm', fields='*', From c21e33b001bad4aef36a3110ba53d55774ee2e0a Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Sat, 21 Mar 2020 16:55:04 +0530 Subject: [PATCH 11/15] fix(postgres): Ignore type casting in default value - return 'Daily' instead of "'Daily'::character varying" --- frappe/database/postgres/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/database/postgres/database.py b/frappe/database/postgres/database.py index 243d0f934e..da489b7340 100644 --- a/frappe/database/postgres/database.py +++ b/frappe/database/postgres/database.py @@ -282,7 +282,7 @@ class PostgresDatabase(Database): ELSE a.data_type END AS type, COUNT(b.indexdef) AS Index, - COALESCE(a.column_default, NULL) AS default, + SPLIT_PART(COALESCE(a.column_default, NULL), '::', 1) AS default, BOOL_OR(b.unique) AS unique FROM information_schema.columns a LEFT JOIN From 8f8e36176683c9306f1a29830affc2bc7e4d100d Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Sat, 21 Mar 2020 17:45:39 +0530 Subject: [PATCH 12/15] style: Fix formatting issues --- .../doctype/customize_form/customize_form.py | 14 +++++++++----- frappe/database/postgres/database.py | 6 +++--- frappe/database/schema.py | 6 ++++-- frappe/model/meta.py | 13 ++++++++----- frappe/tests/test_db_update.py | 6 +----- 5 files changed, 25 insertions(+), 20 deletions(-) diff --git a/frappe/custom/doctype/customize_form/customize_form.py b/frappe/custom/doctype/customize_form/customize_form.py index c8d1bb18b4..3259085781 100644 --- a/frappe/custom/doctype/customize_form/customize_form.py +++ b/frappe/custom/doctype/customize_form/customize_form.py @@ -384,10 +384,10 @@ class CustomizeForm(Document): FROM `tab{doctype}` WHERE LENGTH({fieldname}) > {max_length} '''.format( - fieldname = fieldname, - doctype = self.doc_type, - max_length = max_length - ), as_dict = True) + fieldname=fieldname, + doctype=self.doc_type, + max_length=max_length + ), as_dict=True) links = [] label = df.label for doc in docs: @@ -396,7 +396,11 @@ class CustomizeForm(Document): if docs: frappe.throw(_('Value for field {0} is too long in {1}. Length should be lesser than {2} characters') - .format(frappe.bold(label), links_str, frappe.bold(max_length)), title=_('Data Too Long'), is_minimizable=len(docs) > 1) + .format( + frappe.bold(label), + links_str, + frappe.bold(max_length) + ), title=_('Data Too Long'), is_minimizable=len(docs) > 1) self.flags.update_db = True diff --git a/frappe/database/postgres/database.py b/frappe/database/postgres/database.py index da489b7340..e30ef3293f 100644 --- a/frappe/database/postgres/database.py +++ b/frappe/database/postgres/database.py @@ -92,7 +92,7 @@ class PostgresDatabase(Database): # pylint: disable=W0221 def sql(self, *args, **kwargs): - if len(args): + if args: # since tuple is immutable args = list(args) args[0] = modify_query(args[0]) @@ -276,9 +276,9 @@ class PostgresDatabase(Database): # pylint: disable=W1401 return self.sql(''' SELECT a.column_name AS name, - CASE a.data_type + CASE LOWER(a.data_type) WHEN 'character varying' THEN CONCAT('varchar(', a.character_maximum_length ,')') - WHEN 'timestamp without TIME zone' THEN 'timestamp' + WHEN 'timestamp without time zone' THEN 'timestamp' ELSE a.data_type END AS type, COUNT(b.indexdef) AS Index, diff --git a/frappe/database/schema.py b/frappe/database/schema.py index 42bbcf0b22..8fe1bbfb4d 100644 --- a/frappe/database/schema.py +++ b/frappe/database/schema.py @@ -83,7 +83,8 @@ class DBTable: }) for field in fields: - self.columns[field.get('fieldname')] = DbColumn(self, + self.columns[field.get('fieldname')] = DbColumn( + self, field.get('fieldname'), field.get('fieldtype'), field.get('length'), @@ -91,7 +92,8 @@ class DBTable: field.get('search_index'), field.get('options'), field.get('unique'), - field.get('precision')) + field.get('precision') + ) def validate(self): """Check if change in varchar length isn't truncating the columns""" diff --git a/frappe/model/meta.py b/frappe/model/meta.py index 016e624f37..5065684311 100644 --- a/frappe/model/meta.py +++ b/frappe/model/meta.py @@ -293,9 +293,12 @@ class Meta(Document): if not frappe.db.table_exists('Custom Field'): return - self.extend("fields", frappe.db.sql("""SELECT * FROM `tabCustom Field` - WHERE dt = %s AND docstatus < 2""", (self.name,), as_dict=1, - update={"is_custom_field": 1})) + custom_fields = frappe.db.sql(""" + SELECT * FROM `tabCustom Field` + WHERE dt = %s AND docstatus < 2 + """, (self.name,), as_dict=1, update={"is_custom_field": 1}) + + self.extend("fields", custom_fields) def apply_property_setters(self): if not frappe.db.table_exists('Property Setter'): @@ -371,7 +374,6 @@ class Meta(Document): if frappe.flags.in_patch or frappe.flags.in_install: return - if not self.istable and self.name not in ('DocType', 'DocField', 'DocPerm', 'Custom DocPerm'): custom_perms = frappe.get_all('Custom DocPerm', fields='*', @@ -380,7 +382,8 @@ class Meta(Document): self.permissions = [Document(d) for d in custom_perms] def get_fieldnames_with_value(self, with_field_meta=False): - return [df if with_field_meta else df.fieldname for df in self.fields if df.fieldtype not in no_value_fields] + return [df if with_field_meta else df.fieldname \ + for df in self.fields if df.fieldtype not in no_value_fields] def get_fields_to_check_permissions(self, user_permission_doctypes): diff --git a/frappe/tests/test_db_update.py b/frappe/tests/test_db_update.py index 680c4c0672..1121366884 100644 --- a/frappe/tests/test_db_update.py +++ b/frappe/tests/test_db_update.py @@ -13,8 +13,6 @@ class TestDBUpdate(unittest.TestCase): make_property_setter(doctype, 'bio', 'fieldtype', 'Text', 'Data') make_property_setter(doctype, 'enabled', 'default', '1', 'Int') - meta = frappe.get_meta(doctype, cached=False) - frappe.db.updatedb(doctype) field_defs = get_field_defs(doctype) @@ -30,10 +28,9 @@ class TestDBUpdate(unittest.TestCase): default = field_def.default if field_def.default is not None else 'NULL' - self.assertEquals(fieldtype, table_column.type) + self.assertEqual(fieldtype, table_column.type) self.assertIn(table_column.default or 'NULL', [default, "'{}'".format(default), '0']) - def get_fieldtype_from_def(field_def): fieldtuple = frappe.db.type_map.get(field_def.fieldtype, ('', 0)) fieldtype = fieldtuple[0] @@ -48,7 +45,6 @@ def get_field_defs(doctype): return field_defs def get_other_fields_meta(meta): - from frappe.model import optional_fields default_fields_map = { 'name': ('Data', 0), 'owner': ('Data', 0), From 641580a30003e1982bceca397b7c8de3acc70495 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Sat, 21 Mar 2020 18:11:42 +0530 Subject: [PATCH 13/15] test: Add a fallback default --- frappe/tests/test_db_update.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/frappe/tests/test_db_update.py b/frappe/tests/test_db_update.py index 1121366884..be3e102114 100644 --- a/frappe/tests/test_db_update.py +++ b/frappe/tests/test_db_update.py @@ -26,10 +26,11 @@ class TestDBUpdate(unittest.TestCase): fieldtype = get_fieldtype_from_def(field_def) - default = field_def.default if field_def.default is not None else 'NULL' + fallback_default = '0' if field_def.get('fieldtype') in frappe.model.numeric_fieldtypes else 'NULL' + default = field_def.default if field_def.default is not None else fallback_default self.assertEqual(fieldtype, table_column.type) - self.assertIn(table_column.default or 'NULL', [default, "'{}'".format(default), '0']) + self.assertIn(table_column.default or 'NULL', [default, "'{}'".format(default)]) def get_fieldtype_from_def(field_def): fieldtuple = frappe.db.type_map.get(field_def.fieldtype, ('', 0)) From bf69b98cba33c4f02c9fbf6143d9ab17348e7648 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Sat, 21 Mar 2020 18:24:15 +0530 Subject: [PATCH 14/15] test: Set docstatus type as Check --- frappe/tests/test_db_update.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/tests/test_db_update.py b/frappe/tests/test_db_update.py index be3e102114..34378de3af 100644 --- a/frappe/tests/test_db_update.py +++ b/frappe/tests/test_db_update.py @@ -56,7 +56,7 @@ def get_other_fields_meta(meta): 'creation': ('Datetime', 0), 'modified': ('Datetime', 0), 'idx': ('Int', 8), - 'docstatus': ('Int', 1) + 'docstatus': ('Check', 0) } optional_fields = frappe.db.OPTIONAL_COLUMNS From 1599fc0751a5d00f3149f34074e6cd14378f5627 Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Wed, 25 Mar 2020 12:48:21 +0530 Subject: [PATCH 15/15] fix: Get latest meta while building table --- frappe/database/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/database/schema.py b/frappe/database/schema.py index 8fe1bbfb4d..28e055f382 100644 --- a/frappe/database/schema.py +++ b/frappe/database/schema.py @@ -13,7 +13,7 @@ class DBTable: def __init__(self, doctype, meta=None): self.doctype = doctype self.table_name = 'tab{}'.format(doctype) - self.meta = meta or frappe.get_meta(doctype) + self.meta = meta or frappe.get_meta(doctype, False) self.columns = {} self.current_columns = {}