Merge pull request #9755 from surajshetty3416/fix-customized-form

feat: change fieldtype in db with customize form

frappe/core/doctype/doctype/doctype.py

@@ -24,6 +24,7 @@ from frappe.modules import make_boilerplate, get_doc_path
 from frappe.database.schema import validate_column_name, validate_column_length
 from frappe.model.docfield import supports_translation
 from frappe.modules.import_file import get_file_path
+from frappe.model.meta import Meta
 
 
 class InvalidFieldNameError(frappe.ValidationError): pass
@@ -275,7 +276,7 @@ class DocType(Document):
 		"""Update database schema, make controller templates if `custom` is not set and clear cache."""
 		self.delete_duplicate_custom_fields()
 		try:
-			frappe.db.updatedb(self.name, self)
+			frappe.db.updatedb(self.name, Meta(self))
 		except Exception as e:
 			print("\n\nThere was an issue while migrating the DocType: {}\n".format(self.name))
 			raise e

frappe/custom/doctype/customize_form/customize_form.py

@@ -166,7 +166,6 @@ class CustomizeForm(Document):
 
 		self.flags.update_db = False
 		self.flags.rebuild_doctype_for_global_search = False
-
 		self.set_property_setters()
 		self.update_custom_fields()
 		self.set_name_translation()
@@ -362,13 +361,49 @@ class CustomizeForm(Document):
 
 	def validate_fieldtype_change(self, df, old_value, new_value):
 		allowed = False
+		self.check_length_for_fieldtypes = []
 		for allowed_changes in allowed_fieldtype_change:
 			if (old_value in allowed_changes and new_value in allowed_changes):
 				allowed = True
+				if frappe.db.type_map.get(old_value)[1] > frappe.db.type_map.get(new_value)[1]:
+					self.check_length_for_fieldtypes.append({'df': df, 'old_value': old_value})
+					self.validate_fieldtype_length()
+				else:
+					self.flags.update_db = True
 				break
 		if not allowed:
 			frappe.throw(_("Fieldtype cannot be changed from {0} to {1} in row {2}").format(old_value, new_value, df.idx))
 
+	def validate_fieldtype_length(self):
+		for field in self.check_length_for_fieldtypes:
+			df = field.get('df')
+			max_length = frappe.db.type_map.get(df.fieldtype)[1]
+			fieldname = df.fieldname
+			docs = frappe.db.sql('''
+				SELECT name, {fieldname}, LENGTH({fieldname}) AS len
+				FROM `tab{doctype}`
+				WHERE LENGTH({fieldname}) > {max_length}
+			'''.format(
+				fieldname=fieldname,
+				doctype=self.doc_type,
+				max_length=max_length
+			), as_dict=True)
+			links = []
+			label = df.label
+			for doc in docs:
+				links.append(frappe.utils.get_link_to_form(self.doc_type, doc.name))
+			links_str = ', '.join(links)
+
+			if docs:
+				frappe.throw(_('Value for field {0} is too long in {1}. Length should be lesser than {2} characters')
+					.format(
+						frappe.bold(label),
+						links_str,
+						frappe.bold(max_length)
+					), title=_('Data Too Long'), is_minimizable=len(docs) > 1)
+
+		self.flags.update_db = True
+
 	def reset_to_defaults(self):
 		if not self.doc_type:
 			return

frappe/database/postgres/database.py

@@ -92,7 +92,7 @@ class PostgresDatabase(Database):
 
 	# pylint: disable=W0221
 	def sql(self, *args, **kwargs):
-		if len(args):
+		if args:
 			# since tuple is immutable
 			args = list(args)
 			args[0] = modify_query(args[0])
@@ -276,13 +276,13 @@ class PostgresDatabase(Database):
 		# pylint: disable=W1401
 		return self.sql('''
 			SELECT a.column_name AS name,
-			CASE a.data_type
+			CASE LOWER(a.data_type)
 				WHEN 'character varying' THEN CONCAT('varchar(', a.character_maximum_length ,')')
-				WHEN 'timestamp without TIME zone' THEN 'timestamp'
+				WHEN 'timestamp without time zone' THEN 'timestamp'
 				ELSE a.data_type
 			END AS type,
 			COUNT(b.indexdef) AS Index,
-			COALESCE(a.column_default, NULL) AS default,
+			SPLIT_PART(COALESCE(a.column_default, NULL), '::', 1) AS default,
 			BOOL_OR(b.unique) AS unique
 			FROM information_schema.columns a
 			LEFT JOIN

frappe/database/schema.py

@@ -13,7 +13,7 @@ class DBTable:
 	def __init__(self, doctype, meta=None):
 		self.doctype = doctype
 		self.table_name = 'tab{}'.format(doctype)
-		self.meta = meta or frappe.get_meta(doctype)
+		self.meta = meta or frappe.get_meta(doctype, False)
 		self.columns = {}
 		self.current_columns = {}
 
@@ -65,64 +65,35 @@ class DBTable:
 		"""
 			get columns from docfields and custom fields
 		"""
-		fl = frappe.db.sql("SELECT * FROM `tabDocField` WHERE parent = %s", self.doctype, as_dict = 1)
-		lengths = {}
-		precisions = {}
-		uniques = {}
+		fields = self.meta.get_fieldnames_with_value(True)
 
 		# optional fields like _comments
-		if not self.meta.istable:
+		if not self.meta.get('istable'):
 			for fieldname in frappe.db.OPTIONAL_COLUMNS:
-				fl.append({
+				fields.append({
 					"fieldname": fieldname,
 					"fieldtype": "Text"
 				})
 
 			# add _seen column if track_seen
-			if getattr(self.meta, 'track_seen', False):
-				fl.append({
+			if self.meta.get('track_seen'):
+				fields.append({
 					'fieldname': '_seen',
 					'fieldtype': 'Text'
 				})
 
-		if (not frappe.flags.in_install_db
-			and (frappe.flags.in_install != "frappe"
-			or frappe.flags.ignore_in_install)):
-			custom_fl = frappe.db.sql("""
-				SELECT * FROM `tabCustom Field`
-				WHERE dt = %s AND docstatus < 2
-				""", (self.doctype,), as_dict=1)
-			if custom_fl: fl += custom_fl
-
-			# apply length, precision and unique from property setters
-			for ps in frappe.get_all("Property Setter",
-				fields=["field_name", "property", "value"],
-				filters={
-					"doc_type": self.doctype,
-					"doctype_or_field": "DocField",
-					"property": ["in", ["precision", "length", "unique"]]
-				}):
-
-				if ps.property=="length":
-					lengths[ps.field_name] = cint(ps.value)
-
-				elif ps.property=="precision":
-					precisions[ps.field_name] = cint(ps.value)
-
-				elif ps.property=="unique":
-					uniques[ps.field_name] = cint(ps.value)
-
-		for f in fl:
-			self.columns[f['fieldname']] = DbColumn(self,
-				f['fieldname'],
-				f['fieldtype'],
-				lengths.get(f["fieldname"]) or f.get('length'),
-				f.get('default'),
-				f.get('search_index'),
-				f.get('options'),
-				uniques.get(f["fieldname"],
-				f.get('unique')),
-				precisions.get(f['fieldname']) or f.get('precision'))
+		for field in fields:
+			self.columns[field.get('fieldname')] = DbColumn(
+				self,
+				field.get('fieldname'),
+				field.get('fieldtype'),
+				field.get('length'),
+				field.get('default'),
+				field.get('search_index'),
+				field.get('options'),
+				field.get('unique'),
+				field.get('precision')
+			)
 
 	def validate(self):
 		"""Check if change in varchar length isn't truncating the columns"""

frappe/model/meta.py

@@ -165,7 +165,7 @@ class Meta(Document):
 
 	def get_valid_columns(self):
 		if not hasattr(self, "_valid_columns"):
-			if self.name in ("DocType", "DocField", "DocPerm", 'DocType Action', 'DocType Link', "Property Setter"):
+			if self.name in ("DocType", "DocField", "DocPerm", 'DocType Action', 'DocType Link'):
 				self._valid_columns = get_table_columns(self.name)
 			else:
 				self._valid_columns = self.default_fields + \
@@ -290,17 +290,20 @@ class Meta(Document):
 		return get_workflow_name(self.name)
 
 	def add_custom_fields(self):
-		try:
-			self.extend("fields", frappe.db.sql("""SELECT * FROM `tabCustom Field`
-				WHERE dt = %s AND docstatus < 2""", (self.name,), as_dict=1,
-				update={"is_custom_field": 1}))
-		except Exception as e:
-			if frappe.db.is_table_missing(e):
-				return
-			else:
-				raise
+		if not frappe.db.table_exists('Custom Field'):
+			return
+
+		custom_fields = frappe.db.sql("""
+			SELECT * FROM `tabCustom Field`
+			WHERE dt = %s AND docstatus < 2
+		""", (self.name,), as_dict=1, update={"is_custom_field": 1})
+
+		self.extend("fields", custom_fields)
 
 	def apply_property_setters(self):
+		if not frappe.db.table_exists('Property Setter'):
+			return
+
 		property_setters = frappe.db.sql("""select * from `tabProperty Setter` where
 			doc_type=%s""", (self.name,), as_dict=1)
 
@@ -378,8 +381,9 @@ class Meta(Document):
 			if custom_perms:
 				self.permissions = [Document(d) for d in custom_perms]
 
-	def get_fieldnames_with_value(self):
-		return [df.fieldname for df in self.fields if df.fieldtype not in no_value_fields]
+	def get_fieldnames_with_value(self, with_field_meta=False):
+		return [df if with_field_meta else df.fieldname \
+			for df in self.fields if df.fieldtype not in no_value_fields]
 
 
 	def get_fields_to_check_permissions(self, user_permission_doctypes):

frappe/tests/test_db_update.py

@@ -0,0 +1,70 @@
+import unittest
+import frappe
+
+from frappe.core.utils import find
+from frappe.custom.doctype.property_setter.property_setter import make_property_setter
+
+
+class TestDBUpdate(unittest.TestCase):
+	def test_db_update(self):
+		doctype = 'User'
+		frappe.reload_doctype('User', force=True)
+		frappe.model.meta.trim_tables('User')
+		make_property_setter(doctype, 'bio', 'fieldtype', 'Text', 'Data')
+		make_property_setter(doctype, 'enabled', 'default', '1', 'Int')
+
+		frappe.db.updatedb(doctype)
+
+		field_defs = get_field_defs(doctype)
+		table_columns = frappe.db.get_table_columns_description('tab{}'.format(doctype))
+
+		self.assertEqual(len(field_defs), len(table_columns))
+
+		for field_def in field_defs:
+			fieldname = field_def.get('fieldname')
+			table_column = find(table_columns, lambda d: d.get('name') == fieldname)
+
+			fieldtype = get_fieldtype_from_def(field_def)
+
+			fallback_default = '0' if field_def.get('fieldtype') in frappe.model.numeric_fieldtypes else 'NULL'
+			default = field_def.default if field_def.default is not None else fallback_default
+
+			self.assertEqual(fieldtype, table_column.type)
+			self.assertIn(table_column.default or 'NULL', [default, "'{}'".format(default)])
+
+def get_fieldtype_from_def(field_def):
+	fieldtuple = frappe.db.type_map.get(field_def.fieldtype, ('', 0))
+	fieldtype = fieldtuple[0]
+	if fieldtype in ('varchar', 'datetime', 'int'):
+		fieldtype += '({})'.format(field_def.length or fieldtuple[1])
+	return fieldtype
+
+def get_field_defs(doctype):
+	meta = frappe.get_meta(doctype, cached=False)
+	field_defs = meta.get_fieldnames_with_value(True)
+	field_defs += get_other_fields_meta(meta)
+	return field_defs
+
+def get_other_fields_meta(meta):
+	default_fields_map = {
+		'name': ('Data', 0),
+		'owner': ('Data', 0),
+		'parent': ('Data', 0),
+		'parentfield': ('Data', 0),
+		'modified_by': ('Data', 0),
+		'parenttype': ('Data', 0),
+		'creation': ('Datetime', 0),
+		'modified': ('Datetime', 0),
+		'idx': ('Int', 8),
+		'docstatus': ('Check', 0)
+	}
+
+	optional_fields = frappe.db.OPTIONAL_COLUMNS
+	if meta.track_seen:
+		optional_fields.append('_seen')
+
+	optional_fields_map = {field: ('Text', 0) for field in optional_fields}
+	fields = dict(default_fields_map, **optional_fields_map)
+	field_map =  [frappe._dict({'fieldname': field, 'fieldtype': _type, 'length': _length}) for field, (_type, _length) in fields.items()]
+
+	return field_map