From fe972d0abd0f674ec960cea31989c89a6ca81a36 Mon Sep 17 00:00:00 2001 From: AarDG10 Date: Sat, 1 Nov 2025 17:02:05 +0530 Subject: [PATCH 01/30] feat(ci): add postgres.json config file for Postgresql tests --- .github/helper/db/postgres.json | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 .github/helper/db/postgres.json diff --git a/.github/helper/db/postgres.json b/.github/helper/db/postgres.json new file mode 100644 index 0000000000..73a22b6d49 --- /dev/null +++ b/.github/helper/db/postgres.json @@ -0,0 +1,18 @@ +{ + "db_host": "127.0.0.1", + "db_port": 5432, + "db_name": "test_frappe", + "db_password": "test_frappe", + "db_type": "postgres", + "allow_tests": true, + "auto_email_id": "test@example.com", + "mail_server": "localhost", + "mail_port": 2525, + "mail_login": "test@example.com", + "mail_password": "test", + "admin_password": "admin", + "root_login": "postgres", + "root_password": "travis", + "host_name": "http://test_site:8000", + "server_script_enabled": true +} \ No newline at end of file From c7173147a2070d13c1450a63b93d0fcc9a8459b9 Mon Sep 17 00:00:00 2001 From: AarDG10 Date: Sat, 1 Nov 2025 17:06:49 +0530 Subject: [PATCH 02/30] ci: enable postgres support in github actions setup --- .github/actions/setup/action.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index abf0c8afad..4b63407bb5 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -103,7 +103,7 @@ runs: curl -LsS https://r.mariadb.com/downloads/mariadb_repo_setup | sudo bash sudo apt -qq update sudo apt -qq remove mysql-server mysql-client - sudo apt -qq install libcups2-dev redis-server mariadb-client libmariadb-dev + sudo apt -qq install libcups2-dev redis-server mariadb-client libmariadb-dev postgresql-client libpq-dev wget -q -O /tmp/wkhtmltox.deb https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6.1-2/wkhtmltox_0.12.6.1-2.jammy_amd64.deb sudo apt install /tmp/wkhtmltox.deb @@ -169,6 +169,14 @@ runs: mariadb --host 127.0.0.1 --port 3306 -u root -p${{ inputs.db-root-password }} -e "FLUSH PRIVILEGES"; fi + if [ "$DB" == "postgres" ]; then + export PGPASSWORD='travis' + psql -h 127.0.0.1 -p 5432 -c "CREATE DATABASE test_frappe" -U postgres + psql -h 127.0.0.1 -p 5432 -c "CREATE USER test_frappe WITH PASSWORD 'test_frappe'" -U postgres + psql -h 127.0.0.1 -p 5432 -U postgres -c "GRANT ALL PRIVILEGES ON DATABASE test_frappe TO test_frappe;" + unset PGPASSWORD + fi + - shell: bash -e {0} run: | # Install App(s) From 5ec2d132c582a6df6386c6212f1806d55409edc9 Mon Sep 17 00:00:00 2001 From: AarDG10 Date: Sat, 1 Nov 2025 17:14:23 +0530 Subject: [PATCH 03/30] ci: add postgres 17.0 service and test matrix entry --- .github/workflows/_base-server-tests.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/_base-server-tests.yml b/.github/workflows/_base-server-tests.yml index f7f744a935..0bc499f253 100644 --- a/.github/workflows/_base-server-tests.yml +++ b/.github/workflows/_base-server-tests.yml @@ -62,9 +62,20 @@ jobs: strategy: fail-fast: false matrix: - db: ${{ fromJson(inputs.enable-sqlite && '["mariadb", "sqlite"]' || '["mariadb"]') }} + db: ${{ fromJson(inputs.enable-sqlite && '["mariadb", "postgres", "sqlite"]' || '["mariadb", "postgres"]') }} index: ${{ fromJson(needs.gen-idx-integration.outputs.indices) }} services: + postgres: + image: postgres:17.0 + ports: + - 5432:5432 + env: + POSTGRES_PASSWORD: travis + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 3 mariadb: image: mariadb:11.8 ports: From b955cdc4c0cea650e01f207fbb02dcf64cde417b Mon Sep 17 00:00:00 2001 From: AarDG10 Date: Tue, 4 Nov 2025 12:55:08 +0530 Subject: [PATCH 04/30] test(postgres): fix test_build_match_conditions for Postgres --- frappe/tests/test_db_query.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/tests/test_db_query.py b/frappe/tests/test_db_query.py index 3f196d7214..36cab4f43d 100644 --- a/frappe/tests/test_db_query.py +++ b/frappe/tests/test_db_query.py @@ -221,8 +221,8 @@ class TestDBQuery(IntegrationTestCase): # get as conditions if frappe.db.db_type == "mariadb": assertion_string = """(((ifnull(`tabTest Blog Post`.`name`, '')='' or `tabTest Blog Post`.`name` in ('_Test Blog Post 1', '_Test Blog Post'))))""" - else: - assertion_string = """(((ifnull(cast(`tabBlog Post`.`name` as varchar), '')='' or cast(`tabBlog Post`.`name` as varchar) in ('_Test Blog Post 1', '_Test Blog Post'))))""" + elif frappe.db.db_type == "postgres": + assertion_string = """(((ifnull(cast(`tabTest Blog Post`.`name` as varchar), '')='' or cast(`tabTest Blog Post`.`name` as varchar) in ('_Test Blog Post 1', '_Test Blog Post'))))""" self.assertEqual(build_match_conditions(as_condition=True), assertion_string) From 5ec28f99c6837eb935c7434995cf77ee91e6aa3f Mon Sep 17 00:00:00 2001 From: AarDG10 Date: Tue, 4 Nov 2025 16:45:52 +0530 Subject: [PATCH 05/30] test(postgres): fix test_not_equal_condition_on_none for Postgres --- frappe/tests/test_query.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/frappe/tests/test_query.py b/frappe/tests/test_query.py index 6f205a655e..d3d54aebe7 100644 --- a/frappe/tests/test_query.py +++ b/frappe/tests/test_query.py @@ -63,6 +63,11 @@ def create_tree_docs(): d.insert() +def convert_identifier_quotes(query): + """util to replace mariadb query idenitfifiers with postgres ones""" + return query.replace("`", '"') if frappe.db.db_type == "postgres" else query + + class TestQuery(IntegrationTestCase): def setUp(self): setup_for_tests() @@ -1611,6 +1616,9 @@ class TestQuery(IntegrationTestCase): self.assertIn("Unsupported function or invalid field name: DROP", str(cm.exception)) def test_not_equal_condition_on_none(self): + expected_query = convert_identifier_quotes( + "SELECT `tabDocType`.* FROM `tabDocType` LEFT JOIN `tabDocField` ON `tabDocField`.`parent`=`tabDocType`.`name` AND `tabDocField`.`parenttype`='DocType' AND `tabDocField`.`parentfield`='fields' WHERE `tabDocField`.`name` IS NULL AND `tabDocType`.`parent` IS NOT NULL" + ) self.assertEqual( frappe.qb.get_query( "DocType", @@ -1620,7 +1628,7 @@ class TestQuery(IntegrationTestCase): ["DocType", "parent", "!=", None], ], ).get_sql(), - "SELECT `tabDocType`.* FROM `tabDocType` LEFT JOIN `tabDocField` ON `tabDocField`.`parent`=`tabDocType`.`name` AND `tabDocField`.`parenttype`='DocType' AND `tabDocField`.`parentfield`='fields' WHERE `tabDocField`.`name` IS NULL AND `tabDocType`.`parent` IS NOT NULL", + expected_query, ) From b4d75132d4508bee6a70c5d828591031d950ec87 Mon Sep 17 00:00:00 2001 From: AarDG10 Date: Tue, 4 Nov 2025 17:17:38 +0530 Subject: [PATCH 06/30] test(postgres): fix test_multiple_tables_in_filters for Postgres --- frappe/tests/test_query.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/frappe/tests/test_query.py b/frappe/tests/test_query.py index d3d54aebe7..e7f2c1f0af 100644 --- a/frappe/tests/test_query.py +++ b/frappe/tests/test_query.py @@ -72,8 +72,10 @@ class TestQuery(IntegrationTestCase): def setUp(self): setup_for_tests() - @run_only_if(db_type_is.MARIADB) def test_multiple_tables_in_filters(self): + expected_query = convert_identifier_quotes( + "SELECT `tabDocType`.* FROM `tabDocType` LEFT JOIN `tabDocField` ON `tabDocField`.`parent`=`tabDocType`.`name` AND `tabDocField`.`parenttype`='DocType' AND `tabDocField`.`parentfield`='fields' WHERE `tabDocField`.`name` LIKE 'f%' AND `tabDocType`.`parent`='something'" + ) self.assertEqual( frappe.qb.get_query( "DocType", @@ -83,7 +85,7 @@ class TestQuery(IntegrationTestCase): ["DocType", "parent", "=", "something"], ], ).get_sql(), - "SELECT `tabDocType`.* FROM `tabDocType` LEFT JOIN `tabDocField` ON `tabDocField`.`parent`=`tabDocType`.`name` AND `tabDocField`.`parenttype`='DocType' AND `tabDocField`.`parentfield`='fields' WHERE `tabDocField`.`name` LIKE 'f%' AND `tabDocType`.`parent`='something'", + expected_query, ) @run_only_if(db_type_is.MARIADB) From cb636f7c5cd29baf86ff96b131138f9aadf469ec Mon Sep 17 00:00:00 2001 From: AarDG10 Date: Tue, 4 Nov 2025 17:42:33 +0530 Subject: [PATCH 07/30] test(postgres): fix test_filters for Postgres --- frappe/tests/test_query.py | 47 +++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/frappe/tests/test_query.py b/frappe/tests/test_query.py index e7f2c1f0af..73e2dedee5 100644 --- a/frappe/tests/test_query.py +++ b/frappe/tests/test_query.py @@ -359,88 +359,89 @@ class TestQuery(IntegrationTestCase): .get_sql(), ) - @run_only_if(db_type_is.MARIADB) def test_filters(self): + expected_query = convert_identifier_quotes( + "SELECT `tabDocType`.`name` FROM `tabDocType` LEFT JOIN `tabModule Def` ON `tabModule Def`.`name`=`tabDocType`.`module` WHERE `tabModule Def`.`app_name`='frappe'" + ) self.assertEqual( frappe.qb.get_query( "DocType", fields=["name"], filters={"module.app_name": "frappe"}, ).get_sql(), - "SELECT `tabDocType`.`name` FROM `tabDocType` LEFT JOIN `tabModule Def` ON `tabModule Def`.`name`=`tabDocType`.`module` WHERE `tabModule Def`.`app_name`='frappe'".replace( - "`", '"' if frappe.db.db_type == "postgres" else "`" - ), + expected_query, ) + expected_query = convert_identifier_quotes( + "SELECT `tabDocType`.`name` FROM `tabDocType` LEFT JOIN `tabModule Def` ON `tabModule Def`.`name`=`tabDocType`.`module` WHERE `tabModule Def`.`app_name` LIKE 'frap%'" + ) self.assertEqual( frappe.qb.get_query( "DocType", fields=["name"], filters={"module.app_name": ("like", "frap%")}, ).get_sql(), - "SELECT `tabDocType`.`name` FROM `tabDocType` LEFT JOIN `tabModule Def` ON `tabModule Def`.`name`=`tabDocType`.`module` WHERE `tabModule Def`.`app_name` LIKE 'frap%'".replace( - "`", '"' if frappe.db.db_type == "postgres" else "`" - ), + expected_query, ) + expected_query = convert_identifier_quotes( + "SELECT `tabDocType`.`name` FROM `tabDocType` LEFT JOIN `tabDocPerm` ON `tabDocPerm`.`parent`=`tabDocType`.`name` AND `tabDocPerm`.`parenttype`='DocType' AND `tabDocPerm`.`parentfield`='permissions' WHERE `tabDocPerm`.`role`='System Manager'" + ) self.assertEqual( frappe.qb.get_query( "DocType", fields=["name"], filters={"permissions.role": "System Manager"}, ).get_sql(), - "SELECT `tabDocType`.`name` FROM `tabDocType` LEFT JOIN `tabDocPerm` ON `tabDocPerm`.`parent`=`tabDocType`.`name` AND `tabDocPerm`.`parenttype`='DocType' AND `tabDocPerm`.`parentfield`='permissions' WHERE `tabDocPerm`.`role`='System Manager'".replace( - "`", '"' if frappe.db.db_type == "postgres" else "`" - ), + expected_query, ) + expected_query = convert_identifier_quotes("SELECT `module` FROM `tabDocType` WHERE `name`=''") self.assertEqual( frappe.qb.get_query( "DocType", fields=["module"], filters="", ).get_sql(), - "SELECT `module` FROM `tabDocType` WHERE `name`=''".replace( - "`", '"' if frappe.db.db_type == "postgres" else "`" - ), + expected_query, ) + expected_query = convert_identifier_quotes( + "SELECT `name` FROM `tabDocType` WHERE `name` IN ('ToDo','Note')" + ) self.assertEqual( frappe.qb.get_query( "DocType", filters=["ToDo", "Note"], ).get_sql(), - "SELECT `name` FROM `tabDocType` WHERE `name` IN ('ToDo','Note')".replace( - "`", '"' if frappe.db.db_type == "postgres" else "`" - ), + expected_query, ) + expected_query = convert_identifier_quotes("SELECT `name` FROM `tabDocType` WHERE `name` IN ('')") self.assertEqual( frappe.qb.get_query( "DocType", filters={"name": ("in", [])}, ).get_sql(), - "SELECT `name` FROM `tabDocType` WHERE `name` IN ('')".replace( - "`", '"' if frappe.db.db_type == "postgres" else "`" - ), + expected_query, ) + expected_query = convert_identifier_quotes("SELECT `name` FROM `tabDocType` WHERE `name` IN (1,2,3)") self.assertEqual( frappe.qb.get_query( "DocType", filters=[1, 2, 3], ).get_sql(), - "SELECT `name` FROM `tabDocType` WHERE `name` IN (1,2,3)".replace( - "`", '"' if frappe.db.db_type == "postgres" else "`" - ), + expected_query, ) + expected_query = convert_identifier_quotes("SELECT `name` FROM `tabDocType`") self.assertEqual( frappe.qb.get_query( "DocType", filters=[], ).get_sql(), - "SELECT `name` FROM `tabDocType`".replace("`", '"' if frappe.db.db_type == "postgres" else "`"), + expected_query, ) def test_nested_filters(self): From 4fc7661bfe97c5450b34ab79886d9c9752fd2299 Mon Sep 17 00:00:00 2001 From: AarDG10 Date: Tue, 4 Nov 2025 18:17:52 +0530 Subject: [PATCH 08/30] test(postgres): enable test_string_fields to run on Postgres --- frappe/tests/test_query.py | 1 - 1 file changed, 1 deletion(-) diff --git a/frappe/tests/test_query.py b/frappe/tests/test_query.py index 73e2dedee5..ad8b22f41d 100644 --- a/frappe/tests/test_query.py +++ b/frappe/tests/test_query.py @@ -88,7 +88,6 @@ class TestQuery(IntegrationTestCase): expected_query, ) - @run_only_if(db_type_is.MARIADB) def test_string_fields(self): self.assertEqual( frappe.qb.get_query("User", fields="name, email", filters={"name": "Administrator"}).get_sql(), From af08b40c0952f1aec93d9543282f624f8d09d3d5 Mon Sep 17 00:00:00 2001 From: AarDG10 Date: Tue, 4 Nov 2025 18:48:23 +0530 Subject: [PATCH 09/30] test(postgres): fix test_sql_functions_in_fields for Postgres --- frappe/tests/test_query.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/frappe/tests/test_query.py b/frappe/tests/test_query.py index ad8b22f41d..da84183c60 100644 --- a/frappe/tests/test_query.py +++ b/frappe/tests/test_query.py @@ -1520,7 +1520,7 @@ class TestQuery(IntegrationTestCase): # Test simple function without alias query = frappe.qb.get_query("User", fields=["user_type", {"COUNT": "name"}], group_by="user_type") sql = query.get_sql() - self.assertIn("COUNT(`name`)", sql) + self.assertIn(convert_identifier_quotes("COUNT(`name`)"), sql) self.assertIn("GROUP BY", sql) # Test function with alias @@ -1528,52 +1528,52 @@ class TestQuery(IntegrationTestCase): "User", fields=[{"COUNT": "name", "as": "total_users"}], group_by="user_type" ) sql = query.get_sql() - self.assertIn("COUNT(`name`) `total_users`", sql) + self.assertIn(convert_identifier_quotes("COUNT(`name`) `total_users`"), sql) # Test SUM function with alias query = frappe.qb.get_query( "User", fields=[{"SUM": "enabled", "as": "total_enabled"}], group_by="user_type" ) sql = query.get_sql() - self.assertIn("SUM(`enabled`) `total_enabled`", sql) + self.assertIn(convert_identifier_quotes("SUM(`enabled`) `total_enabled`"), sql) # Test MAX function query = frappe.qb.get_query( "User", fields=[{"MAX": "creation", "as": "latest_user"}], group_by="user_type" ) sql = query.get_sql() - self.assertIn("MAX(`creation`) `latest_user`", sql) + self.assertIn(convert_identifier_quotes("MAX(`creation`) `latest_user`"), sql) # Test MIN function query = frappe.qb.get_query( "User", fields=[{"MIN": "creation", "as": "earliest_user"}], group_by="user_type" ) sql = query.get_sql() - self.assertIn("MIN(`creation`) `earliest_user`", sql) + self.assertIn(convert_identifier_quotes("MIN(`creation`) `earliest_user`"), sql) # Test AVG function query = frappe.qb.get_query( "User", fields=[{"AVG": "enabled", "as": "avg_enabled"}], group_by="user_type" ) sql = query.get_sql() - self.assertIn("AVG(`enabled`) `avg_enabled`", sql) + self.assertIn(convert_identifier_quotes("AVG(`enabled`) `avg_enabled`"), sql) # Test ABS function query = frappe.qb.get_query("User", fields=[{"ABS": "enabled", "as": "abs_enabled"}]) sql = query.get_sql() - self.assertIn("ABS(`enabled`) `abs_enabled`", sql) + self.assertIn(convert_identifier_quotes("ABS(`enabled`) `abs_enabled`"), sql) # Test IFNULL function with two parameters query = frappe.qb.get_query( "User", fields=[{"IFNULL": ["first_name", "'Unknown'"], "as": "safe_name"}] ) sql = query.get_sql() - self.assertIn("IFNULL(`first_name`,'Unknown') `safe_name`", sql) + self.assertIn(convert_identifier_quotes("IFNULL(`first_name`,'Unknown') `safe_name`"), sql) # Test TIMESTAMP function query = frappe.qb.get_query("User", fields=[{"TIMESTAMP": "creation", "as": "ts"}]) sql = query.get_sql() - self.assertIn("TIMESTAMP(`creation`) `ts`", sql) + self.assertIn(convert_identifier_quotes("TIMESTAMP(`creation`) `ts`"), sql) # Test mixed regular fields and function fields query = frappe.qb.get_query( @@ -1586,21 +1586,21 @@ class TestQuery(IntegrationTestCase): group_by="user_type", ) sql = query.get_sql() - self.assertIn("`user_type`", sql) - self.assertIn("COUNT(`name`) `total_users`", sql) - self.assertIn("MAX(`creation`) `latest_creation`", sql) + self.assertIn(convert_identifier_quotes("`user_type`"), sql) + self.assertIn(convert_identifier_quotes("COUNT(`name`) `total_users`"), sql) + self.assertIn(convert_identifier_quotes("MAX(`creation`) `latest_creation`"), sql) # Test NOW function with no arguments query = frappe.qb.get_query("User", fields=[{"NOW": None, "as": "current_time"}]) sql = query.get_sql() - self.assertIn("NOW() `current_time`", sql) + self.assertIn(convert_identifier_quotes("NOW() `current_time`"), sql) # Test CONCAT function (which is supported) query = frappe.qb.get_query( "User", fields=[{"CONCAT": ["first_name", "last_name"], "as": "full_name"}] ) sql = query.get_sql() - self.assertIn("CONCAT(`first_name`,`last_name`) `full_name`", sql) + self.assertIn(convert_identifier_quotes("CONCAT(`first_name`,`last_name`) `full_name`"), sql) # Test unsupported function validation with self.assertRaises(frappe.ValidationError) as cm: From d910dbff19eb2a4e4bf9632ad3e77b1d3f86c5ff Mon Sep 17 00:00:00 2001 From: AarDG10 Date: Tue, 4 Nov 2025 21:17:20 +0530 Subject: [PATCH 10/30] test(postgres): fix test_multiple_dynamic_fields_group_order for Postgres --- frappe/tests/test_query.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/tests/test_query.py b/frappe/tests/test_query.py index da84183c60..218998c54c 100644 --- a/frappe/tests/test_query.py +++ b/frappe/tests/test_query.py @@ -1275,7 +1275,7 @@ class TestQuery(IntegrationTestCase): query = frappe.qb.get_query( "DocType", fields=["module", "module.app_name", "name"], - group_by="module, module.app_name", + group_by="module, module.app_name, name", order_by="module.app_name", ) result = query.run(as_dict=True) From de600a42c83446235c70e86fa02a51546e48830c Mon Sep 17 00:00:00 2001 From: AarDG10 Date: Tue, 4 Nov 2025 22:22:14 +0530 Subject: [PATCH 11/30] test(postgres): fix test_dynamic_fields_in_group_by for Postgres --- frappe/tests/test_query.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/tests/test_query.py b/frappe/tests/test_query.py index 218998c54c..2cf933b7a6 100644 --- a/frappe/tests/test_query.py +++ b/frappe/tests/test_query.py @@ -1198,7 +1198,7 @@ class TestQuery(IntegrationTestCase): query = frappe.qb.get_query( "DocType", fields=["module.app_name", "name"], - group_by="module.app_name", + group_by="module.app_name, name", ) result = query.run(as_dict=True) self.assertTrue(len(result) > 0) @@ -1217,7 +1217,7 @@ class TestQuery(IntegrationTestCase): "Note", fields=["seen_by.user", "name"], filters={"name": note.name}, - group_by="seen_by.user", + group_by="seen_by.user, name", ) result = query.run(as_dict=True) self.assertTrue(len(result) >= 1) From 6481d2aebafcc7ae07867f9eae50e4846fef74c0 Mon Sep 17 00:00:00 2001 From: AarDG10 Date: Tue, 4 Nov 2025 23:37:10 +0530 Subject: [PATCH 12/30] test(postgres): fix test_decimal_field_configuration for Postgres --- frappe/core/doctype/doctype/test_doctype.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/frappe/core/doctype/doctype/test_doctype.py b/frappe/core/doctype/doctype/test_doctype.py index 20820fbaa6..5eeb91a790 100644 --- a/frappe/core/doctype/doctype/test_doctype.py +++ b/frappe/core/doctype/doctype/test_doctype.py @@ -840,7 +840,20 @@ class TestDocType(IntegrationTestCase): ], ).insert(ignore_if_duplicate=True) decimal_field_type = frappe.db.get_column_type(doctype.name, "decimal_field") - self.assertIn("(30,3)", decimal_field_type.lower()) + if frappe.db.db_type == "postgres": + result = frappe.db.sql( + """ + SELECT numeric_precision, numeric_scale + FROM information_schema.columns + WHERE lower(table_name) = lower(%s) + AND column_name = %s + """, + (f"tab{doctype.name}", "decimal_field"), + ) + length, precision = result[0] + self.assertEqual((length, precision), (30, 3)) + elif frappe.db.db_type == "mariadb": + self.assertIn("(30,3)", decimal_field_type.lower()) def test_decimal_field_precision_exceeds_length(self): doctype = new_doctype( From c5d76015131695b3f655c61dea46f37578709824 Mon Sep 17 00:00:00 2001 From: AarDG10 Date: Wed, 5 Nov 2025 10:03:22 +0530 Subject: [PATCH 13/30] test(postgres): fix test_dynamic_fields_in_group_by for Postgres --- frappe/tests/classes/integration_test_case.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/tests/classes/integration_test_case.py b/frappe/tests/classes/integration_test_case.py index 07e82a8992..ac3809b137 100644 --- a/frappe/tests/classes/integration_test_case.py +++ b/frappe/tests/classes/integration_test_case.py @@ -118,7 +118,7 @@ class IntegrationTestCase(UnitTestCase): def _sql_with_count(*args, **kwargs): ret = orig_sql(*args, **kwargs) - queries.append(args[0].last_query) + queries.append(str(args[0].last_query)) return ret try: From c52238087d9017ab0c7672d140f36346e1d37bcb Mon Sep 17 00:00:00 2001 From: AarDG10 Date: Wed, 5 Nov 2025 13:06:07 +0530 Subject: [PATCH 14/30] test(postgres): fix test_smtp for Postgres --- frappe/email/test_smtp.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/frappe/email/test_smtp.py b/frappe/email/test_smtp.py index dc5faccdd2..01c55372e1 100644 --- a/frappe/email/test_smtp.py +++ b/frappe/email/test_smtp.py @@ -26,7 +26,8 @@ class TestSMTP(IntegrationTestCase): # remove mail_server config so that test@example.com is not created mail_server = frappe.conf.get("mail_server") - del frappe.conf["mail_server"] + if "mail_server" in frappe.conf: + del frappe.conf["mail_server"] frappe.local.outgoing_email_account = {} @@ -47,9 +48,9 @@ class TestSMTP(IntegrationTestCase): password="password", enable_outgoing=1, default_outgoing=1, - append_to="Todo", + append_to="ToDo", ) - self.assertEqual(EmailAccount.find_outgoing(match_by_doctype="Todo").email_id, "append_to@gmail.com") + self.assertEqual(EmailAccount.find_outgoing(match_by_doctype="ToDo").email_id, "append_to@gmail.com") # add back the mail_server frappe.conf["mail_server"] = mail_server From 0bff72ed17b39bde94b94e068c7e8554651b9501 Mon Sep 17 00:00:00 2001 From: AarDG10 Date: Wed, 5 Nov 2025 13:53:32 +0530 Subject: [PATCH 15/30] test(postgres): fix mariadb specific identifiers for postgres queries --- .../doctype/user_invitation/test_user_invitation.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/frappe/core/doctype/user_invitation/test_user_invitation.py b/frappe/core/doctype/user_invitation/test_user_invitation.py index e3cab914c0..7345cc300d 100644 --- a/frappe/core/doctype/user_invitation/test_user_invitation.py +++ b/frappe/core/doctype/user_invitation/test_user_invitation.py @@ -13,6 +13,7 @@ from frappe.core.api.user_invitation import ( ) from frappe.core.doctype.user_invitation.user_invitation import mark_expired_invitations from frappe.tests import IntegrationTestCase +from frappe.tests.test_query import convert_identifier_quotes emails = [ "test_user_invite1@example.com", @@ -55,15 +56,18 @@ class IntegrationTestUserInvitation(IntegrationTestCase): @classmethod def delete_all_user_roles(cls): - frappe.db.sql("DELETE FROM `tabUser Role`") + frappe.db.sql(convert_identifier_quotes("DELETE FROM `tabUser Role`")) @classmethod def delete_all_invitations(cls): - frappe.db.sql("DELETE FROM `tabUser Invitation`") + frappe.db.sql(convert_identifier_quotes("DELETE FROM `tabUser Invitation`")) @classmethod def delete_invitation(cls, name: str): - frappe.db.sql(f'DELETE FROM `tabUser Invitation` WHERE name = "{name}"') + frappe.db.sql( + convert_identifier_quotes("DELETE FROM `tabUser Invitation` WHERE name = %s"), + name, + ) def setUp(self): super().setUp() From 675be42169c72a6e6d5aa00426104258dde9a786 Mon Sep 17 00:00:00 2001 From: AarDG10 Date: Wed, 5 Nov 2025 14:52:15 +0530 Subject: [PATCH 16/30] ci(postgres): Fix pg_dump version mismatch by upgrading to v18 --- .github/actions/setup/action.yml | 8 +++++++- .github/workflows/_base-server-tests.yml | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 4b63407bb5..fe19ffe68e 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -101,9 +101,15 @@ runs: # Install System Dependencies start_time=$(date +%s) curl -LsS https://r.mariadb.com/downloads/mariadb_repo_setup | sudo bash + + sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' + wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - + sudo apt -qq update sudo apt -qq remove mysql-server mysql-client - sudo apt -qq install libcups2-dev redis-server mariadb-client libmariadb-dev postgresql-client libpq-dev + sudo apt -qq remove -y postgresql-client postgresql-client-16 postgresql-client-common + sudo apt -qq install libcups2-dev redis-server mariadb-client libmariadb-dev postgresql-client-18 libpq-dev + echo "/usr/lib/postgresql/18/bin" >> $GITHUB_PATH wget -q -O /tmp/wkhtmltox.deb https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6.1-2/wkhtmltox_0.12.6.1-2.jammy_amd64.deb sudo apt install /tmp/wkhtmltox.deb diff --git a/.github/workflows/_base-server-tests.yml b/.github/workflows/_base-server-tests.yml index 0bc499f253..1386466912 100644 --- a/.github/workflows/_base-server-tests.yml +++ b/.github/workflows/_base-server-tests.yml @@ -66,7 +66,7 @@ jobs: index: ${{ fromJson(needs.gen-idx-integration.outputs.indices) }} services: postgres: - image: postgres:17.0 + image: postgres:18.0 ports: - 5432:5432 env: From 420c1e7994046e91788513b58c01515f28c8c6cb Mon Sep 17 00:00:00 2001 From: AarDG10 Date: Wed, 5 Nov 2025 16:52:29 +0530 Subject: [PATCH 17/30] test(postgres): update test_is for postgres --- frappe/tests/test_db.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/frappe/tests/test_db.py b/frappe/tests/test_db.py index 2dc6f2cfe6..286c492140 100644 --- a/frappe/tests/test_db.py +++ b/frappe/tests/test_db.py @@ -1050,14 +1050,13 @@ class TestDDLCommandsPost(IntegrationTestCase): def test_is(self): user = frappe.qb.DocType("User") - self.assertIn( - 'coalesce("name",', - frappe.db.get_values(user, filters={user.name: ("is", "set")}, run=False).lower(), - ) - self.assertIn( - 'coalesce("name",', - frappe.db.get_values(user, filters={user.name: ("is", "not set")}, run=False).lower(), - ) + query_is_set = frappe.db.get_values(user, filters={user.name: ("is", "set")}, run=False).lower() + + query_is_not_set = frappe.db.get_values( + user, filters={user.name: ("is", "not set")}, run=False + ).lower() + self.assertIn('"name"<>%(param1)s', query_is_set) + self.assertIn('"name" is null or "name"=%(param1)s', query_is_not_set) @run_only_if(db_type_is.POSTGRES) From a4304124ec54fe394a89927ac1d1c6e6457641d5 Mon Sep 17 00:00:00 2001 From: AarDG10 Date: Wed, 5 Nov 2025 17:37:12 +0530 Subject: [PATCH 18/30] ci(postgres): add flag to enable/disable postgres ci --- .github/workflows/_base-server-tests.yml | 6 +++++- .github/workflows/server-tests.yml | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/_base-server-tests.yml b/.github/workflows/_base-server-tests.yml index 1386466912..d76a380c41 100644 --- a/.github/workflows/_base-server-tests.yml +++ b/.github/workflows/_base-server-tests.yml @@ -22,6 +22,10 @@ on: required: false type: boolean default: false + enable-postgres: + required: false + type: boolean + default: true enable-coverage: required: false type: boolean @@ -62,7 +66,7 @@ jobs: strategy: fail-fast: false matrix: - db: ${{ fromJson(inputs.enable-sqlite && '["mariadb", "postgres", "sqlite"]' || '["mariadb", "postgres"]') }} + db: ${{ fromJson(inputs.enable-sqlite && (inputs.enable-postgres && '["mariadb", "postgres", "sqlite"]' || '["mariadb", "sqlite"]') || (inputs.enable-postgres && '["mariadb", "postgres"]' || '["mariadb"]')) }} index: ${{ fromJson(needs.gen-idx-integration.outputs.indices) }} services: postgres: diff --git a/.github/workflows/server-tests.yml b/.github/workflows/server-tests.yml index 3bcb763984..6ab68f39d3 100644 --- a/.github/workflows/server-tests.yml +++ b/.github/workflows/server-tests.yml @@ -44,6 +44,7 @@ jobs: name: Tests uses: ./.github/workflows/_base-server-tests.yml with: + enable-postgres: true # This enables PostgreSQL to run tests enable-sqlite: false # This will test against both MariaDB and SQLite if enabled parallel-runs: 2 enable-coverage: ${{ github.event_name != 'pull_request' }} From 886b3b33e3787c1f4c86439ad56b2c7f0519a359 Mon Sep 17 00:00:00 2001 From: AarDG10 Date: Thu, 6 Nov 2025 10:54:07 +0530 Subject: [PATCH 19/30] fix(reportview): remove redundant DISTINCT in get_count for postgres --- frappe/desk/reportview.py | 3 +-- frappe/tests/test_db_query.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/frappe/desk/reportview.py b/frappe/desk/reportview.py index c56bcdadb1..760cfdc3f3 100644 --- a/frappe/desk/reportview.py +++ b/frappe/desk/reportview.py @@ -60,9 +60,8 @@ def get_count() -> int | None: return frappe.call(controller.get_count, args=args, **args) args.distinct = sbool(args.distinct) - distinct = "distinct " if args.distinct else "" args.limit = cint(args.limit) - fieldname = f"{distinct}`tab{args.doctype}`.name" + fieldname = f"`tab{args.doctype}`.name" args.order_by = None # args.limit is specified to avoid getting accurate count. diff --git a/frappe/tests/test_db_query.py b/frappe/tests/test_db_query.py index 36cab4f43d..03ef9bfb65 100644 --- a/frappe/tests/test_db_query.py +++ b/frappe/tests/test_db_query.py @@ -1244,7 +1244,6 @@ class TestDBQuery(IntegrationTestCase): class TestReportView(IntegrationTestCase): - @run_only_if(db_type_is.MARIADB) # TODO: postgres name casting is messed up def test_get_count(self): frappe.local.request = frappe._dict() frappe.local.request.method = "GET" @@ -1282,7 +1281,7 @@ class TestReportView(IntegrationTestCase): child_filter_response = execute_cmd("frappe.desk.reportview.get_count") current_value = frappe.db.sql( # the below query is equivalent to the one in reportview.get_count - "select distinct count(distinct `tabDocType`.name) as total_count" + "select count(distinct `tabDocType`.name) as total_count" " from `tabDocType` left join `tabDocField`" " on (`tabDocField`.parenttype = 'DocType' and `tabDocField`.parent = `tabDocType`.name)" " where `tabDocField`.`fieldtype` = 'Data'" From e1b142614797851e6cc773587ba71a2c7b4ed299 Mon Sep 17 00:00:00 2001 From: AarDG10 Date: Thu, 6 Nov 2025 19:08:22 +0530 Subject: [PATCH 20/30] test(postgres): enable test_backup_no_options for postgres --- frappe/commands/test_commands.py | 1 - 1 file changed, 1 deletion(-) diff --git a/frappe/commands/test_commands.py b/frappe/commands/test_commands.py index cde28456ca..94228e1a80 100644 --- a/frappe/commands/test_commands.py +++ b/frappe/commands/test_commands.py @@ -656,7 +656,6 @@ class TestBackups(BaseTestCommands): except OSError: pass - @run_only_if(db_type_is.MARIADB) def test_backup_no_options(self): """Take a backup without any options""" before_backup = fetch_latest_backups(partial=True) From 907f3de87231887a5f71e6ef5e7b0c8bc78087ec Mon Sep 17 00:00:00 2001 From: AarDG10 Date: Fri, 7 Nov 2025 12:37:55 +0530 Subject: [PATCH 21/30] test(postgres): enable test_varchar_length for postgres --- frappe/tests/test_db_update.py | 1 - 1 file changed, 1 deletion(-) diff --git a/frappe/tests/test_db_update.py b/frappe/tests/test_db_update.py index ec2cb423b7..74afa9cc32 100644 --- a/frappe/tests/test_db_update.py +++ b/frappe/tests/test_db_update.py @@ -176,7 +176,6 @@ class TestDBUpdate(IntegrationTestCase): self.assertEqual(frappe.db.get_column_type(referring_doctype.name, link), "uuid") - @run_only_if(db_type_is.MARIADB) def test_varchar_length(self): from frappe.database.schema import add_column From a9c17f0e841d23e4d2f5af52bb4cd84e5fcdfb60 Mon Sep 17 00:00:00 2001 From: AarDG10 Date: Fri, 7 Nov 2025 12:39:35 +0530 Subject: [PATCH 22/30] test(postgres): add delay in test_backup_no_options to avoid same-second timestamp collision --- frappe/commands/test_commands.py | 1 + 1 file changed, 1 insertion(+) diff --git a/frappe/commands/test_commands.py b/frappe/commands/test_commands.py index 94228e1a80..002cbfe6bc 100644 --- a/frappe/commands/test_commands.py +++ b/frappe/commands/test_commands.py @@ -659,6 +659,7 @@ class TestBackups(BaseTestCommands): def test_backup_no_options(self): """Take a backup without any options""" before_backup = fetch_latest_backups(partial=True) + time.sleep(1) self.execute("bench --site {site} backup") after_backup = fetch_latest_backups(partial=True) From 1ffbe4eaad79a3a45c9477b6ba820a77ef29e34d Mon Sep 17 00:00:00 2001 From: AarDG10 Date: Fri, 7 Nov 2025 15:52:21 +0530 Subject: [PATCH 23/30] test(postgres): add pg-compatible query for unique index check --- frappe/tests/test_db_update.py | 69 ++++++++++++++++++++++------------ 1 file changed, 44 insertions(+), 25 deletions(-) diff --git a/frappe/tests/test_db_update.py b/frappe/tests/test_db_update.py index 74afa9cc32..a6e483e29e 100644 --- a/frappe/tests/test_db_update.py +++ b/frappe/tests/test_db_update.py @@ -93,10 +93,29 @@ class TestDBUpdate(IntegrationTestCase): self.assertEqual(email_sig_column.index, 1) def check_unique_indexes(self, doctype: str, field: str): - indexes = frappe.db.sql( - f"""show index from `tab{doctype}` where column_name = '{field}' and Non_unique = 0""", - as_dict=1, - ) + if frappe.db.db_type == "postgres": + """Check if the column has a unique index (PostgreSQL equivalent of "SHOW INDEX ... WHERE Non_unique = 0")""" + indexes = frappe.db.sql( + """ + SELECT i.relname AS index_name, a.attname AS column_name + FROM + pg_class t + JOIN pg_index ix ON t.oid = ix.indrelid + JOIN pg_class i ON i.oid = ix.indexrelid + JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = ANY(ix.indkey) + WHERE + t.relname = %s + AND a.attname = %s + AND ix.indisunique = true + """, + (f"tab{doctype}", field), + as_dict=1, + ) + elif frappe.db.db_type == "mariadb": + indexes = frappe.db.sql( + f"""show index from `tab{doctype}` where column_name = '{field}' and Non_unique = 0""", + as_dict=1, + ) self.assertEqual( len(indexes), 1, msg=f"There should be 1 index on {doctype}.{field}, found {indexes}" ) @@ -111,7 +130,6 @@ class TestDBUpdate(IntegrationTestCase): doctype.save() frappe.get_doc(doctype=doctype.name, int_field=2**62 - 1).insert() - @run_only_if(db_type_is.MARIADB) def test_unique_index_on_install(self): """Only one unique index should be added""" for dt in frappe.get_all("DocType", {"is_virtual": 0, "issingle": 0}, pluck="name"): @@ -126,30 +144,31 @@ class TestDBUpdate(IntegrationTestCase): """Only one unique index should be added""" doctype = new_doctype(unique=1).insert() - field = "some_fieldname" + try: + field = "some_fieldname" - self.check_unique_indexes(doctype.name, field) - doctype.fields[0].length = 142 - doctype.save() - self.check_unique_indexes(doctype.name, field) + self.check_unique_indexes(doctype.name, field) + doctype.fields[0].length = 142 + doctype.save() + self.check_unique_indexes(doctype.name, field) - doctype.fields[0].unique = 0 - doctype.save() + doctype.fields[0].unique = 0 + doctype.save() - doctype.fields[0].unique = 1 - doctype.save() - self.check_unique_indexes(doctype.name, field) + doctype.fields[0].unique = 1 + doctype.save() + self.check_unique_indexes(doctype.name, field) - # New column with a unique index - # This works because index name is same as fieldname. - new_field = frappe.copy_doc(doctype.fields[0]) - new_field.fieldname = "duplicate_field" - doctype.append("fields", new_field) - doctype.save() - self.check_unique_indexes(doctype.name, new_field.fieldname) - - doctype.delete() - frappe.db.commit() + # New column with a unique index + # This works because index name is same as fieldname. + new_field = frappe.copy_doc(doctype.fields[0]) + new_field.fieldname = "duplicate_field" + doctype.append("fields", new_field) + doctype.save() + self.check_unique_indexes(doctype.name, new_field.fieldname) + finally: + doctype.delete() + frappe.db.commit() def test_uuid_varchar_migration(self): doctype = new_doctype().insert() From 27eca4f74c0f1228197f6d1512d5262ef4c1c4d5 Mon Sep 17 00:00:00 2001 From: AarDG10 Date: Fri, 7 Nov 2025 18:17:06 +0530 Subject: [PATCH 24/30] test(postgres): enable test_store_attachments for postgres --- frappe/tests/test_email.py | 1 - 1 file changed, 1 deletion(-) diff --git a/frappe/tests/test_email.py b/frappe/tests/test_email.py index f0c1a8dd33..392316055c 100644 --- a/frappe/tests/test_email.py +++ b/frappe/tests/test_email.py @@ -371,7 +371,6 @@ class TestEmailIntegrationTest(IntegrationTestCase): self.assertEqual(sent_mail["subject"], subject) self.assertSetEqual(set(recipients.split(",")), {m["to"][0] for m in sent_mails}) - @run_only_if(db_type_is.MARIADB) @IntegrationTestCase.change_settings("System Settings", store_attached_pdf_document=1) def test_store_attachments(self): """ "attach print" feature just tells email queue which document to attach, this is not From 2feb536ec912b6dab33566ee696b9c6403e2f537 Mon Sep 17 00:00:00 2001 From: AarDG10 Date: Sat, 8 Nov 2025 11:46:57 +0530 Subject: [PATCH 25/30] test(postgres): add pg-compatible query for test_db_cli_with_sql --- frappe/commands/test_commands.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/frappe/commands/test_commands.py b/frappe/commands/test_commands.py index 002cbfe6bc..83b5f88b1c 100644 --- a/frappe/commands/test_commands.py +++ b/frappe/commands/test_commands.py @@ -1003,9 +1003,11 @@ class TestDBCli(BaseTestCommands): self.execute("bench --site {site} db-console", kwargs={"cmd_input": cmd_input}) self.assertEqual(self.returncode, 0) - @run_only_if(db_type_is.MARIADB) def test_db_cli_with_sql(self): - self.execute("bench --site {site} db-console -e 'select 1'") + if frappe.db.db_type == "postgres": + self.execute("bench --site {site} db-console -c 'select 1'") + elif frappe.db.db_type == "mariadb": + self.execute("bench --site {site} db-console -e 'select 1'") self.assertEqual(self.returncode, 0) self.assertIn("1", self.stdout) From 103841bd0c856d3c5c12a4c9cb2f738be9c6b796 Mon Sep 17 00:00:00 2001 From: AarDG10 Date: Sun, 9 Nov 2025 18:30:49 +0530 Subject: [PATCH 26/30] test(postgres): enable test_bench_drop_site_should_archive_site for postgres --- frappe/commands/test_commands.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/frappe/commands/test_commands.py b/frappe/commands/test_commands.py index 83b5f88b1c..4b94a2e2fa 100644 --- a/frappe/commands/test_commands.py +++ b/frappe/commands/test_commands.py @@ -469,11 +469,10 @@ class TestCommands(BaseTestCommands): self.assertEqual(check_password("Administrator", original_password), "Administrator") @skipIf( - not (frappe.conf.root_password and frappe.conf.admin_password and frappe.conf.db_type == "mariadb"), + not (frappe.conf.root_password and frappe.conf.admin_password and frappe.conf.db_type != "sqlite"), "DB Root password and Admin password not set in config", ) def test_bench_drop_site_should_archive_site(self): - # TODO: Make this test postgres compatible site = TEST_SITE self.execute( From b8793fac5b23e8fc33330bfdf3c6ef9c19d026c8 Mon Sep 17 00:00:00 2001 From: AarDG10 Date: Sun, 9 Nov 2025 18:46:18 +0530 Subject: [PATCH 27/30] test(postgres): enable test_force_install_app for postgres --- frappe/commands/test_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/commands/test_commands.py b/frappe/commands/test_commands.py index 4b94a2e2fa..ad00bb0933 100644 --- a/frappe/commands/test_commands.py +++ b/frappe/commands/test_commands.py @@ -498,7 +498,7 @@ class TestCommands(BaseTestCommands): self.assertTrue(os.path.exists(archive_directory)) @skipIf( - not (frappe.conf.root_password and frappe.conf.admin_password and frappe.conf.db_type == "mariadb"), + not (frappe.conf.root_password and frappe.conf.admin_password and frappe.conf.db_type != "sqlite"), "DB Root password and Admin password not set in config", ) def test_force_install_app(self): From a991bad767bc6f0f6e696ac41c92af2104002207 Mon Sep 17 00:00:00 2001 From: AarDG10 Date: Mon, 10 Nov 2025 13:15:34 +0530 Subject: [PATCH 28/30] ci: run postgres tests conditionally with postgres label --- .github/workflows/server-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/server-tests.yml b/.github/workflows/server-tests.yml index 6ab68f39d3..311616e4ee 100644 --- a/.github/workflows/server-tests.yml +++ b/.github/workflows/server-tests.yml @@ -44,7 +44,7 @@ jobs: name: Tests uses: ./.github/workflows/_base-server-tests.yml with: - enable-postgres: true # This enables PostgreSQL to run tests + enable-postgres: ${{ contains(github.event.pull_request.labels.*.name, 'postgres') }} # This enables PostgreSQL to run tests enable-sqlite: false # This will test against both MariaDB and SQLite if enabled parallel-runs: 2 enable-coverage: ${{ github.event_name != 'pull_request' }} From a12d855147e39a3667b369972b941bd9d7322a79 Mon Sep 17 00:00:00 2001 From: AarDG10 Date: Mon, 10 Nov 2025 17:46:24 +0530 Subject: [PATCH 29/30] test(postgres): fix mariadb specific identifiers for postgres queries using normalize_sql --- .../user_invitation/test_user_invitation.py | 13 ++- frappe/tests/classes/unit_test_case.py | 2 + frappe/tests/test_query.py | 99 +++++++------------ 3 files changed, 46 insertions(+), 68 deletions(-) diff --git a/frappe/core/doctype/user_invitation/test_user_invitation.py b/frappe/core/doctype/user_invitation/test_user_invitation.py index 7345cc300d..634c739d8e 100644 --- a/frappe/core/doctype/user_invitation/test_user_invitation.py +++ b/frappe/core/doctype/user_invitation/test_user_invitation.py @@ -13,7 +13,6 @@ from frappe.core.api.user_invitation import ( ) from frappe.core.doctype.user_invitation.user_invitation import mark_expired_invitations from frappe.tests import IntegrationTestCase -from frappe.tests.test_query import convert_identifier_quotes emails = [ "test_user_invite1@example.com", @@ -56,18 +55,18 @@ class IntegrationTestUserInvitation(IntegrationTestCase): @classmethod def delete_all_user_roles(cls): - frappe.db.sql(convert_identifier_quotes("DELETE FROM `tabUser Role`")) + query = "DELETE FROM `tabUser Role`" + frappe.db.sql(cls.normalize_sql(query)) @classmethod def delete_all_invitations(cls): - frappe.db.sql(convert_identifier_quotes("DELETE FROM `tabUser Invitation`")) + query = "DELETE FROM `tabUser Invitation`" + frappe.db.sql(cls.normalize_sql(query)) @classmethod def delete_invitation(cls, name: str): - frappe.db.sql( - convert_identifier_quotes("DELETE FROM `tabUser Invitation` WHERE name = %s"), - name, - ) + query = "DELETE FROM `tabUser Invitation` WHERE name = %s" + frappe.db.sql(cls.normalize_sql(query), name) def setUp(self): super().setUp() diff --git a/frappe/tests/classes/unit_test_case.py b/frappe/tests/classes/unit_test_case.py index 45b8e56963..65aa4e616d 100644 --- a/frappe/tests/classes/unit_test_case.py +++ b/frappe/tests/classes/unit_test_case.py @@ -105,6 +105,8 @@ class UnitTestCase(unittest.TestCase, BaseTestCase): """Formats SQL consistently so simple string comparisons can work on them.""" import sqlparse + if frappe.db.db_type == "postgres": + query = query.replace("`", '"') return sqlparse.format(query.strip(), keyword_case="upper", reindent=True, strip_comments=True) diff --git a/frappe/tests/test_query.py b/frappe/tests/test_query.py index 2cf933b7a6..d99bc68b1d 100644 --- a/frappe/tests/test_query.py +++ b/frappe/tests/test_query.py @@ -63,20 +63,12 @@ def create_tree_docs(): d.insert() -def convert_identifier_quotes(query): - """util to replace mariadb query idenitfifiers with postgres ones""" - return query.replace("`", '"') if frappe.db.db_type == "postgres" else query - - class TestQuery(IntegrationTestCase): def setUp(self): setup_for_tests() def test_multiple_tables_in_filters(self): - expected_query = convert_identifier_quotes( - "SELECT `tabDocType`.* FROM `tabDocType` LEFT JOIN `tabDocField` ON `tabDocField`.`parent`=`tabDocType`.`name` AND `tabDocField`.`parenttype`='DocType' AND `tabDocField`.`parentfield`='fields' WHERE `tabDocField`.`name` LIKE 'f%' AND `tabDocType`.`parent`='something'" - ) - self.assertEqual( + self.assertQueryEqual( frappe.qb.get_query( "DocType", ["*"], @@ -85,7 +77,7 @@ class TestQuery(IntegrationTestCase): ["DocType", "parent", "=", "something"], ], ).get_sql(), - expected_query, + "SELECT `tabDocType`.* FROM `tabDocType` LEFT JOIN `tabDocField` ON `tabDocField`.`parent`=`tabDocType`.`name` AND `tabDocField`.`parenttype`='DocType' AND `tabDocField`.`parentfield`='fields' WHERE `tabDocField`.`name` LIKE 'f%' AND `tabDocType`.`parent`='something'", ) def test_string_fields(self): @@ -359,88 +351,72 @@ class TestQuery(IntegrationTestCase): ) def test_filters(self): - expected_query = convert_identifier_quotes( - "SELECT `tabDocType`.`name` FROM `tabDocType` LEFT JOIN `tabModule Def` ON `tabModule Def`.`name`=`tabDocType`.`module` WHERE `tabModule Def`.`app_name`='frappe'" - ) - self.assertEqual( + self.assertQueryEqual( frappe.qb.get_query( "DocType", fields=["name"], filters={"module.app_name": "frappe"}, ).get_sql(), - expected_query, + "SELECT `tabDocType`.`name` FROM `tabDocType` LEFT JOIN `tabModule Def` ON `tabModule Def`.`name`=`tabDocType`.`module` WHERE `tabModule Def`.`app_name`='frappe'", ) - expected_query = convert_identifier_quotes( - "SELECT `tabDocType`.`name` FROM `tabDocType` LEFT JOIN `tabModule Def` ON `tabModule Def`.`name`=`tabDocType`.`module` WHERE `tabModule Def`.`app_name` LIKE 'frap%'" - ) - self.assertEqual( + self.assertQueryEqual( frappe.qb.get_query( "DocType", fields=["name"], filters={"module.app_name": ("like", "frap%")}, ).get_sql(), - expected_query, + "SELECT `tabDocType`.`name` FROM `tabDocType` LEFT JOIN `tabModule Def` ON `tabModule Def`.`name`=`tabDocType`.`module` WHERE `tabModule Def`.`app_name` LIKE 'frap%'", ) - expected_query = convert_identifier_quotes( - "SELECT `tabDocType`.`name` FROM `tabDocType` LEFT JOIN `tabDocPerm` ON `tabDocPerm`.`parent`=`tabDocType`.`name` AND `tabDocPerm`.`parenttype`='DocType' AND `tabDocPerm`.`parentfield`='permissions' WHERE `tabDocPerm`.`role`='System Manager'" - ) - self.assertEqual( + self.assertQueryEqual( frappe.qb.get_query( "DocType", fields=["name"], filters={"permissions.role": "System Manager"}, ).get_sql(), - expected_query, + "SELECT `tabDocType`.`name` FROM `tabDocType` LEFT JOIN `tabDocPerm` ON `tabDocPerm`.`parent`=`tabDocType`.`name` AND `tabDocPerm`.`parenttype`='DocType' AND `tabDocPerm`.`parentfield`='permissions' WHERE `tabDocPerm`.`role`='System Manager'", ) - expected_query = convert_identifier_quotes("SELECT `module` FROM `tabDocType` WHERE `name`=''") - self.assertEqual( + self.assertQueryEqual( frappe.qb.get_query( "DocType", fields=["module"], filters="", ).get_sql(), - expected_query, + "SELECT `module` FROM `tabDocType` WHERE `name`=''", ) - expected_query = convert_identifier_quotes( - "SELECT `name` FROM `tabDocType` WHERE `name` IN ('ToDo','Note')" - ) - self.assertEqual( + self.assertQueryEqual( frappe.qb.get_query( "DocType", filters=["ToDo", "Note"], ).get_sql(), - expected_query, + "SELECT `name` FROM `tabDocType` WHERE `name` IN ('ToDo','Note')", ) - expected_query = convert_identifier_quotes("SELECT `name` FROM `tabDocType` WHERE `name` IN ('')") - self.assertEqual( + self.assertQueryEqual( frappe.qb.get_query( "DocType", filters={"name": ("in", [])}, ).get_sql(), - expected_query, + "SELECT `name` FROM `tabDocType` WHERE `name` IN ('')", ) - expected_query = convert_identifier_quotes("SELECT `name` FROM `tabDocType` WHERE `name` IN (1,2,3)") - self.assertEqual( + self.assertQueryEqual( frappe.qb.get_query( "DocType", filters=[1, 2, 3], ).get_sql(), - expected_query, + "SELECT `name` FROM `tabDocType` WHERE `name` IN (1,2,3)", ) - expected_query = convert_identifier_quotes("SELECT `name` FROM `tabDocType`") - self.assertEqual( + self.assertQueryEqual( frappe.qb.get_query( "DocType", filters=[], ).get_sql(), - expected_query, + "SELECT `name` FROM `tabDocType`", ) def test_nested_filters(self): @@ -1520,7 +1496,7 @@ class TestQuery(IntegrationTestCase): # Test simple function without alias query = frappe.qb.get_query("User", fields=["user_type", {"COUNT": "name"}], group_by="user_type") sql = query.get_sql() - self.assertIn(convert_identifier_quotes("COUNT(`name`)"), sql) + self.assertIn(self.normalize_sql("COUNT(`name`)"), sql) self.assertIn("GROUP BY", sql) # Test function with alias @@ -1528,52 +1504,54 @@ class TestQuery(IntegrationTestCase): "User", fields=[{"COUNT": "name", "as": "total_users"}], group_by="user_type" ) sql = query.get_sql() - self.assertIn(convert_identifier_quotes("COUNT(`name`) `total_users`"), sql) + self.assertIn(self.normalize_sql("COUNT(`name`) `total_users`"), sql) # Test SUM function with alias query = frappe.qb.get_query( "User", fields=[{"SUM": "enabled", "as": "total_enabled"}], group_by="user_type" ) sql = query.get_sql() - self.assertIn(convert_identifier_quotes("SUM(`enabled`) `total_enabled`"), sql) + self.assertIn(self.normalize_sql("SUM(`enabled`) `total_enabled`"), sql) # Test MAX function query = frappe.qb.get_query( "User", fields=[{"MAX": "creation", "as": "latest_user"}], group_by="user_type" ) sql = query.get_sql() - self.assertIn(convert_identifier_quotes("MAX(`creation`) `latest_user`"), sql) + self.assertIn(self.normalize_sql("MAX(`creation`) `latest_user`"), sql) # Test MIN function query = frappe.qb.get_query( "User", fields=[{"MIN": "creation", "as": "earliest_user"}], group_by="user_type" ) sql = query.get_sql() - self.assertIn(convert_identifier_quotes("MIN(`creation`) `earliest_user`"), sql) + self.assertIn(self.normalize_sql("MIN(`creation`) `earliest_user`"), sql) # Test AVG function query = frappe.qb.get_query( "User", fields=[{"AVG": "enabled", "as": "avg_enabled"}], group_by="user_type" ) sql = query.get_sql() - self.assertIn(convert_identifier_quotes("AVG(`enabled`) `avg_enabled`"), sql) + self.assertIn(self.normalize_sql("AVG(`enabled`) `avg_enabled`"), sql) # Test ABS function query = frappe.qb.get_query("User", fields=[{"ABS": "enabled", "as": "abs_enabled"}]) sql = query.get_sql() - self.assertIn(convert_identifier_quotes("ABS(`enabled`) `abs_enabled`"), sql) + self.assertIn(self.normalize_sql("ABS(`enabled`) `abs_enabled`"), sql) # Test IFNULL function with two parameters query = frappe.qb.get_query( "User", fields=[{"IFNULL": ["first_name", "'Unknown'"], "as": "safe_name"}] ) sql = query.get_sql() - self.assertIn(convert_identifier_quotes("IFNULL(`first_name`,'Unknown') `safe_name`"), sql) + self.assertIn( + self.normalize_sql("IFNULL(`first_name`,'Unknown') `safe_name`"), self.normalize_sql(sql) + ) # Test TIMESTAMP function query = frappe.qb.get_query("User", fields=[{"TIMESTAMP": "creation", "as": "ts"}]) sql = query.get_sql() - self.assertIn(convert_identifier_quotes("TIMESTAMP(`creation`) `ts`"), sql) + self.assertIn(self.normalize_sql("TIMESTAMP(`creation`) `ts`"), self.normalize_sql(sql)) # Test mixed regular fields and function fields query = frappe.qb.get_query( @@ -1586,21 +1564,23 @@ class TestQuery(IntegrationTestCase): group_by="user_type", ) sql = query.get_sql() - self.assertIn(convert_identifier_quotes("`user_type`"), sql) - self.assertIn(convert_identifier_quotes("COUNT(`name`) `total_users`"), sql) - self.assertIn(convert_identifier_quotes("MAX(`creation`) `latest_creation`"), sql) + self.assertIn(self.normalize_sql("`user_type`"), sql) + self.assertIn(self.normalize_sql("COUNT(`name`) `total_users`"), sql) + self.assertIn(self.normalize_sql("MAX(`creation`) `latest_creation`"), sql) # Test NOW function with no arguments query = frappe.qb.get_query("User", fields=[{"NOW": None, "as": "current_time"}]) sql = query.get_sql() - self.assertIn(convert_identifier_quotes("NOW() `current_time`"), sql) + self.assertIn(self.normalize_sql("NOW() `current_time`"), sql) # Test CONCAT function (which is supported) query = frappe.qb.get_query( "User", fields=[{"CONCAT": ["first_name", "last_name"], "as": "full_name"}] ) sql = query.get_sql() - self.assertIn(convert_identifier_quotes("CONCAT(`first_name`,`last_name`) `full_name`"), sql) + self.assertIn( + self.normalize_sql("CONCAT(`first_name`,`last_name`) `full_name`"), self.normalize_sql(sql) + ) # Test unsupported function validation with self.assertRaises(frappe.ValidationError) as cm: @@ -1618,10 +1598,7 @@ class TestQuery(IntegrationTestCase): self.assertIn("Unsupported function or invalid field name: DROP", str(cm.exception)) def test_not_equal_condition_on_none(self): - expected_query = convert_identifier_quotes( - "SELECT `tabDocType`.* FROM `tabDocType` LEFT JOIN `tabDocField` ON `tabDocField`.`parent`=`tabDocType`.`name` AND `tabDocField`.`parenttype`='DocType' AND `tabDocField`.`parentfield`='fields' WHERE `tabDocField`.`name` IS NULL AND `tabDocType`.`parent` IS NOT NULL" - ) - self.assertEqual( + self.assertQueryEqual( frappe.qb.get_query( "DocType", ["*"], @@ -1630,7 +1607,7 @@ class TestQuery(IntegrationTestCase): ["DocType", "parent", "!=", None], ], ).get_sql(), - expected_query, + "SELECT `tabDocType`.* FROM `tabDocType` LEFT JOIN `tabDocField` ON `tabDocField`.`parent`=`tabDocType`.`name` AND `tabDocField`.`parenttype`='DocType' AND `tabDocField`.`parentfield`='fields' WHERE `tabDocField`.`name` IS NULL AND `tabDocType`.`parent` IS NOT NULL", ) From 0ca36db9b929c02419daf5a93c754170f52da881 Mon Sep 17 00:00:00 2001 From: AarDG10 Date: Tue, 11 Nov 2025 13:18:42 +0530 Subject: [PATCH 30/30] fix(reportview): remove redundant DISTINCT args in get_count for postgres --- frappe/desk/reportview.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frappe/desk/reportview.py b/frappe/desk/reportview.py index 760cfdc3f3..97fa71b271 100644 --- a/frappe/desk/reportview.py +++ b/frappe/desk/reportview.py @@ -60,8 +60,10 @@ def get_count() -> int | None: return frappe.call(controller.get_count, args=args, **args) args.distinct = sbool(args.distinct) + distinct = "distinct " if args.distinct else "" args.limit = cint(args.limit) - fieldname = f"`tab{args.doctype}`.name" + fieldname = f"{distinct}`tab{args.doctype}`.name" + args.pop("distinct") # to avoid a double DISTINCT concat in db_query args.order_by = None # args.limit is specified to avoid getting accurate count.