perf(install): bulk insert country and currencies (#19084)

frappe/geo/doctype/country/country.py

@@ -1,8 +1,59 @@
 # Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
 
-from frappe.model.document import Document
+import frappe
+from frappe.model.document import Document, bulk_insert
 
 
 class Country(Document):
+	# NOTE: During installation country docs are bulk inserted.
 	pass
+
+
+def import_country_and_currency():
+	from frappe.geo.doctype.currency.currency import enable_default_currencies
+
+	countries, currencies = get_countries_and_currencies()
+
+	bulk_insert("Country", countries, ignore_duplicates=True)
+	bulk_insert("Currency", currencies, ignore_duplicates=True)
+
+	enable_default_currencies()
+
+
+def get_countries_and_currencies():
+	from frappe.geo.country_info import get_all as get_geo_data
+
+	data = get_geo_data()
+
+	countries = []
+	currencies = []
+
+	for name, country in data.items():
+		country = frappe._dict(country)
+		countries.append(
+			frappe.get_doc(
+				doctype="Country",
+				name=name,
+				country_name=name,
+				code=country.code,
+				date_format=country.date_format or "dd-mm-yyyy",
+				time_format=country.time_format or "HH:mm:ss",
+				time_zones="\n".join(country.timezones or []),
+			)
+		)
+		if country.currency:
+			currencies.append(
+				frappe.get_doc(
+					doctype="Currency",
+					name=country.currency,
+					currency_name=country.currency,
+					fraction=country.currency_fraction,
+					symbol=country.currency_symbol,
+					fraction_units=country.currency_fraction_units,
+					smallest_currency_fraction_value=country.smallest_currency_fraction_value,
+					number_format=country.number_format,
+				)
+			)
+
+	return countries, currencies

frappe/geo/doctype/country/test_country.py

@@ -2,5 +2,52 @@
 # License: MIT. See LICENSE
 
 import frappe
+from frappe.geo.doctype.country.country import (
+	get_countries_and_currencies,
+	import_country_and_currency,
+)
+from frappe.geo.doctype.currency.currency import enable_default_currencies
+from frappe.tests.utils import FrappeTestCase
 
 test_records = frappe.get_test_records("Country")
+
+
+def get_table_snapshot(doctype):
+	data = frappe.db.sql(f"select * from `tab{doctype}` order by name", as_dict=True)
+
+	inconsequential_keys = ["modified", "creation"]
+	for row in data:
+		for key in inconsequential_keys:
+			row.pop(key, None)
+	return data
+
+
+class TestCountry(FrappeTestCase):
+	def test_bulk_insert_correctness(self):
+		def clear_tables():
+			frappe.db.delete("Currency")
+			frappe.db.delete("Country")
+
+		# Clear data
+		clear_tables()
+
+		# Reimport and verify same results
+		import_country_and_currency()
+
+		countries_before = get_table_snapshot("Country")
+		currencies_before = get_table_snapshot("Currency")
+
+		clear_tables()
+
+		countries, currencies = get_countries_and_currencies()
+		for country in countries:
+			country.db_insert(ignore_if_duplicate=True)
+		for currency in currencies:
+			currency.db_insert(ignore_if_duplicate=True)
+		enable_default_currencies()
+
+		countries_after = get_table_snapshot("Country")
+		currencies_after = get_table_snapshot("Currency")
+
+		self.assertEqual(countries_before, countries_after)
+		self.assertEqual(currencies_before, currencies_after)

frappe/geo/doctype/currency/currency.py

@@ -4,8 +4,14 @@
 import frappe
 from frappe.model.document import Document
 
+DEFAULT_ENABLED_CURRENCIES = ("INR", "USD", "GBP", "EUR", "AED", "AUD", "JPY", "CNY", "CHF")
+
 
 class Currency(Document):
+	# NOTE: During installation country docs are bulk inserted.
 	def validate(self):
-		if not frappe.flags.in_install_app:
-			frappe.clear_cache()
+		frappe.clear_cache()
+
+
+def enable_default_currencies():
+	frappe.db.set_value("Currency", {"name": ("in", DEFAULT_ENABLED_CURRENCIES)}, "enabled", 1)

frappe/tests/utils.py

@@ -22,6 +22,7 @@ class FrappeTestCase(unittest.TestCase):
 	TEST_SITE = "test_site"
 
 	SHOW_TRANSACTION_COMMIT_WARNINGS = False
+	maxDiff = None  # prints long diffs but useful in CI
 
 	@classmethod
 	def setUpClass(cls) -> None:

frappe/utils/install.py

@@ -3,6 +3,7 @@
 import getpass
 
 import frappe
+from frappe.geo.doctype.country.country import import_country_and_currency
 from frappe.utils.password import update_password
 
 
@@ -22,13 +23,10 @@ def after_install():
 	install_basic_docs()
 
 	from frappe.core.doctype.file.utils import make_home_folder
-
-	make_home_folder()
-
-	import_country_and_currency()
-
 	from frappe.core.doctype.language.language import sync_languages
 
+	make_home_folder()
+	import_country_and_currency()
 	sync_languages()
 
 	# save default print setting
@@ -191,53 +189,6 @@ def complete_setup_wizard():
 	)
 
 
-def import_country_and_currency():
-	from frappe.geo.country_info import get_all
-	from frappe.utils import update_progress_bar
-
-	data = get_all()
-
-	for i, name in enumerate(data):
-		update_progress_bar("Updating country info", i, len(data))
-		country = frappe._dict(data[name])
-		add_country_and_currency(name, country)
-
-	print("")
-
-	# enable frequently used currencies
-	for currency in ("INR", "USD", "GBP", "EUR", "AED", "AUD", "JPY", "CNY", "CHF"):
-		frappe.db.set_value("Currency", currency, "enabled", 1)
-
-
-def add_country_and_currency(name, country):
-	if not frappe.db.exists("Country", name):
-		frappe.get_doc(
-			{
-				"doctype": "Country",
-				"country_name": name,
-				"code": country.code,
-				"date_format": country.date_format or "dd-mm-yyyy",
-				"time_format": country.time_format or "HH:mm:ss",
-				"time_zones": "\n".join(country.timezones or []),
-				"docstatus": 0,
-			}
-		).db_insert()
-
-	if country.currency and not frappe.db.exists("Currency", country.currency):
-		frappe.get_doc(
-			{
-				"doctype": "Currency",
-				"currency_name": country.currency,
-				"fraction": country.currency_fraction,
-				"symbol": country.currency_symbol,
-				"fraction_units": country.currency_fraction_units,
-				"smallest_currency_fraction_value": country.smallest_currency_fraction_value,
-				"number_format": country.number_format,
-				"docstatus": 0,
-			}
-		).db_insert()
-
-
 def add_standard_navbar_items():
 	navbar_settings = frappe.get_single("Navbar Settings")