refactor!: Change internal datastructure of db.value_cache

It's now a defaultdictionary of `[doctype][name/filters][fieldname]`

This allows us to implement granular clearing and improve usage of this cache.

frappe/database/database.py

@@ -34,7 +34,16 @@ from frappe.exceptions import DoesNotExistError, ImplicitCommitError
 from frappe.monitor import get_trace_id
 from frappe.query_builder import Case
 from frappe.query_builder.functions import Count
-from frappe.utils import CallbackManager, cint, get_datetime, get_table_name, getdate, now, sbool
+from frappe.utils import (
+	CallbackManager,
+	cint,
+	get_datetime,
+	get_table_name,
+	getdate,
+	now,
+	recursive_defaultdict,
+	sbool,
+)
 from frappe.utils import cast as cast_fieldtype
 
 if TYPE_CHECKING:
@@ -112,7 +121,7 @@ class Database:
 		self.transaction_writes = 0
 		self.auto_commit_on_many_writes = 0
 
-		self.value_cache = {}
+		self.value_cache = recursive_defaultdict()
 		self.logger = frappe.logger("database")
 		self.logger.setLevel("WARNING")
 
@@ -615,11 +624,8 @@ class Database:
 		        user = frappe.db.get_values("User", "test@example.com", "*")[0]
 		"""
 		out = None
-		cache_key = None
+		if cache and isinstance(filters, str) and fieldname in self.value_cache[doctype][filters]:
-		if cache and isinstance(filters, str):
+			return self.value_cache[doctype][filters][fieldname]
-			cache_key = (doctype, filters, fieldname)
-			if cache_key in self.value_cache:
-				return self.value_cache[cache_key]
 
 		if distinct:
 			order_by = None
@@ -701,8 +707,8 @@ class Database:
 					distinct=distinct,
 				)
 
-		if cache and cache_key:
+		if cache and isinstance(filters, str):
-			self.value_cache[cache_key] = out
+			self.value_cache[doctype][filters][fieldname] = out
 
 		return out
 
@@ -858,8 +864,7 @@ class Database:
 		frappe.qb.into("Singles").columns("doctype", "field", "value").insert(*singles_data).run(debug=debug)
 		frappe.clear_document_cache(doctype, doctype)
 
-		if doctype in self.value_cache:
+		self.value_cache.pop(doctype, None)
-			del self.value_cache[doctype]
 
 	def get_single_value(self, doctype: str, fieldname: str, cache: bool = True):
 		"""Get property of Single DocType. Cache locally by default
@@ -872,10 +877,6 @@ class Database:
 		        # Get the default value of the company from the Global Defaults doctype.
 		        company = frappe.db.get_single_value('Global Defaults', 'default_company')
 		"""
-
-		if doctype not in self.value_cache:
-			self.value_cache[doctype] = {}
-
 		if cache and fieldname in self.value_cache[doctype]:
 			return self.value_cache[doctype][fieldname]
 
@@ -975,8 +976,7 @@ class Database:
 
 		query.run(debug=debug)
 
-		if dt in self.value_cache:
+		self.value_cache.pop(dt, None)
-			del self.value_cache[dt]
 
 	def bulk_update(
 		self,
@@ -1252,10 +1252,10 @@ class Database:
 
 	def count(self, dt, filters=None, debug=False, cache=False, distinct: bool = True):
 		"""Return `COUNT(*)` for given DocType and filters."""
-		cache_key = (dt, "COUNT(*)")
+		cache_key = "COUNT(*)"
-		if cache and not filters:
+		if cache and not filters and cache_key in self.value_cache[dt]:
-			if cache_key in self.value_cache:
+			return self.value_cache[dt][cache_key]
-				return self.value_cache[cache_key]
+
 		count = frappe.qb.get_query(
 			table=dt,
 			filters=filters,
@@ -1263,8 +1263,9 @@ class Database:
 			distinct=distinct,
 			validate_filters=True,
 		).run(debug=debug)[0][0]
+
 		if not filters and cache:
-			self.value_cache[cache_key] = count
+			self.value_cache[dt][cache_key] = count
 		return count
 
 	def estimate_count(self, doctype: str) -> int:

frappe/deprecation_dumpster.py

@@ -576,7 +576,7 @@ def get_tests_CompatFrappeTestCase():
 		traceback.print_stack(limit=10)
 
 	def _rollback_db():
-		frappe.db.value_cache = {}
+		frappe.db.value_cache.clear()
 		frappe.db.rollback()
 
 	def _restore_thread_locals(flags):

frappe/model/document.py

@@ -706,7 +706,7 @@ class Document(BaseDocument, DocRef):
 				)
 
 		if self.doctype in frappe.db.value_cache:
-			del frappe.db.value_cache[self.doctype]
+			frappe.db.value_cache.pop(self.doctype, None)
 
 	def set_user_and_timestamp(self):
 		self._original_modified = self.modified

frappe/tests/classes/context_managers.py

@@ -88,8 +88,6 @@ def change_settings(doctype, settings_dict=None, /, commit=False, **settings) ->
 	for key, value in settings_dict.items():
 		setattr(settings, key, value)
 	settings.save(ignore_permissions=True)
-	# singles are cached by default, clear to avoid flake
-	frappe.db.value_cache[settings] = {}
 	if commit:
 		frappe.db.commit()
 	yield

frappe/tests/classes/integration_test_case.py

@@ -183,7 +183,7 @@ def _commit_watcher():
 
 
 def _rollback_db():
-	frappe.db.value_cache = {}
+	frappe.db.value_cache.clear()
 	frappe.db.rollback()
 
 

frappe/tests/test_website.py

@@ -252,7 +252,6 @@ class TestWebsite(IntegrationTestCase):
 			self.assertEqual(response.status_code, 404)
 
 	def test_printview_page(self):
-		frappe.db.value_cache[("DocType", "Language", "name")] = (("Language",),)
 		frappe.set_user("Administrator")
 		content = get_response_content("/Language/ru")
 		self.assertIn('<div class="print-format">', content)

frappe/utils/data.py

@@ -12,6 +12,7 @@ import re
 import time
 import typing
 from code import compile_command
+from collections import defaultdict
 from enum import Enum
 from functools import lru_cache
 from typing import Any, Literal, Optional, TypeVar
@@ -2711,6 +2712,11 @@ def mock(type, size=1, locale="en"):
 	return squashify(results)
 
 
+# Recursive default dict with arbitrary levels of nesting
+def recursive_defaultdict():
+	return defaultdict(recursive_defaultdict)
+
+
 # This is used in test to count memory overhead of default imports.
 def _get_rss_memory_usage():
 	import psutil

frappe/website/doctype/blog_post/test_blog_post.py

@@ -74,9 +74,6 @@ class TestBlogPost(IntegrationTestCase):
 		category_page_link = next(iter(soup.find_all("a", href=re.compile(blog.blog_category))))
 		category_page_url = category_page_link["href"]
 
-		cached_value = frappe.db.value_cache.get(("DocType", "Blog Post", "name"))
-		frappe.db.value_cache[("DocType", "Blog Post", "name")] = (("Blog Post",),)
-
 		# Visit the category page (by following the link found in above stage)
 		set_request(path=category_page_url)
 		category_page_response = get_response()
@@ -85,7 +82,6 @@ class TestBlogPost(IntegrationTestCase):
 		self.assertIn(blog.title, category_page_html)
 
 		# Cleanup
-		frappe.db.value_cache[("DocType", "Blog Post", "name")] = cached_value
 		frappe.delete_doc("Blog Post", blog.name)
 		frappe.delete_doc("Blog Category", blog.blog_category)