refactor: pipeline wherever possible, optimize calls

Signed-off-by: Akhil Narang <me@akhilnarang.dev>

frappe/cache_manager.py

@@ -82,13 +82,11 @@ def clear_user_cache(user=None):
 	clear_notifications(user)
 
 	if user:
-		for name in user_cache_keys:
-			frappe.cache.hdel(name, user)
+		frappe.cache.hdel_names(user_cache_keys, user)
 		frappe.cache.delete_keys("user:" + user)
 		clear_defaults_cache(user)
 	else:
-		for name in user_cache_keys:
-			frappe.cache.delete_key(name)
+		frappe.cache.delete_key(user_cache_keys)
 		clear_defaults_cache()
 		clear_global_cache()
 
@@ -103,17 +101,15 @@ def clear_global_cache():
 
 	clear_doctype_cache()
 	clear_website_cache()
-	frappe.cache.delete_value(global_cache_keys)
-	frappe.cache.delete_value(bench_cache_keys)
+	frappe.cache.delete_value(global_cache_keys + bench_cache_keys)
 	frappe.setup_module_map()
 
 
 def clear_defaults_cache(user=None):
 	if user:
-		for p in [user] + common_default_keys:
-			frappe.cache.hdel("defaults", p)
+		frappe.cache.hdel("defaults", [user] + common_default_keys)
 	elif frappe.flags.in_install != "frappe":
-		frappe.cache.delete_key("defaults")
+		frappe.cache.delete_value("defaults")
 
 
 def clear_doctype_cache(doctype=None):
@@ -128,15 +124,14 @@ def clear_doctype_cache(doctype=None):
 def _clear_doctype_cache_from_redis(doctype: str | None = None):
 	from frappe.desk.notifications import delete_notification_count_for
 
-	for key in ("is_table", "doctype_modules"):
-		frappe.cache.delete_value(key)
-
-	def clear_single(dt):
-		frappe.clear_document_cache(dt)
-		for name in doctype_cache_keys:
-			frappe.cache.hdel(name, dt)
+	to_del = ["is_table", "doctype_modules"]
 
 	if doctype:
+
+		def clear_single(dt):
+			frappe.clear_document_cache(dt)
+			frappe.cache.hdel_names(doctype_cache_keys, dt)
+
 		clear_single(doctype)
 
 		# clear all parent doctypes
@@ -157,9 +152,10 @@ def _clear_doctype_cache_from_redis(doctype: str | None = None):
 
 	else:
 		# clear all
-		for name in doctype_cache_keys:
-			frappe.cache.delete_value(name)
-		frappe.cache.delete_keys("document_cache::")
+		to_del += doctype_cache_keys
+		to_del += frappe.cache.get_keys("document_cache::")
+
+	frappe.cache.delete_value(to_del)
 
 
 def clear_controller_cache(doctype=None):

frappe/desk/notifications.py

@@ -72,7 +72,7 @@ def get_notifications_for_doctypes(config, notification_count):
 				except frappe.PermissionError:
 					frappe.clear_messages()
 					pass
-					# frappe.msgprint("Permission Error in notifications for {0}".format(d))
+				# frappe.msgprint("Permission Error in notifications for {0}".format(d))
 
 				except Exception as e:
 					# OperationalError: (1412, 'Table definition has changed, please retry transaction')
@@ -149,11 +149,10 @@ def clear_notifications(user=None):
 	for_module = list(config.get("for_module")) if config.get("for_module") else []
 	groups = for_doctype + for_module
 
-	for name in groups:
-		if user:
-			frappe.cache.hdel("notification_count:" + name, user)
-		else:
-			frappe.cache.delete_key("notification_count:" + name)
+	if user:
+		frappe.cache.hdel_names([f"notification_count:{name}" for name in groups], user)
+	else:
+		frappe.cache.delete_value([f"notification_count:{name}" for name in groups])
 
 
 def clear_notification_config(user):

frappe/utils/redis_wrapper.py

@@ -238,22 +238,65 @@ class RedisWrapper(redis.Redis):
 			self.hset(name, key, value, shared=shared)
 		return value
 
-	def hdel(self, name, key, shared=False):
+	def hdel(
+		self,
+		name: str,
+		keys: str | list | tuple,
+		shared=False,
+		pipeline: redis.client.Pipeline | None = None,
+	):
+		"""
+		A wrapper around redis' HDEL command
+
+		:param name: The hash name
+		:param keys: the keys to delete
+		:param shared: shared frappe key or not
+		:param pipeline: A redis.client.Pipeline object, if this transaction is to be run in a pipeline
+		"""
 		_name = self.make_key(name, shared=shared)
 
-		if _name in frappe.local.cache:
-			if key in frappe.local.cache[_name]:
-				del frappe.local.cache[_name][key]
-		try:
-			super().hdel(_name, key)
-		except redis.exceptions.ConnectionError:
-			pass
+		if not isinstance(keys, (list, tuple)):
+			keys = (keys,)
+
+		name_in_local_cache = _name in frappe.local.cache
+
+		local_pipeline = False
+
+		if pipeline is None:
+			pipeline = self.pipeline()
+			local_pipeline = True
+
+		for key in keys:
+			if name_in_local_cache:
+				if key in frappe.local.cache[_name]:
+					del frappe.local.cache[_name][key]
+			try:
+				pipeline.hdel(_name, key)
+			except redis.exceptions.ConnectionError:
+				pass
+
+		if local_pipeline:
+			pipeline.execute()
+
+	def hdel_names(self, names: list | tuple, key: str):
+		"""
+		A function to call HDEL on multiple hash names with a common key, run in a single pipeline
+
+		:param names: The hash names
+		:param key: The common key
+		"""
+		pipeline = self.pipeline()
+		for name in names:
+			self.hdel(name, key, pipeline=pipeline)
+		pipeline.execute()
 
 	def hdel_keys(self, name_starts_with, key):
 		"""Delete hash names with wildcard `*` and key"""
+		pipeline = self.pipeline()
 		for name in self.get_keys(name_starts_with):
 			name = name.split("|", 1)[1]
-			self.hdel(name, key)
+			self.hdel(name, key, pipeline=pipeline)
+		pipeline.execute()
 
 	def hkeys(self, name):
 		try:

frappe/website/utils.py

@@ -10,7 +10,6 @@ import yaml
 from werkzeug.wrappers import Response
 
 import frappe
-from frappe import _
 from frappe.model.document import Document
 from frappe.utils import (
 	cint,
@@ -30,14 +29,13 @@ CLEANUP_PATTERN_3 = re.compile(r"(-)\1+")
 
 
 def delete_page_cache(path):
-	frappe.cache.delete_value("full_index")
-	groups = ("website_page", "page_context")
+	groups = ["website_page", "page_context"]
 	if path:
-		for name in groups:
-			frappe.cache.hdel(name, path)
+		frappe.cache.hdel_names(groups, path)
+		frappe.cache.delete_value("full_index")
 	else:
-		for name in groups:
-			frappe.cache.delete_key(name)
+		groups.append("full_index")
+		frappe.cache.delete_value(groups)
 
 
 def find_first_image(html):
@@ -363,25 +361,24 @@ def clear_cache(path=None):
 	:param path: (optional) for the given path"""
 	from frappe.website.router import clear_routing_cache
 
-	for key in (
+	clear_routing_cache()
+
+	keys = [
 		"website_generator_routes",
 		"website_pages",
 		"website_full_index",
 		"languages_with_name",
 		"languages",
-	):
-		frappe.cache.delete_value(key)
+		"website_404",
+	]
 
-	clear_routing_cache()
-
-	frappe.cache.delete_value("website_404")
 	if path:
 		frappe.cache.hdel("website_redirects", path)
 		delete_page_cache(path)
 	else:
 		clear_sitemap()
 		frappe.clear_cache("Guest")
-		for key in (
+		keys += [
 			"portal_menu_items",
 			"home_page",
 			"website_route_rules",
@@ -389,8 +386,9 @@ def clear_cache(path=None):
 			"website_redirects",
 			"page_context",
 			"website_page",
-		):
-			frappe.cache.delete_value(key)
+		]
+
+	frappe.cache.delete_value(keys)
 
 	for method in frappe.get_hooks("website_clear_cache"):
 		frappe.get_attr(method)(path)