perf: cache dynamic links map in Redis (#28878)

Note about correctness: Once site has seen enough usage this map will rarely change. So the
problem of "cache inconsistency" is very rare, still care is taken to
avoid possible cache inconsistencies.

frappe/model/base_document.py

@@ -18,6 +18,7 @@ from frappe.model import (
 	table_fields,
 )
 from frappe.model.docstatus import DocStatus
+from frappe.model.dynamic_links import invalidate_distinct_link_doctypes
 from frappe.model.naming import set_new_name
 from frappe.model.utils.link_count import notify_link_count
 from frappe.modules import load_doctype_module
@@ -818,6 +819,7 @@ class BaseDocument:
 					doctype = self.get(df.options)
 					if not doctype:
 						frappe.throw(_("{0} must be set first").format(self.meta.get_label(df.options)))
+					invalidate_distinct_link_doctypes(df.parent, df.options, doctype)
 
 				# MySQL is case insensitive. Preserve case of the original docname in the Link Field.
 

frappe/model/dynamic_links.py

@@ -42,9 +42,7 @@ def get_dynamic_link_map(for_delete=False):
 				dynamic_link_map.setdefault(meta.name, []).append(df)
 			else:
 				try:
-					links = frappe.db.sql_list(
-						"""select distinct `{options}` from `tab{parent}`""".format(**df)
-					)
+					links = fetch_distinct_link_doctypes(df.parent, df.options)
 					for doctype in links:
 						dynamic_link_map.setdefault(doctype, []).append(df)
 				except frappe.db.TableMissingError:
@@ -61,3 +59,41 @@ def get_dynamic_links():
 	for query in dynamic_link_queries:
 		df += frappe.db.sql(query, as_dict=True)
 	return df
+
+
+def _dynamic_link_map_key(doctype, fieldname):
+	return f"dynamic_link_map::{doctype}::{fieldname}"
+
+
+def fetch_distinct_link_doctypes(doctype: str, fieldname: str):
+	"""Return all unique doctypes a dynamic link is linking against.
+	Note:
+	- results are cached and can *possibly be outdated*
+	- cache gets updated when a document with different document link is discovered
+	- raw queries adding dynamic link won't update this cache
+	- cache miss can often be VERY expensive on large table.
+	"""
+
+	key = _dynamic_link_map_key(doctype, fieldname)
+	doctypes = frappe.cache.get_value(key)
+
+	if doctypes is None:
+		doctypes = frappe.db.sql(f"""select distinct `{fieldname}` from `tab{doctype}`""", pluck=True)
+		frappe.cache.set_value(key, doctypes, expires_in_sec=12 * 60 * 60)
+
+	return doctypes
+
+
+def invalidate_distinct_link_doctypes(doctype: str, fieldname: str, linked_doctype: str):
+	"""If new linked doctype is discovered for a dynamic link then cache is evicted."""
+
+	key = _dynamic_link_map_key(doctype, fieldname)
+	doctypes = frappe.cache.get_value(key)
+
+	if doctypes is None or not isinstance(doctypes, list):
+		return
+
+	if linked_doctype not in doctypes:
+		# Note: Do NOT "update" cache because it can lead to concurrency bugs.
+		frappe.cache.delete_value(key)
+		frappe.db.after_commit.add(lambda: frappe.cache.delete_value(key))