diff --git a/frappe/recorder.py b/frappe/recorder.py
index 0bcea0c231..96ab502fec 100644
--- a/frappe/recorder.py
+++ b/frappe/recorder.py
@@ -86,10 +86,42 @@ def post_process():
def mark_duplicates(request):
- counts = Counter([call["query"] for call in request["calls"]])
+ exact_duplicates = Counter([call["query"] for call in request["calls"]])
+
+ for sql_call in request["calls"]:
+ sql_call["normalized_query"] = normalize_query(sql_call["query"])
+
+ normalized_duplicates = Counter([call["normalized_query"] for call in request["calls"]])
+
for index, call in enumerate(request["calls"]):
call["index"] = index
- call["exact_copies"] = counts[call["query"]]
+ call["exact_copies"] = exact_duplicates[call["query"]]
+ call["normalized_copies"] = normalized_duplicates[call["normalized_query"]]
+
+
+def normalize_query(query: str) -> str:
+ """Attempt to normalize query by removing variables.
+ This gives a different view of similar duplicate queries.
+
+ Example:
+ These two are distinct queries:
+ `select * from user where name = 'x'`
+ `select * from user where name = 'z'`
+
+ But their "normalized" form would be same:
+ `select * from user where name = ?`
+ """
+
+ try:
+ q = sqlparse.parse(query)[0]
+ for token in q.flatten():
+ if "Token.Literal" in str(token.ttype):
+ token.value = "?"
+ return str(q)
+ except Exception as e:
+ print("Failed to normalize query ", e)
+
+ return query
def record(force=False):
diff --git a/frappe/tests/test_recorder.py b/frappe/tests/test_recorder.py
index e20f165134..71d9b3add5 100644
--- a/frappe/tests/test_recorder.py
+++ b/frappe/tests/test_recorder.py
@@ -5,6 +5,7 @@ import sqlparse
import frappe
import frappe.recorder
+from frappe.recorder import normalize_query
from frappe.tests.utils import FrappeTestCase
from frappe.utils import set_request
from frappe.website.serve import get_response_content
@@ -138,3 +139,17 @@ class TestRecorderDeco(FrappeTestCase):
test()
self.assertTrue(frappe.recorder.get())
+
+
+class TestQueryNormalization(FrappeTestCase):
+ def test_query_normalization(self):
+ test_cases = {
+ "select * from user where name = 'x'": "select * from user where name = ?",
+ "select * from user where a > 5": "select * from user where a > ?",
+ "select * from `user` where a > 5": "select * from `user` where a > ?",
+ "select `name` from `user`": "select `name` from `user`",
+ "select `name` from `user` limit 10": "select `name` from `user` limit ?",
+ }
+
+ for query, normalized in test_cases.items():
+ self.assertEqual(normalize_query(query), normalized)