Merge pull request #33047 from sagarvora/orjson

perf: use `orjson` for faster request processing

frappe/__init__.py

@@ -34,6 +34,7 @@ from typing import (
 )
 
 import click
+import orjson
 from werkzeug.datastructures import Headers
 
 import frappe
@@ -1265,7 +1266,7 @@ def get_installed_apps(*, _ensure_on_bench: bool = False) -> list[str]:
 	if not db:
 		connect()
 
-	installed = json.loads(db.get_global("installed_apps") or "[]")
+	installed = orjson.loads(db.get_global("installed_apps") or "[]")
 
 	if _ensure_on_bench:
 		all_apps = cache.get_value("all_apps", get_all_apps)

frappe/app.py

@@ -5,6 +5,7 @@ import functools
 import logging
 import os
 
+import orjson
 from werkzeug.exceptions import HTTPException, NotFound
 from werkzeug.middleware.profiler import ProfilerMiddleware
 from werkzeug.middleware.proxy_fix import ProxyFix
@@ -297,11 +298,9 @@ def set_cors_headers(response):
 
 
 def make_form_dict(request: Request):
-	import json
-
 	request_data = request.get_data(as_text=True)
 	if request_data and request.is_json:
-		args = json.loads(request_data)
+		args = orjson.loads(request_data)
 	else:
 		args = {}
 		args.update(request.args or {})

frappe/modules/import_file.py

@@ -1,9 +1,10 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
 import hashlib
-import json
 import os
 
+import orjson
+
 import frappe
 from frappe.model.base_document import get_controller
 from frappe.modules import get_module_path, scrub_dt_dn
@@ -173,7 +174,7 @@ def read_doc_from_file(path):
 	if os.path.exists(path):
 		with open(path) as f:
 			try:
-				doc = json.loads(f.read())
+				doc = orjson.loads(f.read())
 			except ValueError:
 				print(f"bad json: {path}")
 				raise

frappe/templates/base.html

@@ -95,7 +95,7 @@
 	{% block base_scripts %}
 	<!-- js should be loaded in body! -->
 	<script>
-		frappe.boot = {{ boot | json }}
+		frappe.boot = {{ frappe.utils.orjson_dumps(boot, default=frappe.json_handler) }}
 		// for backward compatibility of some libs
 		frappe.sys_defaults = frappe.boot.sysdefaults;
 	</script>

frappe/utils/__init__.py

@@ -21,6 +21,7 @@ from email.header import decode_header, make_header
 from email.utils import formataddr, parseaddr
 from typing import Any, Generic, TypeAlias, TypedDict
 
+import orjson
 from werkzeug.test import Client
 
 from frappe.deprecation_dumpster import gzip_compress, gzip_decompress, make_esc
@@ -830,18 +831,7 @@ def get_site_info():
 		site_info.update(frappe.get_attr(method_name)(site_info) or {})
 
 	# dumps -> loads to prevent datatype conflicts
-	return json.loads(frappe.as_json(site_info))
-
-
-def parse_json(val: str):
-	"""
-	Parses json if string else return
-	"""
-	if isinstance(val, str):
-		val = json.loads(val)
-	if isinstance(val, dict):
-		val = frappe._dict(val)
-	return val
+	return orjson.loads(frappe.as_json(site_info))
 
 
 def get_db_count(*args):
@@ -862,7 +852,7 @@ def get_db_count(*args):
 	for doctype in args:
 		db_count[doctype] = frappe.db.count(doctype)
 
-	return json.loads(frappe.as_json(db_count))
+	return orjson.loads(frappe.as_json(db_count))
 
 
 def call(fn, *args, **kwargs):
@@ -878,12 +868,12 @@ def call(fn, *args, **kwargs):
 	        via terminal:
 	                bench --site erpnext.local execute frappe.utils.call --args '''["frappe.get_all", "Activity Log"]''' --kwargs '''{"fields": ["user", "creation", "full_name"], "filters":{"Operation": "Login", "Status": "Success"}, "limit": "10"}'''
 	"""
-	return json.loads(frappe.as_json(frappe.call(fn, *args, **kwargs)))
+	return orjson.loads(frappe.as_json(frappe.call(fn, *args, **kwargs)))
 
 
 def get_safe_filters(filters):
 	try:
-		filters = json.loads(filters)
+		filters = orjson.loads(filters)
 
 		if isinstance(filters, int | float):
 			filters = frappe.as_unicode(filters)
@@ -1043,7 +1033,7 @@ def safe_json_loads(*args):
 
 	for arg in args:
 		try:
-			arg = json.loads(arg)
+			arg = orjson.loads(arg)
 		except Exception:
 			pass
 

frappe/utils/data.py

@@ -19,6 +19,7 @@ from typing import Any, Literal, Optional, TypeVar
 from urllib.parse import parse_qsl, quote, urlencode, urljoin, urlparse, urlunparse
 from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
 
+import orjson
 from click import secho
 from dateutil import parser
 from dateutil.parser import ParserError
@@ -82,6 +83,15 @@ DURATION_PATTERN = re.compile(r"^(?:(\d+d)?((^|\s)\d+h)?((^|\s)\d+m)?((^|\s)\d+s
 HTML_TAG_PATTERN = re.compile("<[^>]+>")
 MARIADB_SPECIFIC_COMMENT = re.compile(r"#.*")
 
+# these options are necessary to use orjson with frappe
+#
+# OPT_PASSTHROUGH_DATETIME allows datetime objects to be passed through
+# to the default function without conversion by orjson
+# frappe converts datetime objects differently (__str__) from orjson (RFC 3339)
+#
+# OPT_NON_STR_KEYS slightly reduces performance of orjson, but allows for non-string keys in dicts
+DEFAULT_ORJSON_OPTIONS = orjson.OPT_PASSTHROUGH_DATETIME | orjson.OPT_NON_STR_KEYS
+
 
 class Weekday(Enum):
 	Sunday = 0
@@ -2439,11 +2449,35 @@ def guess_date_format(date_string: str) -> str:
 
 def validate_json_string(string: str) -> None:
 	try:
-		json.loads(string)
+		orjson.loads(string)
 	except (TypeError, ValueError):
 		raise frappe.ValidationError
 
 
+def parse_json(val: str):
+	"""
+	Parses json if string else return
+	"""
+	if isinstance(val, str):
+		val = orjson.loads(val)
+	if isinstance(val, dict):
+		val = frappe._dict(val)
+	return val
+
+
+def orjson_dumps(obj, default=None, option=None, decode=True):
+	"""A wrapper around `orjson.dumps`, with some default options set"""
+
+	if option is not None:
+		# user defined options are merged with the default options
+		option = option | DEFAULT_ORJSON_OPTIONS
+	else:
+		option = DEFAULT_ORJSON_OPTIONS
+
+	value = orjson.dumps(obj, default, option)
+	return value.decode() if decode else value
+
+
 class _UserInfo(typing.TypedDict):
 	email: str
 	image: str | None

frappe/utils/response.py

@@ -2,18 +2,19 @@
 # License: MIT. See LICENSE
 
 import datetime
-import decimal
-import json
+import functools
 import mimetypes
 import os
 import sys
-import uuid
 from collections.abc import Iterable
+from decimal import Decimal
 from pathlib import Path
 from re import Match
 from typing import TYPE_CHECKING
 from urllib.parse import quote
+from uuid import UUID
 
+import orjson
 import werkzeug.utils
 from werkzeug.exceptions import Forbidden, NotFound
 from werkzeug.local import LocalProxy
@@ -26,12 +27,13 @@ import frappe.sessions
 import frappe.utils
 from frappe import _
 from frappe.core.doctype.access_log.access_log import make_access_log
-from frappe.utils import format_timedelta
+from frappe.utils import format_timedelta, orjson_dumps
 
 if TYPE_CHECKING:
 	from frappe.core.doctype.file.file import File
 
 DateOrTimeTypes = datetime.date | datetime.datetime | datetime.time
+timedelta = datetime.timedelta
 
 
 def report_error(status_code):
@@ -148,7 +150,7 @@ def as_json():
 		del frappe.local.response["http_status_code"]
 
 	response.mimetype = "application/json"
-	response.data = json.dumps(frappe.local.response, default=json_handler, separators=(",", ":"))
+	response.data = orjson_dumps(frappe.local.response, default=json_handler)
 	return response
 
 
@@ -191,13 +193,15 @@ def _make_logs_v1():
 	if frappe.error_log and is_traceback_allowed():
 		if source := guess_exception_source(frappe.local.error_log and frappe.local.error_log[0]["exc"]):
 			response["_exc_source"] = source
-		response["exc"] = json.dumps([frappe.utils.cstr(d["exc"]) for d in frappe.local.error_log])
+		response["exc"] = orjson.dumps([frappe.utils.cstr(d["exc"]) for d in frappe.local.error_log]).decode()
 
 	if frappe.local.message_log:
-		response["_server_messages"] = json.dumps([json.dumps(d) for d in frappe.local.message_log])
+		response["_server_messages"] = orjson.dumps(
+			[orjson.dumps(d).decode() for d in frappe.local.message_log]
+		).decode()
 
 	if frappe.debug_log:
-		response["_debug_messages"] = json.dumps(frappe.local.debug_log)
+		response["_debug_messages"] = orjson.dumps(frappe.local.debug_log).decode()
 
 	if frappe.flags.error_message:
 		response["_error_message"] = frappe.flags.error_message
@@ -219,7 +223,7 @@ def json_handler(obj):
 	if isinstance(obj, DateOrTimeTypes):
 		return str(obj)
 
-	elif isinstance(obj, datetime.timedelta):
+	elif isinstance(obj, timedelta):
 		return format_timedelta(obj)
 
 	elif isinstance(obj, LocalProxy):
@@ -231,7 +235,7 @@ def json_handler(obj):
 	elif isinstance(obj, Iterable):
 		return list(obj)
 
-	elif isinstance(obj, decimal.Decimal):
+	elif isinstance(obj, Decimal):
 		return float(obj)
 
 	elif isinstance(obj, Match):
@@ -243,10 +247,12 @@ def json_handler(obj):
 	elif callable(obj):
 		return repr(obj)
 
-	elif isinstance(obj, uuid.UUID):
+	elif isinstance(obj, Path):
 		return str(obj)
 
-	elif isinstance(obj, Path):
+	# orjson does this already
+	# but json_handler needs to be compatible with built-in json module also
+	elif isinstance(obj, UUID):
 		return str(obj)
 
 	else:

frappe/utils/safe_exec.py

@@ -11,6 +11,7 @@ from itertools import chain
 from types import FunctionType, MethodType, ModuleType
 from typing import TYPE_CHECKING, Any
 
+import orjson
 import RestrictedPython.Guards
 from RestrictedPython import PrintCollector, compile_restricted, safe_globals
 from RestrictedPython.transformer import RestrictingNodeTransformer
@@ -31,6 +32,7 @@ from frappe.model.rename_doc import rename_doc
 from frappe.modules import scrub
 from frappe.utils.background_jobs import enqueue, get_jobs
 from frappe.utils.number_format import NumberFormat
+from frappe.utils.response import json_handler
 from frappe.website.utils import get_next_link, get_toc
 from frappe.www.printview import get_visible_columns
 
@@ -189,6 +191,7 @@ def get_safe_globals():
 	out = NamespaceDict(
 		# make available limited methods of frappe
 		json=NamespaceDict(loads=json.loads, dumps=json.dumps),
+		orjson=SAFE_ORJSON,
 		as_json=frappe.as_json,
 		dict=dict,
 		log=frappe.log,
@@ -278,6 +281,7 @@ def get_safe_globals():
 				get_html_content_based_on_type=frappe.website.utils.get_html_content_based_on_type,
 			),
 			lang=getattr(frappe.local, "lang", "en"),
+			json_handler=json_handler,
 		),
 		FrappeClient=FrappeClient,
 		style=frappe._dict(border_color="#d1d8dd"),
@@ -715,6 +719,8 @@ VALID_UTILS = (
 	"get_abbr",
 	"get_month",
 	"sha256_hash",
+	"parse_json",
+	"orjson_dumps",
 )
 
 
@@ -729,3 +735,8 @@ WHITELISTED_SAFE_EVAL_GLOBALS = {
 	"_getiter_": iter,
 	"_iter_unpack_sequence_": RestrictedPython.Guards.guarded_iter_unpack_sequence,
 }
+
+SAFE_ORJSON = NamespaceDict(loads=orjson.loads, dumps=orjson.dumps)
+for key, val in vars(orjson).items():
+	if key.startswith("OPT_"):
+		SAFE_ORJSON[key] = val

frappe/website/doctype/web_form/templates/web_form.html

@@ -163,7 +163,7 @@
 
 {% block script %}
 	<script>
-		frappe.boot = {{ boot | json }};
+		frappe.boot = {{ frappe.utils.orjson_dumps(boot, default=frappe.json_handler) }};
 		frappe._messages = {{ translated_messages }};
 		frappe.web_form_doc = {{ web_form_doc | json }};
 		frappe.reference_doc = {{ reference_doc | json }};

frappe/website/doctype/web_form/templates/web_list.html

@@ -24,7 +24,7 @@
 
 {% block script %}
 	<script>
-		frappe.boot = {{ boot | json }};
+		frappe.boot = {{ frappe.utils.orjson_dumps(boot, default=frappe.json_handler) }};
 		frappe._messages = {{ translated_messages }};
 		frappe.web_form_doc = {{ web_form_doc | json }};
 	</script>
@@ -41,4 +41,4 @@
 			{{ custom_css }}
 		{% endif %}
 	</style>
-{% endblock %}
+{% endblock %}

frappe/www/app.html

@@ -51,7 +51,7 @@
 
 			if (!window.frappe) window.frappe = {};
 
-			frappe.boot = {{ boot | json }};
+			frappe.boot = {{ frappe.utils.orjson_dumps(boot, default=frappe.json_handler) }};
 			frappe._messages = frappe.boot["__messages"];
 			frappe.csrf_token = "{{ csrf_token }}";
 

pyproject.toml

@@ -47,6 +47,7 @@ dependencies = [
     "num2words~=0.5.14",
     "oauthlib~=3.2.2",
     "openpyxl~=3.1.5",
+    "orjson~=3.10.18",
     "passlib~=1.7.4",
     "pdfkit~=1.0.0",
     "phonenumbers~=9.0.7",