From b857a4099a110603e03f973f335a63de00a03eb7 Mon Sep 17 00:00:00 2001
From: Sagar Vora <16315650+sagarvora@users.noreply.github.com>
Date: Sat, 21 Jun 2025 21:44:11 +0530
Subject: [PATCH 1/4] perf: use `orjson` for faster request processing
---
frappe/__init__.py | 3 ++-
frappe/app.py | 5 ++---
frappe/utils/response.py | 33 ++++++++++++++++++++++-----------
pyproject.toml | 1 +
4 files changed, 27 insertions(+), 15 deletions(-)
diff --git a/frappe/__init__.py b/frappe/__init__.py
index 8f5e2c436f..4f8e8b6e88 100644
--- a/frappe/__init__.py
+++ b/frappe/__init__.py
@@ -34,6 +34,7 @@ from typing import (
)
import click
+import orjson
from werkzeug.datastructures import Headers
import frappe
@@ -1265,7 +1266,7 @@ def get_installed_apps(*, _ensure_on_bench: bool = False) -> list[str]:
if not db:
connect()
- installed = json.loads(db.get_global("installed_apps") or "[]")
+ installed = orjson.loads(db.get_global("installed_apps") or "[]")
if _ensure_on_bench:
all_apps = cache.get_value("all_apps", get_all_apps)
diff --git a/frappe/app.py b/frappe/app.py
index 04e06171ef..1bad66a891 100644
--- a/frappe/app.py
+++ b/frappe/app.py
@@ -5,6 +5,7 @@ import functools
import logging
import os
+import orjson
from werkzeug.exceptions import HTTPException, NotFound
from werkzeug.middleware.profiler import ProfilerMiddleware
from werkzeug.middleware.proxy_fix import ProxyFix
@@ -297,11 +298,9 @@ def set_cors_headers(response):
def make_form_dict(request: Request):
- import json
-
request_data = request.get_data(as_text=True)
if request_data and request.is_json:
- args = json.loads(request_data)
+ args = orjson.loads(request_data)
else:
args = {}
args.update(request.args or {})
diff --git a/frappe/utils/response.py b/frappe/utils/response.py
index dac5237c76..77ce8364f0 100644
--- a/frappe/utils/response.py
+++ b/frappe/utils/response.py
@@ -2,18 +2,19 @@
# License: MIT. See LICENSE
import datetime
-import decimal
-import json
+import functools
import mimetypes
import os
import sys
-import uuid
from collections.abc import Iterable
+from decimal import Decimal
from pathlib import Path
from re import Match
from typing import TYPE_CHECKING
from urllib.parse import quote
+from uuid import UUID
+import orjson
import werkzeug.utils
from werkzeug.exceptions import Forbidden, NotFound
from werkzeug.local import LocalProxy
@@ -32,6 +33,7 @@ if TYPE_CHECKING:
from frappe.core.doctype.file.file import File
DateOrTimeTypes = datetime.date | datetime.datetime | datetime.time
+timedelta = datetime.timedelta
def report_error(status_code):
@@ -148,7 +150,7 @@ def as_json():
del frappe.local.response["http_status_code"]
response.mimetype = "application/json"
- response.data = json.dumps(frappe.local.response, default=json_handler, separators=(",", ":"))
+ response.data = dump_response(frappe.local.response)
return response
@@ -191,13 +193,15 @@ def _make_logs_v1():
if frappe.error_log and is_traceback_allowed():
if source := guess_exception_source(frappe.local.error_log and frappe.local.error_log[0]["exc"]):
response["_exc_source"] = source
- response["exc"] = json.dumps([frappe.utils.cstr(d["exc"]) for d in frappe.local.error_log])
+ response["exc"] = orjson.dumps([frappe.utils.cstr(d["exc"]) for d in frappe.local.error_log]).decode()
if frappe.local.message_log:
- response["_server_messages"] = json.dumps([json.dumps(d) for d in frappe.local.message_log])
+ response["_server_messages"] = orjson.dumps(
+ [orjson.dumps(d).decode() for d in frappe.local.message_log]
+ ).decode()
if frappe.debug_log:
- response["_debug_messages"] = json.dumps(frappe.local.debug_log)
+ response["_debug_messages"] = orjson.dumps(frappe.local.debug_log).decode()
if frappe.flags.error_message:
response["_error_message"] = frappe.flags.error_message
@@ -219,7 +223,7 @@ def json_handler(obj):
if isinstance(obj, DateOrTimeTypes):
return str(obj)
- elif isinstance(obj, datetime.timedelta):
+ elif isinstance(obj, timedelta):
return format_timedelta(obj)
elif isinstance(obj, LocalProxy):
@@ -231,7 +235,7 @@ def json_handler(obj):
elif isinstance(obj, Iterable):
return list(obj)
- elif isinstance(obj, decimal.Decimal):
+ elif isinstance(obj, Decimal):
return float(obj)
elif isinstance(obj, Match):
@@ -243,16 +247,23 @@ def json_handler(obj):
elif callable(obj):
return repr(obj)
- elif isinstance(obj, uuid.UUID):
+ elif isinstance(obj, Path):
return str(obj)
- elif isinstance(obj, Path):
+ # orjson does this already
+ # but json_handler needs to be compatible with built-in json module also
+ elif isinstance(obj, UUID):
return str(obj)
else:
raise TypeError(f"""Object of type {type(obj)} with value of {obj!r} is not JSON serializable""")
+dump_response = functools.partial(
+ orjson.dumps, default=json_handler, option=orjson.OPT_PASSTHROUGH_DATETIME | orjson.OPT_NON_STR_KEYS
+)
+
+
def as_page():
"""print web page"""
from frappe.website.serve import get_response
diff --git a/pyproject.toml b/pyproject.toml
index d9fcd21a32..8187e872e2 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -47,6 +47,7 @@ dependencies = [
"num2words~=0.5.14",
"oauthlib~=3.2.2",
"openpyxl~=3.1.5",
+ "orjson~=3.10.18",
"passlib~=1.7.4",
"pdfkit~=1.0.0",
"phonenumbers~=9.0.7",
From cda7699187ce2d9e488255797c82060c43e8ae68 Mon Sep 17 00:00:00 2001
From: Sagar Vora <16315650+sagarvora@users.noreply.github.com>
Date: Tue, 24 Jun 2025 12:12:25 +0530
Subject: [PATCH 2/4] perf: use `orjson` in utils
---
frappe/utils/__init__.py | 13 +++++++------
1 file changed, 7 insertions(+), 6 deletions(-)
diff --git a/frappe/utils/__init__.py b/frappe/utils/__init__.py
index a9bdee4d1a..0bc0666922 100644
--- a/frappe/utils/__init__.py
+++ b/frappe/utils/__init__.py
@@ -21,6 +21,7 @@ from email.header import decode_header, make_header
from email.utils import formataddr, parseaddr
from typing import Any, Generic, TypeAlias, TypedDict
+import orjson
from werkzeug.test import Client
from frappe.deprecation_dumpster import gzip_compress, gzip_decompress, make_esc
@@ -830,7 +831,7 @@ def get_site_info():
site_info.update(frappe.get_attr(method_name)(site_info) or {})
# dumps -> loads to prevent datatype conflicts
- return json.loads(frappe.as_json(site_info))
+ return orjson.loads(frappe.as_json(site_info))
def parse_json(val: str):
@@ -838,7 +839,7 @@ def parse_json(val: str):
Parses json if string else return
"""
if isinstance(val, str):
- val = json.loads(val)
+ val = orjson.loads(val)
if isinstance(val, dict):
val = frappe._dict(val)
return val
@@ -862,7 +863,7 @@ def get_db_count(*args):
for doctype in args:
db_count[doctype] = frappe.db.count(doctype)
- return json.loads(frappe.as_json(db_count))
+ return orjson.loads(frappe.as_json(db_count))
def call(fn, *args, **kwargs):
@@ -878,12 +879,12 @@ def call(fn, *args, **kwargs):
via terminal:
bench --site erpnext.local execute frappe.utils.call --args '''["frappe.get_all", "Activity Log"]''' --kwargs '''{"fields": ["user", "creation", "full_name"], "filters":{"Operation": "Login", "Status": "Success"}, "limit": "10"}'''
"""
- return json.loads(frappe.as_json(frappe.call(fn, *args, **kwargs)))
+ return orjson.loads(frappe.as_json(frappe.call(fn, *args, **kwargs)))
def get_safe_filters(filters):
try:
- filters = json.loads(filters)
+ filters = orjson.loads(filters)
if isinstance(filters, int | float):
filters = frappe.as_unicode(filters)
@@ -1043,7 +1044,7 @@ def safe_json_loads(*args):
for arg in args:
try:
- arg = json.loads(arg)
+ arg = orjson.loads(arg)
except Exception:
pass
From 2e5c8bea03871f3d71c34b56c6f8246a1e00ccf5 Mon Sep 17 00:00:00 2001
From: Sagar Vora <16315650+sagarvora@users.noreply.github.com>
Date: Thu, 26 Jun 2025 17:33:08 +0530
Subject: [PATCH 3/4] feat: `frappe.utils.orjson_dumps`
---
frappe/templates/base.html | 2 +-
frappe/utils/__init__.py | 11 ------
frappe/utils/data.py | 36 ++++++++++++++++++-
frappe/utils/response.py | 9 ++---
frappe/utils/safe_exec.py | 11 ++++++
.../doctype/web_form/templates/web_form.html | 2 +-
.../doctype/web_form/templates/web_list.html | 4 +--
frappe/www/app.html | 2 +-
8 files changed, 53 insertions(+), 24 deletions(-)
diff --git a/frappe/templates/base.html b/frappe/templates/base.html
index 02753571b1..8dd3f12a47 100644
--- a/frappe/templates/base.html
+++ b/frappe/templates/base.html
@@ -95,7 +95,7 @@
{% block base_scripts %}
diff --git a/frappe/utils/__init__.py b/frappe/utils/__init__.py
index 0bc0666922..aef04764b3 100644
--- a/frappe/utils/__init__.py
+++ b/frappe/utils/__init__.py
@@ -834,17 +834,6 @@ def get_site_info():
return orjson.loads(frappe.as_json(site_info))
-def parse_json(val: str):
- """
- Parses json if string else return
- """
- if isinstance(val, str):
- val = orjson.loads(val)
- if isinstance(val, dict):
- val = frappe._dict(val)
- return val
-
-
def get_db_count(*args):
"""
Pass a doctype or a series of doctypes to get the count of docs in them.
diff --git a/frappe/utils/data.py b/frappe/utils/data.py
index 12dbe8cbb3..ee01baa036 100644
--- a/frappe/utils/data.py
+++ b/frappe/utils/data.py
@@ -19,6 +19,7 @@ from typing import Any, Literal, Optional, TypeVar
from urllib.parse import parse_qsl, quote, urlencode, urljoin, urlparse, urlunparse
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
+import orjson
from click import secho
from dateutil import parser
from dateutil.parser import ParserError
@@ -82,6 +83,15 @@ DURATION_PATTERN = re.compile(r"^(?:(\d+d)?((^|\s)\d+h)?((^|\s)\d+m)?((^|\s)\d+s
HTML_TAG_PATTERN = re.compile("<[^>]+>")
MARIADB_SPECIFIC_COMMENT = re.compile(r"#.*")
+# these options are necessary to use orjson with frappe
+#
+# OPT_PASSTHROUGH_DATETIME allows datetime objects to be passed through
+# to the default function without conversion by orjson
+# frappe converts datetime objects differently (__str__) from orjson (RFC 3339)
+#
+# OPT_NON_STR_KEYS slightly reduces performance of orjson, but allows for non-string keys in dicts
+DEFAULT_ORJSON_OPTIONS = orjson.OPT_PASSTHROUGH_DATETIME | orjson.OPT_NON_STR_KEYS
+
class Weekday(Enum):
Sunday = 0
@@ -2439,11 +2449,35 @@ def guess_date_format(date_string: str) -> str:
def validate_json_string(string: str) -> None:
try:
- json.loads(string)
+ orjson.loads(string)
except (TypeError, ValueError):
raise frappe.ValidationError
+def parse_json(val: str):
+ """
+ Parses json if string else return
+ """
+ if isinstance(val, str):
+ val = orjson.loads(val)
+ if isinstance(val, dict):
+ val = frappe._dict(val)
+ return val
+
+
+def orjson_dumps(obj, default=None, option=None, decode=True):
+ """A wrapper around `orjson.dumps`, with some default options set"""
+
+ if option is not None:
+ # user defined options are merged with the default options
+ option = option | DEFAULT_ORJSON_OPTIONS
+ else:
+ option = DEFAULT_ORJSON_OPTIONS
+
+ value = orjson.dumps(obj, default, option)
+ return value.decode() if decode else value
+
+
class _UserInfo(typing.TypedDict):
email: str
image: str | None
diff --git a/frappe/utils/response.py b/frappe/utils/response.py
index 77ce8364f0..4265ac64e9 100644
--- a/frappe/utils/response.py
+++ b/frappe/utils/response.py
@@ -27,7 +27,7 @@ import frappe.sessions
import frappe.utils
from frappe import _
from frappe.core.doctype.access_log.access_log import make_access_log
-from frappe.utils import format_timedelta
+from frappe.utils import format_timedelta, orjson_dumps
if TYPE_CHECKING:
from frappe.core.doctype.file.file import File
@@ -150,7 +150,7 @@ def as_json():
del frappe.local.response["http_status_code"]
response.mimetype = "application/json"
- response.data = dump_response(frappe.local.response)
+ response.data = orjson_dumps(frappe.local.response, default=json_handler)
return response
@@ -259,11 +259,6 @@ def json_handler(obj):
raise TypeError(f"""Object of type {type(obj)} with value of {obj!r} is not JSON serializable""")
-dump_response = functools.partial(
- orjson.dumps, default=json_handler, option=orjson.OPT_PASSTHROUGH_DATETIME | orjson.OPT_NON_STR_KEYS
-)
-
-
def as_page():
"""print web page"""
from frappe.website.serve import get_response
diff --git a/frappe/utils/safe_exec.py b/frappe/utils/safe_exec.py
index 6cbc9201af..3b89408bcb 100644
--- a/frappe/utils/safe_exec.py
+++ b/frappe/utils/safe_exec.py
@@ -11,6 +11,7 @@ from itertools import chain
from types import FunctionType, MethodType, ModuleType
from typing import TYPE_CHECKING, Any
+import orjson
import RestrictedPython.Guards
from RestrictedPython import PrintCollector, compile_restricted, safe_globals
from RestrictedPython.transformer import RestrictingNodeTransformer
@@ -31,6 +32,7 @@ from frappe.model.rename_doc import rename_doc
from frappe.modules import scrub
from frappe.utils.background_jobs import enqueue, get_jobs
from frappe.utils.number_format import NumberFormat
+from frappe.utils.response import json_handler
from frappe.website.utils import get_next_link, get_toc
from frappe.www.printview import get_visible_columns
@@ -189,6 +191,7 @@ def get_safe_globals():
out = NamespaceDict(
# make available limited methods of frappe
json=NamespaceDict(loads=json.loads, dumps=json.dumps),
+ orjson=SAFE_ORJSON,
as_json=frappe.as_json,
dict=dict,
log=frappe.log,
@@ -278,6 +281,7 @@ def get_safe_globals():
get_html_content_based_on_type=frappe.website.utils.get_html_content_based_on_type,
),
lang=getattr(frappe.local, "lang", "en"),
+ json_handler=json_handler,
),
FrappeClient=FrappeClient,
style=frappe._dict(border_color="#d1d8dd"),
@@ -715,6 +719,8 @@ VALID_UTILS = (
"get_abbr",
"get_month",
"sha256_hash",
+ "parse_json",
+ "orjson_dumps",
)
@@ -729,3 +735,8 @@ WHITELISTED_SAFE_EVAL_GLOBALS = {
"_getiter_": iter,
"_iter_unpack_sequence_": RestrictedPython.Guards.guarded_iter_unpack_sequence,
}
+
+SAFE_ORJSON = NamespaceDict(loads=orjson.loads, dumps=orjson.dumps)
+for key, val in vars(orjson).items():
+ if key.startswith("OPT_"):
+ SAFE_ORJSON[key] = val
diff --git a/frappe/website/doctype/web_form/templates/web_form.html b/frappe/website/doctype/web_form/templates/web_form.html
index 48f22fa646..876e3d280b 100644
--- a/frappe/website/doctype/web_form/templates/web_form.html
+++ b/frappe/website/doctype/web_form/templates/web_form.html
@@ -163,7 +163,7 @@
{% block script %}
@@ -41,4 +41,4 @@
{{ custom_css }}
{% endif %}
-{% endblock %}
\ No newline at end of file
+{% endblock %}
diff --git a/frappe/www/app.html b/frappe/www/app.html
index 4f6d64ec4c..ec4cb99974 100644
--- a/frappe/www/app.html
+++ b/frappe/www/app.html
@@ -51,7 +51,7 @@
if (!window.frappe) window.frappe = {};
- frappe.boot = {{ boot | json }};
+ frappe.boot = {{ frappe.utils.orjson_dumps(boot, default=frappe.json_handler) }};
frappe._messages = frappe.boot["__messages"];
frappe.csrf_token = "{{ csrf_token }}";
From 7660f59c319c418b18e76b23d30b4cf92e6e760e Mon Sep 17 00:00:00 2001
From: Sagar Vora <16315650+sagarvora@users.noreply.github.com>
Date: Thu, 26 Jun 2025 17:37:24 +0530
Subject: [PATCH 4/4] perf(import_file): use `orjson.loads`
---
frappe/modules/import_file.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/frappe/modules/import_file.py b/frappe/modules/import_file.py
index 336652d5b7..eb8f53d0b6 100644
--- a/frappe/modules/import_file.py
+++ b/frappe/modules/import_file.py
@@ -1,9 +1,10 @@
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# License: MIT. See LICENSE
import hashlib
-import json
import os
+import orjson
+
import frappe
from frappe.model.base_document import get_controller
from frappe.modules import get_module_path, scrub_dt_dn
@@ -173,7 +174,7 @@ def read_doc_from_file(path):
if os.path.exists(path):
with open(path) as f:
try:
- doc = json.loads(f.read())
+ doc = orjson.loads(f.read())
except ValueError:
print(f"bad json: {path}")
raise