81 lines
2.6 KiB
Python
81 lines
2.6 KiB
Python
import faulthandler
|
|
import gc
|
|
import io
|
|
import os
|
|
import re
|
|
import signal
|
|
import sys
|
|
|
|
|
|
def optimize_all():
|
|
"""Single entry point to enable all optimizations at right time automatically."""
|
|
|
|
# Note:
|
|
# - This function is ALWAYS executed as soon as `import frappe` ends.
|
|
# - Any deferred work should be deferred using os module's fork hooks.
|
|
# - Respect configurations using environement variables.
|
|
# - fork hooks can not be unregistered, so care should be taken to execute them only when they
|
|
# make sense.
|
|
_optimize_regex_cache()
|
|
_optimize_gc_parameters()
|
|
_optimize_gc_for_copy_on_write()
|
|
_register_fault_handler()
|
|
os.register_at_fork(after_in_child=_register_fault_handler)
|
|
|
|
|
|
def _optimize_gc_parameters():
|
|
from frappe.utils import sbool
|
|
|
|
if not bool(sbool(os.environ.get("FRAPPE_TUNE_GC", True))):
|
|
return
|
|
|
|
# generational GC gets triggered after certain allocs (g0) which is 700 by default.
|
|
# This number is quite small for frappe where a single query can potentially create 700+
|
|
# objects easily.
|
|
# Bump this number higher, this will make GC less aggressive but that improves performance of
|
|
# everything else.
|
|
g0, g1, g2 = gc.get_threshold() # defaults are 700, 10, 10.
|
|
gc.set_threshold(g0 * 10, g1 * 2, g2 * 2)
|
|
|
|
|
|
def _optimize_regex_cache():
|
|
# Remove references to pattern that are pre-compiled and loaded to global scopes.
|
|
# Leave that cache for dynamically generated regex.
|
|
os.register_at_fork(before=re.purge)
|
|
|
|
|
|
def _register_fault_handler():
|
|
# Some libraries monkey patch stderr, we need actual fd
|
|
if isinstance(sys.__stderr__, io.TextIOWrapper):
|
|
faulthandler.register(signal.SIGUSR1, file=sys.__stderr__)
|
|
|
|
|
|
def _optimize_gc_for_copy_on_write():
|
|
from frappe.utils import sbool
|
|
|
|
if not bool(sbool(os.environ.get("FRAPPE_TUNE_GC", True))):
|
|
return
|
|
|
|
os.register_at_fork(before=_freeze_gc)
|
|
|
|
|
|
_gc_frozen = False
|
|
|
|
|
|
def _freeze_gc():
|
|
global _gc_frozen
|
|
if _gc_frozen:
|
|
return
|
|
# Both Gunicorn and RQ use forking to spawn workers. In an ideal world, the fork should be sharing
|
|
# most of the memory if there are no writes made to data because of Copy on Write, however,
|
|
# python's GC is not CoW friendly and writes to data even if user-code doesn't. Specifically, the
|
|
# generational GC which stores and mutates every python object: `PyGC_Head`
|
|
#
|
|
# Calling gc.freeze() moves all the objects imported so far into permanant generation and hence
|
|
# doesn't mutate `PyGC_Head`
|
|
#
|
|
# Refer to issue for more info: https://github.com/frappe/frappe/issues/18927
|
|
gc.collect()
|
|
gc.freeze()
|
|
# RQ workers constantly fork, there' no benefit in doing this in that case.
|
|
_gc_frozen = True
|