Merge pull request #21474 from ankush/gc_freeze

perf: Freeze GC before forking Gunicorn workers

frappe/__init__.py

@@ -11,6 +11,7 @@ be used to build database driven apps.
 Read the documentation: https://frappeframework.com/docs
 """
 import functools
+import gc
 import importlib
 import inspect
 import json
@@ -57,6 +58,7 @@ re._MAXCACHE = (
 	50  # reduced from default 512 given we are already maintaining this on parent worker
 )
 
+_tune_gc = bool(os.environ.get("FRAPPE_TUNE_GC", False))
 
 if _dev_server:
 	warnings.simplefilter("always", DeprecationWarning)
@@ -2435,3 +2437,13 @@ def validate_and_sanitize_search_inputs(fn):
 		return fn(**kwargs)
 
 	return wrapper
+
+
+if _tune_gc:
+	# generational GC gets triggered after certain allocs (g0) which is 700 by default.
+	# This number is quite small for frappe where a single query can potentially create 700+
+	# objects easily.
+	# Bump this number higher, this will make GC less aggressive but that improves performance of
+	# everything else.
+	g0, g1, g2 = gc.get_threshold()  # defaults are 700, 10, 10.
+	gc.set_threshold(g0 * 10, g1 * 2, g2 * 2)

frappe/app.py

@@ -1,6 +1,7 @@
 # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
 # License: MIT. See LICENSE
 
+import gc
 import logging
 import os
 
@@ -394,3 +395,17 @@ def serve(
 		use_evalex=not in_test_env,
 		threaded=not no_threading,
 	)
+
+
+# Both Gunicorn and RQ use forking to spawn workers. In an ideal world, the fork should be sharing
+# most of the memory if there are no writes made to data because of Copy on Write, however,
+# python's GC is not CoW friendly and writes to data even if user-code doesn't. Specifically, the
+# generational GC which stores and mutates every python object: `PyGC_Head`
+#
+# Calling gc.freeze() moves all the objects imported so far into permanant generation and hence
+# doesn't mutate `PyGC_Head`
+#
+# Refer to issue for more info: https://github.com/frappe/frappe/issues/18927
+if frappe._tune_gc:
+	gc.collect()  # clean up any garbage created so far before freeze
+	gc.freeze()

frappe/core/doctype/file/utils.py

@@ -7,6 +7,8 @@ from typing import TYPE_CHECKING, Optional
 from urllib.parse import unquote
 
 import filetype
+import requests
+import requests.exceptions
 from PIL import Image
 
 import frappe
@@ -114,9 +116,6 @@ def get_local_image(file_url: str) -> tuple["ImageFile", str, str]:
 
 
 def get_web_image(file_url: str) -> tuple["ImageFile", str, str]:
-	import requests
-	import requests.exceptions
-
 	# download
 	file_url = frappe.utils.get_url(file_url)
 	r = requests.get(file_url, stream=True)

frappe/integrations/doctype/slack_webhook_url/slack_webhook_url.py

@@ -3,6 +3,8 @@
 
 import json
 
+import requests
+
 import frappe
 from frappe import _
 from frappe.model.document import Document
@@ -22,8 +24,6 @@ class SlackWebhookURL(Document):
 
 
 def send_slack_message(webhook_url, message, reference_doctype, reference_name):
-	import requests
-
 	data = {"text": message, "attachments": []}
 
 	slack_url, show_link = frappe.db.get_value(

frappe/integrations/doctype/webhook/webhook.py

@@ -5,10 +5,11 @@ import base64
 import hashlib
 import hmac
 import json
-import typing
 from time import sleep
 from urllib.parse import urlparse
 
+import requests
+
 import frappe
 from frappe import _
 from frappe.model.document import Document
@@ -17,9 +18,6 @@ from frappe.utils.safe_exec import get_safe_globals
 
 WEBHOOK_SECRET_HEADER = "X-Frappe-Webhook-Signature"
 
-if typing.TYPE_CHECKING:
-	import requests
-
 
 class Webhook(Document):
 	def validate(self):
@@ -114,8 +112,6 @@ def get_context(doc):
 
 
 def enqueue_webhook(doc, webhook) -> None:
-	import requests
-
 	webhook: Webhook = frappe.get_doc("Webhook", webhook.get("name"))
 	headers = get_webhook_headers(doc, webhook)
 	data = get_webhook_data(doc, webhook)
@@ -158,7 +154,7 @@ def log_request(
 	url: str,
 	headers: dict,
 	data: dict,
-	res: typing.Optional["requests.Response"] = None,
+	res: requests.Response | None = None,
 ):
 	request_log = frappe.get_doc(
 		{

frappe/integrations/google_oauth.py

@@ -2,6 +2,7 @@ import json
 
 from google.oauth2.credentials import Credentials
 from googleapiclient.discovery import build
+from requests import get, post
 
 import frappe
 from frappe.utils import get_request_site_address
@@ -55,8 +56,6 @@ class GoogleOAuth:
 			frappe.throw(frappe._("Please update {} before continuing.").format(google_settings))
 
 	def authorize(self, oauth_code: str) -> dict[str, str | int]:
-		import requests
-
 		"""Returns a dict with access and refresh token.
 
 		:param oauth_code: code got back from google upon successful auhtorization
@@ -74,14 +73,13 @@ class GoogleOAuth:
 		}
 
 		return handle_response(
-			requests.post(self.OAUTH_URL, data=data).json(),
+			post(self.OAUTH_URL, data=data).json(),
 			"Google Oauth Authorization Error",
 			"Something went wrong during the authorization.",
 		)
 
 	def refresh_access_token(self, refresh_token: str) -> dict[str, str | int]:
 		"""Refreshes google access token using refresh token"""
-		import requests
 
 		data = {
 			"client_id": self.google_settings.client_id,
@@ -94,7 +92,7 @@ class GoogleOAuth:
 		}
 
 		return handle_response(
-			requests.post(self.OAUTH_URL, data=data).json(),
+			post(self.OAUTH_URL, data=data).json(),
 			"Google Oauth Access Token Refresh Error",
 			"Something went wrong during the access token generation.",
 			raise_err=True,
@@ -160,9 +158,7 @@ def handle_response(
 
 
 def is_valid_access_token(access_token: str) -> bool:
-	import requests
-
-	response = requests.get(
+	response = get(
 		"https://oauth2.googleapis.com/tokeninfo", params={"access_token": access_token}
 	).json()
 

frappe/utils/background_jobs.py

@@ -1,3 +1,4 @@
+import gc
 import os
 import socket
 import time
@@ -234,6 +235,10 @@ def start_worker(
 	"""Wrapper to start rq worker. Connects to redis and monitors these queues."""
 	DEQUEUE_STRATEGIES = {"round_robin": RoundRobinWorker, "random": RandomWorker}
 
+	if frappe._tune_gc:
+		gc.collect()
+		gc.freeze()
+
 	with frappe.init_site():
 		# empty init is required to get redis_queue from common_site_config.json
 		redis_connection = get_redis_conn(username=rq_username, password=rq_password)

frappe/utils/csvutils.py

@@ -4,6 +4,8 @@ import csv
 import json
 from io import StringIO
 
+import requests
+
 import frappe
 from frappe import _, msgprint
 from frappe.utils import cint, comma_or, cstr, flt
@@ -176,8 +178,6 @@ def getlink(doctype, name):
 
 
 def get_csv_content_from_google_sheets(url):
-	import requests
-
 	# https://docs.google.com/spreadsheets/d/{sheetid}}/edit#gid={gid}
 	validate_google_sheets_url(url)
 	# get gid, defaults to first sheet

frappe/www/login.py

@@ -5,7 +5,6 @@ import frappe
 import frappe.utils
 from frappe import _
 from frappe.auth import LoginManager
-from frappe.integrations.doctype.ldap_settings.ldap_settings import LDAPSettings
 from frappe.rate_limiter import rate_limit
 from frappe.utils import cint, get_url
 from frappe.utils.data import escape_html
@@ -85,7 +84,10 @@ def get_context(context):
 			)
 			context["social_login"] = True
 
-	context["ldap_settings"] = LDAPSettings.get_ldap_client_settings()
+	if cint(frappe.db.get_value("LDAP Settings", "LDAP Settings", "enabled")):
+		from frappe.integrations.doctype.ldap_settings.ldap_settings import LDAPSettings
+
+		context["ldap_settings"] = LDAPSettings.get_ldap_client_settings()
 
 	login_label = [_("Email")]