diff --git a/frappe/__init__.py b/frappe/__init__.py index a4ac8111dc..998d881a13 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -11,6 +11,7 @@ be used to build database driven apps. Read the documentation: https://frappeframework.com/docs """ import functools +import gc import importlib import inspect import json @@ -57,6 +58,7 @@ re._MAXCACHE = ( 50 # reduced from default 512 given we are already maintaining this on parent worker ) +_tune_gc = bool(os.environ.get("FRAPPE_TUNE_GC", False)) if _dev_server: warnings.simplefilter("always", DeprecationWarning) @@ -2435,3 +2437,13 @@ def validate_and_sanitize_search_inputs(fn): return fn(**kwargs) return wrapper + + +if _tune_gc: + # generational GC gets triggered after certain allocs (g0) which is 700 by default. + # This number is quite small for frappe where a single query can potentially create 700+ + # objects easily. + # Bump this number higher, this will make GC less aggressive but that improves performance of + # everything else. + g0, g1, g2 = gc.get_threshold() # defaults are 700, 10, 10. + gc.set_threshold(g0 * 10, g1 * 2, g2 * 2) diff --git a/frappe/app.py b/frappe/app.py index ddde313ace..a698331e8e 100644 --- a/frappe/app.py +++ b/frappe/app.py @@ -1,6 +1,7 @@ # Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors # License: MIT. See LICENSE +import gc import logging import os @@ -394,3 +395,17 @@ def serve( use_evalex=not in_test_env, threaded=not no_threading, ) + + +# Both Gunicorn and RQ use forking to spawn workers. In an ideal world, the fork should be sharing +# most of the memory if there are no writes made to data because of Copy on Write, however, +# python's GC is not CoW friendly and writes to data even if user-code doesn't. Specifically, the +# generational GC which stores and mutates every python object: `PyGC_Head` +# +# Calling gc.freeze() moves all the objects imported so far into permanant generation and hence +# doesn't mutate `PyGC_Head` +# +# Refer to issue for more info: https://github.com/frappe/frappe/issues/18927 +if frappe._tune_gc: + gc.collect() # clean up any garbage created so far before freeze + gc.freeze() diff --git a/frappe/core/doctype/file/utils.py b/frappe/core/doctype/file/utils.py index 7b44e94271..1d0d145303 100644 --- a/frappe/core/doctype/file/utils.py +++ b/frappe/core/doctype/file/utils.py @@ -7,6 +7,8 @@ from typing import TYPE_CHECKING, Optional from urllib.parse import unquote import filetype +import requests +import requests.exceptions from PIL import Image import frappe @@ -114,9 +116,6 @@ def get_local_image(file_url: str) -> tuple["ImageFile", str, str]: def get_web_image(file_url: str) -> tuple["ImageFile", str, str]: - import requests - import requests.exceptions - # download file_url = frappe.utils.get_url(file_url) r = requests.get(file_url, stream=True) diff --git a/frappe/integrations/doctype/slack_webhook_url/slack_webhook_url.py b/frappe/integrations/doctype/slack_webhook_url/slack_webhook_url.py index 4ea11aaccb..d71d7075a6 100644 --- a/frappe/integrations/doctype/slack_webhook_url/slack_webhook_url.py +++ b/frappe/integrations/doctype/slack_webhook_url/slack_webhook_url.py @@ -3,6 +3,8 @@ import json +import requests + import frappe from frappe import _ from frappe.model.document import Document @@ -22,8 +24,6 @@ class SlackWebhookURL(Document): def send_slack_message(webhook_url, message, reference_doctype, reference_name): - import requests - data = {"text": message, "attachments": []} slack_url, show_link = frappe.db.get_value( diff --git a/frappe/integrations/doctype/webhook/webhook.py b/frappe/integrations/doctype/webhook/webhook.py index 9e96870857..6fa24bfb67 100644 --- a/frappe/integrations/doctype/webhook/webhook.py +++ b/frappe/integrations/doctype/webhook/webhook.py @@ -5,10 +5,11 @@ import base64 import hashlib import hmac import json -import typing from time import sleep from urllib.parse import urlparse +import requests + import frappe from frappe import _ from frappe.model.document import Document @@ -17,9 +18,6 @@ from frappe.utils.safe_exec import get_safe_globals WEBHOOK_SECRET_HEADER = "X-Frappe-Webhook-Signature" -if typing.TYPE_CHECKING: - import requests - class Webhook(Document): def validate(self): @@ -114,8 +112,6 @@ def get_context(doc): def enqueue_webhook(doc, webhook) -> None: - import requests - webhook: Webhook = frappe.get_doc("Webhook", webhook.get("name")) headers = get_webhook_headers(doc, webhook) data = get_webhook_data(doc, webhook) @@ -158,7 +154,7 @@ def log_request( url: str, headers: dict, data: dict, - res: typing.Optional["requests.Response"] = None, + res: requests.Response | None = None, ): request_log = frappe.get_doc( { diff --git a/frappe/integrations/google_oauth.py b/frappe/integrations/google_oauth.py index 4a6f108150..8bc54e0b1d 100644 --- a/frappe/integrations/google_oauth.py +++ b/frappe/integrations/google_oauth.py @@ -2,6 +2,7 @@ import json from google.oauth2.credentials import Credentials from googleapiclient.discovery import build +from requests import get, post import frappe from frappe.utils import get_request_site_address @@ -55,8 +56,6 @@ class GoogleOAuth: frappe.throw(frappe._("Please update {} before continuing.").format(google_settings)) def authorize(self, oauth_code: str) -> dict[str, str | int]: - import requests - """Returns a dict with access and refresh token. :param oauth_code: code got back from google upon successful auhtorization @@ -74,14 +73,13 @@ class GoogleOAuth: } return handle_response( - requests.post(self.OAUTH_URL, data=data).json(), + post(self.OAUTH_URL, data=data).json(), "Google Oauth Authorization Error", "Something went wrong during the authorization.", ) def refresh_access_token(self, refresh_token: str) -> dict[str, str | int]: """Refreshes google access token using refresh token""" - import requests data = { "client_id": self.google_settings.client_id, @@ -94,7 +92,7 @@ class GoogleOAuth: } return handle_response( - requests.post(self.OAUTH_URL, data=data).json(), + post(self.OAUTH_URL, data=data).json(), "Google Oauth Access Token Refresh Error", "Something went wrong during the access token generation.", raise_err=True, @@ -160,9 +158,7 @@ def handle_response( def is_valid_access_token(access_token: str) -> bool: - import requests - - response = requests.get( + response = get( "https://oauth2.googleapis.com/tokeninfo", params={"access_token": access_token} ).json() diff --git a/frappe/utils/background_jobs.py b/frappe/utils/background_jobs.py index ea3ea4c191..6b0249e720 100755 --- a/frappe/utils/background_jobs.py +++ b/frappe/utils/background_jobs.py @@ -1,3 +1,4 @@ +import gc import os import socket import time @@ -234,6 +235,10 @@ def start_worker( """Wrapper to start rq worker. Connects to redis and monitors these queues.""" DEQUEUE_STRATEGIES = {"round_robin": RoundRobinWorker, "random": RandomWorker} + if frappe._tune_gc: + gc.collect() + gc.freeze() + with frappe.init_site(): # empty init is required to get redis_queue from common_site_config.json redis_connection = get_redis_conn(username=rq_username, password=rq_password) diff --git a/frappe/utils/csvutils.py b/frappe/utils/csvutils.py index 86a0e9776f..4840c806bb 100644 --- a/frappe/utils/csvutils.py +++ b/frappe/utils/csvutils.py @@ -4,6 +4,8 @@ import csv import json from io import StringIO +import requests + import frappe from frappe import _, msgprint from frappe.utils import cint, comma_or, cstr, flt @@ -176,8 +178,6 @@ def getlink(doctype, name): def get_csv_content_from_google_sheets(url): - import requests - # https://docs.google.com/spreadsheets/d/{sheetid}}/edit#gid={gid} validate_google_sheets_url(url) # get gid, defaults to first sheet diff --git a/frappe/www/login.py b/frappe/www/login.py index 0aa1eba0cd..7cf89e7c3c 100644 --- a/frappe/www/login.py +++ b/frappe/www/login.py @@ -5,7 +5,6 @@ import frappe import frappe.utils from frappe import _ from frappe.auth import LoginManager -from frappe.integrations.doctype.ldap_settings.ldap_settings import LDAPSettings from frappe.rate_limiter import rate_limit from frappe.utils import cint, get_url from frappe.utils.data import escape_html @@ -85,7 +84,10 @@ def get_context(context): ) context["social_login"] = True - context["ldap_settings"] = LDAPSettings.get_ldap_client_settings() + if cint(frappe.db.get_value("LDAP Settings", "LDAP Settings", "enabled")): + from frappe.integrations.doctype.ldap_settings.ldap_settings import LDAPSettings + + context["ldap_settings"] = LDAPSettings.get_ldap_client_settings() login_label = [_("Email")]