From c6580b588098377ee7d45b4a84b90b6ca498ba06 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 11 Oct 2024 19:51:08 +0200 Subject: [PATCH 1/7] refactor: Replace pytz to std lib zoneinfo & datetime Signed-off-by: Gavin D'souza --- frappe/core/doctype/rq_worker/rq_worker.py | 4 ++-- frappe/core/doctype/user/user.py | 4 ++-- frappe/email/frappemail.py | 6 ++---- .../doctype/token_cache/token_cache.py | 9 ++++----- frappe/monitor.py | 7 +++---- frappe/oauth.py | 1 - frappe/rate_limiter.py | 5 ++--- frappe/utils/caching.py | 12 +++++++----- frappe/utils/data.py | 16 ++++++---------- frappe/utils/scheduler.py | 3 +-- pyproject.toml | 1 - 11 files changed, 29 insertions(+), 39 deletions(-) diff --git a/frappe/core/doctype/rq_worker/rq_worker.py b/frappe/core/doctype/rq_worker/rq_worker.py index 2d5bc996e1..4f6210ca79 100644 --- a/frappe/core/doctype/rq_worker/rq_worker.py +++ b/frappe/core/doctype/rq_worker/rq_worker.py @@ -4,7 +4,6 @@ import datetime from contextlib import suppress -import pytz from rq import Worker import frappe @@ -104,6 +103,7 @@ def serialize_worker(worker: Worker) -> frappe._dict: def compute_utilization(worker: Worker) -> float: with suppress(Exception): total_time = ( - datetime.datetime.now(pytz.UTC) - worker.birth_date.replace(tzinfo=pytz.UTC) + datetime.datetime.now(datetime.timezone.utc) + - worker.birth_date.replace(tzinfo=datetime.timezone.utc) ).total_seconds() return worker.total_working_time / total_time * 100 diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py index 7954e04ff8..bcb8557c1d 100644 --- a/frappe/core/doctype/user/user.py +++ b/frappe/core/doctype/user/user.py @@ -808,9 +808,9 @@ class User(Document): @frappe.whitelist() def get_timezones(): - import pytz + import zoneinfo - return {"timezones": pytz.all_timezones} + return {"timezones": zoneinfo.available_timezones()} @frappe.whitelist() diff --git a/frappe/email/frappemail.py b/frappe/email/frappemail.py index 90223f338d..6f34b55ec5 100644 --- a/frappe/email/frappemail.py +++ b/frappe/email/frappemail.py @@ -1,8 +1,7 @@ from datetime import datetime from typing import Any from urllib.parse import urljoin - -import pytz +from zoneinfo import ZoneInfo import frappe from frappe import _ @@ -122,9 +121,8 @@ class FrappeMail: def add_or_update_tzinfo(date_time: datetime | str, timezone: str | None = None) -> str: """Adds or updates timezone to the datetime.""" - date_time = get_datetime(date_time) - target_tz = pytz.timezone(timezone or get_system_timezone()) + target_tz = ZoneInfo(timezone or get_system_timezone()) if date_time.tzinfo is None: date_time = target_tz.localize(date_time) diff --git a/frappe/integrations/doctype/token_cache/token_cache.py b/frappe/integrations/doctype/token_cache/token_cache.py index 4cff8bdab7..5be4134ef5 100644 --- a/frappe/integrations/doctype/token_cache/token_cache.py +++ b/frappe/integrations/doctype/token_cache/token_cache.py @@ -2,8 +2,7 @@ # License: MIT. See LICENSE import datetime - -import pytz +from zoneinfo import ZoneInfo import frappe from frappe import _ @@ -73,11 +72,11 @@ class TokenCache(Document): return self def get_expires_in(self): - system_timezone = pytz.timezone(get_system_timezone()) + system_timezone = ZoneInfo(get_system_timezone()) modified = frappe.utils.get_datetime(self.modified) modified = system_timezone.localize(modified) - expiry_utc = modified.astimezone(pytz.utc) + datetime.timedelta(seconds=self.expires_in) - now_utc = datetime.datetime.now(pytz.utc) + expiry_utc = modified.astimezone(datetime.timezone.utc) + datetime.timedelta(seconds=self.expires_in) + now_utc = datetime.datetime.now(datetime.timezone.utc) return cint((expiry_utc - now_utc).total_seconds()) def is_expired(self): diff --git a/frappe/monitor.py b/frappe/monitor.py index 522b743c4c..55512a8abd 100644 --- a/frappe/monitor.py +++ b/frappe/monitor.py @@ -7,7 +7,6 @@ import os import traceback import uuid -import pytz import rq import frappe @@ -52,7 +51,7 @@ class Monitor: self.data = frappe._dict( { "site": frappe.local.site, - "timestamp": datetime.datetime.now(pytz.UTC), + "timestamp": datetime.datetime.now(datetime.timezone.utc), "transaction_type": transaction_type, "uuid": str(uuid.uuid4()), } @@ -85,7 +84,7 @@ class Monitor: if job := rq.get_current_job(): self.data.uuid = job.id - waitdiff = self.data.timestamp - job.enqueued_at.replace(tzinfo=pytz.UTC) + waitdiff = self.data.timestamp - job.enqueued_at.replace(tzinfo=datetime.timezone.utc) self.data.job.wait = int(waitdiff.total_seconds() * 1000000) def add_custom_data(self, **kwargs): @@ -94,7 +93,7 @@ class Monitor: def dump(self, response=None): try: - timediff = datetime.datetime.now(pytz.UTC) - self.data.timestamp + timediff = datetime.datetime.now(datetime.timezone.utc) - self.data.timestamp # Obtain duration in microseconds self.data.duration = int(timediff.total_seconds() * 1000000) diff --git a/frappe/oauth.py b/frappe/oauth.py index 25f058017d..ddd01cc317 100644 --- a/frappe/oauth.py +++ b/frappe/oauth.py @@ -6,7 +6,6 @@ from http import cookies from urllib.parse import unquote, urljoin, urlparse import jwt -import pytz from oauthlib.openid import RequestValidator import frappe diff --git a/frappe/rate_limiter.py b/frappe/rate_limiter.py index fb26128723..8cdc4adb1f 100644 --- a/frappe/rate_limiter.py +++ b/frappe/rate_limiter.py @@ -5,7 +5,6 @@ import datetime from collections.abc import Callable from functools import wraps -import pytz from werkzeug.wrappers import Response import frappe @@ -35,7 +34,7 @@ class RateLimiter: self.limit = int(limit * 1000000) self.window = window - self.start = datetime.datetime.now(pytz.UTC) + self.start = datetime.datetime.now(datetime.timezone.utc) timestamp = int(frappe.utils.now_datetime().timestamp()) self.window_number, self.spent = divmod(timestamp, self.window) @@ -80,7 +79,7 @@ class RateLimiter: def record_request_end(self): if self.end is not None: return - self.end = datetime.datetime.now(pytz.UTC) + self.end = datetime.datetime.now(datetime.timezone.utc) self.duration = int((self.end - self.start).total_seconds() * 1000000) def respond(self): diff --git a/frappe/utils/caching.py b/frappe/utils/caching.py index cb788549fc..2299c95236 100644 --- a/frappe/utils/caching.py +++ b/frappe/utils/caching.py @@ -7,8 +7,6 @@ from collections import defaultdict from collections.abc import Callable from functools import wraps -import pytz - import frappe _SITE_CACHE = defaultdict(lambda: defaultdict(dict)) @@ -115,7 +113,9 @@ def site_cache(ttl: int | None = None, maxsize: int | None = None) -> Callable: if ttl is not None and not callable(ttl): func.ttl = ttl - func.expiration = datetime.datetime.now(pytz.UTC) + datetime.timedelta(seconds=func.ttl) + func.expiration = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta( + seconds=func.ttl + ) if maxsize is not None and not callable(maxsize): func.maxsize = maxsize @@ -125,9 +125,11 @@ def site_cache(ttl: int | None = None, maxsize: int | None = None) -> Callable: if getattr(frappe.local, "initialised", None): func_call_key = json.dumps((args, kwargs)) - if hasattr(func, "ttl") and datetime.datetime.now(pytz.UTC) >= func.expiration: + if hasattr(func, "ttl") and datetime.datetime.now(datetime.timezone.utc) >= func.expiration: func.clear_cache() - func.expiration = datetime.datetime.now(pytz.UTC) + datetime.timedelta(seconds=func.ttl) + func.expiration = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta( + seconds=func.ttl + ) if hasattr(func, "maxsize") and len(_SITE_CACHE[func_key][frappe.local.site]) >= func.maxsize: _SITE_CACHE[func_key][frappe.local.site].pop( diff --git a/frappe/utils/data.py b/frappe/utils/data.py index c5d62029ad..0de7ea92cf 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -15,8 +15,8 @@ from code import compile_command from enum import Enum from typing import Any, Literal, Optional, TypeVar from urllib.parse import parse_qsl, quote, urlencode, urljoin, urlparse, urlunparse +from zoneinfo import ZoneInfo -import pytz from click import secho from dateutil import parser from dateutil.parser import ParserError @@ -350,7 +350,7 @@ def time_diff_in_hours(string_ed_date: DateTimeLikeObject, string_st_date: DateT def now_datetime() -> datetime.datetime: """Return the current datetime in system timezone.""" - dt = convert_utc_to_system_timezone(datetime.datetime.now(pytz.UTC)) + dt = convert_utc_to_system_timezone(datetime.datetime.now(datetime.timezone.utc)) return dt.replace(tzinfo=None) @@ -372,19 +372,15 @@ def get_system_timezone() -> str: def convert_utc_to_timezone(utc_timestamp: datetime.datetime, time_zone: str) -> datetime.datetime: - from pytz import UnknownTimeZoneError, timezone - if utc_timestamp.tzinfo is None: - utc_timestamp = timezone("UTC").localize(utc_timestamp) - try: - return utc_timestamp.astimezone(timezone(time_zone)) - except UnknownTimeZoneError: - return utc_timestamp + utc_timestamp = datetime.datetime.now(ZoneInfo(time_zone)) + + return utc_timestamp def get_datetime_in_timezone(time_zone: str) -> datetime.datetime: """Return the current datetime in the given timezone (e.g. 'Asia/Kolkata').""" - utc_timestamp = datetime.datetime.now(pytz.UTC) + utc_timestamp = datetime.datetime.now(datetime.timezone.utc) return convert_utc_to_timezone(utc_timestamp, time_zone) diff --git a/frappe/utils/scheduler.py b/frappe/utils/scheduler.py index 5d023c8135..be3fec64fd 100644 --- a/frappe/utils/scheduler.py +++ b/frappe/utils/scheduler.py @@ -14,7 +14,6 @@ import random import time from typing import NoReturn -import pytz import setproctitle from croniter import CroniterBadCronError from filelock import FileLock, Timeout @@ -72,7 +71,7 @@ def sleep_duration(tick): # This makes scheduler aligned with real clock, # so event scheduled at 12:00 happen at 12:00 and not 12:00:35. minutes = tick // 60 - now = datetime.datetime.now(pytz.UTC) + now = datetime.datetime.now(datetime.timezone.utc) left_minutes = minutes - now.minute % minutes next_execution = now.replace(second=0) + datetime.timedelta(minutes=left_minutes) diff --git a/pyproject.toml b/pyproject.toml index 0156e4a64d..c0cae33fe2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,7 +57,6 @@ dependencies = [ "pydantic~=2.10.2", "pyotp~=2.8.0", "python-dateutil~=2.8.2", - "pytz==2023.3", "rauth~=0.7.3", "redis~=5.2.0", "hiredis~=3.0.0", From 7e2e5f80b9dac60c7a1c7f5bda28b1d0d6c409f4 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 17 Oct 2024 14:57:14 +0200 Subject: [PATCH 2/7] fix: Usage of tzinfo replace when no tz is specified --- frappe/email/frappemail.py | 2 +- frappe/integrations/doctype/token_cache/token_cache.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/frappe/email/frappemail.py b/frappe/email/frappemail.py index 6f34b55ec5..e653d46039 100644 --- a/frappe/email/frappemail.py +++ b/frappe/email/frappemail.py @@ -125,7 +125,7 @@ def add_or_update_tzinfo(date_time: datetime | str, timezone: str | None = None) target_tz = ZoneInfo(timezone or get_system_timezone()) if date_time.tzinfo is None: - date_time = target_tz.localize(date_time) + date_time = date_time.replace(tzinfo=target_tz) else: date_time = date_time.astimezone(target_tz) diff --git a/frappe/integrations/doctype/token_cache/token_cache.py b/frappe/integrations/doctype/token_cache/token_cache.py index 5be4134ef5..b9ec45ac6f 100644 --- a/frappe/integrations/doctype/token_cache/token_cache.py +++ b/frappe/integrations/doctype/token_cache/token_cache.py @@ -7,7 +7,7 @@ from zoneinfo import ZoneInfo import frappe from frappe import _ from frappe.model.document import Document -from frappe.utils import cint, cstr, get_system_timezone +from frappe.utils import cint, cstr, get_system_timezone, get_datetime class TokenCache(Document): @@ -73,8 +73,7 @@ class TokenCache(Document): def get_expires_in(self): system_timezone = ZoneInfo(get_system_timezone()) - modified = frappe.utils.get_datetime(self.modified) - modified = system_timezone.localize(modified) + modified: datetime.datetime = get_datetime(self.modified).replace(tzinfo=system_timezone) expiry_utc = modified.astimezone(datetime.timezone.utc) + datetime.timedelta(seconds=self.expires_in) now_utc = datetime.datetime.now(datetime.timezone.utc) return cint((expiry_utc - now_utc).total_seconds()) From 00163f5bf414e432de5c70d0cb8227c932f6787f Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 17 Oct 2024 15:59:31 +0200 Subject: [PATCH 3/7] fix: Re-define utils to match previous behaviour --- frappe/tests/classes/context_managers.py | 8 ++++---- frappe/utils/data.py | 7 ++++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/frappe/tests/classes/context_managers.py b/frappe/tests/classes/context_managers.py index dcd49896bd..f259978c6a 100644 --- a/frappe/tests/classes/context_managers.py +++ b/frappe/tests/classes/context_managers.py @@ -2,7 +2,6 @@ import logging from collections.abc import Callable from contextlib import contextmanager from functools import wraps -from inspect import isfunction, ismethod from typing import TYPE_CHECKING, Any import frappe @@ -28,15 +27,16 @@ logger = logging.Logger(__file__) @contextmanager def freeze_time(time_to_freeze: Any, is_utc: bool = False, *args: Any, **kwargs: Any) -> None: """Temporarily: freeze time with freezegun.""" - import pytz + from datetime import UTC + from zoneinfo import ZoneInfo + from freezegun import freeze_time as freezegun_freeze_time from frappe.utils.data import get_datetime, get_system_timezone if not is_utc: # Freeze time expects UTC or tzaware objects. We have neither, so convert to UTC. - timezone = pytz.timezone(get_system_timezone()) - time_to_freeze = timezone.localize(get_datetime(time_to_freeze)).astimezone(pytz.utc) + time_to_freeze = get_datetime(time_to_freeze).replace(tzinfo=ZoneInfo(get_system_timezone())).astimezone(UTC) with freezegun_freeze_time(time_to_freeze, *args, **kwargs): yield diff --git a/frappe/utils/data.py b/frappe/utils/data.py index 0de7ea92cf..e3855984ba 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -350,8 +350,7 @@ def time_diff_in_hours(string_ed_date: DateTimeLikeObject, string_st_date: DateT def now_datetime() -> datetime.datetime: """Return the current datetime in system timezone.""" - dt = convert_utc_to_system_timezone(datetime.datetime.now(datetime.timezone.utc)) - return dt.replace(tzinfo=None) + return datetime.datetime.now(ZoneInfo(get_system_timezone())).replace(tzinfo=None) def get_timestamp(date: Optional["DateTimeLikeObject"] = None) -> float: @@ -373,7 +372,9 @@ def get_system_timezone() -> str: def convert_utc_to_timezone(utc_timestamp: datetime.datetime, time_zone: str) -> datetime.datetime: if utc_timestamp.tzinfo is None: - utc_timestamp = datetime.datetime.now(ZoneInfo(time_zone)) + utc_timestamp = utc_timestamp.replace(tzinfo=ZoneInfo(time_zone)) + else: + utc_timestamp = utc_timestamp.astimezone(ZoneInfo(time_zone)) return utc_timestamp From d521ac124d890676d23a8c8b3c065ae0f870b552 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Fri, 18 Oct 2024 15:28:09 +0200 Subject: [PATCH 4/7] style: Sort imports & move whitespaces to please ruff --- frappe/integrations/doctype/token_cache/token_cache.py | 2 +- frappe/tests/classes/context_managers.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/frappe/integrations/doctype/token_cache/token_cache.py b/frappe/integrations/doctype/token_cache/token_cache.py index b9ec45ac6f..1a12c44008 100644 --- a/frappe/integrations/doctype/token_cache/token_cache.py +++ b/frappe/integrations/doctype/token_cache/token_cache.py @@ -7,7 +7,7 @@ from zoneinfo import ZoneInfo import frappe from frappe import _ from frappe.model.document import Document -from frappe.utils import cint, cstr, get_system_timezone, get_datetime +from frappe.utils import cint, cstr, get_datetime, get_system_timezone class TokenCache(Document): diff --git a/frappe/tests/classes/context_managers.py b/frappe/tests/classes/context_managers.py index f259978c6a..4ed60151a2 100644 --- a/frappe/tests/classes/context_managers.py +++ b/frappe/tests/classes/context_managers.py @@ -36,7 +36,9 @@ def freeze_time(time_to_freeze: Any, is_utc: bool = False, *args: Any, **kwargs: if not is_utc: # Freeze time expects UTC or tzaware objects. We have neither, so convert to UTC. - time_to_freeze = get_datetime(time_to_freeze).replace(tzinfo=ZoneInfo(get_system_timezone())).astimezone(UTC) + time_to_freeze = ( + get_datetime(time_to_freeze).replace(tzinfo=ZoneInfo(get_system_timezone())).astimezone(UTC) + ) with freezegun_freeze_time(time_to_freeze, *args, **kwargs): yield From 401f6dedbd31e0dcab4d04cfc895733c72d74dea Mon Sep 17 00:00:00 2001 From: Akhil Narang Date: Fri, 6 Dec 2024 15:44:51 +0530 Subject: [PATCH 5/7] chore(deps): add back pytz Too many apps depend on it for now Will drop the actual dependency later Signed-off-by: Akhil Narang --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index c0cae33fe2..0156e4a64d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,6 +57,7 @@ dependencies = [ "pydantic~=2.10.2", "pyotp~=2.8.0", "python-dateutil~=2.8.2", + "pytz==2023.3", "rauth~=0.7.3", "redis~=5.2.0", "hiredis~=3.0.0", From ad1ed626526083ad16cb157d9718922405be8762 Mon Sep 17 00:00:00 2001 From: Akhil Narang Date: Fri, 6 Dec 2024 16:01:43 +0530 Subject: [PATCH 6/7] fix: handle `ZoneInfoNotFoundError` Signed-off-by: Akhil Narang --- frappe/utils/data.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index e3855984ba..8a00b336f1 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -15,7 +15,7 @@ from code import compile_command from enum import Enum from typing import Any, Literal, Optional, TypeVar from urllib.parse import parse_qsl, quote, urlencode, urljoin, urlparse, urlunparse -from zoneinfo import ZoneInfo +from zoneinfo import ZoneInfo, ZoneInfoNotFoundError from click import secho from dateutil import parser @@ -373,10 +373,11 @@ def get_system_timezone() -> str: def convert_utc_to_timezone(utc_timestamp: datetime.datetime, time_zone: str) -> datetime.datetime: if utc_timestamp.tzinfo is None: utc_timestamp = utc_timestamp.replace(tzinfo=ZoneInfo(time_zone)) - else: - utc_timestamp = utc_timestamp.astimezone(ZoneInfo(time_zone)) - return utc_timestamp + try: + return utc_timestamp.astimezone(ZoneInfo(time_zone)) + except ZoneInfoNotFoundError: + return utc_timestamp def get_datetime_in_timezone(time_zone: str) -> datetime.datetime: From 94fe90de665f04402d80a6e4e9225a834d2e24db Mon Sep 17 00:00:00 2001 From: Akhil Narang Date: Fri, 6 Dec 2024 16:04:55 +0530 Subject: [PATCH 7/7] refactor: pytz -> ZoneInfo Signed-off-by: Akhil Narang --- frappe/deprecation_dumpster.py | 10 +++++++--- frappe/tests/test_utils.py | 7 +++---- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/frappe/deprecation_dumpster.py b/frappe/deprecation_dumpster.py index 5ede7aef91..9de96e3785 100644 --- a/frappe/deprecation_dumpster.py +++ b/frappe/deprecation_dumpster.py @@ -814,15 +814,19 @@ def get_tests_CompatFrappeTestCase(): @contextmanager def freeze_time(self, time_to_freeze, is_utc=False, *args, **kwargs): - import pytz + from zoneinfo import ZoneInfo + from freezegun import freeze_time from frappe.utils.data import convert_utc_to_timezone, get_datetime, get_system_timezone if not is_utc: # Freeze time expects UTC or tzaware objects. We have neither, so convert to UTC. - timezone = pytz.timezone(get_system_timezone()) - time_to_freeze = timezone.localize(get_datetime(time_to_freeze)).astimezone(pytz.utc) + time_to_freeze = ( + get_datetime(time_to_freeze) + .replace(tzinfo=ZoneInfo(get_system_timezone())) + .astimezone(ZoneInfo("UTC")) + ) with freeze_time(time_to_freeze, *args, **kwargs): yield diff --git a/frappe/tests/test_utils.py b/frappe/tests/test_utils.py index 26b0b85aeb..0575ddab55 100644 --- a/frappe/tests/test_utils.py +++ b/frappe/tests/test_utils.py @@ -5,14 +5,13 @@ import io import json import os import sys -from datetime import date, datetime, time, timedelta +from datetime import date, datetime, time, timedelta, timezone from decimal import ROUND_HALF_UP, Decimal, localcontext from enum import Enum from io import StringIO from mimetypes import guess_type from unittest.mock import patch -import pytz from hypothesis import given from hypothesis import strategies as st from PIL import Image @@ -736,9 +735,9 @@ class TestResponse(IntegrationTestCase): minute=23, second=23, microsecond=23, - tzinfo=pytz.utc, + tzinfo=timezone.utc, ), - time(hour=23, minute=23, second=23, microsecond=23, tzinfo=pytz.utc), + time(hour=23, minute=23, second=23, microsecond=23, tzinfo=timezone.utc), timedelta(days=10, hours=12, minutes=120, seconds=10), ], "float": [