From c80f98c84ce4b936aa0590e1cf63f285cdd1b12b Mon Sep 17 00:00:00 2001 From: Leonard Goertz <49870752+uepselon@users.noreply.github.com> Date: Thu, 2 Mar 2023 13:10:41 +0100 Subject: [PATCH 01/24] fix: frappe.route_options set in quick_list --- frappe/public/js/frappe/widgets/quick_list_widget.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frappe/public/js/frappe/widgets/quick_list_widget.js b/frappe/public/js/frappe/widgets/quick_list_widget.js index a406075af2..90b79ad3f0 100644 --- a/frappe/public/js/frappe/widgets/quick_list_widget.js +++ b/frappe/public/js/frappe/widgets/quick_list_widget.js @@ -241,9 +241,6 @@ export default class QuickListWidget extends Widget { this.footer.empty(); let filters = frappe.utils.get_filter_from_json(this.quick_list_filter); - if (filters) { - frappe.route_options = filters; - } let route = frappe.utils.generate_route({ type: "doctype", name: this.document_type }); this.see_all_button = $(`
${__("View List")}
@@ -253,6 +250,9 @@ export default class QuickListWidget extends Widget { if (e.ctrlKey || e.metaKey) { frappe.open_in_new_tab = true; } + if (filters) { + frappe.route_options = filters; + } frappe.set_route(route); }); } From 3694e654a155c4fda183792897960c986f257031 Mon Sep 17 00:00:00 2001 From: barredterra <14891507+barredterra@users.noreply.github.com> Date: Sat, 4 Mar 2023 19:02:25 +0100 Subject: [PATCH 02/24] refactor: rename `convert_utc_to_user_timezone` to `convert_utc_to_system_timezone` --- frappe/core/doctype/rq_job/rq_job.py | 10 +++++----- frappe/core/doctype/rq_worker/rq_worker.py | 10 +++++----- frappe/desk/page/backups/backups.py | 4 ++-- frappe/email/receive.py | 4 ++-- frappe/utils/data.py | 12 ++++++++++-- 5 files changed, 24 insertions(+), 16 deletions(-) diff --git a/frappe/core/doctype/rq_job/rq_job.py b/frappe/core/doctype/rq_job/rq_job.py index 8e69dd3650..af34955de8 100644 --- a/frappe/core/doctype/rq_job/rq_job.py +++ b/frappe/core/doctype/rq_job/rq_job.py @@ -15,7 +15,7 @@ from frappe.model.document import Document from frappe.utils import ( cint, compare, - convert_utc_to_user_timezone, + convert_utc_to_system_timezone, create_batch, make_filter_dict, ) @@ -132,14 +132,14 @@ def serialize_job(job: Job) -> frappe._dict: queue=job.origin.rsplit(":", 1)[1], job_name=job_name, status=job.get_status(), - started_at=convert_utc_to_user_timezone(job.started_at) if job.started_at else "", - ended_at=convert_utc_to_user_timezone(job.ended_at) if job.ended_at else "", + started_at=convert_utc_to_system_timezone(job.started_at) if job.started_at else "", + ended_at=convert_utc_to_system_timezone(job.ended_at) if job.ended_at else "", time_taken=(job.ended_at - job.started_at).total_seconds() if job.ended_at else "", exc_info=job.exc_info, arguments=frappe.as_json(job.kwargs), timeout=job.timeout, - creation=convert_utc_to_user_timezone(job.created_at), - modified=convert_utc_to_user_timezone(modified), + creation=convert_utc_to_system_timezone(job.created_at), + modified=convert_utc_to_system_timezone(modified), _comment_count=0, owner=job.kwargs.get("user"), modified_by=job.kwargs.get("user"), diff --git a/frappe/core/doctype/rq_worker/rq_worker.py b/frappe/core/doctype/rq_worker/rq_worker.py index 1d24001fc3..ce2f4ca8b2 100644 --- a/frappe/core/doctype/rq_worker/rq_worker.py +++ b/frappe/core/doctype/rq_worker/rq_worker.py @@ -8,7 +8,7 @@ from rq import Worker import frappe from frappe.model.document import Document -from frappe.utils import cint, convert_utc_to_user_timezone +from frappe.utils import cint, convert_utc_to_system_timezone from frappe.utils.background_jobs import get_workers @@ -66,14 +66,14 @@ def serialize_worker(worker: Worker) -> frappe._dict: status=worker.get_state(), pid=worker.pid, current_job_id=worker.get_current_job_id(), - last_heartbeat=convert_utc_to_user_timezone(worker.last_heartbeat), - birth_date=convert_utc_to_user_timezone(worker.birth_date), + last_heartbeat=convert_utc_to_system_timezone(worker.last_heartbeat), + birth_date=convert_utc_to_system_timezone(worker.birth_date), successful_job_count=worker.successful_job_count, failed_job_count=worker.failed_job_count, total_working_time=worker.total_working_time, _comment_count=0, - modified=convert_utc_to_user_timezone(worker.last_heartbeat), - creation=convert_utc_to_user_timezone(worker.birth_date), + modified=convert_utc_to_system_timezone(worker.last_heartbeat), + creation=convert_utc_to_system_timezone(worker.birth_date), utilization_percent=compute_utilization(worker), ) diff --git a/frappe/desk/page/backups/backups.py b/frappe/desk/page/backups/backups.py index 2ef09df900..9554c7b9b7 100644 --- a/frappe/desk/page/backups/backups.py +++ b/frappe/desk/page/backups/backups.py @@ -4,13 +4,13 @@ import os import frappe from frappe import _ from frappe.utils import cint, get_site_path, get_url -from frappe.utils.data import convert_utc_to_user_timezone +from frappe.utils.data import convert_utc_to_system_timezone def get_context(context): def get_time(path): dt = os.path.getmtime(path) - return convert_utc_to_user_timezone(datetime.datetime.utcfromtimestamp(dt)).strftime( + return convert_utc_to_system_timezone(datetime.datetime.utcfromtimestamp(dt)).strftime( "%a %b %d %H:%M %Y" ) diff --git a/frappe/email/receive.py b/frappe/email/receive.py index eddbc14886..538ab7738d 100644 --- a/frappe/email/receive.py +++ b/frappe/email/receive.py @@ -22,7 +22,7 @@ from frappe.email.oauth import Oauth from frappe.utils import ( add_days, cint, - convert_utc_to_user_timezone, + convert_utc_to_system_timezone, cstr, extract_email_id, get_datetime, @@ -458,7 +458,7 @@ class Email: try: utc = email.utils.mktime_tz(email.utils.parsedate_tz(self.mail["Date"])) utc_dt = datetime.datetime.utcfromtimestamp(utc) - self.date = convert_utc_to_user_timezone(utc_dt).strftime("%Y-%m-%d %H:%M:%S") + self.date = convert_utc_to_system_timezone(utc_dt).strftime("%Y-%m-%d %H:%M:%S") except Exception: self.date = now() else: diff --git a/frappe/utils/data.py b/frappe/utils/data.py index 6ec82aba1c..1e75d93fd1 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -18,6 +18,7 @@ from click import secho import frappe from frappe.desk.utils import slug +from frappe.utils.deprecations import deprecation_warning DateTimeLikeObject = Union[str, datetime.date, datetime.datetime] NumericType = Union[int, float] @@ -304,7 +305,7 @@ def time_diff_in_hours(string_ed_date, string_st_date): def now_datetime(): - dt = convert_utc_to_user_timezone(datetime.datetime.utcnow()) + dt = convert_utc_to_system_timezone(datetime.datetime.utcnow()) return dt.replace(tzinfo=None) @@ -343,11 +344,18 @@ def get_datetime_in_timezone(time_zone): return convert_utc_to_timezone(utc_timestamp, time_zone) -def convert_utc_to_user_timezone(utc_timestamp): +def convert_utc_to_system_timezone(utc_timestamp): time_zone = get_time_zone() return convert_utc_to_timezone(utc_timestamp, time_zone) +def convert_utc_to_user_timezone(utc_timestamp): + deprecation_warning( + "`convert_utc_to_user_timezone` is deprecated and will be removed in version 16. Use `convert_utc_to_system_timezone` instead." + ) + return convert_utc_to_system_timezone(utc_timestamp) + + def now() -> str: """return current datetime as yyyy-mm-dd hh:mm:ss""" if frappe.flags.current_date: From b2e36634d67e9ac0de7d95f6dedd49519c9b9543 Mon Sep 17 00:00:00 2001 From: barredterra <14891507+barredterra@users.noreply.github.com> Date: Sat, 4 Mar 2023 19:30:03 +0100 Subject: [PATCH 03/24] refactor: rename `get_time_zone` to `get_system_timezone` --- frappe/boot.py | 6 +++--- frappe/core/doctype/user/user.py | 4 ++-- .../doctype/google_calendar/google_calendar.py | 10 +++++----- .../doctype/token_cache/token_cache.py | 4 ++-- frappe/oauth.py | 3 ++- frappe/utils/data.py | 17 ++++++++++++----- frappe/website/utils.py | 6 +++--- 7 files changed, 29 insertions(+), 21 deletions(-) diff --git a/frappe/boot.py b/frappe/boot.py index 9594635c70..0cab7a060c 100644 --- a/frappe/boot.py +++ b/frappe/boot.py @@ -21,7 +21,7 @@ from frappe.social.doctype.energy_point_settings.energy_point_settings import ( is_energy_point_enabled, ) from frappe.translate import get_lang_dict, get_messages_for_boot, get_translated_doctypes -from frappe.utils import add_user_info, cstr, get_time_zone +from frappe.utils import add_user_info, cstr, get_system_timezone from frappe.utils.change_log import get_versions from frappe.website.doctype.web_page_view.web_page_view import is_tracking_enabled @@ -402,9 +402,9 @@ def get_link_title_doctypes(): def set_time_zone(bootinfo): bootinfo.time_zone = { - "system": get_time_zone(), + "system": get_system_timezone(), "user": bootinfo.get("user_info", {}).get(frappe.session.user, {}).get("time_zone", None) - or get_time_zone(), + or get_system_timezone(), } diff --git a/frappe/core/doctype/user/user.py b/frappe/core/doctype/user/user.py index e04e43051f..c79e1cef63 100644 --- a/frappe/core/doctype/user/user.py +++ b/frappe/core/doctype/user/user.py @@ -23,7 +23,7 @@ from frappe.utils import ( flt, format_datetime, get_formatted_email, - get_time_zone, + get_system_timezone, has_gravatar, now_datetime, today, @@ -647,7 +647,7 @@ class User(Document): def set_time_zone(self): if not self.time_zone: - self.time_zone = get_time_zone() + self.time_zone = get_system_timezone() @frappe.whitelist() diff --git a/frappe/integrations/doctype/google_calendar/google_calendar.py b/frappe/integrations/doctype/google_calendar/google_calendar.py index 5056f536fc..a663c9c593 100644 --- a/frappe/integrations/doctype/google_calendar/google_calendar.py +++ b/frappe/integrations/doctype/google_calendar/google_calendar.py @@ -21,7 +21,7 @@ from frappe.utils import ( add_to_date, get_datetime, get_request_site_address, - get_time_zone, + get_system_timezone, get_weekdays, now_datetime, ) @@ -575,14 +575,14 @@ def google_calendar_to_repeat_on(start, end, recurrence=None): get_datetime(start.get("date")) if start.get("date") else parser.parse(start.get("dateTime")) - .astimezone(ZoneInfo(get_time_zone())) + .astimezone(ZoneInfo(get_system_timezone())) .replace(tzinfo=None) ), "ends_on": ( get_datetime(end.get("date")) if end.get("date") else parser.parse(end.get("dateTime")) - .astimezone(ZoneInfo(get_time_zone())) + .astimezone(ZoneInfo(get_system_timezone())) .replace(tzinfo=None) ), "all_day": 1 if start.get("date") else 0, @@ -648,11 +648,11 @@ def format_date_according_to_google_calendar(all_day, starts_on, ends_on=None): date_format = { "start": { "dateTime": starts_on.isoformat(), - "timeZone": get_time_zone(), + "timeZone": get_system_timezone(), }, "end": { "dateTime": ends_on.isoformat(), - "timeZone": get_time_zone(), + "timeZone": get_system_timezone(), }, } diff --git a/frappe/integrations/doctype/token_cache/token_cache.py b/frappe/integrations/doctype/token_cache/token_cache.py index a4f34b4ad9..b79dfe0abf 100644 --- a/frappe/integrations/doctype/token_cache/token_cache.py +++ b/frappe/integrations/doctype/token_cache/token_cache.py @@ -8,7 +8,7 @@ import pytz import frappe from frappe import _ from frappe.model.document import Document -from frappe.utils import cint, cstr, get_time_zone +from frappe.utils import cint, cstr, get_system_timezone class TokenCache(Document): @@ -52,7 +52,7 @@ class TokenCache(Document): return self def get_expires_in(self): - system_timezone = pytz.timezone(get_time_zone()) + system_timezone = pytz.timezone(get_system_timezone()) modified = frappe.utils.get_datetime(self.modified) modified = system_timezone.localize(modified) expiry_utc = modified.astimezone(pytz.utc) + timedelta(seconds=self.expires_in) diff --git a/frappe/oauth.py b/frappe/oauth.py index 68e21ac88b..8099bdab45 100644 --- a/frappe/oauth.py +++ b/frappe/oauth.py @@ -11,6 +11,7 @@ from oauthlib.openid import RequestValidator import frappe from frappe.auth import LoginManager +from frappe.utils.data import get_system_timezone class OAuthWebRequestValidator(RequestValidator): @@ -248,7 +249,7 @@ class OAuthWebRequestValidator(RequestValidator): # Remember to check expiration and scope membership otoken = frappe.get_doc("OAuth Bearer Token", token) token_expiration_local = otoken.expiration_time.replace( - tzinfo=pytz.timezone(frappe.utils.get_time_zone()) + tzinfo=pytz.timezone(get_system_timezone()) ) token_expiration_utc = token_expiration_local.astimezone(pytz.utc) is_token_valid = ( diff --git a/frappe/utils/data.py b/frappe/utils/data.py index 1e75d93fd1..767d6b4f4b 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -318,15 +318,22 @@ def get_eta(from_time, percent_complete): return str(datetime.timedelta(seconds=(100 - percent_complete) / percent_complete * diff)) -def _get_time_zone(): +def _get_system_timezone(): return frappe.db.get_system_setting("time_zone") or "Asia/Kolkata" # Default to India ?! -def get_time_zone(): +def get_system_timezone(): if frappe.local.flags.in_test: - return _get_time_zone() + return _get_system_timezone() - return frappe.cache().get_value("time_zone", _get_time_zone) + return frappe.cache().get_value("time_zone", _get_system_timezone) + + +def get_time_zone(): + deprecation_warning( + "`get_time_zone` is deprecated and will be removed in version 16. Use `get_system_timezone` instead." + ) + return get_system_timezone() def convert_utc_to_timezone(utc_timestamp, time_zone): @@ -345,7 +352,7 @@ def get_datetime_in_timezone(time_zone): def convert_utc_to_system_timezone(utc_timestamp): - time_zone = get_time_zone() + time_zone = get_system_timezone() return convert_utc_to_timezone(utc_timestamp, time_zone) diff --git a/frappe/website/utils.py b/frappe/website/utils.py index 244fd010b6..71af463c96 100644 --- a/frappe/website/utils.py +++ b/frappe/website/utils.py @@ -12,7 +12,7 @@ from werkzeug.wrappers import Response import frappe from frappe import _ from frappe.model.document import Document -from frappe.utils import cint, get_assets_json, get_time_zone, md_to_html +from frappe.utils import cint, get_assets_json, get_system_timezone, md_to_html FRONTMATTER_PATTERN = re.compile(r"^\s*(?:---|\+\+\+)(.*?)(?:---|\+\+\+)\s*(.+)$", re.S | re.M) H1_TAG_PATTERN = re.compile("

([^<]*)") @@ -167,8 +167,8 @@ def get_boot_data(): "time_format": frappe.get_system_settings("time_format") or "HH:mm:ss", }, "time_zone": { - "system": get_time_zone(), - "user": frappe.db.get_value("User", frappe.session.user, "time_zone") or get_time_zone(), + "system": get_system_timezone(), + "user": frappe.db.get_value("User", frappe.session.user, "time_zone") or get_system_timezone(), }, "assets_json": get_assets_json(), } From c2c5449947c1cab3c35a02547aaadc97b4b732db Mon Sep 17 00:00:00 2001 From: barredterra <14891507+barredterra@users.noreply.github.com> Date: Sun, 5 Mar 2023 15:58:13 +0100 Subject: [PATCH 04/24] chore: deprecate timezone utils in v15 --- frappe/utils/data.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index 767d6b4f4b..acab31bd0d 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -331,7 +331,7 @@ def get_system_timezone(): def get_time_zone(): deprecation_warning( - "`get_time_zone` is deprecated and will be removed in version 16. Use `get_system_timezone` instead." + "`get_time_zone` is deprecated and will be removed in version 15. Use `get_system_timezone` instead." ) return get_system_timezone() @@ -358,7 +358,7 @@ def convert_utc_to_system_timezone(utc_timestamp): def convert_utc_to_user_timezone(utc_timestamp): deprecation_warning( - "`convert_utc_to_user_timezone` is deprecated and will be removed in version 16. Use `convert_utc_to_system_timezone` instead." + "`convert_utc_to_user_timezone` is deprecated and will be removed in version 15. Use `convert_utc_to_system_timezone` instead." ) return convert_utc_to_system_timezone(utc_timestamp) From d1ccfc91b8b65f40cb224ab27efcdc04af80cbcb Mon Sep 17 00:00:00 2001 From: barredterra <14891507+barredterra@users.noreply.github.com> Date: Sun, 5 Mar 2023 16:17:44 +0100 Subject: [PATCH 05/24] refactor: rename timezone utils in safe_exec --- frappe/utils/safe_exec.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/utils/safe_exec.py b/frappe/utils/safe_exec.py index 9e99754c67..8f73efd17c 100644 --- a/frappe/utils/safe_exec.py +++ b/frappe/utils/safe_exec.py @@ -445,8 +445,8 @@ VALID_UTILS = ( "now_datetime", "get_timestamp", "get_eta", - "get_time_zone", - "convert_utc_to_user_timezone", + "get_system_timezone", + "convert_utc_to_system_timezone", "now", "nowdate", "today", From 5581be43b155e487c794dd849994e5fb98c7c6e0 Mon Sep 17 00:00:00 2001 From: Ritwik Puri Date: Mon, 6 Mar 2023 10:44:38 +0530 Subject: [PATCH 06/24] chore: remove manage subscriptions from navbar settings (#20249) * chore: remove manage subscriptions from navbar settings * chore: remove manage subscriptions when adding standard navbar items --- frappe/patches.txt | 1 + .../v14_0/remove_manage_subscriptions_from_navbar.py | 10 ++++++++++ frappe/utils/install.py | 7 ------- 3 files changed, 11 insertions(+), 7 deletions(-) create mode 100644 frappe/patches/v14_0/remove_manage_subscriptions_from_navbar.py diff --git a/frappe/patches.txt b/frappe/patches.txt index b2e2f1392d..9ebb32fea0 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -221,3 +221,4 @@ execute:frappe.delete_doc("Page", "activity", force=1) frappe.patches.v14_0.disable_email_accounts_with_oauth execute:frappe.delete_doc("Page", "translation-tool", force=1) frappe.patches.v15_0.remove_prepared_report_settings_from_system_settings +frappe.patches.v14_0.remove_manage_subscriptions_from_navbar diff --git a/frappe/patches/v14_0/remove_manage_subscriptions_from_navbar.py b/frappe/patches/v14_0/remove_manage_subscriptions_from_navbar.py new file mode 100644 index 0000000000..cd35dd6c9c --- /dev/null +++ b/frappe/patches/v14_0/remove_manage_subscriptions_from_navbar.py @@ -0,0 +1,10 @@ +import frappe + + +def execute(): + navbar_settings = frappe.get_single("Navbar Settings") + for i, l in enumerate(navbar_settings.settings_dropdown): + if l.item_label == "Manage Subscriptions": + navbar_settings.settings_dropdown.pop(i) + navbar_settings.save() + break diff --git a/frappe/utils/install.py b/frappe/utils/install.py index caac5744e8..df918c27e0 100644 --- a/frappe/utils/install.py +++ b/frappe/utils/install.py @@ -210,13 +210,6 @@ def add_standard_navbar_items(): "action": "frappe.ui.toolbar.route_to_user()", "is_standard": 1, }, - { - "item_label": "Manage Subscriptions", - "item_type": "Action", - "action": "frappe.ui.toolbar.redirectToUrl()", - "hidden": 1, - "is_standard": 1, - }, { "item_label": "Session Defaults", "item_type": "Action", From e9dfa80cf03e8537a09afa79c8e0287833fdf4c5 Mon Sep 17 00:00:00 2001 From: Saif Ur Rehman Date: Mon, 6 Mar 2023 10:17:11 +0500 Subject: [PATCH 07/24] fix(Database): clear background jobs and realtime logs on rollback (#20236) --- frappe/database/database.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/frappe/database/database.py b/frappe/database/database.py index 7e82340d23..7e702a8862 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -994,6 +994,9 @@ class Database: if hasattr(obj, "on_rollback"): obj.on_rollback() frappe.local.rollback_observers = [] + + frappe.local.realtime_log = [] + frappe.flags.enqueue_after_commit = [] def field_exists(self, dt, fn): """Return true of field exists.""" From 4ec874ac6ee2aae0d80e044e49972a3140185490 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 6 Mar 2023 11:05:18 +0530 Subject: [PATCH 08/24] style: format --- frappe/database/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/database/database.py b/frappe/database/database.py index 7e702a8862..7e0cb83454 100644 --- a/frappe/database/database.py +++ b/frappe/database/database.py @@ -994,7 +994,7 @@ class Database: if hasattr(obj, "on_rollback"): obj.on_rollback() frappe.local.rollback_observers = [] - + frappe.local.realtime_log = [] frappe.flags.enqueue_after_commit = [] From cd524135c0d93aaf76a21be90e992200308b88e8 Mon Sep 17 00:00:00 2001 From: gavin Date: Mon, 6 Mar 2023 12:40:15 +0530 Subject: [PATCH 09/24] fix: TemplatePage.can_render (#20257) Don't render python executable/loadable files from TemplatePage renderer. This restricts access to reading/downloading possibly private Python source code from Frappe applications --- frappe/tests/test_website.py | 11 +++++++++++ frappe/website/page_renderers/template_page.py | 10 ++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/frappe/tests/test_website.py b/frappe/tests/test_website.py index 2179f4cf13..7af2bfda8e 100644 --- a/frappe/tests/test_website.py +++ b/frappe/tests/test_website.py @@ -331,6 +331,17 @@ class TestWebsite(FrappeTestCase): self.assertIn("test.__test", content) self.assertNotIn("frappe.exceptions.ValidationError: Illegal template", content) + def test_never_render(self): + from pathlib import Path + from random import choices + + WWW = Path(frappe.get_app_path("frappe")) / "www" + FILES_TO_SKIP = choices(list(WWW.glob("**/*.py*")), k=10) + + for suffix in FILES_TO_SKIP: + content = get_response_content(suffix.relative_to(WWW)) + self.assertIn("404", content) + def test_metatags(self): content = get_response_content("/_test/_test_metatags") self.assertIn('', content) diff --git a/frappe/website/page_renderers/template_page.py b/frappe/website/page_renderers/template_page.py index daa4d54cc5..84d376feb4 100644 --- a/frappe/website/page_renderers/template_page.py +++ b/frappe/website/page_renderers/template_page.py @@ -1,5 +1,5 @@ -import io import os +from importlib.machinery import all_suffixes import click @@ -17,6 +17,8 @@ from frappe.website.utils import ( is_binary_file, ) +PY_LOADER_SUFFIXES = tuple(all_suffixes()) + WEBPAGE_PY_MODULE_PROPERTIES = ( "base_template_path", "template", @@ -66,7 +68,11 @@ class TemplatePage(BaseTemplatePage): return def can_render(self): - return hasattr(self, "template_path") and bool(self.template_path) + return ( + hasattr(self, "template_path") + and self.template_path + and not self.template_path.endswith(PY_LOADER_SUFFIXES) + ) @staticmethod def get_index_path_options(search_path): From 3f87ffe4465847c11997c53a1337a9502c89c428 Mon Sep 17 00:00:00 2001 From: barredterra <14891507+barredterra@users.noreply.github.com> Date: Mon, 6 Mar 2023 13:04:20 +0100 Subject: [PATCH 10/24] Revert "refactor: rename timezone utils in safe_exec" This reverts commit d1ccfc91b8b65f40cb224ab27efcdc04af80cbcb. --- frappe/utils/safe_exec.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/utils/safe_exec.py b/frappe/utils/safe_exec.py index 8f73efd17c..9e99754c67 100644 --- a/frappe/utils/safe_exec.py +++ b/frappe/utils/safe_exec.py @@ -445,8 +445,8 @@ VALID_UTILS = ( "now_datetime", "get_timestamp", "get_eta", - "get_system_timezone", - "convert_utc_to_system_timezone", + "get_time_zone", + "convert_utc_to_user_timezone", "now", "nowdate", "today", From c099b67165f0126f5f20c67f9af17124daf4386a Mon Sep 17 00:00:00 2001 From: barredterra <14891507+barredterra@users.noreply.github.com> Date: Mon, 6 Mar 2023 13:07:24 +0100 Subject: [PATCH 11/24] feat: add new timezone utils to safe_exec --- frappe/utils/safe_exec.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frappe/utils/safe_exec.py b/frappe/utils/safe_exec.py index 9e99754c67..a303bed329 100644 --- a/frappe/utils/safe_exec.py +++ b/frappe/utils/safe_exec.py @@ -446,7 +446,9 @@ VALID_UTILS = ( "get_timestamp", "get_eta", "get_time_zone", + "get_system_timezone", "convert_utc_to_user_timezone", + "convert_utc_to_system_timezone", "now", "nowdate", "today", From 036e1c94cd7c4d4bf652e42d4218768fcc78aafe Mon Sep 17 00:00:00 2001 From: Raffael Meyer <14891507+barredterra@users.noreply.github.com> Date: Mon, 6 Mar 2023 15:26:57 +0100 Subject: [PATCH 12/24] feat!: remove deprecated timezone utils (#20255) --- frappe/utils/data.py | 14 -------------- frappe/utils/safe_exec.py | 2 -- 2 files changed, 16 deletions(-) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index acab31bd0d..92467b036b 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -329,13 +329,6 @@ def get_system_timezone(): return frappe.cache().get_value("time_zone", _get_system_timezone) -def get_time_zone(): - deprecation_warning( - "`get_time_zone` is deprecated and will be removed in version 15. Use `get_system_timezone` instead." - ) - return get_system_timezone() - - def convert_utc_to_timezone(utc_timestamp, time_zone): from pytz import UnknownTimeZoneError, timezone @@ -356,13 +349,6 @@ def convert_utc_to_system_timezone(utc_timestamp): return convert_utc_to_timezone(utc_timestamp, time_zone) -def convert_utc_to_user_timezone(utc_timestamp): - deprecation_warning( - "`convert_utc_to_user_timezone` is deprecated and will be removed in version 15. Use `convert_utc_to_system_timezone` instead." - ) - return convert_utc_to_system_timezone(utc_timestamp) - - def now() -> str: """return current datetime as yyyy-mm-dd hh:mm:ss""" if frappe.flags.current_date: diff --git a/frappe/utils/safe_exec.py b/frappe/utils/safe_exec.py index a303bed329..8f73efd17c 100644 --- a/frappe/utils/safe_exec.py +++ b/frappe/utils/safe_exec.py @@ -445,9 +445,7 @@ VALID_UTILS = ( "now_datetime", "get_timestamp", "get_eta", - "get_time_zone", "get_system_timezone", - "convert_utc_to_user_timezone", "convert_utc_to_system_timezone", "now", "nowdate", From b1e08bb26ec8cad090a080cc19520e6da8cac018 Mon Sep 17 00:00:00 2001 From: barredterra <14891507+barredterra@users.noreply.github.com> Date: Mon, 6 Mar 2023 23:12:24 +0100 Subject: [PATCH 13/24] fix: type annotation for filters parameter of get_monthly_goal_graph_data --- frappe/utils/goal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/utils/goal.py b/frappe/utils/goal.py index f60aec4d2b..c794090d03 100644 --- a/frappe/utils/goal.py +++ b/frappe/utils/goal.py @@ -51,7 +51,7 @@ def get_monthly_goal_graph_data( date_field: str, filter_str: str = None, aggregation: str = "sum", - filters: dict | None = None, + filters: str | dict | None = None, ) -> dict: """ Get month-wise graph data for a doctype based on aggregation values of a field in the goal doctype From ce27d7865f44da3296d0cc417e74bdc96a58d06b Mon Sep 17 00:00:00 2001 From: Suraj Shetty Date: Tue, 7 Mar 2023 10:48:23 +0530 Subject: [PATCH 14/24] fix: Remove unnecessary code to avoid timestamp conflict --- frappe/core/doctype/communication/communication.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/frappe/core/doctype/communication/communication.py b/frappe/core/doctype/communication/communication.py index 6b948947a8..adeac35204 100644 --- a/frappe/core/doctype/communication/communication.py +++ b/frappe/core/doctype/communication/communication.py @@ -554,9 +554,6 @@ def update_parent_document_on_communication(doc): parent.db_set("status", "Open") parent.run_method("handle_hold_time", "Replied") apply_assignment_rule(parent) - else: - # update the modified date for document - parent.update_modified() update_first_response_time(parent, doc) set_avg_response_time(parent, doc) From a093f7d4b6c8b9b64af4a5abc03efdd8411d58fc Mon Sep 17 00:00:00 2001 From: MouSoeng <100179677+MouSoeng@users.noreply.github.com> Date: Tue, 7 Mar 2023 16:57:10 +0800 Subject: [PATCH 15/24] chore(py): upgrade babel 2.9.0 -> 2.12.1 (#20251) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 95ef65c0a8..b0205edb22 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ readme = "README.md" dynamic = ["version"] dependencies = [ # core dependencies - "Babel~=2.9.0", + "Babel~=2.12.1", "Click~=8.1.3", "filelock~=3.8.0", "GitPython~=3.1.30", From a00aa51c9620d711a8486e7bf0bb90fa903bdef5 Mon Sep 17 00:00:00 2001 From: barredterra <14891507+barredterra@users.noreply.github.com> Date: Tue, 7 Mar 2023 11:14:48 +0100 Subject: [PATCH 16/24] fix: remove deprecated filter_str parameter of get_monthly_goal_graph_data --- frappe/utils/goal.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/frappe/utils/goal.py b/frappe/utils/goal.py index c794090d03..709fdc1644 100644 --- a/frappe/utils/goal.py +++ b/frappe/utils/goal.py @@ -49,7 +49,6 @@ def get_monthly_goal_graph_data( goal_doctype_link: str, goal_field: str, date_field: str, - filter_str: str = None, aggregation: str = "sum", filters: str | dict | None = None, ) -> dict: @@ -65,17 +64,11 @@ def get_monthly_goal_graph_data( :param goal_doctype: doctype the goal is based on :param goal_doctype_link: doctype link field in goal_doctype :param goal_field: field from which the goal is calculated - :param filter_str: [DEPRECATED] where clause condition. Use filters. :param aggregation: a value like 'count', 'sum', 'avg' :param filters: optional filters :return: dict of graph data """ - if isinstance(filter_str, str): - frappe.throw( - "String filters have been deprecated. Pass Dict filters instead.", exc=DeprecationWarning - ) # nosemgrep - doc = frappe.get_doc(doctype, docname) doc.check_permission() From 640a543dae1929b4873dd38e27ad06839ab00722 Mon Sep 17 00:00:00 2001 From: Ritwik Puri Date: Wed, 8 Mar 2023 00:17:28 +0530 Subject: [PATCH 17/24] chore: translate new button in web form Co-authored-by: Steffen --- frappe/website/doctype/web_form/templates/web_list.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/website/doctype/web_form/templates/web_list.html b/frappe/website/doctype/web_form/templates/web_list.html index 0db5a22c42..e9505605f6 100644 --- a/frappe/website/doctype/web_form/templates/web_list.html +++ b/frappe/website/doctype/web_form/templates/web_list.html @@ -12,7 +12,7 @@
{%- if allow_multiple -%} - New + {{ _("New") }} {%- endif -%}
From 48f63f53abf149785a9249a1f50e7a01eecc0c03 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 6 Mar 2023 11:30:30 +0530 Subject: [PATCH 18/24] feat: configurable rounding methods --- .../system_settings/system_settings.json | 12 ++++- frappe/tests/test_utils.py | 49 ++++++++++++++++++- frappe/utils/data.py | 37 +++++++++----- pyproject.toml | 1 + 4 files changed, 82 insertions(+), 17 deletions(-) diff --git a/frappe/core/doctype/system_settings/system_settings.json b/frappe/core/doctype/system_settings/system_settings.json index ddafd0e9fd..72f6d2345f 100644 --- a/frappe/core/doctype/system_settings/system_settings.json +++ b/frappe/core/doctype/system_settings/system_settings.json @@ -17,10 +17,11 @@ "date_format", "time_format", "number_format", + "first_day_of_the_week", "column_break_7", "float_precision", "currency_precision", - "first_day_of_the_week", + "rounding_method", "sec_backup_limit", "backup_limit", "encrypt_backup", @@ -520,12 +521,19 @@ "fieldname": "login_with_email_link_expiry", "fieldtype": "Int", "label": "Login with email link expiry (in minutes)" + }, + { + "default": "Bankers Rounding", + "fieldname": "rounding_method", + "fieldtype": "Select", + "label": "Rounding Method", + "options": "Bankers Rounding\nRounding half away from zero" } ], "icon": "fa fa-cog", "issingle": 1, "links": [], - "modified": "2022-12-20 21:45:37.651668", + "modified": "2023-03-06 11:31:19.144956", "modified_by": "Administrator", "module": "Core", "name": "System Settings", diff --git a/frappe/tests/test_utils.py b/frappe/tests/test_utils.py index d2d5cdafd7..c41d28d9d1 100644 --- a/frappe/tests/test_utils.py +++ b/frappe/tests/test_utils.py @@ -6,25 +6,28 @@ import json import os import sys from datetime import date, datetime, time, timedelta -from decimal import Decimal +from decimal import ROUND_HALF_UP, Decimal, localcontext from enum import Enum from io import StringIO from mimetypes import guess_type from unittest.mock import patch import pytz +from hypothesis import given +from hypothesis import strategies as st from PIL import Image import frappe from frappe.installer import parse_app_name from frappe.model.document import Document -from frappe.tests.utils import FrappeTestCase +from frappe.tests.utils import FrappeTestCase, change_settings from frappe.utils import ( ceil, dict_to_str, evaluate_filters, execute_in_shell, floor, + flt, format_timedelta, get_bench_path, get_file_timestamp, @@ -1001,3 +1004,45 @@ class TestTBSanitization(FrappeTestCase): self.assertIn("********", traceback) self.assertIn("password =", traceback) self.assertIn("safe_value", traceback) + + +class TestRounding(FrappeTestCase): + @change_settings("System Settings", {"rounding_method": "Rounding half away from zero"}) + def test_normal_rounding(self): + self.assertEqual(flt("what"), 0) + + self.assertEqual(flt("0.5", 0), 1) + self.assertEqual(flt("0.3"), 0.3) + + self.assertEqual(flt("1.5", 0), 2) + + # positive rounding to integers + self.assertEqual(flt(0.4, 0), 0) + self.assertEqual(flt(0.5, 0), 1) + self.assertEqual(flt(1.455, 0), 1) + self.assertEqual(flt(1.5, 0), 2) + + # negative rounding to integers + self.assertEqual(flt(-0.5, 0), -1) + self.assertEqual(flt(-1.5, 0), -2) + + # negative precision i.e. round to nearest 10th + self.assertEqual(flt(123, -1), 120) + self.assertEqual(flt(125, -1), 130) + self.assertEqual(flt(134.45, -1), 130) + self.assertEqual(flt(135, -1), 140) + + # # positive multiple digit rounding + self.assertEqual(flt(1.25, 1), 1.3) + self.assertEqual(flt(0.15, 1), 0.2) + + # # negative multiple digit rounding + self.assertEqual(flt(-1.25, 1), -1.3) + self.assertEqual(flt(-0.15, 1), -0.2) + + @change_settings("System Settings", {"rounding_method": "Rounding half away from zero"}) + @given(st.decimals(min_value=-1e8, max_value=1e8), st.integers(min_value=-2, max_value=4)) + def test_normal_rounding_property(self, number, precision): + with localcontext() as ctx: + ctx.rounding = ROUND_HALF_UP + self.assertEqual(Decimal(str(flt(float(number), precision))), round(number, precision)) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index 92467b036b..dbb8f6294e 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -1047,25 +1047,36 @@ def sbool(x: str) -> bool | Any: def rounded(num, precision=0): - """round method for round halfs to nearest even algorithm aka banker's rounding - compatible with python3""" + """Round according to method set in system setting, defaults to banker's rounding""" precision = cint(precision) - multiplier = 10**precision - # avoid rounding errors - num = round(num * multiplier if precision else num, 8) + rounding_method = frappe.get_system_settings("rounding_method") or "Bankers Rounding" - floor_num = math.floor(num) - decimal_part = num - floor_num + if rounding_method == "Bankers Rounding": + # avoid rounding errors + multiplier = 10**precision + num = round(num * multiplier if precision else num, 8) - if not precision and decimal_part == 0.5: - num = floor_num if (floor_num % 2 == 0) else floor_num + 1 - else: - if decimal_part == 0.5: - num = floor_num + 1 + floor_num = math.floor(num) + decimal_part = num - floor_num + + if not precision and decimal_part == 0.5: + num = floor_num if (floor_num % 2 == 0) else floor_num + 1 else: - num = round(num) + if decimal_part == 0.5: + num = floor_num + 1 + else: + num = round(num) - return (num / multiplier) if precision else num + return (num / multiplier) if precision else num + + elif rounding_method == "Rounding half away from zero": + if num == 0: + return 0.0 + # Epsilon is small correctional value added to correctly round numbers which can't be + # represented in IEEE 754 representation. + epsilon = 2.0 ** (math.log(abs(num), 2) - 52.0) + return round(num + math.copysign(epsilon, num), precision) def remainder(numerator: NumericType, denominator: NumericType, precision: int = 2) -> NumericType: diff --git a/pyproject.toml b/pyproject.toml index b0205edb22..837ea4624a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -102,3 +102,4 @@ Faker = "~=13.12.1" pyngrok = "~=5.0.5" unittest-xml-reporting = "~=3.0.4" watchdog = "~=2.1.9" +hypothesis = "~=6.68.2" From 86b9ff426651b81b273379c4aa122940043aea32 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 6 Mar 2023 17:28:44 +0530 Subject: [PATCH 19/24] feat: Allow specifying rounding method in `flt` --- .../system_settings/system_settings.json | 6 +-- frappe/exceptions.py | 4 ++ frappe/tests/test_utils.py | 37 ++++++++++++++++- frappe/utils/data.py | 41 +++++++++++++++---- 4 files changed, 76 insertions(+), 12 deletions(-) diff --git a/frappe/core/doctype/system_settings/system_settings.json b/frappe/core/doctype/system_settings/system_settings.json index 72f6d2345f..2c9e92d943 100644 --- a/frappe/core/doctype/system_settings/system_settings.json +++ b/frappe/core/doctype/system_settings/system_settings.json @@ -523,11 +523,11 @@ "label": "Login with email link expiry (in minutes)" }, { - "default": "Bankers Rounding", + "default": "Round Half Even", "fieldname": "rounding_method", "fieldtype": "Select", "label": "Rounding Method", - "options": "Bankers Rounding\nRounding half away from zero" + "options": "Round Half Even\nRounding Half Away From Zero" } ], "icon": "fa fa-cog", @@ -552,4 +552,4 @@ "sort_order": "ASC", "states": [], "track_changes": 1 -} \ No newline at end of file +} diff --git a/frappe/exceptions.py b/frappe/exceptions.py index 20e858c543..26c323352d 100644 --- a/frappe/exceptions.py +++ b/frappe/exceptions.py @@ -273,6 +273,10 @@ class ExecutableNotFound(FileNotFoundError): pass +class InvalidRoundingMethod(FileNotFoundError): + pass + + class InvalidRemoteException(Exception): pass diff --git a/frappe/tests/test_utils.py b/frappe/tests/test_utils.py index c41d28d9d1..ac3ed2b913 100644 --- a/frappe/tests/test_utils.py +++ b/frappe/tests/test_utils.py @@ -1007,7 +1007,7 @@ class TestTBSanitization(FrappeTestCase): class TestRounding(FrappeTestCase): - @change_settings("System Settings", {"rounding_method": "Rounding half away from zero"}) + @change_settings("System Settings", {"rounding_method": "Rounding Half Away From Zero"}) def test_normal_rounding(self): self.assertEqual(flt("what"), 0) @@ -1040,7 +1040,40 @@ class TestRounding(FrappeTestCase): self.assertEqual(flt(-1.25, 1), -1.3) self.assertEqual(flt(-0.15, 1), -0.2) - @change_settings("System Settings", {"rounding_method": "Rounding half away from zero"}) + def test_normal_rounding_as_argument(self): + rounding_method = "Rounding Half Away From Zero" + + self.assertEqual(flt("0.5", 0, rounding_method=rounding_method), 1) + self.assertEqual(flt("0.3", rounding_method=rounding_method), 0.3) + + self.assertEqual(flt("1.5", 0, rounding_method=rounding_method), 2) + + # positive rounding to integers + self.assertEqual(flt(0.4, 0, rounding_method=rounding_method), 0) + self.assertEqual(flt(0.5, 0, rounding_method=rounding_method), 1) + self.assertEqual(flt(1.455, 0, rounding_method=rounding_method), 1) + self.assertEqual(flt(1.5, 0, rounding_method=rounding_method), 2) + + # negative rounding to integers + self.assertEqual(flt(-0.5, 0, rounding_method=rounding_method), -1) + self.assertEqual(flt(-1.5, 0, rounding_method=rounding_method), -2) + + # negative precision i.e. round to nearest 10th + self.assertEqual(flt(123, -1, rounding_method=rounding_method), 120) + self.assertEqual(flt(125, -1, rounding_method=rounding_method), 130) + self.assertEqual(flt(134.45, -1, rounding_method=rounding_method), 130) + self.assertEqual(flt(135, -1, rounding_method=rounding_method), 140) + + # # positive multiple digit rounding + self.assertEqual(flt(1.25, 1, rounding_method=rounding_method), 1.3) + self.assertEqual(flt(0.15, 1, rounding_method=rounding_method), 0.2) + self.assertEqual(flt(2.675, 2, rounding_method=rounding_method), 2.68) + + # # negative multiple digit rounding + self.assertEqual(flt(-1.25, 1, rounding_method=rounding_method), -1.3) + self.assertEqual(flt(-0.15, 1, rounding_method=rounding_method), -0.2) + + @change_settings("System Settings", {"rounding_method": "Rounding Half Away From Zero"}) @given(st.decimals(min_value=-1e8, max_value=1e8), st.integers(min_value=-2, max_value=4)) def test_normal_rounding_property(self, number, precision): with localcontext() as ctx: diff --git a/frappe/utils/data.py b/frappe/utils/data.py index dbb8f6294e..6cde2b8868 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -920,7 +920,9 @@ def flt(s: NumericType | str, precision: int | None = None) -> float: ... -def flt(s: NumericType | str, precision: int | None = None) -> float: +def flt( + s: NumericType | str, precision: int | None = None, rounding_method: str | None = None +) -> float: """Convert to float (ignoring commas in string) :param s: Number in string or other numeric format. @@ -946,8 +948,10 @@ def flt(s: NumericType | str, precision: int | None = None) -> float: try: num = float(s) if precision is not None: - num = rounded(num, precision) - except Exception: + num = rounded(num, precision, rounding_method) + except Exception as e: + if isinstance(e, frappe.InvalidRoundingMethod): + raise num = 0.0 return num @@ -1046,13 +1050,15 @@ def sbool(x: str) -> bool | Any: return x -def rounded(num, precision=0): +def rounded(num, precision=0, rounding_method=None): """Round according to method set in system setting, defaults to banker's rounding""" precision = cint(precision) - rounding_method = frappe.get_system_settings("rounding_method") or "Bankers Rounding" + rounding_method = ( + rounding_method or frappe.get_system_settings("rounding_method") or "Round Half Even" + ) - if rounding_method == "Bankers Rounding": + if rounding_method == "Round Half Even": # avoid rounding errors multiplier = 10**precision num = round(num * multiplier if precision else num, 8) @@ -1070,13 +1076,34 @@ def rounded(num, precision=0): return (num / multiplier) if precision else num - elif rounding_method == "Rounding half away from zero": + elif rounding_method == "Rounding Half Away From Zero": if num == 0: return 0.0 # Epsilon is small correctional value added to correctly round numbers which can't be # represented in IEEE 754 representation. + + # In simplified terms, the representation optimizes for absolute errors in representation + # so if a number is not representable it might be represented by a value ever so slighly + # smaller than the value itself. This becomes a problem when breaking ties for numbers + # ending with 5 when it's represented by a smaller number. By adding a very small value + # close to what's "least count" or smallest representable difference in the scale we force + # the number to be bigger than actual value, this increases representation error but + # removes rounding error. + + # References: + # - https://docs.oracle.com/cd/E19957-01/806-3568/ncg_goldberg.html + # - https://docs.python.org/3/tutorial/floatingpoint.html#representation-error + # - https://docs.python.org/3/library/functions.html#round + # - easier to understand: https://www.youtube.com/watch?v=pQs_wx8eoQ8 + epsilon = 2.0 ** (math.log(abs(num), 2) - 52.0) + return round(num + math.copysign(epsilon, num), precision) + else: + frappe.throw( + frappe._("Unknown Rounding Method: {}").format(rounding_method), + exc=frappe.InvalidRoundingMethod, + ) def remainder(numerator: NumericType, denominator: NumericType, precision: int = 2) -> NumericType: From 68d8a8eaddc8c9c85d69819317b969d777049a36 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Wed, 8 Mar 2023 11:49:35 +0530 Subject: [PATCH 20/24] feat: implement custom rounding in JS --- cypress/integration/rounding.js | 44 +++++++++++++++++ .../public/js/frappe/utils/number_format.js | 48 ++++++++++++++----- frappe/tests/test_utils.py | 8 ++-- 3 files changed, 84 insertions(+), 16 deletions(-) create mode 100644 cypress/integration/rounding.js diff --git a/cypress/integration/rounding.js b/cypress/integration/rounding.js new file mode 100644 index 0000000000..647b32a6a5 --- /dev/null +++ b/cypress/integration/rounding.js @@ -0,0 +1,44 @@ +context("Rounding behaviour", () => { + before(() => { + cy.login(); + cy.visit("/app/"); + }); + + it("Rounds floats accurately", () => { + cy.window() + .its("flt") + .then((flt) => { + let rounding_method = "Rounding Half Away From Zero"; + + expect(flt("0.5", 0, null, rounding_method)).eq(1); + expect(flt("0.3", null, null, rounding_method)).eq(0.3); + + expect(flt("1.5", 0, null, rounding_method)).eq(2); + + // positive rounding to integers + expect(flt(0.4, 0, null, rounding_method)).eq(0); + expect(flt(0.5, 0, null, rounding_method)).eq(1); + expect(flt(1.455, 0, null, rounding_method)).eq(1); + expect(flt(1.5, 0, null, rounding_method)).eq(2); + + // negative rounding to integers + expect(flt(-0.5, 0, null, rounding_method)).eq(-1); + expect(flt(-1.5, 0, null, rounding_method)).eq(-2); + + // negative precision i.e. round to nearest 10th + expect(flt(123, -1, null, rounding_method)).eq(120); + expect(flt(125, -1, null, rounding_method)).eq(130); + expect(flt(134.45, -1, null, rounding_method)).eq(130); + expect(flt(135, -1, null, rounding_method)).eq(140); + + // positive multiple digit rounding + expect(flt(1.25, 1, null, rounding_method)).eq(1.3); + expect(flt(0.15, 1, null, rounding_method)).eq(0.2); + expect(flt(2.675, 2, null, rounding_method)).eq(2.68); + + // negative multiple digit rounding + expect(flt(-1.25, 1, null, rounding_method)).eq(-1.3); + expect(flt(-0.15, 1, null, rounding_method)).eq(-0.2); + }); + }); +}); diff --git a/frappe/public/js/frappe/utils/number_format.js b/frappe/public/js/frappe/utils/number_format.js index 63b0cbe451..2c457e75fe 100644 --- a/frappe/public/js/frappe/utils/number_format.js +++ b/frappe/public/js/frappe/utils/number_format.js @@ -5,7 +5,7 @@ import "./datatype"; if (!window.frappe) window.frappe = {}; -function flt(v, decimals, number_format) { +function flt(v, decimals, number_format, rounding_method) { if (v == null || v == "") return 0; if (!(typeof v === "number" || String(parseFloat(v)) == v)) { @@ -30,7 +30,7 @@ function flt(v, decimals, number_format) { } v = parseFloat(v); - if (decimals != null) return _round(v, decimals); + if (decimals != null) return _round(v, decimals, rounding_method); return v; } @@ -173,16 +173,40 @@ function get_number_format_info(format) { return info; } -function _round(num, precision) { - var is_negative = num < 0 ? true : false; - var d = cint(precision); - var m = Math.pow(10, d); - var n = +(d ? Math.abs(num) * m : Math.abs(num)).toFixed(8); // Avoid rounding errors - var i = Math.floor(n), - f = n - i; - var r = !precision && f == 0.5 ? (i % 2 == 0 ? i : i + 1) : Math.round(n); - r = d ? r / m : r; - return is_negative ? -r : r; +function _round(num, precision, rounding_method) { + rounding_method = + rounding_method || frappe.boot.sysdefaults.rounding_method || "Round Half Even"; + + let is_negative = num < 0 ? true : false; + + if (rounding_method == "Round Half Even") { + var d = cint(precision); + var m = Math.pow(10, d); + var n = +(d ? Math.abs(num) * m : Math.abs(num)).toFixed(8); // Avoid rounding errors + var i = Math.floor(n), + f = n - i; + var r = !precision && f == 0.5 ? (i % 2 == 0 ? i : i + 1) : Math.round(n); + r = d ? r / m : r; + return is_negative ? -r : r; + } else if (rounding_method == "Rounding Half Away From Zero") { + if (num == 0) return 0.0; + + let digits = cint(precision); + let multiplier = Math.pow(10, digits); + + num = num * multiplier; + + // For explanation of this method read python flt implementation notes. + let epsilon = 2.0 ** (Math.log2(Math.abs(num)) - 52.0); + if (is_negative) { + epsilon = -1 * epsilon; + } + + num = Math.round(num + epsilon); + return num / multiplier; + } else { + throw new Error(`Unknown rounding method ${rounding_method}`); + } } function roundNumber(num, precision) { diff --git a/frappe/tests/test_utils.py b/frappe/tests/test_utils.py index ac3ed2b913..ce13ee65cd 100644 --- a/frappe/tests/test_utils.py +++ b/frappe/tests/test_utils.py @@ -1032,11 +1032,11 @@ class TestRounding(FrappeTestCase): self.assertEqual(flt(134.45, -1), 130) self.assertEqual(flt(135, -1), 140) - # # positive multiple digit rounding + # positive multiple digit rounding self.assertEqual(flt(1.25, 1), 1.3) self.assertEqual(flt(0.15, 1), 0.2) - # # negative multiple digit rounding + # negative multiple digit rounding self.assertEqual(flt(-1.25, 1), -1.3) self.assertEqual(flt(-0.15, 1), -0.2) @@ -1064,12 +1064,12 @@ class TestRounding(FrappeTestCase): self.assertEqual(flt(134.45, -1, rounding_method=rounding_method), 130) self.assertEqual(flt(135, -1, rounding_method=rounding_method), 140) - # # positive multiple digit rounding + # positive multiple digit rounding self.assertEqual(flt(1.25, 1, rounding_method=rounding_method), 1.3) self.assertEqual(flt(0.15, 1, rounding_method=rounding_method), 0.2) self.assertEqual(flt(2.675, 2, rounding_method=rounding_method), 2.68) - # # negative multiple digit rounding + # negative multiple digit rounding self.assertEqual(flt(-1.25, 1, rounding_method=rounding_method), -1.3) self.assertEqual(flt(-0.15, 1, rounding_method=rounding_method), -0.2) From e4a11fd8cf3b546e6ad20b7084fa183cc4d145a3 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Wed, 8 Mar 2023 12:22:27 +0530 Subject: [PATCH 21/24] fix(UX): Warn about changing rounding method --- .../doctype/system_settings/system_settings.js | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/frappe/core/doctype/system_settings/system_settings.js b/frappe/core/doctype/system_settings/system_settings.js index 1de4dd82e7..1d5ba7ddb0 100644 --- a/frappe/core/doctype/system_settings/system_settings.js +++ b/frappe/core/doctype/system_settings/system_settings.js @@ -39,4 +39,21 @@ frappe.ui.form.on("System Settings", { first_day_of_the_week(frm) { frm.re_setup_moment = true; }, + + rounding_method: function (frm) { + if (frm.doc.rounding_method == frappe.boot.sysdefaults.rounding_method) return; + let msg = __( + "Changing rounding method on site with data can result in unexpected behaviour." + ); + msg += "
"; + msg += __("Do you still want to proceed?"); + + frappe.confirm( + msg, + () => {}, + () => { + frm.set_value("rounding_method", frappe.boot.sysdefaults.rounding_method); + } + ); + }, }); From 9f6a6d74fbf848fa6c49964786d92168c861589d Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Wed, 8 Mar 2023 14:09:50 +0530 Subject: [PATCH 22/24] refactor: split rounding methods in functions --- frappe/utils/data.py | 86 ++++++++++++++++++++++++-------------------- 1 file changed, 47 insertions(+), 39 deletions(-) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index 6cde2b8868..eeae737144 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -1059,46 +1059,9 @@ def rounded(num, precision=0, rounding_method=None): ) if rounding_method == "Round Half Even": - # avoid rounding errors - multiplier = 10**precision - num = round(num * multiplier if precision else num, 8) - - floor_num = math.floor(num) - decimal_part = num - floor_num - - if not precision and decimal_part == 0.5: - num = floor_num if (floor_num % 2 == 0) else floor_num + 1 - else: - if decimal_part == 0.5: - num = floor_num + 1 - else: - num = round(num) - - return (num / multiplier) if precision else num - + return _round_half_even(num, precision) elif rounding_method == "Rounding Half Away From Zero": - if num == 0: - return 0.0 - # Epsilon is small correctional value added to correctly round numbers which can't be - # represented in IEEE 754 representation. - - # In simplified terms, the representation optimizes for absolute errors in representation - # so if a number is not representable it might be represented by a value ever so slighly - # smaller than the value itself. This becomes a problem when breaking ties for numbers - # ending with 5 when it's represented by a smaller number. By adding a very small value - # close to what's "least count" or smallest representable difference in the scale we force - # the number to be bigger than actual value, this increases representation error but - # removes rounding error. - - # References: - # - https://docs.oracle.com/cd/E19957-01/806-3568/ncg_goldberg.html - # - https://docs.python.org/3/tutorial/floatingpoint.html#representation-error - # - https://docs.python.org/3/library/functions.html#round - # - easier to understand: https://www.youtube.com/watch?v=pQs_wx8eoQ8 - - epsilon = 2.0 ** (math.log(abs(num), 2) - 52.0) - - return round(num + math.copysign(epsilon, num), precision) + return _round_away_from_zero(num, precision) else: frappe.throw( frappe._("Unknown Rounding Method: {}").format(rounding_method), @@ -1106,6 +1069,51 @@ def rounded(num, precision=0, rounding_method=None): ) +def _round_half_even(num, precision): + # avoid rounding errors + multiplier = 10**precision + num = round(num * multiplier if precision else num, 8) + + floor_num = math.floor(num) + decimal_part = num - floor_num + + if not precision and decimal_part == 0.5: + num = floor_num if (floor_num % 2 == 0) else floor_num + 1 + else: + if decimal_part == 0.5: + num = floor_num + 1 + else: + num = round(num) + + return (num / multiplier) if precision else num + + +def _round_away_from_zero(num, precision): + if num == 0: + return 0.0 + + # Epsilon is small correctional value added to correctly round numbers which can't be + # represented in IEEE 754 representation. + + # In simplified terms, the representation optimizes for absolute errors in representation + # so if a number is not representable it might be represented by a value ever so slighly + # smaller than the value itself. This becomes a problem when breaking ties for numbers + # ending with 5 when it's represented by a smaller number. By adding a very small value + # close to what's "least count" or smallest representable difference in the scale we force + # the number to be bigger than actual value, this increases representation error but + # removes rounding error. + + # References: + # - https://docs.oracle.com/cd/E19957-01/806-3568/ncg_goldberg.html + # - https://docs.python.org/3/tutorial/floatingpoint.html#representation-error + # - https://docs.python.org/3/library/functions.html#round + # - easier to understand: https://www.youtube.com/watch?v=pQs_wx8eoQ8 + + epsilon = 2.0 ** (math.log(abs(num), 2) - 52.0) + + return round(num + math.copysign(epsilon, num), precision) + + def remainder(numerator: NumericType, denominator: NumericType, precision: int = 2) -> NumericType: precision = cint(precision) multiplier = 10**precision From 92120ea539fe322c8c9ab89d177fa82c2111caef Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Wed, 8 Mar 2023 15:06:13 +0530 Subject: [PATCH 23/24] fix(UX): dont let users select child table in ref_doctype (#20278) Child doctypes have no concept of permission so when ref_doctype is child doctype it will always fail. You should pick ref_doctype that's typical parent used for perm checks. --- frappe/core/doctype/report/report.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/frappe/core/doctype/report/report.js b/frappe/core/doctype/report/report.js index 9850dbf98f..fdbda8de9c 100644 --- a/frappe/core/doctype/report/report.js +++ b/frappe/core/doctype/report/report.js @@ -44,6 +44,14 @@ frappe.ui.form.on("Report", { doc.disabled ? "fa fa-check" : "fa fa-off" ); } + + frm.set_query("ref_doctype", () => { + return { + filters: { + istable: 0, + }, + }; + }); }, ref_doctype: function (frm) { From bf0ea6de89abb52e0c480debecceed0f90a22ad7 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Wed, 8 Mar 2023 16:32:30 +0530 Subject: [PATCH 24/24] chore: use merged action url --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ca46ee2c1e..562437d5d1 100644 --- a/README.md +++ b/README.md @@ -23,8 +23,8 @@ - - + +