From 9d9d76efe626f63a06150ec9519bd8c7becd1d01 Mon Sep 17 00:00:00 2001 From: Fahim Ali Zain Date: Fri, 4 Jun 2021 11:23:02 +0530 Subject: [PATCH 01/21] fix: frappe.local.lang resolution --- frappe/auth.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frappe/auth.py b/frappe/auth.py index ef79d96ddb..7c513aed63 100644 --- a/frappe/auth.py +++ b/frappe/auth.py @@ -35,9 +35,6 @@ class HTTPRequest: else: frappe.local.request_ip = '127.0.0.1' - # language - self.set_lang() - # load cookies frappe.local.cookie_manager = CookieManager() @@ -47,6 +44,9 @@ class HTTPRequest: # login frappe.local.login_manager = LoginManager() + # language + self.set_lang() + if frappe.form_dict._lang: lang = get_lang_code(frappe.form_dict._lang) if lang: From f96ebb1f664e009ed2bf61505067ddacf4b21e78 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 10 Jun 2021 12:47:23 +0530 Subject: [PATCH 02/21] perf: Skip guess_language if _lang is provided --- frappe/auth.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/frappe/auth.py b/frappe/auth.py index 7c513aed63..479ec6bda4 100644 --- a/frappe/auth.py +++ b/frappe/auth.py @@ -10,7 +10,7 @@ import frappe.utils.user from frappe import conf from frappe.sessions import Session, clear_sessions, delete_session from frappe.modules.patch_handler import check_session_stopped -from frappe.translate import get_lang_code +from frappe.translate import get_lang_code, guess_language from frappe.utils.password import check_password, delete_login_failed_cache from frappe.core.doctype.activity_log.activity_log import add_authentication_log from frappe.twofactor import (should_run_2fa, authenticate_for_2factor, @@ -47,11 +47,6 @@ class HTTPRequest: # language self.set_lang() - if frappe.form_dict._lang: - lang = get_lang_code(frappe.form_dict._lang) - if lang: - frappe.local.lang = lang - self.validate_csrf_token() # write out latest cookies @@ -79,8 +74,12 @@ class HTTPRequest: frappe.throw(_("Invalid Request"), frappe.CSRFTokenError) def set_lang(self): - from frappe.translate import guess_language - frappe.local.lang = guess_language() + if frappe.form_dict._lang: + lang = get_lang_code(frappe.form_dict._lang) + if lang: + frappe.local.lang = lang + else: + frappe.local.lang = guess_language() def get_db_name(self): """get database name from conf""" From f5bd21cf46357d5e8bb98f30db7d68610a667b98 Mon Sep 17 00:00:00 2001 From: Fahim Ali Zain Date: Sat, 10 Jul 2021 10:52:43 +0530 Subject: [PATCH 03/21] fix: preferred_language cookie support for all users --- frappe/translate.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/frappe/translate.py b/frappe/translate.py index 4ff50d3fd0..8344436d94 100644 --- a/frappe/translate.py +++ b/frappe/translate.py @@ -22,12 +22,12 @@ from frappe.utils import is_html, strip, strip_html_tags def guess_language(lang_list=None): """Set `frappe.local.lang` from HTTP headers at beginning of request""" - user_preferred_language = frappe.request.cookies.get('preferred_language') - is_guest_user = not frappe.session.user or frappe.session.user == 'Guest' - if is_guest_user and user_preferred_language: - return user_preferred_language + preferred_language_cookie = frappe.request.cookies.get('preferred_language') + lang_codes = list(frappe.request.accept_languages.values()) + + if preferred_language_cookie: + lang_codes.append(preferred_language_cookie) - lang_codes = frappe.request.accept_languages.values() if not lang_codes: return frappe.local.lang From caafd9e2b519204b94b2203444dd1ea73086ac4d Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 13 Jul 2021 13:29:22 +0530 Subject: [PATCH 04/21] refactor: Simplify HTTPRequest class * For the sake of Readability and ease of understanding * Style updates --- frappe/auth.py | 74 ++++++++++++++++++++++++++++++++------------------ 1 file changed, 47 insertions(+), 27 deletions(-) diff --git a/frappe/auth.py b/frappe/auth.py index 479ec6bda4..a95c41906d 100644 --- a/frappe/auth.py +++ b/frappe/auth.py @@ -21,11 +21,40 @@ from urllib.parse import quote class HTTPRequest: def __init__(self): - # Get Environment variables - self.domain = frappe.request.host - if self.domain and self.domain.startswith('www.'): - self.domain = self.domain[4:] + # set frappe.local.request_ip + self.set_request_ip() + # load cookies + self.set_cookies() + + # set frappe.local.db + self.connect() + + # login and start/resume user session + self.set_session() + + # set request language + self.set_lang() + + # match csrf token from current session + self.validate_csrf_token() + + # write out latest cookies + frappe.local.cookie_manager.init_cookies() + + # check session status + check_session_stopped() + + @property + def domain(self): + if not getattr(self, "_domain", None): + self._domain = frappe.request.host + if self._domain and self._domain.startswith('www.'): + self._domain = self._domain[4:] + + return self._domain + + def set_request_ip(self): if frappe.get_request_header('X-Forwarded-For'): frappe.local.request_ip = (frappe.get_request_header('X-Forwarded-For').split(",")[0]).strip() @@ -35,32 +64,21 @@ class HTTPRequest: else: frappe.local.request_ip = '127.0.0.1' - # load cookies + def set_cookies(self): frappe.local.cookie_manager = CookieManager() - # set db - self.connect() - - # login + def set_session(self): frappe.local.login_manager = LoginManager() - # language - self.set_lang() - - self.validate_csrf_token() - - # write out latest cookies - frappe.local.cookie_manager.init_cookies() - - # check status - check_session_stopped() - def validate_csrf_token(self): if frappe.local.request and frappe.local.request.method in ("POST", "PUT", "DELETE"): - if not frappe.local.session: return - if not frappe.local.session.data.csrf_token \ - or frappe.local.session.data.device=="mobile" \ - or frappe.conf.get('ignore_csrf', None): + if not frappe.local.session: + return + if ( + not frappe.local.session.data.csrf_token + or frappe.local.session.data.device == "mobile" + or frappe.conf.get('ignore_csrf', None) + ): # not via boot return @@ -85,10 +103,12 @@ class HTTPRequest: """get database name from conf""" return conf.db_name - def connect(self, ac_name = None): + def connect(self): """connect to db, from ac_name or db_name""" - frappe.local.db = frappe.database.get_db(user = self.get_db_name(), \ - password = getattr(conf, 'db_password', '')) + frappe.local.db = frappe.database.get_db( + user=self.get_db_name(), + password=getattr(conf, 'db_password', '') + ) class LoginManager: def __init__(self): From c47cbfd2efb2f72133494b7b68940fe14410ccf8 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 13 Jul 2021 17:03:30 +0530 Subject: [PATCH 05/21] refactor: Set Language in HTTPHeader Order of priority for setting language: 1. Form Dict => _lang 2. Cookie => preferred_language 3. Request Header => Accept-Language 4. User document => language 5. System Settings => language Cookie is placed at #2 since the language picker in the navbar depends on it. And the Accept-Language header sends values based on the client's locales. --- Form Dict _lang now accepts language codes too. Previously, language names were used...for whatever reason. --- frappe/auth.py | 13 +++----- frappe/translate.py | 78 ++++++++++++++++++++++++++++++++------------- 2 files changed, 59 insertions(+), 32 deletions(-) diff --git a/frappe/auth.py b/frappe/auth.py index a95c41906d..9135b139ae 100644 --- a/frappe/auth.py +++ b/frappe/auth.py @@ -5,13 +5,13 @@ from frappe import _ import frappe import frappe.database import frappe.utils -from frappe.utils import cint, flt, get_datetime, datetime, date_diff, today +from frappe.utils import cint, get_datetime, datetime, date_diff, today import frappe.utils.user from frappe import conf from frappe.sessions import Session, clear_sessions, delete_session from frappe.modules.patch_handler import check_session_stopped -from frappe.translate import get_lang_code, guess_language -from frappe.utils.password import check_password, delete_login_failed_cache +from frappe.translate import guess_language +from frappe.utils.password import check_password from frappe.core.doctype.activity_log.activity_log import add_authentication_log from frappe.twofactor import (should_run_2fa, authenticate_for_2factor, confirm_otp_token, get_cached_user_pass) @@ -92,12 +92,7 @@ class HTTPRequest: frappe.throw(_("Invalid Request"), frappe.CSRFTokenError) def set_lang(self): - if frappe.form_dict._lang: - lang = get_lang_code(frappe.form_dict._lang) - if lang: - frappe.local.lang = lang - else: - frappe.local.lang = guess_language() + frappe.local.lang = guess_language() def get_db_name(self): """get database name from conf""" diff --git a/frappe/translate.py b/frappe/translate.py index 8344436d94..c567d0615c 100644 --- a/frappe/translate.py +++ b/frappe/translate.py @@ -11,6 +11,7 @@ import io import itertools import json import operator +import functools import os import re from csv import reader @@ -21,36 +22,64 @@ from frappe.utils import is_html, strip, strip_html_tags def guess_language(lang_list=None): - """Set `frappe.local.lang` from HTTP headers at beginning of request""" + """Set `frappe.local.lang` from HTTP headers at beginning of request + + Order of priority for setting language: + 1. Form Dict => _lang + 2. Cookie => preferred_language + 3. Request Header => Accept-Language + 4. User document => language + 5. System Settings => language + """ + + # fetch language from form_dict + if frappe.form_dict._lang: + language = get_lang_code( + frappe.form_dict._lang or get_parent_language(frappe.form_dict._lang) + ) + if language: + return language + + lang_set = set(lang_list or get_all_languages() or []) + + # fetch language from cookie preferred_language_cookie = frappe.request.cookies.get('preferred_language') - lang_codes = list(frappe.request.accept_languages.values()) if preferred_language_cookie: - lang_codes.append(preferred_language_cookie) + if preferred_language_cookie in lang_set: + return preferred_language_cookie - if not lang_codes: - return frappe.local.lang + parent_language = get_parent_language(language) + if parent_language in lang_set: + return parent_language - guess = None - if not lang_list: - lang_list = get_all_languages() or [] + # fetch language from request headers + accept_language = list(frappe.request.accept_languages.values()) - for l in lang_codes: - code = l.strip() - if not isinstance(code, str): - code = str(code, 'utf-8') - if code in lang_list or code == "en": - guess = code - break + for language in accept_language: + if language in lang_set: + return language - # check if parent language (pt) is setup, if variant (pt-BR) - if "-" in code: - code = code.split("-")[0] - if code in lang_list: - guess = code - break + parent_language = get_parent_language(language) + if parent_language in lang_set: + return parent_language + + # fallback to language set in User or System Settings + return frappe.local.lang + + +@functools.lru_cache(maxsize=None) +def get_parent_language(lang: str) -> str: + """If the passed language is a variant, return its parent + + Eg: + 1. zh-TW -> zh + 2. sr-BA -> sr + """ + is_language_variant = "-" in lang + if is_language_variant: + return lang[:lang.index("-")] - return guess or frappe.local.lang def get_user_lang(user=None): """Set frappe.local.lang from user preferences on session beginning or resumption""" @@ -75,7 +104,10 @@ def get_user_lang(user=None): return lang def get_lang_code(lang): - return frappe.db.get_value('Language', {'language_name': lang}) or lang + return ( + frappe.db.get_value("Language", {"name": lang}) + or frappe.db.get_value("Language", {"language_name": lang}) + ) def set_default_language(lang): """Set Global default language""" From 76ec9e44e4e9a7ade9509bc1f080c057aa5dbb3c Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 13 Jul 2021 17:09:03 +0530 Subject: [PATCH 06/21] refactor: Rename guess_language as get_language Guess suggests there's some AI involvement. The get_language function has a defined priority. It is deterministic, hence teh name change. --- frappe/auth.py | 4 ++-- frappe/translate.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/frappe/auth.py b/frappe/auth.py index 9135b139ae..7c8c119414 100644 --- a/frappe/auth.py +++ b/frappe/auth.py @@ -10,7 +10,7 @@ import frappe.utils.user from frappe import conf from frappe.sessions import Session, clear_sessions, delete_session from frappe.modules.patch_handler import check_session_stopped -from frappe.translate import guess_language +from frappe.translate import get_language from frappe.utils.password import check_password from frappe.core.doctype.activity_log.activity_log import add_authentication_log from frappe.twofactor import (should_run_2fa, authenticate_for_2factor, @@ -92,7 +92,7 @@ class HTTPRequest: frappe.throw(_("Invalid Request"), frappe.CSRFTokenError) def set_lang(self): - frappe.local.lang = guess_language() + frappe.local.lang = get_language() def get_db_name(self): """get database name from conf""" diff --git a/frappe/translate.py b/frappe/translate.py index c567d0615c..50da6e7297 100644 --- a/frappe/translate.py +++ b/frappe/translate.py @@ -21,7 +21,7 @@ from frappe.model.utils import InvalidIncludePath, render_include from frappe.utils import is_html, strip, strip_html_tags -def guess_language(lang_list=None): +def get_language(lang_list=None): """Set `frappe.local.lang` from HTTP headers at beginning of request Order of priority for setting language: From 736c6c9b8a388c5c9f87509432dd3d53a65de4d9 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 13 Jul 2021 18:18:40 +0530 Subject: [PATCH 07/21] fix: Don't redefine datetime * Sort imports * Update file header --- frappe/auth.py | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/frappe/auth.py b/frappe/auth.py index 7c8c119414..2e92e9fbbb 100644 --- a/frappe/auth.py +++ b/frappe/auth.py @@ -1,22 +1,20 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors -# MIT License. See license.txt -import datetime -from frappe import _ +# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors +# MIT License. See LICENSE +from urllib.parse import quote + import frappe import frappe.database import frappe.utils -from frappe.utils import cint, get_datetime, datetime, date_diff, today import frappe.utils.user -from frappe import conf -from frappe.sessions import Session, clear_sessions, delete_session -from frappe.modules.patch_handler import check_session_stopped -from frappe.translate import get_language -from frappe.utils.password import check_password +from frappe import _, conf from frappe.core.doctype.activity_log.activity_log import add_authentication_log -from frappe.twofactor import (should_run_2fa, authenticate_for_2factor, - confirm_otp_token, get_cached_user_pass) +from frappe.modules.patch_handler import check_session_stopped +from frappe.sessions import Session, clear_sessions, delete_session +from frappe.translate import get_language +from frappe.twofactor import authenticate_for_2factor, confirm_otp_token, get_cached_user_pass, should_run_2fa +from frappe.utils import cint, date_diff, datetime, get_datetime, today +from frappe.utils.password import check_password from frappe.website.utils import get_home_page -from urllib.parse import quote class HTTPRequest: From 0598ddf70ef83cf1bada858ffd7c7b6d8017a098 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Tue, 13 Jul 2021 18:19:38 +0530 Subject: [PATCH 08/21] fix: Clear preferred_language cookie post login If preferred_language was set in cookie pre login, clear it after a successful login so that User or Site specific settings can be applied --- frappe/auth.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/frappe/auth.py b/frappe/auth.py index 2e92e9fbbb..fc1cb09e1a 100644 --- a/frappe/auth.py +++ b/frappe/auth.py @@ -154,8 +154,9 @@ class LoginManager: self.make_session() self.setup_boot_cache() self.set_user_info() + self.clear_preferred_language() - def get_user_info(self, resume=False): + def get_user_info(self): self.info = frappe.db.get_value("User", self.user, ["user_type", "first_name", "last_name", "user_image"], as_dict=1) @@ -193,11 +194,13 @@ class LoginManager: frappe.local.response["redirect_to"] = redirect_to frappe.cache().hdel('redirect_after_login', self.user) - frappe.local.cookie_manager.set_cookie("full_name", self.full_name) frappe.local.cookie_manager.set_cookie("user_id", self.user) frappe.local.cookie_manager.set_cookie("user_image", self.info.user_image or "") + def clear_preferred_language(self): + frappe.local.cookie_manager.delete_cookie("preferred_language") + def make_session(self, resume=False): # start session frappe.local.session_obj = Session(user=self.user, resume=resume, From e00aaf8cc492332ba18fc05c67013eee51f572a7 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 14 Jul 2021 19:30:01 +0530 Subject: [PATCH 09/21] BREAKING CHANGE: Drop frappe.lang in favour of frappe.local.lang --- frappe/__init__.py | 2 -- frappe/boot.py | 2 +- frappe/core/doctype/page/page.py | 2 +- frappe/desk/desk_page.py | 2 +- frappe/desk/query_report.py | 2 +- frappe/sessions.py | 2 +- 6 files changed, 5 insertions(+), 7 deletions(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index 1c978945c7..4015ea8090 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -130,8 +130,6 @@ error_log = local("error_log") debug_log = local("debug_log") message_log = local("message_log") -lang = local("lang") - # This if block is never executed when running the code. It is only used for # telling static code analyzer where to find dynamically defined attributes. if typing.TYPE_CHECKING: diff --git a/frappe/boot.py b/frappe/boot.py index 0589e32ac8..6636cb4329 100644 --- a/frappe/boot.py +++ b/frappe/boot.py @@ -208,7 +208,7 @@ def get_column(doctype): def load_translations(bootinfo): messages = frappe.get_lang_dict("boot") - bootinfo["lang"] = frappe.lang + bootinfo["lang"] = frappe.local.lang # load translated report names for name in bootinfo.user.all_reports: diff --git a/frappe/core/doctype/page/page.py b/frappe/core/doctype/page/page.py index 0ba0e309dd..3a6baef9c4 100644 --- a/frappe/core/doctype/page/page.py +++ b/frappe/core/doctype/page/page.py @@ -141,7 +141,7 @@ class Page(Document): # flag for not caching this page self._dynamic_page = True - if frappe.lang != 'en': + if frappe.local.lang != 'en': from frappe.translate import get_lang_js self.script += get_lang_js("page", self.name) diff --git a/frappe/desk/desk_page.py b/frappe/desk/desk_page.py index d373dbda0e..45e266b957 100644 --- a/frappe/desk/desk_page.py +++ b/frappe/desk/desk_page.py @@ -31,7 +31,7 @@ def getpage(): doc = get(page) # load translations - if frappe.lang != "en": + if frappe.local.lang != "en": send_translations(frappe.get_lang_dict("page", page)) frappe.response.docs.append(doc) diff --git a/frappe/desk/query_report.py b/frappe/desk/query_report.py index 3c0ebf11c1..a127594af1 100644 --- a/frappe/desk/query_report.py +++ b/frappe/desk/query_report.py @@ -187,7 +187,7 @@ def get_script(report_name): script = "frappe.query_reports['%s']={}" % report_name # load translations - if frappe.lang != "en": + if frappe.local.lang != "en": send_translations(frappe.get_lang_dict("report", report_name)) return { diff --git a/frappe/sessions.py b/frappe/sessions.py index 4d922d6769..85a13523ff 100644 --- a/frappe/sessions.py +++ b/frappe/sessions.py @@ -335,7 +335,7 @@ class Session: now = frappe.utils.now() self.data['data']['last_updated'] = now - self.data['data']['lang'] = str(frappe.lang) + self.data['data']['lang'] = str(frappe.local.lang) # update session in db last_updated = frappe.cache().hget("last_db_session_update", self.sid) From e8d50b9d3c6c0a39ed3602ffc7627214f0c7ed72 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 14 Jul 2021 19:32:30 +0530 Subject: [PATCH 10/21] chore: Add types for frappe.translate module *only for recently modified functions --- frappe/translate.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/frappe/translate.py b/frappe/translate.py index 50da6e7297..c3c88f9b0a 100644 --- a/frappe/translate.py +++ b/frappe/translate.py @@ -15,13 +15,14 @@ import functools import os import re from csv import reader +from typing import List, Union import frappe from frappe.model.utils import InvalidIncludePath, render_include from frappe.utils import is_html, strip, strip_html_tags -def get_language(lang_list=None): +def get_language(lang_list: List = None) -> str: """Set `frappe.local.lang` from HTTP headers at beginning of request Order of priority for setting language: @@ -81,7 +82,7 @@ def get_parent_language(lang: str) -> str: return lang[:lang.index("-")] -def get_user_lang(user=None): +def get_user_lang(user: str = None) -> str: """Set frappe.local.lang from user preferences on session beginning or resumption""" if not user: user = frappe.session.user @@ -103,7 +104,7 @@ def get_user_lang(user=None): return lang -def get_lang_code(lang): +def get_lang_code(lang: str) -> Union[str, None]: return ( frappe.db.get_value("Language", {"name": lang}) or frappe.db.get_value("Language", {"language_name": lang}) From 8faf2fd759095b3a54dd3ed0ff16e9ab949e1473 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 14 Jul 2021 19:33:48 +0530 Subject: [PATCH 11/21] refactor(minor): frappe.translate module * Remove unset limit for lru cache in get_parent_language * Simplify get_user_lang and add relevant comment --- frappe/translate.py | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/frappe/translate.py b/frappe/translate.py index c3c88f9b0a..15d771ecdd 100644 --- a/frappe/translate.py +++ b/frappe/translate.py @@ -1,4 +1,4 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors +# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt """ frappe.translate @@ -69,7 +69,7 @@ def get_language(lang_list: List = None) -> str: return frappe.local.lang -@functools.lru_cache(maxsize=None) +@functools.lru_cache() def get_parent_language(lang: str) -> str: """If the passed language is a variant, return its parent @@ -84,21 +84,17 @@ def get_parent_language(lang: str) -> str: def get_user_lang(user: str = None) -> str: """Set frappe.local.lang from user preferences on session beginning or resumption""" - if not user: - user = frappe.session.user - - # via cache + user = user or frappe.session.user lang = frappe.cache().hget("lang", user) if not lang: - - # if defined in user profile - lang = frappe.db.get_value("User", user, "language") - if not lang: - lang = frappe.db.get_default("lang") - - if not lang: - lang = frappe.local.lang or 'en' + # User.language => Session Defaults => frappe.local.lang => 'en' + lang = ( + frappe.db.get_value("User", user, "language") + or frappe.db.get_default("lang") + or frappe.local.lang + or "en" + ) frappe.cache().hset("lang", user, lang) From 2ac1c45c664af135bdaefc934ea258a01d76661f Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 14 Jul 2021 20:12:10 +0530 Subject: [PATCH 12/21] refactor: Maintain common list for Frappe Coverage settings * Maintain common settings for coverage settings of parallel and base test suites * Expand FRAPPE_EXCLUSIONS list based on coveralls.io report --- frappe/commands/utils.py | 24 ++++------------------- frappe/coverage.py | 35 ++++++++++++++++++++++++++++++++++ frappe/parallel_test_runner.py | 25 ++++-------------------- 3 files changed, 43 insertions(+), 41 deletions(-) create mode 100644 frappe/coverage.py diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py index 212642e01b..874ec90d47 100644 --- a/frappe/commands/utils.py +++ b/frappe/commands/utils.py @@ -551,32 +551,16 @@ def run_tests(context, app=None, module=None, doctype=None, test=(), profile=Fal if coverage: from coverage import Coverage + from frappe.coverage import STANDARD_INCLUSIONS, STANDARD_EXCLUSIONS, FRAPPE_EXCLUSIONS # Generate coverage report only for app that is being tested source_path = os.path.join(get_bench_path(), 'apps', app or 'frappe') - incl = [ - '*.py', - ] - omit = [ - '*.js', - '*.xml', - '*.pyc', - '*.css', - '*.less', - '*.scss', - '*.vue', - '*.html', - '*/test_*', - '*/node_modules/*', - '*/doctype/*/*_dashboard.py', - '*/patches/*', - ] + omit = STANDARD_EXCLUSIONS[:] if not app or app == 'frappe': - omit.append('*/tests/*') - omit.append('*/commands/*') + omit.extend(FRAPPE_EXCLUSIONS) - cov = Coverage(source=[source_path], omit=omit, include=incl) + cov = Coverage(source=[source_path], omit=omit, include=STANDARD_INCLUSIONS) cov.start() ret = frappe.test_runner.main(app, module, doctype, context.verbose, tests=tests, diff --git a/frappe/coverage.py b/frappe/coverage.py new file mode 100644 index 0000000000..a59c24a714 --- /dev/null +++ b/frappe/coverage.py @@ -0,0 +1,35 @@ +# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors +# MIT License. See LICENSE +""" + frappe.coverage + ~~~~~~~~~~~~~~~~ + + Coverage settings for frappe +""" + +STANDARD_INCLUSIONS = ["*.py"] + +STANDARD_EXCLUSIONS = [ + '*.js', + '*.xml', + '*.pyc', + '*.css', + '*.less', + '*.scss', + '*.vue', + '*.html', + '*/test_*', + '*/node_modules/*', + '*/doctype/*/*_dashboard.py', + '*/patches/*', +] + +FRAPPE_EXCLUSIONS = [ + "*/tests/*", + "*/commands/*", + "*/frappe/change_log/*", + "*/frappe/exceptions*", + "*frappe/setup.py", + "*/doctype/*/*_dashboard.py", + "*/patches/*", +] diff --git a/frappe/parallel_test_runner.py b/frappe/parallel_test_runner.py index 2f83b88572..6265498c96 100644 --- a/frappe/parallel_test_runner.py +++ b/frappe/parallel_test_runner.py @@ -111,33 +111,16 @@ class ParallelTestRunner(): if self.with_coverage: from coverage import Coverage from frappe.utils import get_bench_path + from frappe.coverage import STANDARD_INCLUSIONS, STANDARD_EXCLUSIONS, FRAPPE_EXCLUSIONS # Generate coverage report only for app that is being tested source_path = os.path.join(get_bench_path(), 'apps', self.app) - incl = [ - '*.py', - ] - omit = [ - '*.js', - '*.xml', - '*.pyc', - '*.css', - '*.less', - '*.scss', - '*.vue', - '*.pyc', - '*.html', - '*/test_*', - '*/node_modules/*', - '*/doctype/*/*_dashboard.py', - '*/patches/*', - ] + omit = STANDARD_EXCLUSIONS[:] if self.app == 'frappe': - omit.append('*/tests/*') - omit.append('*/commands/*') + omit.extend(FRAPPE_EXCLUSIONS) - self.coverage = Coverage(source=[source_path], omit=omit, include=incl) + self.coverage = Coverage(source=[source_path], omit=omit, include=STANDARD_INCLUSIONS) self.coverage.start() def save_coverage(self): From e71eef0ecc1b7601d5df83e007488db0fde236e9 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 14 Jul 2021 20:14:03 +0530 Subject: [PATCH 13/21] chore: Drop dead code (pythonrc file) --- frappe/pythonrc.py | 8 -------- 1 file changed, 8 deletions(-) delete mode 100755 frappe/pythonrc.py diff --git a/frappe/pythonrc.py b/frappe/pythonrc.py deleted file mode 100755 index 6761ead05b..0000000000 --- a/frappe/pythonrc.py +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env python2.7 - -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors -# MIT License. See license.txt - -import os -import frappe -frappe.connect(site=os.environ.get("site")) \ No newline at end of file From 4959dd02f1c8d7cac0ed677eed43ed0bc454a2a5 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 14 Jul 2021 20:14:37 +0530 Subject: [PATCH 14/21] refactor(minor): frappe.translate.get_messages_from_file * Don't re-define frappe util - get_bench_path * Add Python types * Style changes --- frappe/commands/utils.py | 2 +- frappe/translate.py | 20 +++++++++----------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/frappe/commands/utils.py b/frappe/commands/utils.py index 874ec90d47..8fc6877d4f 100644 --- a/frappe/commands/utils.py +++ b/frappe/commands/utils.py @@ -770,7 +770,7 @@ def get_version(output): "table": lambda: render_table( [["App", "Version", "Branch", "Commit"]] + [ - [app_info.app, app_info.version, app_info.branch, app_info.commit] + [app_info.app, app_info.version, app_info.branch, app_info.commit] for app_info in data ] ), diff --git a/frappe/translate.py b/frappe/translate.py index 15d771ecdd..ce4c3abf3d 100644 --- a/frappe/translate.py +++ b/frappe/translate.py @@ -15,11 +15,11 @@ import functools import os import re from csv import reader -from typing import List, Union +from typing import List, Union, Tuple import frappe from frappe.model.utils import InvalidIncludePath, render_include -from frappe.utils import is_html, strip, strip_html_tags +from frappe.utils import get_bench_path, is_html, strip, strip_html_tags def get_language(lang_list: List = None) -> str: @@ -558,7 +558,7 @@ def get_all_messages_from_js_files(app_name=None): return messages -def get_messages_from_file(path): +def get_messages_from_file(path: str) -> List[Tuple[str, str, str, str]]: """Returns a list of transatable strings from a code file :param path: path of the code file @@ -571,7 +571,7 @@ def get_messages_from_file(path): frappe.flags.scanned_files.append(path) - apps_path = get_bench_dir() + bench_path = get_bench_path() if os.path.exists(path): with open(path, 'r') as sourcefile: try: @@ -579,11 +579,12 @@ def get_messages_from_file(path): except Exception: print("Could not scan file for translation: {0}".format(path)) return [] - data = [(os.path.relpath(path, apps_path), message, context, line) \ - for line, message, context in extract_messages_from_code(file_contents)] - return data + + return [ + (os.path.relpath(path, bench_path), message, context, line) + for (line, message, context) in extract_messages_from_code(file_contents) + ] else: - # print "Translate: {0} missing".format(os.path.abspath(path)) return [] def extract_messages_from_code(code): @@ -797,9 +798,6 @@ def deduplicate_messages(messages): ret.append(next(g)) return ret -def get_bench_dir(): - return os.path.join(frappe.__file__, '..', '..', '..', '..') - def rename_language(old_name, new_name): if not frappe.db.exists('Language', new_name): return From 421220a872e07da153b33392005c74a87f08b1ea Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 15 Jul 2021 01:04:04 +0530 Subject: [PATCH 15/21] test: Added tests for frappe.translate.get_language --- frappe/tests/test_translate.py | 54 ++++++++++++++++++++++++++++++++-- frappe/translate.py | 5 +++- 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/frappe/tests/test_translate.py b/frappe/tests/test_translate.py index f51f31d509..717b18ced8 100644 --- a/frappe/tests/test_translate.py +++ b/frappe/tests/test_translate.py @@ -1,13 +1,27 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors +# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt -import frappe, unittest, os +import os +import unittest +from random import choices +from unittest.mock import patch + +import frappe import frappe.translate from frappe import _ +from frappe.auth import HTTPRequest +from frappe.translate import get_parent_language +from frappe.utils import set_request dirname = os.path.dirname(__file__) translation_string_file = os.path.join(dirname, 'translation_test_file.txt') +first_lang, second_lang, third_lang, fourth_lang, fifth_lang = choices( + frappe.get_all("Language", pluck="name"), k=5 +) class TestTranslate(unittest.TestCase): + def tearDown(self): + frappe.form_dict.pop("_lang", None) + def test_extract_message_from_file(self): data = frappe.translate.get_messages_from_file(translation_string_file) self.assertListEqual(data, expected_output) @@ -20,6 +34,42 @@ class TestTranslate(unittest.TestCase): finally: frappe.local.lang = 'en' + def test_request_language_resolution_with_form_dict(self): + """Test for frappe.translate.get_language + + Case 1: frappe.form_dict._lang is set + """ + + frappe.form_dict._lang = first_lang + + with patch.object(frappe.translate, "get_preferred_language_cookie", return_value=second_lang): + set_request(method="POST", path="/", headers=[("Accept-Language", third_lang)]) + HTTPRequest() + + self.assertIn(frappe.local.lang, [first_lang, get_parent_language(first_lang)]) + + def test_request_language_resolution_with_cookie(self): + """Test for frappe.translate.get_language + + Case 2: frappe.form_dict._lang is not set, but preferred_language cookie is + """ + + with patch.object(frappe.translate, "get_preferred_language_cookie", return_value=second_lang): + set_request(method="POST", path="/", headers=[("Accept-Language", third_lang)]) + HTTPRequest() + + self.assertIn(frappe.local.lang, [second_lang, get_parent_language(second_lang)]) + + def test_request_language_resolution_with_request_header(self): + """Test for frappe.translate.get_language + + Case 3: frappe.form_dict._lang & preferred_language cookie is not set, but Accept-Language header is + """ + set_request(method="POST", path="/", headers=[("Accept-Language", third_lang)]) + HTTPRequest() + self.assertIn(frappe.local.lang, [third_lang, get_parent_language(third_lang)]) + + expected_output = [ ('apps/frappe/frappe/tests/translation_test_file.txt', 'Warning: Unable to find {0} in any table related to {1}', 'This is some context', 2), ('apps/frappe/frappe/tests/translation_test_file.txt', 'Warning: Unable to find {0} in any table related to {1}', None, 4), diff --git a/frappe/translate.py b/frappe/translate.py index ce4c3abf3d..d5916f1761 100644 --- a/frappe/translate.py +++ b/frappe/translate.py @@ -44,7 +44,7 @@ def get_language(lang_list: List = None) -> str: lang_set = set(lang_list or get_all_languages() or []) # fetch language from cookie - preferred_language_cookie = frappe.request.cookies.get('preferred_language') + preferred_language_cookie = get_preferred_language_cookie() if preferred_language_cookie: if preferred_language_cookie in lang_set: @@ -906,3 +906,6 @@ def get_all_languages(with_language_name=False): @frappe.whitelist(allow_guest=True) def set_preferred_language_cookie(preferred_language): frappe.local.cookie_manager.set_cookie("preferred_language", preferred_language) + +def get_preferred_language_cookie(): + return frappe.request.cookies.get("preferred_language") From 2ccfb547b50a43e39ce4cbe8d53b81d525689fe6 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 15 Jul 2021 01:11:10 +0530 Subject: [PATCH 16/21] test: Added test to check sanity of HTTPRequest.set_lang --- frappe/tests/test_auth.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/frappe/tests/test_auth.py b/frappe/tests/test_auth.py index bc23cb591c..8447150006 100644 --- a/frappe/tests/test_auth.py +++ b/frappe/tests/test_auth.py @@ -4,8 +4,9 @@ import time import unittest import frappe -from frappe.auth import LoginAttemptTracker +from frappe.auth import HTTPRequest, LoginAttemptTracker from frappe.frappeclient import FrappeClient, AuthError +from frappe.utils import set_request class TestAuth(unittest.TestCase): def __init__(self, *args, **kwargs): @@ -124,3 +125,20 @@ class TestLoginAttemptTracker(unittest.TestCase): tracker.add_failure_attempt() self.assertTrue(tracker.is_user_allowed()) + +class TestFrappeHTTPRequest(unittest.TestCase): + # test frappe.auth.HTTPRequest + def test_set_language(self): + """Check if language is set on object initialization + + This is a test to ensure that language has changed. To test correctness + of frappe.local.lang, check out the tests of frappe.translate.get_language + """ + lang_before_request = frappe.local.lang + random_lang = frappe.get_all("Language", limit=1, pluck="name")[0] + set_request(method='POST', path='/') + frappe.form_dict._lang = random_lang + HTTPRequest() + self.assertTrue(hasattr(frappe.local, "lang")) + self.assertIsInstance(frappe.local.lang, str) + self.assertNotEqual(lang_before_request, frappe.local.lang) From f54894b1e75f30a60e5f611c097f123967901078 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 15 Jul 2021 01:12:13 +0530 Subject: [PATCH 17/21] chore: Update copyright year info in file header --- frappe/sessions.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/frappe/sessions.py b/frappe/sessions.py index 85a13523ff..8e30d0660c 100644 --- a/frappe/sessions.py +++ b/frappe/sessions.py @@ -1,4 +1,4 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors +# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors # MIT License. See license.txt """ Boot session from cache or build @@ -249,7 +249,6 @@ class Session: data = self.get_session_record() if data: - # set language self.data.update({'data': data, 'user':data.user, 'sid': self.sid}) self.user = data.user validate_ip_address(self.user) From 187c777539353b9b9ebfb1cd6089e776a114e5d7 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 15 Jul 2021 14:18:31 +0530 Subject: [PATCH 18/21] chore: Trigger GHA From a6537ce987b208eacc7f1e3b19a2b25de3dae127 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 15 Jul 2021 15:38:36 +0530 Subject: [PATCH 19/21] test: Invoke get_language directly instead of via HTTPRequest --- frappe/tests/test_translate.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/frappe/tests/test_translate.py b/frappe/tests/test_translate.py index 717b18ced8..edab3a82c3 100644 --- a/frappe/tests/test_translate.py +++ b/frappe/tests/test_translate.py @@ -8,8 +8,7 @@ from unittest.mock import patch import frappe import frappe.translate from frappe import _ -from frappe.auth import HTTPRequest -from frappe.translate import get_parent_language +from frappe.translate import get_language, get_parent_language from frappe.utils import set_request dirname = os.path.dirname(__file__) @@ -43,10 +42,9 @@ class TestTranslate(unittest.TestCase): frappe.form_dict._lang = first_lang with patch.object(frappe.translate, "get_preferred_language_cookie", return_value=second_lang): - set_request(method="POST", path="/", headers=[("Accept-Language", third_lang)]) - HTTPRequest() + return_val = get_language() - self.assertIn(frappe.local.lang, [first_lang, get_parent_language(first_lang)]) + self.assertIn(return_val, [first_lang, get_parent_language(first_lang)]) def test_request_language_resolution_with_cookie(self): """Test for frappe.translate.get_language @@ -56,9 +54,9 @@ class TestTranslate(unittest.TestCase): with patch.object(frappe.translate, "get_preferred_language_cookie", return_value=second_lang): set_request(method="POST", path="/", headers=[("Accept-Language", third_lang)]) - HTTPRequest() + return_val = get_language() - self.assertIn(frappe.local.lang, [second_lang, get_parent_language(second_lang)]) + self.assertIn(return_val, [second_lang, get_parent_language(second_lang)]) def test_request_language_resolution_with_request_header(self): """Test for frappe.translate.get_language @@ -66,8 +64,8 @@ class TestTranslate(unittest.TestCase): Case 3: frappe.form_dict._lang & preferred_language cookie is not set, but Accept-Language header is """ set_request(method="POST", path="/", headers=[("Accept-Language", third_lang)]) - HTTPRequest() - self.assertIn(frappe.local.lang, [third_lang, get_parent_language(third_lang)]) + return_val = get_language() + self.assertIn(return_val, [third_lang, get_parent_language(third_lang)]) expected_output = [ From 275a4335c291a56784000139e9e5f32f7afb2eb1 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 15 Jul 2021 15:53:21 +0530 Subject: [PATCH 20/21] ci: Override acceptable semantic commit name types * Add "BREAKING CHANGE" --- .github/semantic.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/semantic.yml b/.github/semantic.yml index e1e53bc1a4..fa15046b4a 100644 --- a/.github/semantic.yml +++ b/.github/semantic.yml @@ -11,3 +11,20 @@ allowRevertCommits: true # For allowed PR types: https://github.com/commitizen/conventional-commit-types/blob/v3.0.0/index.json # Tool Reference: https://github.com/zeke/semantic-pull-requests + +# By default types specified in commitizen/conventional-commit-types is used. +# See: https://github.com/commitizen/conventional-commit-types/blob/v3.0.0/index.json +# You can override the valid types +types: + - BREAKING CHANGE + - feat + - fix + - docs + - style + - refactor + - perf + - test + - build + - ci + - chore + - revert From 42b3c178002e65c7b025561ce3f86e73274b50f3 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 15 Jul 2021 16:02:44 +0530 Subject: [PATCH 21/21] Revert "BREAKING CHANGE: Drop frappe.lang in favour of frappe.local.lang" This reverts commit e00aaf8cc492332ba18fc05c67013eee51f572a7. --- This was pretty much a pointless change since frappe.lang just proxies to frappe.local.lang --- frappe/__init__.py | 2 ++ frappe/boot.py | 2 +- frappe/core/doctype/page/page.py | 2 +- frappe/desk/desk_page.py | 2 +- frappe/desk/query_report.py | 2 +- frappe/sessions.py | 2 +- 6 files changed, 7 insertions(+), 5 deletions(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index 4015ea8090..1c978945c7 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -130,6 +130,8 @@ error_log = local("error_log") debug_log = local("debug_log") message_log = local("message_log") +lang = local("lang") + # This if block is never executed when running the code. It is only used for # telling static code analyzer where to find dynamically defined attributes. if typing.TYPE_CHECKING: diff --git a/frappe/boot.py b/frappe/boot.py index 6636cb4329..0589e32ac8 100644 --- a/frappe/boot.py +++ b/frappe/boot.py @@ -208,7 +208,7 @@ def get_column(doctype): def load_translations(bootinfo): messages = frappe.get_lang_dict("boot") - bootinfo["lang"] = frappe.local.lang + bootinfo["lang"] = frappe.lang # load translated report names for name in bootinfo.user.all_reports: diff --git a/frappe/core/doctype/page/page.py b/frappe/core/doctype/page/page.py index 3a6baef9c4..0ba0e309dd 100644 --- a/frappe/core/doctype/page/page.py +++ b/frappe/core/doctype/page/page.py @@ -141,7 +141,7 @@ class Page(Document): # flag for not caching this page self._dynamic_page = True - if frappe.local.lang != 'en': + if frappe.lang != 'en': from frappe.translate import get_lang_js self.script += get_lang_js("page", self.name) diff --git a/frappe/desk/desk_page.py b/frappe/desk/desk_page.py index 45e266b957..d373dbda0e 100644 --- a/frappe/desk/desk_page.py +++ b/frappe/desk/desk_page.py @@ -31,7 +31,7 @@ def getpage(): doc = get(page) # load translations - if frappe.local.lang != "en": + if frappe.lang != "en": send_translations(frappe.get_lang_dict("page", page)) frappe.response.docs.append(doc) diff --git a/frappe/desk/query_report.py b/frappe/desk/query_report.py index a127594af1..3c0ebf11c1 100644 --- a/frappe/desk/query_report.py +++ b/frappe/desk/query_report.py @@ -187,7 +187,7 @@ def get_script(report_name): script = "frappe.query_reports['%s']={}" % report_name # load translations - if frappe.local.lang != "en": + if frappe.lang != "en": send_translations(frappe.get_lang_dict("report", report_name)) return { diff --git a/frappe/sessions.py b/frappe/sessions.py index 8e30d0660c..0b469616b8 100644 --- a/frappe/sessions.py +++ b/frappe/sessions.py @@ -334,7 +334,7 @@ class Session: now = frappe.utils.now() self.data['data']['last_updated'] = now - self.data['data']['lang'] = str(frappe.local.lang) + self.data['data']['lang'] = str(frappe.lang) # update session in db last_updated = frappe.cache().hget("last_db_session_update", self.sid)