From 22990ff3434dc6cd38e8bde5b6c139a537b73f9c Mon Sep 17 00:00:00 2001 From: Sagar Vora Date: Thu, 27 Oct 2022 13:59:08 +0530 Subject: [PATCH 1/2] fix: use `async...await` when parsing route --- frappe/public/js/frappe/recorder/RecorderRoot.vue | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe/public/js/frappe/recorder/RecorderRoot.vue b/frappe/public/js/frappe/recorder/RecorderRoot.vue index 0aa5a42469..7c802ccec1 100644 --- a/frappe/public/js/frappe/recorder/RecorderRoot.vue +++ b/frappe/public/js/frappe/recorder/RecorderRoot.vue @@ -12,8 +12,8 @@ import { useRoute } from "vue-router" let route = useRoute(); -watch(route, () => { - frappe.router.current_route = frappe.router.parse(); +watch(route, async () => { + frappe.router.current_route = await frappe.router.parse(); frappe.breadcrumbs.update(); frappe.recorder.route = route; }); From 3ddac5fe925ade001e67e3517bc1b857f3c13d17 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Thu, 27 Oct 2022 16:28:02 +0530 Subject: [PATCH 2/2] refactor: accurate translation caching (#18595) * refactor: rename poorly named functions * refactor: getting translation from apps - use generator - use sane name for cache key - avoid manual handling of frappe.local state just use cache() interface --- frappe/__init__.py | 10 +- .../doctype/translation/test_translation.py | 13 ++- .../core/doctype/translation/translation.py | 5 +- frappe/tests/test_search.py | 1 - frappe/tests/utils.py | 1 - frappe/translate.py | 103 ++++++++++-------- 6 files changed, 71 insertions(+), 62 deletions(-) diff --git a/frappe/__init__.py b/frappe/__init__.py index f8e4de34d1..ad07b11caf 100644 --- a/frappe/__init__.py +++ b/frappe/__init__.py @@ -89,7 +89,7 @@ def _(msg: str, lang: str | None = None, context: str | None = None) -> str: _('Change') _('Change', context='Coins') """ - from frappe.translate import get_full_dict + from frappe.translate import get_all_translations from frappe.utils import is_html, strip_html_tags if not hasattr(local, "lang"): @@ -107,14 +107,15 @@ def _(msg: str, lang: str | None = None, context: str | None = None) -> str: msg = as_unicode(msg).strip() translated_string = "" + + all_translations = get_all_translations(lang) if context: string_key = f"{msg}:{context}" - translated_string = get_full_dict(lang).get(string_key) + translated_string = all_translations.get(string_key) if not translated_string: - translated_string = get_full_dict(lang).get(msg) + translated_string = all_translations.get(msg) - # return lang_full_dict according to lang passed parameter return translated_string or non_translated_string @@ -222,7 +223,6 @@ def init(site: str, sites_path: str = ".", new_site: bool = False) -> None: local.conf = _dict(get_site_config()) local.lang = local.conf.lang or "en" - local.lang_full_dict = None local.module_app = None local.app_modules = None diff --git a/frappe/core/doctype/translation/test_translation.py b/frappe/core/doctype/translation/test_translation.py index ebed447b00..5602fa2c2d 100644 --- a/frappe/core/doctype/translation/test_translation.py +++ b/frappe/core/doctype/translation/test_translation.py @@ -3,6 +3,7 @@ import frappe from frappe import _ from frappe.tests.utils import FrappeTestCase +from frappe.translate import clear_cache class TestTranslation(FrappeTestCase): @@ -11,20 +12,17 @@ class TestTranslation(FrappeTestCase): def tearDown(self): frappe.local.lang = "en" - clear_translation_cache() + clear_cache() def test_doctype(self): translation_data = get_translation_data() for key, val in translation_data.items(): frappe.local.lang = key - clear_translation_cache() translation = create_translation(key, val) self.assertEqual(_(val[0]), val[1]) frappe.delete_doc("Translation", translation.name) - clear_translation_cache() - self.assertEqual(_(val[0]), val[0]) def test_parent_language(self): @@ -55,6 +53,10 @@ class TestTranslation(FrappeTestCase): clear_translation_cache() self.assertTrue(_(data[1][0]), data[1][1]) + def test_multi_language_translations(self): + source = "User" + self.assertNotEqual(_(source, lang="de"), _(source, lang="es")) + def test_html_content_data_translation(self): source = """ str: """Set `frappe.local.lang` from HTTP headers at beginning of request @@ -215,7 +221,7 @@ def get_dict(fortype: str, name: str | None = None) -> dict[str, str]: def get_messages_for_boot(): """Return all message translations that are required on boot.""" - messages = get_full_dict(frappe.local.lang) + messages = get_all_translations(frappe.local.lang) messages.update(get_dict_from_hooks("boot", None)) return messages @@ -241,9 +247,9 @@ def make_dict_from_messages(messages, full_dict=None, load_user_translation=True out = {} if full_dict is None: if load_user_translation: - full_dict = get_full_dict(frappe.local.lang) + full_dict = get_all_translations(frappe.local.lang) else: - full_dict = load_lang(frappe.local.lang) + full_dict = get_translations_from_apps(frappe.local.lang) for m in messages: if m[1] in full_dict: @@ -266,31 +272,29 @@ def get_lang_js(fortype: str, name: str) -> str: return f"\n\n$.extend(frappe._messages, {json.dumps(get_dict(fortype, name))})" -def get_full_dict(lang: str) -> dict[str, str]: - """Load and return the entire translations dictionary for a language from :meth:`frape.cache` +def get_all_translations(lang: str) -> dict[str, str]: + """Load and return the entire translations dictionary for a language from apps + user translations. :param lang: Language Code, e.g. `hi` """ if not lang: return {} - # found in local, return! - if getattr(frappe.local, "lang_full_dict", None) is not None: - return frappe.local.lang_full_dict + def _merge_translations(): + all_translations = get_translations_from_apps(lang).copy() + try: + # get user specific translation data + user_translations = get_user_translations(lang) + all_translations.update(user_translations) + except Exception: + pass - frappe.local.lang_full_dict = load_lang(lang) + return all_translations - try: - # get user specific translation data - user_translations = get_user_translations(lang) - frappe.local.lang_full_dict.update(user_translations) - except Exception: - pass - - return frappe.local.lang_full_dict + return frappe.cache().hget(MERGED_TRANSLATION_KEY, lang, generator=_merge_translations) -def load_lang(lang, apps=None): +def get_translations_from_apps(lang, apps=None): """Combine all translations from `.csv` files in all `apps`. For derivative languages (es-GT), take translations from the base language (es) and then update translations from the child (es-GT)""" @@ -298,22 +302,20 @@ def load_lang(lang, apps=None): if lang == "en": return {} - out = frappe.cache().hget("lang_full_dict", lang, shared=True) - if not out: - out = {} + def _get_from_disk(): + translations = {} for app in apps or frappe.get_all_apps(True): path = os.path.join(frappe.get_pymodule_path(app), "translations", lang + ".csv") - out.update(get_translation_dict_from_file(path, lang, app) or {}) - + translations.update(get_translation_dict_from_file(path, lang, app) or {}) if "-" in lang: parent = lang.split("-")[0] - parent_out = load_lang(parent) - parent_out.update(out) - out = parent_out + parent_translations = get_translations_from_apps(parent) + parent_translations.update(translations) + return parent_translations - frappe.cache().hset("lang_full_dict", lang, out, shared=True) + return translations - return out or {} + return frappe.cache().hget(APP_TRANSLATION_KEY, lang, shared=True, generator=_get_from_disk) def get_translation_dict_from_file(path, lang, app, throw=False) -> dict[str, str]: @@ -342,23 +344,22 @@ def get_translation_dict_from_file(path, lang, app, throw=False) -> dict[str, st def get_user_translations(lang): if not frappe.db: frappe.connect() - out = frappe.cache().hget("lang_user_translations", lang) - if out is None: - out = {} - user_translations = frappe.get_all( + + def _read_from_db(): + user_translations = {} + translations = frappe.get_all( "Translation", fields=["source_text", "translated_text", "context"], filters={"language": lang} ) - for translation in user_translations: - key = translation.source_text - value = translation.translated_text - if translation.context: - key += ":" + translation.context - out[key] = value + for t in translations: + key = t.source_text + value = t.translated_text + if t.context: + key += ":" + t.context + user_translations[key] = value + return user_translations - frappe.cache().hset("lang_user_translations", lang, out) - - return out + return frappe.cache().hget(USER_TRANSLATION_KEY, lang, generator=_read_from_db) def clear_cache(): @@ -368,9 +369,10 @@ def clear_cache(): # clear translations saved in boot cache cache.delete_key("bootinfo") - cache.delete_key("lang_full_dict", shared=True) cache.delete_key("translation_assets", shared=True) - cache.delete_key("lang_user_translations") + cache.delete_key(APP_TRANSLATION_KEY, shared=True) + cache.delete_key(USER_TRANSLATION_KEY) + cache.delete_key(MERGED_TRANSLATION_KEY) def get_messages_for_app(app, deduplicate=True): @@ -1050,7 +1052,7 @@ def get_untranslated(lang, untranslated_file, get_all=False, app="_ALL_APPS"): # replace \n with ||| so that internal linebreaks don't get split f.write((escape_newlines(m[1]) + os.linesep).encode("utf-8")) else: - full_dict = get_full_dict(lang) + full_dict = get_all_translations(lang) for m in messages: if not full_dict.get(m[1]): @@ -1073,7 +1075,7 @@ def update_translations(lang, untranslated_file, translated_file, app="_ALL_APPS :param untranslated_file: File path with the messages in English. :param translated_file: File path with messages in language to be updated.""" clear_cache() - full_dict = get_full_dict(lang) + full_dict = get_all_translations(lang) def restore_newlines(s): return ( @@ -1110,7 +1112,7 @@ def update_translations(lang, untranslated_file, translated_file, app="_ALL_APPS def import_translations(lang, path): """Import translations from file in standard format""" clear_cache() - full_dict = get_full_dict(lang) + full_dict = get_all_translations(lang) full_dict.update(get_translation_dict_from_file(path, lang, "import")) for app in frappe.get_all_apps(True): @@ -1140,7 +1142,9 @@ def write_translations_file(app, lang, full_dict=None, app_messages=None): tpath = frappe.get_pymodule_path(app, "translations") frappe.create_folder(tpath) - write_csv_file(os.path.join(tpath, lang + ".csv"), app_messages, full_dict or get_full_dict(lang)) + write_csv_file( + os.path.join(tpath, lang + ".csv"), app_messages, full_dict or get_all_translations(lang) + ) def send_translations(translation_dict): @@ -1302,3 +1306,8 @@ def get_translated_doctypes(): "Property Setter", {"property": "translated_doctype", "value": "1"}, pluck="doc_type" ) return unique(dts + custom_dts) + + +# Backward compatibility +get_full_dict = get_all_translations +load_lang = get_translations_from_apps