feat: gettext utils
This commit is contained in:
barredterra
parent
4de07d0c88
commit
a891feb440
1 changed files with 429 additions and 0 deletions
429
frappe/gettext/translate.py
Normal file
429
frappe/gettext/translate.py
Normal file
|
|
@ -0,0 +1,429 @@
|
|||
import csv
|
||||
import gettext
|
||||
import os
|
||||
from collections import defaultdict
|
||||
from contextlib import suppress
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
from babel.messages.catalog import Catalog
|
||||
from babel.messages.extract import extract_from_dir
|
||||
from babel.messages.mofile import read_mo, write_mo
|
||||
from babel.messages.pofile import read_po, write_po
|
||||
|
||||
import frappe
|
||||
|
||||
DEFAULT_LANG = "en"
|
||||
LOCALE_DIR = "locale"
|
||||
MERGED_TRANSLATION_KEY = "merged_translations"
|
||||
APP_TRANSLATION_KEY = "translations_from_apps"
|
||||
USER_TRANSLATION_KEY = "lang_user_translations"
|
||||
POT_FILE = "main.pot"
|
||||
TRANSLATION_DOMAIN = "messages"
|
||||
|
||||
|
||||
def get_translator(lang: str, localedir: str | None = LOCALE_DIR, context: bool | None = False):
|
||||
t = gettext.translation(TRANSLATION_DOMAIN, localedir=localedir, languages=(lang,), fallback=True)
|
||||
return t.pgettext if context else t.gettext
|
||||
|
||||
|
||||
def new_catalog(app: str, locale: str | None = None) -> Catalog:
|
||||
def get_hook(hook, app):
|
||||
return frappe.get_hooks(hook, [None], app)[0]
|
||||
|
||||
app_email = get_hook("app_email", app)
|
||||
return Catalog(
|
||||
locale=locale,
|
||||
domain="messages",
|
||||
msgid_bugs_address=app_email,
|
||||
language_team=app_email,
|
||||
copyright_holder=get_hook("app_publisher", app),
|
||||
last_translator=app_email,
|
||||
project=get_hook("app_title", app),
|
||||
creation_date=datetime.now(),
|
||||
revision_date=datetime.now(),
|
||||
fuzzy=False,
|
||||
)
|
||||
|
||||
|
||||
def get_locales_dir(app: str) -> Path:
|
||||
return Path(frappe.get_app_path(app)) / LOCALE_DIR
|
||||
|
||||
|
||||
def get_locales(app: str) -> list[str]:
|
||||
return [locale.name for locale in get_locales_dir(app).iterdir() if locale.is_dir()]
|
||||
|
||||
|
||||
def get_po_path(app: str, locale: str | None = None) -> Path:
|
||||
return get_locales_dir(app) / locale / "LC_MESSAGES" / "messages.po"
|
||||
|
||||
|
||||
def get_pot_path(app: str) -> Path:
|
||||
return get_locales_dir(app) / "main.pot"
|
||||
|
||||
|
||||
def get_catalog(app: str, locale: str | None = None) -> Catalog:
|
||||
"""Returns a catatalog for the given app and locale"""
|
||||
po_path = get_po_path(app, locale) if locale else get_pot_path(app)
|
||||
|
||||
if not po_path.exists():
|
||||
return new_catalog(app, locale)
|
||||
|
||||
with open(po_path, "rb") as f:
|
||||
return read_po(f)
|
||||
|
||||
|
||||
def write_catalog(app: str, catalog: Catalog, locale: str | None = None) -> Path:
|
||||
"""Writes a catalog to the given app and locale"""
|
||||
po_path = get_po_path(app, locale) if locale else get_pot_path(app)
|
||||
|
||||
if not po_path.parent.exists():
|
||||
po_path.parent.mkdir(parents=True)
|
||||
|
||||
with open(po_path, "wb") as f:
|
||||
write_po(f, catalog, sort_output=True, ignore_obsolete=True)
|
||||
|
||||
return po_path
|
||||
|
||||
|
||||
def write_binary(app: str, catalog: Catalog, locale: str) -> Path:
|
||||
po_path = get_po_path(app, locale)
|
||||
mo_path = po_path.with_suffix(".mo")
|
||||
with open(mo_path, "wb") as mo_file:
|
||||
write_mo(mo_file, catalog)
|
||||
|
||||
return mo_path
|
||||
|
||||
|
||||
def generate_pot(target_app: str | None = None):
|
||||
"""
|
||||
Generate a POT (PO template) file. This file will contain only messages IDs.
|
||||
https://en.wikipedia.org/wiki/Gettext
|
||||
:param target_app: If specified, limit to `app`
|
||||
"""
|
||||
|
||||
def directory_filter(dirpath: str | os.PathLike[str]) -> bool:
|
||||
if "public/dist" in dirpath:
|
||||
return False
|
||||
|
||||
subdir = os.path.basename(dirpath)
|
||||
return not (subdir.startswith(".") or subdir.startswith("_"))
|
||||
|
||||
apps = [target_app] if target_app else frappe.get_all_apps(True)
|
||||
method_map = [
|
||||
("**.py", "frappe.gettext.extractors.python.extract"),
|
||||
("**.js", "frappe.gettext.extractors.javascript.extract"),
|
||||
("**/doctype/*/*.json", "frappe.gettext.extractors.doctype.extract"),
|
||||
("**/workspace/*/*.json", "frappe.gettext.extractors.workspace.extract"),
|
||||
("**.html", "frappe.gettext.extractors.jinja2.extract"),
|
||||
("hooks.py", "frappe.gettext.extractors.navbar.extract"),
|
||||
("**/report/*/*.json", "frappe.gettext.extractors.report.extract"),
|
||||
]
|
||||
|
||||
for app in apps:
|
||||
app_path = frappe.get_pymodule_path(app)
|
||||
catalog = get_catalog(app)
|
||||
|
||||
for filename, lineno, message, comments, context in extract_from_dir(
|
||||
app_path, method_map, directory_filter=directory_filter
|
||||
):
|
||||
if not message:
|
||||
continue
|
||||
|
||||
catalog.add(message, locations=[(filename, lineno)], auto_comments=comments, context=context)
|
||||
|
||||
pot_path = write_catalog(app, catalog)
|
||||
print(f"POT file created at {pot_path}")
|
||||
|
||||
|
||||
def new_po(locale, target_app: str | None = None):
|
||||
apps = [target_app] if target_app else frappe.get_all_apps(True)
|
||||
|
||||
for target_app in apps:
|
||||
po_path = get_po_path(target_app, locale)
|
||||
if os.path.exists(po_path):
|
||||
print(f"{po_path} exists. Skipping")
|
||||
continue
|
||||
|
||||
pot_catalog = get_catalog(target_app)
|
||||
pot_catalog.locale = locale
|
||||
po_path = write_catalog(target_app, pot_catalog, locale)
|
||||
|
||||
print(f"PO file created_at {po_path}")
|
||||
print(
|
||||
"You will need to add the language in frappe/geo/languages.json, if you haven't done it already."
|
||||
)
|
||||
|
||||
|
||||
def compile(target_app: str | None = None, locale: str | None = None):
|
||||
apps = [target_app] if target_app else frappe.get_all_apps(True)
|
||||
|
||||
for app in apps:
|
||||
locales = [locale] if locale else get_locales(app)
|
||||
for locale in locales:
|
||||
catalog = get_catalog(app, locale)
|
||||
mo_path = write_binary(app, catalog, locale)
|
||||
print(f"MO file created at {mo_path}")
|
||||
|
||||
|
||||
def update_po(target_app: str | None = None, locale: str | None = None):
|
||||
"""
|
||||
Add keys to available PO files, from POT file. This could be used to keep
|
||||
track of available keys, and missing translations
|
||||
:param target_app: Limit operation to `app`, if specified
|
||||
"""
|
||||
apps = [target_app] if target_app else frappe.get_all_apps(True)
|
||||
|
||||
for app in apps:
|
||||
locales = [locale] if locale else get_locales(app)
|
||||
pot_catalog = get_catalog(app)
|
||||
for locale in locales:
|
||||
po_catalog = get_catalog(app, locale)
|
||||
po_catalog.update(pot_catalog)
|
||||
po_path = write_catalog(app, po_catalog, locale)
|
||||
print(f"PO file modified at {po_path}")
|
||||
|
||||
|
||||
def migrate(app: str | None = None, locale: str | None = None):
|
||||
apps = [app] if app else frappe.get_all_apps(True)
|
||||
|
||||
for app in apps:
|
||||
if locale:
|
||||
csv_to_po(app, locale)
|
||||
else:
|
||||
app_path = Path(frappe.get_app_path(app))
|
||||
for filename in (app_path / "translations").iterdir():
|
||||
if filename.suffix != ".csv":
|
||||
continue
|
||||
csv_to_po(app, filename.stem)
|
||||
|
||||
|
||||
def csv_to_po(app: str, locale: str):
|
||||
csv_file = Path(frappe.get_app_path(app)) / "translations" / f"{locale.replace('_', '-')}.csv"
|
||||
locale = locale.replace("-", "_")
|
||||
if not csv_file.exists():
|
||||
return
|
||||
|
||||
catalog: Catalog = get_catalog(app)
|
||||
msgid_context_map = defaultdict(list)
|
||||
for message in catalog:
|
||||
msgid_context_map[message.id].append(message.context)
|
||||
|
||||
with open(csv_file) as f:
|
||||
for row in csv.reader(f):
|
||||
if len(row) < 2:
|
||||
continue
|
||||
|
||||
msgid = escape_percent(row[0])
|
||||
msgstr = escape_percent(row[1])
|
||||
msgctxt = row[2] if len(row) >= 3 else None
|
||||
|
||||
if not msgctxt:
|
||||
# if old context is not defined, add msgstr to all contexts
|
||||
for context in msgid_context_map.get(msgid, []):
|
||||
if message := catalog.get(msgid, context):
|
||||
message.string = msgstr
|
||||
elif message := catalog.get(msgid, msgctxt):
|
||||
message.string = msgstr
|
||||
|
||||
po_path = write_catalog(app, catalog, locale)
|
||||
print(f"PO file created at {po_path}")
|
||||
|
||||
|
||||
def f(msg: str, context: str = None, lang: str = DEFAULT_LANG) -> str:
|
||||
"""
|
||||
Method to translate a string
|
||||
:param msg: Key to translate
|
||||
:param context: Translation context
|
||||
:param lang: Language to fetch
|
||||
:return: Translated string. Could be original string
|
||||
"""
|
||||
from frappe import as_unicode
|
||||
from frappe.utils import is_html, strip_html_tags
|
||||
|
||||
if not lang:
|
||||
lang = DEFAULT_LANG
|
||||
|
||||
msg = as_unicode(msg).strip()
|
||||
|
||||
if is_html(msg):
|
||||
msg = strip_html_tags(msg)
|
||||
|
||||
apps = frappe.get_all_apps()
|
||||
|
||||
for app in apps:
|
||||
app_path = frappe.get_pymodule_path(app)
|
||||
locale_path = os.path.join(app_path, LOCALE_DIR)
|
||||
has_context = context is not None
|
||||
|
||||
if has_context:
|
||||
t = get_translator(lang, localedir=locale_path, context=has_context)
|
||||
r = t(context, msg)
|
||||
if r != msg:
|
||||
return r
|
||||
|
||||
t = get_translator(lang, localedir=locale_path, context=False)
|
||||
r = t(msg)
|
||||
|
||||
if r != msg:
|
||||
return r
|
||||
|
||||
return msg
|
||||
|
||||
|
||||
def get_messages_for_boot():
|
||||
"""
|
||||
Return all message translations that are required on boot
|
||||
"""
|
||||
messages = get_all_translations(frappe.local.lang)
|
||||
messages.update(get_dict_from_hooks("boot", None))
|
||||
|
||||
return messages
|
||||
|
||||
|
||||
def get_dict_from_hooks(fortype: str, name: str) -> dict[str, str]:
|
||||
"""
|
||||
Get and run a custom translator method from hooks for item.
|
||||
Hook example:
|
||||
```
|
||||
get_translated_dict = {
|
||||
("doctype", "Global Defaults"): "frappe.geo.country_info.get_translated_dict",
|
||||
}
|
||||
```
|
||||
:param fortype: Item type. eg: doctype
|
||||
:param name: Item name. eg: User
|
||||
:return: Dictionary with translated messages
|
||||
"""
|
||||
translated_dict = {}
|
||||
hooks = frappe.get_hooks("get_translated_dict")
|
||||
|
||||
for (hook_fortype, fortype_name) in hooks:
|
||||
if hook_fortype == fortype and fortype_name == name:
|
||||
for method in hooks[(hook_fortype, fortype_name)]:
|
||||
translated_dict.update(frappe.get_attr(method)())
|
||||
|
||||
return translated_dict
|
||||
|
||||
|
||||
def make_dict_from_messages(messages, full_dict=None, load_user_translation=True):
|
||||
"""Returns translated messages as a dict in Language specified in `frappe.local.lang`
|
||||
:param messages: List of untranslated messages
|
||||
"""
|
||||
out = {}
|
||||
if full_dict is None:
|
||||
if load_user_translation:
|
||||
full_dict = get_all_translations(frappe.local.lang)
|
||||
else:
|
||||
full_dict = get_translations_from_apps(frappe.local.lang)
|
||||
|
||||
for m in messages:
|
||||
if m[1] in full_dict:
|
||||
out[m[1]] = full_dict[m[1]]
|
||||
# check if msg with context as key exist eg. msg:context
|
||||
if len(m) > 2 and m[2]:
|
||||
key = m[1] + ":" + m[2]
|
||||
if full_dict.get(key):
|
||||
out[key] = full_dict[key]
|
||||
|
||||
return out
|
||||
|
||||
|
||||
def get_all_translations(lang: str) -> dict[str, str]:
|
||||
"""
|
||||
Load and return the entire translations dictionary for a language from apps
|
||||
+ user translations.
|
||||
:param lang: Language Code, e.g. `hi`
|
||||
:return: dictionary of key and value
|
||||
"""
|
||||
if not lang:
|
||||
return {}
|
||||
|
||||
def t():
|
||||
all_translations = get_translations_from_apps(lang)
|
||||
with suppress(Exception):
|
||||
# get user specific translation data
|
||||
user_translations = get_user_translations(lang)
|
||||
all_translations.update(user_translations)
|
||||
|
||||
return all_translations
|
||||
|
||||
try:
|
||||
return frappe.cache().hget(MERGED_TRANSLATION_KEY, lang, generator=t)
|
||||
except Exception:
|
||||
# People mistakenly call translation function on global variables where
|
||||
# locals are not initialized, translations don't make much sense there
|
||||
return {}
|
||||
|
||||
|
||||
def get_translations_from_apps(lang, apps=None):
|
||||
"""
|
||||
Combine all translations from `.mo` files in all `apps`. For derivative
|
||||
languages (es-GT), take translations from the base language (es) and then
|
||||
update translations from the child (es-GT)
|
||||
"""
|
||||
if not lang or lang == DEFAULT_LANG:
|
||||
return {}
|
||||
|
||||
def t():
|
||||
translations = {}
|
||||
|
||||
for app in apps or frappe.get_all_apps(True):
|
||||
app_path = frappe.get_pymodule_path(app)
|
||||
localedir = os.path.join(app_path, LOCALE_DIR)
|
||||
mo_files = gettext.find(TRANSLATION_DOMAIN, localedir, (lang,), True)
|
||||
|
||||
for file in mo_files:
|
||||
with open(file, "rb") as f:
|
||||
po = read_mo(f)
|
||||
for m in po:
|
||||
translations[m.id] = m.string
|
||||
|
||||
return translations
|
||||
|
||||
return frappe.cache().hget(APP_TRANSLATION_KEY, lang, shared=True, generator=t)
|
||||
|
||||
|
||||
def get_user_translations(lang: str) -> dict[str, str]:
|
||||
"""
|
||||
Get translations from db, created by user
|
||||
:param lang: language to fetch
|
||||
:return: translation key/value
|
||||
"""
|
||||
if not frappe.db:
|
||||
frappe.connect()
|
||||
|
||||
def f():
|
||||
user_translations = {}
|
||||
translations = frappe.get_all(
|
||||
"Translation",
|
||||
fields=["source_text", "translated_text", "context"],
|
||||
filters={"language": lang},
|
||||
)
|
||||
|
||||
for t in translations:
|
||||
key = t.source_text
|
||||
value = t.translated_text
|
||||
if t.context:
|
||||
key += ":" + t.context
|
||||
user_translations[key] = value
|
||||
|
||||
return user_translations
|
||||
|
||||
return frappe.cache().hget(USER_TRANSLATION_KEY, lang, generator=f)
|
||||
|
||||
|
||||
def clear_cache():
|
||||
"""Clear all translation assets from :meth:`frappe.cache`"""
|
||||
cache = frappe.cache()
|
||||
cache.delete_key("langinfo")
|
||||
|
||||
# clear translations saved in boot cache
|
||||
cache.delete_key("bootinfo")
|
||||
cache.delete_key("translation_assets", shared=True)
|
||||
cache.delete_key(APP_TRANSLATION_KEY, shared=True)
|
||||
cache.delete_key(USER_TRANSLATION_KEY)
|
||||
cache.delete_key(MERGED_TRANSLATION_KEY)
|
||||
|
||||
|
||||
def escape_percent(s: str):
|
||||
return s.replace("%", "%")
|
||||
Loading…
Add table
Reference in a new issue