From e893312f371b5fe0e55f01304abe2f4e239f2c47 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Wed, 10 Jan 2024 17:21:19 +0530 Subject: [PATCH] perf: parallelize MO compilation This operation is CPU heavy and has little to no Frappe code involvement. So we can use a processpool of 4 to reduce time taken for compilation by 4x. --- frappe/gettext/translate.py | 37 +++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/frappe/gettext/translate.py b/frappe/gettext/translate.py index de4c99f506..4d609eb6fb 100644 --- a/frappe/gettext/translate.py +++ b/frappe/gettext/translate.py @@ -1,5 +1,6 @@ import csv import gettext +import multiprocessing import os from collections import defaultdict from datetime import datetime @@ -169,24 +170,36 @@ def new_po(locale, target_app: str | None = None): def compile_translations(target_app: str | None = None, locale: str | None = None, force=False): apps = [target_app] if target_app else frappe.get_all_apps(True) - + tasks = [] for app in apps: locales = [locale] if locale else get_locales(app) for current_locale in locales: - po_path = get_po_path(app, current_locale) - mo_path = get_mo_path(app, current_locale) - if not po_path.exists(): - continue + tasks.append((app, current_locale, force)) - if mo_path.exists() and po_path.stat().st_mtime < mo_path.stat().st_mtime and not force: - print(f"MO file already up to date at {mo_path}") - continue + # Execute all tasks, doing this sequentially is quite slow hence use processpool of 4 + # processes. + executer = multiprocessing.Pool(processes=4) + executer.starmap(_compile_translation, tasks) - with open(po_path, "rb") as f: - catalog = read_po(f) + executer.close() + executer.join() - mo_path = write_binary(app, catalog, current_locale) - print(f"MO file created at {mo_path}") + +def _compile_translation(app, locale, force=False): + po_path = get_po_path(app, locale) + mo_path = get_mo_path(app, locale) + if not po_path.exists(): + return + + if mo_path.exists() and po_path.stat().st_mtime < mo_path.stat().st_mtime and not force: + print(f"MO file already up to date at {mo_path}") + return + + with open(po_path, "rb") as f: + catalog = read_po(f) + + mo_path = write_binary(app, catalog, locale) + print(f"MO file created at {mo_path}") def update_po(target_app: str | None = None, locale: str | None = None):