diff --git a/webnotes/__init__.py b/webnotes/__init__.py index e9ae70d9b3..250b7e12bb 100644 --- a/webnotes/__init__.py +++ b/webnotes/__init__.py @@ -257,7 +257,9 @@ def clear_cache(user=None, doctype=None): elif user: clear_cache(user) else: # everything + import translate clear_cache() + translate.clear_cache() reset_metadata_version() def get_roles(username=None): diff --git a/webnotes/build.py b/webnotes/build.py index 75e6a4c2a1..768bbc59b9 100644 --- a/webnotes/build.py +++ b/webnotes/build.py @@ -10,12 +10,14 @@ Build the `public` folders and setup languages import os, sys, webnotes, json from cssmin import cssmin +import webnotes.translate def bundle(no_compress): """concat / minify js files""" # build js files make_site_public_dirs() build(no_compress) + webnotes.translate.clear_cache() def watch(no_compress): """watch and rebuild if necessary""" @@ -53,8 +55,7 @@ def make_site_public_dirs(): site_public_assets = os.path.join(site_public_path, "assets") if not os.path.exists(site_public_assets): os.symlink(os.path.abspath(assets_path), site_public_assets) - - + def clear_pyc_files(): from webnotes.utils import get_base_path for path, folders, files in os.walk(get_base_path()): diff --git a/webnotes/cli.py b/webnotes/cli.py index c1a640db70..1e81a4978d 100755 --- a/webnotes/cli.py +++ b/webnotes/cli.py @@ -563,13 +563,13 @@ def mysql(): webnotes.destroy() @cmd -def python(): +def python(site): import webnotes import commands, os python = commands.getoutput('which python') if site: os.environ["site"] = site - os.environ["PYTHONSTARTUP"] = os.path.join(os.path.dirname(__file__), "pythonrc.py") + os.environ["PYTHONSTARTUP"] = os.path.join(os.path.dirname(webnotes.__file__), "pythonrc.py") os.execv(python, [python]) webnotes.destroy() diff --git a/webnotes/client.py b/webnotes/client.py index bd67c6ef7c..7faa20426c 100644 --- a/webnotes/client.py +++ b/webnotes/client.py @@ -6,7 +6,7 @@ import webnotes from webnotes import _ import webnotes.model import webnotes.utils -import json +import json, os @webnotes.whitelist() def get(doctype, name=None, filters=None): @@ -118,4 +118,16 @@ def bulk_update(docs): @webnotes.whitelist() def has_permission(doctype, docname, perm_type="read"): # perm_type can be one of read, write, create, submit, cancel, report - return {"has_permission": webnotes.has_permission(doctype, perm_type.lower(), docname)} \ No newline at end of file + return {"has_permission": webnotes.has_permission(doctype, perm_type.lower(), docname)} + +@webnotes.whitelist() +def get_js(src): + contentpath = os.path.join(webnotes.local.sites_path, src) + with open(contentpath, "r") as srcfile: + code = srcfile.read() + + if webnotes.local.lang != "en": + code += "\n\n$.extend(wn._messages, {})".format(json.dumps(\ + webnotes.get_lang_dict("jsfile", contentpath))) + return code + \ No newline at end of file diff --git a/webnotes/public/js/wn/assets.js b/webnotes/public/js/wn/assets.js index 6f2b28c04e..423957e478 100644 --- a/webnotes/public/js/wn/assets.js +++ b/webnotes/public/js/wn/assets.js @@ -13,10 +13,7 @@ wn.require = function(items) { for(var i=0; i< l; i++) { var src = items[i]; - //if(!(src in wn.assets.executed_)) { - // check if available in localstorage wn.assets.execute(src); - //} } }; @@ -83,23 +80,19 @@ wn.assets = { // load an asset via load: function(src) { // this is virtual page load, only get the the source - // *without* the template - var t = src; - + // *without* the template wn.set_loading(); - $.ajax({ - url: t, - data: { - q: Math.floor(Math.random()*1000) + wn.call({ + method:"webnotes.client.get_js", + args: { + "src": src }, - dataType: 'text', - success: function(txt) { - // add it to localstorage - wn.assets.add(src, txt); + callback: function(r) { + wn.assets.add(src, r.message); }, async: false - }); + }) wn.done_loading(); }, diff --git a/webnotes/public/js/wn/request.js b/webnotes/public/js/wn/request.js index f44341c3da..27befbc7f8 100644 --- a/webnotes/public/js/wn/request.js +++ b/webnotes/public/js/wn/request.js @@ -40,7 +40,8 @@ wn.call = function(opts) { btn: opts.btn, freeze: opts.freeze, show_spinner: !opts.no_spinner, - progress_bar: opts.progress_bar + progress_bar: opts.progress_bar, + async: opts.async }); } @@ -64,7 +65,8 @@ wn.request.call = function(opts) { wn.request.cleanup(opts, {}); show_alert(wn._("Unable to complete request: ") + textStatus) opts.error && opts.error(xhr) - } + }, + async: opts.async }; wn.last_request = ajax_args.data; diff --git a/webnotes/pythonrc.py b/webnotes/pythonrc.py index 6954f2d3ea..72f466aacf 100755 --- a/webnotes/pythonrc.py +++ b/webnotes/pythonrc.py @@ -5,8 +5,6 @@ from __future__ import unicode_literals -import os, sys -sys.path = [".", "lib", "app"] + sys.path - +import os import webnotes -webnotes.connect(site=os.environ.get("site")) +webnotes.connect(site=os.environ.get("site")) \ No newline at end of file diff --git a/webnotes/tests/test_translation.py b/webnotes/tests/test_translation.py index 245c3d690b..b43fa5300e 100644 --- a/webnotes/tests/test_translation.py +++ b/webnotes/tests/test_translation.py @@ -48,7 +48,8 @@ class TestTranslations(unittest.TestCase): def test_write_csv(self): tpath = webnotes.get_pymodule_path("webnotes", "translations", "de.csv") - os.remove(tpath) + if os.path.exists(tpath): + os.remove(tpath) webnotes.translate.write_translations_file("webnotes", "de") self.assertTrue(os.path.exists(tpath)) self.assertEquals(dict(webnotes.translate.read_csv_file(tpath)).get("Row"), "Reihe") diff --git a/webnotes/translate.py b/webnotes/translate.py index 4ca90541ff..ab69bce303 100644 --- a/webnotes/translate.py +++ b/webnotes/translate.py @@ -16,7 +16,7 @@ Contributing: # wn.require # webnotes._ -import webnotes, os, re, codecs +import webnotes, os, re, codecs, json def get_all_languages(): return [a.split()[0] for a in get_lang_info()] @@ -41,17 +41,37 @@ def write_translations_file(app, lang, full_dict=None): def get_dict(fortype, name=None): fortype = fortype.lower() - if fortype=="doctype": - messages = get_messages_from_doctype(name) - elif fortype=="page": - messages = get_messages_from_page(name) - elif fortype=="report": - messages = get_messages_from_report(name) - elif fortype=="include": - messages = get_messages_from_include_files() + cache = webnotes.cache() + cache_key = "translation_assets:" + webnotes.local.lang + asset_key = fortype + ":" + name + translation_assets = cache.get_value(cache_key) or {} + + if not asset_key in translation_assets: + if fortype=="doctype": + messages = get_messages_from_doctype(name) + elif fortype=="page": + messages = get_messages_from_page(name) + elif fortype=="report": + messages = get_messages_from_report(name) + elif fortype=="include": + messages = get_messages_from_include_files() + elif fortype=="jsfile": + messages = get_messages_from_file(name) + translation_assets[asset_key] = make_dict_from_messages(messages) + cache.set_value(cache_key, translation_assets) + + return translation_assets[asset_key] + +def add_lang_dict(code): + messages = extract_messages_from_code(code) + code += "\n\n$.extend(wn._messages, %s)" % json.dumps(make_dict_from_messages(messages)) + return code + +def make_dict_from_messages(messages, full_dict=None): out = {} - full_dict = get_full_dict(webnotes.local.lang) + if full_dict==None: + full_dict = get_full_dict(webnotes.local.lang) for m in messages: if m in full_dict: @@ -60,7 +80,7 @@ def get_dict(fortype, name=None): return out def get_lang_js(fortype, name): - return "\n\n$.extend(wn._messages, %s)" % get_dict(fortype, name) + return "\n\n$.extend(wn._messages, %s)" % json.dumps(get_dict(fortype, name)) def get_full_dict(lang): if lang == "en": return {} @@ -80,6 +100,7 @@ def clear_cache(): cache.delete_value("langinfo") for lang in get_all_languages(): cache.delete_value("lang:" + lang) + cache.delete_value("translation_assets:" + lang) def get_messages_for_app(app): messages = [] @@ -174,14 +195,18 @@ def get_messages_from_include_files(app_name=None): def get_messages_from_file(path): """get list of messages from a code file""" - messages = [] if os.path.exists(path): with open(path, 'r') as sourcefile: - txt = sourcefile.read() - messages += re.findall('_\("([^"]*)".*\)', txt) - messages += re.findall("_\('([^']*)'.*\)", txt) - messages += re.findall('_\("{3}([^"]*)"{3}.*\)', txt, re.S) - + return extract_messages_from_code(sourcefile.read(), path.endswith(".py")) + else: + return [] + +def extract_messages_from_code(code, is_py=False): + messages = [] + messages += re.findall('_\("([^"]*)"\)', code) + messages += re.findall("_\('([^']*)'\)", code) + if is_py: + messages += re.findall('_\("{3}([^"]*)"{3}.*\)', code, re.S) return messages def clean(messages): @@ -257,393 +282,4 @@ def google_translate(lang, untranslated): return dict(zip(untranslated, translated)) else: print "unable to translate" - return {} - - - - - -# import webnotes -# import os -# import codecs -# import json -# import re -# from csv import reader -# from webnotes.modules import get_doc_path,get_doctype_module -# from webnotes.utils import cstr -# -# def translate(lang=None): -# languages = [lang] -# if lang=="all" or lang==None: -# languages = get_all_languages() -# -# print "Extracting / updating translatable strings..." -# build_message_files() -# -# print "Compiling messages in one file..." -# export_messages(lang, '_lang_tmp.csv') -# -# for lang in languages: -# if lang != "en": -# filename = 'app/translations/'+lang+'.csv' -# print "For " + lang + ":" -# print "Translating via Google Translate..." -# google_translate(lang, '_lang_tmp.csv', filename) -# print "Updating language files..." -# import_messages(lang, filename) -# -# print "Deleting temp file..." -# os.remove('_lang_tmp.csv') -# -# def get_all_languages(): -# languages = [a.split()[0] for a in \ -# webnotes.get_file_items(os.path.join(webnotes.local.sites_path, "languages.txt"))] -# -# def get_lang_dict(): -# languages_path = os.path.join(os.path.dirname(webnotes.__file__), "data", "languages.json") -# if os.path.exists(languages_path): -# with open(languages_path, "r") as langfile: -# return json.loads(langfile.read()) -# else: return {} -# -# def update_translations(): -# """ -# compare language file timestamps with last updated timestamps in `.wnf-lang-status` -# if timestamps are missing / changed, build new `.json` files in the `lang folders` -# """ -# langstatus = {} -# languages = get_all_languages() -# message_updated = False -# status_file_path = "app/.wnf-lang-status" -# -# if not os.path.exists(os.path.join('app', 'translations')): -# return -# if os.path.exists(status_file_path): -# with open(status_file_path, "r") as langstatusfile: -# langstatus = eval(langstatusfile.read()) -# -# for lang in languages: -# filename = os.path.join('app', 'translations', lang + '.csv') -# if langstatus.get(lang, None)!=os.path.getmtime(filename): -# print "Setting up lang files for " + lang + "..." -# if not message_updated: -# print "Extracting / updating translatable strings..." -# build_message_files() -# message_updated = True -# print "Writing translations..." -# import_messages(lang, filename) -# langstatus[lang] = os.path.getmtime(filename) -# -# with open(status_file_path, "w") as langstatusfile: -# langstatus = langstatusfile.write(str(langstatus)) -# -# def build_message_files(): -# """build from doctypes, pages, database and framework""" -# if not webnotes.conn: -# webnotes.connect() -# -# for app in webnotes.get_all_apps(True): -# build_for_pages(webnotes.get_pymodule_path(app)) -# build_from_doctype_code(webnotes.get_pymodule_path(app)) -# -# #reports -# build_from_query_report() -# -# # doctype -# build_from_database() -# -# build_for_framework('lib/webnotes', 'py', with_doctype_names=True) -# build_for_framework('lib/public/js/wn', 'js') -# build_for_framework('app/public/js', 'js', with_doctype_names=True) -# -# def build_for_pages(path): -# """make locale files for framework py and js (all)""" -# messages = [] -# for (basepath, folders, files) in os.walk(path): -# if 'locale' in folders: folders.remove('locale') -# if os.path.basename(os.path.dirname(basepath))=="page": -# messages_js, messages_py = [], [] -# for fname in files: -# fname = cstr(fname) -# if fname.endswith('.js'): -# messages_js += get_message_list(os.path.join(basepath, fname)) -# if fname.endswith('.py'): -# messages_py += get_message_list(os.path.join(basepath, fname)) -# if messages_js: -# write_messages_file(basepath, messages_js, "js") -# if messages_py: -# write_messages_file(basepath, messages_py, "py") -# -# def build_from_query_report(): -# """make locale for the query reports from database and the framework js and py files""" -# import re -# for item in webnotes.conn.sql("""select name, report_name,ref_doctype, query -# from `tabReport`""", as_dict=1): -# messages_js, messages_py = [], [] -# -# if item: -# messages_js.append(item.report_name) -# messages_py.append(item.report_name) -# # get the messages from the query using the regex : -# # if we have the string "Production Date:Date:180" in the query then the regex will search for string between " and : . -# # the regex will take "Production Date" and store them into messages -# if item.query : -# messages_query = re.findall('"([^:,^"]*):', item.query) -# messages_js += messages_query -# messages_py += messages_query -# -# module = get_doctype_module(item.ref_doctype) -# if module : -# doctype_path = get_doc_path(module, "Report", item.name) -# if os.path.exists(doctype_path): -# for (basepath, folders, files) in os.walk(doctype_path): -# if 'locale' in folders: folders.remove('locale') -# for fname in files: -# if fname.endswith('.js'): -# messages_js += get_message_list(os.path.join(basepath, fname)) -# if fname.endswith('.py'): -# messages_py += get_message_list(os.path.join(basepath, fname)) -# break -# write_messages_file(doctype_path, messages_js, 'js') -# write_messages_file(doctype_path, messages_py, 'py') -# -# -# def build_from_database(): -# """make doctype labels, names, options, descriptions""" -# def get_select_options(doc): -# if doc.doctype=="DocField" and doc.fieldtype=='Select' and doc.options \ -# and not doc.options.startswith("link:") \ -# and not doc.options.startswith("attach_files:"): -# return doc.options.split('\n') -# else: -# return [] -# -# build_for_doc_from_database(webnotes._dict({ -# "doctype": "DocType", -# "module_field": "module", -# "DocType": ["name", "description", "module"], -# "DocField": ["label", "description"], -# "custom": get_select_options -# })) -# -# def build_for_doc_from_database(fields): -# for item in webnotes.conn.sql("""select name from `tab%s`""" % fields.doctype, as_dict=1): -# messages = [] -# doclist = webnotes.bean(fields.doctype, item.name).doclist -# -# for doc in doclist: -# if doc.doctype in fields: -# messages += map(lambda x: x in fields[doc.doctype] and doc.fields.get(x) or None, -# doc.fields.keys()) -# -# if fields.custom: -# messages += fields.custom(doc) -# -# doc = doclist[0] -# if doc.fields.get(fields.module_field): -# doctype_path = get_doc_path(doc.fields[fields.module_field], -# doc.doctype, doc.name) -# write_messages_file(doctype_path, messages, 'doc') -# -# def build_for_framework(path, mtype, with_doctype_names = False): -# """make locale files for framework py and js (all)""" -# messages = [] -# for (basepath, folders, files) in os.walk(path): -# if 'locale' in folders: folders.remove('locale') -# for fname in files: -# fname = cstr(fname) -# if fname.endswith('.' + mtype): -# messages += get_message_list(os.path.join(basepath, fname)) -# -# -# # append module & doctype names -# if with_doctype_names: -# for m in webnotes.conn.sql("""select name, module from `tabDocType`"""): -# messages.append(m[0]) -# messages.append(m[1]) -# -# # append labels from config.json -# config = webnotes.get_config() -# for moduleinfo in config["modules"].values(): -# if moduleinfo.get("label"): -# messages.append(moduleinfo["label"]) -# -# if messages: -# write_messages_file(path, messages, mtype) -# -# def build_from_doctype_code(path): -# """walk and make locale files in all folders""" -# for (basepath, folders, files) in os.walk(path): -# messagespy = [] -# messagesjs = [] -# for fname in files: -# fname = cstr(fname) -# if fname.endswith('py'): -# messagespy += get_message_list(os.path.join(basepath, fname)) -# if fname.endswith('js'): -# messagesjs += get_message_list(os.path.join(basepath, fname)) -# -# if messagespy: -# write_messages_file(basepath, messagespy, 'py') -# -# if messagespy: -# write_messages_file(basepath, messagesjs, 'js') -# -# def get_message_list(path): -# """get list of messages from a code file""" -# import re -# messages = [] -# with open(path, 'r') as sourcefile: -# txt = sourcefile.read() -# messages += re.findall('_\("([^"]*)".*\)', txt) -# messages += re.findall("_\('([^']*)'.*\)", txt) -# messages += re.findall('_\("{3}([^"]*)"{3}.*\)', txt, re.S) -# -# return messages -# -# def write_messages_file(path, messages, mtype): -# """write messages to translation file""" -# if not os.path.exists(path): -# return -# -# if not os.path.exists(os.path.join(path, 'locale')): -# os.makedirs(os.path.join(path, 'locale')) -# -# fname = os.path.join(path, 'locale', '_messages_' + mtype + '.json') -# messages = list(set(messages)) -# filtered = [] -# for m in messages: -# if m and re.search('[a-zA-Z]+', m): -# filtered.append(m) -# with open(fname, 'w') as msgfile: -# msgfile.write(json.dumps(filtered, indent=1)) -# -# def export_messages(lang, outfile): -# """get list of all messages""" -# messages = {} -# # extract messages -# for (basepath, folders, files) in os.walk('.'): -# def _get_messages(messages, basepath, mtype): -# mlist = get_messages(basepath, mtype) -# if not mlist: -# return -# -# # update messages with already existing translations -# langdata = get_lang_data(basepath, lang, mtype) -# for m in mlist: -# if not messages.get(m): -# messages[m] = langdata.get(m, "") -# -# if os.path.basename(basepath)=='locale': -# _get_messages(messages, basepath, 'doc') -# _get_messages(messages, basepath, 'py') -# _get_messages(messages, basepath, 'js') -# -# # remove duplicates -# if outfile: -# from csv import writer -# with open(outfile, 'w') as msgfile: -# w = writer(msgfile) -# keys = messages.keys() -# keys.sort() -# for m in keys: -# -# w.writerow([m.encode('utf-8'), messages.get(m, '').encode('utf-8')]) -# -# def import_messages(lang, infile): -# """make individual message files for each language""" -# data = dict(get_all_messages_from_file(infile)) -# -# for (basepath, folders, files) in os.walk('.'): -# def _update_lang_file(mtype): -# """create a langauge file for the given message type""" -# messages = get_messages(basepath, mtype) -# if not messages: return -# -# # read existing -# langdata = get_lang_data(basepath, lang, mtype) -# -# # update fresh -# for m in messages: -# if data.get(m): -# langdata[m] = data.get(m) -# -# if langdata: -# # write new langfile -# langfilename = os.path.join(basepath, lang + '-' + mtype + '.json') -# with open(langfilename, 'w') as langfile: -# langfile.write(json.dumps(langdata, indent=1, sort_keys=True).encode('utf-8')) -# #print 'wrote ' + langfilename -# -# if os.path.basename(basepath)=='locale': -# # make / update lang files for each type of message file (doc, js, py) -# # example: hi-doc.json, hi-js.json, hi-py.json -# _update_lang_file('doc') -# _update_lang_file('js') -# _update_lang_file('py') -# -# def load_doc_messages(module, doctype, name): -# if webnotes.lang=="en": -# return {} -# -# if not webnotes.local.translated_docs: -# webnotes.local.translated_docs = [] -# -# doc_path = get_doc_path(module, doctype, name) -# -# # don't repload the same doc again -# if (webnotes.lang + ":" + doc_path) in webnotes.local.translated_docs: -# return -# -# if not docs_loaded: -# webnotes.local.translate_docs_loaded = [] -# webnotes.local.translated_docs.append(webnotes.lang + ":" + doc_path) -# -# webnotes.local.translations.update(get_lang_data(doc_path, None, 'doc')) -# -# def get_lang_data(basepath, lang, mtype): -# """get language dict from langfile""" -# -# # add "locale" folder if reqd -# if os.path.basename(basepath) != 'locale': -# basepath = os.path.join(basepath, 'locale') -# -# if not lang: lang = webnotes.local.lang -# -# path = os.path.join(basepath, lang + '-' + mtype + '.json') -# -# langdata = {} -# if os.path.exists(path): -# with codecs.open(path, 'r', 'utf-8') as langfile: -# langdata = json.loads(langfile.read()) -# -# return langdata -# -# def get_messages(basepath, mtype): -# """load list of messages from _message files""" -# # get message list -# path = os.path.join(basepath, '_messages_' + mtype + '.json') -# messages = [] -# if os.path.exists(path): -# with open(path, 'r') as msgfile: -# messages = json.loads(msgfile.read()) -# -# return messages -# -# def update_lang_js(jscode, path): -# return jscode + "\n\n$.extend(wn._messages, %s)" % \ -# json.dumps(get_lang_data(path, webnotes.lang, 'js')) -# -# def get_all_messages_from_file(path): -# with codecs.open(path, 'r', 'utf-8') as msgfile: -# data = msgfile.read() -# data = reader([r.encode('utf-8') for r in data.splitlines()]) -# newdata = [] -# for row in data: -# newrow = [] -# for val in row: -# newrow.append(unicode(val, 'utf-8')) -# newdata.append(newrow) -# -# return newdata -# + return {} \ No newline at end of file diff --git a/webnotes/widgets/page.py b/webnotes/widgets/page.py index 08f16ea73e..904b87531b 100644 --- a/webnotes/widgets/page.py +++ b/webnotes/widgets/page.py @@ -26,7 +26,7 @@ def getpage(): if has_permission(doclist): # load translations if webnotes.lang != "en": - webnotes.response["__messages"] = webnotes.get_lang_dict("page", d.name) + webnotes.response["__messages"] = webnotes.get_lang_dict("page", page) webnotes.response['docs'] = doclist else: diff --git a/webnotes/widgets/query_report.py b/webnotes/widgets/query_report.py index 9963639a7c..cb0f220ed4 100644 --- a/webnotes/widgets/query_report.py +++ b/webnotes/widgets/query_report.py @@ -63,7 +63,8 @@ def run(report_name, filters=None): else: module = webnotes.conn.get_value("DocType", report.ref_doctype, "module") if report.is_standard=="Yes": - method_name = scrub(module) + ".report." + scrub(report.name) + "." + scrub(report.name) + ".execute" + method_name = webnotes.local.module_app[scrub(module)] + "." + scrub(module) \ + + ".report." + scrub(report.name) + "." + scrub(report.name) + ".execute" columns, result = webnotes.get_attr(method_name)(filters or {}) result = get_filtered_data(report.ref_doctype, columns, result)