From e7f69cc2d76558ea2513e9d610508875d09207be Mon Sep 17 00:00:00 2001 From: Thomas Kluyver Date: Tue, 31 Oct 2017 18:40:16 +0000 Subject: [PATCH] Work on loading UI translations (#2969) * Load translations for Javascript in page template * Normalise language codes to gettext format with underscores * .mo files need to be under LC_MESSAGES as well * remove unused JS code * Normalise result in test * Fix for opening files on Py 2 * Fix location of I18N directory * Add translation files to package_data --- .gitignore | 3 +- notebook/base/handlers.py | 3 + notebook/i18n/README.md | 4 +- notebook/i18n/__init__.py | 99 +++++++++++++++++++ .../i18n/{zh-CN => zh_CN}/LC_MESSAGES/nbjs.po | 0 .../i18n/{zh-CN => zh_CN}/LC_MESSAGES/nbui.po | 0 .../{zh-CN => zh_CN}/LC_MESSAGES/notebook.po | 0 notebook/static/base/js/i18n.js | 46 +-------- notebook/static/base/js/i18nload.js | 26 ----- notebook/templates/page.html | 2 + notebook/tests/test_i18n.py | 10 ++ setupbase.py | 1 + 12 files changed, 123 insertions(+), 71 deletions(-) create mode 100644 notebook/i18n/__init__.py rename notebook/i18n/{zh-CN => zh_CN}/LC_MESSAGES/nbjs.po (100%) rename notebook/i18n/{zh-CN => zh_CN}/LC_MESSAGES/nbui.po (100%) rename notebook/i18n/{zh-CN => zh_CN}/LC_MESSAGES/notebook.po (100%) delete mode 100644 notebook/static/base/js/i18nload.js create mode 100644 notebook/tests/test_i18n.py diff --git a/.gitignore b/.gitignore index 028aba96c..9be814a1c 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,8 @@ docs/man/*.gz docs/source/api/generated docs/source/config.rst docs/gh-pages -notebook/i18n/*/*.mo +notebook/i18n/*/LC_MESSAGES/*.mo +notebook/i18n/*/LC_MESSAGES/nbjs.json notebook/static/components notebook/static/style/*.min.css* notebook/static/*/js/built/ diff --git a/notebook/base/handlers.py b/notebook/base/handlers.py index 771e9e30c..63d90446b 100755 --- a/notebook/base/handlers.py +++ b/notebook/base/handlers.py @@ -34,6 +34,7 @@ from ipython_genutils.py3compat import string_types import notebook from notebook._tz import utcnow +from notebook.i18n import combine_translations from notebook.utils import is_hidden, url_path_join, url_is_absolute, url_escape from notebook.services.security import csp_report_uri @@ -409,6 +410,8 @@ class IPythonHandler(AuthenticatedHandler): xsrf_form_html=self.xsrf_form_html, token=self.token, xsrf_token=self.xsrf_token.decode('utf8'), + nbjs_translations=json.dumps(combine_translations( + self.request.headers.get('Accept-Language', ''))), **self.jinja_template_vars ) diff --git a/notebook/i18n/README.md b/notebook/i18n/README.md index cbe00d8da..04f900409 100644 --- a/notebook/i18n/README.md +++ b/notebook/i18n/README.md @@ -62,8 +62,8 @@ code for your desired language ( i.e. German = "de", Japanese = "ja", etc. ). use at runtime. ```shell -pybabel compile -D notebook -f -l ${LANG} -i ${LANG}/LC_MESSAGES/notebook.po -o ${LANG}/notebook.mo -pybabel compile -D nbui -f -l ${LANG} -i ${LANG}/LC_MESSAGES/nbui.po -o ${LANG}/nbui.mo +pybabel compile -D notebook -f -l ${LANG} -i ${LANG}/LC_MESSAGES/notebook.po -o ${LANG}/LC_MESSAGES/notebook.mo +pybabel compile -D nbui -f -l ${LANG} -i ${LANG}/LC_MESSAGES/nbui.po -o ${LANG}/LC_MESSAGES/nbui.mo ``` *nbjs.po* needs to be converted to JSON for use within the JavaScript code, with *po2json*, as follows: diff --git a/notebook/i18n/__init__.py b/notebook/i18n/__init__.py new file mode 100644 index 000000000..63fde70f0 --- /dev/null +++ b/notebook/i18n/__init__.py @@ -0,0 +1,99 @@ +"""Server functions for loading translations +""" +from collections import defaultdict +import errno +import io +import json +from os.path import dirname, join as pjoin +import re + +I18N_DIR = dirname(__file__) +# Cache structure: +# {'nbjs': { # Domain +# 'zh-CN': { # Language code +# : +# ... +# } +# }} +TRANSLATIONS_CACHE = {'nbjs': {}} + + +_accept_lang_re = re.compile(r''' +(?P[a-zA-Z]{1,8}(-[a-zA-Z]{1,8})?) +(\s*;\s*q\s*=\s* + (?P[01](.\d+)?) +)?''', re.VERBOSE) + +def parse_accept_lang_header(accept_lang): + """Parses the 'Accept-Language' HTTP header. + + Returns a list of language codes in *ascending* order of preference + (with the most preferred language last). + """ + by_q = defaultdict(list) + for part in accept_lang.split(','): + m = _accept_lang_re.match(part.strip()) + if not m: + continue + lang, qvalue = m.group('lang', 'qvalue') + # Browser header format is zh-CN, gettext uses zh_CN + lang = lang.replace('-', '_') + if qvalue is None: + qvalue = 1. + else: + qvalue = float(qvalue) + if qvalue == 0: + continue # 0 means not accepted + by_q[qvalue].append(lang) + + res = [] + for qvalue, langs in sorted(by_q.items()): + res.extend(sorted(langs)) + return res + +def load(language, domain='nbjs'): + """Load translations from an nbjs.json file""" + try: + f = io.open(pjoin(I18N_DIR, language, 'LC_MESSAGES', 'nbjs.json'), + encoding='utf-8') + except IOError as e: + if e.errno != errno.ENOENT: + raise + return {} + + with f: + data = json.load(f) + return data["locale_data"][domain] + +def cached_load(language, domain='nbjs'): + """Load translations for one language, using in-memory cache if available""" + domain_cache = TRANSLATIONS_CACHE[domain] + try: + return domain_cache[language] + except KeyError: + data = load(language, domain) + domain_cache[language] = data + return data + +def combine_translations(accept_language, domain='nbjs'): + """Combine translations for multiple accepted languages. + + Returns data re-packaged in jed1.x format. + """ + lang_codes = parse_accept_lang_header(accept_language) + combined = {} + for language in lang_codes: + if language == 'en': + # en is default, all translations are in frontend. + combined.clear() + else: + combined.update(cached_load(language, domain)) + + combined[''] = {"domain":"nbjs"} + + return { + "domain": domain, + "locale_data": { + domain: combined + } + } diff --git a/notebook/i18n/zh-CN/LC_MESSAGES/nbjs.po b/notebook/i18n/zh_CN/LC_MESSAGES/nbjs.po similarity index 100% rename from notebook/i18n/zh-CN/LC_MESSAGES/nbjs.po rename to notebook/i18n/zh_CN/LC_MESSAGES/nbjs.po diff --git a/notebook/i18n/zh-CN/LC_MESSAGES/nbui.po b/notebook/i18n/zh_CN/LC_MESSAGES/nbui.po similarity index 100% rename from notebook/i18n/zh-CN/LC_MESSAGES/nbui.po rename to notebook/i18n/zh_CN/LC_MESSAGES/nbui.po diff --git a/notebook/i18n/zh-CN/LC_MESSAGES/notebook.po b/notebook/i18n/zh_CN/LC_MESSAGES/notebook.po similarity index 100% rename from notebook/i18n/zh-CN/LC_MESSAGES/notebook.po rename to notebook/i18n/zh_CN/LC_MESSAGES/notebook.po diff --git a/notebook/static/base/js/i18n.js b/notebook/static/base/js/i18n.js index 3785056ec..464d5f236 100644 --- a/notebook/static/base/js/i18n.js +++ b/notebook/static/base/js/i18n.js @@ -4,51 +4,13 @@ // Module to handle i18n ( Internationalization ) and translated UI define([ - 'jed', - 'moment', - 'json!../../../i18n/nbjs.json', - 'base/js/i18nload', - ], function(Jed, moment, nbjs, i18nload) { + 'jed' + ], function(Jed) { "use strict"; - - // Setup language related stuff - var ui_lang = navigator.languages && navigator.languages[0] || // Chrome / Firefox - navigator.language || // All browsers - navigator.userLanguage; // IE <= 10 - var init = function() { - var msg_promise; - if (nbjs.supported_languages.indexOf(ui_lang) >= 0) { - moment.locale(ui_lang); - msg_promise = new Promise( function (resolve, reject) { - require([i18nload.id+"!"+ui_lang], function (data) { - var newi18n = new Jed(data); - newi18n._ = newi18n.gettext; - resolve(newi18n); - }, function (error) { - console.log("Error loading translations for language: "+ui_lang); - var newi18n = new Jed(nbjs); - newi18n._ = newi18n.gettext; - resolve(newi18n); - }); - }); - } else { - msg_promise = new Promise( function (resolve, reject) { - var newi18n = new Jed(nbjs); - newi18n._ = newi18n.gettext; - resolve(newi18n); - }); - } - return msg_promise; - } - var i18n = new Jed(nbjs); + var i18n = new Jed(document.nbjs_translations); i18n._ = i18n.gettext; i18n.msg = i18n; // Just a place holder until the init promise resolves. - - init().then(function (msg) { - i18n.msg = msg; - i18n.msg._ = i18n.msg.gettext; - }); - + return i18n; }); diff --git a/notebook/static/base/js/i18nload.js b/notebook/static/base/js/i18nload.js deleted file mode 100644 index 420a1aa2b..000000000 --- a/notebook/static/base/js/i18nload.js +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Plugin to load a single locale. - */ -define([ - "require", - "module", - // These are only here so that the optimizer knows which ones we MIGHT load. - // We will actually only load the ones we need. There should be one entry - // here for each language you want to support. - // For example, for German.... - // "json!base/../../i18n/de/LC_MESSAGES/nbjs.json" - ], function (require, module) { - return { - id: module.id, - - load: function (locale, callerRequire, onload, loaderConfig) { - - var dependencies = "json!base/../../i18n/"+locale+"/LC_MESSAGES/nbjs.json"; - - // Load the JSON file requested - require([dependencies], function (data) { - onload(data); - }); - } - }; -}); diff --git a/notebook/templates/page.html b/notebook/templates/page.html index 1373e560c..edc48ab1d 100644 --- a/notebook/templates/page.html +++ b/notebook/templates/page.html @@ -97,6 +97,8 @@ return {}; } }) + + document.nbjs_translations = {{ nbjs_translations|safe }}; {% block meta %} diff --git a/notebook/tests/test_i18n.py b/notebook/tests/test_i18n.py new file mode 100644 index 000000000..d2555ac35 --- /dev/null +++ b/notebook/tests/test_i18n.py @@ -0,0 +1,10 @@ +import nose.tools as nt + +from notebook import i18n + +def test_parse_accept_lang_header(): + palh = i18n.parse_accept_lang_header + nt.assert_equal(palh(''), []) + nt.assert_equal(palh('zh-CN,en-GB;q=0.7,en;q=0.3'), + ['en', 'en_GB', 'zh_CN']) + nt.assert_equal(palh('nl,fr;q=0'), ['nl']) diff --git a/setupbase.py b/setupbase.py index 9dc7c603e..1b1372761 100644 --- a/setupbase.py +++ b/setupbase.py @@ -210,6 +210,7 @@ def find_package_data(): 'notebook.tests' : js_tests, 'notebook.bundler.tests': ['resources/*', 'resources/*/*', 'resources/*/*/.*'], 'notebook.services.api': ['api.yaml'], + 'notebook.i18n': ['*/LC_MESSAGES/*.*'], } return package_data