Wrap all strings in _td function for i18n

package.json

@@ -11,7 +11,11 @@
   },
   "license": "Apache-2.0",
   "files": [],
+  "bin": {
+    "matrix-gen-i18n": "scripts/gen-i18n.js"
+  },
   "scripts": {
+    "i18n": "matrix-gen-i18n",
     "mkdirs": "mkdirp packages deploys",
     "fetch": "yarn run mkdirs && node scripts/fetch-package.js",
     "asar-webapp": "asar p webapp webapp.asar",

scripts/gen-i18n.js

@@ -0,0 +1,304 @@
+#!/usr/bin/env node
+
+/*
+Copyright 2017 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/**
+ * Regenerates the translations en_EN file by walking the source tree and
+ * parsing each file with the appropriate parser. Emits a JSON file with the
+ * translatable strings mapped to themselves in the order they appeared
+ * in the files and grouped by the file they appeared in.
+ *
+ * Usage: node scripts/gen-i18n.js
+ */
+const fs = require('fs');
+const path = require('path');
+
+const walk = require('walk');
+
+const parser = require("@babel/parser");
+const traverse = require("@babel/traverse");
+
+const TRANSLATIONS_FUNCS = ['_t', '_td'];
+
+const INPUT_TRANSLATIONS_FILE = 'src/i18n/strings/en_EN.json';
+const OUTPUT_FILE = 'src/i18n/strings/en_EN.json';
+
+// NB. The sync version of walk is broken for single files so we walk
+// all of res rather than just res/home.html.
+// https://git.daplie.com/Daplie/node-walk/merge_requests/1 fixes it,
+// or if we get bored waiting for it to be merged, we could switch
+// to a project that's actively maintained.
+const SEARCH_PATHS = ['src', 'res'];
+
+function getObjectValue(obj, key) {
+    for (const prop of obj.properties) {
+        if (prop.key.type === 'Identifier' && prop.key.name === key) {
+            return prop.value;
+        }
+    }
+    return null;
+}
+
+function getTKey(arg) {
+    if (arg.type === 'Literal' || arg.type === "StringLiteral") {
+        return arg.value;
+    } else if (arg.type === 'BinaryExpression' && arg.operator === '+') {
+        return getTKey(arg.left) + getTKey(arg.right);
+    } else if (arg.type === 'TemplateLiteral') {
+        return arg.quasis.map((q) => {
+            return q.value.raw;
+        }).join('');
+    }
+    return null;
+}
+
+function getFormatStrings(str) {
+    // Match anything that starts with %
+    // We could make a regex that matched the full placeholder, but this
+    // would just not match invalid placeholders and so wouldn't help us
+    // detect the invalid ones.
+    // Also note that for simplicity, this just matches a % character and then
+    // anything up to the next % character (or a single %, or end of string).
+    const formatStringRe = /%([^%]+|%|$)/g;
+    const formatStrings = new Set();
+
+    let match;
+    while ( (match = formatStringRe.exec(str)) !== null ) {
+        const placeholder = match[1]; // Minus the leading '%'
+        if (placeholder === '%') continue; // Literal % is %%
+
+        const placeholderMatch = placeholder.match(/^\((.*?)\)(.)/);
+        if (placeholderMatch === null) {
+            throw new Error("Invalid format specifier: '"+match[0]+"'");
+        }
+        if (placeholderMatch.length < 3) {
+            throw new Error("Malformed format specifier");
+        }
+        const placeholderName = placeholderMatch[1];
+        const placeholderFormat = placeholderMatch[2];
+
+        if (placeholderFormat !== 's') {
+            throw new Error(`'${placeholderFormat}' used as format character: you probably meant 's'`);
+        }
+
+        formatStrings.add(placeholderName);
+    }
+
+    return formatStrings;
+}
+
+function getTranslationsJs(file) {
+    const contents = fs.readFileSync(file, { encoding: 'utf8' });
+
+    const trs = new Set();
+
+    try {
+        const plugins = [
+            // https://babeljs.io/docs/en/babel-parser#plugins
+            "classProperties",
+            "objectRestSpread",
+            "throwExpressions",
+            "exportDefaultFrom",
+            "decorators-legacy",
+        ];
+
+        if (file.endsWith(".js") || file.endsWith(".jsx")) {
+            // all JS is assumed to be flow or react
+            plugins.push("flow", "jsx");
+        } else if (file.endsWith(".ts")) {
+            // TS can't use JSX unless it's a TSX file (otherwise angle casts fail)
+            plugins.push("typescript");
+        } else if (file.endsWith(".tsx")) {
+            // When the file is a TSX file though, enable JSX parsing
+            plugins.push("typescript", "jsx");
+        }
+
+        const babelParsed = parser.parse(contents, {
+            allowImportExportEverywhere: true,
+            errorRecovery: true,
+            sourceFilename: file,
+            tokens: true,
+            plugins,
+        });
+        traverse.default(babelParsed, {
+            enter: (p) => {
+                const node = p.node;
+                if (p.isCallExpression() && node.callee && TRANSLATIONS_FUNCS.includes(node.callee.name)) {
+                    const tKey = getTKey(node.arguments[0]);
+
+                    // This happens whenever we call _t with non-literals (ie. whenever we've
+                    // had to use a _td to compensate) so is expected.
+                    if (tKey === null) return;
+
+                    // check the format string against the args
+                    // We only check _t: _td has no args
+                    if (node.callee.name === '_t') {
+                        try {
+                            const placeholders = getFormatStrings(tKey);
+                            for (const placeholder of placeholders) {
+                                if (node.arguments.length < 2) {
+                                    throw new Error(`Placeholder found ('${placeholder}') but no substitutions given`);
+                                }
+                                const value = getObjectValue(node.arguments[1], placeholder);
+                                if (value === null) {
+                                    throw new Error(`No value found for placeholder '${placeholder}'`);
+                                }
+                            }
+
+                            // Validate tag replacements
+                            if (node.arguments.length > 2) {
+                                const tagMap = node.arguments[2];
+                                for (const prop of tagMap.properties || []) {
+                                    if (prop.key.type === 'Literal') {
+                                        const tag = prop.key.value;
+                                        // RegExp same as in src/languageHandler.js
+                                        const regexp = new RegExp(`(<${tag}>(.*?)<\\/${tag}>|<${tag}>|<${tag}\\s*\\/>)`);
+                                        if (!tKey.match(regexp)) {
+                                            throw new Error(`No match for ${regexp} in ${tKey}`);
+                                        }
+                                    }
+                                }
+                            }
+                        } catch (e) {
+                            console.log();
+                            console.error(`ERROR: ${file}:${node.loc.start.line} ${tKey}`);
+                            console.error(e);
+                            process.exit(1);
+                        }
+                    }
+
+                    let isPlural = false;
+                    if (node.arguments.length > 1 && node.arguments[1].type === 'ObjectExpression') {
+                        const countVal = getObjectValue(node.arguments[1], 'count');
+                        if (countVal) {
+                            isPlural = true;
+                        }
+                    }
+
+                    if (isPlural) {
+                        trs.add(tKey + "|other");
+                        const plurals = enPlurals[tKey];
+                        if (plurals) {
+                            for (const pluralType of Object.keys(plurals)) {
+                                trs.add(tKey + "|" + pluralType);
+                            }
+                        }
+                    } else {
+                        trs.add(tKey);
+                    }
+                }
+            },
+        });
+    } catch (e) {
+        console.error(e);
+        process.exit(1);
+    }
+
+    return trs;
+}
+
+function getTranslationsOther(file) {
+    const contents = fs.readFileSync(file, { encoding: 'utf8' });
+
+    const trs = new Set();
+
+    // Taken from element-web src/components/structures/HomePage.js
+    const translationsRegex = /_t\(['"]([\s\S]*?)['"]\)/mg;
+    let matches;
+    while (matches = translationsRegex.exec(contents)) {
+        trs.add(matches[1]);
+    }
+    return trs;
+}
+
+// gather en_EN plural strings from the input translations file:
+// the en_EN strings are all in the source with the exception of
+// pluralised strings, which we need to pull in from elsewhere.
+const inputTranslationsRaw = JSON.parse(fs.readFileSync(INPUT_TRANSLATIONS_FILE, { encoding: 'utf8' }));
+const enPlurals = {};
+
+for (const key of Object.keys(inputTranslationsRaw)) {
+    const parts = key.split("|");
+    if (parts.length > 1) {
+        const plurals = enPlurals[parts[0]] || {};
+        plurals[parts[1]] = inputTranslationsRaw[key];
+        enPlurals[parts[0]] = plurals;
+    }
+}
+
+const translatables = new Set();
+
+const walkOpts = {
+    listeners: {
+        names: function(root, nodeNamesArray) {
+            // Sort the names case insensitively and alphabetically to
+            // maintain some sense of order between the different strings.
+            nodeNamesArray.sort((a, b) => {
+                a = a.toLowerCase();
+                b = b.toLowerCase();
+                if (a > b) return 1;
+                if (a < b) return -1;
+                return 0;
+            });
+        },
+        file: function(root, fileStats, next) {
+            const fullPath = path.join(root, fileStats.name);
+
+            let trs;
+            if (fileStats.name.endsWith('.js') || fileStats.name.endsWith('.ts') || fileStats.name.endsWith('.tsx')) {
+                trs = getTranslationsJs(fullPath);
+            } else if (fileStats.name.endsWith('.html')) {
+                trs = getTranslationsOther(fullPath);
+            } else {
+                return;
+            }
+            console.log(`${fullPath} (${trs.size} strings)`);
+            for (const tr of trs.values()) {
+                // Convert DOS line endings to unix
+                translatables.add(tr.replace(/\r\n/g, "\n"));
+            }
+        },
+    },
+};
+
+for (const path of SEARCH_PATHS) {
+    if (fs.existsSync(path)) {
+        walk.walkSync(path, walkOpts);
+    }
+}
+
+const trObj = {};
+for (const tr of translatables) {
+    if (tr.includes("|")) {
+        if (inputTranslationsRaw[tr]) {
+            trObj[tr] = inputTranslationsRaw[tr];
+        } else {
+            trObj[tr] = tr.split("|")[0];
+        }
+    } else {
+        trObj[tr] = tr;
+    }
+}
+
+fs.writeFileSync(
+    OUTPUT_FILE,
+    JSON.stringify(trObj, translatables.values(), 4) + "\n",
+);
+
+console.log();
+console.log(`Wrote ${translatables.size} strings to ${OUTPUT_FILE}`);
+

src/electron-main.js

@@ -57,6 +57,8 @@ try {
     }
 }
 
+const { _td } = require('./language-helper');
+
 let seshatSupported = false;
 let Seshat;
 let SeshatRecovery;
@@ -268,8 +270,8 @@ const warnBeforeExit = (event, input) => {
     if (shouldWarnBeforeExit && exitShortcutPressed) {
         const shouldCancelCloseRequest = dialog.showMessageBoxSync(mainWindow, {
             type: "question",
-            buttons: ["Cancel", "Close Element"],
-            message: "Are you sure you want to quit?",
+            buttons: [_td("Cancel"), _td("Close Element")],
+            message: _td("Are you sure you want to quit?"),
             defaultId: 1,
             cancelId: 0,
         }) === 0;

src/i18n/strings/en_EN.json

@@ -0,0 +1,30 @@
+{
+    "Cancel": "Cancel",
+    "Close Element": "Close Element",
+    "Are you sure you want to quit?": "Are you sure you want to quit?",
+    "Show/Hide": "Show/Hide",
+    "Quit": "Quit",
+    "Edit": "Edit",
+    "View": "View",
+    "Preferences": "Preferences",
+    "Window": "Window",
+    "Help": "Help",
+    "Element Help": "Element Help",
+    "Speech": "Speech",
+    "Close": "Close",
+    "Minimize": "Minimize",
+    "Zoom": "Zoom",
+    "Bring All to Front": "Bring All to Front",
+    "File": "File",
+    "Copy image": "Copy image",
+    "Copy email address": "Copy email address",
+    "Copy link address": "Copy link address",
+    "Save image as...": "Save image as...",
+    "Failed to save image": "Failed to save image",
+    "The image failed to save": "The image failed to save",
+    "Add to dictionary": "Add to dictionary",
+    "Cut": "Cut",
+    "Copy": "Copy",
+    "Paste": "Paste",
+    "Select All": "Select All"
+}

src/language-helper.js

@@ -0,0 +1,11 @@
+// Function which only purpose is to mark that a string is translatable
+// Does not actually do anything. It's helpful for automatic extraction of translatable strings
+
+function _td(s) {
+    return s;
+}
+
+
+module.exports = {
+    _td,
+};

src/tray.js

@@ -19,6 +19,7 @@ const {app, Tray, Menu, nativeImage} = require('electron');
 const pngToIco = require('png-to-ico');
 const path = require('path');
 const fs = require('fs');
+const { _td } = require('./language-helper');
 
 let trayIcon = null;
 
@@ -49,12 +50,12 @@ exports.create = function(config) {
 
     const contextMenu = Menu.buildFromTemplate([
         {
-            label: `Show/Hide ${config.brand}`,
+            label: _td('Show/Hide'),
             click: toggleWin,
         },
         { type: 'separator' },
         {
-            label: 'Quit',
+            label: _td('Quit'),
             click: function() {
                 app.quit();
             },

src/vectormenu.js

@@ -15,11 +15,13 @@ limitations under the License.
 */
 
 const {app, shell, Menu} = require('electron');
+const { _td } = require('./language-helper');
 
 // Menu template from http://electron.atom.io/docs/api/menu/, edited
 const template = [
     {
-        label: '&Edit',
+        label: _td('Edit'),
+        accelerator: 'e',
         submenu: [
             { role: 'undo' },
             { role: 'redo' },
@@ -33,7 +35,8 @@ const template = [
         ],
     },
     {
-        label: '&View',
+        label: _td('View'),
+        accelerator: 'V',
         submenu: [
             { type: 'separator' },
             { role: 'resetzoom' },
@@ -41,7 +44,7 @@ const template = [
             { role: 'zoomout' },
             { type: 'separator' },
             {
-                label: 'Preferences',
+                label: _td('Preferences'),
                 accelerator: 'Command+,', // Mac-only accelerator
                 click() { global.mainWindow.webContents.send('preferences'); },
             },
@@ -50,7 +53,8 @@ const template = [
         ],
     },
     {
-        label: '&Window',
+        label: _td('Window'),
+        accelerator: 'w',
         role: 'window',
         submenu: [
             { role: 'minimize' },
@@ -58,11 +62,12 @@ const template = [
         ],
     },
     {
-        label: '&Help',
+        label: _td('Help'),
+        accelerator: 'h',
         role: 'help',
         submenu: [
             {
-                label: 'Element Help',
+                label: _td('Element Help'),
                 click() { shell.openExternal('https://element.io/help'); },
             },
         ],
@@ -94,7 +99,7 @@ if (process.platform === 'darwin') {
     template[1].submenu.push(
         { type: 'separator' },
         {
-            label: 'Speech',
+            label: _td('Speech'),
             submenu: [
                 { role: 'startspeaking' },
                 { role: 'stopspeaking' },
@@ -105,30 +110,31 @@ if (process.platform === 'darwin') {
     // This also has specific functionality on macOS
     template[3].submenu = [
         {
-            label: 'Close',
+            label: _td('Close'),
             accelerator: 'CmdOrCtrl+W',
             role: 'close',
         },
         {
-            label: 'Minimize',
+            label: _td('Minimize'),
             accelerator: 'CmdOrCtrl+M',
             role: 'minimize',
         },
         {
-            label: 'Zoom',
+            label: _td('Zoom'),
             role: 'zoom',
         },
         {
             type: 'separator',
         },
         {
-            label: 'Bring All to Front',
+            label: _td('Bring All to Front'),
             role: 'front',
         },
     ];
 } else {
     template.unshift({
-        label: '&File',
+        label: _td('File'),
+        accelerator: 'f',
         submenu: [
             // For some reason, 'about' does not seem to work on windows.
             /*{

src/webcontents-handler.js

@@ -3,6 +3,7 @@ const url = require('url');
 const fs = require('fs');
 const request = require('request');
 const path = require('path');
+const { _td } = require('./language-helper');
 
 const MAILTO_PREFIX = "mailto:";
 
@@ -73,7 +74,8 @@ function onLinkContextMenu(ev, params) {
 
     if (params.hasImageContents) {
         popupMenu.append(new MenuItem({
-            label: '&Copy image',
+            label: _td('Copy image'),
+            accelerator: 'c',
             click() {
                 ev.sender.copyImageAt(params.x, params.y);
             },
@@ -85,14 +87,16 @@ function onLinkContextMenu(ev, params) {
         // Special-case e-mail URLs to strip the `mailto:` like modern browsers do
         if (url.startsWith(MAILTO_PREFIX)) {
             popupMenu.append(new MenuItem({
-                label: 'Copy email &address',
+                label: _td('Copy email address'),
+                accelerator: 'a',
                 click() {
                     clipboard.writeText(url.substr(MAILTO_PREFIX.length));
                 },
             }));
         } else {
             popupMenu.append(new MenuItem({
-                label: 'Copy link &address',
+                label: _td('Copy link address'),
+                accelerator: 'a',
                 click() {
                     clipboard.writeText(url);
                 },
@@ -104,7 +108,8 @@ function onLinkContextMenu(ev, params) {
     // only the renderer can resolve them so don't give the user an option to.
     if (params.hasImageContents && !url.startsWith('blob:')) {
         popupMenu.append(new MenuItem({
-            label: 'Sa&ve image as...',
+            label: _td('Save image as...'),
+            accelerator: 'a',
             async click() {
                 const targetFileName = params.titleText || "image.png";
                 const {filePath} = await dialog.showSaveDialog({
@@ -123,8 +128,8 @@ function onLinkContextMenu(ev, params) {
                     console.error(err);
                     dialog.showMessageBox({
                         type: "error",
-                        title: "Failed to save image",
-                        message: "The image failed to save",
+                        title: _td("Failed to save image"),
+                        message: _td("The image failed to save"),
                     });
                 }
             },
@@ -151,7 +156,7 @@ function _CutCopyPasteSelectContextMenus(params) {
         options.push({
             type: 'separator',
         }, {
-            label: 'Add to dictionary',
+            label: _td('Add to dictionary'),
             click: (menuItem, browserWindow) => {
                 browserWindow.webContents.session.addWordToSpellCheckerDictionary(params.misspelledWord);
             },
@@ -162,22 +167,26 @@ function _CutCopyPasteSelectContextMenus(params) {
 
     options.push({
         role: 'cut',
-        label: 'Cu&t',
+        label: _td('Cut'),
+        accelerator: 't',
         enabled: params.editFlags.canCut,
     }, {
         role: 'copy',
-        label: '&Copy',
+        label: _td('Copy'),
+        accelerator: 'c',
         enabled: params.editFlags.canCopy,
     }, {
         role: 'paste',
-        label: '&Paste',
+        label: _td('Paste'),
+        accelerator: 'p',
         enabled: params.editFlags.canPaste,
     }, {
         role: 'pasteandmatchstyle',
         enabled: params.editFlags.canPaste,
     }, {
         role: 'selectall',
-        label: "Select &All",
+        label: _td("Select All"),
+        accelerator: 'a',
         enabled: params.editFlags.canSelectAll,
     });
     return options;