Merge pull request #392 from mewtaylor/cleanup/localization-stuff

Cleanup: catch more localization errors that might arise

Makefile

@@ -59,6 +59,8 @@ test:
 	@echo ""
 	@make functional
 	@echo ""
+	@make localization
+	@echo ""
 
 lint:
 	$(ESLINT) ./*.js
@@ -80,6 +82,9 @@ functional:
 integration:
 	$(TAP) ./test/integration/*.js
 
+localization:
+	$(TAP) ./test/localization/*.js
+
 # ------------------------------------
 
 .PHONY: build clean deploy static tag translations webpack watch stop start test lint

bin/build-locales

@@ -39,8 +39,8 @@ var fs = require('fs');
 var glob = require('glob');
 var merge = require('lodash.merge');
 var path = require('path');
-var po2icu = require('po2icu');
 
+var languages = require('../languages.json');
 var localeCompare = require('./lib/locale-compare');
 
 // -----------------------------------------------------------------------------
@@ -54,12 +54,7 @@ if (!args.length) {
     process.stdout.write('A destination directory must be specified.');
     process.exit(1);
 }
-var verbose = false;
-if (args.length > 1) {
-    verbose = (args[1] === '-v') ? true : false;
-}
 
-var poUiDir = path.resolve(__dirname, '../node_modules/scratchr2_translations/ui');
 var outputDir = path.resolve(__dirname, '../', args[0]);
 try {
     fs.accessSync(outputDir, fs.F_OK);
@@ -68,27 +63,19 @@ try {
     fs.mkdirSync(outputDir);
 }
 
-// get global locale strings first.
-var globalTemplateFile = path.resolve(__dirname, '../src/l10n.json');
 // message key with english string values (i.e. default values)
-var generalIds = JSON.parse(fs.readFileSync(globalTemplateFile, 'utf8'));
 var viewLocales = {};
-var generalLocales = {
-    en: generalIds
-};
-
 // FormattedMessage id with english string as value. Use for default values in translations
 // Sample structure: { 'general-general.blah': 'blah', 'about-about.blah': 'blahblah' }
 var idsWithICU = {};
-
 // reverse (i.e. english string with message key as the value) object for searching po files.
 // Sample structure: { 'blah': 'general-general.blah', 'blahblah': 'about-about.blah' }
 var icuWithIds = {};
 
-for (var id in generalIds) {
-    idsWithICU['general-' + id] = generalIds[id];
-    icuWithIds[generalIds[id]] = 'general-' + id;
-}
+// get global locale strings first.
+var globalTemplateFile = path.resolve(__dirname, '../src/l10n.json');
+localeCompare.getIdsForView('general', globalTemplateFile, viewLocales, idsWithICU, icuWithIds);
+
 
 // start with all views, and remove localized ones as they are iterated over
 var views = glob.sync(path.resolve(__dirname, '../src/views/*'));
@@ -102,15 +89,7 @@ files.forEach(function (file) {
     var dirPath = file.split('/');
     dirPath.pop();
     var view = dirPath.pop();
-
-    var viewIds = JSON.parse(fs.readFileSync(file, 'utf8'));
-    viewLocales[view] = {
-        en: viewIds
-    };
-    for (var id in viewIds) {
-        idsWithICU[view + '-' + id] = viewIds[id];
-        icuWithIds[viewIds[id]] = view + '-' + id; // add viewName to identifier for later
-    }
+    localeCompare.getIdsForView(view, file, viewLocales, idsWithICU, icuWithIds);
 });
 
 // md5 of english strings with message key as the value for searching po files.
@@ -118,53 +97,18 @@ files.forEach(function (file) {
 var md5WithIds = localeCompare.getMD5Map(icuWithIds);
 
 // Get ui localization strings first
-glob(poUiDir + '/*', function (err, files) {
-    if (err) throw new Error(err);
-    
-    files.forEach(function (file) {
-        var lang = file.split('/').pop();
-        var jsFile = path.resolve(file, 'LC_MESSAGES/djangojs.po');
-        var pyFile = path.resolve(file, 'LC_MESSAGES/django.po');
-
-        var translations = {};
-
-        try {
-            var jsTranslations = po2icu.poFileToICUSync(lang, jsFile);
-            translations = localeCompare.mergeNewTranslations(translations, jsTranslations, idsWithICU, md5WithIds);
-        } catch (err) {
-            if (verbose) process.stdout.write(lang + ': ' + err + '\n');
-        }
-
-        try {
-            var pyTranslations = po2icu.poFileToICUSync(lang, pyFile);
-            translations = localeCompare.mergeNewTranslations(translations, pyTranslations, idsWithICU, md5WithIds);
-        } catch (err) {
-            if (verbose) process.stdout.write(lang + ': ' + err + '\n');
-        }
+var isoCodes = Object.keys(languages);
+for (i in isoCodes) {
+    var translations = localeCompare.getTranslationsForLanguage(isoCodes[i], idsWithICU, md5WithIds);
+    for (var key in translations) {
+        viewLocales[key] = merge(viewLocales[key], translations[key]);
+    }
+}
 
-        // add new translations to locale object
-        for (var id in translations) {
-            var ids = id.split('-'); // [viewName, stringId]
-            var viewName = ids[0];
-            var stringId = ids[1];
-            if (viewLocales.hasOwnProperty(viewName)) {
-                if (!viewLocales[viewName].hasOwnProperty(lang)) viewLocales[viewName][lang] = {};
-                viewLocales[viewName][lang][stringId] = translations[id];
-            } else {
-                // default to general
-                if (!generalLocales.hasOwnProperty(lang)) generalLocales[lang] = {};
-                generalLocales[lang][stringId] = translations[id];
-            }
-        }
-    });
-    
-    for (var i in views) {
-        var viewTranslations = generalLocales;
-        if (views[i] in viewLocales) {
-            viewTranslations = merge(viewLocales[views[i]], viewTranslations);
-        }
-        var objectString = JSON.stringify(viewTranslations);
-        var fileString = 'window._messages = ' + objectString + ';';
-        fs.writeFileSync(outputDir + '/' + views[i] + '.intl.js', fileString);
+for (i in views) {
+    var viewTranslations = viewLocales['general'];
+    if (views[i] in viewLocales) {
+        viewTranslations = merge(viewLocales[views[i]], viewTranslations);
     }
-});
+    localeCompare.writeTranslationsToJS(outputDir, views[i], viewTranslations);
+}

bin/lib/locale-compare.js

@@ -3,6 +3,9 @@
 // -----------------------------------------------------------------------------
 
 var crypto = require('crypto');
+var fs = require('fs');
+var path = require('path');
+var po2icu = require('po2icu');
 
 var Helpers = {};
 
@@ -62,4 +65,75 @@ Helpers.getMD5Map = function (ICUIdMap) {
     return md5Map;
 };
 
+/**
+ * Grabs the translated strings from the po files for the given language and strings
+ * @param  {str}    lang       iso code of the language to use
+ * @param  {object} idsWithICU key: '<viewName>-<react-intl string id>'.
+ *                             value: english strings for translation
+ * @param  {object} md5WithIds key: md5 hash of the english strings for translation.
+ *                             value: '<viewName>-<react-intl string id>'
+ * @return {object}            translations – sub-objects by view containing:
+ *                                          key: '<react-intl string id>'
+ *                                          value: translated version of string
+ */
+Helpers.getTranslationsForLanguage = function (lang, idsWithICU, md5WithIds) {
+    var poUiDir = path.resolve(__dirname, '../../node_modules/scratchr2_translations/ui');
+    var jsFile = path.resolve(poUiDir, lang + '/LC_MESSAGES/djangojs.po');
+    var pyFile = path.resolve(poUiDir, lang + '/LC_MESSAGES/django.po');
+
+    var translations = {};
+    try {
+        fs.accessSync(jsFile, fs.R_OK);
+        var jsTranslations = po2icu.poFileToICUSync(lang, jsFile);
+        translations = Helpers.mergeNewTranslations(translations, jsTranslations, idsWithICU, md5WithIds);
+    } catch (err) {
+        if (err.code !== 'ENOENT') {
+            throw err;
+        }
+    }
+
+    try {
+        fs.accessSync(pyFile, fs.R_OK);
+        var pyTranslations = po2icu.poFileToICUSync(lang, pyFile);
+        translations = Helpers.mergeNewTranslations(translations, pyTranslations, idsWithICU, md5WithIds);
+    } catch (err) {
+        if (err.code !== 'ENOENT') {
+            throw err;
+        }
+    }
+    
+    var translationsByView = {};
+    for (var id in translations) {
+        var ids = id.split('-'); // [viewName, stringId]
+        var viewName = ids[0];
+        var stringId = ids[1];
+
+        if (!translationsByView.hasOwnProperty(viewName)) {
+            translationsByView[viewName] = {};
+        }
+        if (!translationsByView[viewName].hasOwnProperty(lang)) {
+            translationsByView[viewName][lang] = {};
+        }
+        translationsByView[viewName][lang][stringId] = translations[id];
+    }
+    return translationsByView;
+};
+
+Helpers.writeTranslationsToJS = function (outputDir, viewName, translationObject) {
+    var objectString = JSON.stringify(translationObject);
+    var fileString = 'window._messages = ' + objectString + ';';
+    fs.writeFileSync(outputDir + '/' + viewName + '.intl.js', fileString);
+};
+
+Helpers.getIdsForView = function (viewName, viewFile, localeObject, idsWithICU, icuWithIds) {
+    var ids = JSON.parse(fs.readFileSync(viewFile, 'utf8'));
+    localeObject[viewName] = {
+        en: ids
+    };
+    for (var id in ids) {
+        idsWithICU[viewName + '-' + id] = ids[id];
+        icuWithIds[ids[id]] = viewName + '-' + id; // add viewName to identifier for later
+    }
+};
+
 module.exports = Helpers;

test/localization/spot_check_about_has_strings.js

+/*
+ * spot check that each language has values for the string id keys on About page
+ * that are contained in English (i.e. make sure strings will show up, not ids")
+ */
+var merge = require('lodash.merge');
+var path = require('path');
+var tap = require('tap');
+
+var languages = require('../../languages.json');
+var localeCompare = require('../../bin/lib/locale-compare');
+
+tap.test('spotCheckAboutStrings', function (t) {
+    var isoCodes = Object.keys(languages);
+    isoCodes.splice(isoCodes.indexOf('en'), 1);
+    var viewLocales = {};
+    var idsWithICU = {};
+    var icuWithIds = {};
+    localeCompare.getIdsForView(
+        'about',
+        path.resolve(__dirname, '../../src/views/about/l10n.json'),
+        viewLocales,
+        idsWithICU,
+        icuWithIds
+    );
+    var md5WithIds = localeCompare.getMD5Map(icuWithIds);
+    var keysToCheck = Object.keys(merge(viewLocales['about']['en'])).sort();
+    for (var i in isoCodes) {
+        var translations = localeCompare.getTranslationsForLanguage(isoCodes[i], idsWithICU, md5WithIds);
+        t.same(
+            Object.keys(translations['about'][isoCodes[i]]).sort(),
+            keysToCheck,
+            'check About keys for language ' + isoCodes[i]
+        );
+    }
+    t.end();
+});

test/localization/spot_check_general_has_strings.js

+/*
+ * spot check that each language has values for the string id keys used generally in the site
+ * that are contained in English (i.e. make sure strings will show up, not ids")
+ */
+var merge = require('lodash.merge');
+var path = require('path');
+var tap = require('tap');
+
+var languages = require('../../languages.json');
+var localeCompare = require('../../bin/lib/locale-compare');
+
+tap.test('spotCheckAboutStrings', function (t) {
+    var isoCodes = Object.keys(languages);
+    isoCodes.splice(isoCodes.indexOf('en'), 1);
+    var viewLocales = {};
+    var idsWithICU = {};
+    var icuWithIds = {};
+    localeCompare.getIdsForView(
+        'general',
+        path.resolve(__dirname, '../../src/l10n.json'),
+        viewLocales,
+        idsWithICU,
+        icuWithIds
+    );
+    var md5WithIds = localeCompare.getMD5Map(icuWithIds);
+    var keysToCheck = Object.keys(merge(viewLocales['general']['en'])).sort();
+    for (var i in isoCodes) {
+        var translations = localeCompare.getTranslationsForLanguage(isoCodes[i], idsWithICU, md5WithIds);
+        t.same(
+            Object.keys(translations['general'][isoCodes[i]]).sort(),
+            keysToCheck,
+            'check About keys for language ' + isoCodes[i]
+        );
+    }
+    t.end();
+});

test/localization/spot_check_nav.js

+/*
+ * spot checks the translation of the nav bar for a select set
+ * of languages that cover a number of types of translations.
+ *
+ * Languages checked:
+ *     - Hebrew
+ *     - Edible Scratch (fake language)
+ *     - Mandarin
+ *     - Japanese
+ *     - Brasilian Portuguese
+ *     - Polish
+ *     - Norwegian
+ *     - German
+ */
+var path = require('path');
+var tap = require('tap');
+
+var localeCompare = require('../../bin/lib/locale-compare');
+var viewLocales = {};
+var idsWithICU = {};
+var icuWithIds = {};
+
+var languagesToCheck = [
+    'he', 'zh-cn', 'ja', 'pt-br', 'pl', 'nb'
+];
+var idsToCheck = [
+    'general.about', 'general.create', 'general.help', 'general.joinScratch',
+    'general.signIn', 'general.discuss'
+];
+
+
+// Test nav for real languages.
+localeCompare.getIdsForView(
+    'general',
+    path.resolve(__dirname, '../../src/l10n.json'),
+    viewLocales,
+    idsWithICU,
+    icuWithIds
+);
+var md5WithIds = localeCompare.getMD5Map(icuWithIds);
+
+tap.test('spotCheckNavBar', function (t) {
+    for (var i in languagesToCheck) {
+        var translations = localeCompare.getTranslationsForLanguage(languagesToCheck[i], idsWithICU, md5WithIds);
+        for (var j in idsToCheck) {
+            t.notEqual(
+                translations['general'][languagesToCheck[i]][idsToCheck[j]],
+                viewLocales['general']['en'][idsToCheck[j]],
+                'check localization of ' + idsToCheck[j] + ' for ' + languagesToCheck[i]
+            );
+        }
+    }
+    t.end();
+});
+
+
+// Test splash items for fake language.
+var fakeLanguageIdsToCheck = ['news.scratchNews', 'splash.featuredProjects', 'splash.featuredStudios'];
+
+localeCompare.getIdsForView(
+    'splash',
+    path.resolve(__dirname, '../../src/views/splash/l10n.json'),
+    viewLocales,
+    idsWithICU,
+    icuWithIds
+);
+md5WithIds = localeCompare.getMD5Map(icuWithIds);
+
+tap.test('spotCheckNavBarFakeLanguage', function (t) {
+    var translations = localeCompare.getTranslationsForLanguage('yum', idsWithICU, md5WithIds);
+    for (var i in fakeLanguageIdsToCheck) {
+        t.notEqual(
+            translations['general']['yum'][fakeLanguageIdsToCheck[i]],
+            viewLocales['splash']['en'][fakeLanguageIdsToCheck[i]],
+            'check localization of ' + fakeLanguageIdsToCheck[i] + ' for yum'
+        );
+    }
+    t.end();
+});

test/localization/spot_check_splash_has_strings.js

+/*
+ * spot check that each language has values for the string id keys on Splash page
+ * that are contained in English (i.e. make sure strings will show up, not ids")
+ */
+var merge = require('lodash.merge');
+var path = require('path');
+var tap = require('tap');
+
+var languages = require('../../languages.json');
+var localeCompare = require('../../bin/lib/locale-compare');
+
+tap.test('spotCheckAboutStrings', function (t) {
+    var isoCodes = Object.keys(languages);
+    isoCodes.splice(isoCodes.indexOf('en'), 1);
+    var viewLocales = {};
+    var idsWithICU = {};
+    var icuWithIds = {};
+    localeCompare.getIdsForView(
+        'splash',
+        path.resolve(__dirname, '../../src/views/splash/l10n.json'),
+        viewLocales,
+        idsWithICU,
+        icuWithIds
+    );
+    var md5WithIds = localeCompare.getMD5Map(icuWithIds);
+    var keysToCheck = Object.keys(merge(viewLocales['splash']['en'])).sort();
+    for (var i in isoCodes) {
+        var translations = localeCompare.getTranslationsForLanguage(isoCodes[i], idsWithICU, md5WithIds);
+        t.same(
+            Object.keys(translations['splash'][isoCodes[i]]).sort(),
+            keysToCheck,
+            'check Splash keys for language ' + isoCodes[i]
+        );
+    }
+    t.end();
+});

test/localization/spot_check_wedo2_has_strings.js

+/*
+ * spot check that each language has values for the string id keys on Wedo2 page
+ * that are contained in English (i.e. make sure strings will show up, not ids")
+ */
+var merge = require('lodash.merge');
+var path = require('path');
+var tap = require('tap');
+
+var languages = require('../../languages.json');
+var localeCompare = require('../../bin/lib/locale-compare');
+
+tap.test('spotCheckAboutStrings', function (t) {
+    var isoCodes = Object.keys(languages);
+    isoCodes.splice(isoCodes.indexOf('en'), 1);
+    var viewLocales = {};
+    var idsWithICU = {};
+    var icuWithIds = {};
+    localeCompare.getIdsForView(
+        'wedo2',
+        path.resolve(__dirname, '../../src/views/wedo2/l10n.json'),
+        viewLocales,
+        idsWithICU,
+        icuWithIds
+    );
+    var md5WithIds = localeCompare.getMD5Map(icuWithIds);
+    var keysToCheck = Object.keys(merge(viewLocales['wedo2']['en'])).sort();
+    for (var i in isoCodes) {
+        var translations = localeCompare.getTranslationsForLanguage(isoCodes[i], idsWithICU, md5WithIds);
+        t.same(
+            Object.keys(translations['wedo2'][isoCodes[i]]).sort(),
+            keysToCheck,
+            'check About keys for language ' + isoCodes[i]
+        );
+    }
+    t.end();
+});