Refactor build-locales a bit

This makes a couple of procedure-specific parts into methods in `locale-compare`, so that they can be tested more readily.

Makefile

@@ -73,11 +73,6 @@ lint:
 	$(SASSLINT) ./src/views/**/*.scss
 	$(SASSLINT) ./src/components/**/*.scss
 
-localization-standalone:
-	@make translations
-	@make localization
-	@echo ""
-
 unit:
 	$(TAP) ./test/unit/*.js
 

bin/build-locales

@@ -39,8 +39,8 @@ var fs = require('fs');
 var glob = require('glob');
 var merge = require('lodash.merge');
 var path = require('path');
-var po2icu = require('po2icu');
 
+var languages = require('../languages.json');
 var localeCompare = require('./lib/locale-compare');
 
 // -----------------------------------------------------------------------------
@@ -55,7 +55,6 @@ if (!args.length) {
     process.exit(1);
 }
 
-var poUiDir = path.resolve(__dirname, '../node_modules/scratchr2_translations/ui');
 var outputDir = path.resolve(__dirname, '../', args[0]);
 try {
     fs.accessSync(outputDir, fs.F_OK);
@@ -64,27 +63,19 @@ try {
     fs.mkdirSync(outputDir);
 }
 
-// get global locale strings first.
-var globalTemplateFile = path.resolve(__dirname, '../src/l10n.json');
 // message key with english string values (i.e. default values)
-var generalIds = JSON.parse(fs.readFileSync(globalTemplateFile, 'utf8'));
 var viewLocales = {};
-var generalLocales = {
-    en: generalIds
-};
-
 // FormattedMessage id with english string as value. Use for default values in translations
 // Sample structure: { 'general-general.blah': 'blah', 'about-about.blah': 'blahblah' }
 var idsWithICU = {};
-
 // reverse (i.e. english string with message key as the value) object for searching po files.
 // Sample structure: { 'blah': 'general-general.blah', 'blahblah': 'about-about.blah' }
 var icuWithIds = {};
 
-for (var id in generalIds) {
-    idsWithICU['general-' + id] = generalIds[id];
-    icuWithIds[generalIds[id]] = 'general-' + id;
-}
+// get global locale strings first.
+var globalTemplateFile = path.resolve(__dirname, '../src/l10n.json');
+localeCompare.getIdsForView('general', globalTemplateFile, viewLocales, idsWithICU, icuWithIds);
+
 
 // start with all views, and remove localized ones as they are iterated over
 var views = glob.sync(path.resolve(__dirname, '../src/views/*'));
@@ -98,15 +89,7 @@ files.forEach(function (file) {
     var dirPath = file.split('/');
     dirPath.pop();
     var view = dirPath.pop();
-
-    var viewIds = JSON.parse(fs.readFileSync(file, 'utf8'));
-    viewLocales[view] = {
-        en: viewIds
-    };
-    for (var id in viewIds) {
-        idsWithICU[view + '-' + id] = viewIds[id];
-        icuWithIds[viewIds[id]] = view + '-' + id; // add viewName to identifier for later
-    }
+    localeCompare.getIdsForView(view, file, viewLocales, idsWithICU, icuWithIds);
 });
 
 // md5 of english strings with message key as the value for searching po files.
@@ -114,59 +97,18 @@ files.forEach(function (file) {
 var md5WithIds = localeCompare.getMD5Map(icuWithIds);
 
 // Get ui localization strings first
-glob(poUiDir + '/*', function (err, files) {
-    if (err) throw new Error(err);
-    
-    files.forEach(function (file) {
-        var lang = file.split('/').pop();
-        var jsFile = path.resolve(file, 'LC_MESSAGES/djangojs.po');
-        var pyFile = path.resolve(file, 'LC_MESSAGES/django.po');
-
-        var translations = {};
-
-        try {
-            fs.accessSync(jsFile, fs.R_OK);
-            var jsTranslations = po2icu.poFileToICUSync(lang, jsFile);
-            translations = localeCompare.mergeNewTranslations(translations, jsTranslations, idsWithICU, md5WithIds);
-        } catch (err) {
-            if (err.code !== 'ENOENT') {
-                throw err;
-            }
-        }
-
-        try {
-            fs.accessSync(pyFile, fs.R_OK);
-            var pyTranslations = po2icu.poFileToICUSync(lang, pyFile);
-            translations = localeCompare.mergeNewTranslations(translations, pyTranslations, idsWithICU, md5WithIds);
-        } catch (err) {
-            if (err.code !== 'ENOENT') {
-                throw err;
-            }
-        }
+var isoCodes = Object.keys(languages);
+for (i in isoCodes) {
+    var translations = localeCompare.getTranslationsForLanguage(isoCodes[i], idsWithICU, md5WithIds);
+    for (var key in translations) {
+        viewLocales[key] = merge(viewLocales[key], translations[key]);
+    }
+}
 
-        // add new translations to locale object
-        for (var id in translations) {
-            var ids = id.split('-'); // [viewName, stringId]
-            var viewName = ids[0];
-            var stringId = ids[1];
-            if (viewLocales.hasOwnProperty(viewName)) {
-                if (!viewLocales[viewName].hasOwnProperty(lang)) viewLocales[viewName][lang] = {};
-                viewLocales[viewName][lang][stringId] = translations[id];
-            } else {
-                // default to general
-                if (!generalLocales.hasOwnProperty(lang)) generalLocales[lang] = {};
-                generalLocales[lang][stringId] = translations[id];
-            }
-        }
-    });
-    
-    for (var i in views) {
-        var viewTranslations = generalLocales;
-        if (views[i] in viewLocales) {
-            viewTranslations = merge(viewLocales[views[i]], viewTranslations);
-        }
-        var objectString = JSON.stringify(viewTranslations);
-        var fileString = 'window._messages = ' + objectString + ';';
-        fs.writeFileSync(outputDir + '/' + views[i] + '.intl.js', fileString);
+for (i in views) {
+    var viewTranslations = viewLocales['general'];
+    if (views[i] in viewLocales) {
+        viewTranslations = merge(viewLocales[views[i]], viewTranslations);
     }
-});
+    localeCompare.writeTranslationsToJS(outputDir, views[i], viewTranslations);
+}

bin/lib/locale-compare.js

@@ -3,6 +3,9 @@
 // -----------------------------------------------------------------------------
 
 var crypto = require('crypto');
+var fs = require('fs');
+var path = require('path');
+var po2icu = require('po2icu');
 
 var Helpers = {};
 
@@ -62,4 +65,75 @@ Helpers.getMD5Map = function (ICUIdMap) {
     return md5Map;
 };
 
+/**
+ * Grabs the translated strings from the po files for the given language and strings
+ * @param  {str}    lang       iso code of the language to use
+ * @param  {object} idsWithICU key: '<viewName>-<react-intl string id>'.
+ *                             value: english strings for translation
+ * @param  {object} md5WithIds key: md5 hash of the english strings for translation.
+ *                             value: '<viewName>-<react-intl string id>'
+ * @return {object}            translations – sub-objects by view containing:
+ *                                          key: '<react-intl string id>'
+ *                                          value: translated version of string
+ */
+Helpers.getTranslationsForLanguage = function (lang, idsWithICU, md5WithIds) {
+    var poUiDir = path.resolve(__dirname, '../../node_modules/scratchr2_translations/ui');
+    var jsFile = path.resolve(poUiDir, lang + '/LC_MESSAGES/djangojs.po');
+    var pyFile = path.resolve(poUiDir, lang + '/LC_MESSAGES/django.po');
+
+    var translations = {};
+    try {
+        fs.accessSync(jsFile, fs.R_OK);
+        var jsTranslations = po2icu.poFileToICUSync(lang, jsFile);
+        translations = Helpers.mergeNewTranslations(translations, jsTranslations, idsWithICU, md5WithIds);
+    } catch (err) {
+        if (err.code !== 'ENOENT') {
+            throw err;
+        }
+    }
+
+    try {
+        fs.accessSync(pyFile, fs.R_OK);
+        var pyTranslations = po2icu.poFileToICUSync(lang, pyFile);
+        translations = Helpers.mergeNewTranslations(translations, pyTranslations, idsWithICU, md5WithIds);
+    } catch (err) {
+        if (err.code !== 'ENOENT') {
+            throw err;
+        }
+    }
+    
+    var translationsByView = {};
+    for (var id in translations) {
+        var ids = id.split('-'); // [viewName, stringId]
+        var viewName = ids[0];
+        var stringId = ids[1];
+
+        if (!translationsByView.hasOwnProperty(viewName)) {
+            translationsByView[viewName] = {};
+        }
+        if (!translationsByView[viewName].hasOwnProperty(lang)) {
+            translationsByView[viewName][lang] = {};
+        }
+        translationsByView[viewName][lang][stringId] = translations[id];
+    }
+    return translationsByView;
+};
+
+Helpers.writeTranslationsToJS = function (outputDir, viewName, translationObject) {
+    var objectString = JSON.stringify(translationObject);
+    var fileString = 'window._messages = ' + objectString + ';';
+    fs.writeFileSync(outputDir + '/' + viewName + '.intl.js', fileString);
+};
+
+Helpers.getIdsForView = function (viewName, viewFile, localeObject, idsWithICU, icuWithIds) {
+    var ids = JSON.parse(fs.readFileSync(viewFile, 'utf8'));
+    localeObject[viewName] = {
+        en: ids
+    };
+    for (var id in ids) {
+        idsWithICU[viewName + '-' + id] = ids[id];
+        icuWithIds[ids[id]] = viewName + '-' + id; // add viewName to identifier for later
+    }
+};
+
 module.exports = Helpers;

test/localization/spot_check_about_has_strings.js

@@ -2,17 +2,32 @@
  * spot check that each language has values for the string id keys on About page
  * that are contained in English (i.e. make sure strings will show up, not ids")
  */
+var merge = require('lodash.merge');
+var path = require('path');
 var tap = require('tap');
+
 var languages = require('../../languages.json');
-window = {};
-require('../../intl/about.intl.js');
+var localeCompare = require('../../bin/lib/locale-compare');
 
 tap.test('spotCheckAboutStrings', function (t) {
     var isoCodes = Object.keys(languages);
-    var keysToCheck = Object.keys(window._messages['en']).sort();
+    isoCodes.splice(isoCodes.indexOf('en'), 1);
+    var viewLocales = {};
+    var idsWithICU = {};
+    var icuWithIds = {};
+    localeCompare.getIdsForView(
+        'about',
+        path.resolve(__dirname, '../../src/views/about/l10n.json'),
+        viewLocales,
+        idsWithICU,
+        icuWithIds
+    );
+    var md5WithIds = localeCompare.getMD5Map(icuWithIds);
+    var keysToCheck = Object.keys(merge(viewLocales['about']['en'])).sort();
     for (var i in isoCodes) {
+        var translations = localeCompare.getTranslationsForLanguage(isoCodes[i], idsWithICU, md5WithIds);
         t.same(
-            Object.keys(window._messages[isoCodes[i]]).sort(),
+            Object.keys(translations['about'][isoCodes[i]]).sort(),
             keysToCheck,
             'check About keys for language ' + isoCodes[i]
         );

test/localization/spot_check_general_has_strings.js

+/*
+ * spot check that each language has values for the string id keys used generally in the site
+ * that are contained in English (i.e. make sure strings will show up, not ids")
+ */
+var merge = require('lodash.merge');
+var path = require('path');
+var tap = require('tap');
+
+var languages = require('../../languages.json');
+var localeCompare = require('../../bin/lib/locale-compare');
+
+tap.test('spotCheckAboutStrings', function (t) {
+    var isoCodes = Object.keys(languages);
+    isoCodes.splice(isoCodes.indexOf('en'), 1);
+    var viewLocales = {};
+    var idsWithICU = {};
+    var icuWithIds = {};
+    localeCompare.getIdsForView(
+        'general',
+        path.resolve(__dirname, '../../src/l10n.json'),
+        viewLocales,
+        idsWithICU,
+        icuWithIds
+    );
+    var md5WithIds = localeCompare.getMD5Map(icuWithIds);
+    var keysToCheck = Object.keys(merge(viewLocales['general']['en'])).sort();
+    for (var i in isoCodes) {
+        var translations = localeCompare.getTranslationsForLanguage(isoCodes[i], idsWithICU, md5WithIds);
+        t.same(
+            Object.keys(translations['general'][isoCodes[i]]).sort(),
+            keysToCheck,
+            'check About keys for language ' + isoCodes[i]
+        );
+    }
+    t.end();
+});

test/localization/spot_check_nav.js

@@ -12,9 +12,13 @@
  *     - Norwegian
  *     - German
  */
+var path = require('path');
 var tap = require('tap');
-window = {};
-require('../../intl/splash.intl.js');
+
+var localeCompare = require('../../bin/lib/locale-compare');
+var viewLocales = {};
+var idsWithICU = {};
+var icuWithIds = {};
 
 var languagesToCheck = [
     'he', 'zh-cn', 'ja', 'pt-br', 'pl', 'nb'
@@ -24,12 +28,24 @@ var idsToCheck = [
     'general.signIn', 'general.discuss'
 ];
 
+
+// Test nav for real languages.
+localeCompare.getIdsForView(
+    'general',
+    path.resolve(__dirname, '../../src/l10n.json'),
+    viewLocales,
+    idsWithICU,
+    icuWithIds
+);
+var md5WithIds = localeCompare.getMD5Map(icuWithIds);
+
 tap.test('spotCheckNavBar', function (t) {
     for (var i in languagesToCheck) {
+        var translations = localeCompare.getTranslationsForLanguage(languagesToCheck[i], idsWithICU, md5WithIds);
         for (var j in idsToCheck) {
             t.notEqual(
-                window._messages[languagesToCheck[i]][idsToCheck[j]],
-                window._messages['en'][idsToCheck[j]],
+                translations['general'][languagesToCheck[i]][idsToCheck[j]],
+                viewLocales['general']['en'][idsToCheck[j]],
                 'check localization of ' + idsToCheck[j] + ' for ' + languagesToCheck[i]
             );
         }
@@ -37,12 +53,25 @@ tap.test('spotCheckNavBar', function (t) {
     t.end();
 });
 
+
+// Test splash items for fake language.
 var fakeLanguageIdsToCheck = ['news.scratchNews', 'splash.featuredProjects', 'splash.featuredStudios'];
+
+localeCompare.getIdsForView(
+    'splash',
+    path.resolve(__dirname, '../../src/views/splash/l10n.json'),
+    viewLocales,
+    idsWithICU,
+    icuWithIds
+);
+md5WithIds = localeCompare.getMD5Map(icuWithIds);
+
 tap.test('spotCheckNavBarFakeLanguage', function (t) {
+    var translations = localeCompare.getTranslationsForLanguage('yum', idsWithICU, md5WithIds);
     for (var i in fakeLanguageIdsToCheck) {
         t.notEqual(
-            window._messages['yum'][fakeLanguageIdsToCheck[i]],
-            window._messages['en'][fakeLanguageIdsToCheck[i]],
+            translations['general']['yum'][fakeLanguageIdsToCheck[i]],
+            viewLocales['splash']['en'][fakeLanguageIdsToCheck[i]],
             'check localization of ' + fakeLanguageIdsToCheck[i] + ' for yum'
         );
     }

test/localization/spot_check_splash_has_strings.js

@@ -2,17 +2,32 @@
  * spot check that each language has values for the string id keys on Splash page
  * that are contained in English (i.e. make sure strings will show up, not ids")
  */
+var merge = require('lodash.merge');
+var path = require('path');
 var tap = require('tap');
+
 var languages = require('../../languages.json');
-window = {};
-require('../../intl/splash.intl.js');
+var localeCompare = require('../../bin/lib/locale-compare');
 
-tap.test('spotCheckSplashStrings', function (t) {
+tap.test('spotCheckAboutStrings', function (t) {
     var isoCodes = Object.keys(languages);
-    var keysToCheck = Object.keys(window._messages['en']).sort();
+    isoCodes.splice(isoCodes.indexOf('en'), 1);
+    var viewLocales = {};
+    var idsWithICU = {};
+    var icuWithIds = {};
+    localeCompare.getIdsForView(
+        'splash',
+        path.resolve(__dirname, '../../src/views/splash/l10n.json'),
+        viewLocales,
+        idsWithICU,
+        icuWithIds
+    );
+    var md5WithIds = localeCompare.getMD5Map(icuWithIds);
+    var keysToCheck = Object.keys(merge(viewLocales['splash']['en'])).sort();
     for (var i in isoCodes) {
+        var translations = localeCompare.getTranslationsForLanguage(isoCodes[i], idsWithICU, md5WithIds);
         t.same(
-            Object.keys(window._messages[isoCodes[i]]).sort(),
+            Object.keys(translations['splash'][isoCodes[i]]).sort(),
             keysToCheck,
             'check Splash keys for language ' + isoCodes[i]
         );

test/localization/spot_check_wedo2_has_strings.js

 /*
- * spot check that each language has values for the string id keys on Splash page
+ * spot check that each language has values for the string id keys on Wedo2 page
  * that are contained in English (i.e. make sure strings will show up, not ids")
  */
+var merge = require('lodash.merge');
+var path = require('path');
 var tap = require('tap');
+
 var languages = require('../../languages.json');
-window = {};
-require('../../intl/wedo2.intl.js');
+var localeCompare = require('../../bin/lib/locale-compare');
 
-tap.test('spotCheckWedo2Strings', function (t) {
+tap.test('spotCheckAboutStrings', function (t) {
     var isoCodes = Object.keys(languages);
-    var keysToCheck = Object.keys(window._messages['en']).sort();
+    isoCodes.splice(isoCodes.indexOf('en'), 1);
+    var viewLocales = {};
+    var idsWithICU = {};
+    var icuWithIds = {};
+    localeCompare.getIdsForView(
+        'wedo2',
+        path.resolve(__dirname, '../../src/views/wedo2/l10n.json'),
+        viewLocales,
+        idsWithICU,
+        icuWithIds
+    );
+    var md5WithIds = localeCompare.getMD5Map(icuWithIds);
+    var keysToCheck = Object.keys(merge(viewLocales['wedo2']['en'])).sort();
     for (var i in isoCodes) {
+        var translations = localeCompare.getTranslationsForLanguage(isoCodes[i], idsWithICU, md5WithIds);
         t.same(
-            Object.keys(window._messages[isoCodes[i]]).sort(),
+            Object.keys(translations['wedo2'][isoCodes[i]]).sort(),
             keysToCheck,
-            'check Wedo2 keys for language ' + isoCodes[i]
+            'check About keys for language ' + isoCodes[i]
         );
     }
     t.end();