Skip to content

GitLab

S
scratch-www
Commit 214430b0 authored by Matthew Taylor's avatar Matthew Taylor
Browse files
Options

Migrate to using a loader method 

This moves all locale/translation building to a dependency, `scratch-www-intl-loader`, as well as tests associated with it. Also gets rid of the `make translations` step.
parent d5ffd9fc
Hide whitespace changes
Inline Side-by-side
Showing with 88 additions and 399 deletions
Makefile
View file @ 214430b0
......@@ -12,7 +12,6 @@ GIT_MESSAGE=$(shell git log -1 --pretty=%s 2> /dev/null)
build:
@make clean
@make translations
@make webpack
@make tag
......@@ -55,10 +54,6 @@ test:
@make lint
@make build
@echo ""
@make unit
@echo ""
@make functional
@echo ""
lint:
$(ESLINT) ./*.js
......
bin/build-locales deleted 100755 → 0
View file @ d5ffd9fc
#!/usr/bin/env node
/*
Converts the existing .po translation files in the module to JSON files.
Requires po2json in order to work. Takes as input a directory
in which to store the resulting json translation files.
Takes in as an argument an output directory to put translation files.
Searches for files named `l10n.json` in the `src/views/` directory to get
template english strings (as well as the general template at `src/l10n.json`).
It compiles the template strings into a flat object that is compared against the
translations in the .po files from the `scratchr2_translations` dependency, using
an md5 of the template string without whitespace, and an md5 of the .po msgid string
without whitespace.
The output files are javascript files that declare objects by locale. Each locale
has a sub-object with FormattedMessage ids as keys, and translated strings as
values. If no translation was found for a string, the default english will be the
value.
Output Example:
'''
var message = {
en: {
'general.inAWorld': 'In a world, where bears are invisible...',
'general.question': 'Are there bears here?',
'general.answer': 'I dunno, but there could be...'
},
es: {
'general.inAWorld': 'En un mundo, donde hay osos invisibles',
'general.question': 'Are there bears here?',
'general.answer': 'No sé, pero es posible...'
}
}
'''
*/
var fs = require('fs');
var glob = require('glob');
var merge = require('lodash.merge');
var path = require('path');
var po2icu = require('po2icu');
var localeCompare = require('./lib/locale-compare');
// -----------------------------------------------------------------------------
// Main script
// -----------------------------------------------------------------------------
var args = process.argv.slice(2);
if (!args.length) {
process.stdout.write('A destination directory must be specified.');
process.exit(1);
}
var poUiDir = path.resolve(__dirname, '../node_modules/scratchr2_translations/ui');
var outputDir = path.resolve(__dirname, '../', args[0]);
try {
fs.accessSync(outputDir, fs.F_OK);
} catch (err) {
// Doesn't exist - create it.
fs.mkdirSync(outputDir);
}
// get global locale strings first.
var globalTemplateFile = path.resolve(__dirname, '../src/l10n.json');
// message key with english string values (i.e. default values)
var generalIds = JSON.parse(fs.readFileSync(globalTemplateFile, 'utf8'));
var viewLocales = {};
var generalLocales = {
en: generalIds
};
// FormattedMessage id with english string as value. Use for default values in translations
// Sample structure: { 'general-general.blah': 'blah', 'about-about.blah': 'blahblah' }
var idsWithICU = {};
// reverse (i.e. english string with message key as the value) object for searching po files.
// Sample structure: { 'blah': 'general-general.blah', 'blahblah': 'about-about.blah' }
var icuWithIds = {};
for (var id in generalIds) {
idsWithICU['general-' + id] = generalIds[id];
icuWithIds[generalIds[id]] = 'general-' + id;
}
// get view-specific locale strings.
var files = glob.sync(path.resolve(__dirname, '../src/views/**/l10n.json'));
files.forEach(function (file) {
var dirPath = file.split('/');
dirPath.pop();
var view = dirPath.pop();
var viewIds = JSON.parse(fs.readFileSync(file, 'utf8'));
viewLocales[view] = {
en: viewIds
};
for (var id in viewIds) {
idsWithICU[view + '-' + id] = viewIds[id];
icuWithIds[viewIds[id]] = view + '-' + id; // add viewName to identifier for later
}
});
// md5 of english strings with message key as the value for searching po files.
// Sample structure: { 'sdfas43534sdfasdf': 'general-general.blah', 'lkjfasdf4t342asdfa': 'about-about.blah' }
var md5WithIds = localeCompare.getMD5Map(icuWithIds);
// Get ui localization strings first
glob(poUiDir + '/*', function (err, files) {
if (err) throw new Error(err);
files.forEach(function (file) {
var lang = file.split('/').pop();
var jsFile = path.resolve(file, 'LC_MESSAGES/djangojs.po');
var pyFile = path.resolve(file, 'LC_MESSAGES/django.po');
var translations = {};
try {
var jsTranslations = po2icu.poFileToICUSync(lang, jsFile);
translations = localeCompare.mergeNewTranslations(translations, jsTranslations, idsWithICU, md5WithIds);
} catch (err) {
process.stdout.write(lang + ': ' + err + '\n');
}
try {
var pyTranslations = po2icu.poFileToICUSync(lang, pyFile);
translations = localeCompare.mergeNewTranslations(translations, pyTranslations, idsWithICU, md5WithIds);
} catch (err) {
process.stdout.write(lang + ': ' + err + '\n');
}
// add new translations to locale object
for (var id in translations) {
var ids = id.split('-'); // [viewName, stringId]
var viewName = ids[0];
var stringId = ids[1];
if (viewLocales.hasOwnProperty(viewName)) {
if (!viewLocales[viewName].hasOwnProperty(lang)) viewLocales[viewName][lang] = {};
viewLocales[viewName][lang][stringId] = translations[id];
} else {
// default to general
if (!generalLocales.hasOwnProperty(lang)) generalLocales[lang] = {};
generalLocales[lang][stringId] = translations[id];
}
}
});
for (var view in viewLocales) {
var viewTranslations = merge(viewLocales[view], generalLocales);
var objectString = JSON.stringify(viewTranslations);
var fileString = 'window._messages = ' + objectString + ';';
fs.writeFileSync(outputDir + '/' + view + '.js', fileString);
}
});
bin/lib/locale-compare.js deleted 100644 → 0
View file @ d5ffd9fc
// -----------------------------------------------------------------------------
// Helper Methods for build-locales node script.
// -----------------------------------------------------------------------------
var crypto = require('crypto');
var Helpers = {};
/**
* Get the md5 has of a string with whitespace removed.
*
* @param {string} string a string
* @return {string} an md5 hash of the string in hex.
*/
Helpers.getMD5 = function (string) {
var cleanedString = string.replace(/\s+/g, '');
return crypto.createHash('md5').update(cleanedString, 'utf8').digest('hex');
};
/*
Existing translations should be in the key value format specified by react-intl (i.e.
formatted message id, with icu string as the value). New Translations should be in the
format returned by po2icu (i.e. a source language icu string for key, and a localized
language icu string for value).
ICU Map is an object in the reverse react-intl formatting (icu string as key), which will
help determine if the translation belongs in www currently.
*/
Helpers.mergeNewTranslations = function (existingTranslations, newTranslations, icuTemplate, md5Map) {
for (var id in newTranslations) {
var md5 = Helpers.getMD5(id);
if (md5Map.hasOwnProperty(md5) && newTranslations[id].length > 0) {
existingTranslations[md5Map[md5]] = newTranslations[id];
}
}
//Fill in defaults
for (var id in icuTemplate) {
if (!existingTranslations.hasOwnProperty(id)) existingTranslations[id] = icuTemplate[id];
}
return existingTranslations;
};
/**
* Converts a map of icu strings with react-intl id values into a map
* with md5 hashes of the icu strings as keys and react-intl id values.
* This is done so as to eliminate potential po conversion misses that
* could be caused by different white space formatting between po and icu.
*
* The md5 is generated after all white space is removed from the string.
*
* @param {object} idICUMap map where key=icuString, value=react-intl id
*
* @return {object}
*/
Helpers.getMD5Map = function (ICUIdMap) {
var md5Map = {};
for (var icu in ICUIdMap) {
var md5 = Helpers.getMD5(icu);
md5Map[md5] = ICUIdMap[icu];
}
return md5Map;
};
module.exports = Helpers;
intl-loader.js 0 → 100644
View file @ 214430b0
var glob = require('glob');
var path = require('path');
var po2icu = require('po2icu');
var localeCompare = require('./bin/lib/locale-compare');
module.exports = function (source) {
this.cacheable();
var poUiDir = path.resolve(__dirname, './node_modules/scratchr2_translations/ui');
var viewIds = JSON.parse(source);
var viewLocales = {
en: viewIds
};
var icuWithIds = {};
for (var id in viewIds) {
icuWithIds[viewIds[id]] = id;
}
var md5WithIds = localeCompare.getMD5Map(icuWithIds);
var files = glob.sync(poUiDir + '/*');
files.forEach(function (file) {
var lang = file.split('/').pop();
var jsFile = path.resolve(file, 'LC_MESSAGES/djangojs.po');
var pyFile = path.resolve(file, 'LC_MESSAGES/django.po');
var translations = {};
try {
var jsTranslations = po2icu.poFileToICUSync(lang, jsFile);
translations = localeCompare.mergeNewTranslations(translations, jsTranslations, viewIds, md5WithIds);
} catch (err) {
process.stdout.write(lang + ': ' + err + '\n');
}
try {
var pyTranslations = po2icu.poFileToICUSync(lang, pyFile);
translations = localeCompare.mergeNewTranslations(translations, pyTranslations, viewIds, md5WithIds);
} catch (err) {
process.stdout.write(lang + ': ' + err + '\n');
}
viewLocales[lang] = translations;
});
return 'module.exports = ' + JSON.stringify(viewLocales, undefined, '\t') + ';';
};
server/template.html
View file @ 214430b0
......@@ -51,7 +51,6 @@
<script src="/js/lib/react-intl-with-locales{{min}}.js"></script>
<script src="/js/lib/raven.min.js"></script>
<script src="/js/intl/{{view}}.js"></script>
<script src="/js/{{view}}.bundle.js"></script>
<!-- Error logging (Sentry) -->
......
src/lib/render.jsx
View file @ 214430b0
......@@ -3,22 +3,21 @@ var ReactDOM = require('react-dom');
var ReactIntl = require('./intl.jsx');
var IntlProvider = ReactIntl.IntlProvider;
var render = function (jsx, element) {
var render = function (jsx, element, messages) {
// Get locale and messages from global namespace (see "init.js")
var locale = window._locale || 'en';
if (typeof window._messages[locale] === 'undefined') {
if (typeof messages[locale] === 'undefined') {
// Fall back on the split
locale = locale.split('-')[0];
}
if (typeof window._messages[locale] === 'undefined') {
if (typeof messages[locale] === 'undefined') {
// Language appears to not be supported – fall back to 'en'
locale = 'en';
}
var messages = window._messages[locale];
// Render component
var component = ReactDOM.render(
<IntlProvider locale={locale} messages={messages}>
<IntlProvider locale={locale} messages={messages[locale]}>
{jsx}
</IntlProvider>,
element
......
src/views/about/about.jsx
View file @ 214430b0
var React = require('react');
var FormattedHTMLMessage = require('react-intl').FormattedHTMLMessage;
var FormattedMessage = require('react-intl').FormattedMessage;
var merge = require('lodash.merge');
var React = require('react');
var render = require('../../lib/render.jsx');
require('../../main.scss');
require('./about.scss');
var generalMessages = require('../../main.intl');
var viewMessages = require('./about.intl');
var Navigation = require('../../components/navigation/navigation.jsx');
var Footer = require('../../components/footer/footer.jsx');
......@@ -104,6 +109,6 @@ var About = React.createClass({
}
});
render(<Navigation />, document.getElementById('navigation'));
render(<Footer />, document.getElementById('footer'));
render(<About />, document.getElementById('view'));
render(<Navigation />, document.getElementById('navigation'), generalMessages);
render(<Footer />, document.getElementById('footer'), generalMessages);
render(<About />, document.getElementById('view'), merge(generalMessages, viewMessages));
src/views/components/components.jsx
View file @ 214430b0
......@@ -11,6 +11,8 @@ var Spinner = require('../../components/spinner/spinner.jsx');
require('../../main.scss');
require('./components.scss');
var generalMessages = require('../../main.intl');
var Navigation = require('../../components/navigation/navigation.jsx');
var Footer = require('../../components/footer/footer.jsx');
......@@ -48,6 +50,6 @@ var Components = React.createClass({
}
});
render(<Navigation />, document.getElementById('navigation'));
render(<Footer />, document.getElementById('footer'));
render(<Components />, document.getElementById('view'));
render(<Navigation />, document.getElementById('navigation'), generalMessages);
render(<Footer />, document.getElementById('footer'), generalMessages);
render(<Components />, document.getElementById('view'), generalMessages);
src/views/credits/credits.jsx
View file @ 214430b0
......@@ -4,6 +4,8 @@ var render = require('../../lib/render.jsx');
require('../../main.scss');
require('./credits.scss');
var generalMessages = require('../../main.intl');
var Navigation = require('../../components/navigation/navigation.jsx');
var Footer = require('../../components/footer/footer.jsx');
......@@ -298,6 +300,6 @@ var Credits = React.createClass({
}
});
render(<Navigation />, document.getElementById('navigation'));
render(<Footer />, document.getElementById('footer'));
render(<Credits />, document.getElementById('view'));
render(<Navigation />, document.getElementById('navigation'), generalMessages);
render(<Footer />, document.getElementById('footer'), generalMessages);
render(<Credits />, document.getElementById('view'), generalMessages);
src/views/hoc/hoc.jsx
View file @ 214430b0
var classNames = require('classnames');
var FormattedHTMLMessage = require('react-intl').FormattedHTMLMessage;
var FormattedMessage = require('react-intl').FormattedMessage;
var merge = require('lodash.merge');
var React = require('react');
var render = require('../../lib/render.jsx');
......@@ -11,6 +12,9 @@ var SubNavigation = require('../../components/subnavigation/subnavigation.jsx');
require('../../main.scss');
require('./hoc.scss');
var generalMessages = require('../../main.intl');
var viewMessages = require('./hoc.intl');
var Navigation = require('../../components/navigation/navigation.jsx');
var Footer = require('../../components/footer/footer.jsx');
......@@ -409,6 +413,6 @@ var Hoc = React.createClass({
}
});
render(<Navigation />, document.getElementById('navigation'));
render(<Footer />, document.getElementById('footer'));
render(<Hoc />, document.getElementById('view'));
render(<Navigation />, document.getElementById('navigation'), generalMessages);
render(<Footer />, document.getElementById('footer'), generalMessages);
render(<Hoc />, document.getElementById('view'), merge(generalMessages, viewMessages));
src/views/splash/splash.jsx
View file @ 214430b0
var injectIntl = require('react-intl').injectIntl;
var merge = require('lodash.merge');
var omit = require('lodash.omit');
var React = require('react');
var render = require('../../lib/render.jsx');
......@@ -20,6 +21,9 @@ var Welcome = require('../../components/welcome/welcome.jsx');
require('../../main.scss');
require('./splash.scss');
var generalMessages = require('../../main.intl');
var viewMessages = require('./splash.intl');
var Navigation = require('../../components/navigation/navigation.jsx');
var Footer = require('../../components/footer/footer.jsx');
......@@ -415,6 +419,6 @@ var Splash = injectIntl(React.createClass({
}
}));
render(<Navigation />, document.getElementById('navigation'));
render(<Footer />, document.getElementById('footer'));
render(<Splash />, document.getElementById('view'));
render(<Navigation />, document.getElementById('navigation'), generalMessages);
render(<Footer />, document.getElementById('footer'), generalMessages);
render(<Splash />, document.getElementById('view'), merge(generalMessages, viewMessages));
test/fixtures/build_locales.json deleted 100644 → 0
View file @ d5ffd9fc
{
"<p> <img src=\"{STATIC_URL}/images/help/ask-and-wait.png\" /> asks a question and stores the keyboard input in <img src=\"{STATIC_URL}/images/help/answer.png\" />. The answer is shared by all sprites. </p><p>If you want to save the current answer, you can store it in a variable or list. For example, <img src=\"{STATIC_URL}/images/help/answer-ex2.png\"/> </p><p>To view the value of answer, click the checkbox next to the answer block.<br><img src=\"{STATIC_URL}/images/help/answer-checkbox.png\" /></p>": "test.id1",
"<p> <img src=\"{STATIC_URL}/images/help/ask-and-wait.png\" /> asks a question and stores the keyboard input in <img src=\"{STATIC_URL}/images/help/answer.png\" />. The question appears in a voice balloon on the screen. The program waits as the user types in a response, until the Enter key is pressed or the check mark is clicked. </p>": "test.id2"
}
test/fixtures/test_es.po deleted 100644 → 0
View file @ d5ffd9fc
# SOME DESCRIPTIVE TITLE.
# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER
# This file is distributed under the same license as the PACKAGE package.
# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
msgid ""
msgstr ""
"Project-Id-Version: PACKAGE VERSION\n"
"Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2015-03-20 14:16+0000\n"
"PO-Revision-Date: 2015-09-21 17:37+0000\n"
"Last-Translator: Anonymous Pootle User\n"
"Language-Team: LANGUAGE <LL@li.org>\n"
"Language: es\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Plural-Forms: nplurals=2; plural=(n != 1);\n"
"X-Generator: Pootle 2.5.1.1\n"
"X-POOTLE-MTIME: 1442857052.000000\n"
#: test.html:15
#, python-format
msgid ""
"\n"
"<p> <img src=\"%(STATIC_URL)s/images/help/ask-and-wait.png\" /> asks a "
"question and stores the keyboard input in <img src=\"%(STATIC_URL)s/images/"
"help/answer.png\" />. The answer is shared by all sprites. </p>\n"
"<p>If you want to save the current answer, you can store it in a variable or "
"list. For example, <img src=\"%(STATIC_URL)s/images/help/answer-ex2.png\"/"
"> \n"
"</p>\n"
"\n"
"<p>\n"
"To view the value of answer, click the checkbox next to the answer block."
"<br>\n"
"<img src=\"%(STATIC_URL)s/images/help/answer-checkbox.png\" />\n"
"</p>"
msgstr ""
"\n"
"<p><img src=\"%(STATIC_URL)s/images/help/es/ask-and-wait.png\" /> hace una "
"pregunta y almacena la entrada de teclado en <img src=\"%(STATIC_URL)s/"
"images/help/es/answer.png\" />. La respuesta se comparte para todos los "
"objetos. </p>\n"
"<p>Si deseas guardar la respuesta actual, debes almacenarla en una variable "
"o una lista. Por ejemplo, <img src=\"%(STATIC_URL)s/images/help/es/answer-"
"ex2.png\"/> \n"
"</p>\n"
"\n"
"<p>\n"
"Si deseas ver el valor de una respuesta, haz clic en la casilla que aparece "
"junto al bloque de respuesta.<br>\n"
"<img src=\"%(STATIC_URL)s/images/help/es/answer-checkbox.png\" />\n"
"</p>"
#: test.html:18
#, python-format
msgid ""
"\n"
"<p> <img src=\"%(STATIC_URL)s/images/help/ask-and-wait.png\" /> asks a "
"question and stores the keyboard input in <img src=\"%(STATIC_URL)s/images/"
"help/answer.png\" />. The question appears in a voice balloon on the screen. "
"The program waits as the user types in a response, until the Enter key is "
"pressed or the check mark is clicked. \n"
"</p>"
msgstr ""
"\n"
"<p> <img src=\"%(STATIC_URL)s/images/help/es/ask-and-wait.png\" /> hace una "
"pregunta y almacena la entrada de teclado en <img src=\"%(STATIC_URL)s/"
"images/help/es/answer.png\" />. La pregunta aparece en un globo de voz en la "
"pantalla. El programa espera hasta que el usuario escriba una respuesta y "
"presione Enter o haga clic en la casilla de aprobación.\n"
"</p>"
test/fixtures/test_es_md5map.json deleted 100644 → 0
View file @ d5ffd9fc
{
"2ec20d41b181e1a41c071e13f414a74d": "test.id1",
"37ba6d5ef524504215f478912155f9ba": "test.id2"
}
test/functional/build_locales_complex_strings.js deleted 100644 → 0
View file @ d5ffd9fc
var fs = require('fs');
var path = require('path');
var po2icu = require('po2icu');
var tap = require('tap');
var buildLocales = require('../../bin/lib/locale-compare');
tap.test('buildLocalesFile', function (t) {
var md5map = JSON.parse(fs.readFileSync(path.resolve(__dirname, '../fixtures/test_es_md5map.json'), 'utf8'));
var newTranslations = po2icu.poFileToICUSync('es', path.resolve(__dirname, '../fixtures/test_es.po'));
var translations = buildLocales.mergeNewTranslations({}, newTranslations, {}, md5map);
t.ok(translations['test.id1'] !== undefined);
t.ok(translations['test.id2'] !== undefined);
t.end();
});
test/functional/build_locales_md5map.js deleted 100644 → 0
View file @ d5ffd9fc
var fs = require('fs');
var path = require('path');
var tap = require('tap');
var buildLocales = require('../../bin/lib/locale-compare');
tap.test('buildLocalesFile', function (t) {
var actualMd5map = JSON.parse(fs.readFileSync(path.resolve(__dirname, '../fixtures/test_es_md5map.json'), 'utf8'));
var templates = JSON.parse(fs.readFileSync(path.resolve(__dirname, '../fixtures/build_locales.json'), 'utf8'));
var testMd5map = buildLocales.getMD5Map(templates);
for (var key in actualMd5map) {
t.ok(testMd5map[key] !== undefined);
}
t.end();
});
test/functional/build_locales_mergeTranslations.js deleted 100644 → 0
View file @ d5ffd9fc
var tap = require('tap');
var buildLocales = require('../../bin/lib/locale-compare');
tap.test('buildLocalesMergeTranslations', function (t) {
var existingTranslations = {
'test.test1': 'It\'s like raaayaaain, on your wedding day',
'test.test2': 'Free to flyyy, when you already paid'
};
var newTranslations = {
'Isn\'t it ironic? No.': 'Es irónico? No.'
};
var md5map = {
'c21ce5ceefe167028182032d4255a384': 'test.test1',
'9c40648034e467e16f8d6ae24bd610ab': 'test.test2',
'6885a345adafb3a9dd43d9f549430c88': 'test.test3'
};
var mergedTranslations = buildLocales.mergeNewTranslations(existingTranslations, newTranslations, {}, md5map);
t.ok(mergedTranslations['test.test3'] !== undefined);
t.ok(mergedTranslations['test.test2'] !== undefined);
t.end();
});
test/unit/build_locales_getmd5.js deleted 100644 → 0
View file @ d5ffd9fc
var tap = require('tap');
var buildLocales = require('../../bin/lib/locale-compare');
tap.test('buildLocalesGetMD5', function (t) {
var testString1 = 'are there bears here?';
var testString2 = 'are\nthere\tbears here?';
t.equal(buildLocales.getMD5(testString1), buildLocales.getMD5(testString2));
t.end();
});
webpack.config.js
View file @ 214430b0
......@@ -44,6 +44,10 @@ module.exports = {
{
test: /\.(png|jpg|gif|eot|svg|ttf|woff)$/,
loader: 'url-loader'
},
{
test: /\.intl$/,
loader: 'scratch-www-intl-loader'
}
]
},
......@@ -52,8 +56,7 @@ module.exports = {
},
plugins: [
new CopyWebpackPlugin([
{from: 'static'},
{from: 'intl', to: 'js/intl'}
{from: 'static'}
]),
new webpack.optimize.UglifyJsPlugin({
compress: {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment