scratch-l10n 6.0.75 → 6.1.0

package/CHANGELOG.md

@@ -3,6 +3,18 @@
 All notable changes to this project will be documented in this file. See
 [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+# [6.1.0](https://github.com/scratchfoundation/scratch-l10n/compare/v6.0.75...v6.1.0) (2025-10-30)
+
+
+### Bug Fixes
+
+* apply suggestions from code review ([19fb2c4](https://github.com/scratchfoundation/scratch-l10n/commit/19fb2c424025360955d64b2dec84b22c3bf9c19d))
+
+
+### Features
+
+* continue after validation error ([08be90e](https://github.com/scratchfoundation/scratch-l10n/commit/08be90e3dc362fb0104e29cb96fcd60e8497b6a8))
+
 ## [6.0.75](https://github.com/scratchfoundation/scratch-l10n/compare/v6.0.74...v6.0.75) (2025-10-30)
 
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "scratch-l10n",
-  "version": "6.0.75",
+  "version": "6.1.0",
   "description": "Localization for the Scratch 3.0 components",
   "main": "./dist/l10n.js",
   "browser": "./src/index.mjs",

package/scripts/lib/pull-and-validate.mts

@@ -0,0 +1,142 @@
+import locales, { localeMap } from '../../src/supported-locales.mjs'
+import { poolMap } from './concurrent.mts'
+import { ProgressLogger } from './progress-logger.mts'
+import { TransifexStringsKeyValueJson } from './transifex-formats.mts'
+import { txPull, txResources } from './transifex.mts'
+import { filterInvalidTranslations, TransifexEditorString } from './validate.mts'
+
+const CONCURRENCY_LIMIT = 36
+const SOURCE_LOCALE = 'en' // TODO: don't hardcode this
+
+/**
+ * @param resources - A list of Transifex resource names
+ * @param selectedLocales - A list of Scratch locale codes
+ * @returns The list of all resource/locale combinations to pull. The source locale is always excluded.
+ */
+const expandResourceFiles = (resources: string[], selectedLocales: string[]) => {
+  const files = []
+  for (const resource of resources) {
+    for (const locale of selectedLocales) {
+      if (locale === SOURCE_LOCALE) {
+        continue
+      }
+      files.push({ resource: resource, locale: locale })
+    }
+  }
+  return files
+}
+
+/**
+ * Pull and validate a single Transifex "file" (resource + locale).
+ * @param o - The options for pulling and validating a single file.
+ * @param o.allStrings - All pulled strings so far, used to get source strings for validation
+ * @param o.project - The Transifex project to pull from
+ * @param o.resource - The Transifex resource to pull from
+ * @param o.locale - The locale to pull
+ * @param o.mode - The mode to use when pulling (e.g., "reviewed")
+ * @returns A list of messages about errors encountered during validation, if any.
+ */
+async function pullAndValidateFile({
+  allStrings,
+  project,
+  resource,
+  locale,
+  mode,
+}: {
+  allStrings: Record<string, Record<string, TransifexStringsKeyValueJson>>
+  project: string
+  resource: string
+  locale: string
+  mode?: string
+}) {
+  const messages: string[] = []
+  const txLocale = localeMap[locale] || locale
+  const fileContent = await txPull<TransifexEditorString>(project, resource, txLocale, mode)
+
+  // if fileContent has message & description, we only want the message
+  const translations: TransifexStringsKeyValueJson = {}
+  for (const key of Object.keys(fileContent)) {
+    const tx = fileContent[key]
+    if (typeof tx === 'string') {
+      translations[key] = tx
+    } else {
+      translations[key] = tx.message
+    }
+  }
+
+  if (!(resource in allStrings)) {
+    allStrings[resource] = {}
+  }
+  allStrings[resource][locale] = translations
+
+  // may or may not be the same as `translations`
+  const sourceStrings = allStrings[resource][SOURCE_LOCALE]
+
+  // some of the validation checks may still be relevant even if locale === SOURCE_LOCALE
+  // console.log({ resource, locale, translations, sourceStrings })
+  messages.push(...filterInvalidTranslations(locale, translations, sourceStrings))
+
+  return messages
+}
+
+/**
+ * Pull one or more resource(s) from a transifex project and validate the strings.
+ * Return any error messages from the validation process along with all valid strings.
+ * @param o - The options for pulling, validating, and saving translations.
+ * @param o.project - The Transifex project to pull translations from.
+ * @param o.mode - The mode to use when pulling translations (e.g., "reviewed").
+ * @param o.resources - The resources within the project to pull translations for.
+ * @param o.selectedLocales - The locales to pull translations for. Defaults to all supported locales.
+ * @returns Translation strings and a list of messages about errors encountered during validation, if any.
+ */
+export async function pullAndValidateProject({
+  project,
+  resources,
+  mode,
+  selectedLocales,
+}: {
+  project: string
+  mode?: string
+  resources?: string[]
+  selectedLocales?: string | string[]
+}) {
+  const selectedResources = resources ?? (await txResources(project))
+  selectedLocales =
+    typeof selectedLocales === 'string' ? [selectedLocales] : (selectedLocales ?? Object.keys(locales))
+
+  const files = expandResourceFiles(selectedResources, selectedLocales)
+
+  const allStrings: Record<string, Record<string, TransifexStringsKeyValueJson>> = {}
+  const messages: string[] = []
+
+  const progress = new ProgressLogger(selectedResources.length + files.length)
+
+  const handleFile = async (resource: string, locale: string) => {
+    try {
+      const fileMessages = await pullAndValidateFile({
+        allStrings,
+        project,
+        resource,
+        locale,
+        mode,
+      })
+      for (const message of fileMessages) {
+        // `message` already contains locale and/or string info if appropriate
+        messages.push(`resource ${resource} / ${message}`)
+      }
+    } finally {
+      progress.increment()
+    }
+  }
+
+  // Ensure source locale is available for validation
+  await poolMap(selectedResources, CONCURRENCY_LIMIT, async resource => handleFile(resource, SOURCE_LOCALE))
+
+  // Non-source locales
+  await poolMap(files, CONCURRENCY_LIMIT, async ({ resource, locale }) => handleFile(resource, locale))
+
+  return {
+    allStrings,
+    messages,
+  }
+}

package/scripts/lib/validate.mts

@@ -1,4 +1,3 @@
-import assert from 'assert'
 import parse from 'format-message-parse'
 import {
   TransifexStringChrome,
@@ -84,24 +83,52 @@ export const validMessage = (message: TransifexEditorString, source: TransifexEd
 }
 
 /**
- * @param translation - the translations to validate and their corresponding source strings
- * @param translation.locale - the Transifex locale, for error reporting
- * @param translation.translations - the translations to validate
+ * Validate and filter translations.
+ * WARNING: Modifies the translations object in place by replacing invalid translations with source strings.
+ * @param locale - the Transifex locale, for error reporting
+ * @param translations - the translations to validate and filter
  * @param source - the source strings for the translations
+ * @returns a list of messages about errors encountered during validation. Every removed translation will have a
+ * message. Some messages may not correspond to removed translations (e.g., when the number of keys differ).
  */
-export const validateTranslations = (
-  { locale, translations }: { locale: string; translations: TransifexEditorStrings },
+export const filterInvalidTranslations = (
+  locale: string,
+  translations: TransifexEditorStrings,
   source: TransifexEditorStrings,
-) => {
-  const transKeys = Object.keys(translations)
+): string[] => {
+  const messages: string[] = []
+
   const sourceKeys = Object.keys(source)
-  assert.strictEqual(transKeys.length, sourceKeys.length, `locale ${locale} has a different number of message keys`)
-  transKeys.forEach(item => assert(sourceKeys.includes(item), `locale ${locale} has key ${item} not in the source`))
-  sourceKeys.forEach(item => assert(transKeys.includes(item), `locale ${locale} is missing key ${item}`))
-  sourceKeys.forEach(item =>
-    assert(
-      validMessage(translations[item], source[item]),
-      `locale ${locale} / item ${item}: message validation failed:\n  msg: ${getMessageText(translations[item])}\n  src: ${getMessageText(source[item])}`,
-    ),
-  )
+
+  const transKeys = Object.keys(translations).filter(item => {
+    if (!sourceKeys.includes(item)) {
+      messages.push(`locale ${locale} has key ${item} not in the source`)
+      delete translations[item]
+      return false
+    }
+    return true
+  })
+
+  sourceKeys.forEach(item => {
+    if (!transKeys.includes(item)) {
+      messages.push(`locale ${locale} is missing key ${item}`)
+    }
+  })
+
+  transKeys.forEach(item => {
+    if (!validMessage(translations[item], source[item])) {
+      messages.push(
+        [
+          `locale ${locale} / item ${item}: message validation failed:`,
+          `  msg: ${getMessageText(translations[item])}`,
+          `  src: ${getMessageText(source[item])}`,
+        ].join('\n'),
+      )
+
+      // fall back to source message
+      translations[item] = source[item]
+    }
+  })
+
+  return messages
 }

package/scripts/tx-pull-editor.mts

@@ -8,11 +8,7 @@
  */
 import fs from 'fs'
 import path from 'path'
-import locales, { localeMap } from '../src/supported-locales.mjs'
-import { poolMap } from './lib/concurrent.mts'
-import { TransifexStringsKeyValueJson } from './lib/transifex-formats.mts'
-import { txPull } from './lib/transifex.mts'
-import { TransifexEditorStrings, validateTranslations } from './lib/validate.mts'
+import { pullAndValidateProject } from './lib/pull-and-validate.mts'
 
 const args = process.argv.slice(2)
 
@@ -37,38 +33,20 @@ const PROJECT = args[0]
 const RESOURCE = args[1]
 const OUTPUT_DIR = path.resolve(args[2])
 const MODE = 'reviewed'
-const CONCURRENCY_LIMIT = 36
 
-const getLocaleData = async function (locale: string) {
-  const txLocale = localeMap[locale] || locale
-  const data = (await txPull(PROJECT, RESOURCE, txLocale, MODE)) as TransifexEditorStrings
-  return {
-    locale: locale,
-    translations: data,
-  }
-}
+const validationResults = await pullAndValidateProject({
+  project: PROJECT,
+  resources: [RESOURCE],
+  mode: MODE,
+})
 
-const pullTranslations = async function () {
-  const values = await poolMap(Object.keys(locales), CONCURRENCY_LIMIT, getLocaleData)
-  const source = values.find(elt => elt.locale === 'en')?.translations
-  if (!source) {
-    throw new Error('Could not find source strings')
+for (const resource of Object.values(validationResults.allStrings)) {
+  for (const [locale, strings] of Object.entries(resource)) {
+    const file = JSON.stringify(strings, null, 4)
+    fs.writeFileSync(`${OUTPUT_DIR}/${locale}.json`, file)
   }
-  values.forEach(translation => {
-    validateTranslations({ locale: translation.locale, translations: translation.translations }, source)
-    // if translation has message & description, we only want the message
-    const txs: TransifexStringsKeyValueJson = {}
-    for (const key of Object.keys(translation.translations)) {
-      const tx = translation.translations[key]
-      if (typeof tx === 'string') {
-        txs[key] = tx
-      } else {
-        txs[key] = tx.message
-      }
-    }
-    const file = JSON.stringify(txs, null, 4)
-    fs.writeFileSync(`${OUTPUT_DIR}/${translation.locale}.json`, file)
-  })
 }
 
-await pullTranslations()
+if (validationResults.messages.length > 0) {
+  console.error(validationResults.messages.join('\n\n'))
+}

package/scripts/tx-pull-www.mts

@@ -9,10 +9,7 @@
 import fs from 'fs/promises'
 import { mkdirp } from 'mkdirp'
 import path from 'path'
-import locales, { localeMap } from '../src/supported-locales.mjs'
-import { poolMap } from './lib/concurrent.mts'
-import { ProgressLogger } from './lib/progress-logger.mjs'
-import { txPull, txResources } from './lib/transifex.mts'
+import { pullAndValidateProject } from './lib/pull-and-validate.mts'
 
 const args = process.argv.slice(2)
 
@@ -37,68 +34,34 @@ if (!process.env.TX_TOKEN || args.length < 1) {
 const PROJECT = 'scratch-website'
 const OUTPUT_DIR = path.resolve(args[0])
 // const MODE = {mode: 'reviewed'}; // default is everything for www
-const CONCURRENCY_LIMIT = 36
 
-const lang = args.length === 2 ? args[1] : ''
+const lang = args.length === 2 ? args[1] : undefined
+const validationResults = await pullAndValidateProject({
+  project: PROJECT,
+  selectedLocales: lang,
+})
 
-const getLocaleData = async function (item: { locale: string; resource: string }) {
-  const locale = item.locale
-  const resource = item.resource
-  const txLocale = localeMap[locale] || locale
+for (const [resourceName, resource] of Object.entries(validationResults.allStrings)) {
+  for (const [locale, translations] of Object.entries(resource)) {
+    const txOutdir = `${OUTPUT_DIR}/${PROJECT}.${resourceName}`
+    const fileName = `${txOutdir}/${locale}.json`
 
-  const translations = await txPull(PROJECT, resource, txLocale)
-
-  const txOutdir = `${OUTPUT_DIR}/${PROJECT}.${resource}`
-  const fileName = `${txOutdir}/${locale}.json`
-
-  try {
-    mkdirp.sync(txOutdir)
-    await fs.writeFile(fileName, JSON.stringify(translations, null, 4))
-
-    return {
-      resource,
-      locale,
-      fileName,
-    }
-  } catch (e) {
-    ;(e as Error).cause = {
-      resource,
-      locale,
-      translations,
-      txOutdir,
-      fileName,
-    }
-    throw e
-  }
-}
-
-const expandResourceFiles = (resources: string[]) => {
-  const items = []
-  for (const resource of resources) {
-    if (lang) {
-      items.push({ resource: resource, locale: lang })
-    } else {
-      for (const locale of Object.keys(locales)) {
-        items.push({ resource: resource, locale: locale })
+    try {
+      mkdirp.sync(txOutdir)
+      await fs.writeFile(fileName, JSON.stringify(translations, null, 4))
+    } catch (e) {
+      ;(e as Error).cause = {
+        resourceName,
+        locale,
+        translations,
+        txOutdir,
+        fileName,
       }
+      throw e
     }
   }
-  return items
 }
 
-const pullTranslations = async function () {
-  const resources = await txResources(PROJECT)
-  const allFiles = expandResourceFiles(resources)
-
-  const progress = new ProgressLogger(allFiles.length)
-
-  await poolMap(allFiles, CONCURRENCY_LIMIT, async item => {
-    try {
-      await getLocaleData(item)
-    } finally {
-      progress.increment()
-    }
-  })
+if (validationResults.messages.length > 0) {
+  console.error(validationResults.messages.join('\n\n'))
 }
-
-await pullTranslations()

package/scripts/validate-translations.mts

@@ -7,7 +7,7 @@ import async from 'async'
 import fs from 'fs'
 import path from 'path'
 import locales from '../src/supported-locales.mjs'
-import { TransifexEditorStrings, validateTranslations } from './lib/validate.mts'
+import { filterInvalidTranslations, TransifexEditorStrings } from './lib/validate.mts'
 
 const args = process.argv.slice(2)
 const usage = `
@@ -31,11 +31,10 @@ const validate = (locale: string, callback: async.ErrorCallback) => {
     if (err) callback(err)
     // let this throw an error if invalid json
     const strings = JSON.parse(data) as TransifexEditorStrings
-    const translations = {
-      locale: locale,
-      translations: strings,
+    const messages = filterInvalidTranslations(locale, strings, source)
+    if (messages.length > 0) {
+      callback(new Error(`Locale ${locale} has validation errors:\n${messages.join('\n')}`))
     }
-    validateTranslations(translations, source)
   })
 }
 

package/scripts/validate-www.mts

@@ -9,7 +9,7 @@ import glob from 'glob'
 import path from 'path'
 import locales from '../src/supported-locales.mjs'
 import { TransifexStringsKeyValueJson, TransifexStrings } from './lib/transifex-formats.mts'
-import { validateTranslations } from './lib/validate.mts'
+import { filterInvalidTranslations } from './lib/validate.mts'
 
 const args = process.argv.slice(2)
 const usage = `
@@ -37,11 +37,10 @@ const validate = (localeData: LocaleData, callback: async.ErrorCallback) => {
     if (err) callback(err)
     // let this throw an error if invalid json
     const strings = JSON.parse(data) as TransifexStringsKeyValueJson
-    const translations = {
-      locale: localeData.locale,
-      translations: strings,
+    const messages = filterInvalidTranslations(localeData.locale, strings, localeData.sourceData)
+    if (messages.length > 0) {
+      callback(new Error(`Locale ${localeData.locale} has validation errors:\n${messages.join('\n')}`))
     }
-    validateTranslations(translations, localeData.sourceData)
   })
 }