scratch-l10n 5.0.308 → 6.0.0

package/scripts/lib/transifex.mts

@@ -0,0 +1,284 @@
+/**
+ * @file
+ * Utilities for interfacing with Transifex API 3.
+ */
+import { transifexApi, Collection, JsonApiResource } from '@transifex/api'
+import { TransifexStrings } from './transifex-formats.mts'
+import { TransifexLanguageObject, TransifexResourceObject } from './transifex-objects.mts'
+
+const ORG_NAME = 'llk'
+const SOURCE_LOCALE = 'en'
+
+if (!process.env.TX_TOKEN) {
+  throw new Error('TX_TOKEN is not defined.')
+}
+
+transifexApi.setup({
+  auth: process.env.TX_TOKEN,
+})
+
+/*
+ * The Transifex JS API wraps the Transifex JSON API, and is built around the concept of a `Collection`.
+ * A `Collection` begins as a URL builder: methods like `filter` and `sort` add query parameters to the URL.
+ * The `download` method doesn't actually download anything: it returns the built URL. It seems to be intended
+ * primarily for internal use, but shows up in the documentation despite not being advertised in the .d.ts file.
+ * The `download` method is mainly used to skip the `fetch` method in favor of downloading the resource yourself.
+ * The `fetch` method sends a request to the URL and returns a promise that resolves to the first page of results.
+ * If there's only one page of results, the `data` property of the collection object will be an array of all results.
+ * However, if there are multiple pages of results, the `data` property will only contain the first page of results.
+ * Previous versions of this code would unsafely assume that the `data` property contained all results.
+ * The `all` method returns an async iterator that yields all results, fetching additional pages as needed.
+ */
+
+/**
+ * Collects all resources from all pages of a potentially-paginated JSON API collection.
+ * It's not necessary, but also not harmful, to call `fetch()` on the collection before calling this function.
+ * @param collection A collection of JSON API resources.
+ * @returns An array of all resources in the collection.
+ * @todo This seems necessary with the latest Transifex API..?
+ */
+const collectAll = async function <T extends JsonApiResource>(collection: Collection): Promise<T[]> {
+  await collection.fetch() // fetch the first page if it hasn't already been fetched
+  const collected: T[] = []
+  for (const item of collection.all()) {
+    collected.push(item as T)
+  }
+  return collected
+}
+
+/**
+ * Creates a download event for a specific project, resource, and locale.
+ * Returns the URL to download the resource.
+ * @param projectSlug - project slug (for example,  "scratch-editor")
+ * @param resourceSlug - resource slug (for example,  "blocks")
+ * @param localeCode - language code (for example,  "ko")
+ * @param mode - translation status of strings to include
+ * @returns URL to download the resource
+ */
+const getResourceLocation = async function (
+  projectSlug: string,
+  resourceSlug: string,
+  localeCode: string,
+  mode = 'default',
+): Promise<string> {
+  const resource = {
+    data: {
+      id: `o:${ORG_NAME}:p:${projectSlug}:r:${resourceSlug}`,
+      type: 'resources',
+    },
+  }
+
+  // if locale is English, create a download event of the source file
+  if (localeCode === SOURCE_LOCALE) {
+    return (await transifexApi.ResourceStringsAsyncDownload.download({
+      resource,
+    })) as string
+  }
+
+  const language = {
+    data: {
+      id: `l:${localeCode}`,
+      type: 'languages',
+    },
+  }
+
+  // if locale is not English, create a download event of the translation file
+  return (await transifexApi.ResourceTranslationsAsyncDownload.download({
+    mode,
+    resource,
+    language,
+  })) as string
+}
+
+/**
+ * Pulls a translation JSON from transifex, for a specific project, resource, and locale.
+ * @template T - resource file type, such as `TransifexStringsKeyValueJson`
+ * @param project - project slug (for example, `scratch-editor`)
+ * @param resource - resource slug (for example, `blocks`)
+ * @param locale - language code (for example, `ko`)
+ * @param mode - translation status of strings to include
+ * @returns JSON object of translated resource strings (or, of the original resource strings, if the local is the
+ * source language)
+ */
+export const txPull = async function <T>(
+  project: string,
+  resource: string,
+  locale: string,
+  mode = 'default',
+): Promise<TransifexStrings<T>> {
+  let buffer: string | null = null
+  try {
+    const url = await getResourceLocation(project, resource, locale, mode)
+    for (let i = 0; i < 5; i++) {
+      if (i > 0) {
+        console.log(`Retrying txPull download after ${i} failed attempt(s)`)
+      }
+      try {
+        const response = await fetch(url)
+        if (!response.ok) {
+          throw new Error(`Failed to download resource: ${response.statusText}`)
+        }
+        buffer = await response.text()
+        break
+      } catch (e) {
+        console.error(e, { project, resource, locale, buffer })
+      }
+    }
+    if (!buffer) {
+      throw Error(`txPull download failed after 5 retries: ${url}`)
+    }
+    return JSON.parse(buffer) as TransifexStrings<T>
+  } catch (e) {
+    ;(e as Error).cause = {
+      project,
+      resource,
+      locale,
+      buffer,
+    }
+    throw e
+  }
+}
+
+/**
+ * Given a project, returns a list of the slugs of all resources in the project
+ * @param project - project slug (for example,  "scratch-website")
+ * @returns - array of strings, slugs identifying each resource in the project
+ */
+export const txResources = async function (project: string): Promise<string[]> {
+  const resources = transifexApi.Resource.filter({
+    project: `o:${ORG_NAME}:p:${project}`,
+  })
+
+  const resourcesData = await collectAll<TransifexResourceObject>(resources)
+
+  const slugs = resourcesData.map(
+    r =>
+      // r.id is a longer id string, like "o:llk:p:scratch-website:r:about-l10njson"
+      // We just want the slug that comes after ":r:" ("about-l10njson")
+      r.id.split(':r:')[1],
+  )
+  return slugs
+}
+
+/**
+ * @param project - project slug (for example)
+ * @returns - array of resource objects
+ */
+export const txResourcesObjects = async function (project: string): Promise<TransifexResourceObject[]> {
+  const resources = transifexApi.Resource.filter({
+    project: `o:${ORG_NAME}:p:${project}`,
+  })
+
+  return collectAll<TransifexResourceObject>(resources)
+}
+
+/**
+ * Gets available languages for a project
+ * @param slug - project slug (for example, "scratch-editor")
+ * @returns - list of language codes
+ */
+export const txAvailableLanguages = async function (slug: string): Promise<string[]> {
+  const project = await transifexApi.Project.get({
+    organization: `o:${ORG_NAME}`,
+    slug: slug,
+  })
+
+  const languages = (await project.fetch('languages', false)) as Collection
+  const languagesData = await collectAll<TransifexLanguageObject>(languages)
+  return languagesData.map(l => l.attributes.code)
+}
+
+/**
+ * Uploads English source strings to a resource in transifex
+ * @param project - project slug (for example,  "scratch-editor")
+ * @param resource - resource slug (for example,  "blocks")
+ * @param sourceStrings - json of source strings
+ */
+export const txPush = async function (project: string, resource: string, sourceStrings: TransifexStrings<unknown>) {
+  const resourceObj = {
+    data: {
+      id: `o:${ORG_NAME}:p:${project}:r:${resource}`,
+      type: 'resources',
+    },
+  }
+
+  await transifexApi.ResourceStringsAsyncUpload.upload({
+    resource: resourceObj,
+    content: JSON.stringify(sourceStrings),
+  })
+}
+
+/**
+ * Creates a new resource, and then uploads source strings to it if they are provided
+ * @param project - project slug (for example,  "scratch-editor")
+ * @param resource - object of resource information
+ * @param resource.slug - resource slug (for example,  "blocks")
+ * @param resource.name - human-readable name for the resource
+ * @param resource.i18nType - i18n format id
+ * @param resource.sourceStrings - json object of source strings
+ */
+export const txCreateResource = async function (
+  project: string,
+  {
+    slug,
+    name,
+    i18nType,
+    sourceStrings,
+  }: {
+    slug: string
+    name: string
+    i18nType: string
+    sourceStrings?: TransifexStrings<unknown>
+  },
+) {
+  const i18nFormat = {
+    data: {
+      id: i18nType || 'KEYVALUEJSON',
+      type: 'i18n_formats',
+    },
+  }
+
+  const projectObj = {
+    data: {
+      id: `o:${ORG_NAME}:p:${project}`,
+      type: 'projects',
+    },
+  }
+
+  // @ts-expect-error This omits "required" props but has been like this for ages and I'm not sure how to best fix it
+  await transifexApi.Resource.create({
+    attributes: { slug: slug, name: name },
+    relationships: {
+      i18n_format: i18nFormat,
+      project: projectObj,
+    },
+  })
+
+  if (sourceStrings) {
+    await txPush(project, slug, sourceStrings)
+  }
+}
+
+/**
+ * Information about an error condition generated by Transifex's JSON API
+ * @see https://github.com/transifex/transifex-api-python/blob/master/src/jsonapi/exceptions.py
+ * @see https://github.com/transifex/transifex-javascript/blob/master/packages/jsonapi/src/errors.js
+ */
+export interface JsonApiError {
+  status: number
+  code: string
+  title: string
+  detail: string
+  source?: string
+}
+
+/**
+ * A JS `Error` thrown by Transifex's JSON API
+ * @see https://github.com/transifex/transifex-api-python/blob/master/src/jsonapi/exceptions.py
+ * @see https://github.com/transifex/transifex-javascript/blob/master/packages/jsonapi/src/errors.js
+ */
+export interface JsonApiException extends Error {
+  statusCode: number
+  errors: JsonApiError[]
+  message: string
+}

package/scripts/lib/validate.mts

@@ -0,0 +1,107 @@
+import assert from 'assert'
+import parse from 'format-message-parse'
+import {
+  TransifexStringChrome,
+  TransifexStringKeyValueJson,
+  TransifexStringsChrome,
+  TransifexStringsKeyValueJson,
+} from './transifex-formats.mts'
+
+export type TransifexEditorString = TransifexStringChrome | TransifexStringKeyValueJson
+export type TransifexEditorStrings = TransifexStringsChrome | TransifexStringsKeyValueJson
+
+/**
+ * filter placeholders out of a message
+ * @param message - the message to parse
+ * @returns an array of placeholder information
+ * @example
+ * parse('a message with a {value} and {count, plural, one {one} other {more}}.')
+ * returns an array:
+ * [ 'a message with a ',
+ *   [ 'value' ],
+ *   ' and ',
+ *   [ 'count', 'plural', 0, { one: [Array], other: [Array] } ],
+ *   '.'
+ * ]
+ * placeholders are always an array, so filter for array elements to find the placeholders
+ */
+const placeholders = (message: string): parse.Placeholder[] =>
+  // this will throw an error if the message is not valid ICU
+  // single quote (as in French l'année) messes up the parse and is not
+  // relevant for this check, so strip them out
+  parse(message.replace(/'/g, '')).filter(item => Array.isArray(item))
+
+const getMessageText = (m: TransifexStringKeyValueJson | TransifexStringChrome): string =>
+  typeof m === 'string' ? m : m.message
+
+/**
+ * @param a - one array of items
+ * @param b - another array of items
+ * @returns true if the two arrays contain the same items, without consideration of order and duplicates, judged by
+ * shallow equality.
+ * @example
+ * sameItems(['a', 'b'], ['a', 'b']) === true
+ * sameItems(['a', 'b'], ['b', 'a']) === true
+ * sameItems(['a', 'b'], ['b', 'a', 'b']) === true
+ * sameItems(['a', 'b'], ['a']) === false
+ */
+function sameItems<T>(a: T[], b: T[]): boolean {
+  if (!a.every(x => b.includes(x))) {
+    return false
+  }
+  if (!b.every(x => a.includes(x))) {
+    return false
+  }
+  return true
+}
+
+/**
+ * @param message - the translated message to validate
+ * @param source - the source string for this translated message
+ * @returns `false` if the message definitely has a problem, or `true` if the message might be OK.
+ */
+export const validMessage = (message: TransifexEditorString, source: TransifexEditorString): boolean => {
+  const msgText = getMessageText(message)
+  const srcText = getMessageText(source)
+
+  // Check ICU placeholder names (but not extended plural info)
+  const msgPlaceholderNamesICU = placeholders(msgText).map(x => x[0])
+  const srcPlaceholderNamesICU = placeholders(srcText).map(x => x[0])
+  if (!sameItems(msgPlaceholderNamesICU, srcPlaceholderNamesICU)) {
+    return false
+  }
+
+  // Check scratch-blocks numeric placeholders like '%1'
+  // TODO: apply this only for resources that use numeric placeholders.
+  // Otherwise, sentences with percentages can cause failures in some languages. Example: "göre %48'lik bir artış"
+  // const msgPlaceholdersNumeric: string[] = msgText.match(/%[0-9]+/g) ?? []
+  // const srcPlaceholdersNumeric: string[] = srcText.match(/%[0-9]+/g) ?? []
+  // if (!sameItems(msgPlaceholdersNumeric, srcPlaceholdersNumeric)) {
+  //   return false
+  // }
+
+  return true
+}
+
+/**
+ * @param translation - the translations to validate and their corresponding source strings
+ * @param translation.locale - the Transifex locale, for error reporting
+ * @param translation.translations - the translations to validate
+ * @param source - the source strings for the translations
+ */
+export const validateTranslations = (
+  { locale, translations }: { locale: string; translations: TransifexEditorStrings },
+  source: TransifexEditorStrings,
+) => {
+  const transKeys = Object.keys(translations)
+  const sourceKeys = Object.keys(source)
+  assert.strictEqual(transKeys.length, sourceKeys.length, `locale ${locale} has a different number of message keys`)
+  transKeys.forEach(item => assert(sourceKeys.includes(item), `locale ${locale} has key ${item} not in the source`))
+  sourceKeys.forEach(item => assert(transKeys.includes(item), `locale ${locale} is missing key ${item}`))
+  sourceKeys.forEach(item =>
+    assert(
+      validMessage(translations[item], source[item]),
+      `locale ${locale} / item ${item}: message validation failed:\n  msg: ${getMessageText(translations[item])}\n  src: ${getMessageText(source[item])}`,
+    ),
+  )
+}

package/scripts/tsconfig.json

@@ -0,0 +1,20 @@
+// tsconfig.json
+{
+  "compilerOptions": {
+    "target": "ESNext",
+    "module": "NodeNext",
+    "moduleResolution": "NodeNext",
+    "allowJs": true,
+    "checkJs": true,
+    "allowImportingTsExtensions": true,
+    "esModuleInterop": true,
+    "forceConsistentCasingInFileNames": true,
+    "strict": true,
+    "noEmit": true,
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "incremental": true
+  },
+  "include": ["**/*.js", "**/*.mjs", "**/*.ts", "**/*.mts"],
+  "exclude": ["node_modules"]
+}

package/scripts/tx-pull-editor.mts

@@ -0,0 +1,74 @@
+#!/usr/bin/env tsx
+/**
+ * @file
+ * Script to pull translations from transifex and generate the editor-msgs file.
+ * Expects that the project and resource have already been defined in Transifex, and that
+ * the person running the script has the the TX_TOKEN environment variable set to an api
+ * token that has developer access.
+ */
+import fs from 'fs'
+import path from 'path'
+import locales, { localeMap } from '../src/supported-locales.mjs'
+import { poolMap } from './lib/concurrent.mts'
+import { TransifexStringsKeyValueJson } from './lib/transifex-formats.mts'
+import { txPull } from './lib/transifex.mts'
+import { TransifexEditorStrings, validateTranslations } from './lib/validate.mts'
+
+const args = process.argv.slice(2)
+
+const usage = `
+ Pull supported language translations from Transifex. Usage:
+   node tx-pull-editor.js tx-project tx-resource path
+     tx-project: project on Transifex (e.g., scratch-editor)
+     tx-resource: resource within the project (e.g., interface)
+     path: where to put the downloaded json files
+   NOTE: TX_TOKEN environment variable needs to be set with a Transifex API token. See
+   the Localization page on the GUI wiki for information about setting up Transifex.
+ `
+
+// Fail immediately if the TX_TOKEN is not defined
+if (!process.env.TX_TOKEN || args.length < 3) {
+  process.stdout.write(usage)
+  process.exit(1)
+}
+
+// Globals
+const PROJECT = args[0]
+const RESOURCE = args[1]
+const OUTPUT_DIR = path.resolve(args[2])
+const MODE = 'reviewed'
+const CONCURRENCY_LIMIT = 36
+
+const getLocaleData = async function (locale: string) {
+  const txLocale = localeMap[locale] || locale
+  const data = (await txPull(PROJECT, RESOURCE, txLocale, MODE)) as TransifexEditorStrings
+  return {
+    locale: locale,
+    translations: data,
+  }
+}
+
+const pullTranslations = async function () {
+  const values = await poolMap(Object.keys(locales), CONCURRENCY_LIMIT, getLocaleData)
+  const source = values.find(elt => elt.locale === 'en')?.translations
+  if (!source) {
+    throw new Error('Could not find source strings')
+  }
+  values.forEach(translation => {
+    validateTranslations({ locale: translation.locale, translations: translation.translations }, source)
+    // if translation has message & description, we only want the message
+    const txs: TransifexStringsKeyValueJson = {}
+    for (const key of Object.keys(translation.translations)) {
+      const tx = translation.translations[key]
+      if (typeof tx === 'string') {
+        txs[key] = tx
+      } else {
+        txs[key] = tx.message
+      }
+    }
+    const file = JSON.stringify(txs, null, 4)
+    fs.writeFileSync(`${OUTPUT_DIR}/${translation.locale}.json`, file)
+  })
+}
+
+await pullTranslations()

package/scripts/{tx-pull-help-articles.js → tx-pull-help-articles.mts}

@@ -1,9 +1,9 @@
-#!/usr/bin/env node
-
+#!/usr/bin/env tsx
 /**
  * @file
  * Script to pull scratch-help translations from transifex and push to FreshDesk.
  */
+import { getInputs, saveItem, localizeFolder } from './lib/help-utils.mts'
 
 const args = process.argv.slice(2)
 const usage = `
@@ -21,14 +21,6 @@ if (!process.env.TX_TOKEN || !process.env.FRESHDESK_TOKEN || args.length > 0) {
   process.exit(1)
 }
 
-const { getInputs, saveItem, localizeFolder } = require('./help-utils.js')
-
-getInputs()
-  .then(([languages, folders]) => {
-    process.stdout.write('Processing articles pulled from Transifex\n')
-    return folders.map(item => saveItem(item, languages, localizeFolder))
-  })
-  .catch(e => {
-    process.stdout.write(`Error: ${e.message}\n`)
-    process.exitCode = 1 // not ok
-  })
+const { languages, folders } = await getInputs()
+console.log('Processing articles pulled from Transifex')
+await Promise.all(folders.map(item => saveItem(item, languages, localizeFolder)))

package/scripts/{tx-pull-help-names.js → tx-pull-help-names.mts}

@@ -1,9 +1,9 @@
-#!/usr/bin/env node
-
+#!/usr/bin/env tsx
 /**
  * @file
  * Script to pull scratch-help translations from transifex and push to FreshDesk.
  */
+import { getInputs, saveItem, localizeNames } from './lib/help-utils.mts'
 
 const args = process.argv.slice(2)
 
@@ -22,14 +22,6 @@ if (!process.env.TX_TOKEN || !process.env.FRESHDESK_TOKEN || args.length > 0) {
   process.exit(1)
 }
 
-const { getInputs, saveItem, localizeNames } = require('./help-utils.js')
-
-getInputs()
-  .then(([languages, , names]) => {
-    process.stdout.write('Process Category and Folder Names pulled from Transifex\n')
-    return names.map(item => saveItem(item, languages, localizeNames))
-  })
-  .catch(e => {
-    process.stdout.write(`Error: ${e.message}\n`)
-    process.exitCode = 1 // not ok
-  })
+const { languages, names } = await getInputs()
+console.log('Process Category and Folder Names pulled from Transifex')
+await Promise.all(names.map(item => saveItem(item, languages, localizeNames)))

package/scripts/{tx-pull-locale-articles.js → tx-pull-locale-articles.mts}

@@ -1,9 +1,9 @@
-#!/usr/bin/env node
-
+#!/usr/bin/env tsx
 /**
  * @file
  * Script to pull scratch-help translations from transifex and push to FreshDesk.
  */
+import { getInputs, saveItem, localizeFolder, debugFolder } from './lib/help-utils.mts'
 
 const args = process.argv.slice(2)
 const usage = `
@@ -21,8 +21,6 @@ if (!process.env.TX_TOKEN || !process.env.FRESHDESK_TOKEN || args.length === 0)
   process.exit(1)
 }
 
-const { getInputs, saveItem, localizeFolder, debugFolder } = require('./help-utils.js')
-
 let locale = args[0]
 let debug = false
 if (locale === '-d') {
@@ -31,12 +29,6 @@ if (locale === '-d') {
 }
 const saveFn = debug ? debugFolder : localizeFolder
 
-getInputs()
-  .then(([, folders]) => {
-    process.stdout.write('Processing articles pulled from Transifex\n')
-    return folders.map(item => saveItem(item, [locale], saveFn))
-  })
-  .catch(e => {
-    process.stdout.write(`Error: ${e.message}\n`)
-    process.exitCode = 1 // not ok
-  })
+const { folders } = await getInputs()
+console.log('Processing articles pulled from Transifex')
+await Promise.all(folders.map(item => saveItem(item, [locale], saveFn)))

package/scripts/{tx-pull-www.mjs → tx-pull-www.mts}

@@ -1,4 +1,4 @@
-#!/usr/bin/env node
+#!/usr/bin/env tsx
 /**
  * @file
  * Script to pull www translations from transifex for all resources.
@@ -7,20 +7,12 @@
  * token that has developer access.
  */
 import fs from 'fs/promises'
-import mkdirp from 'mkdirp'
+import { mkdirp } from 'mkdirp'
 import path from 'path'
-import { batchMap } from '../lib/batch.js'
-import { ProgressLogger } from '../lib/progress-logger.mjs'
-import { txPull, txResources } from '../lib/transifex.js'
 import locales, { localeMap } from '../src/supported-locales.mjs'
-
-/**
- * @file
- * Script to pull www translations from transifex for all resources.
- * Expects that the project and that the person running the script
- * has the the TX_TOKEN environment variable set to an api
- * token that has developer access.
- */
+import { poolMap } from './lib/concurrent.mts'
+import { ProgressLogger } from './lib/progress-logger.mjs'
+import { txPull, txResources } from './lib/transifex.mts'
 
 const args = process.argv.slice(2)
 
@@ -49,7 +41,7 @@ const CONCURRENCY_LIMIT = 36
 
 const lang = args.length === 2 ? args[1] : ''
 
-const getLocaleData = async function (item) {
+const getLocaleData = async function (item: { locale: string; resource: string }) {
   const locale = item.locale
   const resource = item.resource
   const txLocale = localeMap[locale] || locale
@@ -69,7 +61,7 @@ const getLocaleData = async function (item) {
       fileName,
     }
   } catch (e) {
-    e.cause = {
+    ;(e as Error).cause = {
       resource,
       locale,
       translations,
@@ -80,7 +72,7 @@ const getLocaleData = async function (item) {
   }
 }
 
-const expandResourceFiles = resources => {
+const expandResourceFiles = (resources: string[]) => {
   const items = []
   for (const resource of resources) {
     if (lang) {
@@ -100,18 +92,13 @@ const pullTranslations = async function () {
 
   const progress = new ProgressLogger(allFiles.length)
 
-  try {
-    await batchMap(allFiles, CONCURRENCY_LIMIT, async item => {
-      try {
-        await getLocaleData(item)
-      } finally {
-        progress.increment()
-      }
-    })
-  } catch (err) {
-    console.error(err)
-    process.exit(1)
-  }
+  await poolMap(allFiles, CONCURRENCY_LIMIT, async item => {
+    try {
+      await getLocaleData(item)
+    } finally {
+      progress.increment()
+    }
+  })
 }
 
-pullTranslations()
+await pullTranslations()