scratch-l10n 5.0.308 → 6.0.0

package/scripts/lib/help-utils.mts

@@ -0,0 +1,221 @@
+/**
+ * @file
+ * Helper functions for syncing Freshdesk knowledge base articles with Transifex
+ */
+import { promises as fsPromises } from 'fs'
+import { mkdirp } from 'mkdirp'
+import FreshdeskApi, { FreshdeskArticleCreate, FreshdeskArticleStatus } from './freshdesk-api.mts'
+import { TransifexStringKeyValueJson, TransifexStringsKeyValueJson, TransifexStrings } from './transifex-formats.mts'
+import { TransifexResourceObject } from './transifex-objects.mts'
+import { txPull, txResourcesObjects, txAvailableLanguages } from './transifex.mts'
+
+const FD = new FreshdeskApi('https://mitscratch.freshdesk.com', process.env.FRESHDESK_TOKEN ?? '')
+const TX_PROJECT = 'scratch-help'
+
+const freshdeskLocale = (locale: string): string => {
+  // map between Transifex locale and Freshdesk. Two letter codes are usually fine
+  const localeMap: Record<string, string> = {
+    es_419: 'es-LA',
+    ja: 'ja-JP',
+    'ja-Hira': 'ja-JP',
+    lv: 'lv-LV',
+    nb: 'nb-NO',
+    nn: 'nb-NO',
+    pt: 'pt-PT',
+    pt_BR: 'pt-BR',
+    ru: 'ru-RU',
+    sv: 'sv-SE',
+    zh_CN: 'zh-CN',
+    zh_TW: 'zh-TW',
+  }
+  return localeMap[locale] || locale
+}
+
+/**
+ * Parse a string into an integer.
+ * If converting the integer back to a string does not result in the same string, throw.
+ * @param str - The (allegedly) numeric string to parse
+ * @param radix - Interpret the string as a number in this base. For example, use 10 for decimal values.
+ * @returns The numeric value of the string
+ */
+const parseIntOrThrow = (str: string, radix: number) => {
+  const num = parseInt(str, radix)
+  if (str != num.toString(radix)) {
+    throw new Error(`Could not parse int safely: ${str}`)
+  }
+  return num
+}
+
+/**
+ * Pull metadata from Transifex for the scratch-help project
+ * @returns Promise for a results object containing:
+ * languages - array of supported languages
+ * folders - array of tx resources corresponding to Freshdesk folders
+ * names - array of tx resources corresponding to the Freshdesk metadata
+ */
+export const getInputs = async () => {
+  const resourcesPromise = txResourcesObjects(TX_PROJECT)
+  const languagesPromise = txAvailableLanguages(TX_PROJECT)
+
+  // there are three types of resources differentiated by the file type
+  const foldersPromise = resourcesPromise.then(resources =>
+    resources.filter(resource => resource.attributes.i18n_type === 'STRUCTURED_JSON'),
+  )
+  const namesPromise = resourcesPromise.then(resources =>
+    resources.filter(resource => resource.attributes.i18n_type === 'KEYVALUEJSON'),
+  )
+  // ignore the yaml type because it's not possible to update via API
+
+  const [languages, folders, names] = await Promise.all([languagesPromise, foldersPromise, namesPromise])
+
+  return {
+    languages,
+    folders,
+    names,
+  }
+}
+
+/**
+ * internal function to serialize saving category and folder name translations to avoid Freshdesk rate limit
+ * @param strings - the string data pulled from Transifex
+ * @param resource - the `attributes` property of the resource object which contains these strings
+ * @param locale - the Transifex locale code corresponding to these strings
+ */
+const serializeNameSave = async (
+  strings: TransifexStringsKeyValueJson,
+  resource: TransifexResourceObject,
+  locale: string,
+): Promise<void> => {
+  for (const [key, value] of Object.entries(strings)) {
+    // key is of the form <name>_<id>
+    const words = key.split('_')
+    const id = parseIntOrThrow(words[words.length - 1], 10)
+    let status
+    if (resource.attributes.name === 'categoryNames_json') {
+      status = await FD.updateCategoryTranslation(id, freshdeskLocale(locale), { name: value })
+    }
+    if (resource.attributes.name === 'folderNames_json') {
+      status = await FD.updateFolderTranslation(id, freshdeskLocale(locale), { name: value })
+    }
+    if (status === -1) {
+      process.exitCode = 1
+    }
+  }
+}
+
+/**
+ * We use this specific structure in the `STRUCTUREDJSON` resources associated with our Freshdesk folders.
+ * This should be compatible with (and stricter than) `TransifexStringStructuredJson`.
+ */
+interface FreshdeskFolderInTransifex {
+  title: { string: string }
+  description: { string: string }
+  tags: { string: string }
+}
+
+/**
+ * Internal function serialize Freshdesk requests to avoid getting rate limited
+ * @param  json   object with keys corresponding to article ids
+ * @param  locale language code
+ * @returns a numeric status code
+ */
+const serializeFolderSave = async (json: TransifexStrings<FreshdeskFolderInTransifex>, locale: string) => {
+  for (const [idString, value] of Object.entries(json)) {
+    const id = parseIntOrThrow(idString, 10)
+    const body: FreshdeskArticleCreate = {
+      title: value.title.string,
+      description: value.description.string,
+      status: FreshdeskArticleStatus.published,
+    }
+    if (Object.prototype.hasOwnProperty.call(value, 'tags')) {
+      const tags = value.tags.string.split(',')
+      const validTags = tags.filter(tag => tag.length < 33)
+      if (validTags.length !== tags.length) {
+        process.stdout.write(`Warning: tags too long in ${id} for ${locale}\n`)
+      }
+      body.tags = validTags
+    }
+    const status = await FD.updateArticleTranslation(id, freshdeskLocale(locale), body)
+    if (status === -1) {
+      // eslint-disable-next-line require-atomic-updates -- `process` will not change across `await`
+      process.exitCode = 1
+    }
+  }
+}
+
+/**
+ * Process Transifex resource corresponding to a Knowledge base folder on Freshdesk
+ * @param  folderAttributes Transifex resource json corresponding to a KB folder
+ * @param  locale locale to pull and submit to Freshdesk
+ */
+export const localizeFolder = async (folderAttributes: TransifexResourceObject, locale: string) => {
+  try {
+    const data = await txPull<FreshdeskFolderInTransifex>(
+      TX_PROJECT,
+      folderAttributes.attributes.slug,
+      locale,
+      'default',
+    )
+    await serializeFolderSave(data, locale)
+  } catch (e) {
+    process.stdout.write(`Error processing ${folderAttributes.attributes.slug}, ${locale}: ${(e as Error).message}\n`)
+    process.exitCode = 1 // not ok
+  }
+}
+
+/**
+ * Save Transifex resource corresponding to a Knowledge base folder locally for debugging
+ * @param  folderAttributes Transifex resource json corresponding to a KB folder
+ * @param  locale locale to pull and save
+ */
+export const debugFolder = async (folderAttributes: TransifexResourceObject, locale: string) => {
+  await mkdirp('tmpDebug')
+  await txPull(TX_PROJECT, folderAttributes.attributes.slug, locale, 'default')
+    .then(data =>
+      fsPromises.writeFile(
+        `tmpDebug/${folderAttributes.attributes.slug}_${locale}.json`,
+        JSON.stringify(data, null, 2),
+      ),
+    )
+    .catch(e => {
+      process.stdout.write(
+        `Error processing ${folderAttributes.attributes.slug}, ${locale}: ${(e as Error).message}\n`,
+      )
+      process.exitCode = 1 // not ok
+    })
+}
+
+/**
+ * Process KEYVALUEJSON resources from scratch-help on transifex
+ * Category and Folder names are stored as plain json
+ * @param resource Transifex resource json for either CategoryNames or FolderNames
+ * @param locale   locale to pull and submit to Freshdesk
+ */
+export const localizeNames = async (resource: TransifexResourceObject, locale: string): Promise<void> => {
+  await txPull<TransifexStringKeyValueJson>(TX_PROJECT, resource.attributes.slug, locale, 'default')
+    .then(data => serializeNameSave(data, resource, locale))
+    .catch(e => {
+      process.stdout.write(`Error saving ${resource.attributes.slug}, ${locale}: ${(e as Error).message}\n`)
+      process.exitCode = 1 // not ok
+    })
+}
+
+const BATCH_SIZE = 2
+
+type SaveFn = (item: TransifexResourceObject, language: string) => Promise<void>
+
+/**
+ * save resource items in batches to reduce rate limiting errors
+ * @param item      Transifex resource json, used for 'slug'
+ * @param languages  Array of languages to save
+ * @param saveFn  Async function to use to save the item
+ */
+export const saveItem = async (item: TransifexResourceObject, languages: string[], saveFn: SaveFn) => {
+  const saveLanguages = languages.filter(l => l !== 'en') // exclude English from update
+  for (let i = 0; i < saveLanguages.length; i += BATCH_SIZE) {
+    await Promise.all(saveLanguages.slice(i, i + BATCH_SIZE).map(l => saveFn(item, l))).catch(err => {
+      process.stdout.write(`Error saving item:${(err as Error).message}\n${JSON.stringify(item, null, 2)}\n`)
+      process.exitCode = 1 // not ok
+    })
+  }
+}

package/{lib/progress-logger.mjs → scripts/lib/progress-logger.mts}

@@ -2,19 +2,24 @@
  * Helper class to log progress.
  */
 export class ProgressLogger {
+  total?: number
+  completed: number
+  percent?: number
+
   /**
-   * @param {number} [total] Optional: expected total number of items to process.
+   * @param [total] - Optional: expected total number of items to process.
    */
-  constructor(total) {
+  constructor(total?: number) {
     this.total = total
     this.completed = 0
+    this.percent = 0
   }
 
   /**
    * Set the expected total number of items to process.
-   * @param {number} total Total number of items to process.
+   * @param total - Total number of items to process.
    */
-  setTotal(total) {
+  setTotal(total: number) {
     if (this.total !== total) {
       this.total = total
       delete this.percent
@@ -25,7 +30,7 @@ export class ProgressLogger {
    * Increment the number of items processed and log progress.
    * If a total is set, progress is logged as a percentage and only when the percentage changes.
    * If no total is set, progress is logged as a count.
-   * @param {number} [count] Number of items processed.
+   * @param [count] - Number of items processed.
    */
   increment(count = 1) {
     this.completed += count

package/scripts/lib/transifex-formats.mts

@@ -0,0 +1,53 @@
+/**
+ * A set of strings from a Transifex resource.
+ */
+export type TransifexStrings<T> = Record<string, T>
+
+/**
+ * A single string from a resource that uses the "CHROME" file type.
+ * Scratch: used for most (but not all) Scratch Editor resources.
+ */
+export interface TransifexStringChrome {
+  /** The source or translation text */
+  message: string
+  /** Description or context information for translators */
+  description?: string
+}
+
+/**
+ * A set of strings from a resource that uses the "CHROME" file type.
+ * Scratch: used for most (but not all) Scratch Editor resources.
+ */
+export type TransifexStringsChrome = TransifexStrings<TransifexStringChrome>
+
+/**
+ * A single string from a resource that uses the "KEYVALUEJSON" file type.
+ * Scratch: used for the Scratch Website project, the Scratch Editor blocks resource, and some Scratch Help resources.
+ */
+export type TransifexStringKeyValueJson = string
+
+/**
+ * A set of strings from a resource that uses the "KEYVALUEJSON" file type.
+ * Scratch: used for the Scratch Website project, the Scratch Editor blocks resource, and some Scratch Help resources.
+ */
+export type TransifexStringsKeyValueJson = TransifexStrings<TransifexStringKeyValueJson>
+
+/**
+ * A single string from a resource that uses the "STRUCTUREDJSON" file type.
+ * Scratch: used for most (but not all) Scratch Help resources.
+ */
+export interface TransifexStringStructuredJson {
+  string?: string
+  context?: string
+  developer_comment?: string
+  character_limit?: number
+  plurals?: object
+
+  [key: string]: TransifexStringStructuredJson | string | number | object | undefined
+}
+
+/**
+ * A set of strings from a resource that uses the "STRUCTUREDJSON" file type.
+ * Scratch: used for most (but not all) Scratch Help resources.
+ */
+export type TransifexStringsStructuredJson = TransifexStrings<TransifexStringStructuredJson>

package/scripts/lib/transifex-objects.mts

@@ -0,0 +1,143 @@
+import { JsonApiResource } from '@transifex/api'
+
+// Writing these types is very manual, so I've only written the ones used by our scripts. The types are adapted from
+// the documentation of the Transifex API, usually from the description of the `data` field in a 200 response to some
+// "list" or "get details" kind of call.
+
+/**
+ * Properties common to all Transifex API objects. I chose the generic term "Object" for what JSON API calls a
+ * "Resource" since "Resource" has a specific meaning in a Transifex context.
+ */
+export interface TransifexObject extends JsonApiResource {
+  /** The type of object. Use this to determine the specific type to use for this object. */
+  type: string
+  /** Unique identifier for this object. */
+  id: string
+  /** The attributes of this object. */
+  attributes: Record<string, unknown>
+  /** The relationships of this object. */
+  relationships: Record<string, unknown>
+  /** The URL links of the object. */
+  links: Record<string, string>
+}
+
+/**
+ * Transifex object representing a Language.
+ * @see https://developers.transifex.com/reference/get_languages-language-id
+ */
+export interface TransifexLanguageObject extends TransifexObject {
+  /** The type of the resource.*/
+  type: 'languages'
+  /** Language identifier. Example: `l:en_US` */
+  id: `l:${string}`
+  attributes: {
+    /** The language code as defined in CLDR. Example: `en`. */
+    code: string
+    /** The name of the language as defined in CLDR. Example: `English`. */
+    name: string
+    /** Whether the language is right-to-left. */
+    rtl: boolean
+    /** The language plural rule equation as defined in CLDR. Example: `(n != 1)`. */
+    plural_equation: string
+    /** Object of plural rules for Language as defined in CLDR. */
+    plural_rules: object
+  }
+}
+
+/**
+ * Transifex object representing a Project.
+ * @see https://developers.transifex.com/reference/get_projects-project-id
+ */
+export interface TransifexProjectObject extends TransifexObject {
+  /** The type of the resource.*/
+  type: 'projects'
+  /** Project identifier. Example: `o:org_slug:p:project_slug` */
+  id: `o:${string}:p:${string}`
+  attributes: {
+    /**
+     * If the project is archived or not.
+     * If a project is archived the pricing will be lower but no action will be available.
+     */
+    archived: boolean
+    /** The date and time the project was created. */
+    datetime_created: string
+    /** The date and time the project was last modified. */
+    datetime_modified: string
+    /** A description of the project. */
+    description: string
+    /** The homepage of the project. */
+    homepage_url: string
+    /**
+     * A web page containing documentation or instructions for translators, or localization tips for your
+     * community.
+     */
+    instructions_url: string
+    /** The license of the project. */
+    license: string
+    /** The URL of the project's logo. */
+    logo_url: string
+    /** A long description of the project. */
+    long_description: string
+    /** If the resources of the project will be filled up from a machine translation. */
+    machine_translation_fillup: boolean
+    /** The name of the project. */
+    name: string
+    /** Whether the project is private. A private project is visible only by you and your team. */
+    private: boolean
+    /** The URL of the public source code repository. */
+    repository_url: string
+    /** The slug of the project. Example: `project_slug`. */
+    slug: string
+    /** List of tags for the project. */
+    tags: string[]
+    /** If the resources of the project will be filled up from common translation memory. */
+    translation_memory_fillup: boolean
+    /** The type of the project. */
+    type: 'live' | 'file'
+  }
+}
+
+/**
+ * Transifex object representing a Resource.
+ * @see https://developers.transifex.com/reference/get_resources-resource-id
+ */
+export interface TransifexResourceObject extends TransifexObject {
+  /** The type of the resource.*/
+  type: 'resources'
+  /** Resource identifier. Example: `o:org_slug:p:project_slug:r:resource_slug` */
+  id: `o:${string}:p:${string}:r:${string}`
+  attributes: {
+    /** The slug of the resource. Example: `resource_slug`. */
+    slug: string
+    /** The name of the resource. */
+    name: string
+    /** The priority of the resource. */
+    priority: 'normal' | 'high' | 'urgent'
+    /** The format of the resource. Example: `STRUCTURED_JSON`. */
+    i18n_type: string
+    /** File format type version. */
+    i18n_version: number
+    /** Whether the resource should accept translations or not. */
+    accept_translations: boolean
+    /** The number of strings in the resource content. */
+    string_count: number
+    /** The number of words in the resource content. */
+    word_count: string
+    /** The date and time the resource was created. */
+    datetime_created: string
+    /** The date and time the resource was last modified. */
+    datetime_modified: string
+    /** List of categories to associate similar resources. */
+    categories: string[]
+    /** Options that determine how the resource will be parsed and compiled. */
+    i18n_options: Record<string, unknown>
+    /** A (public) URL to provide an MP4 video file for subtitle translation. */
+    mp4_url: string
+    /** A (public) URL to provide an OGG video file for subtitle translation. */
+    ogg_url: string
+    /** A (public) URL to provide a YouTube video file for subtitle translation. */
+    youtube_url: string
+    /** A (public) URL to provide a WEBM video file for subtitle translation. */
+    webm_url: string
+  }
+}