waybetter 0.1.0 → 0.6.0

package/cli/i18n.mjs

@@ -0,0 +1,382 @@
+// `waybetter i18n check|extract|rename`: an app's locale catalogs against the framework's rules
+// (waybetter/i18n/check: file shape, keys English does not have, ICU syntax, placeholder parity,
+// plural forms, no markup or control characters, length warnings), the files generated from them,
+// and the keys the app's code looks up.
+//
+//   waybetter i18n check              check every catalog (exit 1 on errors)
+//   waybetter i18n check --write      also regenerate languages.json and the native localization files
+//   waybetter i18n check --verbose    list every warning (untranslated and long messages)
+//   waybetter i18n check tr de        only these locales (plus English)
+//   waybetter i18n extract            keys the code looks up that English lacks (exit 1), then
+//                                     English keys no literal lookup names
+//   waybetter i18n extract --missing  only the missing keys
+//   waybetter i18n rename --map <file>         renames keys in every catalog ({ "old.key": "new.key" }), values carried
+//   waybetter i18n rename --map <file> --copy  the same, keeping the old keys too
+//
+// Config: waybetter.i18n.json in the app's root (or --config <file>); every path is relative to it.
+//
+//   {
+//     "locales": "locales",                      // <bcp47>.json, en.meta.json, the generated languages.json
+//     "framework": "node_modules/waybetter/locales",   // default: this package's catalogs
+//     "sources": ["src"],                        // t('key'), tx('key'), coreText('key'), i18n.t('key')
+//     "nativeSources": ["src-tauri/src"],        // tr(app, "native.key", ...), trf(...)
+//     "bundledPlugins": "src-tauri/bundled-plugins",   // their inline English must match en.json
+//     "runtimeKeys": ["plugin.", "native.app."], // prefixes of keys built at runtime or used by generated files
+//     "supported": ["en", "tr", "de"],           // the supported tier; every other catalog is "community" (default: all supported)
+//     "native": { "tauriDir": "src-tauri", "desktopFile": "linux/app.desktop" }
+//   }
+//
+// With "native", `--write` generates (and the plain check compares): macOS `Info.plist`
+// (CFBundleLocalizations) and `macos/<lang>.lproj/InfoPlist.strings` with `tauri.macos.conf.json`
+// listing them, and the Linux .desktop template with Name[xx]/Comment[xx] (`native.app.name`,
+// `native.app.comment`). The Windows installer's languages ship with this package
+// (windows/languages/*.nsh, generated from the framework catalogs).
+import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from 'node:fs';
+import { dirname, isAbsolute, join, relative, resolve } from 'node:path';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+
+export const I18N_CONFIG_FILE = 'waybetter.i18n.json';
+
+export const I18N_USAGE = `Usage:
+  waybetter i18n check [--write] [--verbose] [locale…] [--config <file>]
+      Checks every locale catalog against English and the framework's rules; with --write also
+      regenerates languages.json and the native localization files (macOS, Linux).
+  waybetter i18n extract [--missing] [--config <file>]
+      Keys the code looks up that English lacks, and English keys without a literal lookup.
+  waybetter i18n rename --map <file> [--copy] [--config <file>]
+      Renames keys in every catalog of the app (and en.meta.json) by a { "old.key": "new.key" } map,
+      carrying each language's value; --copy keeps the old keys as well.
+
+Config: ${I18N_CONFIG_FILE} in the app's root (locales, framework, sources, nativeSources,
+bundledPlugins, runtimeKeys, native).`;
+
+class I18nError extends Error {}
+
+const read = file => JSON.parse(readFileSync(file, 'utf8'));
+const CATALOG = /^[a-z]{2,3}(-[A-Za-z0-9]+)*\.json$/;
+const catalogFiles = dir => readdirSync(dir).filter(file => CATALOG.test(file) && file !== 'languages.json');
+const walk = (dir, pattern) => readdirSync(dir).flatMap(name => {
+  const path = join(dir, name);
+  return statSync(path).isDirectory() ? walk(path, pattern) : pattern.test(name) ? [path] : [];
+});
+
+/** The catalog checks: the framework's sources inside its repository, else the built package. */
+async function loadChecks() {
+  const source = fileURLToPath(new URL('../../runtime/src/i18nCheck.ts', import.meta.url));
+  const built = fileURLToPath(new URL('../dist/runtime/i18nCheck.js', import.meta.url));
+  const file = existsSync(source) ? source : built;
+  if (!existsSync(file)) throw new I18nError('the catalog checks are not built (run the package build)');
+  return import(pathToFileURL(file).href);
+}
+
+/** Reads and resolves the config (paths absolute). */
+export function loadI18nConfig(cwd, file) {
+  const path = resolve(cwd, file ?? I18N_CONFIG_FILE);
+  if (!existsSync(path)) throw new I18nError(`${relative(cwd, path) || path} not found (see \`waybetter i18n --help\`)`);
+  let raw;
+  try { raw = read(path); } catch (error) { throw new I18nError(`${path}: ${error.message}`); }
+  const root = dirname(path);
+  const at = value => (isAbsolute(value) ? value : join(root, value));
+  const list = value => (Array.isArray(value) ? value : value ? [value] : []);
+  if (typeof raw.locales !== 'string') throw new I18nError(`${I18N_CONFIG_FILE}: "locales" (the catalog folder) is required`);
+  const native = raw.native && typeof raw.native === 'object' ? raw.native : null;
+  if (native && typeof native.tauriDir !== 'string') throw new I18nError(`${I18N_CONFIG_FILE}: "native.tauriDir" is required`);
+  return {
+    root,
+    locales: at(raw.locales),
+    framework: raw.framework ? at(raw.framework) : fileURLToPath(new URL('../locales', import.meta.url)),
+    sources: list(raw.sources ?? 'src').map(at),
+    nativeSources: list(raw.nativeSources).map(at),
+    bundledPlugins: raw.bundledPlugins ? at(raw.bundledPlugins) : null,
+    runtimeKeys: list(raw.runtimeKeys ?? ['plugin.', 'native.app.']),
+    supported: Array.isArray(raw.supported) ? raw.supported : null,
+    native: native ? { tauriDir: at(native.tauriDir), desktopFile: native.desktopFile ?? null, macos: native.macos !== false } : null,
+  };
+}
+
+// ---- Native localization files ------------------------------------------------------------------
+
+/** Catalog locale -> Apple localization id (none for private-use tags). */
+const apple = code => (code.includes('-x-') ? null : code);
+const plistEscape = text => text.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
+const stringsEscape = text => text.replace(/\\/g, '\\\\').replace(/"/g, '\\"').replace(/\n/g, '\\n');
+const desktopEscape = text => text.replace(/\\/g, '\\\\').replace(/\n/g, '\\n').replace(/;/g, '\\;');
+
+/** `[path, content]` of every generated native file: `catalogs` are the app's, English first. */
+export function renderNative(config, en, catalogs) {
+  const { tauriDir, desktopFile, macos } = config.native;
+  const from = relative(config.root, config.locales).replace(/\\/g, '/');
+  const out = [];
+  if (macos) {
+    const codes = catalogs.map(catalog => apple(catalog.locale)).filter(Boolean);
+    out.push([join(tauriDir, 'Info.plist'), [
+      '<?xml version="1.0" encoding="UTF-8"?>',
+      '<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">',
+      `<!-- Generated by \`waybetter i18n check --write\` from ${from}. Merged into the app bundle by Tauri. -->`,
+      '<plist version="1.0">',
+      '<dict>',
+      '  <key>CFBundleDevelopmentRegion</key>',
+      '  <string>en</string>',
+      '  <key>CFBundleAllowMixedLocalizations</key>',
+      '  <true/>',
+      '  <key>CFBundleLocalizations</key>',
+      '  <array>',
+      ...codes.map(code => `    <string>${plistEscape(code)}</string>`),
+      '  </array>',
+      '</dict>',
+      '</plist>',
+      '',
+    ].join('\n')]);
+    const files = {};
+    for (const catalog of catalogs) {
+      const code = apple(catalog.locale);
+      if (!code) continue;
+      const name = catalog.messages['native.app.name'] ?? en.messages['native.app.name'];
+      if (name === undefined) throw new I18nError('en.json needs "native.app.name" (the app\'s name on macOS and Linux)');
+      const path = `macos/${code}.lproj/InfoPlist.strings`;
+      files[`Resources/${code}.lproj/InfoPlist.strings`] = path;
+      out.push([join(tauriDir, path), `/* Generated from ${from}/${catalog.locale}.json (\`waybetter i18n check --write\`). */\n"CFBundleDisplayName" = "${stringsEscape(name)}";\n"CFBundleName" = "${stringsEscape(name)}";\n`]);
+    }
+    out.push([join(tauriDir, 'tauri.macos.conf.json'), `${JSON.stringify({ bundle: { macOS: { files } } }, null, 2)}\n`]);
+  }
+  if (desktopFile) {
+    const desktop = [
+      '[Desktop Entry]',
+      `# Generated by \`waybetter i18n check --write\` from ${from} (a Tauri .desktop template).`,
+      'Categories={{categories}}',
+      '{{#if comment}}',
+      'Comment={{comment}}',
+      '{{/if}}',
+    ];
+    for (const catalog of catalogs) {
+      if (catalog.locale === 'en' || !catalog.messages['native.app.comment']) continue;
+      desktop.push(`Comment[${catalog.locale.replace('-', '_')}]=${desktopEscape(catalog.messages['native.app.comment'])}`);
+    }
+    desktop.push('Exec={{exec}}', 'Icon={{icon}}', 'Name={{name}}');
+    for (const catalog of catalogs) {
+      if (catalog.locale === 'en' || !catalog.messages['native.app.name'] || catalog.messages['native.app.name'] === en.messages['native.app.name']) continue;
+      desktop.push(`Name[${catalog.locale.replace('-', '_')}]=${desktopEscape(catalog.messages['native.app.name'])}`);
+    }
+    desktop.push('Terminal=false', 'Type=Application', '{{#if mime_type}}', 'MimeType={{mime_type}}', '{{/if}}', '');
+    out.push([join(tauriDir, desktopFile), desktop.join('\n')]);
+  }
+  return out;
+}
+
+// ---- check ------------------------------------------------------------------------------------------
+
+/**
+ * Checks the catalogs. Returns `{ errors, warnings, lines }` (`lines`: what a terminal shows);
+ * with `write`, out-of-date generated files are rewritten instead of reported.
+ */
+export async function checkCatalogs(config, { write = false, verbose = false, only = [] } = {}) {
+  const { checkCatalog, checkShape, checkSource, coverage } = await loadChecks();
+  const lines = [];
+  let errors = 0;
+  let warnings = 0;
+  const label = path => relative(config.root, path).replace(/\\/g, '/');
+  const report = (issue, where) => {
+    const line = `${issue.level === 'error' ? 'ERROR' : 'warn '} ${where}${issue.key ? ` ${issue.key}` : ''}: ${issue.message}`;
+    if (issue.level === 'error') { errors += 1; lines.push(line); }
+    else { warnings += 1; if (verbose) lines.push(line); }
+  };
+  const metaOf = dir => {
+    if (!existsSync(join(dir, 'en.meta.json'))) return {};
+    const meta = read(join(dir, 'en.meta.json'));
+    delete meta.$schema;
+    return meta;
+  };
+  const APP = config.locales;
+  const KIT = config.framework;
+
+  // ---- Source (English)
+  const en = read(join(APP, 'en.json'));
+  const meta = metaOf(APP);
+  const kitEn = read(join(KIT, 'en.json'));
+  const kitMeta = metaOf(KIT);
+  for (const issue of [...checkShape(en, 'en.json'), ...checkSource(en, meta)]) report(issue, label(join(APP, 'en.json')));
+  for (const issue of [...checkShape(kitEn, 'en.json'), ...checkSource(kitEn, kitMeta)]) report(issue, label(join(KIT, 'en.json')));
+  for (const key of Object.keys(en.messages).filter(name => name in kitEn.messages)) {
+    report({ level: 'error', key, message: 'the framework catalog already defines this key' }, label(join(APP, 'en.json')));
+  }
+
+  // Bundled plugins ship their English inline (so a copied file still works); it must match en.json.
+  if (config.bundledPlugins) {
+    for (const file of readdirSync(config.bundledPlugins).filter(name => name.endsWith('.plugin.js'))) {
+      const ns = file.replace(/\.plugin\.js$/, '');
+      const text = readFileSync(join(config.bundledPlugins, file), 'utf8');
+      const block = /\/\/ i18n:begin\r?\n([\s\S]*?)\/\/ i18n:end/.exec(text)?.[1];
+      const hosted = Object.fromEntries(Object.entries(en.messages).filter(([key]) => key.startsWith(`plugin.${ns}.`)).map(([key, value]) => [key.slice(`plugin.${ns}.`.length), value]));
+      if (!block) {
+        if (Object.keys(hosted).some(key => key !== 'name' && key !== 'description')) report({ level: 'error', message: 'en.json has strings for this plugin but the file has no // i18n:begin … // i18n:end block' }, file);
+        continue;
+      }
+      let inline;
+      try { inline = JSON.parse(/const MESSAGES = ({[\s\S]*});/.exec(block)?.[1] ?? 'null'); } catch (error) { inline = null; report({ level: 'error', message: `MESSAGES is not plain JSON: ${error.message}` }, file); }
+      if (!inline?.en) { report({ level: 'error', message: 'MESSAGES.en is missing' }, file); continue; }
+      for (const [key, value] of Object.entries(inline.en)) {
+        if (hosted[key] !== value) report({ level: 'error', key: `plugin.${ns}.${key}`, message: hosted[key] === undefined ? 'missing from en.json' : 'differs from en.json' }, file);
+      }
+      for (const key of Object.keys(hosted)) {
+        if (key !== 'name' && key !== 'description' && !(key in inline.en)) report({ level: 'error', key: `plugin.${ns}.${key}`, message: 'in en.json but not in the plugin' }, file);
+      }
+    }
+  }
+
+  // ---- Translations
+  const appFiles = catalogFiles(APP).filter(file => file !== 'en.json');
+  const kitFiles = new Set(catalogFiles(KIT));
+  // Tier: "supported" languages are the ones the app resolves to automatically ("Same as system");
+  // every other catalog is "community" (offered, but only when someone picks it). No list = all supported.
+  const supported = config.supported ? new Set(['en', ...config.supported]) : null;
+  const tierOf = code => (!supported || supported.has(code) ? 'supported' : 'community');
+  for (const code of supported ?? []) if (code !== 'en' && !appFiles.includes(`${code}.json`)) report({ level: 'error', message: `"supported" lists ${code}, which has no catalog` }, I18N_CONFIG_FILE);
+  const languages = [{ code: 'en', name: en.name, englishName: en.englishName, dir: 'ltr', status: 'source', tier: 'supported', reviewers: [], coverage: 1 }];
+  const table = [];
+  for (const file of appFiles) {
+    const catalog = read(join(APP, file));
+    if (!kitFiles.has(file)) report({ level: 'error', message: `no framework catalog ${label(join(KIT, file))}` }, file);
+    const kit = kitFiles.has(file) ? read(join(KIT, file)) : null;
+    const fallback = catalog.fallback ? (catalog.fallback === 'en' ? null : read(join(APP, `${catalog.fallback}.json`))) : null;
+    const cover = coverage(catalog, en.messages, fallback ?? undefined);
+    languages.push({
+      code: catalog.locale, name: catalog.name, englishName: catalog.englishName, dir: catalog.dir ?? 'ltr', status: catalog.status, tier: tierOf(catalog.locale),
+      reviewers: catalog.reviewers ?? [], ...(catalog.fallback ? { fallback: catalog.fallback } : {}), coverage: Math.round((catalog.fallback === 'en' ? 1 : cover) * 1000) / 1000,
+    });
+    if (kit && (kit.status !== catalog.status || kit.name !== catalog.name)) report({ level: 'error', message: 'name/status differ from the framework catalog of the same locale' }, file);
+    if (only.length && !only.includes(catalog.locale)) continue;
+    const shape = checkShape(catalog, file);
+    for (const issue of [...shape, ...(shape.length ? [] : checkCatalog(catalog, en.messages, meta))]) report(issue, label(join(APP, file)));
+    // The framework's words in the same language (the shell frame, the panel, the Store, the UI kit).
+    if (kit) {
+      const kitShape = checkShape(kit, file);
+      for (const issue of [...kitShape, ...(kitShape.length ? [] : checkCatalog(kit, kitEn.messages, kitMeta))]) report(issue, label(join(KIT, file)));
+    }
+    table.push(`${catalog.locale.padEnd(8)} ${catalog.status.padEnd(9)} ${(cover * 100).toFixed(1).padStart(5)}%  ${catalog.englishName}`);
+  }
+  languages.sort((a, b) => (a.code === 'en' ? -1 : b.code === 'en' ? 1 : a.englishName.localeCompare(b.englishName, 'en')));
+
+  // ---- Generated files
+  const outputs = [[join(APP, 'languages.json'), `${JSON.stringify(languages, null, 2)}\n`]];
+  if (config.native) outputs.push(...renderNative(config, en, languages.map(language => (language.code === 'en' ? en : read(join(APP, `${language.code}.json`))))));
+  for (const [path, content] of outputs) {
+    let current = null;
+    try { current = readFileSync(path, 'utf8'); } catch { /* missing */ }
+    if (current === content) continue;
+    if (write) { mkdirSync(dirname(path), { recursive: true }); writeFileSync(path, content); lines.push(`wrote ${label(path)}`); }
+    else report({ level: 'error', message: 'out of date: run `waybetter i18n check --write`' }, label(path));
+  }
+
+  lines.push(table.sort().join('\n'));
+  lines.push(`\ni18n check: ${Object.keys(en.messages).length} app + ${Object.keys(kitEn.messages).length} framework messages, ${languages.length} languages; ${errors} error(s), ${warnings} warning(s)${verbose ? '' : ' (--verbose lists them)'}.`);
+  return { errors, warnings, lines, languages };
+}
+
+// ---- extract ----------------------------------------------------------------------------------------
+
+/**
+ * Compares the keys the code looks up with English. Literal lookups are `t('key')`, `tx('key')`,
+ * `coreText('key')` and `i18n.t('key')` in `sources`, and `tr(app, "native.key", …)` / `trf(…)` in
+ * `nativeSources`. Keys built at runtime (`features.${id}.label`) are invisible here, so "no
+ * literal lookup" is a hint, never a reason to delete: search first.
+ */
+export function extractKeys(config, { onlyMissing = false } = {}) {
+  const messages = { ...read(join(config.framework, 'en.json')).messages, ...read(join(config.locales, 'en.json')).messages };
+  const used = new Map();
+  const note = (key, file) => { if (!used.has(key)) used.set(key, relative(config.root, file).replace(/\\/g, '/')); };
+  const call = /\b(?:t|tx|coreText|i18n\.t|i18n\.tx)\(\s*(['"])([a-z][A-Za-z0-9_.-]*)\1/g;
+  const native = /\btrf?\(\s*&?app,\s*"([a-z][A-Za-z0-9_.-]*)"/g;
+  for (const dir of config.sources) {
+    for (const file of walk(dir, /\.(ts|tsx|js|jsx|mjs)$/)) for (const match of readFileSync(file, 'utf8').matchAll(call)) note(match[2], file);
+  }
+  for (const dir of config.nativeSources) {
+    for (const file of walk(dir, /\.rs$/)) for (const match of readFileSync(file, 'utf8').matchAll(native)) note(match[1], file);
+  }
+  const lines = [];
+  const missing = [...used].filter(([key]) => !(key in messages));
+  for (const [key, file] of missing) lines.push(`missing  ${key}  (${file})`);
+  if (onlyMissing) {
+    lines.push(`${missing.length} missing`);
+  } else {
+    // The framework's own words are looked up by the framework (its tests check them).
+    const app = read(join(config.locales, 'en.json')).messages;
+    const unused = Object.keys(app).filter(key => !used.has(key) && !config.runtimeKeys.some(prefix => key.startsWith(prefix)));
+    for (const key of unused) lines.push(`no literal lookup  ${key}`);
+    lines.push(`\n${used.size} keys looked up literally, ${missing.length} missing from en.json, ${unused.length} en.json keys without a literal lookup (many are built at runtime).`);
+  }
+  return { missing: missing.map(([key]) => key), lines };
+}
+
+// ---- rename -----------------------------------------------------------------------------------------
+
+const KEY = /^[a-z][A-Za-z0-9_.-]{0,127}$/;
+
+/**
+ * Renames (or, with `copy`, duplicates) keys in every catalog of the app by `map` ({ old: new }),
+ * carrying each language's own value, so a moved string is never retranslated. A catalog that lacks
+ * the old key is left as it is for that key (it falls back to English, as before). The translator
+ * notes (en.meta.json) follow. Refuses a map that would overwrite an existing English key with a
+ * different value, or that names a key English does not have.
+ */
+export function renameKeys(config, map, { copy = false } = {}) {
+  const entries = Object.entries(map);
+  const en = read(join(config.locales, 'en.json'));
+  for (const [from, to] of entries) {
+    if (typeof to !== 'string' || !KEY.test(from) || !KEY.test(to)) throw new I18nError(`invalid entry ${JSON.stringify(from)}: ${JSON.stringify(to)}`);
+    if (!(from in en.messages)) throw new I18nError(`${from} is not in en.json`);
+    if (to in en.messages && en.messages[to] !== en.messages[from]) throw new I18nError(`${to} already exists in en.json with another value`);
+  }
+  const lines = [];
+  const rewrite = (file, messagesOf) => {
+    const path = join(config.locales, file);
+    if (!existsSync(path)) return;
+    const raw = readFileSync(path, 'utf8');
+    const data = JSON.parse(raw);
+    const messages = messagesOf(data);
+    let changed = 0;
+    for (const [from, to] of entries) {
+      if (!(from in messages)) continue;
+      if (messages[to] !== messages[from]) { messages[to] = messages[from]; changed += 1; }
+      if (!copy && from !== to) { delete messages[from]; changed += 1; }
+    }
+    if (!changed) return;
+    writeFileSync(path, `${JSON.stringify(data, null, 2)}${raw.endsWith('\n') ? '\n' : ''}`);
+    lines.push(`${file}: ${changed} change(s)`);
+  };
+  for (const file of catalogFiles(config.locales)) rewrite(file, data => data.messages);
+  rewrite('en.meta.json', data => data);
+  lines.push(`${copy ? 'copied' : 'renamed'} ${entries.length} key(s) in ${lines.length} file(s).`);
+  return { lines };
+}
+
+// ---- The command ------------------------------------------------------------------------------------
+
+/** `waybetter i18n <check|extract|rename> …`; returns the exit code. */
+export async function i18nCommand(args, { cwd = process.cwd(), log = console.log } = {}) {
+  const [action, ...rest] = args;
+  const flags = new Set(rest.filter(arg => arg.startsWith('--')));
+  const configAt = rest.indexOf('--config');
+  const configFile = configAt >= 0 ? rest[configAt + 1] : undefined;
+  if (configAt >= 0 && !configFile) throw new I18nError('--config needs a file');
+  const mapAt = rest.indexOf('--map');
+  const positional = rest.filter((arg, index) => !arg.startsWith('--') && (configAt < 0 || index !== configAt + 1) && (mapAt < 0 || index !== mapAt + 1));
+  if (action === 'rename') {
+    const mapFile = mapAt >= 0 ? rest[mapAt + 1] : undefined;
+    if (!mapFile) throw new I18nError('rename needs --map <file>');
+    const result = renameKeys(loadI18nConfig(cwd, configFile), read(isAbsolute(mapFile) ? mapFile : resolve(cwd, mapFile)), { copy: flags.has('--copy') });
+    for (const line of result.lines) log(line);
+    return 0;
+  }
+  if (action === 'check') {
+    const result = await checkCatalogs(loadI18nConfig(cwd, configFile), { write: flags.has('--write'), verbose: flags.has('--verbose'), only: positional });
+    for (const line of result.lines) log(line);
+    return result.errors ? 1 : 0;
+  }
+  if (action === 'extract') {
+    const result = extractKeys(loadI18nConfig(cwd, configFile), { onlyMissing: flags.has('--missing') });
+    for (const line of result.lines) log(line);
+    return result.missing.length ? 1 : 0;
+  }
+  log(I18N_USAGE);
+  return action && action !== '--help' && action !== '-h' ? 1 : 0;
+}