prosebook 0.3.0 → 0.5.0

package/src/tools/bundle.ts

@@ -7,12 +7,13 @@
  * so they read the source under `scan.dirs` first (`scanSources`) and load the locale already pruned.
  */
 import { existsSync, readFileSync, watch } from 'node:fs';
-import { relative } from 'node:path';
+import { join, relative } from 'node:path';
 import { fileURLToPath } from 'node:url';
 
 import { compileLocale, localeModule, type Entry } from './compile.ts';
 import type { ResolvedConfig } from './config.ts';
-import { isReferenced, keyRefs, READERS, sourceFiles, unseenReads, type KeyRefs, type Reads, type UnseenRead } from './scan.ts';
+import type { Generated } from './generate.ts';
+import { isReferenced, keyRefs, READERS, SFC, sourceFiles, unseenReads, type KeyRefs, type Reads, type UnseenRead } from './scan.ts';
 
 /** How a build prunes: `strict` also fails on a read whose key it can't see. */
 export type Prune = 'strict' | 'loose' | false;
@@ -77,6 +78,146 @@ export function watchCatalog(config: ResolvedConfig, change: () => void): () =>
   };
 }
 
+/** `virtual:prosebook/area/cart`, and in a build `…/cart/de`, a lazy locale's part of the group `cart` leads. */
+export const AREA = 'virtual:prosebook/area/';
+
+/**
+ * How a split catalog's modules are imported: an area (`cart`), a lazy locale's part of a group of
+ * them, by the group's first area (`cart`, `de`), and a generated module, from its path. Vite's and Bun's are virtual modules (`VIRTUAL`);
+ * Turbopack has none, so Next's are files.
+ */
+export interface Specifiers {
+  area: (area: string, locale?: string) => string;
+  generated: (path: string) => string;
+}
+
+export const VIRTUAL: Specifiers = {
+  area: (area, locale) => AREA + area + (locale === undefined ? '' : `/${locale}`),
+  generated: (path) => path,
+};
+
+/** `path` imported from a module in the folder `from`, as a relative specifier. */
+export function relativeImport(from: string, path: string): string {
+  const to = relative(from, path).replaceAll('\\', '/');
+  return to.startsWith('.') ? to : `./${to}`;
+}
+
+/** An area's file, or a lazy locale's part of a group, where a bundler without virtual modules finds them. */
+export const areaFile = (area: string, locale?: string) => (locale === undefined ? `${area}.js` : `${area}.${locale}.js`);
+
+/** The specifiers of area files in `dir`, from a module in the folder `from`. */
+export const fileSpecifiers = (from: string, dir: string): Specifiers => ({
+  area: (area, locale) => relativeImport(from, join(dir, areaFile(area, locale))),
+  generated: (path) => relativeImport(from, path),
+});
+
+export const areaOf = (key: string) => key.slice(0, key.indexOf('.'));
+
+/** The entries of some areas: their keys start with an area's name. */
+export const inAreas = (entries: Entry[], areas: readonly string[]) => entries.filter(([key]) => areas.includes(areaOf(key)));
+
+/**
+ * Each area to the group whose file its lazy locales load from, named by the group's first area: the
+ * areas a bundler puts in one chunk (`areaGroups` in `chunks.ts`), so they load in one request.
+ */
+export type Groups = ReadonlyMap<string, string>;
+
+/** The areas whose lazy locales load from the group `lead` names. */
+export const membersOf = (groups: Groups, lead: string) => [...groups].filter(([, g]) => g === lead).map(([area]) => area);
+
+export const areasOf = (generated: Generated) => new Set(generated.catalog.messages.map((m) => m.area));
+
+/** Areas `keep` names, which load with the index of a split catalog, as no module may name their keys. */
+export const eagerAreas = (generated: Generated, keep: string[] = []) => {
+  const known = areasOf(generated);
+  return new Set(keep.map(areaOf).filter((a) => known.has(a)));
+};
+
+/** The areas that load with the code naming their keys: all but those `keep` loads with the index. */
+export const lazyAreas = (generated: Generated, keep?: string[]) => {
+  const eager = eagerAreas(generated, keep);
+  return new Set([...areasOf(generated)].filter((a) => !eager.has(a)));
+};
+
+/** Which of `areas` the keys `code` names are in. */
+export function areasNamed(code: string, areas: ReadonlySet<string>): string[] {
+  const named = keyRefs(code);
+  return [...new Set([...named.exact, ...named.prefixes].map(areaOf))].filter((a) => areas.has(a)).toSorted();
+}
+
+/**
+ * `code` importing each of `areas`. At the end, as imports run first wherever they are, so no line
+ * moves; in a component file, at the end of its first script.
+ */
+export function importingAreas(code: string, path: string, areas: string[], specifier = VIRTUAL.area): string {
+  const imports = areas.map((a) => `import ${JSON.stringify(specifier(a))};`).join('');
+  if (!SFC.test(path)) return `${code}\n${imports}\n`;
+  const close = code.indexOf('</script>');
+  return close === -1 ? `${code}\n<script>${imports}</script>\n` : `${code.slice(0, close)};${imports}${code.slice(close)}`;
+}
+
+/** What a split catalog's areas register under: the generated folder, from the config's. */
+const splitId = (config: ResolvedConfig) => relative(config.root, config.out).replaceAll('\\', '/');
+
+/** A generated module's path, with forward slashes. */
+const generatedPath = (config: ResolvedConfig, name: string) =>
+  join(config.out, `${name}.${config.language === 'ts' ? 'ts' : 'js'}`).replaceAll('\\', '/');
+
+/** The index of a split catalog, in the browser: no messages of its own, only the areas `keep` names. */
+export function splitIndex(config: ResolvedConfig, generated: Generated, keep?: string[], where = VIRTUAL): string {
+  const locales = [...generated.entries.keys()];
+  const split = { id: splitId(config), areas: [...areasOf(generated)] };
+  return [
+    "import { defineCatalog } from 'prosebook';",
+    ...[...eagerAreas(generated, keep)].map((a) => `import ${JSON.stringify(where.area(a))};`),
+    `export const catalog = defineCatalog(${JSON.stringify({ source: config.source, locales, messages: {}, loaders: {}, split })});`,
+    '',
+  ].join('\n');
+}
+
+/** How an area's module holds a locale that ships with it in a build: as an expression, from its entries. */
+export type Inline = (locale: string, entries: Entry[]) => string;
+
+/**
+ * An area's module: registers its messages in the locales that ship with it, and loaders for the
+ * rest, and waits until every copy holds it. In a build, `inline` writes the first, and each of the
+ * rest loads its part of the area's group, shared with the areas `groups` puts beside it, which the
+ * copy takes only this area's keys from; in dev, without it, each locale is its generated module
+ * whole, so an edit updates it as it would without splitting.
+ */
+export function areaModule(
+  config: ResolvedConfig,
+  generated: Generated,
+  area: string,
+  inline?: Inline,
+  where = VIRTUAL,
+  groups: Groups = new Map(),
+): string {
+  const lines = ["import { addArea } from 'prosebook';"];
+  const messages: string[] = [];
+  const loaders: string[] = [];
+  for (const [locale, entries] of generated.entries) {
+    const name = JSON.stringify(locale);
+    const own = inAreas(entries, [area]);
+    const whole = JSON.stringify(where.generated(generatedPath(config, locale)));
+    if (locale === config.source || config.bundled.includes(locale)) {
+      if (inline !== undefined) {
+        messages.push(`${name}: ${inline(locale, own)}`);
+      } else {
+        lines.push(`import m${messages.length} from ${whole};`);
+        messages.push(`${name}: m${messages.length}`);
+      }
+    } else if (inline === undefined) {
+      loaders.push(`${name}: () => import(${whole})`);
+    } else if (own.length > 0) {
+      loaders.push(`${name}: () => import(${JSON.stringify(where.area(groups.get(area) ?? area, locale))})`);
+    }
+  }
+  const args = [JSON.stringify(splitId(config)), JSON.stringify(area), `{ ${messages.join(', ')} }`, `{ ${loaders.join(', ')} }`];
+  lines.push(`await addArea(${args.join(', ')});`, '');
+  return lines.join('\n');
+}
+
 /** A locale's module holding only the keys `kept`, as JavaScript, for a bundler to load in its place. */
 export function prunedModule(entries: Entry[], kept: (key: string) => boolean): string {
   return localeModule(compileLocale(entries, kept), '// Pruned by prosebook: only the messages the code names.', false);

package/src/tools/catalog.ts

@@ -312,7 +312,16 @@ function icuError(text: string, e: unknown): string {
   return `is not valid ICU MessageFormat: ${reason}`;
 }
 
-function yamlFiles(dir: string): string[] {
+/** The first line of a catalog file: the YAML language server finds the schema `generate` writes only through it. */
+export const SCHEMA_LINE = '# yaml-language-server: $schema=./catalog.schema.json';
+
+/** `text` opening with the schema line, unless it opens with a schema line of its own already. */
+export function withSchemaLine(text: string): string {
+  if (text.startsWith('# yaml-language-server:')) return text;
+  return `${SCHEMA_LINE}${text.includes('\r\n') ? '\r\n' : '\n'}${text}`;
+}
+
+export function yamlFiles(dir: string): string[] {
   return readdirSync(dir)
     .filter((f) => f.endsWith('.yaml'))
     .sort();

package/src/tools/chunks.ts

@@ -0,0 +1,142 @@
+/**
+ * Which areas of a split catalog a bundler puts in one chunk, so a lazy locale's messages for them
+ * can be one file: a route reading three areas loads German in one request, not three.
+ *
+ * Read from the source's imports ahead of the build, as Bun and Next give a plugin no say in chunks
+ * and Vite loads an area before it has seen every module naming it. Bundlers place a module by the
+ * entries that load it: the app's own, and each dynamic import, less those that only ever load once
+ * it is already in memory, as a route its shell imports. This reads the same from the files under
+ * `scan.dirs`, with relative imports resolved. It is a guess, and only a request rides on it: an
+ * import it can't resolve (an alias, a package) leaves apart areas that may share a chunk, and two
+ * areas grouped in different chunks load each other's messages, which the copy leaves unread.
+ */
+import { readFileSync } from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
+
+import { parseSync } from 'oxc-parser';
+
+import { areasNamed, areasOf, eagerAreas, type Groups } from './bundle.ts';
+import type { ResolvedConfig } from './config.ts';
+import type { Generated } from './generate.ts';
+import { scriptBlocks, SFC, sourceFiles } from './scan.ts';
+
+/** Tests and stories import components but load in no page, and as entries of their own they would part the areas they reach. */
+const NOT_IN_APP = /\.(?:test|spec|stories)\.[^./\\]+$|[\\/]__tests__[\\/]/;
+
+interface Imports {
+  static: string[];
+  dynamic: string[];
+}
+
+/** A module's imports that load code: `import type` and `export type … from` load none. */
+function importsOf(file: string, code: string): Imports {
+  const pieces = SFC.test(file) ? scriptBlocks(code) : [{ code, at: 0 }];
+  const found: Imports = { static: [], dynamic: [] };
+  for (const piece of pieces) {
+    const { module } = parseSync(SFC.test(file) ? 'component.ts' : file, piece.code);
+    for (const i of module.staticImports)
+      if (i.entries.length === 0 || i.entries.some((e) => !e.isType)) found.static.push(i.moduleRequest.value);
+    for (const e of module.staticExports.flatMap((s) => s.entries))
+      if (e.moduleRequest !== null && !e.isType) found.static.push(e.moduleRequest.value);
+    for (const d of module.dynamicImports) {
+      // Only a literal names a module; `import(\`./pages/${name}.tsx\`)` could be any.
+      const literal = /^(['"`])([^'"`$]*)\1$/.exec(piece.code.slice(d.moduleRequest.start, d.moduleRequest.end).trim());
+      if (literal !== null) found.dynamic.push(literal[2]!);
+    }
+  }
+  return found;
+}
+
+/** The file a relative specifier names, as a bundler finds it: as written, with an extension, as `./x.js` for `x.ts`, or as a folder's index. */
+function resolver(files: ReadonlySet<string>, extensions: string[]): (from: string, specifier: string) => string | undefined {
+  return (from, specifier) => {
+    if (!specifier.startsWith('.')) return undefined;
+    const base = resolve(dirname(from), specifier);
+    const stem = base.replace(/\.[cm]?jsx?$/, '');
+    const candidates = [
+      base,
+      ...extensions.map((e) => base + e),
+      ...extensions.map((e) => stem + e),
+      ...extensions.map((e) => join(base, `index${e}`)),
+    ];
+    return candidates.find((c) => files.has(c));
+  };
+}
+
+/**
+ * The groups of a split catalog's areas. Areas `keep` names load with the index, together; one no
+ * file under `scan.dirs` names is a group of its own.
+ */
+export function areaGroups(config: ResolvedConfig, generated: Generated, keep?: string[]): Groups {
+  const eager = eagerAreas(generated, keep);
+  const lazy = new Set([...areasOf(generated)].filter((a) => !eager.has(a)));
+  const files = sourceFiles(config.scan.dirs, config.scan.extensions, [config.out])
+    .filter((f) => !NOT_IN_APP.test(f))
+    .map((f) => resolve(f));
+  const find = resolver(new Set(files), config.scan.extensions);
+
+  // The module graph, an area a node of its own that each file naming its keys imports.
+  const node = (area: string) => `\0${area}`;
+  const statics = new Map<string, string[]>();
+  const dynamics = new Map<string, Set<string>>();
+  const imported = new Set<string>();
+  for (const file of files) {
+    const code = readFileSync(file, 'utf8');
+    const { static: s, dynamic: d } = importsOf(file, code);
+    const to = (specifiers: string[]) => specifiers.map((spec) => find(file, spec)).filter((f) => f !== undefined);
+    statics.set(file, [...to(s), ...areasNamed(code, lazy).map(node)]);
+    for (const target of to(d)) {
+      if (!dynamics.has(target)) dynamics.set(target, new Set());
+      dynamics.get(target)!.add(file);
+    }
+    for (const f of [...to(s), ...to(d)]) imported.add(f);
+  }
+
+  // Entries: each dynamic import, and each file nothing imports, as a page or the app's own entry.
+  const entries = [...new Set([...files.filter((f) => !imported.has(f)), ...dynamics.keys()])];
+  const closure = new Map<string, Set<string>>();
+  const loadedBy = new Map<string, Set<string>>();
+  for (const entry of entries) {
+    const seen = new Set<string>();
+    const visit = (n: string) => {
+      if (seen.has(n)) return;
+      seen.add(n);
+      if (!loadedBy.has(n)) loadedBy.set(n, new Set());
+      loadedBy.get(n)!.add(entry);
+      for (const next of statics.get(n) ?? []) visit(next);
+    };
+    visit(entry);
+    closure.set(entry, seen);
+  }
+
+  // What is in memory whenever a dynamic entry loads: what every module importing it brought, by
+  // every entry that loads that module. It only grows, so it settles.
+  const inMemory = new Map<string, Set<string>>([...dynamics.keys()].map((d) => [d, new Set()]));
+  const nodes = [...loadedBy.keys()];
+  for (let changed = true; changed;) {
+    changed = false;
+    for (const [entry, importers] of dynamics) {
+      const by = [...importers].map((i) => [...(loadedBy.get(i) ?? [])]);
+      if (by.some((e) => e.length === 0)) continue;
+      const held = (n: string) => by.every((e) => e.every((root) => closure.get(root)!.has(n) || inMemory.get(root)?.has(n) === true));
+      const next = new Set(nodes.filter(held));
+      if (next.size > inMemory.get(entry)!.size) {
+        inMemory.set(entry, next);
+        changed = true;
+      }
+    }
+  }
+
+  const groups = new Map<string, string>();
+  const byChunk = new Map<string, string[]>();
+  const sorted = [...lazy].toSorted();
+  for (const area of sorted) {
+    const loading = [...(loadedBy.get(node(area)) ?? [])].filter((e) => inMemory.get(e)?.has(node(area)) !== true).toSorted();
+    // Named by no file read here, so where it lands is unknown: a group of its own.
+    const chunk = loading.length === 0 ? node(area) : loading.join('\n');
+    if (!byChunk.has(chunk)) byChunk.set(chunk, []);
+    byChunk.get(chunk)!.push(area);
+  }
+  for (const members of [...byChunk.values(), [...eager].toSorted()]) for (const area of members) groups.set(area, members[0]!);
+  return groups;
+}

package/src/tools/config.ts

@@ -57,21 +57,35 @@ export interface LintConfig extends WordLists {
 
 /**
  * Native code generated from the catalog, for the parts of an app outside the browser: a tray, a
- * menu bar, a CLI. One file, holding every locale, with a typed function per message.
+ * menu bar, a CLI. One file, holding every locale, with a typed function per message; for Android and
+ * Apple, string resources, a file per locale.
  */
 export interface NativeTarget {
-  language: 'rust' | 'swift' | 'kotlin';
-  /** The file written. */
+  language: 'rust' | 'swift' | 'kotlin' | 'android' | 'apple';
+  /**
+   * The file written; for Android, the `res` folder its `values*` folders go in, and for Apple, the
+   * folder its `.lproj` folders go in.
+   */
   out: string;
   /**
    * Key prefixes the file holds: `['native']` takes every `native.*` key, which it names without the
    * prefix (`native.tray.quit` is `tray_quit`). Every key, named in full, by default.
    */
   keys?: string[];
-  /** The type holding the locales and messages: `Locale` in Rust, `Copy` in Swift and Kotlin. */
+  /**
+   * The type holding the locales and messages: `Locale` in Rust, `Copy` in Swift and Kotlin. For
+   * Android, the resource file's name: `strings`, for `values/strings.xml`. For Apple, the table's:
+   * `Localizable`, or `InfoPlist` with `plist`.
+   */
   name?: string;
   /** Kotlin's package. */
   package?: string;
+  /**
+   * For Apple, the Info.plist the table translates: each key names one of its strings, spelled in
+   * words (`ns-camera-usage-description` is `NSCameraUsageDescription`), and the source must say
+   * what the plist says.
+   */
+  plist?: string;
 }
 
 export interface ProsebookConfig {
@@ -150,7 +164,7 @@ export interface ResolvedConfig {
   scan: { dirs: string[]; extensions: string[]; attributes: string[] };
 }
 
-const NATIVE_NAMES = { rust: 'Locale', swift: 'Copy', kotlin: 'Copy' };
+const NATIVE_NAMES = { rust: 'Locale', swift: 'Copy', kotlin: 'Copy', android: 'strings', apple: 'Localizable' };
 
 /** Typed config, for `prosebook.config.ts`. */
 export function defineConfig(config: ProsebookConfig): ProsebookConfig {
@@ -217,11 +231,15 @@ export function resolveConfig(root: string, config: ProsebookConfig): ResolvedCo
     native: (config.native ?? []).map((target, i) => {
       if (!(target.language in NATIVE_NAMES))
         throw new Error(`native[${i}].language is "${target.language}": name one of ${Object.keys(NATIVE_NAMES).join(', ')}`);
-      return { ...target, out: at(target.out), name: target.name ?? NATIVE_NAMES[target.language] };
+      if (target.plist !== undefined && target.language !== 'apple')
+        throw new Error(`native[${i}].plist names an Info.plist, which only the apple target reads: remove it, or set language to 'apple'`);
+      const plist = target.plist === undefined ? {} : { plist: at(target.plist) };
+      const name = target.name ?? (target.plist === undefined ? NATIVE_NAMES[target.language] : 'InfoPlist');
+      return { ...target, out: at(target.out), name, ...plist };
     }),
     scan: {
       dirs: (config.scan?.dirs ?? ['src']).map(at),
-      extensions: config.scan?.extensions ?? ['.ts', '.tsx', '.js', '.jsx', '.mts', '.mjs'],
+      extensions: config.scan?.extensions ?? ['.ts', '.tsx', '.js', '.jsx', '.mts', '.mjs', '.vue', '.svelte'],
       attributes: config.scan?.attributes ?? ['alt', 'title', 'placeholder', 'label', 'aria-label', 'aria-description', 'aria-placeholder'],
     },
   };

package/src/tools/entry.ts

@@ -15,6 +15,7 @@ import { isMap, isScalar, parseDocument, Scalar, type Document, type YAMLMap } f
 
 import { CatalogError, loadCatalog, type Catalog, type Message, type Problem } from './catalog.ts';
 import type { ResolvedConfig } from './config.ts';
+import { readReturned } from './exchange.ts';
 import { applyEdits, lint, type Edit, type Finding } from './lint.ts';
 
 export interface EntryFields {
@@ -178,46 +179,44 @@ export interface Imported {
 }
 
 /**
- * Translations into `locale` from `prosebook todo`'s YAML filled in, by a person or a model, each
- * written to the file its locale and area put it in. Text around a fenced block is left aside, as a
- * model's reply has it, and an empty value is one left for later. An entry that would break the
- * catalog is left out with its problem, so a reply with one bad entry still lands the others.
+ * Translations into `locale` from `prosebook todo`'s YAML filled in, by a person or a model, or from
+ * a translation tool's XLIFF or JSON (`readReturned`), each written to the file its locale and area
+ * put it in. An empty value is one left for later. An entry that would break the catalog is left out
+ * with its problem, so a reply with one bad entry still lands the others.
  */
 export function importTranslations(config: ResolvedConfig, locale: string, reply: string): Imported {
   const failed = (message: string, key?: string): Problem => ({ severity: 'error', key, message });
   if (locale === config.source)
     return { written: [], unchanged: [], problems: [failed(`${locale} is the source: import takes translations`)] };
-  const fenced = [...reply.matchAll(/^```[^\n]*\n([\s\S]*?)^```/gm)].map((m) => m[1]);
-  const doc = parseDocument(fenced.length > 0 ? fenced.join('\n') : reply);
-  if (doc.errors.length > 0 || !(isMap(doc.contents) || doc.contents === null)) {
-    const why = doc.errors[0]?.message.split('\n', 1)[0] ?? 'it is not a mapping';
-    return {
-      written: [],
-      unchanged: [],
-      problems: [failed(`the reply isn't YAML of \`key: translation\` (${why}): fix it, or ask for one YAML block`)],
-    };
-  }
+  const read = readReturned(reply, locale, config.source);
+  if ('problem' in read) return { written: [], unchanged: [], problems: [failed(read.problem)] };
 
   const catalog = loadCatalog(config);
   const problems: Problem[] = [];
   const unchanged: string[] = [];
   const pending = new Map<string, { file: string; text: string }>();
-  for (const pair of isMap(doc.contents) ? doc.contents.items : []) {
-    const key = isScalar(pair.key) ? String(pair.key.value) : String(pair.key);
+  for (const { key, value, checked, source, problem } of read.returned) {
     const message = catalog.messages.find((m) => m.key === key);
     if (message === undefined) {
-      problems.push(failed('no entry has this key: translate only the keys `prosebook todo` printed', key));
+      problems.push(failed('no entry has this key: translate only the keys `prosebook todo` or `export` gave out', key));
+      continue;
+    }
+    if (problem !== undefined) {
+      problems.push(failed(problem, key));
       continue;
     }
-    const value = isScalar(pair.value) ? pair.value.value : pair.value;
     if (value === null || value === '') continue;
     if (typeof value !== 'string' && typeof value !== 'number') {
       problems.push(failed('the translation is not text: give the message as one string', key));
       continue;
     }
+    if (source !== undefined && source !== message.text) {
+      problems.push(failed(`the source has changed since it was exported, to "${message.text}": export it again`, key));
+      continue;
+    }
     const text = String(value);
-    if (text === message.translations.get(locale)?.text) unchanged.push(key);
-    else pending.set(key, { file: translationFile(config, message, locale), text });
+    if (text !== message.translations.get(locale)?.text) pending.set(key, { file: translationFile(config, message, locale), text });
+    else if (checked) unchanged.push(key);
   }
 
   // An entry whose own file reports it is left out, until what is left loads clean. A problem