prosebook 0.0.1 → 0.2.0

package/src/tools/generate.ts

@@ -0,0 +1,257 @@
+/**
+ * Turns the catalog into the modules an app imports.
+ *
+ * Generated rather than loaded so that a missing key, a misspelled key or a missing argument is a type
+ * error, and so the runtime needs neither a YAML nor an ICU parser. The index carries each key's source
+ * text and context as a doc comment, so hovering a key at its call site shows what the screen says.
+ */
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { dirname, join, relative } from 'node:path';
+
+import { loadCatalog, type Catalog, type Message } from './catalog.ts';
+import { compileLocale, localeModule, type Entry } from './compile.ts';
+import type { ResolvedConfig } from './config.ts';
+import { readLock, reconcile, serializeLock, type Lock, type StaleTranslation } from './lock.ts';
+import { pseudoAst } from './pseudo.ts';
+import { SYNTAX_NAME } from './syntax.ts';
+
+const quote = (s: string) => JSON.stringify(s);
+const prop = (name: string) => (/^[A-Za-z_$][\w$]*$/.test(name) ? name : quote(name));
+
+function banner(config: ResolvedConfig): string {
+  const from = relative(config.root, config.catalog).replaceAll('\\', '/');
+  return `// Generated by prosebook from ${from}. Do not edit: change the catalog and run \`prosebook generate\`.`;
+}
+
+/** Every locale the app can switch to, in order: the source, the translations, the pseudo-locale. */
+export function localesOf(catalog: Catalog, config: ResolvedConfig): string[] {
+  return [config.source, ...catalog.locales, ...(config.pseudo !== false ? [config.pseudo] : [])];
+}
+
+/** Per locale, the messages its module holds. A translated locale holds only what it translates. */
+export function localeEntries(
+  catalog: Catalog,
+  config: ResolvedConfig,
+  stale: Map<string, Map<string, StaleTranslation>>,
+): Map<string, Entry[]> {
+  const out = new Map<string, Entry[]>();
+  out.set(
+    config.source,
+    catalog.messages.map((m) => [m.key, m.ast]),
+  );
+  for (const locale of catalog.locales) {
+    const skip = config.stale === 'fallback' ? stale.get(locale) : undefined;
+    out.set(
+      locale,
+      catalog.messages.flatMap((m): Entry[] => {
+        const t = m.translations.get(locale);
+        return t === undefined || skip?.has(m.key) === true ? [] : [[m.key, t.ast]];
+      }),
+    );
+  }
+  if (config.pseudo !== false) {
+    out.set(
+      config.pseudo,
+      catalog.messages.map((m) => [m.key, pseudoAst(m.ast)]),
+    );
+  }
+  return out;
+}
+
+function doc(lines: string[], indent: string): string {
+  const safe = lines.map((l) => l.replaceAll('*/', '*\\/'));
+  if (safe.length === 1) return `${indent}/** ${safe[0]} */\n`;
+  return `${indent}/**\n${safe.map((l) => (l === '' ? `${indent} *` : `${indent} * ${l}`)).join('\n')}\n${indent} */\n`;
+}
+
+/** A doc written as a phrase (`the person's name`) reads as a sentence in a doc comment. */
+function sentence(text: string): string {
+  const capital = text.charAt(0).toUpperCase() + text.slice(1);
+  return /[.!?]$/.test(capital) ? capital : `${capital}.`;
+}
+
+function shapeOf(m: Message): string {
+  const tags = m.tags.length > 0 ? m.tags.map(quote).join(' | ') : 'never';
+  if (m.args.size === 0) return `{ args: {}; tags: ${tags} }`;
+  const fields = [...m.args].map(([name, type]) => {
+    const held = m.argDocs.get(name);
+    const said = [
+      held === undefined ? undefined : sentence(held),
+      m.argFormats.has(name) ? `Formatted as ${m.argFormats.get(name)}.` : undefined,
+    ].filter(Boolean);
+    return `${said.length > 0 ? doc([said.join(' ')], '      ') : ''}      readonly ${prop(name)}: ${type};`;
+  });
+  return `{\n    args: {\n${fields.join('\n')}\n    };\n    tags: ${tags};\n  }`;
+}
+
+function typesBlock(catalog: Catalog, locales: string[]): string {
+  const rows = catalog.messages.map((m) => {
+    // An MF2 message with variants spans lines; each is a line of the doc comment.
+    const lines = (m.text.length > 400 ? `${m.text.slice(0, 397)}...` : m.text).split('\n');
+    if (m.context) lines.push('', m.context);
+    if (m.remove) lines.push('', 'Marked for removal in the catalog.');
+    if (m.comment) lines.push('', `Comment: ${m.comment}`);
+    return `${doc(lines, '  ')}  ${quote(m.key)}: ${shapeOf(m)};`;
+  });
+  return [
+    `export type Locale = ${locales.map(quote).join(' | ')};`,
+    '',
+    '/** Every message in the catalog: the arguments it takes and the tags it wraps text in. */',
+    `export type Messages = {\n${rows.join('\n')}\n};`,
+  ].join('\n');
+}
+
+const REGISTER = `declare module 'prosebook' {
+  interface Register {
+    messages: Messages;
+    locale: Locale;
+  }
+}`;
+
+function catalogLiteral(config: ResolvedConfig, locales: string[], ext: string): { imports: string[]; literal: string } {
+  const eager = locales.filter((l) => l === config.source || config.bundled.includes(l));
+  const lazy = locales.filter((l) => !eager.includes(l));
+  const ident = (l: string) => `m_${l.replace(/\W/g, '_')}`;
+  return {
+    imports: eager.map((l) => `import ${ident(l)} from './${l}${ext}';`),
+    literal: [
+      '{',
+      `  source: ${quote(config.source)},`,
+      `  locales: [${locales.map(quote).join(', ')}],`,
+      `  messages: { ${eager.map((l) => `${prop(l)}: ${ident(l)}`).join(', ')} },`,
+      `  loaders: { ${lazy.map((l) => `${prop(l)}: () => import('./${l}${ext}')`).join(', ')} },`,
+      '}',
+    ].join('\n'),
+  };
+}
+
+/** The JSON Schema an editor reads to complete and explain catalog entries. */
+export function catalogSchema(config: ResolvedConfig): object {
+  // The shorthand satisfies no required field, so it's offered only where none is.
+  const shorthand = { type: 'string', description: `The message, in ${SYNTAX_NAME[config.messageFormat]}, with nothing said about it.` };
+  const entry = {
+    type: 'object',
+    required: [config.source, ...config.require],
+    additionalProperties: false,
+    properties: {
+      [config.source]: { type: 'string', description: `The message, in ${SYNTAX_NAME[config.messageFormat]}.` },
+      kind: {
+        description: 'What sort of string it is.',
+        oneOf: Object.entries(config.kinds).map(([k, r]) => ({ const: k, description: r.doc })),
+      },
+      surface: {
+        description: 'Where the string sits.',
+        oneOf: Object.entries(config.surfaces).map(([k, d]) => ({ const: k, description: d })),
+      },
+      context: { type: 'string', description: 'One sentence telling a translator where the string appears and what it answers to.' },
+      args: { type: 'object', description: 'What each placeholder holds.', additionalProperties: { type: 'string' } },
+      fragment: { type: 'boolean', description: 'Completes something beside it rather than standing as a sentence.' },
+      comment: { type: 'string', description: 'A change asked of whoever edits the entry next. Make it, then delete the comment.' },
+      remove: { type: 'boolean', description: 'Asks for the entry to go: delete it, its translations and the code that reads it.' },
+      allow: {
+        type: 'object',
+        description: 'Lint rules this entry is exempt from, each with the reason.',
+        propertyNames: { enum: Object.keys(config.lint.rules) },
+        additionalProperties: { type: 'string' },
+      },
+    },
+  };
+  return {
+    $schema: 'https://json-schema.org/draft/2020-12/schema',
+    title: 'prosebook catalog',
+    description: 'Generated by `prosebook generate` from the config. Do not edit.',
+    type: 'object',
+    propertyNames: { pattern: '^[a-z0-9]+(?:[-_][a-z0-9]+)*(?:\\.[a-z0-9]+(?:[-_][a-z0-9]+)*)+$' },
+    additionalProperties: config.require.length === 0 ? { oneOf: [shorthand, entry] } : entry,
+  };
+}
+
+export interface Generated {
+  catalog: Catalog;
+  /** Absolute path to content. */
+  files: Map<string, string>;
+  lock: Lock;
+  stale: Map<string, Map<string, StaleTranslation>>;
+  entries: Map<string, Entry[]>;
+}
+
+/** Everything `generate` writes, in memory. */
+export function render(config: ResolvedConfig, accept?: Map<string, Set<string> | true>): Generated {
+  const catalog = loadCatalog(config);
+  const { lock, stale } = reconcile(catalog, readLock(config.lock), accept);
+  const locales = localesOf(catalog, config);
+  const entries = localeEntries(catalog, config, stale);
+  const ts = config.language === 'ts';
+  const head = banner(config);
+  const files = new Map<string, string>();
+
+  for (const [locale, list] of entries)
+    files.set(join(config.out, `${locale}.${ts ? 'ts' : 'js'}`), localeModule(compileLocale(list), head, ts));
+
+  const { imports, literal } = catalogLiteral(config, locales, config.importExtension);
+  const types = typesBlock(catalog, locales);
+  if (ts) {
+    files.set(
+      join(config.out, 'index.ts'),
+      [
+        head,
+        "import { defineCatalog } from 'prosebook';",
+        ...imports,
+        '',
+        types,
+        '',
+        `export const catalog = defineCatalog<Messages, Locale>(${literal});`,
+        '',
+        REGISTER,
+        '',
+      ].join('\n'),
+    );
+  } else {
+    files.set(
+      join(config.out, 'index.js'),
+      [head, "import { defineCatalog } from 'prosebook';", ...imports, '', `export const catalog = defineCatalog(${literal});`, ''].join(
+        '\n',
+      ),
+    );
+    files.set(
+      join(config.out, 'index.d.ts'),
+      [
+        head,
+        "import type { Catalog } from 'prosebook';",
+        '',
+        types,
+        '',
+        'export declare const catalog: Catalog<Messages, Locale>;',
+        '',
+        REGISTER,
+        '',
+      ].join('\n'),
+    );
+  }
+
+  files.set(join(config.catalog, 'catalog.schema.json'), `${JSON.stringify(catalogSchema(config), null, 2)}\n`);
+  if (Object.keys(lock.translations).length > 0 || existsSync(config.lock)) files.set(config.lock, serializeLock(lock));
+  return { catalog, files, lock, stale, entries };
+}
+
+/** Writes what changed. With `check`, writes nothing and returns what would have changed. */
+export function writeFiles(files: Map<string, string>, check = false): string[] {
+  const changed: string[] = [];
+  for (const [path, content] of files) {
+    const current = existsSync(path) ? readFileSync(path, 'utf8') : undefined;
+    if (current === content) continue;
+    changed.push(path);
+    if (check) continue;
+    mkdirSync(dirname(path), { recursive: true });
+    writeFileSync(path, content);
+  }
+  return changed;
+}
+
+export function generate(
+  config: ResolvedConfig,
+  options: { check?: boolean; accept?: Map<string, Set<string> | true> } = {},
+): Generated & { changed: string[] } {
+  const generated = render(config, options.accept);
+  return { ...generated, changed: writeFiles(generated.files, options.check) };
+}

package/src/tools/icu.ts

@@ -0,0 +1,211 @@
+/**
+ * What an ICU message declares, read off its AST: argument types, tags, and whether each number and
+ * date style is one the formatter honors. Shared by the loader, the compiler and the lint.
+ */
+import { isDateElement, isNumberElement, isTimeElement, parse, TYPE } from '@formatjs/icu-messageformat-parser';
+import type { DateElement, MessageFormatElement, NumberElement, TimeElement } from '@formatjs/icu-messageformat-parser';
+
+export type { MessageFormatElement };
+export { TYPE };
+
+export function parseMessage(text: string): MessageFormatElement[] {
+  const ast = parse(text, { shouldParseSkeletons: true, requiresOtherClause: true });
+  walk(ast, (el) => {
+    if ((isDateElement(el) || isTimeElement(el)) && typeof el.style === 'object' && el.style !== null && el.style.pattern.includes('j'))
+      el.style.parsedOptions = localHour(el.style.pattern);
+  });
+  return ast;
+}
+
+/**
+ * A skeleton's `j` is the hour as the locale writes it, 12 or 24. The parser resolves it only when
+ * given a locale and drops it otherwise, so `::yMMMdjmm` showed the minute alone. Read here as an
+ * hour with no cycle, which `Intl` gives the cycle of whatever locale formats it: "2:05 PM", "14:05".
+ */
+function localHour(pattern: string): Intl.DateTimeFormatOptions {
+  const run = /j+/.exec(pattern)![0];
+  const rest = pattern.replaceAll('j', '');
+  return { ...(rest === '' ? {} : dateFields(rest)), hour: run.length % 2 === 0 ? '2-digit' : 'numeric' };
+}
+
+/** What the parser reads from a date skeleton on its own; nothing where it can't parse it. */
+function dateFields(skeleton: string): Intl.DateTimeFormatOptions {
+  try {
+    const el = parse(`{d, date, ::${skeleton}}`, { shouldParseSkeletons: true })[0];
+    return el !== undefined && isDateElement(el) && typeof el.style === 'object' && el.style !== null ? el.style.parsedOptions : {};
+  } catch {
+    return {};
+  }
+}
+
+/**
+ * ICU argument types, as TypeScript. A bare `{x}` is a string: a number spliced in raw is printed by
+ * `String()`, which knows no locale, so a quantity says `{x, number}` and the type is what asks for it.
+ * A `select` narrows to the arms it names, and still takes any string, since `other` catches the rest.
+ */
+function argType(el: MessageFormatElement): string | undefined {
+  switch (el.type) {
+    case TYPE.argument:
+      return 'string';
+    case TYPE.number:
+    case TYPE.plural:
+      return 'number';
+    case TYPE.date:
+    case TYPE.time:
+      return 'Date | number';
+    case TYPE.select: {
+      const named = Object.keys(el.options).filter((o) => o !== 'other');
+      return named.length > 0 ? `${named.map((o) => JSON.stringify(o)).join(' | ')} | (string & {})` : 'string';
+    }
+    case TYPE.literal:
+    case TYPE.pound:
+    case TYPE.tag:
+    default:
+      return undefined;
+  }
+}
+
+/** Calls `visit` on every element, inside plural and select arms and tags too. */
+export function walk(elements: MessageFormatElement[], visit: (el: MessageFormatElement) => void): void {
+  for (const el of elements) {
+    visit(el);
+    if (el.type === TYPE.select || el.type === TYPE.plural) for (const option of Object.values(el.options)) walk(option.value, visit);
+    if (el.type === TYPE.tag) walk(el.children, visit);
+  }
+}
+
+export function argsOf(elements: MessageFormatElement[], clash: (name: string) => void = () => {}): Map<string, string> {
+  const args = new Map<string, string>();
+  walk(elements, (el) => {
+    const type = argType(el);
+    if (type === undefined || !('value' in el) || typeof el.value !== 'string') return;
+    const seen = args.get(el.value);
+    if (seen !== undefined && seen !== type) clash(el.value);
+    else args.set(el.value, type);
+  });
+  return args;
+}
+
+export function tagsOf(elements: MessageFormatElement[]): string[] {
+  const tags = new Set<string>();
+  walk(elements, (el) => {
+    if (el.type === TYPE.tag) tags.add(el.value);
+  });
+  return [...tags].sort();
+}
+
+/** A message with no placeholders and no tags renders to one literal, which ships as a plain string. */
+export function staticLiteral(ast: MessageFormatElement[]): string | undefined {
+  if (ast.length === 0) return '';
+  if (ast.length === 1 && ast[0]!.type === TYPE.literal) return ast[0]!.value;
+  return undefined;
+}
+
+const NUMBER_STYLES: Record<string, Intl.NumberFormatOptions> = {
+  integer: { maximumFractionDigits: 0 },
+  percent: { style: 'percent' },
+};
+
+const DATE_STYLES = ['short', 'medium', 'long', 'full'] as const;
+
+/** The `Intl` options a number element formats with. Throws, with the fix, where the style is wrong. */
+export function numberOptions(el: NumberElement): Intl.NumberFormatOptions | undefined {
+  const style = el.style;
+  if (style === null || style === undefined) return undefined;
+  if (typeof style === 'string') {
+    if (style === 'currency') throw new Error(`{${el.value}} needs its currency named: {${el.value}, number, ::currency/EUR}`);
+    const named = NUMBER_STYLES[style];
+    if (named === undefined) throw new Error(`{${el.value}} asks for number style "${style}": integer, percent, or a ::skeleton`);
+    return named;
+  }
+  const unknown = style.tokens.filter((t) => !stemIsKnown(t.stem, t.options)).map((t) => t.stem);
+  if (unknown.length > 0) throw new Error(`{${el.value}} has skeleton stems the formatter drops without a word: ${unknown.join(', ')}`);
+  const options = style.parsedOptions as Intl.NumberFormatOptions;
+  try {
+    new Intl.NumberFormat('en', options).resolvedOptions();
+  } catch (e) {
+    throw new Error(`{${el.value}} has a skeleton Intl refuses: ${(e as Error).message}`, { cause: e });
+  }
+  return options;
+}
+
+/** The `Intl` options a date or time element formats with. Throws, with the fix, where the style is wrong. */
+export function dateOptions(el: DateElement | TimeElement): Intl.DateTimeFormatOptions | undefined {
+  const style = el.style;
+  const field = el.type === TYPE.time ? 'timeStyle' : 'dateStyle';
+  if (style === null || style === undefined) return el.type === TYPE.time ? { timeStyle: 'short' } : undefined;
+  if (typeof style === 'string') {
+    if (!(DATE_STYLES as readonly string[]).includes(style)) {
+      const kind = el.type === TYPE.time ? 'time' : 'date';
+      throw new Error(`{${el.value}} asks for ${kind} style "${style}": ${DATE_STYLES.join(', ')}, or a ::skeleton`);
+    }
+    return { [field]: style };
+  }
+  // Every field the parser understands sets an option, and one it doesn't is dropped without a word.
+  const dropped = (style.pattern.match(/([A-Za-z])
*/g) ?? []).filter((f) => !f.startsWith('j') && Object.keys(dateFields(f)).length === 0);
+  if (dropped.length > 0) throw new Error(`{${el.value}} has date fields the formatter drops without a word: ${dropped.join(', ')}`);
+  if (Object.keys(style.parsedOptions).length === 0)
+    throw new Error(`{${el.value}} has a date skeleton the formatter reads nothing from: "${style.pattern}"`);
+  try {
+    new Intl.DateTimeFormat('en', style.parsedOptions).resolvedOptions();
+  } catch (e) {
+    throw new Error(`{${el.value}} has a skeleton Intl refuses: ${(e as Error).message}`, { cause: e });
+  }
+  return style.parsedOptions;
+}
+
+/**
+ * Whether the formatter understands a skeleton stem. One it does not is dropped without a word, so
+ * `::compact-shortt` reaches the screen as `5` where `5K` was written. Every stem it understands sets
+ * at least one option, so an empty result is the answer.
+ */
+function stemIsKnown(stem: string, options: readonly string[]): boolean {
+  try {
+    const el = parse(`{n, number, ::${[stem, ...options].join('/')}}`, { shouldParseSkeletons: true })[0];
+    if (el === undefined || !isNumberElement(el) || typeof el.style !== 'object' || el.style === null) return false;
+    return Object.keys(el.style.parsedOptions).length > 0;
+  } catch {
+    return false;
+  }
+}
+
+/** Every format problem in a message, as sentences. */
+export function formatProblems(ast: MessageFormatElement[]): string[] {
+  const found: string[] = [];
+  walk(ast, (el) => {
+    try {
+      if (isNumberElement(el)) numberOptions(el);
+      if (isDateElement(el) || isTimeElement(el)) dateOptions(el);
+    } catch (e) {
+      found.push((e as Error).message);
+    }
+  });
+  return found;
+}
+
+/** How each number or date argument is formatted, for the generated doc comment: `date medium, time short`. */
+export function formatsOf(ast: MessageFormatElement[]): Map<string, string> {
+  const formats = new Map<string, string[]>();
+  walk(ast, (el) => {
+    if (!isNumberElement(el) && !isDateElement(el) && !isTimeElement(el)) return;
+    const style = el.style;
+    if (style === null || style === undefined) return;
+    const kind = el.type === TYPE.number ? 'number' : el.type === TYPE.date ? 'date' : 'time';
+    let said: string;
+    if (typeof style === 'string') {
+      said = `${kind} \`${style}\``;
+    } else if ('mf2' in style) {
+      // Read from MF2: the function as written.
+      const { name, options } = (style as { mf2: { name: string; options: Record<string, string> } }).mf2;
+      said = `\`${[`:${name}`, ...Object.entries(options).map(([k, v]) => `${k}=${v}`)].join(' ')}\``;
+    } else if ('tokens' in style) {
+      said = `${kind} \`::${style.tokens.map((t) => [t.stem, ...t.options].join('/')).join(' ')}\``;
+    } else {
+      said = `${kind} \`::${style.pattern}\``;
+    }
+    const list = formats.get(el.value) ?? [];
+    if (!list.includes(said)) list.push(said);
+    formats.set(el.value, list);
+  });
+  return new Map([...formats].map(([name, list]) => [name, list.join(', ')]));
+}

package/src/tools/index.ts

@@ -0,0 +1,91 @@
+/**
+ * `prosebook/tools`: the build-time half, for scripts and other bundler integrations. The CLI and the
+ * Vite plugin are written on these.
+ */
+export {
+  CatalogError,
+  entryLocation,
+  KEY_PATTERN,
+  loadCatalog,
+  previewOf,
+  whereOf,
+  type Catalog,
+  type EntryLocation,
+  type Message,
+  type Problem,
+  type Translation,
+} from './catalog.ts';
+export { compileLocale, localeExpression, localeModule, type CompiledLocale, type Entry } from './compile.ts';
+export {
+  applyExtract,
+  areaOf,
+  draftPlurals,
+  planExtract,
+  slugOf,
+  type Extraction,
+  type ExtractPlan,
+  type FilePlan,
+  type NewEntry,
+} from './extract.ts';
+export {
+  applyFixes,
+  editEntry,
+  readEntry,
+  type EntryEdit,
+  type EntryFields,
+  type EntryFinding,
+  type EntryView,
+  type EntryYaml,
+} from './entry.ts';
+export { defineConfig, loadConfig, resolveConfig, type ProsebookConfig, type ResolvedConfig } from './config.ts';
+export { catalogSchema, generate, localeEntries, localesOf, render, writeFiles, type Generated } from './generate.ts';
+export {
+  applyEdits,
+  compileRule,
+  CORE_RULES,
+  definePlugin,
+  HOLE,
+  lint,
+  phrase,
+  sentences,
+  singularOf,
+  words,
+  type CapitalizationRule,
+  type CheckRule,
+  type ConditionalRule,
+  type CoreRule,
+  type Edit,
+  type ExistenceRule,
+  type Finding,
+  type OccurrenceRule,
+  type Plugin,
+  type Readings,
+  type Report,
+  type Rule,
+  type RuleContext,
+  type SubstitutionRule,
+} from './lint.ts';
+export { formatDiff, wordDiff, type DiffPart } from './diff.ts';
+export { emptyLock, hash, readLock, reconcile, serializeLock, type Lock, type StaleTranslation } from './lock.ts';
+export { DEFAULT_KINDS, DEFAULT_SURFACES, STRICT_LISTS, type KindRule, type Severity } from './presets.ts';
+export { pseudoAst, pseudoText } from './pseudo.ts';
+export { intlDate, intlNumber, parseMf2, toMf2, type Mf2Function } from './mf2.ts';
+export { toMf1 } from './mf1.ts';
+export { applyMigrate, planMigrate, type MigratePlan } from './migrate.ts';
+export { readMessage, SYNTAX_NAME, writeMessage, type MessageFormatVersion } from './syntax.ts';
+export { syntaxKind, typescriptOracle, type Span, type TypeOracle, type ValueKind } from './types.ts';
+export { SarifReport, type SarifResult, type SarifTool } from './sarif.ts';
+export {
+  hardcodedText,
+  isReferenced,
+  keyRefs,
+  looksLikeCode,
+  READERS,
+  scan,
+  sourceFiles,
+  unseenReads,
+  type Hardcoded,
+  type KeyRefs,
+  type Reads,
+  type UnseenRead,
+} from './scan.ts';