prosebook 0.0.1 → 0.2.0

package/src/tools/scan.ts

@@ -0,0 +1,303 @@
+/**
+ * What the code says: text a person reads that is still written in a component, and which catalog
+ * keys the code names.
+ *
+ * Key references are found by reading string literals, not by tracing calls, so a key kept in an
+ * array or a map still counts, and `nav.${id}` or `'nav.' + id` keeps every key under `nav.`. That
+ * errs towards keeping a key, which is the safe direction both for `unused` and for pruning a bundle.
+ */
+import { readdirSync, readFileSync, statSync } from 'node:fs';
+import { extname, join, relative } from 'node:path';
+
+import { parseSync } from 'oxc-parser';
+
+const SKIP_DIRS = new Set(['node_modules', 'dist', 'build', 'out', 'coverage', '.git']);
+
+export function sourceFiles(dirs: string[], extensions: string[], skip: string[] = []): string[] {
+  const files: string[] = [];
+  const visit = (dir: string) => {
+    let names: string[];
+    try {
+      names = readdirSync(dir);
+    } catch {
+      return;
+    }
+    for (const name of names) {
+      const path = join(dir, name);
+      if (skip.some((s) => path === s || path.startsWith(`${s}/`) || path.startsWith(`${s}\\`))) continue;
+      if (statSync(path).isDirectory()) {
+        if (!SKIP_DIRS.has(name) && !name.startsWith('.')) visit(path);
+      } else if (extensions.includes(extname(name)) && !name.endsWith('.d.ts')) {
+        files.push(path);
+      }
+    }
+  };
+  for (const dir of dirs) visit(dir);
+  return files.sort();
+}
+
+export interface KeyRefs {
+  exact: Set<string>;
+  /** `nav.` from `nav.${id}` or `'nav.' + id`: every key under it may be read. */
+  prefixes: Set<string>;
+}
+
+const LITERAL_KEY = /(['"`])([a-z0-9][\w-]*(?:\.[\w-]+)+)\1/g;
+const KEY_PREFIX = /(['"`])([a-z0-9][\w-]*(?:\.[\w-]+)*\.)(?:\1|\$\{)/g;
+
+export function keyRefs(code: string, into: KeyRefs = { exact: new Set(), prefixes: new Set() }): KeyRefs {
+  for (const m of code.matchAll(LITERAL_KEY)) into.exact.add(m[2]!);
+  for (const m of code.matchAll(KEY_PREFIX)) into.prefixes.add(m[2]!);
+  return into;
+}
+
+export function isReferenced(key: string, refs: KeyRefs): boolean {
+  if (refs.exact.has(key)) return true;
+  for (const prefix of refs.prefixes) if (key.startsWith(prefix)) return true;
+  return false;
+}
+
+export interface Hardcoded {
+  file: string;
+  line: number;
+  column: number;
+  text: string;
+  /** Where it sits: `text`, or the attribute it is the value of. */
+  where: string;
+}
+
+type Node = { type: string; start: number; end: number; [field: string]: unknown };
+
+/**
+ * Text that reads as code rather than prose: one token holding a path, a call, an identifier with an
+ * underscore or a dotted name, as in `form.json()` or `src/index.ts`. Trailing punctuation doesn't
+ * count, so `Done.` is still prose.
+ */
+export function looksLikeCode(text: string): boolean {
+  const token = text.trim().replace(/[.!?:,…]+$/, '');
+  return token !== '' && !/\s/.test(token) && /[/()_=<>{}[\].]|::/.test(token);
+}
+
+const NON_PROSE_ELEMENTS = new Set(['code', 'pre', 'kbd', 'samp', 'script', 'style', 'svg', 'path']);
+const LETTER = /\p{L}/u;
+
+/** Offsets in `code` as lines and columns, and whether a line is marked `prosebook-ignore`, on it or the line above. */
+function lineIndex(code: string) {
+  const lines = code.split('\n');
+  const lineStarts: number[] = [];
+  let offset = 0;
+  for (const line of lines) {
+    lineStarts.push(offset);
+    offset += line.length + 1;
+  }
+  const position = (at: number) => {
+    let lo = 0;
+    let hi = lineStarts.length - 1;
+    while (lo < hi) {
+      const mid = (lo + hi + 1) >> 1;
+      if (lineStarts[mid]! <= at) lo = mid;
+      else hi = mid - 1;
+    }
+    return { line: lo + 1, column: at - lineStarts[lo]! + 1 };
+  };
+  const ignored = (line: number) =>
+    (lines[line - 1] ?? '').includes('prosebook-ignore') || (lines[line - 2] ?? '').includes('prosebook-ignore');
+  return { position, ignored };
+}
+
+/** Literal text in JSX that a person reads, where it should be a catalog entry. */
+export function hardcodedText(file: string, code: string, attributes: string[]): Hardcoded[] {
+  const { program } = parseSync(file, code);
+  const found: Hardcoded[] = [];
+  const { position, ignored } = lineIndex(code);
+  const report = (node: Node, text: string, where: string) => {
+    const trimmed = text.replace(/\s+/g, ' ').trim();
+    if (!LETTER.test(trimmed) || looksLikeCode(trimmed)) return;
+    const at = position(node.start + Math.max(0, code.slice(node.start, node.end).search(/\S/)));
+    if (ignored(at.line)) return;
+    found.push({ file, ...at, text: trimmed, where });
+  };
+
+  const visit = (node: Node, inProse: boolean) => {
+    let prose = inProse;
+    if (node.type === 'JSXElement') {
+      const opening = node.openingElement as Node & { name: Node & { name?: string } };
+      if (opening.name.type === 'JSXIdentifier' && NON_PROSE_ELEMENTS.has(opening.name.name ?? '')) prose = false;
+    }
+    if (node.type === 'JSXText' && prose) report(node, node.value as string, 'text');
+    if (node.type === 'JSXExpressionContainer' && prose) {
+      const expr = node.expression as Node;
+      if (expr.type === 'Literal' && typeof expr.value === 'string') report(expr, expr.value, 'text');
+      if (expr.type === 'TemplateLiteral' && (expr.expressions as Node[]).length === 0)
+        report(expr, String((expr.quasis as { value: { cooked: string } }[])[0]?.value.cooked), 'text');
+    }
+    if (node.type === 'JSXAttribute') {
+      const name = node.name as Node & { name?: string };
+      const value = node.value as Node | null;
+      const attr = name.type === 'JSXIdentifier' ? (name.name ?? '') : '';
+      if (attributes.includes(attr) && value !== null) {
+        if (value.type === 'Literal' && typeof value.value === 'string') report(value, value.value, attr);
+        if (value.type === 'JSXExpressionContainer') {
+          const expr = value.expression as Node;
+          if (expr.type === 'Literal' && typeof expr.value === 'string') report(expr, expr.value, attr);
+        }
+      }
+      return;
+    }
+    for (const value of Object.values(node)) {
+      if (Array.isArray(value)) {
+        for (const child of value) if (isNode(child)) visit(child, prose);
+      } else if (isNode(value)) {
+        visit(value, prose);
+      }
+    }
+  };
+  visit(program as unknown as Node, true);
+  return found;
+}
+
+function isNode(value: unknown): value is Node {
+  return typeof value === 'object' && value !== null && typeof (value as Node).type === 'string';
+}
+
+export interface ScanResult {
+  hardcoded: Hardcoded[];
+  refs: KeyRefs;
+  files: number;
+}
+
+export function scan(dirs: string[], extensions: string[], attributes: string[], root: string, skip: string[] = []): ScanResult {
+  const refs: KeyRefs = { exact: new Set(), prefixes: new Set() };
+  const hardcoded: Hardcoded[] = [];
+  const files = sourceFiles(dirs, extensions, skip);
+  for (const path of files) {
+    const code = readFileSync(path, 'utf8');
+    keyRefs(code, refs);
+    if (/\.[jt]sx$/.test(path)) hardcoded.push(...hardcodedText(relative(root, path).replaceAll('\\', '/'), code, attributes));
+  }
+  return { hardcoded, refs, files: files.length };
+}
+
+/** A copy's methods that take a key first. */
+export const READERS = new Set(['t', 'rich']);
+
+export interface UnseenRead {
+  file: string;
+  line: number;
+  column: number;
+  /** The read as written, on one line. */
+  code: string;
+  /** What it calls: a reader, the name it was imported by, or `Text`. */
+  name: string;
+}
+
+export interface Reads {
+  /** Reads whose key a build can't see, in calls that may be to a copy's `t` or `rich`. */
+  unseen: UnseenRead[];
+  /** Names a copy's readers are bound to here, as in `const { rich: tRich } = copy`. */
+  aliases: Set<string>;
+}
+
+const PREFIX = /^[a-z0-9][\w-]*(?:\.[\w-]+)*\.$/;
+const WRAPPERS = new Set(['ParenthesizedExpression', 'TSAsExpression', 'TSSatisfiesExpression', 'TSNonNullExpression', 'TSTypeAssertion']);
+
+/** Whether `keyRefs` finds every key an expression can be: a literal, or a template or sum after a key prefix. */
+function keySeen(node: Node | undefined): boolean {
+  if (node === undefined) return false;
+  if (WRAPPERS.has(node.type)) return keySeen(node.expression as Node);
+  switch (node.type) {
+    case 'Literal':
+      return typeof node.value === 'string';
+    case 'TemplateLiteral': {
+      const [head] = node.quasis as { value: { cooked: string | null } }[];
+      return (node.expressions as Node[]).length === 0 || PREFIX.test(head?.value.cooked ?? '');
+    }
+    case 'BinaryExpression': {
+      const left = node.left as Node;
+      return node.operator === '+' && left.type === 'Literal' && typeof left.value === 'string' && PREFIX.test(left.value);
+    }
+    case 'ConditionalExpression':
+      return keySeen(node.consequent as Node) && keySeen(node.alternate as Node);
+    case 'LogicalExpression':
+      return keySeen(node.left as Node) && keySeen(node.right as Node);
+    default:
+      return false;
+  }
+}
+
+const nameOf = (node: Node | undefined): string | undefined =>
+  node?.type === 'Identifier' || node?.type === 'JSXIdentifier'
+    ? (node.name as string)
+    : node?.type === 'Literal'
+      ? String(node.value)
+      : undefined;
+
+/**
+ * The reads in a module whose key a pruned build can't see, as in `t(item.key)` or `<Text id={id} />`:
+ * the build would leave the message out, and the key would show in its place. A build has no types,
+ * so readers are known by name: `t` and `rich` called alone or as methods, and any name imported here,
+ * which may be bound to one elsewhere (`aliases`); the caller keeps those whose name turns out to be a
+ * reader. `<Text>` counts where it is imported from `prosebook/react`. Lines marked `prosebook-ignore`
+ * are skipped.
+ */
+export function unseenReads(file: string, code: string): Reads {
+  const { program } = parseSync(file, code);
+  const { position, ignored } = lineIndex(code);
+  const unseen: UnseenRead[] = [];
+  const aliases = new Set<string>();
+  /** Local name to the name it was imported by. */
+  const imported = new Map<string, string>();
+  const texts = new Set<string>();
+  for (const statement of (program as unknown as Node).body as Node[]) {
+    if (statement.type !== 'ImportDeclaration') continue;
+    for (const s of statement.specifiers as Node[]) {
+      const name = s.type === 'ImportSpecifier' ? nameOf(s.imported as Node) : undefined;
+      const local = nameOf(s.local as Node);
+      if (name === undefined || local === undefined) continue;
+      if (name === 'Text' && (statement.source as Node).value === 'prosebook/react') texts.add(local);
+      else imported.set(local, name);
+    }
+  }
+  const report = (node: Node, name: string) => {
+    const at = position(node.start);
+    if (ignored(at.line)) return;
+    const text = code.slice(node.start, node.end).replace(/\s+/g, ' ');
+    unseen.push({ file, ...at, name, code: text.length > 80 ? `${text.slice(0, 77)}...` : text });
+  };
+  // Calls by a plain name, kept until every alias in the module is known.
+  const calls: [Node, string][] = [];
+  const visit = (node: Node) => {
+    if (node.type === 'ObjectPattern') {
+      for (const p of node.properties as Node[]) {
+        if (p.type === 'Property' && READERS.has(nameOf(p.key as Node) ?? '') && (p.value as Node).type === 'Identifier')
+          aliases.add(nameOf(p.value as Node)!);
+      }
+    }
+    if (node.type === 'CallExpression' && !keySeen((node.arguments as Node[])[0])) {
+      const callee = node.callee as Node;
+      if (callee.type === 'Identifier') calls.push([node, callee.name as string]);
+      else if (callee.type === 'MemberExpression' && !callee.computed && READERS.has(nameOf(callee.property as Node) ?? ''))
+        report(node, nameOf(callee.property as Node)!);
+    }
+    if (node.type === 'JSXOpeningElement' && texts.has(nameOf(node.name as Node) ?? '')) {
+      const id = (node.attributes as Node[]).find((a) => a.type === 'JSXAttribute' && nameOf(a.name as Node) === 'id');
+      const value = id?.value as Node | null | undefined;
+      if (value !== null && value !== undefined && !keySeen(value.type === 'JSXExpressionContainer' ? (value.expression as Node) : value))
+        report(node, 'Text');
+    }
+    for (const value of Object.values(node)) {
+      if (Array.isArray(value)) {
+        for (const child of value) if (isNode(child)) visit(child);
+      } else if (isNode(value)) {
+        visit(value);
+      }
+    }
+  };
+  visit(program as unknown as Node);
+  for (const [node, local] of calls) {
+    if (READERS.has(local) || aliases.has(local)) report(node, local);
+    else if (imported.has(local)) report(node, imported.get(local)!);
+  }
+  unseen.sort((a, b) => (a.line === b.line ? a.column - b.column : a.line - b.line));
+  return { unseen, aliases };
+}

package/src/tools/syntax.ts

@@ -0,0 +1,24 @@
+/**
+ * The two syntaxes a catalog can be written in, behind one pair of calls: read a message into the
+ * element tree, and write a tree back out. Everything past reading works on the tree alone.
+ */
+import { parseMessage, type MessageFormatElement } from './icu.ts';
+import { toMf1 } from './mf1.ts';
+import { parseMf2, toMf2 } from './mf2.ts';
+
+export type MessageFormatVersion = 1 | 2;
+
+export const SYNTAX_NAME: Record<MessageFormatVersion, string> = { 1: 'ICU MessageFormat', 2: 'MessageFormat 2' };
+
+/**
+ * A message as the element tree. MF2 is read per locale: which variants a value can reach depends
+ * on the locale's plural rules (see `matchTree` in mf2.ts).
+ */
+export function readMessage(version: MessageFormatVersion, text: string, locale: string): MessageFormatElement[] {
+  return version === 2 ? parseMf2(text, locale) : parseMessage(text);
+}
+
+/** A tree as text in the given syntax. Throws, saying what the syntax can't express, where it can't. */
+export function writeMessage(version: MessageFormatVersion, ast: MessageFormatElement[], locale: string): string {
+  return version === 2 ? toMf2(ast, locale) : toMf1(ast);
+}

package/src/tools/types.ts

@@ -0,0 +1,137 @@
+/**
+ * What kind of value an expression in JSX holds, so `extract` can write `{n, number}` for a number and
+ * `{d, date, medium}` for a date instead of a bare `{x}`, which is text and prints a number unformatted.
+ *
+ * The project's own TypeScript (7 or later, through its `unstable/async` API) answers where it can; the
+ * syntax of the expression answers where it can't: a JavaScript project, a file outside any tsconfig,
+ * or no TypeScript installed. Not the `unstable/sync` API: on Linux and macOS it reads the compiler's
+ * pipes through a Node internal (`stdout._handle.fd`) that Bun doesn't have, so under Bun it throws.
+ */
+import { existsSync } from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
+
+import type * as TsAsync from 'typescript/unstable/async';
+import type { Checker, Type } from 'typescript/unstable/async';
+
+// oxlint-disable-next-line typescript/no-explicit-any -- a loose ESTree node: the walk reads whichever fields each node type has
+type Node = { type: string; start: number; end: number; [field: string]: any };
+
+type TsNode = { pos: number; end: number; forEachChild(visit: (child: TsNode) => unknown): unknown };
+
+/** `node` is markup or an object, which a message can't hold as text. */
+export type ValueKind = 'string' | 'number' | 'date' | 'node' | 'unknown';
+
+export interface Span {
+  start: number;
+  end: number;
+}
+
+export interface TypeOracle {
+  /** Where the answers come from, for the report. */
+  readonly source: string;
+  /** The kind of each span's expression in `file`, or nothing where the oracle can't say. */
+  kinds(file: string, spans: readonly Span[]): Promise<(ValueKind | undefined)[]>;
+  close(): Promise<void>;
+}
+
+/** The nearest `tsconfig.json` at or above the file's folder. */
+function tsconfigFor(file: string): string | undefined {
+  for (let dir = dirname(resolve(file)); ; dir = dirname(dir)) {
+    const candidate = join(dir, 'tsconfig.json');
+    if (existsSync(candidate)) return candidate;
+    if (dirname(dir) === dir) return undefined;
+  }
+}
+
+// TypeFlags, as the TypeScript 7 API numbers them.
+const ANY_OR_UNKNOWN = 3;
+const NUMBER_LIKE = 67648 | 4224;
+const STRING_LIKE = 12583968;
+const UNION = 134217728;
+const OBJECT = 1048576;
+
+/**
+ * An oracle over the projects the files belong to, or nothing where TypeScript 7 isn't installed or
+ * none of the files has a tsconfig. One compiler process serves every project and is closed by `close`.
+ */
+export async function typescriptOracle(root: string, files: readonly string[]): Promise<TypeOracle | undefined> {
+  const configs = [...new Set(files.map(tsconfigFor).filter((c) => c !== undefined))];
+  if (configs.length === 0) return undefined;
+  let ts: typeof TsAsync;
+  try {
+    ts = await import('typescript/unstable/async');
+  } catch {
+    return undefined;
+  }
+  let api: InstanceType<typeof ts.API>;
+  let snapshot: Awaited<ReturnType<InstanceType<typeof ts.API>['updateSnapshot']>>;
+  try {
+    api = new ts.API({ cwd: root });
+    snapshot = await api.updateSnapshot({ openProjects: configs });
+  } catch {
+    return undefined;
+  }
+
+  const classify = async (checker: Checker, type: Type | undefined): Promise<ValueKind | undefined> => {
+    if (type === undefined) return undefined;
+    const t = (await checker.getNonNullableType(type)) ?? type;
+    if ((t.flags & ANY_OR_UNKNOWN) !== 0) return 'unknown';
+    if ((t.flags & UNION) !== 0) {
+      const members = (await (t as Type & { getTypes(): Promise<readonly Type[] | undefined> }).getTypes()) ?? [];
+      const kinds = new Set(await Promise.all(members.map((m) => classify(checker, m))));
+      if (kinds.size === 1) return [...kinds][0];
+      return kinds.has('node') ? 'node' : 'unknown';
+    }
+    if ((t.flags & NUMBER_LIKE) !== 0) return 'number';
+    if ((t.flags & STRING_LIKE) !== 0) return 'string';
+    if ((await checker.typeToString(t)) === 'Date') return 'date';
+    if ((t.flags & OBJECT) !== 0) return 'node';
+    return 'unknown';
+  };
+
+  return {
+    source: 'TypeScript',
+    async kinds(file, spans) {
+      const project = await snapshot.getDefaultProjectForFile(resolve(file));
+      const sourceFile = await project?.program.getSourceFile(resolve(file));
+      if (project === undefined || sourceFile === undefined) return spans.map(() => undefined);
+      // The smallest node covering each span is the expression itself.
+      const nodes = spans.map(({ start, end }) => {
+        let node: TsNode = sourceFile;
+        for (;;) {
+          let next: TsNode | undefined;
+          node.forEachChild((child) => {
+            if (next === undefined && child.pos <= start && child.end >= end) next = child;
+          });
+          if (next === undefined) return node;
+          node = next;
+        }
+      });
+      const types = await project.checker.getTypeAtLocation(nodes as never[]);
+      return Promise.all(types.map((type) => classify(project.checker, type)));
+    },
+    close() {
+      return api.close();
+    },
+  };
+}
+
+/** The kind an expression's syntax gives away, where it does: `items.length`, `a * b`, `new Date()`. */
+export function syntaxKind(expr: Node): ValueKind | undefined {
+  switch (expr.type) {
+    case 'Literal':
+      return typeof expr.value === 'number' ? 'number' : typeof expr.value === 'string' ? 'string' : undefined;
+    case 'TemplateLiteral':
+      return 'string';
+    case 'MemberExpression':
+      return !expr.computed && ['length', 'size', 'count'].includes(expr.property?.name as string) ? 'number' : undefined;
+    case 'BinaryExpression':
+      return ['-', '*', '/', '%', '**'].includes(expr.operator as string) ? 'number' : undefined;
+    case 'NewExpression':
+      return expr.callee?.type === 'Identifier' && expr.callee.name === 'Date' ? 'date' : undefined;
+    case 'CallExpression':
+      return expr.callee?.type === 'MemberExpression' && expr.callee.object?.name === 'Math' ? 'number' : undefined;
+    default:
+      return undefined;
+  }
+}