unknown-knowledge 2.1.0

package/payload/engine/derive.js

@@ -0,0 +1,34 @@
+#!/usr/bin/env node
+/**
+ * engine/derive.js — the entry shim (UCS-956).
+ *
+ * This file statically imports NOTHING. That is its entire job.
+ *
+ * Node exits 1 on an unhandled ES module load error, and exit 1 means FINDINGS
+ * (PRD §5). A SyntaxError in a lib/ module, or a missing `js-yaml`, must not
+ * tell an agent that the check ran and found problems. So the engine is reached
+ * only through `import()`, where a load failure is an ordinary catchable
+ * rejection rather than a process-level crash.
+ *
+ * Both specifiers are string literals naming the engine's own files. D-014
+ * forbids importing REPO CONTENT — the client's code — and nothing here can
+ * name it: there is no variable to point somewhere else.
+ *
+ * The command lives in commands/derive.js. Its invocation path is unchanged.
+ */
+try {
+  const [{ boot }, command] = await Promise.all([
+    import('./lib/boot.js'),
+    import('./commands/derive.js'),
+  ]);
+  // exitCode, never process.exit(): exit() drops queued async stdout writes, so
+  // piped --json output would truncate at the pipe buffer — corrupt output
+  // wearing a clean exit code. Node exits on its own once stdout drains.
+  process.exitCode = await boot('derive', command);
+} catch (error) {
+  // The engine could not be loaded, so no check ran. Exit 2 — never 1.
+  // Hardcoded, because reading it from lib/exit-codes.js is the very thing
+  // that may have just failed.
+  process.stderr.write(`derive: internal failure — the engine could not be loaded\n${error?.stack ?? error}\n`);
+  process.exitCode = 2;
+}

package/payload/engine/ingest.js

@@ -0,0 +1,34 @@
+#!/usr/bin/env node
+/**
+ * engine/ingest.js — the entry shim (UCS-956).
+ *
+ * This file statically imports NOTHING. That is its entire job.
+ *
+ * Node exits 1 on an unhandled ES module load error, and exit 1 means FINDINGS
+ * (PRD §5). A SyntaxError in a lib/ module, or a missing `js-yaml`, must not
+ * tell an agent that the check ran and found problems. So the engine is reached
+ * only through `import()`, where a load failure is an ordinary catchable
+ * rejection rather than a process-level crash.
+ *
+ * Both specifiers are string literals naming the engine's own files. D-014
+ * forbids importing REPO CONTENT — the client's code — and nothing here can
+ * name it: there is no variable to point somewhere else.
+ *
+ * The command lives in commands/ingest.js.
+ */
+try {
+  const [{ boot }, command] = await Promise.all([
+    import('./lib/boot.js'),
+    import('./commands/ingest.js'),
+  ]);
+  // exitCode, never process.exit(): exit() drops queued async stdout writes, so
+  // piped --json output would truncate at the pipe buffer — corrupt output
+  // wearing a clean exit code. Node exits on its own once stdout drains.
+  process.exitCode = await boot('ingest', command);
+} catch (error) {
+  // The engine could not be loaded, so nothing was adapted. Exit 2 — never 1.
+  // Hardcoded, because reading it from lib/exit-codes.js is the very thing
+  // that may have just failed.
+  process.stderr.write(`ingest: internal failure — the engine could not be loaded\n${error?.stack ?? error}\n`);
+  process.exitCode = 2;
+}

package/payload/engine/lib/anchor-signatures.js

@@ -0,0 +1,126 @@
+/**
+ * Anchor-candidate signatures (KK-25) — ONE regex table shared between the
+ * survey map's pre-scan and the extractor kinds (KK-08/09 import from here).
+ * A kind describes the shape of a declaration, not the file type (PRD §5.1):
+ * .tsx/.jsx are extensions handled by the TS/JS kinds, not separate kinds.
+ *
+ * Each signature is a cheap lexical sniff — "this file plausibly contains an
+ * anchor of this kind" — never a parse. Extraction (and the syntactic-envelope
+ * hard-error discipline) belongs to the extractor recipes; the survey map only
+ * surfaces candidates for agent triage. Patterns are stored as source strings
+ * so the table stays data (importable, testable, greppable), compiled by
+ * consumers. `pattern: null` marks a directory-shape kind (dir-modules) that
+ * is detected structurally, not by content.
+ *
+ * Sorted by kind; a kind may carry several signatures when one anchor shape
+ * lives in more than one file format (strings-keys: legacy `.strings` and the
+ * JSON `.xcstrings` catalog). DEEP-frozen — the entries AND their extensions
+ * arrays — so extending the kind set stays a KK-08/09-style change with
+ * fixtures, never an in-session mutation (D-005).
+ *
+ * Known limitation: legacy UTF-16 .strings files (BOM-marked, common in older
+ * Xcode exports) do not decode as UTF-8, so a content sniff cannot see their
+ * `"key" = "value"` lines. The survey map treats a UTF-16 BOM as "candidate by
+ * extension" instead of skipping silently — see sniffKinds() in survey-map.js.
+ */
+const TS_EXTENSIONS = Object.freeze(['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs']);
+const YAML_EXTENSIONS = Object.freeze(['.yaml', '.yml']);
+
+export const ANCHOR_SIGNATURES = Object.freeze([
+  {
+    kind: 'dir-modules',
+    extensions: null, // directory shape: siblings sharing one extension
+    pattern: null,
+    flags: '',
+    reads: 'subfolder/file names, with pattern/strip options',
+  },
+  {
+    kind: 'json-keys',
+    extensions: ['.json'],
+    pattern: '^\\s*"[^"\\n]+"\\s*:',
+    flags: 'm',
+    reads: 'top-level keys of a JSON object',
+  },
+  {
+    kind: 'json-map-keys',
+    extensions: ['.json'],
+    pattern: '"[^"\\n]+"\\s*:\\s*\\{',
+    flags: 'm',
+    reads: 'keys under a dotted path in a JSON object',
+  },
+  {
+    kind: 'strings-keys',
+    extensions: ['.strings'],
+    pattern: '^\\s*"[^"\\n]+"\\s*=\\s*"',
+    flags: 'm',
+    reads: 'localization keys of a legacy `"key" = "value"` .strings table',
+  },
+  {
+    kind: 'strings-keys',
+    extensions: ['.xcstrings'],
+    pattern: '"strings"\\s*:\\s*\\{',
+    flags: 'm',
+    reads: 'localization keys of an .xcstrings JSON catalog ("strings" map)',
+  },
+  {
+    kind: 'swift-const-array',
+    extensions: ['.swift'],
+    pattern: '\\b(?:static\\s+)?let\\s+\\w+\\s*(?::\\s*\\[[^\\]\\n]*\\])?\\s*=\\s*\\[',
+    flags: 'm',
+    reads: 'static let / let array literals',
+  },
+  {
+    kind: 'swift-enum',
+    extensions: ['.swift'],
+    pattern: '\\benum\\s+\\w+\\s*(?::\\s*[^{\\n]+)?\\{',
+    flags: 'm',
+    reads: 'cases of a Swift enum',
+  },
+  {
+    kind: 'ts-const-array',
+    extensions: TS_EXTENSIONS,
+    pattern: '\\bexport\\s+(?:const|let|var)\\s+\\w+\\s*(?::\\s*[^=\\n]+)?=\\s*\\[',
+    flags: 'm',
+    reads: 'exported const array (works for JS too)',
+  },
+  {
+    kind: 'ts-enum',
+    extensions: TS_EXTENSIONS,
+    pattern: '\\b(?:export\\s+)?(?:const\\s+)?enum\\s+\\w+\\s*\\{',
+    flags: 'm',
+    reads: 'TS enum members',
+  },
+  {
+    kind: 'ts-object-keys',
+    extensions: TS_EXTENSIONS,
+    pattern: '\\bexport\\s+(?:const|let)\\s+\\w+\\s*(?::\\s*[^=\\n]+)?=\\s*\\{',
+    flags: 'm',
+    reads: 'keys of an exported object literal',
+  },
+  {
+    kind: 'ts-union',
+    extensions: TS_EXTENSIONS,
+    pattern: "\\btype\\s+\\w+\\s*=\\s*[^;\\n]*['\"][^'\"\\n]*['\"]\\s*\\|",
+    flags: 'm',
+    reads: 'string-literal union members',
+  },
+  {
+    kind: 'yaml-keys',
+    extensions: YAML_EXTENSIONS,
+    pattern: '^[A-Za-z_][\\w.-]*\\s*:',
+    flags: 'm',
+    reads: 'top-level keys of a YAML document',
+  },
+  {
+    kind: 'yaml-map-keys',
+    extensions: YAML_EXTENSIONS,
+    pattern: '^[A-Za-z_][\\w.-]*\\s*:\\s*$',
+    flags: 'm',
+    reads: 'keys under a dotted path in a YAML document',
+  },
+].map((sig) => {
+  // Deep freeze: a mutable (or aliased) extensions array would let a push
+  // onto one kind silently rewire every kind sharing the same array.
+  if (sig.extensions !== null) Object.freeze(sig.extensions);
+  return Object.freeze(sig);
+}));

package/payload/engine/lib/boot.js

@@ -0,0 +1,39 @@
+/**
+ * Load a command, then run it through the harness (UCS-956).
+ *
+ * `runCli` promises that a crash exits 2, never 1. It cannot keep that promise
+ * for a failure that happens BEFORE it loads: Node exits 1 on an unhandled ES
+ * module load error, so a SyntaxError in any lib/ module — or a missing
+ * `js-yaml` — used to make every engine surface exit 1, the FINDINGS code, with
+ * the harness never having run.
+ *
+ * That is the worst possible failure for the exit-code contract (PRD §5,
+ * D-011). An agent reads exit 1 as "the check ran and found problems", so it
+ * quarantines and continues — past a check that never ran. And it is MORE
+ * reachable than an ordinary crash, because the kit is seeded into a client
+ * repo (D-001): a partial copy, a corrupted file, or an uninstalled dependency
+ * lands exactly here.
+ *
+ * So the entry point of every surface is a shim that statically imports
+ * NOTHING, and reaches the engine only through `import()`, whose failure is an
+ * ordinary catchable rejection. Everything the engine is made of — including
+ * `cli.js`, and this module — loads inside that seam.
+ *
+ * This module is loaded dynamically too. If it cannot load, the shim's own
+ * catch reports it and exits 2.
+ */
+import process from 'node:process';
+import { runCli } from './cli.js';
+
+/**
+ * The shim performs the `import()` itself, with a string-literal specifier, and
+ * hands the loaded module here. Nothing in the engine ever imports a computed
+ * path — there is no variable that could be made to name client code (D-014).
+ *
+ * @param {string} name the command's name, for its messages
+ * @param {{ main: (argv: string[]) => number | Promise<number>, USAGE: string }} command
+ * @returns {Promise<number>} an exit code; 1 only if the command returned it
+ */
+export async function boot(name, command) {
+  return runCli(name, command.main, { usage: command.USAGE });
+}

package/payload/engine/lib/call-numbers.js

@@ -0,0 +1,133 @@
+/**
+ * Synthesized call numbers (UCS-1158) — a display string, and never an identity.
+ *
+ * A call number is what a browse tree PRINTS beside a leaf so a reader scanning
+ * a shelf has a compact handle for where that leaf sits in this particular
+ * projection. It is derived, every time, from the leaf's facets. It is not
+ * stored, not indexed, not resolvable, and not citable.
+ *
+ * That last part is the whole reason this module is careful rather than
+ * cosmetic. The store this engine serves spent its first life with a Dewey-style
+ * dotted notation that fused identity, location and browse order into one
+ * string, and the cost of that fusion is what UCS-1147 spent a migration
+ * undoing: a citation pointed at a position, so moving a leaf broke every
+ * reference to it. A synthesized call number is the SAME SHAPE of string
+ * arriving through a different door — positional, human-friendly, and highly
+ * tempting to paste into a `see-also`. If that paste ever resolved, the whole
+ * inversion would quietly reverse itself one convenient citation at a time.
+ *
+ * So the defense is structural rather than advisory, in three parts:
+ *
+ *   1. The grammar cannot be mistaken for an id. `L-000117` is the accession;
+ *      a call number looks like `DES-COM/REF·L-000117` — uppercase facet
+ *      abbreviations, a slash, and a MIDDLE DOT that no id grammar accepts.
+ *      `assertNoIdSpaceMatches` (below) proves it against every compiled id
+ *      space in ID_GRAMMARS rather than against a remembered list, so a new id
+ *      space cannot silently start matching call numbers.
+ *   2. Nothing accepts one. The citation grammar is `leaf-ref`, accession-only
+ *      since UCS-1147, so a call number in a citation-shaped field is already a
+ *      finding — no new check was needed, and tests/derived-call-numbers pins
+ *      that it stays that way.
+ *   3. The accession travels INSIDE the call number, as its suffix. A reader
+ *      who copies the whole string still carries the identity, and a reader who
+ *      wants the citable part can see exactly which half it is. The tempting
+ *      thing to paste therefore contains the correct thing to paste.
+ *
+ * The call number is NOT stable across projections, and that is a feature: the
+ * same leaf reads `DES-COM/REF·L-000117` in the domain-first tree and
+ * `REF/DES-COM·L-000117` in the form-first one, because the call number
+ * describes a POSITION in a projection, and that leaf holds two of them. A
+ * string that changed meaning between trees while looking stable would be the
+ * worst of both worlds; one that visibly reorders says what it is.
+ */
+import { idPattern, ID_GRAMMARS } from './id-grammars.js';
+
+/**
+ * The separator between the positional part and the accession.
+ *
+ * U+00B7 MIDDLE DOT, chosen because no id grammar in this engine accepts it and
+ * none plausibly will: the id spaces are all `[A-Z]-[0-9]` or dotted decimals,
+ * and a middle dot is not a character an author reaches for when minting an id.
+ * It is also visually distinct from the ASCII period that the LEGACY notation
+ * used, so `362.1` and `DES-COM/REF·L-000117` cannot be skim-confused.
+ */
+export const CALL_NUMBER_SEPARATOR = '·';
+
+/** How many characters of each facet segment survive into the abbreviation. */
+const SEGMENT_WIDTH = 3;
+
+/** What a facet segment abbreviates to when it is missing entirely. */
+const ABSENT_SEGMENT = 'UNC';
+
+/**
+ * Abbreviate one facet segment for display.
+ *
+ * Uppercased and truncated to three characters, with non-alphanumerics dropped
+ * so a segment like `components` reads `COM` rather than `COM-`. Deliberately
+ * LOSSY: two segments can abbreviate to the same three letters, and that is
+ * fine because nothing resolves through this string — the accession suffix is
+ * what disambiguates, and it is exact. Making the abbreviation collision-free
+ * would mean minting and remembering it, which is precisely the property that
+ * would turn a display string back into an identity.
+ *
+ * An absent or non-string segment reads `UNC` (unclassified) rather than
+ * collapsing to an empty string. An empty abbreviation would make
+ * `/REF·L-000117` — a call number with a hole where a facet should be, which
+ * reads as a formatting bug rather than as the fact that the leaf declares no
+ * domain. The tree also demotes such a leaf; the call number says why.
+ *
+ * @param {unknown} segment one facet path segment
+ * @returns {string} the display abbreviation
+ */
+export function abbreviateSegment(segment) {
+  if (typeof segment !== 'string') return ABSENT_SEGMENT;
+  const cleaned = segment.replace(/[^a-zA-Z0-9]/g, '').toUpperCase();
+  return cleaned === '' ? ABSENT_SEGMENT : cleaned.slice(0, SEGMENT_WIDTH);
+}
+
+/**
+ * Synthesize the call number for one leaf, in one projection.
+ *
+ * The positional half is the projection's own path — the same segments the tree
+ * nested this leaf under, abbreviated and slash-joined — so the call number and
+ * the position it describes cannot disagree: both are built from one array.
+ * That is the invariant worth protecting, and it is why this function takes the
+ * path rather than re-deriving it from the leaf's facets. A second derivation
+ * would be a second chance to differ.
+ *
+ * The identity half is the accession, verbatim. A leaf that mints none (which
+ * the schema refuses, but the derived layer never gates on store health) prints
+ * the absent marker instead, so the string stays well-formed and the missing
+ * identity is visible rather than swallowed.
+ *
+ * @param {string[]} path the projection's facet path for this leaf
+ * @param {string|null} accession the leaf's accession id, or null
+ * @returns {string} the synthesized display string
+ */
+export function synthesizeCallNumber(path, accession) {
+  const positional = path.length
+    ? path.map(abbreviateSegment).join('/')
+    : ABSENT_SEGMENT;
+  const identity = typeof accession === 'string' && accession !== '' ? accession : ABSENT_SEGMENT;
+  return `${positional}${CALL_NUMBER_SEPARATOR}${identity}`;
+}
+
+/**
+ * Prove a synthesized call number is not an id in ANY space this engine knows.
+ *
+ * Enumerates ID_GRAMMARS rather than checking a remembered list of spaces, so
+ * adding an id space that happened to accept middle dots would fail here at the
+ * moment it was added rather than the first time somebody cited a shelf label.
+ * Returns the offending spaces rather than throwing: the caller decides whether
+ * that is a test failure or a finding, and a library that threw would make the
+ * check unusable from the one place it matters most — a test asserting the
+ * property holds for every call number a real store generates.
+ *
+ * @param {string} callNumber a synthesized display string
+ * @returns {string[]} the id spaces that would accept it — empty when safe
+ */
+export function idSpacesMatching(callNumber) {
+  return Object.keys(ID_GRAMMARS)
+    .filter((space) => idPattern(space).test(callNumber))
+    .sort();
+}

package/payload/engine/lib/cli.js

@@ -0,0 +1,147 @@
+/**
+ * The CLI shell, once (UCS-944).
+ *
+ * Nine command-line surfaces each hand-roll the same flag grammar, the same
+ * usage-error type, and the same mapping from outcome to exit code. The copies
+ * have already drifted — the `--concepts` filter meant two different things to
+ * two validators until UCS-935 settled it.
+ *
+ * The safety rule they restate is the load-bearing one:
+ *
+ *   EXIT 1 MEANS FINDINGS. A crash must never wear it.
+ *
+ * An engine that dies mid-check and exits 1 tells an agent riding the exit-code
+ * contract that the check RAN and found problems — so it quarantines and
+ * continues, past a check that never ran (PRD §5, D-011). Today that rule is
+ * enforced by each author remembering to copy a catch block, and this session
+ * found FIVE surfaces where the copy was wrong or missing.
+ *
+ * `runCli` owns the epilogue instead: a usage error reports and exits 2, and
+ * ANY unexpected throw reports and exits 2. Exit 1 is reachable only by a main
+ * that deliberately returns it, having actually run and actually found things.
+ *
+ * Every surface reaches this through `lib/boot.js`, from an entry shim that
+ * statically imports nothing (UCS-956). The nine hand-written shells that this
+ * replaced are gone (UCS-952).
+ */
+import process from 'node:process';
+import { EXIT_CODES } from './exit-codes.js';
+import { UsageError } from './usage-error.js';
+
+export { UsageError } from './usage-error.js';
+export { EngineRefusal, rethrowIfBug } from './engine-refusal.js';
+
+/**
+ * Parse the flag grammar every engine CLI shares.
+ *
+ * Both conventional spellings work — `--flag value` and `--flag=value` — and
+ * everything the grammar cannot account for is a UsageError, never a guess:
+ * an unknown flag, a stray positional where none is allowed, a value flag with
+ * no value, a boolean flag given one, or an empty value (`--root=` would
+ * otherwise resolve to the current directory, answering about a repo nobody
+ * named).
+ *
+ * @param {string[]} argv arguments after the script name
+ * @param {object} spec
+ * @param {string[]} [spec.boolean] flags that take no value
+ * @param {string[]} [spec.value] flags taking one value; last wins
+ * @param {string[]} [spec.repeatable] value flags that accumulate into an array
+ * @param {string[]} [spec.allowEmpty] value flags for which "" is a legal value
+ *   — preflight reads an empty `--concepts` as store-health-only (PRD §7)
+ * @param {boolean} [spec.positionals] whether bare arguments are allowed
+ * @returns {{ options: object, positionals: string[] }}
+ */
+export function parseArgs(argv, spec = {}) {
+  const booleans = new Set(spec.boolean ?? []);
+  const values = new Set(spec.value ?? []);
+  const repeatables = new Set(spec.repeatable ?? []);
+  const allowEmpty = new Set(spec.allowEmpty ?? []);
+  const known = new Set([...booleans, ...values, ...repeatables]);
+  const options = {};
+  const positionals = [];
+
+  for (let i = 0; i < argv.length; i += 1) {
+    const arg = argv[i];
+    if (!arg.startsWith('--')) {
+      if (!spec.positionals) {
+        throw new UsageError(`unexpected argument ${JSON.stringify(arg)} — this CLI takes flags only`);
+      }
+      positionals.push(arg);
+      continue;
+    }
+
+    const eq = arg.indexOf('=');
+    const flag = eq === -1 ? arg : arg.slice(0, eq);
+    const name = flag.slice(2);
+    if (!known.has(name)) throw new UsageError(`unknown flag ${flag}`);
+
+    if (booleans.has(name)) {
+      if (eq !== -1) throw new UsageError(`${flag} takes no value`);
+      options[name] = true;
+      continue;
+    }
+
+    let value;
+    if (eq !== -1) {
+      // `--flag=value` carries its value unambiguously, so a value that looks
+      // like a flag is legal here — this spelling is the escape hatch for one.
+      value = arg.slice(eq + 1);
+    } else {
+      const next = argv[i + 1];
+      // In the space form a `--`-prefixed token is the next flag, not a value.
+      if (next === undefined || next.startsWith('--')) {
+        throw new UsageError(`${flag} requires a value`);
+      }
+      value = next;
+      i += 1;
+    }
+    // An empty value is as valueless as none, in EITHER spelling. Both `--root=`
+    // and `--root ""` arrive when a shell expands an unset variable, and both
+    // would otherwise resolve to the current directory — answering about a repo
+    // nobody named.
+    // A flag may declare "" meaningful — preflight's empty `--concepts` selects
+    // store-health-only. Everything else treats it as no value at all.
+    if (value === '' && !allowEmpty.has(name)) throw new UsageError(`${flag} requires a value`);
+
+    if (repeatables.has(name)) (options[name] ??= []).push(value);
+    else options[name] = value;
+  }
+
+  return { options, positionals };
+}
+
+/**
+ * Run a command's main and own the epilogue: nothing it throws can exit 1.
+ *
+ * `main` returns an exit code, or throws. A UsageError reports itself with the
+ * usage line; anything else is an engine failure — the command did not finish,
+ * so it cannot have findings. Both exit 2.
+ *
+ * Returns the code rather than setting it, so the caller stays in charge of
+ * `process.exitCode`. Never calls `process.exit()`: that drops queued async
+ * stdout writes, truncating piped `--json` at the pipe buffer and yielding
+ * corrupt JSON with a clean exit code.
+ *
+ * @param {string} name the command's name, for its messages
+ * @param {(argv: string[]) => number | Promise<number>} main
+ * @param {object} opts
+ * @param {string} opts.usage the usage line printed on a UsageError
+ * @param {string[]} [opts.argv]
+ * @param {{ write: (s: string) => unknown }} [opts.stderr]
+ * @returns {Promise<number>} an exit code; 1 only if `main` returned it
+ */
+export async function runCli(name, main, { usage, argv = process.argv.slice(2), stderr = process.stderr } = {}) {
+  try {
+    return await main(argv);
+  } catch (error) {
+    if (error instanceof UsageError) {
+      stderr.write(`${name}: ${error.message}\n${usage}\n`);
+      return EXIT_CODES.FAILURE;
+    }
+    // The command did not finish. It cannot have findings, so it must not wear
+    // the FINDINGS code — an agent would quarantine-and-continue past a check
+    // that never ran.
+    stderr.write(`${name}: internal failure — the command did not complete\n${error?.stack || error?.message || String(error)}\n`);
+    return EXIT_CODES.FAILURE;
+  }
+}