unknown-knowledge 2.1.0

package/payload/engine/commands/ingest.js

@@ -0,0 +1,124 @@
+/**
+ * Ingest (UCS-1153) — the format-adapter seam at the command line.
+ *
+ * One document in, one intermediate representation out: ordered blocks with
+ * kinds and source locators, normalized from md / txt / html / pdf by the
+ * versioned adapters in lib/format-adapters.js. Everything downstream reads
+ * the IR; nothing downstream re-reads the original bytes.
+ *
+ * The seam exists as its own surface because the whole point is that it is
+ * TESTABLE AND COMPOSABLE: the document coverage map (UCS-1156) is built on
+ * this output, and an agent can run it directly to see exactly what the
+ * engine believes a document contains — the same way `survey-map` shows what
+ * it believes a repo contains.
+ *
+ * EXIT CODES (PRD §5, D-011). This surface has no findings to report — it
+ * either produces an IR or it does not:
+ *   0 — the document was adapted; the IR is on stdout.
+ *   2 — the format has no adapter, the content is outside the adapter's
+ *       envelope, or the file could not be read.
+ *
+ * There is no exit 1 and no partial IR. An unsupported format is a HARD ERROR
+ * WITH CONDUCT, never a best-effort parse: a silent partial poisons what the
+ * team believes was reviewed, which is worse than no review at all (D-005 /
+ * D-012, the false-all-clear class). The refusal names the format and states
+ * every way forward — convert and resubmit, author an adapter, or convert
+ * scanned content upstream and resubmit the text.
+ */
+import { readFileSync } from 'node:fs';
+import { resolve } from 'node:path';
+import { EXIT_CODES } from '../lib/exit-codes.js';
+import { parseArgs as parseFlags, rethrowIfBug, UsageError } from '../lib/cli.js';
+import { AdaptError, UnsupportedFormatError, adapt, adapterFor } from '../lib/format-adapters.js';
+
+export const USAGE = 'usage: node payload/engine/ingest.js <document> [--json]';
+
+function parseArgs(argv) {
+  const { options, positionals } = parseFlags(argv, {
+    boolean: ['json'],
+    positionals: true,
+  });
+  if (positionals.length === 0) {
+    throw new UsageError('nothing to ingest — name one document to adapt');
+  }
+  if (positionals.length > 1) {
+    // Two documents would produce two IRs with no honest way to say which
+    // blocks came from which; the caller runs the command twice.
+    throw new UsageError(`unexpected argument ${JSON.stringify(positionals[1])} — ingest adapts one document at a time`);
+  }
+  return { document: positionals[0], json: !!options.json };
+}
+
+/**
+ * Render the IR for a human: the provenance line, then one line per block.
+ * The locator is printed in the adapter's own scheme (line-based for text
+ * formats, page/object for pdf) — the reader needs to find the block in the
+ * source, and those are the coordinates the source actually has.
+ */
+function printHuman(ir, document) {
+  const locate = ({ line, endLine, page, object }) =>
+    page === undefined
+      ? `L${line}${endLine !== line ? `-${endLine}` : ''}`
+      : `p${page}#${object}`;
+  // Folded, not spread: `Math.max(...blocks.map(…))` passes one argument per
+  // block, and a long document would exceed the engine's argument limit and
+  // throw — a crash on the happy path, from formatting alone.
+  let width = 0;
+  for (const b of ir.blocks) width = Math.max(width, locate(b.locator).length);
+  const lines = [
+    `${document}: ${ir.blocks.length} block(s) via ${ir.adapter} (${ir.hash})`,
+    ...ir.blocks.map((b) => {
+      const text = b.text.replace(/\n/g, ' ');
+      const excerpt = text.length > 72 ? `${text.slice(0, 71)}…` : text;
+      return `  ${locate(b.locator).padEnd(width)}  ${b.kind.padEnd(10)}  ${excerpt}`;
+    }),
+  ];
+  process.stdout.write(`${lines.join('\n')}\n`);
+}
+
+/**
+ * CLI entry. Deterministic by construction: the IR is a pure function of the
+ * document's bytes, so two runs over the same file are byte-identical.
+ */
+export function main(argv) {
+  const opts = parseArgs(argv); // a UsageError reaches the harness
+
+  // DISPATCH BEFORE READ, deliberately. Whether the bytes exist is irrelevant
+  // when no adapter claims the format: reading first would answer a `.docx`
+  // submission with "cannot read", burying the conduct the submitter needs
+  // behind an incidental filesystem detail.
+  try {
+    adapterFor(opts.document);
+  } catch (error) {
+    rethrowIfBug(error); // a bug is not a refusal — the harness prints its stack
+    if (!(error instanceof UnsupportedFormatError)) throw error;
+    process.stderr.write(`error: ${error.message}\n`);
+    return EXIT_CODES.FAILURE;
+  }
+
+  const path = resolve(opts.document);
+  let bytes;
+  try {
+    bytes = readFileSync(path);
+  } catch (error) {
+    rethrowIfBug(error);
+    process.stderr.write(`ingest: cannot read ${opts.document}: ${error.message}\n`);
+    return EXIT_CODES.FAILURE;
+  }
+
+  let ir;
+  try {
+    ir = adapt(opts.document, bytes);
+  } catch (error) {
+    rethrowIfBug(error);
+    if (!(error instanceof UnsupportedFormatError || error instanceof AdaptError)) throw error;
+    // The refusal and its conduct go to stderr, and NOTHING goes to stdout:
+    // a caller piping stdout must receive no IR at all, not a truncated one.
+    process.stderr.write(`error: ${error.message}\n`);
+    return EXIT_CODES.FAILURE;
+  }
+
+  if (opts.json) process.stdout.write(`${JSON.stringify({ document: opts.document, ...ir }, null, 2)}\n`);
+  else printHuman(ir, opts.document);
+  return EXIT_CODES.CLEAN;
+}

package/payload/engine/commands/log-entry.js

@@ -0,0 +1,85 @@
+/**
+ * engine/log-entry.js (KK-13) — CLI over lib/log-entry.js so agents append
+ * and transition fragment-based log entries (PRD §3.4, D-010) without
+ * hand-editing YAML.
+ *
+ *   node engine/log-entry.js create --log findings --date 2026-07-08 \
+ *     --entry '{"trigger":"correction","summary":"K-210 stale per src/tokens.ts"}'
+ *
+ *   node engine/log-entry.js transition --file logs/findings/2026-07-08-a3f2b9c4.yaml \
+ *     --to proposed --date 2026-07-09 [--reason "..."]
+ *
+ * --date is mandatory (injectable dates — the helper never reads the wall
+ * clock, PRD §5); --root defaults to the cwd. Prints the fragment path and
+ * entry as JSON on stdout. Exit codes per PRD §5: 0 clean, 2 failure (an
+ * illegal transition or invalid entry is a hard error, never a silent pass).
+ *
+ * Capture content policy (§3.4): entries carry concept IDs and file paths
+ * only — never verbatim user text or secrets.
+ */
+import process from 'node:process';
+import { createEntry, transitionStatus, LOGS } from '../lib/log-entry.js';
+import { EXIT_CODES } from '../lib/exit-codes.js';
+import { parseArgs as parseFlags, rethrowIfBug, UsageError } from '../lib/cli.js';
+
+export const USAGE = `usage:
+  log-entry.js create --log <${Object.keys(LOGS).join('|')}> --date YYYY-MM-DD --entry '<json fields>' [--suffix hhhhhhhh] [--root dir]
+  log-entry.js transition --file logs/<log>/<entry>.yaml --to <status> --date YYYY-MM-DD [--reason "..."] [--root dir]`;
+
+/** Flags each command understands — a typo (--sufix) is an error, never silence. */
+const KNOWN_FLAGS = Object.freeze({
+  create: Object.freeze(['log', 'date', 'entry', 'suffix', 'root']),
+  transition: Object.freeze(['file', 'to', 'date', 'reason', 'root']),
+});
+
+function parseArgs(argv) {
+  const [command, ...rest] = argv;
+  if (!Object.hasOwn(KNOWN_FLAGS, command ?? '')) {
+    throw new UsageError(`unknown command ${JSON.stringify(command)}`);
+  }
+  // The grammar is per-subcommand: `--file` is meaningful to transition and a
+  // typo to create. A flag the subcommand does not know is an error, never
+  // silence.
+  const { options } = parseFlags(rest, { value: KNOWN_FLAGS[command] });
+  return { command, options };
+}
+
+/** @returns {number} an exit code */
+export function main(argv) {
+  const { command, options } = parseArgs(argv); // a UsageError reaches the harness
+  const root = options.root ?? process.cwd();
+
+  let fields = null;
+  if (command === 'create') {
+    try {
+      fields = JSON.parse(options.entry ?? '');
+    } catch {
+      // fall through to the shape check below
+    }
+    // JSON.parse also accepts null/scalars/arrays — only a plain object is
+    // a set of kind-specific fields.
+    if (typeof fields !== 'object' || fields === null || Array.isArray(fields)) {
+      throw new UsageError('--entry must be a JSON object of kind-specific fields');
+    }
+  }
+
+  let result;
+  try {
+    result = command === 'create'
+      ? createEntry({ root, log: options.log, date: options.date, fields, suffix: options.suffix })
+      : transitionStatus({ root, file: options.file, to: options.to, date: options.date, reason: options.reason });
+  } catch (error) {
+    rethrowIfBug(error); // a bug is not a refusal — the harness prints its stack
+    // An EXPECTED refusal (an illegal transition, an invalid entry, an unknown
+    // log): a hard error, never a silent pass (PRD §5). Exit 2 — the entry was
+    // not written, so nothing was found and nothing was logged.
+    process.stderr.write(`log-entry: ${error.message}\n`);
+    return EXIT_CODES.FAILURE;
+  }
+
+  const { file, entry } = result;
+  // exitCode, never process.exit(): exit() drops queued async stdout writes, so
+  // piped JSON output could truncate (corrupt output with exit 0).
+  console.log(JSON.stringify({ file, status: entry.status, entry }, null, 2));
+  return EXIT_CODES.CLEAN;
+}

package/payload/engine/commands/phoenix.js

@@ -0,0 +1,206 @@
+/**
+ * Phoenix events (KK-34) — apply a governed bulk re-taxonomy (UCS-1154).
+ *
+ * The first engine command that MUTATES a store, which is why almost all of it
+ * is about refusing to.
+ *
+ * A phoenix event rewrites a drifted subtree's facets in bulk from a
+ * leaf-granular mapping, bumps the `edition` of every leaf it moves — the only
+ * thing that ever bumps it in v2 — and lands as an ordinary PR. Citations are
+ * untouched by construction: identity is the accession id, and no event ever
+ * changes one, so nothing that cites a leaf notices it was reclassified.
+ *
+ * Two verbs, and the read-only one is the default:
+ *
+ *   --check   (default) plan the event and report. Writes nothing, ever.
+ *   --apply   plan the event, and write ONLY if the plan is completely clean.
+ *
+ * `--check` is the default because the safe reading of a bare `phoenix` is the
+ * one that cannot damage a store. An agent that runs the command without
+ * reading this file gets a dry run; mutating takes a word typed on purpose.
+ *
+ * Exit codes follow the engine contract (PRD §5, D-011), and the classification
+ * of a rejected mapping is the load-bearing choice here. A mapping that misses
+ * a leaf in its declared scope, or names an accession the store does not carry,
+ * is FINDINGS — exit 1. The precedent is the one every other surface follows:
+ * exit 1 means the run happened and the input has defects its author fixes,
+ * which is exactly what a bad mapping is. Exit 2 stays reserved for a run that
+ * never happened (unreadable store, unparseable mapping, no such event), where
+ * an agent must not conclude anything about the store at all.
+ *
+ * A rejected event NEVER writes. The gate is total and it runs first: the whole
+ * mapping is planned against the whole store before a single byte is written,
+ * and any finding refuses the entire event. There is no partial apply, because
+ * a half-re-taxonomized store is worse than an untouched one — the next run
+ * cannot tell which leaves already moved.
+ */
+import { resolve } from 'node:path';
+import { locateKitRoot } from '../lib/kit-root.js';
+import {
+  loadStores, storeHealth, healthSummary, PHOENIX_DIR, PHOENIX_STORES,
+} from '../lib/load-stores.js';
+import { CHECKS, applyRewrites, planEvent, phoenixPath } from '../lib/phoenix.js';
+import { compare } from '../lib/validate-record.js';
+import { EXIT_CODES } from '../lib/exit-codes.js';
+import { parseArgs as parseFlags, rethrowIfBug, UsageError } from '../lib/cli.js';
+
+export const USAGE = 'usage: node payload/engine/phoenix.js <event> [--apply] [--check] [--root <dir>] [--json]';
+
+function parseArgs(argv) {
+  const { options, positionals } = parseFlags(argv, {
+    boolean: ['json', 'apply', 'check'],
+    value: ['root'],
+    positionals: true,
+  });
+  if (positionals.length > 1) {
+    throw new UsageError(`unexpected argument ${JSON.stringify(positionals[1])} — one event at a time`);
+  }
+  if (!positionals.length) {
+    throw new UsageError(
+      `name the phoenix event to apply, e.g. P-001 (its mapping is ${phoenixPath(PHOENIX_STORES[0], '<event>')})`,
+    );
+  }
+  // Naming both verbs states two intentions and the difference between them is
+  // whether the store gets written to. Refusing is the only reading that cannot
+  // silently pick the destructive one.
+  if (options.apply && options.check) {
+    throw new UsageError('--apply and --check are the two verbs; name one');
+  }
+  return {
+    event: positionals[0],
+    root: resolve(options.root ?? '.'),
+    json: !!options.json,
+    apply: !!options.apply,
+  };
+}
+
+function render(payload) {
+  const { event, verb, findings, rewrites, carried } = payload;
+  const lines = [];
+  if (findings.length) {
+    lines.push(
+      `phoenix ${event} -> REFUSED — ${findings.length} finding(s); nothing was written`,
+      `checks run: ${CHECKS.join(', ')}`,
+    );
+    for (const f of findings) {
+      lines.push(`${f.severity}  ${f.code}  ${f.id}  ${f.file}  ${f.path}`, `    ${f.message}`);
+    }
+    lines.push('a phoenix event applies in full or not at all — fix every finding and re-run');
+    return lines;
+  }
+  const moved = `${rewrites.length} leaf/leaves rewritten, ${carried.length} carried forward`;
+  lines.push(
+    verb === 'apply'
+      ? `phoenix ${event} -> APPLIED — ${moved}`
+      : `phoenix ${event} -> would apply cleanly — ${moved} (nothing written; --apply to write)`,
+    `checks run: ${CHECKS.join(', ')}`,
+  );
+  for (const r of rewrites) {
+    lines.push(`  ${r.id}  ${r.from} -> ${r.to}  edition ${r.edition - 1} -> ${r.edition}  ${r.file}`);
+  }
+  for (const id of carried) lines.push(`  ${id}  carried forward — considered, unmoved, edition unchanged`);
+  lines.push('accession ids and citations are untouched by construction — nothing that cites these leaves moved');
+  return lines;
+}
+
+/**
+ * CLI entry. Exit codes per the engine contract (PRD §5): 0 the event is clean
+ * (and applied, under --apply), 1 the mapping was REFUSED with findings, 2 the
+ * run never happened.
+ *
+ * @param {string[]} argv
+ * @returns {number} an exit code
+ */
+export function main(argv) {
+  const opts = parseArgs(argv); // a UsageError reaches the harness
+
+  let model;
+  try {
+    model = loadStores(locateKitRoot(opts.root));
+  } catch (error) {
+    rethrowIfBug(error); // a bug is not a refusal — the harness prints its stack
+    process.stderr.write(`phoenix: ${error.message}\n`);
+    return EXIT_CODES.FAILURE;
+  }
+
+  // A store the loader rejects cannot be re-taxonomized: the model the plan
+  // would be built from is already known to be wrong, so every verdict about
+  // which leaves are in scope would be untrustworthy — and under --apply we
+  // would write from it. Exit 2: the event never ran.
+  if (!model.ok) {
+    const health = storeHealth(model);
+    process.stderr.write(
+      `phoenix: the store has ${health.errors.length} loader error(s); no event runs against a store that does not load\n`,
+    );
+    for (const d of health.errors) {
+      process.stderr.write(`  ${d.code}  ${d.file}${d.path ? `  ${d.path}` : ''}  ${d.message}\n`);
+    }
+    return EXIT_CODES.FAILURE;
+  }
+
+  // Which stores may carry events is the descriptor table's answer, not this
+  // command's: naming "knowledge" here would leave a second phoenix-capable
+  // store's events loaded into the model and unreachable from the CLI.
+  const event = PHOENIX_STORES.map((store) => model.phoenix.get(`${store}/${opts.event}`))
+    .find((found) => found !== undefined);
+  if (!event) {
+    const known = [...model.phoenix.keys()].map((k) => k.slice(k.indexOf('/') + 1)).sort(compare);
+    const looked = PHOENIX_STORES.map((store) => phoenixPath(store, opts.event)).join(' or ');
+    process.stderr.write(
+      `phoenix: no event "${opts.event}" — expected its mapping at ${looked}`
+      + `${known.length ? `; this store carries ${known.join(', ')}` : `; this store carries no ${PHOENIX_DIR}/ mappings`}\n`,
+    );
+    return EXIT_CODES.FAILURE;
+  }
+
+  const plan = planEvent(model, event);
+  // THE GATE. Everything above decided; nothing above wrote. A single finding
+  // refuses the whole event, and the write below is unreachable.
+  const refused = plan.findings.length > 0;
+  if (opts.apply && !refused) {
+    const touched = [];
+    try {
+      applyRewrites(model.root, plan.rewrites, touched);
+    } catch (error) {
+      rethrowIfBug(error); // a bug is not a refusal — the harness prints its stack
+      // The gate passed and the write began, so this is a filesystem failure
+      // between the probe and the write — rare, and the one path that can leave
+      // a store partially rewritten. Exit 2, never 1: the event did not finish,
+      // and an agent reading 1 would conclude it was cleanly refused and that
+      // nothing was written, which is exactly false here.
+      //
+      // The last entry is the file the write threw on. It is listed with the
+      // rest rather than excused: writeFileSync can truncate and then fail, so
+      // that file may be damaged even though its write did not complete — and
+      // it is the likeliest of all of them to need reverting.
+      const failed = touched[touched.length - 1];
+      process.stderr.write(
+        `phoenix: ${event.event} FAILED PART-WAY THROUGH — the store is partially rewritten\n`
+        + `  ${error.message}\n`
+        + `  revert these ${touched.length} file(s) and re-run:\n`
+        + touched.map((f) => `    ${f}${f === failed ? '   <- the write failed here; this file may be truncated' : ''}\n`).join(''),
+      );
+      return EXIT_CODES.FAILURE;
+    }
+  }
+
+  const payload = {
+    event: event.event,
+    verb: opts.apply && !refused ? 'apply' : 'check',
+    mapping: event.file,
+    decision: event.decision,
+    scope: event.scope,
+    checks: CHECKS,
+    'store-health': healthSummary(storeHealth(model)),
+    counts: { findings: plan.findings.length, rewritten: plan.rewrites.length, carried: plan.carried.length },
+    findings: plan.findings,
+    // The before/after text is deliberately not published: it would make the
+    // JSON enormous and the diff is what `git diff` is for. What ships is the
+    // claim a reviewer checks — which leaf moved where, and to which edition.
+    rewrites: plan.rewrites.map(({ id, file, from, to, edition }) => ({ id, file, from, to, edition })),
+    carried: plan.carried,
+  };
+  const lines = opts.json ? [JSON.stringify(payload, null, 2)] : render(payload);
+  process.stdout.write(`${lines.join('\n').replace(/\n+$/, '')}\n`);
+  return refused ? EXIT_CODES.FINDINGS : EXIT_CODES.CLEAN;
+}