unknown-knowledge 2.1.0

package/payload/engine/lib/derived.js

@@ -0,0 +1,494 @@
+/**
+ * The derived layer (UCS-1158) — browse trees, call numbers, resolution index.
+ *
+ * Everything discovery-shaped is a PROJECTION of the flat store, regenerated on
+ * demand and thrown away without loss. That sentence is the whole design, and
+ * every decision below follows from taking it literally.
+ *
+ * WHY PLURAL TREES. The store's first life filed each leaf at one position in
+ * one tree, and that position was also its identity (§3.2). One tree means one
+ * privileged reading of the material, chosen at bootstrap by whoever wrote the
+ * spine, and permanent because moving a leaf broke every citation to it. The
+ * accession inversion (UCS-1147) removed the cost of being wrong, and this
+ * module spends the refund: the same flat store projects into as many trees as
+ * there are useful orderings, because a projection commits to nothing.
+ *
+ * The two shipped axes are not arbitrary. A steward asks "what do we know about
+ * design tokens?" and wants DOMAIN first, form second — subject matter is how a
+ * person holds a library in their head. An agent asks "what are the constraints
+ * here?" and wants FORM first: it is looking for a kind of thing to act on, and
+ * the domain narrows it afterwards. Those are two genuinely different reading
+ * orders over one set of facts, and before this ticket the store could serve
+ * only whichever one its spine happened to encode.
+ *
+ * WHY DELETING IT LOSES NOTHING. No derived artifact carries a fact that is not
+ * already in a leaf. There is no field an author edits here, no id minted here,
+ * nothing here that another surface reads back. `derived/` is a build output in
+ * the ordinary sense, and `regenerate` proves it: delete the directory, run the
+ * command, and the bytes are identical (tests/derived-round-trip). If that test
+ * ever fails, something started living in the derived layer that had no other
+ * home — which is the exact drift this design exists to prevent.
+ *
+ * WHY DEMOTION IS ANNOTATION, NEVER OMISSION. A stale or draft leaf appears in
+ * every tree it belongs in, wearing a visible marker. Hiding it would make the
+ * tree lie by omission — a browsing steward would conclude the store knows
+ * nothing about a subject when what is true is that what it knows has rotted,
+ * and those two are opposite calls to action. The demotion vocabulary is
+ * imported from the surfaces that already own it (`timeVerdict`,
+ * `isPrePromotionStatus`) rather than re-derived, so a leaf cannot read stale in
+ * a tree and trusted in the resolver.
+ *
+ * WHERE THE PROBABILISTIC LAYER LIVES, AND WHY IT IS HERE AND NOWHERE ELSE.
+ * The resolution index carries a declared SLOT for embedding recall
+ * (`RECALL_SLOT`) and a gate that refuses to let anything through it. The slot
+ * is empty by design and this ticket implements no recall — model selection and
+ * semantic search are out of scope per the spec. What the slot does is fix the
+ * architectural position so it cannot be occupied by accident somewhere worse:
+ * embedding recall belongs in the DERIVED index, is consulted only after the
+ * deterministic layer is exhausted, and produces PROPOSALS that cannot be cited
+ * or persisted without passing a human gate. A proposal that reached frontmatter
+ * would be a probabilistic guess wearing the same clothes as a cited fact, and
+ * no downstream reader could tell them apart. Keeping the slot in the disposable
+ * layer means the strongest possible guarantee: whatever the recall layer ever
+ * computes, deleting `derived/` deletes all of it, and the store is unchanged.
+ */
+import { compare } from './validate-record.js';
+import {
+  DERIVED_DIR, isPrePromotionStatus, leafStage, recordId,
+} from './load-stores.js';
+import { timeVerdict, timeCheckStatus } from './time-verdicts.js';
+import { synthesizeCallNumber } from './call-numbers.js';
+
+/**
+ * The derived layer's home — one directory, clearly disposable.
+ *
+ * Sits INSIDE the knowledge store rather than at the repo root, because the
+ * projection is of this store's leaves and a sibling directory would imply it
+ * spans all three. `derived` rather than `generated` or `build`: the word names
+ * the relationship to the source (it is derived FROM the leaves) rather than
+ * the process that made it, which is what a reader deciding whether it is safe
+ * to delete needs to know.
+ *
+ * Declared in lib/load-stores.js and re-exported here, because the LOADER is
+ * the surface that must ignore it: browse trees are markdown under `knowledge/`
+ * and so are leaves, so nothing but the directory name distinguishes engine
+ * output from a record. One string, named on both sides.
+ */
+export { DERIVED_DIR };
+
+/** The banner every generated artifact opens with — the disposability notice. */
+export const DERIVED_BANNER = 'DERIVED — regenerated by `node payload/engine/derive.js --write`. Do not edit; do not cite. Deleting this directory loses nothing.';
+
+/**
+ * The browse axes, in the order they generate.
+ *
+ * Each is a pure function from a leaf record to a facet path — the same shape
+ * the prototype's pivots use, because the prototype is the executable intent
+ * for this ticket. A missing facet contributes an `unclassified` segment rather
+ * than being skipped, so a leaf with no `form` files under `.../unclassified/`
+ * where a steward will see it, instead of silently flattening up a level into
+ * the domain node and looking correctly classified.
+ *
+ * Adding an axis is one row here. Nothing else in this module names an axis,
+ * and the command enumerates this table — so a third ordering is a data change,
+ * not a code change, which is the property that makes "plural" true rather than
+ * "two".
+ *
+ * @type {ReadonlyArray<{ key: string, label: string, path: (record: object) => string[] }>}
+ */
+export const AXES = Object.freeze([
+  Object.freeze({
+    key: 'domain-form',
+    label: 'domain → form',
+    audience: 'stewards — subject matter first, the way a person holds a library in their head',
+    path: (record) => [...domainSegments(record), formOf(record)],
+  }),
+  Object.freeze({
+    key: 'form-domain',
+    label: 'form → domain',
+    audience: 'agents — kind of knowledge first, narrowed by subject afterwards',
+    path: (record) => [formOf(record), ...domainSegments(record)],
+  }),
+]);
+
+/** The marker a facet path segment carries when the leaf declares no value. */
+export const UNCLASSIFIED = 'unclassified';
+
+/**
+ * A leaf's governed domain path, split into segments.
+ *
+ * Reads `facets.domain` — the REGISTRY-GOVERNED subject path (UCS-1148) — and
+ * never the legacy top-level `domain` free string, which is ungoverned and on
+ * its way out. Reading the legacy field would make the trees project a
+ * vocabulary nothing checks, so a typo would mint a shelf.
+ *
+ * @param {object} record a leaf's front-matter record
+ * @returns {string[]} the domain segments, or one `unclassified` segment
+ */
+export function domainSegments(record) {
+  const domain = record?.facets?.domain;
+  if (typeof domain !== 'string' || domain === '') return [UNCLASSIFIED];
+  const segments = domain.split('/').filter((s) => s !== '');
+  return segments.length ? segments : [UNCLASSIFIED];
+}
+
+/**
+ * A leaf's governed form, or the unclassified marker.
+ *
+ * @param {object} record a leaf's front-matter record
+ * @returns {string}
+ */
+export function formOf(record) {
+  const form = record?.facets?.form;
+  return typeof form === 'string' && form !== '' ? form : UNCLASSIFIED;
+}
+
+/**
+ * Every demotion that applies to one leaf, with its reason.
+ *
+ * The SAME two predicates the resolver demotes on (`isPrePromotionStatus` for
+ * stage, `timeVerdict().stale` for time), read through the same accessors, and
+ * that sharing is the point rather than a convenience: a leaf marked stale in a
+ * browse tree and trusted in resolver output would be one record wearing two
+ * answers, and a reader would have no way to know which surface to believe.
+ * UCS-1150's module header commits to exactly one implementation and names this
+ * ticket as an inheritor; this is that inheritance.
+ *
+ * Returns the same `{reason, detail}` shape the resolver publishes, so a
+ * consumer reading both surfaces sees one vocabulary.
+ *
+ * @param {object} record a leaf's front-matter record
+ * @param {string|null} today the injected date, or null
+ * @returns {Array<{reason: string, detail: string}>}
+ */
+export function demotionsFor(record, today) {
+  const stage = leafStage(record);
+  const time = timeVerdict(record, today);
+  return [
+    ...(isPrePromotionStatus(stage)
+      ? [{ reason: 'stage', detail: `stage "${stage}" is pre-promotion — no moderator has certified this leaf's citations (UCS-1149)` }]
+      : []),
+    ...(time.stale ? [{ reason: 'time', detail: time.reason }] : []),
+  ];
+}
+
+/**
+ * Project one leaf into one axis — the row a tree prints.
+ *
+ * The call number is synthesized from the SAME path array the tree nests the
+ * leaf under, so the printed shelf label and the actual position cannot
+ * disagree. See lib/call-numbers.js for why that string is not an identity.
+ *
+ * @param {object} entry a loader-indexed leaf entry
+ * @param {{ path: (record: object) => string[] }} axis
+ * @param {string|null} today the injected date, or null
+ * @returns {{ id: string, path: string[], callNumber: string, heading: string|null, file: string, stage: string|null, time: object, demoted: boolean, demotions: Array<object> }}
+ */
+export function projectLeaf(entry, axis, today) {
+  const record = entry.record;
+  const path = axis.path(record);
+  const id = recordId(entry);
+  const demotions = demotionsFor(record, today);
+  return {
+    id: typeof id === 'string' ? id : null,
+    path,
+    'call-number': synthesizeCallNumber(path, typeof id === 'string' ? id : null),
+    heading: typeof record?.heading === 'string' ? record.heading : null,
+    file: entry.file,
+    stage: leafStage(record),
+    time: timeVerdict(record, today),
+    // A single boolean answering a single question — does this leaf sort below
+    // the promoted ones — with WHICH demotions fired kept separately, so a leaf
+    // that is both draft and stale reports both rather than having one absorb
+    // the other. Identical to the resolver's contract (UCS-1150).
+    demoted: demotions.length > 0,
+    demotions,
+  };
+}
+
+/**
+ * Build one browse tree from the flat leaf index.
+ *
+ * Nodes are sorted by name and leaves by accession, both through `compare` —
+ * the same collator every other surface stable-sorts with, so a tree's byte
+ * order does not depend on the order the filesystem happened to hand the loader
+ * its leaves. That is what makes the golden files diffable at all, and the
+ * reordered-twin fixture pins it.
+ *
+ * DEMOTED LEAVES SORT LAST WITHIN THEIR NODE, and that is the only way demotion
+ * changes the tree's shape. They are not moved to another node, not collapsed,
+ * and not omitted — a steward browsing `design-system/components` sees every leaf
+ * filed there, with the rotted ones at the bottom wearing their reason. Sorting
+ * them out of the way is a reading aid; hiding them would be a lie about what
+ * the store contains.
+ *
+ * @param {Iterable<object>} entries loader-indexed leaf entries
+ * @param {{ path: (record: object) => string[] }} axis
+ * @param {string|null} today the injected date, or null
+ * @returns {object} the tree root
+ */
+export function buildTree(entries, axis, today) {
+  const root = { name: null, children: new Map(), leaves: [] };
+  for (const entry of entries) {
+    const projected = projectLeaf(entry, axis, today);
+    let node = root;
+    for (const segment of projected.path) {
+      if (!node.children.has(segment)) {
+        node.children.set(segment, { name: segment, children: new Map(), leaves: [] });
+      }
+      node = node.children.get(segment);
+    }
+    node.leaves.push(projected);
+  }
+  return sortNode(root);
+}
+
+/**
+ * Sort one node's children and leaves, recursively, into their published order.
+ *
+ * @param {object} node
+ * @returns {object} the same node, sorted
+ */
+function sortNode(node) {
+  node.leaves.sort((a, b) => {
+    // Demoted last, then by accession. Two demoted leaves keep accession order
+    // between them rather than sorting by WHICH demotion fired: the demotion is
+    // a fact about the leaf, not a rank, and ordering by it would invent a
+    // precedence between "draft" and "stale" that nothing in the store declares.
+    if (a.demoted !== b.demoted) return a.demoted ? 1 : -1;
+    return compare(a.id, b.id);
+  });
+  const sorted = new Map([...node.children.entries()].sort((a, b) => compare(a[0], b[0])));
+  for (const child of sorted.values()) sortNode(child);
+  node.children = sorted;
+  return node;
+}
+
+/**
+ * Count the leaves at or under one node.
+ *
+ * @param {object} node
+ * @returns {number}
+ */
+export function countLeaves(node) {
+  let n = node.leaves.length;
+  for (const child of node.children.values()) n += countLeaves(child);
+  return n;
+}
+
+/**
+ * Render one tree as the markdown artifact written to disk.
+ *
+ * Markdown rather than JSON because the audience for a browse tree is a person
+ * or an agent READING it — the same reason a catalog is browsable. The
+ * machine-shaped projection of the same facts is the resolution index, which is
+ * JSON; the two are generated from one model so they cannot disagree.
+ *
+ * No timestamps, anywhere. The banner and the axis are the only preamble, and
+ * every line below is a function of the store's content — which is what lets
+ * D-012 baseline diffing attribute a changed tree to a changed leaf rather than
+ * to the clock.
+ *
+ * @param {object} tree the sorted tree root
+ * @param {{ key: string, label: string, audience: string }} axis
+ * @param {string|null} today the injected date, or null
+ * @returns {string} the artifact's full text
+ */
+export function renderTree(tree, axis, today) {
+  const lines = [
+    `# Browse tree — ${axis.label}`,
+    '',
+    `> ${DERIVED_BANNER}`,
+    '',
+    `- axis: \`${axis.key}\` (${axis.label})`,
+    `- for: ${axis.audience}`,
+    `- leaves: ${countLeaves(tree)}`,
+    `- time verdicts: ${timeCheckStatus(today)}`,
+    '',
+    'Call numbers are synthesized display strings for this projection only. They',
+    'are NOT identities: cite the accession id (L-NNNNNN), never a call number.',
+    '',
+  ];
+  renderNode(tree, 0, lines);
+  return `${lines.join('\n').replace(/\n+$/, '')}\n`;
+}
+
+/**
+ * Render one node and its subtree into `lines`.
+ *
+ * @param {object} node
+ * @param {number} depth
+ * @param {string[]} lines
+ */
+function renderNode(node, depth, lines) {
+  const indent = '  '.repeat(depth);
+  for (const [name, child] of node.children) {
+    lines.push(`${indent}- **${name}/** (${countLeaves(child)})`);
+    renderNode(child, depth + 1, lines);
+  }
+  for (const leaf of node.leaves) {
+    // The demotion annotation is part of the leaf's LINE, not a separate
+    // section: a reader scanning the shelf must not have to cross-reference a
+    // footnote to learn that what they are looking at has rotted.
+    const marks = leaf.demotions.map((d) => `${d.reason}`).join(', ');
+    const suffix = leaf.demoted ? ` — **demoted** (${marks})` : '';
+    lines.push(`${indent}- \`${leaf['call-number']}\` ${leaf.heading ?? '(no heading)'}${suffix}`);
+    for (const demotion of leaf.demotions) {
+      lines.push(`${indent}  - ${demotion.reason}: ${demotion.detail}`);
+    }
+  }
+}
+
+/**
+ * The declared position of embedding recall — a SLOT, deliberately empty.
+ *
+ * This ticket implements no recall and selects no model: that is out of scope by
+ * spec, and writing similarity code here would be the scope error the spec warns
+ * about. What ships is the ARCHITECTURE — where recall may live, when it may be
+ * consulted, and what it is forbidden to do — so that an implementation arriving
+ * later has exactly one legal shape and cannot quietly acquire a worse one.
+ *
+ * The three constraints are each load-bearing:
+ *
+ *   `location: derived` — recall lives in the disposable layer and nowhere else.
+ *   Deleting `derived/` must delete every probabilistic artifact the system has
+ *   ever computed. An embedding cached in frontmatter would be a guess sitting
+ *   in the same file as cited facts, indistinguishable to every later reader.
+ *
+ *   `consulted: after-deterministic-exhausted` — recall runs only when the
+ *   deterministic layer returns nothing. Consulting it earlier would let a
+ *   similarity score outrank a declared edge, and the store's whole claim is
+ *   that what resolves, resolves for a reason someone can point at.
+ *
+ *   `output: proposals` with `citable: false` and `persistable: false` — recall
+ *   may PROPOSE and may never persist or be cited. A proposal becomes knowledge
+ *   only by passing a human gate, at which point it is an ordinary authored leaf
+ *   with citations, and the probabilistic step is history rather than provenance.
+ *
+ * @type {Readonly<object>}
+ */
+export const RECALL_SLOT = Object.freeze({
+  status: 'declared-unimplemented',
+  location: DERIVED_DIR,
+  consulted: 'after-deterministic-exhausted',
+  output: 'proposals',
+  citable: false,
+  persistable: false,
+  gate: 'human-review — a proposal enters the store only as an authored leaf with citations (UCS-1158)',
+  'in-scope-here': 'the slot and the gate',
+  'out-of-scope-here': 'model selection and semantic-recall implementation (issue #49)',
+});
+
+/**
+ * The gate every probabilistic proposal must pass, and never does automatically.
+ *
+ * Total by construction: it returns a refusal for EVERY proposal, because no
+ * proposal can be admitted without a human, and there is no argument this
+ * function could take that would represent one. An implementation of recall
+ * calls this and gets a refusal; the only way into the store is the ordinary
+ * authoring path, where a human writes a leaf with citations and a moderator
+ * promotes it.
+ *
+ * A function that always refuses may read as a placeholder. It is not: it is the
+ * seam that makes "nothing probabilistic persists without a gate" checkable by a
+ * test rather than promised by a comment, and it is what a future recall
+ * implementation must route through to be reviewable at all.
+ *
+ * @param {object} proposal a candidate produced by probabilistic recall
+ * @returns {{ admitted: false, reason: string, proposal: object }}
+ */
+export function gateProposal(proposal) {
+  return {
+    admitted: false,
+    reason: 'probabilistic proposals are never admitted automatically — they may be read by a human, '
+      + 'who may author an ordinary cited leaf; nothing reaches the store without that gate (UCS-1158)',
+    proposal,
+  };
+}
+
+/**
+ * Build the resolution index — the machine-shaped projection of the same facts.
+ *
+ * One entry per leaf, carrying its identity, its position on every axis, the
+ * call number each axis synthesizes, and its demotion state. The trees are for
+ * reading; this is for joining, and both are generated from one pass over one
+ * model so a leaf cannot be filed one way in a tree and another way here.
+ *
+ * The index is keyed by ACCESSION and by nothing else. Keying it by call number
+ * would create a second resolvable name for a leaf — exactly the fusion the
+ * accession inversion removed — and a lookup table is the most tempting place
+ * in the system to reintroduce it.
+ *
+ * @param {Iterable<object>} entries loader-indexed leaf entries
+ * @param {string|null} today the injected date, or null
+ * @returns {object} the index payload
+ */
+export function buildIndex(entries, today) {
+  const leaves = [];
+  for (const entry of entries) {
+    const record = entry.record;
+    const id = recordId(entry);
+    const demotions = demotionsFor(record, today);
+    const positions = {};
+    for (const axis of AXES) {
+      const path = axis.path(record);
+      positions[axis.key] = {
+        path,
+        'call-number': synthesizeCallNumber(path, typeof id === 'string' ? id : null),
+      };
+    }
+    leaves.push({
+      id: typeof id === 'string' ? id : null,
+      heading: typeof record?.heading === 'string' ? record.heading : null,
+      file: entry.file,
+      stage: leafStage(record),
+      time: timeVerdict(record, today),
+      demoted: demotions.length > 0,
+      demotions,
+      positions,
+    });
+  }
+  leaves.sort((a, b) => compare(a.id, b.id));
+  return {
+    note: DERIVED_BANNER,
+    // Spelled out rather than left implicit, because the index is the artifact
+    // most likely to be read by something that wants to resolve through it.
+    'call-numbers': 'synthesized display strings, per axis — NOT identities and never citable; cite the accession id',
+    'time-check': timeCheckStatus(today),
+    axes: AXES.map((a) => ({ key: a.key, label: a.label, audience: a.audience })),
+    'recall-slot': RECALL_SLOT,
+    counts: {
+      leaves: leaves.length,
+      demoted: leaves.filter((l) => l.demoted).length,
+    },
+    leaves,
+  };
+}
+
+/**
+ * Every artifact the derived layer consists of, as `{path, text}` pairs.
+ *
+ * ONE function computes the whole layer, and both verbs run it: `--check`
+ * compares these bytes against what is on disk, `--write` writes them. That is
+ * what makes the check meaningful — a check that recomputed the artifacts a
+ * second way could pass while the writer produced something else.
+ *
+ * Paths are relative to the store root and always forward-slashed, so the
+ * manifest is identical on every platform.
+ *
+ * @param {Iterable<object>} entries loader-indexed leaf entries
+ * @param {string|null} today the injected date, or null
+ * @returns {Array<{path: string, text: string}>} sorted by path
+ */
+export function deriveArtifacts(entries, today) {
+  const materialized = [...entries];
+  const artifacts = AXES.map((axis) => ({
+    path: `knowledge/${DERIVED_DIR}/tree.${axis.key}.md`,
+    text: renderTree(buildTree(materialized, axis, today), axis, today),
+  }));
+  artifacts.push({
+    path: `knowledge/${DERIVED_DIR}/index.json`,
+    text: `${JSON.stringify(buildIndex(materialized, today), null, 2)}\n`,
+  });
+  return artifacts.sort((a, b) => compare(a.path, b.path));
+}

package/payload/engine/lib/engine-refusal.js

@@ -0,0 +1,40 @@
+/**
+ * The difference between a refusal and a bug (UCS-949).
+ *
+ * A surface may catch an engine throw and print a clean, actionable message:
+ * "no git", "the store loader reported 3 error(s)", "illegal transition". These
+ * are ANTICIPATED conditions. The engine reached them on purpose, so a stack
+ * trace would be noise.
+ *
+ * A `TypeError` from a genuine bug is not that. It must reach `runCli`, which
+ * prints the stack — otherwise a crash hides behind a one-line message that
+ * reads exactly like a considered refusal, and nobody can debug it.
+ *
+ * Both still exit 2. This is about diagnosability, not the exit-code contract.
+ *
+ * The rule: an anticipated refusal is a plain `new Error(msg)` — which is what
+ * every refusal in the engine already throws — or an `EngineRefusal` subclass
+ * for the ones that carry more than a message. Everything else is a bug, or a
+ * `UsageError` that the harness alone gets to interpret.
+ */
+
+/** An anticipated, actionable engine refusal. Reported without a stack trace. */
+export class EngineRefusal extends Error {
+  name = 'EngineRefusal';
+}
+
+/**
+ * Rethrow `error` unless it is an anticipated engine refusal.
+ *
+ * Call this first inside a surface's engine-failure catch, so the catch speaks
+ * only for the failures it can actually explain and everything else — a bug, a
+ * UsageError raised deep in the loader — travels on to the harness.
+ *
+ * @param {unknown} error
+ * @returns {void} when the error is a refusal the caller may report itself
+ * @throws {unknown} the original error, when it is not
+ */
+export function rethrowIfBug(error) {
+  const expected = error instanceof EngineRefusal || error?.constructor === Error;
+  if (!expected) throw error;
+}

package/payload/engine/lib/exit-codes.js

@@ -0,0 +1,14 @@
+/**
+ * Engine exit-code contract (PRD §5, D-011) — uniform across every engine
+ * module: validate, validate-values, audit, resolve, survey-map, preflight.
+ *
+ *   0 — clean
+ *   1 — findings / quarantines
+ *   2 — engine or environment failure. A check that never ran is a
+ *       blocking defect, never a silent pass.
+ */
+export const EXIT_CODES = Object.freeze({
+  CLEAN: 0,
+  FINDINGS: 1,
+  FAILURE: 2,
+});