unknown-knowledge 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (147) hide show
  1. package/LICENSE +202 -0
  2. package/NOTICE +5 -0
  3. package/README.md +167 -0
  4. package/cli/.gitkeep +0 -0
  5. package/cli/commands/init-copy.js +90 -0
  6. package/cli/commands/init.js +386 -0
  7. package/cli/init-copy.js +24 -0
  8. package/cli/init.js +24 -0
  9. package/cli/kit.manifest.yaml +256 -0
  10. package/cli/lib/copy-payload.js +334 -0
  11. package/cli/lib/generate-wrappers.js +162 -0
  12. package/package.json +36 -0
  13. package/payload/adapter-fixtures/README.md +53 -0
  14. package/payload/adapter-fixtures/html/EXPECTED.yaml +50 -0
  15. package/payload/adapter-fixtures/html/sample.html +38 -0
  16. package/payload/adapter-fixtures/md/EXPECTED.yaml +65 -0
  17. package/payload/adapter-fixtures/md/sample.md +32 -0
  18. package/payload/adapter-fixtures/pdf/EXPECTED.yaml +45 -0
  19. package/payload/adapter-fixtures/pdf/sample.pdf +0 -0
  20. package/payload/adapter-fixtures/txt/EXPECTED.yaml +31 -0
  21. package/payload/adapter-fixtures/txt/sample.txt +18 -0
  22. package/payload/docs/README.md +102 -0
  23. package/payload/docs/boundaries.md +60 -0
  24. package/payload/docs/ci-wiring.md +109 -0
  25. package/payload/docs/steward-guide.md +238 -0
  26. package/payload/engine/audit.js +34 -0
  27. package/payload/engine/commands/audit.js +289 -0
  28. package/payload/engine/commands/derive.js +334 -0
  29. package/payload/engine/commands/ingest.js +124 -0
  30. package/payload/engine/commands/log-entry.js +85 -0
  31. package/payload/engine/commands/phoenix.js +206 -0
  32. package/payload/engine/commands/preflight.js +530 -0
  33. package/payload/engine/commands/resolve.js +1678 -0
  34. package/payload/engine/commands/survey-map.js +351 -0
  35. package/payload/engine/commands/validate-values.js +315 -0
  36. package/payload/engine/commands/validate.js +1426 -0
  37. package/payload/engine/derive.js +34 -0
  38. package/payload/engine/ingest.js +34 -0
  39. package/payload/engine/lib/anchor-signatures.js +126 -0
  40. package/payload/engine/lib/boot.js +39 -0
  41. package/payload/engine/lib/call-numbers.js +133 -0
  42. package/payload/engine/lib/cli.js +147 -0
  43. package/payload/engine/lib/coverage.js +849 -0
  44. package/payload/engine/lib/decomposition.js +225 -0
  45. package/payload/engine/lib/derived.js +494 -0
  46. package/payload/engine/lib/engine-refusal.js +40 -0
  47. package/payload/engine/lib/exit-codes.js +14 -0
  48. package/payload/engine/lib/extractor-kinds.js +955 -0
  49. package/payload/engine/lib/format-adapters.js +802 -0
  50. package/payload/engine/lib/id-grammars.js +178 -0
  51. package/payload/engine/lib/iso-date.js +55 -0
  52. package/payload/engine/lib/kit-root.js +101 -0
  53. package/payload/engine/lib/load-stores.js +1624 -0
  54. package/payload/engine/lib/log-entry.js +196 -0
  55. package/payload/engine/lib/phoenix.js +628 -0
  56. package/payload/engine/lib/scoring.js +150 -0
  57. package/payload/engine/lib/suppressions.js +172 -0
  58. package/payload/engine/lib/time-verdicts.js +282 -0
  59. package/payload/engine/lib/usage-error.js +14 -0
  60. package/payload/engine/lib/validate-record.js +504 -0
  61. package/payload/engine/log-entry.js +34 -0
  62. package/payload/engine/phoenix.js +39 -0
  63. package/payload/engine/preflight.js +34 -0
  64. package/payload/engine/resolve.js +34 -0
  65. package/payload/engine/survey-map.js +34 -0
  66. package/payload/engine/validate-values.js +34 -0
  67. package/payload/engine/validate.js +34 -0
  68. package/payload/extractor-fixtures/.gitkeep +0 -0
  69. package/payload/extractor-fixtures/README.md +29 -0
  70. package/payload/extractor-fixtures/swift/strings-keys/EXPECTED.yaml +8 -0
  71. package/payload/extractor-fixtures/swift/strings-keys/sample.strings +15 -0
  72. package/payload/extractor-fixtures/swift/swift-const-array/EXPECTED.yaml +7 -0
  73. package/payload/extractor-fixtures/swift/swift-const-array/sample.swift +21 -0
  74. package/payload/extractor-fixtures/swift/swift-enum/EXPECTED.yaml +8 -0
  75. package/payload/extractor-fixtures/swift/swift-enum/sample.swift +30 -0
  76. package/payload/extractor-fixtures/swift/yaml-keys/EXPECTED.yaml +6 -0
  77. package/payload/extractor-fixtures/swift/yaml-keys/sample.yaml +23 -0
  78. package/payload/extractor-fixtures/swift/yaml-map-keys/EXPECTED.yaml +7 -0
  79. package/payload/extractor-fixtures/swift/yaml-map-keys/sample.yaml +15 -0
  80. package/payload/extractor-fixtures/ts/dir-modules/EXPECTED.yaml +12 -0
  81. package/payload/extractor-fixtures/ts/dir-modules/sample-modules/alpha.widget.ts +1 -0
  82. package/payload/extractor-fixtures/ts/dir-modules/sample-modules/beta.widget.ts +1 -0
  83. package/payload/extractor-fixtures/ts/dir-modules/sample-modules/gamma.widget.ts +1 -0
  84. package/payload/extractor-fixtures/ts/dir-modules/sample-modules/helpers/format.ts +3 -0
  85. package/payload/extractor-fixtures/ts/dir-modules/sample-modules/widgets.test.ts +2 -0
  86. package/payload/extractor-fixtures/ts/json-keys/EXPECTED.yaml +7 -0
  87. package/payload/extractor-fixtures/ts/json-keys/sample.json +5 -0
  88. package/payload/extractor-fixtures/ts/json-map-keys/EXPECTED.yaml +7 -0
  89. package/payload/extractor-fixtures/ts/json-map-keys/sample.json +13 -0
  90. package/payload/extractor-fixtures/ts/ts-const-array/EXPECTED.yaml +6 -0
  91. package/payload/extractor-fixtures/ts/ts-const-array/sample.ts +13 -0
  92. package/payload/extractor-fixtures/ts/ts-enum/EXPECTED.yaml +7 -0
  93. package/payload/extractor-fixtures/ts/ts-enum/sample.ts +11 -0
  94. package/payload/extractor-fixtures/ts/ts-object-keys/EXPECTED.yaml +6 -0
  95. package/payload/extractor-fixtures/ts/ts-object-keys/sample.tsx +23 -0
  96. package/payload/extractor-fixtures/ts/ts-union/EXPECTED.yaml +5 -0
  97. package/payload/extractor-fixtures/ts/ts-union/sample.ts +9 -0
  98. package/payload/hooks/pre-commit +37 -0
  99. package/payload/hooks/reverse-lookup +66 -0
  100. package/payload/package.json +3 -0
  101. package/payload/protocol/.gitkeep +0 -0
  102. package/payload/protocol/AGENTS.md +239 -0
  103. package/payload/protocol/derived-layer.md +174 -0
  104. package/payload/protocol/new-kind-pipeline.md +179 -0
  105. package/payload/protocol/registry-warrant.md +162 -0
  106. package/payload/protocol/skills/kb-build.md +303 -0
  107. package/payload/protocol/skills/knowledge-audit.md +183 -0
  108. package/payload/protocol/skills/knowledge-bootstrap.md +229 -0
  109. package/payload/protocol/skills/knowledge-reflect.md +397 -0
  110. package/payload/schemas/catalog.schema.json +32 -0
  111. package/payload/schemas/decision-entry.schema.json +122 -0
  112. package/payload/schemas/finding.schema.json +77 -0
  113. package/payload/schemas/gap.schema.json +52 -0
  114. package/payload/schemas/graduation-categories.schema.json +64 -0
  115. package/payload/schemas/knowledge-leaf.schema.json +194 -0
  116. package/payload/schemas/miss.schema.json +45 -0
  117. package/payload/schemas/ontology-concept.schema.json +115 -0
  118. package/payload/schemas/phoenix-event.schema.json +76 -0
  119. package/payload/schemas/registry.schema.json +57 -0
  120. package/payload/schemas/rules.schema.json +14 -0
  121. package/payload/schemas/survey-scope.schema.json +23 -0
  122. package/payload/templates/decisions/_catalog.yaml +7 -0
  123. package/payload/templates/decisions/_registries/graduation-categories.yaml +42 -0
  124. package/payload/templates/decisions/phoenix-event.yaml +74 -0
  125. package/payload/templates/decisions/reflect-mint-proposal.yaml +100 -0
  126. package/payload/templates/decisions/registry-minting.yaml +58 -0
  127. package/payload/templates/decisions/trust-graduation.yaml +120 -0
  128. package/payload/templates/decisions/trust-revocation.yaml +106 -0
  129. package/payload/templates/knowledge/_catalog.yaml +9 -0
  130. package/payload/templates/knowledge/_registries/anchor.yaml +42 -0
  131. package/payload/templates/knowledge/_registries/authority-tiers.yaml +32 -0
  132. package/payload/templates/knowledge/_registries/domains.yaml +43 -0
  133. package/payload/templates/knowledge/_registries/form.yaml +38 -0
  134. package/payload/templates/knowledge/_registries/jurisdictions.yaml +20 -0
  135. package/payload/templates/knowledge/_registries/operations.yaml +18 -0
  136. package/payload/templates/knowledge/_registries/stage.yaml +53 -0
  137. package/payload/templates/knowledge/_rules.yaml +6 -0
  138. package/payload/templates/new-kind/README.md +107 -0
  139. package/payload/templates/new-kind/descriptor.example.yaml +18 -0
  140. package/payload/templates/new-kind/fixture/EXPECTED.yaml +6 -0
  141. package/payload/templates/new-kind/fixture/demo-anchor.list +2 -0
  142. package/payload/templates/new-kind/fixture/sample.list +7 -0
  143. package/payload/templates/new-kind/parser.example.js +98 -0
  144. package/payload/templates/ontology/_catalog.yaml +6 -0
  145. package/payload/templates/ontology/_rules.yaml +6 -0
  146. package/payload/wrappers/cursor.mdc +15 -0
  147. package/payload/wrappers/pointer.md +10 -0
@@ -0,0 +1,351 @@
1
+ /**
2
+ * Survey map (KK-25) — deterministic traversal-surface builder (PRD §4), the
3
+ * token-efficiency backbone of bootstrap and audit. Agents TRIAGE this
4
+ * artifact; raw repo traversal is a protocol violation (§6).
5
+ *
6
+ * - Git-tracked files only (`git ls-files --stage`) — never raw fs walks.
7
+ * - Built-in denylist: dot-directories, vendored/generated dirs, lockfiles,
8
+ * `*.gen.*`, binary extensions.
9
+ * - Per-directory extension/count histograms (direct children).
10
+ * - Anchor-candidate pre-scan: lexical sniff per §5.1 kind, sharing ONE
11
+ * regex table with the extractors (lib/anchor-signatures.js). Ranked by
12
+ * path/kind — a stable triage order, not a judgment call.
13
+ * - Proposed include/exclude scope over top-level directories; once the
14
+ * human-confirmed survey-scope.yaml exists it is HONORED: the map is
15
+ * bounded to it, and audit/reflect share the same contract via
16
+ * loadSurveyScope()/inScope(). A malformed scope file is an engine
17
+ * failure (exit 2), never silently ignored.
18
+ * - Explicit `unsurveyed:` disclosure — submodule gitlinks and out-of-root
19
+ * symlinks the map could NOT see (§11.1). Blind spots exit 1 (findings),
20
+ * never a silent pass.
21
+ *
22
+ * Deterministic by construction: stable sorting everywhere, no wall-clock
23
+ * timestamps, no network, and — D-014 — no import/eval/spawn of repo content;
24
+ * candidate scanning is lexical only.
25
+ */
26
+ import { readFileSync, readlinkSync, realpathSync } from 'node:fs';
27
+ import { spawnSync } from 'node:child_process';
28
+ import { basename, dirname, extname, isAbsolute, resolve, sep } from 'node:path';
29
+ import { load, YAMLException } from 'js-yaml';
30
+ import { ANCHOR_SIGNATURES } from '../lib/anchor-signatures.js';
31
+ import { SCOPE_FILE } from '../lib/kit-root.js';
32
+ import { validateStoreFile, compare } from '../lib/validate-record.js';
33
+ import { EXIT_CODES } from '../lib/exit-codes.js';
34
+ import { parseArgs as parseFlags, rethrowIfBug, UsageError } from '../lib/cli.js';
35
+
36
+ export { SCOPE_FILE } from '../lib/kit-root.js';
37
+
38
+ /** Vendored/generated directory names (any path segment). */
39
+ const DENY_DIRS = new Set([
40
+ 'node_modules', 'bower_components', 'vendor', 'vendors', 'third_party',
41
+ 'third-party', 'Pods', 'Carthage', 'DerivedData', 'dist', 'build', 'out',
42
+ 'coverage', '__pycache__', '.build',
43
+ ]);
44
+ const DENY_LOCKFILES = new Set([
45
+ 'package-lock.json', 'npm-shrinkwrap.json', 'yarn.lock', 'pnpm-lock.yaml',
46
+ 'bun.lock', 'bun.lockb', 'deno.lock', 'Podfile.lock', 'Package.resolved',
47
+ 'Cargo.lock', 'Gemfile.lock', 'composer.lock', 'poetry.lock', 'uv.lock',
48
+ 'go.sum',
49
+ ]);
50
+ const BINARY_EXTENSIONS = new Set([
51
+ '.png', '.jpg', '.jpeg', '.gif', '.webp', '.heic', '.ico', '.icns', '.pdf',
52
+ '.zip', '.gz', '.tgz', '.jar', '.woff', '.woff2', '.ttf', '.otf', '.eot',
53
+ '.mp3', '.mp4', '.mov', '.wav', '.so', '.dylib', '.a', '.o', '.bin',
54
+ '.exe', '.dll', '.sqlite', '.realm', '.car', '.dat',
55
+ ]);
56
+ /** Siblings (files sharing one extension, or module subfolders) before a dir reads as dir-modules. */
57
+ const DIR_MODULES_MIN = 3;
58
+
59
+ function denied(path) {
60
+ const name = basename(path);
61
+ if (DENY_LOCKFILES.has(name) || /\.gen\./.test(name)) return true;
62
+ if (BINARY_EXTENSIONS.has(extname(name).toLowerCase())) return true;
63
+ const dirs = path.split('/').slice(0, -1);
64
+ return dirs.some((d) => d.startsWith('.') || DENY_DIRS.has(d));
65
+ }
66
+
67
+ /**
68
+ * `git ls-files --stage` rows: { mode, path }. Throws on any git failure.
69
+ * Deduped by path: a merge-conflicted path emits one row per stage (1/2/3),
70
+ * which would otherwise inflate tracked counts and triplicate candidates.
71
+ */
72
+ function gitLsFiles(root) {
73
+ const result = spawnSync('git', ['-C', root, 'ls-files', '-z', '--stage'], {
74
+ encoding: 'utf8', maxBuffer: 64 * 1024 * 1024,
75
+ });
76
+ if (result.error || result.status !== 0) {
77
+ throw new Error(`git ls-files failed in ${root}: ${result.error?.message ?? result.stderr.trim()}`);
78
+ }
79
+ const rows = [];
80
+ const seen = new Set();
81
+ for (const row of result.stdout.split('\0').filter(Boolean)) {
82
+ const tab = row.indexOf('\t');
83
+ const path = row.slice(tab + 1);
84
+ if (seen.has(path)) continue;
85
+ seen.add(path);
86
+ rows.push({ mode: row.slice(0, 6), path });
87
+ }
88
+ return rows;
89
+ }
90
+
91
+ /**
92
+ * True when a tracked symlink resolves outside the repo root (a blind spot).
93
+ * `realRoot` must be canonical (realpathSync'd once by the caller): comparing
94
+ * against a non-canonical root (/tmp/x when /tmp -> /private/tmp) would flag
95
+ * every absolute in-repo symlink as escaping. The landing point is
96
+ * canonicalized too, so an in-repo absolute target never false-positives.
97
+ */
98
+ function escapesRoot(realRoot, path) {
99
+ try {
100
+ const target = readlinkSync(resolve(realRoot, path));
101
+ let landed = resolve(dirname(resolve(realRoot, path)), target);
102
+ try {
103
+ landed = realpathSync(landed);
104
+ } catch {
105
+ // dangling target: judge by the resolved path as-is
106
+ }
107
+ return landed !== realRoot && !landed.startsWith(realRoot + sep);
108
+ } catch {
109
+ return false; // unreadable link: surveyed as an ordinary tracked file
110
+ }
111
+ }
112
+
113
+ /**
114
+ * Load survey-scope.yaml from `root` — the shared honor-it contract for
115
+ * survey-map, audit (KK-12), and reflect. Returns { present:false } when the
116
+ * file does not exist; THROWS on unreadable/unparseable/invalid content — a
117
+ * scope the engine cannot honor must never degrade to "no scope".
118
+ */
119
+ export function loadSurveyScope(root) {
120
+ let text;
121
+ try {
122
+ text = readFileSync(resolve(root, SCOPE_FILE), 'utf8');
123
+ } catch (error) {
124
+ if (error.code === 'ENOENT') return { present: false, include: [], exclude: [] };
125
+ throw new Error(`${SCOPE_FILE}: cannot read: ${error.message}`);
126
+ }
127
+ let doc;
128
+ try {
129
+ doc = load(text, { filename: SCOPE_FILE });
130
+ } catch (error) {
131
+ const reason = error instanceof YAMLException ? error.reason ?? error.message : error.message;
132
+ throw new Error(`${SCOPE_FILE}: unparseable YAML: ${reason}`);
133
+ }
134
+ const { ok, errors } = validateStoreFile('survey-scope', doc);
135
+ if (!ok) {
136
+ const detail = errors.map((e) => `${e.path}: ${e.message}`).join('; ');
137
+ throw new Error(`${SCOPE_FILE}: invalid scope file — ${detail}`);
138
+ }
139
+ return {
140
+ present: true,
141
+ include: doc.include.map(normalizePrefix).sort(compare),
142
+ exclude: (doc.exclude ?? []).map(normalizePrefix).sort(compare),
143
+ };
144
+ }
145
+
146
+ /** `src/` means `src` — a trailing slash must never make an include silently match nothing. */
147
+ const normalizePrefix = (prefix) => {
148
+ const stripped = prefix.replace(/\/+$/, '');
149
+ return stripped === '' ? '.' : stripped;
150
+ };
151
+
152
+ /** `.` covers root-level files only; any other prefix covers itself and its subtree. */
153
+ const underPrefix = (path, prefix) =>
154
+ prefix === '.' ? !path.includes('/') : path === prefix || path.startsWith(`${prefix}/`);
155
+
156
+ /** The honor-it contract: include prefixes bound the sweep; exclude wins. */
157
+ export function inScope(path, scope) {
158
+ if (!scope.present) return true;
159
+ if (scope.exclude.some((p) => underPrefix(path, p))) return false;
160
+ return scope.include.some((p) => underPrefix(path, p));
161
+ }
162
+
163
+ /** Lexical §5.1 pre-scan of one file; returns matched kinds (never executes). */
164
+ function sniffKinds(root, path) {
165
+ const ext = extname(path).toLowerCase();
166
+ const sigs = ANCHOR_SIGNATURES.filter((s) => s.extensions?.includes(ext));
167
+ if (sigs.length === 0) return [];
168
+ let bytes;
169
+ try {
170
+ bytes = readFileSync(resolve(root, path));
171
+ } catch {
172
+ return []; // tracked but absent from the worktree: nothing to sniff
173
+ }
174
+ // UTF-16 BOM (legacy Xcode .strings exports): the UTF-8 sniff cannot read
175
+ // the content, so treat the file as candidate-by-extension rather than
176
+ // skipping it silently — the extractor decides, the map never hides.
177
+ if (bytes.length >= 2
178
+ && ((bytes[0] === 0xFF && bytes[1] === 0xFE) || (bytes[0] === 0xFE && bytes[1] === 0xFF))) {
179
+ return [...new Set(sigs.map((s) => s.kind))];
180
+ }
181
+ const text = bytes.toString('utf8');
182
+ return sigs.filter((s) => new RegExp(s.pattern, s.flags).test(text)).map((s) => s.kind);
183
+ }
184
+
185
+ /**
186
+ * Build the survey map for a git repo at `root`. Deterministic: same tree in,
187
+ * byte-identical map out. Throws on engine failure (no git, malformed scope).
188
+ */
189
+ export function buildSurveyMap(root) {
190
+ const rows = gitLsFiles(root);
191
+ const scope = loadSurveyScope(root);
192
+ // Canonicalize ONCE: a root reached through a symlink (/tmp -> /private/tmp
193
+ // on macOS) must not flag every absolute in-repo symlink as out-of-root.
194
+ const realRoot = realpathSync(resolve(root));
195
+
196
+ // A scope that matches nothing would emit a plausible-looking empty survey —
197
+ // an engine failure (exit 2), never a silent surveyed:0 pass.
198
+ if (scope.present
199
+ && !rows.some(({ mode, path }) => mode !== '160000' && inScope(path, scope))) {
200
+ throw new Error(`${SCOPE_FILE}: include [${scope.include.join(', ')}] matches zero tracked files — refusing to emit a silent empty survey`);
201
+ }
202
+
203
+ const unsurveyed = [];
204
+ const surveyed = [];
205
+ let deniedCount = 0;
206
+ for (const { mode, path } of rows) {
207
+ if (mode === '160000') {
208
+ unsurveyed.push({ path, reason: 'submodule-gitlink' });
209
+ } else if (mode === '120000' && escapesRoot(realRoot, path)) {
210
+ unsurveyed.push({ path, reason: 'out-of-root-symlink' });
211
+ } else if (denied(path)) {
212
+ deniedCount += 1;
213
+ } else if (inScope(path, scope)) {
214
+ surveyed.push(path);
215
+ }
216
+ }
217
+ surveyed.sort(compare);
218
+ unsurveyed.sort((a, b) => compare(a.path, b.path));
219
+
220
+ // Per-directory histograms over direct children.
221
+ const byDir = new Map();
222
+ for (const path of surveyed) {
223
+ const dir = path.includes('/') ? dirname(path) : '.';
224
+ if (!byDir.has(dir)) byDir.set(dir, new Map());
225
+ const ext = extname(path) || '(none)';
226
+ const hist = byDir.get(dir);
227
+ hist.set(ext, (hist.get(ext) ?? 0) + 1);
228
+ }
229
+ const directories = [...byDir.keys()].sort(compare).map((dir) => {
230
+ const hist = byDir.get(dir);
231
+ const extensions = Object.fromEntries([...hist.entries()].sort((a, b) => compare(a[0], b[0])));
232
+ return { path: dir, files: [...hist.values()].reduce((a, b) => a + b, 0), extensions };
233
+ });
234
+
235
+ // Anchor candidates: content sniffs plus the structural dir-modules shapes —
236
+ // sibling FILES sharing one extension, or per-module SUBFOLDERS (the PRD's
237
+ // canonical modules/editor/, modules/prototyping/, modules/whiteboard/ layout).
238
+ const candidates = [];
239
+ for (const path of surveyed) {
240
+ for (const kind of sniffKinds(root, path)) candidates.push({ kind, path });
241
+ }
242
+ const childDirs = new Map(); // dir -> Set of direct child dirs holding tracked files
243
+ for (const path of surveyed) {
244
+ const segs = path.split('/');
245
+ let parent = '.';
246
+ for (let i = 0; i < segs.length - 1; i += 1) {
247
+ const dir = parent === '.' ? segs[i] : `${parent}/${segs[i]}`;
248
+ if (!childDirs.has(parent)) childDirs.set(parent, new Set());
249
+ childDirs.get(parent).add(dir);
250
+ parent = dir;
251
+ }
252
+ }
253
+ const dirModules = new Set();
254
+ for (const { path, extensions } of directories) {
255
+ if (Object.values(extensions).some((n) => n >= DIR_MODULES_MIN)) dirModules.add(path);
256
+ }
257
+ for (const [dir, kids] of childDirs) {
258
+ if (kids.size >= DIR_MODULES_MIN) dirModules.add(dir);
259
+ }
260
+ for (const path of dirModules) candidates.push({ kind: 'dir-modules', path });
261
+ candidates.sort((a, b) => compare(a.path, b.path) || compare(a.kind, b.kind));
262
+
263
+ // Scope: honor the confirmed file, else propose from top-level directories —
264
+ // include those with surveyed files, exclude those that are pure denylist.
265
+ let scopeOut;
266
+ if (scope.present) {
267
+ scopeOut = { source: SCOPE_FILE, include: scope.include, exclude: scope.exclude };
268
+ } else {
269
+ const topSurveyed = new Set();
270
+ const topDenied = new Set();
271
+ for (const { mode, path } of rows) {
272
+ if (mode === '160000') continue;
273
+ // Root-level files are proposed as '.' (root files only, no subtree) —
274
+ // accepting the proposal must never silently drop package.json et al.
275
+ const top = path.includes('/') ? path.slice(0, path.indexOf('/')) : '.';
276
+ (denied(path) ? topDenied : topSurveyed).add(top);
277
+ }
278
+ scopeOut = {
279
+ source: 'proposed',
280
+ include: [...topSurveyed].sort(compare),
281
+ exclude: [...topDenied].filter((d) => !topSurveyed.has(d)).sort(compare),
282
+ };
283
+ }
284
+
285
+ return {
286
+ counts: { tracked: rows.length, surveyed: surveyed.length, denied: deniedCount, unsurveyed: unsurveyed.length },
287
+ scope: scopeOut,
288
+ directories,
289
+ candidates,
290
+ unsurveyed,
291
+ };
292
+ }
293
+
294
+ // ---------------------------------------------------------------- CLI
295
+
296
+ export const USAGE = 'usage: node payload/engine/survey-map.js [root] [--root <dir>] [--json]';
297
+
298
+ function parseArgs(argv) {
299
+ const { options, positionals } = parseFlags(argv, {
300
+ boolean: ['json'],
301
+ value: ['root'],
302
+ positionals: true,
303
+ });
304
+ if (positionals.length > 1) {
305
+ throw new UsageError(`unexpected argument ${JSON.stringify(positionals[1])}`);
306
+ }
307
+ // The root may be named positionally or by flag, never both: two names for
308
+ // the root is exactly the ambiguity `locateKit` refuses to guess at.
309
+ if (options.root !== undefined && positionals.length) {
310
+ throw new UsageError(`the root is named twice: ${JSON.stringify(positionals[0])} and --root ${JSON.stringify(options.root)}`);
311
+ }
312
+ return { root: resolve(options.root ?? positionals[0] ?? '.'), json: !!options.json };
313
+ }
314
+
315
+ function printHuman(map) {
316
+ const lines = [
317
+ `surveyed ${map.counts.surveyed} tracked file(s) across ${map.directories.length} director(ies); ${map.counts.denied} denylisted`,
318
+ `scope (${map.scope.source}): include ${map.scope.include.join(', ') || '(none)'}; exclude ${map.scope.exclude.join(', ') || '(none)'}`,
319
+ `anchor candidates: ${map.candidates.length}`,
320
+ ...map.candidates.map((c) => ` ${c.kind.padEnd(18)} ${c.path}`),
321
+ map.unsurveyed.length === 0
322
+ ? 'nothing unsurveyed — the map saw every tracked path'
323
+ : `UNSURVEYED (blind spots the map could NOT see — disclose at the scope gate):`,
324
+ ...map.unsurveyed.map((u) => ` ${u.reason.padEnd(22)} ${u.path}`),
325
+ ];
326
+ process.stdout.write(`${lines.join('\n')}\n`);
327
+ }
328
+
329
+ /**
330
+ * CLI entry. Exit codes per the engine contract (PRD §5): 0 clean,
331
+ * 1 blind spots disclosed under unsurveyed:, 2 engine/environment failure.
332
+ */
333
+ export function main(argv) {
334
+ const opts = parseArgs(argv); // a UsageError reaches the harness
335
+
336
+ let map;
337
+ try {
338
+ map = buildSurveyMap(opts.root);
339
+ } catch (error) {
340
+ rethrowIfBug(error); // a bug is not a refusal — the harness prints its stack
341
+ // An EXPECTED environment failure (no git, unparseable scope, a scope whose
342
+ // include matches nothing): the survey never ran, so it cannot report blind
343
+ // spots. Exit 2 — never 1, which would claim it ran and found them.
344
+ process.stderr.write(`survey-map: ${error.message}\n`);
345
+ return EXIT_CODES.FAILURE;
346
+ }
347
+
348
+ if (opts.json) process.stdout.write(`${JSON.stringify(map, null, 2)}\n`);
349
+ else printHuman(map);
350
+ return map.unsurveyed.length === 0 ? EXIT_CODES.CLEAN : EXIT_CODES.FINDINGS;
351
+ }
@@ -0,0 +1,315 @@
1
+ /**
2
+ * Value validator (KK-07) — blocking-grade rung-2 checks (PRD §4). Runs every
3
+ * `enumerates` recipe on the loaded ontology: parses the named source with the
4
+ * descriptor's extractor kind, extracts the actual value set, and diffs it
5
+ * against the claim IN BOTH DIRECTIONS. §3.5 equality: values are strings,
6
+ * compared byte-exact and case-sensitive, as sets — order irrelevant,
7
+ * duplicates in source are a finding.
8
+ *
9
+ * node payload/engine/validate-values.js [--concepts <ids>] [--json] [--root <dir>]
10
+ *
11
+ * Findings (exit 1 when any is error-severity):
12
+ * value-not-in-source a claimed value the source does not carry
13
+ * source-value-missing a source value the claim does not carry
14
+ * duplicate-source-value the source declares the same value twice (§3.5:
15
+ * sets — a duplicate is a defect, never a bigger set)
16
+ * wrong-pointer ALL claimed values missing from a real, parseable
17
+ * file — the descriptor points at the wrong place;
18
+ * one distinct finding, never a per-value cascade
19
+ *
20
+ * Hard errors (exit 2 — a check that never ran is a blocking defect, never a
21
+ * silent pass, PRD §5):
22
+ * - a store the single health model marks unhealthy: every error-severity
23
+ * loader diagnostic (including the KK-02 malformed-descriptor codes
24
+ * non-string-enumerates-value / duplicate-enumerates-value /
25
+ * enumerates-source-not-listed) is surfaced as a hard error and no value
26
+ * check runs — an unprovable claim silently passing would re-open the gap
27
+ * this validator closes
28
+ * unknown-kind the descriptor names an extractor kind the registry does
29
+ * not carry — nothing can re-derive the claim
30
+ * source-missing the named source file does not exist / is unreadable
31
+ * out-of-envelope an out-of-envelope sentinel appears in the matched span
32
+ * (PRD §5: a confident wrong parse is a false all-clear)
33
+ * extract-failed the kind's recipe could not parse the source
34
+ *
35
+ * §3.5 status semantics: draft/proposed concepts are skipped (structural
36
+ * checks only — preflight verdicts them unknown); active is blocking-grade;
37
+ * deprecated demotes value findings AND the path-existence/envelope hard
38
+ * errors to warnings — deprecation is precisely the state that stops the
39
+ * blocking check from dead-ending a legitimate source deletion (§3.5).
40
+ * A malformed descriptor or unknown kind stays hard on every status.
41
+ *
42
+ * Extractor kinds are registered by NAME in lib/extractor-kinds.js — a small
43
+ * deterministic recipe `extract(text, descriptor) -> string[]` that reads a
44
+ * value set out of a reified anchor, throwing EnvelopeError / ExtractError.
45
+ * KK-08 ships the TS/JS + JSON kinds; KK-09 the Swift/config kinds; KK-10
46
+ * dir-modules — the registry's one DIRECTORY kind (`{ reads: 'directory' }`),
47
+ * fed a deterministic listing instead of file text (dispatch seam in
48
+ * checkDescriptor); `test-lines` (newline-delimited registry files) proves
49
+ * dispatch, the envelope hard-error path, and determinism.
50
+ * D-014: kinds parse lexically only — the engine never executes client code.
51
+ *
52
+ * --root is the REPO root (default cwd): descriptor sources and
53
+ * source-of-truth pointers are repo-relative (§9.1 — a post-init repo nests
54
+ * the kit inside the codebase its pointers describe). The stores load from
55
+ * <root>/unknown-knowledge/ when that directory exists, else from <root>
56
+ * itself (the kit repo's own dogfood layout).
57
+ *
58
+ * Consumes the KK-04 loader's model — never re-parses stores. Output is
59
+ * deterministic and stable-sorted (findings by concept/path/code/value), no
60
+ * wall-clock timestamps. Exit codes (PRD §5, D-011): 0 clean, 1 findings,
61
+ * 2 engine failure / check-never-ran.
62
+ */
63
+ import process from 'node:process';
64
+ import { readFileSync } from 'node:fs';
65
+ import { join, resolve } from 'node:path';
66
+ import { fileURLToPath } from 'node:url';
67
+ import { healthSummary, loadStores, isPrePromotionStatus, normalizeConceptIds, selectConcepts, storeHealth, UnknownConceptsError } from '../lib/load-stores.js';
68
+ import { locateKitRoot } from '../lib/kit-root.js';
69
+ import { EXIT_CODES } from '../lib/exit-codes.js';
70
+ import { UsageError, parseArgs as parseFlags, rethrowIfBug } from '../lib/cli.js';
71
+ import { compare } from '../lib/validate-record.js';
72
+ import { KINDS, EnvelopeError, ExtractError, listDirectory } from '../lib/extractor-kinds.js';
73
+
74
+ export const USAGE = 'usage: node payload/engine/validate-values.js [--concepts <ids>] [--json] [--root <dir>]';
75
+
76
+ // ------------------------------------------------------------ the check body
77
+
78
+ const isObject = (v) => typeof v === 'object' && v !== null && !Array.isArray(v);
79
+
80
+ /**
81
+ * Run one descriptor: dispatch to its kind, read the source, diff both ways.
82
+ * Pushes findings/hard-errors into ctx; never throws for check outcomes.
83
+ */
84
+ function checkDescriptor(ctx, concept, descriptor, i) {
85
+ const { id, file } = concept;
86
+ const path = `enumerates[${i}]`;
87
+ const deprecated = concept.record.status === 'deprecated';
88
+ const severity = deprecated ? 'warning' : 'error';
89
+ // deprecated demotes path-existence/envelope to warning findings (§3.5);
90
+ // malformed/unknown-kind stay hard on every status.
91
+ const never = (code, message, extra = {}) => {
92
+ const entry = { concept: id, code, file, path, source: descriptor.source, message, ...extra };
93
+ if (deprecated && (code === 'source-missing' || code === 'out-of-envelope')) {
94
+ ctx.findings.push({ ...entry, severity: 'warning' });
95
+ } else {
96
+ ctx.hardErrors.push(entry);
97
+ }
98
+ };
99
+
100
+ const kind = KINDS[descriptor.kind];
101
+ if (!kind) {
102
+ ctx.hardErrors.push({
103
+ concept: id, code: 'unknown-kind', file, path, source: descriptor.source,
104
+ message: `extractor kind "${descriptor.kind}" is not registered — nothing can re-derive this claim, and an unprovable claim must never silently pass (PRD §4); registered kinds: ${Object.keys(KINDS).sort(compare).join(', ')}`,
105
+ });
106
+ return;
107
+ }
108
+
109
+ // Registry dispatch seam (KK-10): a plain function is a FILE kind fed the
110
+ // source text; a `{ reads: 'directory' }` entry is a DIRECTORY kind fed the
111
+ // deterministic listing. Either way the one filesystem read happens here,
112
+ // and an unreadable source (missing file, missing dir, a file where a
113
+ // directory kind expects a dir — ENOTDIR) is source-missing.
114
+ const readsDirectory = typeof kind !== 'function' && kind.reads === 'directory';
115
+ const extract = readsDirectory ? kind.extract : kind;
116
+
117
+ let input;
118
+ try {
119
+ input = readsDirectory
120
+ ? listDirectory(join(ctx.root, descriptor.source))
121
+ : readFileSync(join(ctx.root, descriptor.source), 'utf8');
122
+ } catch (error) {
123
+ never('source-missing', `cannot ${readsDirectory ? 'list source directory' : 'read source'} ${JSON.stringify(descriptor.source)}: ${error.message}`);
124
+ return;
125
+ }
126
+
127
+ let actual;
128
+ try {
129
+ actual = extract(input, descriptor);
130
+ } catch (error) {
131
+ if (error instanceof EnvelopeError) {
132
+ never('out-of-envelope', error.message);
133
+ } else if (error instanceof ExtractError) {
134
+ never('extract-failed', error.message);
135
+ } else {
136
+ throw error; // a kind bug is an engine failure, not a finding
137
+ }
138
+ return;
139
+ }
140
+
141
+ const claimed = descriptor.values;
142
+ const actualSet = new Set();
143
+ for (const value of actual) {
144
+ if (actualSet.has(value)) {
145
+ ctx.findings.push({
146
+ concept: id, code: 'duplicate-source-value', severity, file, path,
147
+ source: descriptor.source, value,
148
+ message: `source declares ${JSON.stringify(value)} more than once — values compare as sets (§3.5), so a duplicate is a source defect, never a bigger set`,
149
+ });
150
+ }
151
+ actualSet.add(value);
152
+ }
153
+
154
+ // Wrong-pointer signature (PRD §4): the file is real and parseable, yet
155
+ // carries NONE of the claimed values — one finding, not a cascade.
156
+ if (claimed.length && claimed.every((v) => !actualSet.has(v))) {
157
+ ctx.findings.push({
158
+ concept: id, code: 'wrong-pointer', severity, file, path,
159
+ source: descriptor.source,
160
+ message: `all ${claimed.length} claimed value(s) are missing from ${JSON.stringify(descriptor.source)} — the file exists and parses (${actualSet.size} value(s) extracted), so the descriptor points at the wrong place`,
161
+ });
162
+ return;
163
+ }
164
+
165
+ const claimedSet = new Set(claimed);
166
+ for (const value of claimed) {
167
+ if (!actualSet.has(value)) {
168
+ ctx.findings.push({
169
+ concept: id, code: 'value-not-in-source', severity, file, path,
170
+ source: descriptor.source, value,
171
+ message: `claimed value ${JSON.stringify(value)} is not in ${JSON.stringify(descriptor.source)} (byte-exact, case-sensitive, §3.5)`,
172
+ });
173
+ }
174
+ }
175
+ for (const value of actualSet) {
176
+ if (!claimedSet.has(value)) {
177
+ ctx.findings.push({
178
+ concept: id, code: 'source-value-missing', severity, file, path,
179
+ source: descriptor.source, value,
180
+ message: `source value ${JSON.stringify(value)} in ${JSON.stringify(descriptor.source)} is not claimed by the descriptor`,
181
+ });
182
+ }
183
+ }
184
+ }
185
+
186
+ /**
187
+ * Run every enumerates check over a loaded model — the reusable seam
188
+ * preflight (KK-26) consumes, so verdicts and this validator can never
189
+ * disagree. `conceptIds` null = all concepts; an unknown id throws (a check
190
+ * that never ran is a blocking defect). `repoRoot` is where descriptor
191
+ * sources resolve (§9.1 repo-relative); it defaults to the store root, the
192
+ * flat layout where both coincide. Returns { findings, hardErrors,
193
+ * checked }, stable-sorted.
194
+ */
195
+ export function validateValues(model, conceptIds, repoRoot = model.root) {
196
+ // Sources are repo-relative (§9.1), not store-relative: in a seeded repo
197
+ // the stores sit at <repo>/unknown-knowledge/ but point at <repo>/src/....
198
+ const ctx = { root: repoRoot, findings: [], hardErrors: [], checked: [] };
199
+
200
+ // Single health model: the loader already validated every descriptor's
201
+ // shape (KK-02 codes). An unhealthy store means the value check cannot
202
+ // certify anything — surface the diagnostics as hard errors and stop.
203
+ const { errors: storeErrors } = storeHealth(model);
204
+ if (storeErrors.length) {
205
+ ctx.hardErrors = storeErrors.map(({ code, file, path, message }) => ({
206
+ concept: null, code, file, path, source: null, message,
207
+ }));
208
+ return ctx;
209
+ }
210
+
211
+ for (const concept of selectConcepts(model, conceptIds)) {
212
+ const { id, file, record } = concept;
213
+ const descriptors = Array.isArray(record.enumerates) ? record.enumerates : [];
214
+ const entry = { concept: id, status: record.status ?? null, descriptors: descriptors.length };
215
+ if (isPrePromotionStatus(record.status)) {
216
+ // §3.5: structural checks only; the resolver downranks, preflight
217
+ // verdicts unknown — the value check does not run.
218
+ entry.skipped = record.status;
219
+ ctx.checked.push(entry);
220
+ continue;
221
+ }
222
+ ctx.checked.push(entry);
223
+ descriptors.forEach((descriptor, i) => {
224
+ if (!isObject(descriptor)) return; // shape defects already hard-errored via the loader
225
+ checkDescriptor(ctx, { id, file, record }, descriptor, i);
226
+ });
227
+ }
228
+
229
+ ctx.checked.sort((a, b) => compare(a.concept, b.concept));
230
+ ctx.findings.sort((a, b) =>
231
+ compare(a.concept, b.concept) || compare(a.path, b.path) || compare(a.code, b.code)
232
+ || compare(a.value ?? '', b.value ?? ''));
233
+ ctx.hardErrors.sort((a, b) =>
234
+ compare(a.concept ?? '', b.concept ?? '') || compare(a.file ?? '', b.file ?? '')
235
+ || compare(a.path ?? '', b.path ?? '') || compare(a.code, b.code));
236
+ return ctx;
237
+ }
238
+
239
+ // ------------------------------------------------------------- CLI plumbing
240
+
241
+ function parseArgs(argv) {
242
+ const { options } = parseFlags(argv, {
243
+ boolean: ['json'],
244
+ value: ['root'],
245
+ repeatable: ['concepts'],
246
+ allowEmpty: ['concepts'],
247
+ });
248
+ const opts = { json: !!options.json, root: options.root ?? process.cwd(), concepts: null };
249
+ if (options.concepts) {
250
+ opts.concepts = normalizeConceptIds(options.concepts.flatMap((v) => v.split(',')));
251
+ if (!opts.concepts.length) {
252
+ throw new UsageError('--concepts must name at least one concept id — a filter that never ran is a failure, never a silent pass');
253
+ }
254
+ }
255
+ return opts;
256
+ }
257
+
258
+ function renderHuman(payload) {
259
+ const lines = [];
260
+ const f = payload.findings.length;
261
+ const h = payload['hard-errors'].length;
262
+ const checked = payload.checked.filter((c) => !c.skipped).length;
263
+ const skipped = payload.checked.length - checked;
264
+ lines.push(
265
+ `validate-values: ${checked} concept(s) checked, ${skipped} skipped (draft/proposed), `
266
+ + `${f} finding${f === 1 ? '' : 's'}, ${h} hard error${h === 1 ? '' : 's'}`,
267
+ '',
268
+ );
269
+ for (const e of payload['hard-errors']) {
270
+ lines.push(`HARD ERROR ${e.code} ${e.concept ?? e.file}${e.source ? ` (source: ${e.source})` : ''}`);
271
+ lines.push(` ${e.message}`);
272
+ }
273
+ if (h) {
274
+ lines.push('', 'the check never ran on the entries above — fix the descriptors/store first (PRD §4: a malformed descriptor is a hard error, never skipped)', '');
275
+ }
276
+ for (const x of payload.findings) {
277
+ lines.push(`${x.severity === 'warning' ? 'warning' : 'FINDING'} ${x.code} ${x.concept}${x.value !== undefined ? ` ${JSON.stringify(x.value)}` : ''} (source: ${x.source})`);
278
+ lines.push(` ${x.message}`);
279
+ }
280
+ if (!f && !h) lines.push('every enumerates claim agrees with its source (both directions, §3.5 set equality)');
281
+ return lines;
282
+ }
283
+
284
+ export function main(argv) {
285
+ {
286
+ const opts = parseArgs(argv);
287
+
288
+ let model;
289
+ try {
290
+ model = loadStores(locateKitRoot(opts.root));
291
+ } catch (error) {
292
+ // An EXPECTED refusal from the loader — an unreadable root, an ambiguous
293
+ // kit layout, a Store that will not load. The stores this command would
294
+ // check never loaded, so its checks never ran: exit 2, never 1.
295
+ process.stderr.write(`validate-values: ${error.message}\n`);
296
+ rethrowIfBug(error); // a bug, or a UsageError raised deep in the loader, is not ours to speak for
297
+ return EXIT_CODES.FAILURE;
298
+ }
299
+
300
+ const ctx = validateValues(model, opts.concepts, opts.root);
301
+ const hard = ctx.hardErrors.length > 0;
302
+ const blocking = ctx.findings.some((x) => x.severity === 'error');
303
+ const payload = {
304
+ ok: !hard && !blocking,
305
+ 'store-health': healthSummary(storeHealth(model)),
306
+ checked: ctx.checked,
307
+ findings: ctx.findings,
308
+ 'hard-errors': ctx.hardErrors,
309
+ };
310
+
311
+ const lines = opts.json ? [JSON.stringify(payload, null, 2)] : renderHuman(payload);
312
+ process.stdout.write(`${lines.join('\n').replace(/\n+$/, '')}\n`);
313
+ return hard ? EXIT_CODES.FAILURE : blocking ? EXIT_CODES.FINDINGS : EXIT_CODES.CLEAN;
314
+ }
315
+ }