pi-mega-compact 0.17.0 → 0.17.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. package/dist/config/vector-cortex.js +10 -0
  2. package/dist/dedup/digest.js +30 -0
  3. package/dist/dedup/l1-lsh-v2.js +60 -0
  4. package/dist/dedup/l1-lsh.js +52 -0
  5. package/dist/dedup/l1-minhash-v2.js +193 -0
  6. package/dist/dedup/l1-minhash.js +91 -0
  7. package/dist/dedup/l1-verify.js +54 -0
  8. package/dist/dedup/mmr.js +45 -0
  9. package/dist/dedup/normalize.js +39 -0
  10. package/dist/dedup/raptor/buildHistory.js +164 -0
  11. package/dist/dedup/raptor/guardrails.js +83 -0
  12. package/dist/dedup/raptor/incremental.js +304 -0
  13. package/dist/dedup/raptor/index.js +190 -0
  14. package/dist/dedup/raptor/kmeans.js +152 -0
  15. package/dist/dedup/raptor/multilevel.js +184 -0
  16. package/dist/dedup/raptor/retrieval.js +93 -0
  17. package/dist/dedup/raptor/summarizer.js +86 -0
  18. package/dist/dedup/raptor/tree.js +202 -0
  19. package/dist/dedup/topk.js +60 -0
  20. package/dist/extensions/dashboard-server/routes-rag-settings-helpers.js +1 -0
  21. package/dist/src/config/vector-cortex.js +10 -0
  22. package/dist/src/config.js +1 -1
  23. package/dist/src/dedup/l1-lsh-v2.js +60 -0
  24. package/dist/src/dedup/l1-minhash-v2.js +193 -0
  25. package/dist/src/dedup/normalize.js +1 -1
  26. package/dist/src/vector-cortex/conformance/emit.js +46 -0
  27. package/dist/src/vector-cortex/conformance/manifest.js +235 -0
  28. package/dist/src/vector-cortex/conformance/runner.js +111 -0
  29. package/dist/src/vector-cortex/conformance/triadB-reader.js +95 -0
  30. package/dist/src/vector-cortex/migrations/minhash-v2.js +209 -0
  31. package/dist/vector-cortex/conformance/emit.js +46 -0
  32. package/dist/vector-cortex/conformance/manifest.js +235 -0
  33. package/dist/vector-cortex/conformance/runner.js +111 -0
  34. package/dist/vector-cortex/conformance/triadB-reader.js +95 -0
  35. package/dist/vector-cortex/migrations/minhash-v2.js +209 -0
  36. package/extensions/dashboard-client/dist/assets/{AreaChart-CAnWomhw.js → AreaChart-BQXFF7UI.js} +2 -2
  37. package/extensions/dashboard-client/dist/assets/{AreaChart-CAnWomhw.js.map → AreaChart-BQXFF7UI.js.map} +1 -1
  38. package/extensions/dashboard-client/dist/assets/{BarChart-lZ2-s71k.js → BarChart-BVtoz9Eg.js} +2 -2
  39. package/extensions/dashboard-client/dist/assets/{BarChart-lZ2-s71k.js.map → BarChart-BVtoz9Eg.js.map} +1 -1
  40. package/extensions/dashboard-client/dist/assets/{CacheTab-BL2Thh66.js → CacheTab-trI2TIGy.js} +2 -2
  41. package/extensions/dashboard-client/dist/assets/{CacheTab-BL2Thh66.js.map → CacheTab-trI2TIGy.js.map} +1 -1
  42. package/extensions/dashboard-client/dist/assets/{EventsTab-DEh2Z9fp.js → EventsTab-CNUbk9__.js} +2 -2
  43. package/extensions/dashboard-client/dist/assets/{EventsTab-DEh2Z9fp.js.map → EventsTab-CNUbk9__.js.map} +1 -1
  44. package/extensions/dashboard-client/dist/assets/{HealthTab-DTwKclxk.js → HealthTab-K-bd_CM-.js} +2 -2
  45. package/extensions/dashboard-client/dist/assets/{HealthTab-DTwKclxk.js.map → HealthTab-K-bd_CM-.js.map} +1 -1
  46. package/extensions/dashboard-client/dist/assets/{MaintenanceTab-DDfBEdbX.js → MaintenanceTab-CoEioKzx.js} +2 -2
  47. package/extensions/dashboard-client/dist/assets/{MaintenanceTab-DDfBEdbX.js.map → MaintenanceTab-CoEioKzx.js.map} +1 -1
  48. package/extensions/dashboard-client/dist/assets/{MemoryMapTab-Dgfmg-0Q.js → MemoryMapTab-CJo24ulv.js} +2 -2
  49. package/extensions/dashboard-client/dist/assets/{MemoryMapTab-Dgfmg-0Q.js.map → MemoryMapTab-CJo24ulv.js.map} +1 -1
  50. package/extensions/dashboard-client/dist/assets/{MetricsTab-DOcMm0Tx.js → MetricsTab-DCNlQWWP.js} +2 -2
  51. package/extensions/dashboard-client/dist/assets/{MetricsTab-DOcMm0Tx.js.map → MetricsTab-DCNlQWWP.js.map} +1 -1
  52. package/extensions/dashboard-client/dist/assets/{OverviewTab-DHT5aoHg.js → OverviewTab-CQQlyIQ6.js} +2 -2
  53. package/extensions/dashboard-client/dist/assets/{OverviewTab-DHT5aoHg.js.map → OverviewTab-CQQlyIQ6.js.map} +1 -1
  54. package/extensions/dashboard-client/dist/assets/{ReposTab-CGAmtR_c.js → ReposTab-Cqm82_yn.js} +2 -2
  55. package/extensions/dashboard-client/dist/assets/{ReposTab-CGAmtR_c.js.map → ReposTab-Cqm82_yn.js.map} +1 -1
  56. package/extensions/dashboard-client/dist/assets/{SessionsTab-BIgx6iKT.js → SessionsTab-A8xTAdET.js} +2 -2
  57. package/extensions/dashboard-client/dist/assets/{SessionsTab-BIgx6iKT.js.map → SessionsTab-A8xTAdET.js.map} +1 -1
  58. package/extensions/dashboard-client/dist/assets/{SetupTab-ADghKuEI.js → SetupTab-CqaSgaZl.js} +2 -2
  59. package/extensions/dashboard-client/dist/assets/{SetupTab-ADghKuEI.js.map → SetupTab-CqaSgaZl.js.map} +1 -1
  60. package/extensions/dashboard-client/dist/assets/{TimeSavedCard-C33UiiOM.js → TimeSavedCard-C_f3q8GR.js} +2 -2
  61. package/extensions/dashboard-client/dist/assets/{TimeSavedCard-C33UiiOM.js.map → TimeSavedCard-C_f3q8GR.js.map} +1 -1
  62. package/extensions/dashboard-client/dist/assets/{TurnsTab-DdbL9wPN.js → TurnsTab-BEsafACo.js} +2 -2
  63. package/extensions/dashboard-client/dist/assets/{TurnsTab-DdbL9wPN.js.map → TurnsTab-BEsafACo.js.map} +1 -1
  64. package/extensions/dashboard-client/dist/assets/VectorCortexTab-DD1CMK80.js +2 -0
  65. package/extensions/dashboard-client/dist/assets/VectorCortexTab-DD1CMK80.js.map +1 -0
  66. package/extensions/dashboard-client/dist/assets/{WikiTab-Ch00OFKs.js → WikiTab-Bf01M2_Z.js} +2 -2
  67. package/extensions/dashboard-client/dist/assets/{WikiTab-Ch00OFKs.js.map → WikiTab-Bf01M2_Z.js.map} +1 -1
  68. package/extensions/dashboard-client/dist/assets/{button-ByeEdo2g.js → button-BEJPTk_E.js} +2 -2
  69. package/extensions/dashboard-client/dist/assets/{button-ByeEdo2g.js.map → button-BEJPTk_E.js.map} +1 -1
  70. package/extensions/dashboard-client/dist/assets/{card-DesoLXvc.js → card-DK1V382s.js} +2 -2
  71. package/extensions/dashboard-client/dist/assets/{card-DesoLXvc.js.map → card-DK1V382s.js.map} +1 -1
  72. package/extensions/dashboard-client/dist/assets/{generateCategoricalChart-CCAB7Iz4.js → generateCategoricalChart-CRtaPphL.js} +2 -2
  73. package/extensions/dashboard-client/dist/assets/{generateCategoricalChart-CCAB7Iz4.js.map → generateCategoricalChart-CRtaPphL.js.map} +1 -1
  74. package/extensions/dashboard-client/dist/assets/{index-B6cgyAPr.js → index-CB6BslE4.js} +3 -3
  75. package/extensions/dashboard-client/dist/assets/{index-B6cgyAPr.js.map → index-CB6BslE4.js.map} +1 -1
  76. package/extensions/dashboard-client/dist/assets/{switch-DIZC-oe-.js → switch-C4FV6B9k.js} +2 -2
  77. package/extensions/dashboard-client/dist/assets/{switch-DIZC-oe-.js.map → switch-C4FV6B9k.js.map} +1 -1
  78. package/extensions/dashboard-client/dist/assets/{toggle-CIoWaH2e.js → toggle-GBmeENMd.js} +2 -2
  79. package/extensions/dashboard-client/dist/assets/{toggle-CIoWaH2e.js.map → toggle-GBmeENMd.js.map} +1 -1
  80. package/extensions/dashboard-client/dist/assets/{useSSE-Bv4943l1.js → useSSE-BOKZGSn1.js} +2 -2
  81. package/extensions/dashboard-client/dist/assets/{useSSE-Bv4943l1.js.map → useSSE-BOKZGSn1.js.map} +1 -1
  82. package/extensions/dashboard-client/dist/index.html +1 -1
  83. package/extensions/dashboard-client/src/tabs/VectorCortexTab.tsx +1 -1
  84. package/extensions/dashboard-client/tsconfig.json +1 -1
  85. package/extensions/dashboard-server/routes-rag-settings-helpers.ts +6 -0
  86. package/package.json +1 -1
  87. package/src/config/vector-cortex.ts +11 -0
  88. package/src/config.ts +1 -0
  89. package/src/dedup/l1-lsh-v2.ts +78 -0
  90. package/src/dedup/l1-minhash-v2.ts +221 -0
  91. package/src/dedup/normalize.ts +1 -1
  92. package/src/store/sqlite/game-scores.ts +2 -1
  93. package/src/vector-cortex/conformance/emit.ts +56 -0
  94. package/src/vector-cortex/conformance/manifest.ts +298 -0
  95. package/src/vector-cortex/conformance/runner.ts +165 -0
  96. package/src/vector-cortex/conformance/triadB-reader.ts +110 -0
  97. package/src/vector-cortex/migrations/minhash-v2.ts +263 -0
  98. package/extensions/dashboard-client/dist/assets/VectorCortexTab-Cl1pbEdR.js +0 -2
  99. package/extensions/dashboard-client/dist/assets/VectorCortexTab-Cl1pbEdR.js.map +0 -1
@@ -0,0 +1,209 @@
1
+ /**
2
+ * vector-cortex/migrations/minhash-v2.ts — M4 MinHashV2 copy/validate/switch
3
+ * migration (VC1C).
4
+ *
5
+ * M4 versions the L1 dedup signature store: v2 signatures/buckets are written
6
+ * BESIDE v1, backfilled by checkpoint id, verified, and only then switched as
7
+ * the active version. It follows the same copy/validate/switch + resume
8
+ * contract as M2 but for the minhash index:
9
+ *
10
+ * - batch: writes v2 signatures + buckets for a set of checkpoint IDs, each
11
+ * tagged with the frozen version 2; never touches v1 rows.
12
+ * - backfill: resumable by checkpoint id — an interrupted run resumes without
13
+ * duplicate v2 signatures or active-pointer drift.
14
+ * - verify: counts per checkpoint match v1 and every v2 digest re-hashes.
15
+ * - switch: atomically flips the ACTIVE VERSION pointer from v1 to v2; until
16
+ * a verified switch, v1 remains active (interruption keeps old authority).
17
+ *
18
+ * Cross-version compare is REJECTED: a mixed v1-v2 similarity/bucket query
19
+ * returns `MINHASH_VERSION_MISMATCH` (never compares v1/v2 signatures).
20
+ *
21
+ * Pure logic over an injected M4Host (deterministic + testable; no console).
22
+ * PREVENT-002/011/PI-004 honored.
23
+ */
24
+ import { createHash } from "node:crypto";
25
+ import { MINHASH_VERSION, minhashV2Signature, encodeSignatureV2, } from "../../dedup/l1-minhash-v2.js";
26
+ import { lshBandsV2, BANDS_V2, SIGNATURE_BYTES_EXPECTED_V2, } from "../../dedup/l1-lsh-v2.js";
27
+ import { createConformanceReporter, } from "../conformance/emit.js";
28
+ /** M4 failure codes. */
29
+ export const M4_FAIL = {
30
+ VERSION_MISMATCH: "MINHASH_VERSION_MISMATCH",
31
+ BACKFILL_PARTIAL: "M4_BACKFILL_PARTIAL",
32
+ COUNT_MISMATCH: "M4_COUNT_MISMATCH",
33
+ DIGEST_MISMATCH: "M4_DIGEST_MISMATCH",
34
+ };
35
+ /** Reject a cross-version compare. Returns the frozen mismatch code. */
36
+ export function crossVersionError() {
37
+ return M4_FAIL.VERSION_MISMATCH;
38
+ }
39
+ /**
40
+ * Compute v2 signature + buckets for one checkpoint (the authoritative bytes).
41
+ * Pure and cross-language deterministic.
42
+ */
43
+ export function computeV2Row(host, checkpointId) {
44
+ const sessionId = host.sessionOf(checkpointId);
45
+ const source = host.sourceOf(checkpointId);
46
+ const sig = minhashV2Signature(source);
47
+ const bytes = encodeSignatureV2(sig);
48
+ const buckets = lshBandsV2(new Uint8Array(bytes), sessionId);
49
+ return {
50
+ checkpointId,
51
+ sessionId,
52
+ version: MINHASH_VERSION,
53
+ signatureBytes: new Uint8Array(bytes),
54
+ digest: `sha256:${sha256Hex(bytes)}`,
55
+ buckets,
56
+ };
57
+ }
58
+ function sha256Hex(bytes) {
59
+ return createHash("sha256").update(bytes).digest("hex");
60
+ }
61
+ /**
62
+ * Backfill v2 signatures/buckets for all v1 checkpoint ids (resumable): rows
63
+ * already persisted with matching v2 digest/version are left untouched (no
64
+ * duplicates), and the active pointer stays at v1 until a verified switch.
65
+ * Returns the rows written this call (delta), never duplicates.
66
+ *
67
+ * An EXISTING row is re-computed and digest-verified before it is skipped; if
68
+ * the persisted row is corrupt (wrong version, wrong length, or a digest that
69
+ * no longer re-hashes its signatureBytes — e.g. a crash mid-write that left a
70
+ * truncated/bad row for an already-present id), it is recomputed and rewritten
71
+ * so corruption self-heals instead of being frozen by a later `m4Verify` failure.
72
+ */
73
+ export function m4Backfill(host, reporter = createConformanceReporter()) {
74
+ const wanted = host.v1CheckpointIds();
75
+ const existing = new Map(host.storedV2().map((r) => [r.checkpointId, r]));
76
+ const writes = [];
77
+ for (const id of wanted) {
78
+ const row = computeV2Row(host, id);
79
+ const stored = existing.get(id);
80
+ if (stored !== undefined && v2RowMatches(stored, row))
81
+ continue; // healthy, no dup
82
+ writes.push(row); // absent, or stored row is corrupt -> (re)compute authoritative bytes
83
+ }
84
+ if (writes.length > 0) {
85
+ host.putV2(writes);
86
+ reporter.backfilled({
87
+ written: writes.length,
88
+ checkpointIds: writes.map((r) => r.checkpointId),
89
+ });
90
+ }
91
+ return writes;
92
+ }
93
+ /** True when a persisted row already matches the authoritative recompute. */
94
+ function v2RowMatches(stored, fresh) {
95
+ return (stored.version === MINHASH_VERSION &&
96
+ stored.checkpointId === fresh.checkpointId &&
97
+ stored.sessionId === fresh.sessionId &&
98
+ stored.signatureBytes.length === fresh.signatureBytes.length &&
99
+ stored.digest === fresh.digest &&
100
+ bucketsMatch(stored));
101
+ }
102
+ /**
103
+ * Re-derive the 64 bucket keys from the row's signatureBytes + sessionId and
104
+ * compare every value to the stored buckets. This closes a defensive
105
+ * verification gap: a persisted row whose signatureBytes + digest are correct
106
+ * but whose bucket VALUES were written wrong (a writer defect, or a partial
107
+ * bucket write that still held 64 entries) would otherwise pass digest+count
108
+ * checks and be treated as healthy — yet the buckets are exactly the data the
109
+ * live LSH-dedup path queries. Buckets are deterministic from sigBytes +
110
+ * sessionId, so a mismatch is always a real storage/author defect.
111
+ */
112
+ function bucketsMatch(row) {
113
+ // A wrong-length signature can't be banded (lshBandsV2 throws); the caller
114
+ // already reports it as a count/version mismatch, so fail closed here too.
115
+ if (row.signatureBytes.length !== SIGNATURE_BYTES_EXPECTED_V2)
116
+ return false;
117
+ if (row.buckets.length !== BANDS_V2)
118
+ return false;
119
+ const expected = lshBandsV2(new Uint8Array(row.signatureBytes), row.sessionId);
120
+ for (let i = 0; i < BANDS_V2; i++) {
121
+ if (expected[i] !== row.buckets[i])
122
+ return false;
123
+ }
124
+ return true;
125
+ }
126
+ /**
127
+ * Verify the backfilled v2 index: every v1 checkpoint has exactly one v2 row
128
+ * (reset -> resumable partial counts), each stored row's digest re-hashes, and
129
+ * each row has all 64 buckets. Never mutates.
130
+ */
131
+ export function m4Verify(host) {
132
+ const codes = [];
133
+ const wanted = new Set(host.v1CheckpointIds());
134
+ const stored = host.storedV2();
135
+ const counts = new Map();
136
+ for (const r of stored) {
137
+ counts.set(r.checkpointId, (counts.get(r.checkpointId) ?? 0) + 1);
138
+ if (r.version !== MINHASH_VERSION)
139
+ codes.push(M4_FAIL.VERSION_MISMATCH);
140
+ if (r.signatureBytes.length !== 2048)
141
+ codes.push(M4_FAIL.COUNT_MISMATCH);
142
+ if (r.buckets.length !== BANDS_V2)
143
+ codes.push(M4_FAIL.COUNT_MISMATCH);
144
+ if (r.digest !== `sha256:${sha256Hex(r.signatureBytes)}`) {
145
+ codes.push(M4_FAIL.DIGEST_MISMATCH);
146
+ }
147
+ // Bucket VALUE verification: re-derive all 64 keys from signatureBytes +
148
+ // sessionId and compare them. Digest + count alone would let a row with
149
+ // correct sigBytes/digest but wrong bucket values pass; the buckets are the
150
+ // data the live LSH-dedup path queries, so a drifts here is a real defect.
151
+ if (!bucketsMatch(r))
152
+ codes.push(M4_FAIL.COUNT_MISMATCH);
153
+ }
154
+ // Every v1 checkpoint backfilled exactly once (count parity, no dup rows).
155
+ for (const id of wanted) {
156
+ const c = counts.get(id) ?? 0;
157
+ if (c === 0)
158
+ codes.push(M4_FAIL.BACKFILL_PARTIAL);
159
+ if (c > 1)
160
+ codes.push(M4_FAIL.COUNT_MISMATCH);
161
+ }
162
+ // No orphan v2 rows without a v1 checkpoint.
163
+ for (const id of counts.keys()) {
164
+ if (!wanted.has(id))
165
+ codes.push(M4_FAIL.COUNT_MISMATCH);
166
+ }
167
+ const ok = codes.length === 0;
168
+ return { ok, codes: dedupe(codes) };
169
+ }
170
+ /**
171
+ * Switch the active version to v2. Only call after `m4Verify` reports ok; a
172
+ * crash/return before switch leaves v1 active (interruption keeps old
173
+ * authority, resumable idempotently).
174
+ */
175
+ export function m4Switch(host) {
176
+ host.switchToV2();
177
+ }
178
+ /** Full M4 lifecycle: backfill -> verify -> switch; returns verify result. */
179
+ export function migrateMinhashV2(host, reporter = createConformanceReporter()) {
180
+ m4Backfill(host, reporter);
181
+ const v = m4Verify(host);
182
+ if (v.ok)
183
+ m4Switch(host);
184
+ return v;
185
+ }
186
+ /** Registered M4 conformance ID range (M4-001..008 + named M4-*). */
187
+ export const M4_IDS = [
188
+ "M4-001",
189
+ "M4-002",
190
+ "M4-003",
191
+ "M4-004",
192
+ "M4-005",
193
+ "M4-006",
194
+ "M4-007",
195
+ "M4-008",
196
+ ];
197
+ /** Registered named M4 conformance IDs. */
198
+ export const M4_NAMED = [
199
+ "M4-HIGHBIT-001",
200
+ "M4-VERSION-002",
201
+ "M4-RESUME-003",
202
+ ];
203
+ function dedupe(codes) {
204
+ const out = [];
205
+ for (const c of codes)
206
+ if (!out.includes(c))
207
+ out.push(c);
208
+ return out;
209
+ }
@@ -0,0 +1,46 @@
1
+ /**
2
+ * vector-cortex/conformance/emit.ts — VC1C observability seam.
3
+ *
4
+ * Emits the three VC1C structured events, all gated on `MEGACOMPACT_VC1C`
5
+ * (mode C parity: flag OFF => zero emissions). Every event is a JSON line with
6
+ * `ts` + `event` (ENGINEERING_PRACTICES §8); the emitters below capture the
7
+ * event name + fields and are never fatal on consumer failure (non-fatal
8
+ * observability, never breaks the agent loop).
9
+ *
10
+ * vector_cortex_minhash_v2_backfilled — M4 backfill wrote v2 rows
11
+ * vector_cortex_conformance_case_checked — a v2 conformance case dispatched
12
+ * vector_cortex_downgrade_copy_written — a downgrade legacy copy written
13
+ *
14
+ * No network, no side effects beyond the supplied emit callback
15
+ * (PREVENT-PI-004 / PREVENT-011).
16
+ */
17
+ import { VC1C_ENABLED } from "../../config/vector-cortex.js";
18
+ /** A flag-gated no-op reporter (zero emissions, default when none injected). */
19
+ export const NOOP_CONFORMANCE_REPORTER = {
20
+ backfilled: () => { },
21
+ caseChecked: () => { },
22
+ downgradeWritten: () => { },
23
+ };
24
+ /**
25
+ * Flag-gated emit: no-op when VC1C is off or no emitter is supplied. The
26
+ * returned reporter is itself flag-gated (`VC1C_ENABLED`), so wiring it into a
27
+ * runtime seam yields zero emissions when `MEGACOMPACT_VC1C=0` (byte-identical
28
+ * to the predecessor).
29
+ */
30
+ export function createConformanceReporter(emit) {
31
+ const fire = (event, fields) => {
32
+ if (!VC1C_ENABLED())
33
+ return;
34
+ try {
35
+ emit?.(event, { ...fields, ts: new Date().toISOString() });
36
+ }
37
+ catch {
38
+ /* non-fatal observability */
39
+ }
40
+ };
41
+ return {
42
+ backfilled: (fields) => fire("vector_cortex_minhash_v2_backfilled", fields),
43
+ caseChecked: (fields) => fire("vector_cortex_conformance_case_checked", fields),
44
+ downgradeWritten: (fields) => fire("vector_cortex_downgrade_copy_written", fields),
45
+ };
46
+ }
@@ -0,0 +1,235 @@
1
+ /**
2
+ * vector-cortex/conformance/manifest.ts — FixtureManifestV2 (VC1C).
3
+ *
4
+ * Owns the v2 conformance manifest: reads the authoritative
5
+ * `conformance/vector-cortex/v2/manifest.json` plus the on-disk fixture tree
6
+ * and validates it canonically. Each FixtureManifestV2 entry carries:
7
+ * id, domain, inputDigest, expectedDigest, failureCode, algorithmTuple.
8
+ *
9
+ * Canonical validation rejects a mismatched corpus with the frozen codes:
10
+ * CONF_EXTRA_FIXTURE — a file on disk that the manifest does not list;
11
+ * CONF_MISSING_FIXTURE — a manifest entry with no file on disk;
12
+ * CONF_DIGEST_DRIFT — on-disk bytes do not match the manifest SHA-256.
13
+ *
14
+ * Key ordering rule (canonical JSON): object keys are sorted by UTF-8 bytes;
15
+ * a manifest that is not canonical (or a fixture whose canonical re-serialization
16
+ * differs) is itself a digest-drift / noncanonical condition. The invariant:
17
+ * canonical valid manifests converge to ONE digest (the corpus is a single
18
+ * reproducible byte image).
19
+ *
20
+ * Pure FS reads + pure predicates; no network, no side effects on authority
21
+ * data (PREVENT-PI-004 / PREVENT-011).
22
+ */
23
+ import { readFileSync, readdirSync, statSync } from "node:fs";
24
+ import { createHash as sha256 } from "node:crypto";
25
+ import { join, relative, sep } from "node:path";
26
+ /** Frozen VC1C conformance failure codes (the "reject unknown / extra / drifted"). */
27
+ export const CONF_FAIL = {
28
+ EXTRA_FIXTURE: "CONF_EXTRA_FIXTURE",
29
+ MISSING_FIXTURE: "CONF_MISSING_FIXTURE",
30
+ DIGEST_DRIFT: "CONF_DIGEST_DRIFT",
31
+ NONCANONICAL: "CONF_NONCANONICAL",
32
+ UNKNOWN_DOMAIN: "CONF_UNKNOWN_DOMAIN",
33
+ };
34
+ /** Walk a directory tree returning relative POSIX paths (sorted). */
35
+ function walk(dir, base, acc = []) {
36
+ for (const name of readdirSync(dir)) {
37
+ const p = join(dir, name);
38
+ const st = statSync(p);
39
+ if (st.isDirectory())
40
+ walk(p, base, acc);
41
+ else
42
+ acc.push(relative(base, p).split(sep).join("/"));
43
+ }
44
+ return acc.sort();
45
+ }
46
+ /**
47
+ * Canonical JSON bytes for a value (UTF-8, NFC keys, sorted, shortest numbers).
48
+ * Matches the stricter `canonicalNumber` in scripts/vector-cortex-conformance.mjs:
49
+ * a non-finite number or -0 has no canonical shortest representation and is
50
+ * rejected, so the two "canonical" serializers can never diverge for the same
51
+ * value (thereby preserving the "converge to one digest" invariant).
52
+ */
53
+ function canonicalValue(value) {
54
+ if (value === null || typeof value !== "object") {
55
+ if (typeof value === "number") {
56
+ if (!Number.isFinite(value) || Object.is(value, -0)) {
57
+ throw new Error(`non-canonical number: ${value}`);
58
+ }
59
+ if (Number.isInteger(value) && Number.isSafeInteger(value))
60
+ return String(value);
61
+ return JSON.stringify(value);
62
+ }
63
+ return JSON.stringify(value);
64
+ }
65
+ if (Array.isArray(value))
66
+ return `[${value.map((v) => canonicalValue(v)).join(",")}]`;
67
+ const keys = Object.keys(value)
68
+ .map((k) => k.normalize("NFC"))
69
+ .sort();
70
+ const parts = keys.map((k) => `${JSON.stringify(k)}:${canonicalValue(value[k])}`);
71
+ return `{${parts.join(",")}}`;
72
+ }
73
+ function canonicalJson(value) {
74
+ return Buffer.from(`${canonicalValue(value)}\n`, "utf8");
75
+ }
76
+ function sha256Hex(bytes) {
77
+ return sha256("sha256").update(bytes).digest("hex");
78
+ }
79
+ /** parse a manifest.json (must be a valid v2 manifest object). */
80
+ function parseManifest(raw) {
81
+ const obj = JSON.parse(raw);
82
+ if (obj === null || typeof obj !== "object" || !Array.isArray(obj.fixtures)) {
83
+ throw new Error("CONF_MANIFEST_INVALID: manifest.fixtures must be an array");
84
+ }
85
+ return {
86
+ version: typeof obj.version === "string" ? obj.version : "2",
87
+ owner: typeof obj.owner === "string" ? obj.owner : "",
88
+ fixtures: obj.fixtures,
89
+ };
90
+ }
91
+ /**
92
+ * Read + normalize the v2 manifest and on-disk tree into a FixtureManifestV2.
93
+ * `fixtureRoot` is the conformance v2 root directory (injected by callers so
94
+ * `src/` stays runtime-independent of a hardcoded path).
95
+ */
96
+ export function readFixtureManifestV2(fixtureRoot) {
97
+ const manifestRaw = readFileSync(join(fixtureRoot, "manifest.json"), "utf8");
98
+ const manifest = parseManifest(manifestRaw);
99
+ const byId = new Map();
100
+ const entries = [];
101
+ for (const fx of manifest.fixtures) {
102
+ const expectedDigest = String(fx.sha256 ?? "");
103
+ const domain = fx.domain ?? domainOf(fx.path);
104
+ const inputDigest = canonicalFileDigest(join(fixtureRoot, fx.path));
105
+ const entry = {
106
+ id: fx.id,
107
+ domain,
108
+ path: fx.path,
109
+ algorithm: fx.algorithm,
110
+ expected: fx.expected,
111
+ inputDigest,
112
+ expectedDigest,
113
+ expectedOutputDigest: typeof fx.outputDigest === "string" && fx.outputDigest.length > 0
114
+ ? fx.outputDigest
115
+ : undefined,
116
+ failureCode: fx.expected === "ok" ? undefined : fx.expected,
117
+ algorithmTuple: String(fx.algorithm).split(";").filter(Boolean).map((s) => s.trim()),
118
+ };
119
+ entries.push(entry);
120
+ byId.set(fx.id, entry);
121
+ }
122
+ return {
123
+ version: manifest.version,
124
+ owner: manifest.owner.split(",").map((s) => s.trim()).filter(Boolean),
125
+ fixtureEntries: entries,
126
+ byId,
127
+ };
128
+ }
129
+ /** The v2 domain a fixture path belongs to (first path segment). */
130
+ export function domainOf(path) {
131
+ return path.split("/")[0] ?? "";
132
+ }
133
+ /**
134
+ * SHA-256 over the canonical JSON bytes of a fixture file. A missing file
135
+ * returns "" so `validateCanonicalV2` can report CONF_MISSING_FIXTURE instead of
136
+ * throwing ENOENT part-way through manifest normalization.
137
+ */
138
+ function canonicalFileDigest(absPath) {
139
+ let raw;
140
+ try {
141
+ raw = readFileSync(absPath, "utf8");
142
+ }
143
+ catch {
144
+ return "";
145
+ }
146
+ let parsed;
147
+ try {
148
+ parsed = JSON.parse(raw);
149
+ }
150
+ catch {
151
+ parsed = raw; // non-JSON (e.g. seeds as bytes) — digest raw
152
+ }
153
+ return sha256Hex(canonicalJson(parsed));
154
+ }
155
+ /**
156
+ * Validate the on-disk v2 corpus against the manifest. Rejects an extra,
157
+ * missing, or digest-drifted fixture file with the frozen codes. Also verifies
158
+ * the manifest itself is canonical (sorted keys, shortest numbers) — a
159
+ * noncanonical manifest is reported as NONCANONICAL.
160
+ */
161
+ export function validateCanonicalV2(fixtureRoot) {
162
+ const codes = [];
163
+ const issues = [];
164
+ const manifestRaw = readFileSync(join(fixtureRoot, "manifest.json"), "utf8");
165
+ try {
166
+ const canonicalBytes = canonicalJson(JSON.parse(manifestRaw));
167
+ if (Buffer.from(manifestRaw, "utf8").toString("hex") !== canonicalBytes.toString("hex")) {
168
+ codes.push(CONF_FAIL.NONCANONICAL);
169
+ issues.push("manifest.json is not canonical");
170
+ }
171
+ }
172
+ catch {
173
+ codes.push(CONF_FAIL.NONCANONICAL);
174
+ issues.push("manifest.json is not valid JSON");
175
+ }
176
+ const manifest = readFixtureManifestV2(fixtureRoot);
177
+ const listedPaths = new Set(manifest.fixtureEntries.map((e) => e.path));
178
+ // Extra-file detection: every file under the root (except manifest) listed.
179
+ const onDisk = walk(fixtureRoot, fixtureRoot).filter((p) => p !== "manifest.json");
180
+ for (const p of onDisk) {
181
+ if (!listedPaths.has(p)) {
182
+ codes.push(CONF_FAIL.EXTRA_FIXTURE);
183
+ issues.push(`extra fixture: ${p}`);
184
+ }
185
+ }
186
+ // Missing + digest-drift for every listed fixture.
187
+ for (const entry of manifest.fixtureEntries) {
188
+ const abs = join(fixtureRoot, entry.path);
189
+ if (!exists(abs)) {
190
+ codes.push(CONF_FAIL.MISSING_FIXTURE);
191
+ issues.push(`missing fixture: ${entry.path}`);
192
+ continue;
193
+ }
194
+ const raw = readFileSync(abs, "utf8");
195
+ let canonicalHex;
196
+ try {
197
+ canonicalHex = sha256Hex(canonicalJson(JSON.parse(raw)));
198
+ }
199
+ catch {
200
+ canonicalHex = sha256Hex(Buffer.from(raw, "utf8"));
201
+ }
202
+ if (canonicalHex !== entry.expectedDigest) {
203
+ codes.push(CONF_FAIL.DIGEST_DRIFT);
204
+ issues.push(`digest drift for ${entry.path}: expected ${entry.expectedDigest} got ${canonicalHex}`);
205
+ }
206
+ }
207
+ // The canonical-key ordering invariant: a canonical valid manifest is a single
208
+ // reproducible digest. (Extra-file presence also breaks purity but is the
209
+ // extra/missing covers it.) Noncanonical keys are caught above.
210
+ if (codes.length === 0) {
211
+ return { ok: true, entryCount: manifest.fixtureEntries.length };
212
+ }
213
+ return { ok: false, codes: dedupeCodes(codes), issues };
214
+ }
215
+ /** True if the manifest+corpus is canonical (converges to one digest). */
216
+ export function canonicalManifestsConverge(fixtureRoot) {
217
+ const v = validateCanonicalV2(fixtureRoot);
218
+ return v.ok;
219
+ }
220
+ function exists(p) {
221
+ try {
222
+ statSync(p);
223
+ return true;
224
+ }
225
+ catch {
226
+ return false;
227
+ }
228
+ }
229
+ function dedupeCodes(codes) {
230
+ const out = [];
231
+ for (const c of codes)
232
+ if (!out.includes(c))
233
+ out.push(c);
234
+ return out;
235
+ }
@@ -0,0 +1,111 @@
1
+ /**
2
+ * vector-cortex/conformance/runner.ts — v2 conformance runner (VC1C).
3
+ *
4
+ * Dispatches a conformance case STRICTLY by domain/version (no inference, no
5
+ * partial output on unknown) and compares both the canonical success bytes and
6
+ * the expected failure code. Owns `DowngradeReport`.
7
+ *
8
+ * The runner is triad-shaped (mode A client):
9
+ * A = this manifest runner over the real algorithms;
10
+ * B = an independent exact fixture reader (invertible digest recheck);
11
+ * C = reject any unknown domain/version WITHOUT partial output.
12
+ *
13
+ * Downgrade export: `exporter` produces a new legacy copy that never edits
14
+ * authority data; the resulting `DowngradeReport` is deterministic — a second
15
+ * run yields a byte-identical report digest (CONF-DOWN-003).
16
+ *
17
+ * No network, no side effects on authority (PREVENT-PI-004 / PREVENT-011).
18
+ */
19
+ import { CONF_FAIL } from "./manifest.js";
20
+ import { createConformanceReporter, } from "./emit.js";
21
+ const UNKNOWN_DOMAIN = "CONF_UNKNOWN_DOMAIN";
22
+ const UNKNOWN_VERSION = "CONF_UNKNOWN_VERSION";
23
+ /**
24
+ * The runner-pinned expectation mismatch code: a dispatched handler returned an
25
+ * outcome that contradicts the manifest entry's frozen expectation (the wrong
26
+ * failure code, or a success/failure parity error). This is the runner's own
27
+ * cross-check (task 5: "compare both success bytes and expected failure code"),
28
+ * distinct from the domain's own failure codes.
29
+ */
30
+ const EXPECTATION_MISMATCH = "CONF_EXPECTATION_MISMATCH";
31
+ export { UNKNOWN_DOMAIN, UNKNOWN_VERSION, EXPECTATION_MISMATCH };
32
+ /**
33
+ * Run one conformance case by dispatching strictly on `(domain, algorithmTuple)`.
34
+ * If no handler is registered for the exact domain/algorithms, returns
35
+ * `CONF_UNKNOWN_DOMAIN`/`CONF_UNKNOWN_VERSION` WITHOUT partial output.
36
+ *
37
+ * After dispatch the runner cross-checks the handler's outcome against the
38
+ * manifest entry's frozen expectation (task 5):
39
+ * - failure expected (`entry.failureCode` set): the handler must return
40
+ * `ok:false` with that exact code, else `CONF_EXPECTATION_MISMATCH`;
41
+ * - success expected (`entry.expected === "ok"`): the handler must return
42
+ * `ok:true`, and when the manifest pins an `expectedOutputDigest` the
43
+ * handler's `outputDigest` must match it, else `CONF_DIGEST_DRIFT`.
44
+ */
45
+ export function runConformanceCase(entry, handlers, fixture, reporter = createConformanceReporter()) {
46
+ const key = `${entry.domain}::${entry.algorithmTuple.join(";")}`;
47
+ const handler = handlers.get(key);
48
+ let result;
49
+ if (!handler) {
50
+ if (entry.domain === "") {
51
+ result = { ok: false, code: UNKNOWN_DOMAIN, algorithm: entry.algorithm };
52
+ }
53
+ else {
54
+ result = { ok: false, code: UNKNOWN_VERSION, algorithm: entry.algorithm };
55
+ }
56
+ }
57
+ else {
58
+ const out = handler.run(entry, fixture);
59
+ if (out.ok) {
60
+ // Success expected? The manifest entry pins the expected success bytes.
61
+ if (entry.expected !== "ok") {
62
+ result = { ok: false, code: EXPECTATION_MISMATCH, algorithm: entry.algorithm };
63
+ }
64
+ else if (entry.expectedOutputDigest !== undefined &&
65
+ out.outputDigest !== entry.expectedOutputDigest) {
66
+ result = { ok: false, code: CONF_FAIL.DIGEST_DRIFT, algorithm: entry.algorithm };
67
+ }
68
+ else {
69
+ result = { ok: true, outputDigest: out.outputDigest, algorithm: entry.algorithm };
70
+ }
71
+ }
72
+ else {
73
+ // Failure expected? The manifest entry pins the exact failure code.
74
+ if (entry.failureCode !== undefined && out.code === entry.failureCode) {
75
+ result = { ok: false, code: out.code, algorithm: entry.algorithm };
76
+ }
77
+ else {
78
+ result = { ok: false, code: EXPECTATION_MISMATCH, algorithm: entry.algorithm };
79
+ }
80
+ }
81
+ }
82
+ reporter.caseChecked({
83
+ id: entry.id,
84
+ domain: entry.domain,
85
+ algorithm: entry.algorithm,
86
+ ok: result.ok,
87
+ ...(result.ok ? { outputDigest: result.outputDigest } : { code: result.code }),
88
+ });
89
+ return result;
90
+ }
91
+ /**
92
+ * Build a dispatch key for a (domain, algorithmTuple) pair. Callers register
93
+ * handlers under this key.
94
+ */
95
+ export function handlerKey(domain, algorithmTuple) {
96
+ return `${domain}::${algorithmTuple.join(";")}`;
97
+ }
98
+ /**
99
+ * Run a downgrade export and recompute its report digest deterministically.
100
+ * A second invocation produces a byte-identical report (CONF-DOWN-003).
101
+ */
102
+ export function runDowngradeExport(exporter, reporter = createConformanceReporter()) {
103
+ const report = exporter.exportOnce();
104
+ reporter.downgradeWritten({
105
+ exportedCopyId: report.exportedCopyId,
106
+ copiedCount: report.copiedCount,
107
+ unrepresentableIds: report.unrepresentableIds,
108
+ reportDigest: report.reportDigest,
109
+ });
110
+ return report;
111
+ }